mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 02:06:43 +00:00
fix(vault): aggregate provider outage diagnostics (#110908)
* fix(vault): aggregate provider outage diagnostics * fix(vault): classify revoked token outages * fix(vault): preserve scoped ACL failures * fix(vault): keep token probes advisory * test(vault): satisfy gateway proof gates * fix(secrets): attribute web provider outages * test(secrets): prove web outage fan-out in owner suite * fix(vault): scope malformed responses per secret * test(secrets): harden exec fanout fixtures
This commit is contained in:
@@ -615,7 +615,7 @@ Behavior:
|
||||
- Recovered: emitted once after the next successful activation.
|
||||
- Repeated failures while already degraded log warnings but do not re-emit the event.
|
||||
- Startup fail-fast never emits a degraded event, because runtime never became active.
|
||||
- Startup and reload failures emit a structured `SECRETS_DEGRADED` warning for each affected owner. The warning includes the owner kind and id, a redacted reason, `cold` or `stale` state, and the `openclaw secrets reload` retry hint. It never includes resolved values or SecretRef ids.
|
||||
- Ref-scoped startup and reload failures emit a structured `SECRETS_DEGRADED` warning for each affected owner. Provider-scoped outages emit one `SECRETS_PROVIDER_DEGRADED` warning with the provider and complete affected-owner list instead of repeating the provider failure per owner. Warnings include a redacted reason, `cold` or `stale` owner state, and the `openclaw secrets reload` retry hint. They never include resolved values or SecretRef ids.
|
||||
- `openclaw doctor` lists cold and stale owners with their affected config paths, redacted reason, and retry guidance.
|
||||
|
||||
## Command-path resolution
|
||||
|
||||
@@ -0,0 +1,66 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { createServer } from "node:http";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
const resolverPath = fileURLToPath(new URL("../vault-secret-ref-resolver.js", import.meta.url));
|
||||
|
||||
it("keeps malformed successful Vault responses scoped per id", async () => {
|
||||
const server = createServer((_request, response) => {
|
||||
response.setHeader("content-type", "application/json");
|
||||
response.end("not-json");
|
||||
});
|
||||
await new Promise<void>((resolve) => {
|
||||
server.listen(0, "127.0.0.1", resolve);
|
||||
});
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("fixture server did not bind to a TCP port");
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await new Promise<{ stdout: string; stderr: string; code: number | null }>(
|
||||
(resolve, reject) => {
|
||||
const child = spawn(process.execPath, [resolverPath], {
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
env: {
|
||||
...process.env,
|
||||
VAULT_ADDR: `http://127.0.0.1:${address.port}`,
|
||||
VAULT_TOKEN: "not-a-real-auth-header",
|
||||
},
|
||||
});
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
child.stdout.setEncoding("utf8").on("data", (chunk) => (stdout += String(chunk)));
|
||||
child.stderr.setEncoding("utf8").on("data", (chunk) => (stderr += String(chunk)));
|
||||
child.on("error", reject);
|
||||
child.on("close", (code) => resolve({ stdout, stderr, code }));
|
||||
child.stdin.end(
|
||||
`${JSON.stringify({
|
||||
protocolVersion: 1,
|
||||
provider: "vault",
|
||||
ids: ["providers/openai/apiKey", "tts/elevenlabs/apiKey"],
|
||||
})}\n`,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({ code: 0, stderr: "" });
|
||||
expect(JSON.parse(result.stdout)).toEqual({
|
||||
protocolVersion: 1,
|
||||
values: {},
|
||||
errors: {
|
||||
"providers/openai/apiKey": {
|
||||
message: 'Vault read response for "providers/openai/apiKey" was not valid JSON.',
|
||||
},
|
||||
"tts/elevenlabs/apiKey": {
|
||||
message: 'Vault read response for "tts/elevenlabs/apiKey" was not valid JSON.',
|
||||
},
|
||||
},
|
||||
});
|
||||
} finally {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
server.close((error) => (error ? reject(error) : resolve()));
|
||||
});
|
||||
}
|
||||
});
|
||||
@@ -118,11 +118,54 @@ async function startVaultFixture() {
|
||||
};
|
||||
}
|
||||
|
||||
async function startVaultErrorFixture() {
|
||||
const server = createServer((_request, response) => {
|
||||
response.statusCode = 403;
|
||||
async function startVaultErrorFixture(
|
||||
statusCode = 403,
|
||||
errors = ["token not-a-real-sensitive-value denied"],
|
||||
lookupSucceeds = true,
|
||||
lookupErrors = errors,
|
||||
lookupStatusCode = statusCode,
|
||||
) {
|
||||
const requests: string[] = [];
|
||||
const server = createServer((request, response) => {
|
||||
requests.push(request.url ?? "");
|
||||
if (request.url === "/v1/auth/token/lookup-self" && lookupSucceeds) {
|
||||
response.setHeader("content-type", "application/json");
|
||||
response.end(JSON.stringify({ data: { id: "redacted-fixture-token" } }));
|
||||
return;
|
||||
}
|
||||
response.statusCode =
|
||||
request.url === "/v1/auth/token/lookup-self" ? lookupStatusCode : statusCode;
|
||||
response.setHeader("content-type", "application/json");
|
||||
response.end(JSON.stringify({ errors: ["token not-a-real-sensitive-value denied"] }));
|
||||
response.end(
|
||||
JSON.stringify({
|
||||
errors: request.url === "/v1/auth/token/lookup-self" ? lookupErrors : errors,
|
||||
}),
|
||||
);
|
||||
});
|
||||
await new Promise<void>((resolve) => {
|
||||
server.listen(0, "127.0.0.1", resolve);
|
||||
});
|
||||
servers.push({
|
||||
close: () =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
server.close((error) => (error ? reject(error) : resolve()));
|
||||
}),
|
||||
});
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("fixture server did not bind to a TCP port");
|
||||
}
|
||||
return {
|
||||
requests,
|
||||
vaultAddr: `http://127.0.0.1:${address.port}`,
|
||||
};
|
||||
}
|
||||
|
||||
async function startVaultMixedErrorFixture() {
|
||||
const server = createServer((request, response) => {
|
||||
response.statusCode = request.url?.includes("/providers/openai") ? 403 : 503;
|
||||
response.setHeader("content-type", "application/json");
|
||||
response.end(JSON.stringify({ errors: ["not-a-real-sensitive-value"] }));
|
||||
});
|
||||
await new Promise<void>((resolve) => {
|
||||
server.listen(0, "127.0.0.1", resolve);
|
||||
@@ -166,6 +209,37 @@ async function startVaultStalledBodyFixture() {
|
||||
};
|
||||
}
|
||||
|
||||
async function startVaultOversizedErrorBodyFixture() {
|
||||
const server = createServer((request, response) => {
|
||||
if (request.url === "/v1/auth/token/lookup-self") {
|
||||
response.setHeader("content-type", "application/json");
|
||||
response.end(JSON.stringify({ data: { id: "redacted-fixture-token" } }));
|
||||
return;
|
||||
}
|
||||
response.statusCode = 403;
|
||||
response.setHeader("content-type", "application/json");
|
||||
response.setHeader("content-length", String(64 * 1024 + 1));
|
||||
response.write('{"errors":["partial');
|
||||
});
|
||||
await new Promise<void>((resolve) => {
|
||||
server.listen(0, "127.0.0.1", resolve);
|
||||
});
|
||||
servers.push({
|
||||
close: () =>
|
||||
new Promise<void>((resolve, reject) => {
|
||||
server.close((error) => (error ? reject(error) : resolve()));
|
||||
server.closeAllConnections();
|
||||
}),
|
||||
});
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("fixture server did not bind to a TCP port");
|
||||
}
|
||||
return {
|
||||
vaultAddr: `http://127.0.0.1:${address.port}`,
|
||||
};
|
||||
}
|
||||
|
||||
async function readRequestBody(request: import("node:http").IncomingMessage): Promise<string> {
|
||||
return await new Promise((resolve, reject) => {
|
||||
let body = "";
|
||||
@@ -350,12 +424,12 @@ describe("vault SecretRef resolver", () => {
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ code: 0, stderr: "" });
|
||||
expect(result).toMatchObject({ code: 1, stderr: "" });
|
||||
expect(JSON.parse(result.stdout)).toEqual({
|
||||
protocolVersion: 1,
|
||||
values: {},
|
||||
errors: {
|
||||
"providers/openai/apiKey": {
|
||||
request: {
|
||||
message: "VAULT_TOKEN is required.",
|
||||
},
|
||||
},
|
||||
@@ -501,12 +575,12 @@ describe("vault SecretRef resolver", () => {
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ code: 0, stderr: "" });
|
||||
expect(result).toMatchObject({ code: 1, stderr: "" });
|
||||
expect(JSON.parse(result.stdout)).toEqual({
|
||||
protocolVersion: 1,
|
||||
values: {},
|
||||
errors: {
|
||||
"providers/openai/apiKey": {
|
||||
request: {
|
||||
message: expect.stringContaining("exceeds 16384 bytes"),
|
||||
},
|
||||
},
|
||||
@@ -581,12 +655,12 @@ describe("vault SecretRef resolver", () => {
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ code: 0, stderr: "" });
|
||||
expect(result).toMatchObject({ code: 1, stderr: "" });
|
||||
expect(JSON.parse(result.stdout)).toEqual({
|
||||
protocolVersion: 1,
|
||||
values: {},
|
||||
errors: {
|
||||
"providers/openai/apiKey": {
|
||||
request: {
|
||||
message: expect.stringContaining("exceeds 16384 bytes"),
|
||||
},
|
||||
},
|
||||
@@ -641,37 +715,37 @@ describe("vault SecretRef resolver", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("returns per-id errors when Vault auth is not configured", async () => {
|
||||
it("reports one provider failure when Vault auth is unavailable for multiple ids", async () => {
|
||||
const result = await runResolver({
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
provider: "vault",
|
||||
ids: ["providers/anthropic/apiKey"],
|
||||
ids: ["providers/anthropic/apiKey", "tts/elevenlabs/apiKey"],
|
||||
},
|
||||
env: {
|
||||
VAULT_ADDR: "https://vault.example.test",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ code: 0, stderr: "" });
|
||||
expect(result).toMatchObject({ code: 1, stderr: "" });
|
||||
expect(JSON.parse(result.stdout)).toEqual({
|
||||
protocolVersion: 1,
|
||||
values: {},
|
||||
errors: {
|
||||
"providers/anthropic/apiKey": {
|
||||
request: {
|
||||
message: "VAULT_TOKEN is required.",
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("does not echo Vault response bodies in resolver errors", async () => {
|
||||
it("keeps Vault secret read failures scoped per id without echoing response bodies", async () => {
|
||||
const fixture = await startVaultErrorFixture();
|
||||
const result = await runResolver({
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
provider: "vault",
|
||||
ids: ["providers/openai/apiKey"],
|
||||
ids: ["providers/openai/apiKey", "tts/elevenlabs/apiKey"],
|
||||
},
|
||||
env: {
|
||||
VAULT_ADDR: fixture.vaultAddr,
|
||||
@@ -687,6 +761,194 @@ describe("vault SecretRef resolver", () => {
|
||||
"providers/openai/apiKey": {
|
||||
message: 'Vault read failed for "providers/openai/apiKey" (403).',
|
||||
},
|
||||
"tts/elevenlabs/apiKey": {
|
||||
message: 'Vault read failed for "tts/elevenlabs/apiKey" (403).',
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(result.stdout).not.toContain("not-a-real-sensitive-value");
|
||||
expect(fixture.requests.filter((url) => url === "/v1/auth/token/lookup-self")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("reports one provider failure when Vault rejects an invalid token", async () => {
|
||||
const fixture = await startVaultErrorFixture(403, ["permission denied", "invalid token"]);
|
||||
const result = await runResolver({
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
provider: "vault",
|
||||
ids: ["providers/openai/apiKey", "tts/elevenlabs/apiKey"],
|
||||
},
|
||||
env: {
|
||||
VAULT_ADDR: fixture.vaultAddr,
|
||||
VAULT_TOKEN: "not-a-real-auth-header",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ code: 1, stderr: "" });
|
||||
expect(JSON.parse(result.stdout)).toEqual({
|
||||
protocolVersion: 1,
|
||||
values: {},
|
||||
errors: {
|
||||
request: {
|
||||
message: "Vault read failed (403).",
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(result.stdout).not.toContain("permission denied");
|
||||
expect(result.stdout).not.toContain("invalid token");
|
||||
});
|
||||
|
||||
it("keeps ambiguous token self-lookup 403 responses scoped per id", async () => {
|
||||
const fixture = await startVaultErrorFixture(403, ["permission denied"], false);
|
||||
const result = await runResolver({
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
provider: "vault",
|
||||
ids: ["providers/openai/apiKey", "tts/elevenlabs/apiKey"],
|
||||
},
|
||||
env: {
|
||||
VAULT_ADDR: fixture.vaultAddr,
|
||||
VAULT_TOKEN: "not-a-real-auth-header",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ code: 0, stderr: "" });
|
||||
expect(JSON.parse(result.stdout)).toEqual({
|
||||
protocolVersion: 1,
|
||||
values: {},
|
||||
errors: {
|
||||
"providers/openai/apiKey": {
|
||||
message: 'Vault read failed for "providers/openai/apiKey" (403).',
|
||||
},
|
||||
"tts/elevenlabs/apiKey": {
|
||||
message: 'Vault read failed for "tts/elevenlabs/apiKey" (403).',
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(fixture.requests.filter((url) => url === "/v1/auth/token/lookup-self")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("keeps ACL failures scoped when token introspection is unavailable", async () => {
|
||||
const fixture = await startVaultErrorFixture(
|
||||
403,
|
||||
["permission denied"],
|
||||
false,
|
||||
["temporarily unavailable"],
|
||||
503,
|
||||
);
|
||||
const result = await runResolver({
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
provider: "vault",
|
||||
ids: ["providers/openai/apiKey", "tts/elevenlabs/apiKey"],
|
||||
},
|
||||
env: {
|
||||
VAULT_ADDR: fixture.vaultAddr,
|
||||
VAULT_TOKEN: "not-a-real-auth-header",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ code: 0, stderr: "" });
|
||||
expect(JSON.parse(result.stdout)).toEqual({
|
||||
protocolVersion: 1,
|
||||
values: {},
|
||||
errors: {
|
||||
"providers/openai/apiKey": {
|
||||
message: 'Vault read failed for "providers/openai/apiKey" (403).',
|
||||
},
|
||||
"tts/elevenlabs/apiKey": {
|
||||
message: 'Vault read failed for "tts/elevenlabs/apiKey" (403).',
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(fixture.requests.filter((url) => url === "/v1/auth/token/lookup-self")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("promotes an explicit invalid-token self-lookup response to one provider failure", async () => {
|
||||
const fixture = await startVaultErrorFixture(403, ["permission denied"], false, [
|
||||
"invalid token",
|
||||
"permission denied",
|
||||
]);
|
||||
const result = await runResolver({
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
provider: "vault",
|
||||
ids: ["providers/openai/apiKey", "tts/elevenlabs/apiKey"],
|
||||
},
|
||||
env: {
|
||||
VAULT_ADDR: fixture.vaultAddr,
|
||||
VAULT_TOKEN: "not-a-real-auth-header",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ code: 1, stderr: "" });
|
||||
expect(JSON.parse(result.stdout)).toEqual({
|
||||
protocolVersion: 1,
|
||||
values: {},
|
||||
errors: {
|
||||
request: {
|
||||
message: "Vault token is invalid.",
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(fixture.requests.filter((url) => url === "/v1/auth/token/lookup-self")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it.each([412, 472, 473, 503])(
|
||||
"reports one provider failure for Vault availability status %s",
|
||||
async (statusCode) => {
|
||||
const fixture = await startVaultErrorFixture(statusCode);
|
||||
const result = await runResolver({
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
provider: "vault",
|
||||
ids: ["providers/openai/apiKey", "tts/elevenlabs/apiKey"],
|
||||
},
|
||||
env: {
|
||||
VAULT_ADDR: fixture.vaultAddr,
|
||||
VAULT_TOKEN: "not-a-real-auth-header",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ code: 1, stderr: "" });
|
||||
expect(JSON.parse(result.stdout)).toEqual({
|
||||
protocolVersion: 1,
|
||||
values: {},
|
||||
errors: {
|
||||
request: {
|
||||
message: `Vault read failed (${statusCode}).`,
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(result.stdout).not.toContain("not-a-real-sensitive-value");
|
||||
},
|
||||
);
|
||||
|
||||
it("preserves per-id failures when a sibling Vault read has a provider outage", async () => {
|
||||
const fixture = await startVaultMixedErrorFixture();
|
||||
const result = await runResolver({
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
provider: "vault",
|
||||
ids: ["providers/openai/apiKey", "tts/elevenlabs/apiKey"],
|
||||
},
|
||||
env: {
|
||||
VAULT_ADDR: fixture.vaultAddr,
|
||||
VAULT_TOKEN: "not-a-real-auth-header",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ code: 0, stderr: "" });
|
||||
expect(JSON.parse(result.stdout)).toEqual({
|
||||
protocolVersion: 1,
|
||||
values: {},
|
||||
errors: {
|
||||
"providers/openai/apiKey": {
|
||||
message: 'Vault read failed for "providers/openai/apiKey" (403).',
|
||||
},
|
||||
"tts/elevenlabs/apiKey": {
|
||||
message: "Vault read failed (503).",
|
||||
},
|
||||
},
|
||||
});
|
||||
expect(result.stdout).not.toContain("not-a-real-sensitive-value");
|
||||
@@ -709,12 +971,12 @@ describe("vault SecretRef resolver", () => {
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ code: 0, stderr: "" });
|
||||
expect(result).toMatchObject({ code: 1, stderr: "" });
|
||||
expect(JSON.parse(result.stdout)).toEqual({
|
||||
protocolVersion: 1,
|
||||
values: {},
|
||||
errors: {
|
||||
"providers/openai/apiKey": {
|
||||
request: {
|
||||
message: "Vault jwt login failed (403).",
|
||||
},
|
||||
},
|
||||
@@ -737,13 +999,40 @@ describe("vault SecretRef resolver", () => {
|
||||
timeoutMs: 6_500,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ code: 1, stderr: "", timedOut: false });
|
||||
expect(JSON.parse(result.stdout)).toEqual({
|
||||
protocolVersion: 1,
|
||||
values: {},
|
||||
errors: {
|
||||
request: {
|
||||
message: "Vault request failed.",
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("cancels oversized Vault error bodies before clearing the fetch timeout", async () => {
|
||||
const fixture = await startVaultOversizedErrorBodyFixture();
|
||||
const result = await runResolver({
|
||||
request: {
|
||||
protocolVersion: 1,
|
||||
provider: "vault",
|
||||
ids: ["providers/openai/apiKey"],
|
||||
},
|
||||
env: {
|
||||
VAULT_ADDR: fixture.vaultAddr,
|
||||
VAULT_TOKEN: "not-a-real-auth-header",
|
||||
},
|
||||
timeoutMs: 6_500,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ code: 0, stderr: "", timedOut: false });
|
||||
expect(JSON.parse(result.stdout)).toMatchObject({
|
||||
expect(JSON.parse(result.stdout)).toEqual({
|
||||
protocolVersion: 1,
|
||||
values: {},
|
||||
errors: {
|
||||
"providers/openai/apiKey": {
|
||||
message: expect.stringMatching(/abort/iu),
|
||||
message: 'Vault read failed for "providers/openai/apiKey" (403).',
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
@@ -5,6 +5,10 @@ import { parseVaultSecretId } from "./vault-secret-id.js";
|
||||
|
||||
const KUBERNETES_SERVICE_ACCOUNT_TOKEN_PATH = "/var/run/secrets/kubernetes.io/serviceaccount/token";
|
||||
const VAULT_FETCH_TIMEOUT_MS = 5000;
|
||||
const VAULT_ERROR_BODY_MAX_BYTES = 64 * 1024;
|
||||
|
||||
class VaultProviderError extends Error {}
|
||||
class VaultForbiddenError extends Error {}
|
||||
|
||||
function readStdin() {
|
||||
return new Promise((resolve, reject) => {
|
||||
@@ -144,6 +148,47 @@ function assertVaultRequestUrl(baseUrl, requestUrl) {
|
||||
}
|
||||
}
|
||||
|
||||
async function readVaultErrorPayload(response) {
|
||||
const contentLength = Number(response.headers.get("content-length"));
|
||||
if (Number.isFinite(contentLength) && contentLength > VAULT_ERROR_BODY_MAX_BYTES) {
|
||||
try {
|
||||
await response.body?.cancel();
|
||||
} catch {
|
||||
// The fetch timeout still owns a stuck or failed cancellation.
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) {
|
||||
return undefined;
|
||||
}
|
||||
const decoder = new TextDecoder();
|
||||
let bytesRead = 0;
|
||||
let text = "";
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
text += decoder.decode();
|
||||
break;
|
||||
}
|
||||
bytesRead += value.byteLength;
|
||||
if (bytesRead > VAULT_ERROR_BODY_MAX_BYTES) {
|
||||
try {
|
||||
await reader.cancel();
|
||||
} catch {
|
||||
// The fetch timeout still owns a stuck or failed cancellation.
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
text += decoder.decode(value, { stream: true });
|
||||
}
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchVault(baseUrl, url, init) {
|
||||
assertVaultRequestUrl(baseUrl, url);
|
||||
const abortController = new AbortController();
|
||||
@@ -156,13 +201,22 @@ async function fetchVault(baseUrl, url, init) {
|
||||
});
|
||||
return {
|
||||
response,
|
||||
payload: response.ok ? await response.json() : undefined,
|
||||
payload: response.ok ? await response.json() : await readVaultErrorPayload(response),
|
||||
};
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
function isInvalidVaultTokenPayload(payload) {
|
||||
return (
|
||||
Array.isArray(payload?.errors) &&
|
||||
payload.errors.some(
|
||||
(entry) => typeof entry === "string" && entry.trim().toLowerCase() === "invalid token",
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function addVaultNamespaceHeader(headers) {
|
||||
const namespace = process.env.VAULT_NAMESPACE?.trim();
|
||||
if (namespace) {
|
||||
@@ -238,6 +292,31 @@ async function resolveVaultClientToken(baseUrl) {
|
||||
throw new Error("Unsupported Vault auth method.");
|
||||
}
|
||||
|
||||
async function classifyVaultClientToken(baseUrl, vaultToken) {
|
||||
const headers = {
|
||||
"X-Vault-Token": vaultToken,
|
||||
};
|
||||
addVaultNamespaceHeader(headers);
|
||||
let response;
|
||||
let payload;
|
||||
try {
|
||||
({ response, payload } = await fetchVault(baseUrl, `${baseUrl}/v1/auth/token/lookup-self`, {
|
||||
headers,
|
||||
}));
|
||||
} catch {
|
||||
return "unknown";
|
||||
}
|
||||
if (response.ok) {
|
||||
return "valid";
|
||||
}
|
||||
if (response.status === 401 || isInvalidVaultTokenPayload(payload)) {
|
||||
return "invalid";
|
||||
}
|
||||
// Token introspection is advisory. Preserve the concrete per-id ACL failures
|
||||
// when this probe is denied, unavailable, or otherwise inconclusive.
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
function readStringField(payload, parsedId) {
|
||||
const record = payload;
|
||||
const data = resolveKvVersion() === 2 ? record?.data?.data : record?.data;
|
||||
@@ -256,10 +335,35 @@ async function readVaultSecret(baseUrl, vaultToken, id) {
|
||||
"X-Vault-Token": vaultToken,
|
||||
};
|
||||
addVaultNamespaceHeader(headers);
|
||||
const { response, payload } = await fetchVault(baseUrl, buildVaultUrl(baseUrl, parsedId), {
|
||||
headers,
|
||||
});
|
||||
let response;
|
||||
let payload;
|
||||
try {
|
||||
({ response, payload } = await fetchVault(baseUrl, buildVaultUrl(baseUrl, parsedId), {
|
||||
headers,
|
||||
}));
|
||||
} catch (error) {
|
||||
if (error instanceof SyntaxError) {
|
||||
throw new Error(`Vault read response for "${id}" was not valid JSON.`, { cause: error });
|
||||
}
|
||||
throw new VaultProviderError("Vault request failed.", { cause: error });
|
||||
}
|
||||
if (!response.ok) {
|
||||
if (
|
||||
response.status === 401 ||
|
||||
(response.status === 403 && isInvalidVaultTokenPayload(payload)) ||
|
||||
response.status === 408 ||
|
||||
response.status === 412 ||
|
||||
response.status === 425 ||
|
||||
response.status === 429 ||
|
||||
response.status === 472 ||
|
||||
response.status === 473 ||
|
||||
response.status >= 500
|
||||
) {
|
||||
throw new VaultProviderError(`Vault read failed (${response.status}).`);
|
||||
}
|
||||
if (response.status === 403) {
|
||||
throw new VaultForbiddenError(`Vault read failed for "${id}" (403).`);
|
||||
}
|
||||
throw new Error(`Vault read failed for "${id}" (${response.status}).`);
|
||||
}
|
||||
return readStringField(payload, parsedId);
|
||||
@@ -270,25 +374,45 @@ async function resolveFromVault(ids) {
|
||||
if (ids.length === 0) {
|
||||
return response;
|
||||
}
|
||||
const contextPromise = Promise.resolve().then(async () => {
|
||||
const baseUrl = normalizeVaultAddress();
|
||||
return {
|
||||
baseUrl,
|
||||
vaultToken: await resolveVaultClientToken(baseUrl),
|
||||
};
|
||||
});
|
||||
await Promise.all(
|
||||
// Address and authentication are provider-wide. Let those failures terminate the
|
||||
// subprocess so OpenClaw fans one provider diagnostic out to every affected owner.
|
||||
const baseUrl = normalizeVaultAddress();
|
||||
const vaultToken = await resolveVaultClientToken(baseUrl);
|
||||
const results = await Promise.all(
|
||||
ids.map(async (id) => {
|
||||
try {
|
||||
const { baseUrl, vaultToken } = await contextPromise;
|
||||
response.values[id] = await readVaultSecret(baseUrl, vaultToken, id);
|
||||
return { id, value: await readVaultSecret(baseUrl, vaultToken, id) };
|
||||
} catch (error) {
|
||||
response.errors[id] = {
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
return {
|
||||
id,
|
||||
error,
|
||||
providerFailure: error instanceof VaultProviderError,
|
||||
forbidden: error instanceof VaultForbiddenError,
|
||||
};
|
||||
}
|
||||
}),
|
||||
);
|
||||
const providerFailures = results.filter((result) => result.providerFailure);
|
||||
const firstProviderFailure = providerFailures[0];
|
||||
// A batch-wide outage is provider-scoped only when every requested read failed that way.
|
||||
// Mixed results retain their values and per-id failures instead of misclassifying all owners.
|
||||
if (firstProviderFailure && providerFailures.length === results.length) {
|
||||
throw firstProviderFailure.error;
|
||||
}
|
||||
if (results.every((result) => result.forbidden)) {
|
||||
if ((await classifyVaultClientToken(baseUrl, vaultToken)) === "invalid") {
|
||||
throw new VaultProviderError("Vault token is invalid.");
|
||||
}
|
||||
}
|
||||
for (const result of results) {
|
||||
if ("value" in result) {
|
||||
response.values[result.id] = result.value;
|
||||
continue;
|
||||
}
|
||||
response.errors[result.id] = {
|
||||
message: result.error instanceof Error ? result.error.message : String(result.error),
|
||||
};
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
@@ -300,6 +424,7 @@ async function main() {
|
||||
|
||||
/** @param {unknown} error */
|
||||
function handleFatalError(error) {
|
||||
process.exitCode = 1;
|
||||
writeResponse({
|
||||
protocolVersion: 1,
|
||||
values: {},
|
||||
|
||||
@@ -1559,6 +1559,73 @@ describe("gateway startup config secret preflight", () => {
|
||||
expect(emitStateEvent).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("publishes one provider outage diagnostic with its affected owner list", async () => {
|
||||
const sourceConfig = gatewayTokenConfig({});
|
||||
const providerFailures = [{ source: "exec" as const, provider: "vault" }];
|
||||
const prepared = {
|
||||
...preparedSnapshot(sourceConfig),
|
||||
warnings: [
|
||||
{
|
||||
code: "SECRETS_OWNER_UNAVAILABLE" as const,
|
||||
path: "models.providers.openai.apiKey",
|
||||
message: "Secret owner provider:openai is configured-unavailable.",
|
||||
},
|
||||
{
|
||||
code: "SECRETS_OWNER_UNAVAILABLE" as const,
|
||||
path: "messages.tts.providers.elevenlabs.apiKey",
|
||||
message: "Secret owner capability:tts is configured-unavailable.",
|
||||
},
|
||||
],
|
||||
degradedOwners: [
|
||||
{
|
||||
ownerKind: "provider" as const,
|
||||
ownerId: "openai",
|
||||
state: "unavailable" as const,
|
||||
degradationState: "cold" as const,
|
||||
paths: ["models.providers.openai.apiKey"],
|
||||
refKeys: ["exec:vault:models/openai"],
|
||||
reason: "secret provider failed",
|
||||
providerFailures,
|
||||
},
|
||||
{
|
||||
ownerKind: "capability" as const,
|
||||
ownerId: "tts",
|
||||
state: "unavailable" as const,
|
||||
degradationState: "stale" as const,
|
||||
paths: ["messages.tts.providers.elevenlabs.apiKey"],
|
||||
refKeys: ["exec:vault:tts/elevenlabs"],
|
||||
reason: "secret provider failed",
|
||||
providerFailures,
|
||||
},
|
||||
],
|
||||
};
|
||||
const logSecrets = mockLogSecretsForTest();
|
||||
const activateRuntimeSecrets = runtimeSecretsActivatorForTest({
|
||||
logSecrets,
|
||||
prepareRuntimeSecretsSnapshot: vi.fn(async () => prepared),
|
||||
});
|
||||
|
||||
await activateRuntimeSecrets(sourceConfig, { reason: "startup", activate: true });
|
||||
|
||||
expect(logSecrets.warn).toHaveBeenCalledOnce();
|
||||
expect(logSecrets.warn).toHaveBeenCalledWith(
|
||||
"[SECRETS_PROVIDER_DEGRADED] exec:vault: secret provider failed. " +
|
||||
"Affected owners: stale capability:tts, cold provider:openai. " +
|
||||
"Retry: openclaw secrets reload.",
|
||||
{
|
||||
event: "secrets.provider_degraded",
|
||||
source: "exec",
|
||||
provider: "vault",
|
||||
reason: "secret provider failed",
|
||||
affectedOwners: [
|
||||
{ ownerKind: "capability", ownerId: "tts", state: "stale" },
|
||||
{ ownerKind: "provider", ownerId: "openai", state: "cold" },
|
||||
],
|
||||
retryHint: "openclaw secrets reload",
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it.each(["reload", "restart-check"] as const)(
|
||||
"does not classify untyped %s errors as secret degradation",
|
||||
async (reason) => {
|
||||
|
||||
@@ -11,9 +11,6 @@ import {
|
||||
classifySecretResolutionErrorDegradations,
|
||||
isRetryableSecretDegradationReason,
|
||||
listSecretResolutionErrorOwners,
|
||||
redactSecretDegradationReason,
|
||||
SECRET_DEGRADATION_RETRY_HINT,
|
||||
type SecretDegradation,
|
||||
} from "../secrets/runtime-degraded-state.js";
|
||||
import { prepareSecretsRuntimeFastPathSnapshot } from "../secrets/runtime-fast-path.js";
|
||||
import { registerProviderAuthRuntimeSnapshotActivationOwner } from "../secrets/runtime-provider-auth-activation.js";
|
||||
@@ -44,6 +41,10 @@ import {
|
||||
type GatewayStartupConfigMeasure,
|
||||
type GatewayStartupLog,
|
||||
} from "./server-startup-config-helpers.js";
|
||||
import {
|
||||
logPreparedSecretDegradations,
|
||||
logThrownSecretDegradations,
|
||||
} from "./server-startup-secret-diagnostics.js";
|
||||
export {
|
||||
loadGatewayStartupConfigSnapshot,
|
||||
type GatewayStartupConfigSnapshotLoadResult,
|
||||
@@ -118,22 +119,6 @@ export function publishRuntimeSecretsStateTransition(
|
||||
runtimeSecretsStatePublishers.get(activateRuntimeSecrets)?.(snapshot, options);
|
||||
}
|
||||
|
||||
function logSecretDegradation(log: GatewayStartupLog, degradation: SecretDegradation): void {
|
||||
const reason = redactSecretDegradationReason(degradation.reason);
|
||||
log.warn(
|
||||
`[SECRETS_DEGRADED] ${degradation.state} ${degradation.kind}:${degradation.id}: ` +
|
||||
`${reason}. Retry: ${degradation.retryHint}.`,
|
||||
{
|
||||
event: "secrets.degraded",
|
||||
ownerKind: degradation.kind,
|
||||
ownerId: degradation.id,
|
||||
reason,
|
||||
state: degradation.state,
|
||||
retryHint: degradation.retryHint,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/** Create the serialized secrets activation function used by startup and reload paths. */
|
||||
export function createRuntimeSecretsActivator(params: {
|
||||
logSecrets: GatewayStartupLog;
|
||||
@@ -212,15 +197,7 @@ export function createRuntimeSecretsActivator(params: {
|
||||
scope: SecretsStateScope = "full",
|
||||
activationScope: SecretsStateScope = "full",
|
||||
) => {
|
||||
for (const owner of prepared.degradedOwners ?? []) {
|
||||
logSecretDegradation(params.logSecrets, {
|
||||
kind: owner.ownerKind,
|
||||
id: owner.ownerId,
|
||||
reason: owner.reason,
|
||||
state: owner.degradationState ?? "cold",
|
||||
retryHint: SECRET_DEGRADATION_RETRY_HINT,
|
||||
});
|
||||
}
|
||||
logPreparedSecretDegradations(params.logSecrets, prepared.degradedOwners ?? []);
|
||||
if (reason === "startup") {
|
||||
return;
|
||||
}
|
||||
@@ -332,9 +309,7 @@ export function createRuntimeSecretsActivator(params: {
|
||||
retryableDegradations.length > 0 &&
|
||||
(activationParams.reason === "startup" || mayPublishReloadDegradation)
|
||||
) {
|
||||
for (const degradation of retryableDegradations) {
|
||||
logSecretDegradation(params.logSecrets, degradation);
|
||||
}
|
||||
logThrownSecretDegradations(params.logSecrets, err, retryableDegradations);
|
||||
if (activationParams.reason !== "startup") {
|
||||
if (!secretsDegraded) {
|
||||
params.emitStateEvent(
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
/** Tests Gateway aggregation of provider-scoped SecretRef diagnostics. */
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { logPreparedSecretDegradations } from "./server-startup-secret-diagnostics.js";
|
||||
|
||||
describe("Gateway SecretRef diagnostics", () => {
|
||||
it("includes a multi-provider owner in every provider diagnostic", () => {
|
||||
const warn = vi.fn();
|
||||
|
||||
logPreparedSecretDegradations({ info: vi.fn(), warn }, [
|
||||
{
|
||||
ownerKind: "provider",
|
||||
ownerId: "example",
|
||||
state: "unavailable",
|
||||
degradationState: "cold",
|
||||
paths: ["models.providers.example.apiKey", "models.providers.example.headers.X-Secondary"],
|
||||
refKeys: ["exec:first:api-key", "exec:second:secondary"],
|
||||
reason: "secret provider failed",
|
||||
providerFailures: [
|
||||
{ source: "exec", provider: "first" },
|
||||
{ source: "exec", provider: "second" },
|
||||
],
|
||||
},
|
||||
]);
|
||||
|
||||
expect(warn).toHaveBeenCalledTimes(2);
|
||||
for (const provider of ["first", "second"]) {
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining(`[SECRETS_PROVIDER_DEGRADED] exec:${provider}`),
|
||||
expect.objectContaining({
|
||||
event: "secrets.provider_degraded",
|
||||
provider,
|
||||
affectedOwners: [{ ownerKind: "provider", ownerId: "example", state: "cold" }],
|
||||
}),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("keeps an owner diagnostic when provider and ref failures are mixed", () => {
|
||||
const warn = vi.fn();
|
||||
|
||||
logPreparedSecretDegradations({ info: vi.fn(), warn }, [
|
||||
{
|
||||
ownerKind: "provider",
|
||||
ownerId: "example",
|
||||
state: "unavailable",
|
||||
degradationState: "cold",
|
||||
paths: ["models.providers.example.apiKey"],
|
||||
refKeys: ["exec:vault:api-key"],
|
||||
reason: "secret reference was not found",
|
||||
providerFailures: [{ source: "exec", provider: "vault" }],
|
||||
refFailureReason: "secret reference was not found",
|
||||
},
|
||||
]);
|
||||
|
||||
expect(warn).toHaveBeenCalledTimes(2);
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining("[SECRETS_DEGRADED] cold provider:example"),
|
||||
expect.objectContaining({
|
||||
event: "secrets.degraded",
|
||||
reason: "secret reference was not found",
|
||||
}),
|
||||
);
|
||||
expect(warn).toHaveBeenCalledWith(
|
||||
expect.stringContaining("[SECRETS_PROVIDER_DEGRADED] exec:vault"),
|
||||
expect.objectContaining({
|
||||
event: "secrets.provider_degraded",
|
||||
reason: "secret provider failed",
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,120 @@
|
||||
/** Aggregates redacted SecretRef degradation diagnostics at the Gateway activation boundary. */
|
||||
import { isProviderScopedSecretResolutionError } from "../secrets/resolve-errors.js";
|
||||
import {
|
||||
redactSecretDegradationReason,
|
||||
SECRET_DEGRADATION_RETRY_HINT,
|
||||
type DegradedSecretOwner,
|
||||
type SecretDegradation,
|
||||
} from "../secrets/runtime-degraded-state.js";
|
||||
import type { GatewayStartupLog } from "./server-startup-config-helpers.js";
|
||||
|
||||
function logSecretDegradation(log: GatewayStartupLog, degradation: SecretDegradation): void {
|
||||
const reason = redactSecretDegradationReason(degradation.reason);
|
||||
log.warn(
|
||||
`[SECRETS_DEGRADED] ${degradation.state} ${degradation.kind}:${degradation.id}: ` +
|
||||
`${reason}. Retry: ${degradation.retryHint}.`,
|
||||
{
|
||||
event: "secrets.degraded",
|
||||
ownerKind: degradation.kind,
|
||||
ownerId: degradation.id,
|
||||
reason,
|
||||
state: degradation.state,
|
||||
retryHint: degradation.retryHint,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function logSecretProviderDegradation(
|
||||
log: GatewayStartupLog,
|
||||
providerFailure: NonNullable<DegradedSecretOwner["providerFailures"]>[number],
|
||||
degradations: SecretDegradation[],
|
||||
): void {
|
||||
const reason = redactSecretDegradationReason(
|
||||
degradations[0]?.reason ?? "secret resolution failed",
|
||||
);
|
||||
const affectedOwners = degradations
|
||||
.map(({ kind, id, state }) => ({ ownerKind: kind, ownerId: id, state }))
|
||||
.toSorted(
|
||||
(left, right) =>
|
||||
left.ownerKind.localeCompare(right.ownerKind) || left.ownerId.localeCompare(right.ownerId),
|
||||
);
|
||||
const affectedOwnerSummary = affectedOwners
|
||||
.map((owner) => `${owner.state} ${owner.ownerKind}:${owner.ownerId}`)
|
||||
.join(", ");
|
||||
log.warn(
|
||||
`[SECRETS_PROVIDER_DEGRADED] ${providerFailure.source}:${providerFailure.provider}: ${reason}. ` +
|
||||
`Affected owners: ${affectedOwnerSummary}. Retry: ${SECRET_DEGRADATION_RETRY_HINT}.`,
|
||||
{
|
||||
event: "secrets.provider_degraded",
|
||||
source: providerFailure.source,
|
||||
provider: providerFailure.provider,
|
||||
reason,
|
||||
affectedOwners,
|
||||
retryHint: SECRET_DEGRADATION_RETRY_HINT,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/** Logs one provider diagnostic per failed provider and owner diagnostics for ref failures. */
|
||||
export function logPreparedSecretDegradations(
|
||||
log: GatewayStartupLog,
|
||||
owners: DegradedSecretOwner[],
|
||||
): void {
|
||||
const providerDegradations = new Map<
|
||||
string,
|
||||
{
|
||||
providerFailure: NonNullable<DegradedSecretOwner["providerFailures"]>[number];
|
||||
degradations: SecretDegradation[];
|
||||
}
|
||||
>();
|
||||
for (const owner of owners) {
|
||||
const degradation: SecretDegradation = {
|
||||
kind: owner.ownerKind,
|
||||
id: owner.ownerId,
|
||||
reason: owner.reason,
|
||||
state: owner.degradationState ?? "cold",
|
||||
retryHint: SECRET_DEGRADATION_RETRY_HINT,
|
||||
};
|
||||
if (!owner.providerFailures?.length) {
|
||||
logSecretDegradation(log, degradation);
|
||||
continue;
|
||||
}
|
||||
if (owner.refFailureReason) {
|
||||
logSecretDegradation(log, { ...degradation, reason: owner.refFailureReason });
|
||||
}
|
||||
for (const providerFailure of owner.providerFailures) {
|
||||
const key = `${providerFailure.source}\0${providerFailure.provider}`;
|
||||
const group = providerDegradations.get(key);
|
||||
if (group) {
|
||||
group.degradations.push({ ...degradation, reason: "secret provider failed" });
|
||||
} else {
|
||||
providerDegradations.set(key, {
|
||||
providerFailure,
|
||||
degradations: [{ ...degradation, reason: "secret provider failed" }],
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const group of providerDegradations.values()) {
|
||||
logSecretProviderDegradation(log, group.providerFailure, group.degradations);
|
||||
}
|
||||
}
|
||||
|
||||
/** Logs typed thrown failures with the same provider-level aggregation as prepared snapshots. */
|
||||
export function logThrownSecretDegradations(
|
||||
log: GatewayStartupLog,
|
||||
error: unknown,
|
||||
degradations: SecretDegradation[],
|
||||
): void {
|
||||
if (isProviderScopedSecretResolutionError(error)) {
|
||||
logSecretProviderDegradation(
|
||||
log,
|
||||
{ source: error.source, provider: error.provider },
|
||||
degradations,
|
||||
);
|
||||
return;
|
||||
}
|
||||
for (const degradation of degradations) {
|
||||
logSecretDegradation(log, degradation);
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,9 @@
|
||||
/** Real Gateway startup coverage for SecretRef owner isolation boundaries. */
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { createServer } from "node:http";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import { resolveDefaultAgentDir } from "../agents/agent-scope-config.js";
|
||||
import { getRuntimeAuthProfileStoreSnapshot } from "../agents/auth-profiles/runtime-snapshots.js";
|
||||
import { saveAuthProfileStore } from "../agents/auth-profiles/store.js";
|
||||
@@ -17,7 +21,65 @@ import {
|
||||
testState,
|
||||
} from "./test-helpers.js";
|
||||
|
||||
const { webSearchProviders } = vi.hoisted(() => {
|
||||
const credentialPath = "plugins.entries.google.config.webSearch.apiKey";
|
||||
return {
|
||||
webSearchProviders: [
|
||||
{
|
||||
pluginId: "google",
|
||||
id: "gemini",
|
||||
label: "Gemini",
|
||||
hint: "Gateway startup owner-isolation provider",
|
||||
envVars: ["GEMINI_API_KEY"],
|
||||
placeholder: "gemini-...",
|
||||
signupUrl: "https://example.com/gemini",
|
||||
autoDetectOrder: 20,
|
||||
credentialPath,
|
||||
inactiveSecretPaths: [credentialPath],
|
||||
getCredentialValue: (config: { apiKey?: unknown } | undefined) => config?.apiKey,
|
||||
setCredentialValue: (config: { apiKey?: unknown }, value: unknown) => {
|
||||
config.apiKey = value;
|
||||
},
|
||||
getConfiguredCredentialValue: (config: OpenClawConfig | undefined) => {
|
||||
const pluginConfig = config?.plugins?.entries?.google?.config;
|
||||
return pluginConfig && typeof pluginConfig === "object"
|
||||
? (pluginConfig as { webSearch?: { apiKey?: unknown } }).webSearch?.apiKey
|
||||
: undefined;
|
||||
},
|
||||
setConfiguredCredentialValue: () => {},
|
||||
createTool: () => null,
|
||||
},
|
||||
],
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("../secrets/runtime-web-tools-manifest.runtime.js", () => ({
|
||||
resolveManifestContractPluginIds: ({ contract }: { contract: string }) =>
|
||||
contract === "webSearchProviders" ? ["google"] : [],
|
||||
resolveManifestContractOwnerPluginId: ({ value }: { value: string }) =>
|
||||
value === "gemini" ? "google" : undefined,
|
||||
resolveManifestContractPluginIdsByCompatibilityRuntimePath: () => [],
|
||||
}));
|
||||
|
||||
vi.mock("../plugins/web-provider-public-artifacts.explicit.js", () => ({
|
||||
resolveBundledExplicitWebSearchProvidersFromPublicArtifacts: () => webSearchProviders,
|
||||
resolveBundledExplicitWebFetchProvidersFromPublicArtifacts: () => [],
|
||||
}));
|
||||
|
||||
vi.mock("../secrets/runtime-web-tools-public-artifacts.runtime.js", () => ({
|
||||
resolveBundledWebSearchProvidersFromPublicArtifacts: () => webSearchProviders,
|
||||
resolveBundledWebFetchProvidersFromPublicArtifacts: () => [],
|
||||
}));
|
||||
|
||||
vi.mock("../secrets/runtime-web-tools-fallback.runtime.js", () => ({
|
||||
runtimeWebToolsFallbackProviders: {
|
||||
resolvePluginWebSearchProviders: () => webSearchProviders,
|
||||
resolvePluginWebFetchProviders: () => [],
|
||||
},
|
||||
}));
|
||||
|
||||
installGatewayTestHooks({ scope: "suite" });
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
async function writeConfig(config: OpenClawConfig): Promise<void> {
|
||||
const { writeConfigFile } = await import("../config/config.js");
|
||||
@@ -34,6 +96,31 @@ function baseConfig(): OpenClawConfig {
|
||||
};
|
||||
}
|
||||
|
||||
async function startVaultAclFixture() {
|
||||
const requests: string[] = [];
|
||||
const vault = createServer((request, response) => {
|
||||
requests.push(request.url ?? "");
|
||||
response.setHeader("content-type", "application/json");
|
||||
response.statusCode = 403;
|
||||
response.end(JSON.stringify({ errors: ["permission denied"] }));
|
||||
});
|
||||
await new Promise<void>((resolve) => {
|
||||
vault.listen(0, "127.0.0.1", resolve);
|
||||
});
|
||||
const address = vault.address();
|
||||
if (!address || typeof address === "string") {
|
||||
throw new Error("Vault ACL fixture did not bind to a TCP port");
|
||||
}
|
||||
return {
|
||||
requests,
|
||||
vaultAddr: `http://127.0.0.1:${address.port}`,
|
||||
close: () =>
|
||||
new Promise<void>((resolve) => {
|
||||
vault.close(() => resolve());
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
describe("Gateway startup SecretRef owner isolation", () => {
|
||||
let server: Awaited<ReturnType<typeof startGatewayServer>> | undefined;
|
||||
|
||||
@@ -102,6 +189,172 @@ describe("Gateway startup SecretRef owner isolation", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("fans one Vault auth outage out to standard and web-tool owners", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
const root = tempDirs.make("openclaw-gateway-provider-outage-");
|
||||
const callLogPath = path.join(root, "calls.log");
|
||||
const commandPath = path.join(root, "provider.sh");
|
||||
const resolverPath = path.resolve("extensions/vault/vault-secret-ref-resolver.js");
|
||||
writeFileSync(
|
||||
commandPath,
|
||||
`#!/bin/sh\nprintf 'call\\n' >> ${JSON.stringify(callLogPath)}\n` +
|
||||
`exec ${JSON.stringify(process.execPath)} ${JSON.stringify(resolverPath)}\n`,
|
||||
{ encoding: "utf8", mode: 0o700 },
|
||||
);
|
||||
await withEnvAsync({ VAULT_ADDR: "https://vault.example.test" }, async () => {
|
||||
await writeConfig({
|
||||
...baseConfig(),
|
||||
secrets: {
|
||||
providers: {
|
||||
vault: { source: "exec", command: commandPath, passEnv: ["PATH", "VAULT_ADDR"] },
|
||||
},
|
||||
},
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
apiKey: { source: "exec", provider: "vault", id: "models/openai" },
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
messages: {
|
||||
tts: {
|
||||
providers: {
|
||||
elevenlabs: {
|
||||
apiKey: { source: "exec", provider: "vault", id: "tts/elevenlabs" },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
tools: { web: { search: { provider: "gemini" } } },
|
||||
plugins: {
|
||||
enabled: true,
|
||||
entries: {
|
||||
google: {
|
||||
enabled: true,
|
||||
config: {
|
||||
webSearch: {
|
||||
apiKey: { source: "exec", provider: "vault", id: "web/gemini" },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const port = await getFreePort();
|
||||
server = await startGatewayServer(port, { auth: { mode: "none" } });
|
||||
const ready = await fetch(`http://127.0.0.1:${port}/readyz`);
|
||||
|
||||
expect(ready.status).toBe(200);
|
||||
await expect(ready.json()).resolves.toMatchObject({ ready: true });
|
||||
expect(readFileSync(callLogPath, "utf8").trim().split("\n")).toHaveLength(2);
|
||||
expect(getActiveSecretsRuntimeSnapshot()?.degradedOwners).toMatchObject([
|
||||
{
|
||||
ownerKind: "provider",
|
||||
ownerId: "openai",
|
||||
providerFailures: [{ source: "exec", provider: "vault" }],
|
||||
},
|
||||
{
|
||||
ownerKind: "capability",
|
||||
ownerId: "tts",
|
||||
providerFailures: [{ source: "exec", provider: "vault" }],
|
||||
},
|
||||
{
|
||||
ownerKind: "capability",
|
||||
ownerId: "web-search:gemini",
|
||||
providerFailures: [{ source: "exec", provider: "vault" }],
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps Vault path ACL failures scoped when token introspection is denied", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
const root = tempDirs.make("openclaw-gateway-vault-acl-");
|
||||
const commandPath = path.join(root, "provider.sh");
|
||||
const resolverPath = path.resolve("extensions/vault/vault-secret-ref-resolver.js");
|
||||
writeFileSync(
|
||||
commandPath,
|
||||
`#!/bin/sh\nexec ${JSON.stringify(process.execPath)} ${JSON.stringify(resolverPath)}\n`,
|
||||
{ encoding: "utf8", mode: 0o700 },
|
||||
);
|
||||
const vault = await startVaultAclFixture();
|
||||
try {
|
||||
await withEnvAsync(
|
||||
{ VAULT_ADDR: vault.vaultAddr, VAULT_TOKEN: "not-a-real-auth-header" },
|
||||
async () => {
|
||||
await writeConfig({
|
||||
...baseConfig(),
|
||||
secrets: {
|
||||
providers: {
|
||||
vault: {
|
||||
source: "exec",
|
||||
command: commandPath,
|
||||
passEnv: ["PATH", "VAULT_ADDR", "VAULT_TOKEN"],
|
||||
},
|
||||
},
|
||||
},
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
apiKey: { source: "exec", provider: "vault", id: "models/openai" },
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
messages: {
|
||||
tts: {
|
||||
providers: {
|
||||
elevenlabs: {
|
||||
apiKey: { source: "exec", provider: "vault", id: "tts/elevenlabs" },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const port = await getFreePort();
|
||||
server = await startGatewayServer(port, { auth: { mode: "none" } });
|
||||
const ready = await fetch(`http://127.0.0.1:${port}/readyz`);
|
||||
|
||||
expect(ready.status).toBe(200);
|
||||
await expect(ready.json()).resolves.toMatchObject({ ready: true });
|
||||
const snapshot = getActiveSecretsRuntimeSnapshot();
|
||||
const degradedOwners = snapshot?.degradedOwners ?? [];
|
||||
expect(degradedOwners).toMatchObject([
|
||||
{ ownerKind: "provider", ownerId: "openai", state: "unavailable" },
|
||||
{ ownerKind: "capability", ownerId: "tts", state: "unavailable" },
|
||||
]);
|
||||
expect(degradedOwners.every((owner) => !owner.providerFailures)).toBe(true);
|
||||
expect(snapshot?.warnings).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
code: "SECRETS_OWNER_UNAVAILABLE",
|
||||
path: "models.providers.openai.apiKey",
|
||||
}),
|
||||
expect.objectContaining({
|
||||
code: "SECRETS_OWNER_UNAVAILABLE",
|
||||
path: "messages.tts.providers.elevenlabs.apiKey",
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(
|
||||
vault.requests.filter((url) => url === "/v1/auth/token/lookup-self").length,
|
||||
).toBeGreaterThan(0);
|
||||
},
|
||||
);
|
||||
} finally {
|
||||
await vault.close();
|
||||
}
|
||||
});
|
||||
|
||||
it("reaches /readyz with a cold memory provider and rejects only that owner", async () => {
|
||||
await withEnvAsync({ MISSING_MEMORY_KEY: undefined }, async () => {
|
||||
await writeConfig({
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
/** Process-local registry for SecretRef owners isolated during cold startup. */
|
||||
import type { SecretRefSource } from "../config/types.secrets.js";
|
||||
import {
|
||||
describeSecretResolutionError,
|
||||
isSecretResolutionError,
|
||||
@@ -31,6 +32,13 @@ export type DegradedSecretOwner = {
|
||||
paths: string[];
|
||||
refKeys: string[];
|
||||
reason: string;
|
||||
/** Shared provider failure that made this owner unavailable. Runtime-internal diagnostic data. */
|
||||
providerFailures?: Array<{
|
||||
source: SecretRefSource;
|
||||
provider: string;
|
||||
}>;
|
||||
/** Ref-scoped failure retained when this owner also has a provider-scoped outage. */
|
||||
refFailureReason?: string;
|
||||
};
|
||||
|
||||
/** SecretRef identities resolved for one owner in an active runtime snapshot. */
|
||||
|
||||
@@ -142,6 +142,8 @@ function createDegradedOwner(
|
||||
assignments: SecretAssignment[],
|
||||
reason: SecretDegradationReason,
|
||||
degradationState: "cold" | "stale" = "cold",
|
||||
providerFailures?: DegradedSecretOwner["providerFailures"],
|
||||
refFailureReason?: string,
|
||||
): DegradedSecretOwner {
|
||||
const owner = assignments[0]!;
|
||||
if (owner.ownerKind === "unknown") {
|
||||
@@ -155,6 +157,8 @@ function createDegradedOwner(
|
||||
paths: assignments.map((assignment) => assignment.path),
|
||||
refKeys: assignments.map((assignment) => secretRefKey(assignment.ref)),
|
||||
reason,
|
||||
...(providerFailures?.length ? { providerFailures } : {}),
|
||||
...(refFailureReason ? { refFailureReason } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -387,7 +391,14 @@ export async function resolveAndApplySecretAssignments(params: {
|
||||
);
|
||||
registerResolvedValuesForRedaction(resolution.resolved);
|
||||
|
||||
const failedOwners = new Map<SecretAssignment[], SecretDegradationReason>();
|
||||
const failedOwners = new Map<
|
||||
SecretAssignment[],
|
||||
{
|
||||
reason: SecretDegradationReason;
|
||||
providerFailures: NonNullable<DegradedSecretOwner["providerFailures"]>;
|
||||
refFailureReason?: SecretDegradationReason;
|
||||
}
|
||||
>();
|
||||
for (const failure of resolution.failures) {
|
||||
associateAssignmentFailureOwners({
|
||||
assignments: pendingOwners.flat(),
|
||||
@@ -403,8 +414,28 @@ export async function resolveAndApplySecretAssignments(params: {
|
||||
throw failure.error;
|
||||
}
|
||||
for (const assignments of matchingOwners) {
|
||||
if (!failedOwners.has(assignments)) {
|
||||
failedOwners.set(assignments, assertOwnerCanBeIsolated(assignments, failure.error));
|
||||
const reason = assertOwnerCanBeIsolated(assignments, failure.error);
|
||||
const existing = failedOwners.get(assignments);
|
||||
const providerFailure = isProviderScopedSecretResolutionError(failure.error)
|
||||
? { source: failure.error.source, provider: failure.error.provider }
|
||||
: undefined;
|
||||
if (!existing) {
|
||||
failedOwners.set(assignments, {
|
||||
reason,
|
||||
providerFailures: providerFailure ? [providerFailure] : [],
|
||||
...(!providerFailure ? { refFailureReason: reason } : {}),
|
||||
});
|
||||
} else if (
|
||||
providerFailure &&
|
||||
!existing.providerFailures.some(
|
||||
(entry) =>
|
||||
entry.source === providerFailure.source &&
|
||||
entry.provider === providerFailure.provider,
|
||||
)
|
||||
) {
|
||||
existing.providerFailures.push(providerFailure);
|
||||
} else if (!providerFailure && !existing.refFailureReason) {
|
||||
existing.refFailureReason = reason;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -437,8 +468,8 @@ export async function resolveAndApplySecretAssignments(params: {
|
||||
|
||||
const nextPendingOwners: SecretAssignment[][] = [];
|
||||
for (const assignments of pendingOwners) {
|
||||
const failureReason = failedOwners.get(assignments);
|
||||
if (failureReason) {
|
||||
const failure = failedOwners.get(assignments);
|
||||
if (failure) {
|
||||
const owner = assignments[0]!;
|
||||
let degradationState = classifySecretOwnerDegradationState({
|
||||
ownerKind: owner.ownerKind as Exclude<typeof owner.ownerKind, "unknown">,
|
||||
@@ -480,7 +511,13 @@ export async function resolveAndApplySecretAssignments(params: {
|
||||
assignment.apply({ ...assignment.ref });
|
||||
}
|
||||
}
|
||||
const degradedOwner = createDegradedOwner(assignments, failureReason, degradationState);
|
||||
const degradedOwner = createDegradedOwner(
|
||||
assignments,
|
||||
failure.refFailureReason ?? failure.reason,
|
||||
degradationState,
|
||||
failure.providerFailures,
|
||||
failure.refFailureReason,
|
||||
);
|
||||
degradedOwners.push(degradedOwner);
|
||||
warnDegradedSecretOwner(params.context, degradedOwner);
|
||||
continue;
|
||||
|
||||
@@ -0,0 +1,244 @@
|
||||
/** Tests provider-scoped SecretRef failure fan-out across runtime owners. */
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import { clearSecretsRuntimeSnapshot } from "./runtime-state.js";
|
||||
import { asConfig, setupSecretsRuntimeSnapshotTestHooks } from "./runtime.test-support.js";
|
||||
|
||||
const EMPTY_LOADABLE_PLUGIN_ORIGINS = new Map();
|
||||
const { prepareSecretsRuntimeSnapshot } = setupSecretsRuntimeSnapshotTestHooks();
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
const EXEC_FIXTURE_TIMEOUT_MS = 20_000;
|
||||
|
||||
function execFixtureProvider(command: string) {
|
||||
return {
|
||||
source: "exec" as const,
|
||||
command,
|
||||
passEnv: ["PATH"],
|
||||
timeoutMs: EXEC_FIXTURE_TIMEOUT_MS,
|
||||
noOutputTimeoutMs: EXEC_FIXTURE_TIMEOUT_MS,
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
clearSecretsRuntimeSnapshot();
|
||||
});
|
||||
|
||||
describe("provider-scoped SecretRef failure fan-out", () => {
|
||||
it("preserves provider provenance for an unavailable web-search owner", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
const root = tempDirs.make("openclaw-web-secret-provider-failure-");
|
||||
const commandPath = path.join(root, "provider.sh");
|
||||
await fs.writeFile(commandPath, "#!/bin/sh\nexit 1\n", {
|
||||
encoding: "utf8",
|
||||
mode: 0o700,
|
||||
});
|
||||
const ref = { source: "exec" as const, provider: "vault", id: "web/gemini" };
|
||||
|
||||
const snapshot = await prepareSecretsRuntimeSnapshot({
|
||||
config: asConfig({
|
||||
secrets: {
|
||||
providers: {
|
||||
vault: execFixtureProvider(commandPath),
|
||||
},
|
||||
},
|
||||
tools: { web: { search: { provider: "gemini" } } },
|
||||
plugins: {
|
||||
entries: {
|
||||
google: { config: { webSearch: { apiKey: ref } } },
|
||||
},
|
||||
},
|
||||
}),
|
||||
env: { PATH: process.env.PATH ?? "" },
|
||||
includeAuthStoreRefs: false,
|
||||
allowUnavailableSecretOwners: true,
|
||||
loadablePluginOrigins: EMPTY_LOADABLE_PLUGIN_ORIGINS,
|
||||
});
|
||||
|
||||
expect(snapshot.degradedOwners).toContainEqual(
|
||||
expect.objectContaining({
|
||||
ownerKind: "capability",
|
||||
ownerId: "web-search:gemini",
|
||||
reason: "secret provider failed",
|
||||
providerFailures: [{ source: "exec", provider: "vault" }],
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("reuses one provider failure across isolated owners", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
const root = tempDirs.make("openclaw-owner-secret-provider-failure-");
|
||||
const callLogPath = path.join(root, "calls.log");
|
||||
const commandPath = path.join(root, "provider.sh");
|
||||
await fs.writeFile(
|
||||
commandPath,
|
||||
`#!/bin/sh\nprintf 'call\\n' >> ${JSON.stringify(callLogPath)}\nexit 1\n`,
|
||||
{ encoding: "utf8", mode: 0o700 },
|
||||
);
|
||||
const input = {
|
||||
modelRef: { source: "exec" as const, provider: "shared", id: "models/openai" },
|
||||
ttsRef: { source: "exec" as const, provider: "shared", id: "tts/elevenlabs" },
|
||||
};
|
||||
|
||||
const snapshot = await prepareSecretsRuntimeSnapshot({
|
||||
config: asConfig({
|
||||
secrets: {
|
||||
providers: {
|
||||
shared: execFixtureProvider(commandPath),
|
||||
},
|
||||
},
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
apiKey: input.modelRef,
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
messages: {
|
||||
tts: { providers: { elevenlabs: { apiKey: input.ttsRef } } },
|
||||
},
|
||||
}),
|
||||
env: { PATH: process.env.PATH ?? "" },
|
||||
includeAuthStoreRefs: false,
|
||||
allowUnavailableSecretOwners: true,
|
||||
loadablePluginOrigins: EMPTY_LOADABLE_PLUGIN_ORIGINS,
|
||||
});
|
||||
|
||||
expect(snapshot.config.models?.providers?.openai?.apiKey).toEqual(input.modelRef);
|
||||
expect(snapshot.config.messages?.tts?.providers?.elevenlabs?.apiKey).toEqual(input.ttsRef);
|
||||
expect(snapshot.degradedOwners).toMatchObject([
|
||||
{
|
||||
ownerKind: "provider",
|
||||
ownerId: "openai",
|
||||
reason: "secret provider failed",
|
||||
providerFailures: [{ source: "exec", provider: "shared" }],
|
||||
},
|
||||
{
|
||||
ownerKind: "capability",
|
||||
ownerId: "tts",
|
||||
reason: "secret provider failed",
|
||||
providerFailures: [{ source: "exec", provider: "shared" }],
|
||||
},
|
||||
]);
|
||||
expect((await fs.readFile(callLogPath, "utf8")).trim().split("\n")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("records every failed provider used by one isolated owner", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
const root = tempDirs.make("openclaw-owner-multi-provider-failure-");
|
||||
const command = async (name: string) => {
|
||||
const commandPath = path.join(root, `${name}.sh`);
|
||||
await fs.writeFile(commandPath, "#!/bin/sh\nexit 1\n", {
|
||||
encoding: "utf8",
|
||||
mode: 0o700,
|
||||
});
|
||||
return commandPath;
|
||||
};
|
||||
const firstCommand = await command("first");
|
||||
const secondCommand = await command("second");
|
||||
|
||||
const snapshot = await prepareSecretsRuntimeSnapshot({
|
||||
config: asConfig({
|
||||
secrets: {
|
||||
providers: {
|
||||
first: execFixtureProvider(firstCommand),
|
||||
second: execFixtureProvider(secondCommand),
|
||||
},
|
||||
},
|
||||
models: {
|
||||
providers: {
|
||||
example: {
|
||||
apiKey: { source: "exec", provider: "first", id: "api-key" },
|
||||
headers: {
|
||||
"X-Secondary": { source: "exec", provider: "second", id: "secondary" },
|
||||
},
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
env: { PATH: process.env.PATH ?? "" },
|
||||
includeAuthStoreRefs: false,
|
||||
allowUnavailableSecretOwners: true,
|
||||
loadablePluginOrigins: EMPTY_LOADABLE_PLUGIN_ORIGINS,
|
||||
});
|
||||
|
||||
expect(snapshot.degradedOwners).toMatchObject([
|
||||
{
|
||||
ownerKind: "provider",
|
||||
ownerId: "example",
|
||||
providerFailures: [
|
||||
{ source: "exec", provider: "first" },
|
||||
{ source: "exec", provider: "second" },
|
||||
],
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("retains a ref failure when the same owner also has a provider outage", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
const root = tempDirs.make("openclaw-owner-mixed-provider-failure-");
|
||||
const providerCommand = path.join(root, "provider.sh");
|
||||
const refCommand = path.join(root, "ref.sh");
|
||||
await fs.writeFile(providerCommand, "#!/bin/sh\nexit 1\n", {
|
||||
encoding: "utf8",
|
||||
mode: 0o700,
|
||||
});
|
||||
const refResponse = JSON.stringify({
|
||||
protocolVersion: 1,
|
||||
values: {},
|
||||
errors: { secondary: { code: "NOT_FOUND" } },
|
||||
});
|
||||
await fs.writeFile(refCommand, `#!/bin/sh\nprintf '%s\\n' ${JSON.stringify(refResponse)}\n`, {
|
||||
encoding: "utf8",
|
||||
mode: 0o700,
|
||||
});
|
||||
|
||||
const snapshot = await prepareSecretsRuntimeSnapshot({
|
||||
config: asConfig({
|
||||
secrets: {
|
||||
providers: {
|
||||
unavailable: execFixtureProvider(providerCommand),
|
||||
partial: execFixtureProvider(refCommand),
|
||||
},
|
||||
},
|
||||
models: {
|
||||
providers: {
|
||||
example: {
|
||||
apiKey: { source: "exec", provider: "unavailable", id: "api-key" },
|
||||
headers: {
|
||||
"X-Secondary": { source: "exec", provider: "partial", id: "secondary" },
|
||||
},
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
env: { PATH: process.env.PATH ?? "" },
|
||||
includeAuthStoreRefs: false,
|
||||
allowUnavailableSecretOwners: true,
|
||||
loadablePluginOrigins: EMPTY_LOADABLE_PLUGIN_ORIGINS,
|
||||
});
|
||||
|
||||
expect(snapshot.degradedOwners).toMatchObject([
|
||||
{
|
||||
ownerKind: "provider",
|
||||
ownerId: "example",
|
||||
reason: "secret reference was not found",
|
||||
providerFailures: [{ source: "exec", provider: "unavailable" }],
|
||||
refFailureReason: "secret reference was not found",
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -187,6 +187,12 @@ function cloneDegradedSecretOwner(owner: DegradedSecretOwner): DegradedSecretOwn
|
||||
if (owner.degradationState) {
|
||||
cloned.degradationState = owner.degradationState;
|
||||
}
|
||||
if (owner.providerFailures) {
|
||||
cloned.providerFailures = owner.providerFailures.map((failure) => ({ ...failure }));
|
||||
}
|
||||
if (owner.refFailureReason) {
|
||||
cloned.refFailureReason = owner.refFailureReason;
|
||||
}
|
||||
return cloned;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
/** Tests warning suppression when provider failures have aggregate Gateway diagnostics. */
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { logRuntimeSecretWarnings } from "./runtime-warning-log.js";
|
||||
|
||||
function snapshot(refFailureReason?: string) {
|
||||
const path = "models.providers.example.apiKey";
|
||||
return {
|
||||
warnings: [
|
||||
{
|
||||
code: "SECRETS_OWNER_UNAVAILABLE" as const,
|
||||
path,
|
||||
message: "Secret owner provider:example is configured-unavailable.",
|
||||
},
|
||||
],
|
||||
degradedOwners: [
|
||||
{
|
||||
ownerKind: "provider" as const,
|
||||
ownerId: "example",
|
||||
state: "unavailable" as const,
|
||||
paths: [path],
|
||||
refKeys: ["exec:vault:api-key"],
|
||||
reason: refFailureReason ?? "secret provider failed",
|
||||
providerFailures: [{ source: "exec" as const, provider: "vault" }],
|
||||
...(refFailureReason ? { refFailureReason } : {}),
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe("runtime SecretRef warning logging", () => {
|
||||
it("suppresses per-owner warnings for a pure provider outage", () => {
|
||||
const warn = vi.fn();
|
||||
logRuntimeSecretWarnings({ snapshot: snapshot(), log: { warn }, ownerUnavailable: "include" });
|
||||
expect(warn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("retains per-owner warnings when a ref failure is also present", () => {
|
||||
const warn = vi.fn();
|
||||
logRuntimeSecretWarnings({
|
||||
snapshot: snapshot("secret reference was not found"),
|
||||
log: { warn },
|
||||
ownerUnavailable: "include",
|
||||
});
|
||||
expect(warn).toHaveBeenCalledOnce();
|
||||
});
|
||||
});
|
||||
@@ -8,6 +8,11 @@ export function logRuntimeSecretWarnings(params: {
|
||||
log: { warn: (message: string) => void };
|
||||
ownerUnavailable: OwnerUnavailableWarningMode;
|
||||
}): void {
|
||||
const providerFailurePaths = new Set(
|
||||
(params.snapshot.degradedOwners ?? []).flatMap((owner) =>
|
||||
owner.providerFailures?.length && !owner.refFailureReason ? owner.paths : [],
|
||||
),
|
||||
);
|
||||
const activeDegradedPaths =
|
||||
params.ownerUnavailable === "active-only"
|
||||
? new Set((params.snapshot.degradedOwners ?? []).flatMap((owner) => owner.paths))
|
||||
@@ -20,6 +25,10 @@ export function logRuntimeSecretWarnings(params: {
|
||||
if (activeDegradedPaths && !activeDegradedPaths.has(warning.path)) {
|
||||
continue;
|
||||
}
|
||||
// Provider-scoped outages are published once with their complete affected-owner list.
|
||||
if (providerFailurePaths.has(warning.path)) {
|
||||
continue;
|
||||
}
|
||||
} else if (params.ownerUnavailable === "active-only") {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/** Typed credential ownership and unavailable-provider results for runtime web tools. */
|
||||
import type { SecretRef } from "../config/types.secrets.js";
|
||||
import type { SecretRef, SecretRefSource } from "../config/types.secrets.js";
|
||||
import type { SecretDegradationReason } from "./runtime-degraded-state.js";
|
||||
import type { SecretResolverWarningCode } from "./runtime-shared.js";
|
||||
import type { RuntimeWebDiagnosticCode } from "./runtime-web-tools.types.js";
|
||||
@@ -32,6 +32,7 @@ export type RuntimeWebSecretOwner = {
|
||||
contractDigest: string;
|
||||
resolvedValue?: string;
|
||||
reason?: SecretDegradationReason;
|
||||
providerFailure?: { source: SecretRefSource; provider: string };
|
||||
restoreResolvedValue?: (value: string) => void;
|
||||
};
|
||||
|
||||
|
||||
@@ -19,7 +19,10 @@ import { sortWebSearchProvidersForAutoDetect } from "../plugins/web-search-provi
|
||||
import { createLazyRuntimeSurface } from "../shared/lazy-runtime.js";
|
||||
import { normalizeSecretInput } from "../utils/normalize-secret-input.js";
|
||||
import { secretRefKey } from "./ref-contract.js";
|
||||
import { describeSecretResolutionError } from "./resolve-errors.js";
|
||||
import {
|
||||
describeSecretResolutionError,
|
||||
isProviderScopedSecretResolutionError,
|
||||
} from "./resolve-errors.js";
|
||||
import { resolveSecretRefValues } from "./resolve.js";
|
||||
import {
|
||||
associateSecretResolutionErrorOwners,
|
||||
@@ -87,10 +90,17 @@ type ResolvedRuntimeWebTools = {
|
||||
type RuntimeWebProviderFailure = Omit<RuntimeWebUnavailableProvider, "contractDigest"> & {
|
||||
contractDigest?: string;
|
||||
};
|
||||
type RuntimeWebProviderFailureByRefKey = Map<
|
||||
string,
|
||||
NonNullable<RuntimeWebUnavailableProvider["providerFailure"]>
|
||||
>;
|
||||
|
||||
function createUnavailableWebProviderOwner(params: {
|
||||
kind: "search" | "fetch";
|
||||
unavailable: Pick<RuntimeWebUnavailableProvider, "providerId" | "path" | "refKey" | "reason">;
|
||||
unavailable: Pick<
|
||||
RuntimeWebUnavailableProvider,
|
||||
"providerId" | "path" | "refKey" | "reason" | "providerFailure"
|
||||
>;
|
||||
degradationState?: "cold" | "stale";
|
||||
}): DegradedSecretOwner {
|
||||
return {
|
||||
@@ -101,9 +111,21 @@ function createUnavailableWebProviderOwner(params: {
|
||||
paths: [params.unavailable.path],
|
||||
refKeys: [params.unavailable.refKey],
|
||||
reason: params.unavailable.reason,
|
||||
...(params.unavailable.providerFailure
|
||||
? { providerFailures: [params.unavailable.providerFailure] }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
function attachWebProviderFailures(
|
||||
unavailableProviders: RuntimeWebProviderFailure[],
|
||||
providerFailuresByRefKey: RuntimeWebProviderFailureByRefKey,
|
||||
): void {
|
||||
for (const unavailable of unavailableProviders) {
|
||||
unavailable.providerFailure = providerFailuresByRefKey.get(unavailable.refKey);
|
||||
}
|
||||
}
|
||||
|
||||
function collectUnavailableWebProviders(params: {
|
||||
kind: "search" | "fetch";
|
||||
result: RuntimeWebProviderSelectionResult;
|
||||
@@ -240,6 +262,7 @@ function associateWebProviderResolutionError(params: {
|
||||
}),
|
||||
failureMatched: true,
|
||||
source: "config" as const,
|
||||
...(firstMatch.providerFailure ? { providerFailures: [firstMatch.providerFailure] } : {}),
|
||||
},
|
||||
];
|
||||
},
|
||||
@@ -392,6 +415,7 @@ async function resolveSecretInputWithEnvFallback(params: {
|
||||
path: string;
|
||||
envVars: string[];
|
||||
contractDigest: string;
|
||||
providerFailuresByRefKey: RuntimeWebProviderFailureByRefKey;
|
||||
restrictEnvRefsToEnvVars?: boolean;
|
||||
}): Promise<SecretResolutionResult<SecretResolutionSource>> {
|
||||
const { ref } = resolveSecretInputRef({
|
||||
@@ -486,6 +510,12 @@ async function resolveSecretInputWithEnvFallback(params: {
|
||||
throw error;
|
||||
}
|
||||
unresolvedRefReason = reason;
|
||||
if (isProviderScopedSecretResolutionError(error)) {
|
||||
params.providerFailuresByRefKey.set(secretRefKey(ref), {
|
||||
source: error.source,
|
||||
provider: error.provider,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -710,6 +740,7 @@ export async function resolveRuntimeWebTools(params: {
|
||||
const diagnostics: RuntimeWebDiagnostic[] = [];
|
||||
const degradedOwners: DegradedSecretOwner[] = [];
|
||||
const secretOwners: SecretOwnerRefState[] = [];
|
||||
const providerFailuresByRefKey: RuntimeWebProviderFailureByRefKey = new Map();
|
||||
const finish = (metadata: RuntimeWebToolsMetadata): ResolvedRuntimeWebTools => ({
|
||||
metadata,
|
||||
degradedOwners,
|
||||
@@ -762,6 +793,7 @@ export async function resolveRuntimeWebTools(params: {
|
||||
value: legacyXSearchSourceRecord.apiKey,
|
||||
path: "tools.web.x_search.apiKey",
|
||||
envVars: ["XAI_API_KEY"],
|
||||
providerFailuresByRefKey,
|
||||
contractDigest: digestRuntimeWebOwnerContract({
|
||||
scopePath: "tools.web.x_search",
|
||||
configuredProvider: "grok",
|
||||
@@ -877,13 +909,15 @@ export async function resolveRuntimeWebTools(params: {
|
||||
allowKeylessAutoSelect: false,
|
||||
deferKeylessFallback: true,
|
||||
allowUnavailableProviders: params.allowUnavailableSecretOwners,
|
||||
onUnavailableProviders: (error) =>
|
||||
onUnavailableProviders: (error) => {
|
||||
attachWebProviderFailures(error.unavailableProviders, providerFailuresByRefKey);
|
||||
associateWebProviderResolutionError({
|
||||
kind: "search",
|
||||
config: params.sourceConfig,
|
||||
error,
|
||||
unavailableProviders: error.unavailableProviders,
|
||||
}),
|
||||
});
|
||||
},
|
||||
noFallbackCode: "WEB_SEARCH_KEY_UNRESOLVED_NO_FALLBACK",
|
||||
autoDetectSelectedCode: "WEB_SEARCH_AUTODETECT_SELECTED",
|
||||
readConfiguredCredential: ({ provider, config, toolConfig }) =>
|
||||
@@ -909,6 +943,7 @@ export async function resolveRuntimeWebTools(params: {
|
||||
path,
|
||||
envVars,
|
||||
contractDigest,
|
||||
providerFailuresByRefKey,
|
||||
}),
|
||||
setResolvedCredential: ({ resolvedConfig, provider, value }) =>
|
||||
setResolvedWebSearchApiKey({
|
||||
@@ -939,6 +974,7 @@ export async function resolveRuntimeWebTools(params: {
|
||||
);
|
||||
},
|
||||
});
|
||||
attachWebProviderFailures(searchSelection.unavailableProviders, providerFailuresByRefKey);
|
||||
collectUnavailableWebProviders({
|
||||
kind: "search",
|
||||
result: searchSelection,
|
||||
@@ -1011,13 +1047,15 @@ export async function resolveRuntimeWebTools(params: {
|
||||
allowKeylessAutoSelect: true,
|
||||
deferKeylessFallback: false,
|
||||
allowUnavailableProviders: params.allowUnavailableSecretOwners,
|
||||
onUnavailableProviders: (error) =>
|
||||
onUnavailableProviders: (error) => {
|
||||
attachWebProviderFailures(error.unavailableProviders, providerFailuresByRefKey);
|
||||
associateWebProviderResolutionError({
|
||||
kind: "fetch",
|
||||
config: params.sourceConfig,
|
||||
error,
|
||||
unavailableProviders: error.unavailableProviders,
|
||||
}),
|
||||
});
|
||||
},
|
||||
noFallbackCode: "WEB_FETCH_PROVIDER_KEY_UNRESOLVED_NO_FALLBACK",
|
||||
autoDetectSelectedCode: "WEB_FETCH_AUTODETECT_SELECTED",
|
||||
readConfiguredCredential: ({ provider, config, toolConfig }) =>
|
||||
@@ -1043,6 +1081,7 @@ export async function resolveRuntimeWebTools(params: {
|
||||
path,
|
||||
envVars,
|
||||
contractDigest,
|
||||
providerFailuresByRefKey,
|
||||
restrictEnvRefsToEnvVars: true,
|
||||
}),
|
||||
setResolvedCredential: ({ resolvedConfig, provider, value }) =>
|
||||
@@ -1074,6 +1113,7 @@ export async function resolveRuntimeWebTools(params: {
|
||||
);
|
||||
},
|
||||
});
|
||||
attachWebProviderFailures(fetchSelection.unavailableProviders, providerFailuresByRefKey);
|
||||
collectUnavailableWebProviders({
|
||||
kind: "fetch",
|
||||
result: fetchSelection,
|
||||
|
||||
@@ -797,58 +797,6 @@ describe("secrets runtime snapshot", () => {
|
||||
).rejects.toThrow("not allowlisted");
|
||||
});
|
||||
|
||||
it("reuses provider-scoped failures across isolated owners", async () => {
|
||||
if (process.platform === "win32") {
|
||||
return;
|
||||
}
|
||||
const root = tempDirs.make("openclaw-owner-secret-provider-failure-");
|
||||
const callLogPath = path.join(root, "calls.log");
|
||||
const commandPath = path.join(root, "provider.sh");
|
||||
await fs.writeFile(
|
||||
commandPath,
|
||||
`#!/bin/sh\nprintf 'call\\n' >> ${JSON.stringify(callLogPath)}\nexit 1\n`,
|
||||
{ encoding: "utf8", mode: 0o700 },
|
||||
);
|
||||
const input = {
|
||||
modelRef: { source: "exec" as const, provider: "shared", id: "models/openai" },
|
||||
ttsRef: { source: "exec" as const, provider: "shared", id: "tts/elevenlabs" },
|
||||
};
|
||||
|
||||
const snapshot = await prepareSecretsRuntimeSnapshot({
|
||||
config: asConfig({
|
||||
secrets: {
|
||||
providers: {
|
||||
shared: { source: "exec", command: commandPath, passEnv: ["PATH"] },
|
||||
},
|
||||
},
|
||||
models: {
|
||||
providers: {
|
||||
openai: {
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
apiKey: input.modelRef,
|
||||
models: [],
|
||||
},
|
||||
},
|
||||
},
|
||||
messages: {
|
||||
tts: { providers: { elevenlabs: { apiKey: input.ttsRef } } },
|
||||
},
|
||||
}),
|
||||
env: { PATH: process.env.PATH ?? "" },
|
||||
includeAuthStoreRefs: false,
|
||||
allowUnavailableSecretOwners: true,
|
||||
loadablePluginOrigins: EMPTY_LOADABLE_PLUGIN_ORIGINS,
|
||||
});
|
||||
|
||||
expect(snapshot.config.models?.providers?.openai?.apiKey).toEqual(input.modelRef);
|
||||
expect(snapshot.config.messages?.tts?.providers?.elevenlabs?.apiKey).toEqual(input.ttsRef);
|
||||
expect(snapshot.degradedOwners).toMatchObject([
|
||||
{ ownerKind: "provider", ownerId: "openai", reason: "secret provider failed" },
|
||||
{ ownerKind: "capability", ownerId: "tts", reason: "secret provider failed" },
|
||||
]);
|
||||
expect((await fs.readFile(callLogPath, "utf8")).trim().split("\n")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("keeps invalid TTS SecretRef ids fail-closed", async () => {
|
||||
await expect(
|
||||
prepareSecretsRuntimeSnapshot({
|
||||
|
||||
Reference in New Issue
Block a user