fix(qa): repair WhatsApp live scenario regressions (#110754)

* fix(qa): make WhatsApp scenarios account-aware

* test(qa): cover configured WhatsApp SUT account

* refactor(qa): centralize QA target encoding

* fix(qa): configure scenario policies at startup
This commit is contained in:
Marcus Castro
2026-07-19 01:43:38 -03:00
committed by GitHub
parent 721aee1f19
commit d5cb708623
13 changed files with 270 additions and 67 deletions
+6 -13
View File
@@ -3,7 +3,11 @@ import http from "node:http";
import https from "node:https";
import { resolvePositiveTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime";
import { readProviderJsonResponse } from "openclaw/plugin-sdk/provider-http";
import { parseQaTarget, type QaTargetParts } from "openclaw/plugin-sdk/qa-channel-protocol";
import {
buildQaTarget,
parseQaTarget,
type QaTargetParts,
} from "openclaw/plugin-sdk/qa-channel-protocol";
import { readByteStreamWithLimit } from "openclaw/plugin-sdk/response-limit-runtime";
import { fetchWithSsrFGuard } from "openclaw/plugin-sdk/ssrf-runtime";
import type {
@@ -16,7 +20,7 @@ import type {
QaBusToolCall,
} from "./protocol.js";
export { parseQaTarget };
export { buildQaTarget, parseQaTarget };
export type {
QaBusAttachment,
@@ -152,17 +156,6 @@ export function resolveQaTargetThread(params: {
};
}
export function buildQaTarget(params: {
chatType: "direct" | "channel" | "group";
conversationId: string;
threadId?: string | null;
}) {
if (params.threadId) {
return `thread:${params.conversationId}/${params.threadId}`;
}
return `${params.chatType === "direct" ? "dm" : params.chatType}:${params.conversationId}`;
}
export async function pollQaBus(params: {
baseUrl: string;
accountId: string;
@@ -2,7 +2,7 @@
import { randomUUID } from "node:crypto";
import path from "node:path";
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import { buildQaTarget } from "openclaw/plugin-sdk/qa-channel";
import { buildQaTarget } from "openclaw/plugin-sdk/qa-channel-protocol";
import type { QaRunnerCliRegistration } from "openclaw/plugin-sdk/qa-runner-runtime";
import { readQaScenarioExecutionConfig } from "../../scenario-catalog.js";
import { createMatrixQaScenarioEnvironment } from "./scenarios/scenario-environment.js";
@@ -3,7 +3,7 @@ import fs from "node:fs/promises";
import path from "node:path";
import type { WhatsAppQaDriverSession } from "@openclaw/whatsapp/api.js";
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import { buildQaTarget } from "openclaw/plugin-sdk/qa-channel";
import { buildQaTarget } from "openclaw/plugin-sdk/qa-channel-protocol";
import type { QaRunnerCliRegistration } from "openclaw/plugin-sdk/qa-runner-runtime";
import { resolvePreferredOpenClawTmpDir } from "openclaw/plugin-sdk/temp-path";
import {
@@ -193,6 +193,7 @@ export async function createWhatsAppQaTransportAdapter(
authDir: sutAuthDir,
dmPolicy: "allowlist",
groupJid: runtimeEnv.groupJid,
ownerAllowFrom: [runtimeEnv.driverPhoneE164],
overrides: options.transportPolicy?.topLevelReplies ? { replyToMode: "off" } : undefined,
sutAccountId: accountId,
}),
@@ -30,6 +30,19 @@ export type WhatsAppQaScenarioEnvironment = {
sutAuthDir: string;
};
function resolveWhatsAppQaReplacePaths(accountId: string): string[] {
return [
"agents",
"approvals",
"broadcast",
"channels.whatsapp",
`channels.whatsapp.accounts.${accountId}.allowFrom`,
"messages",
"plugins",
"tools",
];
}
export function createWhatsAppQaScenarioEnvironment(params: {
accountId: string;
driverAuthDir: string;
@@ -44,7 +57,7 @@ export function createWhatsAppQaScenarioEnvironment(params: {
const prepareFlow = async (input: FlowPreparationInput) => {
const scenarioId = input.config.whatsappScenarioId;
if (typeof scenarioId !== "string") {
throw new Error("WhatsApp QA module flow requires config.whatsappScenarioId");
return undefined;
}
const scenario = getWhatsAppQaScenarioDefinition(scenarioId);
if (scenario.requiresGroupJid && !params.runtimeEnv.groupJid) {
@@ -86,21 +99,14 @@ export function createWhatsAppQaScenarioEnvironment(params: {
authDir: params.sutAuthDir,
dmPolicy,
groupJid,
ownerAllowFrom: [params.runtimeEnv.driverPhoneE164],
overrides: scenario.configOverrides,
sutAccountId: params.accountId,
});
await patchLiveQaGatewayConfig({
gateway: input.gateway,
patch: cfg as Record<string, unknown>,
replacePaths: [
"agents",
"approvals",
"broadcast",
"channels.whatsapp",
"messages",
"plugins",
"tools",
],
replacePaths: resolveWhatsAppQaReplacePaths(params.accountId),
timeoutMs: input.timeoutMs,
waitForConfigRestartSettle: input.waitForConfigRestartSettle,
});
@@ -163,6 +163,7 @@ export function buildWhatsAppQaConfig(
authDir: string;
dmPolicy: "allowlist" | "disabled" | "open" | "pairing";
groupJid?: string;
ownerAllowFrom: string[];
overrides?: WhatsAppQaConfigOverrides;
sutAccountId: string;
},
@@ -249,6 +250,13 @@ export function buildWhatsAppQaConfig(
...audioPreflightConfig,
...broadcastConfig,
...actionToolConfig,
commands: {
...baseCfg.commands,
ownerAllowFrom: uniqueStrings([
...normalizeStringEntries(baseCfg.commands?.ownerAllowFrom),
...params.ownerAllowFrom,
]),
},
plugins: {
...baseCfg.plugins,
allow: pluginAllow,
@@ -5,12 +5,16 @@ import os from "node:os";
import path from "node:path";
import { promisify } from "node:util";
import { expectDefined } from "@openclaw/normalization-core";
import type {
WhatsAppQaDriverObservedMessage,
WhatsAppQaDriverSession,
import {
resolveWhatsAppAccount,
type WhatsAppQaDriverObservedMessage,
type WhatsAppQaDriverSession,
} from "@openclaw/whatsapp/api.js";
import { describe, expect, it, vi } from "vitest";
import { fingerprintQaCredentialId } from "../../qa-credentials-fingerprint.runtime.js";
import { readQaScenarioById } from "../../scenario-catalog.js";
import { applyQaMergePatch, collectQaSuiteGatewayConfigPatch } from "../../suite-planning.js";
import { createWhatsAppQaScenarioEnvironment } from "./scenario-environment.js";
import { resolveWhatsAppQaScenarioIds } from "./scenario-selection.js";
import { runWhatsAppApprovalScenario } from "./whatsapp-live.approvals.js";
import { buildWhatsAppQaConfig, parseWhatsAppQaCredentialPayload } from "./whatsapp-live.config.js";
@@ -149,6 +153,7 @@ function buildWhatsAppQaConfigFixture(
allowFrom: ["+15550000001"],
authDir: "/tmp/openclaw-whatsapp-qa-auth",
dmPolicy: "allowlist",
ownerAllowFrom: ["+15550000001"],
sutAccountId: "sut",
...options,
});
@@ -1106,11 +1111,185 @@ describe("WhatsApp QA live runtime", () => {
});
const account = cfg.channels?.whatsapp?.accounts?.sut;
expect(account?.allowFrom).toEqual(["+15550000001"]);
expect(cfg.commands?.ownerAllowFrom).toEqual(["+15550000001"]);
expect(account?.groupPolicy).toBe("open");
expect(account?.groups?.[groupJid]?.requireMention).toBe(true);
}
});
it("authorizes the exact active WhatsApp account allowlist replacement path", async () => {
const gatewayCall = vi.fn(async (method: string, _params?: unknown) => {
if (method === "config.get") {
return { config: {}, hash: "config-hash" };
}
if (method === "config.patch") {
return { noop: true };
}
if (method === "channels.status") {
return {
channelAccounts: {
whatsapp: [
{
accountId: "work",
busy: false,
connected: true,
lastConnectedAt: Date.now() - 30_000,
restartPending: false,
running: true,
},
],
},
};
}
throw new Error(`unexpected gateway method: ${method}`);
});
const { prepareFlow } = createWhatsAppQaScenarioEnvironment({
accountId: "work",
driverAuthDir: "/tmp/whatsapp-driver",
explicitScenarioSelection: true,
getDriver: vi.fn(() => undefined as never),
replaceDriver: vi.fn(),
runtimeEnv: {
driverAuthArchiveBase64: "driver-auth",
driverPhoneE164: "+15550000001",
sutAuthArchiveBase64: "sut-auth",
sutPhoneE164: "+15550000002",
},
sutAuthDir: "/tmp/whatsapp-sut",
});
await prepareFlow({
config: { whatsappScenarioId: "whatsapp-canary" },
gateway: { call: gatewayCall } as never,
outputDir: "/tmp/whatsapp-output",
primaryModel: "mock-openai/gpt-5.6-luna",
timeoutMs: 60_000,
waitForConfigRestartSettle: vi.fn(),
});
const patchCall = gatewayCall.mock.calls.find(([method]) => method === "config.patch");
if (!patchCall) {
throw new Error("config.patch was not called");
}
expect(patchCall[1]).toMatchObject({
replacePaths: expect.arrayContaining(["channels.whatsapp.accounts.work.allowFrom"]),
});
expect((patchCall[1] as { replacePaths?: string[] }).replacePaths).not.toContain(
"channels.whatsapp.accounts.sut.allowFrom",
);
});
it("leaves generic declarative flows to their own config preparation", async () => {
const gatewayCall = vi.fn();
const { prepareFlow } = createWhatsAppQaScenarioEnvironment({
accountId: "work",
driverAuthDir: "/tmp/whatsapp-driver",
explicitScenarioSelection: true,
getDriver: vi.fn(() => undefined as never),
replaceDriver: vi.fn(),
runtimeEnv: {
driverAuthArchiveBase64: "driver-auth",
driverPhoneE164: "+15550000001",
sutAuthArchiveBase64: "sut-auth",
sutPhoneE164: "+15550000002",
},
sutAuthDir: "/tmp/whatsapp-sut",
});
await expect(
prepareFlow({
config: { policyKey: "dmPolicy", policyValue: "disabled" },
gateway: { call: gatewayCall } as never,
outputDir: "/tmp/whatsapp-output",
primaryModel: "mock-openai/gpt-5.6-luna",
timeoutMs: 60_000,
waitForConfigRestartSettle: vi.fn(),
}),
).resolves.toBeUndefined();
expect(gatewayCall).not.toHaveBeenCalled();
});
it("patches the effective WhatsApp policy for default and named SUT accounts", async () => {
const policyScenarios = [
{
id: "whatsapp-access-control-dm-disabled",
policyKey: "dmPolicy",
policyValue: "disabled",
staleValue: "allowlist",
},
{
id: "whatsapp-access-control-dm-open",
policyKey: "dmPolicy",
policyValue: "open",
staleValue: "allowlist",
},
{
id: "whatsapp-access-control-group-disabled",
policyKey: "groupPolicy",
policyValue: "disabled",
staleValue: "open",
},
{
id: "whatsapp-access-control-group-open",
policyKey: "groupPolicy",
policyValue: "open",
staleValue: "disabled",
},
{
id: "whatsapp-pairing-block",
policyKey: "dmPolicy",
policyValue: "pairing",
staleValue: "allowlist",
},
] as const;
for (const accountId of ["default", "work"]) {
for (const policyScenario of policyScenarios) {
const staleAccount = {
[policyScenario.policyKey]: policyScenario.staleValue,
};
const initialConfig = buildWhatsAppQaConfigFixture(
{ sutAccountId: accountId },
{
channels: {
whatsapp: {
accounts: { [accountId]: staleAccount },
},
},
},
);
const scenario = readQaScenarioById(policyScenario.id);
const flow = scenario.execution.kind === "flow" ? scenario.execution.flow : undefined;
expect(JSON.stringify(flow), policyScenario.id).not.toContain('"patchConfig"');
const startupPatch = collectQaSuiteGatewayConfigPatch([scenario], accountId);
const patchedConfig = applyQaMergePatch(
initialConfig,
startupPatch ?? {},
) as WhatsAppQaConfigBase;
const effective = resolveWhatsAppAccount({ cfg: patchedConfig, accountId });
expect(
effective[policyScenario.policyKey],
`${policyScenario.id}:${accountId}:effective`,
).toBe(policyScenario.policyValue);
if (policyScenario.id === "whatsapp-pairing-block") {
expect(effective.allowFrom).toEqual(["+15550000000"]);
}
}
}
});
it("preserves configured command owners while adding the WhatsApp QA driver", () => {
const cfg = buildWhatsAppQaConfigFixture(
{},
{
commands: { ownerAllowFrom: ["telegram:existing-owner"] },
},
);
expect(cfg.commands?.ownerAllowFrom).toEqual(["telegram:existing-owner", "+15550000001"]);
});
it("models activation always through visible group behavior and restores mention gating", async () => {
const scenario = findMockWhatsAppScenario("whatsapp-group-activation-always");
const run = scenario.buildRun();
@@ -7,6 +7,12 @@ scenario:
primary:
- channels.access-policy
objective: Verify dmPolicy disabled rejects direct messages.
gatewayConfigPatch:
channels:
whatsapp:
accounts:
$selectedAccount:
dmPolicy: disabled
successCriteria:
- The direct message produces no outbound reply.
docsRefs:
@@ -19,8 +25,6 @@ scenario:
suiteIsolation: isolated
summary: Verify WhatsApp dmPolicy disabled behavior through the canonical adapter.
config:
policyKey: dmPolicy
policyValue: disabled
conversationKind: direct
conversationId: 15550000001@s.whatsapp.net
senderId: 15550000002@s.whatsapp.net
@@ -37,11 +41,6 @@ flow:
args: [{ ref: env }, 60000]
- call: waitForTransportReady
args: [{ ref: env }, 60000]
- call: patchConfig
args:
- env: { ref: env }
patch:
expr: "({ channels: { whatsapp: { [config.policyKey]: config.policyValue, ...(env.transport.accountId === 'default' ? {} : { accounts: { [env.transport.accountId]: { [config.policyKey]: config.policyValue } } }) } } })"
- resetTransport: true
- set: marker
value:
@@ -7,6 +7,12 @@ scenario:
primary:
- channels.access-policy
objective: Verify dmPolicy open accepts a direct message.
gatewayConfigPatch:
channels:
whatsapp:
accounts:
$selectedAccount:
dmPolicy: open
successCriteria:
- The direct message receives one marker reply.
docsRefs:
@@ -19,8 +25,6 @@ scenario:
suiteIsolation: isolated
summary: Verify WhatsApp dmPolicy open behavior through the canonical adapter.
config:
policyKey: dmPolicy
policyValue: open
conversationKind: direct
conversationId: 15550000001@s.whatsapp.net
senderId: 15550000002@s.whatsapp.net
@@ -37,11 +41,6 @@ flow:
args: [{ ref: env }, 60000]
- call: waitForTransportReady
args: [{ ref: env }, 60000]
- call: patchConfig
args:
- env: { ref: env }
patch:
expr: "({ channels: { whatsapp: { [config.policyKey]: config.policyValue, ...(env.transport.accountId === 'default' ? {} : { accounts: { [env.transport.accountId]: { [config.policyKey]: config.policyValue } } }) } } })"
- resetTransport: true
- set: marker
value:
@@ -7,6 +7,12 @@ scenario:
primary:
- channels.access-policy
objective: Verify groupPolicy disabled rejects group messages.
gatewayConfigPatch:
channels:
whatsapp:
accounts:
$selectedAccount:
groupPolicy: disabled
successCriteria:
- The group message produces no outbound reply.
docsRefs:
@@ -19,8 +25,6 @@ scenario:
suiteIsolation: isolated
summary: Verify WhatsApp groupPolicy disabled behavior through the canonical adapter.
config:
policyKey: groupPolicy
policyValue: disabled
conversationKind: group
conversationId: 120363000000000000@g.us
senderId: 15550000002@s.whatsapp.net
@@ -37,11 +41,6 @@ flow:
args: [{ ref: env }, 60000]
- call: waitForTransportReady
args: [{ ref: env }, 60000]
- call: patchConfig
args:
- env: { ref: env }
patch:
expr: "({ channels: { whatsapp: { [config.policyKey]: config.policyValue, ...(env.transport.accountId === 'default' ? {} : { accounts: { [env.transport.accountId]: { [config.policyKey]: config.policyValue } } }) } } })"
- resetTransport: true
- set: marker
value:
@@ -7,6 +7,12 @@ scenario:
primary:
- channels.access-policy
objective: Verify groupPolicy open accepts a mentioned group message.
gatewayConfigPatch:
channels:
whatsapp:
accounts:
$selectedAccount:
groupPolicy: open
successCriteria:
- The mentioned group message receives one marker reply.
docsRefs:
@@ -19,12 +25,10 @@ scenario:
suiteIsolation: isolated
summary: Verify WhatsApp groupPolicy open behavior through the canonical adapter.
config:
policyKey: groupPolicy
policyValue: open
conversationKind: group
conversationId: 120363000000000000@g.us
senderId: 15550000002@s.whatsapp.net
mentionPrefix: "@openclaw "
mentionPrefix: "openclawqa "
expectReply: true
markerPrefix: WHATSAPP_QA_GROUP_OPEN
timeoutMs: 60000
@@ -37,11 +41,6 @@ flow:
args: [{ ref: env }, 60000]
- call: waitForTransportReady
args: [{ ref: env }, 60000]
- call: patchConfig
args:
- env: { ref: env }
patch:
expr: "({ channels: { whatsapp: { [config.policyKey]: config.policyValue, ...(env.transport.accountId === 'default' ? {} : { accounts: { [env.transport.accountId]: { [config.policyKey]: config.policyValue } } }) } } })"
- resetTransport: true
- set: marker
value:
@@ -7,6 +7,14 @@ scenario:
primary:
- channels.pairing-gate
objective: Verify an unpaired WhatsApp DM receives the pairing gate instead of reaching the agent.
gatewayConfigPatch:
channels:
whatsapp:
accounts:
$selectedAccount:
dmPolicy: pairing
allowFrom:
- "+15550000000"
successCriteria:
- The inbound DM receives access-not-configured or pairing-code guidance.
- The agent marker is not emitted.
@@ -20,7 +28,6 @@ scenario:
suiteIsolation: isolated
summary: Verify pairing gate output through the canonical WhatsApp adapter.
config:
accountId: sut
conversationId: 15550000001@s.whatsapp.net
senderId: 15550000002@s.whatsapp.net
@@ -32,14 +39,6 @@ flow:
args: [{ ref: env }, 60000]
- call: waitForTransportReady
args: [{ ref: env }, 60000]
- call: patchConfig
args:
- env: { ref: env }
patch:
expr: "({ channels: { whatsapp: { dmPolicy: 'pairing', allowFrom: ['+15550000000'], ...(env.transport.accountId === 'default' ? {} : { accounts: { [env.transport.accountId]: { dmPolicy: 'pairing', allowFrom: ['+15550000000'] } } }) } } })"
replacePaths:
- channels.whatsapp.allowFrom
- channels.whatsapp.accounts.sut.allowFrom
- resetTransport: true
- set: marker
value:
+10 -1
View File
@@ -1,8 +1,17 @@
// QA channel protocol tests cover synthetic channel payload validation and parsing.
import { describe, expect, it } from "vitest";
import { parseQaTarget, sanitizeQaBusToolCalls } from "./qa-channel-protocol.js";
import { buildQaTarget, parseQaTarget, sanitizeQaBusToolCalls } from "./qa-channel-protocol.js";
describe("qa-channel protocol", () => {
it("builds canonical targets", () => {
expect(buildQaTarget({ chatType: "direct", conversationId: "Alice" })).toBe("dm:Alice");
expect(buildQaTarget({ chatType: "group", conversationId: "Room" })).toBe("group:Room");
expect(buildQaTarget({ chatType: "channel", conversationId: "Room" })).toBe("channel:Room");
expect(buildQaTarget({ chatType: "channel", conversationId: "Room", threadId: "Topic" })).toBe(
"thread:Room/Topic",
);
});
it("parses canonical targets without folding ids or prefix casing", () => {
expect(parseQaTarget("channel:CaseSensitive")).toEqual({
chatType: "channel",
+12
View File
@@ -11,6 +11,18 @@ export type QaTargetParts = {
threadId?: string;
};
/** Encode a canonical QA channel target. */
export function buildQaTarget(params: {
chatType: QaBusConversationKind;
conversationId: string;
threadId?: string | null;
}): string {
if (params.threadId) {
return `thread:${params.conversationId}/${params.threadId}`;
}
return `${params.chatType === "direct" ? "dm" : params.chatType}:${params.conversationId}`;
}
/** Parse the lowercase, prefix-scoped target grammar shared by QA Channel and QA Lab. */
export function parseQaTarget(
raw: string,