furyhawk/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-07-21 10:16:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(qa): resolve profile channels through selected driver (#109350)

Browse Source 
* fix(qa): keep generic profiles off channel-specific lanes

* fix(qa): resolve profile channels through selected driver
This commit is contained in:
Dallin Romney
2026-07-17 16:42:18 -07:00
committed by GitHub
parent 03d5d88a14
commit 82fa956d5a
4 changed files with 151 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/qa-lab/src/cli.runtime.test.ts
+24 -2
View File
@@ -521,6 +521,23 @@ describe("qa cli runtime", () => {
expect(suiteArgs.channelDriverSelection).toBeUndefined();
});
it("keeps portable channel scenarios in driver-selected profile runs", async () => {
await runQaProfileCommand({
repoRoot: "/tmp/openclaw-repo",
profile: "release",
surface: "channel-framework",
providerMode: "mock-openai",
scenarioIds: ["channel-chat-baseline", "telegram-help-command"],
});
const suiteArgs = mockFirstObjectArg(runQaSuite);
expect(suiteArgs.scenarioIds).toContain("channel-chat-baseline");
expect(suiteArgs.scenarioIds).toContain("telegram-help-command");
expect(suiteArgs.adapterFactories).toBe(
listLiveTransportQaAdapterFactories.mock.results[0]?.value,
);
});
it("runs the all profile through the live taxonomy profile path", async () => {
await runQaProfileCommand({
repoRoot: "/tmp/openclaw-repo",
@@ -767,14 +784,19 @@ describe("qa cli runtime", () => {
expect(runQaSuite).not.toHaveBeenCalled();
});
it("keeps live taxonomy metadata unchanged without an explicit adapter channel", async () => {
it("loads contributed adapters without preselecting a scenario channel", async () => {
await runQaSuiteCommand({
channelDriver: "live",
scenarioIds: ["channel-chat-baseline"],
});
expect(runQaSuite).toHaveBeenCalledWith(
expect.not.objectContaining({ adapterFactories: expect.anything() }),
expect.objectContaining({
adapterFactories: listLiveTransportQaAdapterFactories.mock.results[0]?.value,
}),
);
expect(runQaSuite).toHaveBeenCalledWith(
expect.not.objectContaining({ channelId: expect.anything() }),
);
});
extensions/qa-lab/src/cli.runtime.ts
+4 -3
View File
@@ -994,7 +994,8 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) {
throw new Error("--channel override requires --channel-driver crabline or live.");
}
const liveChannelId = channelDriver === "live" ? opts.channel?.trim() : undefined;
const liveAdapterFactories = liveChannelId ? listLiveTransportQaAdapterFactories() : undefined;
const liveAdapterFactories =
channelDriver === "live" ? listLiveTransportQaAdapterFactories() : undefined;
const liveAdapterFactory = liveChannelId
? liveAdapterFactories?.find((factory) => factory.id === liveChannelId)
: undefined;
@@ -1125,10 +1126,10 @@ export async function runQaSuiteCommand(opts: QaSuiteCommandOptions) {
evidenceMode: opts.evidenceMode,
transportId,
channelDriver,
...(liveChannelId
...(liveAdapterFactories
? {
adapterFactories: liveAdapterFactories,
channelId: liveChannelId,
...(liveChannelId ? { channelId: liveChannelId } : {}),
adapterOptions: {
repoRoot,
sutAccountId: opts.sutAccountId,
extensions/qa-lab/src/suite-launch.runtime.test.ts
+76 -1
View File
@@ -186,7 +186,7 @@ describe("qa suite runtime launcher", () => {
expect(runQaTestFileScenarios).not.toHaveBeenCalled();
});
it("runs runtime-specific live scenarios in dedicated workers", async () => {
it("runs runtime-specific channel scenarios in dedicated workers", async () => {
const repoRoot = await makeTempRepo("qa-suite-live-runtime-");
await runQaSuite({
repoRoot,
@@ -215,6 +215,81 @@ describe("qa suite runtime launcher", () => {
);
});
it("partitions portable scenarios by channel for a pluggable driver", async () => {
const repoRoot = await makeTempRepo("qa-suite-pluggable-channels-");
const adapterFactories = [
{
id: "portable-driver",
matches: vi.fn(),
create: vi.fn(),
},
];
await runQaSuite({
repoRoot,
outputDir: ".artifacts/qa-e2e/pluggable-channels",
providerMode: "mock-openai",
channelDriver: "live",
adapterFactories,
scenarioIds: ["channel-chat-baseline", "telegram-help-command", "matrix-restart-resume"],
});
const outputDir = path.join(repoRoot, ".artifacts", "qa-e2e", "pluggable-channels");
expect(runQaFlowSuite).toHaveBeenCalledTimes(3);
expect(runQaFlowSuite).toHaveBeenCalledWith(
expect.objectContaining({
adapterFactories,
channelId: undefined,
outputDir: path.join(outputDir, "flow"),
scenarioIds: ["channel-chat-baseline"],
}),
);
expect(runQaFlowSuite).toHaveBeenCalledWith(
expect.objectContaining({
adapterFactories,
channelId: "telegram",
outputDir: path.join(outputDir, "flow", "telegram"),
scenarioIds: ["telegram-help-command"],
}),
);
expect(runQaFlowSuite).toHaveBeenCalledWith(
expect.objectContaining({
adapterFactories,
channelId: "matrix",
outputDir: path.join(outputDir, "flow", "matrix"),
scenarioIds: ["matrix-restart-resume"],
}),
);
});
it("binds one portable channel scenario without an explicit channel override", async () => {
const adapterFactories = [
{
id: "portable-driver",
matches: vi.fn(),
create: vi.fn(),
},
];
const result = await runQaSuite({
repoRoot: process.cwd(),
providerMode: "mock-openai",
channelDriver: "live",
adapterFactories,
scenarioIds: ["telegram-help-command"],
});
expect(result.executionKind).toBe("suite");
expect(runQaFlowSuite).toHaveBeenCalledTimes(1);
expect(runQaFlowSuite).toHaveBeenCalledWith(
expect.objectContaining({
adapterFactories,
channelId: "telegram",
scenarioIds: ["telegram-help-command"],
}),
);
});
it("partitions mixed Crabline flow channels into one aggregate suite", async () => {
const repoRoot = await makeTempRepo("qa-suite-crabline-channels-");
const defaultFlowImplementation = runQaFlowSuite.getMockImplementation();
extensions/qa-lab/src/suite-launch.runtime.ts
+47 -6
View File
@@ -93,6 +93,7 @@ type QaUnifiedPartitionTask = {
type QaFlowChannelGroup = {
channel: string | undefined;
channelId: string | undefined;
channelDriverSelection: QaSuiteRunParams["channelDriverSelection"];
scenarios: QaSeedScenarioWithSource[];
};
@@ -132,10 +133,37 @@ async function resolveQaFlowChannelGroups(
runParams: QaSuiteRunParams | undefined,
scenarios: readonly QaSeedScenarioWithSource[],
): Promise<QaFlowChannelGroup[]> {
if (runParams?.adapterFactories) {
const explicitChannel = runParams.channelId?.trim().toLowerCase();
if (explicitChannel) {
return [
{
channel: explicitChannel,
channelId: explicitChannel,
channelDriverSelection: runParams.channelDriverSelection,
scenarios: [...scenarios],
},
];
}
const groups = new Map<string | undefined, QaSeedScenarioWithSource[]>();
for (const scenario of scenarios) {
const channel = normalizeQaSuiteScenarioChannel(scenario);
const group = groups.get(channel) ?? [];
group.push(scenario);
groups.set(channel, group);
}
return [...groups].map(([channel, groupedScenarios]) => ({
channel,
channelId: channel,
channelDriverSelection: runParams.channelDriverSelection,
scenarios: groupedScenarios,
}));
}
if (runParams?.channelDriver !== "crabline") {
return [
{
channel: runParams?.channelDriverSelection?.channel,
channel: runParams?.channelId ?? runParams?.channelDriverSelection?.channel,
channelId: runParams?.channelId,
channelDriverSelection: runParams?.channelDriverSelection,
scenarios: [...scenarios],
},
@@ -155,6 +183,7 @@ async function resolveQaFlowChannelGroups(
return [
{
channel: singleChannel,
channelId: undefined,
channelDriverSelection:
runParams.channelDriverSelection ??
resolveOpenClawCrablineChannelDriverSelection({ channel: singleChannel }),
@@ -166,6 +195,7 @@ async function resolveQaFlowChannelGroups(
// launch one flow partition per channel and aggregate them at this owner.
return channels.map((channel) => ({
channel,
channelId: undefined,
channelDriverSelection: resolveOpenClawCrablineChannelDriverSelection({ channel }),
scenarios: scenarios.filter(
(scenario) =>
@@ -196,10 +226,14 @@ async function resolveSuiteExecutionPlan(
scenarios.push(scenario);
testFileScenariosByKind.set(scenario.execution.kind, scenarios);
}
const channelGroups = (await resolveQaFlowChannelGroups(params, flowScenarios)).filter(
(group) => group.scenarios.length > 0,
);
const requiresFlowPartitions =
(await resolveQaFlowChannelGroups(params, flowScenarios)).filter(
(group) => group.scenarios.length > 0,
).length > 1 ||
channelGroups.length > 1 ||
channelGroups.some(
(group) => group.channelId !== undefined && group.channelId !== params?.channelId,
) ||
(flowScenarios.length > 1 && flowScenarios.some(scenarioRequiresIsolatedQaSuiteWorker));
if (testFileScenariosByKind.size === 0 && !requiresFlowPartitions) {
return { kind: "flow" };
@@ -560,12 +594,18 @@ async function runUnifiedQaSuite(params: {
const ordinaryIsolatedFlowScenarios = isolatedFlowScenarios.filter(
(scenario) => !runtimeScenarioSet.has(scenario),
);
const sharedFlowPartitions = partitionSharedFlowScenarios(sharedFlowScenarios, concurrency);
const usesContributedChannelDriver = Boolean(
channelGroup.channelId && params.runParams?.adapterFactories,
);
const sharedFlowPartitions = partitionSharedFlowScenarios(
sharedFlowScenarios,
usesContributedChannelDriver ? 1 : concurrency,
);
// Channel-driver flow workers each launch a gateway plus transport harness.
// Serializing their isolated workers keeps state-mutating smoke checks from
// flaking under concurrent child gateways while preserving non-driver speed.
const channelDriverFlowRequiresExclusiveWorkers = Boolean(
channelGroup.channelDriverSelection,
channelGroup.channelDriverSelection || usesContributedChannelDriver,
);
const isolatedFlowConcurrencyLimit = channelDriverFlowRequiresExclusiveWorkers
? 1
@@ -633,6 +673,7 @@ async function runUnifiedQaSuite(params: {
? (partition.scenarios[0].execution.runtime ?? params.runParams?.forcedRuntime)
: params.runParams?.forcedRuntime,
concurrency: partition.concurrency,
channelId: channelGroup.channelId,
channelDriverSelection: channelGroup.channelDriverSelection,
workerStartStaggerMs: isolatedPartition
? (params.runParams?.workerStartStaggerMs ??