Revert "fix(qa): keep smoke profile on one channel (#101173)" (#101184)

This reverts commit b773f7981a.
This commit is contained in:
Dallin Romney
2026-07-06 13:29:07 -07:00
committed by GitHub
parent b773f7981a
commit 19491fe34c
2 changed files with 9 additions and 48 deletions
+1 -28
View File
@@ -110,7 +110,6 @@ import { loadNonYamlScenarioRefs } from "./live-transports/shared/live-transport
import { runQaTelegramCommand } from "./live-transports/telegram/cli.runtime.js";
import { defaultQaModelForMode as defaultQaProviderModelForMode } from "./model-selection.js";
import type { QaProviderModeInput } from "./run-config.js";
import { readQaScenarioPack } from "./scenario-catalog.js";
function mockFirstObjectArg(mock: unknown): Record<string, unknown> {
const calls = (mock as { mock?: { calls?: Array<Array<unknown>> } }).mock?.calls ?? [];
@@ -567,7 +566,7 @@ describe("qa cli runtime", () => {
expectWriteContains(stdoutWrite, "QA run profile: all; categories: 1; scenarios:");
});
it("keeps the automatic Crabline smoke profile on its default channel", async () => {
it("filters QA-channel-pinned scenarios from the Crabline smoke profile", async () => {
runQaSuite.mockImplementationOnce(async () => {
await fs.writeFile(suiteEvidencePath, JSON.stringify(makeQaEvidence()), "utf8");
return flowSuiteRuntimeResult({
@@ -584,17 +583,6 @@ describe("qa cli runtime", () => {
const suiteArgs = mockFirstObjectArg(runQaSuite);
expect(suiteArgs.channelDriver).toBe("crabline");
expect(suiteArgs.scenarioIds).toEqual(expect.arrayContaining(["dm-chat-baseline"]));
const scenarioChannelById = new Map(
readQaScenarioPack().scenarios.map((scenario) => [scenario.id, scenario.execution.channel]),
);
const selectedChannels = [
...new Set(
(suiteArgs.scenarioIds as string[])
.map((scenarioId) => scenarioChannelById.get(scenarioId))
.filter((channel): channel is string => Boolean(channel)),
),
];
expect(selectedChannels).toEqual(["telegram"]);
expect(suiteArgs.scenarioIds).not.toEqual(
expect.arrayContaining([
"instruction-followthrough-repo-contract",
@@ -625,21 +613,6 @@ describe("qa cli runtime", () => {
);
});
it("keeps an explicitly requested non-default Crabline channel scenario", async () => {
await runQaProfileCommand({
repoRoot: "/tmp/openclaw-repo",
profile: "smoke-ci",
scenarioIds: ["slack-restart-resume"],
});
const suiteArgs = mockFirstObjectArg(runQaSuite);
expect(suiteArgs.scenarioIds).toEqual(["slack-restart-resume"]);
expect(suiteArgs.channelDriverSelection).toMatchObject({
channel: "slack",
channelDriver: "crabline",
});
});
it("rejects qa profile runs that do not match taxonomy categories", async () => {
await expect(
runQaProfileCommand({
+8 -20
View File
@@ -734,26 +734,14 @@ export async function runQaProfileCommand(opts: QaProfileCommandOptions) {
const providerMode = opts.providerMode ?? defaultQaRunProfileProviderMode(profile);
const normalizedProviderMode = normalizeQaProviderMode(providerMode);
const primaryModel = opts.primaryModel?.trim() || defaultQaModelForMode(normalizedProviderMode);
// Automatic Crabline profiles share one driver instance, so keep unpinned scenarios plus the
// default channel. Explicit scenario requests still infer their declared execution channel.
const automaticProfileChannel =
requestedScenarioIds.length === 0 && profileReport.channelDriver === "crabline"
? OPENCLAW_CRABLINE_DEFAULT_CHANNEL
: undefined;
const scenarios = taxonomyScenarios.filter((scenario) => {
const scenarioChannel = scenario.execution.channel?.trim().toLowerCase();
const matchesProfileChannel =
!automaticProfileChannel || !scenarioChannel || scenarioChannel === automaticProfileChannel;
return (
matchesProfileChannel &&
scenarioMatchesQaProviderLane({
scenario,
providerMode: normalizedProviderMode,
primaryModel,
channelDriver: profileReport.channelDriver,
})
);
});
const scenarios = taxonomyScenarios.filter((scenario) =>
scenarioMatchesQaProviderLane({
scenario,
providerMode: normalizedProviderMode,
primaryModel,
channelDriver: profileReport.channelDriver,
}),
);
if (scenarios.length === 0) {
throw new Error(
`qa run --qa-profile ${profile} did not resolve any executable QA scenarios for provider mode ${normalizedProviderMode}.`,