diff --git a/extensions/qa-lab/src/scenario-lane.test.ts b/extensions/qa-lab/src/scenario-lane.test.ts new file mode 100644 index 000000000000..e9f94ee1ef2d --- /dev/null +++ b/extensions/qa-lab/src/scenario-lane.test.ts @@ -0,0 +1,66 @@ +// Qa Lab tests cover canonical scenario lane matching behavior. +import { describe, expect, it } from "vitest"; +import { + describeQaProviderLaneMismatches, + scenarioMatchesQaProviderLane, +} from "./scenario-lane.js"; +import { makeQaSuiteTestScenario } from "./suite-test-helpers.js"; + +describe("QA scenario lane matching", () => { + it("reports every declared mismatch in one decision", () => { + const scenario = makeQaSuiteTestScenario("strict-live-lane", { + channel: "matrix", + driver: "live", + runtimeParityTier: "live-only", + config: { + requiredProviderMode: "live-frontier", + requiredProvider: "claude-cli", + requiredModel: "claude-sonnet-4-6", + authMode: "subscription", + }, + }); + + expect( + describeQaProviderLaneMismatches({ + scenario, + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.6-luna", + channelDriver: "crabline", + channel: "telegram", + claudeCliAuthMode: "api-key", + }), + ).toEqual([ + "live provider mode", + "providerMode=live-frontier", + "channelDriver=live", + "channel=matrix", + "provider=claude-cli", + "model=claude-sonnet-4-6", + "authMode=subscription", + ]); + }); + + it("accepts a mock lane only when its selected provider and model satisfy the contract", () => { + const scenario = makeQaSuiteTestScenario("mock-anthropic", { + config: { + requiredProvider: "anthropic", + requiredModel: "claude-opus-4-8", + }, + }); + + expect( + scenarioMatchesQaProviderLane({ + scenario, + providerMode: "mock-openai", + primaryModel: "anthropic/claude-opus-4-8", + }), + ).toBe(true); + expect( + scenarioMatchesQaProviderLane({ + scenario, + providerMode: "mock-openai", + primaryModel: "mock-openai/gpt-5.6-luna", + }), + ).toBe(false); + }); +}); diff --git a/extensions/qa-lab/src/scenario-lane.ts b/extensions/qa-lab/src/scenario-lane.ts index d0bbfd4549ab..94e92dc73447 100644 --- a/extensions/qa-lab/src/scenario-lane.ts +++ b/extensions/qa-lab/src/scenario-lane.ts @@ -40,9 +40,6 @@ export function describeQaProviderLaneMismatches(params: { ) { mismatches.push(`channel=${scenarioChannel}`); } - if (provider.kind !== "live") { - return mismatches; - } const selected = splitQaModelRef(params.primaryModel); const requiredProvider = normalizeQaConfigString(config.requiredProvider); if (requiredProvider && selected?.provider !== requiredProvider) { diff --git a/extensions/qa-lab/src/suite-planning.test.ts b/extensions/qa-lab/src/suite-planning.test.ts index 1cbbeea186a4..c2c783b5e083 100644 --- a/extensions/qa-lab/src/suite-planning.test.ts +++ b/extensions/qa-lab/src/suite-planning.test.ts @@ -281,13 +281,103 @@ describe("qa suite planning helpers", () => { ).toBe(25); }); - it("keeps explicitly requested provider-specific scenarios", () => { + it("rejects an explicitly requested scenario for the wrong provider", () => { const scenarios = [ makeQaSuiteTestScenario("generic"), makeQaSuiteTestScenario("anthropic-only", { config: { requiredProvider: "anthropic", - requiredModel: "claude-opus-4-8", + }, + }), + ]; + + expect(() => + selectQaFlowSuiteScenarios({ + scenarios, + scenarioIds: ["anthropic-only"], + providerMode: "live-frontier", + primaryModel: "openai/gpt-5.6-luna", + }), + ).toThrow( + "selected QA scenario(s) do not match the current QA lane: anthropic-only (provider=anthropic)", + ); + }); + + it("rejects an explicitly requested scenario for the wrong provider mode", () => { + const scenarios = [ + makeQaSuiteTestScenario("mock-only", { + config: { requiredProviderMode: "mock-openai" }, + }), + ]; + + expect(() => + selectQaFlowSuiteScenarios({ + scenarios, + scenarioIds: ["mock-only"], + providerMode: "live-frontier", + primaryModel: "openai/gpt-5.6-luna", + }), + ).toThrow( + "selected QA scenario(s) do not match the current QA lane: mock-only (providerMode=mock-openai)", + ); + }); + + it("rejects an explicitly requested scenario for the wrong model", () => { + const scenarios = [ + makeQaSuiteTestScenario("openai-model", { + config: { + requiredProvider: "openai", + requiredModel: "gpt-5.6-luna", + }, + }), + ]; + + expect(() => + selectQaFlowSuiteScenarios({ + scenarios, + scenarioIds: ["openai-model"], + providerMode: "live-frontier", + primaryModel: "openai/gpt-5.6-terra", + }), + ).toThrow( + "selected QA scenario(s) do not match the current QA lane: openai-model (model=gpt-5.6-luna)", + ); + }); + + it("rejects an explicitly requested scenario for the wrong auth mode", () => { + const scenarios = [ + makeQaSuiteTestScenario("claude-subscription", { + config: { + requiredProvider: "claude-cli", + authMode: "subscription", + }, + }), + ]; + + expect(() => + selectQaFlowSuiteScenarios({ + scenarios, + scenarioIds: ["claude-subscription"], + providerMode: "live-frontier", + primaryModel: "claude-cli/claude-sonnet-4-6", + claudeCliAuthMode: "api-key", + }), + ).toThrow( + "selected QA scenario(s) do not match the current QA lane: claude-subscription (authMode=subscription)", + ); + }); + + it("keeps an explicitly requested scenario when every lane contract matches", () => { + const scenarios = [ + makeQaSuiteTestScenario("strict-live-lane", { + channel: "matrix", + driver: "live", + runtimeParityTier: "live-only", + config: { + requiredProviderMode: "live-frontier", + requiredProvider: "claude-cli", + requiredModel: "claude-sonnet-4-6", + authMode: "subscription", }, }), ]; @@ -295,11 +385,14 @@ describe("qa suite planning helpers", () => { expect( selectQaFlowSuiteScenarios({ scenarios, - scenarioIds: ["anthropic-only"], + scenarioIds: ["strict-live-lane"], providerMode: "live-frontier", - primaryModel: "openai/gpt-5.6-luna", + primaryModel: "claude-cli/claude-sonnet-4-6", + claudeCliAuthMode: "subscription", + channelDriver: "live", + channel: "matrix", }).map((scenario) => scenario.id), - ).toEqual(["anthropic-only"]); + ).toEqual(["strict-live-lane"]); }); it("keeps explicitly requested scenarios in request order", () => { @@ -319,6 +412,36 @@ describe("qa suite planning helpers", () => { ).toEqual(["third", "first"]); }); + it("applies the same lane contract to explicit and implicit selection", () => { + const scenarios = [ + makeQaSuiteTestScenario("generic"), + makeQaSuiteTestScenario("openai-only", { + config: { requiredProvider: "openai", requiredModel: "gpt-5.6-luna" }, + }), + makeQaSuiteTestScenario("anthropic-only", { + config: { requiredProvider: "anthropic", requiredModel: "claude-opus-4-8" }, + }), + ]; + const lane = { + scenarios, + providerMode: "live-frontier" as const, + primaryModel: "openai/gpt-5.6-luna", + }; + + expect(selectQaFlowSuiteScenarios(lane).map((scenario) => scenario.id)).toEqual([ + "generic", + "openai-only", + ]); + expect( + selectQaFlowSuiteScenarios({ ...lane, scenarioIds: ["openai-only"] }).map( + (scenario) => scenario.id, + ), + ).toEqual(["openai-only"]); + expect(() => selectQaFlowSuiteScenarios({ ...lane, scenarioIds: ["anthropic-only"] })).toThrow( + "selected QA scenario(s) do not match the current QA lane: anthropic-only (provider=anthropic, model=claude-opus-4-8)", + ); + }); + it("resolves driver channels from scenario execution with explicit and default fallbacks", () => { expect( resolveQaSuiteScenarioChannel({ @@ -781,7 +904,7 @@ describe("qa suite planning helpers", () => { ).toEqual(["matrix-transport"]); }); - it("keeps live-only runtime parity scenarios out of implicit mock selections", () => { + it("keeps live-only runtime parity scenarios out of every mock selection", () => { const scenarios = [ makeQaSuiteTestScenario("generic"), makeQaSuiteTestScenario("live-runtime", { @@ -797,13 +920,15 @@ describe("qa suite planning helpers", () => { }).map((scenario) => scenario.id), ).toEqual(["generic"]); - expect( + expect(() => selectQaFlowSuiteScenarios({ scenarios, scenarioIds: ["live-runtime"], providerMode: "mock-openai", primaryModel: "mock-openai/gpt-5.6-luna", - }).map((scenario) => scenario.id), - ).toEqual(["live-runtime"]); + }), + ).toThrow( + "selected QA scenario(s) do not match the current QA lane: live-runtime (live provider mode)", + ); }); }); diff --git a/extensions/qa-lab/src/suite-planning.ts b/extensions/qa-lab/src/suite-planning.ts index e70ee0842742..aad8d0ffe35c 100644 --- a/extensions/qa-lab/src/suite-planning.ts +++ b/extensions/qa-lab/src/suite-planning.ts @@ -59,7 +59,7 @@ function selectQaFlowSuiteScenarios(params: { `suite execution requires flow scenarios; unsupported scenario(s): ${scenarioList}`, ); } - const channelDriverMismatches = selectedScenarios.flatMap((scenario) => { + const laneMismatches = selectedScenarios.flatMap((scenario) => { const mismatches = describeQaProviderLaneMismatches({ scenario, providerMode: params.providerMode, @@ -67,14 +67,12 @@ function selectQaFlowSuiteScenarios(params: { channelDriver: params.channelDriver, channel: params.channel, claudeCliAuthMode: params.claudeCliAuthMode, - }).filter( - (mismatch) => mismatch.startsWith("channelDriver=") || mismatch.startsWith("channel="), - ); + }); return mismatches.length > 0 ? [`${scenario.id} (${mismatches.join(", ")})`] : []; }); - if (channelDriverMismatches.length > 0) { + if (laneMismatches.length > 0) { throw new Error( - `selected QA scenario(s) do not match the current QA lane: ${channelDriverMismatches.join(", ")}`, + `selected QA scenario(s) do not match the current QA lane: ${laneMismatches.join(", ")}`, ); } return selectedScenarios;