fix(qa): enforce scenario lane contracts (#108539)

extensions/qa-lab/src/scenario-lane.test.ts

@@ -0,0 +1,66 @@
+// Qa Lab tests cover canonical scenario lane matching behavior.
+import { describe, expect, it } from "vitest";
+import {
+  describeQaProviderLaneMismatches,
+  scenarioMatchesQaProviderLane,
+} from "./scenario-lane.js";
+import { makeQaSuiteTestScenario } from "./suite-test-helpers.js";
+
+describe("QA scenario lane matching", () => {
+  it("reports every declared mismatch in one decision", () => {
+    const scenario = makeQaSuiteTestScenario("strict-live-lane", {
+      channel: "matrix",
+      driver: "live",
+      runtimeParityTier: "live-only",
+      config: {
+        requiredProviderMode: "live-frontier",
+        requiredProvider: "claude-cli",
+        requiredModel: "claude-sonnet-4-6",
+        authMode: "subscription",
+      },
+    });
+
+    expect(
+      describeQaProviderLaneMismatches({
+        scenario,
+        providerMode: "mock-openai",
+        primaryModel: "mock-openai/gpt-5.6-luna",
+        channelDriver: "crabline",
+        channel: "telegram",
+        claudeCliAuthMode: "api-key",
+      }),
+    ).toEqual([
+      "live provider mode",
+      "providerMode=live-frontier",
+      "channelDriver=live",
+      "channel=matrix",
+      "provider=claude-cli",
+      "model=claude-sonnet-4-6",
+      "authMode=subscription",
+    ]);
+  });
+
+  it("accepts a mock lane only when its selected provider and model satisfy the contract", () => {
+    const scenario = makeQaSuiteTestScenario("mock-anthropic", {
+      config: {
+        requiredProvider: "anthropic",
+        requiredModel: "claude-opus-4-8",
+      },
+    });
+
+    expect(
+      scenarioMatchesQaProviderLane({
+        scenario,
+        providerMode: "mock-openai",
+        primaryModel: "anthropic/claude-opus-4-8",
+      }),
+    ).toBe(true);
+    expect(
+      scenarioMatchesQaProviderLane({
+        scenario,
+        providerMode: "mock-openai",
+        primaryModel: "mock-openai/gpt-5.6-luna",
+      }),
+    ).toBe(false);
+  });
+});

extensions/qa-lab/src/scenario-lane.ts

@@ -40,9 +40,6 @@ export function describeQaProviderLaneMismatches(params: {
   ) {
     mismatches.push(`channel=${scenarioChannel}`);
   }
-  if (provider.kind !== "live") {
-    return mismatches;
-  }
   const selected = splitQaModelRef(params.primaryModel);
   const requiredProvider = normalizeQaConfigString(config.requiredProvider);
   if (requiredProvider && selected?.provider !== requiredProvider) {

extensions/qa-lab/src/suite-planning.test.ts

@@ -281,13 +281,103 @@ describe("qa suite planning helpers", () => {
     ).toBe(25);
   });
 
-  it("keeps explicitly requested provider-specific scenarios", () => {
+  it("rejects an explicitly requested scenario for the wrong provider", () => {
     const scenarios = [
       makeQaSuiteTestScenario("generic"),
       makeQaSuiteTestScenario("anthropic-only", {
         config: {
           requiredProvider: "anthropic",
-          requiredModel: "claude-opus-4-8",
+        },
+      }),
+    ];
+
+    expect(() =>
+      selectQaFlowSuiteScenarios({
+        scenarios,
+        scenarioIds: ["anthropic-only"],
+        providerMode: "live-frontier",
+        primaryModel: "openai/gpt-5.6-luna",
+      }),
+    ).toThrow(
+      "selected QA scenario(s) do not match the current QA lane: anthropic-only (provider=anthropic)",
+    );
+  });
+
+  it("rejects an explicitly requested scenario for the wrong provider mode", () => {
+    const scenarios = [
+      makeQaSuiteTestScenario("mock-only", {
+        config: { requiredProviderMode: "mock-openai" },
+      }),
+    ];
+
+    expect(() =>
+      selectQaFlowSuiteScenarios({
+        scenarios,
+        scenarioIds: ["mock-only"],
+        providerMode: "live-frontier",
+        primaryModel: "openai/gpt-5.6-luna",
+      }),
+    ).toThrow(
+      "selected QA scenario(s) do not match the current QA lane: mock-only (providerMode=mock-openai)",
+    );
+  });
+
+  it("rejects an explicitly requested scenario for the wrong model", () => {
+    const scenarios = [
+      makeQaSuiteTestScenario("openai-model", {
+        config: {
+          requiredProvider: "openai",
+          requiredModel: "gpt-5.6-luna",
+        },
+      }),
+    ];
+
+    expect(() =>
+      selectQaFlowSuiteScenarios({
+        scenarios,
+        scenarioIds: ["openai-model"],
+        providerMode: "live-frontier",
+        primaryModel: "openai/gpt-5.6-terra",
+      }),
+    ).toThrow(
+      "selected QA scenario(s) do not match the current QA lane: openai-model (model=gpt-5.6-luna)",
+    );
+  });
+
+  it("rejects an explicitly requested scenario for the wrong auth mode", () => {
+    const scenarios = [
+      makeQaSuiteTestScenario("claude-subscription", {
+        config: {
+          requiredProvider: "claude-cli",
+          authMode: "subscription",
+        },
+      }),
+    ];
+
+    expect(() =>
+      selectQaFlowSuiteScenarios({
+        scenarios,
+        scenarioIds: ["claude-subscription"],
+        providerMode: "live-frontier",
+        primaryModel: "claude-cli/claude-sonnet-4-6",
+        claudeCliAuthMode: "api-key",
+      }),
+    ).toThrow(
+      "selected QA scenario(s) do not match the current QA lane: claude-subscription (authMode=subscription)",
+    );
+  });
+
+  it("keeps an explicitly requested scenario when every lane contract matches", () => {
+    const scenarios = [
+      makeQaSuiteTestScenario("strict-live-lane", {
+        channel: "matrix",
+        driver: "live",
+        runtimeParityTier: "live-only",
+        config: {
+          requiredProviderMode: "live-frontier",
+          requiredProvider: "claude-cli",
+          requiredModel: "claude-sonnet-4-6",
+          authMode: "subscription",
         },
       }),
     ];
@@ -295,11 +385,14 @@ describe("qa suite planning helpers", () => {
     expect(
       selectQaFlowSuiteScenarios({
         scenarios,
-        scenarioIds: ["anthropic-only"],
+        scenarioIds: ["strict-live-lane"],
         providerMode: "live-frontier",
-        primaryModel: "openai/gpt-5.6-luna",
+        primaryModel: "claude-cli/claude-sonnet-4-6",
+        claudeCliAuthMode: "subscription",
+        channelDriver: "live",
+        channel: "matrix",
       }).map((scenario) => scenario.id),
-    ).toEqual(["anthropic-only"]);
+    ).toEqual(["strict-live-lane"]);
   });
 
   it("keeps explicitly requested scenarios in request order", () => {
@@ -319,6 +412,36 @@ describe("qa suite planning helpers", () => {
     ).toEqual(["third", "first"]);
   });
 
+  it("applies the same lane contract to explicit and implicit selection", () => {
+    const scenarios = [
+      makeQaSuiteTestScenario("generic"),
+      makeQaSuiteTestScenario("openai-only", {
+        config: { requiredProvider: "openai", requiredModel: "gpt-5.6-luna" },
+      }),
+      makeQaSuiteTestScenario("anthropic-only", {
+        config: { requiredProvider: "anthropic", requiredModel: "claude-opus-4-8" },
+      }),
+    ];
+    const lane = {
+      scenarios,
+      providerMode: "live-frontier" as const,
+      primaryModel: "openai/gpt-5.6-luna",
+    };
+
+    expect(selectQaFlowSuiteScenarios(lane).map((scenario) => scenario.id)).toEqual([
+      "generic",
+      "openai-only",
+    ]);
+    expect(
+      selectQaFlowSuiteScenarios({ ...lane, scenarioIds: ["openai-only"] }).map(
+        (scenario) => scenario.id,
+      ),
+    ).toEqual(["openai-only"]);
+    expect(() => selectQaFlowSuiteScenarios({ ...lane, scenarioIds: ["anthropic-only"] })).toThrow(
+      "selected QA scenario(s) do not match the current QA lane: anthropic-only (provider=anthropic, model=claude-opus-4-8)",
+    );
+  });
+
   it("resolves driver channels from scenario execution with explicit and default fallbacks", () => {
     expect(
       resolveQaSuiteScenarioChannel({
@@ -781,7 +904,7 @@ describe("qa suite planning helpers", () => {
     ).toEqual(["matrix-transport"]);
   });
 
-  it("keeps live-only runtime parity scenarios out of implicit mock selections", () => {
+  it("keeps live-only runtime parity scenarios out of every mock selection", () => {
     const scenarios = [
       makeQaSuiteTestScenario("generic"),
       makeQaSuiteTestScenario("live-runtime", {
@@ -797,13 +920,15 @@ describe("qa suite planning helpers", () => {
       }).map((scenario) => scenario.id),
     ).toEqual(["generic"]);
 
-    expect(
+    expect(() =>
       selectQaFlowSuiteScenarios({
         scenarios,
         scenarioIds: ["live-runtime"],
         providerMode: "mock-openai",
         primaryModel: "mock-openai/gpt-5.6-luna",
-      }).map((scenario) => scenario.id),
-    ).toEqual(["live-runtime"]);
+      }),
+    ).toThrow(
+      "selected QA scenario(s) do not match the current QA lane: live-runtime (live provider mode)",
+    );
   });
 });

extensions/qa-lab/src/suite-planning.ts

@@ -59,7 +59,7 @@ function selectQaFlowSuiteScenarios(params: {
         `suite execution requires flow scenarios; unsupported scenario(s): ${scenarioList}`,
       );
     }
-    const channelDriverMismatches = selectedScenarios.flatMap((scenario) => {
+    const laneMismatches = selectedScenarios.flatMap((scenario) => {
       const mismatches = describeQaProviderLaneMismatches({
         scenario,
         providerMode: params.providerMode,
@@ -67,14 +67,12 @@ function selectQaFlowSuiteScenarios(params: {
         channelDriver: params.channelDriver,
         channel: params.channel,
         claudeCliAuthMode: params.claudeCliAuthMode,
-      }).filter(
-        (mismatch) => mismatch.startsWith("channelDriver=") || mismatch.startsWith("channel="),
-      );
+      });
       return mismatches.length > 0 ? [`${scenario.id} (${mismatches.join(", ")})`] : [];
     });
-    if (channelDriverMismatches.length > 0) {
+    if (laneMismatches.length > 0) {
       throw new Error(
-        `selected QA scenario(s) do not match the current QA lane: ${channelDriverMismatches.join(", ")}`,
+        `selected QA scenario(s) do not match the current QA lane: ${laneMismatches.join(", ")}`,
       );
     }
     return selectedScenarios;