fix(qa): isolate script scenarios from flow gateways (#99736)

extensions/qa-lab/src/suite-launch.runtime.test.ts

@@ -374,6 +374,78 @@ describe("qa suite runtime launcher", () => {
     expect(runQaTestFileScenarios).toHaveBeenCalledTimes(1);
   });
 
+  it("runs script scenarios after flow Gateways stop without serializing Playwright", async () => {
+    const repoRoot = await makeTempRepo("qa-suite-script-isolation-");
+    let releaseFlow!: () => void;
+    let markFlowStarted!: () => void;
+    const flowStarted = new Promise<void>((resolve) => {
+      markFlowStarted = resolve;
+    });
+    const flowBlocked = new Promise<void>((resolve) => {
+      releaseFlow = resolve;
+    });
+    runQaFlowSuite.mockImplementationOnce(
+      async (params: { outputDir?: string; scenarioIds?: string[] } | undefined) => {
+        markFlowStarted();
+        await flowBlocked;
+        const outputDir = params?.outputDir ?? "/tmp/qa-flow";
+        const evidencePath = path.join(outputDir, "qa-evidence.json");
+        await writeEvidence(evidencePath);
+        const scenarioIds = params?.scenarioIds ?? ["channel-chat-baseline"];
+        return {
+          outputDir,
+          evidencePath,
+          reportPath: path.join(outputDir, "qa-suite-report.md"),
+          summaryPath: path.join(outputDir, "qa-suite-summary.json"),
+          report: "# QA Suite Report\n",
+          scenarios: scenarioIds.map((scenarioId) => ({
+            name: scenarioId,
+            status: "pass",
+            steps: [],
+          })),
+          watchUrl: "http://127.0.0.1:43124",
+        };
+      },
+    );
+
+    const runPromise = runQaSuite({
+      repoRoot,
+      outputDir: ".artifacts/qa-e2e/script-isolation",
+      concurrency: 8,
+      scenarioIds: [
+        "channel-chat-baseline",
+        "control-ui-chat-flow-playwright",
+        "docker-npm-onboard-channel-agent",
+      ],
+    });
+    await flowStarted;
+    await vi.waitFor(() => {
+      expect(runQaTestFileScenarios).toHaveBeenCalledTimes(1);
+    });
+
+    expect(runQaTestFileScenarios).toHaveBeenNthCalledWith(
+      1,
+      expect.objectContaining({
+        scenarios: [
+          expect.objectContaining({ execution: expect.objectContaining({ kind: "playwright" }) }),
+        ],
+      }),
+    );
+
+    releaseFlow();
+    await runPromise;
+
+    expect(runQaTestFileScenarios).toHaveBeenCalledTimes(2);
+    expect(runQaTestFileScenarios).toHaveBeenNthCalledWith(
+      2,
+      expect.objectContaining({
+        scenarios: [
+          expect.objectContaining({ execution: expect.objectContaining({ kind: "script" }) }),
+        ],
+      }),
+    );
+  });
+
   it("keeps multiple isolated flow scenarios in separate serial partitions", async () => {
     const repoRoot = await makeTempRepo("qa-suite-serial-isolated-");
     await runQaSuite({

extensions/qa-lab/src/suite-launch.runtime.ts

@@ -451,6 +451,7 @@ async function runUnifiedQaSuite(params: {
   const sharedFlowPartitionTasks: QaUnifiedPartitionTask[] = [];
   const isolatedFlowPartitionTasks: QaUnifiedPartitionTask[] = [];
   const testFilePartitionTasks: QaUnifiedPartitionTask[] = [];
+  const scriptPartitionTasks: QaUnifiedPartitionTask[] = [];
   if (params.plan.flowScenarios.length > 0) {
     const sharedFlowScenarios = params.plan.flowScenarios.filter(
       (scenario) => !scenarioRequiresIsolatedQaSuiteWorker(scenario),
@@ -535,13 +536,15 @@ async function runUnifiedQaSuite(params: {
       }
     }
   }
-  if (params.plan.testFileScenariosByKind.size > 0) {
-    testFilePartitionTasks.push({
+  const createTestFilePartitionTask = (
+    scenariosByKind: ReadonlyMap<QaTestFileExecutionKind, QaTestFileScenario[]>,
+  ) =>
+    ({
       weight: 1,
       run: async () => {
         const testFileEvidenceSummaries: QaEvidenceSummaryJson[] = [];
         const testFileScenarioResults: QaUnifiedPartitionResult["scenarioResults"] = [];
-        for (const [kind, testFileScenarios] of params.plan.testFileScenariosByKind) {
+        for (const [kind, testFileScenarios] of scenariosByKind) {
           const result = await runQaTestFileSuiteFromRuntime({
             runParams: {
               ...params.runParams,
@@ -566,14 +569,30 @@ async function runUnifiedQaSuite(params: {
           scenarioResults: testFileScenarioResults,
         };
       },
-    });
+    }) satisfies QaUnifiedPartitionTask;
+  const concurrentTestFileScenariosByKind = new Map(
+    [...params.plan.testFileScenariosByKind].filter(([kind]) => kind !== "script"),
+  );
+  if (concurrentTestFileScenariosByKind.size > 0) {
+    testFilePartitionTasks.push(createTestFilePartitionTask(concurrentTestFileScenariosByKind));
   }
-  const partitionTasks = [
+  const scriptScenarios = params.plan.testFileScenariosByKind.get("script");
+  if (scriptScenarios?.length) {
+    scriptPartitionTasks.push(createTestFilePartitionTask(new Map([["script", scriptScenarios]])));
+  }
+  const concurrentPartitionTasks = [
     ...sharedFlowPartitionTasks,
     ...testFilePartitionTasks,
     ...isolatedFlowPartitionTasks,
   ];
-  const partitionResults = await runWeightedUnifiedPartitionTasks(partitionTasks, concurrency);
+  const concurrentPartitionResults = await runWeightedUnifiedPartitionTasks(
+    concurrentPartitionTasks,
+    concurrency,
+  );
+  // Script scenarios may rebuild the checkout's shared dist tree. Wait until every
+  // flow Gateway has stopped so package postbuild cannot invalidate its loaded chunks.
+  const scriptPartitionResults = await runWeightedUnifiedPartitionTasks(scriptPartitionTasks, 1);
+  const partitionResults = [...concurrentPartitionResults, ...scriptPartitionResults];
   for (const partitionResult of partitionResults) {
     for (const scenarioResult of partitionResult.scenarioResults) {
       scenarioResultsById.set(scenarioResult.scenarioId, scenarioResult.result);