fix(ci): give vitest no-output retries the full watchdog window

Shards that pin a short no-output timeout (agents-core stripes) die fast
on the known warm-cache stall, but a cold Vitest module cache makes the
same imports legitimately silent for minutes - the in-job retry then
re-dies at the short limit without running a single test (observed 4x
on agents-core-runner-cli-2 today). Raise the retry attempt's window to
the 300s watchdog floor while keeping the fast first kill.
This commit is contained in:
Peter Steinberger
2026-07-16 16:42:01 +01:00
parent 9c08675852
commit 89a16c93b5
4 changed files with 38 additions and 1 deletions
+2 -1
View File
@@ -39,6 +39,7 @@ import {
resolveChangedTargetArgs,
shouldAcquireLocalHeavyCheckLock,
shouldRetryVitestNoOutputTimeout,
withRetryNoOutputTimeout,
writeVitestIncludeFile,
} from "./test-projects.test-support.mjs";
import { forceKillVitestProcessGroup } from "./vitest-process-group.mjs";
@@ -164,7 +165,7 @@ async function runLoggedVitestSpec(spec) {
let result = await runVitestSpec(spec);
if (result.noOutputTimedOut && !spec.watchMode && shouldRetryVitestNoOutputTimeout(spec.env)) {
console.error(`[test] retrying ${spec.config} after no-output timeout`);
result = await runVitestSpec(spec);
result = await runVitestSpec(withRetryNoOutputTimeout(spec));
}
const durationMs = performance.now() - startedAt;
if (result.noOutputTimedOut && result.signal) {
+3
View File
@@ -161,6 +161,9 @@ export function applyParallelVitestCachePaths<T extends { config: string; env: N
): Array<Omit<T, "env"> & { env: NodeJS.ProcessEnv }>;
export function shouldRetryVitestNoOutputTimeout(env?: Record<string, string | undefined>): boolean;
export function withRetryNoOutputTimeout<T extends { env?: Record<string, string | undefined> }>(
spec: T,
): T;
export function shouldAcquireLocalHeavyCheckLock(
runSpecs: Array<Pick<VitestRunSpec, "config" | "includePatterns" | "watchMode">>,
+21
View File
@@ -4578,6 +4578,27 @@ export function shouldRetryVitestNoOutputTimeout(env = process.env) {
return !["0", "false", "no", "off"].includes(value ?? "");
}
// Shards may pin a short no-output window so the known warm-cache stall dies
// fast (see AGENTS_CORE_RUNTIME_ENV in ci-node-test-plan.mjs). A cold Vitest
// module cache makes those same imports legitimately silent for minutes, so
// the retry attempt must get the full watchdog window or it re-dies at the
// short limit and the job fails without ever running a test.
const RETRY_NO_OUTPUT_TIMEOUT_FLOOR_MS = 300_000;
export function withRetryNoOutputTimeout(spec) {
const current = Number(spec.env?.[VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY]);
if (!Number.isFinite(current) || current <= 0 || current >= RETRY_NO_OUTPUT_TIMEOUT_FLOOR_MS) {
return spec;
}
return {
...spec,
env: {
...spec.env,
[VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY]: String(RETRY_NO_OUTPUT_TIMEOUT_FLOOR_MS),
},
};
}
export function createVitestRunSpecs(args, params = {}) {
const cwd = params.cwd ?? process.cwd();
const baseEnv = params.baseEnv ?? process.env;
+12
View File
@@ -27,6 +27,7 @@ import {
resolveChangedTargetArgs,
resolveParallelFullSuiteConcurrency,
shouldRetryVitestNoOutputTimeout,
withRetryNoOutputTimeout,
writeVitestIncludeFile,
} from "../../scripts/test-projects.test-support.mjs";
import { captureReaddirSyncCallsDuring } from "../../src/test-utils/fs-scan-assertions.js";
@@ -5000,6 +5001,17 @@ describe("scripts/test-projects Vitest stall watchdog", () => {
it("allows changed checks to disable automatic silent-run retries", () => {
expect(shouldRetryVitestNoOutputTimeout({})).toBe(true);
expect(shouldRetryVitestNoOutputTimeout({ CI: "true" })).toBe(false);
});
it("raises short shard no-output timeouts for the retry attempt", () => {
const spec = { env: { OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: "60000" } };
expect(withRetryNoOutputTimeout(spec).env.OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS).toBe("300000");
const generous = { env: { OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: "600000" } };
expect(withRetryNoOutputTimeout(generous)).toBe(generous);
const disabled = { env: { OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: "0" } };
expect(withRetryNoOutputTimeout(disabled)).toBe(disabled);
const unset = { env: {} };
expect(withRetryNoOutputTimeout(unset)).toBe(unset);
expect(shouldRetryVitestNoOutputTimeout({ GITHUB_ACTIONS: "true" })).toBe(false);
expect(shouldRetryVitestNoOutputTimeout({ OPENCLAW_VITEST_NO_OUTPUT_RETRY: "1" })).toBe(true);
expect(shouldRetryVitestNoOutputTimeout({ OPENCLAW_VITEST_NO_OUTPUT_RETRY: "0" })).toBe(false);