mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 02:06:43 +00:00
fix(ci): give vitest no-output retries the full watchdog window
Shards that pin a short no-output timeout (agents-core stripes) die fast on the known warm-cache stall, but a cold Vitest module cache makes the same imports legitimately silent for minutes - the in-job retry then re-dies at the short limit without running a single test (observed 4x on agents-core-runner-cli-2 today). Raise the retry attempt's window to the 300s watchdog floor while keeping the fast first kill.
This commit is contained in:
@@ -39,6 +39,7 @@ import {
|
||||
resolveChangedTargetArgs,
|
||||
shouldAcquireLocalHeavyCheckLock,
|
||||
shouldRetryVitestNoOutputTimeout,
|
||||
withRetryNoOutputTimeout,
|
||||
writeVitestIncludeFile,
|
||||
} from "./test-projects.test-support.mjs";
|
||||
import { forceKillVitestProcessGroup } from "./vitest-process-group.mjs";
|
||||
@@ -164,7 +165,7 @@ async function runLoggedVitestSpec(spec) {
|
||||
let result = await runVitestSpec(spec);
|
||||
if (result.noOutputTimedOut && !spec.watchMode && shouldRetryVitestNoOutputTimeout(spec.env)) {
|
||||
console.error(`[test] retrying ${spec.config} after no-output timeout`);
|
||||
result = await runVitestSpec(spec);
|
||||
result = await runVitestSpec(withRetryNoOutputTimeout(spec));
|
||||
}
|
||||
const durationMs = performance.now() - startedAt;
|
||||
if (result.noOutputTimedOut && result.signal) {
|
||||
|
||||
@@ -161,6 +161,9 @@ export function applyParallelVitestCachePaths<T extends { config: string; env: N
|
||||
): Array<Omit<T, "env"> & { env: NodeJS.ProcessEnv }>;
|
||||
|
||||
export function shouldRetryVitestNoOutputTimeout(env?: Record<string, string | undefined>): boolean;
|
||||
export function withRetryNoOutputTimeout<T extends { env?: Record<string, string | undefined> }>(
|
||||
spec: T,
|
||||
): T;
|
||||
|
||||
export function shouldAcquireLocalHeavyCheckLock(
|
||||
runSpecs: Array<Pick<VitestRunSpec, "config" | "includePatterns" | "watchMode">>,
|
||||
|
||||
@@ -4578,6 +4578,27 @@ export function shouldRetryVitestNoOutputTimeout(env = process.env) {
|
||||
return !["0", "false", "no", "off"].includes(value ?? "");
|
||||
}
|
||||
|
||||
// Shards may pin a short no-output window so the known warm-cache stall dies
|
||||
// fast (see AGENTS_CORE_RUNTIME_ENV in ci-node-test-plan.mjs). A cold Vitest
|
||||
// module cache makes those same imports legitimately silent for minutes, so
|
||||
// the retry attempt must get the full watchdog window or it re-dies at the
|
||||
// short limit and the job fails without ever running a test.
|
||||
const RETRY_NO_OUTPUT_TIMEOUT_FLOOR_MS = 300_000;
|
||||
|
||||
export function withRetryNoOutputTimeout(spec) {
|
||||
const current = Number(spec.env?.[VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY]);
|
||||
if (!Number.isFinite(current) || current <= 0 || current >= RETRY_NO_OUTPUT_TIMEOUT_FLOOR_MS) {
|
||||
return spec;
|
||||
}
|
||||
return {
|
||||
...spec,
|
||||
env: {
|
||||
...spec.env,
|
||||
[VITEST_NO_OUTPUT_TIMEOUT_ENV_KEY]: String(RETRY_NO_OUTPUT_TIMEOUT_FLOOR_MS),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createVitestRunSpecs(args, params = {}) {
|
||||
const cwd = params.cwd ?? process.cwd();
|
||||
const baseEnv = params.baseEnv ?? process.env;
|
||||
|
||||
@@ -27,6 +27,7 @@ import {
|
||||
resolveChangedTargetArgs,
|
||||
resolveParallelFullSuiteConcurrency,
|
||||
shouldRetryVitestNoOutputTimeout,
|
||||
withRetryNoOutputTimeout,
|
||||
writeVitestIncludeFile,
|
||||
} from "../../scripts/test-projects.test-support.mjs";
|
||||
import { captureReaddirSyncCallsDuring } from "../../src/test-utils/fs-scan-assertions.js";
|
||||
@@ -5000,6 +5001,17 @@ describe("scripts/test-projects Vitest stall watchdog", () => {
|
||||
it("allows changed checks to disable automatic silent-run retries", () => {
|
||||
expect(shouldRetryVitestNoOutputTimeout({})).toBe(true);
|
||||
expect(shouldRetryVitestNoOutputTimeout({ CI: "true" })).toBe(false);
|
||||
});
|
||||
|
||||
it("raises short shard no-output timeouts for the retry attempt", () => {
|
||||
const spec = { env: { OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: "60000" } };
|
||||
expect(withRetryNoOutputTimeout(spec).env.OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS).toBe("300000");
|
||||
const generous = { env: { OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: "600000" } };
|
||||
expect(withRetryNoOutputTimeout(generous)).toBe(generous);
|
||||
const disabled = { env: { OPENCLAW_VITEST_NO_OUTPUT_TIMEOUT_MS: "0" } };
|
||||
expect(withRetryNoOutputTimeout(disabled)).toBe(disabled);
|
||||
const unset = { env: {} };
|
||||
expect(withRetryNoOutputTimeout(unset)).toBe(unset);
|
||||
expect(shouldRetryVitestNoOutputTimeout({ GITHUB_ACTIONS: "true" })).toBe(false);
|
||||
expect(shouldRetryVitestNoOutputTimeout({ OPENCLAW_VITEST_NO_OUTPUT_RETRY: "1" })).toBe(true);
|
||||
expect(shouldRetryVitestNoOutputTimeout({ OPENCLAW_VITEST_NO_OUTPUT_RETRY: "0" })).toBe(false);
|
||||
|
||||
Reference in New Issue
Block a user