mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 02:06:43 +00:00
perf(ci): run the deadcode Knip scans concurrently (#109151)
* perf(ci): run the deadcode Knip scans concurrently check-dependencies spent ~194s running seven Knip child processes strictly serially (three package scripts, each iterating its scans). The scans are independent processes over separate configs: both deadcode scripts now Promise.all their scans, the CI task launches the three package scripts concurrently with buffered logs, and the lane moves to the 16 vCPU class for core/memory headroom. The missing-exports-script contract violation now fails fast before launching scans. Local: unused-files 218% CPU, 37s wall. * test(ci): align prerelease plan with the 16 vCPU dependencies lane
This commit is contained in:
@@ -1820,7 +1820,9 @@ jobs:
|
||||
runner: blacksmith-16vcpu-ubuntu-2404
|
||||
- check_name: check-dependencies
|
||||
task: dependencies
|
||||
runner: blacksmith-4vcpu-ubuntu-2404
|
||||
# Concurrent Knip scans need cores and memory headroom; the wall
|
||||
# clock roughly halves for similar billed core-minutes.
|
||||
runner: blacksmith-16vcpu-ubuntu-2404
|
||||
- check_name: check-test-types
|
||||
task: test-types
|
||||
# Slowest static lane (~3.5min on 4 vCPU); extra cores shorten the
|
||||
@@ -1923,14 +1925,38 @@ jobs:
|
||||
dependencies)
|
||||
if has_package_script "deadcode:dependencies" &&
|
||||
has_package_script "deadcode:unused-files"; then
|
||||
pnpm deadcode:dependencies
|
||||
pnpm deadcode:unused-files
|
||||
# The three deadcode scripts spawn independent Knip scans over
|
||||
# separate configs; run them concurrently (with buffered logs
|
||||
# so output stays readable) instead of paying ~3 minutes of
|
||||
# serial scanning.
|
||||
dc_scripts=(deadcode:dependencies deadcode:unused-files)
|
||||
if has_package_script "deadcode:exports"; then
|
||||
pnpm deadcode:exports
|
||||
dc_scripts+=(deadcode:exports)
|
||||
elif [[ "$HISTORICAL_TARGET" != "true" ]]; then
|
||||
echo "Current CI targets must provide the deadcode:exports package script." >&2
|
||||
exit 1
|
||||
fi
|
||||
dc_pids=()
|
||||
dc_logs=()
|
||||
for dc in "${dc_scripts[@]}"; do
|
||||
dc_log="$(mktemp -t deadcode-log.XXXXXX)"
|
||||
dc_logs+=("$dc_log")
|
||||
pnpm "$dc" >"$dc_log" 2>&1 &
|
||||
dc_pids+=($!)
|
||||
done
|
||||
dc_failures=0
|
||||
for i in "${!dc_scripts[@]}"; do
|
||||
if wait "${dc_pids[$i]}"; then
|
||||
echo "[ok] ${dc_scripts[$i]}"
|
||||
else
|
||||
echo "::error title=${dc_scripts[$i]} failed::${dc_scripts[$i]} failed"
|
||||
dc_failures=1
|
||||
fi
|
||||
cat "${dc_logs[$i]}"
|
||||
done
|
||||
if [ "$dc_failures" -ne 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
elif [[ "$HISTORICAL_TARGET" == "true" ]] && has_package_script "deadcode:ci"; then
|
||||
pnpm deadcode:ci
|
||||
else
|
||||
|
||||
@@ -96,9 +96,16 @@ export function checkUnusedExports(output) {
|
||||
}
|
||||
|
||||
async function main() {
|
||||
for (const scan of KNIP_SCANS) {
|
||||
const ok = await runUnusedExportScan(scan);
|
||||
if (!ok) {
|
||||
// The scans are independent Knip child processes over separate configs;
|
||||
// running them concurrently cuts the lane's serial wall clock roughly 2x.
|
||||
const results = await Promise.all(
|
||||
KNIP_SCANS.map(async (scan) => ({
|
||||
scan,
|
||||
result: await runKnip([...scan.args, ...KNIP_COMMON_ARGS], { scanName: scan.name }),
|
||||
})),
|
||||
);
|
||||
for (const { scan, result } of results) {
|
||||
if (!reportUnusedExportScan(scan, result)) {
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
@@ -108,8 +115,7 @@ async function main() {
|
||||
);
|
||||
}
|
||||
|
||||
async function runUnusedExportScan(scan) {
|
||||
const result = await runKnip([...scan.args, ...KNIP_COMMON_ARGS], { scanName: scan.name });
|
||||
function reportUnusedExportScan(scan, result) {
|
||||
if (result.errorCode || result.status === null) {
|
||||
console.error(
|
||||
`deadcode ${scan.name} failed: ${result.errorCode ?? result.signal ?? "unknown"}${
|
||||
|
||||
@@ -91,9 +91,16 @@ export function checkKnipUnusedFileScanResult(result) {
|
||||
}
|
||||
|
||||
async function main() {
|
||||
for (const scan of KNIP_SCANS) {
|
||||
const ok = await runUnusedFileScan(scan);
|
||||
if (!ok) {
|
||||
// The scans are independent Knip child processes over separate configs;
|
||||
// running them concurrently halves the lane's serial wall clock.
|
||||
const results = await Promise.all(
|
||||
KNIP_SCANS.map(async (scan) => ({
|
||||
scan,
|
||||
result: await runKnip([...scan.args, ...KNIP_COMMON_ARGS], { scanName: scan.name }),
|
||||
})),
|
||||
);
|
||||
for (const { scan, result } of results) {
|
||||
if (!reportUnusedFileScan(scan, result)) {
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
@@ -101,8 +108,7 @@ async function main() {
|
||||
console.log("[deadcode] Knip production and full-tree unused-file checks passed with 0 entries.");
|
||||
}
|
||||
|
||||
async function runUnusedFileScan(scan) {
|
||||
const result = await runKnip([...scan.args, ...KNIP_COMMON_ARGS], { scanName: scan.name });
|
||||
function reportUnusedFileScan(scan, result) {
|
||||
const validation = checkKnipUnusedFileScanResult(result);
|
||||
if (validation.failureReason) {
|
||||
console.error(
|
||||
|
||||
@@ -1798,7 +1798,8 @@ describe("ci workflow guards", () => {
|
||||
expect(workflow.jobs["check-shard"].strategy.matrix.include).toContainEqual({
|
||||
check_name: "check-dependencies",
|
||||
task: "dependencies",
|
||||
runner: "blacksmith-4vcpu-ubuntu-2404",
|
||||
// Concurrent Knip scans need cores and memory headroom.
|
||||
runner: "blacksmith-16vcpu-ubuntu-2404",
|
||||
});
|
||||
expect(workflow.jobs["check-additional-shard"]["runs-on"]).toContain("matrix.runner");
|
||||
expect(workflow.jobs["check-additional-shard"].strategy.matrix.include).toContainEqual({
|
||||
@@ -2340,10 +2341,11 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}"
|
||||
scripts: ["deadcode:dependencies", "deadcode:unused-files", "deadcode:exports"],
|
||||
});
|
||||
expect(modern.status, modern.output).toBe(0);
|
||||
expect(modern.calls).toEqual([
|
||||
// The scripts launch concurrently; completion order is nondeterministic.
|
||||
expect(modern.calls.toSorted()).toEqual([
|
||||
"deadcode:dependencies",
|
||||
"deadcode:unused-files",
|
||||
"deadcode:exports",
|
||||
"deadcode:unused-files",
|
||||
]);
|
||||
|
||||
const frozenWithExports = runDependencyCheckFixture({
|
||||
@@ -2351,10 +2353,10 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}"
|
||||
scripts: ["deadcode:dependencies", "deadcode:unused-files", "deadcode:exports"],
|
||||
});
|
||||
expect(frozenWithExports.status, frozenWithExports.output).toBe(0);
|
||||
expect(frozenWithExports.calls).toEqual([
|
||||
expect(frozenWithExports.calls.toSorted()).toEqual([
|
||||
"deadcode:dependencies",
|
||||
"deadcode:unused-files",
|
||||
"deadcode:exports",
|
||||
"deadcode:unused-files",
|
||||
]);
|
||||
|
||||
const frozen = runDependencyCheckFixture({
|
||||
@@ -2367,14 +2369,16 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}"
|
||||
],
|
||||
});
|
||||
expect(frozen.status, frozen.output).toBe(0);
|
||||
expect(frozen.calls).toEqual(["deadcode:dependencies", "deadcode:unused-files"]);
|
||||
expect(frozen.calls.toSorted()).toEqual(["deadcode:dependencies", "deadcode:unused-files"]);
|
||||
|
||||
const currentWithoutExports = runDependencyCheckFixture({
|
||||
historicalTarget: false,
|
||||
scripts: ["deadcode:dependencies", "deadcode:unused-files"],
|
||||
});
|
||||
expect(currentWithoutExports.status).toBe(1);
|
||||
expect(currentWithoutExports.calls).toEqual(["deadcode:dependencies", "deadcode:unused-files"]);
|
||||
// The missing-script contract violation now fails fast before launching
|
||||
// the concurrent scans instead of wasting two Knip runs first.
|
||||
expect(currentWithoutExports.calls).toEqual([]);
|
||||
expect(currentWithoutExports.output).toContain(
|
||||
"Current CI targets must provide the deadcode:exports package script.",
|
||||
);
|
||||
@@ -2829,7 +2833,8 @@ printf '%s\n' "\${CURL_SUCCESS_IP:-203.0.113.7}"
|
||||
expect(checkShard.run).toContain('has_package_script "deadcode:dependencies"');
|
||||
expect(checkShard.run).toContain('has_package_script "deadcode:unused-files"');
|
||||
expect(checkShard.run).toContain('has_package_script "deadcode:exports"');
|
||||
expect(checkShard.run).toContain("pnpm deadcode:exports");
|
||||
// The concurrent launcher invokes scripts through the dc_scripts array.
|
||||
expect(checkShard.run).toContain("dc_scripts+=(deadcode:exports)");
|
||||
expect(checkShard.run).toContain(
|
||||
"Current CI targets must provide the deadcode:exports package script.",
|
||||
);
|
||||
|
||||
@@ -413,7 +413,8 @@ describe("scripts/lib/plugin-prerelease-test-plan.mjs", () => {
|
||||
).toEqual({
|
||||
check_name: "check-dependencies",
|
||||
task: "dependencies",
|
||||
runner: "blacksmith-4vcpu-ubuntu-2404",
|
||||
// Concurrent Knip scans need cores and memory headroom.
|
||||
runner: "blacksmith-16vcpu-ubuntu-2404",
|
||||
});
|
||||
expect(
|
||||
workflow.jobs["check-shard"].steps.find(
|
||||
|
||||
Reference in New Issue
Block a user