furyhawk/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-07-21 02:06:43 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf(ci): reuse CLI startup metadata cache (#109863)

Browse Source
This commit is contained in:
Peter Steinberger
2026-07-17 10:25:26 +01:00
committed by GitHub
parent 065223c438
commit 5024c100a3
5 changed files with 223 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/build-all.d.mts
+1
View File
@@ -23,6 +23,7 @@ export type BuildAllStep = {
restore?: "always";
runOnHit?: {
env?: NodeJS.ProcessEnv;
finalize?: "refresh";
};
};
};
scripts/build-all.mjs
+13 -2
View File
@@ -263,6 +263,15 @@ export const BUILD_ALL_STEPS = [
label: "write-cli-startup-metadata",
kind: "node",
args: ["--import", "tsx", "scripts/write-cli-startup-metadata.ts"],
cache: {
inputs: [
"scripts/write-cli-startup-metadata.ts",
"scripts/lib/cli-startup-root-help-bundle.ts",
],
outputs: ["dist/cli-startup-metadata.json"],
restore: "always",
runOnHit: { finalize: "refresh" },
},
},
];
@@ -791,9 +800,11 @@ export function restoreBuildAllStepCacheOutputs(cacheState, params = {}) {
}
export function finalizeBuildAllStepCache(step, cacheState, params = {}) {
if (params.reusedCache) {
if (params.reusedCache && step.cache?.runOnHit?.finalize !== "refresh") {
return restoreBuildAllStepCacheOutputs(cacheState, params);
}
// Validator-style cache hits may update a restored seed. Capture that result;
// restoring the old seed here would silently discard the validated refresh.
writeBuildAllStepCacheStamp(
step,
resolveBuildAllStepCacheStampState(step, cacheState, params),
@@ -869,7 +880,7 @@ if (isMainModule()) {
reusedCache = true;
stepToRun = cacheHitStep;
}
console.error(`[build-all] ${step.label}${reusedCache ? " (cached declarations)" : ""}`);
console.error(`[build-all] ${step.label}${reusedCache ? " (cache restored)" : ""}`);
const invocation = resolveBuildAllStep(stepToRun, { env: buildEnv });
const result = spawnSync(invocation.command, invocation.args, invocation.options);
const durationMs = performance.now() - startedAt;
scripts/write-cli-startup-metadata.ts
+136 -81
View File
@@ -75,6 +75,20 @@ type RootHelpRenderContext = Pick<RootHelpRenderOptions, "config" | "env">;
type Awaitable<T> = T | Promise<T>;
type SourceCommandHelpCommand = "browser" | "nodes" | "secrets" | PrecomputedSubcommandHelpCommand;
type SourceCommandHelpText = Record<SourceCommandHelpCommand, string>;
type ExistingCliStartupMetadata = {
rootHelpBundleSignature?: unknown;
generatorSignature?: unknown;
browserHelpSourceSignature?: unknown;
secretsHelpSourceSignature?: unknown;
nodesHelpSourceSignature?: unknown;
subcommandHelpSourceSignature?: unknown;
channelCatalogSignature?: unknown;
browserHelpText?: unknown;
secretsHelpText?: unknown;
nodesHelpText?: unknown;
subcommandHelpText?: unknown;
rootHelpText?: unknown;
};
type SpawnTextParentSignalState = {
done: boolean;
signal: NodeJS.Signals | null;
@@ -742,97 +756,138 @@ export async function writeCliStartupMetadata(options?: {
);
const channelOptions = dedupe([...CORE_CHANNEL_ORDER, ...channelCatalog.ids]);
let existing: ExistingCliStartupMetadata | undefined;
try {
const existing = JSON.parse(readFileSync(resolvedOutputPath, "utf8")) as {
rootHelpBundleSignature?: unknown;
generatorSignature?: unknown;
browserHelpSourceSignature?: unknown;
secretsHelpSourceSignature?: unknown;
nodesHelpSourceSignature?: unknown;
subcommandHelpSourceSignature?: unknown;
channelCatalogSignature?: unknown;
browserHelpText?: unknown;
secretsHelpText?: unknown;
nodesHelpText?: unknown;
subcommandHelpText?: unknown;
};
if (
bundleIdentity &&
existing.rootHelpBundleSignature === bundleIdentity.signature &&
existing.generatorSignature === generatorSignature &&
existing.browserHelpSourceSignature === browserHelpSourceSignature &&
existing.secretsHelpSourceSignature === secretsHelpSourceSignature &&
existing.nodesHelpSourceSignature === nodesHelpSourceSignature &&
existing.subcommandHelpSourceSignature === subcommandHelpSourceSignature &&
existing.channelCatalogSignature === channelCatalog.signature &&
typeof existing.browserHelpText === "string" &&
existing.browserHelpText.length > 0 &&
typeof existing.secretsHelpText === "string" &&
existing.secretsHelpText.length > 0 &&
typeof existing.nodesHelpText === "string" &&
existing.nodesHelpText.length > 0 &&
hasAllPrecomputedSubcommandHelpText(existing.subcommandHelpText)
) {
return;
}
existing = JSON.parse(readFileSync(resolvedOutputPath, "utf8")) as ExistingCliStartupMetadata;
} catch {
// Missing or malformed existing metadata means we should regenerate it.
}
const rootHelpTextPromise = (async () => {
try {
return await (options?.renderBundledRootHelpText ?? renderBundledRootHelpText)(
resolvedDistDir,
renderContext,
);
} catch {
// The spawnSync source fallback blocks the event loop; that is fine for
// this rare recovery path (missing/broken bundle) and only delays
// draining sibling render output, not its correctness.
return (options?.renderSourceRootHelpText ?? renderSourceRootHelpText)(renderContext);
}
})();
const reusableExisting =
existing?.generatorSignature === generatorSignature &&
existing.channelCatalogSignature === channelCatalog.signature
? existing
: undefined;
const reusableRootHelpText =
reusableExisting &&
bundleIdentity &&
reusableExisting.rootHelpBundleSignature === bundleIdentity.signature &&
typeof reusableExisting.rootHelpText === "string" &&
reusableExisting.rootHelpText.length > 0
? reusableExisting.rootHelpText
: undefined;
const reusableBrowserHelpText =
reusableExisting &&
reusableExisting.browserHelpSourceSignature === browserHelpSourceSignature &&
typeof reusableExisting.browserHelpText === "string" &&
reusableExisting.browserHelpText.length > 0
? reusableExisting.browserHelpText
: undefined;
const reusableSecretsHelpText =
reusableExisting &&
reusableExisting.secretsHelpSourceSignature === secretsHelpSourceSignature &&
typeof reusableExisting.secretsHelpText === "string" &&
reusableExisting.secretsHelpText.length > 0
? reusableExisting.secretsHelpText
: undefined;
const reusableNodesHelpText =
reusableExisting &&
reusableExisting.nodesHelpSourceSignature === nodesHelpSourceSignature &&
typeof reusableExisting.nodesHelpText === "string" &&
reusableExisting.nodesHelpText.length > 0
? reusableExisting.nodesHelpText
: undefined;
const reusableSubcommandHelpText =
reusableExisting &&
reusableExisting.subcommandHelpSourceSignature === subcommandHelpSourceSignature &&
hasAllPrecomputedSubcommandHelpText(reusableExisting.subcommandHelpText)
? (reusableExisting.subcommandHelpText as PrecomputedSubcommandHelpText)
: undefined;
if (
reusableRootHelpText &&
reusableBrowserHelpText &&
reusableSecretsHelpText &&
reusableNodesHelpText &&
reusableSubcommandHelpText
) {
return;
}
const rootHelpTextPromise = reusableRootHelpText
? Promise.resolve(reusableRootHelpText)
: (async () => {
try {
return await (options?.renderBundledRootHelpText ?? renderBundledRootHelpText)(
resolvedDistDir,
renderContext,
);
} catch {
// The spawnSync source fallback blocks the event loop; that is fine for
// this rare recovery path (missing/broken bundle) and only delays
// draining sibling render output, not its correctness.
return (options?.renderSourceRootHelpText ?? renderSourceRootHelpText)(renderContext);
}
})();
const hasCustomCommandRenderer =
options?.renderSourceBrowserHelpText ||
options?.renderSourceSecretsHelpText ||
options?.renderSourceNodesHelpText ||
options?.renderSourceSubcommandHelpTextRecord;
const commandHelpTextPromise = hasCustomCommandRenderer
? null
: renderSourceCommandHelpTextRecord(
["browser", "secrets", "nodes", ...PRECOMPUTED_SUBCOMMAND_HELP_COMMANDS],
renderContext,
);
const browserHelpTextPromise = commandHelpTextPromise
? commandHelpTextPromise.then((commandHelpText) => commandHelpText.browser)
: Promise.resolve(
(options?.renderSourceBrowserHelpText ?? renderSourceBrowserHelpText)(renderContext),
);
const secretsHelpTextPromise = commandHelpTextPromise
? commandHelpTextPromise.then((commandHelpText) => commandHelpText.secrets)
: Promise.resolve(
(options?.renderSourceSecretsHelpText ?? renderSourceSecretsHelpText)(renderContext),
);
const nodesHelpTextPromise = commandHelpTextPromise
? commandHelpTextPromise.then((commandHelpText) => commandHelpText.nodes)
: Promise.resolve(
(options?.renderSourceNodesHelpText ?? renderSourceNodesHelpText)(renderContext),
);
const subcommandHelpTextPromise = commandHelpTextPromise
? commandHelpTextPromise.then(
(commandHelpText) =>
Object.fromEntries(
PRECOMPUTED_SUBCOMMAND_HELP_COMMANDS.map((commandName) => [
commandName,
commandHelpText[commandName],
]),
) as PrecomputedSubcommandHelpText,
)
: Promise.resolve(
(options?.renderSourceSubcommandHelpTextRecord ?? renderSourceSubcommandHelpTextRecord)(
renderContext,
),
);
const sourceCommandsToRender: SourceCommandHelpCommand[] = [];
if (!reusableBrowserHelpText) {
sourceCommandsToRender.push("browser");
}
if (!reusableSecretsHelpText) {
sourceCommandsToRender.push("secrets");
}
if (!reusableNodesHelpText) {
sourceCommandsToRender.push("nodes");
}
if (!reusableSubcommandHelpText) {
sourceCommandsToRender.push(...PRECOMPUTED_SUBCOMMAND_HELP_COMMANDS);
}
const commandHelpTextPromise =
hasCustomCommandRenderer || sourceCommandsToRender.length === 0
? null
: renderSourceCommandHelpTextRecord(sourceCommandsToRender, renderContext);
const browserHelpTextPromise = reusableBrowserHelpText
? Promise.resolve(reusableBrowserHelpText)
: commandHelpTextPromise
? commandHelpTextPromise.then((commandHelpText) => commandHelpText.browser)
: Promise.resolve(
(options?.renderSourceBrowserHelpText ?? renderSourceBrowserHelpText)(renderContext),
);
const secretsHelpTextPromise = reusableSecretsHelpText
? Promise.resolve(reusableSecretsHelpText)
: commandHelpTextPromise
? commandHelpTextPromise.then((commandHelpText) => commandHelpText.secrets)
: Promise.resolve(
(options?.renderSourceSecretsHelpText ?? renderSourceSecretsHelpText)(renderContext),
);
const nodesHelpTextPromise = reusableNodesHelpText
? Promise.resolve(reusableNodesHelpText)
: commandHelpTextPromise
? commandHelpTextPromise.then((commandHelpText) => commandHelpText.nodes)
: Promise.resolve(
(options?.renderSourceNodesHelpText ?? renderSourceNodesHelpText)(renderContext),
);
const subcommandHelpTextPromise = reusableSubcommandHelpText
? Promise.resolve(reusableSubcommandHelpText)
: commandHelpTextPromise
? commandHelpTextPromise.then(
(commandHelpText) =>
Object.fromEntries(
PRECOMPUTED_SUBCOMMAND_HELP_COMMANDS.map((commandName) => [
commandName,
commandHelpText[commandName],
]),
) as PrecomputedSubcommandHelpText,
)
: Promise.resolve(
(options?.renderSourceSubcommandHelpTextRecord ?? renderSourceSubcommandHelpTextRecord)(
renderContext,
),
);
const [rootHelpText, browserHelpText, secretsHelpText, nodesHelpText, subcommandHelpText] =
await Promise.all([
rootHelpTextPromise,
test/scripts/build-all.test.ts
+46
View File
@@ -59,6 +59,10 @@ function withBuildCacheFixture(
}
>;
restore?: "always";
runOnHit?: {
env?: NodeJS.ProcessEnv;
finalize?: "refresh";
};
};
};
}) => void,
@@ -276,6 +280,21 @@ describe("resolveBuildAllStep", () => {
expect(step.cache?.outputs).toEqual(["dist/export-html"]);
});
it("restores startup metadata as a validator seed and refreshes it after validation", () => {
const step = getBuildAllStep("write-cli-startup-metadata");
expect(step.cache).toMatchObject({
inputs: [
"scripts/write-cli-startup-metadata.ts",
"scripts/lib/cli-startup-root-help-bundle.ts",
],
outputs: ["dist/cli-startup-metadata.json"],
restore: "always",
runOnHit: { finalize: "refresh" },
});
expect(resolveBuildAllStepOnCacheHit(step)).not.toBeNull();
});
});
describe("resolveBuildAllSteps", () => {
@@ -1015,6 +1034,33 @@ describe("resolveBuildAllStepCacheState", () => {
});
});
it("refreshes validator cache outputs after a cache-hit command updates them", () => {
withBuildCacheFixture(({ rootDir, outputPath, step }) => {
const refreshStep = {
...step,
cache: {
...step.cache,
restore: "always" as const,
runOnHit: { finalize: "refresh" as const },
},
};
const cacheState = resolveBuildAllStepCacheState(refreshStep, { rootDir });
writeBuildAllStepCacheStamp(refreshStep, cacheState, { rootDir });
const restorable = resolveBuildAllStepCacheState(refreshStep, { rootDir });
expect(restoreBuildAllStepCacheOutputs(restorable, { rootDir })).toBe(true);
fs.writeFileSync(outputPath, "validated refresh");
expect(
finalizeBuildAllStepCache(refreshStep, restorable, { rootDir, reusedCache: true }),
).toBe(true);
fs.rmSync(outputPath);
const refreshed = resolveBuildAllStepCacheState(refreshStep, { rootDir });
expect(restoreBuildAllStepCacheOutputs(refreshed, { rootDir })).toBe(true);
expect(fs.readFileSync(outputPath, "utf8")).toBe("validated refresh");
});
});
it("can cache only direct directory files for generated flat outputs", () => {
withBuildCacheFixture(({ rootDir, step }) => {
const nestedPath = path.join(rootDir, "dist/nested/output.d.ts");
test/scripts/write-cli-startup-metadata.test.ts
+27 -11
View File
@@ -626,6 +626,7 @@ describe("write-cli-startup-metadata", () => {
const extensionsDir = path.join(tempRoot, "extensions");
const outputPath = path.join(distDir, "cli-startup-metadata.json");
let renderCount = 0;
let commandRenderCount = 0;
writeStartupMetadataSourceSignatureFixture(tempRoot);
writeFixtureFile(distDir, "root-help-fixture.js", "export function outputRootHelp() {}\n");
@@ -640,17 +641,29 @@ describe("write-cli-startup-metadata", () => {
renderCount += 1;
return `Usage: openclaw ${renderCount}\n`;
},
renderSourceBrowserHelpText: () => "Usage: openclaw browser\n",
renderSourceSecretsHelpText: () => "Usage: openclaw secrets\n",
renderSourceNodesHelpText: () => "Usage: openclaw nodes\n",
renderSourceSubcommandHelpTextRecord: () => ({
doctor: "Usage: openclaw doctor\n",
gateway: "Usage: openclaw gateway\n",
models: "Usage: openclaw models\n",
plugins: "Usage: openclaw plugins\n",
sessions: "Usage: openclaw sessions\n",
tasks: "Usage: openclaw tasks\n",
}),
renderSourceBrowserHelpText: () => {
commandRenderCount += 1;
return "Usage: openclaw browser\n";
},
renderSourceSecretsHelpText: () => {
commandRenderCount += 1;
return "Usage: openclaw secrets\n";
},
renderSourceNodesHelpText: () => {
commandRenderCount += 1;
return "Usage: openclaw nodes\n";
},
renderSourceSubcommandHelpTextRecord: () => {
commandRenderCount += 1;
return {
doctor: "Usage: openclaw doctor\n",
gateway: "Usage: openclaw gateway\n",
models: "Usage: openclaw models\n",
plugins: "Usage: openclaw plugins\n",
sessions: "Usage: openclaw sessions\n",
tasks: "Usage: openclaw tasks\n",
};
},
});
};
@@ -662,6 +675,7 @@ describe("write-cli-startup-metadata", () => {
await writeMetadata();
await writeMetadata();
expect(renderCount).toBe(1);
expect(commandRenderCount).toBe(4);
writeFixtureFile(
distDir,
@@ -670,6 +684,7 @@ describe("write-cli-startup-metadata", () => {
);
await writeMetadata();
expect(renderCount).toBe(2);
expect(commandRenderCount).toBe(4);
writeFixtureFile(
distDir,
@@ -678,5 +693,6 @@ describe("write-cli-startup-metadata", () => {
);
await writeMetadata();
expect(renderCount).toBe(3);
expect(commandRenderCount).toBe(4);
});
});