test(qa): prove gateway package self-upgrade (#109399)

This commit is contained in:
Dallin Romney
2026-07-17 12:26:35 -07:00
committed by GitHub
parent 1d999b817f
commit 71a02e16af
12 changed files with 1490 additions and 113 deletions
+54 -16
View File
@@ -1,5 +1,7 @@
// Qa Lab tests cover scenario catalog plugin behavior.
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import { resolveQaParityPackScenarioIds } from "./agentic-parity.js";
import {
@@ -10,6 +12,7 @@ import {
readQaScenarioPack,
validateQaScenarioExecutionConfig,
} from "./scenario-catalog.js";
import { runQaTestFileScenarios } from "./test-file-scenario-runner.js";
type CatalogScenario = ReturnType<typeof readQaScenarioPack>["scenarios"][number];
type FlowCatalogScenario = CatalogScenario & {
@@ -530,27 +533,62 @@ describe("qa scenario catalog", () => {
]);
});
it("loads the opt-in update.run package self-upgrade sentinel", () => {
it("loads the opt-in update.run package self-upgrade script proof", () => {
const scenario = readQaScenarioById("update-run-package-self-upgrade");
const config = readQaScenarioExecutionConfig(scenario.id) as
| {
requiredProviderMode?: string;
allowEnv?: string;
sourceVersion?: string;
targetTag?: string;
}
| undefined;
expect(scenario.sourcePath).toBe("qa/scenarios/runtime/update-run-package-self-upgrade.yaml");
expect(scenario.coverage?.primary).toContain("runtime.update-run");
expect(scenario.coverage?.secondary).toContain("runtime.package-update");
expect(config?.requiredProviderMode).toBe("live-frontier");
expect(config?.allowEnv).toBe("OPENCLAW_QA_ALLOW_UPDATE_RUN_SELF");
expect(config?.sourceVersion).toBe("2026.4.26");
expect(config?.targetTag).toBe("latest");
expect(scenario.execution.flow?.steps.map((step) => step.name)).toEqual([
"asks the agent to self-update through update.run",
]);
expect(scenario.execution.kind).toBe("script");
if (scenario.execution.kind !== "script") {
throw new Error(`expected script execution, got ${scenario.execution.kind}`);
}
expect(scenario.execution.path).toBe(
"test/e2e/qa-lab/runtime/update-run-package-self-upgrade.ts",
);
expect(scenario.execution.allowBlockedEvidence).toBe(true);
expect(scenario.execution.timeoutMs).toBe(3_600_000);
expect(scenario.execution.args).toEqual(["--artifact-base", "${outputDir}"]);
expect(scenario.execution.flow).toBeUndefined();
});
it("accepts the update.run producer's blocked evidence without destructive opt-in", async () => {
const outputDir = await fs.promises.mkdtemp(
path.join(os.tmpdir(), "openclaw-update-run-blocked-"),
);
try {
const result = await runQaTestFileScenarios({
repoRoot: process.cwd(),
outputDir,
providerMode: "mock-openai",
primaryModel: "mock-openai/gpt-5.6-luna",
scenarios: [readQaScenarioById("update-run-package-self-upgrade")],
env: {
OPENCLAW_QA_ALLOW_UPDATE_RUN_SELF: "0",
OPENCLAW_QA_REF: "blocked-evidence-test",
},
});
expect(result.results[0]).toMatchObject({
status: "pass",
producerEvidence: {
entries: [
{
test: { id: "update-run-package-self-upgrade" },
result: {
status: "blocked",
failure: {
reason:
"blocked destructive package self-upgrade; set OPENCLAW_QA_ALLOW_UPDATE_RUN_SELF=1 to run",
},
},
},
],
},
});
} finally {
await fs.promises.rm(outputDir, { recursive: true, force: true });
}
});
it("loads Codex plugin lifecycle scenarios into the standard runtime tier", () => {
+1
View File
@@ -1905,6 +1905,7 @@
"test:docker:update-corrupt-plugin": "bash scripts/e2e/update-corrupt-plugin-docker.sh",
"test:docker:update-migration": "env OPENCLAW_UPGRADE_SURVIVOR_PUBLISHED_BASELINE=1 OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC=${OPENCLAW_UPGRADE_SURVIVOR_BASELINE_SPEC:-openclaw@2026.4.23} OPENCLAW_UPGRADE_SURVIVOR_SCENARIO=${OPENCLAW_UPGRADE_SURVIVOR_SCENARIO:-plugin-deps-cleanup} bash scripts/e2e/upgrade-survivor-docker.sh",
"test:docker:update-restart-auth": "env OPENCLAW_UPGRADE_SURVIVOR_UPDATE_RESTART_MODE=auto-auth OPENCLAW_UPGRADE_SURVIVOR_DOCKER_RUN_TIMEOUT=${OPENCLAW_UPGRADE_SURVIVOR_DOCKER_RUN_TIMEOUT:-1500s} bash scripts/e2e/upgrade-survivor-docker.sh",
"test:docker:update-run-package-self-upgrade": "bash scripts/e2e/update-run-package-self-upgrade-docker.sh",
"test:docker:upgrade-survivor": "bash scripts/e2e/upgrade-survivor-docker.sh",
"test:env-mutations:report": "node --import tsx scripts/test-env-mutation-report.ts",
"test:skip-inventory:report": "node --import tsx scripts/test-skip-inventory.ts",
@@ -1,4 +1,4 @@
title: Update run package self-upgrade
title: Gateway update.run package self-upgrade
scenario:
id: update-run-package-self-upgrade
@@ -9,108 +9,30 @@ scenario:
secondary:
- runtime.gateway-restart
- runtime.package-update
objective: Verify an agent can self-update an installed OpenClaw package from 2026.4.26 to latest by using the gateway update.run action, then recover through the forced restart.
objective: Verify an installed OpenClaw 2026.4.26 package upgrades to the resolved latest package through the authenticated Gateway update.run RPC and returns healthy after restart.
successCriteria:
- The agent is explicitly instructed to use the gateway tool action update.run instead of shell package-manager commands.
- The update request carries a restart note marker that can be observed after the gateway restart.
- Gateway and qa-channel return healthy after update.run restarts the process.
- The package-backed Docker lane installs and verifies openclaw@2026.4.26 before invoking the authenticated Gateway update.run RPC directly.
- The lane resolves openclaw@latest before the RPC and rejects a no-op or any installed version that differs from the resolved target.
- The update RPC succeeds with source and target versions and writes the exact requested restart note into the update sentinel.
- The restarted Gateway reports the resolved target version, passes health, readiness, and RPC status probes, and reports the QA channel account running.
- Missing OPENCLAW_QA_ALLOW_UPDATE_RUN_SELF=1 produces blocked evidence without starting the destructive lane.
docsRefs:
- docs/cli/update.md
- docs/install/updating.md
- docs/gateway/protocol.md
- docs/help/testing-updates-plugins.md
codeRefs:
- src/agents/tools/gateway-tool.ts
- src/gateway/server-methods/update.ts
- src/infra/restart.ts
- scripts/e2e/update-run-package-self-upgrade-docker.sh
- scripts/e2e/lib/upgrade-survivor/update-run-package-self-upgrade.sh
- scripts/e2e/lib/upgrade-survivor/assertions.mjs
execution:
kind: flow
summary: "Opt-in destructive package-update lane: ask the agent to update a 2026.4.26 install to latest via gateway action update.run and verify the restart marker after recovery."
config:
requiredProviderMode: live-frontier
sourceVersion: "2026.4.26"
targetTag: latest
allowEnv: OPENCLAW_QA_ALLOW_UPDATE_RUN_SELF
channelId: qa-room
flow:
steps:
- name: asks the agent to self-update through update.run
actions:
- if:
expr: "env.gateway.runtimeEnv[config.allowEnv] !== '1'"
then:
- assert: "true"
else:
- call: waitForGatewayHealthy
args:
- ref: env
- 60000
- call: waitForQaChannelReady
args:
- ref: env
- 60000
- call: reset
- set: sessionKey
value:
expr: "buildAgentSessionKey({ agentId: 'qa', channel: 'qa-channel', peer: { kind: 'channel', id: config.channelId } })"
- call: createSession
args:
- ref: env
- Update run package self-upgrade
- ref: sessionKey
- call: readEffectiveTools
saveAs: tools
args:
- ref: env
- ref: sessionKey
- assert:
expr: "tools.has('gateway')"
message: gateway tool not present for update.run self-upgrade scenario
- set: startIndex
value:
expr: state.getSnapshot().messages.length
- set: marker
value:
expr: "`QA-UPDATE-RUN-${randomUUID().slice(0, 8)}`"
- call: startAgentRun
saveAs: started
args:
- ref: env
- sessionKey:
ref: sessionKey
to:
expr: "`channel:${config.channelId}`"
message:
expr: |-
`Update-run self-upgrade QA check. The OpenClaw package under test was installed from openclaw@${config.sourceVersion} and must update itself to openclaw@${config.targetTag}. Use the gateway tool with action=update.run. Do not run npm, pnpm, bun, git pull, or shell package-manager commands yourself. Set note exactly to "${marker} update.run complete" and restartDelayMs to 0 so the post-restart channel message proves recovery.`
timeoutMs:
expr: liveTurnTimeoutMs(env, 180000)
- call: waitForGatewayHealthy
args:
- ref: env
- 180000
- call: waitForQaChannelReady
args:
- ref: env
- 180000
- call: waitForOutboundMessage
saveAs: outbound
args:
- ref: state
- lambda:
params: [candidate]
expr: "candidate.text.includes(marker)"
- expr: liveTurnTimeoutMs(env, 180000)
- sinceIndex:
ref: startIndex
- call: env.gateway.call
saveAs: updateStatus
args:
- update.status
- {}
- timeoutMs: 30000
- assert:
expr: "Boolean(updateStatus?.sentinel)"
message:
expr: "`update.status did not report a restart sentinel after update.run: ${JSON.stringify(updateStatus)}`"
detailsExpr: "env.gateway.runtimeEnv[config.allowEnv] !== '1' ? `skipped destructive package self-update; set ${config.allowEnv}=1 to run` : `runId=${started.runId} marker=${marker} outbound=${outbound.text}`"
kind: script
path: test/e2e/qa-lab/runtime/update-run-package-self-upgrade.ts
summary: Runs the opt-in package-backed Docker lane for authenticated Gateway update.run self-upgrade and emits source/target, restart sentinel, Gateway health, and QA channel evidence.
allowBlockedEvidence: true
timeoutMs: 3600000
args:
- --artifact-base
- ${outputDir}
@@ -691,6 +691,161 @@ function assertStatusJson([file]) {
assert(/running|connected|ok|ready/u.test(text), "gateway status did not report a healthy state");
}
function parseStableVersion(version) {
const match = /^(\d+)\.(\d+)\.(\d+)(?:-(\d+))?$/u.exec(version ?? "");
assert(match, `invalid stable package version: ${String(version)}`);
return match.slice(1).map((part) => Number(part ?? 0));
}
function compareStableVersions(left, right) {
const leftParts = parseStableVersion(left);
const rightParts = parseStableVersion(right);
for (let index = 0; index < Math.max(leftParts.length, rightParts.length); index += 1) {
const difference = (leftParts[index] ?? 0) - (rightParts[index] ?? 0);
if (difference !== 0) {
return difference;
}
}
return 0;
}
function normalizeSystemctlInvocation(line) {
const parts = String(line ?? "")
.trim()
.split(/\s+/u)
.filter(Boolean);
const normalized = [];
for (let index = 0; index < parts.length; index += 1) {
const part = parts[index];
if (["--user", "--quiet", "--no-page", "--no-pager", "--now"].includes(part)) {
continue;
}
if (part === "--property") {
index += 1;
continue;
}
normalized.push(part);
}
return normalized.join(" ");
}
function assertUpdateRunSelfUpgrade([file]) {
assert(file, "assert-update-run-self-upgrade requires a summary path");
const summary = readJson(file);
const sourceVersion = summary?.source?.version;
const targetVersion = summary?.target?.resolvedVersion;
const updateRpc = summary?.updateRpcResult;
const sentinel = summary?.restartSentinel;
const qaChannelInstallRecord = summary?.qaChannelInstallRecord;
const targetQaChannelInstallRecord = summary?.targetPluginIndex?.installRecords?.["qa-channel"];
const gatewayStatus = summary?.gateway?.status;
const qaAccounts = summary?.qaChannel?.status?.channelAccounts?.["qa-channel"];
const targetServiceStarts = (summary?.supervisorHandoff?.systemctlInvocations ?? [])
.map(normalizeSystemctlInvocation)
.filter((invocation) => invocation === "start openclaw-gateway.service");
assert(summary?.status === "passed", "update.run self-upgrade summary did not pass");
assert(sourceVersion === "2026.4.26", `unexpected source version: ${String(sourceVersion)}`);
assert(summary?.source?.spec === "openclaw@2026.4.26", "source package spec was not exact");
assert(summary?.target?.tag === "latest", "target tag was not latest");
assert(
compareStableVersions(targetVersion, sourceVersion) > 0,
`target version did not advance beyond source: ${String(sourceVersion)} -> ${String(targetVersion)}`,
);
assert(
summary?.installedVersion === targetVersion,
`installed version mismatch: expected ${String(targetVersion)}, got ${String(summary?.installedVersion)}`,
);
assert(qaChannelInstallRecord?.source === "path", "QA channel was not path-installed");
assert(
typeof qaChannelInstallRecord?.sourcePath === "string" &&
qaChannelInstallRecord.sourcePath.includes("/extensions/qa-channel"),
"QA channel install record omitted its source path",
);
assert(
typeof qaChannelInstallRecord?.installPath === "string" &&
qaChannelInstallRecord.installPath.includes("/dist/extensions/qa-channel"),
"QA channel install record omitted its compiled local install path",
);
assert(
qaChannelInstallRecord?.version === "2026.4.25",
"QA channel install record version mismatch",
);
assert(
summary?.sourcePluginInspect?.plugin?.status === "loaded",
"source package did not load the compiled QA channel plugin",
);
assert(
targetQaChannelInstallRecord?.source === "path" &&
targetQaChannelInstallRecord?.installPath === qaChannelInstallRecord?.installPath,
"target SQLite index did not preserve the QA channel path install record",
);
assert(updateRpc?.ok === true, `update.run RPC did not report ok: ${JSON.stringify(updateRpc)}`);
assert(updateRpc?.result?.status === "ok", "update.run did not execute the package update");
assert(
updateRpc?.result?.before?.version === sourceVersion,
"update.run source version mismatch",
);
assert(updateRpc?.result?.after?.version === targetVersion, "update.run target version mismatch");
assert(
Array.isArray(updateRpc?.result?.steps) && updateRpc.result.steps.length > 0,
"update.run reported no executed update steps",
);
assert(updateRpc?.restart, "update.run did not schedule a Gateway restart");
assert(
updateRpc?.sentinel?.payload?.message === summary.expectedRestartNote,
"update.run response sentinel note mismatch",
);
assert(sentinel?.kind === "update", "final restart sentinel kind was not update");
assert(sentinel?.status === "ok", "final restart sentinel did not report ok");
assert(sentinel?.message === summary.expectedRestartNote, "final restart sentinel note mismatch");
assert(
sentinel?.stats?.before?.version === sourceVersion,
"restart sentinel source version mismatch",
);
assert(
sentinel?.stats?.after?.version === targetVersion,
"restart sentinel target version mismatch",
);
assert(
Number.isSafeInteger(summary?.supervisorHandoff?.servicePid) &&
summary.supervisorHandoff.servicePid > 1,
"supervisor handoff did not record the target service PID",
);
assert(targetServiceStarts.length === 1, "systemctl shim did not start the target exactly once");
assert(
summary?.supervisorHandoff?.monitorEvents?.some((line) =>
line.includes("source Gateway exited through supervised update handoff"),
),
"supervisor monitor did not prove the source supervised handoff",
);
assert(
summary?.gateway?.healthz?.body?.ok === true &&
summary?.gateway?.healthz?.body?.status === "live",
"post-restart /healthz was not live",
);
assert(summary?.gateway?.readyz?.body?.ready === true, "post-restart /readyz was not ready");
assert(
gatewayStatus?.rpc?.ok === true &&
gatewayStatus?.rpc?.version === targetVersion &&
gatewayStatus?.gateway?.version === targetVersion &&
gatewayStatus?.cli?.version === targetVersion,
`post-restart Gateway did not report target version ${String(targetVersion)}`,
);
assert(Array.isArray(qaAccounts), "post-restart channels.status omitted qa-channel");
assert(
qaAccounts.some((account) => account?.running === true && account?.restartPending !== true),
"post-restart QA channel account was not running",
);
assert(
Number(summary?.qaChannel?.busPollsAfterRestart) > 0,
"QA channel did not poll its bus after the target Gateway restart",
);
}
if (command === "list-scenarios") {
process.stdout.write(`${JSON.stringify([...SCENARIOS])}\n`);
} else if (command === "seed") {
@@ -702,6 +857,8 @@ if (command === "list-scenarios") {
assertConfiguredPluginInstalls();
} else if (command === "assert-status-json") {
assertStatusJson(process.argv.slice(3));
} else if (command === "assert-update-run-self-upgrade") {
assertUpdateRunSelfUpgrade(process.argv.slice(3));
} else {
throw new Error(`unknown upgrade-survivor assertion command: ${command ?? "<missing>"}`);
}
@@ -0,0 +1,73 @@
// Minimal QA bus poll endpoint for package/restart channel lifecycle proof.
import fs from "node:fs";
import http from "node:http";
function readOption(name, fallback) {
const index = process.argv.indexOf(name);
return index === -1 ? fallback : process.argv[index + 1];
}
const port = Number(readOption("--port", "43123"));
const readyFile = readOption("--ready-file", "");
const logFile = readOption("--log-file", "");
if (!Number.isInteger(port) || port < 1 || port > 65535) {
throw new Error(`invalid --port: ${String(port)}`);
}
if (!readyFile || !logFile) {
throw new Error("--ready-file and --log-file are required");
}
function appendEvent(event) {
fs.appendFileSync(logFile, `${JSON.stringify({ atMs: Date.now(), ...event })}\n`);
}
function writeJson(response, status, value) {
const body = JSON.stringify(value);
response.writeHead(status, {
"content-length": Buffer.byteLength(body),
"content-type": "application/json",
});
response.end(body);
}
const server = http.createServer((request, response) => {
if (request.method === "GET" && request.url === "/healthz") {
writeJson(response, 200, { ok: true });
return;
}
if (request.method !== "POST" || request.url !== "/v1/poll") {
writeJson(response, 404, { error: "not found" });
return;
}
const chunks = [];
request.on("data", (chunk) => chunks.push(Buffer.from(chunk)));
request.on("end", () => {
try {
const input = JSON.parse(Buffer.concat(chunks).toString("utf8"));
appendEvent({ accountId: input.accountId ?? null, path: "/v1/poll" });
setTimeout(() => {
if (!response.writableEnded) {
writeJson(response, 200, {
cursor: Number.isInteger(input.cursor) ? input.cursor : 0,
events: [],
});
}
}, 100);
} catch (error) {
writeJson(response, 400, {
error: error instanceof Error ? error.message : String(error),
});
}
});
});
server.listen(port, "127.0.0.1", () => {
fs.writeFileSync(readyFile, `${port}\n`);
});
function shutdown() {
server.close(() => process.exit(0));
}
process.on("SIGINT", shutdown);
process.on("SIGTERM", shutdown);
@@ -0,0 +1,563 @@
#!/usr/bin/env bash
set -Eeuo pipefail
source scripts/lib/openclaw-e2e-instance.sh
source scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh
if [ "${OPENCLAW_QA_ALLOW_UPDATE_RUN_SELF:-0}" != "1" ]; then
echo "blocked destructive package self-upgrade; set OPENCLAW_QA_ALLOW_UPDATE_RUN_SELF=1 to run" >&2
exit 2
fi
export CI=true
export OPENCLAW_NO_ONBOARD=1
export OPENCLAW_NO_PROMPT=1
export OPENCLAW_SKIP_PROVIDERS=1
export OPENCLAW_DISABLE_BONJOUR=1
export npm_config_audit=false
export npm_config_fund=false
export npm_config_loglevel=error
SOURCE_VERSION="${OPENCLAW_UPDATE_RUN_SELF_UPGRADE_SOURCE_VERSION:-2026.4.26}"
SOURCE_SPEC="openclaw@$SOURCE_VERSION"
TARGET_TAG="latest"
RESTART_NOTE="QA-UPDATE-RUN-PACKAGE-SELF-UPGRADE"
PORT=18789
QA_BUS_PORT=43123
ARTIFACT_DIR="${OPENCLAW_UPDATE_RUN_SELF_UPGRADE_ARTIFACT_DIR:-/tmp/openclaw-update-run-artifacts}"
RUNTIME_ROOT="${OPENCLAW_UPDATE_RUN_SELF_UPGRADE_RUNTIME_ROOT:-/tmp/openclaw-update-run-runtime}"
export HOME="$RUNTIME_ROOT/home"
export OPENCLAW_STATE_DIR="$HOME/.openclaw"
export OPENCLAW_CONFIG_PATH="$OPENCLAW_STATE_DIR/openclaw.json"
export OPENCLAW_TEST_WORKSPACE_DIR="$HOME/workspace"
export npm_config_prefix="$RUNTIME_ROOT/npm-prefix"
export NPM_CONFIG_PREFIX="$npm_config_prefix"
export npm_config_cache="$RUNTIME_ROOT/npm-cache"
export NPM_CONFIG_CACHE="$npm_config_cache"
export PATH="$npm_config_prefix/bin:$PATH"
BASELINE_INSTALL_LOG="$ARTIFACT_DIR/baseline-install.log"
PLUGIN_INSTALL_LOG="$ARTIFACT_DIR/plugin-install.log"
SOURCE_PLUGIN_INSPECT_JSON="$ARTIFACT_DIR/source-plugin-inspect.json"
QA_CHANNEL_INSTALL_RECORD_JSON="$ARTIFACT_DIR/qa-channel-install-record.json"
SOURCE_PLUGIN_INDEX_JSON="$ARTIFACT_DIR/source-plugin-index.json"
TARGET_PLUGIN_INDEX_JSON="$ARTIFACT_DIR/target-plugin-index.json"
TARGET_RESOLUTION_JSON="$ARTIFACT_DIR/target-resolution.json"
GATEWAY_LOG="$ARTIFACT_DIR/gateway.log"
QA_BUS_LOG="$ARTIFACT_DIR/qa-bus.jsonl"
QA_BUS_STDIO_LOG="$ARTIFACT_DIR/qa-bus.log"
QA_BUS_READY_FILE="$RUNTIME_ROOT/qa-bus.ready"
UPDATE_RPC_JSON="$ARTIFACT_DIR/update-rpc.json"
UPDATE_RPC_ERR="$ARTIFACT_DIR/update-rpc.err"
UPDATE_STATUS_JSON="$ARTIFACT_DIR/update-status.json"
UPDATE_STATUS_ERR="$ARTIFACT_DIR/update-status.err"
HEALTHZ_JSON="$ARTIFACT_DIR/healthz.json"
READYZ_JSON="$ARTIFACT_DIR/readyz.json"
GATEWAY_STATUS_JSON="$ARTIFACT_DIR/gateway-status.json"
GATEWAY_STATUS_ERR="$ARTIFACT_DIR/gateway-status.err"
CHANNELS_STATUS_JSON="$ARTIFACT_DIR/channels-status.json"
CHANNELS_STATUS_ERR="$ARTIFACT_DIR/channels-status.err"
SUMMARY_JSON="$ARTIFACT_DIR/summary.json"
SYSTEMCTL_SHIM_LOG="$ARTIFACT_DIR/systemctl-shim.log"
SYSTEMCTL_SHIM_SETUP_LOG="$ARTIFACT_DIR/systemctl-shim-setup.log"
SYSTEMCTL_SHIM_PID_FILE="$ARTIFACT_DIR/systemctl-shim.pid"
SYSTEMCTL_SHIM_DAEMON_LOG="$ARTIFACT_DIR/systemctl-shim-gateway.log"
SUPERVISOR_MONITOR_LOG="$ARTIFACT_DIR/supervisor-monitor.log"
SERVICE_INSTALL_JSON="$ARTIFACT_DIR/gateway-service-install.json"
SERVICE_INSTALL_ERR="$ARTIFACT_DIR/gateway-service-install.err"
SERVICE_UNIT_ARTIFACT="$ARTIFACT_DIR/openclaw-gateway.service"
export OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_LOG="$SYSTEMCTL_SHIM_LOG"
export OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_PID_FILE="$SYSTEMCTL_SHIM_PID_FILE"
export OPENCLAW_UPGRADE_SURVIVOR_SYSTEMCTL_SHIM_DAEMON_LOG="$SYSTEMCTL_SHIM_DAEMON_LOG"
gateway_pid=""
qa_bus_pid=""
supervisor_monitor_pid=""
mkdir -p \
"$ARTIFACT_DIR" \
"$HOME" \
"$OPENCLAW_STATE_DIR" \
"$OPENCLAW_TEST_WORKSPACE_DIR" \
"$npm_config_prefix" \
"$npm_config_cache"
rm -f \
"$QA_BUS_READY_FILE" \
"$QA_BUS_LOG" \
"$UPDATE_STATUS_JSON" \
"$UPDATE_STATUS_ERR" \
"$ARTIFACT_DIR/update-status.candidate.json" \
"$SUMMARY_JSON"
: >"$SYSTEMCTL_SHIM_DAEMON_LOG"
cleanup() {
if [ -n "${supervisor_monitor_pid:-}" ]; then
kill "$supervisor_monitor_pid" >/dev/null 2>&1 || true
wait "$supervisor_monitor_pid" >/dev/null 2>&1 || true
fi
openclaw_e2e_terminate_gateways "${gateway_pid:-}"
if [ -s "$SYSTEMCTL_SHIM_PID_FILE" ]; then
openclaw_e2e_terminate_gateways "$(cat "$SYSTEMCTL_SHIM_PID_FILE" 2>/dev/null || true)"
fi
if [ -n "${qa_bus_pid:-}" ]; then
kill "$qa_bus_pid" >/dev/null 2>&1 || true
wait "$qa_bus_pid" >/dev/null 2>&1 || true
fi
}
trap cleanup EXIT
package_root() {
printf '%s/lib/node_modules/openclaw\n' "$npm_config_prefix"
}
read_installed_version() {
node -p \
'JSON.parse(require("node:fs").readFileSync(process.argv[1] + "/package.json", "utf8")).version' \
"$(package_root)"
}
echo "Installing declared source package $SOURCE_SPEC"
openclaw_e2e_maybe_timeout 600s \
npm install -g --prefix "$npm_config_prefix" "$SOURCE_SPEC" --no-fund --no-audit \
>"$BASELINE_INSTALL_LOG" 2>&1
installed_source_version="$(read_installed_version)"
if [ "$installed_source_version" != "$SOURCE_VERSION" ]; then
echo "source package version mismatch: expected $SOURCE_VERSION, got $installed_source_version" >&2
exit 1
fi
if ! openclaw --version | grep -Fq "$SOURCE_VERSION"; then
echo "source openclaw --version did not report $SOURCE_VERSION" >&2
exit 1
fi
target_version="$(
npm view "openclaw@$TARGET_TAG" version --json --prefer-online --cache "$npm_config_cache" |
node -e '
let raw = "";
process.stdin.on("data", (chunk) => (raw += chunk));
process.stdin.on("end", () => {
const parsed = JSON.parse(raw);
if (typeof parsed !== "string" || !parsed.trim()) process.exit(1);
process.stdout.write(parsed.trim());
});
'
)"
if [ "$target_version" = "$SOURCE_VERSION" ]; then
echo "resolved target $TARGET_TAG is a no-op at $target_version" >&2
exit 1
fi
TARGET_VERSION="$target_version" TARGET_TAG="$TARGET_TAG" node -e '
const fs = require("node:fs");
fs.writeFileSync(
process.argv[1],
`${JSON.stringify({ tag: process.env.TARGET_TAG, version: process.env.TARGET_VERSION }, null, 2)}\n`,
);
' "$TARGET_RESOLUTION_JSON"
qa_plugin_source="/tmp/openclaw-update-run-build/dist/extensions/qa-channel"
qa_plugin_dir="$qa_plugin_source"
if [ ! -f "$qa_plugin_source/openclaw.plugin.json" ] || [ ! -f "$qa_plugin_source/index.js" ]; then
echo "compiled tagged QA channel fixture is missing" >&2
exit 1
fi
QA_PLUGIN_SOURCE="$qa_plugin_source" node -e '
const fs = require("node:fs");
const path = require("node:path");
const packageJson = JSON.parse(
fs.readFileSync(path.join(process.env.QA_PLUGIN_SOURCE, "package.json"), "utf8"),
);
const entries = [
...(packageJson.openclaw?.extensions ?? []),
packageJson.openclaw?.setupEntry,
].filter(Boolean);
if (entries.length === 0 || entries.some((entry) => /\.[cm]?ts$/u.test(entry))) {
throw new Error(`compiled QA channel retained TypeScript entrypoints: ${JSON.stringify(entries)}`);
}
for (const entry of entries) {
const relative = entry.replace(/^\.\//u, "");
if (!fs.existsSync(path.join(process.env.QA_PLUGIN_SOURCE, relative))) {
throw new Error(`compiled QA channel entry is missing: ${entry}`);
}
}
'
openclaw_e2e_maybe_timeout 300s \
openclaw plugins install "$qa_plugin_source" --link \
>"$PLUGIN_INSTALL_LOG" 2>&1
openclaw plugins inspect qa-channel --json >"$SOURCE_PLUGIN_INSPECT_JSON"
node -e '
const fs = require("node:fs");
const inspect = JSON.parse(fs.readFileSync(process.argv[1], "utf8"));
if (inspect?.plugin?.status !== "loaded") {
throw new Error(`source plugin inspection did not load qa-channel: ${JSON.stringify(inspect)}`);
}
' "$SOURCE_PLUGIN_INSPECT_JSON"
QA_PLUGIN_SOURCE="$qa_plugin_source" \
QA_PLUGIN_INSTALL="$qa_plugin_dir" \
QA_PLUGIN_RECORD_OUT="$QA_CHANNEL_INSTALL_RECORD_JSON" \
SOURCE_PLUGIN_INDEX_OUT="$SOURCE_PLUGIN_INDEX_JSON" \
node -e '
const fs = require("node:fs");
const indexPath = `${process.env.OPENCLAW_STATE_DIR}/plugins/installs.json`;
const index = JSON.parse(fs.readFileSync(indexPath, "utf8"));
const record = index.installRecords?.["qa-channel"];
if (
record?.source !== "path" ||
record.sourcePath !== process.env.QA_PLUGIN_SOURCE ||
record.installPath !== process.env.QA_PLUGIN_INSTALL ||
record.version !== "2026.4.25"
) {
throw new Error(`unexpected qa-channel install record: ${JSON.stringify(record)}`);
}
fs.writeFileSync(process.env.QA_PLUGIN_RECORD_OUT, `${JSON.stringify(record, null, 2)}\n`);
fs.copyFileSync(indexPath, process.env.SOURCE_PLUGIN_INDEX_OUT);
'
node scripts/e2e/lib/upgrade-survivor/mock-server.mjs \
--port "$QA_BUS_PORT" \
--ready-file "$QA_BUS_READY_FILE" \
--log-file "$QA_BUS_LOG" \
>"$QA_BUS_STDIO_LOG" 2>&1 &
qa_bus_pid="$!"
for _ in $(seq 1 100); do
if [ -s "$QA_BUS_READY_FILE" ]; then
break
fi
if ! kill -0 "$qa_bus_pid" 2>/dev/null; then
openclaw_e2e_print_log "$QA_BUS_STDIO_LOG" >&2
exit 1
fi
sleep 0.1
done
if [ ! -s "$QA_BUS_READY_FILE" ]; then
echo "timed out waiting for QA bus fixture" >&2
exit 1
fi
CONFIG_PATH="$OPENCLAW_CONFIG_PATH" \
QA_BUS_PORT="$QA_BUS_PORT" \
node -e '
const fs = require("node:fs");
const existing = fs.existsSync(process.env.CONFIG_PATH)
? JSON.parse(fs.readFileSync(process.env.CONFIG_PATH, "utf8"))
: {};
const config = {
...existing,
gateway: {
mode: "local",
port: 18789,
bind: "loopback",
auth: { mode: "token", token: "test-token" },
},
update: { channel: "stable" },
plugins: {
...existing.plugins,
allow: [...new Set([...(existing.plugins?.allow ?? []), "qa-channel"])],
entries: {
...existing.plugins?.entries,
"qa-channel": { enabled: true },
},
},
channels: {
"qa-channel": {
enabled: true,
baseUrl: `http://127.0.0.1:${process.env.QA_BUS_PORT}`,
botUserId: "openclaw",
botDisplayName: "OpenClaw QA",
allowFrom: ["*"],
pollTimeoutMs: 250,
},
},
};
fs.writeFileSync(process.env.CONFIG_PATH, `${JSON.stringify(config, null, 2)}\n`);
'
openclaw config validate >"$ARTIFACT_DIR/config-validate.log" 2>&1
install_update_restart_systemctl_shim
if ! openclaw_e2e_maybe_timeout 120s \
openclaw gateway install --force --json \
>"$SERVICE_INSTALL_JSON" 2>"$SERVICE_INSTALL_ERR"; then
echo "historical Gateway service install failed" >&2
openclaw_e2e_print_log "$SERVICE_INSTALL_ERR" >&2
exit 1
fi
service_unit="$HOME/.config/systemd/user/openclaw-gateway.service"
if [ ! -f "$service_unit" ] || ! grep -q '^ExecStart=' "$service_unit"; then
echo "historical Gateway install did not create a service unit" >&2
exit 1
fi
if grep -q 'OPENCLAW_SKIP_PROVIDERS' "$service_unit"; then
echo "service-owned target environment unexpectedly suppresses providers" >&2
exit 1
fi
if ! grep -q 'OPENCLAW_SYSTEMD_UNIT=openclaw-gateway.service' "$service_unit"; then
echo "service-owned target environment omitted its systemd marker" >&2
exit 1
fi
cp "$service_unit" "$SERVICE_UNIT_ARTIFACT"
systemctl --user stop openclaw-gateway.service
if systemctl --user is-active openclaw-gateway.service >/dev/null 2>&1; then
echo "setup service remained active before the update.run proof" >&2
exit 1
fi
if [ -s "$SYSTEMCTL_SHIM_PID_FILE" ]; then
echo "setup service PID remained recorded before the update.run proof" >&2
exit 1
fi
cp "$SYSTEMCTL_SHIM_LOG" "$SYSTEMCTL_SHIM_SETUP_LOG"
: >"$SYSTEMCTL_SHIM_LOG"
env \
OPENCLAW_SYSTEMD_UNIT=openclaw-gateway.service \
openclaw gateway --port "$PORT" --bind loopback --allow-unconfigured \
>"$GATEWAY_LOG" 2>&1 &
gateway_pid="$!"
printf '%s\n' "$gateway_pid" >"$SYSTEMCTL_SHIM_PID_FILE"
openclaw_e2e_wait_gateway_ready "$gateway_pid" "$GATEWAY_LOG" 360 "$PORT"
gateway_call() {
local method="$1"
local params="$2"
local output="$3"
local error_output="$4"
local timeout_ms="${5:-30000}"
openclaw gateway call "$method" \
--url "ws://127.0.0.1:$PORT" \
--token "test-token" \
--timeout "$timeout_ms" \
--json \
--params "$params" \
>"$output" 2>"$error_output"
}
gateway_call channels.status '{"probe":false,"timeoutMs":2000}' \
"$ARTIFACT_DIR/channels-status-before.json" \
"$ARTIFACT_DIR/channels-status-before.err"
update_params="$(
RESTART_NOTE="$RESTART_NOTE" node -e '
process.stdout.write(
JSON.stringify({
note: process.env.RESTART_NOTE,
restartDelayMs: 0,
timeoutMs: 1200000,
}),
);
'
)"
source_gateway_pid="$gateway_pid"
(
while kill -0 "$source_gateway_pid" >/dev/null 2>&1; do
source_state="$(ps -o stat= -p "$source_gateway_pid" 2>/dev/null | tr -d '[:space:]' || true)"
[[ "$source_state" == Z* ]] && break
sleep 0.1
done
if ! grep -q 'restart mode: update process respawn (supervisor restart)' "$GATEWAY_LOG"; then
echo "source Gateway did not select the supervised update handoff" >&2
exit 1
fi
if grep -q 'restart mode: update process respawn (spawned pid' "$GATEWAY_LOG"; then
echo "source Gateway unexpectedly detached-spawned a target" >&2
exit 1
fi
echo "source Gateway exited through supervised update handoff"
echo "starting installed service without provider suppression"
env \
-u OPENCLAW_SKIP_PROVIDERS \
systemctl --user start openclaw-gateway.service
service_pid="$(cat "$SYSTEMCTL_SHIM_PID_FILE" 2>/dev/null || true)"
[[ "$service_pid" =~ ^[0-9]+$ ]] || exit 1
echo "service Gateway started pid=$service_pid"
) >"$SUPERVISOR_MONITOR_LOG" 2>&1 &
supervisor_monitor_pid="$!"
echo "Invoking authenticated Gateway RPC update.run"
gateway_call update.run "$update_params" "$UPDATE_RPC_JSON" "$UPDATE_RPC_ERR" 1200000
update_rpc_completed_at_ms="$(node -e 'process.stdout.write(String(Date.now()))')"
if ! wait "$source_gateway_pid"; then
echo "historical Gateway did not exit cleanly for supervised update handoff" >&2
exit 1
fi
gateway_pid=""
if ! wait "$supervisor_monitor_pid"; then
echo "service monitor did not restart the target Gateway" >&2
openclaw_e2e_print_log "$SUPERVISOR_MONITOR_LOG" >&2
exit 1
fi
supervisor_monitor_pid=""
gateway_pid="$(cat "$SYSTEMCTL_SHIM_PID_FILE" 2>/dev/null || true)"
if ! [[ "$gateway_pid" =~ ^[0-9]+$ ]]; then
echo "target service Gateway PID was not recorded" >&2
exit 1
fi
openclaw_e2e_wait_gateway_ready "$gateway_pid" "$SYSTEMCTL_SHIM_DAEMON_LOG" 180 "$PORT"
deadline=$((SECONDS + 180))
update_status_candidate="$ARTIFACT_DIR/update-status.candidate.json"
while [ "$SECONDS" -lt "$deadline" ]; do
if gateway_call update.status '{}' "$update_status_candidate" "$UPDATE_STATUS_ERR"; then
if TARGET_VERSION="$target_version" RESTART_NOTE="$RESTART_NOTE" node -e '
const fs = require("node:fs");
const payload = JSON.parse(fs.readFileSync(process.argv[1], "utf8"));
const sentinel = payload?.sentinel;
if (
sentinel?.kind === "update" &&
sentinel.status === "ok" &&
sentinel.message === process.env.RESTART_NOTE &&
sentinel.stats?.after?.version === process.env.TARGET_VERSION
) {
process.exit(0);
}
process.exit(1);
' "$update_status_candidate"; then
mv "$update_status_candidate" "$UPDATE_STATUS_JSON"
break
fi
fi
sleep 1
done
if [ ! -f "$UPDATE_STATUS_JSON" ]; then
echo "timed out waiting for target Gateway update sentinel" >&2
openclaw_e2e_print_log "$UPDATE_STATUS_ERR" >&2
openclaw_e2e_print_log "$GATEWAY_LOG" >&2
openclaw_e2e_print_log "$SYSTEMCTL_SHIM_DAEMON_LOG" >&2
exit 1
fi
post_restart_observed_at_ms="$(node -e 'process.stdout.write(String(Date.now()))')"
deadline=$((SECONDS + 60))
while [ "$SECONDS" -lt "$deadline" ]; do
if POST_RESTART_AT_MS="$post_restart_observed_at_ms" node -e '
const fs = require("node:fs");
const lines = fs.existsSync(process.argv[1])
? fs.readFileSync(process.argv[1], "utf8").trim().split("\n").filter(Boolean)
: [];
const count = lines
.map((line) => JSON.parse(line))
.filter((event) => event.path === "/v1/poll" && event.atMs >= Number(process.env.POST_RESTART_AT_MS))
.length;
process.exit(count > 0 ? 0 : 1);
' "$QA_BUS_LOG"; then
break
fi
sleep 0.25
done
node scripts/e2e/lib/upgrade-survivor/probe-gateway.mjs \
--base-url "http://127.0.0.1:$PORT" \
--path /healthz \
--expect live \
--out "$HEALTHZ_JSON"
node scripts/e2e/lib/upgrade-survivor/probe-gateway.mjs \
--base-url "http://127.0.0.1:$PORT" \
--path /readyz \
--expect ready \
--out "$READYZ_JSON"
openclaw gateway status \
--url "ws://127.0.0.1:$PORT" \
--token "test-token" \
--timeout 30000 \
--json \
>"$GATEWAY_STATUS_JSON" 2>"$GATEWAY_STATUS_ERR"
gateway_call channels.status '{"probe":false,"timeoutMs":2000}' \
"$CHANNELS_STATUS_JSON" \
"$CHANNELS_STATUS_ERR"
TARGET_PLUGIN_INDEX_OUT="$TARGET_PLUGIN_INDEX_JSON" node --input-type=module -e '
import fs from "node:fs";
import { readPluginInstallIndex } from "./scripts/e2e/lib/plugin-index-sqlite.mjs";
const index = readPluginInstallIndex();
const record = index.installRecords?.["qa-channel"];
if (
record?.source !== "path" ||
record.installPath !== "/tmp/openclaw-update-run-build/dist/extensions/qa-channel"
) {
throw new Error(`target SQLite index omitted qa-channel path install: ${JSON.stringify(record)}`);
}
fs.writeFileSync(process.env.TARGET_PLUGIN_INDEX_OUT, `${JSON.stringify(index, null, 2)}\n`);
'
installed_version="$(read_installed_version)"
SOURCE_VERSION="$SOURCE_VERSION" \
SOURCE_SPEC="$SOURCE_SPEC" \
TARGET_TAG="$TARGET_TAG" \
TARGET_VERSION="$target_version" \
INSTALLED_VERSION="$installed_version" \
RESTART_NOTE="$RESTART_NOTE" \
UPDATE_RPC_COMPLETED_AT_MS="$update_rpc_completed_at_ms" \
POST_RESTART_OBSERVED_AT_MS="$post_restart_observed_at_ms" \
UPDATE_RPC_JSON="$UPDATE_RPC_JSON" \
UPDATE_STATUS_JSON="$UPDATE_STATUS_JSON" \
QA_CHANNEL_INSTALL_RECORD_JSON="$QA_CHANNEL_INSTALL_RECORD_JSON" \
TARGET_PLUGIN_INDEX_JSON="$TARGET_PLUGIN_INDEX_JSON" \
SOURCE_PLUGIN_INSPECT_JSON="$SOURCE_PLUGIN_INSPECT_JSON" \
SYSTEMCTL_SHIM_LOG="$SYSTEMCTL_SHIM_LOG" \
SUPERVISOR_MONITOR_LOG="$SUPERVISOR_MONITOR_LOG" \
SERVICE_PID="$gateway_pid" \
HEALTHZ_JSON="$HEALTHZ_JSON" \
READYZ_JSON="$READYZ_JSON" \
GATEWAY_STATUS_JSON="$GATEWAY_STATUS_JSON" \
CHANNELS_STATUS_JSON="$CHANNELS_STATUS_JSON" \
QA_BUS_LOG="$QA_BUS_LOG" \
SUMMARY_JSON="$SUMMARY_JSON" \
node -e '
const fs = require("node:fs");
const readJson = (file) => JSON.parse(fs.readFileSync(file, "utf8"));
const readJsonLines = (file) =>
fs.existsSync(file)
? fs.readFileSync(file, "utf8").trim().split("\n").filter(Boolean).map(JSON.parse)
: [];
const updateStatus = readJson(process.env.UPDATE_STATUS_JSON);
const readLines = (file) =>
fs.existsSync(file)
? fs.readFileSync(file, "utf8").split("\n").map((line) => line.trim()).filter(Boolean)
: [];
const postRestartAtMs = Number(process.env.POST_RESTART_OBSERVED_AT_MS);
const qaBusPollsAfterRestart = readJsonLines(process.env.QA_BUS_LOG).filter(
(event) => event.path === "/v1/poll" && event.atMs >= postRestartAtMs,
).length;
const summary = {
status: "passed",
source: {
spec: process.env.SOURCE_SPEC,
version: process.env.SOURCE_VERSION,
},
target: {
tag: process.env.TARGET_TAG,
resolvedVersion: process.env.TARGET_VERSION,
},
installedVersion: process.env.INSTALLED_VERSION,
expectedRestartNote: process.env.RESTART_NOTE,
updateRpcCompletedAtMs: Number(process.env.UPDATE_RPC_COMPLETED_AT_MS),
postRestartObservedAtMs: postRestartAtMs,
updateRpcResult: readJson(process.env.UPDATE_RPC_JSON),
restartSentinel: updateStatus.sentinel,
qaChannelInstallRecord: readJson(process.env.QA_CHANNEL_INSTALL_RECORD_JSON),
sourcePluginInspect: readJson(process.env.SOURCE_PLUGIN_INSPECT_JSON),
targetPluginIndex: readJson(process.env.TARGET_PLUGIN_INDEX_JSON),
supervisorHandoff: {
servicePid: Number(process.env.SERVICE_PID),
systemctlInvocations: readLines(process.env.SYSTEMCTL_SHIM_LOG),
monitorEvents: readLines(process.env.SUPERVISOR_MONITOR_LOG),
},
gateway: {
healthz: readJson(process.env.HEALTHZ_JSON),
readyz: readJson(process.env.READYZ_JSON),
status: readJson(process.env.GATEWAY_STATUS_JSON),
},
qaChannel: {
status: readJson(process.env.CHANNELS_STATUS_JSON),
busPollsAfterRestart: qaBusPollsAfterRestart,
},
};
fs.writeFileSync(process.env.SUMMARY_JSON, `${JSON.stringify(summary, null, 2)}\n`);
'
node scripts/e2e/lib/upgrade-survivor/assertions.mjs \
assert-update-run-self-upgrade \
"$SUMMARY_JSON"
echo "Gateway update.run package self-upgrade passed source=$SOURCE_VERSION target=$target_version installed=$installed_version note=$RESTART_NOTE."
+129
View File
@@ -0,0 +1,129 @@
#!/usr/bin/env bash
# Proves a published package can update itself through Gateway update.run and restart healthy.
set -euo pipefail
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
source "$ROOT_DIR/scripts/lib/docker-e2e-image.sh"
source "$ROOT_DIR/scripts/lib/docker-e2e-package.sh"
ALLOW_ENV="OPENCLAW_QA_ALLOW_UPDATE_RUN_SELF"
SOURCE_VERSION="2026.4.26"
SOURCE_TAG="v$SOURCE_VERSION"
SOURCE_COMMIT="be8c24633aaa7ef0425ae1178f096ee8dd6226c0"
if [ "${OPENCLAW_QA_ALLOW_UPDATE_RUN_SELF:-0}" != "1" ]; then
echo "blocked destructive package self-upgrade; set $ALLOW_ENV=1 to run" >&2
exit 2
fi
IMAGE_NAME="$(
docker_e2e_resolve_image \
"openclaw-update-run-package-self-upgrade-e2e" \
OPENCLAW_UPDATE_RUN_SELF_UPGRADE_E2E_IMAGE
)"
SKIP_BUILD="${OPENCLAW_UPDATE_RUN_SELF_UPGRADE_E2E_SKIP_BUILD:-0}"
DOCKER_RUN_TIMEOUT="${OPENCLAW_UPDATE_RUN_SELF_UPGRADE_DOCKER_RUN_TIMEOUT:-1800s}"
ARTIFACT_DIR="${OPENCLAW_UPDATE_RUN_SELF_UPGRADE_ARTIFACT_DIR:-$ROOT_DIR/.artifacts/update-run-package-self-upgrade}"
QA_CHANNEL_FIXTURE_ROOT="$(mktemp -d "${TMPDIR:-/tmp}/openclaw-update-run-qa-channel.XXXXXX")"
cleanup() {
rm -rf "$QA_CHANNEL_FIXTURE_ROOT"
}
trap cleanup EXIT
prepare_qa_channel_fixture() {
local source_repo="$ROOT_DIR"
local source_ref="$SOURCE_COMMIT"
local clone_root=""
local resolved_commit=""
local tag_commit=""
local tag_object=""
if ! git -C "$source_repo" cat-file -e "$SOURCE_COMMIT^{commit}" 2>/dev/null || \
! git -C "$source_repo" cat-file -e "$SOURCE_TAG^{commit}" 2>/dev/null; then
clone_root="$QA_CHANNEL_FIXTURE_ROOT/source"
git clone \
--depth=1 \
--filter=blob:none \
--single-branch \
--branch "$SOURCE_TAG" \
https://github.com/openclaw/openclaw.git \
"$clone_root"
source_repo="$clone_root"
source_ref="HEAD"
fi
resolved_commit="$(git -C "$source_repo" rev-parse "$source_ref^{commit}")"
tag_commit="$(git -C "$source_repo" rev-parse "$SOURCE_TAG^{commit}")"
tag_object="$(git -C "$source_repo" rev-parse "$SOURCE_TAG")"
if [ "$resolved_commit" != "$SOURCE_COMMIT" ] || [ "$tag_commit" != "$SOURCE_COMMIT" ]; then
echo "$SOURCE_TAG/source fixture resolved to unexpected commits: tag=$tag_commit source=$resolved_commit" >&2
return 1
fi
SOURCE_TAG="$SOURCE_TAG" \
SOURCE_COMMIT="$SOURCE_COMMIT" \
TAG_OBJECT="$tag_object" \
node -e '
const fs = require("node:fs");
fs.writeFileSync(
process.argv[1],
`${JSON.stringify({
tag: process.env.SOURCE_TAG,
tagObject: process.env.TAG_OBJECT,
commit: process.env.SOURCE_COMMIT,
buildCommand: "OPENCLAW_BUILD_PRIVATE_QA=1 corepack pnpm build:docker",
}, null, 2)}\n`,
);
' "$ARTIFACT_DIR/qa-channel-fixture-provenance.json"
local checkout_root="$QA_CHANNEL_FIXTURE_ROOT/checkout"
mkdir -p "$checkout_root"
git -C "$source_repo" archive "$source_ref" | tar -x -C "$checkout_root"
local package_json="$checkout_root/extensions/qa-channel/package.json"
local fixture_version
fixture_version="$(node -p 'JSON.parse(require("node:fs").readFileSync(process.argv[1], "utf8")).version' "$package_json")"
if [ "$fixture_version" != "2026.4.25" ]; then
echo "historical QA channel fixture version mismatch: expected 2026.4.25, got $fixture_version" >&2
return 1
fi
echo "Building the tagged QA channel with the shipped Docker build" | tee "$ARTIFACT_DIR/historical-qa-channel-build.log"
(
cd "$checkout_root"
COREPACK_ENABLE_DOWNLOAD_PROMPT=0 corepack pnpm install --frozen-lockfile
COREPACK_ENABLE_DOWNLOAD_PROMPT=0 OPENCLAW_BUILD_PRIVATE_QA=1 corepack pnpm build:docker
) >>"$ARTIFACT_DIR/historical-qa-channel-build.log" 2>&1
local compiled_plugin="$checkout_root/dist/extensions/qa-channel"
for required_file in package.json openclaw.plugin.json index.js setup-entry.js; do
if [ ! -f "$compiled_plugin/$required_file" ]; then
echo "shipped build omitted QA channel artifact $required_file" >&2
return 1
fi
done
}
mkdir -p "$ARTIFACT_DIR"
chmod -R a+rwX "$ARTIFACT_DIR" || true
prepare_qa_channel_fixture
docker_e2e_build_or_reuse \
"$IMAGE_NAME" \
update-run-package-self-upgrade \
"$ROOT_DIR/scripts/e2e/Dockerfile" \
"$ROOT_DIR" \
bare \
"$SKIP_BUILD"
echo "Running Gateway update.run package self-upgrade Docker E2E..."
docker_e2e_run_with_harness \
-e COREPACK_ENABLE_DOWNLOAD_PROMPT=0 \
-e OPENCLAW_QA_ALLOW_UPDATE_RUN_SELF=1 \
-e OPENCLAW_UPDATE_RUN_SELF_UPGRADE_ARTIFACT_DIR=/tmp/openclaw-update-run-artifacts \
-e OPENCLAW_UPDATE_RUN_SELF_UPGRADE_SOURCE_VERSION="$SOURCE_VERSION" \
-v "$ARTIFACT_DIR:/tmp/openclaw-update-run-artifacts" \
-v "$QA_CHANNEL_FIXTURE_ROOT/checkout:/tmp/openclaw-update-run-build:ro" \
"$IMAGE_NAME" \
timeout --kill-after=30s "$DOCKER_RUN_TIMEOUT" \
bash scripts/e2e/lib/upgrade-survivor/update-run-package-self-upgrade.sh
+8
View File
@@ -21,6 +21,8 @@ const rootManagedVpsUpgradeCommand =
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:root-managed-vps-upgrade";
const updateRestartAuthCommand =
"OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:update-restart-auth";
const updateRunPackageSelfUpgradeCommand =
"OPENCLAW_QA_ALLOW_UPDATE_RUN_SELF=1 OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:update-run-package-self-upgrade";
const CODEX_HARNESS_API_KEY_ENV = "OPENCLAW_LIVE_CODEX_HARNESS_AUTH=api-key";
const LIVE_RETRY_PATTERNS = [
@@ -165,6 +167,12 @@ function createPackageUpdateMaintenanceLanes() {
timeoutMs: 25 * 60 * 1000,
weight: 3,
}),
npmLane("update-run-package-self-upgrade", updateRunPackageSelfUpgradeCommand, {
resources: ["service"],
stateScenario: "upgrade-survivor",
timeoutMs: 45 * 60 * 1000,
weight: 3,
}),
];
}
@@ -0,0 +1,104 @@
import fs from "node:fs/promises";
import path from "node:path";
import { describe, expect, it } from "vitest";
import {
formatUpdateRunSelfUpgradeDetails,
parseUpdateRunSelfUpgradeOptions,
resolveUpdateRunSelfUpgradePermission,
} from "./update-run-package-self-upgrade.js";
describe("update.run package self-upgrade producer", () => {
it("requires an explicit destructive opt-in", () => {
expect(resolveUpdateRunSelfUpgradePermission({})).toEqual({
allowed: false,
reason:
"blocked destructive package self-upgrade; set OPENCLAW_QA_ALLOW_UPDATE_RUN_SELF=1 to run",
});
expect(
resolveUpdateRunSelfUpgradePermission({ OPENCLAW_QA_ALLOW_UPDATE_RUN_SELF: "1" }),
).toEqual({ allowed: true });
});
it("parses the evidence artifact directory", () => {
expect(
parseUpdateRunSelfUpgradeOptions(["--artifact-base", ".artifacts/update-run"]).artifactBase,
).toContain(".artifacts/update-run");
expect(() => parseUpdateRunSelfUpgradeOptions([])).toThrow("--artifact-base is required");
});
it("uses one supervised update handoff without a manual target restart", async () => {
const script = await fs.readFile(
path.join(
process.cwd(),
"scripts/e2e/lib/upgrade-survivor/update-run-package-self-upgrade.sh",
),
"utf8",
);
expect(script).toContain("source scripts/e2e/lib/upgrade-survivor/update-restart-auth.sh");
expect(script).toContain("-u OPENCLAW_SKIP_PROVIDERS");
expect(script).toContain("systemctl --user start openclaw-gateway.service");
expect(script).toContain("OPENCLAW_SYSTEMD_UNIT=openclaw-gateway.service");
expect(script).toContain("restart mode: update process respawn (supervisor restart)");
expect(script).toContain("service-owned target environment unexpectedly suppresses providers");
expect(script).not.toContain("target_gateway_pid");
expect(script).not.toContain("openclaw_e2e_stop_process");
expect(script).toContain("systemctl --user stop openclaw-gateway.service");
expect(script).toContain(': >"$SYSTEMCTL_SHIM_LOG"');
const runCleanup = script.slice(
script.indexOf("rm -f \\"),
script.indexOf(': >"$SYSTEMCTL_SHIM_DAEMON_LOG"'),
);
expect(runCleanup).toContain('"$UPDATE_STATUS_JSON"');
expect(runCleanup).toContain('"$ARTIFACT_DIR/update-status.candidate.json"');
expect(script.indexOf("openclaw gateway install --force --json")).toBeLessThan(
script.indexOf("OPENCLAW_SYSTEMD_UNIT=openclaw-gateway.service"),
);
});
it("gives update.run matching server and CLI timeout budgets", async () => {
const script = await fs.readFile(
path.join(
process.cwd(),
"scripts/e2e/lib/upgrade-survivor/update-run-package-self-upgrade.sh",
),
"utf8",
);
expect(script).toContain('local timeout_ms="${5:-30000}"');
expect(script).toContain('--timeout "$timeout_ms"');
expect(script).toContain(
'gateway_call update.run "$update_params" "$UPDATE_RPC_JSON" "$UPDATE_RPC_ERR" 1200000',
);
});
it("falls back to the exact tag clone when commit or tag objects are missing", async () => {
const script = await fs.readFile(
path.join(process.cwd(), "scripts/e2e/update-run-package-self-upgrade-docker.sh"),
"utf8",
);
expect(script).toContain(
'if ! git -C "$source_repo" cat-file -e "$SOURCE_COMMIT^{commit}" 2>/dev/null ||',
);
expect(script).toContain(
'! git -C "$source_repo" cat-file -e "$SOURCE_TAG^{commit}" 2>/dev/null',
);
});
it("formats the proven version transition and sentinel", () => {
expect(
formatUpdateRunSelfUpgradeDetails({
installedVersion: "2026.7.2",
source: { version: "2026.4.26" },
target: { resolvedVersion: "2026.7.2", tag: "latest" },
restartSentinel: {
message: "QA-UPDATE-RUN-PACKAGE-SELF-UPGRADE",
status: "ok",
},
}),
).toBe(
"source=2026.4.26; target=latest:2026.7.2; installed=2026.7.2; sentinel=ok:QA-UPDATE-RUN-PACKAGE-SELF-UPGRADE",
);
});
});
@@ -0,0 +1,221 @@
// Produces QA evidence for the destructive package-backed update.run Docker lane.
import { spawn } from "node:child_process";
import fs from "node:fs/promises";
import path from "node:path";
import { pathToFileURL } from "node:url";
import {
QA_EVIDENCE_FILENAME,
type QaEvidenceSummaryJson,
} from "../../../../extensions/qa-lab/api.js";
import { createQaScriptEvidenceWriter } from "./script-evidence.js";
const SOURCE_PATH = "test/e2e/qa-lab/runtime/update-run-package-self-upgrade.ts";
const SCENARIO_ID = "update-run-package-self-upgrade";
const ALLOW_ENV = "OPENCLAW_QA_ALLOW_UPDATE_RUN_SELF";
type ProducerOptions = {
artifactBase: string;
repoRoot: string;
};
type UpdateRunSelfUpgradeSummary = {
installedVersion?: string;
source?: { version?: string };
target?: { resolvedVersion?: string; tag?: string };
restartSentinel?: { message?: string; status?: string };
};
function formatErrorMessage(error: unknown) {
return error instanceof Error ? error.message : String(error);
}
export function parseUpdateRunSelfUpgradeOptions(args: string[]): ProducerOptions {
let artifactBase: string | undefined;
for (let index = 0; index < args.length; index += 1) {
const option = args[index];
const value = args[index + 1];
if (option !== "--artifact-base") {
throw new Error(`unknown argument: ${option}`);
}
if (!value || value.startsWith("--")) {
throw new Error("--artifact-base requires a value");
}
artifactBase = value;
index += 1;
}
if (!artifactBase) {
throw new Error("--artifact-base is required");
}
return { artifactBase: path.resolve(artifactBase), repoRoot: process.cwd() };
}
export function resolveUpdateRunSelfUpgradePermission(
env: NodeJS.ProcessEnv = process.env,
): { allowed: true } | { allowed: false; reason: string } {
if (env[ALLOW_ENV] === "1") {
return { allowed: true };
}
return {
allowed: false,
reason: `blocked destructive package self-upgrade; set ${ALLOW_ENV}=1 to run`,
};
}
export function formatUpdateRunSelfUpgradeDetails(summary: UpdateRunSelfUpgradeSummary) {
return [
`source=${summary.source?.version ?? "unknown"}`,
`target=${summary.target?.tag ?? "unknown"}:${summary.target?.resolvedVersion ?? "unknown"}`,
`installed=${summary.installedVersion ?? "unknown"}`,
`sentinel=${summary.restartSentinel?.status ?? "unknown"}:${summary.restartSentinel?.message ?? "missing"}`,
].join("; ");
}
async function runDockerLane(options: ProducerOptions, appendLog: (chunk: unknown) => void) {
const dockerRunDir = path.join(options.artifactBase, "docker-run");
const laneArtifactDir = path.join(options.artifactBase, "lane");
await fs.mkdir(dockerRunDir, { recursive: true });
await fs.mkdir(laneArtifactDir, { recursive: true });
return await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>(
(resolve, reject) => {
const child = spawn(process.execPath, ["scripts/test-docker-all.mjs"], {
cwd: options.repoRoot,
env: {
...process.env,
[ALLOW_ENV]: "1",
OPENCLAW_DOCKER_ALL_BUILD: "1",
OPENCLAW_DOCKER_ALL_DRY_RUN: "0",
OPENCLAW_DOCKER_ALL_LANES: SCENARIO_ID,
OPENCLAW_DOCKER_ALL_LOG_DIR: dockerRunDir,
OPENCLAW_DOCKER_ALL_PARALLELISM: "1",
OPENCLAW_DOCKER_ALL_PREFLIGHT: "1",
OPENCLAW_DOCKER_ALL_TIMINGS_FILE: path.join(dockerRunDir, "lane-timings.json"),
OPENCLAW_UPDATE_RUN_SELF_UPGRADE_ARTIFACT_DIR: laneArtifactDir,
},
stdio: ["ignore", "pipe", "pipe"],
});
child.on("error", reject);
child.stdout.on("data", (chunk: Buffer) => {
process.stdout.write(chunk);
appendLog(chunk);
});
child.stderr.on("data", (chunk: Buffer) => {
process.stderr.write(chunk);
appendLog(chunk);
});
child.on("exit", (code, signal) => resolve({ code, signal }));
},
);
}
async function runProducer(options: ProducerOptions): Promise<QaEvidenceSummaryJson> {
const writer = createQaScriptEvidenceWriter({
artifactBase: options.artifactBase,
logFileName: "update-run-package-self-upgrade.log",
primaryModel: "gateway/update.run",
providerMode: "mock-openai",
repoRoot: options.repoRoot,
target: {
codeRefs: [
SOURCE_PATH,
"scripts/e2e/update-run-package-self-upgrade-docker.sh",
"scripts/e2e/lib/upgrade-survivor/update-run-package-self-upgrade.sh",
"scripts/e2e/lib/upgrade-survivor/assertions.mjs",
"scripts/lib/docker-e2e-scenarios.mjs",
"src/gateway/server-methods/update.ts",
],
docsRefs: [
"docs/cli/update.md",
"docs/install/updating.md",
"docs/gateway/protocol.md",
"docs/help/testing-updates-plugins.md",
],
id: SCENARIO_ID,
primaryCoverageIds: ["runtime.update-run"],
secondaryCoverageIds: ["runtime.gateway-restart", "runtime.package-update"],
sourcePath: SOURCE_PATH,
title: "Gateway update.run package self-upgrade",
},
});
const startedAt = Date.now();
const permission = resolveUpdateRunSelfUpgradePermission();
if (!permission.allowed) {
writer.appendLog(`${permission.reason}\n`);
return await writer.write({
details: permission.reason,
durationMs: Math.max(1, Date.now() - startedAt),
status: "blocked",
});
}
try {
const result = await runDockerLane(options, (chunk) => writer.appendLog(chunk));
if (result.code !== 0 || result.signal) {
throw new Error(
`Docker lane ${SCENARIO_ID} failed: code=${String(result.code)} signal=${String(result.signal)}`,
);
}
const laneDir = path.join(options.artifactBase, "lane");
const summary = JSON.parse(
await fs.readFile(path.join(laneDir, "summary.json"), "utf8"),
) as UpdateRunSelfUpgradeSummary;
return await writer.write({
artifacts: [
{ kind: "summary", filePath: path.join("lane", "summary.json") },
{ kind: "rpc", filePath: path.join("lane", "update-rpc.json") },
{ kind: "sentinel", filePath: path.join("lane", "update-status.json") },
{
kind: "summary",
filePath: path.join("lane", "qa-channel-install-record.json"),
},
{ kind: "summary", filePath: path.join("lane", "source-plugin-index.json") },
{ kind: "summary", filePath: path.join("lane", "source-plugin-inspect.json") },
{ kind: "summary", filePath: path.join("lane", "target-plugin-index.json") },
{ kind: "log", filePath: path.join("lane", "historical-qa-channel-build.log") },
{
kind: "summary",
filePath: path.join("lane", "qa-channel-fixture-provenance.json"),
},
{ kind: "log", filePath: path.join("lane", "systemctl-shim.log") },
{ kind: "log", filePath: path.join("lane", "systemctl-shim-setup.log") },
{ kind: "log", filePath: path.join("lane", "supervisor-monitor.log") },
{ kind: "log", filePath: path.join("lane", "systemctl-shim-gateway.log") },
{ kind: "summary", filePath: path.join("lane", "openclaw-gateway.service") },
{ kind: "health", filePath: path.join("lane", "healthz.json") },
{ kind: "health", filePath: path.join("lane", "readyz.json") },
{ kind: "health", filePath: path.join("lane", "gateway-status.json") },
{ kind: "channel-status", filePath: path.join("lane", "channels-status.json") },
{ kind: "summary", filePath: path.join("docker-run", "summary.json") },
],
details: formatUpdateRunSelfUpgradeDetails(summary),
durationMs: Math.max(1, Date.now() - startedAt),
status: "pass",
});
} catch (error) {
const details = formatErrorMessage(error);
writer.appendLog(`\nfail: ${details}\n`);
return await writer.write({
details,
durationMs: Math.max(1, Date.now() - startedAt),
status: "fail",
});
}
}
async function main(argv: string[]) {
const evidence = await runProducer(parseUpdateRunSelfUpgradeOptions(argv));
const status = evidence.entries[0]?.result.status;
console.log(`Update run package self-upgrade evidence: ${QA_EVIDENCE_FILENAME}`);
console.log(`Update run package self-upgrade status: ${status}`);
return status === "pass" || status === "blocked" ? 0 : 1;
}
if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
main(process.argv.slice(2))
.then((exitCode) => {
process.exitCode = exitCode;
})
.catch((error: unknown) => {
console.error(formatErrorMessage(error));
process.exitCode = 1;
});
}
+12
View File
@@ -499,6 +499,17 @@ describe("scripts/lib/docker-e2e-plan", () => {
timeoutMs: 1_500_000,
weight: 3,
},
{
command:
"OPENCLAW_QA_ALLOW_UPDATE_RUN_SELF=1 OPENCLAW_SKIP_DOCKER_BUILD=1 pnpm test:docker:update-run-package-self-upgrade",
imageKind: "bare",
live: false,
name: "update-run-package-self-upgrade",
resources: ["docker", "npm", "service"],
stateScenario: "upgrade-survivor",
timeoutMs: 2_700_000,
weight: 3,
},
]);
expect(pluginsRuntimePlugins.lanes.map((lane) => lane.name)).toEqual(["plugins"]);
expect(pluginsRuntimeServices.lanes.map(summarizeLane)).toEqual([
@@ -656,6 +667,7 @@ describe("scripts/lib/docker-e2e-plan", () => {
"published-upgrade-survivor",
"root-managed-vps-upgrade",
"update-restart-auth",
"update-run-package-self-upgrade",
]);
expect(pluginsRuntime.lanes.map((lane) => lane.name)).toEqual([
"plugins",
@@ -154,6 +154,99 @@ function assertConfiguredPluginState(params: { installPath?: string } = {}): voi
}
}
function createUpdateRunSelfUpgradeSummary() {
const sourceVersion = "2026.4.26";
const targetVersion = "2026.7.2";
const note = "QA-UPDATE-RUN-PACKAGE-SELF-UPGRADE";
return {
status: "passed",
source: { spec: `openclaw@${sourceVersion}`, version: sourceVersion },
target: { tag: "latest", resolvedVersion: targetVersion },
installedVersion: targetVersion,
expectedRestartNote: note,
updateRpcResult: {
ok: true,
result: {
status: "ok",
before: { version: sourceVersion },
after: { version: targetVersion },
steps: [{ name: "package manager install" }],
},
restart: { scheduled: true },
sentinel: { payload: { message: note } },
},
restartSentinel: {
kind: "update",
status: "ok",
message: note,
stats: {
before: { version: sourceVersion },
after: { version: targetVersion },
},
},
qaChannelInstallRecord: {
source: "path",
sourcePath: "/tmp/source/dist/extensions/qa-channel",
installPath: "/tmp/source/dist/extensions/qa-channel",
version: "2026.4.25",
},
sourcePluginInspect: {
plugin: { id: "qa-channel", status: "loaded" },
},
targetPluginIndex: {
installRecords: {
"qa-channel": {
source: "path",
sourcePath: "/tmp/source/dist/extensions/qa-channel",
installPath: "/tmp/source/dist/extensions/qa-channel",
version: "2026.4.25",
},
},
},
supervisorHandoff: {
servicePid: 4242,
systemctlInvocations: ["--user start openclaw-gateway.service"],
monitorEvents: [
"source Gateway exited through supervised update handoff",
"starting installed service without provider suppression",
"service Gateway started pid=4242",
],
},
gateway: {
healthz: { body: { ok: true, status: "live" } },
readyz: { body: { ready: true } },
status: {
cli: { version: targetVersion },
gateway: { version: targetVersion },
rpc: { ok: true, version: targetVersion },
},
},
qaChannel: {
status: {
channelAccounts: {
"qa-channel": [{ accountId: "default", running: true, restartPending: false }],
},
},
busPollsAfterRestart: 2,
},
};
}
function assertUpdateRunSelfUpgrade(summary: ReturnType<typeof createUpdateRunSelfUpgradeSummary>) {
const root = mkdtempSync(join(tmpdir(), "openclaw-update-run-self-upgrade-"));
try {
const summaryPath = join(root, "summary.json");
writeJson(summaryPath, summary);
execFileSync(
process.execPath,
[ASSERTIONS_PATH, "assert-update-run-self-upgrade", summaryPath],
{ stdio: "pipe" },
);
} finally {
rmSync(root, { force: true, recursive: true });
}
}
describe("upgrade survivor assertions", () => {
it("lists the dependency-free scenario contract", () => {
const scenarios = JSON.parse(
@@ -240,4 +333,60 @@ describe("upgrade survivor assertions", () => {
rmSync(root, { force: true, recursive: true });
}
});
it("accepts executed update.run package transition and post-restart health evidence", () => {
expect(() => assertUpdateRunSelfUpgrade(createUpdateRunSelfUpgradeSummary())).not.toThrow();
});
it("rejects no-op update.run package transitions", () => {
const summary = createUpdateRunSelfUpgradeSummary();
summary.target.resolvedVersion = summary.source.version;
summary.installedVersion = summary.source.version;
summary.updateRpcResult.result.after.version = summary.source.version;
summary.restartSentinel.stats.after.version = summary.source.version;
summary.gateway.status.gateway.version = summary.source.version;
expect(() => assertUpdateRunSelfUpgrade(summary)).toThrow(/did not advance beyond source/);
});
it("rejects unsupported update.run paths that did not execute package steps", () => {
const summary = createUpdateRunSelfUpgradeSummary();
summary.updateRpcResult.ok = false;
summary.updateRpcResult.result.status = "skipped";
summary.updateRpcResult.result.steps = [];
expect(() => assertUpdateRunSelfUpgrade(summary)).toThrow(/did not report ok/);
});
it("rejects QA channel payloads without a canonical path install record", () => {
const summary = createUpdateRunSelfUpgradeSummary();
summary.qaChannelInstallRecord.source = "npm";
expect(() => assertUpdateRunSelfUpgrade(summary)).toThrow(/was not path-installed/);
});
it("rejects upgrades that lose the path install during SQLite migration", () => {
const summary = createUpdateRunSelfUpgradeSummary();
Reflect.deleteProperty(summary.targetPluginIndex.installRecords, "qa-channel");
expect(() => assertUpdateRunSelfUpgrade(summary)).toThrow(
/target SQLite index did not preserve/,
);
});
it("rejects source fixtures that were never runtime-loaded", () => {
const summary = createUpdateRunSelfUpgradeSummary();
summary.sourcePluginInspect.plugin.status = "error";
expect(() => assertUpdateRunSelfUpgrade(summary)).toThrow(/source package did not load/);
});
it("rejects duplicate target service starts during the supervised handoff", () => {
const summary = createUpdateRunSelfUpgradeSummary();
summary.supervisorHandoff.systemctlInvocations.push(
"--user --quiet start openclaw-gateway.service",
);
expect(() => assertUpdateRunSelfUpgrade(summary)).toThrow(/target exactly once/);
});
});