perf(sqlite): fast schema gates, background integrity verification, and downgrade protection (#110271)

* perf(sqlite): gate schema version before integrity scans and quarantine terminal open failures

* feat(gateway): refuse incompatible database schemas at boot and verify integrity in the background

* feat(update): refuse installing builds that cannot open the current databases

* docs: add database schema reference with version history and downgrade guidance

* test(update): tolerate optional runner options in git-mutation mocks

* refactor(update): split npm package-target metadata out of update-check

* chore: model verifier internals for knip and regenerate docs map

* chore(i18n): resync stale iOS localization catalog

* chore: allowlist read-only preflight and verifier raw SQLite access
This commit is contained in:
Peter Steinberger
2026-07-18 05:49:39 +01:00
committed by GitHub
parent 4e0cc187a2
commit a89f01abf4
46 changed files with 2875 additions and 531 deletions
@@ -7702,38 +7702,6 @@
"Configuração"
]
},
{
"locale": "de",
"source": "Show reasoning & tool activity",
"translations": [
"Denkprozess und Tool-Aktivität anzeigen",
"Gedankengang und Werkzeugaktivität anzeigen"
]
},
{
"locale": "th",
"source": "Show reasoning & tool activity",
"translations": [
"แสดงกระบวนการให้เหตุผลและการทำงานของเครื่องมือ",
"แสดงการให้เหตุผลและกิจกรรมของเครื่องมือ"
]
},
{
"locale": "tr",
"source": "Show reasoning & tool activity",
"translations": [
"Akıl yürütme ve araç etkinliğini göster",
"Akıl yürütmeyi ve araç etkinliğini göster"
]
},
{
"locale": "uk",
"source": "Show reasoning & tool activity",
"translations": [
"Показувати міркування та активність інструментів",
"Показувати міркування та дії інструментів"
]
},
{
"locale": "de",
"source": "Skill Workshop",
+136
View File
@@ -152457,6 +152457,142 @@
}
}
},
"Show reasoning & tool activity": {
"localizations": {
"en": {
"stringUnit": {
"state": "translated",
"value": "Show reasoning & tool activity"
}
},
"zh-CN": {
"stringUnit": {
"state": "translated",
"value": "显示推理和工具活动"
}
},
"zh-TW": {
"stringUnit": {
"state": "translated",
"value": "顯示推理與工具活動"
}
},
"pt-BR": {
"stringUnit": {
"state": "translated",
"value": "Mostrar raciocínio e atividade das ferramentas"
}
},
"de": {
"stringUnit": {
"state": "translated",
"value": "Denkprozess und Tool-Aktivität anzeigen"
}
},
"es": {
"stringUnit": {
"state": "translated",
"value": "Mostrar el razonamiento y la actividad de las herramientas"
}
},
"ja-JP": {
"stringUnit": {
"state": "translated",
"value": "推論とツールのアクティビティを表示"
}
},
"ko": {
"stringUnit": {
"state": "translated",
"value": "추론 및 도구 활동 표시"
}
},
"fr": {
"stringUnit": {
"state": "translated",
"value": "Afficher le raisonnement et lactivité des outils"
}
},
"hi": {
"stringUnit": {
"state": "translated",
"value": "तर्क और टूल गतिविधि दिखाएँ"
}
},
"ar": {
"stringUnit": {
"state": "translated",
"value": "إظهار الاستدلال ونشاط الأدوات"
}
},
"it": {
"stringUnit": {
"state": "translated",
"value": "Mostra il ragionamento e l'attività degli strumenti"
}
},
"tr": {
"stringUnit": {
"state": "translated",
"value": "Akıl yürütmeyi ve araç etkinliğini göster"
}
},
"uk": {
"stringUnit": {
"state": "translated",
"value": "Показувати міркування та активність інструментів"
}
},
"id": {
"stringUnit": {
"state": "translated",
"value": "Tampilkan penalaran & aktivitas alat"
}
},
"pl": {
"stringUnit": {
"state": "translated",
"value": "Pokaż tok rozumowania i aktywność narzędzi"
}
},
"th": {
"stringUnit": {
"state": "translated",
"value": "แสดงการให้เหตุผลและกิจกรรมของเครื่องมือ"
}
},
"vi": {
"stringUnit": {
"state": "translated",
"value": "Hiển thị quá trình suy luận và hoạt động của công cụ"
}
},
"nl": {
"stringUnit": {
"state": "translated",
"value": "Redenering en toolactiviteit tonen"
}
},
"fa": {
"stringUnit": {
"state": "translated",
"value": "نمایش استدلال و فعالیت ابزارها"
}
},
"ru": {
"stringUnit": {
"state": "translated",
"value": "Показывать ход рассуждений и действия инструментов"
}
},
"sv": {
"stringUnit": {
"state": "translated",
"value": "Visa resonemang och verktygsaktivitet"
}
}
}
},
"Show talk transcript": {
"localizations": {
"en": {
+1
View File
@@ -111,6 +111,7 @@ const rootEntries = [
"scripts/e2e/*.{js,mjs,ts}!",
"scripts/e2e/lib/**/{assertions,probe,mock-server}.{js,mjs,ts}!",
"src/audit/audit-event-writer.worker.ts!",
"src/state/openclaw-database-verify.worker.ts!",
"src/agents/model-provider-auth.worker.ts!",
// Split runtime loaded through a path assembled in subagent-registry.ts.
"src/agents/subagent-registry.runtime.ts!",
+2
View File
@@ -207,6 +207,8 @@ This includes retired MCP OAuth files under `<state-dir>/mcp-oauth/*.json`. Stop
## Shared state SQLite compaction
See [Database schemas](/reference/database-schemas) for schema versioning, integrity checks, and downgrade recovery.
`openclaw doctor --state-sqlite compact` is explicit offline maintenance for
the canonical shared state database at
`<state-dir>/state/openclaw.sqlite`. It does not accept an arbitrary database
+1
View File
@@ -1946,6 +1946,7 @@
"reference/prompt-caching",
"reference/api-usage-costs",
"reference/transcript-hygiene",
"reference/database-schemas",
"reference/memory-config",
"reference/rich-output-protocol",
"reference/session-management-compaction",
+12
View File
@@ -8638,6 +8638,18 @@ Do not edit it by hand; run `pnpm docs:map:gen`.
- H2: License
- H2: Related
## reference/database-schemas.md
- Route: /reference/database-schemas
- Headings:
- H2: Database layout
- H2: Versioning contract
- H2: Agent schema history
- H2: State schema history
- H2: Integrity checks
- H2: Downgrades are unsupported
- H3: Example: agent schema 11 to 9
## reference/device-models.md
- Route: /reference/device-models
+105
View File
@@ -0,0 +1,105 @@
---
summary: "OpenClaw SQLite database locations, schema versions, integrity checks, and downgrade recovery"
read_when:
- Diagnosing a newer database schema error
- Checking database compatibility before an update or downgrade
- Recovering a database for an older OpenClaw release
title: "Database schemas"
---
OpenClaw stores control-plane state in a global SQLite database and agent data in one SQLite database per agent. Schema migrations run forward when a database opens. Older OpenClaw builds refuse databases written by a newer schema.
## Database layout
| Scope | Default path | Contents |
| -------------------- | ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------- |
| Global control plane | `~/.openclaw/state/openclaw.sqlite` | Shared configuration state, registries, approvals, plugin state, and verification history |
| Per-agent data plane | `~/.openclaw/agents/<agentId>/agent/openclaw-agent.sqlite` | Sessions, transcripts, memory indexes, auth state, conversation state, and agent-scoped runtime state |
A few high-volume or lifecycle-specific features use dedicated SQLite stores, including the task registry and trajectory data.
## Versioning contract
Each database records its schema in two places:
- `PRAGMA user_version` is the SQLite schema version.
- The primary `schema_meta` row records `role`, `agent_id`, `schema_version`, and `app_version`. `app_version` is the OpenClaw build that last wrote the schema metadata.
OpenClaw applies forward-only migrations when it opens an older supported database. It refuses a database whose `user_version` is newer than the running build and reports a `newer schema version` error. The Gateway checks all registered databases before startup. `openclaw update` also refuses a package or source target whose declared schema support is older than an on-disk database. Target packages published before schema metadata was added cannot be preflighted.
Installing OpenClaw manually through npm bypasses the updater guard. Database open checks still refuse an incompatible build.
## Agent schema history
| Version | Change | First release |
| ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------- |
| 1 | Initial per-agent store ([#88349](https://github.com/openclaw/openclaw/pull/88349)) | `v2026.5.30-beta.1`, stable through `v2026.7.1` |
| 2 | Memory index identity ([#104449](https://github.com/openclaw/openclaw/pull/104449)) | `v2026.7.2-beta.1` |
| 4 | Sessions and transcripts moved into SQLite ([#98236](https://github.com/openclaw/openclaw/pull/98236)) | `v2026.7.2-beta.1` |
| 5-6 | Terminal freshness and state lifecycle ([#104859](https://github.com/openclaw/openclaw/pull/104859)) | `v2026.7.2-beta.1` |
| 7 | Per-entry lifecycle status projection ([#106151](https://github.com/openclaw/openclaw/pull/106151)) | `v2026.7.2-beta.1` |
| 8 | Per-transcript session provenance ([#106766](https://github.com/openclaw/openclaw/pull/106766)) | `v2026.7.2-beta.2` |
| 9 | `STRICT` tables ([#108663](https://github.com/openclaw/openclaw/pull/108663)) | `v2026.7.2-beta.2` |
| 10 | Materialized active transcript paths ([#108851](https://github.com/openclaw/openclaw/pull/108851)) | Unreleased |
| 11 | Leases, durable delivery, conversation addresses, and heartbeat outcomes ([#109636](https://github.com/openclaw/openclaw/pull/109636), [#95838](https://github.com/openclaw/openclaw/pull/95838), [#109999](https://github.com/openclaw/openclaw/pull/109999)) | Unreleased |
Version 3 was an unshipped development step folded into version 4.
## State schema history
| Version | Change | First release |
| ------- | -------------------------------------------------------------------------------------------------------- | ------------------- |
| 1 | Initial shared state database | `v2026.5.30-beta.1` |
| 2 | Metadata-only message audit events ([#103903](https://github.com/openclaw/openclaw/pull/103903)) | `v2026.7.2-beta.1` |
| 3 | `STRICT` tables and schema-drift hardening ([#108663](https://github.com/openclaw/openclaw/pull/108663)) | `v2026.7.2-beta.2` |
## Integrity checks
| When | Check |
| ------------------------------------------- | -------------------------------------------------------------------------------------------------- |
| Every open | Validate the `schema_meta` table and primary metadata row |
| Before a pending migration | Run a full integrity, foreign-key, role, schema, and index scan |
| Gateway background verifier | Run the full scan about once daily and record results in the global `database_verifications` table |
| Doctor, backup verification, and compaction | Run the full scan before accepting or rewriting the database |
The Gateway preflight reads schema headers only. The background verifier owns the slower full scan for databases that do not need migration.
## Downgrades are unsupported
Manual schema downgrades are for agents and operators who accept the risk. [Create and verify a backup](/cli/backup) before editing any database. Stop the Gateway and every process that can open the database.
The general procedure is:
1. Read the target release's schema and migrations.
2. In one transaction, drop every table, index, trigger, and column introduced after the target version.
3. Set `PRAGMA user_version` and `schema_meta.schema_version` to the target version.
4. Run the target release's full database verification before starting the Gateway.
### Example: agent schema 11 to 9
Schema 10 added the active transcript projection. Schema 11 added leases, durable delivery, conversation-address state, and heartbeat outcomes. QMD coordination uses rows in `state_leases`; there is no separate QMD table to preserve.
Run equivalent SQL against each affected per-agent database after inspecting the exact schema that wrote it:
```sql
BEGIN IMMEDIATE;
DROP TABLE IF EXISTS heartbeat_outcomes;
DROP TABLE IF EXISTS conversation_deliveries;
DROP TABLE IF EXISTS state_leases;
DROP TABLE IF EXISTS session_transcript_active_events;
ALTER TABLE session_transcript_index_state DROP COLUMN active_event_count;
ALTER TABLE session_transcript_index_state DROP COLUMN active_message_count;
ALTER TABLE conversations DROP COLUMN delivery_target;
PRAGMA user_version = 9;
UPDATE schema_meta
SET schema_version = 9,
updated_at = unixepoch('now') * 1000
WHERE meta_key = 'primary';
COMMIT;
```
This discards version 10-11 state, including in-flight delivery operations, leases, heartbeat outcomes, and the derived active transcript projection. A botched downgrade means restore from the verified backup.
+6
View File
@@ -1,6 +1,12 @@
{
"name": "openclaw",
"version": "2026.7.2",
"openclaw": {
"schemaVersions": {
"state": 3,
"agent": 11
}
},
"description": "Multi-channel AI gateway with extensible messaging integrations",
"keywords": [],
"homepage": "https://github.com/openclaw/openclaw#readme",
+4
View File
@@ -53,6 +53,10 @@ const rawSqliteAllowPathGroups = {
],
"agent auth profile read-only bootstrap": ["src/agents/auth-profiles/sqlite.ts"],
"read-only shared state database access": ["src/state/openclaw-state-db-readonly.ts"],
"read-only schema preflight and integrity verification access": [
"src/state/openclaw-database-preflight.ts",
"src/state/openclaw-database-verify.worker.ts",
],
"read-only SQLite status probes": [
"src/commands/doctor-db-bloat.ts",
"src/commands/status.scan.shared.ts",
+1
View File
@@ -113,6 +113,7 @@ const requiredPathGroups = [
"dist/agents/model-provider-auth.worker.js",
"dist/audit/audit-event-writer.worker.js",
"dist/config/sessions/session-transcript-reconcile.worker.js",
"dist/state/openclaw-database-verify.worker.js",
"dist/task-registry-control.runtime.js",
"dist/telegram-ingress-worker.runtime.js",
"dist/build-info.json",
+327 -20
View File
@@ -71,6 +71,9 @@ const legacyConfigRepairMocks = vi.hoisted(() => ({
const launchdUpdateCleanupMocks = vi.hoisted(() => ({
disableCurrentOpenClawUpdateLaunchdJob: vi.fn(async () => false),
}));
const databasePreflightMocks = vi.hoisted(() => ({
preflightOpenClawDatabaseSchemas: vi.fn(),
}));
const restartHealthTestControl = vi.hoisted(() => ({
snapshot: undefined as unknown,
}));
@@ -104,6 +107,11 @@ vi.mock("../infra/update-runner.js", async (importOriginal) => ({
runGatewayUpdate: vi.fn(),
}));
vi.mock("../state/openclaw-database-preflight.js", () => ({
OPENCLAW_DATABASE_SCHEMA_DOCS_URL: "https://docs.openclaw.ai/reference/database-schemas",
preflightOpenClawDatabaseSchemas: databasePreflightMocks.preflightOpenClawDatabaseSchemas,
}));
vi.mock("../infra/openclaw-root.js", () => ({
resolveOpenClawPackageRoot: vi.fn(),
resolveOpenClawPackageRootSync: vi.fn(() => process.cwd()),
@@ -177,12 +185,15 @@ vi.mock("../infra/update-check.js", () => ({
}
return 0;
}),
fetchNpmPackageTargetStatus: vi.fn(),
fetchNpmTagVersion: vi.fn(),
resolveExtendedStablePackage: vi.fn(),
resolveNpmChannelTag: vi.fn(),
}));
vi.mock("../infra/update-check-package-target.js", () => ({
fetchNpmPackageTargetStatus: vi.fn(),
}));
vi.mock("../infra/runtime-guard.js", () => ({
nodeVersionSatisfiesEngine,
parseSemver: (version: string | null) => {
@@ -416,11 +427,11 @@ const {
} = await import("../config/config.js");
const {
checkUpdateStatus,
fetchNpmPackageTargetStatus,
fetchNpmTagVersion,
resolveExtendedStablePackage,
resolveNpmChannelTag,
} = await import("../infra/update-check.js");
const { fetchNpmPackageTargetStatus } = await import("../infra/update-check-package-target.js");
const { CONTROL_PLANE_UPDATE_SENTINEL_META_ENV } =
await import("../infra/update-control-plane-sentinel.js");
const { runCommandWithTimeout, runExec } = await import("../process/exec.js");
@@ -757,7 +768,7 @@ describe("update-cli", () => {
allowGatewayActivation?: boolean;
} | void> = [];
vi.mocked(runGatewayUpdate).mockImplementationOnce(async (opts) => {
preparations.push(await opts?.beforeGitMutation?.());
preparations.push(await opts?.beforeGitMutation?.({}));
return result;
});
return preparations;
@@ -883,6 +894,10 @@ describe("update-cli", () => {
return child;
});
vi.mocked(defaultRuntime.exit).mockImplementation(() => {});
databasePreflightMocks.preflightOpenClawDatabaseSchemas.mockReturnValue({
incompatible: [],
indeterminate: [],
});
vi.mocked(resolveOpenClawPackageRoot).mockResolvedValue(process.cwd());
vi.mocked(readConfigFileSnapshot).mockResolvedValue(baseSnapshot);
vi.mocked(readSourceConfigBestEffort).mockResolvedValue(baseSnapshot.config);
@@ -891,11 +906,11 @@ describe("update-cli", () => {
tag: "latest",
version: "9999.0.0",
});
vi.mocked(fetchNpmPackageTargetStatus).mockResolvedValue({
target: "latest",
version: "9999.0.0",
vi.mocked(fetchNpmPackageTargetStatus).mockImplementation(async ({ target }) => ({
target,
version: /^\d/u.test(target) ? target : "9999.0.0",
nodeEngine: ">=22.19.0",
});
}));
vi.mocked(resolveExtendedStablePackage).mockResolvedValue({
status: "resolved",
selector: "extended-stable",
@@ -2491,6 +2506,260 @@ describe("update-cli", () => {
},
] as const)("updateCommand dry-run behavior: $name", runUpdateCliScenario);
it("refuses an incompatible package target before service stop or install", async () => {
mockPackageInstallStatus(createCaseDir("openclaw-schema-refusal"));
vi.mocked(fetchNpmPackageTargetStatus).mockResolvedValue({
target: "9999.0.0",
version: "9999.0.0",
nodeEngine: ">=22.19.0",
schemaVersions: { state: 3, agent: 9 },
});
databasePreflightMocks.preflightOpenClawDatabaseSchemas.mockReturnValue({
incompatible: [
{
kind: "agent",
path: "/tmp/openclaw/agents/main/agent/openclaw-agent.sqlite",
agentId: "main",
foundVersion: 11,
supportedVersion: 9,
writerAppVersion: "2026.7.2",
},
],
indeterminate: [],
});
await updateCommand({ yes: true });
expect(databasePreflightMocks.preflightOpenClawDatabaseSchemas).toHaveBeenCalledWith({
env: process.env,
supportedVersions: { state: 3, agent: 9 },
});
expect(serviceStop).not.toHaveBeenCalled();
expect(packageInstallCommandCall()).toBeUndefined();
expect(defaultRuntime.error).toHaveBeenCalledWith(
expect.stringContaining("agent database (agent main)"),
);
expect(defaultRuntime.exit).toHaveBeenCalledWith(1);
});
it("skips package schema preflight when target metadata is missing", async () => {
mockPackageInstallStatus(createCaseDir("openclaw-schema-missing"));
vi.mocked(fetchNpmPackageTargetStatus).mockResolvedValue({
target: "9999.0.0",
version: "9999.0.0",
nodeEngine: ">=22.19.0",
});
await updateCommand({ yes: true, restart: false });
expect(databasePreflightMocks.preflightOpenClawDatabaseSchemas).not.toHaveBeenCalled();
expect(packageInstallCommandCall()).toBeDefined();
});
it("refuses a package update when exact target metadata lookup fails", async () => {
mockPackageInstallStatus(createCaseDir("openclaw-schema-metadata-failure"));
vi.mocked(fetchNpmPackageTargetStatus).mockResolvedValue({
target: "9999.0.0",
version: null,
nodeEngine: null,
error: "registry timeout",
});
await updateCommand({ yes: true });
expect(databasePreflightMocks.preflightOpenClawDatabaseSchemas).not.toHaveBeenCalled();
expect(serviceStop).not.toHaveBeenCalled();
expect(packageInstallCommandCall()).toBeUndefined();
expect(defaultRuntime.error).toHaveBeenCalledWith(
expect.stringContaining("could not inspect exact package target openclaw@9999.0.0"),
);
expect(defaultRuntime.exit).toHaveBeenCalledWith(1);
});
it("continues a package update when target schemas are compatible", async () => {
mockPackageInstallStatus(createCaseDir("openclaw-schema-compatible"));
vi.mocked(fetchNpmPackageTargetStatus).mockResolvedValue({
target: "9999.0.0",
version: "9999.0.0",
nodeEngine: ">=22.19.0",
schemaVersions: { state: 3, agent: 11 },
});
await updateCommand({ yes: true, restart: false });
expect(databasePreflightMocks.preflightOpenClawDatabaseSchemas).toHaveBeenCalledTimes(2);
expect(packageInstallCommandCall()?.[0]).toContain("openclaw@9999.0.0");
});
it("reports an incompatible package target during dry-run", async () => {
mockPackageInstallStatus(createCaseDir("openclaw-schema-dry-run"));
vi.mocked(fetchNpmPackageTargetStatus).mockResolvedValue({
target: "9999.0.0",
version: "9999.0.0",
nodeEngine: ">=22.19.0",
schemaVersions: { state: 2, agent: 9 },
});
databasePreflightMocks.preflightOpenClawDatabaseSchemas.mockReturnValue({
incompatible: [
{
kind: "state",
path: "/tmp/openclaw/state/openclaw.sqlite",
foundVersion: 3,
supportedVersion: 2,
},
],
indeterminate: [],
});
await updateCommand({ dryRun: true });
const logs = vi
.mocked(defaultRuntime.log)
.mock.calls.map((call) => String(call[0]))
.join("\n");
expect(logs).toContain("Would refuse update: state database");
expect(logs).toContain("https://docs.openclaw.ai/reference/database-schemas");
expect(serviceStop).not.toHaveBeenCalled();
expect(packageInstallCommandCall()).toBeUndefined();
expect(defaultRuntime.exit).not.toHaveBeenCalledWith(1);
});
it("refuses an incompatible git target before stopping the service", async () => {
mockOwnedGitService();
serviceLoaded.mockResolvedValue(true);
vi.mocked(runGatewayUpdate).mockImplementationOnce(async (options) => {
await options?.beforeGitMutation?.({ schemaVersions: { state: 3, agent: 9 } });
return makeOkUpdateResult({ mode: "git" });
});
databasePreflightMocks.preflightOpenClawDatabaseSchemas.mockReturnValue({
incompatible: [
{
kind: "agent",
path: "/tmp/openclaw/agents/main/agent/openclaw-agent.sqlite",
foundVersion: 11,
supportedVersion: 9,
},
],
indeterminate: [],
});
await updateCommand({ yes: true });
expect(serviceStop).not.toHaveBeenCalled();
expect(serviceRestart).not.toHaveBeenCalled();
expect(defaultRuntime.exit).toHaveBeenCalledWith(1);
});
it("reports indeterminate package databases during dry-run", async () => {
mockPackageInstallStatus(createCaseDir("openclaw-schema-indeterminate-dry-run"));
vi.mocked(fetchNpmPackageTargetStatus).mockResolvedValue({
target: "9999.0.0",
version: "9999.0.0",
nodeEngine: ">=22.19.0",
schemaVersions: { state: 3, agent: 11 },
});
databasePreflightMocks.preflightOpenClawDatabaseSchemas.mockReturnValue({
incompatible: [],
indeterminate: [
{ kind: "state", path: "/tmp/openclaw/state/openclaw.sqlite", reason: "database busy" },
],
});
await updateCommand({ dryRun: true });
const logs = vi
.mocked(defaultRuntime.log)
.mock.calls.map((call) => String(call[0]))
.join("\n");
expect(logs).toContain(
"could not inspect state database /tmp/openclaw/state/openclaw.sqlite: database busy; retry once the gateway releases it",
);
});
it("refuses a package target that changes after the service stops", async () => {
mockPackageInstallStatus(process.cwd());
mockOwnedGitService();
serviceReadCommand.mockResolvedValue({
programArguments: ["node", path.join(process.cwd(), "dist", "index.js"), "gateway", "run"],
environment: { OPENCLAW_STATE_DIR: "/managed/state" },
});
serviceLoaded.mockResolvedValue(true);
vi.mocked(fetchNpmPackageTargetStatus).mockResolvedValue({
target: "9999.0.0",
version: "9999.0.0",
nodeEngine: ">=22.19.0",
schemaVersions: { state: 3, agent: 11 },
});
databasePreflightMocks.preflightOpenClawDatabaseSchemas
.mockReturnValueOnce({ incompatible: [], indeterminate: [] })
.mockReturnValueOnce({
incompatible: [
{
kind: "agent",
path: "/tmp/openclaw/agents/main/agent/openclaw-agent.sqlite",
foundVersion: 12,
supportedVersion: 11,
},
],
indeterminate: [],
});
await updateCommand({ yes: true });
expect(serviceStop).toHaveBeenCalledOnce();
expect(databasePreflightMocks.preflightOpenClawDatabaseSchemas.mock.calls[1]?.[0].env).toEqual(
expect.objectContaining({ OPENCLAW_STATE_DIR: "/managed/state" }),
);
expect(packageInstallCommandCall()).toBeUndefined();
expect(serviceRestart).toHaveBeenCalledOnce();
expect(defaultRuntime.exit).toHaveBeenCalledWith(1);
});
it("refuses a git target that changes after the service stops", async () => {
mockOwnedGitService();
serviceLoaded.mockResolvedValue(true);
vi.mocked(runGatewayUpdate).mockImplementationOnce(async (options) => {
await options?.beforeGitMutation?.({ schemaVersions: { state: 3, agent: 11 } });
return makeOkUpdateResult({ mode: "git" });
});
databasePreflightMocks.preflightOpenClawDatabaseSchemas
.mockReturnValueOnce({ incompatible: [], indeterminate: [] })
.mockReturnValueOnce({
incompatible: [],
indeterminate: [
{ kind: "agent", path: "/tmp/openclaw-agent.sqlite", reason: "database busy" },
],
});
await updateCommand({ yes: true });
expect(serviceStop).toHaveBeenCalledOnce();
expect(serviceRestart).toHaveBeenCalledOnce();
expect(defaultRuntime.exit).toHaveBeenCalledWith(1);
expect(serviceRestart.mock.invocationCallOrder[0]).toBeLessThan(
vi.mocked(defaultRuntime.exit).mock.invocationCallOrder.at(-1) ?? 0,
);
});
it("fails a post-stop git refusal when no managed service was running", async () => {
vi.mocked(runGatewayUpdate).mockImplementationOnce(async (options) => {
await options?.beforeGitMutation?.({ schemaVersions: { state: 3, agent: 11 } });
return makeOkUpdateResult({ mode: "git" });
});
databasePreflightMocks.preflightOpenClawDatabaseSchemas
.mockReturnValueOnce({ incompatible: [], indeterminate: [] })
.mockReturnValueOnce({
incompatible: [],
indeterminate: [{ kind: "state", path: "/tmp/openclaw.sqlite", reason: "database busy" }],
});
await updateCommand({ yes: true });
expect(serviceStop).not.toHaveBeenCalled();
expect(serviceRestart).not.toHaveBeenCalled();
expect(defaultRuntime.exit).toHaveBeenCalledWith(1);
});
it.each([
{
name: "table output",
@@ -2635,7 +2904,7 @@ describe("update-cli", () => {
expect(call?.tag).toBe(expectedTag);
}
} else {
expectPackageInstallSpec("openclaw@latest");
expectPackageInstallSpec("openclaw@9999.0.0");
}
if (expectedPersistedChannel !== undefined) {
@@ -2662,7 +2931,7 @@ describe("update-cli", () => {
});
await updateCommand({});
expectPackageInstallSpec("openclaw@latest");
expectPackageInstallSpec("openclaw@1.2.3-1");
});
it("installs the verified exact package and persists an explicit extended-stable channel", async () => {
@@ -2920,12 +3189,14 @@ describe("update-cli", () => {
await updateCommand({ yes: true });
expectPackageInstallSpec("openclaw@latest");
const preflightParams = vi.mocked(fetchNpmPackageTargetStatus).mock.calls[0]?.[0];
expectPackageInstallSpec("openclaw@9999.0.0");
const preflightParams = vi
.mocked(fetchNpmPackageTargetStatus)
.mock.calls.find(([params]) => params.target === "latest")?.[0];
expect(preflightParams).toEqual(
expect.objectContaining({
target: "latest",
spec: "openclaw@latest",
spec: "openclaw@9999.0.0",
cwd: process.cwd(),
}),
);
@@ -2963,7 +3234,7 @@ describe("update-cli", () => {
"Run `openclaw update` from a shell outside the gateway service, or stop the gateway service first and then update.",
].join("\n"),
);
expectPackageInstallSpec("openclaw@latest");
expectPackageInstallSpec("openclaw@9999.0.0");
});
it("refuses package updates from inherited gateway service env when --no-restart leaves the gateway running", async () => {
@@ -3129,6 +3400,10 @@ describe("update-cli", () => {
it("blocks package updates when the target requires a newer Node runtime", async () => {
mockPackageInstallStatus(createCaseDir("openclaw-update"));
vi.mocked(resolveNpmChannelTag).mockResolvedValue({
tag: "latest",
version: "2026.3.23-2",
});
vi.mocked(fetchNpmPackageTargetStatus).mockResolvedValue({
target: "latest",
version: "2026.3.23-2",
@@ -3155,7 +3430,7 @@ describe("update-cli", () => {
mockPackageInstallStatus(createCaseDir("openclaw-update"));
await updateCommand({ tag: "next" });
},
expectedSpec: "openclaw@next",
expectedSpec: "openclaw@9999.0.0",
},
{
name: "main shorthand",
@@ -3399,6 +3674,10 @@ describe("update-cli", () => {
readPackageVersion.mockImplementation(async (packageRoot: string) =>
packageRoot === pkgRoot ? "2026.4.21" : "1.0.0",
);
vi.mocked(resolveNpmChannelTag).mockResolvedValue({
tag: "latest",
version: "2026.4.21",
});
pathExists.mockImplementation(async (candidate: string) => {
try {
await fs.access(candidate);
@@ -3473,6 +3752,10 @@ describe("update-cli", () => {
);
await fs.writeFile(entryPath, "export {};\n", "utf-8");
await writePackageDistInventory(pkgRoot);
vi.mocked(resolveNpmChannelTag).mockResolvedValue({
tag: "latest",
version: "2026.4.21",
});
pathExists.mockImplementation(async (candidate: string) => {
try {
await fs.access(candidate);
@@ -3574,6 +3857,10 @@ describe("update-cli", () => {
);
await fs.writeFile(entryPath, "export {};\n", "utf-8");
await writePackageDistInventory(pkgRoot);
vi.mocked(resolveNpmChannelTag).mockResolvedValue({
tag: "latest",
version: "2026.4.21",
});
pathExists.mockImplementation(async (candidate: string) => {
try {
await fs.access(candidate);
@@ -3646,6 +3933,10 @@ describe("update-cli", () => {
);
await fs.writeFile(entryPath, "export {};\n", "utf-8");
await writePackageDistInventory(pkgRoot);
vi.mocked(resolveNpmChannelTag).mockResolvedValue({
tag: "latest",
version: "2026.5.14",
});
pathExists.mockImplementation(async (candidate: string) => {
try {
await fs.access(candidate);
@@ -4547,7 +4838,7 @@ describe("update-cli", () => {
await updateCommand({ yes: true, restart: false });
expectPackageInstallSpec("openclaw@latest");
expectPackageInstallSpec("openclaw@2026.4.23");
const doctorCall = doctorCommandCall();
expect(doctorCall?.[0][0]).toContain("node");
expect(doctorCall?.[0].slice(1)).toEqual([entryPath, "doctor", "--non-interactive"]);
@@ -4574,9 +4865,15 @@ describe("update-cli", () => {
await fs.mkdir(pkgRoot, { recursive: true });
await fs.writeFile(
path.join(pkgRoot, "package.json"),
JSON.stringify({ name: "openclaw", version: "1.0.0" }),
JSON.stringify({ name: "openclaw", version: "9999.0.0" }),
"utf-8",
);
for (const relativePath of TEST_BUNDLED_RUNTIME_SIDECAR_PATHS) {
const absolutePath = path.join(pkgRoot, relativePath);
await fs.mkdir(path.dirname(absolutePath), { recursive: true });
await fs.writeFile(absolutePath, "export {};\n", "utf-8");
}
await writePackageDistInventory(pkgRoot);
vi.mocked(runCommandWithTimeout).mockImplementation(async (argv) => {
if (Array.isArray(argv) && argv[0] === "npm" && argv[1] === "root" && argv[2] === "-g") {
@@ -4625,7 +4922,7 @@ describe("update-cli", () => {
"i",
"-g",
"--allow-scripts=openclaw",
"openclaw@latest",
"openclaw@9999.0.0",
"--no-fund",
"--no-audit",
"--loglevel=error",
@@ -4636,7 +4933,7 @@ describe("update-cli", () => {
"i",
"-g",
"--allow-scripts=openclaw",
"openclaw@latest",
"openclaw@9999.0.0",
"--omit=optional",
"--no-fund",
"--no-audit",
@@ -4715,7 +5012,7 @@ describe("update-cli", () => {
isOwningNpmCommand(argv[0], brewPrefix) &&
argv[1] === "i" &&
argv[2] === "-g" &&
argv.includes("openclaw@latest"),
argv.includes("openclaw@9999.0.0"),
);
const requiredInstallCall = requireValue(installCall, "brew npm install call");
@@ -4902,6 +5199,7 @@ describe("update-cli", () => {
serviceReadCommand.mockResolvedValue({
programArguments: [serviceNode, path.join(serviceRoot, "dist", "index.js"), "gateway"],
});
vi.mocked(resolveNpmChannelTag).mockResolvedValue({ tag: "latest", version: "2026.5.20" });
vi.mocked(fetchNpmPackageTargetStatus).mockResolvedValue({
target: "latest",
version: "2026.5.20",
@@ -4967,6 +5265,10 @@ describe("update-cli", () => {
programArguments: [serviceNode, entrypoint, "gateway"],
});
serviceLoaded.mockResolvedValue(true);
vi.mocked(resolveNpmChannelTag).mockResolvedValue({
tag: "latest",
version: "2026.5.20",
});
pathExists.mockImplementation(async (candidate: string) => {
try {
await fs.access(candidate);
@@ -5093,6 +5395,7 @@ describe("update-cli", () => {
programArguments: [serviceNode, entrypoint, "gateway"],
});
serviceLoaded.mockResolvedValue(true);
vi.mocked(resolveNpmChannelTag).mockResolvedValue({ tag: "latest", version: "2026.7.1" });
vi.mocked(fetchNpmPackageTargetStatus).mockResolvedValue({
target: "latest",
version: "2026.7.1",
@@ -5160,7 +5463,7 @@ describe("update-cli", () => {
await fs.mkdir(path.dirname(stageEntryPoint), { recursive: true });
await fs.writeFile(
path.join(stageRoot, "package.json"),
JSON.stringify({ name: "openclaw", version: "2026.5.20" }),
JSON.stringify({ name: "openclaw", version: "2026.7.1" }),
"utf-8",
);
await fs.writeFile(stageEntryPoint, "export {};\n", "utf-8");
@@ -5218,6 +5521,10 @@ describe("update-cli", () => {
programArguments: [serviceNode, entrypoint, "gateway"],
});
serviceLoaded.mockResolvedValue(true);
vi.mocked(resolveNpmChannelTag).mockResolvedValue({
tag: "latest",
version: "2026.5.20",
});
pathExists.mockImplementation(async (candidate: string) => {
try {
await fs.access(candidate);
+1 -1
View File
@@ -36,7 +36,7 @@ import {
import { parseStrictPositiveInteger } from "../../infra/parse-finite-number.js";
import { getSelfAndAncestorPidsSync } from "../../infra/restart-stale-pids.js";
import { nodeVersionSatisfiesEngine } from "../../infra/runtime-guard.js";
import { fetchNpmPackageTargetStatus } from "../../infra/update-check.js";
import { fetchNpmPackageTargetStatus } from "../../infra/update-check-package-target.js";
import { canResolveRegistryVersionForPackageTarget } from "../../infra/update-global.js";
import type { UpdateRunResult } from "../../infra/update-runner.js";
import { runCommandWithTimeout } from "../../process/exec.js";
+209 -55
View File
@@ -37,6 +37,7 @@ import {
normalizeUpdateChannel,
type UpdateChannel,
} from "../../infra/update-channels.js";
import { fetchNpmPackageTargetStatus } from "../../infra/update-check-package-target.js";
import {
checkUpdateStatus,
compareSemverStrings,
@@ -71,6 +72,14 @@ import {
} from "../../infra/update-runner.js";
import { loadInstalledPluginIndexInstallRecords } from "../../plugins/installed-plugin-index-records.js";
import { defaultRuntime } from "../../runtime.js";
import {
OPENCLAW_DATABASE_SCHEMA_DOCS_URL,
preflightOpenClawDatabaseSchemas,
type IncompatibleOpenClawDatabase,
type IndeterminateOpenClawDatabase,
type OpenClawDatabaseSchemaPreflight,
} from "../../state/openclaw-database-preflight.js";
import type { OpenClawSchemaVersions } from "../../state/openclaw-schema-versions.js";
import { VERSION } from "../../version.js";
import { replaceCliName, resolveCliName } from "../cli-name.js";
import { formatCliCommand } from "../command-format.js";
@@ -234,6 +243,41 @@ function printDryRunPreview(preview: UpdateDryRunPreview, jsonMode: boolean): vo
}
}
function formatSchemaRefusalLines(
schemas: {
incompatible: readonly IncompatibleOpenClawDatabase[];
indeterminate: readonly IndeterminateOpenClawDatabase[];
},
dryRun = false,
): string[] {
const prefix = dryRun ? "Would refuse update" : "Update refused";
return [
...schemas.incompatible.map((database) => {
const agent = database.agentId ? ` (agent ${database.agentId})` : "";
return `${prefix}: ${database.kind} database${agent} ${database.path} has schema ${database.foundVersion}; target supports ${database.supportedVersion}; writer build ${database.writerAppVersion ?? "unknown"}.`;
}),
...schemas.indeterminate.map(
(database) =>
`${prefix}: could not inspect ${database.kind} database ${database.path}: ${database.reason}; retry once the gateway releases it.`,
),
OPENCLAW_DATABASE_SCHEMA_DOCS_URL,
"Installing manually via npm bypasses this guard; back up first and verify compatibility.",
];
}
function checkTargetDatabaseSchemas(
supportedVersions: OpenClawSchemaVersions | undefined,
env: NodeJS.ProcessEnv = process.env,
): OpenClawDatabaseSchemaPreflight {
return supportedVersions
? preflightOpenClawDatabaseSchemas({ env, supportedVersions })
: { incompatible: [], indeterminate: [] };
}
function hasSchemaRefusal(schemas: OpenClawDatabaseSchemaPreflight): boolean {
return schemas.incompatible.length > 0 || schemas.indeterminate.length > 0;
}
async function runPackageInstallUpdate(params: {
root: string;
installKind: "git" | "package" | "unknown";
@@ -410,7 +454,10 @@ async function runGitUpdate(params: {
opts: UpdateCommandOptions;
stop: () => void;
devTargetRef?: string;
beforeGitMutation?: () => Promise<{
beforeGitMutation?: (target: {
schemaVersions?: OpenClawSchemaVersions;
metadataUnreadable?: string;
}) => Promise<{
allowGatewayServiceRepair?: boolean;
allowGatewayActivation?: boolean;
} | void>;
@@ -740,6 +787,7 @@ async function updateCommandInternal(
let packageInstallTarget: ResolvedGlobalInstallTarget | undefined;
let installedPackageName = DEFAULT_PACKAGE_NAME;
let packageAlreadyCurrent = false;
let packageTargetSchemaVersions: OpenClawSchemaVersions | undefined;
let managedServiceRootRedirect: ManagedServiceRootRedirect | null = null;
// Resolved independently of the root redirect so it covers the common case
// where the package root is the same but the user's PATH-resolved node
@@ -880,6 +928,46 @@ async function updateCommandInternal(
tag,
env: packageInstallEnv,
});
if (targetVersion) {
const targetMetadata = await fetchNpmPackageTargetStatus({
target: targetVersion,
spec: resolveGlobalInstallSpec({
packageName: DEFAULT_PACKAGE_NAME,
tag: targetVersion,
env: packageInstallEnv,
}),
command: npmMetadataCommand,
timeoutMs,
cwd: packageInstallCwd,
env: packageInstallEnv,
});
if (targetMetadata.error || targetMetadata.version !== targetVersion) {
defaultRuntime.error(
`Update refused: could not inspect exact package target openclaw@${targetVersion}: ${targetMetadata.error ?? `registry returned version ${targetMetadata.version ?? "unknown"}`}.`,
);
defaultRuntime.exit(1);
return;
}
packageTargetSchemaVersions = targetMetadata.schemaVersions;
// Always install the exact inspected version: a dist-tag can move between
// this lookup and the install, and an uninspected version would bypass
// the schema and runtime decisions made here. Missing schema metadata
// only means the schema preflight cannot run (legacy target).
if (updateInstallKind === "package" && canResolveRegistryVersionForPackageTarget(tag)) {
packageInstallSpec = resolveGlobalInstallSpec({
packageName: DEFAULT_PACKAGE_NAME,
tag: targetVersion,
env: packageInstallEnv,
});
}
}
}
const packageSchemaPreflight = checkTargetDatabaseSchemas(packageTargetSchemaVersions);
if (!opts.dryRun && hasSchemaRefusal(packageSchemaPreflight)) {
defaultRuntime.error(formatSchemaRefusalLines(packageSchemaPreflight).join("\n"));
defaultRuntime.exit(1);
return;
}
if (opts.dryRun) {
@@ -934,6 +1022,16 @@ async function updateCommandInternal(
if (explicitTag && !canResolveRegistryVersionForPackageTarget(tag)) {
notes.push("Non-registry package specs skip npm version lookup and downgrade previews.");
}
if (hasSchemaRefusal(packageSchemaPreflight)) {
notes.push(...formatSchemaRefusalLines(packageSchemaPreflight, true));
}
if (updateInstallKind === "git") {
// The git target revision is resolved inside the real update run, so its
// schema support cannot be previewed here without duplicating that flow.
notes.push(
"Database schema compatibility of the git target is verified during the real update; this preview does not check it.",
);
}
printDryRunPreview(
{
@@ -1044,6 +1142,7 @@ async function updateCommandInternal(
const preUpdatePluginInstallRecords = await loadInstalledPluginIndexInstallRecords();
let preManagedServiceStop: PreManagedServiceStop | undefined;
let schemaRefusalAfterStop = false;
const gitMutationRoots =
updateInstallKind === "git" ? (switchToGit ? [root, resolveGitInstallDir()] : [root]) : null;
const stopManagedServiceBeforeMutableUpdate = async (
@@ -1119,67 +1218,122 @@ async function updateCommandInternal(
}
}
const postStopPackageSchemaPreflight =
updateInstallKind === "package"
? checkTargetDatabaseSchemas(
packageTargetSchemaVersions,
preManagedServiceStop?.serviceEnv ?? process.env,
)
: { incompatible: [], indeterminate: [] };
if (hasSchemaRefusal(postStopPackageSchemaPreflight)) {
schemaRefusalAfterStop = true;
defaultRuntime.error(formatSchemaRefusalLines(postStopPackageSchemaPreflight).join("\n"));
}
let result: UpdateRunResult;
try {
result =
updateInstallKind === "package"
? await runPackageInstallUpdate({
updateInstallKind === "package" && hasSchemaRefusal(postStopPackageSchemaPreflight)
? {
status: "error",
mode: packageInstallTarget?.manager ?? "unknown",
root,
installKind,
tag,
installSpec: packageInstallSpec ?? undefined,
timeoutMs: updateStepTimeoutMs,
startedAt,
progress,
jsonMode: Boolean(opts.json),
allowGatewayServiceRepair: preManagedServiceStop?.serviceMatchesMutationRoot === true,
allowGatewayActivation:
shouldRestart &&
preManagedServiceStop?.stopped === true &&
preManagedServiceStop.serviceMatchesMutationRoot === true,
managedServiceEnv: preManagedServiceStop?.serviceEnv,
invocationCwd,
honorPackageRoot:
managedServiceRootRedirect !== null || managedServiceNodeRunner !== undefined,
nodeRunner: packageUpdateNodeRunner,
installEnv: packageInstallEnv,
installTarget: packageInstallTarget,
})
: await runGitUpdate({
root,
switchToGit,
installKind,
timeoutMs,
startedAt,
progress,
channel,
tag,
showProgress,
opts,
stop,
devTargetRef,
beforeGitMutation:
updateInstallKind === "git"
? async () => {
await stopManagedServiceBeforeMutableUpdate(gitMutationRoots ?? [root]);
return {
// Only a positively owned service may be rewritten. Activation
// additionally requires this update to have stopped it.
allowGatewayServiceRepair:
preManagedServiceStop?.serviceMatchesMutationRoot === true,
allowGatewayActivation:
shouldRestart &&
preManagedServiceStop?.stopped === true &&
preManagedServiceStop.serviceMatchesMutationRoot === true,
};
}
: undefined,
allowGatewayServiceRepair: false,
allowGatewayActivation: false,
});
reason: "database-schema-preflight",
steps: [],
durationMs: Date.now() - startedAt,
}
: updateInstallKind === "package"
? await runPackageInstallUpdate({
root,
installKind,
tag,
installSpec: packageInstallSpec ?? undefined,
timeoutMs: updateStepTimeoutMs,
startedAt,
progress,
jsonMode: Boolean(opts.json),
allowGatewayServiceRepair: preManagedServiceStop?.serviceMatchesMutationRoot === true,
allowGatewayActivation:
shouldRestart &&
preManagedServiceStop?.stopped === true &&
preManagedServiceStop.serviceMatchesMutationRoot === true,
managedServiceEnv: preManagedServiceStop?.serviceEnv,
invocationCwd,
honorPackageRoot:
managedServiceRootRedirect !== null || managedServiceNodeRunner !== undefined,
nodeRunner: packageUpdateNodeRunner,
installEnv: packageInstallEnv,
installTarget: packageInstallTarget,
})
: await runGitUpdate({
root,
switchToGit,
installKind,
timeoutMs,
startedAt,
progress,
channel,
tag,
showProgress,
opts,
stop,
devTargetRef,
beforeGitMutation:
updateInstallKind === "git"
? async (target) => {
if (target?.metadataUnreadable) {
defaultRuntime.error(
`Update refused: could not inspect the target's schema support (${target.metadataUnreadable}). Retry, or see ${OPENCLAW_DATABASE_SCHEMA_DOCS_URL}.`,
);
defaultRuntime.exit(1);
throw new UpdateCommandAbort();
}
const preStopSchemas = checkTargetDatabaseSchemas(target?.schemaVersions);
if (hasSchemaRefusal(preStopSchemas)) {
defaultRuntime.error(formatSchemaRefusalLines(preStopSchemas).join("\n"));
defaultRuntime.exit(1);
throw new UpdateCommandAbort();
}
await stopManagedServiceBeforeMutableUpdate(gitMutationRoots ?? [root]);
const postStopSchemas = checkTargetDatabaseSchemas(
target?.schemaVersions,
preManagedServiceStop?.serviceEnv ?? process.env,
);
if (hasSchemaRefusal(postStopSchemas)) {
schemaRefusalAfterStop = true;
defaultRuntime.error(formatSchemaRefusalLines(postStopSchemas).join("\n"));
throw new UpdateCommandAbort();
}
return {
// Only a positively owned service may be rewritten. Activation
// additionally requires this update to have stopped it.
allowGatewayServiceRepair:
preManagedServiceStop?.serviceMatchesMutationRoot === true,
allowGatewayActivation:
shouldRestart &&
preManagedServiceStop?.stopped === true &&
preManagedServiceStop.serviceMatchesMutationRoot === true,
};
}
: undefined,
allowGatewayServiceRepair: false,
allowGatewayActivation: false,
});
} catch (err) {
stop();
if (err instanceof UpdateCommandAbort) {
if (schemaRefusalAfterStop) {
if (preManagedServiceStop?.stopped === true) {
await maybeResumeWindowsTaskAutoStartAfterPackageUpdate(preManagedServiceStop).catch(
() => undefined,
);
await maybeRestartServiceAfterFailedMutableUpdate({
preManagedServiceStop,
jsonMode: Boolean(opts.json),
});
}
defaultRuntime.exit(1);
}
return;
}
try {
+13 -2
View File
@@ -3,6 +3,7 @@ import fs from "node:fs";
import type { SessionStoreTarget } from "../config/sessions/targets.js";
import {
assertOpenClawAgentDatabaseForMaintenance,
clearOpenClawAgentDatabaseOpenFailure,
ensureOpenClawAgentDatabasePermissions,
isOpenClawAgentDatabaseOpen,
migrateOpenClawAgentDatabaseForMaintenance,
@@ -40,19 +41,29 @@ export function compactDoctorSessionSqliteTarget(
`OpenClaw agent database ${sqlitePath} is already open in this process. Stop OpenClaw and retry.`,
);
}
const requireQuarantineCleared = () => {
if (!clearOpenClawAgentDatabaseOpenFailure(sqlitePath)) {
throw new Error(
`OpenClaw agent database ${sqlitePath} was repaired, but its persisted quarantine record could not be cleared. Rerun openclaw doctor --fix so the database is not refused again.`,
);
}
};
if (options.migrateOlderSchema) {
migrateOpenClawAgentDatabaseForMaintenance({
agentId: target.agentId,
pathname: sqlitePath,
});
requireQuarantineCleared();
}
const compact = compactDoctorSqliteFile({
afterMutation: () =>
afterMutation: () => {
requireQuarantineCleared();
ensureOpenClawAgentDatabasePermissions(sqlitePath, {
agentId: target.agentId,
path: sqlitePath,
}),
});
},
sqlitePath,
validateBeforeMutation: (database) =>
assertOpenClawAgentDatabaseForMaintenance(database, {
+1 -3
View File
@@ -17,7 +17,6 @@ type DoctorSqliteCompactResult = {
after: DoctorSqliteCompactSnapshot;
before: DoctorSqliteCompactSnapshot;
integrityCheck: "ok";
quickCheck: "ok";
reclaimedBytes: number;
};
@@ -56,7 +55,7 @@ export function compactDoctorSqliteFile(
database.exec("PRAGMA auto_vacuum = INCREMENTAL;");
database.exec("VACUUM;");
checkpointTruncate(database, options.sqlitePath);
const { quickCheck, integrityCheck } = assertSqliteIntegrity(database, options.sqlitePath);
const { integrityCheck } = assertSqliteIntegrity(database, options.sqlitePath);
const after = readCompactSnapshot(database, options.sqlitePath);
const beforeBytes = before.dbSizeBytes + before.walSizeBytes;
const afterBytes = after.dbSizeBytes + after.walSizeBytes;
@@ -64,7 +63,6 @@ export function compactDoctorSqliteFile(
after,
before,
integrityCheck,
quickCheck,
reclaimedBytes: Math.max(0, beforeBytes - afterBytes),
};
} catch (error) {
@@ -163,7 +163,6 @@ describe("runDoctorStateSqliteCompact", () => {
expect(report.after.walSizeBytes).toBe(0);
expect(report.after.pageSizeBytes).toBeGreaterThan(0);
expect(report.reclaimedBytes).toBeGreaterThan(0);
expect(report.quickCheck).toBe("ok");
expect(report.integrityCheck).toBe("ok");
});
+11 -2
View File
@@ -2,6 +2,8 @@
import fs from "node:fs";
import {
assertOpenClawStateDatabaseForMaintenance,
clearOpenClawDatabaseVerification,
clearOpenClawStateDatabaseOpenFailure,
ensureOpenClawStatePermissions,
isOpenClawStateDatabaseOpen,
} from "../state/openclaw-state-db.js";
@@ -25,7 +27,6 @@ type DoctorStateSqliteCompactReport =
integrityCheck: "ok";
mode: "compact";
path: string;
quickCheck: "ok";
reclaimedBytes: number;
skipped: false;
};
@@ -70,7 +71,15 @@ export async function runDoctorStateSqliteCompact(
}
const compact = compactDoctorSqliteFile({
afterMutation: () => ensureOpenClawStatePermissions(sqlitePath, env),
afterMutation: () => {
if (!clearOpenClawDatabaseVerification(sqlitePath, { path: sqlitePath })) {
throw new Error(
`OpenClaw state database ${sqlitePath} was compacted, but its persisted quarantine record could not be cleared. Rerun openclaw doctor --fix so the database is not refused again.`,
);
}
clearOpenClawStateDatabaseOpenFailure(sqlitePath);
ensureOpenClawStatePermissions(sqlitePath, env);
},
...(deps.busyTimeoutMs !== undefined ? { busyTimeoutMs: deps.busyTimeoutMs } : {}),
sqlitePath,
validateBeforeMutation: (database) =>
-1
View File
@@ -256,7 +256,6 @@ describe("doctorCommand", () => {
integrityCheck: "ok",
mode: "compact",
path: "/tmp/openclaw/state/openclaw.sqlite",
quickCheck: "ok",
reclaimedBytes: 12_288,
skipped: false,
};
+1 -3
View File
@@ -58,9 +58,7 @@ export async function doctorCommand(runtime?: RuntimeEnv, options?: DoctorOption
outputRuntime.log(
`- freelist=${report.before.freelistPages}->${report.after.freelistPages} pages, page-size=${report.after.pageSizeBytes} bytes, auto-vacuum=${report.before.autoVacuum}->${report.after.autoVacuum}`,
);
outputRuntime.log(
`- quick-check=${report.quickCheck}, integrity-check=${report.integrityCheck}, path=${report.path}`,
);
outputRuntime.log(`- integrity-check=${report.integrityCheck}, path=${report.path}`);
}
outputRuntime.exit(0);
return;
+49
View File
@@ -574,6 +574,48 @@ export async function startGatewayServer(
opts: GatewayServerOptions = {},
): Promise<GatewayServer> {
normalizeStateDirEnv(process.env);
const [
{
OPENCLAW_DATABASE_SCHEMA_DOCS_URL,
OpenClawDatabaseSchemaPreflightError,
preflightOpenClawDatabaseSchemas,
},
agentDatabase,
stateDatabase,
] = await Promise.all([
import("../state/openclaw-database-preflight.js"),
import("../state/openclaw-agent-db.js"),
import("../state/openclaw-state-db.js"),
]);
const databaseSchemas = preflightOpenClawDatabaseSchemas({
env: process.env,
supportedVersions: {
state: stateDatabase.OPENCLAW_STATE_SCHEMA_VERSION,
agent: agentDatabase.OPENCLAW_AGENT_SCHEMA_VERSION,
},
});
if (databaseSchemas.incompatible.length > 0) {
for (const database of databaseSchemas.incompatible) {
log.error("database schema preflight rejected newer schema", {
kind: database.kind,
path: database.path,
...(database.agentId ? { agentId: database.agentId } : {}),
foundVersion: database.foundVersion,
supportedVersion: database.supportedVersion,
writerAppVersion: database.writerAppVersion ?? "unknown",
docsUrl: OPENCLAW_DATABASE_SCHEMA_DOCS_URL,
});
}
throw new OpenClawDatabaseSchemaPreflightError(databaseSchemas.incompatible);
}
for (const database of databaseSchemas.indeterminate) {
log.warn("database schema preflight could not inspect database; continuing to real open", {
kind: database.kind,
path: database.path,
reason: database.reason,
docsUrl: OPENCLAW_DATABASE_SCHEMA_DOCS_URL,
});
}
const { bootstrapGatewayNetworkRuntime } = await import("./server-network-runtime.js");
bootstrapGatewayNetworkRuntime();
@@ -2180,6 +2222,13 @@ export async function startGatewayServer(
log.info("gateway ready");
}
finishGatewayRestartTrace("restart.ready", collectGatewayProcessMemoryUsageMb());
if (!minimalTestGateway) {
const { startOpenClawDatabaseIntegrityVerifier } =
await import("../state/openclaw-database-verify.js");
runtimeState.gatewayLifetimeSidecars.push(
startOpenClawDatabaseIntegrityVerifier({ env: process.env }),
);
}
postAttachRuntimeReturned = true;
activateScheduledServicesWhenReady();
+52 -3
View File
@@ -1,6 +1,7 @@
import type { DatabaseSync } from "node:sqlite";
import { describe, expect, it } from "vitest";
import { requireNodeSqlite } from "./node-sqlite.js";
import { assertSqliteIntegrity } from "./sqlite-integrity.js";
import { assertSqliteIntegrity, isTerminalSqliteIntegrityError } from "./sqlite-integrity.js";
describe("assertSqliteIntegrity", () => {
it("accepts structurally and referentially consistent databases", () => {
@@ -20,7 +21,6 @@ describe("assertSqliteIntegrity", () => {
expect(assertSqliteIntegrity(database, "test database")).toEqual({
integrityCheck: "ok",
quickCheck: "ok",
});
} finally {
database.close();
@@ -45,7 +45,14 @@ describe("assertSqliteIntegrity", () => {
integrity_check: "ok",
});
expect(() => assertSqliteIntegrity(database, "test database")).toThrow(
let failure: unknown;
try {
assertSqliteIntegrity(database, "test database");
} catch (error) {
failure = error;
}
expect(failure).toMatchObject({ name: "SqliteIntegrityError" });
expect(String(failure)).toMatch(
/foreign_key_check failed for test database: children row 1 references parents \(foreign key 0\)/u,
);
} finally {
@@ -53,6 +60,24 @@ describe("assertSqliteIntegrity", () => {
}
});
it("names integrity-check failures", () => {
const database = {
prepare: () => ({ all: () => [{ integrity_check: "broken index" }] }),
} as unknown as DatabaseSync;
let failure: unknown;
try {
assertSqliteIntegrity(database, "test database");
} catch (error) {
failure = error;
}
expect(failure).toMatchObject({
name: "SqliteIntegrityError",
message: "SQLite integrity_check failed for test database: broken index",
});
});
it("reports violations deterministically without truncating 64-bit rowids", () => {
const sqlite = requireNodeSqlite();
const database = new sqlite.DatabaseSync(":memory:");
@@ -152,3 +177,27 @@ describe("assertSqliteIntegrity", () => {
}
});
});
describe("isTerminalSqliteIntegrityError", () => {
it("distinguishes persistent damage from transient pragma failures", () => {
const corrupt = new Error("integrity check found damage");
corrupt.name = "SqliteIntegrityError";
const busy = new Error("integrity check could not run", {
cause: Object.assign(new Error("database is locked"), { errcode: 5 }),
});
busy.name = "SqliteIntegrityError";
const malformed = new Error("integrity check could not read the database", {
cause: Object.assign(new Error("database disk image is malformed"), { errcode: 11 }),
});
malformed.name = "SqliteIntegrityError";
const corruptIndex = new Error("integrity check found a corrupt index", {
cause: Object.assign(new Error("database index is malformed"), { errcode: 779 }),
});
corruptIndex.name = "SqliteIntegrityError";
expect(isTerminalSqliteIntegrityError(corrupt)).toBe(true);
expect(isTerminalSqliteIntegrityError(busy)).toBe(false);
expect(isTerminalSqliteIntegrityError(malformed)).toBe(true);
expect(isTerminalSqliteIntegrityError(corruptIndex)).toBe(true);
});
});
+47 -12
View File
@@ -2,10 +2,9 @@ import type { DatabaseSync } from "node:sqlite";
type SqliteIntegrityChecks = {
integrityCheck: "ok";
quickCheck: "ok";
};
type SqliteCheckPragma = "integrity_check" | "quick_check";
type SqliteCheckPragma = "integrity_check";
type SqliteForeignKeyViolation = {
fkid: bigint;
parent: string;
@@ -15,15 +14,35 @@ type SqliteForeignKeyViolation = {
const MAX_REPORTED_FOREIGN_KEY_VIOLATIONS = 5;
const SQLITE_CORRUPT_ERRCODE = 11;
const SQLITE_NOTADB_ERRCODE = 26;
/** Return whether a named integrity failure proves persistent database damage. */
export function isTerminalSqliteIntegrityError(error: Error): boolean {
if (error.name !== "SqliteIntegrityError") {
return false;
}
const cause = error.cause as { errcode?: unknown } | undefined;
if (!cause) {
// No cause means the check pragma itself reported corruption rows: persistent.
return true;
}
if (typeof cause.errcode !== "number") {
return false;
}
// Mask extended codes to the primary; transient lock/busy failures must not latch.
const primaryCode = cause.errcode & 0xff;
return primaryCode === SQLITE_CORRUPT_ERRCODE || primaryCode === SQLITE_NOTADB_ERRCODE;
}
/** Require structural, table/index, and referential consistency before trusting a database. */
export function assertSqliteIntegrity(
database: DatabaseSync,
databaseLabel: string,
): SqliteIntegrityChecks {
const quickCheck = runSqliteCheck(database, databaseLabel, "quick_check");
const integrityCheck = runSqliteCheck(database, databaseLabel, "integrity_check");
runSqliteForeignKeyCheck(database, databaseLabel);
return { integrityCheck, quickCheck };
return { integrityCheck };
}
/** Require table and associated index consistency before trusting indexed reads. */
@@ -42,15 +61,22 @@ function runSqliteCheck(
tableName?: string,
): "ok" {
const argument = tableName ? `('${tableName.replaceAll("'", "''")}')` : "";
const rows = database.prepare(`PRAGMA ${pragma}${argument};`).all() as Array<
Record<string, unknown>
>;
let rows: Array<Record<string, unknown>>;
try {
rows = database.prepare(`PRAGMA ${pragma}${argument};`).all() as Array<Record<string, unknown>>;
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
throw createSqliteIntegrityError(
`SQLite ${pragma} failed for ${databaseLabel}: ${message}`,
error,
);
}
const results = rows.map((row) => row[pragma] ?? Object.values(row)[0]);
if (results.length === 1 && results[0] === "ok") {
return "ok";
}
const details = results.map((result) => String(result)).join("; ") || "no result";
throw new Error(`SQLite ${pragma} failed for ${databaseLabel}: ${details}`);
throw createSqliteIntegrityError(`SQLite ${pragma} failed for ${databaseLabel}: ${details}`);
}
function runSqliteForeignKeyCheck(database: DatabaseSync, databaseLabel: string): void {
@@ -68,9 +94,10 @@ function runSqliteForeignKeyCheck(database: DatabaseSync, databaseLabel: string)
}
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
throw new Error(`SQLite foreign_key_check failed for ${databaseLabel}: ${message}`, {
cause: error,
});
throw createSqliteIntegrityError(
`SQLite foreign_key_check failed for ${databaseLabel}: ${message}`,
error,
);
}
if (violations.length === 0) {
return;
@@ -80,7 +107,15 @@ function runSqliteForeignKeyCheck(database: DatabaseSync, databaseLabel: string)
if (violationCount > MAX_REPORTED_FOREIGN_KEY_VIOLATIONS) {
details.push("additional violations omitted");
}
throw new Error(`SQLite foreign_key_check failed for ${databaseLabel}: ${details.join("; ")}`);
throw createSqliteIntegrityError(
`SQLite foreign_key_check failed for ${databaseLabel}: ${details.join("; ")}`,
);
}
function createSqliteIntegrityError(message: string, cause?: unknown): Error {
const error = cause === undefined ? new Error(message) : new Error(message, { cause });
error.name = "SqliteIntegrityError";
return error;
}
function retainSortedForeignKeyViolation(
+13 -1
View File
@@ -1,6 +1,9 @@
// Tests for SQLite user_version pragma helper.
import { describe, expect, it } from "vitest";
import { readSqliteUserVersion } from "./sqlite-user-version.js";
import {
createNewerSqliteSchemaVersionError,
readSqliteUserVersion,
} from "./sqlite-user-version.js";
describe("readSqliteUserVersion", () => {
it("returns 0 when row is undefined", () => {
@@ -45,3 +48,12 @@ describe("readSqliteUserVersion", () => {
expect(readSqliteUserVersion(db)).toBe(0);
});
});
describe("createNewerSqliteSchemaVersionError", () => {
it("returns a stable named error with the schema guide", () => {
const error = createNewerSqliteSchemaVersionError("test database", "/tmp/test.sqlite", 12, 11);
expect(error.name).toBe("SqliteSchemaVersionError");
expect(error.message).toContain("https://docs.openclaw.ai/reference/database-schemas");
});
});
+4 -2
View File
@@ -13,7 +13,9 @@ export function createNewerSqliteSchemaVersionError(
schemaVersion: number,
supportedVersion: number,
): Error {
return new Error(
`${databaseLabel} ${pathname} uses newer schema version ${schemaVersion}; this OpenClaw build supports ${supportedVersion}. Upgrade OpenClaw before opening this database. Do not downgrade OpenClaw or modify the database. To run this older build, use a separate state directory or restore a compatible backup.`,
const error = new Error(
`${databaseLabel} ${pathname} uses newer schema version ${schemaVersion}; this OpenClaw build supports ${supportedVersion}. Upgrade OpenClaw before opening this database. Do not downgrade OpenClaw or modify the database. To run this older build, use a separate state directory or restore a compatible backup. See https://docs.openclaw.ai/reference/database-schemas.`,
);
error.name = "SqliteSchemaVersionError";
return error;
}
+1
View File
@@ -102,6 +102,7 @@ describe("tsdown config", () => {
"cli/gateway-lifecycle.runtime",
"agents/compaction-planning.worker",
"agents/model-provider-auth.worker",
"state/openclaw-database-verify.worker",
"plugins/memory-state",
"subagent-registry.runtime",
"task-registry-control.runtime",
+199
View File
@@ -0,0 +1,199 @@
import { readProviderJsonResponse } from "../agents/provider-http-errors.js";
import { runCommandWithTimeout } from "../process/exec.js";
import {
parseOpenClawSchemaVersions,
type OpenClawSchemaVersions,
} from "../state/openclaw-schema-versions.js";
import { buildTimeoutAbortSignal } from "../utils/fetch-timeout.js";
type NpmPackageTargetStatus = {
target: string;
version: string | null;
nodeEngine: string | null;
schemaVersions?: OpenClawSchemaVersions;
error?: string;
};
export type NpmMetadataCommandRunner = (
argv: string[],
options: {
timeoutMs: number;
cwd?: string;
env?: NodeJS.ProcessEnv;
maxOutputBytes?: number;
},
) => Promise<{
stdout: string;
stderr: string;
code: number | null;
}>;
function toOptionalTrimmedString(value: unknown): string | null {
return typeof value === "string" && value.trim() ? value.trim() : null;
}
function parseNpmPackageTargetMetadata(raw: string): {
version: string | null;
nodeEngine: string | null;
schemaVersions?: OpenClawSchemaVersions;
} {
let parsed: unknown;
try {
parsed = JSON.parse(raw.trim()) as unknown;
} catch (err) {
throw new Error(`npm view returned invalid JSON: ${String(err)}`, { cause: err });
}
// npm 12 wraps `npm view --json` results in a singleton array.
const entry = Array.isArray(parsed) && parsed.length === 1 ? parsed[0] : parsed;
if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
return { version: null, nodeEngine: null };
}
const rec = entry as Record<string, unknown>;
const engines = rec.engines && typeof rec.engines === "object" ? rec.engines : null;
const nodeEngine =
toOptionalTrimmedString(rec["engines.node"]) ??
(engines ? toOptionalTrimmedString((engines as Record<string, unknown>).node) : null);
const openclaw = rec.openclaw && typeof rec.openclaw === "object" ? rec.openclaw : null;
const schemaVersions =
parseOpenClawSchemaVersions(rec["openclaw.schemaVersions"]) ??
(openclaw
? parseOpenClawSchemaVersions((openclaw as Record<string, unknown>).schemaVersions)
: undefined);
return {
version: toOptionalTrimmedString(rec.version),
nodeEngine,
...(schemaVersions ? { schemaVersions } : {}),
};
}
function formatNpmViewError(res: { stdout: string; stderr: string }): string {
const raw = (res.stderr.trim() || res.stdout.trim()).split("\n").slice(-3).join("\n");
return raw ? `npm view failed: ${raw}` : "npm view failed";
}
function packageTargetSpec(params: { target: string; spec?: string }): string {
const spec = params.spec?.trim();
return spec || `openclaw@${params.target.trim() || "latest"}`;
}
const PUBLIC_NPM_REGISTRY_URL = "https://registry.npmjs.org/";
const PUBLIC_NPM_PACKAGE_NAME = "openclaw";
function npmRegistryTargetUrl(params: {
registryUrl: string;
packageName: string;
target: string;
}): string {
const baseUrl = params.registryUrl.endsWith("/") ? params.registryUrl : `${params.registryUrl}/`;
return new URL(
`${encodeURIComponent(params.packageName)}/${encodeURIComponent(params.target)}`,
baseUrl,
).toString();
}
async function fetchNpmPackageTargetStatusFromRegistry(params: {
target: string;
timeoutMs: number;
registryUrl?: string;
packageName?: string;
}): Promise<NpmPackageTargetStatus> {
const url = npmRegistryTargetUrl({
registryUrl: params.registryUrl ?? PUBLIC_NPM_REGISTRY_URL,
packageName: params.packageName ?? PUBLIC_NPM_PACKAGE_NAME,
target: params.target,
});
const { signal, cleanup } = buildTimeoutAbortSignal({
timeoutMs: Math.max(250, params.timeoutMs),
operation: "npm-registry-update-check",
url,
});
let res: Response | undefined;
try {
res = await fetch(url, { signal });
if (!res.ok) {
return {
target: params.target,
version: null,
nodeEngine: null,
error: `HTTP ${res.status}`,
};
}
// Keep the deadline active through body consumption. Fetch resolves at
// headers, so clearing it earlier would leave a stalled registry body unbounded.
const json = await readProviderJsonResponse<{
version?: unknown;
engines?: { node?: unknown };
openclaw?: { schemaVersions?: unknown };
}>(res, "npm package target status");
const schemaVersions = parseOpenClawSchemaVersions(json.openclaw?.schemaVersions);
return {
target: params.target,
version: toOptionalTrimmedString(json.version),
nodeEngine: toOptionalTrimmedString(json.engines?.node),
...(schemaVersions ? { schemaVersions } : {}),
};
} catch (err) {
return { target: params.target, version: null, nodeEngine: null, error: String(err) };
} finally {
if (res?.bodyUsed !== true) {
await res?.body?.cancel().catch(() => undefined);
}
cleanup();
}
}
export async function fetchNpmPackageTargetStatus(params: {
target: string;
timeoutMs?: number;
spec?: string;
command?: string;
cwd?: string;
env?: NodeJS.ProcessEnv;
runCommand?: NpmMetadataCommandRunner;
registryUrl?: string;
packageName?: string;
}): Promise<NpmPackageTargetStatus> {
const timeoutMs = params.timeoutMs ?? 3500;
const target = params.target;
if (!params.command && !params.runCommand) {
return await fetchNpmPackageTargetStatusFromRegistry({
target,
timeoutMs,
registryUrl: params.registryUrl,
packageName: params.packageName,
});
}
const runCommand = params.runCommand ?? runCommandWithTimeout;
try {
const res = await runCommand(
[
params.command ?? "npm",
"view",
packageTargetSpec({ target, spec: params.spec }),
"version",
"engines.node",
"openclaw.schemaVersions",
"--json",
"--global",
],
{
timeoutMs: Math.max(250, timeoutMs),
cwd: params.cwd,
env: params.env,
maxOutputBytes: 1024 * 1024,
},
);
if (res.code !== 0) {
return {
target,
version: null,
nodeEngine: null,
error: formatNpmViewError(res),
};
}
const { version, nodeEngine, schemaVersions } = parseNpmPackageTargetMetadata(res.stdout);
return { target, version, nodeEngine, ...(schemaVersions ? { schemaVersions } : {}) };
} catch (err) {
return { target, version: null, nodeEngine: null, error: String(err) };
}
}
+4 -1
View File
@@ -7,10 +7,10 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { runCommandWithTimeout } from "../process/exec.js";
import { withTempDir } from "../test-helpers/temp-dir.js";
import { useMockHttp } from "../test-utils/mock-http.js";
import { fetchNpmPackageTargetStatus } from "./update-check-package-target.js";
import {
checkUpdateStatus,
compareSemverStrings,
fetchNpmPackageTargetStatus,
fetchNpmTagVersion,
formatGitInstallLabel,
resolveExtendedStablePackage,
@@ -101,6 +101,7 @@ describe("resolveNpmChannelTag", () => {
"openclaw@latest",
"version",
"engines.node",
"openclaw.schemaVersions",
"--json",
"--global",
],
@@ -118,6 +119,7 @@ describe("resolveNpmChannelTag", () => {
{
version: "2026.7.1",
engines: { node: ">=22.22.3" },
openclaw: { schemaVersions: { state: 3, agent: 11 } },
},
]),
stderr: "",
@@ -134,6 +136,7 @@ describe("resolveNpmChannelTag", () => {
target: "latest",
version: "2026.7.1",
nodeEngine: ">=22.22.3",
schemaVersions: { state: 3, agent: 11 },
});
});
+6 -173
View File
@@ -1,13 +1,15 @@
// Computes git, dependency, and registry update status for OpenClaw installs.
import fs from "node:fs/promises";
import path from "node:path";
import { readProviderJsonResponse } from "../agents/provider-http-errors.js";
import { runCommandWithTimeout } from "../process/exec.js";
import { buildTimeoutAbortSignal } from "../utils/fetch-timeout.js";
import { detectPackageManager as detectPackageManagerImpl } from "./detect-package-manager.js";
import { compareOpenClawReleaseVersions } from "./npm-registry-spec.js";
import { compareValidSemver, normalizeLegacyDotBetaVersion } from "./semver.js";
import { channelToNpmTag, type UpdateChannel } from "./update-channels.js";
import {
fetchNpmPackageTargetStatus,
type NpmMetadataCommandRunner,
} from "./update-check-package-target.js";
type PackageManager = "pnpm" | "bun" | "npm" | "unknown";
@@ -63,13 +65,6 @@ type NpmTagStatus = {
error?: string;
};
type NpmPackageTargetStatus = {
target: string;
version: string | null;
nodeEngine: string | null;
error?: string;
};
export type UpdateCheckResult = {
root: string | null;
installKind: "git" | "package" | "unknown";
@@ -79,60 +74,6 @@ export type UpdateCheckResult = {
registry?: RegistryStatus;
};
type NpmMetadataCommandRunner = (
argv: string[],
options: {
timeoutMs: number;
cwd?: string;
env?: NodeJS.ProcessEnv;
maxOutputBytes?: number;
},
) => Promise<{
stdout: string;
stderr: string;
code: number | null;
}>;
function toOptionalTrimmedString(value: unknown): string | null {
return typeof value === "string" && value.trim() ? value.trim() : null;
}
function parseNpmPackageTargetMetadata(raw: string): {
version: string | null;
nodeEngine: string | null;
} {
let parsed: unknown;
try {
parsed = JSON.parse(raw.trim()) as unknown;
} catch (err) {
throw new Error(`npm view returned invalid JSON: ${String(err)}`, { cause: err });
}
// npm 12 wraps `npm view --json` results in a singleton array.
const entry = Array.isArray(parsed) && parsed.length === 1 ? parsed[0] : parsed;
if (!entry || typeof entry !== "object" || Array.isArray(entry)) {
return { version: null, nodeEngine: null };
}
const rec = entry as Record<string, unknown>;
const engines = rec.engines && typeof rec.engines === "object" ? rec.engines : null;
const nodeEngine =
toOptionalTrimmedString(rec["engines.node"]) ??
(engines ? toOptionalTrimmedString((engines as Record<string, unknown>).node) : null);
return {
version: toOptionalTrimmedString(rec.version),
nodeEngine,
};
}
function formatNpmViewError(res: { stdout: string; stderr: string }): string {
const raw = (res.stderr.trim() || res.stdout.trim()).split("\n").slice(-3).join("\n");
return raw ? `npm view failed: ${raw}` : "npm view failed";
}
function packageTargetSpec(params: { target: string; spec?: string }): string {
const spec = params.spec?.trim();
return spec || `openclaw@${params.target.trim() || "latest"}`;
}
const PUBLIC_NPM_REGISTRY_URL = "https://registry.npmjs.org/";
const PUBLIC_NPM_PACKAGE_NAME = "openclaw";
@@ -168,66 +109,6 @@ function resolveExtendedStableRegistryTarget(params: {
};
}
function npmRegistryTargetUrl(params: {
registryUrl: string;
packageName: string;
target: string;
}): string {
const baseUrl = params.registryUrl.endsWith("/") ? params.registryUrl : `${params.registryUrl}/`;
return new URL(
`${encodeURIComponent(params.packageName)}/${encodeURIComponent(params.target)}`,
baseUrl,
).toString();
}
async function fetchNpmPackageTargetStatusFromRegistry(params: {
target: string;
timeoutMs: number;
registryUrl?: string;
packageName?: string;
}): Promise<NpmPackageTargetStatus> {
const url = npmRegistryTargetUrl({
registryUrl: params.registryUrl ?? PUBLIC_NPM_REGISTRY_URL,
packageName: params.packageName ?? PUBLIC_NPM_PACKAGE_NAME,
target: params.target,
});
const { signal, cleanup } = buildTimeoutAbortSignal({
timeoutMs: Math.max(250, params.timeoutMs),
operation: "npm-registry-update-check",
url,
});
let res: Response | undefined;
try {
res = await fetch(url, { signal });
if (!res.ok) {
return {
target: params.target,
version: null,
nodeEngine: null,
error: `HTTP ${res.status}`,
};
}
// Keep the deadline active through body consumption. Fetch resolves at
// headers, so clearing it earlier would leave a stalled registry body unbounded.
const json = await readProviderJsonResponse<{
version?: unknown;
engines?: { node?: unknown };
}>(res, "npm package target status");
return {
target: params.target,
version: toOptionalTrimmedString(json.version),
nodeEngine: toOptionalTrimmedString(json.engines?.node),
};
} catch (err) {
return { target: params.target, version: null, nodeEngine: null, error: String(err) };
} finally {
if (res?.bodyUsed !== true) {
await res?.body?.cancel().catch(() => undefined);
}
cleanup();
}
}
/** Resolves the extended-stable selector and verifies its exact package manifest. */
export async function resolveExtendedStablePackage(params: {
installKind: "git" | "package" | "unknown";
@@ -241,7 +122,7 @@ export async function resolveExtendedStablePackage(params: {
const timeoutMs = params.timeoutMs ?? 3500;
const registryTarget = resolveExtendedStableRegistryTarget(params);
const selector = await fetchNpmPackageTargetStatusFromRegistry({
const selector = await fetchNpmPackageTargetStatus({
target: "extended-stable",
timeoutMs,
...registryTarget,
@@ -253,7 +134,7 @@ export async function resolveExtendedStablePackage(params: {
};
}
const exact = await fetchNpmPackageTargetStatusFromRegistry({
const exact = await fetchNpmPackageTargetStatus({
target: selector.version,
timeoutMs,
...registryTarget,
@@ -549,54 +430,6 @@ async function fetchNpmRegistryVersionForChannel(params: {
};
}
export async function fetchNpmPackageTargetStatus(params: {
target: string;
timeoutMs?: number;
spec?: string;
command?: string;
cwd?: string;
env?: NodeJS.ProcessEnv;
runCommand?: NpmMetadataCommandRunner;
}): Promise<NpmPackageTargetStatus> {
const timeoutMs = params.timeoutMs ?? 3500;
const target = params.target;
if (!params.command && !params.runCommand) {
return await fetchNpmPackageTargetStatusFromRegistry({ target, timeoutMs });
}
const runCommand = params.runCommand ?? runCommandWithTimeout;
try {
const res = await runCommand(
[
params.command ?? "npm",
"view",
packageTargetSpec({ target, spec: params.spec }),
"version",
"engines.node",
"--json",
"--global",
],
{
timeoutMs: Math.max(250, timeoutMs),
cwd: params.cwd,
env: params.env,
maxOutputBytes: 1024 * 1024,
},
);
if (res.code !== 0) {
return {
target,
version: null,
nodeEngine: null,
error: formatNpmViewError(res),
};
}
const { version, nodeEngine } = parseNpmPackageTargetMetadata(res.stdout);
return { target, version, nodeEngine };
} catch (err) {
return { target, version: null, nodeEngine: null, error: String(err) };
}
}
export async function fetchNpmTagVersion(params: {
tag: string;
timeoutMs?: number;
+44 -2
View File
@@ -412,7 +412,9 @@ describe("runGatewayUpdate", () => {
deferConfiguredPluginInstallRepair?: boolean;
allowGatewayServiceRepair?: boolean;
allowGatewayActivation?: boolean;
beforeGitMutation?: () => Promise<{
beforeGitMutation?: (target: {
schemaVersions?: { state: number; agent: number };
}) => Promise<{
allowGatewayServiceRepair?: boolean;
allowGatewayActivation?: boolean;
} | void>;
@@ -444,7 +446,9 @@ describe("runGatewayUpdate", () => {
cwd?: string;
devTargetRef?: string;
deferConfiguredPluginInstallRepair?: boolean;
beforeGitMutation?: () => Promise<{
beforeGitMutation?: (target: {
schemaVersions?: { state: number; agent: number };
}) => Promise<{
allowGatewayServiceRepair?: boolean;
allowGatewayActivation?: boolean;
} | void>;
@@ -640,6 +644,11 @@ describe("runGatewayUpdate", () => {
[`git -C ${tempDir} rev-list --max-count=10 ${upstreamSha}`]: {
stdout: `${upstreamSha}\n`,
},
[`git -C ${tempDir} show ${upstreamSha}:package.json`]: {
stdout: JSON.stringify({
openclaw: { schemaVersions: { state: 3, agent: 11 } },
}),
},
[`git -C ${tempDir} rebase ${upstreamSha}`]: { stdout: "" },
"pnpm --version": { stdout: "10.0.0" },
"pnpm install": { stdout: "" },
@@ -652,6 +661,9 @@ describe("runGatewayUpdate", () => {
expect(result.status).toBe("ok");
expect(beforeGitMutation).toHaveBeenCalledTimes(1);
expect(beforeGitMutation).toHaveBeenCalledWith({
schemaVersions: { state: 3, agent: 11 },
});
expect(calls).toContain(`git -C ${tempDir} fetch --all --prune --no-tags`);
expect(calls).not.toContain(`git -C ${tempDir} fetch --all --prune --tags`);
const cleanupIndex = calls.findIndex(
@@ -666,6 +678,36 @@ describe("runGatewayUpdate", () => {
);
});
it("hands beforeGitMutation an unreadable marker when target metadata cannot be read", async () => {
await setupGitCheckout();
const upstreamSha = "b".repeat(40);
const beforeGitMutation = vi.fn(async () => {
throw new Error("refused by caller");
});
const { runner } = createRunner({
...buildGitWorktreeProbeResponses(),
[`git -C ${tempDir} fetch --all --prune --no-tags`]: { stdout: "" },
[`git -C ${tempDir} rev-parse --abbrev-ref --symbolic-full-name @{upstream}`]: {
stdout: "origin/main",
},
[`git -C ${tempDir} rev-parse @{upstream}`]: { stdout: upstreamSha },
[`git -C ${tempDir} rev-list --max-count=10 ${upstreamSha}`]: {
stdout: `${upstreamSha}\n`,
},
[`git -C ${tempDir} show ${upstreamSha}:package.json`]: {
code: 128,
stderr: "fatal: path 'package.json' does not exist",
},
});
await expect(runWithRunner(runner, { channel: "dev", beforeGitMutation })).rejects.toThrow(
"refused by caller",
);
expect(beforeGitMutation).toHaveBeenCalledWith({
metadataUnreadable: expect.stringContaining("exited 128"),
});
});
it("does not use remote main fallback when existing local main has no upstream", async () => {
await setupGitPackageManagerFixture();
const beforeGitMutation = vi.fn<() => Promise<void>>();
+60 -6
View File
@@ -8,6 +8,10 @@ import {
} from "@openclaw/normalization-core/string-normalization";
import { resolveGatewayInstallEntrypoint } from "../daemon/gateway-entrypoint.js";
import { type CommandOptions, runCommandWithTimeout } from "../process/exec.js";
import {
parsePackageOpenClawSchemaVersions,
type OpenClawSchemaVersions,
} from "../state/openclaw-schema-versions.js";
import {
resolveControlUiDistIndexHealth,
resolveControlUiDistIndexPathForRoot,
@@ -163,7 +167,10 @@ type UpdateRunnerOptions = {
deferConfiguredPluginInstallRepair?: boolean;
allowGatewayServiceRepair?: boolean;
allowGatewayActivation?: boolean;
beforeGitMutation?: () => Promise<{
beforeGitMutation?: (target: {
schemaVersions?: OpenClawSchemaVersions;
metadataUnreadable?: string;
}) => Promise<{
allowGatewayServiceRepair?: boolean;
allowGatewayActivation?: boolean;
} | void>;
@@ -198,6 +205,41 @@ type UpdateInstallSurface =
packageRoot?: undefined;
};
// Only a target we actually read may skip the schema guard as legacy; a failed
// read must abort before mutation or the guard is silently bypassed.
type GitTargetSchemaMetadata =
| { status: "ok"; schemaVersions?: OpenClawSchemaVersions }
| { status: "unreadable"; reason: string };
async function readGitTargetSchemaVersions(params: {
runCommand: CommandRunner;
root: string;
revision: string;
timeoutMs: number;
}): Promise<GitTargetSchemaMetadata> {
let result: Awaited<ReturnType<CommandRunner>>;
try {
result = await params.runCommand(
["git", "-C", params.root, "show", `${params.revision}:package.json`],
{ cwd: params.root, timeoutMs: params.timeoutMs },
);
} catch (error) {
return { status: "unreadable", reason: String(error) };
}
if (result.code !== 0) {
return {
status: "unreadable",
reason: `git show ${params.revision}:package.json exited ${result.code}`,
};
}
try {
const schemaVersions = parsePackageOpenClawSchemaVersions(JSON.parse(result.stdout) as unknown);
return { status: "ok", ...(schemaVersions ? { schemaVersions } : {}) };
} catch (error) {
return { status: "unreadable", reason: `target package.json unparseable: ${String(error)}` };
}
}
function mapManagerResolutionFailure(
reason: UpdatePackageManagerFailureReason,
): NonNullable<UpdateRunResult["reason"]> {
@@ -896,11 +938,23 @@ export async function runGatewayUpdate(opts: UpdateRunnerOptions = {}): Promise<
let gitMutationPrepared = false;
let createdDevBranchDuringUpdate = false;
let localDevBranchExists: boolean | null = null;
const prepareGitMutation = async () => {
const prepareGitMutation = async (targetRevision: string) => {
if (gitMutationPrepared) {
return;
}
const preparation = await opts.beforeGitMutation?.();
const targetMetadata = await readGitTargetSchemaVersions({
runCommand,
root: gitRoot,
revision: targetRevision,
timeoutMs,
});
const preparation = await opts.beforeGitMutation?.(
targetMetadata.status === "ok"
? targetMetadata.schemaVersions
? { schemaVersions: targetMetadata.schemaVersions }
: {}
: { metadataUnreadable: targetMetadata.reason },
);
if (typeof preparation?.allowGatewayServiceRepair === "boolean") {
allowGatewayServiceRepair = preparation.allowGatewayServiceRepair;
}
@@ -1420,7 +1474,7 @@ export async function runGatewayUpdate(opts: UpdateRunnerOptions = {}): Promise<
}
if (devTargetRef) {
await prepareGitMutation();
await prepareGitMutation(selectedSha);
const failure = await runRequiredGitStep(
`git checkout ${selectedSha}`,
["git", "-C", gitRoot, "checkout", "--detach", selectedSha],
@@ -1430,7 +1484,7 @@ export async function runGatewayUpdate(opts: UpdateRunnerOptions = {}): Promise<
return failure;
}
} else {
await prepareGitMutation();
await prepareGitMutation(selectedSha);
let checkedOutSelectedSha = false;
if (needsCheckoutMain) {
const hasLocalDevBranch = localDevBranchExists !== false;
@@ -1530,7 +1584,7 @@ export async function runGatewayUpdate(opts: UpdateRunnerOptions = {}): Promise<
};
}
await prepareGitMutation();
await prepareGitMutation(tag);
const failure = await runRequiredGitStep(
`git checkout ${tag}`,
["git", "-C", gitRoot, "checkout", "--detach", tag],
+116 -98
View File
@@ -13,9 +13,11 @@ import {
import { requireNodeSqlite } from "../infra/node-sqlite.js";
import { listOpenFileDescriptorsForPath } from "../infra/open-file-descriptors.test-support.js";
import { readSqliteNumberPragma } from "../infra/sqlite-pragma.test-support.js";
import { VERSION } from "../version.js";
import type { DB as OpenClawAgentKyselyDatabase } from "./openclaw-agent-db.generated.js";
import {
assertOpenClawAgentDatabaseForMaintenance,
clearOpenClawAgentDatabaseOpenFailure,
closeOpenClawAgentDatabaseByPath,
closeOpenClawAgentDatabasesForTest,
disposeOpenClawAgentDatabaseByPath,
@@ -186,6 +188,25 @@ function createUnsafeIndexDrift(databasePath: string): void {
}
}
function createUnsafeSchemaMetaIndexDrift(databasePath: string): void {
const { DatabaseSync } = requireNodeSqlite();
const database = new DatabaseSync(databasePath);
try {
database.exec("CREATE INDEX unsafe_schema_meta_role ON schema_meta(role);");
database.enableDefensive?.(false);
database.exec("PRAGMA writable_schema = ON;");
database
.prepare(
"UPDATE sqlite_schema SET sql = 'CREATE INDEX unsafe_schema_meta_role ON schema_meta(app_version)' WHERE name = 'unsafe_schema_meta_role'",
)
.run();
const schemaVersion = readSqliteNumberPragma(database, "schema_version");
database.exec(`PRAGMA writable_schema = OFF; PRAGMA schema_version = ${schemaVersion + 1};`);
} finally {
database.close();
}
}
function createTranscriptIdempotencyIndexDrift(
databasePath: string,
options: { duplicateRows?: boolean } = {},
@@ -1475,12 +1496,15 @@ describe("openclaw agent database", () => {
expect(
executeSqliteQueryTakeFirstSync(
database.db,
agentDb.selectFrom("schema_meta").select(["role", "schema_version", "agent_id"]),
agentDb
.selectFrom("schema_meta")
.select(["role", "schema_version", "agent_id", "app_version"]),
),
).toEqual({
role: "agent",
schema_version: OPENCLAW_AGENT_SCHEMA_VERSION,
agent_id: "worker-1",
app_version: VERSION,
});
});
@@ -1978,38 +2002,17 @@ describe("openclaw agent database", () => {
]);
});
it("rejects stale secondary indexes before writable initialization", () => {
it("rejects stale schema_meta indexes before writable initialization", () => {
const stateDir = createTempStateDir();
const env = { OPENCLAW_STATE_DIR: stateDir };
const created = openOpenClawAgentDatabase({ agentId: "worker-1", env });
const databasePath = created.path;
const databasePath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path;
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
createUnsafeIndexDrift(databasePath);
createUnsafeSchemaMetaIndexDrift(databasePath);
const { DatabaseSync } = requireNodeSqlite();
const before = new DatabaseSync(databasePath, { readOnly: true });
let metadataBefore: unknown;
try {
expect(before.prepare("PRAGMA quick_check").get()).toEqual({ quick_check: "ok" });
expect(before.prepare("PRAGMA integrity_check").all()).toEqual(
expect.arrayContaining([
expect.objectContaining({
integrity_check: expect.stringMatching(/missing from index unsafe_index_records_value/),
}),
]),
);
metadataBefore = before
.prepare(
"SELECT schema_version, updated_at FROM schema_meta WHERE meta_key = 'primary' LIMIT 1",
)
.get();
} finally {
before.close();
}
expect(() => openOpenClawAgentDatabase({ agentId: "worker-1", env })).toThrow(
/integrity_check failed.*missing from index unsafe_index_records_value/iu,
/integrity_check failed.*unsafe_schema_meta_role/iu,
);
const independentlyManaged = new DatabaseSync(databasePath);
try {
@@ -2018,33 +2021,66 @@ describe("openclaw agent database", () => {
agentId: "worker-1",
env,
}),
).toThrow(/integrity_check failed.*missing from index unsafe_index_records_value/iu);
).toThrow(/integrity_check failed.*unsafe_schema_meta_role/iu);
} finally {
independentlyManaged.close();
}
const after = new DatabaseSync(databasePath, { readOnly: true });
try {
expect(
after
.prepare(
"SELECT schema_version, updated_at FROM schema_meta WHERE meta_key = 'primary' LIMIT 1",
)
.get(),
).toEqual(metadataBefore);
expect(after.prepare("PRAGMA integrity_check").all()).toEqual(
expect.arrayContaining([
expect.objectContaining({
integrity_check: expect.stringMatching(/missing from index unsafe_index_records_value/),
}),
]),
);
} finally {
after.close();
}
});
it("rejects foreign-key violations before writable initialization", () => {
it("defers unrelated current-schema index corruption to background verification", () => {
const stateDir = createTempStateDir();
const env = { OPENCLAW_STATE_DIR: stateDir };
const databasePath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path;
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
createUnsafeIndexDrift(databasePath);
expect(openOpenClawAgentDatabase({ agentId: "worker-1", env }).db.isOpen).toBe(true);
});
it("runs full integrity before a pending agent schema migration", () => {
const stateDir = createTempStateDir();
const env = { OPENCLAW_STATE_DIR: stateDir };
const databasePath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path;
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
createUnsafeIndexDrift(databasePath);
const { DatabaseSync } = requireNodeSqlite();
const before = new DatabaseSync(databasePath);
try {
before.exec(`PRAGMA user_version = ${OPENCLAW_AGENT_SCHEMA_VERSION - 1};`);
} finally {
before.close();
}
expect(() => openOpenClawAgentDatabase({ agentId: "worker-1", env })).toThrow(
/integrity_check failed.*missing from index unsafe_index_records_value/iu,
);
});
it("runs full integrity before mutating a nonempty unversioned agent database", () => {
const stateDir = createTempStateDir();
const env = { OPENCLAW_STATE_DIR: stateDir };
const databasePath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path;
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
createUnsafeIndexDrift(databasePath);
const { DatabaseSync } = requireNodeSqlite();
const before = new DatabaseSync(databasePath);
try {
before.exec("PRAGMA user_version = 0;");
} finally {
before.close();
}
expect(() => openOpenClawAgentDatabase({ agentId: "worker-1", env })).toThrow(
/integrity_check failed.*missing from index unsafe_index_records_value/iu,
);
});
it("defers current-schema foreign-key violations to background verification", () => {
const stateDir = createTempStateDir();
const env = { OPENCLAW_STATE_DIR: stateDir };
const created = openOpenClawAgentDatabase({ agentId: "worker-1", env });
@@ -2075,42 +2111,11 @@ describe("openclaw agent database", () => {
corrupted.close();
}
const before = new DatabaseSync(databasePath, { readOnly: true });
let metadataBefore: unknown;
try {
metadataBefore = before
.prepare(
"SELECT schema_version, updated_at FROM schema_meta WHERE meta_key = 'primary' LIMIT 1",
)
.get();
} finally {
before.close();
}
const failure =
/foreign_key_check failed.*session_entries row 1 references sessions \(foreign key 0\)/iu;
expect(() => openOpenClawAgentDatabase({ agentId: "worker-1", env })).toThrow(failure);
const independentlyManaged = new DatabaseSync(databasePath);
try {
expect(() =>
ensureOpenClawAgentDatabaseSchema(independentlyManaged, {
agentId: "worker-1",
env,
}),
).toThrow(failure);
} finally {
independentlyManaged.close();
}
expect(openOpenClawAgentDatabase({ agentId: "worker-1", env }).db.isOpen).toBe(true);
closeOpenClawAgentDatabasesForTest();
const after = new DatabaseSync(databasePath, { readOnly: true });
try {
expect(
after
.prepare(
"SELECT schema_version, updated_at FROM schema_meta WHERE meta_key = 'primary' LIMIT 1",
)
.get(),
).toEqual(metadataBefore);
expect(after.prepare("PRAGMA foreign_key_check").get()).toEqual({
table: "session_entries",
rowid: 1,
@@ -2122,29 +2127,42 @@ describe("openclaw agent database", () => {
}
});
it("refuses to open newer per-agent schema versions", () => {
it("latches newer per-agent schema failures before integrity scans", () => {
const stateDir = createTempStateDir();
const databasePath = path.join(
stateDir,
"agents",
"worker-1",
"agent",
"openclaw-agent.sqlite",
);
fs.mkdirSync(path.dirname(databasePath), { recursive: true });
const env = { OPENCLAW_STATE_DIR: stateDir };
const databasePath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path;
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
createUnsafeIndexDrift(databasePath);
const { DatabaseSync } = requireNodeSqlite();
const db = new DatabaseSync(databasePath);
db.exec(`PRAGMA user_version = ${OPENCLAW_AGENT_SCHEMA_VERSION + 1};`);
db.close();
expect(() =>
openOpenClawAgentDatabase({
agentId: "worker-1",
env: { OPENCLAW_STATE_DIR: stateDir },
}),
).toThrow(
`OpenClaw agent database ${databasePath} uses newer schema version ${OPENCLAW_AGENT_SCHEMA_VERSION + 1}; this OpenClaw build supports ${OPENCLAW_AGENT_SCHEMA_VERSION}. Upgrade OpenClaw before opening this database. Do not downgrade OpenClaw or modify the database. To run this older build, use a separate state directory or restore a compatible backup.`,
);
let firstFailure: unknown;
try {
openOpenClawAgentDatabase({ agentId: "worker-1", env });
} catch (error) {
firstFailure = error;
}
expect(firstFailure).toMatchObject({
name: "SqliteSchemaVersionError",
message: expect.stringContaining("https://docs.openclaw.ai/reference/database-schemas"),
});
for (const candidate of [databasePath, `${databasePath}-wal`, `${databasePath}-shm`]) {
fs.rmSync(candidate, { force: true });
}
let secondFailure: unknown;
try {
openOpenClawAgentDatabase({ agentId: "worker-1", env });
} catch (error) {
secondFailure = error;
}
expect(secondFailure).toBe(firstFailure);
clearOpenClawAgentDatabaseOpenFailure(databasePath);
expect(openOpenClawAgentDatabase({ agentId: "worker-1", env }).db.isOpen).toBe(true);
});
it("closes cached handles on normal process exit so no stale WAL remains", () => {
+108 -11
View File
@@ -18,7 +18,11 @@ import {
repairCanonicalSqliteUniqueIndexes,
type CanonicalSqliteUniqueIndex,
} from "../infra/sqlite-index-schema.js";
import { assertSqliteIntegrity } from "../infra/sqlite-integrity.js";
import {
assertSqliteIntegrity,
assertSqliteTableIntegrity,
isTerminalSqliteIntegrityError,
} from "../infra/sqlite-integrity.js";
import {
assertSqliteSchemaContains,
type SqliteSchemaCompatibility,
@@ -40,6 +44,7 @@ import {
} from "../infra/sqlite-wal.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import { normalizeAgentId } from "../routing/session-key.js";
import { VERSION } from "../version.js";
import {
backfillSessionConversations,
migrateConversationDeliveryTargetColumn,
@@ -56,9 +61,12 @@ import { resolveOpenClawAgentSqlitePath } from "./openclaw-agent-db.paths.js";
import { OPENCLAW_AGENT_SCHEMA_SQL } from "./openclaw-agent-schema.generated.js";
import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.generated.js";
import {
clearOpenClawDatabaseVerification,
createOpenClawDatabaseVerificationError,
detectOpenClawStateDatabaseSchemaMigrations,
OPENCLAW_STATE_SCHEMA_VERSION,
OPENCLAW_SQLITE_BUSY_TIMEOUT_MS,
readOpenClawDatabaseVerification,
runOpenClawStateWriteTransaction,
type OpenClawStateDatabaseOptions,
} from "./openclaw-state-db.js";
@@ -154,6 +162,31 @@ type OpenClawAgentMetadataDatabase = Pick<OpenClawAgentKyselyDatabase, "schema_m
type OpenClawAgentRegistryDatabase = Pick<OpenClawStateKyselyDatabase, "agent_databases">;
const cachedDatabases = new Map<string, OpenClawAgentDatabase>();
const terminalOpenFailures = new Map<string, Error>();
/** Latch background verification damage so later opens fail without rescanning. */
export function recordOpenClawAgentDatabaseOpenFailure(pathname: string, error: Error): void {
const resolvedPath = path.resolve(pathname);
terminalOpenFailures.set(resolvedPath, error);
// Quarantine: writing into a proven-corrupt file compounds damage, so close
// any live handle too; the latch then fails every later open fast.
closeOpenClawAgentDatabaseByPath(resolvedPath);
}
/**
* Clear a terminal open failure after doctor rewrites the database file.
* Returns false when the persisted quarantine row survived; callers must
* surface that, or the next open re-quarantines the repaired file.
*/
export function clearOpenClawAgentDatabaseOpenFailure(
pathname: string,
options: OpenClawStateDatabaseOptions = {},
): boolean {
const resolvedPath = path.resolve(pathname);
const cleared = clearOpenClawDatabaseVerification(resolvedPath, options);
terminalOpenFailures.delete(resolvedPath);
return cleared;
}
type ExistingSchemaMeta = {
agentId: string | null;
@@ -722,8 +755,8 @@ function ensureAgentSchema(db: DatabaseSync, agentId: string, pathname: string):
db.exec("PRAGMA foreign_keys = OFF;");
try {
runSqliteImmediateTransactionSync(db, () => {
// Ownership and version checks must share the write transaction with the
// schema update; concurrent openers must not overwrite another agent.
// Repeat preflight ownership/version gates inside the write transaction;
// concurrent openers must not overwrite another agent after the scan.
// Role/ownership gates before version: user_version is only meaningful
// within one schema role, and the global state DB now carries version 3.
assertExistingSchemaOwner(readExistingSchemaMeta(db), agentId, pathname);
@@ -764,7 +797,7 @@ function ensureAgentSchema(db: DatabaseSync, agentId: string, pathname: string):
role: "agent",
schema_version: OPENCLAW_AGENT_SCHEMA_VERSION,
agent_id: agentId,
app_version: null,
app_version: VERSION,
created_at: now,
updated_at: now,
})
@@ -773,7 +806,7 @@ function ensureAgentSchema(db: DatabaseSync, agentId: string, pathname: string):
role: "agent",
schema_version: OPENCLAW_AGENT_SCHEMA_VERSION,
agent_id: agentId,
app_version: null,
app_version: VERSION,
updated_at: now,
}),
),
@@ -793,6 +826,9 @@ export function ensureOpenClawAgentDatabaseSchema(
const databaseOptions = { ...options, agentId };
const pathname = resolveOpenClawAgentSqlitePath(databaseOptions);
ensureOpenClawAgentDatabasePermissions(pathname, databaseOptions);
db.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`);
assertSupportedAgentSchemaVersion(db, pathname);
assertExistingSchemaOwner(readExistingSchemaMeta(db), agentId, pathname);
assertAgentDatabaseIntegrityBeforeMutation(db, pathname);
configureSqlitePreSchemaPragmas(db, {
busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS,
@@ -994,9 +1030,30 @@ function assertAgentDatabaseIntegrityBeforeMutation(
pathname: string,
): void {
database.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`);
// Writable open permits interrupted journal recovery. Schema and connection
// setup must wait until full table/index consistency is proven afterward.
assertSqliteIntegrity(database, pathname);
const userVersion = readSqliteUserVersion(database);
const hasApplicationSchema = database
.prepare("SELECT 1 FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' LIMIT 1")
.get();
if (
(userVersion === 0 && hasApplicationSchema) ||
(userVersion > 0 && userVersion < OPENCLAW_AGENT_SCHEMA_VERSION)
) {
// Migration rewrites the schema; prove the whole file before that mutation.
// Only a truly empty v0 file may skip; legacy v0 files need the same proof.
agentDbLog.info("agent database schema migration pending; verifying integrity first", {
fromVersion: userVersion,
path: pathname,
toVersion: OPENCLAW_AGENT_SCHEMA_VERSION,
});
assertSqliteIntegrity(database, pathname);
return;
}
const schemaMetaExists = database
.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'schema_meta'")
.get();
if (schemaMetaExists) {
assertSqliteTableIntegrity(database, pathname, "schema_meta");
}
}
/** Open or return a cached per-agent database after schema and owner validation. */
@@ -1006,6 +1063,8 @@ export function openOpenClawAgentDatabase(
const agentId = normalizeAgentId(options.agentId);
const databaseOptions = { ...options, agentId };
const pathname = resolveOpenClawAgentSqlitePath(databaseOptions);
// A live cached handle is authoritative: the recorder closes handles when it
// latches, so a cache hit implies the path is not quarantined in this process.
const cached = cachedDatabases.get(pathname);
if (cached?.db.isOpen) {
if (cached.agentId !== agentId) {
@@ -1015,13 +1074,38 @@ export function openOpenClawAgentDatabase(
}
return cached;
}
// Latched paths are quarantined; every fresh open fails fast here until
// doctor repairs the file and clears the latch plus the persisted row.
const terminalFailure = terminalOpenFailures.get(pathname);
if (terminalFailure) {
throw terminalFailure;
}
let persistedFailure: Error | undefined;
try {
const verification = readOpenClawDatabaseVerification(pathname, { env: databaseOptions.env });
if (verification?.result === "error") {
persistedFailure = createOpenClawDatabaseVerificationError(
"agent",
pathname,
verification.error,
);
}
} catch {
// Accepted tradeoff: a locked/unavailable state DB must not block agent
// opens, or every transient state hiccup takes all agents down. The
// in-process latch still covers this process; a missed cross-process
// quarantine is re-detected by the next daily verifier pass.
}
if (persistedFailure) {
recordOpenClawAgentDatabaseOpenFailure(pathname, persistedFailure);
throw persistedFailure;
}
if (cached) {
// A closed handle can leave Kysely and WAL helpers cached; clear both before reopening.
cached.walMaintenance.close();
clearNodeSqliteKyselyCacheForDatabase(cached.db);
cachedDatabases.delete(pathname);
}
const openStartedAt = Date.now();
ensureOpenClawAgentDatabasePermissions(pathname, databaseOptions);
const sqlite = requireNodeSqlite();
@@ -1029,6 +1113,9 @@ export function openOpenClawAgentDatabase(
const walMaintenance = (() => {
let maintenance: SqliteWalMaintenance | undefined;
try {
db.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`);
assertSupportedAgentSchemaVersion(db, pathname);
assertExistingSchemaOwner(readExistingSchemaMeta(db), agentId, pathname);
assertAgentDatabaseIntegrityBeforeMutation(db, pathname);
configureSqlitePreSchemaPragmas(db, {
busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS,
@@ -1045,6 +1132,12 @@ export function openOpenClawAgentDatabase(
} catch (err) {
maintenance?.close();
db.close();
if (
err instanceof Error &&
(err.name === "SqliteSchemaVersionError" || isTerminalSqliteIntegrityError(err))
) {
recordOpenClawAgentDatabaseOpenFailure(pathname, err);
}
throw err;
}
})();
@@ -1057,6 +1150,7 @@ export function openOpenClawAgentDatabase(
throw error;
}
cachedDatabases.set(pathname, database);
terminalOpenFailures.delete(pathname);
// Safety net for processes that end without an orderly close: agent DBs have
// no shutdown owner like the ACP/gateway state DB closes. Closing unregisters.
unregisterExitClose ??= registerSqliteCacheExitClose(closeOpenClawAgentDatabases);
@@ -1207,6 +1301,9 @@ export function closeOpenClawAgentDatabases(): void {
cachedDatabases.clear();
}
/** Test alias for closing cached agent database handles from teardown code. */
export const closeOpenClawAgentDatabasesForTest = closeOpenClawAgentDatabases;
/** Close cached agent handles and clear terminal failure latches for test isolation. */
export function closeOpenClawAgentDatabasesForTest(): void {
closeOpenClawAgentDatabases();
terminalOpenFailures.clear();
}
/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */
@@ -0,0 +1,172 @@
import { afterAll, afterEach, describe, expect, it } from "vitest";
import packageJson from "../../package.json" with { type: "json" };
import { cleanupTempDirs, makeTempDir } from "../../test/helpers/temp-dir.js";
import { requireNodeSqlite } from "../infra/node-sqlite.js";
import {
closeOpenClawAgentDatabasesForTest,
OPENCLAW_AGENT_SCHEMA_VERSION,
openOpenClawAgentDatabase,
} from "./openclaw-agent-db.js";
import { preflightOpenClawDatabaseSchemas } from "./openclaw-database-preflight.js";
import {
closeOpenClawStateDatabaseForTest,
OPENCLAW_STATE_SCHEMA_VERSION,
openOpenClawStateDatabase,
} from "./openclaw-state-db.js";
const tempDirs: string[] = [];
afterEach(() => {
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
});
afterAll(() => cleanupTempDirs(tempDirs));
describe("OpenClaw database schema preflight", () => {
it("keeps package schema support metadata aligned", () => {
expect(packageJson.openclaw.schemaVersions).toEqual({
state: OPENCLAW_STATE_SCHEMA_VERSION,
agent: OPENCLAW_AGENT_SCHEMA_VERSION,
});
});
it("collects newer state and registered agent schemas with writer builds", () => {
const stateDir = makeTempDir(tempDirs, "openclaw-database-preflight-");
const env = { OPENCLAW_STATE_DIR: stateDir };
const statePath = openOpenClawStateDatabase({ env }).path;
const agentPath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path;
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
const { DatabaseSync } = requireNodeSqlite();
const state = new DatabaseSync(statePath);
try {
state.exec(`PRAGMA user_version = ${OPENCLAW_STATE_SCHEMA_VERSION + 1};`);
state
.prepare("UPDATE schema_meta SET app_version = ? WHERE meta_key = 'primary'")
.run("state-writer-build");
} finally {
state.close();
}
const agent = new DatabaseSync(agentPath);
try {
agent.exec(`PRAGMA user_version = ${OPENCLAW_AGENT_SCHEMA_VERSION + 1};`);
agent
.prepare("UPDATE schema_meta SET app_version = ? WHERE meta_key = 'primary'")
.run("agent-writer-build");
} finally {
agent.close();
}
expect(
preflightOpenClawDatabaseSchemas({
env,
supportedVersions: {
state: OPENCLAW_STATE_SCHEMA_VERSION,
agent: OPENCLAW_AGENT_SCHEMA_VERSION,
},
}),
).toEqual({
incompatible: [
{
kind: "state",
path: statePath,
foundVersion: OPENCLAW_STATE_SCHEMA_VERSION + 1,
supportedVersion: OPENCLAW_STATE_SCHEMA_VERSION,
writerAppVersion: "state-writer-build",
},
{
kind: "agent",
path: agentPath,
agentId: "worker-1",
foundVersion: OPENCLAW_AGENT_SCHEMA_VERSION + 1,
supportedVersion: OPENCLAW_AGENT_SCHEMA_VERSION,
writerAppVersion: "agent-writer-build",
},
],
indeterminate: [],
});
});
it("reports an existing unreadable state database as indeterminate", () => {
const stateDir = makeTempDir(tempDirs, "openclaw-database-preflight-unreadable-state-");
const env = { OPENCLAW_STATE_DIR: stateDir };
const statePath = openOpenClawStateDatabase({ env }).path;
closeOpenClawStateDatabaseForTest();
fs.writeFileSync(statePath, "not a sqlite database");
expect(
preflightOpenClawDatabaseSchemas({
env,
supportedVersions: {
state: OPENCLAW_STATE_SCHEMA_VERSION,
agent: OPENCLAW_AGENT_SCHEMA_VERSION,
},
}),
).toEqual({
incompatible: [],
indeterminate: [
{ kind: "state", path: statePath, reason: expect.stringMatching(/database|file/iu) },
],
});
});
it("reports a failed agent registry query as indeterminate", () => {
const stateDir = makeTempDir(tempDirs, "openclaw-database-preflight-registry-");
const env = { OPENCLAW_STATE_DIR: stateDir };
const statePath = openOpenClawStateDatabase({ env }).path;
closeOpenClawStateDatabaseForTest();
const { DatabaseSync } = requireNodeSqlite();
const state = new DatabaseSync(statePath);
try {
state.exec("DROP TABLE agent_databases; CREATE TABLE agent_databases (bad TEXT) STRICT;");
} finally {
state.close();
}
expect(
preflightOpenClawDatabaseSchemas({
env,
supportedVersions: {
state: OPENCLAW_STATE_SCHEMA_VERSION,
agent: OPENCLAW_AGENT_SCHEMA_VERSION,
},
}),
).toEqual({
incompatible: [],
indeterminate: [
{
kind: "state",
path: statePath,
reason: expect.stringContaining("agent database registry query failed"),
},
],
});
});
it("reports an existing unreadable registered agent database as indeterminate", () => {
const stateDir = makeTempDir(tempDirs, "openclaw-database-preflight-unreadable-agent-");
const env = { OPENCLAW_STATE_DIR: stateDir };
const agentPath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path;
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
fs.writeFileSync(agentPath, "not a sqlite database");
expect(
preflightOpenClawDatabaseSchemas({
env,
supportedVersions: {
state: OPENCLAW_STATE_SCHEMA_VERSION,
agent: OPENCLAW_AGENT_SCHEMA_VERSION,
},
}),
).toEqual({
incompatible: [],
indeterminate: [
{ kind: "agent", path: agentPath, reason: expect.stringMatching(/database|file/iu) },
],
});
});
});
import fs from "node:fs";
+173
View File
@@ -0,0 +1,173 @@
import { existsSync } from "node:fs";
import path from "node:path";
import type { DatabaseSync } from "node:sqlite";
import {
clearNodeSqliteKyselyCacheForDatabase,
executeSqliteQuerySync,
getNodeSqliteKysely,
} from "../infra/kysely-sync.js";
import { requireNodeSqlite } from "../infra/node-sqlite.js";
import { readSqliteUserVersion } from "../infra/sqlite-user-version.js";
import type { OpenClawSchemaVersions } from "./openclaw-schema-versions.js";
import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.generated.js";
import {
OPENCLAW_DATABASE_SCHEMA_DOCS_URL,
OPENCLAW_SQLITE_BUSY_TIMEOUT_MS,
} from "./openclaw-state-db.js";
import { resolveOpenClawStateSqlitePath } from "./openclaw-state-db.paths.js";
export { OPENCLAW_DATABASE_SCHEMA_DOCS_URL } from "./openclaw-state-db.js";
export type IncompatibleOpenClawDatabase = {
kind: "agent" | "state";
path: string;
agentId?: string;
foundVersion: number;
supportedVersion: number;
writerAppVersion?: string;
};
export type IndeterminateOpenClawDatabase = {
kind: "agent" | "state";
path: string;
reason: string;
};
export type OpenClawDatabaseSchemaPreflight = {
incompatible: IncompatibleOpenClawDatabase[];
indeterminate: IndeterminateOpenClawDatabase[];
};
type AgentRegistryDatabase = Pick<OpenClawStateKyselyDatabase, "agent_databases">;
/** Fatal Gateway refusal when persisted schemas were written by a newer build. */
export class OpenClawDatabaseSchemaPreflightError extends Error {
constructor(readonly incompatibleDatabases: readonly IncompatibleOpenClawDatabase[]) {
super(
`Gateway refused startup because ${incompatibleDatabases.length} OpenClaw database schema(s) are newer than this build. See ${OPENCLAW_DATABASE_SCHEMA_DOCS_URL}.`,
);
this.name = "OpenClawDatabaseSchemaPreflightError";
}
}
function readWriterAppVersion(database: DatabaseSync): string | undefined {
try {
const row = database
.prepare("SELECT app_version FROM schema_meta WHERE meta_key = 'primary' LIMIT 1")
.get() as { app_version?: unknown } | undefined;
return typeof row?.app_version === "string" && row.app_version.length > 0
? row.app_version
: undefined;
} catch {
return undefined;
}
}
function readRegisteredAgentDatabases(database: DatabaseSync): Array<{
agentId: string;
path: string;
}> {
const table = database
.prepare("SELECT 1 AS ok FROM sqlite_master WHERE type = 'table' AND name = 'agent_databases'")
.get();
if (!table) {
return [];
}
const db = getNodeSqliteKysely<AgentRegistryDatabase>(database);
return executeSqliteQuerySync(
database,
db.selectFrom("agent_databases").select(["agent_id", "path"]),
).rows.flatMap((row) =>
typeof row.agent_id === "string" && typeof row.path === "string"
? [{ agentId: row.agent_id, path: row.path }]
: [],
);
}
function errorReason(error: unknown): string {
return error instanceof Error ? error.message : String(error);
}
/** Read schema headers; report unreadable existing files without diagnosing or repairing them. */
export function preflightOpenClawDatabaseSchemas(options: {
env: NodeJS.ProcessEnv;
supportedVersions: OpenClawSchemaVersions;
}): OpenClawDatabaseSchemaPreflight {
const result: OpenClawDatabaseSchemaPreflight = { incompatible: [], indeterminate: [] };
const statePath = path.resolve(resolveOpenClawStateSqlitePath(options.env));
if (!existsSync(statePath)) {
return result;
}
const sqlite = requireNodeSqlite();
let stateDatabase: DatabaseSync | undefined;
try {
stateDatabase = new sqlite.DatabaseSync(statePath, { readOnly: true });
stateDatabase.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`);
const stateVersion = readSqliteUserVersion(stateDatabase);
if (stateVersion > options.supportedVersions.state) {
const writerAppVersion = readWriterAppVersion(stateDatabase);
result.incompatible.push({
kind: "state",
path: statePath,
foundVersion: stateVersion,
supportedVersion: options.supportedVersions.state,
...(writerAppVersion ? { writerAppVersion } : {}),
});
}
let registeredDatabases: ReturnType<typeof readRegisteredAgentDatabases>;
try {
registeredDatabases = readRegisteredAgentDatabases(stateDatabase);
} catch (error) {
result.indeterminate.push({
kind: "state",
path: statePath,
reason: `agent database registry query failed: ${errorReason(error)}`,
});
return result;
}
for (const row of registeredDatabases) {
const agentPath = path.resolve(row.path);
if (!existsSync(agentPath)) {
continue;
}
let agentDatabase: DatabaseSync | undefined;
try {
agentDatabase = new sqlite.DatabaseSync(agentPath, { readOnly: true });
agentDatabase.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`);
const agentVersion = readSqliteUserVersion(agentDatabase);
if (agentVersion <= options.supportedVersions.agent) {
continue;
}
const writerAppVersion = readWriterAppVersion(agentDatabase);
result.incompatible.push({
kind: "agent",
path: agentPath,
agentId: row.agentId,
foundVersion: agentVersion,
supportedVersion: options.supportedVersions.agent,
...(writerAppVersion ? { writerAppVersion } : {}),
});
} catch (error) {
result.indeterminate.push({
kind: "agent",
path: agentPath,
reason: errorReason(error),
});
} finally {
agentDatabase?.close();
}
}
return result;
} catch (error) {
result.indeterminate.push({ kind: "state", path: statePath, reason: errorReason(error) });
return result;
} finally {
if (stateDatabase) {
clearNodeSqliteKyselyCacheForDatabase(stateDatabase);
stateDatabase.close();
}
}
}
+256
View File
@@ -0,0 +1,256 @@
import { existsSync } from "node:fs";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { Worker } from "node:worker_threads";
import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import {
listOpenClawRegisteredAgentDatabases,
recordOpenClawAgentDatabaseOpenFailure,
} from "./openclaw-agent-db.js";
import type {
OpenClawDatabaseVerifyResult,
OpenClawDatabaseVerifyTarget,
} from "./openclaw-database-verify.worker.js";
import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.generated.js";
import {
recordOpenClawStateDatabaseOpenFailure,
runOpenClawStateWriteTransaction,
} from "./openclaw-state-db.js";
import { resolveOpenClawStateSqlitePath } from "./openclaw-state-db.paths.js";
export const OPENCLAW_DATABASE_VERIFY_INITIAL_DELAY_MS = 5 * 60_000;
export const OPENCLAW_DATABASE_VERIFY_INTERVAL_MS = 24 * 60 * 60_000;
const log = createSubsystemLogger("state/database-verify");
type VerificationDatabase = Pick<OpenClawStateKyselyDatabase, "database_verifications">;
function toError(error: unknown): Error {
return error instanceof Error ? error : new Error(String(error));
}
function resolveDatabaseVerifyWorkerUrl(currentModuleUrl = import.meta.url): URL {
const currentPath = fileURLToPath(currentModuleUrl);
const normalized = currentPath.replaceAll(path.sep, "/");
const distMarker = "/dist/";
const distIndex = normalized.lastIndexOf(distMarker);
if (distIndex >= 0) {
const distRoot = currentPath.slice(0, distIndex + distMarker.length);
return pathToFileURL(path.join(distRoot, "state", "openclaw-database-verify.worker.js"));
}
const extension = path.extname(currentPath) || ".js";
return new URL(`./openclaw-database-verify.worker${extension}`, currentModuleUrl);
}
function isVerifyResult(value: unknown): value is OpenClawDatabaseVerifyResult {
if (!value || typeof value !== "object" || Array.isArray(value)) {
return false;
}
const result = value as Record<string, unknown>;
return (
typeof result.path === "string" &&
typeof result.ok === "boolean" &&
(result.error === undefined || typeof result.error === "string") &&
(result.terminal === undefined || typeof result.terminal === "boolean")
);
}
export function runDatabaseVerifyWorker(
targets: readonly OpenClawDatabaseVerifyTarget[],
options: { onWorker?: (worker: Worker | undefined) => void; workerUrl?: URL } = {},
): Promise<OpenClawDatabaseVerifyResult[]> {
const workerUrl = options.workerUrl ?? resolveDatabaseVerifyWorkerUrl();
const execArgv = workerUrl.pathname.endsWith(".ts") ? ["--import", "tsx"] : undefined;
let worker: Worker;
try {
worker = new Worker(workerUrl, { workerData: targets, execArgv });
} catch (error) {
return Promise.reject(toError(error));
}
options.onWorker?.(worker);
return new Promise((resolve, reject) => {
let settled = false;
const settle = (finish: () => void) => {
if (settled) {
return;
}
settled = true;
worker.removeAllListeners();
options.onWorker?.(undefined);
finish();
};
worker.once("message", (message: unknown) => {
settle(() => {
if (!Array.isArray(message) || !message.every(isVerifyResult)) {
reject(new Error("database verification worker returned invalid results"));
return;
}
resolve(message);
});
});
worker.once("error", (error) => settle(() => reject(toError(error))));
worker.once("exit", (code) => {
if (code !== 0) {
settle(() => reject(new Error(`database verification worker exited with code ${code}`)));
} else {
settle(() => reject(new Error("database verification worker exited without results")));
}
});
});
}
/** Resolve the state database and current registered agent database paths. */
export function collectOpenClawDatabaseVerifyTargets(options: {
env: NodeJS.ProcessEnv;
}): OpenClawDatabaseVerifyTarget[] {
const targets = new Map<string, OpenClawDatabaseVerifyTarget>();
const statePath = path.resolve(resolveOpenClawStateSqlitePath(options.env));
if (existsSync(statePath)) {
targets.set(statePath, { kind: "state", label: "OpenClaw state database", path: statePath });
}
let registeredDatabases: ReturnType<typeof listOpenClawRegisteredAgentDatabases> = [];
try {
registeredDatabases = listOpenClawRegisteredAgentDatabases({ env: options.env });
} catch (error) {
log.warn("failed to collect registered agent databases for integrity verification", {
error: String(error),
});
}
for (const registered of registeredDatabases) {
const agentPath = path.resolve(registered.path);
if (!existsSync(agentPath) || targets.has(agentPath)) {
continue;
}
targets.set(agentPath, {
kind: "agent",
label: `OpenClaw agent database ${registered.agentId}`,
path: agentPath,
});
}
return [...targets.values()];
}
function createVerificationFailure(result: OpenClawDatabaseVerifyResult): Error {
const error = new Error(
result.error ?? `SQLite integrity verification failed for ${result.path}`,
);
error.name = "SqliteIntegrityError";
return error;
}
/** Persist one worker batch, then latch failures at each database owner. */
export function applyOpenClawDatabaseVerificationResults(options: {
env: NodeJS.ProcessEnv;
results: readonly OpenClawDatabaseVerifyResult[];
targets: readonly OpenClawDatabaseVerifyTarget[];
verifiedAt?: number;
}): void {
const targetByPath = new Map(options.targets.map((target) => [target.path, target]));
const verifiedAt = options.verifiedAt ?? Date.now();
// One best-effort row write into a possibly corrupt state file buys restart-durable quarantine.
// Doctor rebuilds the file during repair, so this bounded mutation is an accepted tradeoff.
const persistResults = (results: readonly OpenClawDatabaseVerifyResult[]) => {
runOpenClawStateWriteTransaction(
(database) => {
const db = getNodeSqliteKysely<VerificationDatabase>(database.db);
for (const result of results) {
const target = targetByPath.get(result.path);
if (!target) {
continue;
}
const existing = executeSqliteQuerySync(
database.db,
db
.selectFrom("database_verifications")
.select("result")
.where("path", "=", result.path)
.limit(1),
).rows[0];
if (existing?.result === "error") {
continue;
}
executeSqliteQuerySync(
database.db,
db
.insertInto("database_verifications")
.values({
path: result.path,
kind: target.kind,
verified_at: verifiedAt,
result: result.ok ? "ok" : result.terminal ? "error" : "inconclusive",
error: result.error ?? null,
})
.onConflict((conflict) =>
conflict.column("path").doUpdateSet({
kind: target.kind,
verified_at: verifiedAt,
result: result.ok ? "ok" : result.terminal ? "error" : "inconclusive",
error: result.error ?? null,
}),
),
);
}
},
{ env: options.env },
{ operationLabel: "state.database-verifications.record" },
);
};
try {
persistResults(options.results);
} catch (error) {
// Terminal rows are the restart-durable quarantine; retry those once so a
// transient state-DB lock does not silently downgrade them to process-local.
const terminalResults = options.results.filter((result) => !result.ok && result.terminal);
let retried = false;
if (terminalResults.length > 0) {
try {
persistResults(terminalResults);
retried = true;
} catch {
// fall through to the process-local warning below
}
}
if (!retried) {
// Accepted tradeoff: when this write fails the quarantine stays process-local
// until the next daily verifier pass re-detects and re-persists. A sidecar
// marker would break the SQLite-only storage contract, and startup full
// scans would reintroduce the multi-second opens this design removes.
log.error("failed to persist database verification results; quarantine is process-local", {
error: String(error),
});
}
}
for (const result of options.results) {
if (result.ok) {
continue;
}
const target = targetByPath.get(result.path);
if (!target) {
continue;
}
if (!result.terminal) {
log.warn("database integrity verification was inconclusive", {
kind: target.kind,
label: target.label,
path: result.path,
error: result.error,
});
continue;
}
const error = createVerificationFailure(result);
if (target.kind === "state") {
recordOpenClawStateDatabaseOpenFailure(result.path, error);
} else {
recordOpenClawAgentDatabaseOpenFailure(result.path, error);
}
log.error("database integrity verification failed", {
kind: target.kind,
label: target.label,
path: result.path,
error: error.message,
});
}
}
+238
View File
@@ -0,0 +1,238 @@
import fs from "node:fs";
import { afterAll, afterEach, describe, expect, it } from "vitest";
import { cleanupTempDirs, makeTempDir } from "../../test/helpers/temp-dir.js";
import { requireNodeSqlite } from "../infra/node-sqlite.js";
import { readSqliteNumberPragma } from "../infra/sqlite-pragma.test-support.js";
import {
clearOpenClawAgentDatabaseOpenFailure,
closeOpenClawAgentDatabasesForTest,
openOpenClawAgentDatabase,
} from "./openclaw-agent-db.js";
import {
applyOpenClawDatabaseVerificationResults,
runDatabaseVerifyWorker,
} from "./openclaw-database-verify.impl.js";
import {
type OpenClawDatabaseVerifyTarget,
verifyOpenClawDatabases,
} from "./openclaw-database-verify.worker.js";
import {
closeOpenClawStateDatabaseForTest,
openOpenClawStateDatabase,
repairOpenClawStateDatabaseSchema,
} from "./openclaw-state-db.js";
const tempDirs: string[] = [];
afterEach(() => {
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
});
afterAll(() => cleanupTempDirs(tempDirs));
function createUnsafeIndexDrift(databasePath: string): void {
const { DatabaseSync } = requireNodeSqlite();
const database = new DatabaseSync(databasePath);
try {
database.exec(`
CREATE TABLE unsafe_index_records (
id INTEGER PRIMARY KEY,
indexed_value TEXT NOT NULL,
alternate_value TEXT NOT NULL
);
CREATE INDEX unsafe_index_records_value ON unsafe_index_records(indexed_value);
INSERT INTO unsafe_index_records (indexed_value, alternate_value)
VALUES ('alpha', 'zeta'), ('beta', 'eta'), ('gamma', 'theta');
`);
database.enableDefensive?.(false);
database.exec("PRAGMA writable_schema = ON;");
database
.prepare(
"UPDATE sqlite_schema SET sql = 'CREATE INDEX unsafe_index_records_value ON unsafe_index_records(alternate_value)' WHERE name = 'unsafe_index_records_value'",
)
.run();
const schemaVersion = readSqliteNumberPragma(database, "schema_version");
database.exec(`PRAGMA writable_schema = OFF; PRAGMA schema_version = ${schemaVersion + 1};`);
} finally {
database.close();
}
}
describe("OpenClaw database integrity verifier", () => {
it("detects corruption off-thread, persists it, and latches later opens", async () => {
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-");
const env = { OPENCLAW_STATE_DIR: stateDir };
const agentPath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path;
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
createUnsafeIndexDrift(agentPath);
const targets: OpenClawDatabaseVerifyTarget[] = [
{ kind: "agent", label: "OpenClaw agent database worker-1", path: agentPath },
];
const directResults = verifyOpenClawDatabases(targets);
expect(directResults).toEqual([
{
path: agentPath,
ok: false,
error: expect.stringMatching(/missing from index unsafe_index_records_value/iu),
terminal: true,
},
]);
await expect(runDatabaseVerifyWorker(targets)).resolves.toEqual(directResults);
// The drift lives outside schema_meta, so the rescoped open still succeeds;
// the recorder must then quarantine this live handle, not just future opens.
const liveHandle = openOpenClawAgentDatabase({ agentId: "worker-1", env });
expect(liveHandle.db.isOpen).toBe(true);
applyOpenClawDatabaseVerificationResults({
env,
results: directResults,
targets,
verifiedAt: 1234,
});
expect(liveHandle.db.isOpen).toBe(false);
expect(
openOpenClawStateDatabase({ env })
.db.prepare(
"SELECT path, kind, verified_at, result, error FROM database_verifications WHERE path = ?",
)
.get(agentPath),
).toEqual({
path: agentPath,
kind: "agent",
verified_at: 1234,
result: "error",
error: directResults[0]?.error,
});
applyOpenClawDatabaseVerificationResults({
env,
results: [{ path: agentPath, ok: false, error: "database busy", terminal: false }],
targets,
verifiedAt: 1235,
});
expect(
openOpenClawStateDatabase({ env })
.db.prepare("SELECT verified_at, result, error FROM database_verifications WHERE path = ?")
.get(agentPath),
).toEqual({ verified_at: 1234, result: "error", error: directResults[0]?.error });
expect(() => openOpenClawAgentDatabase({ agentId: "worker-1", env })).toThrow(
expect.objectContaining({ name: "SqliteIntegrityError" }),
);
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
expect(() => openOpenClawAgentDatabase({ agentId: "worker-1", env })).toThrow(
expect.objectContaining({
name: "SqliteIntegrityError",
message: expect.stringContaining(directResults[0]?.error ?? ""),
}),
);
clearOpenClawAgentDatabaseOpenFailure(agentPath, { env });
expect(openOpenClawAgentDatabase({ agentId: "worker-1", env }).db.isOpen).toBe(true);
});
it("reports an uncleared quarantine row instead of claiming repair success", () => {
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-clear-failure-");
const env = { OPENCLAW_STATE_DIR: stateDir };
const agentPath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path;
const statePath = openOpenClawStateDatabase({ env }).path;
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
applyOpenClawDatabaseVerificationResults({
env,
results: [{ path: agentPath, ok: false, error: "corrupt index", terminal: true }],
targets: [{ kind: "agent", label: "OpenClaw agent database worker-1", path: agentPath }],
verifiedAt: 4567,
});
closeOpenClawStateDatabaseForTest();
// A read-only state DB cannot drop the quarantine row; the clear must say so
// instead of letting doctor report success while the next open still refuses.
fs.chmodSync(statePath, 0o444);
try {
expect(clearOpenClawAgentDatabaseOpenFailure(agentPath, { env })).toBe(false);
} finally {
// WAL sidecars minted during the read-only attempt inherit its mode.
for (const sidecar of [statePath, `${statePath}-wal`, `${statePath}-shm`]) {
if (fs.existsSync(sidecar)) {
fs.chmodSync(sidecar, 0o600);
}
}
}
expect(clearOpenClawAgentDatabaseOpenFailure(agentPath, { env })).toBe(true);
expect(openOpenClawAgentDatabase({ agentId: "worker-1", env }).db.isOpen).toBe(true);
});
it("persists transient verifier errors as inconclusive without latching", () => {
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-transient-");
const env = { OPENCLAW_STATE_DIR: stateDir };
const agentPath = openOpenClawAgentDatabase({ agentId: "worker-1", env }).path;
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
const targets: OpenClawDatabaseVerifyTarget[] = [
{ kind: "agent", label: "OpenClaw agent database worker-1", path: agentPath },
];
applyOpenClawDatabaseVerificationResults({
env,
results: [{ path: agentPath, ok: false, error: "Error: database is busy", terminal: false }],
targets,
verifiedAt: 2345,
});
expect(
openOpenClawStateDatabase({ env })
.db.prepare("SELECT result, error FROM database_verifications WHERE path = ?")
.get(agentPath),
).toEqual({ result: "inconclusive", error: "Error: database is busy" });
expect(openOpenClawAgentDatabase({ agentId: "worker-1", env }).db.isOpen).toBe(true);
});
it("persists state failure quarantine across restart until doctor repair", () => {
const stateDir = makeTempDir(tempDirs, "openclaw-database-verify-state-failure-");
const env = { OPENCLAW_STATE_DIR: stateDir };
const statePath = openOpenClawStateDatabase({ env }).path;
closeOpenClawStateDatabaseForTest();
const targets: OpenClawDatabaseVerifyTarget[] = [
{ kind: "state", label: "OpenClaw state database", path: statePath },
];
applyOpenClawDatabaseVerificationResults({
env,
results: [{ path: statePath, ok: false, error: "corrupt index", terminal: true }],
targets,
verifiedAt: 3456,
});
const { DatabaseSync } = requireNodeSqlite();
const raw = new DatabaseSync(statePath, { readOnly: true });
try {
expect(
raw
.prepare("SELECT result, error FROM database_verifications WHERE path = ?")
.get(statePath),
).toEqual({ result: "error", error: "corrupt index" });
} finally {
raw.close();
}
expect(() => openOpenClawStateDatabase({ env })).toThrow(
expect.objectContaining({
name: "SqliteIntegrityError",
message: expect.stringContaining("corrupt index"),
}),
);
closeOpenClawStateDatabaseForTest();
expect(() => openOpenClawStateDatabase({ env })).toThrow(
expect.objectContaining({
name: "SqliteIntegrityError",
message: expect.stringContaining("corrupt index"),
}),
);
expect(repairOpenClawStateDatabaseSchema({ env }).warnings).toEqual([]);
expect(openOpenClawStateDatabase({ env }).db.isOpen).toBe(true);
});
});
+63
View File
@@ -0,0 +1,63 @@
import type { Worker } from "node:worker_threads";
import { createSubsystemLogger } from "../logging/subsystem.js";
import {
applyOpenClawDatabaseVerificationResults,
collectOpenClawDatabaseVerifyTargets,
OPENCLAW_DATABASE_VERIFY_INITIAL_DELAY_MS,
OPENCLAW_DATABASE_VERIFY_INTERVAL_MS,
runDatabaseVerifyWorker,
} from "./openclaw-database-verify.impl.js";
const log = createSubsystemLogger("state/database-verify");
/** Start the Gateway-owned delayed daily integrity verifier. */
export function startOpenClawDatabaseIntegrityVerifier(options: { env: NodeJS.ProcessEnv }): {
stop: () => Promise<void>;
} {
let activeWorker: Worker | undefined;
let stopped = false;
let timer: ReturnType<typeof setTimeout> | undefined;
const schedule = (delayMs: number) => {
timer = setTimeout(() => void run(), delayMs);
timer.unref?.();
};
const run = async () => {
timer = undefined;
try {
const targets = collectOpenClawDatabaseVerifyTargets(options);
if (targets.length > 0) {
const results = await runDatabaseVerifyWorker(targets, {
onWorker: (worker) => {
activeWorker = worker;
},
});
if (!stopped) {
applyOpenClawDatabaseVerificationResults({ ...options, results, targets });
}
}
} catch (error) {
if (!stopped) {
log.error("database integrity verifier failed", { error: String(error) });
}
} finally {
activeWorker = undefined;
if (!stopped) {
schedule(OPENCLAW_DATABASE_VERIFY_INTERVAL_MS);
}
}
};
schedule(OPENCLAW_DATABASE_VERIFY_INITIAL_DELAY_MS);
return {
stop: async () => {
stopped = true;
if (timer) {
clearTimeout(timer);
timer = undefined;
}
await activeWorker?.terminate();
activeWorker = undefined;
},
};
}
@@ -0,0 +1,59 @@
import { parentPort, workerData } from "node:worker_threads";
import { requireNodeSqlite } from "../infra/node-sqlite.js";
import {
assertSqliteIntegrity,
isTerminalSqliteIntegrityError,
} from "../infra/sqlite-integrity.js";
import { OPENCLAW_SQLITE_BUSY_TIMEOUT_MS } from "./openclaw-state-db.js";
export type OpenClawDatabaseVerifyTarget = {
path: string;
kind: "agent" | "state";
label: string;
};
export type OpenClawDatabaseVerifyResult = {
path: string;
ok: boolean;
error?: string;
terminal?: boolean;
};
function isVerifyTarget(value: unknown): value is OpenClawDatabaseVerifyTarget {
if (!value || typeof value !== "object" || Array.isArray(value)) {
return false;
}
const target = value as Record<string, unknown>;
return (
typeof target.path === "string" &&
(target.kind === "agent" || target.kind === "state") &&
typeof target.label === "string"
);
}
/** Verify database files serially so large agent scans never compete for I/O. */
export function verifyOpenClawDatabases(
targets: readonly OpenClawDatabaseVerifyTarget[],
): OpenClawDatabaseVerifyResult[] {
const sqlite = requireNodeSqlite();
return targets.map((target) => {
let database: InstanceType<typeof sqlite.DatabaseSync> | undefined;
try {
database = new sqlite.DatabaseSync(target.path, { readOnly: true });
database.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`);
assertSqliteIntegrity(database, target.label);
return { path: target.path, ok: true };
} catch (error) {
const detail = error instanceof Error ? `${error.name}: ${error.message}` : String(error);
const terminal = error instanceof Error && isTerminalSqliteIntegrityError(error);
return { path: target.path, ok: false, error: detail, terminal };
} finally {
database?.close();
}
});
}
if (parentPort) {
const targets = Array.isArray(workerData) ? workerData.filter(isVerifyTarget) : [];
parentPort.postMessage(verifyOpenClawDatabases(targets), []);
}
+33
View File
@@ -0,0 +1,33 @@
export type OpenClawSchemaVersions = {
state: number;
agent: number;
};
export function parseOpenClawSchemaVersions(value: unknown): OpenClawSchemaVersions | undefined {
if (!value || typeof value !== "object" || Array.isArray(value)) {
return undefined;
}
const record = value as Record<string, unknown>;
if (
!Number.isInteger(record.state) ||
(record.state as number) < 0 ||
!Number.isInteger(record.agent) ||
(record.agent as number) < 0
) {
return undefined;
}
return { state: record.state as number, agent: record.agent as number };
}
export function parsePackageOpenClawSchemaVersions(
packageJson: unknown,
): OpenClawSchemaVersions | undefined {
if (!packageJson || typeof packageJson !== "object" || Array.isArray(packageJson)) {
return undefined;
}
const openclaw = (packageJson as Record<string, unknown>).openclaw;
if (!openclaw || typeof openclaw !== "object" || Array.isArray(openclaw)) {
return undefined;
}
return parseOpenClawSchemaVersions((openclaw as Record<string, unknown>).schemaVersions);
}
+9
View File
@@ -401,6 +401,14 @@ export interface CurrentConversationBindings {
updated_at: number;
}
export interface DatabaseVerifications {
error: string | null;
kind: string;
path: string;
result: string;
verified_at: number;
}
export interface DeliveryQueueEntries {
account_id: string | null;
channel: string | null;
@@ -1329,6 +1337,7 @@ export interface DB {
config_health_entries: ConfigHealthEntries;
cron_jobs: CronJobs;
current_conversation_bindings: CurrentConversationBindings;
database_verifications: DatabaseVerifications;
delivery_queue_entries: DeliveryQueueEntries;
device_auth_tokens: DeviceAuthTokens;
device_bootstrap_tokens: DeviceBootstrapTokens;
+111 -89
View File
@@ -17,9 +17,11 @@ import { listOpenFileDescriptorsForPath } from "../infra/open-file-descriptors.t
import { readSqliteNumberPragma } from "../infra/sqlite-pragma.test-support.js";
import { loadTaskRegistryStateFromSqlite } from "../tasks/task-registry.store.sqlite.js";
import { withOpenClawTestState } from "../test-utils/openclaw-test-state.js";
import { VERSION } from "../version.js";
import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.generated.js";
import {
assertOpenClawStateDatabaseForMaintenance,
clearOpenClawStateDatabaseOpenFailure,
closeOpenClawStateDatabaseForTest,
detectOpenClawStateDatabaseSchemaMigrations,
OPENCLAW_SQLITE_BUSY_TIMEOUT_MS,
@@ -308,6 +310,25 @@ function createUnsafeIndexDrift(databasePath: string): void {
}
}
function createUnsafeSchemaMetaIndexDrift(databasePath: string): void {
const { DatabaseSync } = requireNodeSqlite();
const database = new DatabaseSync(databasePath);
try {
database.exec("CREATE INDEX unsafe_schema_meta_role ON schema_meta(role);");
database.enableDefensive?.(false);
database.exec("PRAGMA writable_schema = ON;");
database
.prepare(
"UPDATE sqlite_schema SET sql = 'CREATE INDEX unsafe_schema_meta_role ON schema_meta(app_version)' WHERE name = 'unsafe_schema_meta_role'",
)
.run();
const schemaVersion = readSqliteNumberPragma(database, "schema_version");
database.exec(`PRAGMA writable_schema = OFF; PRAGMA schema_version = ${schemaVersion + 1};`);
} finally {
database.close();
}
}
function runHotRollbackJournalRecoveryProbe(params: { moduleUrl: string; rootDir: string }): {
integrity: string;
journalExistsAfterRecovery: boolean;
@@ -1750,35 +1771,25 @@ describe("openclaw state database", () => {
expect(listOpenFileDescriptorsForPath(databasePath)).toEqual([]);
});
it("rejects stale secondary indexes before writable initialization", () => {
it("rejects stale schema_meta indexes before writable initialization", () => {
const stateDir = createTempStateDir();
const databasePath = createCanonicalAuditStateDatabase(stateDir);
const options = { env: { OPENCLAW_STATE_DIR: stateDir } };
createUnsafeSchemaMetaIndexDrift(databasePath);
expect(() => openOpenClawStateDatabase(options)).toThrow(
/integrity_check failed.*unsafe_schema_meta_role/iu,
);
});
it("defers unrelated current-schema index corruption but keeps doctor scans full", () => {
const stateDir = createTempStateDir();
const databasePath = createCanonicalAuditStateDatabase(stateDir);
const options = { env: { OPENCLAW_STATE_DIR: stateDir } };
createUnsafeIndexDrift(databasePath);
const { DatabaseSync } = requireNodeSqlite();
const before = new DatabaseSync(databasePath, { readOnly: true });
let metadataBefore: unknown;
try {
expect(before.prepare("PRAGMA quick_check").get()).toEqual({ quick_check: "ok" });
expect(before.prepare("PRAGMA integrity_check").all()).toEqual(
expect.arrayContaining([
expect.objectContaining({
integrity_check: expect.stringMatching(/missing from index unsafe_index_records_value/),
}),
]),
);
metadataBefore = before
.prepare(
"SELECT schema_version, updated_at FROM schema_meta WHERE meta_key = 'primary' LIMIT 1",
)
.get();
} finally {
before.close();
}
expect(() => openOpenClawStateDatabase(options)).toThrow(
/integrity_check failed.*missing from index unsafe_index_records_value/iu,
);
expect(openOpenClawStateDatabase(options).db.isOpen).toBe(true);
closeOpenClawStateDatabaseForTest();
expect(repairOpenClawStateDatabaseSchema(options)).toEqual({
changes: [],
warnings: [
@@ -1792,29 +1803,47 @@ describe("openclaw state database", () => {
withOpenClawStateStartupMigrationCheckpointDatabase(checkpointCallback, options),
).toThrow(/integrity_check failed.*missing from index unsafe_index_records_value/iu);
expect(checkpointCallback).not.toHaveBeenCalled();
const after = new DatabaseSync(databasePath, { readOnly: true });
try {
expect(
after
.prepare(
"SELECT schema_version, updated_at FROM schema_meta WHERE meta_key = 'primary' LIMIT 1",
)
.get(),
).toEqual(metadataBefore);
expect(after.prepare("PRAGMA integrity_check").all()).toEqual(
expect.arrayContaining([
expect.objectContaining({
integrity_check: expect.stringMatching(/missing from index unsafe_index_records_value/),
}),
]),
);
} finally {
after.close();
}
});
it("rejects foreign-key violations before writable initialization", () => {
it("runs full integrity before a pending state schema migration", () => {
const stateDir = createTempStateDir();
const databasePath = createCanonicalAuditStateDatabase(stateDir);
const options = { env: { OPENCLAW_STATE_DIR: stateDir } };
createUnsafeIndexDrift(databasePath);
const { DatabaseSync } = requireNodeSqlite();
const before = new DatabaseSync(databasePath);
try {
before.exec(`PRAGMA user_version = ${OPENCLAW_STATE_SCHEMA_VERSION - 1};`);
} finally {
before.close();
}
expect(() => openOpenClawStateDatabase(options)).toThrow(
/integrity_check failed.*missing from index unsafe_index_records_value/iu,
);
});
it("runs full integrity before mutating a nonempty unversioned state database", () => {
const stateDir = createTempStateDir();
const databasePath = createCanonicalAuditStateDatabase(stateDir);
const options = { env: { OPENCLAW_STATE_DIR: stateDir } };
createUnsafeIndexDrift(databasePath);
const { DatabaseSync } = requireNodeSqlite();
const before = new DatabaseSync(databasePath);
try {
before.exec("PRAGMA user_version = 0;");
} finally {
before.close();
}
expect(() => openOpenClawStateDatabase(options)).toThrow(
/integrity_check failed.*missing from index unsafe_index_records_value/iu,
);
});
it("defers current-schema foreign-key violations but keeps doctor scans full", () => {
const stateDir = createTempStateDir();
const databasePath = createCanonicalAuditStateDatabase(stateDir);
const options = { env: { OPENCLAW_STATE_DIR: stateDir } };
@@ -1837,21 +1866,10 @@ describe("openclaw state database", () => {
corrupted.close();
}
const before = new DatabaseSync(databasePath, { readOnly: true });
let metadataBefore: unknown;
try {
metadataBefore = before
.prepare(
"SELECT schema_version, updated_at FROM schema_meta WHERE meta_key = 'primary' LIMIT 1",
)
.get();
} finally {
before.close();
}
const failure =
/foreign_key_check failed.*task_delivery_state row 1 references task_runs \(foreign key 0\)/iu;
expect(() => openOpenClawStateDatabase(options)).toThrow(failure);
expect(openOpenClawStateDatabase(options).db.isOpen).toBe(true);
closeOpenClawStateDatabaseForTest();
expect(repairOpenClawStateDatabaseSchema(options)).toEqual({
changes: [],
warnings: [expect.stringMatching(failure)],
@@ -1861,25 +1879,6 @@ describe("openclaw state database", () => {
withOpenClawStateStartupMigrationCheckpointDatabase(checkpointCallback, options),
).toThrow(failure);
expect(checkpointCallback).not.toHaveBeenCalled();
const after = new DatabaseSync(databasePath, { readOnly: true });
try {
expect(
after
.prepare(
"SELECT schema_version, updated_at FROM schema_meta WHERE meta_key = 'primary' LIMIT 1",
)
.get(),
).toEqual(metadataBefore);
expect(after.prepare("PRAGMA foreign_key_check").get()).toEqual({
table: "task_delivery_state",
rowid: 1,
parent: "task_runs",
fkid: 0,
});
} finally {
after.close();
}
});
it.skipIf(process.platform === "win32")(
@@ -2921,27 +2920,50 @@ describe("openclaw state database", () => {
expect(
executeSqliteQueryTakeFirstSync(
database.db,
stateDb.selectFrom("schema_meta").select(["role", "schema_version"]),
stateDb.selectFrom("schema_meta").select(["role", "schema_version", "app_version"]),
),
).toEqual({ role: "global", schema_version: OPENCLAW_STATE_SCHEMA_VERSION });
).toEqual({
role: "global",
schema_version: OPENCLAW_STATE_SCHEMA_VERSION,
app_version: VERSION,
});
});
it("refuses to open newer global schema versions", () => {
it("latches newer global schema failures before integrity scans", () => {
const stateDir = createTempStateDir();
const databasePath = path.join(stateDir, "state", "openclaw.sqlite");
fs.mkdirSync(path.dirname(databasePath), { recursive: true });
const options = { env: { OPENCLAW_STATE_DIR: stateDir } };
const databasePath = openOpenClawStateDatabase(options).path;
closeOpenClawStateDatabaseForTest();
createUnsafeIndexDrift(databasePath);
const { DatabaseSync } = requireNodeSqlite();
const db = new DatabaseSync(databasePath);
db.exec(`PRAGMA user_version = ${OPENCLAW_STATE_SCHEMA_VERSION + 1};`);
db.close();
expect(() =>
openOpenClawStateDatabase({
env: { OPENCLAW_STATE_DIR: stateDir },
}),
).toThrow(
`OpenClaw state database ${databasePath} uses newer schema version ${OPENCLAW_STATE_SCHEMA_VERSION + 1}; this OpenClaw build supports ${OPENCLAW_STATE_SCHEMA_VERSION}. Upgrade OpenClaw before opening this database. Do not downgrade OpenClaw or modify the database. To run this older build, use a separate state directory or restore a compatible backup.`,
);
let firstFailure: unknown;
try {
openOpenClawStateDatabase(options);
} catch (error) {
firstFailure = error;
}
expect(firstFailure).toMatchObject({
name: "SqliteSchemaVersionError",
message: expect.stringContaining("https://docs.openclaw.ai/reference/database-schemas"),
});
for (const candidate of [databasePath, `${databasePath}-wal`, `${databasePath}-shm`]) {
fs.rmSync(candidate, { force: true });
}
let secondFailure: unknown;
try {
openOpenClawStateDatabase(options);
} catch (error) {
secondFailure = error;
}
expect(secondFailure).toBe(firstFailure);
clearOpenClawStateDatabaseOpenFailure(databasePath);
expect(openOpenClawStateDatabase(options).db.isOpen).toBe(true);
});
it("does not chmod shared parent directories for explicit database paths", () => {
+204 -13
View File
@@ -16,7 +16,11 @@ import {
repairCanonicalSqliteUniqueIndexes,
type CanonicalSqliteUniqueIndex,
} from "../infra/sqlite-index-schema.js";
import { assertSqliteIntegrity } from "../infra/sqlite-integrity.js";
import {
assertSqliteIntegrity,
assertSqliteTableIntegrity,
isTerminalSqliteIntegrityError,
} from "../infra/sqlite-integrity.js";
import {
assertSqliteSchemaContains,
type SqliteSchemaCompatibility,
@@ -37,6 +41,7 @@ import {
} from "../infra/sqlite-wal.js";
import { migrateLegacyCronRunLogsToTaskRuns } from "../infra/state-migrations.cron-run-logs.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import { VERSION } from "../version.js";
import * as operatorApprovalMigration from "./openclaw-state-db-operator-approval-migration.js";
import {
ensureColumn,
@@ -59,9 +64,13 @@ import { OPENCLAW_STATE_SCHEMA_SQL } from "./openclaw-state-schema.generated.js"
* migrations/backups that operate on local state.
*/
// v3 rebuilds every OpenClaw-owned table with SQLite STRICT type enforcement.
// database_verifications is additive derived cache; no bump preserves safe downgrades.
export const OPENCLAW_STATE_SCHEMA_VERSION = 3;
/** Maximum time one synchronous SQLite call may wait for a lock. */
export const OPENCLAW_SQLITE_BUSY_TIMEOUT_MS = 5_000;
/** User-facing guide for schema refusals; lives here so error sites avoid import cycles. */
export const OPENCLAW_DATABASE_SCHEMA_DOCS_URL =
"https://docs.openclaw.ai/reference/database-schemas";
const OPENCLAW_STATE_DIR_MODE = 0o700;
const OPENCLAW_STATE_FILE_MODE = 0o600;
const OPENCLAW_STATE_CANONICAL_UNIQUE_INDEXES = [
@@ -125,7 +134,142 @@ export type OpenClawStateDatabaseSchemaMigration = {
path: string;
};
const cachedDatabases = new Map<string, OpenClawStateDatabase>();
const terminalOpenFailures = new Map<string, Error>();
/** Latch background verification damage so later opens fail without rescanning. */
export function recordOpenClawStateDatabaseOpenFailure(pathname: string, error: Error): void {
const resolvedPath = path.resolve(pathname);
terminalOpenFailures.set(resolvedPath, error);
// Quarantine: writing into a proven-corrupt file compounds damage, so close
// any live handle too; the latch then fails every later open fast.
const cached = cachedDatabases.get(resolvedPath);
if (cached) {
cached.walMaintenance.close();
clearNodeSqliteKyselyCacheForDatabase(cached.db);
if (cached.db.isOpen) {
cached.db.close();
}
cachedDatabases.delete(resolvedPath);
}
}
/** Clear a terminal open failure after doctor rewrites the database file. */
export function clearOpenClawStateDatabaseOpenFailure(pathname: string): void {
terminalOpenFailures.delete(path.resolve(pathname));
}
type OpenClawStateMetadataDatabase = Pick<OpenClawStateKyselyDatabase, "schema_meta">;
type OpenClawDatabaseVerificationDatabase = Pick<
OpenClawStateKyselyDatabase,
"database_verifications"
>;
type OpenClawDatabaseVerification = {
error: string | null;
result: string;
};
function readDatabaseVerificationRow(
database: DatabaseSync,
pathname: string,
): OpenClawDatabaseVerification | undefined {
if (!tableExists(database, "database_verifications")) {
return undefined;
}
const db = getNodeSqliteKysely<OpenClawDatabaseVerificationDatabase>(database);
return executeSqliteQuerySync(
database,
db
.selectFrom("database_verifications")
.select(["result", "error"])
.where("path", "=", path.resolve(pathname))
.limit(1),
).rows[0];
}
function clearDatabaseVerificationRow(database: DatabaseSync, pathname: string): void {
if (!tableExists(database, "database_verifications")) {
return;
}
const db = getNodeSqliteKysely<OpenClawDatabaseVerificationDatabase>(database);
executeSqliteQuerySync(
database,
db.deleteFrom("database_verifications").where("path", "=", path.resolve(pathname)),
);
}
/** Read one durable verification row through the cached shared state database. */
export function readOpenClawDatabaseVerification(
pathname: string,
options: OpenClawStateDatabaseOptions = {},
): OpenClawDatabaseVerification | undefined {
const statePath = path.resolve(
options.path ?? resolveOpenClawStateSqlitePath(options.env ?? process.env),
);
if (!existsSync(statePath)) {
return undefined;
}
return readDatabaseVerificationRow(openOpenClawStateDatabase(options).db, pathname);
}
/**
* Remove a persisted quarantine row after doctor repairs a database.
* Returns false when the row could not be cleared: callers must surface that,
* or the next open re-quarantines a healthy repaired file while doctor
* reported success.
*/
export function clearOpenClawDatabaseVerification(
pathname: string,
options: OpenClawStateDatabaseOptions = {},
): boolean {
const statePath = path.resolve(
options.path ?? resolveOpenClawStateSqlitePath(options.env ?? process.env),
);
if (!existsSync(statePath)) {
return true;
}
const sqlite = requireNodeSqlite();
let database: DatabaseSync | undefined;
try {
database = new sqlite.DatabaseSync(statePath);
database.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`);
runSqliteImmediateTransactionSync(
database,
() => clearDatabaseVerificationRow(database!, pathname),
{
busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS,
databaseLabel: statePath,
operationLabel: "state.database-verifications.clear",
},
);
return true;
} catch {
return false;
} finally {
if (database) {
try {
clearNodeSqliteKyselyCacheForDatabase(database);
database.close();
} catch {
// Best-effort cleanup must not turn completed doctor repair into failure.
}
}
}
}
/** Build the shared refusal used by state and agent database opens. */
export function createOpenClawDatabaseVerificationError(
kind: "agent" | "state",
pathname: string,
storedError: string | null,
): Error {
// Doctor's clearing hooks run after a full integrity assertion, so a still-
// corrupt file cannot be cleared directly: the file must be healthy first.
const error = new Error(
`OpenClaw ${kind} database ${pathname} is quarantined after integrity verification failed: ${storedError ?? "unknown integrity error"}. Restore the database from a backup or repair it, then run openclaw doctor --fix to clear the quarantine. See ${OPENCLAW_DATABASE_SCHEMA_DOCS_URL}.`,
);
error.name = "SqliteIntegrityError";
return error;
}
function assertSupportedSchemaVersion(db: DatabaseSync, pathname: string): void {
const userVersion = readSqliteUserVersion(db);
if (userVersion > OPENCLAW_STATE_SCHEMA_VERSION) {
@@ -808,7 +952,7 @@ export function repairOpenClawStateDatabaseSchema(options: OpenClawStateDatabase
const sqlite = requireNodeSqlite();
const db = new sqlite.DatabaseSync(pathname);
try {
assertStateDatabaseIntegrityBeforeMutation(db, pathname);
assertSqliteIntegrity(db, pathname);
assertSupportedSchemaVersion(db, pathname);
db.exec("PRAGMA foreign_keys = OFF;");
const changes = runSqliteImmediateTransactionSync(
@@ -848,7 +992,16 @@ export function repairOpenClawStateDatabaseSchema(options: OpenClawStateDatabase
operationLabel: "state.schema.repair",
},
);
return { changes, warnings: [] };
const quarantineCleared = clearOpenClawDatabaseVerification(pathname, { path: pathname });
clearOpenClawStateDatabaseOpenFailure(pathname);
return {
changes,
warnings: quarantineCleared
? []
: [
`Persisted quarantine record for ${pathname} could not be cleared; rerun openclaw doctor --fix so the repaired database is not refused again.`,
],
};
} catch (err) {
// Reaching this catch inside doctor means repair itself refused or failed,
// so the runtime asserts' "run openclaw doctor --fix" advice is circular here.
@@ -921,7 +1074,7 @@ export function withOpenClawStateStartupMigrationCheckpointDatabase<T>(
const sqlite = requireNodeSqlite();
const db = new sqlite.DatabaseSync(pathname);
try {
assertStateDatabaseIntegrityBeforeMutation(db, pathname);
assertSqliteIntegrity(db, pathname);
ensureStartupMigrationCheckpointSchema(db, pathname);
return callback(db);
} finally {
@@ -1615,7 +1768,7 @@ function ensureSchema(db: DatabaseSync, pathname: string): void {
role: "global",
schema_version: OPENCLAW_STATE_SCHEMA_VERSION,
agent_id: null,
app_version: null,
app_version: VERSION,
created_at: now,
updated_at: now,
})
@@ -1624,7 +1777,7 @@ function ensureSchema(db: DatabaseSync, pathname: string): void {
role: "global",
schema_version: OPENCLAW_STATE_SCHEMA_VERSION,
agent_id: null,
app_version: null,
app_version: VERSION,
updated_at: now,
}),
),
@@ -1650,10 +1803,27 @@ function assertStateDatabaseIntegrityBeforeMutation(
pathname: string,
): void {
database.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`);
// A writable handle lets SQLite recover a hot journal or WAL after an
// interrupted writer. OpenClaw mutations start only after recovery and a
// full table/index consistency check both succeed.
assertSqliteIntegrity(database, pathname);
const userVersion = readSqliteUserVersion(database);
const hasApplicationSchema = database
.prepare("SELECT 1 FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' LIMIT 1")
.get();
if (
(userVersion === 0 && hasApplicationSchema) ||
(userVersion > 0 && userVersion < OPENCLAW_STATE_SCHEMA_VERSION)
) {
// Migration rewrites the schema; prove the whole file before that mutation.
// Only a truly empty v0 file may skip; legacy v0 files need the same proof.
stateDbLog.info("state database schema migration pending; verifying integrity first", {
fromVersion: userVersion,
path: pathname,
toVersion: OPENCLAW_STATE_SCHEMA_VERSION,
});
assertSqliteIntegrity(database, pathname);
return;
}
if (tableExists(database, "schema_meta")) {
assertSqliteTableIntegrity(database, pathname, "schema_meta");
}
}
/** Open or return a cached shared state database after schema and migration checks. */
@@ -1662,6 +1832,12 @@ export function openOpenClawStateDatabase(
): OpenClawStateDatabase {
const env = options.env ?? process.env;
const pathname = resolveDatabasePath(options);
// Latched paths are quarantined: the recorder closed any live handle, and
// every open fails fast here until doctor repairs the file and clears it.
const terminalFailure = terminalOpenFailures.get(pathname);
if (terminalFailure) {
throw terminalFailure;
}
const cached = cachedDatabases.get(pathname);
if (cached?.db.isOpen) {
return cached;
@@ -1672,14 +1848,19 @@ export function openOpenClawStateDatabase(
clearNodeSqliteKyselyCacheForDatabase(cached.db);
cachedDatabases.delete(pathname);
}
ensureOpenClawStatePermissions(pathname, env);
const sqlite = requireNodeSqlite();
const db = new sqlite.DatabaseSync(pathname);
const walMaintenance = (() => {
let maintenance: SqliteWalMaintenance | undefined;
try {
db.exec(`PRAGMA busy_timeout = ${OPENCLAW_SQLITE_BUSY_TIMEOUT_MS};`);
assertSupportedSchemaVersion(db, pathname);
assertStateDatabaseIntegrityBeforeMutation(db, pathname);
const verification = readDatabaseVerificationRow(db, pathname);
if (verification?.result === "error") {
throw createOpenClawDatabaseVerificationError("state", pathname, verification.error);
}
configureSqlitePreSchemaPragmas(db, {
busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS,
});
@@ -1695,12 +1876,19 @@ export function openOpenClawStateDatabase(
} catch (err) {
maintenance?.close();
db.close();
if (
err instanceof Error &&
(err.name === "SqliteSchemaVersionError" || isTerminalSqliteIntegrityError(err))
) {
recordOpenClawStateDatabaseOpenFailure(pathname, err);
}
throw err;
}
})();
ensureOpenClawStatePermissions(pathname, env);
const database = { db, path: pathname, walMaintenance };
cachedDatabases.set(pathname, database);
terminalOpenFailures.delete(pathname);
return database;
}
@@ -1746,6 +1934,9 @@ export function isOpenClawStateDatabaseOpen(): boolean {
return Array.from(cachedDatabases.values()).some((database) => database.db.isOpen);
}
/** Test alias for closing shared state handles from teardown code. */
export const closeOpenClawStateDatabaseForTest = closeOpenClawStateDatabase;
/** Close shared state handles and clear terminal failure latches for test isolation. */
export function closeOpenClawStateDatabaseForTest(): void {
closeOpenClawStateDatabase();
terminalOpenFailures.clear();
}
/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */
@@ -958,6 +958,16 @@ CREATE TABLE IF NOT EXISTS agent_databases (
PRIMARY KEY (agent_id, path)
) STRICT;
-- Additive derived cache: older builds safely ignore this table.
-- Keep state schema v3 so verification never makes downgrades refuse startup.
CREATE TABLE IF NOT EXISTS database_verifications (
path TEXT NOT NULL PRIMARY KEY,
kind TEXT NOT NULL,
verified_at INTEGER NOT NULL,
result TEXT NOT NULL,
error TEXT
) STRICT;
CREATE TABLE IF NOT EXISTS plugin_state_entries (
plugin_id TEXT NOT NULL,
namespace TEXT NOT NULL,
+10
View File
@@ -953,6 +953,16 @@ CREATE TABLE IF NOT EXISTS agent_databases (
PRIMARY KEY (agent_id, path)
) STRICT;
-- Additive derived cache: older builds safely ignore this table.
-- Keep state schema v3 so verification never makes downgrades refuse startup.
CREATE TABLE IF NOT EXISTS database_verifications (
path TEXT NOT NULL PRIMARY KEY,
kind TEXT NOT NULL,
verified_at INTEGER NOT NULL,
result TEXT NOT NULL,
error TEXT
) STRICT;
CREATE TABLE IF NOT EXISTS plugin_state_entries (
plugin_id TEXT NOT NULL,
namespace TEXT NOT NULL,
+2
View File
@@ -711,6 +711,7 @@ describe("collectMissingPackPaths", () => {
"dist/agents/model-provider-auth.worker.js",
"dist/audit/audit-event-writer.worker.js",
"dist/config/sessions/session-transcript-reconcile.worker.js",
"dist/state/openclaw-database-verify.worker.js",
"dist/task-registry-control.runtime.js",
"dist/telegram-ingress-worker.runtime.js",
bundledDistPluginFile("telegram", "runtime-api.js"),
@@ -748,6 +749,7 @@ describe("collectMissingPackPaths", () => {
"dist/agents/model-provider-auth.worker.js",
"dist/audit/audit-event-writer.worker.js",
"dist/config/sessions/session-transcript-reconcile.worker.js",
"dist/state/openclaw-database-verify.worker.js",
"dist/task-registry-control.runtime.js",
"dist/telegram-ingress-worker.runtime.js",
"dist/build-info.json",
+1
View File
@@ -277,6 +277,7 @@ function buildCoreDistEntries(): Record<string, string> {
"audit/audit-event-writer.worker": "src/audit/audit-event-writer.worker.ts",
"config/sessions/session-transcript-reconcile.worker":
"src/config/sessions/session-transcript-reconcile.worker.ts",
"state/openclaw-database-verify.worker": "src/state/openclaw-database-verify.worker.ts",
"acp/control-plane/manager": "src/acp/control-plane/manager.ts",
"cli/gateway-lifecycle.runtime": "src/cli/gateway-cli/lifecycle.runtime.ts",
"provider-dispatcher.runtime": "src/auto-reply/reply/provider-dispatcher.runtime.ts",