diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index de6ae4e7b1b..a690e4faf61 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -1,2 +1,2 @@ -37477c8761123ab9b1dbc0b5923b77c0e6713d23e79d3cb7a4d3c349494fe2df plugin-sdk-api-baseline.json -67a0f46755da5f57de1d08882e1445b34dbf38af3bdf757149a1dfa3ee7763b1 plugin-sdk-api-baseline.jsonl +c4770214933b0ed42066f781c4f18a991af979bc87d2e0ca495a32439b524d1c plugin-sdk-api-baseline.json +31459c3d236e28a9474ada640a412796392d1eedf0f71b20c75513d61a6dba39 plugin-sdk-api-baseline.jsonl diff --git a/docs/.generated/sqlite-session-transcript-schema-baseline.sha256 b/docs/.generated/sqlite-session-transcript-schema-baseline.sha256 index b7a3bc90a41..442bc86a6ac 100644 --- a/docs/.generated/sqlite-session-transcript-schema-baseline.sha256 +++ b/docs/.generated/sqlite-session-transcript-schema-baseline.sha256 @@ -1 +1 @@ -b378fb02792c2147b8804006cc6efc7eb5409ccfa96f1ae727b641e03bf843b8 sqlite-session-transcript-schema-baseline.sql +1754707609cb8b5248307fc7082f3a213e0069883b6be5f392180f80bc0364c5 sqlite-session-transcript-schema-baseline.sql diff --git a/docs/plugins/sdk-subpaths.md b/docs/plugins/sdk-subpaths.md index eda187d1e21..af76423f51c 100644 --- a/docs/plugins/sdk-subpaths.md +++ b/docs/plugins/sdk-subpaths.md @@ -277,7 +277,7 @@ usage endpoint failed or returned no usable usage data. | `plugin-sdk/sqlite-runtime` | Focused SQLite agent-schema, path, and transaction helpers for first-party runtime, without database lifecycle controls | | `plugin-sdk/cron-store-runtime` | Cron store path/load/save helpers | | `plugin-sdk/state-paths` | State/OAuth dir path helpers | - | `plugin-sdk/plugin-state-runtime` | Plugin sidecar SQLite keyed-state types plus centralized connection pragma and WAL maintenance setup for plugin-owned databases | + | `plugin-sdk/plugin-state-runtime` | Plugin sidecar SQLite keyed-state types plus centralized connection pragma, verified WAL maintenance, and atomic STRICT-schema migration helpers for plugin-owned databases | | `plugin-sdk/routing` | Route/session-key/account binding helpers such as `resolveAgentRoute`, `buildAgentSessionKey`, and `resolveDefaultAgentBoundAccountId` | | `plugin-sdk/status-helpers` | Shared channel/account status summary helpers, runtime-state defaults, and issue metadata helpers | | `plugin-sdk/target-resolver-runtime` | Shared target resolver helpers | diff --git a/extensions/logbook/src/store.test.ts b/extensions/logbook/src/store.test.ts index 5dee634a6fc..218c634dffc 100644 --- a/extensions/logbook/src/store.test.ts +++ b/extensions/logbook/src/store.test.ts @@ -1,6 +1,7 @@ import { existsSync, mkdirSync, mkdtempSync, rmSync, statSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; import { expectDefined } from "@openclaw/normalization-core"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { LogbookStore, dayKeyFor } from "./store.js"; @@ -72,6 +73,111 @@ describe("LogbookStore", () => { expect(store.batchFrames(batchId)).toHaveLength(2); }); + it("creates every owned table as STRICT with foreign keys enabled", () => { + const database = new DatabaseSync(path.join(dir, "logbook.sqlite"), { readOnly: true }); + try { + const ordinaryTables = database + .prepare( + `SELECT name, strict FROM pragma_table_list + WHERE schema = 'main' AND type = 'table' AND name NOT LIKE 'sqlite_%' + ORDER BY name`, + ) + .all(); + expect(ordinaryTables).toEqual([ + { name: "batches", strict: 1 }, + { name: "cards", strict: 1 }, + { name: "frames", strict: 1 }, + { name: "observations", strict: 1 }, + { name: "standups", strict: 1 }, + ]); + expect(database.prepare("PRAGMA user_version").get()).toEqual({ user_version: 1 }); + expect(database.prepare("PRAGMA foreign_key_list(observations)").all()).toContainEqual( + expect.objectContaining({ table: "batches", on_delete: "CASCADE" }), + ); + expect(database.prepare("PRAGMA foreign_key_list(cards)").all()).toContainEqual( + expect.objectContaining({ table: "frames", on_delete: "SET NULL" }), + ); + } finally { + database.close(); + } + }); + + it("rejects values that violate STRICT column types", () => { + const database = new DatabaseSync(path.join(dir, "logbook.sqlite")); + try { + expect(() => + database + .prepare("INSERT INTO standups (day, text, updated_ms) VALUES (?, ?, ?)") + .run(DAY, "bad timestamp", "not-an-integer"), + ).toThrow(); + expect(database.prepare("SELECT COUNT(*) AS count FROM standups").get()).toEqual({ + count: 0, + }); + } finally { + database.close(); + } + }); + + it("rolls back the whole batch when any frame is missing", () => { + const t0 = Date.now(); + const frameId = insertFrame(t0); + + expect(() => + store.createBatch({ + day: dayKeyFor(t0), + startMs: t0, + endMs: t0 + 1000, + frameIds: [frameId, 999_999], + }), + ).toThrow("Logbook frame 999999 is missing or already batched"); + + expect(store.latestBatch()).toBeNull(); + expect(store.unbatchedActiveFrames(10).map((frame) => frame.id)).toEqual([frameId]); + }); + + it("does not steal a frame from an existing batch", () => { + const t0 = Date.now(); + const firstFrame = insertFrame(t0); + const secondFrame = insertFrame(t0 + 1000); + const firstBatch = store.createBatch({ + day: dayKeyFor(t0), + startMs: t0, + endMs: t0 + 1000, + frameIds: [firstFrame], + }); + + expect(() => + store.createBatch({ + day: dayKeyFor(t0), + startMs: t0, + endMs: t0 + 2000, + frameIds: [firstFrame, secondFrame], + }), + ).toThrow(`Logbook frame ${firstFrame} is missing or already batched`); + + expect(store.latestBatch()?.id).toBe(firstBatch); + expect(store.batchFrames(firstBatch).map((frame) => frame.id)).toEqual([firstFrame]); + expect(store.unbatchedActiveFrames(10).map((frame) => frame.id)).toEqual([secondFrame]); + }); + + it("rejects empty and duplicate frame claims without leaving a batch", () => { + const t0 = Date.now(); + expect(() => + store.createBatch({ day: DAY, startMs: t0, endMs: t0 + 1000, frameIds: [] }), + ).toThrow("Logbook batch requires at least one frame"); + const frameId = insertFrame(t0); + expect(() => + store.createBatch({ + day: DAY, + startMs: t0, + endMs: t0 + 1000, + frameIds: [frameId, frameId], + }), + ).toThrow(`Logbook frame ${frameId} is missing or already batched`); + expect(store.latestBatch()).toBeNull(); + expect(store.countUnbatchedActiveFrames()).toBe(1); + }); + it("resets running batches to pending on startup recovery", () => { const t0 = Date.now(); insertFrame(t0); @@ -129,6 +235,24 @@ describe("LogbookStore", () => { expect(store.frameById(newId)).not.toBeNull(); }); + it("keeps frame metadata when a retained file cannot be removed", () => { + const now = Date.now(); + const firstId = insertFrame(now - 21 * 24 * 60 * 60_000); + const blockedId = insertFrame(now - 20 * 24 * 60 * 60_000); + const blockedPath = expectDefined(store.frameById(blockedId), "blocked frame").path; + rmSync(blockedPath); + mkdirSync(blockedPath); + + expect(() => store.pruneFrames(now - 14 * 24 * 60 * 60_000)).toThrow(); + expect(store.frameById(firstId)).not.toBeNull(); + expect(store.frameById(blockedId)).not.toBeNull(); + + rmSync(blockedPath, { recursive: true }); + expect(store.pruneFrames(now - 14 * 24 * 60 * 60_000)).toBe(2); + expect(store.frameById(firstId)).toBeNull(); + expect(store.frameById(blockedId)).toBeNull(); + }); + it("detaches pruned keyframes from surviving cards", () => { const now = Date.now(); const oldId = insertFrame(now - 20 * 24 * 60 * 60_000); @@ -153,6 +277,27 @@ describe("LogbookStore", () => { expect(expectDefined(observations[0], "retried observation").text).toBe("retry run"); }); + it("rejects observations for a missing batch", () => { + expect(() => + store.replaceObservations(999_999, DAY, [ + { startMs: 1, endMs: 2, text: "orphan observation" }, + ]), + ).toThrow(); + expect(store.observationsInRange(DAY, 0, Number.MAX_SAFE_INTEGER)).toEqual([]); + }); + + it("rolls back a card replacement with a missing keyframe", () => { + const base = new Date(`${DAY}T10:00:00`).getTime(); + store.replaceCardsInWindow(DAY, base, base + 60_000, [draft({ title: "kept" })]); + + expect(() => + store.replaceCardsInWindow(DAY, base, base + 60_000, [ + draft({ title: "invalid", keyframeId: 999_999 }), + ]), + ).toThrow(); + expect(store.cardsForDay(DAY).map((card) => card.title)).toEqual(["kept"]); + }); + it("requeues errored batches for explicit retry", () => { const t0 = Date.now(); const frameId = insertFrame(t0); @@ -182,4 +327,95 @@ describe("LogbookStore", () => { store.saveStandup(DAY, "## Done\n- shipped more"); expect(store.getStandup(DAY)?.text).toContain("shipped more"); }); + + it("migrates legacy tables to STRICT without losing batch assignments", () => { + store.close(); + const databasePath = path.join(dir, "logbook.sqlite"); + rmSync(databasePath, { force: true }); + const legacy = new DatabaseSync(databasePath); + legacy.exec(` + CREATE TABLE batches ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + day TEXT NOT NULL, + start_ms INTEGER NOT NULL, + end_ms INTEGER NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + error TEXT, + frame_count INTEGER NOT NULL DEFAULT 0, + model TEXT, + created_ms INTEGER NOT NULL, + updated_ms INTEGER NOT NULL + ); + CREATE TABLE frames ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + captured_at_ms INTEGER NOT NULL, + day TEXT NOT NULL, + path TEXT NOT NULL, + screen_index INTEGER NOT NULL DEFAULT 0, + width INTEGER, + height INTEGER, + byte_size INTEGER NOT NULL DEFAULT 0, + content_hash TEXT NOT NULL, + idle INTEGER NOT NULL DEFAULT 0, + batch_id INTEGER + ); + INSERT INTO batches VALUES (7, '${DAY}', 10, 20, 'pending', NULL, 1, NULL, 10, 10); + INSERT INTO frames VALUES (11, 10, '${DAY}', '/tmp/frame.jpg', 0, NULL, NULL, 1, 'hash', 0, 7); + `); + legacy.close(); + + store = new LogbookStore(dir); + + expect(store.batchFrames(7).map((frame) => frame.id)).toEqual([11]); + const migrated = new DatabaseSync(databasePath, { readOnly: true }); + try { + expect( + migrated + .prepare( + `SELECT name, strict FROM pragma_table_list + WHERE schema = 'main' AND type = 'table' AND name NOT LIKE 'sqlite_%' + ORDER BY name`, + ) + .all(), + ).toEqual([ + { name: "batches", strict: 1 }, + { name: "cards", strict: 1 }, + { name: "frames", strict: 1 }, + { name: "observations", strict: 1 }, + { name: "standups", strict: 1 }, + ]); + } finally { + migrated.close(); + } + }); + + it("rolls back a legacy STRICT migration when stored data has the wrong type", () => { + const legacyDir = path.join(dir, "invalid-legacy"); + mkdirSync(legacyDir); + const databasePath = path.join(legacyDir, "logbook.sqlite"); + const legacy = new DatabaseSync(databasePath); + legacy.exec(` + CREATE TABLE standups (day TEXT PRIMARY KEY, text TEXT NOT NULL, updated_ms TEXT NOT NULL); + INSERT INTO standups VALUES ('${DAY}', 'legacy', 'not-an-integer'); + `); + legacy.close(); + + expect(() => new LogbookStore(legacyDir)).toThrow( + "Failed migrating SQLite table standups to STRICT", + ); + + const preserved = new DatabaseSync(databasePath, { readOnly: true }); + try { + expect( + preserved.prepare("SELECT strict FROM pragma_table_list WHERE name = 'standups'").get(), + ).toEqual({ strict: 0 }); + expect(preserved.prepare("SELECT * FROM standups").get()).toEqual({ + day: DAY, + text: "legacy", + updated_ms: "not-an-integer", + }); + } finally { + preserved.close(); + } + }); }); diff --git a/extensions/logbook/src/store.ts b/extensions/logbook/src/store.ts index e880454bbcb..ea0efc7a498 100644 --- a/extensions/logbook/src/store.ts +++ b/extensions/logbook/src/store.ts @@ -4,6 +4,11 @@ import { chmodSync, mkdirSync, rmdirSync, rmSync } from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; +import { + configureSqliteConnectionPragmas, + migrateSqliteSchemaToStrict, +} from "openclaw/plugin-sdk/plugin-state-runtime"; +import { runSqliteImmediateTransactionSync } from "openclaw/plugin-sdk/sqlite-runtime"; import type { LogbookBatch, LogbookBatchStatus, @@ -23,7 +28,22 @@ function loadNodeSqlite(): SqliteModule { return req("node:sqlite") as SqliteModule; } +const LOGBOOK_SCHEMA_VERSION = 1; +const LOGBOOK_SQLITE_BUSY_TIMEOUT_MS = 5_000; const SCHEMA = ` +CREATE TABLE IF NOT EXISTS batches ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + day TEXT NOT NULL, + start_ms INTEGER NOT NULL, + end_ms INTEGER NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' CHECK (status IN ('pending', 'running', 'done', 'error')), + error TEXT, + frame_count INTEGER NOT NULL DEFAULT 0, + model TEXT, + created_ms INTEGER NOT NULL, + updated_ms INTEGER NOT NULL +) STRICT; +CREATE INDEX IF NOT EXISTS idx_logbook_batches_day ON batches (day, start_ms); CREATE TABLE IF NOT EXISTS frames ( id INTEGER PRIMARY KEY AUTOINCREMENT, captured_at_ms INTEGER NOT NULL, @@ -34,32 +54,19 @@ CREATE TABLE IF NOT EXISTS frames ( height INTEGER, byte_size INTEGER NOT NULL DEFAULT 0, content_hash TEXT NOT NULL, - idle INTEGER NOT NULL DEFAULT 0, - batch_id INTEGER -); + idle INTEGER NOT NULL DEFAULT 0 CHECK (idle IN (0, 1)), + batch_id INTEGER REFERENCES batches(id) ON DELETE SET NULL +) STRICT; CREATE INDEX IF NOT EXISTS idx_logbook_frames_day ON frames (day, captured_at_ms); CREATE INDEX IF NOT EXISTS idx_logbook_frames_unbatched ON frames (batch_id) WHERE batch_id IS NULL; -CREATE TABLE IF NOT EXISTS batches ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - day TEXT NOT NULL, - start_ms INTEGER NOT NULL, - end_ms INTEGER NOT NULL, - status TEXT NOT NULL DEFAULT 'pending', - error TEXT, - frame_count INTEGER NOT NULL DEFAULT 0, - model TEXT, - created_ms INTEGER NOT NULL, - updated_ms INTEGER NOT NULL -); -CREATE INDEX IF NOT EXISTS idx_logbook_batches_day ON batches (day, start_ms); CREATE TABLE IF NOT EXISTS observations ( id INTEGER PRIMARY KEY AUTOINCREMENT, - batch_id INTEGER NOT NULL, + batch_id INTEGER NOT NULL REFERENCES batches(id) ON DELETE CASCADE, day TEXT NOT NULL, start_ms INTEGER NOT NULL, end_ms INTEGER NOT NULL, text TEXT NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_logbook_observations_day ON observations (day, start_ms); CREATE TABLE IF NOT EXISTS cards ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -73,15 +80,15 @@ CREATE TABLE IF NOT EXISTS cards ( app_primary TEXT, app_secondary TEXT, distractions TEXT NOT NULL DEFAULT '[]', - keyframe_id INTEGER, + keyframe_id INTEGER REFERENCES frames(id) ON DELETE SET NULL, updated_ms INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_logbook_cards_day ON cards (day, start_ms); CREATE TABLE IF NOT EXISTS standups ( day TEXT PRIMARY KEY, text TEXT NOT NULL, updated_ms INTEGER NOT NULL -); +) STRICT; `; type FrameRow = { @@ -195,6 +202,7 @@ export function dayKeyFor(ms: number): string { export class LogbookStore { private readonly db: Database; + private readonly walMaintenance: ReturnType; readonly framesDir: string; constructor(readonly dataDir: string) { @@ -207,15 +215,47 @@ export class LogbookStore { chmodSync(this.framesDir, 0o700); const { DatabaseSync } = loadNodeSqlite(); const dbPath = path.join(dataDir, "logbook.sqlite"); - this.db = new DatabaseSync(dbPath); - // WAL/SHM sidecars inherit the main DB file's permissions. - chmodSync(dbPath, 0o600); - this.db.exec("PRAGMA journal_mode = WAL"); - this.db.exec("PRAGMA busy_timeout = 1000"); - this.db.exec(SCHEMA); + const db = new DatabaseSync(dbPath); + let walMaintenance: ReturnType | undefined; + try { + // WAL/SHM sidecars inherit the main DB file's permissions. + chmodSync(dbPath, 0o600); + walMaintenance = configureSqliteConnectionPragmas(db, { + busyTimeoutMs: LOGBOOK_SQLITE_BUSY_TIMEOUT_MS, + databaseLabel: "logbook", + databasePath: dbPath, + foreignKeys: true, + synchronous: "NORMAL", + }); + const versionRow = db.prepare("PRAGMA user_version").get() as + | { user_version?: unknown } + | undefined; + const schemaVersion = Number(versionRow?.user_version ?? 0); + if (schemaVersion > LOGBOOK_SCHEMA_VERSION) { + throw new Error( + `Logbook database uses newer schema version ${schemaVersion}; this build supports ${LOGBOOK_SCHEMA_VERSION}`, + ); + } + db.exec(SCHEMA); + if (schemaVersion < LOGBOOK_SCHEMA_VERSION) { + migrateSqliteSchemaToStrict(db, SCHEMA, { databaseLabel: dbPath }); + db.exec(`PRAGMA user_version = ${LOGBOOK_SCHEMA_VERSION};`); + } + } catch (error) { + walMaintenance?.close(); + db.close(); + throw error; + } + if (!walMaintenance) { + db.close(); + throw new Error("Logbook SQLite maintenance failed to initialize"); + } + this.db = db; + this.walMaintenance = walMaintenance; } close(): void { + this.walMaintenance.close(); this.db.close(); } @@ -302,19 +342,43 @@ export class LogbookStore { } createBatch(params: { day: string; startMs: number; endMs: number; frameIds: number[] }): number { - const now = Date.now(); - const result = this.db - .prepare( - `INSERT INTO batches (day, start_ms, end_ms, status, frame_count, created_ms, updated_ms) - VALUES (?, ?, ?, 'pending', ?, ?, ?)`, - ) - .run(params.day, params.startMs, params.endMs, params.frameIds.length, now, now); - const batchId = Number(result.lastInsertRowid); - const assign = this.db.prepare(`UPDATE frames SET batch_id = ? WHERE id = ?`); - for (const frameId of params.frameIds) { - assign.run(batchId, frameId); + if (params.frameIds.length === 0) { + throw new Error("Logbook batch requires at least one frame"); } - return batchId; + const now = Date.now(); + const insertBatch = this.db.prepare( + `INSERT INTO batches (day, start_ms, end_ms, status, frame_count, created_ms, updated_ms) + VALUES (?, ?, ?, 'pending', ?, ?, ?)`, + ); + const assignFrame = this.db.prepare( + `UPDATE frames SET batch_id = ? WHERE id = ? AND batch_id IS NULL`, + ); + return runSqliteImmediateTransactionSync( + this.db, + () => { + const result = insertBatch.run( + params.day, + params.startMs, + params.endMs, + params.frameIds.length, + now, + now, + ); + const batchId = Number(result.lastInsertRowid); + for (const frameId of params.frameIds) { + const assignment = assignFrame.run(batchId, frameId); + if (assignment.changes !== 1) { + throw new Error(`Logbook frame ${frameId} is missing or already batched`); + } + } + return batchId; + }, + { + busyTimeoutMs: LOGBOOK_SQLITE_BUSY_TIMEOUT_MS, + databaseLabel: "logbook", + operationLabel: "logbook.batch.create", + }, + ); } setBatchStatus( @@ -387,20 +451,24 @@ export class LogbookStore { day: string, segments: Array<{ startMs: number; endMs: number; text: string }>, ): void { - this.db.exec("BEGIN"); - try { - this.db.prepare(`DELETE FROM observations WHERE batch_id = ?`).run(batchId); - const insert = this.db.prepare( - `INSERT INTO observations (batch_id, day, start_ms, end_ms, text) VALUES (?, ?, ?, ?, ?)`, - ); - for (const segment of segments) { - insert.run(batchId, day, segment.startMs, segment.endMs, segment.text); - } - this.db.exec("COMMIT"); - } catch (err) { - this.db.exec("ROLLBACK"); - throw err; - } + const deleteBatch = this.db.prepare(`DELETE FROM observations WHERE batch_id = ?`); + const insert = this.db.prepare( + `INSERT INTO observations (batch_id, day, start_ms, end_ms, text) VALUES (?, ?, ?, ?, ?)`, + ); + runSqliteImmediateTransactionSync( + this.db, + () => { + deleteBatch.run(batchId); + for (const segment of segments) { + insert.run(batchId, day, segment.startMs, segment.endMs, segment.text); + } + }, + { + busyTimeoutMs: LOGBOOK_SQLITE_BUSY_TIMEOUT_MS, + databaseLabel: "logbook", + operationLabel: "logbook.observations.replace", + }, + ); } observationsInRange(day: string, startMs: number, endMs: number): LogbookObservation[] { @@ -459,36 +527,40 @@ export class LogbookStore { drafts: LogbookCardDraft[], ): void { const now = Date.now(); - this.db.exec("BEGIN"); - try { - this.db - .prepare(`DELETE FROM cards WHERE day = ? AND end_ms > ? AND start_ms < ?`) - .run(day, startMs, endMs); - const insert = this.db.prepare( - `INSERT INTO cards (day, start_ms, end_ms, title, summary, detail, category, app_primary, app_secondary, distractions, keyframe_id, updated_ms) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - ); - for (const draft of drafts) { - insert.run( - draft.day, - draft.startMs, - draft.endMs, - draft.title, - draft.summary, - draft.detail, - draft.category, - draft.appPrimary ?? null, - draft.appSecondary ?? null, - JSON.stringify(draft.distractions), - draft.keyframeId ?? null, - now, - ); - } - this.db.exec("COMMIT"); - } catch (err) { - this.db.exec("ROLLBACK"); - throw err; - } + const deleteWindow = this.db.prepare( + `DELETE FROM cards WHERE day = ? AND end_ms > ? AND start_ms < ?`, + ); + const insert = this.db.prepare( + `INSERT INTO cards (day, start_ms, end_ms, title, summary, detail, category, app_primary, app_secondary, distractions, keyframe_id, updated_ms) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ); + runSqliteImmediateTransactionSync( + this.db, + () => { + deleteWindow.run(day, startMs, endMs); + for (const draft of drafts) { + insert.run( + draft.day, + draft.startMs, + draft.endMs, + draft.title, + draft.summary, + draft.detail, + draft.category, + draft.appPrimary ?? null, + draft.appSecondary ?? null, + JSON.stringify(draft.distractions), + draft.keyframeId ?? null, + now, + ); + } + }, + { + busyTimeoutMs: LOGBOOK_SQLITE_BUSY_TIMEOUT_MS, + databaseLabel: "logbook", + operationLabel: "logbook.cards.replace", + }, + ); } listDays(): Array<{ day: string; cards: number; firstMs: number; lastMs: number }> { @@ -552,32 +624,56 @@ export class LogbookStore { /** Deletes frame rows and files older than the retention window. */ pruneFrames(olderThanMs: number): number { - const rows = this.db - .prepare(`SELECT id, path, day FROM frames WHERE captured_at_ms < ?`) - .all(olderThanMs) as Array<{ id: number; path: string; day: string }>; + const selectExpired = this.db.prepare( + `SELECT id, path, day FROM frames WHERE captured_at_ms < ?`, + ); + const rows = selectExpired.all(olderThanMs) as Array<{ + id: number; + path: string; + day: string; + }>; if (rows.length === 0) { return 0; } const days = new Set(); for (const row of rows) { + // Keep metadata until every file operation succeeds. A later retry can + // then find rows whose earlier files were already removed with force. rmSync(row.path, { force: true }); days.add(row.day); } - // Cards outlive their frames; a dangling keyframe_id would make the UI - // retry a preview fetch forever, so detach it before the rows disappear. - this.db - .prepare( - `UPDATE cards SET keyframe_id = NULL - WHERE keyframe_id IN (SELECT id FROM frames WHERE captured_at_ms < ?)`, - ) - .run(olderThanMs); - this.db.prepare(`DELETE FROM frames WHERE captured_at_ms < ?`).run(olderThanMs); + const selectCurrent = this.db.prepare(`SELECT path FROM frames WHERE id = ?`); + const deleteById = this.db.prepare(`DELETE FROM frames WHERE id = ?`); + const deleted = runSqliteImmediateTransactionSync( + this.db, + () => { + let count = 0; + for (const row of rows) { + const current = selectCurrent.get(row.id) as { path: string } | undefined; + if (!current) { + continue; + } + if (current.path !== row.path) { + throw new Error(`Logbook frame ${row.id} changed path while pruning`); + } + // keyframe_id uses ON DELETE SET NULL, so the same commit cannot + // leave surviving cards pointed at removed frame rows. + count += Number(deleteById.run(row.id).changes); + } + return count; + }, + { + busyTimeoutMs: LOGBOOK_SQLITE_BUSY_TIMEOUT_MS, + databaseLabel: "logbook", + operationLabel: "logbook.frames.prune", + }, + ); for (const day of days) { // Best-effort: removes now-empty day directories, keeps non-empty ones. try { rmdirSync(path.join(this.framesDir, day)); } catch {} } - return rows.length; + return deleted; } } diff --git a/extensions/memory-core/doctor-contract-api.ts b/extensions/memory-core/doctor-contract-api.ts index 4994f10c8ce..fb86fa41171 100644 --- a/extensions/memory-core/doctor-contract-api.ts +++ b/extensions/memory-core/doctor-contract-api.ts @@ -500,7 +500,7 @@ function copyLegacyMemoryIndexRows( dims INTEGER, updated_at INTEGER NOT NULL, PRIMARY KEY (provider, model, provider_key, hash) - ); + ) STRICT; INSERT OR IGNORE INTO main.${MEMORY_EMBEDDING_CACHE_TABLE} ( provider, model, provider_key, hash, embedding, dims, updated_at ) diff --git a/extensions/memory-core/src/memory/manager-session-sync-state.test.ts b/extensions/memory-core/src/memory/manager-session-sync-state.test.ts index 3075f7857cf..19ec2a60dfd 100644 --- a/extensions/memory-core/src/memory/manager-session-sync-state.test.ts +++ b/extensions/memory-core/src/memory/manager-session-sync-state.test.ts @@ -85,7 +85,13 @@ describe("memory session sync state", () => { { absPath: "/tmp/sessions/unchanged.jsonl", path: "sessions/unchanged.jsonl", - mtimeMs: 100, + mtimeMs: 100.75, + size: 10, + }, + { + absPath: "/tmp/sessions/sub-ms-newer.jsonl", + path: "sessions/sub-ms-newer.jsonl", + mtimeMs: 100.75, size: 10, }, { @@ -108,13 +114,15 @@ describe("memory session sync state", () => { }, ], existingRows: [ - { path: "sessions/unchanged.jsonl", hash: "hash-unchanged", mtime: 100, size: 10 }, + { path: "sessions/unchanged.jsonl", hash: "hash-unchanged", mtime: 100.75, size: 10 }, + { path: "sessions/sub-ms-newer.jsonl", hash: "hash-sub-ms", mtime: 100.25, size: 10 }, { path: "sessions/newer.jsonl", hash: "hash-newer", mtime: 200, size: 20 }, { path: "sessions/resized.jsonl", hash: "hash-resized", mtime: 300, size: 30 }, ], }); expect(dirtyFiles).toEqual([ + "/tmp/sessions/sub-ms-newer.jsonl", "/tmp/sessions/newer.jsonl", "/tmp/sessions/resized.jsonl", "/tmp/sessions/missing.jsonl", diff --git a/extensions/memory-core/src/memory/manager-sync-base.ts b/extensions/memory-core/src/memory/manager-sync-base.ts index f8d080f9c76..3130bc9f0ea 100644 --- a/extensions/memory-core/src/memory/manager-sync-base.ts +++ b/extensions/memory-core/src/memory/manager-sync-base.ts @@ -19,6 +19,7 @@ import { type MemorySyncParams, type MemorySyncProgressUpdate, } from "openclaw/plugin-sdk/memory-core-host-engine-storage"; +import { runSqliteImmediateTransactionSync } from "openclaw/plugin-sdk/sqlite-runtime"; import type { MemoryCoreAcquireLocalService } from "./embedding-local-service.js"; import { resolveEmbeddingProviderAdapterId, @@ -94,6 +95,7 @@ const META_KEY = MEMORY_INDEX_META_KEY; const VECTOR_TABLE = MEMORY_INDEX_VECTOR_TABLE; const LEGACY_VECTOR_TABLE = "chunks_vec"; const EMBEDDING_CACHE_TABLE = MEMORY_EMBEDDING_CACHE_TABLE; +const EMBEDDING_CACHE_SEED_BATCH_SIZE = 1_000; const VECTOR_LOAD_TIMEOUT_MS = 30_000; const log = createSubsystemLogger("memory"); @@ -581,62 +583,63 @@ export abstract class MemoryManagerSyncBase { if (!this.cache.enabled) { return; } - let transactionStarted = false; - try { - const rows = sourceDb - .prepare( - `SELECT provider, model, provider_key, hash, embedding, dims, updated_at FROM ${EMBEDDING_CACHE_TABLE}`, - ) - .iterate() as IterableIterator<{ - provider: string; - model: string; - provider_key: string; - hash: string; - embedding: string; - dims: number | null; - updated_at: number; - }>; - let rowCount = 0; - let insert: ReturnType | null = null; - for (const row of rows) { - if (!insert) { - insert = this.db.prepare( - `INSERT INTO ${EMBEDDING_CACHE_TABLE} (provider, model, provider_key, hash, embedding, dims, updated_at) - VALUES (?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(provider, model, provider_key, hash) DO UPDATE SET - embedding=excluded.embedding, - dims=excluded.dims, - updated_at=excluded.updated_at`, - ); - this.db.exec("BEGIN"); - transactionStarted = true; - } - insert.run( - row.provider, - row.model, - row.provider_key, - row.hash, - row.embedding, - row.dims, - row.updated_at, - ); - rowCount += 1; - if (rowCount % 1000 === 0) { - await new Promise((resolve) => { - setImmediate(resolve); - }); - } + type CacheRow = { + rowid: number; + provider: string; + model: string; + provider_key: string; + hash: string; + embedding: string; + dims: number | null; + updated_at: number; + }; + const selectBatch = sourceDb.prepare( + `SELECT rowid, provider, model, provider_key, hash, embedding, dims, updated_at + FROM ${EMBEDDING_CACHE_TABLE} + WHERE rowid > ? + ORDER BY rowid + LIMIT ?`, + ); + const insert = this.db.prepare( + `INSERT INTO ${EMBEDDING_CACHE_TABLE} (provider, model, provider_key, hash, embedding, dims, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(provider, model, provider_key, hash) DO UPDATE SET + embedding=excluded.embedding, + dims=excluded.dims, + updated_at=excluded.updated_at`, + ); + let lastRowid = 0; + while (true) { + // Materialize each source page so neither a read cursor nor a write + // transaction remains open when control returns to the event loop. + const batch = selectBatch.all(lastRowid, EMBEDDING_CACHE_SEED_BATCH_SIZE) as CacheRow[]; + if (batch.length === 0) { + return; } - if (transactionStarted) { - this.db.exec("COMMIT"); + runSqliteImmediateTransactionSync( + this.db, + () => { + for (const row of batch) { + insert.run( + row.provider, + row.model, + row.provider_key, + row.hash, + row.embedding, + row.dims, + row.updated_at, + ); + } + }, + { operationLabel: "memory.embedding-cache.seed" }, + ); + lastRowid = batch[batch.length - 1]?.rowid ?? lastRowid; + if (batch.length < EMBEDDING_CACHE_SEED_BATCH_SIZE) { + return; } - } catch (err) { - if (transactionStarted) { - try { - this.db.exec("ROLLBACK"); - } catch {} - } - throw err; + await new Promise((resolve) => { + setImmediate(resolve); + }); } } diff --git a/extensions/memory-core/src/memory/manager-sync-yield.test.ts b/extensions/memory-core/src/memory/manager-sync-yield.test.ts index 1706559783c..3f5b0280477 100644 --- a/extensions/memory-core/src/memory/manager-sync-yield.test.ts +++ b/extensions/memory-core/src/memory/manager-sync-yield.test.ts @@ -7,7 +7,11 @@ import { type OpenClawConfig, type ResolvedMemorySearchConfig, } from "openclaw/plugin-sdk/memory-core-host-engine-foundation"; -import type { MemorySource } from "openclaw/plugin-sdk/memory-core-host-engine-storage"; +import { + ensureMemoryIndexSchema, + requireNodeSqlite, + type MemorySource, +} from "openclaw/plugin-sdk/memory-core-host-engine-storage"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const { buildSessionEntryMock } = vi.hoisted(() => ({ @@ -182,6 +186,20 @@ class SessionSyncYieldHarness extends MemoryManagerSyncOps { } } +class EmbeddingCacheSeedHarness extends SessionSyncYieldHarness { + protected override readonly cache = { enabled: true }; + protected override db: DatabaseSync; + + constructor(db: DatabaseSync) { + super(() => {}); + this.db = db; + } + + async seedCache(sourceDb: DatabaseSync): Promise { + await this.seedEmbeddingCache(sourceDb); + } +} + describe("session sync responsiveness", () => { beforeEach(() => { setSyncYieldStateDir(); @@ -229,3 +247,75 @@ describe("session sync responsiveness", () => { await immediate; }); }); + +describe("embedding cache seed responsiveness", () => { + const { DatabaseSync: NodeDatabaseSync } = requireNodeSqlite(); + + function createCacheDb(): DatabaseSync { + const db = new NodeDatabaseSync(":memory:"); + ensureMemoryIndexSchema({ + db, + cacheEnabled: true, + ftsEnabled: false, + ftsTokenizer: "unicode61", + }); + return db; + } + + function countCacheRows(db: DatabaseSync): number { + const row = db.prepare("SELECT count(*) AS count FROM memory_embedding_cache").get() as { + count: number; + }; + return row.count; + } + + it("commits each materialized page before yielding", async () => { + const sourceDb = createCacheDb(); + const targetDb = createCacheDb(); + try { + const insert = sourceDb.prepare( + `INSERT INTO memory_embedding_cache + (provider, model, provider_key, hash, embedding, dims, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?)`, + ); + sourceDb.exec("BEGIN"); + for (let index = 0; index < 1_001; index += 1) { + insert.run("test", "model", "key", `hash-${index}`, "[0.5]", 1, index); + } + sourceDb.exec("COMMIT"); + + let duringYield: { + sourceInTransaction: boolean; + targetInTransaction: boolean; + rows: number; + } | null = null; + const observedYield = new Promise((resolve, reject) => { + setImmediate(() => { + try { + duringYield = { + sourceInTransaction: sourceDb.isTransaction, + targetInTransaction: targetDb.isTransaction, + rows: countCacheRows(targetDb), + }; + resolve(); + } catch (error) { + reject(error instanceof Error ? error : new Error(String(error))); + } + }); + }); + + await new EmbeddingCacheSeedHarness(targetDb).seedCache(sourceDb); + await observedYield; + + expect(duringYield).toEqual({ + sourceInTransaction: false, + targetInTransaction: false, + rows: 1_000, + }); + expect(countCacheRows(targetDb)).toBe(1_001); + } finally { + sourceDb.close(); + targetDb.close(); + } + }); +}); diff --git a/extensions/memory-core/src/qmd-session-artifacts.ts b/extensions/memory-core/src/qmd-session-artifacts.ts index de1f8f07085..05d7681c57c 100644 --- a/extensions/memory-core/src/qmd-session-artifacts.ts +++ b/extensions/memory-core/src/qmd-session-artifacts.ts @@ -3,8 +3,23 @@ import path from "node:path"; import type { DatabaseSync } from "node:sqlite"; import { requireNodeSqlite } from "openclaw/plugin-sdk/memory-core-host-engine-storage"; import type { MemorySearchResult } from "openclaw/plugin-sdk/memory-core-host-runtime-files"; +import { migrateSqliteSchemaToStrict } from "openclaw/plugin-sdk/plugin-state-runtime"; const QMD_SESSION_ARTIFACT_TABLE = "openclaw_qmd_session_artifacts"; +const QMD_SESSION_ARTIFACT_SCHEMA = ` + CREATE TABLE IF NOT EXISTS ${QMD_SESSION_ARTIFACT_TABLE} ( + collection TEXT NOT NULL, + artifact_path TEXT NOT NULL, + search_path TEXT NOT NULL, + docid TEXT, + memory_key TEXT NOT NULL, + agent_id TEXT NOT NULL, + session_id TEXT NOT NULL, + archived INTEGER NOT NULL DEFAULT 0, + updated_at INTEGER NOT NULL, + PRIMARY KEY (collection, artifact_path) + ) STRICT; +`; const QMD_SESSION_ARTIFACT_HIT: unique symbol = Symbol("openclaw.qmdSessionArtifactHit"); @@ -49,26 +64,21 @@ type QmdSessionArtifactRow = { }; function ensureQmdSessionArtifactSchema(db: DatabaseSync): void { - db.exec( - `CREATE TABLE IF NOT EXISTS ${QMD_SESSION_ARTIFACT_TABLE} ( - collection TEXT NOT NULL, - artifact_path TEXT NOT NULL, - search_path TEXT NOT NULL, - docid TEXT, - memory_key TEXT NOT NULL, - agent_id TEXT NOT NULL, - session_id TEXT NOT NULL, - archived INTEGER NOT NULL DEFAULT 0, - updated_at INTEGER NOT NULL, - PRIMARY KEY (collection, artifact_path) - )`, - ); + db.exec(QMD_SESSION_ARTIFACT_SCHEMA); try { db.exec( `ALTER TABLE ${QMD_SESSION_ARTIFACT_TABLE} ADD COLUMN archived INTEGER NOT NULL DEFAULT 0`, ); } catch {} + const table = db + .prepare("SELECT strict FROM pragma_table_list WHERE schema = 'main' AND name = ?") + .get(QMD_SESSION_ARTIFACT_TABLE) as { strict?: unknown } | undefined; + if (Number(table?.strict ?? 0) !== 1) { + migrateSqliteSchemaToStrict(db, QMD_SESSION_ARTIFACT_SCHEMA, { + databaseLabel: "QMD session artifact mappings", + }); + } db.exec( `CREATE INDEX IF NOT EXISTS idx_openclaw_qmd_session_artifacts_docid ON ${QMD_SESSION_ARTIFACT_TABLE} (docid)`, diff --git a/extensions/memory-core/src/session-search-visibility.test.ts b/extensions/memory-core/src/session-search-visibility.test.ts index 0c58885a83e..40fc8bfbbcc 100644 --- a/extensions/memory-core/src/session-search-visibility.test.ts +++ b/extensions/memory-core/src/session-search-visibility.test.ts @@ -2,6 +2,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; import type { MemorySearchResult } from "openclaw/plugin-sdk/memory-core-host-runtime-files"; import * as sessionTranscriptHit from "openclaw/plugin-sdk/session-transcript-hit"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -97,6 +98,78 @@ describe("filterMemorySearchHitsBySessionVisibility", () => { return attachQmdSessionArtifactHit(hit, identity); } + it("migrates legacy QMD artifact mappings to STRICT without losing rows", async () => { + const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-qmd-session-artifact-")); + tempRoots.push(root); + const indexPath = path.join(root, "index.sqlite"); + const legacy = new DatabaseSync(indexPath); + legacy.exec(` + CREATE TABLE openclaw_qmd_session_artifacts ( + collection TEXT NOT NULL, + artifact_path TEXT NOT NULL, + search_path TEXT NOT NULL, + docid TEXT, + memory_key TEXT NOT NULL, + agent_id TEXT NOT NULL, + session_id TEXT NOT NULL, + updated_at INTEGER NOT NULL, + PRIMARY KEY (collection, artifact_path) + ); + INSERT INTO openclaw_qmd_session_artifacts ( + collection, artifact_path, search_path, docid, memory_key, agent_id, session_id, updated_at + ) VALUES ('legacy', 'old.md', 'qmd/legacy/old.md', NULL, 'old-key', 'main', 'old', 1); + `); + legacy.close(); + + replaceQmdSessionArtifactMappings({ + collection: "current", + indexPath, + mappings: [ + { + agentId: "main", + archived: true, + artifactPath: "new.md", + collection: "current", + memoryKey: "new-key", + searchPath: "qmd/current/new.md", + sessionId: "new", + }, + ], + }); + + const migrated = new DatabaseSync(indexPath); + try { + expect( + migrated + .prepare( + "SELECT strict FROM pragma_table_list WHERE name = 'openclaw_qmd_session_artifacts'", + ) + .get(), + ).toEqual({ strict: 1 }); + expect( + migrated + .prepare( + `SELECT collection, artifact_path, archived + FROM openclaw_qmd_session_artifacts + ORDER BY collection`, + ) + .all(), + ).toEqual([ + { collection: "current", artifact_path: "new.md", archived: 1 }, + { collection: "legacy", artifact_path: "old.md", archived: 0 }, + ]); + expect(() => + migrated + .prepare( + "UPDATE openclaw_qmd_session_artifacts SET archived = ? WHERE collection = 'legacy'", + ) + .run("not-an-integer"), + ).toThrow(); + } finally { + migrated.close(); + } + }); + it("drops sessions-sourced hits when requester key is missing (fail closed)", async () => { const cfg = asOpenClawConfig({ tools: { sessions: { visibility: "all" } } }); const hits: MemorySearchResult[] = [ diff --git a/extensions/workboard/src/sqlite-store.ts b/extensions/workboard/src/sqlite-store.ts index 19789598e23..69384db1348 100644 --- a/extensions/workboard/src/sqlite-store.ts +++ b/extensions/workboard/src/sqlite-store.ts @@ -16,7 +16,10 @@ import type { WorkboardRunAttempt, WorkboardWorkerLog, } from "@openclaw/workboard-contract"; -import { configureSqliteConnectionPragmas } from "openclaw/plugin-sdk/plugin-state-runtime"; +import { + configureSqliteConnectionPragmas, + migrateSqliteSchemaToStrict, +} from "openclaw/plugin-sdk/plugin-state-runtime"; import { resolveStateDir } from "openclaw/plugin-sdk/state-paths"; import type { PersistedWorkboardAttachment, @@ -26,7 +29,7 @@ import type { WorkboardKeyedStore, } from "./persistence-types.js"; const WORKBOARD_DB_RELATIVE_PATH = ["plugins", "workboard", "workboard.sqlite"] as const; -const SCHEMA_VERSION = 2; +const SCHEMA_VERSION = 3; const WORKBOARD_SQLITE_BUSY_TIMEOUT_MS = 5000; const WORKBOARD_SQLITE_DIR_MODE = 0o700; const WORKBOARD_SQLITE_FILE_MODE = 0o600; @@ -132,13 +135,11 @@ function ensureColumn(db: DatabaseSync, tableName: string, columnName: string, d db.exec(`ALTER TABLE ${tableName} ADD COLUMN ${definition}`); } -function ensureWorkboardSchema(db: DatabaseSync): void { - db.exec(` - PRAGMA foreign_keys = ON; +const WORKBOARD_SCHEMA_SQL = ` CREATE TABLE IF NOT EXISTS workboard_schema_migrations ( id TEXT PRIMARY KEY, applied_at INTEGER NOT NULL - ); + ) STRICT; CREATE TABLE IF NOT EXISTS workboard_boards ( id TEXT PRIMARY KEY, @@ -151,7 +152,7 @@ function ensureWorkboardSchema(db: DatabaseSync): void { created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, archived_at INTEGER - ); + ) STRICT; CREATE TABLE IF NOT EXISTS workboard_cards ( id TEXT PRIMARY KEY, @@ -187,7 +188,7 @@ function ensureWorkboardSchema(db: DatabaseSync): void { stale_json TEXT, lifecycle_status_source_updated_at INTEGER, failure_count INTEGER - ); + ) STRICT; CREATE INDEX IF NOT EXISTS workboard_cards_board_status_idx ON workboard_cards(board_id, status, position); CREATE INDEX IF NOT EXISTS workboard_cards_session_idx @@ -198,7 +199,7 @@ function ensureWorkboardSchema(db: DatabaseSync): void { ordinal INTEGER NOT NULL, label TEXT NOT NULL, PRIMARY KEY(card_id, ordinal) - ); + ) STRICT; CREATE TABLE IF NOT EXISTS workboard_card_events ( id TEXT PRIMARY KEY, @@ -210,7 +211,7 @@ function ensureWorkboardSchema(db: DatabaseSync): void { to_status TEXT, session_key TEXT, run_id TEXT - ); + ) STRICT; CREATE TABLE IF NOT EXISTS workboard_card_attempts ( id TEXT PRIMARY KEY, @@ -225,7 +226,7 @@ function ensureWorkboardSchema(db: DatabaseSync): void { session_key TEXT, run_id TEXT, error TEXT - ); + ) STRICT; CREATE TABLE IF NOT EXISTS workboard_card_comments ( id TEXT PRIMARY KEY, @@ -234,7 +235,7 @@ function ensureWorkboardSchema(db: DatabaseSync): void { body TEXT NOT NULL, created_at INTEGER NOT NULL, updated_at INTEGER - ); + ) STRICT; CREATE TABLE IF NOT EXISTS workboard_card_links ( id TEXT PRIMARY KEY, @@ -245,7 +246,7 @@ function ensureWorkboardSchema(db: DatabaseSync): void { title TEXT, url TEXT, created_at INTEGER NOT NULL - ); + ) STRICT; CREATE TABLE IF NOT EXISTS workboard_card_proof ( id TEXT PRIMARY KEY, @@ -257,7 +258,7 @@ function ensureWorkboardSchema(db: DatabaseSync): void { url TEXT, note TEXT, created_at INTEGER NOT NULL - ); + ) STRICT; CREATE TABLE IF NOT EXISTS workboard_card_artifacts ( id TEXT PRIMARY KEY, @@ -268,7 +269,7 @@ function ensureWorkboardSchema(db: DatabaseSync): void { path TEXT, mime_type TEXT, created_at INTEGER NOT NULL - ); + ) STRICT; CREATE TABLE IF NOT EXISTS workboard_card_diagnostics ( card_id TEXT NOT NULL REFERENCES workboard_cards(id) ON DELETE CASCADE, @@ -282,7 +283,7 @@ function ensureWorkboardSchema(db: DatabaseSync): void { count INTEGER NOT NULL, actions_json TEXT NOT NULL, PRIMARY KEY(card_id, ordinal) - ); + ) STRICT; CREATE TABLE IF NOT EXISTS workboard_card_notifications ( id TEXT PRIMARY KEY, @@ -294,7 +295,7 @@ function ensureWorkboardSchema(db: DatabaseSync): void { sequence INTEGER, session_key TEXT, run_id TEXT - ); + ) STRICT; CREATE TABLE IF NOT EXISTS workboard_worker_logs ( id TEXT PRIMARY KEY, @@ -305,14 +306,14 @@ function ensureWorkboardSchema(db: DatabaseSync): void { created_at INTEGER NOT NULL, session_key TEXT, run_id TEXT - ); + ) STRICT; CREATE TABLE IF NOT EXISTS workboard_worker_protocol ( card_id TEXT PRIMARY KEY REFERENCES workboard_cards(id) ON DELETE CASCADE, state TEXT NOT NULL, updated_at INTEGER NOT NULL, detail TEXT - ); + ) STRICT; CREATE TABLE IF NOT EXISTS workboard_card_attachments ( id TEXT PRIMARY KEY, @@ -323,14 +324,14 @@ function ensureWorkboardSchema(db: DatabaseSync): void { mime_type TEXT, note TEXT, created_at INTEGER NOT NULL - ); + ) STRICT; CREATE INDEX IF NOT EXISTS workboard_card_attachments_card_idx ON workboard_card_attachments(card_id, ordinal); CREATE TABLE IF NOT EXISTS workboard_attachment_blobs ( attachment_id TEXT PRIMARY KEY, content BLOB NOT NULL - ); + ) STRICT; CREATE TABLE IF NOT EXISTS workboard_notification_subscriptions ( id TEXT PRIMARY KEY, @@ -346,17 +347,29 @@ function ensureWorkboardSchema(db: DatabaseSync): void { delivered_event_ids_json TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL - ); - `); + ) STRICT; + `; + +function ensureWorkboardSchema(db: DatabaseSync): void { + db.exec(WORKBOARD_SCHEMA_SQL); ensureColumn( db, "workboard_cards", "lifecycle_status_source_updated_at", "lifecycle_status_source_updated_at INTEGER", ); - db.prepare( - "INSERT OR IGNORE INTO workboard_schema_migrations (id, applied_at) VALUES (?, ?)", - ).run(`schema-${SCHEMA_VERSION}`, Date.now()); + const migrationId = `schema-${SCHEMA_VERSION}`; + const current = db + .prepare("SELECT 1 AS found FROM workboard_schema_migrations WHERE id = ?") + .get(migrationId); + if (!current) { + migrateSqliteSchemaToStrict(db, WORKBOARD_SCHEMA_SQL, { + databaseLabel: "workboard database", + }); + db.prepare( + "INSERT OR IGNORE INTO workboard_schema_migrations (id, applied_at) VALUES (?, ?)", + ).run(migrationId, Date.now()); + } } function chmodIfExists(targetPath: string, mode: number): void { diff --git a/extensions/workboard/src/store.test.ts b/extensions/workboard/src/store.test.ts index 04e3c287a05..efca65a35a3 100644 --- a/extensions/workboard/src/store.test.ts +++ b/extensions/workboard/src/store.test.ts @@ -197,6 +197,22 @@ describe("WorkboardStore", () => { expect(rawDb.prepare("PRAGMA journal_mode").get()).toMatchObject({ journal_mode: "wal", }); + expect( + rawDb + .prepare( + `SELECT name FROM pragma_table_list + WHERE schema = 'main' + AND type = 'table' + AND name NOT LIKE 'sqlite_%' + AND strict <> 1`, + ) + .all(), + ).toEqual([]); + expect(() => + rawDb + .prepare("INSERT INTO workboard_attachment_blobs (attachment_id, content) VALUES (?, ?)") + .run("wrong-type", "text-not-blob"), + ).toThrow(); rawDb.close(); const reopenedStores = createWorkboardSqliteStores({ dbPath }); @@ -239,6 +255,70 @@ describe("WorkboardStore", () => { } }); + it("migrates a version 2 workboard table to STRICT without losing rows", async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-workboard-strict-migration-")); + const dbPath = path.join(dir, "workboard.sqlite"); + const initialized = createWorkboardSqliteStores({ dbPath }); + initialized.close(); + const legacy = new DatabaseSync(dbPath); + try { + legacy.exec(` + INSERT INTO workboard_boards ( + id, name, description, icon, color, default_workspace_json, orchestration_json, + created_at, updated_at, archived_at + ) VALUES ('legacy', 'Legacy board', NULL, NULL, NULL, NULL, NULL, 1, 2, NULL); + ALTER TABLE workboard_boards RENAME TO workboard_boards_strict; + CREATE TABLE workboard_boards ( + id TEXT PRIMARY KEY, + name TEXT, + description TEXT, + icon TEXT, + color TEXT, + default_workspace_json TEXT, + orchestration_json TEXT, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL, + archived_at INTEGER + ); + INSERT INTO workboard_boards SELECT * FROM workboard_boards_strict; + DROP TABLE workboard_boards_strict; + DELETE FROM workboard_schema_migrations WHERE id = 'schema-3'; + INSERT OR IGNORE INTO workboard_schema_migrations (id, applied_at) + VALUES ('schema-2', 1); + `); + } finally { + legacy.close(); + } + + try { + const migratedStores = createWorkboardSqliteStores({ dbPath }); + try { + await expect(migratedStores.boards.lookup("legacy")).resolves.toMatchObject({ + board: { id: "legacy", name: "Legacy board" }, + }); + } finally { + migratedStores.close(); + } + const migrated = new DatabaseSync(dbPath, { readOnly: true }); + try { + expect( + migrated + .prepare("SELECT strict FROM pragma_table_list WHERE name = 'workboard_boards'") + .get(), + ).toEqual({ strict: 1 }); + expect( + migrated + .prepare("SELECT 1 AS found FROM workboard_schema_migrations WHERE id = 'schema-3'") + .get(), + ).toEqual({ found: 1 }); + } finally { + migrated.close(); + } + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + }); + it("uses rollback journaling on network-backed volumes", () => { const dir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-workboard-sqlite-network-")); const dbPath = path.join(dir, "workboard.sqlite"); diff --git a/extensions/workspaces/src/store.test.ts b/extensions/workspaces/src/store.test.ts index 0a482a23c24..548d28a29df 100644 --- a/extensions/workspaces/src/store.test.ts +++ b/extensions/workspaces/src/store.test.ts @@ -1,7 +1,9 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; -import { describe, expect, it } from "vitest"; +import { DatabaseSync } from "node:sqlite"; +import { describe, expect, it, vi } from "vitest"; +import { DEFAULT_WORKSPACE } from "./default-workspace.js"; import { validateWorkspaceDoc, type WorkspaceDoc } from "./schema.js"; import { WorkspaceStore } from "./store.js"; @@ -38,6 +40,95 @@ describe("WorkspaceStore", () => { }); }); + it("uses STRICT tables and durable connection pragmas", async () => { + await withStore((store) => { + const database = new DatabaseSync(store.dbPath, { readOnly: true }); + try { + expect( + database + .prepare( + `SELECT name, strict FROM pragma_table_list + WHERE schema = 'main' AND type = 'table' AND name NOT LIKE 'sqlite_%' + ORDER BY name`, + ) + .all(), + ).toEqual([ + { name: "undo", strict: 1 }, + { name: "workspace", strict: 1 }, + ]); + expect(database.prepare("PRAGMA user_version").get()).toEqual({ user_version: 1 }); + expect(database.prepare("PRAGMA journal_mode").get()).toEqual({ journal_mode: "wal" }); + } finally { + database.close(); + } + }); + }); + + it("closes its WAL maintenance timer", async () => { + vi.useFakeTimers(); + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-workspace-timer-")); + try { + const store = new WorkspaceStore({ stateDir }); + expect(vi.getTimerCount()).toBe(1); + store.close(); + expect(vi.getTimerCount()).toBe(0); + } finally { + vi.useRealTimers(); + await fs.rm(stateDir, { recursive: true, force: true }); + } + }); + + it("migrates an existing workspace and undo history to STRICT", async () => { + const stateDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-workspace-legacy-")); + const workspaceDir = path.join(stateDir, "workspaces"); + const databasePath = path.join(workspaceDir, "workspaces.sqlite"); + await fs.mkdir(workspaceDir, { recursive: true }); + const legacy = new DatabaseSync(databasePath); + legacy.exec(` + CREATE TABLE workspace ( + id INTEGER PRIMARY KEY CHECK (id = 1), + version INTEGER NOT NULL, + doc TEXT NOT NULL, + updated_ms INTEGER NOT NULL + ); + CREATE TABLE undo ( + version INTEGER PRIMARY KEY, + doc TEXT NOT NULL, + created_ms INTEGER NOT NULL + ); + `); + const serialized = JSON.stringify(DEFAULT_WORKSPACE); + legacy.prepare("INSERT INTO workspace VALUES (1, 1, ?, 10)").run(serialized); + legacy.prepare("INSERT INTO undo VALUES (1, ?, 10)").run(serialized); + legacy.close(); + + const store = new WorkspaceStore({ stateDir }); + try { + expect(store.read()).toEqual(DEFAULT_WORKSPACE); + const migrated = new DatabaseSync(databasePath, { readOnly: true }); + try { + expect( + migrated + .prepare( + `SELECT name, strict FROM pragma_table_list + WHERE schema = 'main' AND type = 'table' AND name NOT LIKE 'sqlite_%' + ORDER BY name`, + ) + .all(), + ).toEqual([ + { name: "undo", strict: 1 }, + { name: "workspace", strict: 1 }, + ]); + expect(migrated.prepare("SELECT COUNT(*) AS count FROM undo").get()).toEqual({ count: 1 }); + } finally { + migrated.close(); + } + } finally { + store.close(); + await fs.rm(stateDir, { recursive: true, force: true }); + } + }); + it("keeps a 20-entry undo ring and restores the newest snapshot as a NEW version", async () => { await withStore((store) => { store.read(); diff --git a/extensions/workspaces/src/store.ts b/extensions/workspaces/src/store.ts index 73077b51528..be2b475a3d4 100644 --- a/extensions/workspaces/src/store.ts +++ b/extensions/workspaces/src/store.ts @@ -22,7 +22,10 @@ import { chmodSync, mkdirSync } from "node:fs"; import path from "node:path"; import { DatabaseSync } from "node:sqlite"; -import { configureSqliteConnectionPragmas } from "openclaw/plugin-sdk/plugin-state-runtime"; +import { + configureSqliteConnectionPragmas, + migrateSqliteSchemaToStrict, +} from "openclaw/plugin-sdk/plugin-state-runtime"; import { resolveStateDir } from "openclaw/plugin-sdk/state-paths"; import { WidgetAssetTokens } from "./asset-tokens.js"; import { DEFAULT_WORKSPACE } from "./default-workspace.js"; @@ -41,6 +44,7 @@ const UNDO_RING_SIZE = 20; const DIR_MODE = 0o700; const FILE_MODE = 0o600; const BUSY_TIMEOUT_MS = 5000; +const SCHEMA_VERSION = 1; const SCHEMA = ` CREATE TABLE IF NOT EXISTS workspace ( @@ -48,12 +52,12 @@ CREATE TABLE IF NOT EXISTS workspace ( version INTEGER NOT NULL, doc TEXT NOT NULL, updated_ms INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS undo ( version INTEGER PRIMARY KEY, doc TEXT NOT NULL, created_ms INTEGER NOT NULL -); +) STRICT; `; function serializeWorkspaceDoc(doc: WorkspaceDoc): string { @@ -112,6 +116,7 @@ export class WorkspaceStore { readonly workspaceDir: string; readonly dbPath: string; private readonly db: DatabaseSync; + private readonly walMaintenance: ReturnType; readonly assetTokens = new WidgetAssetTokens(); /** * Single-slot cache of the parsed document. This process is the only writer @@ -126,19 +131,47 @@ export class WorkspaceStore { this.workspaceDir = path.join(this.stateDir, "workspaces"); this.dbPath = path.join(this.workspaceDir, "workspaces.sqlite"); mkdirSync(this.workspaceDir, { recursive: true, mode: DIR_MODE }); - this.db = new DatabaseSync(this.dbPath); + const db = new DatabaseSync(this.dbPath); + let walMaintenance: ReturnType | undefined; try { - configureSqliteConnectionPragmas(this.db, { busyTimeoutMs: BUSY_TIMEOUT_MS }); + walMaintenance = configureSqliteConnectionPragmas(db, { + busyTimeoutMs: BUSY_TIMEOUT_MS, + databaseLabel: "workspaces", + databasePath: this.dbPath, + foreignKeys: true, + synchronous: "NORMAL", + }); // WAL/SHM sidecars inherit the main DB file's permissions. chmodSync(this.dbPath, FILE_MODE); - this.db.exec(SCHEMA); + const versionRow = db.prepare("PRAGMA user_version").get() as + | { user_version?: unknown } + | undefined; + const schemaVersion = Number(versionRow?.user_version ?? 0); + if (schemaVersion > SCHEMA_VERSION) { + throw new Error( + `Workspaces database uses newer schema version ${schemaVersion}; this build supports ${SCHEMA_VERSION}`, + ); + } + db.exec(SCHEMA); + if (schemaVersion < SCHEMA_VERSION) { + migrateSqliteSchemaToStrict(db, SCHEMA, { databaseLabel: this.dbPath }); + db.exec(`PRAGMA user_version = ${SCHEMA_VERSION};`); + } } catch (error) { - this.db.close(); + walMaintenance?.close(); + db.close(); throw error; } + if (!walMaintenance) { + db.close(); + throw new Error("Workspaces SQLite maintenance failed to initialize"); + } + this.db = db; + this.walMaintenance = walMaintenance; } close(): void { + this.walMaintenance.close(); this.db.close(); } diff --git a/packages/memory-host-sdk/src/host/memory-schema.test.ts b/packages/memory-host-sdk/src/host/memory-schema.test.ts index 09322e11dc2..f8d7430af4a 100644 --- a/packages/memory-host-sdk/src/host/memory-schema.test.ts +++ b/packages/memory-host-sdk/src/host/memory-schema.test.ts @@ -46,7 +46,7 @@ describe("memory index schema", () => { start_line UNINDEXED, end_line UNINDEXED ); INSERT INTO meta VALUES ('memory_index_meta_v1', '{"vectorDims":3}'); - INSERT INTO files VALUES ('MEMORY.md', 'memory', 'file-hash', 10, 20); + INSERT INTO files VALUES ('MEMORY.md', 'memory', 'file-hash', 10.75, 20); INSERT INTO chunks VALUES ( 'chunk-1', 'MEMORY.md', 'memory', 1, 2, 'chunk-hash', 'embed-model', 'remember this', '[1,0,0]', 30 @@ -67,7 +67,14 @@ describe("memory index schema", () => { expect(result.ftsAvailable).toBe(true); expect(db.prepare("SELECT * FROM memory_index_sources").all()).toEqual([ - { id: 1, path: "MEMORY.md", source: "memory", hash: "file-hash", mtime: 10, size: 20 }, + { + id: 1, + path: "MEMORY.md", + source: "memory", + hash: "file-hash", + mtime: 10.75, + size: 20, + }, ]); expect(db.prepare("SELECT id, text FROM memory_index_chunks").all()).toEqual([ { id: "chunk-1", text: "remember this" }, @@ -78,6 +85,17 @@ describe("memory index schema", () => { expect(db.prepare("SELECT provider, hash FROM memory_embedding_cache").all()).toEqual([ { provider: "openai", hash: "chunk-hash" }, ]); + expect( + db + .prepare( + `SELECT name FROM pragma_table_list + WHERE schema = 'main' + AND type = 'table' + AND name LIKE 'memory_%' + AND strict <> 1`, + ) + .all(), + ).toEqual([]); expect( db .prepare( @@ -90,6 +108,81 @@ describe("memory index schema", () => { } }); + it("upgrades already-canonical memory tables to STRICT and preserves precise mtimes", () => { + const db = new DatabaseSync(":memory:"); + try { + db.exec(` + CREATE TABLE memory_index_meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ); + CREATE TABLE memory_index_sources ( + id INTEGER PRIMARY KEY, + path TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'memory', + hash TEXT NOT NULL, + mtime INTEGER NOT NULL, + size INTEGER NOT NULL, + UNIQUE (path, source) + ); + CREATE TABLE memory_index_chunks ( + id TEXT PRIMARY KEY, + path TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'memory', + start_line INTEGER NOT NULL, + end_line INTEGER NOT NULL, + hash TEXT NOT NULL, + model TEXT NOT NULL, + text TEXT NOT NULL, + embedding TEXT NOT NULL, + updated_at INTEGER NOT NULL + ); + CREATE TABLE memory_index_state ( + id INTEGER PRIMARY KEY CHECK (id = 1), + revision INTEGER NOT NULL + ); + INSERT INTO memory_index_meta VALUES ('memory_index_meta_v1', '{}'); + INSERT INTO memory_index_sources + (path, source, hash, mtime, size) + VALUES ('MEMORY.md', 'memory', 'source-hash', 10.75, 20); + INSERT INTO memory_index_chunks VALUES ( + 'chunk-1', 'MEMORY.md', 'memory', 1, 1, 'chunk-hash', 'model', 'body', '[]', 30 + ); + INSERT INTO memory_index_state VALUES (1, 3); + `); + + ensureMemoryIndexSchema({ db, cacheEnabled: false, ftsEnabled: false }); + + expect( + db + .prepare( + `SELECT name FROM pragma_table_list + WHERE schema = 'main' + AND type = 'table' + AND name LIKE 'memory_index_%' + AND strict <> 1`, + ) + .all(), + ).toEqual([]); + expect( + db.prepare("SELECT mtime, typeof(mtime) AS storage_type FROM memory_index_sources").get(), + ).toEqual({ mtime: 10.75, storage_type: "real" }); + expect( + db + .prepare( + "SELECT type FROM pragma_table_info('memory_index_sources') WHERE name = 'mtime'", + ) + .get(), + ).toEqual({ type: "REAL" }); + expect(db.prepare("SELECT id, text FROM memory_index_chunks").get()).toEqual({ + id: "chunk-1", + text: "body", + }); + } finally { + db.close(); + } + }); + it("does not import a legacy sidecar memory database during schema startup", () => { const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-memory-sidecar-")); const legacyPath = path.join(rootDir, "memory", "main.sqlite"); @@ -406,7 +499,7 @@ describe("memory index schema", () => { mtime INTEGER NOT NULL, size INTEGER NOT NULL ); - INSERT INTO memory_index_sources VALUES ('shared.md', 'memory', 'memory-hash', 10, 20); + INSERT INTO memory_index_sources VALUES ('shared.md', 'memory', 'memory-hash', 10.75, 20); `); ensureMemoryIndexSchema({ @@ -420,10 +513,12 @@ describe("memory index schema", () => { ).run("shared.md", "sessions", "session-hash", 30, 40); expect( - db.prepare("SELECT id, path, source, hash FROM memory_index_sources ORDER BY source").all(), + db + .prepare("SELECT id, path, source, hash, mtime FROM memory_index_sources ORDER BY source") + .all(), ).toEqual([ - { id: 1, path: "shared.md", source: "memory", hash: "memory-hash" }, - { id: 2, path: "shared.md", source: "sessions", hash: "session-hash" }, + { id: 1, path: "shared.md", source: "memory", hash: "memory-hash", mtime: 10.75 }, + { id: 2, path: "shared.md", source: "sessions", hash: "session-hash", mtime: 30 }, ]); ensureMemoryIndexSchema({ db, cacheEnabled: false, ftsEnabled: false }); expect(db.prepare("SELECT id FROM memory_index_sources ORDER BY id").all()).toEqual([ diff --git a/packages/memory-host-sdk/src/host/memory-schema.ts b/packages/memory-host-sdk/src/host/memory-schema.ts index a56da362542..a79b2e4a383 100644 --- a/packages/memory-host-sdk/src/host/memory-schema.ts +++ b/packages/memory-host-sdk/src/host/memory-schema.ts @@ -1,6 +1,7 @@ // Memory Host SDK module implements memory schema behavior. import type { DatabaseSync } from "node:sqlite"; import { formatErrorMessage } from "./error-utils.js"; +import { migrateSqliteSchemaToStrict } from "./openclaw-runtime-sqlite.js"; // SQLite schema setup for builtin memory index, embedding cache, and FTS. @@ -68,7 +69,7 @@ const MEMORY_INDEX_SOURCE_COLUMN_TYPES = new Map([ ["path", "TEXT"], ["source", "TEXT"], ["hash", "TEXT"], - ["mtime", "INTEGER"], + ["mtime", "REAL"], ["size", "INTEGER"], ]); @@ -191,7 +192,7 @@ function tableHasCanonicalSourceColumnTypes(db: DatabaseSync): boolean { const expectedType = MEMORY_INDEX_SOURCE_COLUMN_TYPES.get(column.name); const expectedDefault = column.name === "source" ? "'memory'" : null; if ( - column.type !== expectedType || + (column.type !== expectedType && !(column.name === "mtime" && column.type === "INTEGER")) || column.defaultValue !== expectedDefault || column.hidden !== 0 ) { @@ -295,10 +296,10 @@ export function migrateMemoryIndexSourcesIdentity(db: DatabaseSync): void { path TEXT NOT NULL, source TEXT NOT NULL DEFAULT 'memory', hash TEXT NOT NULL, - mtime INTEGER NOT NULL, + mtime REAL NOT NULL, size INTEGER NOT NULL, UNIQUE (path, source) - ); + ) STRICT; INSERT INTO ${MEMORY_INDEX_SOURCES_TABLE} (id, path, source, hash, mtime, size) SELECT rowid, path, source, hash, mtime, size FROM memory_index_sources_identity_migration; @@ -363,7 +364,8 @@ function copyLegacyMemoryIndexRows( SELECT key, value FROM ${schema}.meta; INSERT OR IGNORE INTO main.${MEMORY_INDEX_SOURCES_TABLE} (path, source, hash, mtime, size) - SELECT path, source, hash, mtime, size FROM ${schema}.files; + SELECT path, source, hash, mtime, size + FROM ${schema}.files; INSERT OR IGNORE INTO main.${MEMORY_INDEX_CHUNKS_TABLE} ( id, path, source, start_line, end_line, hash, model, text, embedding, updated_at @@ -428,7 +430,7 @@ function copyLegacyMemoryIndexRows( dims INTEGER, updated_at INTEGER NOT NULL, PRIMARY KEY (provider, model, provider_key, hash) - ); + ) STRICT; INSERT OR IGNORE INTO main.${MEMORY_EMBEDDING_CACHE_TABLE} ( provider, model, provider_key, hash, embedding, dims, updated_at ) @@ -526,6 +528,58 @@ function ensureMemoryPathFtsSchema(params: { db: DatabaseSync; tokenizeClause: s } } +function buildMemoryIndexStrictSchema(params: { + embeddingCacheTable: string; + includeEmbeddingCache: boolean; +}): string { + const embeddingCacheSql = params.includeEmbeddingCache + ? ` + CREATE TABLE IF NOT EXISTS ${params.embeddingCacheTable} ( + provider TEXT NOT NULL, + model TEXT NOT NULL, + provider_key TEXT NOT NULL, + hash TEXT NOT NULL, + embedding TEXT NOT NULL, + dims INTEGER, + updated_at INTEGER NOT NULL, + PRIMARY KEY (provider, model, provider_key, hash) + ) STRICT; + ` + : ""; + return ` + CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_META_TABLE} ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL + ) STRICT; + CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_SOURCES_TABLE} ( + id INTEGER PRIMARY KEY, + path TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'memory', + hash TEXT NOT NULL, + mtime REAL NOT NULL, + size INTEGER NOT NULL, + UNIQUE (path, source) + ) STRICT; + CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_CHUNKS_TABLE} ( + id TEXT PRIMARY KEY, + path TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'memory', + start_line INTEGER NOT NULL, + end_line INTEGER NOT NULL, + hash TEXT NOT NULL, + model TEXT NOT NULL, + text TEXT NOT NULL, + embedding TEXT NOT NULL, + updated_at INTEGER NOT NULL + ) STRICT; + CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_STATE_TABLE} ( + id INTEGER PRIMARY KEY CHECK (id = 1), + revision INTEGER NOT NULL + ) STRICT; + ${embeddingCacheSql} + `; +} + /** Ensure canonical memory index tables and the optional FTS table exist. */ export function ensureMemoryIndexSchema(params: { db: DatabaseSync; @@ -539,36 +593,13 @@ export function ensureMemoryIndexSchema(params: { }): { ftsAvailable: boolean; ftsError?: string } { const embeddingCacheTable = params.embeddingCacheTable ?? MEMORY_EMBEDDING_CACHE_TABLE; const ftsTable = params.ftsTable ?? MEMORY_INDEX_FTS_TABLE; + params.db.exec( + buildMemoryIndexStrictSchema({ + embeddingCacheTable, + includeEmbeddingCache: params.cacheEnabled, + }), + ); params.db.exec(` - CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_META_TABLE} ( - key TEXT PRIMARY KEY, - value TEXT NOT NULL - ); - CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_SOURCES_TABLE} ( - id INTEGER PRIMARY KEY, - path TEXT NOT NULL, - source TEXT NOT NULL DEFAULT 'memory', - hash TEXT NOT NULL, - mtime INTEGER NOT NULL, - size INTEGER NOT NULL, - UNIQUE (path, source) - ); - CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_CHUNKS_TABLE} ( - id TEXT PRIMARY KEY, - path TEXT NOT NULL, - source TEXT NOT NULL DEFAULT 'memory', - start_line INTEGER NOT NULL, - end_line INTEGER NOT NULL, - hash TEXT NOT NULL, - model TEXT NOT NULL, - text TEXT NOT NULL, - embedding TEXT NOT NULL, - updated_at INTEGER NOT NULL - ); - CREATE TABLE IF NOT EXISTS ${MEMORY_INDEX_STATE_TABLE} ( - id INTEGER PRIMARY KEY CHECK (id = 1), - revision INTEGER NOT NULL - ); INSERT OR IGNORE INTO ${MEMORY_INDEX_STATE_TABLE} (id, revision) VALUES (1, 0); `); migrateMemoryIndexSourcesIdentity(params.db); @@ -622,20 +653,18 @@ export function ensureMemoryIndexSchema(params: { ? "idx_memory_embedding_cache_updated_at" : "idx_embedding_cache_updated_at"; params.db.exec(` - CREATE TABLE IF NOT EXISTS ${embeddingCacheTable} ( - provider TEXT NOT NULL, - model TEXT NOT NULL, - provider_key TEXT NOT NULL, - hash TEXT NOT NULL, - embedding TEXT NOT NULL, - dims INTEGER, - updated_at INTEGER NOT NULL, - PRIMARY KEY (provider, model, provider_key, hash) - ); CREATE INDEX IF NOT EXISTS ${updatedAtIndex} ON ${embeddingCacheTable}(updated_at); `); } + migrateSqliteSchemaToStrict( + params.db, + buildMemoryIndexStrictSchema({ + embeddingCacheTable, + includeEmbeddingCache: params.cacheEnabled || tableExists(params.db, embeddingCacheTable), + }), + { databaseLabel: "memory index" }, + ); let ftsAvailable = false; let ftsError: string | undefined; diff --git a/packages/memory-host-sdk/src/host/openclaw-runtime-sqlite.ts b/packages/memory-host-sdk/src/host/openclaw-runtime-sqlite.ts new file mode 100644 index 00000000000..2c2b1c9354a --- /dev/null +++ b/packages/memory-host-sdk/src/host/openclaw-runtime-sqlite.ts @@ -0,0 +1,3 @@ +// Narrow core bridge for shared SQLite schema migration primitives. + +export { migrateSqliteSchemaToStrict } from "../../../../src/infra/sqlite-strict.js"; diff --git a/scripts/check-kysely-guardrails.mjs b/scripts/check-kysely-guardrails.mjs index c75498badf2..dd6f47abf03 100644 --- a/scripts/check-kysely-guardrails.mjs +++ b/scripts/check-kysely-guardrails.mjs @@ -34,6 +34,7 @@ const rawSqliteAllowPathGroups = { "src/infra/sqlite-integrity.ts", "src/infra/sqlite-pragma.test-support.ts", "src/infra/sqlite-schema-contract.ts", + "src/infra/sqlite-strict.ts", "src/infra/sqlite-transaction.ts", "src/infra/sqlite-user-version.ts", "src/infra/sqlite-wal.ts", diff --git a/scripts/plugin-sdk-surface-report.mjs b/scripts/plugin-sdk-surface-report.mjs index 0fe3f7a7720..8ab64c798bc 100644 --- a/scripts/plugin-sdk-surface-report.mjs +++ b/scripts/plugin-sdk-surface-report.mjs @@ -222,6 +222,7 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // +1: matchesNoProxy exposes canonical Undici-compatible bypass selection to plugins. // +4: group scope encoder/key builder (channel-policy + compat mirror). // +9: app-guided provider setup context/candidate/hook types and their public mirrors. + // +3: atomic SQLite STRICT migration function, options, and result for plugin stores. // Harvest: channel-ingress -64; dead channel-message dispatch aliases -23. // Harvest: retired qa-live-transport-scenarios subpath -6. // +12: typed plan step/status and checklist formatter across channel barrels. @@ -231,13 +232,14 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // +6: app-guided provider setup types retained by plugin-entry and mirrors. // Used-union narrowing: 31 wildcard barrels drop to explicit used exports; // proxy stream API and codex marker/scaffold pins retained. - 7942, + 7945, env, ), publicFunctionExports: readPluginSdkSurfaceBudgetEnv( "OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_FUNCTION_EXPORTS", // +2: materializeRequesterScopedMcpToolsForHarnessRun (agent-harness-runtime + compat mirror). // +4: group scope encoder/key builder (channel-policy + compat mirror). + // +1: atomic SQLite STRICT migration for plugin stores. // Harvest: channel-ingress -19; dead channel-message dispatch aliases -23. // Harvest: retired qa-live-transport-scenarios subpath -3. // +4: shared plan checklist formatter across channel barrels. @@ -245,7 +247,7 @@ export function readPluginSdkSurfaceBudgets(env = process.env) { // +4: dual-field plan payload builder for the steps deprecation window. // +6: active plan-step helpers pinned through channel-outbound and mirrors. // Used-union narrowing of the 31 wildcard barrels. - 4433, + 4434, env, ), publicDeprecatedExports: readPluginSdkSurfaceBudgetEnv( diff --git a/src/commands/doctor/cron/legacy-run-log-migration.ts b/src/commands/doctor/cron/legacy-run-log-migration.ts index f06d783141a..351dca222d2 100644 --- a/src/commands/doctor/cron/legacy-run-log-migration.ts +++ b/src/commands/doctor/cron/legacy-run-log-migration.ts @@ -73,7 +73,7 @@ export async function migrateLegacyCronRunLogsToSqlite( entry_json TEXT NOT NULL, created_at INTEGER NOT NULL, PRIMARY KEY (store_key, job_id, seq) - ); + ) STRICT; `); const insert = db.prepare( `INSERT INTO cron_run_logs diff --git a/src/infra/sqlite-strict.test.ts b/src/infra/sqlite-strict.test.ts new file mode 100644 index 00000000000..85eafefa181 --- /dev/null +++ b/src/infra/sqlite-strict.test.ts @@ -0,0 +1,417 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { requireNodeSqlite } from "./node-sqlite.js"; +import { migrateSqliteSchemaToStrict } from "./sqlite-strict.js"; + +const openDatabases: Array = []; + +function createDatabase(): import("node:sqlite").DatabaseSync { + const sqlite = requireNodeSqlite(); + const database = new sqlite.DatabaseSync(":memory:"); + openDatabases.push(database); + return database; +} + +function readStrictFlag(database: import("node:sqlite").DatabaseSync, tableName: string): number { + const row = database + .prepare("SELECT strict FROM pragma_table_list WHERE schema = 'main' AND name = ?") + .get(tableName) as { strict?: unknown } | undefined; + return Number(row?.strict ?? -1); +} + +afterEach(() => { + for (const database of openDatabases.splice(0)) { + if (database.isOpen) { + database.close(); + } + } +}); + +describe("migrateSqliteSchemaToStrict", () => { + it("atomically rebuilds related tables and preserves their rows", () => { + const database = createDatabase(); + database.exec(` + PRAGMA foreign_keys = ON; + CREATE TABLE parents (id INTEGER PRIMARY KEY, name TEXT NOT NULL); + CREATE TABLE children ( + id INTEGER PRIMARY KEY, + parent_id INTEGER NOT NULL REFERENCES parents(id) ON DELETE CASCADE, + score INTEGER NOT NULL + ); + INSERT INTO parents (id, name) VALUES (1, 'parent'); + INSERT INTO children (id, parent_id, score) VALUES (2, 1, 3); + `); + + const result = migrateSqliteSchemaToStrict( + database, + ` + CREATE TABLE IF NOT EXISTS parents ( + id INTEGER PRIMARY KEY, + name TEXT NOT NULL + ) STRICT; + CREATE TABLE IF NOT EXISTS children ( + id INTEGER PRIMARY KEY, + parent_id INTEGER NOT NULL REFERENCES parents(id) ON DELETE CASCADE, + score INTEGER NOT NULL + ) STRICT; + `, + { databaseLabel: "test.sqlite" }, + ); + + expect(result.migratedTables).toEqual(["children", "parents"]); + expect(readStrictFlag(database, "parents")).toBe(1); + expect(readStrictFlag(database, "children")).toBe(1); + expect(database.prepare("SELECT * FROM parents").all()).toEqual([{ id: 1, name: "parent" }]); + expect(database.prepare("SELECT * FROM children").all()).toEqual([ + { id: 2, parent_id: 1, score: 3 }, + ]); + expect(database.prepare("PRAGMA foreign_keys").get()).toEqual({ foreign_keys: 1 }); + }); + + it("keeps database-local indexes and triggers", () => { + const database = createDatabase(); + database.exec(` + CREATE TABLE items (id INTEGER PRIMARY KEY, name TEXT NOT NULL); + CREATE TABLE item_audit (item_id INTEGER NOT NULL, action TEXT NOT NULL); + CREATE UNIQUE INDEX items_name_custom ON items(name); + CREATE TRIGGER items_insert_custom AFTER INSERT ON items BEGIN + INSERT INTO item_audit (item_id, action) VALUES (NEW.id, 'insert'); + END; + INSERT INTO items (id, name) VALUES (1, 'first'); + `); + + migrateSqliteSchemaToStrict( + database, + `CREATE TABLE IF NOT EXISTS items (id INTEGER PRIMARY KEY, name TEXT NOT NULL) STRICT;`, + ); + database.prepare("INSERT INTO items (id, name) VALUES (?, ?)").run(2, "second"); + + expect( + database + .prepare( + "SELECT type, name FROM sqlite_schema WHERE name IN ('items_name_custom', 'items_insert_custom') ORDER BY type", + ) + .all(), + ).toEqual([ + { type: "index", name: "items_name_custom" }, + { type: "trigger", name: "items_insert_custom" }, + ]); + expect(database.prepare("SELECT * FROM item_audit ORDER BY item_id").all()).toEqual([ + { item_id: 1, action: "insert" }, + { item_id: 2, action: "insert" }, + ]); + }); + + it("preserves cross-table triggers and views while their target is rebuilt", () => { + const database = createDatabase(); + database.exec(` + CREATE TABLE counters (id INTEGER PRIMARY KEY, value INTEGER NOT NULL); + CREATE TABLE events (id INTEGER PRIMARY KEY, amount INTEGER NOT NULL) STRICT; + INSERT INTO counters (id, value) VALUES (1, 0); + CREATE VIEW counter_totals AS SELECT id, value FROM counters; + CREATE TRIGGER events_update_counter AFTER INSERT ON events BEGIN + UPDATE counters SET value = value + NEW.amount WHERE id = 1; + END; + `); + + migrateSqliteSchemaToStrict( + database, + ` + CREATE TABLE IF NOT EXISTS counters ( + id INTEGER PRIMARY KEY, + value INTEGER NOT NULL + ) STRICT; + CREATE TABLE IF NOT EXISTS events ( + id INTEGER PRIMARY KEY, + amount INTEGER NOT NULL + ) STRICT; + `, + ); + database.prepare("INSERT INTO events (id, amount) VALUES (?, ?)").run(1, 7); + + expect(database.prepare("SELECT * FROM counter_totals").all()).toEqual([{ id: 1, value: 7 }]); + expect( + database + .prepare( + "SELECT type, name FROM sqlite_schema WHERE name IN ('counter_totals', 'events_update_counter') ORDER BY type", + ) + .all(), + ).toEqual([ + { type: "trigger", name: "events_update_counter" }, + { type: "view", name: "counter_totals" }, + ]); + }); + + it("preserves AUTOINCREMENT high-water marks", () => { + const database = createDatabase(); + database.exec(` + CREATE TABLE entries (id INTEGER PRIMARY KEY AUTOINCREMENT, value TEXT NOT NULL); + INSERT INTO entries (value) VALUES ('first'), ('second'), ('third'); + DELETE FROM entries WHERE id = 3; + UPDATE sqlite_sequence SET seq = 40.0 WHERE name = 'entries'; + `); + + migrateSqliteSchemaToStrict( + database, + `CREATE TABLE IF NOT EXISTS entries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + value TEXT NOT NULL + ) STRICT;`, + ); + database.prepare("INSERT INTO entries (value) VALUES (?)").run("fourth"); + + expect(database.prepare("SELECT id, value FROM entries ORDER BY id").all()).toEqual([ + { id: 1, value: "first" }, + { id: 2, value: "second" }, + { id: 41, value: "fourth" }, + ]); + }); + + it("preserves implicit and declared row identities across table shapes", () => { + const database = createDatabase(); + database.exec(` + CREATE TABLE implicit_items (value TEXT NOT NULL); + INSERT INTO implicit_items (rowid, value) VALUES (100, 'implicit'); + + CREATE TABLE composite_items ( + left_key TEXT NOT NULL, + right_key TEXT NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY (left_key, right_key) + ); + INSERT INTO composite_items (rowid, left_key, right_key, value) + VALUES (200, 'left', 'right', 'composite'); + + CREATE TABLE shadowed_items (rowid TEXT NOT NULL, value TEXT NOT NULL); + INSERT INTO shadowed_items (_rowid_, rowid, value) VALUES (300, 'named', 'shadowed'); + + CREATE TABLE integer_items (id INTEGER PRIMARY KEY, value TEXT NOT NULL); + INSERT INTO integer_items (id, value) VALUES (400, 'integer-primary-key'); + + CREATE TABLE without_items ( + id TEXT PRIMARY KEY, + value TEXT NOT NULL + ) WITHOUT ROWID; + INSERT INTO without_items (id, value) VALUES ('without', 'without-rowid'); + `); + + migrateSqliteSchemaToStrict( + database, + ` + CREATE TABLE IF NOT EXISTS implicit_items (value TEXT NOT NULL) STRICT; + CREATE TABLE IF NOT EXISTS composite_items ( + left_key TEXT NOT NULL, + right_key TEXT NOT NULL, + value TEXT NOT NULL, + PRIMARY KEY (left_key, right_key) + ) STRICT; + CREATE TABLE IF NOT EXISTS shadowed_items ( + rowid TEXT NOT NULL, + value TEXT NOT NULL + ) STRICT; + CREATE TABLE IF NOT EXISTS integer_items ( + id INTEGER PRIMARY KEY, + value TEXT NOT NULL + ) STRICT; + CREATE TABLE IF NOT EXISTS without_items ( + id TEXT PRIMARY KEY, + value TEXT NOT NULL + ) STRICT, WITHOUT ROWID; + `, + ); + + expect(database.prepare("SELECT rowid, value FROM implicit_items").get()).toEqual({ + rowid: 100, + value: "implicit", + }); + expect(database.prepare("SELECT rowid, value FROM composite_items").get()).toEqual({ + rowid: 200, + value: "composite", + }); + expect( + database.prepare("SELECT _rowid_ AS hidden_rowid, rowid, value FROM shadowed_items").get(), + ).toEqual({ hidden_rowid: 300, rowid: "named", value: "shadowed" }); + expect( + database.prepare("SELECT rowid AS stored_rowid, id, value FROM integer_items").get(), + ).toEqual({ + stored_rowid: 400, + id: 400, + value: "integer-primary-key", + }); + expect(database.prepare("SELECT id, value FROM without_items").get()).toEqual({ + id: "without", + value: "without-rowid", + }); + }); + + it("rejects implicit-rowid tables that shadow every usable alias", () => { + const database = createDatabase(); + database.exec("CREATE TABLE aliases (rowid TEXT, _rowid_ TEXT, oid TEXT);"); + + expect(() => + migrateSqliteSchemaToStrict( + database, + "CREATE TABLE IF NOT EXISTS aliases (rowid TEXT, _rowid_ TEXT, oid TEXT) STRICT;", + ), + ).toThrow("shadows every rowid alias"); + expect(readStrictFlag(database, "aliases")).toBe(0); + }); + + it("rejects migrations that change the table rowid model", () => { + const database = createDatabase(); + database.exec("CREATE TABLE entries (id TEXT PRIMARY KEY, value TEXT NOT NULL);"); + + expect(() => + migrateSqliteSchemaToStrict( + database, + `CREATE TABLE IF NOT EXISTS entries ( + id INTEGER PRIMARY KEY, + value TEXT NOT NULL + ) STRICT;`, + ), + ).toThrow("changes rowid storage from implicit to integer-primary-key"); + expect(readStrictFlag(database, "entries")).toBe(0); + }); + + it("accepts values that SQLite can losslessly coerce", () => { + const database = createDatabase(); + database.exec("CREATE TABLE counters (id TEXT PRIMARY KEY, value TEXT NOT NULL);"); + database.prepare("INSERT INTO counters (id, value) VALUES (?, ?)").run("one", "42"); + + migrateSqliteSchemaToStrict( + database, + `CREATE TABLE IF NOT EXISTS counters ( + id TEXT PRIMARY KEY, + value INTEGER NOT NULL + ) STRICT;`, + ); + + expect( + database.prepare("SELECT id, value, typeof(value) AS value_type FROM counters").get(), + ).toEqual({ id: "one", value: 42, value_type: "integer" }); + }); + + it("rolls back when an existing value violates the canonical type", () => { + const database = createDatabase(); + database.exec("PRAGMA foreign_keys = ON; CREATE TABLE counters (id TEXT, value TEXT);"); + database.prepare("INSERT INTO counters (id, value) VALUES (?, ?)").run("one", "not-a-number"); + + expect(() => + migrateSqliteSchemaToStrict( + database, + `CREATE TABLE IF NOT EXISTS counters ( + id TEXT, + value INTEGER NOT NULL + ) STRICT;`, + ), + ).toThrow("Failed migrating SQLite table counters to STRICT"); + + expect(readStrictFlag(database, "counters")).toBe(0); + expect(database.prepare("SELECT id, value FROM counters").get()).toEqual({ + id: "one", + value: "not-a-number", + }); + expect(database.prepare("PRAGMA foreign_keys").get()).toEqual({ foreign_keys: 1 }); + }); + + it("rolls back when existing rows violate a foreign key", () => { + const database = createDatabase(); + database.exec(` + PRAGMA foreign_keys = OFF; + CREATE TABLE parents (id INTEGER PRIMARY KEY); + CREATE TABLE children (id INTEGER PRIMARY KEY, parent_id INTEGER REFERENCES parents(id)); + INSERT INTO children (id, parent_id) VALUES (1, 99); + PRAGMA foreign_keys = ON; + `); + + expect(() => + migrateSqliteSchemaToStrict( + database, + ` + CREATE TABLE IF NOT EXISTS parents (id INTEGER PRIMARY KEY) STRICT; + CREATE TABLE IF NOT EXISTS children ( + id INTEGER PRIMARY KEY, + parent_id INTEGER REFERENCES parents(id) + ) STRICT; + `, + ), + ).toThrow("foreign_key_check failed"); + + expect(readStrictFlag(database, "parents")).toBe(0); + expect(readStrictFlag(database, "children")).toBe(0); + expect(database.prepare("SELECT * FROM children").get()).toEqual({ id: 1, parent_id: 99 }); + }); + + it("rejects schema drift without changing the existing table", () => { + const database = createDatabase(); + database.exec("CREATE TABLE entries (id TEXT PRIMARY KEY, obsolete TEXT);"); + + expect(() => + migrateSqliteSchemaToStrict( + database, + `CREATE TABLE IF NOT EXISTS entries ( + id TEXT PRIMARY KEY, + current TEXT + ) STRICT;`, + ), + ).toThrow("SQLite table entries does not match its canonical columns"); + + expect(readStrictFlag(database, "entries")).toBe(0); + expect( + database.prepare("SELECT name FROM pragma_table_info('entries') ORDER BY cid").all(), + ).toEqual([{ name: "id" }, { name: "obsolete" }]); + }); + + it("rejects canonical schemas that still contain non-STRICT tables", () => { + const database = createDatabase(); + + expect(() => + migrateSqliteSchemaToStrict( + database, + "CREATE TABLE IF NOT EXISTS entries (id TEXT PRIMARY KEY);", + ), + ).toThrow("Canonical SQLite schema contains non-STRICT tables: entries"); + expect(database.prepare("PRAGMA table_list").all()).not.toContainEqual( + expect.objectContaining({ name: "entries" }), + ); + }); + + it("ignores virtual tables while migrating ordinary tables", () => { + const database = createDatabase(); + database.exec(` + CREATE TABLE documents (id INTEGER PRIMARY KEY, body TEXT NOT NULL); + CREATE VIRTUAL TABLE documents_fts USING fts5(body); + INSERT INTO documents (id, body) VALUES (1, 'hello'); + INSERT INTO documents_fts (rowid, body) VALUES (1, 'hello'); + `); + + const result = migrateSqliteSchemaToStrict( + database, + ` + CREATE TABLE IF NOT EXISTS documents ( + id INTEGER PRIMARY KEY, + body TEXT NOT NULL + ) STRICT; + CREATE VIRTUAL TABLE IF NOT EXISTS documents_fts USING fts5(body); + `, + ); + + expect(result.migratedTables).toEqual(["documents"]); + expect(readStrictFlag(database, "documents")).toBe(1); + expect(database.prepare("SELECT rowid, body FROM documents_fts").all()).toEqual([ + { rowid: 1, body: "hello" }, + ]); + }); + + it("is idempotent once every canonical table is STRICT", () => { + const database = createDatabase(); + const schema = "CREATE TABLE IF NOT EXISTS entries (id TEXT PRIMARY KEY, value BLOB) STRICT;"; + database.exec(schema); + database.prepare("INSERT INTO entries (id, value) VALUES (?, ?)").run("one", null); + + expect(migrateSqliteSchemaToStrict(database, schema)).toEqual({ migratedTables: [] }); + expect(database.prepare("SELECT id, value FROM entries").get()).toEqual({ + id: "one", + value: null, + }); + }); +}); diff --git a/src/infra/sqlite-strict.ts b/src/infra/sqlite-strict.ts new file mode 100644 index 00000000000..8c2fb84957e --- /dev/null +++ b/src/infra/sqlite-strict.ts @@ -0,0 +1,404 @@ +// Migrates OpenClaw-owned SQLite tables to canonical STRICT schemas. +import type { DatabaseSync } from "node:sqlite"; +import { requireNodeSqlite } from "./node-sqlite.js"; +import { assertSqliteIntegrity } from "./sqlite-integrity.js"; +import { runSqliteImmediateTransactionSync } from "./sqlite-transaction.js"; + +type TableListRow = { + name?: unknown; + schema?: unknown; + strict?: unknown; + type?: unknown; + wr?: unknown; +}; + +type TableColumnRow = { + hidden?: unknown; + name?: unknown; + pk?: unknown; + type?: unknown; +}; + +type SchemaObjectRow = { + name?: unknown; + sql?: unknown; + tbl_name?: unknown; + type?: unknown; +}; + +type PreservedSchemaObject = { + name: string; + sql: string; + type: "index" | "trigger" | "view"; +}; + +type CanonicalStrictTable = { + columns: string[]; + createSql: string; + name: string; + rowidAlias: string | null; + rowidStorage: TableRowidStorage; + usesAutoincrement: boolean; +}; + +type TableRowidStorage = "implicit" | "integer-primary-key" | "without-rowid"; + +type TableRowidModel = { + alias: string | null; + storage: TableRowidStorage; +}; + +export type SqliteStrictMigrationOptions = { + busyTimeoutMs?: number; + databaseLabel?: string; +}; + +export type SqliteStrictMigrationResult = { + migratedTables: string[]; +}; + +const DEFAULT_STRICT_MIGRATION_BUSY_TIMEOUT_MS = 5_000; +const STRICT_MIGRATION_TABLE_PREFIX = "__openclaw_strict_migration_"; +const SQLITE_ROWID_ALIASES = ["_rowid_", "rowid", "oid"] as const; + +function quoteSqliteIdentifier(identifier: string): string { + return `"${identifier.replaceAll('"', '""')}"`; +} + +function readMainTableList(db: DatabaseSync): TableListRow[] { + return (db.prepare("PRAGMA table_list").all() as TableListRow[]).filter( + (row) => + row.schema === "main" && typeof row.name === "string" && !row.name.startsWith("sqlite_"), + ); +} + +function readTableColumns(db: DatabaseSync, tableName: string): TableColumnRow[] { + return db + .prepare(`PRAGMA table_xinfo(${quoteSqliteIdentifier(tableName)})`) + .all() as TableColumnRow[]; +} + +function readVisibleColumns(db: DatabaseSync, tableName: string): string[] { + return readTableColumns(db, tableName) + .filter((row) => Number(row.hidden ?? 0) === 0) + .map((row) => { + if (typeof row.name !== "string" || row.name.length === 0) { + throw new Error(`SQLite table ${tableName} has an invalid column name`); + } + return row.name; + }); +} + +function readTableRowidModel( + db: DatabaseSync, + tableName: string, + tableRow: TableListRow, +): TableRowidModel { + if (Number(tableRow.wr ?? 0) === 1) { + return { alias: null, storage: "without-rowid" }; + } + const columns = readTableColumns(db, tableName); + const primaryKeyColumns = columns.filter((column) => Number(column.pk ?? 0) > 0); + const primaryKeyIndex = db + .prepare(`SELECT 1 AS found FROM pragma_index_list(?) WHERE origin = 'pk' LIMIT 1`) + .get(tableName); + const primaryKeyType = primaryKeyColumns[0]?.type; + if ( + primaryKeyColumns.length === 1 && + typeof primaryKeyType === "string" && + primaryKeyType.toUpperCase() === "INTEGER" && + !primaryKeyIndex + ) { + return { alias: null, storage: "integer-primary-key" }; + } + const declaredNames = new Set( + columns.flatMap((column) => + typeof column.name === "string" ? [column.name.toLowerCase()] : [], + ), + ); + const alias = SQLITE_ROWID_ALIASES.find((candidate) => !declaredNames.has(candidate)) ?? null; + if (!alias) { + throw new Error( + `SQLite table ${tableName} shadows every rowid alias; its implicit rowids cannot be migrated safely`, + ); + } + return { alias, storage: "implicit" }; +} + +function readCanonicalStrictTables(schemaSql: string): CanonicalStrictTable[] { + const sqlite = requireNodeSqlite(); + const canonical = new sqlite.DatabaseSync(":memory:"); + try { + canonical.exec(schemaSql); + const tables = readMainTableList(canonical).filter((row) => row.type === "table"); + const nonStrict = tables.flatMap((row) => + Number(row.strict ?? 0) === 1 || typeof row.name !== "string" ? [] : [row.name], + ); + if (nonStrict.length > 0) { + throw new Error( + `Canonical SQLite schema contains non-STRICT tables: ${nonStrict.toSorted().join(", ")}`, + ); + } + return tables + .map((row) => { + if (typeof row.name !== "string") { + throw new Error("Canonical SQLite schema contains an unnamed table"); + } + const schemaRow = canonical + .prepare("SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = ?") + .get(row.name) as { sql?: unknown } | undefined; + if (typeof schemaRow?.sql !== "string") { + throw new Error(`Canonical SQLite table ${row.name} has no CREATE statement`); + } + const rowidModel = readTableRowidModel(canonical, row.name, row); + return { + columns: readVisibleColumns(canonical, row.name), + createSql: schemaRow.sql, + name: row.name, + rowidAlias: rowidModel.alias, + rowidStorage: rowidModel.storage, + usesAutoincrement: /\bAUTOINCREMENT\b/iu.test(schemaRow.sql), + }; + }) + .toSorted((left, right) => left.name.localeCompare(right.name)); + } finally { + canonical.close(); + } +} + +function rewriteCreateTableName(createSql: string, replacementName: string): string { + const openingParen = createSql.indexOf("("); + if (openingParen === -1) { + throw new Error("Canonical SQLite table CREATE statement has no column list"); + } + return `CREATE TABLE ${quoteSqliteIdentifier(replacementName)} ${createSql.slice(openingParen)}`; +} + +function readPreservedSchemaObjects( + db: DatabaseSync, + tableNames: ReadonlySet, +): PreservedSchemaObject[] { + return ( + db + .prepare( + "SELECT type, name, tbl_name, sql FROM sqlite_schema WHERE type IN ('index', 'trigger', 'view')", + ) + .all() as SchemaObjectRow[] + ) + .flatMap((row) => { + if ( + (row.type !== "index" && row.type !== "trigger" && row.type !== "view") || + typeof row.name !== "string" || + typeof row.tbl_name !== "string" || + typeof row.sql !== "string" || + (row.type === "index" && !tableNames.has(row.tbl_name)) + ) { + return []; + } + return [{ name: row.name, sql: row.sql, type: row.type }]; + }) + .toSorted((left, right) => { + const typeOrder = { view: 0, index: 1, trigger: 2 } as const; + return typeOrder[left.type] - typeOrder[right.type] || left.name.localeCompare(right.name); + }); +} + +function readAutoincrementHighWater(db: DatabaseSync, tableName: string): string | null { + const sequenceTable = db + .prepare( + "SELECT 1 AS found FROM sqlite_schema WHERE type = 'table' AND name = 'sqlite_sequence'", + ) + .get(); + if (!sequenceTable) { + return null; + } + const row = db + .prepare("SELECT CAST(seq AS TEXT) AS seq FROM sqlite_sequence WHERE name = ?") + .get(tableName) as { seq?: unknown } | undefined; + if (row === undefined) { + return null; + } + const normalized = typeof row.seq === "string" ? /^(\d+)(?:\.0+)?$/u.exec(row.seq)?.[1] : null; + if (!normalized) { + throw new Error( + `SQLite table ${tableName} has an invalid AUTOINCREMENT high-water mark (${typeof row.seq}: ${String(row.seq)})`, + ); + } + return normalized; +} + +function restoreAutoincrementHighWater( + db: DatabaseSync, + tableName: string, + previousHighWater: string | null, +): void { + if (previousHighWater === null) { + return; + } + const currentHighWater = readAutoincrementHighWater(db, tableName); + const restored = + currentHighWater === null || BigInt(previousHighWater) > BigInt(currentHighWater) + ? previousHighWater + : currentHighWater; + db.prepare("DELETE FROM sqlite_sequence WHERE name = ?").run(tableName); + db.prepare("INSERT INTO sqlite_sequence (name, seq) VALUES (?, CAST(? AS INTEGER))").run( + tableName, + restored, + ); +} + +function assertMatchingColumns( + tableName: string, + currentColumns: readonly string[], + canonicalColumns: readonly string[], +): void { + const current = new Set(currentColumns); + const canonical = new Set(canonicalColumns); + const missing = canonicalColumns.filter((column) => !current.has(column)); + const extra = currentColumns.filter((column) => !canonical.has(column)); + if (missing.length === 0 && extra.length === 0) { + return; + } + const details = [ + missing.length > 0 ? `missing ${missing.join(", ")}` : "", + extra.length > 0 ? `extra ${extra.join(", ")}` : "", + ] + .filter(Boolean) + .join("; "); + throw new Error(`SQLite table ${tableName} does not match its canonical columns (${details})`); +} + +function readForeignKeysEnabled(db: DatabaseSync): boolean { + const row = db.prepare("PRAGMA foreign_keys").get() as { foreign_keys?: unknown } | undefined; + return Number(row?.foreign_keys ?? 0) === 1; +} + +/** + * Rebuild canonical non-STRICT tables inside the caller's transaction. + * Foreign-key enforcement must be disabled before BEGIN; integrity is checked + * before this function returns so any bad row or relationship rolls back. + */ +export function migrateSqliteSchemaToStrictInTransaction( + db: DatabaseSync, + schemaSql: string, + options: Pick = {}, +): SqliteStrictMigrationResult { + if (!db.isTransaction) { + throw new Error("SQLite STRICT schema migration requires an active transaction"); + } + const canonicalTables = readCanonicalStrictTables(schemaSql); + db.exec(schemaSql); + const currentTableRows = new Map( + readMainTableList(db) + .filter((row) => row.type === "table" && typeof row.name === "string") + .map((row) => [row.name as string, row]), + ); + const tablesToMigrate = canonicalTables.filter( + (table) => Number(currentTableRows.get(table.name)?.strict ?? 0) !== 1, + ); + if (tablesToMigrate.length === 0) { + return { migratedTables: [] }; + } + if (readForeignKeysEnabled(db)) { + throw new Error("SQLite STRICT schema migration requires foreign_keys=OFF before BEGIN"); + } + + const names = new Set(tablesToMigrate.map((table) => table.name)); + const preservedObjects = readPreservedSchemaObjects(db, names); + // SQLite reparses every trigger and view during ALTER TABLE. Temporarily + // remove them so a referenced table can be absent between DROP and RENAME. + for (const object of preservedObjects) { + if (object.type === "trigger") { + db.exec(`DROP TRIGGER ${quoteSqliteIdentifier(object.name)};`); + } + } + for (const object of preservedObjects) { + if (object.type === "view") { + db.exec(`DROP VIEW ${quoteSqliteIdentifier(object.name)};`); + } + } + for (const [index, table] of tablesToMigrate.entries()) { + const migrationTable = `${STRICT_MIGRATION_TABLE_PREFIX}${index}_${table.name}`; + if (currentTableRows.has(migrationTable)) { + throw new Error(`SQLite STRICT migration table already exists: ${migrationTable}`); + } + const currentColumns = readVisibleColumns(db, table.name); + assertMatchingColumns(table.name, currentColumns, table.columns); + const currentTableRow = currentTableRows.get(table.name); + if (!currentTableRow) { + throw new Error(`SQLite table ${table.name} disappeared during STRICT migration`); + } + const currentRowidModel = readTableRowidModel(db, table.name, currentTableRow); + if (currentRowidModel.storage !== table.rowidStorage) { + throw new Error( + `SQLite table ${table.name} changes rowid storage from ${currentRowidModel.storage} to ${table.rowidStorage}; refusing an identity-changing STRICT migration`, + ); + } + const previousHighWater = table.usesAutoincrement + ? readAutoincrementHighWater(db, table.name) + : null; + db.exec(rewriteCreateTableName(table.createSql, migrationTable)); + const columns = table.columns.map(quoteSqliteIdentifier); + if (table.rowidAlias) { + columns.unshift(quoteSqliteIdentifier(table.rowidAlias)); + } + const copyColumns = columns.join(", "); + try { + db.exec( + `INSERT INTO ${quoteSqliteIdentifier(migrationTable)} (${copyColumns}) ` + + `SELECT ${copyColumns} FROM ${quoteSqliteIdentifier(table.name)};`, + ); + } catch (error) { + throw new Error(`Failed migrating SQLite table ${table.name} to STRICT`, { cause: error }); + } + db.exec(`DROP TABLE ${quoteSqliteIdentifier(table.name)};`); + db.exec( + `ALTER TABLE ${quoteSqliteIdentifier(migrationTable)} RENAME TO ${quoteSqliteIdentifier(table.name)};`, + ); + restoreAutoincrementHighWater(db, table.name, previousHighWater); + } + + // Recreate canonical objects first, then retain any database-local indexes or + // triggers that are not part of the checked-in schema. + db.exec(schemaSql); + const findObject = db.prepare( + "SELECT 1 AS found FROM sqlite_schema WHERE type = ? AND name = ? LIMIT 1", + ); + for (const object of preservedObjects) { + if (!findObject.get(object.type, object.name)) { + db.exec(object.sql); + } + } + assertSqliteIntegrity(db, options.databaseLabel ?? "SQLite STRICT schema migration"); + return { migratedTables: tablesToMigrate.map((table) => table.name) }; +} + +/** Atomically upgrade OpenClaw-owned tables described by a canonical STRICT schema. */ +export function migrateSqliteSchemaToStrict( + db: DatabaseSync, + schemaSql: string, + options: SqliteStrictMigrationOptions = {}, +): SqliteStrictMigrationResult { + if (db.isTransaction) { + throw new Error("SQLite STRICT schema migration cannot start inside a transaction"); + } + const foreignKeysWereEnabled = readForeignKeysEnabled(db); + if (foreignKeysWereEnabled) { + db.exec("PRAGMA foreign_keys = OFF;"); + } + try { + return runSqliteImmediateTransactionSync( + db, + () => migrateSqliteSchemaToStrictInTransaction(db, schemaSql, options), + { + busyTimeoutMs: options.busyTimeoutMs ?? DEFAULT_STRICT_MIGRATION_BUSY_TIMEOUT_MS, + databaseLabel: options.databaseLabel, + operationLabel: "sqlite.strict-schema-migration", + }, + ); + } finally { + if (foreignKeysWereEnabled) { + db.exec("PRAGMA foreign_keys = ON;"); + } + } +} diff --git a/src/infra/sqlite-wal.test.ts b/src/infra/sqlite-wal.test.ts index 2d17f66541f..6c7314d6362 100644 --- a/src/infra/sqlite-wal.test.ts +++ b/src/infra/sqlite-wal.test.ts @@ -24,7 +24,7 @@ function createMockDb(): DatabaseSync { get: vi.fn(() => sql.includes("wal_checkpoint") ? { busy: 0, log: 0, checkpointed: 0 } - : { journal_mode: "delete" }, + : { journal_mode: sql === "PRAGMA journal_mode;" ? "wal" : "delete" }, ), })), } as unknown as DatabaseSync; @@ -142,7 +142,7 @@ describe("sqlite WAL maintenance", () => { }); expect(realpath).toHaveBeenCalledWith(databasePath); - expect(db["prepare"]).not.toHaveBeenCalled(); + expect(db["prepare"]).toHaveBeenCalledWith("PRAGMA journal_mode;"); expect(db["exec"]).toHaveBeenNthCalledWith(1, "PRAGMA journal_mode = WAL;"); }); @@ -184,6 +184,45 @@ describe("sqlite WAL maintenance", () => { } }); + it("accepts SQLite's memory journal for an in-memory database", () => { + const sqlite = requireNodeSqlite(); + const db = new sqlite.DatabaseSync(":memory:"); + try { + const maintenance = configureSqliteWalMaintenance(db, { + checkpointIntervalMs: 0, + databaseLabel: "in-memory-test-db", + }); + + expect(db.prepare("PRAGMA journal_mode;").get()).toEqual({ journal_mode: "memory" }); + expect(maintenance.checkpoint()).toBe(true); + expect(maintenance.close()).toBe(true); + } finally { + db.close(); + } + }); + + it("rejects a memory journal for a file-backed database", () => { + const db = createMockDb(); + vi.mocked(db["prepare"]).mockImplementation( + (sql) => + ({ + all: vi.fn(() => + sql === "PRAGMA database_list;" + ? [{ seq: 0, name: "main", file: "/tmp/file-backed.sqlite" }] + : [], + ), + get: vi.fn(() => ({ journal_mode: "memory" })), + }) as unknown as ReturnType, + ); + + expect(() => + configureSqliteWalMaintenance(db, { + checkpointIntervalMs: 0, + databaseLabel: "file-backed-test-db", + }), + ).toThrow("file-backed-test-db could not enable WAL; SQLite kept journal_mode=memory"); + }); + it("uses mountinfo filesystem names when statfs magic is not enough", () => { const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-sqlite-nfs-")); try { @@ -474,7 +513,7 @@ describe("sqlite WAL maintenance", () => { get: vi.fn(() => sql.includes("wal_checkpoint") ? { busy: 1, log: 4, checkpointed: 3 } - : { journal_mode: "delete" }, + : { journal_mode: sql === "PRAGMA journal_mode;" ? "wal" : "delete" }, ), }) as unknown as ReturnType, ); @@ -538,7 +577,7 @@ describe("sqlite WAL maintenance", () => { throw error; } return { - get: vi.fn(() => ({ journal_mode: "delete" })), + get: vi.fn(() => ({ journal_mode: sql === "PRAGMA journal_mode;" ? "wal" : "delete" })), } as unknown as ReturnType; }); @@ -579,6 +618,26 @@ describe("sqlite WAL maintenance", () => { ]); }); + it("rejects a WAL transition that SQLite silently declines", () => { + const db = createMockDb(); + vi.spyOn(process, "platform", "get").mockReturnValue("linux"); + vi.mocked(db["prepare"]).mockImplementation( + (sql) => + ({ + get: vi.fn(() => ({ journal_mode: sql === "PRAGMA journal_mode;" ? "delete" : "wal" })), + }) as unknown as ReturnType, + ); + + expect(() => + configureSqliteConnectionPragmas(db, { + busyTimeoutMs: 50, + checkpointIntervalMs: 0, + databaseLabel: "test-db", + }), + ).toThrow("test-db could not enable WAL; SQLite kept journal_mode=delete"); + expect(db["prepare"]).toHaveBeenCalledWith("PRAGMA journal_mode;"); + }); + it("configures lock retry before inspecting a fresh database header", () => { const db = createMockDb(); vi.mocked(db["prepare"]).mockImplementation( diff --git a/src/infra/sqlite-wal.ts b/src/infra/sqlite-wal.ts index fe5b68f55c6..f8622738648 100644 --- a/src/infra/sqlite-wal.ts +++ b/src/infra/sqlite-wal.ts @@ -300,6 +300,15 @@ function readJournalModeResult(row: unknown): string | null { return typeof value === "string" ? value.toLowerCase() : null; } +function hasInMemoryMainDatabase(db: DatabaseSync): boolean { + const rows = db.prepare("PRAGMA database_list;").all() as Array<{ + file?: unknown; + name?: unknown; + }>; + const main = rows.find((row) => row.name === "main"); + return main?.file === ""; +} + function readCheckpointBusyResult(row: unknown): boolean { if (!row || typeof row !== "object") { return false; @@ -322,14 +331,31 @@ function requireRollbackJournalMode(db: DatabaseSync, options: SqliteWalMaintena } } -function enableWalJournalMode(db: DatabaseSync, retryTimeoutMs: number): void { +function enableWalJournalMode( + db: DatabaseSync, + retryTimeoutMs: number, + options: SqliteWalMaintenanceOptions, +): boolean { const deadline = Date.now() + retryTimeoutMs; let restoreBusyTimeout = false; try { while (true) { try { db.exec("PRAGMA journal_mode = WAL;"); - return; + const journalMode = readJournalModeResult(db.prepare("PRAGMA journal_mode;").get()); + if (journalMode === "wal") { + return true; + } + // SQLite's in-memory databases cannot use WAL and correctly retain + // journal_mode=memory. They have no sidecars or checkpoint work. + if (journalMode === "memory" && hasInMemoryMainDatabase(db)) { + return false; + } + const label = options.databaseLabel ?? "sqlite database"; + const location = options.databasePath ? ` at ${options.databasePath}` : ""; + throw new Error( + `${label}${location} could not enable WAL; SQLite kept journal_mode=${journalMode ?? "unknown"}.`, + ); } catch (error) { const remainingMs = deadline - Date.now(); if (!isSqliteLockError(error) || remainingMs <= 0) { @@ -408,7 +434,12 @@ export function configureSqliteWalMaintenance( close: () => true, }; } - enableWalJournalMode(db, busyTimeoutMs); + if (!enableWalJournalMode(db, busyTimeoutMs, options)) { + return { + checkpoint: () => true, + close: () => true, + }; + } enableMacosCheckpointFullfsync(db); db.exec(`PRAGMA wal_autocheckpoint = ${autoCheckpointPages};`); diff --git a/src/infra/update-managed-service-handoff.test.ts b/src/infra/update-managed-service-handoff.test.ts index d884d1d2993..360843a2365 100644 --- a/src/infra/update-managed-service-handoff.test.ts +++ b/src/infra/update-managed-service-handoff.test.ts @@ -798,6 +798,41 @@ describe("managed service update handoff", () => { } }); + it("waits for a concurrent state writer before persisting the fallback failure", async () => { + let lockReleased: Promise | undefined; + const { result, env } = await runHelperWithExistingSentinel({ + handoffId: "handoff-locked", + metaHandoffId: "handoff-locked", + prepareStateDatabase: async (stateEnv) => { + openOpenClawStateDatabase({ env: stateEnv }); + closeOpenClawStateDatabaseForTest(); + const sqlite = await import("node:sqlite"); + const lock = new sqlite.DatabaseSync(resolveOpenClawStateSqlitePath(stateEnv)); + lock.exec("BEGIN IMMEDIATE;"); + lockReleased = new Promise((resolve) => { + setTimeout(() => { + lock.exec("COMMIT;"); + lock.close(); + resolve(); + }, 200); + }); + }, + }); + await lockReleased; + + expect(result).toEqual({ code: 1, signal: null }); + expect(readRestartSentinelPayload(env)).toMatchObject({ + version: 1, + payload: { + status: "error", + stats: { + handoffId: "handoff-locked", + reason: "managed-service-handoff-parent-timeout", + }, + }, + }); + }); + it("repairs legacy restart sentinel columns before writing fallback failures", async () => { const { result, env } = await runHelperWithExistingSentinel({ handoffId: "handoff-123", diff --git a/src/infra/update-managed-service-handoff.ts b/src/infra/update-managed-service-handoff.ts index e5b8c6f242e..db7a5467165 100644 --- a/src/infra/update-managed-service-handoff.ts +++ b/src/infra/update-managed-service-handoff.ts @@ -25,6 +25,7 @@ import type { UpdateRestartSentinelMeta } from "./update-restart-sentinel-payloa const PARENT_EXIT_SHUTDOWN_RESERVE_MS = 30_000; const HANDOFF_READY_TIMEOUT_MS = 30_000; const HANDOFF_READY_MARKER = "OPENCLAW_UPDATE_HANDOFF_READY\n"; +const HANDOFF_STATE_DATABASE_BUSY_TIMEOUT_MS = 5_000; const SYSTEMD_RUN_CANDIDATE_PATHS = ["/usr/bin/systemd-run", "/bin/systemd-run"] as const; const SERVICE_IDENTITY_ENV_VARS = new Set([ "OPENCLAW_LAUNCHD_LABEL", @@ -123,6 +124,7 @@ function openStateDatabase() { const sqlite = require("node:sqlite"); fs.mkdirSync(path.dirname(params.stateDatabasePath), { recursive: true, mode: 0o700 }); const db = new sqlite.DatabaseSync(params.stateDatabasePath); + db.exec("PRAGMA busy_timeout = ${HANDOFF_STATE_DATABASE_BUSY_TIMEOUT_MS};"); db.exec([ "CREATE TABLE IF NOT EXISTS gateway_restart_sentinel (", "sentinel_key TEXT NOT NULL PRIMARY KEY,", @@ -141,7 +143,7 @@ function openStateDatabase() { "stats_json TEXT,", "payload_json TEXT NOT NULL,", "updated_at_ms INTEGER NOT NULL", - ");", + ") STRICT;", "CREATE INDEX IF NOT EXISTS idx_gateway_restart_sentinel_ts", "ON gateway_restart_sentinel(ts DESC, sentinel_key);", ].join(" ")); diff --git a/src/plugin-sdk/plugin-state-runtime.ts b/src/plugin-sdk/plugin-state-runtime.ts index 82a4feffde6..149340347f6 100644 --- a/src/plugin-sdk/plugin-state-runtime.ts +++ b/src/plugin-sdk/plugin-state-runtime.ts @@ -2,6 +2,11 @@ * Runtime SDK type surface for plugin-scoped keyed state stores. */ export { configureSqliteConnectionPragmas } from "../infra/sqlite-wal.js"; +export { + migrateSqliteSchemaToStrict, + type SqliteStrictMigrationOptions, + type SqliteStrictMigrationResult, +} from "../infra/sqlite-strict.js"; export type { OpenKeyedStoreOptions, PluginStateEntry, diff --git a/src/plugins/contracts/extension-package-project-boundaries.test.ts b/src/plugins/contracts/extension-package-project-boundaries.test.ts index 43ca302fabc..177efa83d6c 100644 --- a/src/plugins/contracts/extension-package-project-boundaries.test.ts +++ b/src/plugins/contracts/extension-package-project-boundaries.test.ts @@ -61,6 +61,7 @@ const MEMORY_HOST_SDK_EXPORTS = [ const MEMORY_HOST_SDK_ALLOWED_CORE_BRIDGE_FILES = [ "packages/memory-host-sdk/src/host/openclaw-runtime-auth.ts", "packages/memory-host-sdk/src/host/openclaw-runtime-network.ts", + "packages/memory-host-sdk/src/host/openclaw-runtime-sqlite.ts", "packages/memory-host-sdk/src/host/openclaw-runtime.ts", ] as const; const MEMORY_HOST_SDK_RUNTIME_ADAPTER_FILES = [ diff --git a/src/proxy-capture/store.sqlite.test.ts b/src/proxy-capture/store.sqlite.test.ts index deede91cacb..719b2a5a3a9 100644 --- a/src/proxy-capture/store.sqlite.test.ts +++ b/src/proxy-capture/store.sqlite.test.ts @@ -137,6 +137,19 @@ describe("DebugProxyCaptureStore", () => { .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'capture_blobs'") .get(), ).toBeUndefined(); + expect( + lease.store.db + .prepare( + `SELECT name, strict FROM pragma_table_list + WHERE schema = 'main' AND type = 'table' AND name NOT LIKE 'sqlite_%' + ORDER BY name`, + ) + .all(), + ).toEqual([ + { name: "capture_events", strict: 1 }, + { name: "capture_sessions", strict: 1 }, + ]); + expect(lease.store.db.prepare("PRAGMA user_version").get()).toEqual({ user_version: 1 }); expect(lease.store.deleteSessions(["legacy-sdk-session"])).toEqual({ sessions: 1, events: 1, diff --git a/src/proxy-capture/store.sqlite.ts b/src/proxy-capture/store.sqlite.ts index 80731facf94..e920b739022 100644 --- a/src/proxy-capture/store.sqlite.ts +++ b/src/proxy-capture/store.sqlite.ts @@ -10,6 +10,7 @@ import { sha256Hex } from "../infra/crypto-digest.js"; import { requireNodeSqlite } from "../infra/node-sqlite.js"; import { applyPrivateModeSync } from "../infra/private-mode.js"; import { resolveSqliteDatabaseFilePaths } from "../infra/sqlite-files.js"; +import { migrateSqliteSchemaToStrict } from "../infra/sqlite-strict.js"; import { runSqliteImmediateTransactionSync } from "../infra/sqlite-transaction.js"; import { configureSqliteConnectionPragmas, @@ -41,6 +42,45 @@ type PathBasedDebugProxyCaptureStore = { const DEBUG_PROXY_CAPTURE_DIR_MODE = 0o700; const DEBUG_PROXY_CAPTURE_FILE_MODE = 0o600; +const DEBUG_PROXY_CAPTURE_LEGACY_SCHEMA_VERSION = 1; +const DEBUG_PROXY_CAPTURE_LEGACY_SCHEMA_SQL = ` + CREATE TABLE IF NOT EXISTS capture_sessions ( + id TEXT PRIMARY KEY, + started_at INTEGER NOT NULL, + ended_at INTEGER, + mode TEXT NOT NULL, + source_scope TEXT NOT NULL, + source_process TEXT NOT NULL, + proxy_url TEXT, + db_path TEXT NOT NULL, + blob_dir TEXT NOT NULL + ) STRICT; + CREATE TABLE IF NOT EXISTS capture_events ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + ts INTEGER NOT NULL, + source_scope TEXT NOT NULL, + source_process TEXT NOT NULL, + protocol TEXT NOT NULL, + direction TEXT NOT NULL, + kind TEXT NOT NULL, + flow_id TEXT NOT NULL, + method TEXT, + host TEXT, + path TEXT, + status INTEGER, + close_code INTEGER, + content_type TEXT, + headers_json TEXT, + data_text TEXT, + data_blob_id TEXT, + data_sha256 TEXT, + error_text TEXT, + meta_json TEXT + ) STRICT; + CREATE INDEX IF NOT EXISTS capture_events_session_ts_idx ON capture_events(session_id, ts); + CREATE INDEX IF NOT EXISTS capture_events_flow_idx ON capture_events(flow_id, ts); +`; function isInMemoryDatabasePath(dbPath: string): boolean { if (dbPath === ":memory:") { @@ -101,44 +141,22 @@ function openPathBasedDebugProxyCaptureStore( ...(fileBackedPath ? { databasePath: fileBackedPath } : {}), foreignKeys: true, }); - db.exec(` - CREATE TABLE IF NOT EXISTS capture_sessions ( - id TEXT PRIMARY KEY, - started_at INTEGER NOT NULL, - ended_at INTEGER, - mode TEXT NOT NULL, - source_scope TEXT NOT NULL, - source_process TEXT NOT NULL, - proxy_url TEXT, - db_path TEXT NOT NULL, - blob_dir TEXT NOT NULL + const versionRow = db.prepare("PRAGMA user_version").get() as + | { user_version?: unknown } + | undefined; + const schemaVersion = Number(versionRow?.user_version ?? 0); + if (schemaVersion > DEBUG_PROXY_CAPTURE_LEGACY_SCHEMA_VERSION) { + throw new Error( + `Legacy debug proxy capture database uses newer schema version ${schemaVersion}; this build supports ${DEBUG_PROXY_CAPTURE_LEGACY_SCHEMA_VERSION}`, ); - CREATE TABLE IF NOT EXISTS capture_events ( - id INTEGER PRIMARY KEY, - session_id TEXT NOT NULL, - ts INTEGER NOT NULL, - source_scope TEXT NOT NULL, - source_process TEXT NOT NULL, - protocol TEXT NOT NULL, - direction TEXT NOT NULL, - kind TEXT NOT NULL, - flow_id TEXT NOT NULL, - method TEXT, - host TEXT, - path TEXT, - status INTEGER, - close_code INTEGER, - content_type TEXT, - headers_json TEXT, - data_text TEXT, - data_blob_id TEXT, - data_sha256 TEXT, - error_text TEXT, - meta_json TEXT - ); - CREATE INDEX IF NOT EXISTS capture_events_session_ts_idx ON capture_events(session_id, ts); - CREATE INDEX IF NOT EXISTS capture_events_flow_idx ON capture_events(flow_id, ts); - `); + } + db.exec(DEBUG_PROXY_CAPTURE_LEGACY_SCHEMA_SQL); + if (schemaVersion < DEBUG_PROXY_CAPTURE_LEGACY_SCHEMA_VERSION) { + migrateSqliteSchemaToStrict(db, DEBUG_PROXY_CAPTURE_LEGACY_SCHEMA_SQL, { + databaseLabel: fileBackedPath ?? dbPath, + }); + db.exec(`PRAGMA user_version = ${DEBUG_PROXY_CAPTURE_LEGACY_SCHEMA_VERSION};`); + } if (fileBackedPath) { hardenLegacyDatabaseFiles(fileBackedPath); } diff --git a/src/state/openclaw-agent-db.generated.d.ts b/src/state/openclaw-agent-db.generated.d.ts index c8860d54614..ede360f7c78 100644 --- a/src/state/openclaw-agent-db.generated.d.ts +++ b/src/state/openclaw-agent-db.generated.d.ts @@ -61,7 +61,7 @@ export interface MemoryIndexChunks { embedding: string; end_line: number; hash: string; - id: string | null; + id: string; model: string; path: string; source: Generated; @@ -71,7 +71,7 @@ export interface MemoryIndexChunks { } export interface MemoryIndexMeta { - key: string | null; + key: string; value: string; } diff --git a/src/state/openclaw-agent-db.test.ts b/src/state/openclaw-agent-db.test.ts index aa4ad62ef30..1bea38c1140 100644 --- a/src/state/openclaw-agent-db.test.ts +++ b/src/state/openclaw-agent-db.test.ts @@ -458,6 +458,22 @@ describe("openclaw agent database", () => { expect(collectSqliteSchemaShape(database.db)).toEqual( createSqliteSchemaShapeFromSql(new URL("./openclaw-agent-schema.sql", import.meta.url)), ); + expect( + database.db + .prepare( + `SELECT name FROM pragma_table_list + WHERE schema = 'main' + AND type = 'table' + AND name NOT LIKE 'sqlite_%' + AND strict <> 1`, + ) + .all(), + ).toEqual([]); + expect(() => + database.db + .prepare("UPDATE schema_meta SET schema_version = ? WHERE meta_key = 'primary'") + .run("not-an-integer"), + ).toThrow(); expect(database.agentId).toBe("worker-1"); expect(database.path).toBe( path.join(stateDir, "agents", "worker-1", "agent", "openclaw-agent.sqlite"), @@ -475,6 +491,77 @@ describe("openclaw agent database", () => { expect(registered?.sizeBytes).toBeGreaterThan(0); }); + it("migrates version 8 tables to STRICT without losing agent state", () => { + const stateDir = createTempStateDir(); + const env = { OPENCLAW_STATE_DIR: stateDir }; + const opened = openOpenClawAgentDatabase({ agentId: "worker-1", env }); + const databasePath = opened.path; + opened.db + .prepare( + "INSERT INTO auth_profile_state (state_key, state_json, updated_at) VALUES (?, ?, ?)", + ) + .run("last-good", '{"profile":"primary"}', 10); + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + + const { DatabaseSync } = requireNodeSqlite(); + const legacy = new DatabaseSync(databasePath); + legacy.exec(` + ALTER TABLE auth_profile_state RENAME TO auth_profile_state_strict; + CREATE TABLE auth_profile_state ( + state_key TEXT NOT NULL PRIMARY KEY, + state_json TEXT NOT NULL, + updated_at INTEGER NOT NULL + ); + INSERT INTO auth_profile_state SELECT * FROM auth_profile_state_strict; + DROP TABLE auth_profile_state_strict; + DROP TRIGGER memory_index_sources_revision_after_insert; + DROP TRIGGER memory_index_sources_revision_after_update; + DROP TRIGGER memory_index_sources_revision_after_delete; + ALTER TABLE memory_index_sources RENAME TO memory_index_sources_strict; + CREATE TABLE memory_index_sources ( + id INTEGER PRIMARY KEY, + path TEXT NOT NULL, + source TEXT NOT NULL DEFAULT 'memory', + hash TEXT NOT NULL, + mtime INTEGER NOT NULL, + size INTEGER NOT NULL, + UNIQUE (path, source) + ); + INSERT INTO memory_index_sources SELECT * FROM memory_index_sources_strict; + DROP TABLE memory_index_sources_strict; + INSERT INTO memory_index_sources (path, source, hash, mtime, size) + VALUES ('MEMORY.md', 'memory', 'legacy-hash', 10.75, 20); + PRAGMA user_version = 8; + UPDATE schema_meta SET schema_version = 8 WHERE meta_key = 'primary'; + `); + legacy.close(); + + const migrated = openOpenClawAgentDatabase({ agentId: "worker-1", env }); + expect( + migrated.db + .prepare("SELECT strict FROM pragma_table_list WHERE name = 'auth_profile_state'") + .get(), + ).toEqual({ strict: 1 }); + expect(migrated.db.prepare("SELECT * FROM auth_profile_state").get()).toEqual({ + state_key: "last-good", + state_json: '{"profile":"primary"}', + updated_at: 10, + }); + expect( + migrated.db + .prepare("SELECT mtime, typeof(mtime) AS storage_type FROM memory_index_sources") + .get(), + ).toEqual({ mtime: 10.75, storage_type: "real" }); + expect(readSqliteNumberPragma(migrated.db, "user_version")).toBe(OPENCLAW_AGENT_SCHEMA_VERSION); + expect( + migrated.db + .prepare("SELECT schema_version FROM schema_meta WHERE meta_key = 'primary'") + .get(), + ).toEqual({ schema_version: OPENCLAW_AGENT_SCHEMA_VERSION }); + expect(migrated.db.prepare("PRAGMA foreign_keys").get()).toEqual({ foreign_keys: 1 }); + }); + it("generates stable typed memory source identities", () => { const stateDir = createTempStateDir(); const database = openOpenClawAgentDatabase({ @@ -1470,7 +1557,7 @@ describe("openclaw agent database", () => { acp_owned: 1, plugin_owner_id: "history-owner", }); - expect(readSqliteNumberPragma(database.db, "user_version")).toBe(8); + expect(readSqliteNumberPragma(database.db, "user_version")).toBe(OPENCLAW_AGENT_SCHEMA_VERSION); }); it("inspects registered database ownership without mutating the database", () => { diff --git a/src/state/openclaw-agent-db.ts b/src/state/openclaw-agent-db.ts index bbe8badb5a3..e3b976f886e 100644 --- a/src/state/openclaw-agent-db.ts +++ b/src/state/openclaw-agent-db.ts @@ -23,6 +23,7 @@ import { assertSqliteSchemaContains, type SqliteSchemaCompatibility, } from "../infra/sqlite-schema-contract.js"; +import { migrateSqliteSchemaToStrictInTransaction } from "../infra/sqlite-strict.js"; import { runSqliteImmediateTransactionSync, type SqliteTransactionOptions, @@ -69,13 +70,14 @@ export { resolveOpenClawAgentSqlitePath } from "./openclaw-agent-db.paths.js"; * per pathname, protected with private file modes, and registered in the shared * OpenClaw state database for discovery and maintenance. */ -// v8 = per-transcript session provenance. v7 added per-entry lifecycle status projection. +// v9 = SQLite STRICT tables. v8 added per-transcript session provenance. +// v7 added per-entry lifecycle status projection. // v6 added session/transcript hot-path indexes. // v5 added transcript mutation watermarks. // The v4 session/transcript flip and main's v2 memory-identity // change is folded in structure-gated (migrateMemoryIndexSourcesIdentity), so // v2 main DBs and pre-merge v4 flip DBs both converge on this schema. -export const OPENCLAW_AGENT_SCHEMA_VERSION = 8; +export const OPENCLAW_AGENT_SCHEMA_VERSION = 9; const OPENCLAW_AGENT_DB_DIR_MODE = 0o700; const OPENCLAW_AGENT_DB_FILE_MODE = 0o600; const OPENCLAW_AGENT_DB_SLOW_OPEN_MS = 1_000; @@ -679,7 +681,7 @@ function ensureAgentSchema(db: DatabaseSync, agentId: string, pathname: string): // Ownership and version checks must share the write transaction with the // schema update; concurrent openers must not overwrite another agent. // Role/ownership gates before version: user_version is only meaningful - // within one schema role, and the global state DB now carries version 2. + // within one schema role, and the global state DB now carries version 3. assertExistingSchemaOwner(readExistingSchemaMeta(db), agentId, pathname); assertSupportedAgentSchemaVersion(db, pathname); const previousVersion = readSqliteUserVersion(db); @@ -693,6 +695,11 @@ function ensureAgentSchema(db: DatabaseSync, agentId: string, pathname: string): migrateMemoryIndexSourcesIdentity(db); migrateOpenClawAgentSchema(db); db.exec(OPENCLAW_AGENT_SCHEMA_SQL); + if (previousVersion < OPENCLAW_AGENT_SCHEMA_VERSION) { + migrateSqliteSchemaToStrictInTransaction(db, OPENCLAW_AGENT_SCHEMA_SQL, { + databaseLabel: pathname, + }); + } repairCanonicalSqliteUniqueIndexes(db, pathname, OPENCLAW_AGENT_CANONICAL_UNIQUE_INDEXES); backfillOpenClawAgentSchema(db, previousVersion); backfillSessionEntryProvenance(db, previousVersion); diff --git a/src/state/openclaw-agent-schema.generated.ts b/src/state/openclaw-agent-schema.generated.ts index 6e23aaeb272..0bf2ee8c6a8 100644 --- a/src/state/openclaw-agent-schema.generated.ts +++ b/src/state/openclaw-agent-schema.generated.ts @@ -11,7 +11,7 @@ export const OPENCLAW_AGENT_SCHEMA_SQL = `CREATE TABLE IF NOT EXISTS schema_meta app_version TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS sessions ( session_id TEXT NOT NULL PRIMARY KEY, @@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS sessions ( spawned_by TEXT, display_name TEXT, FOREIGN KEY (primary_conversation_id) REFERENCES conversations(conversation_id) ON DELETE SET NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_agent_sessions_updated_at ON sessions(updated_at DESC, session_id); @@ -56,7 +56,7 @@ CREATE TABLE IF NOT EXISTS session_routes ( session_id TEXT NOT NULL, updated_at INTEGER NOT NULL, FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_agent_session_routes_session_id ON session_routes(session_id); @@ -75,7 +75,7 @@ CREATE TABLE IF NOT EXISTS conversations ( metadata_json TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_agent_conversations_lookup ON conversations(channel, account_id, kind, peer_id, thread_id); @@ -102,7 +102,7 @@ CREATE TABLE IF NOT EXISTS session_conversations ( PRIMARY KEY (session_id, conversation_id, role), FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE, FOREIGN KEY (conversation_id) REFERENCES conversations(conversation_id) ON DELETE CASCADE -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_agent_session_conversations_conversation ON session_conversations(conversation_id, last_seen_at DESC, session_id); @@ -118,7 +118,7 @@ CREATE TABLE IF NOT EXISTS session_entries ( updated_at INTEGER NOT NULL, status TEXT CHECK (status IS NULL OR status IN ('running', 'done', 'failed', 'killed', 'timeout')), FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_agent_session_entries_updated_at ON session_entries(updated_at DESC, session_key); @@ -137,7 +137,7 @@ CREATE TABLE IF NOT EXISTS transcript_events ( created_at INTEGER NOT NULL, PRIMARY KEY (session_id, seq), FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE -); +) STRICT; CREATE TABLE IF NOT EXISTS trajectory_runtime_events ( session_id TEXT NOT NULL, @@ -147,7 +147,7 @@ CREATE TABLE IF NOT EXISTS trajectory_runtime_events ( created_at INTEGER NOT NULL, PRIMARY KEY (session_id, seq), FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_agent_trajectory_runtime_run ON trajectory_runtime_events(session_id, run_id, seq) @@ -163,7 +163,7 @@ CREATE TABLE IF NOT EXISTS transcript_event_identities ( created_at INTEGER NOT NULL, PRIMARY KEY (session_id, event_id), FOREIGN KEY (session_id, seq) REFERENCES transcript_events(session_id, seq) ON DELETE CASCADE -); +) STRICT; CREATE UNIQUE INDEX IF NOT EXISTS idx_agent_transcript_message_idempotency ON transcript_event_identities(session_id, message_idempotency_key) @@ -184,7 +184,7 @@ CREATE TABLE IF NOT EXISTS cache_entries ( expires_at INTEGER, updated_at INTEGER NOT NULL, PRIMARY KEY (scope, key) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_agent_cache_expiry ON cache_entries(scope, expires_at, key) @@ -197,28 +197,28 @@ CREATE TABLE IF NOT EXISTS auth_profile_store ( store_key TEXT NOT NULL PRIMARY KEY, store_json TEXT NOT NULL, updated_at INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS auth_profile_state ( state_key TEXT NOT NULL PRIMARY KEY, state_json TEXT NOT NULL, updated_at INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS memory_index_meta ( key TEXT PRIMARY KEY, value TEXT NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS memory_index_sources ( id INTEGER PRIMARY KEY, path TEXT NOT NULL, source TEXT NOT NULL DEFAULT 'memory', hash TEXT NOT NULL, - mtime INTEGER NOT NULL, + mtime REAL NOT NULL, size INTEGER NOT NULL, UNIQUE (path, source) -); +) STRICT; CREATE TABLE IF NOT EXISTS memory_index_chunks ( id TEXT PRIMARY KEY, @@ -231,7 +231,7 @@ CREATE TABLE IF NOT EXISTS memory_index_chunks ( text TEXT NOT NULL, embedding TEXT NOT NULL, updated_at INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS memory_embedding_cache ( provider TEXT NOT NULL, @@ -242,12 +242,12 @@ CREATE TABLE IF NOT EXISTS memory_embedding_cache ( dims INTEGER, updated_at INTEGER NOT NULL, PRIMARY KEY (provider, model, provider_key, hash) -); +) STRICT; CREATE TABLE IF NOT EXISTS memory_index_state ( id INTEGER PRIMARY KEY CHECK (id = 1), revision INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS session_transcript_index_state ( session_id TEXT NOT NULL PRIMARY KEY, @@ -255,7 +255,7 @@ CREATE TABLE IF NOT EXISTS session_transcript_index_state ( leaf_event_id TEXT, needs_rebuild INTEGER NOT NULL DEFAULT 0, updated_at INTEGER NOT NULL -); +) STRICT; CREATE VIRTUAL TABLE IF NOT EXISTS session_transcript_fts USING fts5( text, diff --git a/src/state/openclaw-agent-schema.sql b/src/state/openclaw-agent-schema.sql index f84b2ac89f0..a7c17995b8c 100644 --- a/src/state/openclaw-agent-schema.sql +++ b/src/state/openclaw-agent-schema.sql @@ -6,7 +6,7 @@ CREATE TABLE IF NOT EXISTS schema_meta ( app_version TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS sessions ( session_id TEXT NOT NULL PRIMARY KEY, @@ -34,7 +34,7 @@ CREATE TABLE IF NOT EXISTS sessions ( spawned_by TEXT, display_name TEXT, FOREIGN KEY (primary_conversation_id) REFERENCES conversations(conversation_id) ON DELETE SET NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_agent_sessions_updated_at ON sessions(updated_at DESC, session_id); @@ -51,7 +51,7 @@ CREATE TABLE IF NOT EXISTS session_routes ( session_id TEXT NOT NULL, updated_at INTEGER NOT NULL, FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_agent_session_routes_session_id ON session_routes(session_id); @@ -70,7 +70,7 @@ CREATE TABLE IF NOT EXISTS conversations ( metadata_json TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_agent_conversations_lookup ON conversations(channel, account_id, kind, peer_id, thread_id); @@ -97,7 +97,7 @@ CREATE TABLE IF NOT EXISTS session_conversations ( PRIMARY KEY (session_id, conversation_id, role), FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE, FOREIGN KEY (conversation_id) REFERENCES conversations(conversation_id) ON DELETE CASCADE -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_agent_session_conversations_conversation ON session_conversations(conversation_id, last_seen_at DESC, session_id); @@ -113,7 +113,7 @@ CREATE TABLE IF NOT EXISTS session_entries ( updated_at INTEGER NOT NULL, status TEXT CHECK (status IS NULL OR status IN ('running', 'done', 'failed', 'killed', 'timeout')), FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_agent_session_entries_updated_at ON session_entries(updated_at DESC, session_key); @@ -132,7 +132,7 @@ CREATE TABLE IF NOT EXISTS transcript_events ( created_at INTEGER NOT NULL, PRIMARY KEY (session_id, seq), FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE -); +) STRICT; CREATE TABLE IF NOT EXISTS trajectory_runtime_events ( session_id TEXT NOT NULL, @@ -142,7 +142,7 @@ CREATE TABLE IF NOT EXISTS trajectory_runtime_events ( created_at INTEGER NOT NULL, PRIMARY KEY (session_id, seq), FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_agent_trajectory_runtime_run ON trajectory_runtime_events(session_id, run_id, seq) @@ -158,7 +158,7 @@ CREATE TABLE IF NOT EXISTS transcript_event_identities ( created_at INTEGER NOT NULL, PRIMARY KEY (session_id, event_id), FOREIGN KEY (session_id, seq) REFERENCES transcript_events(session_id, seq) ON DELETE CASCADE -); +) STRICT; CREATE UNIQUE INDEX IF NOT EXISTS idx_agent_transcript_message_idempotency ON transcript_event_identities(session_id, message_idempotency_key) @@ -179,7 +179,7 @@ CREATE TABLE IF NOT EXISTS cache_entries ( expires_at INTEGER, updated_at INTEGER NOT NULL, PRIMARY KEY (scope, key) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_agent_cache_expiry ON cache_entries(scope, expires_at, key) @@ -192,28 +192,28 @@ CREATE TABLE IF NOT EXISTS auth_profile_store ( store_key TEXT NOT NULL PRIMARY KEY, store_json TEXT NOT NULL, updated_at INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS auth_profile_state ( state_key TEXT NOT NULL PRIMARY KEY, state_json TEXT NOT NULL, updated_at INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS memory_index_meta ( key TEXT PRIMARY KEY, value TEXT NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS memory_index_sources ( id INTEGER PRIMARY KEY, path TEXT NOT NULL, source TEXT NOT NULL DEFAULT 'memory', hash TEXT NOT NULL, - mtime INTEGER NOT NULL, + mtime REAL NOT NULL, size INTEGER NOT NULL, UNIQUE (path, source) -); +) STRICT; CREATE TABLE IF NOT EXISTS memory_index_chunks ( id TEXT PRIMARY KEY, @@ -226,7 +226,7 @@ CREATE TABLE IF NOT EXISTS memory_index_chunks ( text TEXT NOT NULL, embedding TEXT NOT NULL, updated_at INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS memory_embedding_cache ( provider TEXT NOT NULL, @@ -237,12 +237,12 @@ CREATE TABLE IF NOT EXISTS memory_embedding_cache ( dims INTEGER, updated_at INTEGER NOT NULL, PRIMARY KEY (provider, model, provider_key, hash) -); +) STRICT; CREATE TABLE IF NOT EXISTS memory_index_state ( id INTEGER PRIMARY KEY CHECK (id = 1), revision INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS session_transcript_index_state ( session_id TEXT NOT NULL PRIMARY KEY, @@ -250,7 +250,7 @@ CREATE TABLE IF NOT EXISTS session_transcript_index_state ( leaf_event_id TEXT, needs_rebuild INTEGER NOT NULL DEFAULT 0, updated_at INTEGER NOT NULL -); +) STRICT; CREATE VIRTUAL TABLE IF NOT EXISTS session_transcript_fts USING fts5( text, diff --git a/src/state/openclaw-state-db-operator-approval-migration.test.ts b/src/state/openclaw-state-db-operator-approval-migration.test.ts index 1ca847580a9..e86318796a7 100644 --- a/src/state/openclaw-state-db-operator-approval-migration.test.ts +++ b/src/state/openclaw-state-db-operator-approval-migration.test.ts @@ -9,19 +9,19 @@ import { OPENCLAW_STATE_SCHEMA_SQL } from "./openclaw-state-schema.generated.js" function canonicalOperatorApprovalCreateSql(): string { const marker = "CREATE TABLE IF NOT EXISTS operator_approvals ("; + const tableTerminator = "\n) STRICT;"; const start = OPENCLAW_STATE_SCHEMA_SQL.indexOf(marker); const end = OPENCLAW_STATE_SCHEMA_SQL.indexOf( - "\n);\n\nCREATE INDEX IF NOT EXISTS idx_operator_approvals_status_expiry", + `${tableTerminator}\n\nCREATE INDEX IF NOT EXISTS idx_operator_approvals_status_expiry`, start, ); - return OPENCLAW_STATE_SCHEMA_SQL.slice(start, end + 3); + return OPENCLAW_STATE_SCHEMA_SQL.slice(start, end + tableTerminator.length); } function legacyTwoKindCreateSql(): string { - return canonicalOperatorApprovalCreateSql().replace( - /'exec',\s*'plugin',\s*'system-agent'/, - "'exec', 'plugin'", - ); + return canonicalOperatorApprovalCreateSql() + .replace(/\) STRICT;$/u, ");") + .replace(/'exec',\s*'plugin',\s*'system-agent'/, "'exec', 'plugin'"); } function seedRow(db: DatabaseSync, kind: string): void { @@ -53,6 +53,9 @@ describe("repairOperatorApprovalKinds", () => { expect(() => assertCanonicalOperatorApprovalKinds(db, ":memory:")).not.toThrow(); const rows = db.prepare("SELECT approval_id, kind FROM operator_approvals").all(); expect(rows).toEqual([{ approval_id: "a1", kind: "exec" }]); + expect( + db.prepare("SELECT strict FROM pragma_table_list WHERE name = 'operator_approvals'").get(), + ).toEqual({ strict: 1 }); db.close(); }); diff --git a/src/state/openclaw-state-db-operator-approval-migration.ts b/src/state/openclaw-state-db-operator-approval-migration.ts index c70bb5d0a9f..b70869bc6c7 100644 --- a/src/state/openclaw-state-db-operator-approval-migration.ts +++ b/src/state/openclaw-state-db-operator-approval-migration.ts @@ -77,15 +77,16 @@ function normalizeDdl(sql: string): string { function canonicalOperatorApprovalCreateSql(): string { const marker = "CREATE TABLE IF NOT EXISTS operator_approvals ("; + const tableTerminator = "\n) STRICT;"; const start = OPENCLAW_STATE_SCHEMA_SQL.indexOf(marker); const end = OPENCLAW_STATE_SCHEMA_SQL.indexOf( - "\n);\n\nCREATE INDEX IF NOT EXISTS idx_operator_approvals_status_expiry", + `${tableTerminator}\n\nCREATE INDEX IF NOT EXISTS idx_operator_approvals_status_expiry`, start, ); if (start < 0 || end < 0) { throw new Error("canonical operator approval schema is unavailable"); } - return OPENCLAW_STATE_SCHEMA_SQL.slice(start, end + 3); + return OPENCLAW_STATE_SCHEMA_SQL.slice(start, end + tableTerminator.length); } // The only legacy shape this repair may destructively replace is the exact @@ -98,16 +99,14 @@ function hasExactLegacyOperatorApprovalSchema(db: DatabaseSync): boolean { if (!live) { return false; } - const expectedLegacy = normalizeDdl( - canonicalOperatorApprovalCreateSql() - // sqlite_master stores the CREATE statement without "IF NOT EXISTS". - .replace( - "CREATE TABLE IF NOT EXISTS operator_approvals (", - "CREATE TABLE operator_approvals (", - ) - .replace(/'exec',\s*'plugin',\s*'system-agent'/, "'exec', 'plugin'"), - ); - return normalizeDdl(live) === expectedLegacy; + const exactStrictLegacy = canonicalOperatorApprovalCreateSql() + // sqlite_master stores the CREATE statement without "IF NOT EXISTS". + .replace("CREATE TABLE IF NOT EXISTS operator_approvals (", "CREATE TABLE operator_approvals (") + .replace(/'exec',\s*'plugin',\s*'system-agent'/, "'exec', 'plugin'"); + const expectedStrictLegacy = normalizeDdl(exactStrictLegacy); + const expectedFlexibleLegacy = normalizeDdl(exactStrictLegacy.replace(/\) STRICT;$/u, ");")); + const normalizedLive = normalizeDdl(live); + return normalizedLive === expectedStrictLegacy || normalizedLive === expectedFlexibleLegacy; } function canonicalCreateSql(): string { diff --git a/src/state/openclaw-state-db.test.ts b/src/state/openclaw-state-db.test.ts index 81a336d83cd..424a8f00e8b 100644 --- a/src/state/openclaw-state-db.test.ts +++ b/src/state/openclaw-state-db.test.ts @@ -719,6 +719,78 @@ describe("openclaw state database", () => { createSqliteSchemaShapeFromSql(new URL("./openclaw-state-schema.sql", import.meta.url)), ); expect(database.path).toBe(path.join(stateDir, "state", "openclaw.sqlite")); + expect( + database.db + .prepare( + `SELECT name FROM pragma_table_list + WHERE schema = 'main' + AND type = 'table' + AND name NOT LIKE 'sqlite_%' + AND strict <> 1`, + ) + .all(), + ).toEqual([]); + expect(() => + database.db + .prepare("UPDATE schema_meta SET schema_version = ? WHERE meta_key = 'primary'") + .run("not-an-integer"), + ).toThrow(); + }); + + it("doctor migrates version 2 tables to STRICT without losing rows", () => { + const stateDir = createTempStateDir(); + const options = { env: { OPENCLAW_STATE_DIR: stateDir } }; + const opened = openOpenClawStateDatabase(options); + const databasePath = opened.path; + opened.db + .prepare( + `INSERT INTO skill_curator_state ( + id, last_attempt_at_ms, last_success_at_ms, last_error, last_result_json + ) VALUES (1, 10, 20, NULL, '{}')`, + ) + .run(); + closeOpenClawStateDatabaseForTest(); + + const { DatabaseSync } = requireNodeSqlite(); + const legacy = new DatabaseSync(databasePath); + legacy.exec(` + ALTER TABLE skill_curator_state RENAME TO skill_curator_state_strict; + CREATE TABLE skill_curator_state ( + id INTEGER NOT NULL PRIMARY KEY CHECK (id = 1), + last_attempt_at_ms INTEGER NOT NULL, + last_success_at_ms INTEGER, + last_error TEXT, + last_result_json TEXT NOT NULL + ); + INSERT INTO skill_curator_state SELECT * FROM skill_curator_state_strict; + DROP TABLE skill_curator_state_strict; + PRAGMA user_version = 2; + UPDATE schema_meta SET schema_version = 2 WHERE meta_key = 'primary'; + `); + legacy.close(); + + expect(detectOpenClawStateDatabaseSchemaMigrations(options)).toEqual([ + { kind: "strict-tables-v3", path: databasePath }, + ]); + expect(repairOpenClawStateDatabaseSchema(options)).toEqual({ + changes: ["Migrated shared state tables to SQLite STRICT typing (1)"], + warnings: [], + }); + expect(detectOpenClawStateDatabaseSchemaMigrations(options)).toEqual([]); + + const migrated = openOpenClawStateDatabase(options); + expect( + migrated.db + .prepare("SELECT strict FROM pragma_table_list WHERE name = 'skill_curator_state'") + .get(), + ).toEqual({ strict: 1 }); + expect(migrated.db.prepare("SELECT * FROM skill_curator_state").get()).toEqual({ + id: 1, + last_attempt_at_ms: 10, + last_success_at_ms: 20, + last_error: null, + last_result_json: "{}", + }); }); it("rejects a placement turn claim tuple without an owner", () => { @@ -1131,11 +1203,15 @@ describe("openclaw state database", () => { expect(detectOpenClawStateDatabaseSchemaMigrations(options)).toEqual([ { kind: "audit-events-v2", path: databasePath }, + { kind: "strict-tables-v3", path: databasePath }, ]); expect(() => openOpenClawStateDatabase(options)).toThrow(/legacy audit event schema/); expect(repairOpenClawStateDatabaseSchema(options)).toEqual({ - changes: ["Migrated shared state audit event ledger → versioned message lifecycle schema"], + changes: [ + "Migrated shared state audit event ledger → versioned message lifecycle schema", + "Migrated shared state tables to SQLite STRICT typing (3)", + ], warnings: [], }); expect(repairOpenClawStateDatabaseSchema(options)).toEqual({ changes: [], warnings: [] }); diff --git a/src/state/openclaw-state-db.ts b/src/state/openclaw-state-db.ts index e0d3e790a73..c0fe1b4a95f 100644 --- a/src/state/openclaw-state-db.ts +++ b/src/state/openclaw-state-db.ts @@ -20,6 +20,7 @@ import { assertSqliteSchemaContains, type SqliteSchemaCompatibility, } from "../infra/sqlite-schema-contract.js"; +import { migrateSqliteSchemaToStrictInTransaction } from "../infra/sqlite-strict.js"; import { runSqliteImmediateTransactionSync } from "../infra/sqlite-transaction.js"; import { createNewerSqliteSchemaVersionError, @@ -53,7 +54,8 @@ import { OPENCLAW_STATE_SCHEMA_SQL } from "./openclaw-state-schema.generated.js" * tables, private file permissions, cached handles, and audit rows for * migrations/backups that operate on local state. */ -export const OPENCLAW_STATE_SCHEMA_VERSION = 2; +// v3 rebuilds every OpenClaw-owned table with SQLite STRICT type enforcement. +export const OPENCLAW_STATE_SCHEMA_VERSION = 3; /** Maximum time one synchronous SQLite call may wait for a lock. */ export const OPENCLAW_SQLITE_BUSY_TIMEOUT_MS = 5_000; const OPENCLAW_STATE_DIR_MODE = 0o700; @@ -114,7 +116,8 @@ export type OpenClawStateDatabaseSchemaMigration = { kind: | "agent-databases-composite-primary-key" | "audit-events-v2" - | "operator-approvals-system-agent"; + | "operator-approvals-system-agent" + | "strict-tables-v3"; path: string; }; const cachedDatabases = new Map(); @@ -771,6 +774,12 @@ export function detectOpenClawStateDatabaseSchemaMigrations( if (!hasCanonicalAuditEventsSchema(db)) { migrations.push({ kind: "audit-events-v2", path: pathname }); } + if ( + tableExists(db, "audit_events") && + readSqliteUserVersion(db) < OPENCLAW_STATE_SCHEMA_VERSION + ) { + migrations.push({ kind: "strict-tables-v3", path: pathname }); + } migrations.push( ...operatorApprovalMigration.detectOperatorApprovalSchemaMigration(db, pathname), ); @@ -795,6 +804,7 @@ export function repairOpenClawStateDatabaseSchema(options: OpenClawStateDatabase try { assertStateDatabaseIntegrityBeforeMutation(db, pathname); assertSupportedSchemaVersion(db, pathname); + db.exec("PRAGMA foreign_keys = OFF;"); const changes = runSqliteImmediateTransactionSync( db, () => { @@ -809,6 +819,20 @@ export function repairOpenClawStateDatabaseSchema(options: OpenClawStateDatabase } applied.push(...operatorApprovalMigration.repairOperatorApprovalSchema(db)); assertCanonicalStateSchemaShape(db, pathname); + if (tableExists(db, "audit_events")) { + ensureAdditiveStateColumns(db); + db.exec(OPENCLAW_STATE_SCHEMA_SQL); + const strictMigration = migrateSqliteSchemaToStrictInTransaction( + db, + OPENCLAW_STATE_SCHEMA_SQL, + { databaseLabel: pathname }, + ); + if (strictMigration.migratedTables.length > 0) { + applied.push( + `Migrated shared state tables to SQLite STRICT typing (${strictMigration.migratedTables.length})`, + ); + } + } markCurrentStateSchemaVersion(db); return applied; }, @@ -825,6 +849,9 @@ export function repairOpenClawStateDatabaseSchema(options: OpenClawStateDatabase warnings: [`Failed migrating shared state database schema at ${pathname}: ${String(err)}`], }; } finally { + if (db.isOpen) { + db.exec("PRAGMA foreign_keys = ON;"); + } db.close(); ensureOpenClawStatePermissions(pathname, env); } @@ -1530,52 +1557,64 @@ function ensureAdditiveStateColumns(db: DatabaseSync): void { function ensureSchema(db: DatabaseSync, pathname: string): void { const now = Date.now(); const kysely = getNodeSqliteKysely(db); - runSqliteImmediateTransactionSync( - db, - () => { - assertSupportedSchemaVersion(db, pathname); - ensureAdditiveStateColumns(db); - assertCanonicalStateSchemaShape(db, pathname); - db.exec(OPENCLAW_STATE_SCHEMA_SQL); - migrateLegacyCronRunLogsToTaskRuns(db); - repairCanonicalSqliteUniqueIndexes(db, pathname, OPENCLAW_STATE_CANONICAL_UNIQUE_INDEXES); - // Retired node_pairing_* tables were created by earlier schema revisions but - // never had a shipped writer (the node surface lives on device_pairing_paired - // records), so dropping the always-empty tables is safe, not destructive. - db.exec( - "DROP TABLE IF EXISTS node_pairing_pending; DROP TABLE IF EXISTS node_pairing_paired;", - ); - db.exec(`PRAGMA user_version = ${OPENCLAW_STATE_SCHEMA_VERSION};`); - executeSqliteQuerySync( - db, - kysely - .insertInto("schema_meta") - .values({ - meta_key: "primary", - role: "global", - schema_version: OPENCLAW_STATE_SCHEMA_VERSION, - agent_id: null, - app_version: null, - created_at: now, - updated_at: now, - }) - .onConflict((conflict) => - conflict.column("meta_key").doUpdateSet({ + // Rebuilding referenced tables requires disabling FK enforcement before BEGIN. + db.exec("PRAGMA foreign_keys = OFF;"); + try { + runSqliteImmediateTransactionSync( + db, + () => { + assertSupportedSchemaVersion(db, pathname); + const previousVersion = readSqliteUserVersion(db); + ensureAdditiveStateColumns(db); + assertCanonicalStateSchemaShape(db, pathname); + db.exec(OPENCLAW_STATE_SCHEMA_SQL); + migrateLegacyCronRunLogsToTaskRuns(db); + if (previousVersion < OPENCLAW_STATE_SCHEMA_VERSION) { + migrateSqliteSchemaToStrictInTransaction(db, OPENCLAW_STATE_SCHEMA_SQL, { + databaseLabel: pathname, + }); + } + repairCanonicalSqliteUniqueIndexes(db, pathname, OPENCLAW_STATE_CANONICAL_UNIQUE_INDEXES); + // Retired node_pairing_* tables were created by earlier schema revisions but + // never had a shipped writer (the node surface lives on device_pairing_paired + // records), so dropping the always-empty tables is safe, not destructive. + db.exec( + "DROP TABLE IF EXISTS node_pairing_pending; DROP TABLE IF EXISTS node_pairing_paired;", + ); + db.exec(`PRAGMA user_version = ${OPENCLAW_STATE_SCHEMA_VERSION};`); + executeSqliteQuerySync( + db, + kysely + .insertInto("schema_meta") + .values({ + meta_key: "primary", role: "global", schema_version: OPENCLAW_STATE_SCHEMA_VERSION, agent_id: null, app_version: null, + created_at: now, updated_at: now, - }), - ), - ); - }, - { - busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, - databaseLabel: pathname, - operationLabel: "state.schema.ensure", - }, - ); + }) + .onConflict((conflict) => + conflict.column("meta_key").doUpdateSet({ + role: "global", + schema_version: OPENCLAW_STATE_SCHEMA_VERSION, + agent_id: null, + app_version: null, + updated_at: now, + }), + ), + ); + }, + { + busyTimeoutMs: OPENCLAW_SQLITE_BUSY_TIMEOUT_MS, + databaseLabel: pathname, + operationLabel: "state.schema.ensure", + }, + ); + } finally { + db.exec("PRAGMA foreign_keys = ON;"); + } } function resolveDatabasePath(options: OpenClawStateDatabaseOptions = {}): string { diff --git a/src/state/openclaw-state-schema.generated.ts b/src/state/openclaw-state-schema.generated.ts index ddd8221f7a2..f9e14cd6a3c 100644 --- a/src/state/openclaw-state-schema.generated.ts +++ b/src/state/openclaw-state-schema.generated.ts @@ -7,13 +7,13 @@ export const OPENCLAW_STATE_SCHEMA_SQL = `CREATE TABLE IF NOT EXISTS auth_profil store_key TEXT NOT NULL PRIMARY KEY, store_json TEXT NOT NULL, updated_at INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS auth_profile_state ( store_key TEXT NOT NULL PRIMARY KEY, state_json TEXT NOT NULL, updated_at INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS diagnostic_events ( scope TEXT NOT NULL, @@ -21,7 +21,7 @@ CREATE TABLE IF NOT EXISTS diagnostic_events ( payload_json TEXT NOT NULL, created_at INTEGER NOT NULL, PRIMARY KEY (scope, event_key) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_diagnostic_events_scope_created ON diagnostic_events(scope, created_at, event_key); @@ -35,7 +35,7 @@ CREATE TABLE IF NOT EXISTS skill_usage ( last_used_at_ms INTEGER NOT NULL, use_count INTEGER NOT NULL, last_agent_id TEXT -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_skill_usage_key ON skill_usage(skill_key, skill_file); @@ -49,7 +49,7 @@ CREATE TABLE IF NOT EXISTS skill_lifecycle ( state_changed_at_ms INTEGER NOT NULL, created_at_ms INTEGER NOT NULL, archived_reason TEXT -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_skill_lifecycle_key ON skill_lifecycle(skill_key, skill_file); @@ -63,7 +63,7 @@ CREATE TABLE IF NOT EXISTS skill_curator_state ( last_success_at_ms INTEGER, last_error TEXT, last_result_json TEXT NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS audit_events ( sequence INTEGER PRIMARY KEY AUTOINCREMENT, @@ -97,7 +97,7 @@ CREATE TABLE IF NOT EXISTS audit_events ( conversation_ref TEXT, message_ref TEXT, target_ref TEXT -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_audit_events_time ON audit_events(occurred_at DESC, sequence DESC); @@ -128,7 +128,7 @@ CREATE TABLE IF NOT EXISTS audit_identity_keys ( key_id TEXT NOT NULL, key BLOB NOT NULL, created_at INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS session_state_events ( sequence INTEGER PRIMARY KEY AUTOINCREMENT, @@ -143,7 +143,7 @@ CREATE TABLE IF NOT EXISTS session_state_events ( occurred_at INTEGER NOT NULL, summary TEXT NOT NULL, payload_json TEXT -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_session_state_events_session_sequence ON session_state_events(session_key, sequence DESC); @@ -158,7 +158,7 @@ CREATE TABLE IF NOT EXISTS session_state_heads ( pruned_max_sequence INTEGER NOT NULL DEFAULT 0, updated_at INTEGER NOT NULL, PRIMARY KEY (session_key, agent_id) -); +) STRICT; -- Watcher identity is the bare session key, matching the process-local system-event -- queue it feeds. Producers only create rows for agent-qualified watcher keys; @@ -172,7 +172,7 @@ CREATE TABLE IF NOT EXISTS session_watch_cursors ( material_sequence INTEGER NOT NULL DEFAULT 0, updated_at INTEGER NOT NULL, PRIMARY KEY (watcher_session_key, target_session_key) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_session_watch_cursors_target ON session_watch_cursors(target_session_key); @@ -192,7 +192,7 @@ CREATE TABLE IF NOT EXISTS session_upstream_links ( -- (session_key, agent_id) composite identity: under session.scope="global" agents -- share bare keys; a key-only row would let one agent overwrite another's upstream. PRIMARY KEY (session_key, agent_id) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_session_upstream_links_catalog_id ON session_upstream_links(catalog_id); @@ -203,7 +203,7 @@ CREATE TABLE IF NOT EXISTS diagnostic_stability_bundles ( generated_at TEXT NOT NULL, bundle_json TEXT NOT NULL, created_at INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_diagnostic_stability_bundles_created ON diagnostic_stability_bundles(created_at DESC, bundle_key); @@ -218,7 +218,7 @@ CREATE TABLE IF NOT EXISTS state_leases ( created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, PRIMARY KEY (scope, lease_key) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_state_leases_expiry ON state_leases(expires_at, scope, lease_key) @@ -239,7 +239,7 @@ CREATE TABLE IF NOT EXISTS exec_approvals_config ( agent_count INTEGER NOT NULL, allowlist_count INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS operator_approvals ( approval_id TEXT NOT NULL PRIMARY KEY CHECK ( @@ -345,7 +345,7 @@ CREATE TABLE IF NOT EXISTS operator_approvals ( AND consumed_by IS NOT NULL ) ) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_operator_approvals_status_expiry ON operator_approvals(status, expires_at_ms, approval_id); @@ -372,7 +372,7 @@ CREATE TABLE IF NOT EXISTS schema_meta ( app_version TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS device_pairing_pending ( request_id TEXT NOT NULL PRIMARY KEY, @@ -391,7 +391,7 @@ CREATE TABLE IF NOT EXISTS device_pairing_pending ( is_repair INTEGER, ts INTEGER NOT NULL, refreshed_at_ms INTEGER -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_device_pairing_pending_device ON device_pairing_pending(device_id, ts DESC); @@ -418,7 +418,7 @@ CREATE TABLE IF NOT EXISTS device_pairing_paired ( approved_at_ms INTEGER NOT NULL, last_seen_at_ms INTEGER, last_seen_reason TEXT -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_device_pairing_paired_approved ON device_pairing_paired(approved_at_ms DESC, device_id); @@ -434,7 +434,7 @@ CREATE TABLE IF NOT EXISTS device_bootstrap_tokens ( pending_profile_json TEXT, issued_at_ms INTEGER NOT NULL, last_used_at_ms INTEGER -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_device_bootstrap_tokens_ts ON device_bootstrap_tokens(ts); @@ -446,7 +446,7 @@ CREATE TABLE IF NOT EXISTS device_identities ( private_key_pem TEXT NOT NULL, created_at_ms INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_device_identities_device ON device_identities(device_id, updated_at_ms DESC); @@ -458,7 +458,7 @@ CREATE TABLE IF NOT EXISTS device_auth_tokens ( scopes_json TEXT NOT NULL, updated_at_ms INTEGER NOT NULL, PRIMARY KEY (device_id, role) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_device_auth_tokens_updated ON device_auth_tokens(updated_at_ms DESC, device_id, role); @@ -467,7 +467,7 @@ CREATE TABLE IF NOT EXISTS android_notification_recent_packages ( package_name TEXT NOT NULL PRIMARY KEY, sort_order INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_android_notification_recent_packages_order ON android_notification_recent_packages(sort_order, package_name); @@ -478,7 +478,7 @@ CREATE TABLE IF NOT EXISTS macos_port_guardian_records ( command TEXT NOT NULL, mode TEXT NOT NULL, timestamp REAL NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_macos_port_guardian_records_port ON macos_port_guardian_records(port, timestamp DESC); @@ -490,7 +490,7 @@ CREATE TABLE IF NOT EXISTS workspace_setup_state ( bootstrap_seeded_at TEXT, setup_completed_at TEXT, updated_at INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_workspace_setup_state_path ON workspace_setup_state(workspace_path); @@ -503,7 +503,7 @@ CREATE TABLE IF NOT EXISTS native_hook_relay_bridges ( token TEXT NOT NULL, expires_at_ms INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_native_hook_relay_bridges_expires ON native_hook_relay_bridges(expires_at_ms, relay_id); @@ -524,7 +524,7 @@ CREATE TABLE IF NOT EXISTS model_capability_cache ( cost_cache_write REAL NOT NULL, updated_at_ms INTEGER NOT NULL, PRIMARY KEY (provider_id, model_id) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_model_capability_cache_provider_updated ON model_capability_cache(provider_id, updated_at_ms DESC, model_id); @@ -534,7 +534,7 @@ CREATE TABLE IF NOT EXISTS agent_model_catalogs ( agent_dir TEXT NOT NULL, raw_json TEXT NOT NULL, updated_at INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_agent_model_catalogs_agent_dir ON agent_model_catalogs(agent_dir, updated_at DESC); @@ -558,7 +558,7 @@ CREATE TABLE IF NOT EXISTS managed_outgoing_image_records ( original_filename TEXT, record_json TEXT NOT NULL, cleanup_pending INTEGER NOT NULL DEFAULT 0 CHECK (cleanup_pending IN (0, 1)) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_managed_outgoing_images_session ON managed_outgoing_image_records(session_key, created_at DESC, attachment_id); @@ -583,7 +583,7 @@ CREATE TABLE IF NOT EXISTS channel_pairing_requests ( last_seen_at TEXT NOT NULL, meta_json TEXT, PRIMARY KEY (channel_key, account_id, request_id) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_channel_pairing_requests_code ON channel_pairing_requests(channel_key, code); @@ -598,7 +598,7 @@ CREATE TABLE IF NOT EXISTS channel_pairing_allow_entries ( sort_order INTEGER NOT NULL, updated_at INTEGER NOT NULL, PRIMARY KEY (channel_key, account_id, entry) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_channel_pairing_allow_account ON channel_pairing_allow_entries(channel_key, account_id, sort_order, entry); @@ -611,7 +611,7 @@ CREATE TABLE IF NOT EXISTS web_push_subscriptions ( auth TEXT NOT NULL, created_at_ms INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_web_push_subscriptions_updated ON web_push_subscriptions(updated_at_ms DESC, subscription_id); @@ -622,7 +622,7 @@ CREATE TABLE IF NOT EXISTS web_push_vapid_keys ( private_key TEXT NOT NULL, subject TEXT NOT NULL, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS apns_registrations ( node_id TEXT NOT NULL PRIMARY KEY, @@ -636,7 +636,7 @@ CREATE TABLE IF NOT EXISTS apns_registrations ( distribution TEXT, token_debug_suffix TEXT, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_apns_registrations_updated ON apns_registrations(updated_at_ms DESC, node_id); @@ -653,7 +653,7 @@ CREATE TABLE IF NOT EXISTS node_host_config ( gateway_tls_fingerprint TEXT, gateway_context_path TEXT, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS voicewake_triggers ( config_key TEXT NOT NULL, @@ -661,7 +661,7 @@ CREATE TABLE IF NOT EXISTS voicewake_triggers ( trigger TEXT NOT NULL, updated_at_ms INTEGER NOT NULL, PRIMARY KEY (config_key, position) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_voicewake_triggers_trigger ON voicewake_triggers(config_key, trigger); @@ -673,7 +673,7 @@ CREATE TABLE IF NOT EXISTS voicewake_routing_config ( default_target_agent_id TEXT, default_target_session_key TEXT, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS voicewake_routing_routes ( config_key TEXT NOT NULL, @@ -685,7 +685,7 @@ CREATE TABLE IF NOT EXISTS voicewake_routing_routes ( updated_at_ms INTEGER NOT NULL, PRIMARY KEY (config_key, position), FOREIGN KEY (config_key) REFERENCES voicewake_routing_config(config_key) ON DELETE CASCADE -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_voicewake_routing_routes_trigger ON voicewake_routing_routes(config_key, trigger); @@ -706,7 +706,7 @@ CREATE TABLE IF NOT EXISTS update_check_state ( auto_last_success_version TEXT, auto_last_success_at TEXT, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS config_health_entries ( config_path TEXT NOT NULL PRIMARY KEY, @@ -714,7 +714,7 @@ CREATE TABLE IF NOT EXISTS config_health_entries ( last_promoted_good_json TEXT, last_observed_suspicious_signature TEXT, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS clawhub_promotions_feed_state ( state_key TEXT NOT NULL PRIMARY KEY, @@ -724,7 +724,7 @@ CREATE TABLE IF NOT EXISTS clawhub_promotions_feed_state ( last_checked_at_ms INTEGER, notified_slugs_json TEXT NOT NULL DEFAULT '[]', updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS clawhub_promotion_claims ( slug TEXT NOT NULL PRIMARY KEY, @@ -732,7 +732,7 @@ CREATE TABLE IF NOT EXISTS clawhub_promotion_claims ( model_keys_json TEXT NOT NULL, ends_at_ms INTEGER NOT NULL, claimed_at_ms INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS installed_plugin_index ( index_key TEXT NOT NULL PRIMARY KEY, @@ -748,7 +748,7 @@ CREATE TABLE IF NOT EXISTS installed_plugin_index ( diagnostics_json TEXT NOT NULL, warning TEXT, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_installed_plugin_index_generated ON installed_plugin_index(generated_at_ms DESC, index_key); @@ -767,7 +767,7 @@ CREATE TABLE IF NOT EXISTS official_external_plugin_catalog_snapshots ( trust_threshold INTEGER, trust_verified_at TEXT, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_official_external_plugin_catalog_snapshots_updated ON official_external_plugin_catalog_snapshots(updated_at_ms DESC, feed_url); @@ -789,7 +789,7 @@ CREATE TABLE IF NOT EXISTS gateway_restart_sentinel ( stats_json TEXT, payload_json TEXT NOT NULL, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_gateway_restart_sentinel_ts ON gateway_restart_sentinel(ts DESC, sentinel_key); @@ -803,7 +803,7 @@ CREATE TABLE IF NOT EXISTS gateway_restart_intent ( force INTEGER, wait_ms INTEGER, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS gateway_restart_handoff ( handoff_key TEXT NOT NULL PRIMARY KEY, @@ -821,7 +821,7 @@ CREATE TABLE IF NOT EXISTS gateway_restart_handoff ( restart_kind TEXT NOT NULL, supervisor_mode TEXT NOT NULL, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_gateway_restart_handoff_expiry ON gateway_restart_handoff(expires_at, pid); @@ -834,7 +834,7 @@ CREATE TABLE IF NOT EXISTS gateway_boot_lifecycle ( outcome TEXT, startup_reason TEXT, reason TEXT -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_gateway_boot_lifecycle_started ON gateway_boot_lifecycle(started_at_ms); @@ -853,7 +853,7 @@ CREATE TABLE IF NOT EXISTS acp_sessions ( last_activity_at INTEGER NOT NULL, last_error TEXT, updated_at INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_acp_sessions_state_activity ON acp_sessions(state, last_activity_at DESC, session_key); @@ -873,7 +873,7 @@ CREATE TABLE IF NOT EXISTS acp_replay_sessions ( -- all event rows), maintained at insert/trim so budget checks never scan -- acp_replay_events (#100622). estimated_bytes INTEGER NOT NULL DEFAULT 0 -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_acp_replay_sessions_key_updated ON acp_replay_sessions(session_key, complete, updated_at DESC, session_id); @@ -891,7 +891,7 @@ CREATE TABLE IF NOT EXISTS acp_replay_events ( estimated_bytes INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (session_id, seq), FOREIGN KEY (session_id) REFERENCES acp_replay_sessions(session_id) ON DELETE CASCADE -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_acp_replay_events_session_seq ON acp_replay_events(session_id, seq); @@ -903,7 +903,7 @@ CREATE TABLE IF NOT EXISTS agent_databases ( last_seen_at INTEGER NOT NULL, size_bytes INTEGER, PRIMARY KEY (agent_id, path) -); +) STRICT; CREATE TABLE IF NOT EXISTS plugin_state_entries ( plugin_id TEXT NOT NULL, @@ -913,7 +913,7 @@ CREATE TABLE IF NOT EXISTS plugin_state_entries ( created_at INTEGER NOT NULL, expires_at INTEGER, PRIMARY KEY (plugin_id, namespace, entry_key) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_plugin_state_expiry ON plugin_state_entries(expires_at) @@ -944,7 +944,7 @@ CREATE TABLE IF NOT EXISTS channel_ingress_events ( completed_at INTEGER, completed_metadata_json TEXT, PRIMARY KEY (queue_name, event_id) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_channel_ingress_pending ON channel_ingress_events(queue_name, status, received_at, event_id); @@ -964,7 +964,7 @@ CREATE TABLE IF NOT EXISTS plugin_blob_entries ( created_at INTEGER NOT NULL, expires_at INTEGER, PRIMARY KEY (plugin_id, namespace, entry_key) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_plugin_blob_expiry ON plugin_blob_entries(expires_at) @@ -982,7 +982,7 @@ CREATE TABLE IF NOT EXISTS media_blobs ( created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, PRIMARY KEY (subdir, id) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_media_blobs_created ON media_blobs(created_at); @@ -1002,7 +1002,7 @@ CREATE TABLE IF NOT EXISTS skill_uploads ( committed INTEGER NOT NULL, committed_at INTEGER, idempotency_key_hash TEXT UNIQUE -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_skill_uploads_expiry ON skill_uploads(expires_at); @@ -1019,7 +1019,7 @@ CREATE TABLE IF NOT EXISTS skill_upload_chunks ( PRIMARY KEY (upload_id, byte_offset), FOREIGN KEY (upload_id) REFERENCES skill_uploads(upload_id) ON DELETE CASCADE, CHECK (length(chunk_blob) = size_bytes) -); +) STRICT; CREATE TABLE IF NOT EXISTS capture_sessions ( id TEXT NOT NULL PRIMARY KEY, @@ -1029,7 +1029,7 @@ CREATE TABLE IF NOT EXISTS capture_sessions ( source_scope TEXT NOT NULL, source_process TEXT NOT NULL, proxy_url TEXT -); +) STRICT; CREATE TABLE IF NOT EXISTS capture_blobs ( blob_id TEXT NOT NULL PRIMARY KEY, @@ -1039,7 +1039,7 @@ CREATE TABLE IF NOT EXISTS capture_blobs ( sha256 TEXT NOT NULL, data BLOB NOT NULL, created_at INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS capture_events ( id INTEGER NOT NULL PRIMARY KEY, @@ -1065,7 +1065,7 @@ CREATE TABLE IF NOT EXISTS capture_events ( meta_json TEXT, FOREIGN KEY (session_id) REFERENCES capture_sessions(id) ON DELETE CASCADE, FOREIGN KEY (data_blob_id) REFERENCES capture_blobs(blob_id) ON DELETE SET NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS capture_events_session_ts_idx ON capture_events(session_id, ts); @@ -1089,7 +1089,7 @@ CREATE TABLE IF NOT EXISTS sandbox_registry_entries ( entry_json TEXT NOT NULL, updated_at INTEGER NOT NULL, PRIMARY KEY (registry_kind, container_name) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_sandbox_registry_updated ON sandbox_registry_entries(registry_kind, updated_at DESC, container_name); @@ -1133,7 +1133,7 @@ CREATE TABLE IF NOT EXISTS commitments ( snoozed_until_ms INTEGER, expired_at_ms INTEGER, record_json TEXT NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_commitments_scope_due ON commitments(agent_id, session_key, status, due_earliest_ms, due_latest_ms); @@ -1227,7 +1227,7 @@ CREATE TABLE IF NOT EXISTS cron_jobs ( sort_order INTEGER NOT NULL DEFAULT 0, updated_at INTEGER NOT NULL, PRIMARY KEY (store_key, job_id) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_cron_jobs_store_updated ON cron_jobs(store_key, sort_order ASC, updated_at DESC, job_id); @@ -1251,7 +1251,7 @@ CREATE TABLE IF NOT EXISTS command_log_entries ( sender_id TEXT NOT NULL, source TEXT NOT NULL, entry_json TEXT NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_command_log_entries_timestamp ON command_log_entries(timestamp_ms DESC, id); @@ -1278,7 +1278,7 @@ CREATE TABLE IF NOT EXISTS delivery_queue_entries ( updated_at INTEGER NOT NULL, failed_at INTEGER, PRIMARY KEY (queue_name, id) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_delivery_queue_pending ON delivery_queue_entries(queue_name, status, enqueued_at, id); @@ -1325,7 +1325,7 @@ CREATE TABLE IF NOT EXISTS task_runs ( terminal_summary TEXT, terminal_outcome TEXT, detail_json TEXT -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_task_runs_run_id ON task_runs(run_id); CREATE INDEX IF NOT EXISTS idx_task_runs_status ON task_runs(status); @@ -1386,7 +1386,7 @@ CREATE TABLE IF NOT EXISTS subagent_runs ( pending_final_delivery_payload_json TEXT, completion_announced_at INTEGER, payload_json TEXT NOT NULL DEFAULT '{}' -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_subagent_runs_child_session_key ON subagent_runs(child_session_key, created_at DESC, run_id); @@ -1417,7 +1417,7 @@ CREATE TABLE IF NOT EXISTS current_conversation_bindings ( metadata_json TEXT, record_json TEXT NOT NULL, updated_at INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_current_conversation_bindings_target ON current_conversation_bindings(target_agent_id, target_session_key, updated_at DESC, binding_key); @@ -1434,7 +1434,7 @@ CREATE TABLE IF NOT EXISTS plugin_binding_approvals ( plugin_name TEXT, approved_at INTEGER NOT NULL, PRIMARY KEY (plugin_root, channel, account_id) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_plugin_binding_approvals_plugin ON plugin_binding_approvals(plugin_id, approved_at DESC); @@ -1443,7 +1443,7 @@ CREATE TABLE IF NOT EXISTS tui_last_sessions ( scope_key TEXT NOT NULL PRIMARY KEY, session_key TEXT NOT NULL, updated_at INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_tui_last_sessions_session_key ON tui_last_sessions(session_key, updated_at DESC, scope_key); @@ -1453,7 +1453,7 @@ CREATE TABLE IF NOT EXISTS task_delivery_state ( requester_origin_json TEXT, last_notified_event_at INTEGER, FOREIGN KEY (task_id) REFERENCES task_runs(task_id) ON DELETE CASCADE -); +) STRICT; CREATE TABLE IF NOT EXISTS flow_runs ( flow_id TEXT NOT NULL PRIMARY KEY, @@ -1475,7 +1475,7 @@ CREATE TABLE IF NOT EXISTS flow_runs ( created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, ended_at INTEGER -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_flow_runs_status ON flow_runs(status); CREATE INDEX IF NOT EXISTS idx_flow_runs_owner_key ON flow_runs(owner_key); @@ -1487,7 +1487,7 @@ CREATE TABLE IF NOT EXISTS migration_runs ( finished_at INTEGER, status TEXT NOT NULL, report_json TEXT NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_migration_runs_started ON migration_runs(started_at DESC, id); @@ -1506,7 +1506,7 @@ CREATE TABLE IF NOT EXISTS migration_sources ( removed_source INTEGER NOT NULL DEFAULT 0, report_json TEXT NOT NULL, FOREIGN KEY (last_run_id) REFERENCES migration_runs(id) ON DELETE CASCADE -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_migration_sources_path ON migration_sources(source_path, migration_kind, target_table); @@ -1520,7 +1520,7 @@ CREATE TABLE IF NOT EXISTS backup_runs ( archive_path TEXT NOT NULL, status TEXT NOT NULL, manifest_json TEXT NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_backup_runs_created ON backup_runs(created_at DESC, id); @@ -1538,7 +1538,7 @@ CREATE TABLE IF NOT EXISTS worktrees ( created_at INTEGER NOT NULL, last_active_at INTEGER NOT NULL, removed_at INTEGER -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_worktrees_repo_fingerprint ON worktrees(repo_fingerprint); @@ -1553,7 +1553,7 @@ CREATE TABLE IF NOT EXISTS session_groups ( name TEXT NOT NULL PRIMARY KEY, position INTEGER NOT NULL, created_at INTEGER NOT NULL -); +) STRICT; -- Gateway-owned durable cloud worker lifecycle. Provider-specific execution -- stays in plugins; this table records only core reconciliation facts. @@ -1596,7 +1596,7 @@ CREATE TABLE IF NOT EXISTS worker_environments ( idle_since_at_ms INTEGER, destroy_requested_at_ms INTEGER, last_error TEXT -); +) STRICT; CREATE UNIQUE INDEX IF NOT EXISTS idx_worker_environments_provider_lease ON worker_environments(provider_id, lease_id) @@ -1710,7 +1710,7 @@ CREATE TABLE IF NOT EXISTS worker_session_placements ( (turn_claim_owner IS 'worker' AND state IN ('active', 'draining') AND turn_claim_owner_epoch IS active_owner_epoch) ) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_worker_session_placements_session_key ON worker_session_placements(agent_id, session_key); @@ -1731,7 +1731,7 @@ CREATE TABLE IF NOT EXISTS worker_workspace_reconciliations ( base_pack BLOB NOT NULL CHECK (length(base_pack) <= 268435456), created_at_ms INTEGER NOT NULL, FOREIGN KEY (session_id) REFERENCES worker_session_placements(session_id) ON DELETE CASCADE -); +) STRICT; -- A completed remote turn is fenced from stale-claim teardown until its -- workspace result is durably reconciled into the managed worktree. @@ -1747,7 +1747,7 @@ CREATE TABLE IF NOT EXISTS worker_workspace_pending_results ( workspace_accepted_at_ms INTEGER, created_at_ms INTEGER NOT NULL, FOREIGN KEY (session_id) REFERENCES worker_session_placements(session_id) ON DELETE CASCADE -); +) STRICT; -- One active, opaque admission credential per worker environment. Plaintext -- may be retried until delivery acknowledgement but never enters durable state. @@ -1761,7 +1761,7 @@ CREATE TABLE IF NOT EXISTS worker_environment_credentials ( expires_at_ms INTEGER NOT NULL CHECK (expires_at_ms >= 0), delivered_at_ms INTEGER CHECK (delivered_at_ms >= 0), FOREIGN KEY (environment_id) REFERENCES worker_environments(environment_id) ON DELETE CASCADE -); +) STRICT; -- One durable sequence cursor per attached session owner epoch. The environment -- binding prevents independent workers with coincident epochs from sharing replay state. @@ -1772,7 +1772,7 @@ CREATE TABLE IF NOT EXISTS worker_transcript_commit_heads ( next_seq INTEGER NOT NULL CHECK (next_seq >= 1), updated_at_ms INTEGER NOT NULL CHECK (updated_at_ms >= 0), PRIMARY KEY (session_id, run_epoch) -); +) STRICT; -- Pending rows preserve a claimed request across gateway restarts. Terminal rows -- cache the exact result returned for deterministic at-least-once replay. @@ -1793,7 +1793,7 @@ CREATE TABLE IF NOT EXISTS worker_transcript_commits ( (state = 'pending' AND result_json IS NULL) OR (state = 'terminal' AND result_json IS NOT NULL) ) -); +) STRICT; -- Pending rows preserve a claimed inference turn across gateway restarts. -- Terminal rows cache the exact outcome returned for deterministic replay. @@ -1814,7 +1814,7 @@ CREATE TABLE IF NOT EXISTS worker_inference_turns ( (state = 'pending' AND terminal_json IS NULL) OR (state = 'terminal' AND terminal_json IS NOT NULL) ) -); +) STRICT; CREATE UNIQUE INDEX IF NOT EXISTS idx_worker_inference_turns_pending_run ON worker_inference_turns(session_id, run_epoch, run_id) @@ -1828,4 +1828,4 @@ CREATE TABLE IF NOT EXISTS fleet_cells ( host_port INTEGER NOT NULL, container_name TEXT NOT NULL, data_dir TEXT NOT NULL -);\n`; +) STRICT;\n`; diff --git a/src/state/openclaw-state-schema.sql b/src/state/openclaw-state-schema.sql index 0906fbd5df8..ea909114a5d 100644 --- a/src/state/openclaw-state-schema.sql +++ b/src/state/openclaw-state-schema.sql @@ -2,13 +2,13 @@ CREATE TABLE IF NOT EXISTS auth_profile_stores ( store_key TEXT NOT NULL PRIMARY KEY, store_json TEXT NOT NULL, updated_at INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS auth_profile_state ( store_key TEXT NOT NULL PRIMARY KEY, state_json TEXT NOT NULL, updated_at INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS diagnostic_events ( scope TEXT NOT NULL, @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS diagnostic_events ( payload_json TEXT NOT NULL, created_at INTEGER NOT NULL, PRIMARY KEY (scope, event_key) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_diagnostic_events_scope_created ON diagnostic_events(scope, created_at, event_key); @@ -30,7 +30,7 @@ CREATE TABLE IF NOT EXISTS skill_usage ( last_used_at_ms INTEGER NOT NULL, use_count INTEGER NOT NULL, last_agent_id TEXT -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_skill_usage_key ON skill_usage(skill_key, skill_file); @@ -44,7 +44,7 @@ CREATE TABLE IF NOT EXISTS skill_lifecycle ( state_changed_at_ms INTEGER NOT NULL, created_at_ms INTEGER NOT NULL, archived_reason TEXT -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_skill_lifecycle_key ON skill_lifecycle(skill_key, skill_file); @@ -58,7 +58,7 @@ CREATE TABLE IF NOT EXISTS skill_curator_state ( last_success_at_ms INTEGER, last_error TEXT, last_result_json TEXT NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS audit_events ( sequence INTEGER PRIMARY KEY AUTOINCREMENT, @@ -92,7 +92,7 @@ CREATE TABLE IF NOT EXISTS audit_events ( conversation_ref TEXT, message_ref TEXT, target_ref TEXT -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_audit_events_time ON audit_events(occurred_at DESC, sequence DESC); @@ -123,7 +123,7 @@ CREATE TABLE IF NOT EXISTS audit_identity_keys ( key_id TEXT NOT NULL, key BLOB NOT NULL, created_at INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS session_state_events ( sequence INTEGER PRIMARY KEY AUTOINCREMENT, @@ -138,7 +138,7 @@ CREATE TABLE IF NOT EXISTS session_state_events ( occurred_at INTEGER NOT NULL, summary TEXT NOT NULL, payload_json TEXT -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_session_state_events_session_sequence ON session_state_events(session_key, sequence DESC); @@ -153,7 +153,7 @@ CREATE TABLE IF NOT EXISTS session_state_heads ( pruned_max_sequence INTEGER NOT NULL DEFAULT 0, updated_at INTEGER NOT NULL, PRIMARY KEY (session_key, agent_id) -); +) STRICT; -- Watcher identity is the bare session key, matching the process-local system-event -- queue it feeds. Producers only create rows for agent-qualified watcher keys; @@ -167,7 +167,7 @@ CREATE TABLE IF NOT EXISTS session_watch_cursors ( material_sequence INTEGER NOT NULL DEFAULT 0, updated_at INTEGER NOT NULL, PRIMARY KEY (watcher_session_key, target_session_key) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_session_watch_cursors_target ON session_watch_cursors(target_session_key); @@ -187,7 +187,7 @@ CREATE TABLE IF NOT EXISTS session_upstream_links ( -- (session_key, agent_id) composite identity: under session.scope="global" agents -- share bare keys; a key-only row would let one agent overwrite another's upstream. PRIMARY KEY (session_key, agent_id) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_session_upstream_links_catalog_id ON session_upstream_links(catalog_id); @@ -198,7 +198,7 @@ CREATE TABLE IF NOT EXISTS diagnostic_stability_bundles ( generated_at TEXT NOT NULL, bundle_json TEXT NOT NULL, created_at INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_diagnostic_stability_bundles_created ON diagnostic_stability_bundles(created_at DESC, bundle_key); @@ -213,7 +213,7 @@ CREATE TABLE IF NOT EXISTS state_leases ( created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, PRIMARY KEY (scope, lease_key) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_state_leases_expiry ON state_leases(expires_at, scope, lease_key) @@ -234,7 +234,7 @@ CREATE TABLE IF NOT EXISTS exec_approvals_config ( agent_count INTEGER NOT NULL, allowlist_count INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS operator_approvals ( approval_id TEXT NOT NULL PRIMARY KEY CHECK ( @@ -340,7 +340,7 @@ CREATE TABLE IF NOT EXISTS operator_approvals ( AND consumed_by IS NOT NULL ) ) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_operator_approvals_status_expiry ON operator_approvals(status, expires_at_ms, approval_id); @@ -367,7 +367,7 @@ CREATE TABLE IF NOT EXISTS schema_meta ( app_version TEXT, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS device_pairing_pending ( request_id TEXT NOT NULL PRIMARY KEY, @@ -386,7 +386,7 @@ CREATE TABLE IF NOT EXISTS device_pairing_pending ( is_repair INTEGER, ts INTEGER NOT NULL, refreshed_at_ms INTEGER -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_device_pairing_pending_device ON device_pairing_pending(device_id, ts DESC); @@ -413,7 +413,7 @@ CREATE TABLE IF NOT EXISTS device_pairing_paired ( approved_at_ms INTEGER NOT NULL, last_seen_at_ms INTEGER, last_seen_reason TEXT -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_device_pairing_paired_approved ON device_pairing_paired(approved_at_ms DESC, device_id); @@ -429,7 +429,7 @@ CREATE TABLE IF NOT EXISTS device_bootstrap_tokens ( pending_profile_json TEXT, issued_at_ms INTEGER NOT NULL, last_used_at_ms INTEGER -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_device_bootstrap_tokens_ts ON device_bootstrap_tokens(ts); @@ -441,7 +441,7 @@ CREATE TABLE IF NOT EXISTS device_identities ( private_key_pem TEXT NOT NULL, created_at_ms INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_device_identities_device ON device_identities(device_id, updated_at_ms DESC); @@ -453,7 +453,7 @@ CREATE TABLE IF NOT EXISTS device_auth_tokens ( scopes_json TEXT NOT NULL, updated_at_ms INTEGER NOT NULL, PRIMARY KEY (device_id, role) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_device_auth_tokens_updated ON device_auth_tokens(updated_at_ms DESC, device_id, role); @@ -462,7 +462,7 @@ CREATE TABLE IF NOT EXISTS android_notification_recent_packages ( package_name TEXT NOT NULL PRIMARY KEY, sort_order INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_android_notification_recent_packages_order ON android_notification_recent_packages(sort_order, package_name); @@ -473,7 +473,7 @@ CREATE TABLE IF NOT EXISTS macos_port_guardian_records ( command TEXT NOT NULL, mode TEXT NOT NULL, timestamp REAL NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_macos_port_guardian_records_port ON macos_port_guardian_records(port, timestamp DESC); @@ -485,7 +485,7 @@ CREATE TABLE IF NOT EXISTS workspace_setup_state ( bootstrap_seeded_at TEXT, setup_completed_at TEXT, updated_at INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_workspace_setup_state_path ON workspace_setup_state(workspace_path); @@ -498,7 +498,7 @@ CREATE TABLE IF NOT EXISTS native_hook_relay_bridges ( token TEXT NOT NULL, expires_at_ms INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_native_hook_relay_bridges_expires ON native_hook_relay_bridges(expires_at_ms, relay_id); @@ -519,7 +519,7 @@ CREATE TABLE IF NOT EXISTS model_capability_cache ( cost_cache_write REAL NOT NULL, updated_at_ms INTEGER NOT NULL, PRIMARY KEY (provider_id, model_id) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_model_capability_cache_provider_updated ON model_capability_cache(provider_id, updated_at_ms DESC, model_id); @@ -529,7 +529,7 @@ CREATE TABLE IF NOT EXISTS agent_model_catalogs ( agent_dir TEXT NOT NULL, raw_json TEXT NOT NULL, updated_at INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_agent_model_catalogs_agent_dir ON agent_model_catalogs(agent_dir, updated_at DESC); @@ -553,7 +553,7 @@ CREATE TABLE IF NOT EXISTS managed_outgoing_image_records ( original_filename TEXT, record_json TEXT NOT NULL, cleanup_pending INTEGER NOT NULL DEFAULT 0 CHECK (cleanup_pending IN (0, 1)) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_managed_outgoing_images_session ON managed_outgoing_image_records(session_key, created_at DESC, attachment_id); @@ -578,7 +578,7 @@ CREATE TABLE IF NOT EXISTS channel_pairing_requests ( last_seen_at TEXT NOT NULL, meta_json TEXT, PRIMARY KEY (channel_key, account_id, request_id) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_channel_pairing_requests_code ON channel_pairing_requests(channel_key, code); @@ -593,7 +593,7 @@ CREATE TABLE IF NOT EXISTS channel_pairing_allow_entries ( sort_order INTEGER NOT NULL, updated_at INTEGER NOT NULL, PRIMARY KEY (channel_key, account_id, entry) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_channel_pairing_allow_account ON channel_pairing_allow_entries(channel_key, account_id, sort_order, entry); @@ -606,7 +606,7 @@ CREATE TABLE IF NOT EXISTS web_push_subscriptions ( auth TEXT NOT NULL, created_at_ms INTEGER NOT NULL, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_web_push_subscriptions_updated ON web_push_subscriptions(updated_at_ms DESC, subscription_id); @@ -617,7 +617,7 @@ CREATE TABLE IF NOT EXISTS web_push_vapid_keys ( private_key TEXT NOT NULL, subject TEXT NOT NULL, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS apns_registrations ( node_id TEXT NOT NULL PRIMARY KEY, @@ -631,7 +631,7 @@ CREATE TABLE IF NOT EXISTS apns_registrations ( distribution TEXT, token_debug_suffix TEXT, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_apns_registrations_updated ON apns_registrations(updated_at_ms DESC, node_id); @@ -648,7 +648,7 @@ CREATE TABLE IF NOT EXISTS node_host_config ( gateway_tls_fingerprint TEXT, gateway_context_path TEXT, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS voicewake_triggers ( config_key TEXT NOT NULL, @@ -656,7 +656,7 @@ CREATE TABLE IF NOT EXISTS voicewake_triggers ( trigger TEXT NOT NULL, updated_at_ms INTEGER NOT NULL, PRIMARY KEY (config_key, position) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_voicewake_triggers_trigger ON voicewake_triggers(config_key, trigger); @@ -668,7 +668,7 @@ CREATE TABLE IF NOT EXISTS voicewake_routing_config ( default_target_agent_id TEXT, default_target_session_key TEXT, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS voicewake_routing_routes ( config_key TEXT NOT NULL, @@ -680,7 +680,7 @@ CREATE TABLE IF NOT EXISTS voicewake_routing_routes ( updated_at_ms INTEGER NOT NULL, PRIMARY KEY (config_key, position), FOREIGN KEY (config_key) REFERENCES voicewake_routing_config(config_key) ON DELETE CASCADE -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_voicewake_routing_routes_trigger ON voicewake_routing_routes(config_key, trigger); @@ -701,7 +701,7 @@ CREATE TABLE IF NOT EXISTS update_check_state ( auto_last_success_version TEXT, auto_last_success_at TEXT, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS config_health_entries ( config_path TEXT NOT NULL PRIMARY KEY, @@ -709,7 +709,7 @@ CREATE TABLE IF NOT EXISTS config_health_entries ( last_promoted_good_json TEXT, last_observed_suspicious_signature TEXT, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS clawhub_promotions_feed_state ( state_key TEXT NOT NULL PRIMARY KEY, @@ -719,7 +719,7 @@ CREATE TABLE IF NOT EXISTS clawhub_promotions_feed_state ( last_checked_at_ms INTEGER, notified_slugs_json TEXT NOT NULL DEFAULT '[]', updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS clawhub_promotion_claims ( slug TEXT NOT NULL PRIMARY KEY, @@ -727,7 +727,7 @@ CREATE TABLE IF NOT EXISTS clawhub_promotion_claims ( model_keys_json TEXT NOT NULL, ends_at_ms INTEGER NOT NULL, claimed_at_ms INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS installed_plugin_index ( index_key TEXT NOT NULL PRIMARY KEY, @@ -743,7 +743,7 @@ CREATE TABLE IF NOT EXISTS installed_plugin_index ( diagnostics_json TEXT NOT NULL, warning TEXT, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_installed_plugin_index_generated ON installed_plugin_index(generated_at_ms DESC, index_key); @@ -762,7 +762,7 @@ CREATE TABLE IF NOT EXISTS official_external_plugin_catalog_snapshots ( trust_threshold INTEGER, trust_verified_at TEXT, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_official_external_plugin_catalog_snapshots_updated ON official_external_plugin_catalog_snapshots(updated_at_ms DESC, feed_url); @@ -784,7 +784,7 @@ CREATE TABLE IF NOT EXISTS gateway_restart_sentinel ( stats_json TEXT, payload_json TEXT NOT NULL, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_gateway_restart_sentinel_ts ON gateway_restart_sentinel(ts DESC, sentinel_key); @@ -798,7 +798,7 @@ CREATE TABLE IF NOT EXISTS gateway_restart_intent ( force INTEGER, wait_ms INTEGER, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS gateway_restart_handoff ( handoff_key TEXT NOT NULL PRIMARY KEY, @@ -816,7 +816,7 @@ CREATE TABLE IF NOT EXISTS gateway_restart_handoff ( restart_kind TEXT NOT NULL, supervisor_mode TEXT NOT NULL, updated_at_ms INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_gateway_restart_handoff_expiry ON gateway_restart_handoff(expires_at, pid); @@ -829,7 +829,7 @@ CREATE TABLE IF NOT EXISTS gateway_boot_lifecycle ( outcome TEXT, startup_reason TEXT, reason TEXT -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_gateway_boot_lifecycle_started ON gateway_boot_lifecycle(started_at_ms); @@ -848,7 +848,7 @@ CREATE TABLE IF NOT EXISTS acp_sessions ( last_activity_at INTEGER NOT NULL, last_error TEXT, updated_at INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_acp_sessions_state_activity ON acp_sessions(state, last_activity_at DESC, session_key); @@ -868,7 +868,7 @@ CREATE TABLE IF NOT EXISTS acp_replay_sessions ( -- all event rows), maintained at insert/trim so budget checks never scan -- acp_replay_events (#100622). estimated_bytes INTEGER NOT NULL DEFAULT 0 -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_acp_replay_sessions_key_updated ON acp_replay_sessions(session_key, complete, updated_at DESC, session_id); @@ -886,7 +886,7 @@ CREATE TABLE IF NOT EXISTS acp_replay_events ( estimated_bytes INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (session_id, seq), FOREIGN KEY (session_id) REFERENCES acp_replay_sessions(session_id) ON DELETE CASCADE -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_acp_replay_events_session_seq ON acp_replay_events(session_id, seq); @@ -898,7 +898,7 @@ CREATE TABLE IF NOT EXISTS agent_databases ( last_seen_at INTEGER NOT NULL, size_bytes INTEGER, PRIMARY KEY (agent_id, path) -); +) STRICT; CREATE TABLE IF NOT EXISTS plugin_state_entries ( plugin_id TEXT NOT NULL, @@ -908,7 +908,7 @@ CREATE TABLE IF NOT EXISTS plugin_state_entries ( created_at INTEGER NOT NULL, expires_at INTEGER, PRIMARY KEY (plugin_id, namespace, entry_key) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_plugin_state_expiry ON plugin_state_entries(expires_at) @@ -939,7 +939,7 @@ CREATE TABLE IF NOT EXISTS channel_ingress_events ( completed_at INTEGER, completed_metadata_json TEXT, PRIMARY KEY (queue_name, event_id) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_channel_ingress_pending ON channel_ingress_events(queue_name, status, received_at, event_id); @@ -959,7 +959,7 @@ CREATE TABLE IF NOT EXISTS plugin_blob_entries ( created_at INTEGER NOT NULL, expires_at INTEGER, PRIMARY KEY (plugin_id, namespace, entry_key) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_plugin_blob_expiry ON plugin_blob_entries(expires_at) @@ -977,7 +977,7 @@ CREATE TABLE IF NOT EXISTS media_blobs ( created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, PRIMARY KEY (subdir, id) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_media_blobs_created ON media_blobs(created_at); @@ -997,7 +997,7 @@ CREATE TABLE IF NOT EXISTS skill_uploads ( committed INTEGER NOT NULL, committed_at INTEGER, idempotency_key_hash TEXT UNIQUE -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_skill_uploads_expiry ON skill_uploads(expires_at); @@ -1014,7 +1014,7 @@ CREATE TABLE IF NOT EXISTS skill_upload_chunks ( PRIMARY KEY (upload_id, byte_offset), FOREIGN KEY (upload_id) REFERENCES skill_uploads(upload_id) ON DELETE CASCADE, CHECK (length(chunk_blob) = size_bytes) -); +) STRICT; CREATE TABLE IF NOT EXISTS capture_sessions ( id TEXT NOT NULL PRIMARY KEY, @@ -1024,7 +1024,7 @@ CREATE TABLE IF NOT EXISTS capture_sessions ( source_scope TEXT NOT NULL, source_process TEXT NOT NULL, proxy_url TEXT -); +) STRICT; CREATE TABLE IF NOT EXISTS capture_blobs ( blob_id TEXT NOT NULL PRIMARY KEY, @@ -1034,7 +1034,7 @@ CREATE TABLE IF NOT EXISTS capture_blobs ( sha256 TEXT NOT NULL, data BLOB NOT NULL, created_at INTEGER NOT NULL -); +) STRICT; CREATE TABLE IF NOT EXISTS capture_events ( id INTEGER NOT NULL PRIMARY KEY, @@ -1060,7 +1060,7 @@ CREATE TABLE IF NOT EXISTS capture_events ( meta_json TEXT, FOREIGN KEY (session_id) REFERENCES capture_sessions(id) ON DELETE CASCADE, FOREIGN KEY (data_blob_id) REFERENCES capture_blobs(blob_id) ON DELETE SET NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS capture_events_session_ts_idx ON capture_events(session_id, ts); @@ -1084,7 +1084,7 @@ CREATE TABLE IF NOT EXISTS sandbox_registry_entries ( entry_json TEXT NOT NULL, updated_at INTEGER NOT NULL, PRIMARY KEY (registry_kind, container_name) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_sandbox_registry_updated ON sandbox_registry_entries(registry_kind, updated_at DESC, container_name); @@ -1128,7 +1128,7 @@ CREATE TABLE IF NOT EXISTS commitments ( snoozed_until_ms INTEGER, expired_at_ms INTEGER, record_json TEXT NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_commitments_scope_due ON commitments(agent_id, session_key, status, due_earliest_ms, due_latest_ms); @@ -1222,7 +1222,7 @@ CREATE TABLE IF NOT EXISTS cron_jobs ( sort_order INTEGER NOT NULL DEFAULT 0, updated_at INTEGER NOT NULL, PRIMARY KEY (store_key, job_id) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_cron_jobs_store_updated ON cron_jobs(store_key, sort_order ASC, updated_at DESC, job_id); @@ -1246,7 +1246,7 @@ CREATE TABLE IF NOT EXISTS command_log_entries ( sender_id TEXT NOT NULL, source TEXT NOT NULL, entry_json TEXT NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_command_log_entries_timestamp ON command_log_entries(timestamp_ms DESC, id); @@ -1273,7 +1273,7 @@ CREATE TABLE IF NOT EXISTS delivery_queue_entries ( updated_at INTEGER NOT NULL, failed_at INTEGER, PRIMARY KEY (queue_name, id) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_delivery_queue_pending ON delivery_queue_entries(queue_name, status, enqueued_at, id); @@ -1320,7 +1320,7 @@ CREATE TABLE IF NOT EXISTS task_runs ( terminal_summary TEXT, terminal_outcome TEXT, detail_json TEXT -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_task_runs_run_id ON task_runs(run_id); CREATE INDEX IF NOT EXISTS idx_task_runs_status ON task_runs(status); @@ -1381,7 +1381,7 @@ CREATE TABLE IF NOT EXISTS subagent_runs ( pending_final_delivery_payload_json TEXT, completion_announced_at INTEGER, payload_json TEXT NOT NULL DEFAULT '{}' -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_subagent_runs_child_session_key ON subagent_runs(child_session_key, created_at DESC, run_id); @@ -1412,7 +1412,7 @@ CREATE TABLE IF NOT EXISTS current_conversation_bindings ( metadata_json TEXT, record_json TEXT NOT NULL, updated_at INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_current_conversation_bindings_target ON current_conversation_bindings(target_agent_id, target_session_key, updated_at DESC, binding_key); @@ -1429,7 +1429,7 @@ CREATE TABLE IF NOT EXISTS plugin_binding_approvals ( plugin_name TEXT, approved_at INTEGER NOT NULL, PRIMARY KEY (plugin_root, channel, account_id) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_plugin_binding_approvals_plugin ON plugin_binding_approvals(plugin_id, approved_at DESC); @@ -1438,7 +1438,7 @@ CREATE TABLE IF NOT EXISTS tui_last_sessions ( scope_key TEXT NOT NULL PRIMARY KEY, session_key TEXT NOT NULL, updated_at INTEGER NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_tui_last_sessions_session_key ON tui_last_sessions(session_key, updated_at DESC, scope_key); @@ -1448,7 +1448,7 @@ CREATE TABLE IF NOT EXISTS task_delivery_state ( requester_origin_json TEXT, last_notified_event_at INTEGER, FOREIGN KEY (task_id) REFERENCES task_runs(task_id) ON DELETE CASCADE -); +) STRICT; CREATE TABLE IF NOT EXISTS flow_runs ( flow_id TEXT NOT NULL PRIMARY KEY, @@ -1470,7 +1470,7 @@ CREATE TABLE IF NOT EXISTS flow_runs ( created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL, ended_at INTEGER -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_flow_runs_status ON flow_runs(status); CREATE INDEX IF NOT EXISTS idx_flow_runs_owner_key ON flow_runs(owner_key); @@ -1482,7 +1482,7 @@ CREATE TABLE IF NOT EXISTS migration_runs ( finished_at INTEGER, status TEXT NOT NULL, report_json TEXT NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_migration_runs_started ON migration_runs(started_at DESC, id); @@ -1501,7 +1501,7 @@ CREATE TABLE IF NOT EXISTS migration_sources ( removed_source INTEGER NOT NULL DEFAULT 0, report_json TEXT NOT NULL, FOREIGN KEY (last_run_id) REFERENCES migration_runs(id) ON DELETE CASCADE -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_migration_sources_path ON migration_sources(source_path, migration_kind, target_table); @@ -1515,7 +1515,7 @@ CREATE TABLE IF NOT EXISTS backup_runs ( archive_path TEXT NOT NULL, status TEXT NOT NULL, manifest_json TEXT NOT NULL -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_backup_runs_created ON backup_runs(created_at DESC, id); @@ -1533,7 +1533,7 @@ CREATE TABLE IF NOT EXISTS worktrees ( created_at INTEGER NOT NULL, last_active_at INTEGER NOT NULL, removed_at INTEGER -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_worktrees_repo_fingerprint ON worktrees(repo_fingerprint); @@ -1548,7 +1548,7 @@ CREATE TABLE IF NOT EXISTS session_groups ( name TEXT NOT NULL PRIMARY KEY, position INTEGER NOT NULL, created_at INTEGER NOT NULL -); +) STRICT; -- Gateway-owned durable cloud worker lifecycle. Provider-specific execution -- stays in plugins; this table records only core reconciliation facts. @@ -1591,7 +1591,7 @@ CREATE TABLE IF NOT EXISTS worker_environments ( idle_since_at_ms INTEGER, destroy_requested_at_ms INTEGER, last_error TEXT -); +) STRICT; CREATE UNIQUE INDEX IF NOT EXISTS idx_worker_environments_provider_lease ON worker_environments(provider_id, lease_id) @@ -1705,7 +1705,7 @@ CREATE TABLE IF NOT EXISTS worker_session_placements ( (turn_claim_owner IS 'worker' AND state IN ('active', 'draining') AND turn_claim_owner_epoch IS active_owner_epoch) ) -); +) STRICT; CREATE INDEX IF NOT EXISTS idx_worker_session_placements_session_key ON worker_session_placements(agent_id, session_key); @@ -1726,7 +1726,7 @@ CREATE TABLE IF NOT EXISTS worker_workspace_reconciliations ( base_pack BLOB NOT NULL CHECK (length(base_pack) <= 268435456), created_at_ms INTEGER NOT NULL, FOREIGN KEY (session_id) REFERENCES worker_session_placements(session_id) ON DELETE CASCADE -); +) STRICT; -- A completed remote turn is fenced from stale-claim teardown until its -- workspace result is durably reconciled into the managed worktree. @@ -1742,7 +1742,7 @@ CREATE TABLE IF NOT EXISTS worker_workspace_pending_results ( workspace_accepted_at_ms INTEGER, created_at_ms INTEGER NOT NULL, FOREIGN KEY (session_id) REFERENCES worker_session_placements(session_id) ON DELETE CASCADE -); +) STRICT; -- One active, opaque admission credential per worker environment. Plaintext -- may be retried until delivery acknowledgement but never enters durable state. @@ -1756,7 +1756,7 @@ CREATE TABLE IF NOT EXISTS worker_environment_credentials ( expires_at_ms INTEGER NOT NULL CHECK (expires_at_ms >= 0), delivered_at_ms INTEGER CHECK (delivered_at_ms >= 0), FOREIGN KEY (environment_id) REFERENCES worker_environments(environment_id) ON DELETE CASCADE -); +) STRICT; -- One durable sequence cursor per attached session owner epoch. The environment -- binding prevents independent workers with coincident epochs from sharing replay state. @@ -1767,7 +1767,7 @@ CREATE TABLE IF NOT EXISTS worker_transcript_commit_heads ( next_seq INTEGER NOT NULL CHECK (next_seq >= 1), updated_at_ms INTEGER NOT NULL CHECK (updated_at_ms >= 0), PRIMARY KEY (session_id, run_epoch) -); +) STRICT; -- Pending rows preserve a claimed request across gateway restarts. Terminal rows -- cache the exact result returned for deterministic at-least-once replay. @@ -1788,7 +1788,7 @@ CREATE TABLE IF NOT EXISTS worker_transcript_commits ( (state = 'pending' AND result_json IS NULL) OR (state = 'terminal' AND result_json IS NOT NULL) ) -); +) STRICT; -- Pending rows preserve a claimed inference turn across gateway restarts. -- Terminal rows cache the exact outcome returned for deterministic replay. @@ -1809,7 +1809,7 @@ CREATE TABLE IF NOT EXISTS worker_inference_turns ( (state = 'pending' AND terminal_json IS NULL) OR (state = 'terminal' AND terminal_json IS NOT NULL) ) -); +) STRICT; CREATE UNIQUE INDEX IF NOT EXISTS idx_worker_inference_turns_pending_run ON worker_inference_turns(session_id, run_epoch, run_id) @@ -1823,4 +1823,4 @@ CREATE TABLE IF NOT EXISTS fleet_cells ( host_port INTEGER NOT NULL, container_name TEXT NOT NULL, data_dir TEXT NOT NULL -); +) STRICT; diff --git a/src/state/sqlite-schema-shape.test-support.ts b/src/state/sqlite-schema-shape.test-support.ts index 684c5aa6e4a..b2ecb4a164e 100644 --- a/src/state/sqlite-schema-shape.test-support.ts +++ b/src/state/sqlite-schema-shape.test-support.ts @@ -38,6 +38,7 @@ type SqliteSchemaShape = Record< { columns: ColumnShape[]; indexes: IndexShape[]; + strict: number; } >; @@ -100,11 +101,22 @@ export function collectSqliteSchemaShape(db: DatabaseSync): SqliteSchemaShape { { columns: collectColumns(db, table.name), indexes: collectIndexes(db, table.name), + strict: collectStrictFlag(db, table.name), }, ]), ); } +function collectStrictFlag(db: DatabaseSync, tableName: string): number { + const row = db + .prepare("SELECT strict FROM pragma_table_list WHERE schema = 'main' AND name = ?") + .get(tableName) as { strict?: unknown } | undefined; + if (typeof row?.strict !== "number") { + throw new Error(`SQLite table ${tableName} has no table_list entry`); + } + return row.strict; +} + function collectColumns(db: DatabaseSync, tableName: string): ColumnShape[] { return ( db.prepare(`PRAGMA table_info(${quoteSqliteIdentifier(tableName)})`).all() as TableInfoRow[]