mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 10:16:44 +00:00
fix(ci): drop unreferenced snapshot repository module
src/snapshot/{local-repository,manifest,snapshot-provider}.ts landed in #105525
with no production caller (only their own test), which fails the
check-dependencies unused-files gate and the runtime topology lane on main and
on every PR, blocking all landings. src/infra/sqlite-snapshot.ts stays: it
predates the PR and is used by backup-create. The module can reland together
with the code that wires it in.
This commit is contained in:
Peter Steinberger
@@ -1,958 +0,0 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { requireNodeSqlite } from "../infra/node-sqlite.js";
|
||||
import { OPENCLAW_AGENT_SCHEMA_VERSION } from "../state/openclaw-agent-db.js";
|
||||
import { OPENCLAW_STATE_SCHEMA_VERSION } from "../state/openclaw-state-db.js";
|
||||
import { createLocalSqliteSnapshotProvider } from "./local-repository.js";
|
||||
import { hashSnapshotArtifact, parseSnapshotManifest, readSnapshotManifest } from "./manifest.js";
|
||||
import {
|
||||
SNAPSHOT_MANIFEST_FILENAME,
|
||||
SNAPSHOT_SQLITE_FILENAME,
|
||||
type SnapshotManifest,
|
||||
type SnapshotResult,
|
||||
} from "./snapshot-provider.js";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
async function createTempDir(): Promise<string> {
|
||||
const tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-snapshot-repository-"));
|
||||
tempDirs.push(tempDir);
|
||||
return tempDir;
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(tempDirs.splice(0).map((tempDir) => fs.rm(tempDir, { recursive: true })));
|
||||
});
|
||||
|
||||
function createGenericDatabase(
|
||||
databasePath: string,
|
||||
options: { userVersion?: number; values?: string[]; wal?: boolean } = {},
|
||||
): void {
|
||||
const sqlite = requireNodeSqlite();
|
||||
const database = new sqlite.DatabaseSync(databasePath);
|
||||
try {
|
||||
database.exec(`
|
||||
${options.wal ? "PRAGMA journal_mode = WAL; PRAGMA wal_autocheckpoint = 0;" : ""}
|
||||
PRAGMA user_version = ${options.userVersion ?? 7};
|
||||
CREATE TABLE entries (
|
||||
id INTEGER PRIMARY KEY,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
`);
|
||||
const insert = database.prepare("INSERT INTO entries (value) VALUES (?)");
|
||||
for (const value of options.values ?? ["one"]) {
|
||||
insert.run(value);
|
||||
}
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
}
|
||||
|
||||
function createGlobalDatabase(databasePath: string): void {
|
||||
const sqlite = requireNodeSqlite();
|
||||
const database = new sqlite.DatabaseSync(databasePath);
|
||||
try {
|
||||
database.exec(`
|
||||
PRAGMA user_version = ${OPENCLAW_STATE_SCHEMA_VERSION};
|
||||
CREATE TABLE schema_meta (
|
||||
meta_key TEXT PRIMARY KEY,
|
||||
role TEXT NOT NULL,
|
||||
schema_version INTEGER NOT NULL
|
||||
);
|
||||
INSERT INTO schema_meta VALUES ('primary', 'global', ${OPENCLAW_STATE_SCHEMA_VERSION});
|
||||
CREATE TABLE delivery_queue_entries (
|
||||
id TEXT PRIMARY KEY,
|
||||
payload TEXT NOT NULL
|
||||
);
|
||||
INSERT INTO delivery_queue_entries VALUES ('queued', 'do-not-restore');
|
||||
`);
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
}
|
||||
|
||||
function createAgentDatabase(databasePath: string, agentId: string): void {
|
||||
const sqlite = requireNodeSqlite();
|
||||
const database = new sqlite.DatabaseSync(databasePath);
|
||||
try {
|
||||
database.exec(`
|
||||
PRAGMA user_version = ${OPENCLAW_AGENT_SCHEMA_VERSION};
|
||||
CREATE TABLE schema_meta (
|
||||
meta_key TEXT PRIMARY KEY,
|
||||
role TEXT NOT NULL,
|
||||
schema_version INTEGER NOT NULL,
|
||||
agent_id TEXT
|
||||
);
|
||||
`);
|
||||
database
|
||||
.prepare("INSERT INTO schema_meta VALUES ('primary', 'agent', ?, ?)")
|
||||
.run(OPENCLAW_AGENT_SCHEMA_VERSION, agentId);
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
}
|
||||
|
||||
function createUnsafeIndexDrift(databasePath: string): void {
|
||||
const sqlite = requireNodeSqlite();
|
||||
const database = new sqlite.DatabaseSync(databasePath);
|
||||
try {
|
||||
database.exec(`
|
||||
CREATE TABLE records (
|
||||
id INTEGER PRIMARY KEY,
|
||||
indexed_value TEXT NOT NULL,
|
||||
alternate_value TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX records_value ON records(indexed_value);
|
||||
INSERT INTO records (indexed_value, alternate_value)
|
||||
VALUES ('alpha', 'zeta'), ('beta', 'eta'), ('gamma', 'theta');
|
||||
PRAGMA writable_schema = ON;
|
||||
`);
|
||||
database
|
||||
.prepare(
|
||||
"UPDATE sqlite_schema SET sql = 'CREATE INDEX records_value ON records(alternate_value)' WHERE name = 'records_value'",
|
||||
)
|
||||
.run();
|
||||
const schemaVersion = Number(
|
||||
Object.values(database.prepare("PRAGMA schema_version").get() as Record<string, unknown>)[0],
|
||||
);
|
||||
database.exec(`PRAGMA writable_schema = OFF; PRAGMA schema_version = ${schemaVersion + 1};`);
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function rewriteManifest(
|
||||
result: SnapshotResult,
|
||||
mutate: (manifest: SnapshotManifest) => SnapshotManifest,
|
||||
): Promise<void> {
|
||||
const manifestPath = path.join(result.ref.path, SNAPSHOT_MANIFEST_FILENAME);
|
||||
const manifest = await readSnapshotManifest(result.ref.path);
|
||||
await fs.writeFile(manifestPath, `${JSON.stringify(mutate(manifest), null, 2)}\n`);
|
||||
}
|
||||
|
||||
async function refreshArtifactManifest(result: SnapshotResult): Promise<void> {
|
||||
const digest = await hashSnapshotArtifact(result.ref.path);
|
||||
await rewriteManifest(result, (manifest) => ({
|
||||
...manifest,
|
||||
artifact: {
|
||||
...manifest.artifact,
|
||||
sha256: digest.sha256,
|
||||
sizeBytes: digest.sizeBytes,
|
||||
},
|
||||
}));
|
||||
}
|
||||
|
||||
describe("local SQLite snapshot repository", () => {
|
||||
it("creates, lists, verifies, and fresh-restores committed WAL state", async () => {
|
||||
const tempDir = await createTempDir();
|
||||
const sourcePath = path.join(tempDir, "source.sqlite");
|
||||
const repositoryPath = path.join(tempDir, "snapshots");
|
||||
const restorePath = path.join(tempDir, "restore", "source.sqlite");
|
||||
const sqlite = requireNodeSqlite();
|
||||
const source = new sqlite.DatabaseSync(sourcePath);
|
||||
try {
|
||||
source.exec(`
|
||||
PRAGMA journal_mode = WAL;
|
||||
PRAGMA wal_autocheckpoint = 0;
|
||||
PRAGMA user_version = 42;
|
||||
CREATE TABLE entries (id INTEGER PRIMARY KEY, value TEXT NOT NULL);
|
||||
INSERT INTO entries (value) VALUES ('checkpointed');
|
||||
PRAGMA wal_checkpoint(TRUNCATE);
|
||||
INSERT INTO entries (value) VALUES ('committed-in-wal');
|
||||
`);
|
||||
const provider = createLocalSqliteSnapshotProvider({
|
||||
repositoryPath,
|
||||
now: () => new Date("2026-07-12T14:00:00.000Z"),
|
||||
});
|
||||
const snapshot = await provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "generic", id: "test-database" },
|
||||
});
|
||||
|
||||
expect(snapshot.manifest).toMatchObject({
|
||||
schemaVersion: 1,
|
||||
createdAt: "2026-07-12T14:00:00.000Z",
|
||||
database: {
|
||||
role: "generic",
|
||||
id: "test-database",
|
||||
basename: "source.sqlite",
|
||||
userVersion: 42,
|
||||
},
|
||||
artifact: {
|
||||
path: SNAPSHOT_SQLITE_FILENAME,
|
||||
},
|
||||
});
|
||||
expect(snapshot.manifest.artifact.sha256).toMatch(/^[a-f0-9]{64}$/u);
|
||||
await expect(provider.verify(snapshot.ref)).resolves.toEqual({
|
||||
ok: true,
|
||||
manifest: snapshot.manifest,
|
||||
});
|
||||
await expect(provider.list()).resolves.toEqual([snapshot]);
|
||||
await expect(provider.restoreFresh(snapshot.ref, restorePath)).resolves.toEqual({
|
||||
ok: true,
|
||||
manifest: snapshot.manifest,
|
||||
});
|
||||
await expect(fs.readFile(restorePath)).resolves.toEqual(
|
||||
await fs.readFile(path.join(snapshot.ref.path, SNAPSHOT_SQLITE_FILENAME)),
|
||||
);
|
||||
expect((await fs.readdir(repositoryPath)).every((name) => !name.startsWith(".tmp-"))).toBe(
|
||||
true,
|
||||
);
|
||||
await expect(fs.readdir(path.dirname(restorePath))).resolves.toEqual(["source.sqlite"]);
|
||||
} finally {
|
||||
source.close();
|
||||
}
|
||||
|
||||
const restored = new sqlite.DatabaseSync(restorePath, { readOnly: true });
|
||||
try {
|
||||
expect(restored.prepare("SELECT value FROM entries ORDER BY id").all()).toEqual([
|
||||
{ value: "checkpointed" },
|
||||
{ value: "committed-in-wal" },
|
||||
]);
|
||||
expect(restored.prepare("PRAGMA user_version").get()).toEqual({ user_version: 42 });
|
||||
} finally {
|
||||
restored.close();
|
||||
}
|
||||
if (process.platform !== "win32") {
|
||||
expect((await fs.stat(repositoryPath)).mode & 0o777).toBe(0o700);
|
||||
expect((await fs.stat(restorePath)).mode & 0o777).toBe(0o600);
|
||||
}
|
||||
});
|
||||
|
||||
it("sorts snapshots newest first and ignores incomplete staging directories", async () => {
|
||||
const tempDir = await createTempDir();
|
||||
const sourcePath = path.join(tempDir, "source.sqlite");
|
||||
const repositoryPath = path.join(tempDir, "snapshots");
|
||||
createGenericDatabase(sourcePath);
|
||||
const dates = [new Date("2026-07-12T14:00:00.000Z"), new Date("2026-07-12T14:01:00.000Z")];
|
||||
const provider = createLocalSqliteSnapshotProvider({
|
||||
repositoryPath,
|
||||
now: () => dates.shift() ?? new Date("invalid"),
|
||||
});
|
||||
const first = await provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "generic", id: "test-database" },
|
||||
});
|
||||
const second = await provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "generic", id: "test-database" },
|
||||
});
|
||||
await fs.mkdir(path.join(repositoryPath, ".tmp-interrupted"));
|
||||
await fs.mkdir(path.join(repositoryPath, "interrupted-final"));
|
||||
await fs.writeFile(path.join(repositoryPath, "interrupted-final", ".pending"), "");
|
||||
await fs.mkdir(path.join(repositoryPath, "empty-final"));
|
||||
|
||||
await expect(provider.list()).resolves.toEqual([second, first]);
|
||||
});
|
||||
|
||||
it("never replaces a snapshot directory raced into place", async () => {
|
||||
const tempDir = await createTempDir();
|
||||
const sourcePath = path.join(tempDir, "source.sqlite");
|
||||
const repositoryPath = path.join(tempDir, "snapshots");
|
||||
createGenericDatabase(sourcePath);
|
||||
const provider = createLocalSqliteSnapshotProvider({
|
||||
repositoryPath,
|
||||
now: () => new Date("2026-07-12T14:00:00.000Z"),
|
||||
});
|
||||
const originalMkdir = fs.mkdir.bind(fs);
|
||||
let racedPath: string | undefined;
|
||||
const mkdirSpy = vi.spyOn(fs, "mkdir").mockImplementation(async (directoryPath, options) => {
|
||||
const resolvedPath = path.resolve(String(directoryPath));
|
||||
if (
|
||||
path.dirname(resolvedPath) === repositoryPath &&
|
||||
!path.basename(resolvedPath).startsWith(".tmp-")
|
||||
) {
|
||||
racedPath = resolvedPath;
|
||||
await originalMkdir(resolvedPath, options);
|
||||
await fs.writeFile(path.join(resolvedPath, "keep"), "racer");
|
||||
}
|
||||
return await originalMkdir(directoryPath, options);
|
||||
});
|
||||
|
||||
try {
|
||||
await expect(
|
||||
provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "generic", id: "directory-race" },
|
||||
}),
|
||||
).rejects.toThrow(/directory already exists/u);
|
||||
} finally {
|
||||
mkdirSpy.mockRestore();
|
||||
}
|
||||
expect(racedPath).toBeDefined();
|
||||
await expect(fs.readFile(path.join(racedPath!, "keep"), "utf8")).resolves.toBe("racer");
|
||||
});
|
||||
|
||||
it("rejects an artifact changed after entering the final directory", async () => {
|
||||
const tempDir = await createTempDir();
|
||||
const sourcePath = path.join(tempDir, "source.sqlite");
|
||||
const repositoryPath = path.join(tempDir, "snapshots");
|
||||
createGenericDatabase(sourcePath);
|
||||
const provider = createLocalSqliteSnapshotProvider({ repositoryPath });
|
||||
const originalLink = fs.link.bind(fs);
|
||||
const linkSpy = vi.spyOn(fs, "link").mockImplementation(async (source, target) => {
|
||||
await originalLink(source, target);
|
||||
if (
|
||||
path.basename(String(target)) === SNAPSHOT_MANIFEST_FILENAME &&
|
||||
!path.basename(path.dirname(String(target))).startsWith(".tmp-")
|
||||
) {
|
||||
await fs.appendFile(
|
||||
path.join(path.dirname(String(target)), SNAPSHOT_SQLITE_FILENAME),
|
||||
"changed-after-final-move",
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
await expect(
|
||||
provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "generic", id: "final-directory-race" },
|
||||
}),
|
||||
).rejects.toThrow(/size mismatch/u);
|
||||
await expect(provider.list()).resolves.toEqual([]);
|
||||
} finally {
|
||||
linkSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("cleans a linked entry when post-link inspection fails", async () => {
|
||||
const tempDir = await createTempDir();
|
||||
const sourcePath = path.join(tempDir, "source.sqlite");
|
||||
const repositoryPath = path.join(tempDir, "snapshots");
|
||||
createGenericDatabase(sourcePath);
|
||||
const provider = createLocalSqliteSnapshotProvider({ repositoryPath });
|
||||
const originalLink = fs.link.bind(fs);
|
||||
const originalLstat = fs.lstat.bind(fs);
|
||||
let linkedArtifactPath: string | undefined;
|
||||
let failedInspection = false;
|
||||
const linkSpy = vi.spyOn(fs, "link").mockImplementation(async (source, target) => {
|
||||
await originalLink(source, target);
|
||||
if (
|
||||
path.basename(String(target)) === SNAPSHOT_SQLITE_FILENAME &&
|
||||
!path.basename(path.dirname(String(target))).startsWith(".tmp-")
|
||||
) {
|
||||
linkedArtifactPath = path.resolve(String(target));
|
||||
}
|
||||
});
|
||||
const lstatSpy = vi.spyOn(fs, "lstat").mockImplementation(async (filePath) => {
|
||||
if (
|
||||
linkedArtifactPath &&
|
||||
!failedInspection &&
|
||||
path.resolve(String(filePath)) === linkedArtifactPath
|
||||
) {
|
||||
failedInspection = true;
|
||||
throw Object.assign(new Error("post-link inspection failed"), { code: "EIO" });
|
||||
}
|
||||
return await originalLstat(filePath);
|
||||
});
|
||||
|
||||
try {
|
||||
await expect(
|
||||
provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "generic", id: "post-link-inspection" },
|
||||
}),
|
||||
).rejects.toThrow(/post-link inspection failed/u);
|
||||
await expect(provider.list()).resolves.toEqual([]);
|
||||
await expect(fs.readdir(repositoryPath)).resolves.toEqual([]);
|
||||
} finally {
|
||||
lstatSpy.mockRestore();
|
||||
linkSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("never overwrites a file raced into the final snapshot directory", async () => {
|
||||
const tempDir = await createTempDir();
|
||||
const sourcePath = path.join(tempDir, "source.sqlite");
|
||||
const repositoryPath = path.join(tempDir, "snapshots");
|
||||
createGenericDatabase(sourcePath);
|
||||
const provider = createLocalSqliteSnapshotProvider({ repositoryPath });
|
||||
const originalLink = fs.link.bind(fs);
|
||||
let racedPath: string | undefined;
|
||||
const linkSpy = vi.spyOn(fs, "link").mockImplementation(async (source, target) => {
|
||||
const targetPath = path.resolve(String(target));
|
||||
if (
|
||||
path.basename(targetPath) === SNAPSHOT_SQLITE_FILENAME &&
|
||||
path.dirname(targetPath) !== repositoryPath &&
|
||||
!path.basename(path.dirname(targetPath)).startsWith(".tmp-")
|
||||
) {
|
||||
racedPath = targetPath;
|
||||
await fs.writeFile(targetPath, "racer", { flag: "wx" });
|
||||
}
|
||||
await originalLink(source, target);
|
||||
});
|
||||
|
||||
try {
|
||||
await expect(
|
||||
provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "generic", id: "entry-race" },
|
||||
}),
|
||||
).rejects.toThrow(/EEXIST/u);
|
||||
} finally {
|
||||
linkSpy.mockRestore();
|
||||
}
|
||||
expect(racedPath).toBeDefined();
|
||||
await expect(fs.readFile(racedPath!, "utf8")).resolves.toBe("racer");
|
||||
});
|
||||
|
||||
it("sanitizes transient global delivery rows and enforces the global owner", async () => {
|
||||
const tempDir = await createTempDir();
|
||||
const sourcePath = path.join(tempDir, "openclaw.sqlite");
|
||||
const repositoryPath = path.join(tempDir, "snapshots");
|
||||
createGlobalDatabase(sourcePath);
|
||||
const provider = createLocalSqliteSnapshotProvider({ repositoryPath });
|
||||
const snapshot = await provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "global" },
|
||||
});
|
||||
const artifactPath = path.join(snapshot.ref.path, SNAPSHOT_SQLITE_FILENAME);
|
||||
expect((await fs.readFile(artifactPath)).includes("do-not-restore")).toBe(false);
|
||||
const sqlite = requireNodeSqlite();
|
||||
const artifact = new sqlite.DatabaseSync(artifactPath, { readOnly: true });
|
||||
try {
|
||||
expect(
|
||||
artifact.prepare("SELECT COUNT(*) AS count FROM delivery_queue_entries").get(),
|
||||
).toEqual({ count: 0 });
|
||||
} finally {
|
||||
artifact.close();
|
||||
}
|
||||
|
||||
const wrongRolePath = path.join(tempDir, "wrong-role.sqlite");
|
||||
createAgentDatabase(wrongRolePath, "main");
|
||||
const wrongRole = new sqlite.DatabaseSync(wrongRolePath);
|
||||
wrongRole.exec(`PRAGMA user_version = ${OPENCLAW_STATE_SCHEMA_VERSION};`);
|
||||
wrongRole.close();
|
||||
await expect(
|
||||
provider.create({ path: wrongRolePath, identity: { role: "global" } }),
|
||||
).rejects.toThrow(/expected global/u);
|
||||
});
|
||||
|
||||
it("enforces the exact agent owner and canonical agent id", async () => {
|
||||
const tempDir = await createTempDir();
|
||||
const sourcePath = path.join(tempDir, "openclaw-agent.sqlite");
|
||||
const repositoryPath = path.join(tempDir, "snapshots");
|
||||
createAgentDatabase(sourcePath, "worker-1");
|
||||
const provider = createLocalSqliteSnapshotProvider({ repositoryPath });
|
||||
|
||||
await expect(
|
||||
provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "agent", agentId: "worker-2" },
|
||||
}),
|
||||
).rejects.toThrow(/belongs to agent worker-1/u);
|
||||
await expect(
|
||||
provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "agent", agentId: "Worker-1" },
|
||||
}),
|
||||
).rejects.toThrow(/must be canonical/u);
|
||||
await expect(
|
||||
provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "agent", agentId: "worker-1" },
|
||||
}),
|
||||
).resolves.toMatchObject({
|
||||
manifest: {
|
||||
database: {
|
||||
role: "agent",
|
||||
agentId: "worker-1",
|
||||
userVersion: OPENCLAW_AGENT_SCHEMA_VERSION,
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects foreign-key violations and unsafe index definitions at creation", async () => {
|
||||
const tempDir = await createTempDir();
|
||||
const repositoryPath = path.join(tempDir, "snapshots");
|
||||
const foreignKeyPath = path.join(tempDir, "foreign-key.sqlite");
|
||||
const sqlite = requireNodeSqlite();
|
||||
const foreignKeyDatabase = new sqlite.DatabaseSync(foreignKeyPath);
|
||||
try {
|
||||
foreignKeyDatabase.exec(`
|
||||
PRAGMA foreign_keys = OFF;
|
||||
CREATE TABLE parents (id INTEGER PRIMARY KEY);
|
||||
CREATE TABLE children (
|
||||
id INTEGER PRIMARY KEY,
|
||||
parent_id INTEGER REFERENCES parents(id)
|
||||
);
|
||||
INSERT INTO children VALUES (1, 99);
|
||||
`);
|
||||
} finally {
|
||||
foreignKeyDatabase.close();
|
||||
}
|
||||
const unsafeIndexPath = path.join(tempDir, "unsafe-index.sqlite");
|
||||
createUnsafeIndexDrift(unsafeIndexPath);
|
||||
const provider = createLocalSqliteSnapshotProvider({ repositoryPath });
|
||||
|
||||
await expect(
|
||||
provider.create({
|
||||
path: foreignKeyPath,
|
||||
identity: { role: "generic", id: "foreign-key" },
|
||||
}),
|
||||
).rejects.toThrow(/foreign_key_check failed/u);
|
||||
await expect(
|
||||
provider.create({
|
||||
path: unsafeIndexPath,
|
||||
identity: { role: "generic", id: "unsafe-index" },
|
||||
}),
|
||||
).rejects.toThrow(/integrity_check failed|malformed database schema/iu);
|
||||
await expect(provider.list()).resolves.toEqual([]);
|
||||
});
|
||||
|
||||
it("detects artifact hash, user_version, and unsafe-index drift after creation", async () => {
|
||||
const tempDir = await createTempDir();
|
||||
const sourcePath = path.join(tempDir, "source.sqlite");
|
||||
const repositoryPath = path.join(tempDir, "snapshots");
|
||||
createGenericDatabase(sourcePath);
|
||||
const provider = createLocalSqliteSnapshotProvider({ repositoryPath });
|
||||
|
||||
const hashSnapshot = await provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "generic", id: "hash" },
|
||||
});
|
||||
await fs.appendFile(path.join(hashSnapshot.ref.path, SNAPSHOT_SQLITE_FILENAME), "tamper");
|
||||
await expect(provider.verify(hashSnapshot.ref)).rejects.toThrow(/size mismatch/u);
|
||||
|
||||
const versionSnapshot = await provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "generic", id: "version" },
|
||||
});
|
||||
const sqlite = requireNodeSqlite();
|
||||
const versionDatabase = new sqlite.DatabaseSync(
|
||||
path.join(versionSnapshot.ref.path, SNAPSHOT_SQLITE_FILENAME),
|
||||
);
|
||||
versionDatabase.exec("PRAGMA user_version = 99;");
|
||||
versionDatabase.close();
|
||||
await refreshArtifactManifest(versionSnapshot);
|
||||
await expect(provider.verify(versionSnapshot.ref)).rejects.toThrow(/user_version mismatch/u);
|
||||
|
||||
const unsafeSnapshot = await provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "generic", id: "unsafe" },
|
||||
});
|
||||
const unsafePath = path.join(unsafeSnapshot.ref.path, SNAPSHOT_SQLITE_FILENAME);
|
||||
const unsafeDatabase = new sqlite.DatabaseSync(unsafePath);
|
||||
try {
|
||||
unsafeDatabase.exec(`
|
||||
CREATE TABLE indexed_records (
|
||||
id INTEGER PRIMARY KEY,
|
||||
indexed_value TEXT NOT NULL,
|
||||
alternate_value TEXT NOT NULL
|
||||
);
|
||||
CREATE INDEX indexed_records_value ON indexed_records(indexed_value);
|
||||
INSERT INTO indexed_records (indexed_value, alternate_value)
|
||||
VALUES ('alpha', 'zeta'), ('beta', 'eta');
|
||||
PRAGMA writable_schema = ON;
|
||||
`);
|
||||
unsafeDatabase
|
||||
.prepare(
|
||||
"UPDATE sqlite_schema SET sql = 'CREATE INDEX indexed_records_value ON indexed_records(alternate_value)' WHERE name = 'indexed_records_value'",
|
||||
)
|
||||
.run();
|
||||
const schemaVersion = Number(
|
||||
Object.values(
|
||||
unsafeDatabase.prepare("PRAGMA schema_version").get() as Record<string, unknown>,
|
||||
)[0],
|
||||
);
|
||||
unsafeDatabase.exec(
|
||||
`PRAGMA writable_schema = OFF; PRAGMA schema_version = ${schemaVersion + 1};`,
|
||||
);
|
||||
} finally {
|
||||
unsafeDatabase.close();
|
||||
}
|
||||
await refreshArtifactManifest(unsafeSnapshot);
|
||||
await expect(provider.verify(unsafeSnapshot.ref)).rejects.toThrow(
|
||||
/integrity_check failed|malformed database schema/iu,
|
||||
);
|
||||
});
|
||||
|
||||
it("never overwrites an existing target or orphan SQLite sidecar", async () => {
|
||||
const tempDir = await createTempDir();
|
||||
const sourcePath = path.join(tempDir, "source.sqlite");
|
||||
const repositoryPath = path.join(tempDir, "snapshots");
|
||||
const restorePath = path.join(tempDir, "restore", "source.sqlite");
|
||||
createGenericDatabase(sourcePath);
|
||||
const provider = createLocalSqliteSnapshotProvider({ repositoryPath });
|
||||
const snapshot = await provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "generic", id: "restore" },
|
||||
});
|
||||
await fs.mkdir(path.dirname(restorePath), { recursive: true });
|
||||
await fs.writeFile(restorePath, "keep");
|
||||
|
||||
await expect(provider.restoreFresh(snapshot.ref, restorePath)).rejects.toThrow(
|
||||
/restore path already exists/u,
|
||||
);
|
||||
await expect(fs.readFile(restorePath, "utf8")).resolves.toBe("keep");
|
||||
|
||||
await fs.unlink(restorePath);
|
||||
await fs.writeFile(`${restorePath}-wal`, "keep-wal");
|
||||
await expect(provider.restoreFresh(snapshot.ref, restorePath)).rejects.toThrow(
|
||||
/restore path already exists/u,
|
||||
);
|
||||
await expect(fs.readFile(`${restorePath}-wal`, "utf8")).resolves.toBe("keep-wal");
|
||||
await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
});
|
||||
|
||||
it("fails closed when fresh restore cannot publish atomically", async () => {
|
||||
const tempDir = await createTempDir();
|
||||
const sourcePath = path.join(tempDir, "source.sqlite");
|
||||
const repositoryPath = path.join(tempDir, "snapshots");
|
||||
const restorePath = path.join(tempDir, "restore", "source.sqlite");
|
||||
createGenericDatabase(sourcePath);
|
||||
const provider = createLocalSqliteSnapshotProvider({ repositoryPath });
|
||||
const snapshot = await provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "generic", id: "atomic-restore" },
|
||||
});
|
||||
const linkSpy = vi
|
||||
.spyOn(fs, "link")
|
||||
.mockRejectedValue(Object.assign(new Error("hard links unsupported"), { code: "ENOTSUP" }));
|
||||
|
||||
try {
|
||||
await expect(provider.restoreFresh(snapshot.ref, restorePath)).rejects.toThrow(
|
||||
/requires hard-link support/u,
|
||||
);
|
||||
await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
} finally {
|
||||
linkSpy.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects restore targets inside the snapshot repository", async () => {
|
||||
const tempDir = await createTempDir();
|
||||
const sourcePath = path.join(tempDir, "source.sqlite");
|
||||
const repositoryPath = path.join(tempDir, "snapshots");
|
||||
createGenericDatabase(sourcePath);
|
||||
const provider = createLocalSqliteSnapshotProvider({ repositoryPath });
|
||||
const snapshot = await provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "generic", id: "repository-boundary" },
|
||||
});
|
||||
|
||||
await expect(
|
||||
provider.restoreFresh(snapshot.ref, path.join(repositoryPath, "restored.sqlite")),
|
||||
).rejects.toThrow(/outside snapshot repository/u);
|
||||
await expect(
|
||||
provider.restoreFresh(snapshot.ref, path.join(snapshot.ref.path, "restored.sqlite")),
|
||||
).rejects.toThrow(/outside snapshot repository/u);
|
||||
await expect(provider.verify(snapshot.ref)).resolves.toMatchObject({ ok: true });
|
||||
});
|
||||
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"binds restore to the exact artifact bytes recorded by the manifest",
|
||||
async () => {
|
||||
const tempDir = await createTempDir();
|
||||
const sourcePath = path.join(tempDir, "source.sqlite");
|
||||
const repositoryPath = path.join(tempDir, "snapshots");
|
||||
const restorePath = path.join(tempDir, "restore", "source.sqlite");
|
||||
createGenericDatabase(sourcePath);
|
||||
const provider = createLocalSqliteSnapshotProvider({ repositoryPath });
|
||||
const snapshot = await provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "generic", id: "verified-bytes" },
|
||||
});
|
||||
const artifactPath = path.join(snapshot.ref.path, SNAPSHOT_SQLITE_FILENAME);
|
||||
const originalOpen = fs.open.bind(fs);
|
||||
const openSpy = vi.spyOn(fs, "open").mockImplementation(async (filePath, flags, mode) => {
|
||||
const handle = await originalOpen(filePath, flags, mode);
|
||||
if (
|
||||
flags === "wx+" &&
|
||||
path.basename(String(filePath)) === SNAPSHOT_SQLITE_FILENAME &&
|
||||
path.basename(path.dirname(String(filePath))).startsWith(".tmp-restore-")
|
||||
) {
|
||||
const sqlite = requireNodeSqlite();
|
||||
const database = new sqlite.DatabaseSync(artifactPath);
|
||||
database.prepare("INSERT INTO entries (value) VALUES (?)").run("raced");
|
||||
database.close();
|
||||
}
|
||||
return handle;
|
||||
});
|
||||
|
||||
try {
|
||||
await expect(provider.restoreFresh(snapshot.ref, restorePath)).rejects.toThrow(
|
||||
/hash mismatch|size mismatch/u,
|
||||
);
|
||||
await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
} finally {
|
||||
openSpy.mockRestore();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"never publishes replacement bytes when the pinned staging pathname changes",
|
||||
async () => {
|
||||
const tempDir = await createTempDir();
|
||||
const sourcePath = path.join(tempDir, "source.sqlite");
|
||||
const repositoryPath = path.join(tempDir, "snapshots");
|
||||
const restorePath = path.join(tempDir, "restore", "source.sqlite");
|
||||
createGenericDatabase(sourcePath, { values: ["original"] });
|
||||
const provider = createLocalSqliteSnapshotProvider({ repositoryPath });
|
||||
const snapshot = await provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "generic", id: "pinned-staging" },
|
||||
});
|
||||
const originalOpen = fs.open.bind(fs);
|
||||
const openSpy = vi.spyOn(fs, "open").mockImplementation(async (filePath, flags, mode) => {
|
||||
if (
|
||||
flags === "wx+" &&
|
||||
path.basename(path.dirname(String(filePath))).startsWith(".sqlite-publish-")
|
||||
) {
|
||||
const stagingEntry = (await fs.readdir(repositoryPath)).find((entry) =>
|
||||
entry.startsWith(".tmp-restore-"),
|
||||
);
|
||||
if (!stagingEntry) {
|
||||
throw new Error("restore staging directory was not created");
|
||||
}
|
||||
const stagedPath = path.join(repositoryPath, stagingEntry, SNAPSHOT_SQLITE_FILENAME);
|
||||
await fs.unlink(stagedPath);
|
||||
createGenericDatabase(stagedPath, { values: ["replacement"] });
|
||||
}
|
||||
return await originalOpen(filePath, flags, mode);
|
||||
});
|
||||
|
||||
let restored = false;
|
||||
try {
|
||||
await provider.restoreFresh(snapshot.ref, restorePath);
|
||||
restored = true;
|
||||
} catch (error) {
|
||||
expect(String(error)).toMatch(/file changed while reading/u);
|
||||
} finally {
|
||||
openSpy.mockRestore();
|
||||
}
|
||||
if (!restored) {
|
||||
await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
return;
|
||||
}
|
||||
const sqlite = requireNodeSqlite();
|
||||
const database = new sqlite.DatabaseSync(restorePath, { readOnly: true });
|
||||
try {
|
||||
expect(database.prepare("SELECT value FROM entries").all()).toEqual([
|
||||
{ value: "original" },
|
||||
]);
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"removes only its restored target when a sidecar races publication",
|
||||
async () => {
|
||||
const tempDir = await createTempDir();
|
||||
const sourcePath = path.join(tempDir, "source.sqlite");
|
||||
const repositoryPath = path.join(tempDir, "snapshots");
|
||||
const restorePath = path.join(tempDir, "restore", "source.sqlite");
|
||||
createGenericDatabase(sourcePath);
|
||||
const provider = createLocalSqliteSnapshotProvider({ repositoryPath });
|
||||
const snapshot = await provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "generic", id: "restore-race" },
|
||||
});
|
||||
const originalLink = fs.link.bind(fs);
|
||||
const linkSpy = vi.spyOn(fs, "link").mockImplementation(async (source, target) => {
|
||||
await originalLink(source, target);
|
||||
if (path.resolve(String(target)) === restorePath) {
|
||||
await fs.writeFile(`${restorePath}-wal`, "racer");
|
||||
}
|
||||
});
|
||||
|
||||
try {
|
||||
await expect(provider.restoreFresh(snapshot.ref, restorePath)).rejects.toThrow(
|
||||
/unexpected sidecar/u,
|
||||
);
|
||||
await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await expect(fs.readFile(`${restorePath}-wal`, "utf8")).resolves.toBe("racer");
|
||||
} finally {
|
||||
linkSpy.mockRestore();
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("rejects snapshots outside the configured repository and unexpected contents", async () => {
|
||||
const tempDir = await createTempDir();
|
||||
const sourcePath = path.join(tempDir, "source.sqlite");
|
||||
const repositoryPath = path.join(tempDir, "snapshots");
|
||||
createGenericDatabase(sourcePath);
|
||||
const provider = createLocalSqliteSnapshotProvider({ repositoryPath });
|
||||
const snapshot = await provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "generic", id: "boundary" },
|
||||
});
|
||||
|
||||
await expect(provider.verify({ path: tempDir })).rejects.toThrow(/immediate child/u);
|
||||
await fs.writeFile(path.join(snapshot.ref.path, `${SNAPSHOT_SQLITE_FILENAME}-wal`), "orphan");
|
||||
await expect(provider.verify(snapshot.ref)).rejects.toThrow(/unexpected entry/u);
|
||||
await expect(provider.list()).rejects.toThrow(/unexpected entry/u);
|
||||
});
|
||||
|
||||
it("bounds manifest reads before parsing untrusted snapshot metadata", async () => {
|
||||
const tempDir = await createTempDir();
|
||||
const sourcePath = path.join(tempDir, "source.sqlite");
|
||||
const repositoryPath = path.join(tempDir, "snapshots");
|
||||
createGenericDatabase(sourcePath);
|
||||
const provider = createLocalSqliteSnapshotProvider({ repositoryPath });
|
||||
const snapshot = await provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "generic", id: "bounded-manifest" },
|
||||
});
|
||||
await fs.writeFile(
|
||||
path.join(snapshot.ref.path, SNAPSHOT_MANIFEST_FILENAME),
|
||||
Buffer.alloc(1024 * 1024 + 1, 0x20),
|
||||
);
|
||||
|
||||
await expect(provider.verify(snapshot.ref)).rejects.toThrow(/1048576 bytes/u);
|
||||
});
|
||||
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"rejects symlinked repositories, snapshot files, restore parents, and hardlinked artifacts",
|
||||
async () => {
|
||||
const tempDir = await createTempDir();
|
||||
const sourcePath = path.join(tempDir, "source.sqlite");
|
||||
const realRepositoryPath = path.join(tempDir, "real-snapshots");
|
||||
const repositoryLink = path.join(tempDir, "snapshot-link");
|
||||
createGenericDatabase(sourcePath);
|
||||
await fs.mkdir(realRepositoryPath);
|
||||
await fs.symlink(realRepositoryPath, repositoryLink);
|
||||
const linkedProvider = createLocalSqliteSnapshotProvider({
|
||||
repositoryPath: repositoryLink,
|
||||
});
|
||||
await expect(
|
||||
linkedProvider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "generic", id: "symlink-repository" },
|
||||
}),
|
||||
).rejects.toThrow(/symlink|Invalid path/iu);
|
||||
|
||||
const provider = createLocalSqliteSnapshotProvider({
|
||||
repositoryPath: realRepositoryPath,
|
||||
});
|
||||
const snapshot = await provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "generic", id: "links" },
|
||||
});
|
||||
const artifactPath = path.join(snapshot.ref.path, SNAPSHOT_SQLITE_FILENAME);
|
||||
const externalArtifact = path.join(tempDir, "external.sqlite");
|
||||
await fs.link(artifactPath, externalArtifact);
|
||||
await expect(provider.verify(snapshot.ref)).rejects.toThrow(/hardlink/iu);
|
||||
await fs.unlink(externalArtifact);
|
||||
|
||||
const manifestPath = path.join(snapshot.ref.path, SNAPSHOT_MANIFEST_FILENAME);
|
||||
const realManifest = path.join(tempDir, "manifest.json");
|
||||
await fs.rename(manifestPath, realManifest);
|
||||
await fs.symlink(realManifest, manifestPath);
|
||||
await expect(provider.verify(snapshot.ref)).rejects.toThrow(/regular file|symlink/iu);
|
||||
|
||||
await fs.unlink(manifestPath);
|
||||
await fs.rename(realManifest, manifestPath);
|
||||
const realRestoreParent = path.join(tempDir, "real-restore");
|
||||
const restoreParentLink = path.join(tempDir, "restore-link");
|
||||
await fs.mkdir(realRestoreParent);
|
||||
await fs.symlink(realRestoreParent, restoreParentLink);
|
||||
await expect(
|
||||
provider.restoreFresh(snapshot.ref, path.join(restoreParentLink, "restored.sqlite")),
|
||||
).rejects.toThrow(/symlink|Invalid path/iu);
|
||||
},
|
||||
);
|
||||
|
||||
it.runIf(process.platform !== "win32")(
|
||||
"rejects repository restore targets reached through another filesystem spelling",
|
||||
async () => {
|
||||
const tempDir = await createTempDir();
|
||||
const sourcePath = path.join(tempDir, "source.sqlite");
|
||||
const repositoryPath = path.join(tempDir, "snapshots");
|
||||
createGenericDatabase(sourcePath);
|
||||
const provider = createLocalSqliteSnapshotProvider({ repositoryPath });
|
||||
const snapshot = await provider.create({
|
||||
path: sourcePath,
|
||||
identity: { role: "generic", id: "canonical-boundary" },
|
||||
});
|
||||
const aliasRoot = path.join(tempDir, "alias");
|
||||
await fs.symlink(tempDir, aliasRoot);
|
||||
const aliasRepositoryPath = path.join(aliasRoot, "snapshots");
|
||||
const aliasProvider = createLocalSqliteSnapshotProvider({
|
||||
repositoryPath: aliasRepositoryPath,
|
||||
});
|
||||
const aliasSnapshot = {
|
||||
path: path.join(aliasRepositoryPath, path.basename(snapshot.ref.path)),
|
||||
};
|
||||
|
||||
await expect(
|
||||
aliasProvider.restoreFresh(aliasSnapshot, path.join(repositoryPath, "restored.sqlite")),
|
||||
).rejects.toThrow(/outside snapshot repository/u);
|
||||
await expect(provider.verify(snapshot.ref)).resolves.toMatchObject({ ok: true });
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
describe("snapshot manifest parser", () => {
|
||||
const manifestPath = "/snapshots/snapshot/manifest.json";
|
||||
const snapshotId = "snapshot";
|
||||
const validManifest: SnapshotManifest = {
|
||||
schemaVersion: 1,
|
||||
snapshotId,
|
||||
createdAt: "2026-07-12T14:00:00.000Z",
|
||||
database: {
|
||||
role: "agent",
|
||||
agentId: "worker-1",
|
||||
basename: "openclaw-agent.sqlite",
|
||||
userVersion: OPENCLAW_AGENT_SCHEMA_VERSION,
|
||||
},
|
||||
artifact: {
|
||||
path: SNAPSHOT_SQLITE_FILENAME,
|
||||
sha256: "a".repeat(64),
|
||||
sizeBytes: 4096,
|
||||
},
|
||||
};
|
||||
|
||||
it.each([
|
||||
["unknown top-level field", { ...validManifest, extra: true }, /fields must be exactly/u],
|
||||
["wrong directory id", validManifest, /does not match directory/u, "other"],
|
||||
[
|
||||
"noncanonical timestamp",
|
||||
{ ...validManifest, createdAt: "2026-07-12T14:00:00Z" },
|
||||
/not canonical/u,
|
||||
],
|
||||
[
|
||||
"artifact path traversal",
|
||||
{ ...validManifest, artifact: { ...validManifest.artifact, path: "../database.sqlite" } },
|
||||
/artifact\.path must be database\.sqlite/u,
|
||||
],
|
||||
[
|
||||
"prefixed digest",
|
||||
{
|
||||
...validManifest,
|
||||
artifact: { ...validManifest.artifact, sha256: `sha256:${"a".repeat(64)}` },
|
||||
},
|
||||
/sha256 is invalid/u,
|
||||
],
|
||||
[
|
||||
"noncanonical agent id",
|
||||
{ ...validManifest, database: { ...validManifest.database, agentId: "Worker-1" } },
|
||||
/agentId is invalid/u,
|
||||
],
|
||||
[
|
||||
"unsafe basename",
|
||||
{ ...validManifest, database: { ...validManifest.database, basename: "../db.sqlite" } },
|
||||
/basename is invalid/u,
|
||||
],
|
||||
[
|
||||
"out-of-range user version",
|
||||
{ ...validManifest, database: { ...validManifest.database, userVersion: 2 ** 31 } },
|
||||
/userVersion is invalid/u,
|
||||
],
|
||||
[
|
||||
"zero-byte artifact",
|
||||
{ ...validManifest, artifact: { ...validManifest.artifact, sizeBytes: 0 } },
|
||||
/sizeBytes is invalid/u,
|
||||
],
|
||||
])("rejects %s", (_name, value, error, expectedId = snapshotId) => {
|
||||
expect(() => parseSnapshotManifest(value, manifestPath, expectedId)).toThrow(error);
|
||||
});
|
||||
});
|
||||
@@ -1,863 +0,0 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fsSync, { type Stats } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import type { FileHandle } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import { loadSqliteVecExtension } from "../../packages/memory-host-sdk/src/engine-storage.js";
|
||||
import { sameFileIdentity } from "../infra/fs-safe-advanced.js";
|
||||
import {
|
||||
canonicalPathFromExistingAncestor,
|
||||
ensureAbsoluteDirectory,
|
||||
isPathInside,
|
||||
} from "../infra/fs-safe.js";
|
||||
import { requireNodeSqlite } from "../infra/node-sqlite.js";
|
||||
import { applyPrivateModeSync } from "../infra/private-mode.js";
|
||||
import { assertSqliteIntegrity } from "../infra/sqlite-integrity.js";
|
||||
import {
|
||||
createVerifiedSqliteSnapshot,
|
||||
publishVerifiedSqliteFile,
|
||||
syncDirectoryBestEffort,
|
||||
type SqliteSnapshotValidator,
|
||||
} from "../infra/sqlite-snapshot.js";
|
||||
import { readSqliteUserVersion } from "../infra/sqlite-user-version.js";
|
||||
import { isValidAgentId, normalizeAgentId } from "../routing/session-key.js";
|
||||
import { assertOpenClawAgentDatabaseForMaintenance } from "../state/openclaw-agent-db.js";
|
||||
import { assertOpenClawStateDatabaseForMaintenance } from "../state/openclaw-state-db.js";
|
||||
import {
|
||||
containsAsciiControlCharacter,
|
||||
copySnapshotArtifact,
|
||||
hashSnapshotArtifact,
|
||||
readSnapshotManifest,
|
||||
type SnapshotArtifactDigest,
|
||||
writeSnapshotManifest,
|
||||
} from "./manifest.js";
|
||||
import {
|
||||
SNAPSHOT_MANIFEST_FILENAME,
|
||||
SNAPSHOT_SQLITE_FILENAME,
|
||||
type SnapshotDatabaseIdentity,
|
||||
type SnapshotDatabaseManifest,
|
||||
type SnapshotDatabaseRef,
|
||||
type SnapshotManifest,
|
||||
type SnapshotRef,
|
||||
type SnapshotResult,
|
||||
type SnapshotSummary,
|
||||
type SnapshotVerificationResult,
|
||||
type SqliteSnapshotProvider,
|
||||
} from "./snapshot-provider.js";
|
||||
|
||||
const SNAPSHOT_DIRECTORY_MODE = 0o700;
|
||||
const SNAPSHOT_FILE_MODE = 0o600;
|
||||
const SNAPSHOT_ID_BASENAME_MAX_LENGTH = 80;
|
||||
const SNAPSHOT_PENDING_FILENAME = ".pending";
|
||||
const SQLITE_SIDECAR_SUFFIXES = ["-wal", "-shm", "-journal"] as const;
|
||||
const SNAPSHOT_ARTIFACT_ENTRIES = new Set([
|
||||
SNAPSHOT_MANIFEST_FILENAME,
|
||||
SNAPSHOT_PENDING_FILENAME,
|
||||
SNAPSHOT_SQLITE_FILENAME,
|
||||
]);
|
||||
const RESTORE_STAGING_ENTRIES = new Set([SNAPSHOT_SQLITE_FILENAME]);
|
||||
|
||||
export type LocalSqliteSnapshotProviderOptions = {
|
||||
readonly repositoryPath: string;
|
||||
readonly now?: () => Date;
|
||||
};
|
||||
|
||||
export function createLocalSqliteSnapshotProvider(
|
||||
options: LocalSqliteSnapshotProviderOptions,
|
||||
): SqliteSnapshotProvider {
|
||||
return new LocalSqliteSnapshotProvider(options);
|
||||
}
|
||||
|
||||
class LocalSqliteSnapshotProvider implements SqliteSnapshotProvider {
|
||||
readonly #repositoryPath: string;
|
||||
readonly #now: () => Date;
|
||||
|
||||
constructor(options: LocalSqliteSnapshotProviderOptions) {
|
||||
this.#repositoryPath = path.resolve(options.repositoryPath);
|
||||
this.#now = options.now ?? (() => new Date());
|
||||
}
|
||||
|
||||
async create(database: SnapshotDatabaseRef): Promise<SnapshotResult> {
|
||||
await ensurePrivateDirectory(this.#repositoryPath, "SQLite snapshot repository");
|
||||
const sourcePath = path.resolve(database.path);
|
||||
const identity = normalizeSnapshotIdentity(database.identity);
|
||||
const now = this.#now();
|
||||
if (!Number.isFinite(now.getTime())) {
|
||||
throw new Error("SQLite snapshot timestamp is invalid.");
|
||||
}
|
||||
const snapshotId = buildSnapshotId(now, sourcePath);
|
||||
const snapshotDir = path.join(this.#repositoryPath, snapshotId);
|
||||
const stagingDir = path.join(this.#repositoryPath, `.tmp-${snapshotId}-${randomUUID()}`);
|
||||
const artifactPath = path.join(stagingDir, SNAPSHOT_SQLITE_FILENAME);
|
||||
await fs.mkdir(stagingDir, { mode: SNAPSHOT_DIRECTORY_MODE });
|
||||
|
||||
let stagingIdentity: Stats | undefined;
|
||||
let publishedDirectory: FileHandle | undefined;
|
||||
let publishedIdentity: Stats | undefined;
|
||||
const publishedEntries = new Map<string, Stats>();
|
||||
let snapshotDirectoryCreated = false;
|
||||
try {
|
||||
stagingIdentity = await fs.lstat(stagingDir);
|
||||
applyPrivateModeSync(stagingDir, SNAPSHOT_DIRECTORY_MODE);
|
||||
const result = await createVerifiedSqliteSnapshot({
|
||||
sourcePath,
|
||||
targetPath: artifactPath,
|
||||
transform: identity.role === "global" ? sanitizeGlobalStateSnapshot : undefined,
|
||||
validate: buildDatabaseValidator(identity),
|
||||
});
|
||||
applyPrivateModeSync(artifactPath, SNAPSHOT_FILE_MODE);
|
||||
const artifact = await hashSnapshotArtifact(stagingDir);
|
||||
const manifest: SnapshotManifest = {
|
||||
schemaVersion: 1,
|
||||
snapshotId,
|
||||
createdAt: now.toISOString(),
|
||||
database: buildDatabaseManifest(identity, sourcePath, result.userVersion),
|
||||
artifact: {
|
||||
path: SNAPSHOT_SQLITE_FILENAME,
|
||||
sha256: artifact.sha256,
|
||||
sizeBytes: artifact.sizeBytes,
|
||||
},
|
||||
};
|
||||
await writeSnapshotManifest(stagingDir, manifest);
|
||||
applyPrivateModeSync(path.join(stagingDir, SNAPSHOT_MANIFEST_FILENAME), SNAPSHOT_FILE_MODE);
|
||||
await readSnapshotManifest(stagingDir, snapshotId);
|
||||
await syncDirectoryBestEffort(stagingDir);
|
||||
|
||||
try {
|
||||
await fs.mkdir(snapshotDir, { mode: SNAPSHOT_DIRECTORY_MODE });
|
||||
snapshotDirectoryCreated = true;
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "EEXIST") {
|
||||
throw new Error(`SQLite snapshot directory already exists: ${snapshotDir}`, {
|
||||
cause: error,
|
||||
});
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
publishedIdentity = await fs.lstat(snapshotDir);
|
||||
applyPrivateModeSync(snapshotDir, SNAPSHOT_DIRECTORY_MODE);
|
||||
publishedDirectory = await fs.open(snapshotDir, "r");
|
||||
await assertOpenDirectoryIdentity(publishedDirectory, snapshotDir, publishedIdentity);
|
||||
const pendingPath = path.join(snapshotDir, SNAPSHOT_PENDING_FILENAME);
|
||||
await fs.writeFile(pendingPath, "", {
|
||||
flag: "wx",
|
||||
mode: SNAPSHOT_FILE_MODE,
|
||||
});
|
||||
publishedEntries.set(SNAPSHOT_PENDING_FILENAME, await fs.lstat(pendingPath));
|
||||
await assertOpenDirectoryIdentity(publishedDirectory, snapshotDir, publishedIdentity);
|
||||
await publishSnapshotEntryNoOverwrite(
|
||||
path.join(stagingDir, SNAPSHOT_SQLITE_FILENAME),
|
||||
path.join(snapshotDir, SNAPSHOT_SQLITE_FILENAME),
|
||||
SNAPSHOT_SQLITE_FILENAME,
|
||||
publishedEntries,
|
||||
);
|
||||
await assertOpenDirectoryIdentity(publishedDirectory, snapshotDir, publishedIdentity);
|
||||
await publishSnapshotEntryNoOverwrite(
|
||||
path.join(stagingDir, SNAPSHOT_MANIFEST_FILENAME),
|
||||
path.join(snapshotDir, SNAPSHOT_MANIFEST_FILENAME),
|
||||
SNAPSHOT_MANIFEST_FILENAME,
|
||||
publishedEntries,
|
||||
);
|
||||
await assertOpenDirectoryIdentity(publishedDirectory, snapshotDir, publishedIdentity);
|
||||
await syncDirectoryBestEffort(snapshotDir);
|
||||
await assertPendingSnapshotContents(snapshotDir);
|
||||
const publishedManifest = await readSnapshotManifest(snapshotDir, snapshotId);
|
||||
if (!isDeepStrictEqual(publishedManifest, manifest)) {
|
||||
throw new Error(`SQLite snapshot manifest changed during publication: ${snapshotDir}`);
|
||||
}
|
||||
const publishedArtifact = await hashSnapshotArtifact(snapshotDir);
|
||||
const publishedArtifactPath = path.join(snapshotDir, SNAPSHOT_SQLITE_FILENAME);
|
||||
assertArtifactMatchesManifest(publishedArtifactPath, publishedArtifact, publishedManifest);
|
||||
await verifySnapshotDatabaseFile(
|
||||
publishedArtifactPath,
|
||||
publishedArtifact.stat,
|
||||
publishedManifest,
|
||||
);
|
||||
const expectedPendingIdentity = publishedEntries.get(SNAPSHOT_PENDING_FILENAME);
|
||||
const currentPendingIdentity = fsSync.lstatSync(pendingPath);
|
||||
if (
|
||||
!expectedPendingIdentity ||
|
||||
!sameFileIdentity(expectedPendingIdentity, currentPendingIdentity)
|
||||
) {
|
||||
throw new Error(`SQLite snapshot pending marker changed: ${pendingPath}`);
|
||||
}
|
||||
fsSync.unlinkSync(pendingPath);
|
||||
publishedEntries.delete(SNAPSHOT_PENDING_FILENAME);
|
||||
await syncDirectoryBestEffort(snapshotDir);
|
||||
await publishedDirectory.close();
|
||||
publishedDirectory = undefined;
|
||||
const committedManifest = await readSnapshotManifest(snapshotDir, snapshotId);
|
||||
if (!isDeepStrictEqual(committedManifest, manifest)) {
|
||||
throw new Error(`SQLite snapshot manifest changed after commit: ${snapshotDir}`);
|
||||
}
|
||||
const committedArtifact = await hashSnapshotArtifact(snapshotDir);
|
||||
assertArtifactMatchesManifest(
|
||||
path.join(snapshotDir, SNAPSHOT_SQLITE_FILENAME),
|
||||
committedArtifact,
|
||||
committedManifest,
|
||||
);
|
||||
const currentIdentity = await fs.lstat(snapshotDir);
|
||||
if (!sameFileIdentity(publishedIdentity, currentIdentity)) {
|
||||
throw new Error(`SQLite snapshot directory changed during publication: ${snapshotDir}`);
|
||||
}
|
||||
await assertExactSnapshotContents(snapshotDir);
|
||||
await syncDirectoryBestEffort(this.#repositoryPath);
|
||||
return { ref: { path: snapshotDir }, manifest };
|
||||
} catch (error) {
|
||||
await publishedDirectory?.close().catch(() => undefined);
|
||||
publishedDirectory = undefined;
|
||||
if (snapshotDirectoryCreated) {
|
||||
publishedIdentity ??= await fs.lstat(snapshotDir).catch(() => undefined);
|
||||
}
|
||||
if (publishedIdentity) {
|
||||
const removed = await removePublishedSnapshotDirectoryIfOwned(
|
||||
snapshotDir,
|
||||
publishedIdentity,
|
||||
publishedEntries,
|
||||
);
|
||||
if (removed) {
|
||||
await syncDirectoryBestEffort(this.#repositoryPath);
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
const removed = stagingIdentity
|
||||
? await removePrivateDirectoryIfOwned(
|
||||
stagingDir,
|
||||
stagingIdentity,
|
||||
SNAPSHOT_ARTIFACT_ENTRIES,
|
||||
).catch(() => false)
|
||||
: await fs
|
||||
.rmdir(stagingDir)
|
||||
.then(() => true)
|
||||
.catch(() => false);
|
||||
if (removed) {
|
||||
await syncDirectoryBestEffort(this.#repositoryPath).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async verify(snapshot: SnapshotRef): Promise<SnapshotVerificationResult> {
|
||||
const snapshotDir = await this.#resolveSnapshotDirectory(snapshot);
|
||||
const manifest = await readVerifiedSnapshotManifest(snapshotDir);
|
||||
const artifact = await hashSnapshotArtifact(snapshotDir);
|
||||
const artifactPath = path.join(snapshotDir, SNAPSHOT_SQLITE_FILENAME);
|
||||
assertArtifactMatchesManifest(artifactPath, artifact, manifest);
|
||||
await verifySnapshotDatabaseFile(artifactPath, artifact.stat, manifest);
|
||||
await assertExactSnapshotContents(snapshotDir);
|
||||
return { ok: true, manifest };
|
||||
}
|
||||
|
||||
async restoreFresh(
|
||||
snapshot: SnapshotRef,
|
||||
targetPath: string,
|
||||
): Promise<SnapshotVerificationResult> {
|
||||
const snapshotDir = await this.#resolveSnapshotDirectory(snapshot);
|
||||
const manifest = await readVerifiedSnapshotManifest(snapshotDir);
|
||||
const resolvedTargetPath = path.resolve(targetPath);
|
||||
const canonicalRepositoryPath = await fs.realpath(this.#repositoryPath);
|
||||
const canonicalTargetPath = await canonicalPathFromExistingAncestor(resolvedTargetPath);
|
||||
if (isPathInside(canonicalRepositoryPath, canonicalTargetPath)) {
|
||||
throw new Error(
|
||||
`SQLite restore target must be outside snapshot repository ${this.#repositoryPath}: ${resolvedTargetPath}`,
|
||||
);
|
||||
}
|
||||
const restoreParentPath = path.dirname(resolvedTargetPath);
|
||||
await ensureRestoreParentDirectory(restoreParentPath);
|
||||
const restoreParentIdentity = await fs.lstat(restoreParentPath);
|
||||
applyPrivateModeSync(this.#repositoryPath, SNAPSHOT_DIRECTORY_MODE);
|
||||
// Existing databases need a crash-recoverable main/WAL/SHM swap protocol.
|
||||
// This path is deliberately fresh-only and refuses every preexisting sidecar.
|
||||
await assertFreshRestorePathsAbsent(resolvedTargetPath);
|
||||
|
||||
const stagingDir = await fs.mkdtemp(path.join(this.#repositoryPath, ".tmp-restore-"));
|
||||
const stagedSourcePath = path.join(stagingDir, SNAPSHOT_SQLITE_FILENAME);
|
||||
let stagingIdentity: Stats | undefined;
|
||||
try {
|
||||
stagingIdentity = await fs.lstat(stagingDir);
|
||||
applyPrivateModeSync(stagingDir, SNAPSHOT_DIRECTORY_MODE);
|
||||
const stagedArtifact = await copySnapshotArtifact(snapshotDir, stagedSourcePath);
|
||||
await assertDirectoryIdentity(stagingDir, stagingIdentity);
|
||||
assertArtifactMatchesManifest(stagedSourcePath, stagedArtifact, manifest);
|
||||
await assertExactSnapshotContents(snapshotDir);
|
||||
await verifySnapshotDatabaseFile(stagedSourcePath, stagedArtifact.stat, manifest);
|
||||
await publishVerifiedSqliteFile({
|
||||
sourceIdentity: stagedArtifact.stat,
|
||||
sourcePath: stagedSourcePath,
|
||||
targetPath: resolvedTargetPath,
|
||||
expectedContent: manifest.artifact,
|
||||
requireAtomicPublication: true,
|
||||
beforePublish: async () => {
|
||||
await assertDirectoryIdentity(restoreParentPath, restoreParentIdentity);
|
||||
await assertFreshRestorePathsAbsent(resolvedTargetPath);
|
||||
},
|
||||
afterPublish: (guard) => {
|
||||
guard.assertTargetMatchesExpectedContent(() => {
|
||||
assertDirectoryIdentitySync(restoreParentPath, restoreParentIdentity);
|
||||
assertNoSqliteSidecarsSync(resolvedTargetPath);
|
||||
});
|
||||
},
|
||||
});
|
||||
return { ok: true, manifest };
|
||||
} finally {
|
||||
const removed = stagingIdentity
|
||||
? await removePrivateDirectoryIfOwned(
|
||||
stagingDir,
|
||||
stagingIdentity,
|
||||
RESTORE_STAGING_ENTRIES,
|
||||
).catch(() => false)
|
||||
: await fs
|
||||
.rmdir(stagingDir)
|
||||
.then(() => true)
|
||||
.catch(() => false);
|
||||
if (removed) {
|
||||
await syncDirectoryBestEffort(this.#repositoryPath).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async list(): Promise<SnapshotSummary[]> {
|
||||
const repositoryStat = await lstatIfExists(this.#repositoryPath);
|
||||
if (!repositoryStat) {
|
||||
return [];
|
||||
}
|
||||
assertDirectory(repositoryStat, this.#repositoryPath, "SQLite snapshot repository");
|
||||
applyPrivateModeSync(this.#repositoryPath, SNAPSHOT_DIRECTORY_MODE);
|
||||
|
||||
const entries = await fs.readdir(this.#repositoryPath, { withFileTypes: true });
|
||||
const snapshots: SnapshotSummary[] = [];
|
||||
for (const entry of entries) {
|
||||
if (entry.name.startsWith(".tmp-")) {
|
||||
if (entry.isSymbolicLink() || !entry.isDirectory()) {
|
||||
throw new Error(
|
||||
`SQLite snapshot repository contains unsafe staging entry: ${path.join(this.#repositoryPath, entry.name)}`,
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (entry.isSymbolicLink() || !entry.isDirectory()) {
|
||||
throw new Error(
|
||||
`SQLite snapshot repository contains unexpected entry: ${path.join(this.#repositoryPath, entry.name)}`,
|
||||
);
|
||||
}
|
||||
const snapshotPath = path.join(this.#repositoryPath, entry.name);
|
||||
if (await isIncompleteSnapshotDirectory(snapshotPath)) {
|
||||
continue;
|
||||
}
|
||||
await assertExactSnapshotContents(snapshotPath);
|
||||
snapshots.push({
|
||||
ref: { path: snapshotPath },
|
||||
manifest: await readSnapshotManifest(snapshotPath),
|
||||
});
|
||||
}
|
||||
return snapshots.toSorted(
|
||||
(left, right) =>
|
||||
right.manifest.createdAt.localeCompare(left.manifest.createdAt) ||
|
||||
right.manifest.snapshotId.localeCompare(left.manifest.snapshotId),
|
||||
);
|
||||
}
|
||||
|
||||
async #resolveSnapshotDirectory(snapshot: SnapshotRef): Promise<string> {
|
||||
const snapshotDir = path.resolve(snapshot.path);
|
||||
if (path.dirname(snapshotDir) !== this.#repositoryPath) {
|
||||
throw new Error(
|
||||
`SQLite snapshot must be an immediate child of repository ${this.#repositoryPath}: ${snapshotDir}`,
|
||||
);
|
||||
}
|
||||
const repositoryStat = await fs.lstat(this.#repositoryPath);
|
||||
assertDirectory(repositoryStat, this.#repositoryPath, "SQLite snapshot repository");
|
||||
const snapshotStat = await fs.lstat(snapshotDir);
|
||||
assertDirectory(snapshotStat, snapshotDir, "SQLite snapshot");
|
||||
return snapshotDir;
|
||||
}
|
||||
}
|
||||
|
||||
async function readVerifiedSnapshotManifest(snapshotDir: string): Promise<SnapshotManifest> {
|
||||
await assertExactSnapshotContents(snapshotDir);
|
||||
return await readSnapshotManifest(snapshotDir);
|
||||
}
|
||||
|
||||
function assertArtifactMatchesManifest(
|
||||
artifactPath: string,
|
||||
artifact: SnapshotArtifactDigest,
|
||||
manifest: SnapshotManifest,
|
||||
): void {
|
||||
if (artifact.sizeBytes !== manifest.artifact.sizeBytes) {
|
||||
throw new Error(
|
||||
`Snapshot artifact size mismatch for ${artifactPath}: expected ${manifest.artifact.sizeBytes}, got ${artifact.sizeBytes}`,
|
||||
);
|
||||
}
|
||||
if (artifact.sha256 !== manifest.artifact.sha256) {
|
||||
throw new Error(
|
||||
`Snapshot artifact hash mismatch for ${artifactPath}: expected ${manifest.artifact.sha256}, got ${artifact.sha256}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function verifySnapshotDatabaseFile(
|
||||
artifactPath: string,
|
||||
expectedIdentity: Stats,
|
||||
manifest: SnapshotManifest,
|
||||
): Promise<void> {
|
||||
const beforeOpen = await fs.lstat(artifactPath);
|
||||
if (
|
||||
beforeOpen.isSymbolicLink() ||
|
||||
!beforeOpen.isFile() ||
|
||||
beforeOpen.nlink > 1 ||
|
||||
!sameFileIdentity(expectedIdentity, beforeOpen)
|
||||
) {
|
||||
throw new Error(`Snapshot artifact changed before SQLite verification: ${artifactPath}`);
|
||||
}
|
||||
|
||||
const validationDir = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-snapshot-verify-"));
|
||||
const validationPath = path.join(validationDir, SNAPSHOT_SQLITE_FILENAME);
|
||||
try {
|
||||
await fs.chmod(validationDir, SNAPSHOT_DIRECTORY_MODE);
|
||||
const validationArtifact = await copySnapshotArtifact(
|
||||
path.dirname(artifactPath),
|
||||
validationPath,
|
||||
);
|
||||
assertArtifactMatchesManifest(validationPath, validationArtifact, manifest);
|
||||
const sqlite = requireNodeSqlite();
|
||||
const database = new sqlite.DatabaseSync(validationPath, {
|
||||
allowExtension: true,
|
||||
readOnly: true,
|
||||
});
|
||||
try {
|
||||
database.exec("PRAGMA busy_timeout = 30000; PRAGMA trusted_schema = OFF;");
|
||||
await loadSqliteVecExtension({ db: database });
|
||||
assertSqliteIntegrity(database, artifactPath);
|
||||
buildManifestDatabaseValidator(manifest.database)(database, artifactPath);
|
||||
} finally {
|
||||
database.close();
|
||||
}
|
||||
const validatedArtifact = await hashSnapshotArtifact(validationDir);
|
||||
if (!sameFileIdentity(validationArtifact.stat, validatedArtifact.stat)) {
|
||||
throw new Error(`Snapshot validation copy changed: ${validationPath}`);
|
||||
}
|
||||
assertArtifactMatchesManifest(validationPath, validatedArtifact, manifest);
|
||||
} finally {
|
||||
await fs.unlink(validationPath).catch(() => undefined);
|
||||
await fs.rmdir(validationDir).catch(() => undefined);
|
||||
}
|
||||
const afterOpen = await fs.lstat(artifactPath);
|
||||
if (
|
||||
afterOpen.isSymbolicLink() ||
|
||||
!afterOpen.isFile() ||
|
||||
afterOpen.nlink > 1 ||
|
||||
!sameFileIdentity(expectedIdentity, afterOpen)
|
||||
) {
|
||||
throw new Error(`Snapshot artifact changed during SQLite verification: ${artifactPath}`);
|
||||
}
|
||||
const verifiedArtifact = await hashSnapshotArtifact(path.dirname(artifactPath));
|
||||
if (!sameFileIdentity(expectedIdentity, verifiedArtifact.stat)) {
|
||||
throw new Error(`Snapshot artifact changed after SQLite verification: ${artifactPath}`);
|
||||
}
|
||||
assertArtifactMatchesManifest(artifactPath, verifiedArtifact, manifest);
|
||||
}
|
||||
|
||||
function normalizeSnapshotIdentity(identity: SnapshotDatabaseIdentity): SnapshotDatabaseIdentity {
|
||||
if (identity.role === "global") {
|
||||
return identity;
|
||||
}
|
||||
if (identity.role === "agent") {
|
||||
const agentId = normalizeAgentId(identity.agentId);
|
||||
if (!isValidAgentId(identity.agentId) || agentId !== identity.agentId) {
|
||||
throw new Error(`SQLite snapshot agent id must be canonical: ${identity.agentId}`);
|
||||
}
|
||||
return { role: "agent", agentId };
|
||||
}
|
||||
const id = identity.id.trim();
|
||||
if (!id || id !== identity.id || id.length > 256 || containsAsciiControlCharacter(id)) {
|
||||
throw new Error("SQLite snapshot generic database id is invalid.");
|
||||
}
|
||||
return { role: "generic", id };
|
||||
}
|
||||
|
||||
function buildDatabaseManifest(
|
||||
identity: SnapshotDatabaseIdentity,
|
||||
sourcePath: string,
|
||||
userVersion: number,
|
||||
): SnapshotDatabaseManifest {
|
||||
const basename = path.basename(sourcePath);
|
||||
if (identity.role === "global") {
|
||||
return { role: "global", basename, userVersion };
|
||||
}
|
||||
if (identity.role === "agent") {
|
||||
return { role: "agent", agentId: identity.agentId, basename, userVersion };
|
||||
}
|
||||
return { role: "generic", id: identity.id, basename, userVersion };
|
||||
}
|
||||
|
||||
function buildDatabaseValidator(
|
||||
identity: SnapshotDatabaseIdentity | SnapshotDatabaseManifest,
|
||||
): SqliteSnapshotValidator {
|
||||
if (identity.role === "global") {
|
||||
return (database, pathname) =>
|
||||
assertOpenClawStateDatabaseForMaintenance(database, { pathname });
|
||||
}
|
||||
if (identity.role === "agent") {
|
||||
return (database, pathname) =>
|
||||
assertOpenClawAgentDatabaseForMaintenance(database, {
|
||||
agentId: identity.agentId,
|
||||
pathname,
|
||||
});
|
||||
}
|
||||
return () => undefined;
|
||||
}
|
||||
|
||||
function buildManifestDatabaseValidator(
|
||||
manifest: SnapshotDatabaseManifest,
|
||||
): SqliteSnapshotValidator {
|
||||
const validateOwner = buildDatabaseValidator(manifest);
|
||||
return (database, pathname) => {
|
||||
validateOwner(database, pathname);
|
||||
const userVersion = readSqliteUserVersion(database);
|
||||
if (userVersion !== manifest.userVersion) {
|
||||
throw new Error(
|
||||
`Snapshot database user_version mismatch for ${pathname}: expected ${manifest.userVersion}, got ${userVersion}`,
|
||||
);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function sanitizeGlobalStateSnapshot(database: DatabaseSync): void {
|
||||
database.prepare("DELETE FROM delivery_queue_entries").run();
|
||||
}
|
||||
|
||||
function buildSnapshotId(now: Date, sourcePath: string): string {
|
||||
const timestamp = now.toISOString().replaceAll(/[:.]/g, "-");
|
||||
const basename =
|
||||
path
|
||||
.basename(sourcePath)
|
||||
.replaceAll(/[^a-zA-Z0-9._-]/g, "-")
|
||||
.slice(0, SNAPSHOT_ID_BASENAME_MAX_LENGTH) || "database.sqlite";
|
||||
return `${timestamp}-${basename}-${randomUUID()}`;
|
||||
}
|
||||
|
||||
async function ensurePrivateDirectory(directoryPath: string, scopeLabel: string): Promise<void> {
|
||||
const result = await ensureAbsoluteDirectory(directoryPath, {
|
||||
mode: SNAPSHOT_DIRECTORY_MODE,
|
||||
scopeLabel,
|
||||
});
|
||||
if (!result.ok) {
|
||||
throw result.error;
|
||||
}
|
||||
applyPrivateModeSync(result.path, SNAPSHOT_DIRECTORY_MODE);
|
||||
}
|
||||
|
||||
async function ensureRestoreParentDirectory(directoryPath: string): Promise<void> {
|
||||
const result = await ensureAbsoluteDirectory(directoryPath, {
|
||||
mode: SNAPSHOT_DIRECTORY_MODE,
|
||||
scopeLabel: "SQLite restore target",
|
||||
});
|
||||
if (!result.ok) {
|
||||
throw result.error;
|
||||
}
|
||||
}
|
||||
|
||||
function assertDirectory(stat: Stats, pathname: string, label: string): void {
|
||||
if (stat.isSymbolicLink() || !stat.isDirectory()) {
|
||||
throw new Error(`${label} must be a real directory: ${pathname}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function assertDirectoryIdentity(
|
||||
directoryPath: string,
|
||||
expectedIdentity: Stats,
|
||||
): Promise<void> {
|
||||
const currentIdentity = await fs.lstat(directoryPath);
|
||||
assertDirectory(currentIdentity, directoryPath, "SQLite restore target directory");
|
||||
if (!sameFileIdentity(currentIdentity, expectedIdentity)) {
|
||||
throw new Error(`SQLite restore target directory changed during restore: ${directoryPath}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function assertOpenDirectoryIdentity(
|
||||
handle: FileHandle,
|
||||
directoryPath: string,
|
||||
expectedIdentity: Stats,
|
||||
): Promise<void> {
|
||||
const openedIdentity = await handle.stat();
|
||||
const currentIdentity = await fs.lstat(directoryPath);
|
||||
assertDirectory(openedIdentity, directoryPath, "SQLite snapshot directory");
|
||||
assertDirectory(currentIdentity, directoryPath, "SQLite snapshot directory");
|
||||
if (
|
||||
!sameFileIdentity(openedIdentity, expectedIdentity) ||
|
||||
!sameFileIdentity(currentIdentity, expectedIdentity)
|
||||
) {
|
||||
throw new Error(`SQLite snapshot directory changed during publication: ${directoryPath}`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertDirectoryIdentitySync(directoryPath: string, expectedIdentity: Stats): void {
|
||||
const currentIdentity = fsSync.lstatSync(directoryPath);
|
||||
assertDirectory(currentIdentity, directoryPath, "SQLite restore target directory");
|
||||
if (!sameFileIdentity(currentIdentity, expectedIdentity)) {
|
||||
throw new Error(`SQLite restore target directory changed during restore: ${directoryPath}`);
|
||||
}
|
||||
}
|
||||
|
||||
function isSnapshotEntryLinkFallbackError(error: unknown): boolean {
|
||||
const code = (error as NodeJS.ErrnoException).code;
|
||||
return (
|
||||
code === "EPERM" ||
|
||||
code === "EXDEV" ||
|
||||
code === "ENOTSUP" ||
|
||||
code === "EOPNOTSUPP" ||
|
||||
code === "ENOSYS"
|
||||
);
|
||||
}
|
||||
|
||||
async function publishSnapshotEntryNoOverwrite(
|
||||
sourcePath: string,
|
||||
targetPath: string,
|
||||
entryName: string,
|
||||
publishedEntries: Map<string, Stats>,
|
||||
): Promise<void> {
|
||||
let linked = false;
|
||||
let linkedSourceIdentity: Stats | undefined;
|
||||
try {
|
||||
linkedSourceIdentity = await fs.lstat(sourcePath);
|
||||
await fs.link(sourcePath, targetPath);
|
||||
publishedEntries.set(entryName, linkedSourceIdentity);
|
||||
linked = true;
|
||||
} catch (error) {
|
||||
if (!isSnapshotEntryLinkFallbackError(error)) {
|
||||
throw error;
|
||||
}
|
||||
const copiedIdentity = await copySnapshotEntryExclusive(sourcePath, targetPath);
|
||||
publishedEntries.set(entryName, copiedIdentity);
|
||||
}
|
||||
const expectedTargetIdentity = publishedEntries.get(entryName);
|
||||
const initialTargetIdentity = await fs.lstat(targetPath);
|
||||
if (!expectedTargetIdentity || !sameFileIdentity(expectedTargetIdentity, initialTargetIdentity)) {
|
||||
throw new Error(`SQLite snapshot entry changed during publication: ${targetPath}`);
|
||||
}
|
||||
if (linked) {
|
||||
if (!linkedSourceIdentity || !sameFileIdentity(linkedSourceIdentity, initialTargetIdentity)) {
|
||||
throw new Error(`SQLite snapshot entry changed during publication: ${targetPath}`);
|
||||
}
|
||||
const sourceIdentity = await fs.lstat(sourcePath);
|
||||
if (!sameFileIdentity(sourceIdentity, initialTargetIdentity)) {
|
||||
throw new Error(`SQLite snapshot entry changed during publication: ${targetPath}`);
|
||||
}
|
||||
}
|
||||
await fs.unlink(sourcePath);
|
||||
const finalTargetIdentity = await fs.lstat(targetPath);
|
||||
if (!sameFileIdentity(initialTargetIdentity, finalTargetIdentity)) {
|
||||
throw new Error(`SQLite snapshot entry changed after publication: ${targetPath}`);
|
||||
}
|
||||
publishedEntries.set(entryName, finalTargetIdentity);
|
||||
}
|
||||
|
||||
async function copySnapshotEntryExclusive(sourcePath: string, targetPath: string): Promise<Stats> {
|
||||
const source = await fs.open(sourcePath, "r");
|
||||
let target: FileHandle | undefined;
|
||||
let targetIdentity: Stats | undefined;
|
||||
try {
|
||||
target = await fs.open(targetPath, "wx+", SNAPSHOT_FILE_MODE);
|
||||
targetIdentity = await target.stat();
|
||||
const buffer = Buffer.allocUnsafe(1024 * 1024);
|
||||
let offset = 0;
|
||||
while (true) {
|
||||
const { bytesRead } = await source.read(buffer, 0, buffer.length, offset);
|
||||
if (bytesRead === 0) {
|
||||
break;
|
||||
}
|
||||
let bytesWritten = 0;
|
||||
while (bytesWritten < bytesRead) {
|
||||
const result = await target.write(
|
||||
buffer,
|
||||
bytesWritten,
|
||||
bytesRead - bytesWritten,
|
||||
offset + bytesWritten,
|
||||
);
|
||||
if (result.bytesWritten === 0) {
|
||||
throw new Error(`SQLite snapshot entry copy made no progress: ${targetPath}`);
|
||||
}
|
||||
bytesWritten += result.bytesWritten;
|
||||
}
|
||||
offset += bytesRead;
|
||||
}
|
||||
await target.sync();
|
||||
const finalIdentity = await target.stat();
|
||||
const currentIdentity = await fs.lstat(targetPath);
|
||||
if (
|
||||
!sameFileIdentity(targetIdentity, finalIdentity) ||
|
||||
!sameFileIdentity(targetIdentity, currentIdentity)
|
||||
) {
|
||||
throw new Error(`SQLite snapshot entry changed during copy: ${targetPath}`);
|
||||
}
|
||||
return finalIdentity;
|
||||
} catch (error) {
|
||||
if (targetIdentity) {
|
||||
const currentIdentity = await fs.lstat(targetPath).catch(() => undefined);
|
||||
if (currentIdentity && sameFileIdentity(currentIdentity, targetIdentity)) {
|
||||
await fs.unlink(targetPath).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
await target?.close().catch(() => undefined);
|
||||
await source.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
async function assertExactSnapshotContents(snapshotDir: string): Promise<void> {
|
||||
await assertSnapshotContents(
|
||||
snapshotDir,
|
||||
new Set([SNAPSHOT_MANIFEST_FILENAME, SNAPSHOT_SQLITE_FILENAME]),
|
||||
);
|
||||
}
|
||||
|
||||
async function assertPendingSnapshotContents(snapshotDir: string): Promise<void> {
|
||||
await assertSnapshotContents(
|
||||
snapshotDir,
|
||||
new Set([SNAPSHOT_MANIFEST_FILENAME, SNAPSHOT_PENDING_FILENAME, SNAPSHOT_SQLITE_FILENAME]),
|
||||
);
|
||||
}
|
||||
|
||||
async function assertSnapshotContents(snapshotDir: string, expected: Set<string>): Promise<void> {
|
||||
const entries = await fs.readdir(snapshotDir, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
if (!expected.delete(entry.name)) {
|
||||
throw new Error(
|
||||
`SQLite snapshot contains unexpected entry: ${path.join(snapshotDir, entry.name)}`,
|
||||
);
|
||||
}
|
||||
if (entry.isSymbolicLink() || !entry.isFile()) {
|
||||
throw new Error(
|
||||
`SQLite snapshot entry must be a regular file: ${path.join(snapshotDir, entry.name)}`,
|
||||
);
|
||||
}
|
||||
const stat = await fs.lstat(path.join(snapshotDir, entry.name));
|
||||
if (stat.nlink > 1) {
|
||||
throw new Error(
|
||||
`SQLite snapshot entry must not be hardlinked: ${path.join(snapshotDir, entry.name)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (expected.size > 0) {
|
||||
throw new Error(`SQLite snapshot is missing ${[...expected].join(", ")}: ${snapshotDir}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function isIncompleteSnapshotDirectory(snapshotDir: string): Promise<boolean> {
|
||||
const entries = await fs.readdir(snapshotDir, { withFileTypes: true });
|
||||
const names = new Set(entries.map((entry) => entry.name));
|
||||
if (names.has(SNAPSHOT_PENDING_FILENAME)) {
|
||||
return true;
|
||||
}
|
||||
if (names.has(SNAPSHOT_MANIFEST_FILENAME)) {
|
||||
return false;
|
||||
}
|
||||
return entries.length === 0;
|
||||
}
|
||||
|
||||
async function assertFreshRestorePathsAbsent(databasePath: string): Promise<void> {
|
||||
for (const candidate of [
|
||||
databasePath,
|
||||
...SQLITE_SIDECAR_SUFFIXES.map((suffix) => `${databasePath}${suffix}`),
|
||||
]) {
|
||||
if (await lstatIfExists(candidate)) {
|
||||
throw new Error(`Fresh SQLite restore path already exists: ${candidate}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function assertNoSqliteSidecarsSync(databasePath: string): void {
|
||||
for (const suffix of SQLITE_SIDECAR_SUFFIXES) {
|
||||
const sidecarPath = `${databasePath}${suffix}`;
|
||||
try {
|
||||
fsSync.lstatSync(sidecarPath);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
continue;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
throw new Error(`Restored SQLite database has unexpected sidecar: ${sidecarPath}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function lstatIfExists(pathname: string): Promise<Stats | undefined> {
|
||||
try {
|
||||
return await fs.lstat(pathname);
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
|
||||
return undefined;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function removePrivateDirectoryIfOwned(
|
||||
directoryPath: string,
|
||||
expectedIdentity: Stats,
|
||||
allowedEntries: ReadonlySet<string>,
|
||||
): Promise<boolean> {
|
||||
const currentIdentity = await lstatIfExists(directoryPath);
|
||||
if (!currentIdentity) {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
currentIdentity.isSymbolicLink() ||
|
||||
!currentIdentity.isDirectory() ||
|
||||
!sameFileIdentity(currentIdentity, expectedIdentity)
|
||||
) {
|
||||
throw new Error(`Private SQLite staging directory changed before cleanup: ${directoryPath}`);
|
||||
}
|
||||
const entries = await fs.readdir(directoryPath, { withFileTypes: true });
|
||||
const verifiedPaths: string[] = [];
|
||||
for (const entry of entries) {
|
||||
const entryPath = path.join(directoryPath, entry.name);
|
||||
if (!allowedEntries.has(entry.name) || entry.isSymbolicLink() || !entry.isFile()) {
|
||||
throw new Error(`Private SQLite staging directory has unexpected entry: ${entryPath}`);
|
||||
}
|
||||
const stat = await fs.lstat(entryPath);
|
||||
if (stat.nlink > 1) {
|
||||
throw new Error(`Private SQLite staging file must not be hardlinked: ${entryPath}`);
|
||||
}
|
||||
verifiedPaths.push(entryPath);
|
||||
}
|
||||
await Promise.all(verifiedPaths.map(async (entryPath) => await fs.unlink(entryPath)));
|
||||
await fs.rmdir(directoryPath);
|
||||
return true;
|
||||
}
|
||||
|
||||
async function removePublishedSnapshotDirectoryIfOwned(
|
||||
directoryPath: string,
|
||||
expectedIdentity: Stats,
|
||||
publishedEntries: ReadonlyMap<string, Stats>,
|
||||
): Promise<boolean> {
|
||||
const currentIdentity = await lstatIfExists(directoryPath);
|
||||
if (
|
||||
!currentIdentity ||
|
||||
currentIdentity.isSymbolicLink() ||
|
||||
!currentIdentity.isDirectory() ||
|
||||
!sameFileIdentity(currentIdentity, expectedIdentity)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
const entries = await fs.readdir(directoryPath, { withFileTypes: true });
|
||||
for (const entry of entries) {
|
||||
const expectedEntryIdentity = publishedEntries.get(entry.name);
|
||||
if (!expectedEntryIdentity || entry.isSymbolicLink() || !entry.isFile()) {
|
||||
continue;
|
||||
}
|
||||
const entryPath = path.join(directoryPath, entry.name);
|
||||
const currentEntryIdentity = await fs.lstat(entryPath);
|
||||
if (sameFileIdentity(currentEntryIdentity, expectedEntryIdentity)) {
|
||||
await fs.unlink(entryPath);
|
||||
}
|
||||
}
|
||||
if ((await fs.readdir(directoryPath)).length > 0) {
|
||||
return false;
|
||||
}
|
||||
await fs.rmdir(directoryPath);
|
||||
return true;
|
||||
}
|
||||
@@ -1,316 +0,0 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import type { BigIntStats, Stats } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { sameFileIdentity } from "../infra/fs-safe-advanced.js";
|
||||
import { root } from "../infra/fs-safe.js";
|
||||
import { isValidAgentId, normalizeAgentId } from "../routing/session-key.js";
|
||||
import {
|
||||
SNAPSHOT_MANIFEST_FILENAME,
|
||||
SNAPSHOT_SQLITE_FILENAME,
|
||||
type SnapshotDatabaseManifest,
|
||||
type SnapshotManifest,
|
||||
} from "./snapshot-provider.js";
|
||||
|
||||
const MAX_MANIFEST_BYTES = 1024 * 1024;
|
||||
const MAX_SQLITE_USER_VERSION = 2_147_483_647;
|
||||
const MIN_SQLITE_USER_VERSION = -2_147_483_648;
|
||||
const SNAPSHOT_ID_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]{0,254}$/;
|
||||
const SHA256_PATTERN = /^[a-f0-9]{64}$/;
|
||||
|
||||
export type SnapshotArtifactDigest = {
|
||||
sha256: string;
|
||||
sizeBytes: number;
|
||||
stat: Stats;
|
||||
};
|
||||
|
||||
type OpenFileHandle = Awaited<ReturnType<typeof fs.open>>;
|
||||
|
||||
export function containsAsciiControlCharacter(value: string): boolean {
|
||||
for (let index = 0; index < value.length; index += 1) {
|
||||
const code = value.charCodeAt(index);
|
||||
if (code <= 0x1f || code === 0x7f) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export async function hashSnapshotArtifact(snapshotDir: string): Promise<SnapshotArtifactDigest> {
|
||||
const snapshotRoot = await root(snapshotDir);
|
||||
const opened = await snapshotRoot.open(SNAPSHOT_SQLITE_FILENAME, {
|
||||
hardlinks: "reject",
|
||||
symlinks: "reject",
|
||||
});
|
||||
try {
|
||||
return { ...(await hashFileHandle(opened.handle)), stat: opened.stat };
|
||||
} finally {
|
||||
await opened.handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function copySnapshotArtifact(
|
||||
snapshotDir: string,
|
||||
targetPath: string,
|
||||
): Promise<SnapshotArtifactDigest> {
|
||||
const snapshotRoot = await root(snapshotDir);
|
||||
const source = await snapshotRoot.open(SNAPSHOT_SQLITE_FILENAME, {
|
||||
hardlinks: "reject",
|
||||
symlinks: "reject",
|
||||
});
|
||||
let target: OpenFileHandle | undefined;
|
||||
let targetIdentity: Stats | undefined;
|
||||
try {
|
||||
target = await fs.open(targetPath, "wx+", 0o600);
|
||||
targetIdentity = await target.stat();
|
||||
const digest = await hashFileHandle(source.handle, target);
|
||||
await target.sync();
|
||||
const finalIdentity = await target.stat();
|
||||
const currentIdentity = await fs.lstat(targetPath);
|
||||
if (
|
||||
!sameFileIdentity(targetIdentity, finalIdentity) ||
|
||||
!sameFileIdentity(targetIdentity, currentIdentity)
|
||||
) {
|
||||
throw new Error(`Snapshot restore staging file changed during copy: ${targetPath}`);
|
||||
}
|
||||
return { ...digest, stat: finalIdentity };
|
||||
} catch (error) {
|
||||
await target?.close().catch(() => undefined);
|
||||
target = undefined;
|
||||
if (targetIdentity) {
|
||||
const currentIdentity = await fs.lstat(targetPath).catch(() => undefined);
|
||||
if (currentIdentity && sameFileIdentity(targetIdentity, currentIdentity)) {
|
||||
await fs.unlink(targetPath).catch(() => undefined);
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
} finally {
|
||||
await target?.close().catch(() => undefined);
|
||||
await source.handle.close().catch(() => undefined);
|
||||
}
|
||||
}
|
||||
|
||||
async function hashFileHandle(
|
||||
source: OpenFileHandle,
|
||||
target?: OpenFileHandle,
|
||||
): Promise<Omit<SnapshotArtifactDigest, "stat">> {
|
||||
const initialStat = await source.stat({ bigint: true });
|
||||
const hash = createHash("sha256");
|
||||
const buffer = Buffer.allocUnsafe(1024 * 1024);
|
||||
let sizeBytes = 0;
|
||||
while (true) {
|
||||
const { bytesRead } = await source.read(buffer, 0, buffer.length, sizeBytes);
|
||||
if (bytesRead === 0) {
|
||||
break;
|
||||
}
|
||||
hash.update(buffer.subarray(0, bytesRead));
|
||||
let bytesWritten = 0;
|
||||
if (target) {
|
||||
while (bytesWritten < bytesRead) {
|
||||
const result = await target.write(
|
||||
buffer,
|
||||
bytesWritten,
|
||||
bytesRead - bytesWritten,
|
||||
sizeBytes + bytesWritten,
|
||||
);
|
||||
if (result.bytesWritten === 0) {
|
||||
throw new Error("Snapshot restore staging copy made no progress.");
|
||||
}
|
||||
bytesWritten += result.bytesWritten;
|
||||
}
|
||||
}
|
||||
sizeBytes += bytesRead;
|
||||
}
|
||||
const finalStat = await source.stat({ bigint: true });
|
||||
if (!sameMutationFingerprint(initialStat, finalStat)) {
|
||||
throw new Error("Snapshot artifact changed while being read.");
|
||||
}
|
||||
return { sha256: hash.digest("hex"), sizeBytes };
|
||||
}
|
||||
|
||||
function sameMutationFingerprint(left: BigIntStats, right: BigIntStats): boolean {
|
||||
return (
|
||||
left.birthtimeNs === right.birthtimeNs &&
|
||||
left.ctimeNs === right.ctimeNs &&
|
||||
left.dev === right.dev &&
|
||||
left.ino === right.ino &&
|
||||
left.mtimeNs === right.mtimeNs &&
|
||||
left.size === right.size
|
||||
);
|
||||
}
|
||||
|
||||
export async function writeSnapshotManifest(
|
||||
snapshotDir: string,
|
||||
manifest: SnapshotManifest,
|
||||
): Promise<void> {
|
||||
const manifestPath = path.join(snapshotDir, SNAPSHOT_MANIFEST_FILENAME);
|
||||
const handle = await fs.open(manifestPath, "wx+", 0o600);
|
||||
try {
|
||||
await handle.writeFile(`${JSON.stringify(manifest, null, 2)}\n`, "utf8");
|
||||
await handle.sync();
|
||||
} finally {
|
||||
await handle.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function readSnapshotManifest(
|
||||
snapshotDir: string,
|
||||
expectedSnapshotId = path.basename(snapshotDir),
|
||||
): Promise<SnapshotManifest> {
|
||||
const snapshotRoot = await root(snapshotDir);
|
||||
const manifestPath = path.join(snapshotDir, SNAPSHOT_MANIFEST_FILENAME);
|
||||
const result = await snapshotRoot.read(SNAPSHOT_MANIFEST_FILENAME, {
|
||||
hardlinks: "reject",
|
||||
maxBytes: MAX_MANIFEST_BYTES,
|
||||
symlinks: "reject",
|
||||
});
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(result.buffer.toString("utf8")) as unknown;
|
||||
} catch (error) {
|
||||
throw new Error(`Snapshot manifest is not valid JSON: ${manifestPath}`, { cause: error });
|
||||
}
|
||||
return parseSnapshotManifest(parsed, manifestPath, expectedSnapshotId);
|
||||
}
|
||||
|
||||
export function parseSnapshotManifest(
|
||||
value: unknown,
|
||||
manifestPath: string,
|
||||
expectedSnapshotId: string,
|
||||
): SnapshotManifest {
|
||||
const record = requireRecord(value, "manifest", manifestPath);
|
||||
requireExactKeys(record, ["schemaVersion", "snapshotId", "createdAt", "database", "artifact"]);
|
||||
if (record.schemaVersion !== 1) {
|
||||
throw new Error(
|
||||
`Unsupported snapshot manifest schemaVersion ${String(record.schemaVersion)}: ${manifestPath}`,
|
||||
);
|
||||
}
|
||||
const snapshotId = requireSnapshotId(record.snapshotId, manifestPath);
|
||||
if (snapshotId !== expectedSnapshotId) {
|
||||
throw new Error(
|
||||
`Snapshot manifest id ${snapshotId} does not match directory ${expectedSnapshotId}: ${manifestPath}`,
|
||||
);
|
||||
}
|
||||
const createdAt = requireCanonicalTimestamp(record.createdAt, manifestPath);
|
||||
const database = parseSnapshotDatabase(record.database, manifestPath);
|
||||
const artifactRecord = requireRecord(record.artifact, "artifact", manifestPath);
|
||||
requireExactKeys(artifactRecord, ["path", "sha256", "sizeBytes"]);
|
||||
if (artifactRecord.path !== SNAPSHOT_SQLITE_FILENAME) {
|
||||
throw new Error(
|
||||
`Snapshot manifest artifact.path must be ${SNAPSHOT_SQLITE_FILENAME}: ${manifestPath}`,
|
||||
);
|
||||
}
|
||||
if (typeof artifactRecord.sha256 !== "string" || !SHA256_PATTERN.test(artifactRecord.sha256)) {
|
||||
throw new Error(`Snapshot manifest artifact.sha256 is invalid: ${manifestPath}`);
|
||||
}
|
||||
if (!Number.isSafeInteger(artifactRecord.sizeBytes) || Number(artifactRecord.sizeBytes) <= 0) {
|
||||
throw new Error(`Snapshot manifest artifact.sizeBytes is invalid: ${manifestPath}`);
|
||||
}
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
snapshotId,
|
||||
createdAt,
|
||||
database,
|
||||
artifact: {
|
||||
path: SNAPSHOT_SQLITE_FILENAME,
|
||||
sha256: artifactRecord.sha256,
|
||||
sizeBytes: Number(artifactRecord.sizeBytes),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function parseSnapshotDatabase(value: unknown, manifestPath: string): SnapshotDatabaseManifest {
|
||||
const database = requireRecord(value, "database", manifestPath);
|
||||
const role = database.role;
|
||||
const basename = requireSafeText(database.basename, "database.basename", manifestPath, 255);
|
||||
if (path.basename(basename) !== basename || basename === "." || basename === "..") {
|
||||
throw new Error(`Snapshot manifest database.basename is invalid: ${manifestPath}`);
|
||||
}
|
||||
const userVersion = requireSqliteUserVersion(database.userVersion, manifestPath);
|
||||
if (role === "global") {
|
||||
requireExactKeys(database, ["role", "basename", "userVersion"]);
|
||||
return { role, basename, userVersion };
|
||||
}
|
||||
if (role === "agent") {
|
||||
requireExactKeys(database, ["role", "agentId", "basename", "userVersion"]);
|
||||
const agentId = requireSafeText(database.agentId, "database.agentId", manifestPath, 64);
|
||||
if (!isValidAgentId(agentId) || normalizeAgentId(agentId) !== agentId) {
|
||||
throw new Error(`Snapshot manifest database.agentId is invalid: ${manifestPath}`);
|
||||
}
|
||||
return { role, agentId, basename, userVersion };
|
||||
}
|
||||
if (role === "generic") {
|
||||
requireExactKeys(database, ["role", "id", "basename", "userVersion"]);
|
||||
const id = requireSafeText(database.id, "database.id", manifestPath, 256);
|
||||
return { role, id, basename, userVersion };
|
||||
}
|
||||
throw new Error(`Snapshot manifest database.role is invalid: ${manifestPath}`);
|
||||
}
|
||||
|
||||
function requireRecord(
|
||||
value: unknown,
|
||||
field: string,
|
||||
manifestPath: string,
|
||||
): Record<string, unknown> {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
throw new Error(`Snapshot manifest ${field} must be an object: ${manifestPath}`);
|
||||
}
|
||||
return value as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function requireExactKeys(record: Record<string, unknown>, expectedKeys: readonly string[]): void {
|
||||
const actual = Object.keys(record).toSorted();
|
||||
const expected = [...expectedKeys].toSorted();
|
||||
if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) {
|
||||
throw new Error(
|
||||
`Snapshot manifest fields must be exactly ${expectedKeys.join(", ")}; got ${actual.join(", ")}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function requireSnapshotId(value: unknown, manifestPath: string): string {
|
||||
if (typeof value !== "string" || !SNAPSHOT_ID_PATTERN.test(value)) {
|
||||
throw new Error(`Snapshot manifest snapshotId is invalid: ${manifestPath}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requireCanonicalTimestamp(value: unknown, manifestPath: string): string {
|
||||
if (typeof value !== "string") {
|
||||
throw new Error(`Snapshot manifest createdAt is invalid: ${manifestPath}`);
|
||||
}
|
||||
const parsed = new Date(value);
|
||||
if (!Number.isFinite(parsed.getTime()) || parsed.toISOString() !== value) {
|
||||
throw new Error(`Snapshot manifest createdAt is not canonical ISO 8601: ${manifestPath}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requireSafeText(
|
||||
value: unknown,
|
||||
field: string,
|
||||
manifestPath: string,
|
||||
maxLength: number,
|
||||
): string {
|
||||
if (
|
||||
typeof value !== "string" ||
|
||||
value.length === 0 ||
|
||||
value.length > maxLength ||
|
||||
value.trim() !== value ||
|
||||
containsAsciiControlCharacter(value)
|
||||
) {
|
||||
throw new Error(`Snapshot manifest ${field} is invalid: ${manifestPath}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function requireSqliteUserVersion(value: unknown, manifestPath: string): number {
|
||||
if (
|
||||
!Number.isSafeInteger(value) ||
|
||||
Number(value) < MIN_SQLITE_USER_VERSION ||
|
||||
Number(value) > MAX_SQLITE_USER_VERSION
|
||||
) {
|
||||
throw new Error(`Snapshot manifest database.userVersion is invalid: ${manifestPath}`);
|
||||
}
|
||||
return Number(value);
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
export const SNAPSHOT_MANIFEST_FILENAME = "manifest.json";
|
||||
export const SNAPSHOT_SQLITE_FILENAME = "database.sqlite";
|
||||
|
||||
export type SnapshotDatabaseIdentity =
|
||||
| { readonly role: "global" }
|
||||
| { readonly role: "agent"; readonly agentId: string }
|
||||
| { readonly role: "generic"; readonly id: string };
|
||||
|
||||
export type SnapshotDatabaseRef = {
|
||||
readonly path: string;
|
||||
readonly identity: SnapshotDatabaseIdentity;
|
||||
};
|
||||
|
||||
export type SnapshotDatabaseManifest =
|
||||
| {
|
||||
readonly role: "global";
|
||||
readonly basename: string;
|
||||
readonly userVersion: number;
|
||||
}
|
||||
| {
|
||||
readonly role: "agent";
|
||||
readonly agentId: string;
|
||||
readonly basename: string;
|
||||
readonly userVersion: number;
|
||||
}
|
||||
| {
|
||||
readonly role: "generic";
|
||||
readonly id: string;
|
||||
readonly basename: string;
|
||||
readonly userVersion: number;
|
||||
};
|
||||
|
||||
export type SnapshotManifest = {
|
||||
readonly schemaVersion: 1;
|
||||
readonly snapshotId: string;
|
||||
readonly createdAt: string;
|
||||
readonly database: SnapshotDatabaseManifest;
|
||||
readonly artifact: {
|
||||
readonly path: typeof SNAPSHOT_SQLITE_FILENAME;
|
||||
readonly sha256: string;
|
||||
readonly sizeBytes: number;
|
||||
};
|
||||
};
|
||||
|
||||
export type SnapshotRef = {
|
||||
readonly path: string;
|
||||
};
|
||||
|
||||
export type SnapshotResult = {
|
||||
readonly ref: SnapshotRef;
|
||||
readonly manifest: SnapshotManifest;
|
||||
};
|
||||
|
||||
export type SnapshotVerificationResult = {
|
||||
readonly ok: true;
|
||||
readonly manifest: SnapshotManifest;
|
||||
};
|
||||
|
||||
export type SnapshotSummary = SnapshotResult;
|
||||
|
||||
export type SqliteSnapshotProvider = {
|
||||
create(database: SnapshotDatabaseRef): Promise<SnapshotResult>;
|
||||
list(): Promise<SnapshotSummary[]>;
|
||||
restoreFresh(snapshot: SnapshotRef, targetPath: string): Promise<SnapshotVerificationResult>;
|
||||
verify(snapshot: SnapshotRef): Promise<SnapshotVerificationResult>;
|
||||
};
|
||||
Reference in New Issue
Block a user