refactor: move non-session runtime journals to SQLite (#109427)

* refactor: migrate runtime JSONL state to SQLite

* chore: refresh SQLite migration validation

* fix: preserve safe audit migration order

* fix: sanitize retired audit archives safely

* fix: satisfy SQLite storage architecture gates

* fix: serialize plugin doctor state migrations

* fix: bound memory event SQLite projections

* fix: rotate memory event projections safely

* fix: preserve imported event retention age

* fix: harden memory event export projection

* fix: canonicalize memory event migrations

* fix: harden legacy journal migrations

* fix: report unsafe legacy journal paths

* fix: preserve journal ownership during migration

* fix: keep legacy file imports doctor-only

* fix: align SQLite audit CI coverage

* refactor: keep state helpers module-private

* fix: harden legacy state migration recovery

* fix: reconcile SQLite state audit migration

* test: use neutral audit redaction fixtures

* fix: close SQLite migration recovery gaps

* refactor: split audit migration recovery helpers

* test: prove completed audit pads stay out of backups

* fix: preserve audit record ordinals across blank lines

* fix: align SQLite audit CI contracts

* build: package SQLite audit E2E entries

* fix: preserve audit ordinals in backup snapshots
This commit is contained in:
Peter Steinberger
2026-07-18 11:42:14 +01:00
committed by GitHub
parent bc6d9eb16a
commit 9d97e10efe
90 changed files with 10450 additions and 716 deletions
@@ -166,7 +166,7 @@ ae67c256150b14f1a58738dce0a703071035b86ec26fc344ab03e942253aa72c module/memory-
4205ab767b79e740c7c7137079b19b315c85f7f8c748592840f00bbf70a71709 module/memory-core-host-engine-foundation
cb8dcef652580ca82242d6e9506eda094006c85848f9b16b8b1c1aadff1706a8 module/memory-core-host-engine-qmd
bbc06f2bb23d9e4a3445744711f691de9f902270e61c9740fd746f222d10fccb module/memory-core-host-engine-storage
b3255578cdf355d224d840a57e4ee7e190e60c02cbed1bfff2a222c08c3df3a8 module/memory-core-host-events
39e86819441281365dedb3d7d59979672bb90196f49206edb50b879bd747c9f0 module/memory-core-host-events
fe87e7d794bbbdaed3c834b218a787e98c0e69b26b0fac582b0668b8effab9b4 module/memory-core-host-multimodal
746e155bb06b398749a6a1bfdc1b435b1ec1941dca3854554a912577731218dd module/memory-core-host-query
e2377479799d52d8f67dcccf8f499f28a63948b0c03fa6043fc67e476a22ec0d module/memory-core-host-runtime-cli
@@ -175,7 +175,7 @@ f8eb835f0769eba8324033141ecddb9f45bdb3d3241d2a5959fe8426ba6a2280 module/memory-
ebbf937b2258487f773b8a6fbd6c648feea96399cae5fbea18ec7d29e027523d module/memory-core-host-secret
16bdd82751898ffa98157fd27c86fa0eb9ea415fb8ed703eb4e9b170f0a9501b module/memory-core-host-status
646773d8282a2ac6a89101685c406200935ddb3459a2830e820da132fa433b3c module/memory-host-core
7d68cbdfa0d5fad1ab32ba3792787daf386bc4ff7975fd5e25117e92563a2897 module/memory-host-events
f7bde3a0231ac75c128d6ae367d5020de886715fe9edbb2faaf673c6bd168398 module/memory-host-events
059856bc3671ccba7a2191f47c0295e0a91a374f7048cc2bc22784b389d15182 module/memory-host-files
ba07f497aa5469915ebb5b6787148ff1f0f267d7e48c48f0c95d32b7dda8d91a module/memory-host-markdown
896d315fb78af09effd0af467e2b6e84331272b0123d4132e5b1f2254e80108e module/memory-host-search
+3 -4
View File
@@ -548,10 +548,9 @@ Two ways to start an ACP session:
</ParamField>
<ParamField path="streamTo" type='"parent"'>
`"parent"` streams initial ACP run progress summaries back to the requester
session as system events. Accepted responses include `streamLogPath`
pointing to a session-scoped JSONL log (`<sessionId>.acp-stream.jsonl`) you
can tail for full relay history. Parent progress streams show assistant
commentary and ACP status progress by default unless
session as system events. OpenClaw records the full relay history in the
child agent's SQLite state and removes it with the child session. Parent
progress streams show assistant commentary and ACP status progress by default unless
`streaming.progress.commentary=false`. Discord also defaults parent
previews to progress mode when no stream mode is configured. Status
progress still honors `acp.stream.tagVisibility`, so tags such as `plan`
@@ -3,14 +3,16 @@ import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { DatabaseSync } from "node:sqlite";
import { expectDefined } from "@openclaw/normalization-core";
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import {
ensureMemoryIndexSchema,
loadSqliteVecExtension,
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
import { readMemoryHostEventRecords } from "openclaw/plugin-sdk/memory-host-events";
import {
createPluginStateKeyedStoreForTests,
getPluginStateCapacityForTests,
importPluginStateEntriesForDoctorForTests,
resetPluginStateStoreForTests,
} from "openclaw/plugin-sdk/plugin-state-test-runtime";
import type {
@@ -32,12 +34,18 @@ import {
shortTermTestState as shortTermTesting,
} from "./src/test-helpers.js";
function requireStateMigration(index: number) {
return expectDefined(stateMigrations[index], `Memory Core state migration ${index}`);
}
function createDoctorContext(env: NodeJS.ProcessEnv): PluginDoctorStateMigrationContext {
return {
getPluginStateCapacity() {
return getPluginStateCapacityForTests("memory-core", env);
},
importPluginStateEntries(options, entries) {
importPluginStateEntriesForDoctorForTests(
"memory-core",
{ ...options, env: options.env ?? env },
entries,
);
},
openPluginStateKeyedStore<T>(options: OpenKeyedStoreOptions) {
return createPluginStateKeyedStoreForTests<T>("memory-core", {
...options,
@@ -57,6 +65,26 @@ function legacyMemoryIndexMigration() {
return migration;
}
function dreamingStateMigration() {
const migration = stateMigrations.find(
(entry) => entry.id === "memory-core-dreams-json-to-sqlite",
);
if (!migration) {
throw new Error("expected memory-core dreaming state migration");
}
return migration;
}
function hostEventsMigration() {
const migration = stateMigrations.find(
(entry) => entry.id === "memory-core-host-events-jsonl-to-sqlite",
);
if (!migration) {
throw new Error("expected memory-core host events migration");
}
return migration;
}
function qmdFileLockMigration() {
const migration = stateMigrations.find(
(entry) => entry.id === "memory-core-qmd-file-locks-to-sqlite-leases",
@@ -458,6 +486,628 @@ describe("memory-core doctor dreaming migration", () => {
};
}
it("treats a missing legacy host event directory as no state", async () => {
await fs.rm(path.join(workspaceDir, "memory"), { recursive: true });
const migration = hostEventsMigration();
await expect(migration.detectLegacyState(migrationParams())).resolves.toBeNull();
await expect(migration.migrateLegacyState(migrationParams())).resolves.toEqual({
changes: [],
warnings: [],
});
});
it("imports legacy memory host events into plugin state", async () => {
const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl");
await fs.writeFile(
eventPath,
`${JSON.stringify({
type: "memory.recall.recorded",
timestamp: "2026-07-01T00:00:00.000Z",
query: "sqlite policy",
resultCount: 0,
results: [],
})}\n`,
"utf8",
);
const migration = hostEventsMigration();
await expect(migration.detectLegacyState(migrationParams())).resolves.toEqual({
preview: [expect.stringContaining("Memory Core host events")],
});
const store = context().openPluginStateKeyedStore<{
kind: "event";
workspaceKey: string;
event: { type: string; query: string };
recordedAt: number;
sequence: number;
}>({ namespace: "memory-host.events", maxEntries: 10_000 });
await store.register("runtime-event", {
kind: "event",
workspaceKey: path.resolve(workspaceDir).replace(/\\/g, "/"),
event: {
type: "memory.recall.recorded",
query: "runtime after upgrade",
},
recordedAt: Date.parse("2026-07-02T00:00:00.000Z"),
sequence: 1,
});
const result = await migration.migrateLegacyState(migrationParams());
expect(result.warnings).toEqual([]);
expect(result.changes).toEqual([
"Migrated Memory Core host events -> SQLite plugin state (1 new row(s))",
expect.stringContaining("Archived Memory Core host events legacy source"),
]);
const entries = await store.entries();
const events = entries
.flatMap((entry) => (entry.value.kind === "event" ? [entry.value] : []))
.toSorted((left, right) => left.sequence - right.sequence);
expect(events.map((entry) => entry.event.query)).toEqual([
"sqlite policy",
"runtime after upgrade",
]);
expect(events[0]?.sequence).toBeLessThan(0);
const migratedEntry = entries.find((entry) => entry.value.sequence < 0);
expect(migratedEntry?.createdAt).toBe(migratedEntry?.value.sequence);
const cursors = await context()
.openPluginStateKeyedStore<{ kind: "cursor"; lastSequence: number }>({
namespace: "memory-host.event-cursors",
maxEntries: 1_000,
})
.entries();
expect(cursors).toHaveLength(1);
expect(cursors[0]?.value).toEqual({ kind: "cursor", lastSequence: 1 });
await expect(fs.access(`${eventPath}.migrated`)).resolves.toBeUndefined();
});
it("keeps identical events recreated by an older writer as a new archive generation", async () => {
const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl");
const event = {
type: "memory.recall.recorded",
timestamp: "2026-07-01T00:00:00.000Z",
query: "repeated after downgrade",
resultCount: 0,
results: [],
};
const migration = hostEventsMigration();
await fs.writeFile(eventPath, `${JSON.stringify(event)}\n`, "utf8");
await migration.migrateLegacyState(migrationParams());
await fs.writeFile(eventPath, `${JSON.stringify(event)}\n`, "utf8");
const repeated = await migration.migrateLegacyState(migrationParams());
expect(repeated.warnings).toEqual([]);
expect(repeated.changes).toEqual([
"Migrated Memory Core host events -> SQLite plugin state (1 new row(s))",
expect.stringContaining("events.jsonl.migrated.2"),
]);
await expect(readMemoryHostEventRecords({ workspaceDir, env })).resolves.toHaveLength(2);
await expect(fs.access(`${eventPath}.migrated`)).resolves.toBeUndefined();
await expect(fs.access(`${eventPath}.migrated.2`)).resolves.toBeUndefined();
});
it("recovers appends written through an open legacy descriptor after archival", async () => {
const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl");
const event = (query: string) => ({
type: "memory.recall.recorded" as const,
timestamp: "2026-07-01T00:00:00.000Z",
query,
resultCount: 0,
results: [],
});
await fs.writeFile(eventPath, `${JSON.stringify(event("before claim"))}\n`, "utf8");
const oldWriter = await fs.open(eventPath, "a");
const migration = hostEventsMigration();
try {
await migration.migrateLegacyState(migrationParams());
await fs.writeFile(eventPath, `${JSON.stringify(event("newer generation"))}\n`, "utf8");
await migration.migrateLegacyState(migrationParams());
await oldWriter.appendFile(`${JSON.stringify(event("late append"))}\n`, "utf8");
await oldWriter.sync();
} finally {
await oldWriter.close();
}
await expect(migration.detectLegacyState(migrationParams())).resolves.toEqual({
preview: [expect.stringContaining("events.jsonl.migrated")],
});
const recovered = await migration.migrateLegacyState(migrationParams());
expect(recovered.warnings).toEqual([]);
expect(recovered.changes).toEqual([
expect.stringContaining("Recovered 1 later Memory Core host event row"),
]);
await expect(readMemoryHostEventRecords({ workspaceDir, env })).resolves.toMatchObject([
{ query: "before claim" },
{ query: "newer generation" },
{ query: "late append" },
]);
await expect(
readMemoryHostEventRecords({ workspaceDir, env, limit: 1 }),
).resolves.toMatchObject([{ query: "late append" }]);
await expect(migration.detectLegacyState(migrationParams())).resolves.toBeNull();
await fs.writeFile(eventPath, `${JSON.stringify(event("after recovery generation"))}\n`);
await migration.migrateLegacyState(migrationParams());
await expect(
readMemoryHostEventRecords({ workspaceDir, env, limit: 1 }),
).resolves.toMatchObject([{ query: "after recovery generation" }]);
});
it("fails closed when a checkpointed host event archive changes other than by append", async () => {
const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl");
const archivedPath = `${eventPath}.migrated`;
await fs.writeFile(
eventPath,
`${JSON.stringify({
type: "memory.recall.recorded",
timestamp: "2026-07-01T00:00:00.000Z",
query: "original archive row",
resultCount: 0,
results: [],
})}\n`,
"utf8",
);
const migration = hostEventsMigration();
await migration.migrateLegacyState(migrationParams());
await fs.writeFile(
archivedPath,
`${JSON.stringify({
type: "memory.recall.recorded",
timestamp: "2026-07-01T00:00:00.000Z",
query: "rewritten archive row",
resultCount: 0,
results: [],
})}\n`,
"utf8",
);
const result = await migration.migrateLegacyState(migrationParams());
expect(result.changes).toEqual([]);
expect(result.warnings).toEqual([expect.stringContaining("changed other than by append")]);
await expect(readMemoryHostEventRecords({ workspaceDir, env })).resolves.toMatchObject([
{ query: "original archive row" },
]);
await expect(fs.readFile(archivedPath, "utf8")).resolves.toContain("rewritten archive row");
});
it("orders and limits migrated host events by archive generation", async () => {
const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl");
const event = (query: string) => ({
type: "memory.recall.recorded",
timestamp: "2026-07-01T00:00:00.000Z",
query,
resultCount: 0,
results: [],
});
const migration = hostEventsMigration();
await fs.writeFile(eventPath, `${JSON.stringify(event("older generation"))}\n`, "utf8");
await migration.migrateLegacyState(migrationParams());
await fs.writeFile(eventPath, `${JSON.stringify(event("newer generation"))}\n`, "utf8");
await migration.migrateLegacyState(migrationParams());
await expect(readMemoryHostEventRecords({ workspaceDir, env })).resolves.toMatchObject([
{ query: "older generation" },
{ query: "newer generation" },
]);
await expect(
readMemoryHostEventRecords({ workspaceDir, env, limit: 1 }),
).resolves.toMatchObject([{ query: "newer generation" }]);
});
it("refuses to replay a checkpointless older archive after a newer generation", async () => {
const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl");
const event = (query: string) => ({
type: "memory.recall.recorded" as const,
timestamp: "2026-07-01T00:00:00.000Z",
query,
resultCount: 0,
results: [],
});
const migration = hostEventsMigration();
await fs.writeFile(eventPath, `${JSON.stringify(event("older generation"))}\n`, "utf8");
await migration.migrateLegacyState(migrationParams());
await context()
.openPluginStateKeyedStore({ namespace: "memory-host.events", maxEntries: 10_000 })
.clear();
await fs.writeFile(eventPath, `${JSON.stringify(event("newer generation"))}\n`, "utf8");
await migration.migrateLegacyState(migrationParams());
await context()
.openPluginStateKeyedStore({
namespace: "memory-host.event-migration-checkpoints",
maxEntries: 10_000,
overflowPolicy: "reject-new",
})
.clear();
const replay = await migration.migrateLegacyState(migrationParams());
expect(replay.changes).toEqual([]);
expect(replay.warnings).toEqual([
expect.stringContaining(
"has no durable checkpoint and later generations are already imported",
),
]);
await expect(readMemoryHostEventRecords({ workspaceDir, env })).resolves.toMatchObject([
{ query: "newer generation" },
]);
});
it("defers host event import when a source contains invalid rows", async () => {
const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl");
await fs.writeFile(
eventPath,
`${JSON.stringify({
type: "memory.recall.recorded",
timestamp: "2026-07-01T00:00:00.000Z",
query: "valid before malformed",
resultCount: 0,
results: [],
})}\n${JSON.stringify({
type: "memory.recall.recorded",
timestamp: "2026-07-01T00:00:01.000Z",
})}\n{malformed\n`,
"utf8",
);
const result = await hostEventsMigration().migrateLegacyState(migrationParams());
expect(result.changes).toEqual([]);
expect(result.warnings).toEqual([
expect.stringContaining("Skipped invalid Memory Core host event"),
expect.stringContaining("Skipped malformed Memory Core host event"),
expect.stringContaining("invalid rows still require repair"),
]);
await expect(readMemoryHostEventRecords({ workspaceDir, env })).resolves.toEqual([]);
await expect(fs.access(eventPath)).resolves.toBeUndefined();
await expect(fs.access(`${eventPath}.migrated`)).rejects.toMatchObject({ code: "ENOENT" });
});
it("does not import newer host event generations before an older source is repaired", async () => {
const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl");
const archivedPath = `${eventPath}.migrated`;
const event = (query: string) => ({
type: "memory.recall.recorded" as const,
timestamp: "2026-07-01T00:00:00.000Z",
query,
resultCount: 0,
results: [],
});
await fs.writeFile(archivedPath, "{malformed\n", "utf8");
await fs.writeFile(eventPath, `${JSON.stringify(event("newer generation"))}\n`, "utf8");
const migration = hostEventsMigration();
const blocked = await migration.migrateLegacyState(migrationParams());
expect(blocked.changes).toEqual([]);
expect(blocked.warnings).toEqual([
expect.stringContaining("Skipped malformed Memory Core host event"),
expect.stringContaining("invalid rows still require repair"),
]);
await expect(readMemoryHostEventRecords({ workspaceDir, env })).resolves.toEqual([]);
await expect(fs.access(eventPath)).resolves.toBeUndefined();
await fs.writeFile(
archivedPath,
`${JSON.stringify(event("repaired older generation"))}\n`,
"utf8",
);
const repaired = await migration.migrateLegacyState(migrationParams());
expect(repaired.warnings).toEqual([]);
expect(repaired.changes).toEqual([
expect.stringContaining("Recovered 1 later Memory Core host event row"),
"Migrated Memory Core host events -> SQLite plugin state (1 new row(s))",
expect.stringContaining("Archived Memory Core host events legacy source"),
]);
await expect(readMemoryHostEventRecords({ workspaceDir, env })).resolves.toMatchObject([
{ query: "repaired older generation" },
{ query: "newer generation" },
]);
});
it("imports host events after malformed rows are repaired", async () => {
const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl");
const event = {
type: "memory.recall.recorded",
timestamp: "2026-07-01T00:00:00.000Z",
query: "stable after repair",
resultCount: 0,
results: [],
};
await fs.writeFile(eventPath, `{malformed\n${JSON.stringify(event)}\n`, "utf8");
const migration = hostEventsMigration();
await migration.migrateLegacyState(migrationParams());
await fs.writeFile(
eventPath,
` ${JSON.stringify({
results: [],
resultCount: 0,
query: event.query,
timestamp: event.timestamp,
type: event.type,
})} \n`,
"utf8",
);
const repaired = await migration.migrateLegacyState(migrationParams());
expect(repaired.warnings).toEqual([]);
expect(repaired.changes).toEqual([
"Migrated Memory Core host events -> SQLite plugin state (1 new row(s))",
expect.stringContaining("Archived Memory Core host events legacy source"),
]);
await expect(readMemoryHostEventRecords({ workspaceDir, env })).resolves.toMatchObject([
{ query: "stable after repair" },
]);
await expect(fs.access(`${eventPath}.migrated`)).resolves.toBeUndefined();
});
it("preserves legacy host event append order when timestamps move backward", async () => {
const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl");
const events = [
{
type: "memory.recall.recorded",
timestamp: "2026-07-02T00:00:00.000Z",
query: "appended first",
resultCount: 0,
results: [],
},
{
type: "memory.recall.recorded",
timestamp: "2026-07-01T00:00:00.000Z",
query: "appended last after clock rollback",
resultCount: 0,
results: [],
},
];
await fs.writeFile(eventPath, `${events.map((event) => JSON.stringify(event)).join("\n")}\n`);
const result = await hostEventsMigration().migrateLegacyState(migrationParams());
expect(result.warnings).toEqual([]);
await expect(readMemoryHostEventRecords({ workspaceDir, env })).resolves.toMatchObject([
{ query: "appended first" },
{ query: "appended last after clock rollback" },
]);
await expect(
readMemoryHostEventRecords({ workspaceDir, env, limit: 1 }),
).resolves.toMatchObject([{ query: "appended last after clock rollback" }]);
});
it.runIf(process.platform !== "win32")(
"canonicalizes and deduplicates aliased legacy host event sources",
async () => {
const workspaceAlias = path.join(rootDir, "workspace-alias");
const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl");
await fs.symlink(workspaceDir, workspaceAlias);
await fs.writeFile(
eventPath,
`${JSON.stringify({
type: "memory.recall.recorded",
timestamp: "2026-07-01T00:00:00.000Z",
query: "canonical alias",
resultCount: 0,
results: [],
})}\n`,
"utf8",
);
const params = migrationParams({
agents: {
list: [
{ id: "main", workspace: workspaceDir },
{ id: "alias", workspace: workspaceAlias },
],
},
});
const migration = hostEventsMigration();
await expect(migration.detectLegacyState(params)).resolves.toEqual({
preview: [expect.stringContaining("Memory Core host events")],
});
const result = await migration.migrateLegacyState(params);
expect(result.warnings).toEqual([]);
expect(result.changes).toEqual([
"Migrated Memory Core host events -> SQLite plugin state (1 new row(s))",
expect.stringContaining("Archived Memory Core host events legacy source"),
]);
await expect(
readMemoryHostEventRecords({ workspaceDir: workspaceAlias, env }),
).resolves.toMatchObject([{ query: "canonical alias" }]);
await expect(fs.access(`${eventPath}.migrated`)).resolves.toBeUndefined();
},
);
it.runIf(process.platform !== "win32")(
"rejects legacy host events beneath symlinked workspace parents",
async () => {
const externalMemoryDir = await fs.mkdtemp(
path.join(os.tmpdir(), "openclaw-memory-core-external-events-"),
);
const externalEventPath = path.join(externalMemoryDir, ".dreams", "events.jsonl");
try {
await fs.rm(path.join(workspaceDir, "memory"), { recursive: true });
await fs.mkdir(path.dirname(externalEventPath), { recursive: true });
await fs.writeFile(
externalEventPath,
`${JSON.stringify({
type: "memory.recall.recorded",
timestamp: "2026-07-01T00:00:00.000Z",
query: "outside workspace",
resultCount: 0,
results: [],
})}\n`,
"utf8",
);
await fs.symlink(externalMemoryDir, path.join(workspaceDir, "memory"));
await expect(hostEventsMigration().detectLegacyState(migrationParams())).resolves.toEqual({
preview: [expect.stringContaining("requires safe-path repair")],
});
const result = await hostEventsMigration().migrateLegacyState(migrationParams());
expect(result.changes).toEqual([]);
expect(result.warnings).toEqual([
expect.stringContaining("Skipped unsafe Memory Core host event source"),
]);
await expect(fs.readFile(externalEventPath, "utf8")).resolves.toContain(
"outside workspace",
);
await expect(fs.access(`${externalEventPath}.migrated`)).rejects.toMatchObject({
code: "ENOENT",
});
} finally {
await fs.rm(externalMemoryDir, { recursive: true, force: true });
}
},
);
it("imports the newest retained tail from an oversized legacy host event log", async () => {
const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl");
const events = Array.from({ length: 10_002 }, (_, index) => ({
type: "memory.recall.recorded",
timestamp: "2026-07-01T00:00:00.000Z",
query: `oversized-${index}`,
resultCount: 0,
results: [],
}));
await fs.writeFile(eventPath, `${events.map((event) => JSON.stringify(event)).join("\n")}\n`);
const result = await hostEventsMigration().migrateLegacyState(migrationParams());
expect(result.warnings).toEqual([]);
expect(result.changes).toEqual([
"Migrated Memory Core host events -> SQLite plugin state (10000 new row(s))",
expect.stringContaining("Archived Memory Core host events legacy source"),
]);
const imported = await readMemoryHostEventRecords({ workspaceDir, env });
expect(imported).toHaveLength(10_000);
expect(imported[0]).toMatchObject({ query: "oversized-2" });
expect(imported.at(-1)).toMatchObject({ query: "oversized-10001" });
await context()
.openPluginStateKeyedStore({
namespace: "memory-host.event-migration-checkpoints",
maxEntries: 10_000,
overflowPolicy: "reject-new",
})
.clear();
const retriedWithoutCheckpoint =
await hostEventsMigration().migrateLegacyState(migrationParams());
expect(retriedWithoutCheckpoint.warnings).toEqual([]);
const afterCheckpointRetry = await readMemoryHostEventRecords({ workspaceDir, env });
expect(afterCheckpointRetry[0]).toMatchObject({ query: "oversized-2" });
expect(afterCheckpointRetry.at(-1)).toMatchObject({ query: "oversized-10001" });
await fs.writeFile(
eventPath,
`${JSON.stringify({
type: "memory.recall.recorded",
timestamp: "2026-07-02T00:00:00.000Z",
query: "newer recreated generation",
resultCount: 0,
results: [],
})}\n`,
);
const repeated = await hostEventsMigration().migrateLegacyState(migrationParams());
expect(repeated.warnings).toEqual([]);
expect(repeated.changes[0]).toContain("1 new row");
const afterRepeated = await readMemoryHostEventRecords({ workspaceDir, env });
expect(afterRepeated).toHaveLength(10_000);
expect(afterRepeated[0]).toMatchObject({ query: "oversized-3" });
expect(afterRepeated.at(-1)).toMatchObject({ query: "newer recreated generation" });
await expect(fs.access(`${eventPath}.migrated`)).resolves.toBeUndefined();
await expect(fs.access(`${eventPath}.migrated.2`)).resolves.toBeUndefined();
});
it("leaves legacy host events in place when plugin-wide SQLite capacity is exhausted", async () => {
const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl");
await fs.writeFile(
eventPath,
`${JSON.stringify({
type: "memory.recall.recorded",
timestamp: "2026-07-01T00:00:00.000Z",
query: "sqlite capacity",
resultCount: 0,
results: [],
})}\n`,
"utf8",
);
const params = migrationParams();
params.context = {
...params.context,
getPluginStateCapacity: () => ({ liveEntries: 50_000, maxEntries: 50_000 }),
};
const result = await hostEventsMigration().migrateLegacyState(params);
expect(result.changes).toEqual([]);
expect(result.warnings).toEqual([expect.stringContaining("no room for its workspace cursor")]);
await expect(fs.access(eventPath)).resolves.toBeUndefined();
await expect(fs.access(`${eventPath}.migrated`)).rejects.toMatchObject({ code: "ENOENT" });
});
it("reserves plugin-wide capacity for the migrated workspace cursor and checkpoint", async () => {
const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl");
const events = Array.from({ length: 3 }, (_, index) => ({
type: "memory.recall.recorded",
timestamp: "2026-07-01T00:00:00.000Z",
query: `capacity-${index}`,
resultCount: 0,
results: [],
}));
await fs.writeFile(eventPath, `${events.map((event) => JSON.stringify(event)).join("\n")}\n`);
const params = migrationParams();
params.context = {
...params.context,
getPluginStateCapacity: () => ({ liveEntries: 49_997, maxEntries: 50_000 }),
};
const result = await hostEventsMigration().migrateLegacyState(params);
expect(result.warnings).toEqual([]);
expect(result.changes).toEqual([
"Migrated Memory Core host events -> SQLite plugin state (1 new row(s))",
expect.stringContaining("Archived Memory Core host events legacy source"),
]);
await expect(readMemoryHostEventRecords({ workspaceDir, env })).resolves.toMatchObject([
{ query: "capacity-2" },
]);
const cursors = await context()
.openPluginStateKeyedStore<{ kind: "cursor"; lastSequence: number }>({
namespace: "memory-host.event-cursors",
maxEntries: 1_000,
})
.entries();
expect(cursors).toHaveLength(1);
const checkpoints = await context()
.openPluginStateKeyedStore({
namespace: "memory-host.event-migration-checkpoints",
maxEntries: 10_000,
overflowPolicy: "reject-new",
})
.entries();
expect(checkpoints).toHaveLength(1);
});
it("retires an empty legacy memory host event source without claiming an import", async () => {
const eventPath = path.join(workspaceDir, "memory", ".dreams", "events.jsonl");
await fs.writeFile(eventPath, "\n", "utf8");
const result = await hostEventsMigration().migrateLegacyState(migrationParams());
expect(result.warnings).toEqual([]);
expect(result.changes).toEqual([
"Retired empty Memory Core host events legacy source",
expect.stringContaining("Archived Memory Core host events legacy source"),
]);
const entries = await context()
.openPluginStateKeyedStore({ namespace: "memory-host.events", maxEntries: 10_000 })
.entries();
expect(entries).toEqual([]);
await expect(fs.access(`${eventPath}.migrated`)).resolves.toBeUndefined();
});
it("imports persistent legacy dreaming state and ignores transient locks", async () => {
const dreamsDir = path.join(workspaceDir, "memory", ".dreams");
const dailyPath = path.join(dreamsDir, "daily-ingestion.json");
@@ -542,7 +1192,7 @@ describe("memory-core doctor dreaming migration", () => {
);
await fs.writeFile(lockPath, `${process.pid}:${Date.now()}\n`, "utf8");
const migration = requireStateMigration(0);
const migration = dreamingStateMigration();
const preview = await migration.detectLegacyState(migrationParams());
expect(preview?.preview).toEqual([
expect.stringContaining("Memory Core daily ingestion"),
@@ -624,7 +1274,7 @@ describe("memory-core doctor dreaming migration", () => {
const recallPath = path.join(workspaceDir, "memory", ".dreams", "short-term-recall.json");
await fs.writeFile(recallPath, "{", "utf8");
const result = await requireStateMigration(0).migrateLegacyState(migrationParams());
const result = await dreamingStateMigration().migrateLegacyState(migrationParams());
expect(result.changes).toEqual([]);
expect(result.warnings).toEqual([
@@ -666,10 +1316,10 @@ describe("memory-core doctor dreaming migration", () => {
);
const config = { agents: { list: [{ id: "main", default: true }] } };
const preview = await requireStateMigration(0).detectLegacyState(migrationParams(config));
const preview = await dreamingStateMigration().detectLegacyState(migrationParams(config));
expect(preview?.preview).toEqual([expect.stringContaining("Memory Core short-term recall")]);
const result = await requireStateMigration(0).migrateLegacyState(migrationParams(config));
const result = await dreamingStateMigration().migrateLegacyState(migrationParams(config));
expect(result.warnings).toEqual([]);
expect(result.changes).toEqual([
+772 -1
View File
@@ -6,7 +6,7 @@ import fs from "node:fs/promises";
import path from "node:path";
import type { DatabaseSync } from "node:sqlite";
import { reclaimDefinitelyStaleFileLock } from "openclaw/plugin-sdk/file-lock";
import { resolveUserPath } from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
import { resolveUserPath, root } from "openclaw/plugin-sdk/memory-core-host-engine-foundation";
import {
ensureMemoryIndexSchema,
loadSqliteVecExtension,
@@ -19,10 +19,15 @@ import {
requireNodeSqlite,
} from "openclaw/plugin-sdk/memory-core-host-engine-storage";
import { resolveMemoryDreamingWorkspaces } from "openclaw/plugin-sdk/memory-core-host-status";
import {
normalizeMemoryHostEventRecordForStorage,
resolveMemoryHostEventLogPath,
} from "openclaw/plugin-sdk/memory-host-events";
import { normalizeAgentId } from "openclaw/plugin-sdk/routing";
import {
archiveLegacyStateSource,
legacyStateFileExists,
type PluginDoctorStateMigrationContext,
type PluginDoctorStateMigration,
} from "openclaw/plugin-sdk/runtime-doctor";
import {
@@ -67,6 +72,70 @@ type LegacySource = {
filePath: string;
};
type LegacyMemoryHostEventSource =
| {
kind: "ready";
workspaceDir: string;
filePath: string;
relativePath: string;
root: Awaited<ReturnType<typeof root>>;
storage: "active" | "claim" | "archive";
archiveRelativePath?: string;
generationKey?: string;
}
| {
kind: "rejected";
workspaceDir: string;
filePath: string;
reason: string;
};
type ReadyLegacyMemoryHostEventSource = Extract<LegacyMemoryHostEventSource, { kind: "ready" }>;
type StoredMemoryHostEvent = {
kind: "event";
event: NonNullable<ReturnType<typeof normalizeMemoryHostEventRecordForStorage>>;
recordedAt: number;
sequence: number;
};
type StoredMemoryHostCursor = {
kind: "cursor";
lastSequence: number;
};
type StoredMemoryHostMigrationCheckpoint = {
kind: "migration-checkpoint";
contentHash: string;
recordCount: number;
sequenceBase: number;
size: number;
};
const MEMORY_HOST_EVENTS_NAMESPACE = "memory-host.events";
const MEMORY_HOST_EVENT_CURSORS_NAMESPACE = "memory-host.event-cursors";
const MEMORY_HOST_EVENT_MIGRATION_CHECKPOINTS_NAMESPACE = "memory-host.event-migration-checkpoints";
// Keep migration aligned with event-store.ts retention so legacy import cannot
// consume memory-core's plugin-wide budget or starve sibling state namespaces.
const MAX_MEMORY_HOST_EVENTS = 10_000;
const MAX_MEMORY_HOST_EVENT_CURSORS = 1_000;
const MAX_MEMORY_HOST_EVENT_MIGRATION_CHECKPOINTS = 10_000;
const MAX_LEGACY_MEMORY_HOST_EVENT_VALUE_BYTES = 65_536;
const LEGACY_MEMORY_HOST_SEQUENCE_BASE = Number.MIN_SAFE_INTEGER;
function normalizeMemoryHostWorkspaceKey(workspaceDir: string): string {
const resolved = path.resolve(workspaceDir).replace(/\\/g, "/");
return process.platform === "win32" ? resolved.toLowerCase() : resolved;
}
function memoryHostWorkspacePrefix(workspaceDir: string): string {
return crypto
.createHash("sha256")
.update(normalizeMemoryHostWorkspaceKey(workspaceDir))
.digest("hex")
.slice(0, 24);
}
type LegacyMemorySidecarSource = {
agentId: string;
legacyPath: string;
@@ -1175,7 +1244,709 @@ async function migrateSource(source: LegacySource): Promise<number> {
return await migratePhaseSignals(source);
}
async function collectLegacyMemoryHostEventSources(
config: unknown,
env: NodeJS.ProcessEnv,
): Promise<LegacyMemoryHostEventSource[]> {
const sources: LegacyMemoryHostEventSource[] = [];
const seenWorkspaces = new Set<string>();
for (const workspaceDir of resolveConfiguredWorkspaces(config, env)) {
let canonicalWorkspaceDir = path.resolve(workspaceDir);
let filePath = resolveMemoryHostEventLogPath(canonicalWorkspaceDir);
try {
const workspaceRoot = await root(workspaceDir, {
hardlinks: "reject",
// Legacy doctor import previously read the complete JSONL source.
maxBytes: Number.MAX_SAFE_INTEGER,
mkdir: false,
symlinks: "reject",
});
canonicalWorkspaceDir = workspaceRoot.rootReal;
if (seenWorkspaces.has(canonicalWorkspaceDir)) {
continue;
}
seenWorkspaces.add(canonicalWorkspaceDir);
filePath = resolveMemoryHostEventLogPath(canonicalWorkspaceDir);
const relativePath = path.relative(canonicalWorkspaceDir, filePath);
const directoryRelativePath = path.dirname(relativePath);
if (!(await workspaceRoot.exists(directoryRelativePath))) {
continue;
}
const directoryStat = await workspaceRoot.stat(directoryRelativePath);
if (!directoryStat.isDirectory) {
continue;
}
const baseName = path.basename(relativePath).replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
const archivePattern = new RegExp(`^${baseName}\\.migrated(?:\\.([2-9]|[1-9][0-9]+))?$`, "u");
const claimPattern = new RegExp(
`^\\.${baseName}\\.doctor-importing(?:\\.([2-9]|[1-9][0-9]+))?$`,
"u",
);
const entries = await fs.readdir(path.join(workspaceRoot.rootReal, directoryRelativePath));
const candidates: Array<{
entry: string;
storage: "active" | "claim" | "archive";
generation: bigint | undefined;
}> = [];
for (const entry of entries) {
if (entry === path.basename(relativePath)) {
candidates.push({ entry, storage: "active", generation: undefined });
continue;
}
const claim = claimPattern.exec(entry);
if (claim) {
candidates.push({ entry, storage: "claim", generation: BigInt(claim[1] ?? "1") });
continue;
}
const archive = archivePattern.exec(entry);
if (archive) {
candidates.push({ entry, storage: "archive", generation: BigInt(archive[1] ?? "1") });
}
}
candidates.sort((left, right) => {
if (left.generation === undefined) {
return 1;
}
if (right.generation === undefined) {
return -1;
}
return left.generation < right.generation ? -1 : left.generation > right.generation ? 1 : 0;
});
for (const candidate of candidates) {
const candidateRelativePath = path.join(directoryRelativePath, candidate.entry);
const stat = await workspaceRoot.stat(candidateRelativePath);
if (!stat.isFile) {
continue;
}
const generationText = candidate.generation?.toString();
const generationKey = generationText?.padStart(20, "0");
sources.push({
kind: "ready",
workspaceDir: canonicalWorkspaceDir,
filePath: path.join(canonicalWorkspaceDir, candidateRelativePath),
relativePath: candidateRelativePath,
root: workspaceRoot,
storage: candidate.storage,
...(candidate.storage === "active"
? {}
: {
archiveRelativePath: `${relativePath}.migrated${candidate.generation === 1n ? "" : `.${candidate.generation}`}`,
generationKey,
}),
});
}
} catch (error) {
const code = (error as { code?: unknown }).code;
if (code === "ENOENT" || code === "ENOTDIR" || code === "not-found") {
continue;
}
if (!seenWorkspaces.has(canonicalWorkspaceDir)) {
seenWorkspaces.add(canonicalWorkspaceDir);
}
sources.push({
kind: "rejected",
workspaceDir: canonicalWorkspaceDir,
filePath,
reason: String(error),
});
}
}
return sources;
}
async function resolveMemoryHostEventArchivePath(
source: ReadyLegacyMemoryHostEventSource,
): Promise<{ archiveRelativePath: string; claimRelativePath: string; generationKey: string }> {
const activeRelativePath = path.relative(
source.workspaceDir,
resolveMemoryHostEventLogPath(source.workspaceDir),
);
const directoryPath = path.join(source.root.rootReal, path.dirname(activeRelativePath));
const baseName = path.basename(activeRelativePath).replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
const archivePattern = new RegExp(`^${baseName}\\.migrated(?:\\.([2-9]|[1-9][0-9]+))?$`, "u");
const claimPattern = new RegExp(
`^\\.${baseName}\\.doctor-importing(?:\\.([2-9]|[1-9][0-9]+))?$`,
"u",
);
let latestGeneration = 0n;
for (const entry of await fs.readdir(directoryPath)) {
const match = archivePattern.exec(entry) ?? claimPattern.exec(entry);
if (!match) {
continue;
}
const generation = BigInt(match[1] ?? "1");
if (generation > latestGeneration) {
latestGeneration = generation;
}
}
const generation = latestGeneration + 1n;
const generationText = generation.toString();
if (generationText.length > 20) {
throw new RangeError("Memory Core host event archive generation is too large");
}
const generationSuffix = generation === 1n ? "" : `.${generation}`;
return {
archiveRelativePath: `${activeRelativePath}.migrated${generationSuffix}`,
claimRelativePath: path.join(
path.dirname(activeRelativePath),
`.${path.basename(activeRelativePath)}.doctor-importing${generationSuffix}`,
),
// Fixed-width decimal order keeps key-range reads chronological across archives.
generationKey: generationText.padStart(20, "0"),
};
}
function memoryHostMigrationCheckpointKey(source: ReadyLegacyMemoryHostEventSource): string {
if (!source.generationKey) {
throw new Error(`Missing Memory Core host event archive generation for ${source.filePath}`);
}
return `${memoryHostWorkspacePrefix(source.workspaceDir)}:archive:${source.generationKey}`;
}
function memoryHostMigrationSnapshot(raw: string, recordCount: number, sequenceBase: number) {
return {
kind: "migration-checkpoint" as const,
contentHash: crypto.createHash("sha256").update(raw).digest("hex"),
recordCount,
sequenceBase,
size: Buffer.byteLength(raw, "utf8"),
};
}
function isMemoryHostMigrationCheckpoint(
value: StoredMemoryHostMigrationCheckpoint | undefined,
): value is StoredMemoryHostMigrationCheckpoint {
return (
value?.kind === "migration-checkpoint" &&
typeof value.contentHash === "string" &&
Number.isSafeInteger(value.recordCount) &&
value.recordCount >= 0 &&
Number.isSafeInteger(value.sequenceBase) &&
Number.isSafeInteger(value.size) &&
value.size >= 0
);
}
async function memoryHostEventSourceNeedsMigration(params: {
source: ReadyLegacyMemoryHostEventSource;
context: PluginDoctorStateMigrationContext;
}): Promise<boolean> {
if (params.source.storage !== "archive") {
return true;
}
const checkpoint = await params.context
.openPluginStateKeyedStore<StoredMemoryHostMigrationCheckpoint>({
namespace: MEMORY_HOST_EVENT_MIGRATION_CHECKPOINTS_NAMESPACE,
maxEntries: MAX_MEMORY_HOST_EVENT_MIGRATION_CHECKPOINTS,
overflowPolicy: "reject-new",
})
.lookup(memoryHostMigrationCheckpointKey(params.source));
if (!isMemoryHostMigrationCheckpoint(checkpoint)) {
return true;
}
const raw = await params.source.root.readText(params.source.relativePath);
return (
Buffer.byteLength(raw, "utf8") !== checkpoint.size ||
crypto.createHash("sha256").update(raw).digest("hex") !== checkpoint.contentHash
);
}
async function finalizeLegacyMemoryHostEventSource(params: {
source: ReadyLegacyMemoryHostEventSource;
changes: string[];
warnings: string[];
}): Promise<boolean> {
if (params.source.storage === "archive") {
return true;
}
const archivedRelativePath = params.source.archiveRelativePath;
if (!archivedRelativePath) {
throw new Error(`Missing Memory Core host event archive path for ${params.source.filePath}`);
}
try {
await params.source.root.move(params.source.relativePath, archivedRelativePath);
params.changes.push(
`Archived Memory Core host events legacy source -> ${path.join(params.source.workspaceDir, archivedRelativePath)}`,
);
return true;
} catch (error) {
params.warnings.push(
`Failed archiving Memory Core host events legacy source: ${String(error)}`,
);
return false;
}
}
async function restoreClaimedMemoryHostEventSource(params: {
source: ReadyLegacyMemoryHostEventSource;
activeRelativePath: string;
warnings: string[];
}): Promise<void> {
try {
if (!(await params.source.root.exists(params.source.relativePath))) {
return;
}
if (!(await params.source.root.exists(params.activeRelativePath))) {
await params.source.root.move(params.source.relativePath, params.activeRelativePath);
return;
}
params.warnings.push(
`Left claimed Memory Core host events at ${params.source.filePath} because an old writer recreated the active source`,
);
} catch (error) {
params.warnings.push(
`Failed restoring claimed Memory Core host events ${params.source.filePath}: ${String(error)}`,
);
}
}
async function migrateLegacyMemoryHostEventSource(params: {
source: ReadyLegacyMemoryHostEventSource;
context: PluginDoctorStateMigrationContext;
changes: string[];
warnings: string[];
}): Promise<"completed" | "blocked"> {
const activeRelativePath = path.relative(
params.source.workspaceDir,
resolveMemoryHostEventLogPath(params.source.workspaceDir),
);
let source = params.source;
let restoreNewClaim = false;
let claimFinalized = source.storage === "archive";
if (source.storage === "active") {
const generation = await resolveMemoryHostEventArchivePath(source);
await source.root.move(source.relativePath, generation.claimRelativePath);
source = {
...source,
filePath: path.join(source.workspaceDir, generation.claimRelativePath),
relativePath: generation.claimRelativePath,
storage: "claim",
archiveRelativePath: generation.archiveRelativePath,
generationKey: generation.generationKey,
};
restoreNewClaim = true;
}
try {
if (!source.generationKey) {
throw new Error(`Missing Memory Core host event generation for ${source.filePath}`);
}
const warningStart = params.warnings.length;
const raw = await source.root.readText(source.relativePath);
const prefix = memoryHostWorkspacePrefix(source.workspaceDir);
const records: Array<{
digest: string;
ordinal: number;
value: StoredMemoryHostEvent;
}> = [];
for (const [lineIndex, line] of raw.split(/\r?\n/u).entries()) {
const trimmed = line.trim();
if (!trimmed) {
continue;
}
let event: unknown;
try {
event = JSON.parse(trimmed) as unknown;
} catch (error) {
params.warnings.push(
`Skipped malformed Memory Core host event at ${source.filePath}:${lineIndex + 1}: ${String(error)}`,
);
continue;
}
const normalizedEvent = normalizeMemoryHostEventRecordForStorage(event);
if (!normalizedEvent) {
params.warnings.push(
`Skipped invalid Memory Core host event at ${source.filePath}:${lineIndex + 1}`,
);
continue;
}
const canonicalEvent = JSON.stringify(normalizedEvent);
const parsedTimestamp = Date.parse(normalizedEvent.timestamp);
const recordedAt = Number.isSafeInteger(parsedTimestamp) ? parsedTimestamp : 0;
const ordinal = records.length;
const digest = crypto.createHash("sha256").update(canonicalEvent).digest("hex");
const value: StoredMemoryHostEvent = {
kind: "event",
event: normalizedEvent,
recordedAt,
sequence: LEGACY_MEMORY_HOST_SEQUENCE_BASE + ordinal + 1,
};
if (
Buffer.byteLength(JSON.stringify(value), "utf8") > MAX_LEGACY_MEMORY_HOST_EVENT_VALUE_BYTES
) {
params.warnings.push(
`Skipped oversized Memory Core host event at ${source.filePath}:${lineIndex + 1}`,
);
continue;
}
records.push({
// Runtime keys use the adjacent `1:` range. Generation plus valid-row
// ordinal makes interrupted imports and later raw-archive recovery stable.
digest,
ordinal,
value,
});
}
if (params.warnings.length > warningStart) {
params.warnings.push(
"Left Memory Core host events legacy source in place because invalid rows still require repair",
);
return "blocked";
}
const checkpointStore =
params.context.openPluginStateKeyedStore<StoredMemoryHostMigrationCheckpoint>({
namespace: MEMORY_HOST_EVENT_MIGRATION_CHECKPOINTS_NAMESPACE,
maxEntries: MAX_MEMORY_HOST_EVENT_MIGRATION_CHECKPOINTS,
overflowPolicy: "reject-new",
});
// Plugin-wide shedding is namespace-local. `reject-new` therefore keeps
// retained raw-archive checkpoints out of every sibling namespace's
// eviction budget and fails before this namespace can rotate its own rows.
const checkpointKey = memoryHostMigrationCheckpointKey(source);
const checkpointValue = await checkpointStore.lookup(checkpointKey);
const previousCheckpoint = isMemoryHostMigrationCheckpoint(checkpointValue)
? checkpointValue
: undefined;
if (source.storage === "archive" && previousCheckpoint) {
const bytes = Buffer.from(raw, "utf8");
const prefixHash = crypto
.createHash("sha256")
.update(bytes.subarray(0, previousCheckpoint.size))
.digest("hex");
if (
bytes.length < previousCheckpoint.size ||
prefixHash !== previousCheckpoint.contentHash ||
records.length < previousCheckpoint.recordCount
) {
params.warnings.push(
`Skipped Memory Core host event recovery because ${source.filePath} changed other than by append; left the archive in place`,
);
return "blocked";
}
}
const firstCandidateOrdinal =
source.storage === "archive" && previousCheckpoint ? previousCheckpoint.recordCount : 0;
const candidateRecords = records.slice(firstCandidateOrdinal);
const store = params.context.openPluginStateKeyedStore<StoredMemoryHostEvent>({
namespace: MEMORY_HOST_EVENTS_NAMESPACE,
maxEntries: MAX_MEMORY_HOST_EVENTS,
});
const existingEntries = await store.entries();
const existingKeys = new Set(existingEntries.map((entry) => entry.key));
const latestLegacySequence = existingEntries.reduce(
(latest, entry) =>
entry.value.sequence < 0 ? Math.max(latest, entry.value.sequence) : latest,
LEGACY_MEMORY_HOST_SEQUENCE_BASE,
);
const legacyKeyPrefix = `${prefix}:event:0:s:`;
const existingByIdentity = new Map<string, (typeof existingEntries)[number]>(
existingEntries.flatMap((entry) => {
if (!entry.key.startsWith(legacyKeyPrefix) || entry.value.sequence >= 0) {
return [];
}
const parts = entry.key.split(":");
return parts.length === 8 ? [[`${parts[5]}:${parts[6]}:${parts[7]}`, entry] as const] : [];
}),
);
const existingSourceBase = candidateRecords.flatMap((record) => {
const identity = `${source.generationKey}:${record.ordinal.toString().padStart(16, "0")}:${record.digest}`;
const existing = existingByIdentity.get(identity);
return existing ? [existing.value.sequence - (record.ordinal + 1)] : [];
})[0];
const laterGenerationExists = existingEntries.some((entry) => {
if (!entry.key.startsWith(legacyKeyPrefix) || entry.value.sequence >= 0) {
return false;
}
const generationKey = entry.key.split(":")[5];
return generationKey !== undefined && generationKey > source.generationKey!;
});
if (
source.storage === "archive" &&
!previousCheckpoint &&
existingSourceBase === undefined &&
laterGenerationExists
) {
params.warnings.push(
`Skipped Memory Core host event recovery because ${source.filePath} has no durable checkpoint and later generations are already imported; left the archive in place`,
);
return "blocked";
}
const sourceSequenceBase =
previousCheckpoint?.sequenceBase ?? existingSourceBase ?? latestLegacySequence;
let nextSequence = latestLegacySequence;
const sequencedRecords = candidateRecords.map((record) => {
const ordinalKey = record.ordinal.toString().padStart(16, "0");
const identity = `${source.generationKey}:${ordinalKey}:${record.digest}`;
const existing = existingByIdentity.get(identity);
if (existing) {
return {
...record,
key: existing.key,
value: { ...record.value, sequence: existing.value.sequence },
};
}
const sequence = previousCheckpoint
? (nextSequence += 1)
: sourceSequenceBase + record.ordinal + 1;
const sequenceKey = (sequence - LEGACY_MEMORY_HOST_SEQUENCE_BASE)
.toString()
.padStart(16, "0");
return {
...record,
key: `${legacyKeyPrefix}${sequenceKey}:${identity}`,
value: { ...record.value, sequence },
};
});
const sourceKeys = new Set(sequencedRecords.map((record) => record.key));
if (
sequencedRecords.some(
(record) => !Number.isSafeInteger(record.value.sequence) || record.value.sequence >= 0,
)
) {
params.warnings.push(
"Skipped Memory Core host event migration because legacy sequence capacity is exhausted; left legacy source in place",
);
return "blocked";
}
const nativeCount = existingEntries.filter((entry) => entry.value.sequence >= 0).length;
const legacyRetentionLimit = Math.max(0, MAX_MEMORY_HOST_EVENTS - nativeCount);
const combinedLegacy = [
...existingEntries
.filter((entry) => entry.value.sequence < 0 && !sourceKeys.has(entry.key))
.map((entry) => ({ key: entry.key, sequence: entry.value.sequence })),
...sequencedRecords.map((record) => ({
key: record.key,
sequence: record.value.sequence,
})),
].toSorted(
(left, right) => left.sequence - right.sequence || left.key.localeCompare(right.key),
);
const desiredLegacyKeys = new Set(
legacyRetentionLimit === 0
? []
: combinedLegacy.slice(-legacyRetentionLimit).map((record) => record.key),
);
let retainedRecords = sequencedRecords.filter((record) => desiredLegacyKeys.has(record.key));
const capacity = params.context.getPluginStateCapacity?.();
if (!capacity) {
params.warnings.push(
"Skipped Memory Core host event migration because plugin-wide SQLite capacity is unavailable; left legacy source in place",
);
return "blocked";
}
const pluginRemainingCapacity = Math.max(0, capacity.maxEntries - capacity.liveEntries);
const cursorStore = params.context.openPluginStateKeyedStore<StoredMemoryHostCursor>({
namespace: MEMORY_HOST_EVENT_CURSORS_NAMESPACE,
maxEntries: MAX_MEMORY_HOST_EVENT_CURSORS,
});
const cursorKey = `${prefix}:cursor`;
const existingCursor = await cursorStore.lookup(cursorKey);
const cursorCapacity = candidateRecords.length > 0 && existingCursor?.kind !== "cursor" ? 1 : 0;
if (cursorCapacity > pluginRemainingCapacity) {
params.warnings.push(
"Skipped Memory Core host event migration because SQLite plugin state has no room for its workspace cursor; left legacy source in place",
);
return "blocked";
}
const checkpointCapacity = checkpointValue ? 0 : 1;
if (
checkpointCapacity > 0 &&
(await checkpointStore.entries()).length >= MAX_MEMORY_HOST_EVENT_MIGRATION_CHECKPOINTS
) {
// Checkpoints use reject-new and never expire while their raw archives remain.
// Stop before import/archive once durable processed-generation capacity is full.
params.warnings.push(
"Skipped Memory Core host event migration because durable raw-archive checkpoint capacity is exhausted; left legacy source in place",
);
return "blocked";
}
if (cursorCapacity + checkpointCapacity > pluginRemainingCapacity) {
params.warnings.push(
"Skipped Memory Core host event migration because SQLite plugin state has no room for its raw-archive checkpoint; left legacy source in place",
);
return "blocked";
}
const importEntries = params.context.importPluginStateEntries;
if (candidateRecords.length > 0 && !importEntries) {
params.warnings.push(
"Skipped Memory Core host event migration because retention-aware SQLite import is unavailable; left legacy source in place",
);
return "blocked";
}
let retainedKeys = new Set(retainedRecords.map((record) => record.key));
let missing = retainedRecords.filter((record) => !existingKeys.has(record.key));
let replaceableLegacyRows = existingEntries.filter(
(entry) => entry.value.sequence < 0 && !retainedKeys.has(entry.key),
).length;
const reservedCapacity = cursorCapacity + checkpointCapacity;
const eventCapacity = pluginRemainingCapacity - reservedCapacity + replaceableLegacyRows;
const retentionDeficit = Math.max(0, missing.length - eventCapacity);
if (retentionDeficit > 0) {
retainedRecords = retainedRecords.slice(Math.min(retentionDeficit, retainedRecords.length));
retainedKeys = new Set(retainedRecords.map((record) => record.key));
missing = retainedRecords.filter((record) => !existingKeys.has(record.key));
replaceableLegacyRows = existingEntries.filter(
(entry) => entry.value.sequence < 0 && !retainedKeys.has(entry.key),
).length;
}
const availableEventCapacity =
pluginRemainingCapacity - reservedCapacity + replaceableLegacyRows;
if (missing.length > availableEventCapacity) {
params.warnings.push(
`Skipped Memory Core host event migration because SQLite plugin state has room for ${availableEventCapacity} of ${missing.length} missing rows after reserving its cursor and raw-archive checkpoint; left legacy source in place`,
);
return "blocked";
}
if (cursorCapacity > 0) {
const lastSequence = existingEntries.reduce(
(maximum, entry) =>
Number.isSafeInteger(entry.value.sequence)
? Math.max(maximum, entry.value.sequence)
: maximum,
0,
);
await cursorStore.register(cursorKey, { kind: "cursor", lastSequence });
const registeredCursor = await cursorStore.lookup(cursorKey);
if (registeredCursor?.kind !== "cursor") {
params.warnings.push(
"Skipped Memory Core host event migration because its workspace cursor could not be verified; left legacy source in place",
);
return "blocked";
}
}
importEntries?.(
{ namespace: MEMORY_HOST_EVENTS_NAMESPACE, maxEntries: MAX_MEMORY_HOST_EVENTS },
missing.map((record) => ({
key: record.key,
value: record.value,
// Negative legacy sequence keeps imported rows older than live runtime rows.
createdAt: record.value.sequence,
})),
);
const importedKeys = new Set((await store.entries()).map((entry) => entry.key));
const missingKey = retainedRecords.find((record) => !importedKeys.has(record.key))?.key;
if (missingKey) {
params.warnings.push(
`Skipped archiving Memory Core host events because SQLite verification missed ${missingKey}`,
);
return "blocked";
}
if (source.storage === "archive") {
if (missing.length > 0) {
params.changes.push(
`Recovered ${missing.length} later Memory Core host event row(s) from ${source.filePath}`,
);
}
} else {
params.changes.push(
records.length === 0
? "Retired empty Memory Core host events legacy source"
: `Migrated Memory Core host events -> SQLite plugin state (${missing.length} new row(s))`,
);
claimFinalized = await finalizeLegacyMemoryHostEventSource({
source,
changes: params.changes,
warnings: params.warnings,
});
if (!claimFinalized) {
params.changes.pop();
return "blocked";
}
}
const checkpoint = memoryHostMigrationSnapshot(raw, records.length, sourceSequenceBase);
await checkpointStore.register(checkpointKey, checkpoint);
const registeredCheckpoint = await checkpointStore.lookup(checkpointKey);
if (
!isMemoryHostMigrationCheckpoint(registeredCheckpoint) ||
registeredCheckpoint.contentHash !== checkpoint.contentHash ||
registeredCheckpoint.recordCount !== checkpoint.recordCount ||
registeredCheckpoint.sequenceBase !== checkpoint.sequenceBase ||
registeredCheckpoint.size !== checkpoint.size
) {
params.warnings.push(
`Failed verifying Memory Core host event raw-archive checkpoint for ${source.filePath}`,
);
return "blocked";
}
if (source.storage !== "archive" && (await source.root.exists(activeRelativePath))) {
params.warnings.push(
"An old writer recreated the Memory Core host event source; rerun openclaw doctor --fix to import the retained rows",
);
}
return "completed";
} finally {
if (restoreNewClaim && !claimFinalized) {
await restoreClaimedMemoryHostEventSource({
source,
activeRelativePath,
warnings: params.warnings,
});
}
}
}
export const stateMigrations: PluginDoctorStateMigration[] = [
{
id: "memory-core-host-events-jsonl-to-sqlite",
label: "Memory Core host events",
doctorOnly: true,
async detectLegacyState(params) {
const sources = await collectLegacyMemoryHostEventSources(params.config, params.env);
const pending: LegacyMemoryHostEventSource[] = [];
for (const source of sources) {
if (
source.kind === "rejected" ||
(await memoryHostEventSourceNeedsMigration({ source, context: params.context }))
) {
pending.push(source);
}
}
if (pending.length === 0) {
return null;
}
return {
preview: pending.map((source) =>
source.kind === "ready"
? `- Memory Core host events: ${source.filePath} -> SQLite plugin state (${MEMORY_HOST_EVENTS_NAMESPACE})`
: `- Memory Core host events: ${source.filePath} requires safe-path repair (${source.reason})`,
),
};
},
async migrateLegacyState(params) {
const changes: string[] = [];
const warnings: string[] = [];
const blockedWorkspaces = new Set<string>();
for (const source of await collectLegacyMemoryHostEventSources(params.config, params.env)) {
if (blockedWorkspaces.has(source.workspaceDir)) {
continue;
}
if (source.kind === "rejected") {
warnings.push(
`Skipped unsafe Memory Core host event source for ${source.workspaceDir}: ${source.reason}`,
);
blockedWorkspaces.add(source.workspaceDir);
continue;
}
if (!(await memoryHostEventSourceNeedsMigration({ source, context: params.context }))) {
continue;
}
const result = await migrateLegacyMemoryHostEventSource({
source,
context: params.context,
changes,
warnings,
});
if (result === "blocked") {
// Archive generations encode append order. A later generation cannot
// overtake an older source that still needs repair or durable import.
blockedWorkspaces.add(source.workspaceDir);
}
}
return { changes, warnings };
},
},
{
id: "memory-core-dreams-json-to-sqlite",
label: "Memory Core dreaming state",
@@ -2,9 +2,9 @@
import fs from "node:fs/promises";
import path from "node:path";
import {
appendMemoryHostEvent,
readMemoryHostEventRecords,
readMemoryHostEvents,
resolveMemoryHostEventLogPath,
} from "openclaw/plugin-sdk/memory-host-events";
import { describe, expect, it } from "vitest";
import { writeDailyDreamingPhaseBlock } from "./dreaming-markdown.js";
@@ -193,20 +193,14 @@ describe("memory host event journal integration", () => {
it("keeps legacy dreaming completion events without outcome readable", async () => {
const workspaceDir = await createTempWorkspace("memory-core-legacy-dream-events-");
const eventLogPath = resolveMemoryHostEventLogPath(workspaceDir);
await fs.mkdir(path.dirname(eventLogPath), { recursive: true });
await fs.writeFile(
eventLogPath,
`${JSON.stringify({
type: "memory.dream.completed",
timestamp: "2026-04-05T13:00:00.000Z",
phase: "deep",
inlinePath: path.join(workspaceDir, "DREAMS.md"),
lineCount: 2,
storageMode: "inline",
})}\n`,
"utf8",
);
await appendMemoryHostEvent(workspaceDir, {
type: "memory.dream.completed",
timestamp: "2026-04-05T13:00:00.000Z",
phase: "deep",
inlinePath: path.join(workspaceDir, "DREAMS.md"),
lineCount: 2,
storageMode: "inline",
});
const events = await readMemoryHostEvents({ workspaceDir });
@@ -2,11 +2,9 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import {
appendMemoryHostEvent,
resolveMemoryHostEventLogPath,
} from "openclaw/plugin-sdk/memory-host-events";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { appendMemoryHostEvent } from "openclaw/plugin-sdk/memory-host-events";
import { resetPluginStateStoreForTests } from "openclaw/plugin-sdk/plugin-state-test-runtime";
import { afterAll, afterEach, beforeAll, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../api.js";
import { listMemoryCorePublicArtifacts } from "./public-artifacts.js";
@@ -24,8 +22,15 @@ describe("listMemoryCorePublicArtifacts", () => {
await fs.rm(fixtureRoot, { recursive: true, force: true });
});
afterEach(() => {
resetPluginStateStoreForTests();
vi.unstubAllEnvs();
vi.restoreAllMocks();
});
it("lists public workspace artifacts with stable kinds", async () => {
const workspaceDir = path.join(fixtureRoot, "workspace-stable-kinds");
vi.stubEnv("OPENCLAW_STATE_DIR", fixtureRoot);
await fs.mkdir(path.join(workspaceDir, "memory", "dreaming"), { recursive: true });
await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), "# Durable Memory\n", "utf8");
await fs.writeFile(
@@ -38,6 +43,8 @@ describe("listMemoryCorePublicArtifacts", () => {
"# Dream Report\n",
"utf8",
);
const eventStoredAt = Date.parse("2026-04-07T09:30:00.000Z");
vi.spyOn(Date, "now").mockReturnValue(eventStoredAt);
await appendMemoryHostEvent(workspaceDir, {
type: "memory.recall.recorded",
timestamp: "2026-04-06T12:00:00.000Z",
@@ -52,7 +59,12 @@ describe("listMemoryCorePublicArtifacts", () => {
},
};
await expect(listMemoryCorePublicArtifacts({ cfg })).resolves.toEqual([
const artifacts = await listMemoryCorePublicArtifacts({ cfg });
const eventArtifact = artifacts.find((artifact) => artifact.kind === "event-log");
if (!eventArtifact) {
throw new Error("expected memory event export");
}
expect(artifacts.filter((artifact) => artifact.kind !== "event-log")).toEqual([
{
kind: "memory-root",
workspaceDir,
@@ -77,15 +89,19 @@ describe("listMemoryCorePublicArtifacts", () => {
agentIds: ["main"],
contentType: "markdown",
},
{
kind: "event-log",
workspaceDir,
relativePath: "memory/.dreams/events.jsonl",
absolutePath: resolveMemoryHostEventLogPath(workspaceDir),
agentIds: ["main"],
contentType: "json",
},
]);
expect(eventArtifact.relativePath).toMatch(
/^memory\/events\/[a-f0-9]{32}\/memory-host-events\.jsonl$/u,
);
await expect(fs.readFile(eventArtifact.absolutePath, "utf8")).resolves.toBe(
`${JSON.stringify({
type: "memory.recall.recorded",
timestamp: "2026-04-06T12:00:00.000Z",
query: "alpha",
resultCount: 0,
results: [],
})}\n`,
);
});
it("ignores lowercase memory root when only the legacy filename exists", async () => {
+7 -8
View File
@@ -7,10 +7,6 @@ import {
type MemoryPluginPublicArtifact,
registerMemoryCapability,
} from "openclaw/plugin-sdk/memory-host-core";
import {
appendMemoryHostEvent,
resolveMemoryHostEventLogPath,
} from "openclaw/plugin-sdk/memory-host-events";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import type { OpenClawConfig } from "../api.js";
import { syncMemoryWikiBridgeSources } from "./bridge.js";
@@ -436,7 +432,7 @@ describe("syncMemoryWikiBridgeSources", () => {
},
});
await appendMemoryHostEvent(workspaceDir, {
const eventContent = `${JSON.stringify({
type: "memory.recall.recorded",
timestamp: "2026-04-05T12:00:00.000Z",
query: "bridge events",
@@ -449,13 +445,16 @@ describe("syncMemoryWikiBridgeSources", () => {
score: 0.8,
},
],
});
})}\n`;
const eventPath = path.join(workspaceDir, "memory", "events", "memory-host-events.jsonl");
await fs.mkdir(path.dirname(eventPath), { recursive: true });
await fs.writeFile(eventPath, eventContent, "utf8");
registerBridgeArtifacts([
{
kind: "event-log",
workspaceDir,
relativePath: "memory/.dreams/events.jsonl",
absolutePath: resolveMemoryHostEventLogPath(workspaceDir),
relativePath: "memory/events/memory-host-events.jsonl",
absolutePath: eventPath,
agentIds: ["main"],
contentType: "json",
},
@@ -36,6 +36,7 @@ export async function writeImportedSourcePage(params: {
vaultRoot: string;
syncKey: string;
sourcePath: string;
sourceContent?: string;
sourceUpdatedAtMs: number;
sourceSize: number;
renderFingerprint: string;
@@ -70,7 +71,7 @@ export async function writeImportedSourcePage(params: {
});
const created = !pageStat;
const updatedAt = timestampMsToIsoString(params.sourceUpdatedAtMs) ?? new Date().toISOString();
const raw = await fs.readFile(params.sourcePath, "utf8");
const raw = params.sourceContent ?? (await fs.readFile(params.sourcePath, "utf8"));
const rendered = params.buildRendered(raw, updatedAt);
const existing = pageStat ? await readExistingImportedSourcePage(vault, params.pagePath) : "";
const nextRendered = existing ? preserveHumanNotesBlock(rendered, existing) : rendered;
+1 -1
View File
@@ -4,7 +4,7 @@
"openclaw": {
"schemaVersions": {
"state": 4,
"agent": 11
"agent": 12
}
},
"description": "Multi-channel AI gateway with extensible messaging integrations",
@@ -126,7 +126,7 @@ const legacyStorePatterns = [
/\b(?:crestodian|openclaw)\/rescue-pending\/[^"'`]*\.json\b/u,
/\bcron\/(?:runs\/[^"'`]+\.jsonl|jobs\.json|jobs-state\.json)\b/u,
/\b(?:process-leases|session-toggles|known-users|msteams-conversations|msteams-polls|msteams-sso-tokens|bot-storage|sync-store|thread-bindings|inbound-dedupe|startup-verification|storage-meta|crypto-idb-snapshot|command-deploy-cache|plugin-binding-approvals|plugins\/installs|config-health|port-guard|restart-sentinel|gateway-restart-intent|gateway-supervisor-restart-handoff)\.json\b/u,
/\b(?:calls|ref-index|audit\/file-transfer|audit\/openclaw)\.jsonl\b/u,
/\b(?:calls|ref-index|config-audit|audit\/(?:file-transfer|openclaw|system-agent|crestodian))\.jsonl\b/u,
/\b(?:reply-cache|sent-echoes|events|claims)\.jsonl\b/u,
/\bplugin-state\/state\.sqlite\b/u,
/\btasks\/(?:runs\.sqlite|flows\/registry\.sqlite)\b/u,
+7 -4
View File
@@ -265,9 +265,10 @@ export function readPluginSdkSurfaceBudgets(env = process.env) {
// +81: meeting-runtime barrel: browser meeting-bot core behind MeetingPlatformAdapter.
// +3: question-gateway-runtime resolver plus request/result types.
// +1: async memory prompt preparation registration.
// +1: canonical memory host event normalization for SQLite storage.
// +4: gateway-backed harness question runner, claim/cancel helpers, and caller type.
// Harvest: internal question runtime exports -2.
8155,
8156,
env,
),
publicFunctionExports: readPluginSdkSurfaceBudgetEnv(
@@ -303,9 +304,10 @@ export function readPluginSdkSurfaceBudgets(env = process.env) {
// +32: meeting-runtime barrel: browser meeting-bot core behind MeetingPlatformAdapter.
// +1: question-gateway-runtime resolver.
// +1: async memory prompt preparation registration.
// +1: canonical memory host event normalization for SQLite storage.
// +3: gateway-backed harness question runner and claim/cancel helpers.
// Harvest: internal question runtime callable -1.
4537,
4538,
env,
),
publicDeprecatedExports: readPluginSdkSurfaceBudgetEnv(
@@ -330,8 +332,9 @@ export function readPluginSdkSurfaceBudgets(env = process.env) {
publicWildcardReexports: readPluginSdkSurfaceBudgetEnv(
"OPENCLAW_PLUGIN_SDK_MAX_PUBLIC_WILDCARD_REEXPORTS",
// Used-union narrowing removes 103 wildcard re-exports.
// Harvest: freeze the compat config-schema barrel to explicit exports -1.
104,
// Harvest: freeze the compat config-schema barrel to explicit exports -1;
// retire the Memory Core facade's event-store wildcard -1.
103,
env,
),
};
@@ -0,0 +1,25 @@
import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js";
import type { DB as OpenClawAgentKyselyDatabase } from "../state/openclaw-agent-db.generated.js";
import {
openOpenClawAgentDatabase,
type OpenClawAgentDatabaseOptions,
} from "../state/openclaw-agent-db.js";
import type { AcpParentStreamEvent } from "./acp-parent-stream-store.sqlite.js";
type AcpParentStreamDatabase = Pick<OpenClawAgentKyselyDatabase, "acp_parent_stream_events">;
export function listAcpParentStreamEventsForTest(
options: OpenClawAgentDatabaseOptions & { sessionId: string; runId: string },
): AcpParentStreamEvent[] {
const database = openOpenClawAgentDatabase(options);
const db = getNodeSqliteKysely<AcpParentStreamDatabase>(database.db);
return executeSqliteQuerySync(
database.db,
db
.selectFrom("acp_parent_stream_events")
.select("event_json")
.where("session_id", "=", options.sessionId)
.where("run_id", "=", options.runId)
.orderBy("seq", "asc"),
).rows.map((row) => JSON.parse(row.event_json) as AcpParentStreamEvent);
}
@@ -0,0 +1,105 @@
import { afterEach, describe, expect, it } from "vitest";
import { executeSqliteQuerySync, getNodeSqliteKysely } from "../infra/kysely-sync.js";
import type { DB as OpenClawAgentKyselyDatabase } from "../state/openclaw-agent-db.generated.js";
import {
closeOpenClawAgentDatabasesForTest,
runOpenClawAgentWriteTransaction,
} from "../state/openclaw-agent-db.js";
import { withTempDir } from "../test-helpers/temp-dir.js";
import { recordAcpParentStreamEvents } from "./acp-parent-stream-store.sqlite.js";
import { listAcpParentStreamEventsForTest } from "./acp-parent-stream-store.sqlite.test-support.js";
describe("ACP parent stream SQLite store", () => {
afterEach(() => {
closeOpenClawAgentDatabasesForTest();
});
it("orders run events and removes them with the child session", async () => {
await withTempDir({ prefix: "openclaw-acp-parent-stream-" }, async (stateDir) => {
const options = {
agentId: "codex",
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
};
runOpenClawAgentWriteTransaction((database) => {
const db = getNodeSqliteKysely<Pick<OpenClawAgentKyselyDatabase, "sessions">>(database.db);
executeSqliteQuerySync(
database.db,
db.insertInto("sessions").values({
session_id: "session-1",
session_key: "agent:codex:acp:child",
session_scope: "conversation",
created_at: 1,
updated_at: 1,
}),
);
}, options);
recordAcpParentStreamEvents({
...options,
sessionId: "session-1",
runId: "run-1",
events: [
{ createdAt: 10, event: { kind: "assistant_delta", delta: "one" } },
{ createdAt: 11, event: { kind: "lifecycle", phase: "end" } },
],
});
expect(
listAcpParentStreamEventsForTest({ ...options, sessionId: "session-1", runId: "run-1" }),
).toEqual([
{ kind: "assistant_delta", delta: "one" },
{ kind: "lifecycle", phase: "end" },
]);
runOpenClawAgentWriteTransaction((database) => {
const db = getNodeSqliteKysely<Pick<OpenClawAgentKyselyDatabase, "sessions">>(database.db);
executeSqliteQuerySync(
database.db,
db.deleteFrom("sessions").where("session_id", "=", "session-1"),
);
}, options);
expect(
listAcpParentStreamEventsForTest({ ...options, sessionId: "session-1", runId: "run-1" }),
).toEqual([]);
});
});
it("drops unserializable events without blocking later diagnostics", async () => {
await withTempDir({ prefix: "openclaw-acp-parent-stream-invalid-" }, async (stateDir) => {
const options = {
agentId: "codex",
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
};
runOpenClawAgentWriteTransaction((database) => {
const db = getNodeSqliteKysely<Pick<OpenClawAgentKyselyDatabase, "sessions">>(database.db);
executeSqliteQuerySync(
database.db,
db.insertInto("sessions").values({
session_id: "session-1",
session_key: "agent:codex:acp:invalid",
session_scope: "conversation",
created_at: 1,
updated_at: 1,
}),
);
}, options);
const circular: Record<string, unknown> = { kind: "circular" };
circular.self = circular;
recordAcpParentStreamEvents({
...options,
sessionId: "session-1",
runId: "run-1",
events: [
{ createdAt: 10, event: { toJSON: () => undefined } },
{ createdAt: 11, event: circular },
{ createdAt: 12, event: { kind: "lifecycle", phase: "end" } },
],
});
expect(
listAcpParentStreamEventsForTest({ ...options, sessionId: "session-1", runId: "run-1" }),
).toEqual([{ kind: "lifecycle", phase: "end" }]);
});
});
});
@@ -0,0 +1,77 @@
// ACP parent-stream diagnostics live with their child session in the per-agent database.
import {
executeSqliteQuerySync,
executeSqliteQueryTakeFirstSync,
getNodeSqliteKysely,
} from "../infra/kysely-sync.js";
import type { DB as OpenClawAgentKyselyDatabase } from "../state/openclaw-agent-db.generated.js";
import {
runOpenClawAgentWriteTransaction,
type OpenClawAgentDatabaseOptions,
} from "../state/openclaw-agent-db.js";
type AcpParentStreamDatabase = Pick<OpenClawAgentKyselyDatabase, "acp_parent_stream_events">;
export type AcpParentStreamEvent = Record<string, unknown>;
function getAcpParentStreamKysely(database: import("node:sqlite").DatabaseSync) {
return getNodeSqliteKysely<AcpParentStreamDatabase>(database);
}
function normalizeSqliteNumber(value: number | bigint): number {
return typeof value === "bigint" ? Number(value) : value;
}
/** Records one ordered batch in the same synchronous commit section as sequence allocation. */
export function recordAcpParentStreamEvents(
options: OpenClawAgentDatabaseOptions & {
sessionId: string;
runId: string;
events: Array<{ event: AcpParentStreamEvent; createdAt: number }>;
},
): void {
if (options.events.length === 0) {
return;
}
const prepared = options.events.flatMap((entry) => {
try {
const eventJson = JSON.stringify(entry.event);
if (eventJson !== undefined) {
return [{ eventJson, createdAt: entry.createdAt }];
}
} catch {
// One malformed diagnostic must not poison later valid events or retries.
}
return [];
});
if (prepared.length === 0) {
return;
}
runOpenClawAgentWriteTransaction((database) => {
const db = getAcpParentStreamKysely(database.db);
const row = executeSqliteQueryTakeFirstSync(
database.db,
db
.selectFrom("acp_parent_stream_events")
.select((eb) => eb.fn.max<number | bigint>("seq").as("max_seq"))
.where("session_id", "=", options.sessionId)
.where("run_id", "=", options.runId),
);
const firstSeq =
row?.max_seq === null || row?.max_seq === undefined
? 0
: normalizeSqliteNumber(row.max_seq) + 1;
executeSqliteQuerySync(
database.db,
db.insertInto("acp_parent_stream_events").values(
prepared.map((entry, index) => ({
session_id: options.sessionId,
run_id: options.runId,
seq: firstSeq + index,
event_json: entry.eventJson,
created_at: entry.createdAt,
})),
),
);
}, options);
}
+53 -58
View File
@@ -1,13 +1,11 @@
/** Tests ACP child-to-parent stream relay notices, routing, and log path resolution. */
/** Tests ACP child-to-parent stream relay notices and routing. */
import { afterEach, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { mergeMockedModule } from "../test-utils/vitest-module-mocks.js";
const enqueueSystemEventMock = vi.fn();
const requestHeartbeatMock = vi.fn();
const readAcpSessionEntryMock = vi.fn();
const resolveSessionFilePathMock = vi.fn();
const resolveSessionFilePathOptionsMock = vi.fn();
const recordAcpParentStreamEventsMock = vi.fn();
vi.mock("../infra/system-events.js", () => ({
enqueueSystemEvent: (...args: unknown[]) => enqueueSystemEventMock(...args),
@@ -24,32 +22,18 @@ vi.mock("../infra/heartbeat-wake.js", async () => {
);
});
vi.mock("../acp/runtime/session-meta.js", async () => {
vi.mock("./acp-parent-stream-store.sqlite.js", async () => {
return await mergeMockedModule(
await vi.importActual<typeof import("../acp/runtime/session-meta.js")>(
"../acp/runtime/session-meta.js",
await vi.importActual<typeof import("./acp-parent-stream-store.sqlite.js")>(
"./acp-parent-stream-store.sqlite.js",
),
() => ({
readAcpSessionEntry: (...args: unknown[]) => readAcpSessionEntryMock(...args),
}),
);
});
vi.mock("../config/sessions/paths.js", async () => {
return await mergeMockedModule(
await vi.importActual<typeof import("../config/sessions/paths.js")>(
"../config/sessions/paths.js",
),
() => ({
resolveSessionFilePath: (...args: unknown[]) => resolveSessionFilePathMock(...args),
resolveSessionFilePathOptions: (...args: unknown[]) =>
resolveSessionFilePathOptionsMock(...args),
recordAcpParentStreamEvents: (...args: unknown[]) => recordAcpParentStreamEventsMock(...args),
}),
);
});
let emitAgentEvent: typeof import("../infra/agent-events.js").emitAgentEvent;
let resolveAcpSpawnStreamLogPath: typeof import("./acp-spawn-parent-stream.js").resolveAcpSpawnStreamLogPath;
let startAcpSpawnParentStreamRelay: typeof import("./acp-spawn-parent-stream.js").startAcpSpawnParentStreamRelay;
const progressCommentaryDeliveryContext = {
@@ -103,17 +87,14 @@ function firstMockCall(
describe("startAcpSpawnParentStreamRelay", () => {
beforeAll(async () => {
({ emitAgentEvent } = await import("../infra/agent-events.js"));
({ resolveAcpSpawnStreamLogPath, startAcpSpawnParentStreamRelay } =
await import("./acp-spawn-parent-stream.js"));
({ startAcpSpawnParentStreamRelay } = await import("./acp-spawn-parent-stream.js"));
});
beforeEach(() => {
enqueueSystemEventMock.mockClear();
requestHeartbeatMock.mockClear();
readAcpSessionEntryMock.mockReset();
resolveSessionFilePathMock.mockReset();
resolveSessionFilePathOptionsMock.mockReset();
resolveSessionFilePathOptionsMock.mockImplementation((value: unknown) => value);
recordAcpParentStreamEventsMock.mockReset();
recordAcpParentStreamEventsMock.mockImplementation(() => undefined);
vi.useFakeTimers();
vi.setSystemTime(new Date("2026-03-04T01:00:00.000Z"));
});
@@ -222,6 +203,50 @@ describe("startAcpSpawnParentStreamRelay", () => {
relay.dispose();
});
it("backs off and caps SQLite diagnostic retries", () => {
recordAcpParentStreamEventsMock
.mockImplementationOnce(() => {
throw new Error("database unavailable");
})
.mockImplementation(() => undefined);
const relay = startAcpSpawnParentStreamRelay({
runId: "run-diagnostic-retry",
parentSessionKey: "agent:main:main",
childSessionKey: "agent:codex:acp:diagnostic-retry",
childSessionId: "session-diagnostic-retry",
agentId: "codex",
streamFlushMs: 120_000,
noOutputNoticeMs: 120_000,
});
emitAgentEvent({
runId: "run-diagnostic-retry",
stream: "assistant",
data: { delta: "first" },
});
vi.advanceTimersByTime(1_000);
expect(recordAcpParentStreamEventsMock).toHaveBeenCalledTimes(1);
for (let index = 0; index < 300; index += 1) {
emitAgentEvent({
runId: "run-diagnostic-retry",
stream: "assistant",
data: { delta: `event-${index}` },
});
}
expect(recordAcpParentStreamEventsMock).toHaveBeenCalledTimes(1);
vi.advanceTimersByTime(1_999);
expect(recordAcpParentStreamEventsMock).toHaveBeenCalledTimes(1);
vi.advanceTimersByTime(1);
expect(recordAcpParentStreamEventsMock).toHaveBeenCalledTimes(2);
const retried = recordAcpParentStreamEventsMock.mock.calls[1]?.[0] as
| { events?: unknown[] }
| undefined;
expect(retried?.events).toHaveLength(256);
relay.dispose();
});
it("remaps cron-run parent session keys while relaying stream events", () => {
const relay = startAcpSpawnParentStreamRelay({
runId: "run-cron",
@@ -1436,35 +1461,5 @@ describe("startAcpSpawnParentStreamRelay", () => {
expect(collectedTexts()[1]).toBe(`codex: ${expected}`);
relay.dispose();
});
it("resolves ACP spawn stream log path from session metadata", () => {
readAcpSessionEntryMock.mockReturnValue({
storePath: "/tmp/openclaw/agents/codex/sessions/sessions.json",
entry: {
sessionId: "sess-123",
sessionFile: "/tmp/openclaw/agents/codex/sessions/sess-123.jsonl",
},
});
resolveSessionFilePathMock.mockReturnValue(
"/tmp/openclaw/agents/codex/sessions/sess-123.jsonl",
);
const resolved = resolveAcpSpawnStreamLogPath({
childSessionKey: "agent:codex:acp:child-1",
});
expect(resolved).toBe("/tmp/openclaw/agents/codex/sessions/sess-123.acp-stream.jsonl");
expect(readAcpSessionEntryMock).toHaveBeenCalledWith({
sessionKey: "agent:codex:acp:child-1",
});
expect(resolveSessionFilePathMock).toHaveBeenCalledTimes(1);
const [sessionId, entry, options] = firstMockCall(
resolveSessionFilePathMock,
"session file path resolution",
) as [string, { sessionId?: unknown }, { storePath?: unknown }];
expect(sessionId).toBe("sess-123");
expect(entry.sessionId).toBe("sess-123");
expect(options.storePath).toBe("/tmp/openclaw/agents/codex/sessions/sessions.json");
});
});
/* oxlint-disable max-lines -- TODO: split this grandfathered oversized file. */
+98 -89
View File
@@ -1,10 +1,7 @@
/** Relays child ACP session stream updates back into the requester parent session. */
import { mkdir } from "node:fs/promises";
import path from "node:path";
import { asFiniteNumber } from "@openclaw/normalization-core/number-coercion";
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import { sliceUtf16Safe, truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
import { readAcpSessionEntry } from "../acp/runtime/session-meta.js";
import {
isAcpTagVisible,
resolveAcpProjectionSettings,
@@ -14,7 +11,6 @@ import {
resolveChannelStreamingProgressCommentary,
type StreamingCompatEntry,
} from "../channels/streaming.js";
import { resolveSessionFilePath, resolveSessionFilePathOptions } from "../config/sessions/paths.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { onAgentEvent } from "../infra/agent-events.js";
import {
@@ -23,13 +19,17 @@ import {
scopedHeartbeatWakeOptionsForPolicy,
} from "../infra/event-session-routing.js";
import { requestHeartbeat } from "../infra/heartbeat-wake.js";
import { appendRegularFile } from "../infra/regular-file.js";
import { enqueueSystemEvent } from "../infra/system-events.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import { resolveNormalizedAccountEntry } from "../routing/account-lookup.js";
import { normalizeAccountId } from "../routing/session-key.js";
import { normalizeAssistantPhase } from "../shared/chat-message-content.js";
import { recordTaskRunProgressByRunId } from "../tasks/detached-task-runtime.js";
import type { DeliveryContext } from "../utils/delivery-context.types.js";
import {
recordAcpParentStreamEvents,
type AcpParentStreamEvent,
} from "./acp-parent-stream-store.sqlite.js";
const DEFAULT_STREAM_FLUSH_MS = 2_500;
const DEFAULT_NO_OUTPUT_NOTICE_MS = 60_000;
@@ -37,6 +37,11 @@ const DEFAULT_NO_OUTPUT_POLL_MS = 15_000;
const DEFAULT_MAX_RELAY_LIFETIME_MS = 6 * 60 * 60 * 1000;
const STREAM_BUFFER_MAX_CHARS = 4_000;
const STREAM_SNIPPET_MAX_CHARS = 220;
const STREAM_LOG_BATCH_SIZE = 100;
const STREAM_LOG_FLUSH_MS = 1_000;
const STREAM_LOG_MAX_PENDING_EVENTS = 256;
const STREAM_LOG_MAX_RETRY_MS = 30_000;
const log = createSubsystemLogger("agents/acp-parent-stream");
type AcpParentProgressStreamingConfig = StreamingCompatEntry & {
accounts?: Record<string, StreamingCompatEntry | undefined>;
@@ -206,46 +211,14 @@ function shouldRelayAcpStatusProgress(params: {
return isAcpTagVisible(params.projectionSettings, params.tag);
}
function resolveAcpStreamLogPathFromSessionFile(sessionFile: string, sessionId: string): string {
const baseDir = path.dirname(path.resolve(sessionFile));
return path.join(baseDir, `${sessionId}.acp-stream.jsonl`);
}
/** Resolves the JSONL stream log path for an ACP child session when metadata exists. */
export function resolveAcpSpawnStreamLogPath(params: {
childSessionKey: string;
}): string | undefined {
const childSessionKey = normalizeOptionalString(params.childSessionKey);
if (!childSessionKey) {
return undefined;
}
const storeEntry = readAcpSessionEntry({
sessionKey: childSessionKey,
});
const sessionId = normalizeOptionalString(storeEntry?.entry?.sessionId);
if (!storeEntry || !sessionId) {
return undefined;
}
try {
const sessionFile = resolveSessionFilePath(
sessionId,
storeEntry.entry,
resolveSessionFilePathOptions({
storePath: storeEntry.storePath,
}),
);
return resolveAcpStreamLogPathFromSessionFile(sessionFile, sessionId);
} catch {
return undefined;
}
}
/** Starts a bounded parent-session relay for child ACP output and progress notices. */
export function startAcpSpawnParentStreamRelay(params: {
runId: string;
parentSessionKey: string;
childSessionKey: string;
childSessionId?: string;
agentId: string;
env?: NodeJS.ProcessEnv;
/**
* Optional `session.mainKey` from the runtime config. Used to remap
* cron-run parent session keys to the agent's main queue when relaying
@@ -262,7 +235,6 @@ export function startAcpSpawnParentStreamRelay(params: {
*/
sessionScope?: "per-sender" | "global";
eventRouting?: EventSessionRoutingPolicy;
logPath?: string;
deliveryContext?: DeliveryContext;
surfaceUpdates?: boolean;
streamFlushMs?: number;
@@ -300,67 +272,105 @@ export function startAcpSpawnParentStreamRelay(params: {
const relayLabel = truncate(compactWhitespace(params.agentId), 40) || "ACP child";
const contextPrefix = `acp-spawn:${runId}`;
const logPath = normalizeOptionalString(params.logPath);
let logDirReady = false;
let pendingLogLines = "";
let logFlushScheduled = false;
let logWriteChain: Promise<void> = Promise.resolve();
const flushLogBuffer = () => {
if (!logPath || !pendingLogLines) {
const childSessionId = normalizeOptionalString(params.childSessionId);
// Delayed flushes must keep the state database selected when the relay started.
const stateEnv = { ...(params.env ?? process.env) };
const pendingLogEvents: Array<{ event: AcpParentStreamEvent; createdAt: number }> = [];
let logFlushTimer: NodeJS.Timeout | undefined;
let logFailureWarned = false;
let logBufferWarned = false;
let consecutiveLogFailures = 0;
let disposed = false;
const capPendingLogEvents = () => {
const overflow = pendingLogEvents.length - STREAM_LOG_MAX_PENDING_EVENTS;
if (overflow <= 0) {
return;
}
const chunk = pendingLogLines;
pendingLogLines = "";
logWriteChain = logWriteChain
.then(async () => {
if (!logDirReady) {
await mkdir(path.dirname(logPath), {
recursive: true,
});
logDirReady = true;
}
await appendRegularFile({ filePath: logPath, content: chunk });
})
.catch(() => {
// Best-effort diagnostics; never break relay flow.
pendingLogEvents.splice(0, overflow);
if (!logBufferWarned) {
log.warn("Capped ACP parent stream diagnostic buffer", {
runId,
childSessionId,
maxPendingEvents: STREAM_LOG_MAX_PENDING_EVENTS,
});
logBufferWarned = true;
}
};
const scheduleLogFlush = () => {
if (!logPath || logFlushScheduled) {
const clearLogFlushTimer = () => {
if (!logFlushTimer) {
return;
}
logFlushScheduled = true;
queueMicrotask(() => {
logFlushScheduled = false;
flushLogBuffer();
});
clearTimeout(logFlushTimer);
logFlushTimer = undefined;
};
const writeLogLine = (entry: Record<string, unknown>) => {
if (!logPath) {
function flushLogEvents(options: { terminal?: boolean } = {}) {
clearLogFlushTimer();
if (!childSessionId || pendingLogEvents.length === 0) {
return;
}
const events = pendingLogEvents.splice(0);
try {
pendingLogLines += `${JSON.stringify(entry)}\n`;
if (pendingLogLines.length >= 16_384) {
flushLogBuffer();
return;
recordAcpParentStreamEvents({
agentId: params.agentId,
env: stateEnv,
sessionId: childSessionId,
runId,
events,
});
logFailureWarned = false;
logBufferWarned = false;
consecutiveLogFailures = 0;
} catch (error) {
if (!options.terminal) {
pendingLogEvents.unshift(...events);
capPendingLogEvents();
consecutiveLogFailures += 1;
scheduleLogFlush(
Math.min(STREAM_LOG_FLUSH_MS * 2 ** consecutiveLogFailures, STREAM_LOG_MAX_RETRY_MS),
);
}
if (!logFailureWarned || options.terminal) {
log.warn("Failed to persist ACP parent stream diagnostics", {
runId,
childSessionId,
retrying: !options.terminal,
error: String(error),
});
logFailureWarned = true;
}
scheduleLogFlush();
} catch {
// Best-effort diagnostics; never break relay flow.
}
};
}
function scheduleLogFlush(delayMs = STREAM_LOG_FLUSH_MS) {
if (disposed || logFlushTimer || pendingLogEvents.length === 0) {
return;
}
logFlushTimer = setTimeout(() => flushLogEvents(), delayMs);
logFlushTimer.unref?.();
}
const logEvent = (kind: string, fields?: Record<string, unknown>) => {
writeLogLine({
ts: new Date().toISOString(),
epochMs: Date.now(),
runId,
parentSessionKey,
childSessionKey: params.childSessionKey,
agentId: params.agentId,
kind,
...fields,
if (!childSessionId) {
return;
}
const createdAt = Date.now();
pendingLogEvents.push({
createdAt,
event: {
ts: new Date(createdAt).toISOString(),
epochMs: createdAt,
runId,
parentSessionKey,
childSessionKey: params.childSessionKey,
agentId: params.agentId,
kind,
...fields,
},
});
capPendingLogEvents();
if (consecutiveLogFailures === 0 && pendingLogEvents.length >= STREAM_LOG_BATCH_SIZE) {
flushLogEvents();
return;
}
scheduleLogFlush();
};
const shouldSurfaceUpdates = params.surfaceUpdates !== false;
const shouldRelayProgressCommentary = resolveParentProgressCommentary({
@@ -418,7 +428,6 @@ export function startAcpSpawnParentStreamRelay(params: {
);
};
let disposed = false;
let pendingText = "";
let pendingProgressKind: string | undefined;
let replaceableAssistantSnapshot: string | undefined;
@@ -757,8 +766,8 @@ export function startAcpSpawnParentStreamRelay(params: {
}
disposed = true;
clearFlushTimer();
flushLogEvents({ terminal: true });
clearRelayLifetimeTimer();
flushLogBuffer();
clearInterval(noOutputWatcherTimer);
unsubscribe();
};
+2 -25
View File
@@ -58,7 +58,6 @@ const hoisted = vi.hoisted(() => {
const initializeSessionMock = vi.fn();
const getAcpSessionManagerMock = vi.fn();
const startAcpSpawnParentStreamRelayMock = vi.fn();
const resolveAcpSpawnStreamLogPathMock = vi.fn();
const loadSessionStoreMock = vi.fn();
const readAcpSessionMetaMock = vi.fn();
const resolveStorePathMock = vi.fn();
@@ -150,7 +149,6 @@ const hoisted = vi.hoisted(() => {
initializeSessionMock,
getAcpSessionManagerMock,
startAcpSpawnParentStreamRelayMock,
resolveAcpSpawnStreamLogPathMock,
loadSessionStoreMock,
readAcpSessionMetaMock,
resolveStorePathMock,
@@ -229,7 +227,6 @@ vi.mock("../tasks/detached-task-runtime.js", () => ({
}));
vi.mock("./acp-spawn-parent-stream.js", () => ({
resolveAcpSpawnStreamLogPath: hoisted.resolveAcpSpawnStreamLogPathMock,
startAcpSpawnParentStreamRelay: hoisted.startAcpSpawnParentStreamRelayMock,
}));
@@ -811,9 +808,6 @@ describe("spawnAcpDirect", () => {
hoisted.startAcpSpawnParentStreamRelayMock
.mockReset()
.mockImplementation(() => createRelayHandle());
hoisted.resolveAcpSpawnStreamLogPathMock
.mockReset()
.mockReturnValue("/tmp/sess-main.acp-stream.jsonl");
hoisted.resolveStorePathMock.mockReset().mockReturnValue("/tmp/codex-sessions.json");
hoisted.readAcpSessionMetaMock.mockReset().mockReturnValue(undefined);
hoisted.loadSessionStoreMock.mockReset().mockImplementation(() => {
@@ -2424,7 +2418,6 @@ describe("spawnAcpDirect", () => {
const accepted = expectAcceptedSpawn(result);
expect(accepted.mode).toBe("run");
expect(accepted.streamLogPath).toBeUndefined();
expect(hoisted.startAcpSpawnParentStreamRelayMock).not.toHaveBeenCalled();
if (expectTranscriptPersistence) {
expectRecordFields(
@@ -2625,7 +2618,6 @@ describe("spawnAcpDirect", () => {
);
const accepted = expectAcceptedSpawn(result);
expect(accepted.streamLogPath).toBe("/tmp/sess-main.acp-stream.jsonl");
const agentCall = hoisted.callGatewayMock.mock.calls
.map((call: unknown[]) => call[0] as { method?: string; params?: Record<string, unknown> })
.find((request) => request.method === "agent");
@@ -2647,7 +2639,7 @@ describe("spawnAcpDirect", () => {
expectRelayCallFields({
parentSessionKey: "agent:main:main",
agentId: "codex",
logPath: "/tmp/sess-main.acp-stream.jsonl",
childSessionId: "sess-123",
emitStartNotice: false,
});
const relayRuns = hoisted.startAcpSpawnParentStreamRelayMock.mock.calls.map(
@@ -2655,11 +2647,6 @@ describe("spawnAcpDirect", () => {
);
expect(relayRuns).toContain(agentCall?.params?.idempotencyKey);
expect(relayRuns).toContain(accepted.runId);
const streamPathInput = expectRecordFields(
firstMockCall(hoisted.resolveAcpSpawnStreamLogPathMock, "stream log path resolution")[0],
{},
);
expect(streamPathInput.childSessionKey).toMatch(/^agent:codex:acp:/);
expect(firstHandle.dispose).toHaveBeenCalledTimes(1);
expect(firstHandle.notifyStarted).not.toHaveBeenCalled();
expect(secondHandle.notifyStarted).toHaveBeenCalledTimes(1);
@@ -2724,7 +2711,6 @@ describe("spawnAcpDirect", () => {
const accepted = expectAcceptedSpawn(result);
expect(accepted.mode).toBe("run");
expect(accepted.streamLogPath).toBe("/tmp/sess-main.acp-stream.jsonl");
const agentCall = hoisted.callGatewayMock.mock.calls
.map((call: unknown[]) => call[0] as { method?: string; params?: Record<string, unknown> })
.find((request) => request.method === "agent");
@@ -2735,7 +2721,7 @@ describe("spawnAcpDirect", () => {
expectRelayCallFields({
parentSessionKey: "agent:main:subagent:parent",
agentId: "codex",
logPath: "/tmp/sess-main.acp-stream.jsonl",
childSessionId: "sess-123",
deliveryContext: {
channel: "discord",
to: "channel:parent-channel",
@@ -2812,7 +2798,6 @@ describe("spawnAcpDirect", () => {
const accepted = expectAcceptedSpawn(result);
expect(accepted.mode).toBe("run");
expect(accepted.streamLogPath).toBeUndefined();
expect(hoisted.startAcpSpawnParentStreamRelayMock).not.toHaveBeenCalled();
});
@@ -2843,7 +2828,6 @@ describe("spawnAcpDirect", () => {
const accepted = expectAcceptedSpawn(result);
expect(accepted.mode).toBe("run");
expect(accepted.streamLogPath).toBeUndefined();
expect(hoisted.startAcpSpawnParentStreamRelayMock).not.toHaveBeenCalled();
});
@@ -2877,7 +2861,6 @@ describe("spawnAcpDirect", () => {
const accepted = expectAcceptedSpawn(result);
expect(accepted.mode).toBe("run");
expect(accepted.streamLogPath).toBeUndefined();
expect(hoisted.startAcpSpawnParentStreamRelayMock).not.toHaveBeenCalled();
});
@@ -2933,7 +2916,6 @@ describe("spawnAcpDirect", () => {
const accepted = expectAcceptedSpawn(result);
expect(accepted.mode).toBe("run");
expect(accepted.streamLogPath).toBeUndefined();
expect(hoisted.startAcpSpawnParentStreamRelayMock).not.toHaveBeenCalled();
});
@@ -2963,7 +2945,6 @@ describe("spawnAcpDirect", () => {
const accepted = expectAcceptedSpawn(result);
expect(accepted.mode).toBe("run");
expect(accepted.streamLogPath).toBeUndefined();
expect(hoisted.startAcpSpawnParentStreamRelayMock).not.toHaveBeenCalled();
});
@@ -2982,7 +2963,6 @@ describe("spawnAcpDirect", () => {
const accepted = expectAcceptedSpawn(result);
expect(accepted.mode).toBe("run");
expect(accepted.streamLogPath).toBeUndefined();
expect(hoisted.startAcpSpawnParentStreamRelayMock).not.toHaveBeenCalled();
});
@@ -2999,7 +2979,6 @@ describe("spawnAcpDirect", () => {
const accepted = expectAcceptedSpawn(result);
expect(accepted.mode).toBe("run");
expect(accepted.streamLogPath).toBeUndefined();
expect(hoisted.startAcpSpawnParentStreamRelayMock).not.toHaveBeenCalled();
});
@@ -3020,7 +2999,6 @@ describe("spawnAcpDirect", () => {
const accepted = expectAcceptedSpawn(result);
expect(accepted.mode).toBe("run");
expect(accepted.streamLogPath).toBeUndefined();
expect(hoisted.startAcpSpawnParentStreamRelayMock).not.toHaveBeenCalled();
});
@@ -3053,7 +3031,6 @@ describe("spawnAcpDirect", () => {
const accepted = expectAcceptedSpawn(result);
expect(accepted.mode).toBe("run");
expect(accepted.streamLogPath).toBeUndefined();
expect(hoisted.startAcpSpawnParentStreamRelayMock).not.toHaveBeenCalled();
});
+7 -11
View File
@@ -77,7 +77,6 @@ import { listTasksForOwnerKey } from "../tasks/runtime-internal.js";
import { deliveryContextFromSession, normalizeDeliveryContext } from "../utils/delivery-context.js";
import {
type AcpSpawnParentRelayHandle,
resolveAcpSpawnStreamLogPath,
startAcpSpawnParentStreamRelay,
} from "./acp-spawn-parent-stream.js";
import { listAgentIds, resolveAgentConfig, resolveDefaultAgentId } from "./agent-scope.js";
@@ -206,7 +205,6 @@ type SpawnAcpResultFields = {
mode?: SpawnAcpMode;
runTimeoutSeconds?: number;
inlineDelivery?: boolean;
streamLogPath?: string;
note?: string;
};
@@ -1478,6 +1476,7 @@ export async function spawnAcpDirect(
let binding: SessionBindingRecord | null = null;
let sessionCreated = false;
let childSessionId: string | undefined;
let initializedRuntime: AcpSpawnRuntimeCloseHandle | undefined;
try {
await callGateway({
@@ -1504,6 +1503,7 @@ export async function spawnAcpDirect(
cwd: runtimeCwd,
});
initializedRuntime = initializedSession.runtimeCloseHandle;
childSessionId = initializedSession.sessionId;
if (preparedBinding) {
({ binding } = await bindPreparedAcpThread({
@@ -1548,12 +1548,6 @@ export async function spawnAcpDirect(
requesterSessionKey: requesterInternalKey,
agentId: targetAgentId,
});
const streamLogPath =
effectiveStreamToParent && parentSessionKey
? resolveAcpSpawnStreamLogPath({
childSessionKey: sessionKey,
})
: undefined;
const parentAgentId = parentSessionKey
? resolveAgentIdFromSessionKey(parentSessionKey)
: undefined;
@@ -1571,6 +1565,7 @@ export async function spawnAcpDirect(
: undefined;
let parentRelay: AcpSpawnParentRelayHandle | undefined;
const parentRelayStateEnv = { ...process.env };
const parentEventRouting = parentSessionKey
? resolveEventSessionRoutingPolicy({ cfg, sessionKey: parentSessionKey })
: undefined;
@@ -1580,11 +1575,12 @@ export async function spawnAcpDirect(
runId: childIdem,
parentSessionKey,
childSessionKey: sessionKey,
childSessionId,
agentId: targetAgentId,
env: parentRelayStateEnv,
mainKey: cfg.session?.mainKey,
sessionScope: cfg.session?.scope,
eventRouting: parentEventRouting,
logPath: streamLogPath,
deliveryContext: parentDeliveryCtx,
emitStartNotice: false,
cfg,
@@ -1640,11 +1636,12 @@ export async function spawnAcpDirect(
runId: childRunId,
parentSessionKey,
childSessionKey: sessionKey,
childSessionId,
agentId: targetAgentId,
env: parentRelayStateEnv,
mainKey: cfg.session?.mainKey,
sessionScope: cfg.session?.scope,
eventRouting: parentEventRouting,
logPath: streamLogPath,
deliveryContext: parentDeliveryCtx,
emitStartNotice: false,
cfg,
@@ -1687,7 +1684,6 @@ export async function spawnAcpDirect(
runId: childRunId,
mode: spawnMode,
runTimeoutSeconds,
...(streamLogPath ? { streamLogPath } : {}),
note: spawnMode === "session" ? ACP_SPAWN_SESSION_ACCEPTED_NOTE : ACP_SPAWN_ACCEPTED_NOTE,
};
}
@@ -1,7 +1,7 @@
// Gateway run option collision tests cover gateway run flag registration boundaries.
import path from "node:path";
import { Command } from "commander";
import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from "vitest";
import { CONFIG_AUDIT_STORE_LABEL } from "../../config/io.audit.js";
import type { ConfigFileSnapshot } from "../../config/types.js";
import { GATEWAY_SERVICE_RUNTIME_PID_ENV } from "../../daemon/constants.js";
import { SUPERVISOR_HINT_ENV_VARS } from "../../infra/supervisor-markers.js";
@@ -1589,9 +1589,7 @@ describe("gateway run option collisions", () => {
expect(runtimeErrors).toContain(
"Gateway start blocked: existing config is missing gateway.mode. Treat this as suspicious or clobbered config. Re-run `openclaw onboard --mode local` or `openclaw setup`, set gateway.mode=local manually, or pass --allow-unconfigured.",
);
expect(runtimeErrors).toContain(
`Config write audit: ${path.join("/tmp", "logs", "config-audit.jsonl")}`,
);
expect(runtimeErrors).toContain(`Config write audit: ${CONFIG_AUDIT_STORE_LABEL}`);
expect(startGatewayServer).not.toHaveBeenCalled();
expect(readBestEffortConfig).not.toHaveBeenCalled();
});
@@ -1613,9 +1611,7 @@ describe("gateway run option collisions", () => {
expect(runtimeErrors).toContain(
"Gateway start blocked: existing config is missing gateway.mode. Treat this as suspicious or clobbered config. Re-run `openclaw onboard --mode local` or `openclaw setup`, set gateway.mode=local manually, or pass --allow-unconfigured.",
);
expect(runtimeErrors).toContain(
`Config write audit: ${path.join("/tmp", "logs", "config-audit.jsonl")}`,
);
expect(runtimeErrors).toContain(`Config write audit: ${CONFIG_AUDIT_STORE_LABEL}`);
expect(readConfigFileSnapshotWithPluginMetadata).toHaveBeenCalledOnce();
expect(startGatewayServer).not.toHaveBeenCalled();
});
+6 -12
View File
@@ -2,7 +2,6 @@
import fs from "node:fs";
import { request as httpRequest } from "node:http";
import { request as httpsRequest } from "node:https";
import path from "node:path";
import { TLSSocket } from "node:tls";
import { expectDefined } from "@openclaw/normalization-core";
import {
@@ -17,16 +16,12 @@ import type {
ReadConfigFileSnapshotWithPluginMetadataResult,
} from "../../config/config.js";
import { ALLOW_OLDER_BINARY_DESTRUCTIVE_ACTIONS_ENV } from "../../config/future-version-guard.js";
import { CONFIG_AUDIT_STORE_LABEL } from "../../config/io.audit.js";
import {
isDoctorRecoverableInvalidConfigError,
isInvalidConfigError,
} from "../../config/io.invalid-config.js";
import {
CONFIG_PATH,
normalizeStateDirEnv,
resolveGatewayPort,
resolveStateDir,
} from "../../config/paths.js";
import { CONFIG_PATH, normalizeStateDirEnv, resolveGatewayPort } from "../../config/paths.js";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
import { hasConfiguredSecretInput } from "../../config/types.secrets.js";
import { GATEWAY_SERVICE_RUNTIME_PID_ENV } from "../../daemon/constants.js";
@@ -250,7 +245,7 @@ function shouldBlockGatewayBindWithoutExplicitAuth(params: {
function getGatewayStartGuardErrors(params: {
allowUnconfigured?: boolean;
configExists: boolean;
configAuditPath: string;
configAuditLocation: string;
mode: string | undefined;
}): string[] {
if (params.allowUnconfigured || params.mode === "local") {
@@ -268,12 +263,12 @@ function getGatewayStartGuardErrors(params: {
"Treat this as suspicious or clobbered config.",
`Re-run \`${formatCliCommand("openclaw onboard --mode local")}\` or \`${formatCliCommand("openclaw setup")}\`, set gateway.mode=local manually, or pass --allow-unconfigured.`,
].join(" "),
`Config write audit: ${params.configAuditPath}`,
`Config write audit: ${params.configAuditLocation}`,
];
}
return [
`Gateway start blocked: set gateway.mode=local (current: ${params.mode}) or pass --allow-unconfigured.`,
`Config write audit: ${params.configAuditPath}`,
`Config write audit: ${params.configAuditLocation}`,
];
}
@@ -959,13 +954,12 @@ async function runGatewayCommandOnce(opts: GatewayRunOpts, hooks: GatewayRunRunt
gatewayLog.info("resolving authentication…");
const configExists = snapshot?.exists ?? fs.existsSync(CONFIG_PATH);
const configAuditPath = path.join(resolveStateDir(process.env), "logs", "config-audit.jsonl");
const effectiveCfg = snapshot?.valid ? snapshot.config : cfg;
const mode = effectiveCfg.gateway?.mode;
const guardErrors = getGatewayStartGuardErrors({
allowUnconfigured: opts.allowUnconfigured,
configExists,
configAuditPath,
configAuditLocation: CONFIG_AUDIT_STORE_LABEL,
mode,
});
if (guardErrors.length > 0) {
+2 -2
View File
@@ -3,7 +3,7 @@ import fs from "node:fs/promises";
import os from "node:os";
import { note } from "../../packages/terminal-core/src/note.js";
import {
resolveConfigAuditLogPath,
resolveLegacyConfigAuditLogPath,
scrubConfigAuditLog,
type ConfigAuditScrubResult,
} from "../config/io.audit.js";
@@ -30,7 +30,7 @@ export async function detectConfigAuditScrubIssue(params?: {
});
return {
...result,
auditPath: resolveConfigAuditLogPath(env, homedir),
auditPath: resolveLegacyConfigAuditLogPath(env, homedir),
};
}
+4
View File
@@ -289,6 +289,10 @@ function createLegacyStateMigrationDetectionResult(params?: {
sourcePath: "/tmp/state/commitments/commitments.json",
hasLegacy: false,
},
auditLogs: {
sources: [],
hasLegacy: false,
},
acpReplayLedger: {
sourcePath: "/tmp/state/acp/event-ledger.json",
hasLegacy: false,
+23
View File
@@ -0,0 +1,23 @@
import { createSqliteAuditRecordStore } from "../infra/sqlite-audit-record-store.js";
import {
CONFIG_AUDIT_MAX_ENTRIES,
CONFIG_AUDIT_SCOPE,
type ConfigAuditRecord,
} from "./io.audit.js";
import { resolveStateDir } from "./paths.js";
export function listConfigAuditRecordsForTests(params: {
env: NodeJS.ProcessEnv;
homedir: () => string;
}): ConfigAuditRecord[] {
return createSqliteAuditRecordStore<ConfigAuditRecord>({
scope: CONFIG_AUDIT_SCOPE,
maxEntries: CONFIG_AUDIT_MAX_ENTRIES,
env: {
...params.env,
OPENCLAW_STATE_DIR: resolveStateDir(params.env, params.homedir),
},
})
.entries()
.map((entry) => entry.value);
}
+118 -14
View File
@@ -2,16 +2,18 @@
import fs from "node:fs";
import { promises as fsPromises } from "node:fs";
import path from "node:path";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { afterAll, afterEach, beforeAll, describe, expect, it } from "vitest";
import { resetPluginStateStoreForTests } from "../plugin-state/plugin-state-store.js";
import { createSuiteTempRootTracker } from "../test-helpers/temp-dir.js";
import {
appendConfigAuditRecord,
createConfigWriteAuditRecordBase,
finalizeConfigWriteAuditRecord,
formatConfigOverwriteLogMessage,
resolveConfigAuditLogPath,
resolveLegacyConfigAuditLogPath,
scrubConfigAuditLog,
} from "./io.audit.js";
import { listConfigAuditRecordsForTests } from "./io.audit.test-support.js";
function createAuditRecordBase(configPath: string, argv?: string[]) {
return createConfigWriteAuditRecordBase({
@@ -66,7 +68,7 @@ function createRenameAuditRecord(home: string) {
});
}
function readAuditLog(home: string): unknown[] {
function readLegacyAuditLog(home: string): unknown[] {
const auditPath = path.join(home, ".openclaw", "logs", "config-audit.jsonl");
return fs
.readFileSync(auditPath, "utf-8")
@@ -93,9 +95,13 @@ describe("config io audit helpers", () => {
await suiteRootTracker.cleanup();
});
afterEach(() => {
resetPluginStateStoreForTests();
});
it('ignores literal "undefined" home env values when choosing the audit log path', async () => {
const home = await suiteRootTracker.make("home");
const auditPath = resolveConfigAuditLogPath(
const auditPath = resolveLegacyConfigAuditLogPath(
{
HOME: "undefined",
USERPROFILE: "null",
@@ -196,18 +202,20 @@ describe("config io audit helpers", () => {
expect(record.errorMessage).toBe("disk full");
});
it("appends JSONL audit entries to the resolved audit path", async () => {
it("appends audit entries to shared SQLite state", async () => {
const home = await suiteRootTracker.make("append");
const record = createRenameAuditRecord(home);
await appendConfigAuditRecord({
fs,
env: {} as NodeJS.ProcessEnv,
homedir: () => home,
record,
});
const records = readAuditLog(home);
const records = listConfigAuditRecordsForTests({
env: {} as NodeJS.ProcessEnv,
homedir: () => home,
});
expect(records).toHaveLength(1);
const written = requireAuditRecord(records[0]);
expect(written.event).toBe("config.write");
@@ -230,15 +238,16 @@ describe("config io audit helpers", () => {
});
await appendConfigAuditRecord({
fs,
env: {} as NodeJS.ProcessEnv,
homedir: () => home,
record,
});
const raw = fs.readFileSync(
path.join(home, ".openclaw", "logs", "config-audit.jsonl"),
"utf-8",
const raw = JSON.stringify(
listConfigAuditRecordsForTests({
env: {} as NodeJS.ProcessEnv,
homedir: () => home,
}),
);
expect(raw).not.toContain("AIzaSyD-very-real-looking");
expect(raw).not.toContain("ya29.fake-access-token");
@@ -355,6 +364,99 @@ describe("config io audit helpers", () => {
argv: ["openclaw", "--token"],
expected: ["openclaw", "--token"],
},
{
name: "sensitive config set positional value",
argv: ["openclaw", "config", "set", "channels.slack.token", "secret-value"],
expected: ["openclaw", "config", "set", "channels.slack.token", "***"],
},
{
name: "sensitive config set value after boolean option",
argv: ["openclaw", "config", "set", "--json", "channels.slack.token", '"secret-value"'],
expected: ["openclaw", "config", "set", "--json", "channels.slack.token", "***"],
},
{
name: "sensitive config set value after root value option",
argv: [
"openclaw",
"config",
"set",
"--profile",
"work",
"channels.slack.token",
"secret-value",
],
expected: ["openclaw", "config", "set", "--profile", "work", "channels.slack.token", "***"],
},
{
name: "sensitive config set value after option before subcommand",
argv: [
"openclaw",
"config",
"--profile",
"work",
"set",
"channels.slack.token",
"secret-value",
],
expected: ["openclaw", "config", "--profile", "work", "set", "channels.slack.token", "***"],
},
{
name: "sensitive config set value after config parent option",
argv: [
"openclaw",
"config",
"--section",
"channels",
"set",
"channels.slack.token",
"secret-value",
],
expected: [
"openclaw",
"config",
"--section",
"channels",
"set",
"channels.slack.token",
"***",
],
},
{
name: "sensitive config set value when a root option value is config",
argv: [
"openclaw",
"--profile",
"config",
"config",
"set",
"channels.slack.token",
"secret-value",
],
expected: ["openclaw", "--profile", "config", "config", "set", "channels.slack.token", "***"],
},
{
name: "sensitive config set value after interleaved option",
argv: [
"openclaw",
"config",
"set",
"channels.slack.token",
"--strict-json",
'"secret-value"',
],
expected: ["openclaw", "config", "set", "channels.slack.token", "--strict-json", "***"],
},
{
name: "config set batch JSON",
argv: [
"openclaw",
"config",
"set",
"--batch-json",
'[{"path":"channels.slack.token","value":"secret-value"}]',
],
expected: ["openclaw", "config", "set", "--batch-json", "***"],
},
])("redacts $name in persisted audit process info", ({ argv, expected }) => {
expect(createAuditRecordBase("/tmp/openclaw.json", argv).argv).toEqual(expected);
});
@@ -364,13 +466,15 @@ describe("config io audit helpers", () => {
const record = createRenameAuditRecord(home);
await appendConfigAuditRecord({
fs,
env: {} as NodeJS.ProcessEnv,
homedir: () => home,
...record,
});
const records = readAuditLog(home);
const records = listConfigAuditRecordsForTests({
env: {} as NodeJS.ProcessEnv,
homedir: () => home,
});
expect(records).toHaveLength(1);
const written = requireAuditRecord(records[0]);
expect(written.event).toBe("config.write");
@@ -429,7 +533,7 @@ describe("config io audit helpers", () => {
});
expect(result).toEqual({ scanned: 2, rewritten: 1, skipped: 0, aborted: false });
const after = readAuditLog(home);
const after = readLegacyAuditLog(home);
expect(after).toHaveLength(2);
const firstAfter = requireAuditRecord(after[0]);
const secondAfter = requireAuditRecord(after[1]);
+173 -43
View File
@@ -1,13 +1,127 @@
// Audits config paths and values for diagnostics and safety checks.
import { randomUUID } from "node:crypto";
import path from "node:path";
import { createSqliteAuditRecordStore } from "../infra/sqlite-audit-record-store.js";
import { redactSecrets } from "../logging/redact.js";
import { resolveStateDir } from "./paths.js";
import { redactSensitiveArgv } from "./redact-argv.js";
import { isSensitiveConfigPath } from "./sensitive-paths.js";
const CONFIG_AUDIT_ARGV_CAP = 8;
const CONFIG_SET_VALUE_OPTIONS = new Set([
"--batch-file",
"--batch-json",
"--container",
"--log-level",
"--profile",
"--provider-allowlist",
"--provider-arg",
"--provider-command",
"--provider-env",
"--provider-max-bytes",
"--provider-max-output-bytes",
"--provider-mode",
"--provider-no-output-timeout-ms",
"--provider-pass-env",
"--provider-path",
"--provider-source",
"--provider-timeout-ms",
"--provider-trusted-dir",
"--ref-id",
"--ref-provider",
"--ref-source",
"--section",
]);
function findConfigSetPositionals(argv: readonly string[], setIndex: number): number[] {
const positionals: number[] = [];
let optionsEnded = false;
for (let index = setIndex + 1; index < argv.length && positionals.length < 2; index += 1) {
const arg = argv[index];
if (arg === undefined) {
break;
}
if (!optionsEnded && arg === "--") {
optionsEnded = true;
continue;
}
if (!optionsEnded && arg.startsWith("-")) {
const equalsIndex = arg.indexOf("=");
const optionName = equalsIndex < 0 ? arg : arg.slice(0, equalsIndex);
if (equalsIndex < 0 && CONFIG_SET_VALUE_OPTIONS.has(optionName)) {
index += 1;
}
continue;
}
positionals.push(index);
}
return positionals;
}
function findConfigSetCommandIndex(argv: readonly string[], configIndex: number): number {
let optionsEnded = false;
for (let index = configIndex + 1; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === undefined) {
return -1;
}
if (!optionsEnded && arg === "--") {
optionsEnded = true;
continue;
}
if (!optionsEnded && arg.startsWith("-")) {
const equalsIndex = arg.indexOf("=");
const optionName = equalsIndex < 0 ? arg : arg.slice(0, equalsIndex);
if (equalsIndex < 0 && CONFIG_SET_VALUE_OPTIONS.has(optionName)) {
index += 1;
}
continue;
}
return arg === "set" ? index : -1;
}
return -1;
}
function redactConfigAuditArgv(argv: readonly string[]): string[] {
return redactSensitiveArgv(argv);
const redacted = redactSensitiveArgv(argv);
let setIndex = -1;
for (let index = 0; index < redacted.length; index += 1) {
if (redacted[index] !== "config") {
continue;
}
setIndex = findConfigSetCommandIndex(redacted, index);
if (setIndex >= 0) {
break;
}
}
if (setIndex < 0) {
return redacted;
}
for (let index = setIndex + 1; index < redacted.length; index += 1) {
const arg = redacted[index];
if (arg === undefined) {
break;
}
if (arg === "--batch-json" && index + 1 < redacted.length) {
redacted[index + 1] = "***";
index += 1;
continue;
}
if (arg.startsWith("--batch-json=")) {
redacted[index] = "--batch-json=***";
}
}
const positionals = findConfigSetPositionals(redacted, setIndex);
if (positionals.length < 2) {
return redacted;
}
const pathIndex = positionals[0]!;
const valueIndex = positionals[1]!;
const configPath = redacted[pathIndex];
if (typeof configPath === "string" && isSensitiveConfigPath(configPath)) {
redacted[valueIndex] = "***";
}
return redacted;
}
function capArgv(argv: readonly string[] | undefined): string[] {
@@ -27,7 +141,11 @@ export function snapshotConfigAuditProcessInfo(): ConfigAuditProcessInfo {
};
}
const CONFIG_AUDIT_LOG_FILENAME = "config-audit.jsonl";
export const CONFIG_AUDIT_SCOPE = "config-audit";
export const CONFIG_AUDIT_MAX_ENTRIES = 50_000;
export const CONFIG_AUDIT_STORE_LABEL =
"SQLite diagnostic_events/config-audit state (latest 50000 rows)";
const LEGACY_CONFIG_AUDIT_LOG_FILENAME = ["config-audit", "jsonl"].join(".");
export type ConfigWriteAuditResult = "rename" | "copy-fallback" | "failed" | "rejected";
@@ -127,7 +245,7 @@ export type ConfigObserveAuditRecord = {
restoreErrorMessage: string | null;
};
type ConfigAuditRecord = ConfigWriteAuditRecord | ConfigObserveAuditRecord;
export type ConfigAuditRecord = ConfigWriteAuditRecord | ConfigObserveAuditRecord;
type ConfigAuditStatMetadata = {
dev: string | null;
@@ -162,23 +280,6 @@ type ConfigWriteAuditRecordBase = Omit<
nextBytes: number;
};
type ConfigAuditFs = {
promises: {
mkdir(path: string, options?: { recursive?: boolean; mode?: number }): Promise<unknown>;
appendFile(
path: string,
data: string,
options?: { encoding?: BufferEncoding; mode?: number },
): Promise<unknown>;
};
mkdirSync(path: string, options?: { recursive?: boolean; mode?: number }): unknown;
appendFileSync(
path: string,
data: string,
options?: { encoding?: BufferEncoding; mode?: number },
): unknown;
};
function normalizeAuditLabel(value: string | undefined): string | null {
if (typeof value !== "string") {
return null;
@@ -200,8 +301,11 @@ function resolveConfigAuditProcessInfo(
return snapshotConfigAuditProcessInfo();
}
export function resolveConfigAuditLogPath(env: NodeJS.ProcessEnv, homedir: () => string): string {
return path.join(resolveStateDir(env, homedir), "logs", CONFIG_AUDIT_LOG_FILENAME);
export function resolveLegacyConfigAuditLogPath(
env: NodeJS.ProcessEnv,
homedir: () => string,
): string {
return path.join(resolveStateDir(env, homedir), "logs", LEGACY_CONFIG_AUDIT_LOG_FILENAME);
}
export function formatConfigOverwriteLogMessage(params: {
@@ -307,13 +411,12 @@ export function finalizeConfigWriteAuditRecord(params: {
nextNlink: success ? nextMetadata.nlink : null,
nextUid: success ? nextMetadata.uid : null,
nextGid: success ? nextMetadata.gid : null,
errorCode,
errorMessage,
...(errorCode !== undefined ? { errorCode } : {}),
...(errorMessage !== undefined ? { errorMessage } : {}),
};
}
type ConfigAuditAppendContext = {
fs: ConfigAuditFs;
env: NodeJS.ProcessEnv;
homedir: () => string;
};
@@ -328,10 +431,10 @@ type ConfigAuditAppendParams = ConfigAuditAppendContext &
function resolveConfigAuditAppendRecord(params: ConfigAuditAppendParams): ConfigAuditRecord {
if ("record" in params) {
return redactSecrets(params.record);
return params.record;
}
const { fs: _fs, env: _env, homedir: _homedir, ...record } = params;
return redactSecrets(record as ConfigAuditRecord);
const { env: _env, homedir: _homedir, ...record } = params;
return record as ConfigAuditRecord;
}
export type ConfigAuditScrubResult = {
@@ -376,7 +479,7 @@ export async function scrubConfigAuditLog(params: {
homedir: () => string;
dryRun?: boolean;
}): Promise<ConfigAuditScrubResult> {
const auditPath = resolveConfigAuditLogPath(params.env, params.homedir);
const auditPath = resolveLegacyConfigAuditLogPath(params.env, params.homedir);
let raw: string;
try {
raw = await params.fs.promises.readFile(auditPath, "utf-8");
@@ -503,15 +606,43 @@ export async function scrubConfigAuditLog(params: {
return { scanned, rewritten, skipped, aborted: false };
}
function resolveConfigAuditStoreEnv(params: {
env: NodeJS.ProcessEnv;
homedir: () => string;
}): NodeJS.ProcessEnv {
return {
...params.env,
OPENCLAW_STATE_DIR: resolveStateDir(params.env, params.homedir),
};
}
function openConfigAuditStore(env: NodeJS.ProcessEnv) {
return createSqliteAuditRecordStore<ConfigAuditRecord>({
scope: CONFIG_AUDIT_SCOPE,
maxEntries: CONFIG_AUDIT_MAX_ENTRIES,
env,
});
}
function configAuditEntryKey(record: ConfigAuditRecord): string {
return `${record.ts}:${record.event}:${randomUUID()}`;
}
export function sanitizeConfigAuditRecord(record: ConfigAuditRecord): ConfigAuditRecord {
const sanitized = structuredClone(record);
sanitized.argv = redactConfigAuditArgv(capArgv(sanitized.argv));
sanitized.execArgv = redactConfigAuditArgv(capArgv(sanitized.execArgv));
return redactSecrets(sanitized);
}
export async function appendConfigAuditRecord(params: ConfigAuditAppendParams): Promise<void> {
try {
const auditPath = resolveConfigAuditLogPath(params.env, params.homedir);
const record = resolveConfigAuditAppendRecord(params);
await params.fs.promises.mkdir(path.dirname(auditPath), { recursive: true, mode: 0o700 });
await params.fs.promises.appendFile(auditPath, `${JSON.stringify(record)}\n`, {
encoding: "utf-8",
mode: 0o600,
});
const record = sanitizeConfigAuditRecord(resolveConfigAuditAppendRecord(params));
openConfigAuditStore(resolveConfigAuditStoreEnv(params)).register(
configAuditEntryKey(record),
record,
Date.parse(record.ts),
);
} catch {
// best-effort
}
@@ -519,13 +650,12 @@ export async function appendConfigAuditRecord(params: ConfigAuditAppendParams):
export function appendConfigAuditRecordSync(params: ConfigAuditAppendParams): void {
try {
const auditPath = resolveConfigAuditLogPath(params.env, params.homedir);
const record = resolveConfigAuditAppendRecord(params);
params.fs.mkdirSync(path.dirname(auditPath), { recursive: true, mode: 0o700 });
params.fs.appendFileSync(auditPath, `${JSON.stringify(record)}\n`, {
encoding: "utf-8",
mode: 0o600,
});
const record = sanitizeConfigAuditRecord(resolveConfigAuditAppendRecord(params));
openConfigAuditStore(resolveConfigAuditStoreEnv(params)).register(
configAuditEntryKey(record),
record,
Date.parse(record.ts),
);
} catch {
// best-effort
}
+6 -11
View File
@@ -11,6 +11,7 @@ import {
closeOpenClawStateDatabaseForTest,
openOpenClawStateDatabase,
} from "../state/openclaw-state-db.js";
import { listConfigAuditRecordsForTests } from "./io.audit.test-support.js";
import { createConfigIO } from "./io.js";
import {
maybeRecoverSuspiciousConfigRead,
@@ -101,17 +102,11 @@ describe("config observe recovery", () => {
}
async function readObserveEvents(auditPath: string): Promise<Record<string, unknown>[]> {
const events: Record<string, unknown>[] = [];
for (const line of (await fsp.readFile(auditPath, "utf-8")).trim().split("\n")) {
if (!line) {
continue;
}
const parsed = JSON.parse(line) as Record<string, unknown>;
if (parsed.event === "config.observe") {
events.push(parsed);
}
}
return events;
const stateDir = path.dirname(path.dirname(auditPath));
return listConfigAuditRecordsForTests({
env: { OPENCLAW_STATE_DIR: stateDir },
homedir: () => stateDir,
}).filter((event) => event.event === "config.observe");
}
async function listClobberFiles(configPath: string): Promise<string[]> {
-1
View File
@@ -240,7 +240,6 @@ function createConfigObserveAuditAppendParams(
params: ConfigObserveAuditRecordParams,
) {
return {
fs: deps.fs,
env: deps.env,
homedir: deps.homedir,
record: createConfigObserveAuditRecord(params),
-2
View File
@@ -259,7 +259,6 @@ export async function observeConfigSnapshot(
(baseline?.hash ? baseline : null) ?? (await readConfigFingerprintForPath(deps, backupPath));
deps.logger.warn(`Config observe anomaly: ${snapshot.path} (${suspicious.join(", ")})`);
await appendConfigAuditRecord({
fs: deps.fs,
env: deps.env,
homedir: deps.homedir,
record: createObserveAuditRecord({ snapshot, current, suspicious, entry, backup }),
@@ -306,7 +305,6 @@ export function observeConfigSnapshotSync(
(baseline?.hash ? baseline : null) ?? readConfigFingerprintForPathSync(deps, backupPath);
deps.logger.warn(`Config observe anomaly: ${snapshot.path} (${suspicious.join(", ")})`);
appendConfigAuditRecordSync({
fs: deps.fs,
env: deps.env,
homedir: deps.homedir,
record: createObserveAuditRecord({ snapshot, current, suspicious, entry, backup }),
-1
View File
@@ -294,7 +294,6 @@ export async function writeConfigFileFromContext(
nextStat?: fs.Stats | null,
) => {
await appendConfigAuditRecord({
fs: deps.fs,
env: deps.env,
homedir: deps.homedir,
record: finalizeConfigWriteAuditRecord({
+253
View File
@@ -8,6 +8,7 @@ import * as tar from "tar";
import { describe, expect, it, vi } from "vitest";
import { saveAuthProfileStore } from "../agents/auth-profiles/store.js";
import { backupVerifyCommand } from "../commands/backup-verify.js";
import { CONFIG_AUDIT_MAX_ENTRIES, CONFIG_AUDIT_SCOPE } from "../config/io.audit.js";
import type { RuntimeEnv } from "../runtime.js";
import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js";
import {
@@ -29,6 +30,8 @@ import { writeTarArchiveWithRetry } from "./backup-tar-retry.js";
import { isVolatileBackupPath } from "./backup-volatile-filter.js";
import { createBackupVolatileStatCache } from "./backup-volatile-stat-cache.js";
import { requireNodeSqlite } from "./node-sqlite.js";
import { createSqliteAuditRecordStore } from "./sqlite-audit-record-store.js";
import { detectLegacyAuditLogs, migrateLegacyAuditLogs } from "./state-migrations.audit-logs.js";
function makeResult(overrides: Partial<BackupCreateResult> = {}): BackupCreateResult {
return {
@@ -239,6 +242,27 @@ describe("sanitizeOpenClawGlobalStateSnapshot", () => {
database.close();
}
});
it("leaves diagnostic state to its backup-specific sanitizer", () => {
const sqlite = requireNodeSqlite();
const database = new sqlite.DatabaseSync(":memory:");
try {
database.exec(`
CREATE TABLE diagnostic_events (scope TEXT);
INSERT INTO diagnostic_events VALUES ('migration.legacy-audit-raw');
INSERT INTO diagnostic_events VALUES ('system-agent.audit');
`);
sanitizeOpenClawGlobalStateSnapshot(database);
expect(database.prepare("SELECT scope FROM diagnostic_events").all()).toEqual([
{ scope: "migration.legacy-audit-raw" },
{ scope: "system-agent.audit" },
]);
} finally {
database.close();
}
});
});
describe("writeTarArchiveWithRetry", () => {
@@ -621,6 +645,235 @@ describe("createBackupArchive", () => {
);
});
it("replaces legacy audit raw archives with sanitized restorable snapshots", async () => {
await withOpenClawTestState(
{
layout: "state-only",
prefix: "openclaw-backup-audit-raw-",
scenario: "minimal",
},
async (state) => {
const outputDir = state.path("backups");
const extractDir = state.path("extract");
const rawRelativePath = "logs/config-audit.jsonl.migrated.raw";
const marker = "audit-value-7f3c";
await fs.mkdir(outputDir, { recursive: true });
await fs.mkdir(extractDir, { recursive: true });
await state.writeText(
rawRelativePath,
`${JSON.stringify({
ts: "2026-07-01T00:00:00.000Z",
source: "config-io",
event: "config.write",
argv: ["openclaw", "config", "set", "token", marker],
execArgv: [],
})}\n`,
);
const { db } = openOpenClawStateDatabase({ env: state.env });
db.prepare(
`
INSERT INTO diagnostic_events (
scope, event_key, payload_json, created_at, sequence
) VALUES ('migration.legacy-audit-raw', 'checkpoint', '{}', 1, 1)
`,
).run();
try {
const result = await createBackupArchive({
output: outputDir,
includeWorkspace: false,
nowMs: Date.UTC(2026, 4, 9, 8, 15, 0),
});
const entries = await listArchiveEntries(result.archivePath);
const rawEntry = expectDefined(
entries.find((entry) => entry.endsWith(`/state/${rawRelativePath}`)),
"sanitized raw archive entry",
);
const databaseEntry = expectDefined(
entries.find((entry) => entry.endsWith("/state/state/openclaw.sqlite")),
"global state database entry",
);
expect(entries.some((entry) => entry.endsWith(".doctor-scrub-restore"))).toBe(false);
await tar.x({ file: result.archivePath, gzip: true, cwd: extractDir });
const archivedRaw = await fs.readFile(path.join(extractDir, rawEntry), "utf8");
expect(archivedRaw).not.toContain(marker);
expect(JSON.parse(archivedRaw.trim())).toMatchObject({
argv: ["openclaw", "config", "set", "token", "***"],
});
const sqlite = requireNodeSqlite();
const archivedDb = new sqlite.DatabaseSync(path.join(extractDir, databaseEntry), {
readOnly: true,
});
try {
expect(
archivedDb
.prepare(
"SELECT COUNT(*) AS count FROM diagnostic_events WHERE scope = 'migration.legacy-audit-raw'",
)
.get(),
).toEqual({ count: 0 });
} finally {
archivedDb.close();
}
} finally {
closeOpenClawStateDatabase();
}
},
);
});
it("omits completed blank audit append pads when dropping their checkpoints", async () => {
await withOpenClawTestState(
{
layout: "state-only",
prefix: "openclaw-backup-completed-audit-pad-",
scenario: "minimal",
},
async (state) => {
const outputDir = state.path("backups");
const extractDir = state.path("extract");
const sourcePath = state.statePath("logs/config-audit.jsonl");
const rawRelativePath = "logs/config-audit.jsonl.migrated.raw";
await fs.mkdir(outputDir, { recursive: true });
await fs.mkdir(extractDir, { recursive: true });
await fs.mkdir(path.dirname(sourcePath), { recursive: true });
await fs.writeFile(
sourcePath,
`${JSON.stringify({
ts: "2026-07-01T00:00:00.000Z",
source: "config-io",
event: "config.write",
argv: ["openclaw", "config", "set", "safe", "value"],
execArgv: [],
})}\n`,
);
await migrateLegacyAuditLogs({
detected: detectLegacyAuditLogs({
stateDir: state.stateDir,
doctorOnlyStateMigrations: true,
}),
stateDir: state.stateDir,
});
expect(
detectLegacyAuditLogs({
stateDir: state.stateDir,
doctorOnlyStateMigrations: true,
}).hasLegacy,
).toBe(false);
const { db } = openOpenClawStateDatabase({ env: state.env });
expect(
db
.prepare(
"SELECT COUNT(*) AS count FROM diagnostic_events WHERE scope = 'migration.legacy-audit-raw'",
)
.get(),
).toEqual({ count: 1 });
try {
const result = await createBackupArchive({
output: outputDir,
includeWorkspace: false,
nowMs: Date.UTC(2026, 4, 9, 8, 20, 0),
});
const entries = await listArchiveEntries(result.archivePath);
expect(entries.some((entry) => entry.endsWith(`/state/${rawRelativePath}`))).toBe(false);
const databaseEntry = expectDefined(
entries.find((entry) => entry.endsWith("/state/state/openclaw.sqlite")),
"global state database entry",
);
await tar.x({ file: result.archivePath, gzip: true, cwd: extractDir });
const sqlite = requireNodeSqlite();
const archivedDb = new sqlite.DatabaseSync(path.join(extractDir, databaseEntry), {
readOnly: true,
});
try {
expect(
archivedDb
.prepare(
"SELECT COUNT(*) AS count FROM diagnostic_events WHERE scope = 'migration.legacy-audit-raw'",
)
.get(),
).toEqual({ count: 0 });
} finally {
archivedDb.close();
}
} finally {
closeOpenClawStateDatabase();
}
},
);
});
it("preserves audit ordinals for identical later appends across backup restore", async () => {
await withOpenClawTestState(
{
layout: "state-only",
prefix: "openclaw-backup-audit-ordinal-",
scenario: "minimal",
},
async (state) => {
const outputDir = state.path("backups");
const extractDir = state.path("extract");
const sourcePath = state.statePath("logs/config-audit.jsonl");
const rawRelativePath = "logs/config-audit.jsonl.migrated.raw";
const record = {
ts: "2026-07-01T00:00:00.000Z",
source: "config-io",
event: "config.write",
argv: ["openclaw", "config", "set", "safe", "same"],
execArgv: [],
};
await fs.mkdir(outputDir, { recursive: true });
await fs.mkdir(extractDir, { recursive: true });
await fs.mkdir(path.dirname(sourcePath), { recursive: true });
await fs.writeFile(sourcePath, `${JSON.stringify(record)}\n`);
await migrateLegacyAuditLogs({
detected: detectLegacyAuditLogs({
stateDir: state.stateDir,
doctorOnlyStateMigrations: true,
}),
stateDir: state.stateDir,
});
await fs.appendFile(state.statePath(rawRelativePath), `${JSON.stringify(record)}\n`);
const result = await createBackupArchive({
output: outputDir,
includeWorkspace: false,
nowMs: Date.UTC(2026, 4, 9, 8, 25, 0),
});
const entries = await listArchiveEntries(result.archivePath);
const databaseEntry = expectDefined(
entries.find((entry) => entry.endsWith("/state/state/openclaw.sqlite")),
"global state database entry",
);
await tar.x({ file: result.archivePath, gzip: true, cwd: extractDir });
closeOpenClawStateDatabase();
const restoredDatabasePath = path.join(extractDir, databaseEntry);
const restoredStateDir = path.dirname(path.dirname(restoredDatabasePath));
const restoredDetection = detectLegacyAuditLogs({
stateDir: restoredStateDir,
doctorOnlyStateMigrations: true,
});
expect(restoredDetection.hasLegacy).toBe(true);
await migrateLegacyAuditLogs({
detected: restoredDetection,
stateDir: restoredStateDir,
});
const restoredEntries = createSqliteAuditRecordStore({
scope: CONFIG_AUDIT_SCOPE,
maxEntries: CONFIG_AUDIT_MAX_ENTRIES,
env: { ...process.env, OPENCLAW_STATE_DIR: restoredStateDir },
}).entries();
expect(restoredEntries).toHaveLength(2);
expect(new Set(restoredEntries.map((entry) => entry.key)).size).toBe(2);
expect(restoredEntries.map((entry) => entry.value)).toEqual([record, record]);
closeOpenClawStateDatabase();
},
);
});
it("scrubs transient SQLite queue and plugin blob rows from archive snapshots", async () => {
await withOpenClawTestState(
{
+58 -11
View File
@@ -33,6 +33,14 @@ import { formatErrorMessage } from "./errors.js";
import { sameFileIdentity } from "./fs-safe-advanced.js";
import { writeJson } from "./json-files.js";
import { createVerifiedSqliteSnapshot } from "./sqlite-snapshot.js";
import {
createLegacyAuditBackupSnapshots,
hasLegacyAuditBackupSources,
isLegacyAuditMigrationBackupPath,
rewriteLegacyAuditBackupCheckpoints,
type LegacyAuditBackupSnapshot,
} from "./state-migrations.audit-backup.js";
import { withLegacyAuditMigrationLease } from "./state-migrations.audit-coordination.js";
const loadTarRuntime = createLazyRuntimeModule(() => import("tar"));
@@ -618,6 +626,7 @@ async function createStateSqliteBackupPlan(params: {
stateDir: string;
tempDir: string;
preservedStatePaths?: readonly string[];
legacyAuditSnapshots: readonly LegacyAuditBackupSnapshot[];
}): Promise<StateSqliteBackupPlan> {
// Complete discovery before writing snapshots. chooseBackupTempRoot keeps
// tempDir outside stateDir, and this ordering prevents future overlap from
@@ -676,7 +685,10 @@ async function createStateSqliteBackupPlan(params: {
// Agent coordination is transient, while unrelated plugin databases
// remain owner-defined. Queue and TTL-blob policy is global-only.
transform: isGlobalStateDatabase
? sanitizeOpenClawGlobalStateSnapshot
? (database) => {
sanitizeOpenClawGlobalStateSnapshot(database);
rewriteLegacyAuditBackupCheckpoints(database, params.legacyAuditSnapshots);
}
: canonicalAgentSource
? sanitizeOpenClawStateLeaseRows
: undefined,
@@ -771,19 +783,47 @@ export async function createBackupArchive(
.map((asset) => asset.sourcePath),
].filter((entry) => stateAsset && isPathWithin(entry, stateAsset.sourcePath));
try {
const stateSqliteBackup = stateAsset
? await createStateSqliteBackupPlan({
stateDir: stateAsset.sourcePath,
tempDir,
preservedStatePaths,
})
: { snapshots: [], discoveredSourcePaths: new Set<string>() };
// Capture every legacy file first, including active and claimed sources.
// A concurrent Doctor then leaves each row in this snapshot, the later
// SQLite snapshot, or both; restore-side import keys make overlap harmless.
const hasLegacyAuditSources = stateAsset
? await hasLegacyAuditBackupSources(stateAsset.sourcePath)
: false;
const createSnapshotPlans = async () => {
const legacyAuditSnapshots =
stateAsset && hasLegacyAuditSources
? await createLegacyAuditBackupSnapshots({
stateDir: stateAsset.sourcePath,
tempDir,
})
: [];
const stateSqliteBackup = stateAsset
? await createStateSqliteBackupPlan({
stateDir: stateAsset.sourcePath,
tempDir,
preservedStatePaths,
legacyAuditSnapshots,
})
: { snapshots: [], discoveredSourcePaths: new Set<string>() };
return { legacyAuditSnapshots, stateSqliteBackup };
};
const snapshotPlans =
stateAsset && hasLegacyAuditSources
? await withLegacyAuditMigrationLease(stateAsset.sourcePath, createSnapshotPlans)
: await createSnapshotPlans();
const { legacyAuditSnapshots, stateSqliteBackup } = snapshotPlans;
const sourcePathRemaps = new Map<string, string>();
const skippedSqliteSourcePaths = new Set<string>();
const skippedStateSourcePaths = new Set<string>();
for (const snapshot of stateSqliteBackup.snapshots) {
sourcePathRemaps.set(path.resolve(snapshot.sourcePath), snapshot.archiveSourcePath);
for (const skippedSourcePath of snapshot.skippedSourcePaths) {
skippedSqliteSourcePaths.add(skippedSourcePath);
skippedStateSourcePaths.add(skippedSourcePath);
}
}
for (const snapshot of legacyAuditSnapshots) {
sourcePathRemaps.set(path.resolve(snapshot.sourcePath), snapshot.archiveSourcePath);
for (const skippedSourcePath of snapshot.skippedSourcePaths) {
skippedStateSourcePaths.add(skippedSourcePath);
}
}
const manifest = buildManifest({
@@ -822,13 +862,19 @@ export async function createBackupArchive(
if (stateFilter && !stateFilter(entryPath)) {
return false;
}
if (
stateAsset &&
isLegacyAuditMigrationBackupPath(resolvedEntryPath, stateAsset.sourcePath)
) {
return false;
}
const sqliteSourceKind = stateAsset
? classifyStateSqliteBackupSourcePath(resolvedEntryPath, stateAsset.sourcePath)
: undefined;
if (sqliteSourceKind === "excluded") {
return false;
}
if (skippedSqliteSourcePaths.has(resolvedEntryPath)) {
if (skippedStateSourcePaths.has(resolvedEntryPath)) {
return false;
}
if (
@@ -878,6 +924,7 @@ export async function createBackupArchive(
[
manifestPath,
...stateSqliteBackup.snapshots.map((snapshot) => snapshot.sourcePath),
...legacyAuditSnapshots.map((snapshot) => snapshot.sourcePath),
...result.assets.map((asset) => asset.sourcePath),
],
),
+7
View File
@@ -22,6 +22,12 @@ describe("isVolatileBackupPath", () => {
[`${stateDir}/ipc/gateway.sock`, true],
[`${stateDir}/gateway.pid`, true],
[`${stateDir}/tmp/pending.tmp`, true],
[`${stateDir}/audit/system-agent.jsonl.migrated.raw`, false],
[`${stateDir}/audit/system-agent.jsonl.migrated.2.raw`, false],
[`${stateDir}/audit/system-agent.jsonl.migrated.10.raw`, false],
[`${stateDir}/audit/system-agent.jsonl.migrated.raw.doctor-scrub-restore`, false],
[`${stateDir}/logs/config-audit.jsonl.migrated.raw.doctor-scrub-staging`, false],
[`${stateDir}/logs/config-audit.jsonl.migrated.raw.doctor-scrub-progress`, false],
[`${stateDir}/delivery-queue/pending.tmp`, true],
[`${stateDir}/session-delivery-queue/pending.tmp`, true],
@@ -38,6 +44,7 @@ describe("isVolatileBackupPath", () => {
["/home/user/project/README.md", false],
["/home/user/project/Cargo.lock", false],
["/home/user/project/pending.tmp", false],
["/home/user/project/config.jsonl.doctor-scrub-restore", false],
// non-volatile: log-like name outside scope
["/home/user/notes/daily.log", false],
])("classifies %s as volatile=%s", (p, expected) => {
+1
View File
@@ -5,6 +5,7 @@ import "./fs-safe-defaults.js";
export {
assertNoSymlinkParents,
assertNoSymlinkParentsSync,
type FileIdentityStat,
sameFileIdentity,
sanitizeUntrustedFileName,
} from "@openclaw/fs-safe/advanced";
-1
View File
@@ -4,7 +4,6 @@ import "./fs-safe-defaults.js";
// Regular-file IO helpers reject symlinks and non-file targets before reads or
// appends touch user-controlled paths.
export {
appendRegularFile,
appendRegularFileSync,
readRegularFile,
readRegularFileSync,
+120
View File
@@ -0,0 +1,120 @@
import { afterEach, describe, expect, it } from "vitest";
import { closeOpenClawStateDatabase } from "../state/openclaw-state-db.js";
import { withTempDir } from "../test-helpers/temp-dir.js";
import { createSqliteAuditRecordStore } from "./sqlite-audit-record-store.js";
describe("SQLite audit record store", () => {
afterEach(() => {
closeOpenClawStateDatabase();
});
it("keeps the newest configured number of rows per scope", async () => {
await withTempDir({ prefix: "openclaw-audit-store-" }, async (stateDir) => {
const store = createSqliteAuditRecordStore<{ value: number }>({
scope: "bounded-test",
maxEntries: 2,
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
});
store.register("one", { value: 1 }, 1);
store.register("two", { value: 2 }, 2);
store.register("three", { value: 3 }, 3);
expect(store.size()).toBe(2);
expect(store.entries().map((entry) => entry.key)).toEqual(["two", "three"]);
});
});
it("preserves insertion order and prunes the oldest row when timestamps tie", async () => {
await withTempDir({ prefix: "openclaw-audit-store-ties-" }, async (stateDir) => {
const store = createSqliteAuditRecordStore<{ value: number }>({
scope: "tied-timestamps",
maxEntries: 2,
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
});
store.register("z-first", { value: 1 }, 1);
store.register("a-second", { value: 2 }, 1);
expect(store.entries().map((entry) => entry.key)).toEqual(["z-first", "a-second"]);
store.register("m-third", { value: 3 }, 1);
expect(store.entries().map((entry) => entry.key)).toEqual(["a-second", "m-third"]);
});
});
it("prunes by insertion order when wall-clock timestamps move", async () => {
await withTempDir({ prefix: "openclaw-audit-store-clock-skew-" }, async (stateDir) => {
const store = createSqliteAuditRecordStore<{ value: number }>({
scope: "clock-skew",
maxEntries: 2,
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
});
store.register("future-first", { value: 1 }, 4_000_000_000_000);
store.register("past-second", { value: 2 }, 1);
store.register("current-third", { value: 3 }, 2_000_000_000_000);
expect(store.entries().map((entry) => entry.key)).toEqual(["past-second", "current-third"]);
});
});
it("commits batch inserts with one retention pass", async () => {
await withTempDir({ prefix: "openclaw-audit-store-batch-" }, async (stateDir) => {
const store = createSqliteAuditRecordStore<{ value: number }>({
scope: "batch-test",
maxEntries: 2,
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
});
store.registerLegacyMany([
{ key: "one", value: { value: 1 }, createdAt: 1 },
{ key: "two", value: { value: 2 }, createdAt: 2 },
{ key: "three", value: { value: 3 }, createdAt: 3 },
]);
expect(store.entries().map((entry) => entry.key)).toEqual(["two", "three"]);
});
});
it("updates a retained key without consuming another bounded entry", async () => {
await withTempDir({ prefix: "openclaw-audit-store-upsert-" }, async (stateDir) => {
const store = createSqliteAuditRecordStore<{ value: number }>({
scope: "upsert-test",
maxEntries: 2,
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
});
store.register("one", { value: 1 }, 1);
store.register("two", { value: 2 }, 2);
store.upsert("one", { value: 3 }, 3);
expect(store.size()).toBe(2);
expect(store.entries()).toEqual([
{ key: "one", value: { value: 3 }, createdAt: 3 },
{ key: "two", value: { value: 2 }, createdAt: 2 },
]);
});
});
it("orders legacy batches before existing runtime rows", async () => {
await withTempDir({ prefix: "openclaw-audit-store-legacy-order-" }, async (stateDir) => {
const store = createSqliteAuditRecordStore<{ value: number }>({
scope: "legacy-order",
maxEntries: 4,
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
});
store.register("runtime", { value: 3 }, 3);
store.registerLegacyMany([
{ key: "legacy-one", value: { value: 1 }, createdAt: 1 },
{ key: "legacy-two", value: { value: 2 }, createdAt: 2 },
]);
expect(store.entries().map((entry) => entry.key)).toEqual([
"legacy-one",
"legacy-two",
"runtime",
]);
});
});
});
+222
View File
@@ -0,0 +1,222 @@
// Shared SQLite storage for append-only diagnostic audit records.
import type { DatabaseSync } from "node:sqlite";
import type { Selectable } from "kysely";
import type { DB as OpenClawStateKyselyDatabase } from "../state/openclaw-state-db.generated.js";
import {
openOpenClawStateDatabase,
runOpenClawStateWriteTransaction,
type OpenClawStateDatabaseOptions,
} from "../state/openclaw-state-db.js";
import {
executeSqliteQuerySync,
executeSqliteQueryTakeFirstSync,
getNodeSqliteKysely,
} from "./kysely-sync.js";
type DiagnosticEventsTable = OpenClawStateKyselyDatabase["diagnostic_events"];
type AuditRecordDatabase = Pick<OpenClawStateKyselyDatabase, "diagnostic_events">;
type DiagnosticEventRow = Pick<
Selectable<DiagnosticEventsTable>,
"event_key" | "payload_json" | "created_at" | "sequence"
>;
type PreparedDiagnosticEventRow = Omit<DiagnosticEventRow, "sequence">;
const LEGACY_AUDIT_SEQUENCE_BASE = Number.MIN_SAFE_INTEGER;
type SqliteAuditRecordEntry<T> = {
key: string;
value: T;
createdAt: number;
};
function getAuditRecordKysely(database: DatabaseSync) {
return getNodeSqliteKysely<AuditRecordDatabase>(database);
}
function parseAuditRecord<T>(row: DiagnosticEventRow): SqliteAuditRecordEntry<T> {
return {
key: row.event_key,
value: JSON.parse(row.payload_json) as T,
createdAt: row.created_at,
};
}
function countAuditRecords(database: DatabaseSync, scope: string): number {
const row = executeSqliteQueryTakeFirstSync(
database,
getAuditRecordKysely(database)
.selectFrom("diagnostic_events")
.select((eb) => eb.fn.countAll<number | bigint>().as("count"))
.where("scope", "=", scope),
);
return typeof row?.count === "bigint" ? Number(row.count) : (row?.count ?? 0);
}
function nextAuditSequence(params: {
database: DatabaseSync;
scope: string;
legacy: boolean;
}): number {
const row = executeSqliteQueryTakeFirstSync(
params.database,
getAuditRecordKysely(params.database)
.selectFrom("diagnostic_events")
.select((eb) => eb.fn.max<number>("sequence").as("sequence"))
.where("scope", "=", params.scope)
.where("sequence", params.legacy ? "<" : ">=", 0),
);
const current = row?.sequence ?? (params.legacy ? LEGACY_AUDIT_SEQUENCE_BASE : 0);
const next = current + 1;
if (!Number.isSafeInteger(next) || (params.legacy && next >= 0)) {
throw new Error(`Audit sequence exhausted for scope ${params.scope}`);
}
return next;
}
function pruneAuditRecords(params: {
database: DatabaseSync;
scope: string;
maxEntries: number;
protectedKey?: string;
}): void {
const overflow = countAuditRecords(params.database, params.scope) - params.maxEntries;
if (overflow <= 0) {
return;
}
const protectedKey = params.protectedKey;
const baseCandidates = getAuditRecordKysely(params.database)
.selectFrom("diagnostic_events")
.select("event_key")
.where("scope", "=", params.scope);
const candidates = (
protectedKey === undefined
? baseCandidates
: baseCandidates.where("event_key", "!=", protectedKey)
)
.orderBy("sequence", "asc")
.limit(overflow);
const rows = executeSqliteQuerySync(params.database, candidates).rows;
for (const row of rows) {
executeSqliteQuerySync(
params.database,
getAuditRecordKysely(params.database)
.deleteFrom("diagnostic_events")
.where("scope", "=", params.scope)
.where("event_key", "=", row.event_key),
);
}
}
/** Opens one bounded append-only audit scope in the shared state database. */
export function createSqliteAuditRecordStore<T>(
options: OpenClawStateDatabaseOptions & { scope: string; maxEntries: number },
) {
const scope = options.scope;
const maxEntries = Math.max(1, Math.floor(options.maxEntries));
function prepareRecord(record: SqliteAuditRecordEntry<T>): PreparedDiagnosticEventRow {
const payloadJson = JSON.stringify(record.value);
if (payloadJson === undefined) {
throw new Error(`Audit record ${scope}/${record.key} is not JSON-serializable`);
}
return {
event_key: record.key,
payload_json: payloadJson,
created_at: record.createdAt,
};
}
function insertRecord(database: DatabaseSync, record: DiagnosticEventRow): void {
executeSqliteQuerySync(
database,
getAuditRecordKysely(database)
.insertInto("diagnostic_events")
.values({
scope,
event_key: record.event_key,
payload_json: record.payload_json,
created_at: record.created_at,
sequence: record.sequence,
})
.onConflict((conflict) => conflict.columns(["scope", "event_key"]).doNothing()),
);
}
return {
register(key: string, value: T, createdAt = Date.now()): void {
const record = prepareRecord({ key, value, createdAt });
runOpenClawStateWriteTransaction((database) => {
insertRecord(database.db, {
...record,
sequence: nextAuditSequence({ database: database.db, scope, legacy: false }),
});
// Audit retention is scope-local. Keep the just-written row and evict the oldest
// prior rows in the same synchronous commit section.
pruneAuditRecords({
database: database.db,
scope,
maxEntries,
protectedKey: key,
});
}, options);
},
upsert(key: string, value: T, createdAt = Date.now()): void {
const record = prepareRecord({ key, value, createdAt });
runOpenClawStateWriteTransaction((database) => {
executeSqliteQuerySync(
database.db,
getAuditRecordKysely(database.db)
.insertInto("diagnostic_events")
.values({
scope,
event_key: record.event_key,
payload_json: record.payload_json,
created_at: record.created_at,
sequence: nextAuditSequence({ database: database.db, scope, legacy: false }),
})
.onConflict((conflict) =>
conflict.columns(["scope", "event_key"]).doUpdateSet({
payload_json: record.payload_json,
created_at: record.created_at,
}),
),
);
pruneAuditRecords({
database: database.db,
scope,
maxEntries,
protectedKey: key,
});
}, options);
},
registerLegacyMany(records: readonly SqliteAuditRecordEntry<T>[]): void {
const prepared = records.map(prepareRecord);
if (prepared.length === 0) {
return;
}
// Legacy imports can contain tens of thousands of rows. Serialize first,
// then assign ordered negative sequences before runtime audit history.
runOpenClawStateWriteTransaction((database) => {
let sequence = nextAuditSequence({ database: database.db, scope, legacy: true });
for (const record of prepared) {
insertRecord(database.db, { ...record, sequence });
sequence += 1;
}
pruneAuditRecords({ database: database.db, scope, maxEntries });
}, options);
},
size(): number {
return countAuditRecords(openOpenClawStateDatabase(options).db, scope);
},
entries(): SqliteAuditRecordEntry<T>[] {
const database = openOpenClawStateDatabase(options);
return executeSqliteQuerySync(
database.db,
getAuditRecordKysely(database.db)
.selectFrom("diagnostic_events")
.select(["event_key", "payload_json", "created_at", "sequence"])
.where("scope", "=", scope)
.orderBy("sequence", "asc"),
).rows.map((row) => parseAuditRecord<T>(row));
},
};
}
@@ -0,0 +1,242 @@
import { createHash } from "node:crypto";
import fs from "node:fs/promises";
import path from "node:path";
import { expectDefined } from "@openclaw/normalization-core";
import { afterEach, describe, expect, it } from "vitest";
import { resetPluginStateStoreForTests } from "../plugin-state/plugin-state-store.js";
import { withTempDir } from "../test-helpers/temp-dir.js";
import {
createLegacyAuditBackupSnapshots,
hasLegacyAuditBackupSources,
isLegacyAuditMigrationBackupPath,
} from "./state-migrations.audit-backup.js";
const TEST_SCRUB_PATTERN = Buffer.from(
Array.from({ length: 32 }, (_, index) => (index % 2 === 0 ? 0x20 : 0x09)),
);
function configAuditRecord(value: string) {
return {
ts: "2026-07-01T00:00:00.000Z",
source: "config-io",
event: "config.write",
argv: ["openclaw", "config", "set", "token", value],
execArgv: [],
};
}
async function buildTestAuditRestoreJournal(
rawPath: string,
sourceRaw: Buffer,
scrubbedBytes = 0,
): Promise<string> {
const stat = await fs.stat(rawPath);
const journal = `${JSON.stringify({
schemaVersion: 6,
rawBase64: sourceRaw.toString("base64"),
scrubPatternBase64: TEST_SCRUB_PATTERN.toString("base64"),
target: { dev: stat.dev, ino: stat.ino, size: sourceRaw.length },
})}\n`;
await fs.writeFile(
`${rawPath}.doctor-scrub-progress`,
`${JSON.stringify({
schemaVersion: 1,
journalHash: createHash("sha256").update(journal).digest("hex"),
direction: "scrubbing",
committedBytes: scrubbedBytes,
pendingEnd: scrubbedBytes,
extentBytes: sourceRaw.length,
})}\n`,
);
return journal;
}
describe("legacy audit raw backup snapshots", () => {
afterEach(() => {
resetPluginStateStoreForTests();
});
it("recognizes only supported audit migration paths", () => {
const stateDir = "/opt/openclaw/state";
expect(
isLegacyAuditMigrationBackupPath(
`${stateDir}/logs/config-audit.jsonl.migrated.10.raw.doctor-scrub-restore`,
stateDir,
),
).toBe(true);
expect(
isLegacyAuditMigrationBackupPath(
`${stateDir}/audit/.system-agent.jsonl.doctor-importing.2`,
stateDir,
),
).toBe(true);
expect(
isLegacyAuditMigrationBackupPath(
`${stateDir}/logs/config-audit.jsonl.migrated.raw.doctor-scrub-progress`,
stateDir,
),
).toBe(true);
expect(
isLegacyAuditMigrationBackupPath(
`${stateDir}/plugins/example/cache.jsonl.migrated.raw`,
stateDir,
),
).toBe(false);
});
it("propagates audit-directory inspection failures", async () => {
await withTempDir({ prefix: "openclaw-audit-backup-inspection-" }, async (stateDir) => {
await fs.writeFile(path.join(stateDir, "logs"), "not a directory");
await expect(hasLegacyAuditBackupSources(stateDir)).rejects.toMatchObject({
code: "ENOTDIR",
});
});
});
it("captures an active legacy source before the later SQLite snapshot", async () => {
await withTempDir({ prefix: "openclaw-audit-backup-active-" }, async (rootDir) => {
const stateDir = path.join(rootDir, "state");
const tempDir = path.join(rootDir, "backup-temp");
const sourcePath = path.join(stateDir, "logs", "config-audit.jsonl");
await fs.mkdir(path.dirname(sourcePath), { recursive: true });
await fs.mkdir(tempDir);
await fs.writeFile(sourcePath, `${JSON.stringify(configAuditRecord("active-value-7f3c"))}\n`);
const snapshots = await createLegacyAuditBackupSnapshots({ stateDir, tempDir });
const snapshotAsset = expectDefined(snapshots[0], "snapshot");
const snapshot = await fs.readFile(snapshotAsset.sourcePath, "utf8");
expect(snapshotAsset.archiveSourcePath).toBe(sourcePath);
expect(snapshot).not.toContain("active-value-7f3c");
expect(JSON.parse(snapshot.trim())).toMatchObject({
argv: ["openclaw", "config", "set", "token", "***"],
});
});
});
it("captures a stable active prefix while an old writer keeps appending", async () => {
await withTempDir({ prefix: "openclaw-audit-backup-appending-" }, async (rootDir) => {
const stateDir = path.join(rootDir, "state");
const tempDir = path.join(rootDir, "backup-temp");
const sourcePath = path.join(stateDir, "logs", "config-audit.jsonl");
await fs.mkdir(path.dirname(sourcePath), { recursive: true });
await fs.mkdir(tempDir);
await fs.writeFile(
sourcePath,
Buffer.concat([
Buffer.from(`${JSON.stringify(configAuditRecord("initial-value-7f3c"))}\n`),
Buffer.alloc(4 * 1024 * 1024, 0x20),
]),
);
const snapshotPromise = createLegacyAuditBackupSnapshots({ stateDir, tempDir });
for (let index = 0; index < 8; index += 1) {
await fs.appendFile(
sourcePath,
`${JSON.stringify(configAuditRecord(`late-value-${index}-9a21`))}\n`,
);
}
const snapshots = await snapshotPromise;
const snapshotAsset = expectDefined(snapshots[0], "snapshot");
const snapshot = await fs.readFile(snapshotAsset.sourcePath, "utf8");
expect(snapshot).not.toContain("initial-value-7f3c");
expect(snapshot).not.toContain("late-value-");
expect(
snapshot
.trim()
.split("\n")
.map((line) => JSON.parse(line)),
).not.toHaveLength(0);
});
});
it("captures an unimported append without archiving its secret", async () => {
await withTempDir({ prefix: "openclaw-audit-backup-" }, async (rootDir) => {
const stateDir = path.join(rootDir, "state");
const tempDir = path.join(rootDir, "backup-temp");
const rawPath = path.join(stateDir, "logs", "config-audit.jsonl.migrated.raw");
await fs.mkdir(path.dirname(rawPath), { recursive: true });
await fs.mkdir(tempDir);
await fs.writeFile(rawPath, `${JSON.stringify(configAuditRecord("late-value-7f3c"))}\n`);
const snapshots = await createLegacyAuditBackupSnapshots({ stateDir, tempDir });
expect(snapshots).toHaveLength(1);
const snapshotAsset = expectDefined(snapshots[0], "snapshot");
expect(snapshotAsset.archiveSourcePath).toBe(rawPath);
const snapshot = await fs.readFile(snapshotAsset.sourcePath, "utf8");
expect(snapshot).not.toContain("late-value-7f3c");
expect(JSON.parse(snapshot.trim())).toMatchObject({
argv: ["openclaw", "config", "set", "token", "***"],
});
});
});
it("reconstructs a scrub-in-progress source and sanitizes its later append", async () => {
await withTempDir({ prefix: "openclaw-audit-backup-recovery-" }, async (rootDir) => {
const stateDir = path.join(rootDir, "state");
const tempDir = path.join(rootDir, "backup-temp");
const rawPath = path.join(stateDir, "logs", "config-audit.jsonl.migrated.raw");
const original = Buffer.from(`${JSON.stringify(configAuditRecord("original-value-7f3c"))}\n`);
const later = `${JSON.stringify(configAuditRecord("later-value-9a21"))}\n`;
const partial = Buffer.from(original);
for (let index = 0; index < Math.floor(partial.length / 2); index += 1) {
partial[index] = TEST_SCRUB_PATTERN[index % TEST_SCRUB_PATTERN.length]!;
}
await fs.mkdir(path.dirname(rawPath), { recursive: true });
await fs.mkdir(tempDir);
await fs.writeFile(rawPath, Buffer.concat([partial, Buffer.from(later)]));
await fs.writeFile(
`${rawPath}.doctor-scrub-restore`,
await buildTestAuditRestoreJournal(rawPath, original, Math.floor(partial.length / 2)),
);
const snapshots = await createLegacyAuditBackupSnapshots({ stateDir, tempDir });
const snapshotAsset = expectDefined(snapshots[0], "snapshot");
const snapshot = await fs.readFile(snapshotAsset.sourcePath, "utf8");
expect(snapshot).not.toContain("original-value-7f3c");
expect(snapshot).not.toContain("later-value-9a21");
expect(
snapshot
.trim()
.split("\n")
.map((line) => JSON.parse(line)),
).toMatchObject([
{ argv: ["openclaw", "config", "set", "token", "***"] },
{ argv: ["openclaw", "config", "set", "token", "***"] },
]);
});
});
it("ignores a stale restore journal after the raw archive is replaced", async () => {
await withTempDir({ prefix: "openclaw-audit-backup-stale-journal-" }, async (rootDir) => {
const stateDir = path.join(rootDir, "state");
const tempDir = path.join(rootDir, "backup-temp");
const rawPath = path.join(stateDir, "logs", "config-audit.jsonl.migrated.raw");
const original = Buffer.from(`${JSON.stringify(configAuditRecord("old-value-7f3c"))}\n`);
const replacement = {
...configAuditRecord("replacement-value-9a21"),
event: "config.delete",
};
await fs.mkdir(path.dirname(rawPath), { recursive: true });
await fs.mkdir(tempDir);
await fs.writeFile(rawPath, `${JSON.stringify(replacement)}\n`);
await fs.writeFile(
`${rawPath}.doctor-scrub-restore`,
await buildTestAuditRestoreJournal(rawPath, original),
);
const snapshots = await createLegacyAuditBackupSnapshots({ stateDir, tempDir });
const snapshotAsset = expectDefined(snapshots[0], "snapshot");
const snapshot = JSON.parse(await fs.readFile(snapshotAsset.sourcePath, "utf8"));
expect(snapshot).toMatchObject({
event: "config.delete",
argv: ["openclaw", "config", "set", "token", "***"],
});
});
});
});
+228
View File
@@ -0,0 +1,228 @@
// Builds secret-sanitized backup replacements for legacy audit append archives.
import { createHash } from "node:crypto";
import fs from "node:fs/promises";
import path from "node:path";
import type { DatabaseSync } from "node:sqlite";
import { root as createFsSafeRoot } from "./fs-safe.js";
import {
detectLegacyAuditLogs,
legacyAuditRawCheckpointKey,
legacyAuditSourceGenerationKey,
type LegacyAuditRawCheckpoint,
} from "./state-migrations.audit-checkpoints.js";
import {
prepareLegacyAuditRecords,
serializePreparedAuditRecords,
} from "./state-migrations.audit-records.js";
import {
findPreviousLegacyAuditRawCheckpoint,
readLegacyAuditRecoverySourceForBackup,
readLegacyAuditSourcePrefixSnapshotForBackup,
} from "./state-migrations.audit-recovery.js";
const LEGACY_AUDIT_LOGICAL_PATHS = [
{ directory: "logs", basename: "config-audit.jsonl" },
{ directory: "audit", basename: "system-agent.jsonl" },
{ directory: "audit", basename: "crestodian.jsonl" },
] as const;
export async function hasLegacyAuditBackupSources(stateDir: string): Promise<boolean> {
for (const logical of LEGACY_AUDIT_LOGICAL_PATHS) {
let entries: string[];
try {
entries = await fs.readdir(path.join(stateDir, logical.directory));
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT") {
continue;
}
throw error;
}
const escaped = logical.basename.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
const sourcePattern = new RegExp(
`^(?:${escaped}|\\.${escaped}\\.doctor-importing(?:\\.(?:[2-9]|[1-9][0-9]+))?|${escaped}\\.migrated(?:\\.(?:[2-9]|[1-9][0-9]+))?\\.raw(?:\\.doctor-scrub-(?:progress|restore|staging))?)$`,
"u",
);
if (entries.some((entry) => sourcePattern.test(entry))) {
return true;
}
}
return false;
}
export function isLegacyAuditMigrationBackupPath(sourcePath: string, stateDir: string): boolean {
const relativePath = path.relative(path.resolve(stateDir), path.resolve(sourcePath));
if (!relativePath || relativePath.startsWith(`..${path.sep}`) || path.isAbsolute(relativePath)) {
return false;
}
const directory = path.dirname(relativePath);
const basename = path.basename(relativePath);
for (const logical of LEGACY_AUDIT_LOGICAL_PATHS) {
if (directory !== logical.directory) {
continue;
}
if (basename === logical.basename) {
return true;
}
const escaped = logical.basename.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
const claimPattern = new RegExp(
`^\\.${escaped}\\.doctor-importing(?:\\.(?:[2-9]|[1-9][0-9]+))?$`,
"u",
);
const rawPattern = new RegExp(
`^${escaped}\\.migrated(?:\\.(?:[2-9]|[1-9][0-9]+))?\\.raw(?:\\.doctor-scrub-(?:progress|restore|staging))?$`,
"u",
);
if (claimPattern.test(basename) || rawPattern.test(basename)) {
return true;
}
}
return false;
}
type LegacyAuditBackupCheckpoint = {
key: string;
value: LegacyAuditRawCheckpoint;
};
export type LegacyAuditBackupSnapshot = {
sourcePath: string;
archiveSourcePath: string;
skippedSourcePaths: Set<string>;
checkpoint?: LegacyAuditBackupCheckpoint;
};
/** Replaces live raw checkpoints with metadata for the transformed backup files. */
export function rewriteLegacyAuditBackupCheckpoints(
database: DatabaseSync,
snapshots: readonly LegacyAuditBackupSnapshot[],
): void {
const hasDiagnosticEvents = database // sqlite-allow-raw -- Offline snapshot maintenance boundary.
.prepare("SELECT 1 AS ok FROM sqlite_master WHERE type = 'table' AND name = ?")
.get("diagnostic_events") as { ok?: unknown } | undefined;
if (hasDiagnosticEvents?.ok !== 1) {
return;
}
const scope = "migration.legacy-audit-raw";
database.prepare("DELETE FROM diagnostic_events WHERE scope = ?").run(scope); // sqlite-allow-raw -- Offline snapshot maintenance boundary.
const insert = database // sqlite-allow-raw -- Offline snapshot maintenance boundary.
.prepare(
`INSERT INTO diagnostic_events (
scope, event_key, payload_json, created_at, sequence
) VALUES (?, ?, ?, ?, ?)`,
);
let sequence = 1;
for (const snapshot of snapshots) {
if (!snapshot.checkpoint) {
continue;
}
insert.run(
scope,
snapshot.checkpoint.key,
JSON.stringify(snapshot.checkpoint.value),
0,
sequence,
);
sequence += 1;
}
}
async function createLegacyAuditBackupSnapshotsOnce(params: {
stateDir: string;
tempDir: string;
}): Promise<LegacyAuditBackupSnapshot[]> {
const detected = detectLegacyAuditLogs({
stateDir: params.stateDir,
doctorOnlyStateMigrations: true,
});
if (detected.sources.length === 0) {
return [];
}
const root = await createFsSafeRoot(params.stateDir, {
hardlinks: "reject",
maxBytes: Number.MAX_SAFE_INTEGER,
mkdir: false,
mode: 0o600,
symlinks: "reject",
});
const snapshots: LegacyAuditBackupSnapshot[] = [];
for (const [index, source] of detected.sources.entries()) {
const sourceRelativePath = path.relative(path.resolve(params.stateDir), source.sourcePath);
const snapshot =
source.storage === "raw-archive"
? await readLegacyAuditRecoverySourceForBackup(root, sourceRelativePath)
: await readLegacyAuditSourcePrefixSnapshotForBackup(root, sourceRelativePath);
const sourceGeneration = legacyAuditSourceGenerationKey(sourceRelativePath);
const previousCheckpoint =
source.storage === "raw-archive"
? findPreviousLegacyAuditRawCheckpoint(params.stateDir, sourceRelativePath)
: undefined;
const prepared = prepareLegacyAuditRecords(
source,
snapshot.raw,
sourceGeneration,
previousCheckpoint?.recordOrdinalBase ?? 0,
);
if (!prepared.ok) {
throw new Error(
`Legacy ${source.label} append archive cannot be sanitized for backup: ${prepared.warnings.join("; ")}`,
);
}
const sourcePath = path.join(params.tempDir, `legacy-audit-raw-${index}.jsonl`);
await fs.writeFile(sourcePath, prepared.sanitizedJsonl, { mode: 0o600 });
let checkpoint: LegacyAuditBackupCheckpoint | undefined;
if (previousCheckpoint) {
if (previousCheckpoint.recordCount > prepared.records.length) {
throw new Error(
`Legacy ${source.label} append archive is shorter than its durable checkpoint`,
);
}
// Backup rewrites raw bytes to sanitized JSONL. Preserve the source ordinal
// and rebase the checkpoint hash onto the equivalent transformed prefix.
const transformedPrefix = Buffer.from(
serializePreparedAuditRecords(prepared.records.slice(0, previousCheckpoint.recordCount)),
"utf8",
);
const value: LegacyAuditRawCheckpoint = {
...previousCheckpoint,
dev: 0,
ino: 0,
mtimeMs: 0,
size: transformedPrefix.length,
contentHash: createHash("sha256").update(transformedPrefix).digest("hex"),
};
checkpoint = { key: legacyAuditRawCheckpointKey(value), value };
}
snapshots.push({
sourcePath,
archiveSourcePath: source.sourcePath,
...(checkpoint ? { checkpoint } : {}),
skippedSourcePaths: new Set([
path.resolve(source.sourcePath),
path.resolve(`${source.sourcePath}.doctor-scrub-progress`),
path.resolve(`${source.sourcePath}.doctor-scrub-restore`),
path.resolve(`${source.sourcePath}.doctor-scrub-staging`),
]),
});
}
return snapshots;
}
export async function createLegacyAuditBackupSnapshots(params: {
stateDir: string;
tempDir: string;
}): Promise<LegacyAuditBackupSnapshot[]> {
let lastError: unknown;
for (let attempt = 0; attempt < 3; attempt += 1) {
try {
return await createLegacyAuditBackupSnapshotsOnce(params);
} catch (error) {
lastError = error;
if (attempt < 2) {
await new Promise<void>((resolve) => {
setTimeout(resolve, 25);
});
}
}
}
throw lastError;
}
@@ -0,0 +1,273 @@
import { createHash } from "node:crypto";
import fs from "node:fs";
import path from "node:path";
import { createSqliteAuditRecordStore } from "./sqlite-audit-record-store.js";
import type {
LegacyAuditLogSource,
LegacyAuditLogsDetection,
} from "./state-migrations.audit-logs.types.js";
export type LegacyAuditFileCheckpoint = {
dev: number;
ino: number;
mtimeMs: number;
size: number;
};
export type LegacyAuditRawCheckpoint = LegacyAuditFileCheckpoint & {
phase: "merge-intent" | "raw";
generationKey: string;
recordCount: number;
recordOrdinalBase: number;
contentHash: string;
sanitizedContentHash: string;
sanitizedSize: number;
};
const LEGACY_AUDIT_RAW_CHECKPOINT_SCOPE = "migration.legacy-audit-raw";
const LEGACY_AUDIT_RAW_CHECKPOINT_MAX_ENTRIES = 10_000;
export function legacyAuditRawCheckpointKey(checkpoint: LegacyAuditRawCheckpoint): string {
return checkpoint.generationKey;
}
export function legacyAuditSourceGenerationKey(rawArchiveRelativePath: string): string {
// The numbered raw archive path is the durable generation identity. Unlike
// device/inode metadata, it survives backup restore and cross-device moves.
return createHash("sha256")
.update(rawArchiveRelativePath.replace(/\\/gu, "/"))
.digest("hex")
.slice(0, 16);
}
export function openLegacyAuditRawCheckpointStore(stateDir: string) {
return createSqliteAuditRecordStore<LegacyAuditRawCheckpoint>({
scope: LEGACY_AUDIT_RAW_CHECKPOINT_SCOPE,
maxEntries: LEGACY_AUDIT_RAW_CHECKPOINT_MAX_ENTRIES,
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
});
}
export function hasLegacyAuditRawCheckpointCapacity(
stateDir: string,
rawArchiveRelativePath: string,
): boolean {
const generationKey = legacyAuditSourceGenerationKey(rawArchiveRelativePath);
const entries = openLegacyAuditRawCheckpointStore(stateDir).entries();
return (
entries.some((entry) => entry.value.generationKey === generationKey) ||
entries.length < LEGACY_AUDIT_RAW_CHECKPOINT_MAX_ENTRIES
);
}
function statLegacyAuditRawCheckpoint(sourcePath: string): LegacyAuditFileCheckpoint | undefined {
try {
const stat = fs.lstatSync(sourcePath);
if (!stat.isFile() || stat.isSymbolicLink()) {
return undefined;
}
return { dev: stat.dev, ino: stat.ino, mtimeMs: stat.mtimeMs, size: stat.size };
} catch {
return undefined;
}
}
export function legacyAuditRawCheckpointsMatch(
left: LegacyAuditFileCheckpoint | undefined,
right: LegacyAuditFileCheckpoint | undefined,
): boolean {
return (
left !== undefined &&
right !== undefined &&
left.dev === right.dev &&
left.ino === right.ino &&
left.mtimeMs === right.mtimeMs &&
left.size === right.size
);
}
function legacyAuditRawCheckpointIsCurrent(
sourcePath: string,
checkpoint: LegacyAuditRawCheckpoint,
): boolean {
let fd: number | undefined;
try {
fd = fs.openSync(sourcePath, "r");
const beforeStat = fs.fstatSync(fd);
const before = {
dev: beforeStat.dev,
ino: beforeStat.ino,
mtimeMs: beforeStat.mtimeMs,
size: beforeStat.size,
};
if (!beforeStat.isFile() || !legacyAuditRawCheckpointsMatch(checkpoint, before)) {
return false;
}
const hash = createHash("sha256");
const chunk = Buffer.allocUnsafe(64 * 1024);
let offset = 0;
while (offset < checkpoint.size) {
const bytesRead = fs.readSync(
fd,
chunk,
0,
Math.min(chunk.byteLength, checkpoint.size - offset),
offset,
);
if (bytesRead === 0) {
return false;
}
hash.update(chunk.subarray(0, bytesRead));
offset += bytesRead;
}
const afterStat = fs.fstatSync(fd);
const after = {
dev: afterStat.dev,
ino: afterStat.ino,
mtimeMs: afterStat.mtimeMs,
size: afterStat.size,
};
return (
legacyAuditRawCheckpointsMatch(before, after) &&
offset === checkpoint.size &&
hash.digest("hex") === checkpoint.contentHash
);
} catch {
return false;
} finally {
if (fd !== undefined) {
fs.closeSync(fd);
}
}
}
export function detectLegacyAuditLogs(params: {
stateDir: string;
doctorOnlyStateMigrations?: boolean;
}): LegacyAuditLogsDetection {
const logicalSources: Array<Pick<LegacyAuditLogSource, "kind" | "label" | "sourcePath">> = [
{
kind: "config",
label: "config audit log",
sourcePath: path.join(params.stateDir, "logs", "config-audit.jsonl"),
},
{
kind: "system-agent",
label: "system-agent audit log",
sourcePath: path.join(params.stateDir, "audit", "system-agent.jsonl"),
},
{
kind: "crestodian",
label: "Crestodian audit log",
sourcePath: path.join(params.stateDir, "audit", "crestodian.jsonl"),
},
];
// Startup migrates every detected source. Retired audit imports belong only
// to explicit `doctor --fix` repair.
if (params.doctorOnlyStateMigrations !== true) {
return { sources: [], hasLegacy: false };
}
let checkpoints: LegacyAuditRawCheckpoint[] | undefined;
const loadCheckpoints = () => {
if (checkpoints) {
return checkpoints;
}
try {
checkpoints = openLegacyAuditRawCheckpointStore(params.stateDir)
.entries()
.map((entry) => entry.value);
} catch {
checkpoints = [];
}
return checkpoints;
};
const sources: LegacyAuditLogSource[] = [];
for (const logical of logicalSources) {
let directoryEntries: string[] = [];
try {
directoryEntries = fs.readdirSync(path.dirname(logical.sourcePath));
} catch {
// The active-path check below still preserves the ordinary detection result.
}
const baseName = path.basename(logical.sourcePath).replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
const rawArchivePattern = new RegExp(
`^${baseName}\\.migrated(?:\\.([2-9]|[1-9][0-9]+))?\\.raw$`,
"u",
);
const claimPattern = new RegExp(
`^\\.${baseName}\\.doctor-importing(?:\\.([2-9]|[1-9][0-9]+))?$`,
"u",
);
const rawArchives = directoryEntries
.flatMap((entry) => {
const match = rawArchivePattern.exec(entry);
return match ? [{ entry, generation: BigInt(match[1] ?? "1") }] : [];
})
.toSorted(
(left, right) =>
(left.generation < right.generation ? -1 : left.generation > right.generation ? 1 : 0) ||
left.entry.localeCompare(right.entry),
);
for (const { entry } of rawArchives) {
const rawPath = path.join(path.dirname(logical.sourcePath), entry);
const rawRelativePath = path.relative(path.resolve(params.stateDir), rawPath);
const generationKey = legacyAuditSourceGenerationKey(rawRelativePath);
const checkpoint = statLegacyAuditRawCheckpoint(rawPath);
const hasRecoveryJournal =
statLegacyAuditRawCheckpoint(`${rawPath}.doctor-scrub-restore`) !== undefined;
if (
!hasRecoveryJournal &&
checkpoint &&
loadCheckpoints().some(
(candidate) =>
candidate.generationKey === generationKey &&
candidate.phase === "raw" &&
candidate.recordCount === 0 &&
legacyAuditRawCheckpointsMatch(candidate, checkpoint) &&
legacyAuditRawCheckpointIsCurrent(rawPath, candidate),
)
) {
continue;
}
sources.push({
...logical,
sourcePath: rawPath,
logicalSourcePath: logical.sourcePath,
storage: "raw-archive",
sanitizedArchivePath: rawPath.slice(0, -".raw".length),
});
}
// Claims reserve their archive generation across a crash. An older
// sanitized-only generation cannot be reused by a later claim.
const claims = directoryEntries
.flatMap((entry) => {
const match = claimPattern.exec(entry);
return match ? [{ entry, generation: BigInt(match[1] ?? "1") }] : [];
})
.toSorted(
(left, right) =>
(left.generation < right.generation ? -1 : left.generation > right.generation ? 1 : 0) ||
left.entry.localeCompare(right.entry),
);
for (const { entry, generation } of claims) {
const generationSuffix = generation === 1n ? "" : `.${generation}`;
const sanitizedArchivePath = `${logical.sourcePath}.migrated${generationSuffix}`;
sources.push({
...logical,
sourcePath: path.join(path.dirname(logical.sourcePath), entry),
logicalSourcePath: logical.sourcePath,
storage: "claim",
sanitizedArchivePath,
rawArchivePath: `${sanitizedArchivePath}.raw`,
});
}
if (fs.existsSync(logical.sourcePath)) {
sources.push({
...logical,
logicalSourcePath: logical.sourcePath,
storage: "active",
});
}
}
return { sources, hasLegacy: sources.length > 0 };
}
@@ -0,0 +1,25 @@
import { withOpenClawStateLease } from "../state/openclaw-state-lease.js";
const LEGACY_AUDIT_COORDINATION_SCOPE = "migration.legacy-audit";
const LEGACY_AUDIT_COORDINATION_KEY = "filesystem-sqlite-boundary";
export function withLegacyAuditMigrationLease<T>(
stateDir: string,
run: () => Promise<T>,
): Promise<T> {
return withOpenClawStateLease(
{
scope: LEGACY_AUDIT_COORDINATION_SCOPE,
key: LEGACY_AUDIT_COORDINATION_KEY,
database: {
scope: "shared",
options: { env: { ...process.env, OPENCLAW_STATE_DIR: stateDir } },
},
leaseMs: 60_000,
waitMs: 5_000,
leaseLabel: "legacy audit migration lease",
operationLabel: "migration.legacy-audit.lease",
},
async () => await run(),
);
}
File diff suppressed because it is too large Load Diff
+669
View File
@@ -0,0 +1,669 @@
// Doctor-only import for retired core JSONL audit stores.
import fs from "node:fs";
import path from "node:path";
import {
CONFIG_AUDIT_MAX_ENTRIES,
CONFIG_AUDIT_SCOPE,
type ConfigAuditRecord,
} from "../config/io.audit.js";
import {
SYSTEM_AGENT_AUDIT_MAX_ENTRIES,
SYSTEM_AGENT_AUDIT_SCOPE,
type SystemAgentAuditEntry,
} from "../system-agent/audit.js";
import { root as createFsSafeRoot } from "./fs-safe.js";
import { acquireGatewayLock } from "./gateway-lock.js";
import { createSqliteAuditRecordStore } from "./sqlite-audit-record-store.js";
import {
hasLegacyAuditRawCheckpointCapacity,
legacyAuditSourceGenerationKey,
} from "./state-migrations.audit-checkpoints.js";
import { withLegacyAuditMigrationLease } from "./state-migrations.audit-coordination.js";
import type {
LegacyAuditLogSource,
LegacyAuditLogsDetection,
} from "./state-migrations.audit-logs.types.js";
import {
prepareLegacyAuditRecords,
serializePreparedAuditRecords,
type PreparedAuditRecord,
} from "./state-migrations.audit-records.js";
import {
finalizeLegacyAuditRecoveryArchive,
findPreviousLegacyAuditRawCheckpoint,
readLegacyAuditSourceSnapshot,
recordLegacyAuditRawCheckpoint,
recordsAfterLegacyAuditRawCheckpoint,
restoreInterruptedAuditRecoveryArchive,
scrubLegacyAuditRecoveryArchive,
type AuditMigrationRoot,
type LegacyAuditSourceSnapshot,
} from "./state-migrations.audit-recovery.js";
import { writeRecoveredSanitizedAuditArchive } from "./state-migrations.audit-sanitized.js";
import type { MigrationMessages } from "./state-migrations.types.js";
function legacyAuditClaimPathForArchive(sourcePath: string, sanitizedArchivePath: string): string {
const archivePrefix = `${sourcePath}.migrated`;
if (!sanitizedArchivePath.startsWith(archivePrefix)) {
throw new Error(`Invalid legacy audit archive path ${sanitizedArchivePath}`);
}
const generationSuffix = sanitizedArchivePath.slice(archivePrefix.length);
return path.join(
path.dirname(sourcePath),
`.${path.basename(sourcePath)}.doctor-importing${generationSuffix}`,
);
}
export { detectLegacyAuditLogs } from "./state-migrations.audit-checkpoints.js";
type AuditArchiveRelativePaths = {
sanitized: string;
raw: string;
resumeSanitized: boolean;
};
async function resolveAuditArchiveRelativePaths(
root: AuditMigrationRoot,
sourceRelativePath: string,
): Promise<AuditArchiveRelativePaths> {
const directoryPath = path.join(root.rootReal, path.dirname(sourceRelativePath));
const baseName = path.basename(sourceRelativePath).replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
const archivePattern = new RegExp(
`^${baseName}\\.migrated(?:\\.([2-9]|[1-9][0-9]+))?(?:\\.raw)?$`,
"u",
);
const claimPattern = new RegExp(
`^\\.${baseName}\\.doctor-importing(?:\\.([2-9]|[1-9][0-9]+))?$`,
"u",
);
let latestGeneration = 0n;
for (const entry of fs.readdirSync(directoryPath)) {
const match = archivePattern.exec(entry) ?? claimPattern.exec(entry);
if (!match) {
continue;
}
const generation = BigInt(match[1] ?? "1");
if (generation > latestGeneration) {
latestGeneration = generation;
}
}
const generation = latestGeneration + 1n;
const sanitized = `${sourceRelativePath}.migrated${generation === 1n ? "" : `.${generation}`}`;
return { sanitized, raw: `${sanitized}.raw`, resumeSanitized: false };
}
async function secureAuditArchiveFile(params: {
root: AuditMigrationRoot;
relativePath: string;
label: string;
warnings: string[];
}): Promise<boolean> {
try {
const opened = await params.root.open(params.relativePath);
try {
await opened.handle.chmod(0o600);
await opened.handle.sync();
} finally {
await opened.handle.close();
}
return true;
} catch (error) {
params.warnings.push(`Failed securing ${params.label} legacy source: ${String(error)}`);
return false;
}
}
async function archiveLegacyAuditClaim(params: {
source: LegacyAuditLogSource;
claimRelativePath: string;
archivePaths: { sanitized: string; raw: string; resumeSanitized: boolean };
snapshot: LegacyAuditSourceSnapshot;
sanitizedJsonl: string;
root: AuditMigrationRoot;
changes: string[];
warnings: string[];
}): Promise<{
moved: boolean;
rawRelativePath?: string;
scrubbedSnapshot?: LegacyAuditSourceSnapshot;
}> {
let moved = false;
let sanitizedCreated = false;
const archivePaths = params.archivePaths;
try {
if (archivePaths.resumeSanitized) {
await params.root.write(archivePaths.sanitized, params.sanitizedJsonl, {
mkdir: false,
mode: 0o600,
});
} else {
await params.root.create(archivePaths.sanitized, params.sanitizedJsonl, { mode: 0o600 });
}
sanitizedCreated = true;
if (
!(await secureAuditArchiveFile({
root: params.root,
relativePath: archivePaths.sanitized,
label: `sanitized ${params.source.label}`,
warnings: params.warnings,
}))
) {
return { moved: false };
}
// Keep the claimed inode intact. A predecessor CLI may already hold an append
// descriptor across the claim; moving that inode to a named migration backup
// preserves any late write while the sanitized sibling remains safe to inspect.
await params.root.move(params.claimRelativePath, archivePaths.raw);
if (
!(await secureAuditArchiveFile({
root: params.root,
relativePath: archivePaths.raw,
label: `raw archived ${params.source.label}`,
warnings: params.warnings,
}))
) {
try {
await params.root.move(archivePaths.raw, params.claimRelativePath);
} catch (error) {
params.warnings.push(
`Failed restoring unsecured ${params.source.label} legacy source: ${String(error)}`,
);
}
return { moved: false };
}
moved = true;
const scrubbedSnapshot = await scrubLegacyAuditRecoveryArchive({
root: params.root,
relativePath: archivePaths.raw,
expectedSnapshot: params.snapshot,
label: params.source.label,
warnings: params.warnings,
});
params.changes.push(
`Archived sanitized ${params.source.label} legacy source → ${path.join(path.dirname(params.source.logicalSourcePath), path.basename(archivePaths.sanitized))}; ${scrubbedSnapshot ? "scrubbed same-inode append recovery archive" : "retained same-inode append recovery archive for Doctor retry"}${path.join(path.dirname(params.source.logicalSourcePath), path.basename(archivePaths.raw))}`,
);
return {
moved: true,
rawRelativePath: archivePaths.raw,
...(scrubbedSnapshot ? { scrubbedSnapshot } : {}),
};
} catch (error) {
params.warnings.push(
`Failed archiving ${params.source.label} ${params.source.logicalSourcePath}: ${String(error)}`,
);
} finally {
if (!moved && sanitizedCreated) {
await params.root.remove(archivePaths.sanitized).catch(() => undefined);
}
}
return { moved, ...(moved ? { rawRelativePath: archivePaths.raw } : {}) };
}
async function restoreOrPreserveLegacyAuditClaim(params: {
source: LegacyAuditLogSource;
claimRelativePath: string;
sourceRelativePath: string;
archivePaths: AuditArchiveRelativePaths;
root: AuditMigrationRoot;
warnings: string[];
}): Promise<void> {
try {
if (!(await params.root.exists(params.claimRelativePath))) {
return;
}
if (!(await params.root.exists(params.sourceRelativePath))) {
await params.root.move(params.claimRelativePath, params.sourceRelativePath);
await secureAuditArchiveFile({
root: params.root,
relativePath: params.sourceRelativePath,
label: params.source.label,
warnings: params.warnings,
});
return;
}
await params.root.move(params.claimRelativePath, params.archivePaths.raw);
await secureAuditArchiveFile({
root: params.root,
relativePath: params.archivePaths.raw,
label: `preserved ${params.source.label}`,
warnings: params.warnings,
});
params.warnings.push(
`Preserved claimed ${params.source.label} at ${path.join(path.dirname(params.source.logicalSourcePath), path.basename(params.archivePaths.raw))} because an old writer recreated ${params.source.logicalSourcePath}`,
);
} catch (error) {
params.warnings.push(
`Failed restoring claimed ${params.source.label} ${params.source.logicalSourcePath}: ${String(error)}`,
);
}
}
async function migrateLegacyAuditLogSource(params: {
source: LegacyAuditLogSource;
stateDir: string;
recreatedSourceScheduled?: boolean;
}): Promise<MigrationMessages & { completed: boolean }> {
const changes: string[] = [];
const warnings: string[] = [];
const result = (completed: boolean) => ({ changes, warnings, completed });
const root = await createFsSafeRoot(params.stateDir, {
hardlinks: "reject",
// Doctor previously accepted the complete legacy log; keep that migration
// contract while root operations enforce path and symlink boundaries.
maxBytes: Number.MAX_SAFE_INTEGER,
mkdir: false,
mode: 0o600,
symlinks: "reject",
});
const sourceRelativePath = path.relative(
path.resolve(params.stateDir),
params.source.logicalSourcePath,
);
const detectedRelativePath = path.relative(
path.resolve(params.stateDir),
params.source.sourcePath,
);
let archivePaths: AuditArchiveRelativePaths | undefined;
let claimRelativePath = detectedRelativePath;
if (params.source.storage === "active") {
archivePaths = await resolveAuditArchiveRelativePaths(root, sourceRelativePath);
claimRelativePath = path.relative(
path.resolve(params.stateDir),
legacyAuditClaimPathForArchive(
params.source.logicalSourcePath,
path.join(params.stateDir, archivePaths.sanitized),
),
);
await root.move(detectedRelativePath, claimRelativePath);
} else if (params.source.storage === "claim") {
if (!params.source.sanitizedArchivePath || !params.source.rawArchivePath) {
throw new Error(`Missing reserved archive generation for ${params.source.sourcePath}`);
}
const sanitized = path.relative(
path.resolve(params.stateDir),
params.source.sanitizedArchivePath,
);
const raw = path.relative(path.resolve(params.stateDir), params.source.rawArchivePath);
archivePaths = {
sanitized,
raw,
resumeSanitized: (await root.exists(sanitized)) && !(await root.exists(raw)),
};
}
let claimFinalized = params.source.storage === "raw-archive";
try {
if (
!(await secureAuditArchiveFile({
root,
relativePath: claimRelativePath,
label: `claimed ${params.source.label}`,
warnings,
}))
) {
return result(false);
}
const rawArchiveRelativePath = archivePaths?.raw ?? detectedRelativePath;
if (!hasLegacyAuditRawCheckpointCapacity(params.stateDir, rawArchiveRelativePath)) {
warnings.push(
`Skipped ${params.source.label} migration because durable raw-archive checkpoint capacity is exhausted; left the legacy source in place`,
);
return result(false);
}
if (
!(await restoreInterruptedAuditRecoveryArchive({
root,
relativePath: claimRelativePath,
label: params.source.label,
warnings,
}))
) {
return result(false);
}
const snapshot = await readLegacyAuditSourceSnapshot(root, claimRelativePath);
const sourceGeneration = legacyAuditSourceGenerationKey(rawArchiveRelativePath);
const sanitizedRelativePath =
params.source.storage === "raw-archive" && params.source.sanitizedArchivePath
? path.relative(path.resolve(params.stateDir), params.source.sanitizedArchivePath)
: undefined;
const previousCheckpoint =
params.source.storage === "raw-archive"
? findPreviousLegacyAuditRawCheckpoint(params.stateDir, rawArchiveRelativePath)
: undefined;
if (params.source.storage === "raw-archive" && !previousCheckpoint) {
if (!sanitizedRelativePath) {
throw new Error(`Missing sanitized archive path for ${params.source.sourcePath}`);
}
const firstContentByte = snapshot.rawBytes.findIndex(
(byte) => byte !== 0x20 && byte !== 0x09 && byte !== 0x0a && byte !== 0x0d,
);
if (snapshot.rawBytes.length > 0 && firstContentByte !== 0) {
warnings.push(
`Skipped ${params.source.label} recovery because its checkpointless raw archive begins with ambiguous whitespace; left the archive in place`,
);
return result(false);
}
}
const prepared = prepareLegacyAuditRecords(
params.source,
snapshot.raw,
sourceGeneration,
previousCheckpoint?.recordOrdinalBase ?? 0,
);
if (!prepared.ok) {
warnings.push(...prepared.warnings);
return result(false);
}
const env = { ...process.env, OPENCLAW_STATE_DIR: params.stateDir };
const maxEntries =
params.source.kind === "config" ? CONFIG_AUDIT_MAX_ENTRIES : SYSTEM_AGENT_AUDIT_MAX_ENTRIES;
const store = createSqliteAuditRecordStore<ConfigAuditRecord | SystemAgentAuditEntry>({
scope: params.source.kind === "config" ? CONFIG_AUDIT_SCOPE : SYSTEM_AGENT_AUDIT_SCOPE,
maxEntries,
env,
});
const existingEntries = store.entries();
const existingKeys = new Set(existingEntries.map((entry) => entry.key));
let candidateRecords: readonly PreparedAuditRecord[] = prepared.records;
if (params.source.storage === "raw-archive") {
if (previousCheckpoint) {
const appendedRecords = recordsAfterLegacyAuditRawCheckpoint({
checkpoint: previousCheckpoint,
snapshot,
records: prepared.records,
});
if (!appendedRecords) {
warnings.push(
`Skipped ${params.source.label} recovery because ${params.source.sourcePath} changed other than by append; left the raw archive in place`,
);
return result(false);
}
candidateRecords = appendedRecords;
}
}
if (!previousCheckpoint && candidateRecords === prepared.records) {
const lastRetainedSourceIndex = prepared.records.findLastIndex((record) =>
existingKeys.has(record.key),
);
if (lastRetainedSourceIndex >= 0) {
// A crash can occur after bounded insertion but before its raw checkpoint.
// Continue after the latest retained source ordinal instead of resurrecting
// the pruned head as newly appended audit history.
candidateRecords = prepared.records.slice(lastRetainedSourceIndex + 1);
}
}
const missing = candidateRecords.filter((record) => !existingKeys.has(record.key));
// SQLite may commit before the sanitized merge/checkpoint. Keep raw-derived
// candidates intact so retry can still prove and materialize the artifact tail.
store.registerLegacyMany(missing);
const importedKeys = new Set(store.entries().map((entry) => entry.key));
const retainedNewRows = missing.filter((record) => importedKeys.has(record.key)).length;
const retentionNote =
retainedNewRows === missing.length
? ""
: `; ${retainedNewRows} retained after bounded retention`;
if (params.source.storage === "raw-archive") {
if (!sanitizedRelativePath) {
throw new Error(`Missing sanitized archive path for ${params.source.sourcePath}`);
}
if (
!(await writeRecoveredSanitizedAuditArchive({
sourceLabel: params.source.label,
root,
relativePath: sanitizedRelativePath,
allRecordsJsonl: prepared.sanitizedJsonl,
candidateRecordsJsonl: serializePreparedAuditRecords(candidateRecords),
previousCheckpoint,
warnings,
}))
) {
return result(false);
}
// Checkpoint the unscrubbed append before hardening/scrubbing. A retry can
// then prove the sanitized tail was already written instead of duplicating it.
// A merge-intent count describes its exact raw snapshot. Keep an existing
// intent untouched when retry has no later rows; the final raw checkpoint replaces it.
if (previousCheckpoint?.phase !== "merge-intent" || candidateRecords.length > 0) {
if (
!(await recordLegacyAuditRawCheckpoint({
stateDir: params.stateDir,
rawPath: params.source.sourcePath,
rawRelativePath: claimRelativePath,
sanitizedRelativePath,
root,
snapshot,
phase: "merge-intent",
recordCount: prepared.records.length,
recordOrdinalBase: previousCheckpoint?.recordOrdinalBase ?? 0,
warnings,
}))
) {
return result(false);
}
}
if (
!(await secureAuditArchiveFile({
root,
relativePath: sanitizedRelativePath,
label: `sanitized ${params.source.label}`,
warnings,
}))
) {
return result(false);
}
if (missing.length > 0) {
changes.push(
`Recovered ${missing.length} later ${params.source.label} row(s) from ${params.source.sourcePath}${retentionNote}`,
);
}
const scrubbedSnapshot = await scrubLegacyAuditRecoveryArchive({
root,
relativePath: claimRelativePath,
expectedSnapshot: snapshot,
label: params.source.label,
warnings,
});
if (!scrubbedSnapshot) {
return result(false);
}
const scrubbedRecords = prepareLegacyAuditRecords(
params.source,
scrubbedSnapshot.raw,
legacyAuditSourceGenerationKey(rawArchiveRelativePath),
);
if (!scrubbedRecords.ok) {
warnings.push(...scrubbedRecords.warnings);
warnings.push(
`Retained uncheckpointed ${params.source.label} recovery archive; rerun openclaw doctor --fix`,
);
return result(false);
}
if (scrubbedRecords.records.length !== 0) {
warnings.push(
`A legacy ${params.source.label} writer appended during recovery; rerun openclaw doctor --fix to import the retained rows`,
);
return result(false);
}
const checkpointed = await recordLegacyAuditRawCheckpoint({
stateDir: params.stateDir,
rawPath: params.source.sourcePath,
rawRelativePath: claimRelativePath,
sanitizedRelativePath,
root,
snapshot: scrubbedSnapshot,
phase: "raw",
recordCount: 0,
recordOrdinalBase:
(previousCheckpoint?.recordOrdinalBase ?? 0) +
Math.max(previousCheckpoint?.recordCount ?? 0, prepared.records.length),
warnings,
});
if (checkpointed) {
await finalizeLegacyAuditRecoveryArchive({ root, relativePath: claimRelativePath }).catch(
(error: unknown) => {
warnings.push(
`Failed removing completed ${params.source.label} recovery journal: ${String(error)}`,
);
},
);
}
return result(checkpointed);
}
if (!archivePaths) {
throw new Error(`Missing archive generation for ${params.source.sourcePath}`);
}
changes.push(
`Migrated ${params.source.label} -> shared SQLite state (${missing.length} new row(s)${retentionNote})`,
);
const archived = await archiveLegacyAuditClaim({
source: params.source,
claimRelativePath,
archivePaths,
snapshot,
sanitizedJsonl: prepared.sanitizedJsonl,
root,
changes,
warnings,
});
claimFinalized = archived.moved;
if (!archived.moved || !archived.rawRelativePath) {
changes.pop();
return result(false);
}
if (!archived.scrubbedSnapshot) {
return result(false);
}
const scrubbedRecords = prepareLegacyAuditRecords(
params.source,
archived.scrubbedSnapshot.raw,
legacyAuditSourceGenerationKey(archived.rawRelativePath),
);
if (!scrubbedRecords.ok) {
warnings.push(...scrubbedRecords.warnings);
warnings.push(
`Retained uncheckpointed ${params.source.label} recovery archive; rerun openclaw doctor --fix`,
);
return result(false);
}
if (scrubbedRecords.records.length !== 0) {
warnings.push(
`A legacy ${params.source.label} writer appended during migration; rerun openclaw doctor --fix to import the retained rows`,
);
return result(false);
}
const rawPath = path.join(params.stateDir, archived.rawRelativePath);
const checkpointed = await recordLegacyAuditRawCheckpoint({
stateDir: params.stateDir,
rawPath,
rawRelativePath: archived.rawRelativePath,
sanitizedRelativePath: archivePaths.sanitized,
root,
snapshot: archived.scrubbedSnapshot,
phase: "raw",
recordCount: 0,
recordOrdinalBase: prepared.records.length,
warnings,
});
if (checkpointed) {
await finalizeLegacyAuditRecoveryArchive({
root,
relativePath: archived.rawRelativePath,
}).catch((error: unknown) => {
warnings.push(
`Failed removing completed ${params.source.label} recovery journal: ${String(error)}`,
);
});
}
if ((await root.exists(sourceRelativePath)) && !params.recreatedSourceScheduled) {
warnings.push(
`An old writer recreated ${params.source.label} at ${params.source.logicalSourcePath}; rerun openclaw doctor --fix to import the retained rows`,
);
}
return result(checkpointed);
} finally {
if (!claimFinalized && params.source.storage === "active" && archivePaths) {
await restoreOrPreserveLegacyAuditClaim({
source: params.source,
claimRelativePath,
sourceRelativePath,
archivePaths,
root,
warnings,
});
}
}
}
export async function migrateLegacyAuditLogs(params: {
detected: LegacyAuditLogsDetection;
stateDir: string;
}): Promise<MigrationMessages> {
const changes: string[] = [];
const warnings: string[] = [];
if (params.detected.sources.length === 0) {
return { changes, warnings };
}
const env = { ...process.env, OPENCLAW_STATE_DIR: params.stateDir };
let lock: Awaited<ReturnType<typeof acquireGatewayLock>>;
try {
// Exclusive state ownership excludes a predecessor Gateway and sibling doctor.
// Each source is also atomically claimed because old short-lived CLI processes
// can append config audit rows without participating in the Gateway lock.
lock = await acquireGatewayLock({
allowInTests: true,
env,
pollIntervalMs: 25,
role: "sqlite-maintenance",
timeoutMs: 250,
});
} catch (error) {
warnings.push(
`Skipped legacy audit migration because exclusive state ownership is unavailable: ${String(error)}`,
);
return { changes, warnings };
}
if (!lock) {
warnings.push(
"Skipped legacy audit migration because exclusive state ownership is unavailable",
);
return { changes, warnings };
}
try {
await withLegacyAuditMigrationLease(params.stateDir, async () => {
const blockedLogicalSources = new Set<string>();
for (const [index, source] of params.detected.sources.entries()) {
if (blockedLogicalSources.has(source.logicalSourcePath)) {
continue;
}
try {
const recreatedSourceScheduled = params.detected.sources
.slice(index + 1)
.some(
(candidate) =>
candidate.storage === "active" &&
candidate.logicalSourcePath === source.logicalSourcePath,
);
const result = await migrateLegacyAuditLogSource({
source,
stateDir: params.stateDir,
...(recreatedSourceScheduled ? { recreatedSourceScheduled: true } : {}),
});
changes.push(...result.changes);
warnings.push(...result.warnings);
if (!result.completed) {
// Generations encode append order. A later archive must not overtake
// an older source that still needs repair or durable checkpointing.
blockedLogicalSources.add(source.logicalSourcePath);
}
} catch (error) {
warnings.push(`Failed migrating ${source.label}: ${String(error)}`);
blockedLogicalSources.add(source.logicalSourcePath);
}
}
});
} catch (error) {
warnings.push(`Skipped legacy audit migration because coordination failed: ${String(error)}`);
} finally {
await lock.release();
}
return { changes, warnings };
}
@@ -0,0 +1,14 @@
export type LegacyAuditLogSource = {
kind: "config" | "system-agent" | "crestodian";
label: string;
sourcePath: string;
logicalSourcePath: string;
storage: "active" | "claim" | "raw-archive";
sanitizedArchivePath?: string;
rawArchivePath?: string;
};
export type LegacyAuditLogsDetection = {
sources: LegacyAuditLogSource[];
hasLegacy: boolean;
};
@@ -0,0 +1,90 @@
import { createHash } from "node:crypto";
import { sanitizeConfigAuditRecord, type ConfigAuditRecord } from "../config/io.audit.js";
import { redactSecrets } from "../logging/redact.js";
import type { SystemAgentAuditEntry } from "../system-agent/audit.js";
import type { LegacyAuditLogSource } from "./state-migrations.audit-logs.types.js";
export type PreparedAuditRecord = {
key: string;
value: ConfigAuditRecord | SystemAgentAuditEntry;
createdAt: number;
};
export function serializePreparedAuditRecords(records: readonly PreparedAuditRecord[]): string {
return records.length > 0
? `${records.map((record) => JSON.stringify(record.value)).join("\n")}\n`
: "";
}
function legacyAuditRecordCreatedAt(
source: LegacyAuditLogSource,
value: ConfigAuditRecord | SystemAgentAuditEntry,
): number {
const timestamp =
source.kind === "config"
? (value as Partial<ConfigAuditRecord>).ts
: (value as Partial<SystemAgentAuditEntry>).timestamp;
if (typeof timestamp !== "string") {
return 0;
}
const parsed = Date.parse(timestamp);
return Number.isFinite(parsed) ? parsed : 0;
}
type PreparedLegacyAuditRecords =
| { ok: false; warnings: string[] }
| {
ok: true;
records: PreparedAuditRecord[];
sanitizedJsonl: string;
};
export function prepareLegacyAuditRecords(
source: LegacyAuditLogSource,
raw: string,
sourceGeneration: string,
sourceOrdinalBase = 0,
): PreparedLegacyAuditRecords {
const records: PreparedAuditRecord[] = [];
const warnings: string[] = [];
for (const [index, line] of raw.split(/\r?\n/u).entries()) {
const trimmed = line.trim();
if (!trimmed) {
continue;
}
let parsed: unknown;
try {
parsed = JSON.parse(trimmed) as unknown;
} catch (error) {
warnings.push(
`Failed reading ${source.label} record at ${source.sourcePath}:${index + 1}: ${String(error)}`,
);
continue;
}
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
warnings.push(
`Skipped non-object ${source.label} record at ${source.sourcePath}:${index + 1}`,
);
continue;
}
const value =
source.kind === "config"
? sanitizeConfigAuditRecord(parsed as ConfigAuditRecord)
: (redactSecrets(parsed) as SystemAgentAuditEntry);
const digest = createHash("sha256").update(JSON.stringify(value)).digest("hex").slice(0, 16);
const recordOrdinal = sourceOrdinalBase + records.length + 1;
records.push({
key: `legacy:${source.kind}:${sourceGeneration}:${recordOrdinal}:${digest}`,
value,
createdAt: legacyAuditRecordCreatedAt(source, value),
});
}
if (warnings.length > 0) {
return { ok: false, warnings };
}
return {
ok: true,
records,
sanitizedJsonl: serializePreparedAuditRecords(records),
};
}
@@ -0,0 +1,161 @@
import { createHash } from "node:crypto";
import type { LegacyAuditFileCheckpoint } from "./state-migrations.audit-checkpoints.js";
type AuditRecoveryRestoreJournal = {
schemaVersion: 6;
rawBase64: string;
scrubPatternBase64: string;
target: Pick<LegacyAuditFileCheckpoint, "dev" | "ino" | "size">;
};
export type ParsedAuditRecoveryRestoreJournal = {
sourceRaw: Buffer;
scrubPattern: Buffer;
target: AuditRecoveryRestoreJournal["target"];
journalHash: string;
};
export type AuditRecoveryProgress = {
schemaVersion: 1;
journalHash: string;
direction: "restoring" | "scrubbing";
committedBytes: number;
pendingEnd: number;
extentBytes: number;
};
const AUDIT_RECOVERY_SCRUB_PATTERN_BYTES = 32;
export function serializeAuditRecoveryRestoreJournal(params: {
rawBytes: Buffer;
scrubPattern: Buffer;
target: AuditRecoveryRestoreJournal["target"];
}): string {
const journal: AuditRecoveryRestoreJournal = {
schemaVersion: 6,
rawBase64: params.rawBytes.toString("base64"),
scrubPatternBase64: params.scrubPattern.toString("base64"),
target: params.target,
};
return `${JSON.stringify(journal)}\n`;
}
export function parseAuditRecoveryRestoreJournal(raw: string): ParsedAuditRecoveryRestoreJournal {
const parsed = JSON.parse(raw) as Partial<AuditRecoveryRestoreJournal>;
if (
parsed.schemaVersion !== 6 ||
typeof parsed.rawBase64 !== "string" ||
typeof parsed.scrubPatternBase64 !== "string" ||
!parsed.target ||
typeof parsed.target.dev !== "number" ||
typeof parsed.target.ino !== "number" ||
typeof parsed.target.size !== "number"
) {
throw new Error("invalid legacy audit recovery restore journal");
}
const scrubPattern = Buffer.from(parsed.scrubPatternBase64, "base64");
if (
scrubPattern.length !== AUDIT_RECOVERY_SCRUB_PATTERN_BYTES ||
scrubPattern.some((byte) => byte !== 0x09 && byte !== 0x20)
) {
throw new Error("invalid legacy audit recovery scrub pattern");
}
const sourceRaw = Buffer.from(parsed.rawBase64, "base64");
if (sourceRaw.length !== parsed.target.size) {
throw new Error("invalid legacy audit recovery source size");
}
return {
sourceRaw,
scrubPattern,
target: parsed.target,
journalHash: createHash("sha256").update(raw).digest("hex"),
};
}
export function serializeAuditRecoveryProgress(progress: AuditRecoveryProgress): string {
return `${JSON.stringify(progress)}\n`;
}
export function parseAuditRecoveryProgress(
raw: string,
journal: ParsedAuditRecoveryRestoreJournal,
): AuditRecoveryProgress {
const parsed = JSON.parse(raw) as Partial<AuditRecoveryProgress>;
if (
parsed.schemaVersion !== 1 ||
parsed.journalHash !== journal.journalHash ||
(parsed.direction !== "restoring" && parsed.direction !== "scrubbing") ||
!Number.isSafeInteger(parsed.committedBytes) ||
!Number.isSafeInteger(parsed.pendingEnd) ||
!Number.isSafeInteger(parsed.extentBytes) ||
parsed.committedBytes! < 0 ||
parsed.pendingEnd! < parsed.committedBytes! ||
parsed.extentBytes! < parsed.pendingEnd! ||
parsed.extentBytes! > journal.target.size ||
(parsed.direction === "scrubbing" && parsed.extentBytes !== journal.target.size)
) {
throw new Error("invalid legacy audit recovery progress");
}
return parsed as AuditRecoveryProgress;
}
function auditRecoveryTransitionMatches(
current: Buffer,
previous: Buffer,
desired: Buffer,
start: number,
end: number,
): boolean {
let boundary = start;
while (boundary < end && current[boundary] === desired[boundary]) {
boundary += 1;
}
return current.subarray(boundary, end).equals(previous.subarray(boundary, end));
}
export function auditRecoveryStateMatchesJournal(params: {
current: Buffer;
original: Buffer;
scrubbed: Buffer;
progress: AuditRecoveryProgress;
}): boolean {
const { current, original, scrubbed, progress } = params;
if (current.length < original.length) {
return false;
}
if (progress.direction === "scrubbing") {
return (
current
.subarray(0, progress.committedBytes)
.equals(scrubbed.subarray(0, progress.committedBytes)) &&
auditRecoveryTransitionMatches(
current,
original,
scrubbed,
progress.committedBytes,
progress.pendingEnd,
) &&
current
.subarray(progress.pendingEnd, original.length)
.equals(original.subarray(progress.pendingEnd))
);
}
return (
current
.subarray(0, progress.committedBytes)
.equals(original.subarray(0, progress.committedBytes)) &&
auditRecoveryTransitionMatches(
current,
scrubbed,
original,
progress.committedBytes,
progress.pendingEnd,
) &&
current
.subarray(progress.pendingEnd, progress.extentBytes)
.equals(scrubbed.subarray(progress.pendingEnd, progress.extentBytes)) &&
current
.subarray(progress.extentBytes, original.length)
.equals(original.subarray(progress.extentBytes))
);
}
@@ -0,0 +1,542 @@
import { createHash } from "node:crypto";
import fs from "node:fs/promises";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { resetPluginStateStoreForTests } from "../plugin-state/plugin-state-store.js";
import { runOpenClawStateWriteTransaction } from "../state/openclaw-state-db.js";
import { listSystemAgentAuditEntriesForTests } from "../system-agent/audit.test-support.js";
import { withTempDir } from "../test-helpers/temp-dir.js";
import { openLegacyAuditRawCheckpointStore } from "./state-migrations.audit-checkpoints.js";
import { detectLegacyAuditLogs, migrateLegacyAuditLogs } from "./state-migrations.audit-logs.js";
const TEST_AUDIT_SCRUB_PATTERN = Buffer.from(
Array.from({ length: 32 }, (_, index) => (index % 2 === 0 ? 0x20 : 0x09)),
);
function buildTestAuditScrubbedContent(length: number): Buffer {
const content = Buffer.allocUnsafe(length);
for (let offset = 0; offset < length; offset += TEST_AUDIT_SCRUB_PATTERN.length) {
TEST_AUDIT_SCRUB_PATTERN.copy(
content,
offset,
0,
Math.min(TEST_AUDIT_SCRUB_PATTERN.length, length - offset),
);
}
return content;
}
async function buildTestAuditRestoreJournal(
rawPath: string,
sourceRaw: Buffer,
progress: { restoredBytes: number; scrubbedBytes: number } = {
restoredBytes: 0,
scrubbedBytes: 0,
},
): Promise<string> {
const stat = await fs.stat(rawPath);
const journal = `${JSON.stringify({
schemaVersion: 6,
rawBase64: sourceRaw.toString("base64"),
scrubPatternBase64: TEST_AUDIT_SCRUB_PATTERN.toString("base64"),
target: { dev: stat.dev, ino: stat.ino, size: sourceRaw.length },
})}\n`;
await fs.writeFile(
`${rawPath}.doctor-scrub-progress`,
`${JSON.stringify({
schemaVersion: 1,
journalHash: createHash("sha256").update(journal).digest("hex"),
direction: progress.restoredBytes > 0 ? "restoring" : "scrubbing",
committedBytes: progress.restoredBytes > 0 ? progress.restoredBytes : progress.scrubbedBytes,
pendingEnd: progress.restoredBytes > 0 ? progress.restoredBytes : progress.scrubbedBytes,
extentBytes: progress.restoredBytes > 0 ? progress.scrubbedBytes : sourceRaw.length,
})}\n`,
);
return journal;
}
describe("legacy audit recovery byte handling", () => {
afterEach(() => {
resetPluginStateStoreForTests();
});
it("blanks a zero-slack source within the fixed-size recovery inode", async () => {
await withTempDir({ prefix: "openclaw-audit-migration-short-secret-" }, async (stateDir) => {
const sourcePath = path.join(stateDir, "logs", "config-audit.jsonl");
const record = {
ts: "2026-07-01T00:00:00.000Z",
source: "config-io",
event: "config.write",
argv: ["openclaw", "config", "set", "token", "x"],
execArgv: [],
};
const original = `${JSON.stringify(record)}\n`;
await fs.mkdir(path.dirname(sourcePath), { recursive: true });
await fs.writeFile(sourcePath, original);
const result = await migrateLegacyAuditLogs({
detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }),
stateDir,
});
expect(result.warnings).toEqual([]);
const sanitized = await fs.readFile(`${sourcePath}.migrated`, "utf8");
const recovery = await fs.readFile(`${sourcePath}.migrated.raw`, "utf8");
expect(Buffer.byteLength(recovery)).toBe(Buffer.byteLength(original));
expect(JSON.parse(sanitized.trim())).toMatchObject({
argv: ["openclaw", "config", "set", "token", "***"],
});
expect(recovery.trim()).toBe("");
await expect(
fs.access(`${sourcePath}.migrated.raw.doctor-scrub-restore`),
).rejects.toMatchObject({ code: "ENOENT" });
});
});
it("uses original byte offsets when decoded audit text contains replacement characters", async () => {
await withTempDir({ prefix: "openclaw-audit-migration-invalid-utf8-" }, async (stateDir) => {
const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl");
const rawPath = `${sourcePath}.migrated.raw`;
const original = Buffer.concat([
Buffer.from(
'{"timestamp":"2026-07-03T00:00:00.000Z","operation":"gateway.restart","summary":"',
),
Buffer.from([0x80]),
Buffer.from('"}'),
]);
await fs.mkdir(path.dirname(sourcePath), { recursive: true });
await fs.writeFile(sourcePath, original);
const migrated = await migrateLegacyAuditLogs({
detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }),
stateDir,
});
expect(migrated.warnings).toEqual([]);
const blanked = await fs.readFile(rawPath);
expect(blanked).toHaveLength(original.length);
expect(blanked.every((byte) => byte === 0x20 || byte === 0x09)).toBe(true);
await fs.appendFile(
rawPath,
`${JSON.stringify({
timestamp: "2026-07-04T00:00:00.000Z",
operation: "gateway.restart",
summary: "later",
})}\n`,
);
const recovered = await migrateLegacyAuditLogs({
detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }),
stateDir,
});
expect(recovered.warnings).toEqual([]);
expect(
listSystemAgentAuditEntriesForTests({
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
}).map((entry) => entry.value.summary),
).toEqual(["", "later"]);
});
});
it("upgrades a legacy nonzero raw checkpoint to the blank append pad", async () => {
await withTempDir(
{ prefix: "openclaw-audit-migration-checkpoint-upgrade-" },
async (stateDir) => {
const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl");
await fs.mkdir(path.dirname(sourcePath), { recursive: true });
await fs.writeFile(
sourcePath,
`${JSON.stringify({
timestamp: "2026-07-03T00:00:00.000Z",
operation: "gateway.restart",
summary: "original",
})}\n`,
);
await migrateLegacyAuditLogs({
detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }),
stateDir,
});
const rawPath = `${sourcePath}.migrated.raw`;
const legacyRaw = Buffer.concat([
Buffer.from(
'{"timestamp":"2026-07-03T00:00:00.000Z","operation":"gateway.restart","summary":"',
),
Buffer.from([0x80]),
Buffer.from('"}\n'),
]);
await fs.writeFile(rawPath, legacyRaw);
const legacyStat = await fs.stat(rawPath);
const checkpointStore = openLegacyAuditRawCheckpointStore(stateDir);
const checkpoint = checkpointStore.entries()[0]!;
checkpointStore.upsert(checkpoint.key, {
...checkpoint.value,
dev: legacyStat.dev,
ino: legacyStat.ino,
mtimeMs: legacyStat.mtimeMs,
size: legacyStat.size,
contentHash: createHash("sha256").update(legacyRaw.toString("utf8")).digest("hex"),
recordCount: 1,
});
const detected = detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true });
expect(detected.hasLegacy).toBe(true);
const result = await migrateLegacyAuditLogs({ detected, stateDir });
expect(result.warnings).toEqual([]);
expect(openLegacyAuditRawCheckpointStore(stateDir).entries()[0]?.value.recordCount).toBe(0);
},
);
});
it("does not confuse an older prefix checkpoint with a later scrub generation", async () => {
await withTempDir(
{ prefix: "openclaw-audit-migration-scrub-generation-" },
async (stateDir) => {
const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl");
const rawPath = `${sourcePath}.migrated.raw`;
const restorePath = `${rawPath}.doctor-scrub-restore`;
await fs.mkdir(path.dirname(sourcePath), { recursive: true });
await fs.writeFile(
sourcePath,
`${JSON.stringify({
timestamp: "2026-07-03T00:00:00.000Z",
operation: "gateway.restart",
summary: "original",
})}\n`,
);
await migrateLegacyAuditLogs({
detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }),
stateDir,
});
await fs.appendFile(
rawPath,
`${JSON.stringify({
timestamp: "2026-07-04T00:00:00.000Z",
operation: "gateway.restart",
summary: "later",
})}\n`,
);
const interruptedRaw = await fs.readFile(rawPath);
await fs.writeFile(
restorePath,
await buildTestAuditRestoreJournal(rawPath, interruptedRaw, {
restoredBytes: 0,
scrubbedBytes: interruptedRaw.length,
}),
);
await fs.writeFile(rawPath, buildTestAuditScrubbedContent(interruptedRaw.length));
const result = await migrateLegacyAuditLogs({
detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }),
stateDir,
});
expect(result.warnings).toEqual([]);
expect(
listSystemAgentAuditEntriesForTests({
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
}).map((entry) => entry.value.summary),
).toEqual(["original", "later"]);
await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" });
},
);
});
it("does not replay a scrub journal over an equal-width space redaction", async () => {
await withTempDir({ prefix: "openclaw-audit-migration-scrub-redacted-" }, async (stateDir) => {
const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl");
const rawPath = `${sourcePath}.migrated.raw`;
const restorePath = `${rawPath}.doctor-scrub-restore`;
const originalContent = `${JSON.stringify({
timestamp: "2026-07-03T00:00:00.000Z",
operation: "gateway.restart",
summary: "secret archive value",
})}\n`;
const replacementContent = originalContent.replace("secret archive value", " ".repeat(20));
await fs.mkdir(path.dirname(sourcePath), { recursive: true });
await fs.writeFile(`${sourcePath}.migrated`, "{}\n");
await fs.writeFile(rawPath, replacementContent);
await fs.writeFile(
restorePath,
await buildTestAuditRestoreJournal(rawPath, Buffer.from(originalContent, "utf8")),
);
const result = await migrateLegacyAuditLogs({
detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }),
stateDir,
});
expect(result.warnings.join("\n")).toContain("no longer matches its restore journal target");
await expect(fs.readFile(rawPath, "utf8")).resolves.toBe(replacementContent);
await expect(fs.access(restorePath)).resolves.toBeUndefined();
});
});
it("resumes a one-byte scrub interruption using exact journal progress", async () => {
await withTempDir({ prefix: "openclaw-audit-migration-scrub-one-byte-" }, async (stateDir) => {
const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl");
const rawPath = `${sourcePath}.migrated.raw`;
const restorePath = `${rawPath}.doctor-scrub-restore`;
const originalContent = `${JSON.stringify({
timestamp: "2026-07-03T00:00:00.000Z",
operation: "gateway.restart",
summary: "original archive",
})}\n`;
const replacementBytes = Buffer.from(originalContent);
replacementBytes[0] = TEST_AUDIT_SCRUB_PATTERN[0]!;
await fs.mkdir(path.dirname(sourcePath), { recursive: true });
await fs.writeFile(`${sourcePath}.migrated`, "{}\n");
await fs.writeFile(rawPath, replacementBytes);
await fs.writeFile(
restorePath,
await buildTestAuditRestoreJournal(rawPath, Buffer.from(originalContent), {
restoredBytes: 0,
scrubbedBytes: 1,
}),
);
const result = await migrateLegacyAuditLogs({
detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }),
stateDir,
});
expect(result.warnings).toEqual([]);
expect((await fs.readFile(rawPath, "utf8")).trim()).toBe("");
await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" });
});
});
it("resumes restoration after an interrupted rollback write", async () => {
await withTempDir({ prefix: "openclaw-audit-migration-restore-restart-" }, async (stateDir) => {
const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl");
const rawPath = `${sourcePath}.migrated.raw`;
const restorePath = `${rawPath}.doctor-scrub-restore`;
const originalContent = `${JSON.stringify({
timestamp: "2026-07-03T00:00:00.000Z",
operation: "gateway.restart",
summary: "restore after restart",
})}\n`;
const originalBytes = Buffer.from(originalContent, "utf8");
const interruptedRestore = buildTestAuditScrubbedContent(originalBytes.length);
originalBytes.subarray(0, Math.floor(originalBytes.length / 2)).copy(interruptedRestore);
await fs.mkdir(path.dirname(sourcePath), { recursive: true });
await fs.writeFile(`${sourcePath}.migrated`, "{}\n");
await fs.writeFile(rawPath, interruptedRestore);
await fs.writeFile(
restorePath,
await buildTestAuditRestoreJournal(rawPath, originalBytes, {
restoredBytes: Math.floor(originalBytes.length / 2),
scrubbedBytes: originalBytes.length,
}),
);
const result = await migrateLegacyAuditLogs({
detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }),
stateDir,
});
expect(result.warnings).toEqual([]);
expect(
listSystemAgentAuditEntriesForTests({
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
}).map((entry) => entry.value.summary),
).toEqual(["restore after restart"]);
await expect(fs.access(restorePath)).rejects.toMatchObject({ code: "ENOENT" });
});
});
it("retries raw archive recovery when sanitized archive hardening fails", async () => {
await withTempDir(
{ prefix: "openclaw-audit-migration-recovery-permissions-" },
async (stateDir) => {
const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl");
const event = (summary: string) => ({
timestamp: "2026-07-03T00:00:00.000Z",
operation: "gateway.restart",
summary,
});
await fs.mkdir(path.dirname(sourcePath), { recursive: true });
await fs.writeFile(sourcePath, `${JSON.stringify(event("before archive"))}\n`);
await migrateLegacyAuditLogs({
detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }),
stateDir,
});
await fs.appendFile(
`${sourcePath}.migrated.raw`,
`${JSON.stringify(event("later row"))}\n`,
);
const probe = await fs.open(path.join(stateDir, "recovery-chmod-probe"), "w");
const fileHandlePrototype = Object.getPrototypeOf(probe) as {
chmod(mode: number): Promise<void>;
};
await probe.close();
const originalChmod = Reflect.get(fileHandlePrototype, "chmod") as (
this: typeof fileHandlePrototype,
mode: number,
) => Promise<void>;
let chmodCalls = 0;
const chmodSpy = vi.spyOn(fileHandlePrototype, "chmod").mockImplementation(function (
this: typeof fileHandlePrototype,
mode: number,
) {
chmodCalls += 1;
if (chmodCalls === 2) {
return Promise.reject(new Error("simulated recovery chmod failure"));
}
return originalChmod.call(this, mode);
});
let failed: Awaited<ReturnType<typeof migrateLegacyAuditLogs>>;
try {
failed = await migrateLegacyAuditLogs({
detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }),
stateDir,
});
} finally {
chmodSpy.mockRestore();
}
expect(failed.changes).toEqual([]);
expect(failed.warnings.join("\n")).toContain(
"Failed securing sanitized system-agent audit log",
);
expect(
detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }).sources,
).toMatchObject([{ storage: "raw-archive" }]);
const recovered = await migrateLegacyAuditLogs({
detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }),
stateDir,
});
expect(recovered.warnings).toEqual([]);
expect(
listSystemAgentAuditEntriesForTests({
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
}).map((entry) => entry.value.summary),
).toEqual(["before archive", "later row"]);
const sanitizedRows = (await fs.readFile(`${sourcePath}.migrated`, "utf8"))
.trim()
.split("\n")
.map((line) => JSON.parse(line) as { summary: string });
expect(sanitizedRows.map((row) => row.summary)).toEqual(["before archive", "later row"]);
expect(
detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }).sources,
).toEqual([]);
},
);
});
it("completes a verified partial sanitized tail after an interrupted write", async () => {
await withTempDir(
{ prefix: "openclaw-audit-migration-partial-sanitized-tail-" },
async (stateDir) => {
const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl");
const sanitizedPath = `${sourcePath}.migrated`;
const rawPath = `${sanitizedPath}.raw`;
const event = (day: string, summary: string) => ({
timestamp: `2026-07-${day}T00:00:00.000Z`,
operation: "gateway.restart",
summary,
});
await fs.mkdir(path.dirname(sourcePath), { recursive: true });
await fs.writeFile(sourcePath, `${JSON.stringify(event("01", "before archive"))}\n`);
await migrateLegacyAuditLogs({
detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }),
stateDir,
});
const firstLater = event("02", "first later row");
const secondLater = event("03", "second later row");
await fs.appendFile(
rawPath,
`${JSON.stringify(firstLater)}\n${JSON.stringify(secondLater)}\n`,
);
// Simulate a stopped sanitized write: the durable checkpoint prefix plus
// one complete candidate row is a byte-for-byte prefix of the desired file.
await fs.appendFile(sanitizedPath, `${JSON.stringify(firstLater)}\n`);
const recovered = await migrateLegacyAuditLogs({
detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }),
stateDir,
});
expect(recovered.warnings).toEqual([]);
expect(
listSystemAgentAuditEntriesForTests({
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
}).map((entry) => entry.value.summary),
).toEqual(["before archive", "first later row", "second later row"]);
const sanitizedRows = (await fs.readFile(sanitizedPath, "utf8"))
.trim()
.split("\n")
.map((line) => JSON.parse(line) as { summary: string });
expect(sanitizedRows.map((row) => row.summary)).toEqual([
"before archive",
"first later row",
"second later row",
]);
},
);
});
it("preserves identical appends and blocks checkpointless whitespace ambiguity", async () => {
await withTempDir({ prefix: "openclaw-audit-migration-duplicate-tail-" }, async (stateDir) => {
const sourcePath = path.join(stateDir, "audit", "system-agent.jsonl");
const rawPath = `${sourcePath}.migrated.raw`;
const event = {
timestamp: "2026-07-03T00:00:00.000Z",
operation: "gateway.restart",
summary: "Repeated operation",
};
await fs.mkdir(path.dirname(sourcePath), { recursive: true });
await fs.writeFile(sourcePath, `${JSON.stringify(event)}\n\n${JSON.stringify(event)}\n`);
await migrateLegacyAuditLogs({
detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }),
stateDir,
});
await fs.appendFile(rawPath, `${JSON.stringify(event)}\n`);
const recovered = await migrateLegacyAuditLogs({
detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }),
stateDir,
});
expect(recovered.warnings).toEqual([]);
expect(
listSystemAgentAuditEntriesForTests({
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
}).map((entry) => entry.value.summary),
).toEqual(["Repeated operation", "Repeated operation", "Repeated operation"]);
const sanitizedRows = (await fs.readFile(`${sourcePath}.migrated`, "utf8"))
.trim()
.split("\n");
expect(sanitizedRows).toHaveLength(3);
runOpenClawStateWriteTransaction(
(database) => {
database.db
.prepare("DELETE FROM diagnostic_events WHERE scope = ?")
.run("migration.legacy-audit-raw");
},
{ env: { ...process.env, OPENCLAW_STATE_DIR: stateDir } },
);
await fs.appendFile(rawPath, `${JSON.stringify(event)}\n`);
const ambiguous = await migrateLegacyAuditLogs({
detected: detectLegacyAuditLogs({ stateDir, doctorOnlyStateMigrations: true }),
stateDir,
});
expect(ambiguous.changes).toEqual([]);
expect(ambiguous.warnings).toEqual([
expect.stringContaining("checkpointless raw archive begins with ambiguous whitespace"),
]);
expect(
listSystemAgentAuditEntriesForTests({
env: { ...process.env, OPENCLAW_STATE_DIR: stateDir },
}),
).toHaveLength(3);
expect((await fs.readFile(`${sourcePath}.migrated`, "utf8")).trim().split("\n")).toHaveLength(
3,
);
});
});
});
@@ -0,0 +1,710 @@
import { createHash, randomBytes } from "node:crypto";
import path from "node:path";
import { root as createFsSafeRoot } from "./fs-safe.js";
import { syncDirectoryBestEffort } from "./sqlite-snapshot.js";
import {
legacyAuditRawCheckpointKey,
legacyAuditRawCheckpointsMatch,
legacyAuditSourceGenerationKey,
openLegacyAuditRawCheckpointStore,
type LegacyAuditFileCheckpoint,
type LegacyAuditRawCheckpoint,
} from "./state-migrations.audit-checkpoints.js";
import {
auditRecoveryStateMatchesJournal,
parseAuditRecoveryProgress,
parseAuditRecoveryRestoreJournal,
serializeAuditRecoveryProgress,
serializeAuditRecoveryRestoreJournal,
type AuditRecoveryProgress,
type ParsedAuditRecoveryRestoreJournal,
} from "./state-migrations.audit-recovery-protocol.js";
export type AuditMigrationRoot = Awaited<ReturnType<typeof createFsSafeRoot>>;
export type LegacyAuditSourceSnapshot = LegacyAuditFileCheckpoint & {
raw: string;
rawBytes: Buffer;
};
const AUDIT_RECOVERY_RESTORE_SUFFIX = ".doctor-scrub-restore";
const AUDIT_RECOVERY_STAGING_SUFFIX = ".doctor-scrub-staging";
const AUDIT_RECOVERY_PROGRESS_SUFFIX = ".doctor-scrub-progress";
const AUDIT_RECOVERY_SCRUB_PATTERN_BYTES = 32;
function auditRecoverySiblingPath(relativePath: string, suffix: string): string {
return `${relativePath}${suffix}`;
}
function auditRecoveryJournalTargetsSnapshot(
snapshot: LegacyAuditSourceSnapshot,
journal: Pick<ParsedAuditRecoveryRestoreJournal, "target">,
): boolean {
return (
snapshot.dev === journal.target.dev &&
snapshot.ino === journal.target.ino &&
snapshot.rawBytes.length >= journal.target.size
);
}
function auditRecoveryCheckpointPrefixMatches(
snapshot: LegacyAuditSourceSnapshot,
checkpoint: LegacyAuditRawCheckpoint,
): boolean {
if (snapshot.rawBytes.length < checkpoint.size) {
return false;
}
return (
createHash("sha256").update(snapshot.rawBytes.subarray(0, checkpoint.size)).digest("hex") ===
checkpoint.contentHash
);
}
async function syncAuditRecoveryDirectory(
root: AuditMigrationRoot,
relativePath: string,
): Promise<void> {
await syncDirectoryBestEffort(path.join(root.rootReal, path.dirname(relativePath)));
}
export async function readLegacyAuditSourceSnapshot(
root: AuditMigrationRoot,
relativePath: string,
): Promise<LegacyAuditSourceSnapshot> {
const opened = await root.open(relativePath);
try {
const before = await opened.handle.stat();
if (!before.isFile()) {
throw new Error("legacy audit source is not a regular file");
}
const rawBytes = await opened.handle.readFile();
const after = await opened.handle.stat();
const beforeCheckpoint = {
dev: before.dev,
ino: before.ino,
mtimeMs: before.mtimeMs,
size: before.size,
};
const afterCheckpoint = {
dev: after.dev,
ino: after.ino,
mtimeMs: after.mtimeMs,
size: after.size,
};
if (!legacyAuditRawCheckpointsMatch(beforeCheckpoint, afterCheckpoint)) {
throw new Error("legacy audit source changed while Doctor was reading it");
}
return { ...afterCheckpoint, raw: rawBytes.toString("utf8"), rawBytes };
} finally {
await opened.handle.close();
}
}
export async function readLegacyAuditSourcePrefixSnapshotForBackup(
root: AuditMigrationRoot,
relativePath: string,
): Promise<LegacyAuditSourceSnapshot> {
const opened = await root.open(relativePath);
try {
const before = await opened.handle.stat();
if (!before.isFile()) {
throw new Error("legacy audit source is not a regular file");
}
const rawBytes = Buffer.allocUnsafe(before.size);
let offset = 0;
while (offset < rawBytes.length) {
const length = Math.min(64 * 1024, rawBytes.length - offset);
const { bytesRead } = await opened.handle.read(rawBytes, offset, length, offset);
if (bytesRead === 0) {
throw new Error("legacy audit source was truncated while backup was reading it");
}
offset += bytesRead;
}
const after = await opened.handle.stat();
if (before.dev !== after.dev || before.ino !== after.ino || after.size < before.size) {
throw new Error("legacy audit source changed other than by append during backup");
}
const checkpoint = {
dev: before.dev,
ino: before.ino,
mtimeMs: before.mtimeMs,
size: before.size,
};
return { ...checkpoint, raw: rawBytes.toString("utf8"), rawBytes };
} finally {
await opened.handle.close();
}
}
export async function readLegacyAuditRecoverySourceForBackup(
root: AuditMigrationRoot,
relativePath: string,
): Promise<LegacyAuditSourceSnapshot> {
const current = await readLegacyAuditSourcePrefixSnapshotForBackup(root, relativePath);
const restoreRelativePath = auditRecoverySiblingPath(relativePath, AUDIT_RECOVERY_RESTORE_SUFFIX);
if (!(await root.exists(restoreRelativePath))) {
return current;
}
const restoreSnapshot = await readLegacyAuditSourceSnapshot(root, restoreRelativePath);
const journal = parseAuditRecoveryRestoreJournal(restoreSnapshot.raw);
const progress = await readAuditRecoveryProgress({ root, relativePath, journal });
const scrubbedContent = buildScrubbedAuditRecoveryContent(
journal.sourceRaw,
journal.scrubPattern,
);
if (
!auditRecoveryJournalTargetsSnapshot(current, journal) ||
!auditRecoveryStateMatchesJournal({
current: current.rawBytes,
original: journal.sourceRaw,
scrubbed: scrubbedContent,
progress,
})
) {
return current;
}
const rawBytes = Buffer.concat([
journal.sourceRaw,
current.rawBytes.subarray(journal.sourceRaw.length),
]);
return { ...current, raw: rawBytes.toString("utf8"), rawBytes };
}
function createAuditRecoveryScrubPattern(): Buffer {
const pattern = randomBytes(AUDIT_RECOVERY_SCRUB_PATTERN_BYTES);
for (let index = 0; index < pattern.length; index += 1) {
pattern[index] = (pattern[index]! & 1) === 0 ? 0x20 : 0x09;
}
for (let offset = 0; offset < pattern.length; offset += 8) {
const block = pattern.subarray(offset, offset + 8);
if (block.every((byte) => byte === 0x20)) {
pattern[offset + 7] = 0x09;
} else if (block.every((byte) => byte === 0x09)) {
pattern[offset + 7] = 0x20;
}
}
return pattern;
}
function buildScrubbedAuditRecoveryContent(rawBytes: Buffer, scrubPattern: Buffer): Buffer {
if (rawBytes.length === 0) {
return Buffer.alloc(0);
}
// The readable sanitized sibling owns migrated history. This same-inode file
// is only an append landing pad for predecessor writers, so blank the complete
// fixed-size prefix and checkpoint it with zero records. Leading whitespace is
// valid before any late JSONL row and preserves an open O_APPEND offset.
const scrubbed = Buffer.allocUnsafe(rawBytes.length);
for (let offset = 0; offset < scrubbed.length; offset += scrubPattern.length) {
scrubPattern.copy(scrubbed, offset, 0, Math.min(scrubPattern.length, scrubbed.length - offset));
}
return scrubbed;
}
async function writeAuditRecoveryRange(
handle: Awaited<ReturnType<AuditMigrationRoot["openWritable"]>>["handle"],
content: Buffer,
position: number,
): Promise<void> {
let offset = 0;
while (offset < content.byteLength) {
const { bytesWritten } = await handle.write(
content,
offset,
content.byteLength - offset,
position + offset,
);
if (bytesWritten === 0) {
throw new Error("zero-byte write while updating legacy recovery archive");
}
offset += bytesWritten;
}
}
const AUDIT_RECOVERY_WRITE_CHUNK_BYTES = 64 * 1024;
async function writeAuditRecoveryProgress(params: {
root: AuditMigrationRoot;
relativePath: string;
progress: AuditRecoveryProgress;
}): Promise<void> {
const progressRelativePath = auditRecoverySiblingPath(
params.relativePath,
AUDIT_RECOVERY_PROGRESS_SUFFIX,
);
await params.root.write(progressRelativePath, serializeAuditRecoveryProgress(params.progress), {
mkdir: false,
mode: 0o600,
});
const opened = await params.root.open(progressRelativePath);
try {
await opened.handle.chmod(0o600);
await opened.handle.sync();
} finally {
await opened.handle.close();
}
await syncAuditRecoveryDirectory(params.root, params.relativePath);
}
async function readAuditRecoveryProgress(params: {
root: AuditMigrationRoot;
relativePath: string;
journal: ReturnType<typeof parseAuditRecoveryRestoreJournal>;
}): Promise<AuditRecoveryProgress> {
const progressRelativePath = auditRecoverySiblingPath(
params.relativePath,
AUDIT_RECOVERY_PROGRESS_SUFFIX,
);
if (!(await params.root.exists(progressRelativePath))) {
return {
schemaVersion: 1,
journalHash: params.journal.journalHash,
direction: "scrubbing",
committedBytes: 0,
pendingEnd: 0,
extentBytes: params.journal.target.size,
};
}
const snapshot = await readLegacyAuditSourceSnapshot(params.root, progressRelativePath);
return parseAuditRecoveryProgress(snapshot.raw, params.journal);
}
async function advanceAuditRecoveryWrite(params: {
root: AuditMigrationRoot;
relativePath: string;
progress: AuditRecoveryProgress;
desiredContent: Buffer;
handle: Awaited<ReturnType<AuditMigrationRoot["openWritable"]>>["handle"];
}): Promise<AuditRecoveryProgress> {
let progress = params.progress;
if (progress.pendingEnd > progress.committedBytes) {
await writeAuditRecoveryRange(
params.handle,
params.desiredContent.subarray(progress.committedBytes, progress.pendingEnd),
progress.committedBytes,
);
await params.handle.sync();
progress = { ...progress, committedBytes: progress.pendingEnd };
await writeAuditRecoveryProgress({ ...params, progress });
}
while (progress.committedBytes < progress.extentBytes) {
const end = Math.min(
progress.committedBytes + AUDIT_RECOVERY_WRITE_CHUNK_BYTES,
progress.extentBytes,
);
// Commit intent before target bytes. A crash can leave any prefix of this
// range changed; pendingEnd lets recovery finish it without guessing.
progress = { ...progress, pendingEnd: end };
await writeAuditRecoveryProgress({ ...params, progress });
await writeAuditRecoveryRange(
params.handle,
params.desiredContent.subarray(progress.committedBytes, end),
progress.committedBytes,
);
await params.handle.sync();
progress = { ...progress, committedBytes: end };
await writeAuditRecoveryProgress({ ...params, progress });
}
return progress;
}
async function reconcileAuditRecoveryPendingWrite(params: {
root: AuditMigrationRoot;
relativePath: string;
progress: AuditRecoveryProgress;
desiredContent: Buffer;
handle: Awaited<ReturnType<AuditMigrationRoot["openWritable"]>>["handle"];
}): Promise<AuditRecoveryProgress> {
if (params.progress.pendingEnd === params.progress.committedBytes) {
return params.progress;
}
await writeAuditRecoveryRange(
params.handle,
params.desiredContent.subarray(params.progress.committedBytes, params.progress.pendingEnd),
params.progress.committedBytes,
);
await params.handle.sync();
const progress = { ...params.progress, committedBytes: params.progress.pendingEnd };
await writeAuditRecoveryProgress({ ...params, progress });
return progress;
}
async function stageAuditRecoveryRestore(params: {
root: AuditMigrationRoot;
relativePath: string;
snapshot: LegacyAuditSourceSnapshot;
scrubPattern: Buffer;
}): Promise<AuditRecoveryProgress> {
const restoreRelativePath = auditRecoverySiblingPath(
params.relativePath,
AUDIT_RECOVERY_RESTORE_SUFFIX,
);
const stagingRelativePath = auditRecoverySiblingPath(
params.relativePath,
AUDIT_RECOVERY_STAGING_SUFFIX,
);
await params.root.remove(stagingRelativePath).catch(() => undefined);
const journalRaw = serializeAuditRecoveryRestoreJournal({
rawBytes: params.snapshot.rawBytes,
scrubPattern: params.scrubPattern,
target: {
dev: params.snapshot.dev,
ino: params.snapshot.ino,
size: params.snapshot.size,
},
});
await params.root.create(stagingRelativePath, journalRaw, { mode: 0o600 });
const staged = await params.root.open(stagingRelativePath);
try {
await staged.handle.chmod(0o600);
await staged.handle.sync();
} finally {
await staged.handle.close();
}
await params.root.move(stagingRelativePath, restoreRelativePath);
await syncAuditRecoveryDirectory(params.root, params.relativePath);
const journal = parseAuditRecoveryRestoreJournal(journalRaw);
const progress: AuditRecoveryProgress = {
schemaVersion: 1,
journalHash: journal.journalHash,
direction: "scrubbing",
committedBytes: 0,
pendingEnd: 0,
extentBytes: journal.target.size,
};
await writeAuditRecoveryProgress({
root: params.root,
relativePath: params.relativePath,
progress,
});
return progress;
}
export async function restoreInterruptedAuditRecoveryArchive(params: {
root: AuditMigrationRoot;
relativePath: string;
label: string;
warnings: string[];
}): Promise<boolean> {
const restoreRelativePath = auditRecoverySiblingPath(
params.relativePath,
AUDIT_RECOVERY_RESTORE_SUFFIX,
);
const stagingRelativePath = auditRecoverySiblingPath(
params.relativePath,
AUDIT_RECOVERY_STAGING_SUFFIX,
);
const progressRelativePath = auditRecoverySiblingPath(
params.relativePath,
AUDIT_RECOVERY_PROGRESS_SUFFIX,
);
if (!(await params.root.exists(restoreRelativePath))) {
await params.root.remove(stagingRelativePath).catch(() => undefined);
await params.root.remove(progressRelativePath).catch(() => undefined);
return true;
}
try {
const currentSnapshot = await readLegacyAuditSourceSnapshot(params.root, params.relativePath);
const restoreSnapshot = await readLegacyAuditSourceSnapshot(params.root, restoreRelativePath);
const journal = parseAuditRecoveryRestoreJournal(restoreSnapshot.raw);
let progress = await readAuditRecoveryProgress({
root: params.root,
relativePath: params.relativePath,
journal,
});
const scrubbedContent = buildScrubbedAuditRecoveryContent(
journal.sourceRaw,
journal.scrubPattern,
);
let completedCheckpoint: LegacyAuditRawCheckpoint | undefined;
try {
completedCheckpoint = findPreviousLegacyAuditRawCheckpoint(
params.root.rootReal,
params.relativePath,
);
} catch {
completedCheckpoint = undefined;
}
if (
completedCheckpoint &&
completedCheckpoint.phase === "raw" &&
completedCheckpoint.recordCount === 0 &&
completedCheckpoint.size === journal.sourceRaw.length &&
auditRecoveryCheckpointPrefixMatches(currentSnapshot, completedCheckpoint)
) {
// Checkpoint commit won the crash race; the restore journal is stale and
// must not roll the already-checkpointed sanitized inode backward.
await params.root.remove(progressRelativePath).catch(() => undefined);
await params.root.remove(stagingRelativePath).catch(() => undefined);
await params.root.remove(restoreRelativePath);
await syncAuditRecoveryDirectory(params.root, params.relativePath);
return true;
}
const writable = await params.root.openWritable(params.relativePath, {
mode: 0o600,
writeMode: "update",
});
try {
const verification = await readLegacyAuditSourceSnapshot(params.root, params.relativePath);
if (
writable.stat.dev !== verification.dev ||
writable.stat.ino !== verification.ino ||
!auditRecoveryJournalTargetsSnapshot(verification, journal) ||
!auditRecoveryStateMatchesJournal({
current: verification.rawBytes,
original: journal.sourceRaw,
scrubbed: scrubbedContent,
progress,
})
) {
throw new Error("legacy recovery archive no longer matches its restore journal target");
}
progress = await reconcileAuditRecoveryPendingWrite({
root: params.root,
relativePath: params.relativePath,
progress,
desiredContent: progress.direction === "scrubbing" ? scrubbedContent : journal.sourceRaw,
handle: writable.handle,
});
if (progress.direction === "scrubbing") {
progress = {
schemaVersion: 1,
journalHash: journal.journalHash,
direction: "restoring",
committedBytes: 0,
pendingEnd: 0,
extentBytes: progress.committedBytes,
};
await writeAuditRecoveryProgress({
root: params.root,
relativePath: params.relativePath,
progress,
});
}
await advanceAuditRecoveryWrite({
root: params.root,
relativePath: params.relativePath,
progress,
desiredContent: journal.sourceRaw,
handle: writable.handle,
});
await writable.handle.chmod(0o600);
await writable.handle.sync();
} finally {
await writable.handle.close().catch(() => undefined);
}
await params.root.remove(progressRelativePath).catch(() => undefined);
await params.root.remove(stagingRelativePath).catch(() => undefined);
await params.root.remove(restoreRelativePath);
await syncAuditRecoveryDirectory(params.root, params.relativePath);
return true;
} catch (error) {
params.warnings.push(
`Failed restoring interrupted ${params.label} legacy recovery archive: ${String(error)}`,
);
return false;
}
}
export async function finalizeLegacyAuditRecoveryArchive(params: {
root: AuditMigrationRoot;
relativePath: string;
}): Promise<void> {
await params.root
.remove(auditRecoverySiblingPath(params.relativePath, AUDIT_RECOVERY_PROGRESS_SUFFIX))
.catch(() => undefined);
await params.root
.remove(auditRecoverySiblingPath(params.relativePath, AUDIT_RECOVERY_STAGING_SUFFIX))
.catch(() => undefined);
await params.root.remove(
auditRecoverySiblingPath(params.relativePath, AUDIT_RECOVERY_RESTORE_SUFFIX),
);
await syncAuditRecoveryDirectory(params.root, params.relativePath);
}
export async function scrubLegacyAuditRecoveryArchive(params: {
root: AuditMigrationRoot;
relativePath: string;
expectedSnapshot: LegacyAuditSourceSnapshot;
label: string;
warnings: string[];
}): Promise<LegacyAuditSourceSnapshot | undefined> {
const scrubPattern = createAuditRecoveryScrubPattern();
const scrubbedContent = buildScrubbedAuditRecoveryContent(
params.expectedSnapshot.rawBytes,
scrubPattern,
);
let progress: AuditRecoveryProgress;
try {
progress = await stageAuditRecoveryRestore({
root: params.root,
relativePath: params.relativePath,
snapshot: params.expectedSnapshot,
scrubPattern,
});
} catch (error) {
params.warnings.push(
`Failed staging ${params.label} legacy recovery restore journal: ${String(error)}`,
);
return undefined;
}
let writable: Awaited<ReturnType<AuditMigrationRoot["openWritable"]>>;
try {
writable = await params.root.openWritable(params.relativePath, {
mode: 0o600,
writeMode: "update",
});
} catch (error) {
params.warnings.push(
`Failed scrubbing ${params.label} legacy recovery archive: ${String(error)}`,
);
return undefined;
}
try {
if (!legacyAuditRawCheckpointsMatch(params.expectedSnapshot, writable.stat)) {
params.warnings.push(
`Skipped scrubbing changed ${params.label} legacy recovery archive; rerun openclaw doctor --fix`,
);
return undefined;
}
await advanceAuditRecoveryWrite({
root: params.root,
relativePath: params.relativePath,
progress,
desiredContent: scrubbedContent,
handle: writable.handle,
});
await writable.handle.chmod(0o600);
await writable.handle.sync();
} catch (error) {
await writable.handle.close().catch(() => undefined);
const recoveryWarnings: string[] = [];
const restored = await restoreInterruptedAuditRecoveryArchive({
root: params.root,
relativePath: params.relativePath,
label: params.label,
warnings: recoveryWarnings,
});
if (restored) {
params.warnings.push(
`Failed scrubbing ${params.label} legacy recovery archive; restored it for Doctor retry: ${String(error)}`,
);
} else {
params.warnings.push(...recoveryWarnings);
params.warnings.push(
`Failed scrubbing ${params.label} legacy recovery archive; left its progress journal for Doctor retry: ${String(error)}`,
);
}
return undefined;
} finally {
await writable.handle.close().catch(() => undefined);
}
let scrubbedSnapshot: LegacyAuditSourceSnapshot;
try {
scrubbedSnapshot = await readLegacyAuditSourceSnapshot(params.root, params.relativePath);
} catch (error) {
params.warnings.push(
`Changed ${params.label} legacy recovery archive during scrub verification; rerun openclaw doctor --fix: ${String(error)}`,
);
return undefined;
}
const scrubbedPrefix = scrubbedSnapshot.rawBytes.subarray(0, scrubbedContent.length);
if (!scrubbedPrefix.equals(scrubbedContent)) {
params.warnings.push(
`Failed verifying scrubbed ${params.label} legacy recovery archive; rerun openclaw doctor --fix`,
);
return undefined;
}
return scrubbedSnapshot;
}
export async function recordLegacyAuditRawCheckpoint(params: {
stateDir: string;
rawPath: string;
rawRelativePath: string;
sanitizedRelativePath: string;
root: AuditMigrationRoot;
snapshot: LegacyAuditSourceSnapshot;
phase: "merge-intent" | "raw";
recordCount: number;
recordOrdinalBase: number;
warnings: string[];
}): Promise<boolean> {
try {
const sanitizedSnapshot = await readLegacyAuditSourceSnapshot(
params.root,
params.sanitizedRelativePath,
);
const opened = await params.root.open(params.rawRelativePath);
let checkpoint: LegacyAuditRawCheckpoint;
try {
const stat = await opened.handle.stat();
checkpoint = {
dev: stat.dev,
ino: stat.ino,
mtimeMs: stat.mtimeMs,
size: stat.size,
phase: params.phase,
generationKey: legacyAuditSourceGenerationKey(params.rawRelativePath),
recordCount: params.recordCount,
recordOrdinalBase: params.recordOrdinalBase,
contentHash: createHash("sha256").update(params.snapshot.rawBytes).digest("hex"),
sanitizedContentHash: createHash("sha256").update(sanitizedSnapshot.rawBytes).digest("hex"),
sanitizedSize: sanitizedSnapshot.rawBytes.length,
};
} finally {
await opened.handle.close();
}
if (!legacyAuditRawCheckpointsMatch(checkpoint, params.snapshot)) {
params.warnings.push(
`Retained changed legacy audit backup ${params.rawPath}; rerun openclaw doctor --fix to import its later rows`,
);
return false;
}
openLegacyAuditRawCheckpointStore(params.stateDir).upsert(
legacyAuditRawCheckpointKey(checkpoint),
checkpoint,
);
return true;
} catch (error) {
params.warnings.push(
`Failed recording legacy audit backup checkpoint for ${params.rawPath}: ${String(error)}`,
);
return false;
}
}
export function findPreviousLegacyAuditRawCheckpoint(
stateDir: string,
rawRelativePath: string,
): LegacyAuditRawCheckpoint | undefined {
const generationKey = legacyAuditSourceGenerationKey(rawRelativePath);
return openLegacyAuditRawCheckpointStore(stateDir)
.entries()
.toReversed()
.find((entry) => entry.value.generationKey === generationKey)?.value;
}
export function recordsAfterLegacyAuditRawCheckpoint<T>(params: {
checkpoint: LegacyAuditRawCheckpoint;
snapshot: LegacyAuditSourceSnapshot;
records: readonly T[];
}): readonly T[] | undefined {
const rawBytes = params.snapshot.rawBytes;
if (rawBytes.length < params.checkpoint.size) {
return undefined;
}
const prefixHash = createHash("sha256")
.update(rawBytes.subarray(0, params.checkpoint.size))
.digest("hex");
const legacyUtf8PrefixHash = createHash("sha256")
.update(rawBytes.subarray(0, params.checkpoint.size).toString("utf8"))
.digest("hex");
if (
(prefixHash !== params.checkpoint.contentHash &&
legacyUtf8PrefixHash !== params.checkpoint.contentHash) ||
params.records.length < params.checkpoint.recordCount
) {
return undefined;
}
return params.records.slice(params.checkpoint.recordCount);
}
@@ -0,0 +1,72 @@
import { createHash } from "node:crypto";
import type { LegacyAuditRawCheckpoint } from "./state-migrations.audit-checkpoints.js";
import {
readLegacyAuditSourceSnapshot,
type AuditMigrationRoot,
} from "./state-migrations.audit-recovery.js";
export async function writeRecoveredSanitizedAuditArchive(params: {
sourceLabel: string;
root: AuditMigrationRoot;
relativePath: string;
allRecordsJsonl: string;
candidateRecordsJsonl: string;
previousCheckpoint: LegacyAuditRawCheckpoint | undefined;
warnings: string[];
}): Promise<boolean> {
const current = (await params.root.exists(params.relativePath))
? await readLegacyAuditSourceSnapshot(params.root, params.relativePath)
: undefined;
let desired: Buffer;
if (params.previousCheckpoint) {
if (!current || current.rawBytes.length < params.previousCheckpoint.sanitizedSize) {
params.warnings.push(
`Skipped ${params.sourceLabel} recovery because its sanitized archive is missing or truncated`,
);
return false;
}
const checkpointedPrefix = current.rawBytes.subarray(
0,
params.previousCheckpoint.sanitizedSize,
);
if (
createHash("sha256").update(checkpointedPrefix).digest("hex") !==
params.previousCheckpoint.sanitizedContentHash
) {
params.warnings.push(
`Skipped ${params.sourceLabel} recovery because its sanitized archive changed after checkpoint`,
);
return false;
}
// The caller already removed checkpointed raw records. For merge-intent,
// this sanitized prefix also contains that previously materialized batch.
desired = Buffer.concat([
checkpointedPrefix,
Buffer.from(params.candidateRecordsJsonl, "utf8"),
]);
if (current.rawBytes.equals(desired)) {
return true;
}
const currentIsVerifiedDesiredPrefix = desired
.subarray(0, current.rawBytes.length)
.equals(current.rawBytes);
if (
current.rawBytes.length !== params.previousCheckpoint.sanitizedSize &&
!currentIsVerifiedDesiredPrefix
) {
params.warnings.push(
`Skipped ${params.sourceLabel} recovery because its sanitized archive has an uncheckpointed tail`,
);
return false;
}
} else {
// A checkpointless unscrubbed raw archive is authoritative. Replacing from
// the complete sanitized projection is idempotent before checkpoint commit.
desired = Buffer.from(params.allRecordsJsonl, "utf8");
if (current?.rawBytes.equals(desired)) {
return true;
}
}
await params.root.write(params.relativePath, desired, { mkdir: false, mode: 0o600 });
return true;
}
+121 -30
View File
@@ -11,6 +11,8 @@ import type { OpenClawConfig } from "../config/types.openclaw.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import {
createPluginStateKeyedStore,
getPluginStateCapacity as resolvePluginStateCapacity,
importPluginStateEntriesForDoctor,
type OpenKeyedStoreOptions,
} from "../plugin-state/plugin-state-store.js";
import {
@@ -27,6 +29,7 @@ import {
repairOpenClawStateDatabaseSchema,
type OpenClawStateDatabaseSchemaMigration,
} from "../state/openclaw-state-db.js";
import { acquireGatewayLock } from "./gateway-lock.js";
import {
detectLegacyAcpReplayLedger,
migrateLegacyAcpReplayLedger,
@@ -35,6 +38,7 @@ import {
detectLegacyApnsRegistrations,
migrateLegacyApnsRegistrations,
} from "./state-migrations.apns.js";
import { detectLegacyAuditLogs, migrateLegacyAuditLogs } from "./state-migrations.audit-logs.js";
import {
detectLegacyChannelPairingState,
migrateLegacyChannelPairingState,
@@ -164,6 +168,9 @@ function describeStateSchemaMigration(migration: OpenClawStateDatabaseSchemaMigr
let autoMigrateChecked = false;
const PLUGIN_DOCTOR_MIGRATION_LOCK_TIMEOUT_MS = 250;
const PLUGIN_DOCTOR_MIGRATION_LOCK_POLL_INTERVAL_MS = 25;
export function resetAutoMigrateLegacyStateForTest(): void {
autoMigrateChecked = false;
resetAutoMigrateLegacyTaskStateSidecarsForTest();
@@ -206,6 +213,7 @@ async function collectPluginDoctorStateMigrationPlans(params: {
env: NodeJS.ProcessEnv;
stateDir: string;
oauthDir: string;
includeDoctorOnly?: boolean;
warnings?: string[];
}): Promise<DetectedPluginDoctorStateMigrationPlan[]> {
const plans: DetectedPluginDoctorStateMigrationPlan[] = [];
@@ -214,6 +222,9 @@ async function collectPluginDoctorStateMigrationPlans(params: {
config,
env: params.env,
})) {
if (entry.migration.doctorOnly === true && params.includeDoctorOnly !== true) {
continue;
}
let detected: PluginDoctorStateMigrationDetection | null;
try {
detected = await entry.migration.detectLegacyState({
@@ -243,6 +254,15 @@ function createPluginDoctorStateMigrationContext(
env: NodeJS.ProcessEnv,
): PluginDoctorStateMigrationContext {
return {
getPluginStateCapacity() {
return resolvePluginStateCapacity(pluginId, env);
},
importPluginStateEntries(
options: OpenKeyedStoreOptions,
entries: readonly { key: string; value: unknown; createdAt: number }[],
) {
importPluginStateEntriesForDoctor(pluginId, { ...options, env: options.env ?? env }, entries);
},
openPluginStateKeyedStore<T>(options: OpenKeyedStoreOptions) {
return createPluginStateKeyedStore<T>(pluginId, {
...options,
@@ -406,6 +426,10 @@ export async function detectLegacyStateMigrations(params: {
stateDir,
doctorOnlyStateMigrations: params.doctorOnlyStateMigrations,
});
const auditLogs = detectLegacyAuditLogs({
stateDir,
doctorOnlyStateMigrations: params.doctorOnlyStateMigrations,
});
const acpReplayLedger = detectLegacyAcpReplayLedger({
stateDir,
doctorOnlyStateMigrations: params.doctorOnlyStateMigrations,
@@ -522,6 +546,7 @@ export async function detectLegacyStateMigrations(params: {
env,
stateDir,
oauthDir,
includeDoctorOnly: params.doctorOnlyStateMigrations === true,
warnings: pluginPlanWarnings,
});
@@ -593,6 +618,9 @@ export async function detectLegacyStateMigrations(params: {
if (commitments.hasLegacy) {
preview.push("- Commitments: legacy JSON file → shared SQLite state");
}
for (const source of auditLogs.sources) {
preview.push(`- ${source.label}: legacy JSONL file → shared SQLite state`);
}
if (acpReplayLedger.hasLegacy) {
preview.push("- ACP replay ledger: legacy JSON file → shared SQLite state");
}
@@ -634,6 +662,7 @@ export async function detectLegacyStateMigrations(params: {
}
return {
doctorOnlyStateMigrations: params.doctorOnlyStateMigrations === true,
targetAgentId,
targetMainKey,
targetScope,
@@ -707,6 +736,7 @@ export async function detectLegacyStateMigrations(params: {
},
tuiLastSessions,
commitments,
auditLogs,
acpReplayLedger,
managedOutgoingImages,
apns,
@@ -737,6 +767,7 @@ async function runPluginDoctorStateMigrationPlans(params: {
env: params.env,
stateDir: params.detected.stateDir,
oauthDir: params.detected.oauthDir,
includeDoctorOnly: params.detected.doctorOnlyStateMigrations,
warnings,
});
const hasDetectorFailure = warnings.length > 0;
@@ -746,21 +777,80 @@ async function runPluginDoctorStateMigrationPlans(params: {
refreshedPlans.length > 0 || hasDetectorFailure
? refreshedPlans
: (params.detected.pluginPlans?.plans ?? []);
for (const plan of plans) {
try {
const result = await plan.migration.migrateLegacyState({
config: params.config,
env: params.env,
stateDir: params.detected.stateDir,
oauthDir: params.detected.oauthDir,
context: createPluginDoctorStateMigrationContext(plan.pluginId, params.env),
});
changes.push(...result.changes);
warnings.push(...result.warnings);
notices.push(...(result.notices ?? []));
} catch (err) {
warnings.push(`Failed migrating ${plan.migration.label}: ${String(err)}`);
const migrated = await migratePluginDoctorStatePlans({
plans,
config: params.config,
env: params.env,
stateDir: params.detected.stateDir,
oauthDir: params.detected.oauthDir,
});
changes.push(...migrated.changes);
warnings.push(...migrated.warnings);
notices.push(...(migrated.notices ?? []));
return notices.length > 0 ? { changes, warnings, notices } : { changes, warnings };
}
async function migratePluginDoctorStatePlans(params: {
plans: readonly DetectedPluginDoctorStateMigrationPlan[];
config: OpenClawConfig;
env: NodeJS.ProcessEnv;
stateDir: string;
oauthDir: string;
}): Promise<MigrationMessages> {
const changes: string[] = [];
const warnings: string[] = [];
const notices: string[] = [];
if (params.plans.length === 0) {
return { changes, warnings };
}
let lock: Awaited<ReturnType<typeof acquireGatewayLock>>;
try {
lock = await acquireGatewayLock({
allowInTests: true,
env: { ...params.env, OPENCLAW_STATE_DIR: params.stateDir },
pollIntervalMs: PLUGIN_DOCTOR_MIGRATION_LOCK_POLL_INTERVAL_MS,
role: "sqlite-maintenance",
timeoutMs: PLUGIN_DOCTOR_MIGRATION_LOCK_TIMEOUT_MS,
});
} catch (error) {
return {
changes,
warnings: [
`Skipped plugin doctor state migrations because exclusive state ownership is unavailable: ${String(error)}`,
],
};
}
if (!lock) {
return {
changes,
warnings: [
"Skipped plugin doctor state migrations because exclusive state ownership is unavailable",
],
};
}
try {
// Plugin migrations may claim retired files after verified import. Keep the
// predecessor Gateway excluded for the full read, import, and archive window.
for (const plan of params.plans) {
try {
const result = await plan.migration.migrateLegacyState({
config: params.config,
env: params.env,
stateDir: params.stateDir,
oauthDir: params.oauthDir,
context: createPluginDoctorStateMigrationContext(plan.pluginId, params.env),
});
changes.push(...result.changes);
warnings.push(...result.warnings);
notices.push(...(result.notices ?? []));
} catch (err) {
warnings.push(`Failed migrating ${plan.migration.label}: ${String(err)}`);
}
}
} finally {
await lock.release();
}
return notices.length > 0 ? { changes, warnings, notices } : { changes, warnings };
}
@@ -807,22 +897,16 @@ export async function autoMigrateLegacyPluginDoctorState(params: {
oauthDir,
warnings,
});
for (const plan of plans) {
try {
const result = await plan.migration.migrateLegacyState({
config: params.config,
env,
stateDir,
oauthDir,
context: createPluginDoctorStateMigrationContext(plan.pluginId, env),
});
changes.push(...result.changes);
warnings.push(...result.warnings);
notices.push(...(result.notices ?? []));
} catch (err) {
warnings.push(`Failed migrating ${plan.migration.label}: ${String(err)}`);
}
}
const migrated = await migratePluginDoctorStatePlans({
plans,
config: params.config,
env,
stateDir,
oauthDir,
});
changes.push(...migrated.changes);
warnings.push(...migrated.warnings);
notices.push(...(migrated.notices ?? []));
return {
migrated: stateDirResult.migrated || stateSchema.changes.length > 0 || plans.length > 0,
skipped: false,
@@ -902,6 +986,10 @@ export async function runLegacyStateMigrations(params: {
detected: detected.commitments,
stateDir: detected.stateDir,
});
const auditLogs = await migrateLegacyAuditLogs({
detected: detected.auditLogs,
stateDir: detected.stateDir,
});
const acpReplayLedger = await migrateLegacyAcpReplayLedger({
detected: detected.acpReplayLedger,
stateDir: detected.stateDir,
@@ -980,6 +1068,7 @@ export async function runLegacyStateMigrations(params: {
updateCheck,
tuiLastSessions,
commitments,
auditLogs,
acpReplayLedger,
managedOutgoingImages,
apns,
@@ -1006,6 +1095,7 @@ export async function runLegacyStateMigrations(params: {
...currentConversationBindings.changes,
...tuiLastSessions.changes,
...commitments.changes,
...auditLogs.changes,
...acpReplayLedger.changes,
...managedOutgoingImages.changes,
...apns.changes,
@@ -1039,6 +1129,7 @@ export async function runLegacyStateMigrations(params: {
...currentConversationBindings.warnings,
...tuiLastSessions.warnings,
...commitments.warnings,
...auditLogs.warnings,
...acpReplayLedger.warnings,
...managedOutgoingImages.warnings,
...apns.warnings,
+99
View File
@@ -16,6 +16,7 @@ import {
OPENCLAW_STATE_SCHEMA_VERSION,
} from "../state/openclaw-state-db.js";
import { createTrackedTempDirs } from "../test-utils/tracked-temp-dirs.js";
import { acquireGatewayLock } from "./gateway-lock.js";
import {
executeSqliteQuerySync,
executeSqliteQueryTakeFirstSync,
@@ -48,6 +49,7 @@ const pluginDoctorStateMigrationEntries = vi.hoisted(
migration: {
id: string;
label: string;
doctorOnly?: boolean;
detectLegacyState: (params: {
config: OpenClawConfig;
env: NodeJS.ProcessEnv;
@@ -546,6 +548,54 @@ describe("state migrations", () => {
expect(result.changes).toContain("Migrated fixture environment");
});
it("runs doctor-only plugin file imports only during explicit Doctor repair", async () => {
const root = await createTempDir();
const stateDir = path.join(root, ".openclaw");
const env = createEnv(stateDir);
const cfg = createConfig();
const detectLegacyState = vi.fn(() => ({ preview: ["doctor-only plugin state"] }));
const migrateLegacyState = vi.fn(() => ({
changes: ["doctor-only plugin state migrated"],
warnings: [],
}));
pluginDoctorStateMigrationEntries.entries = [
{
pluginId: "memory-core",
migration: {
id: "memory-core-doctor-only-test",
label: "Memory Core doctor-only test migration",
doctorOnly: true,
detectLegacyState,
migrateLegacyState,
},
},
];
const automatic = await autoMigrateLegacyPluginDoctorState({
config: cfg,
env,
homedir: () => root,
});
expect(automatic.changes).not.toContain("doctor-only plugin state migrated");
expect(detectLegacyState).not.toHaveBeenCalled();
expect(migrateLegacyState).not.toHaveBeenCalled();
const detected = await detectLegacyStateMigrations({
cfg,
env,
homedir: () => root,
doctorOnlyStateMigrations: true,
});
expect(detected.pluginPlans).toMatchObject({ hasLegacy: true });
expect(detected.preview).toContain("doctor-only plugin state");
const repaired = await runLegacyStateMigrations({ detected, config: cfg, env });
expect(repaired.warnings).toStrictEqual([]);
expect(repaired.changes).toContain("doctor-only plugin state migrated");
expect(detectLegacyState).toHaveBeenCalledTimes(2);
expect(migrateLegacyState).toHaveBeenCalledOnce();
});
it("detects legacy sessions, agent files, channel auth, and pairing state", () => {
expect(detectionCase.targetAgentId).toBe("worker-1");
expect(detectionCase.targetMainKey).toBe("desk");
@@ -2011,6 +2061,55 @@ describe("state migrations", () => {
expect(migrateLegacyState).toHaveBeenCalledOnce();
});
it("requires exclusive state ownership before plugin doctor migrations", async () => {
const root = await createTempDir();
const stateDir = path.join(root, ".openclaw");
const env = createEnv(stateDir);
const cfg = createConfig();
openOpenClawStateDatabase({ env });
closeOpenClawStateDatabaseForTest();
const migrateLegacyState = vi.fn(() => ({
changes: ["plugin state migrated"],
warnings: [],
}));
pluginDoctorStateMigrationEntries.entries = [
{
pluginId: "memory-core",
migration: {
id: "memory-core-lock-test",
label: "Memory Core lock test migration",
detectLegacyState: () => ({ preview: ["plugin state"] }),
migrateLegacyState,
},
},
];
const gatewayLock = await acquireGatewayLock({
allowInTests: true,
env,
pollIntervalMs: 10,
port: 18_791,
timeoutMs: 100,
});
if (!gatewayLock) {
throw new Error("expected test Gateway lock");
}
let result: Awaited<ReturnType<typeof autoMigrateLegacyPluginDoctorState>>;
try {
result = await autoMigrateLegacyPluginDoctorState({
config: cfg,
env,
homedir: () => root,
});
} finally {
await gatewayLock.release();
}
expect(result.changes).not.toContain("plugin state migrated");
expect(result.warnings.join("\n")).toContain("exclusive state ownership is unavailable");
expect(migrateLegacyState).not.toHaveBeenCalled();
});
it("skips stale plugin doctor plans when refresh detection fails", async () => {
const root = await createTempDir();
const stateDir = path.join(root, ".openclaw");
+3
View File
@@ -1,6 +1,7 @@
import type { ChannelLegacyStateMigrationPlan } from "../channels/plugins/types.core.js";
import type { SessionScope } from "../config/sessions/types.js";
import type { PluginDoctorStateMigration } from "../plugins/doctor-contract-registry.js";
import type { LegacyAuditLogsDetection } from "./state-migrations.audit-logs.types.js";
import type { LegacyChannelPairingStateDetection } from "./state-migrations.channel-pairing.js";
import type { LegacyMcpOAuthDetection } from "./state-migrations.mcp-oauth.types.js";
import type { LegacyRestartSentinelDetection } from "./state-migrations.restart-sentinel.types.js";
@@ -18,6 +19,7 @@ export type SessionStoreAliasPlan = {
};
export type LegacyStateDetection = {
doctorOnlyStateMigrations?: boolean;
targetAgentId: string;
targetMainKey: string;
targetScope?: SessionScope;
@@ -103,6 +105,7 @@ export type LegacyStateDetection = {
sourcePath: string;
hasLegacy: boolean;
};
auditLogs: LegacyAuditLogsDetection;
acpReplayLedger: {
sourcePath: string;
hasLegacy: boolean;
+20
View File
@@ -0,0 +1,20 @@
import type { PersistedMemoryHostEvent } from "./event-store.js";
export const MAX_MEMORY_HOST_PUBLIC_EXPORT_BYTES = 1024 * 1024;
export function serializeMemoryHostEventExport(
storedEvents: readonly PersistedMemoryHostEvent[],
): string {
const lines: string[] = [];
let sizeBytes = 0;
for (const entry of storedEvents.toReversed()) {
const line = JSON.stringify(entry.value.event);
const lineBytes = Buffer.byteLength(line, "utf8") + 1;
if (sizeBytes + lineBytes > MAX_MEMORY_HOST_PUBLIC_EXPORT_BYTES) {
break;
}
lines.push(line);
sizeBytes += lineBytes;
}
return lines.toReversed().join("\n") + "\n";
}
+359
View File
@@ -0,0 +1,359 @@
import { createHash } from "node:crypto";
import { resolveWorkspaceStateIdentity } from "../agents/workspace-state-store.js";
import {
pluginStateEntriesInKeyRange,
registerPluginStateSyncSequencedJournalEntry,
} from "../plugin-state/plugin-state-store.js";
import type { MemoryHostEventRecord } from "./event-types.js";
const MEMORY_HOST_EVENTS_PLUGIN_ID = "memory-core";
const MEMORY_HOST_EVENTS_NAMESPACE = "memory-host.events";
const MEMORY_HOST_EVENT_CURSORS_NAMESPACE = "memory-host.event-cursors";
// Event rotation retains deep diagnostics without consuming memory-core's
// plugin-wide 50,000-row budget or starving sibling state namespaces.
const MAX_MEMORY_HOST_EVENTS = 10_000;
const MAX_MEMORY_HOST_EVENT_CURSORS = 1_000;
const MAX_MEMORY_HOST_EVENT_JSON_BYTES = 8 * 1024;
const MAX_MEMORY_HOST_EVENT_ITEMS = 10;
const MAX_MEMORY_HOST_EVENT_TEXT_BYTES = 2 * 1024;
const MAX_MEMORY_HOST_EVENT_PATH_BYTES = 256;
const WORKSPACE_HASH_BYTES = 24;
type StoredMemoryHostEvent = {
kind: "event";
event: MemoryHostEventRecord;
recordedAt: number;
sequence: number;
};
let maxMemoryHostEventsForTests: number | undefined;
export type PersistedMemoryHostEvent = {
key: string;
value: StoredMemoryHostEvent;
createdAt: number;
};
function normalizeMemoryHostWorkspaceKey(workspaceDir: string): string {
// Workspace aliases must share one event/cursor namespace. Otherwise two
// configured paths to the same workspace can publish conflicting exports.
const resolved = resolveWorkspaceStateIdentity(workspaceDir).workspacePath.replace(/\\/g, "/");
return process.platform === "win32" ? resolved.toLowerCase() : resolved;
}
function memoryHostWorkspacePrefix(workspaceDir: string): string {
return createHash("sha256")
.update(normalizeMemoryHostWorkspaceKey(workspaceDir))
.digest("hex")
.slice(0, WORKSPACE_HASH_BYTES);
}
function eventKeyPrefix(workspaceDir: string): string {
return `${memoryHostWorkspacePrefix(workspaceDir)}:event:`;
}
function eventKeyRangeEnd(workspaceDir: string): string {
return `${memoryHostWorkspacePrefix(workspaceDir)}:event;`;
}
function memoryHostEventStorageKey(workspaceDir: string, sequence: number): string {
if (!Number.isSafeInteger(sequence)) {
throw new Error("Memory host event sequence must be a safe integer");
}
return `${eventKeyPrefix(workspaceDir)}1:${sequence.toString().padStart(16, "0")}`;
}
function cursorKey(workspaceDir: string): string {
return `${memoryHostWorkspacePrefix(workspaceDir)}:cursor`;
}
function truncateUtf8(value: string, maxBytes: number): { value: string; truncated: boolean } {
if (Buffer.byteLength(value, "utf8") <= maxBytes) {
return { value, truncated: false };
}
let low = 0;
let high = value.length;
while (low < high) {
const middle = Math.ceil((low + high) / 2);
if (Buffer.byteLength(value.slice(0, middle), "utf8") <= maxBytes - 3) {
low = middle;
} else {
high = middle - 1;
}
}
const end = low > 0 && /[\uD800-\uDBFF]/u.test(value.charAt(low - 1)) ? low - 1 : low;
return { value: `${value.slice(0, end)}`, truncated: true };
}
function isFiniteNumber(value: unknown): value is number {
return typeof value === "number" && Number.isFinite(value);
}
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
}
/** Validate and bound one diagnostic event before storing it in plugin state. */
export function normalizeMemoryHostEventRecordForStorage(
value: unknown,
): MemoryHostEventRecord | null {
if (!isRecord(value) || typeof value.type !== "string" || typeof value.timestamp !== "string") {
return null;
}
const timestamp = truncateUtf8(value.timestamp, 128);
let truncated = timestamp.truncated || value.storageTruncated === true;
if (value.type === "memory.recall.recorded" || value.type === "memory.recall.skipped") {
if (
typeof value.query !== "string" ||
!Array.isArray(value.results) ||
(value.type === "memory.recall.recorded"
? !isFiniteNumber(value.resultCount)
: !isFiniteNumber(value.skippedResultCount))
) {
return null;
}
if (
value.type === "memory.recall.skipped" &&
(value.reason !== "non-short-term-memory-path" ||
!isFiniteNumber(value.eligibleResultCount) ||
!isFiniteNumber(value.skippedResultCount))
) {
return null;
}
const query = truncateUtf8(value.query, MAX_MEMORY_HOST_EVENT_TEXT_BYTES);
truncated ||= query.truncated || value.results.length > MAX_MEMORY_HOST_EVENT_ITEMS;
const results: Array<{
path: string;
startLine: number;
endLine: number;
score: number;
reason?: "non-short-term-memory-path";
}> = [];
for (const result of value.results.slice(0, MAX_MEMORY_HOST_EVENT_ITEMS)) {
if (
!isRecord(result) ||
typeof result.path !== "string" ||
!isFiniteNumber(result.startLine) ||
!isFiniteNumber(result.endLine) ||
!isFiniteNumber(result.score) ||
(value.type === "memory.recall.skipped" && result.reason !== "non-short-term-memory-path")
) {
return null;
}
const resultPath = truncateUtf8(result.path, MAX_MEMORY_HOST_EVENT_PATH_BYTES);
truncated ||= resultPath.truncated;
results.push({
path: resultPath.value,
startLine: result.startLine,
endLine: result.endLine,
score: result.score,
...(value.type === "memory.recall.skipped"
? { reason: "non-short-term-memory-path" as const }
: {}),
});
}
const normalized =
value.type === "memory.recall.recorded"
? {
type: "memory.recall.recorded" as const,
timestamp: timestamp.value,
query: query.value,
resultCount: value.resultCount as number,
results: results.map((result) => ({
path: result.path,
startLine: result.startLine,
endLine: result.endLine,
score: result.score,
})),
...(truncated ? { storageTruncated: true as const } : {}),
}
: {
type: "memory.recall.skipped" as const,
timestamp: timestamp.value,
query: query.value,
reason: "non-short-term-memory-path" as const,
eligibleResultCount: value.eligibleResultCount as number,
skippedResultCount: value.skippedResultCount as number,
results: results.map((result) => ({
path: result.path,
startLine: result.startLine,
endLine: result.endLine,
score: result.score,
reason: "non-short-term-memory-path" as const,
})),
...(truncated ? { storageTruncated: true as const } : {}),
};
return Buffer.byteLength(JSON.stringify(normalized), "utf8") <= MAX_MEMORY_HOST_EVENT_JSON_BYTES
? normalized
: { ...normalized, results: [], storageTruncated: true as const };
}
if (value.type === "memory.promotion.applied") {
if (
typeof value.memoryPath !== "string" ||
!isFiniteNumber(value.applied) ||
!Array.isArray(value.candidates)
) {
return null;
}
const memoryPath = truncateUtf8(value.memoryPath, MAX_MEMORY_HOST_EVENT_PATH_BYTES);
truncated ||= memoryPath.truncated || value.candidates.length > MAX_MEMORY_HOST_EVENT_ITEMS;
const candidates: Array<{
key: string;
path: string;
startLine: number;
endLine: number;
score: number;
recallCount: number;
}> = [];
for (const candidate of value.candidates.slice(0, MAX_MEMORY_HOST_EVENT_ITEMS)) {
if (
!isRecord(candidate) ||
typeof candidate.key !== "string" ||
typeof candidate.path !== "string" ||
!isFiniteNumber(candidate.startLine) ||
!isFiniteNumber(candidate.endLine) ||
!isFiniteNumber(candidate.score) ||
!isFiniteNumber(candidate.recallCount)
) {
return null;
}
const key = truncateUtf8(candidate.key, MAX_MEMORY_HOST_EVENT_PATH_BYTES);
const candidatePath = truncateUtf8(candidate.path, MAX_MEMORY_HOST_EVENT_PATH_BYTES);
truncated ||= key.truncated || candidatePath.truncated;
candidates.push({
key: key.value,
path: candidatePath.value,
startLine: candidate.startLine,
endLine: candidate.endLine,
score: candidate.score,
recallCount: candidate.recallCount,
});
}
const normalized = {
type: "memory.promotion.applied" as const,
timestamp: timestamp.value,
memoryPath: memoryPath.value,
applied: value.applied,
candidates,
...(truncated ? { storageTruncated: true as const } : {}),
};
return Buffer.byteLength(JSON.stringify(normalized), "utf8") <= MAX_MEMORY_HOST_EVENT_JSON_BYTES
? normalized
: { ...normalized, candidates: [], storageTruncated: true as const };
}
if (value.type === "memory.dream.completed") {
if (
(value.phase !== "light" && value.phase !== "deep" && value.phase !== "rem") ||
(value.outcome !== undefined &&
value.outcome !== "completed" &&
value.outcome !== "failed") ||
(value.error !== undefined && typeof value.error !== "string") ||
(value.inlinePath !== undefined && typeof value.inlinePath !== "string") ||
(value.reportPath !== undefined && typeof value.reportPath !== "string") ||
!isFiniteNumber(value.lineCount) ||
(value.storageMode !== "inline" &&
value.storageMode !== "separate" &&
value.storageMode !== "both")
) {
return null;
}
const error = value.error
? truncateUtf8(value.error, MAX_MEMORY_HOST_EVENT_TEXT_BYTES)
: undefined;
const inlinePath = value.inlinePath
? truncateUtf8(value.inlinePath, MAX_MEMORY_HOST_EVENT_PATH_BYTES)
: undefined;
const reportPath = value.reportPath
? truncateUtf8(value.reportPath, MAX_MEMORY_HOST_EVENT_PATH_BYTES)
: undefined;
truncated ||= Boolean(error?.truncated || inlinePath?.truncated || reportPath?.truncated);
return {
type: value.type,
timestamp: timestamp.value,
phase: value.phase,
...(value.outcome ? { outcome: value.outcome } : {}),
...(error ? { error: error.value } : {}),
...(inlinePath ? { inlinePath: inlinePath.value } : {}),
...(reportPath ? { reportPath: reportPath.value } : {}),
lineCount: value.lineCount,
storageMode: value.storageMode,
...(truncated ? { storageTruncated: true } : {}),
};
}
return null;
}
export function registerMemoryHostEvent(params: {
workspaceDir: string;
event: MemoryHostEventRecord;
env?: NodeJS.ProcessEnv;
}): void {
const event = normalizeMemoryHostEventRecordForStorage(params.event);
if (!event) {
throw new TypeError("Memory host event is invalid");
}
const initialSequence = Math.max(
0,
listStoredMemoryHostEvents({
workspaceDir: params.workspaceDir,
limit: 1,
...(params.env ? { env: params.env } : {}),
}).at(-1)?.value.sequence ?? 0,
);
const recordedAt = Date.now();
registerPluginStateSyncSequencedJournalEntry({
pluginId: MEMORY_HOST_EVENTS_PLUGIN_ID,
cursorOptions: {
namespace: MEMORY_HOST_EVENT_CURSORS_NAMESPACE,
maxEntries: MAX_MEMORY_HOST_EVENT_CURSORS,
...(params.env ? { env: params.env } : {}),
},
cursorKey: cursorKey(params.workspaceDir),
journalOptions: {
namespace: MEMORY_HOST_EVENTS_NAMESPACE,
maxEntries: maxMemoryHostEventsForTests ?? MAX_MEMORY_HOST_EVENTS,
...(params.env ? { env: params.env } : {}),
},
initialSequence,
journalKey: (sequence) => memoryHostEventStorageKey(params.workspaceDir, sequence),
journalValue: (sequence) => ({ kind: "event", event, recordedAt, sequence }),
});
}
export function listStoredMemoryHostEvents(params: {
workspaceDir: string;
limit?: number;
env?: NodeJS.ProcessEnv;
}): PersistedMemoryHostEvent[] {
const limit = Number.isFinite(params.limit)
? Math.max(
1,
Math.min(
maxMemoryHostEventsForTests ?? MAX_MEMORY_HOST_EVENTS,
Math.floor(params.limit as number),
),
)
: (maxMemoryHostEventsForTests ?? MAX_MEMORY_HOST_EVENTS);
const entries = pluginStateEntriesInKeyRange({
pluginId: MEMORY_HOST_EVENTS_PLUGIN_ID,
namespace: MEMORY_HOST_EVENTS_NAMESPACE,
keyStartInclusive: eventKeyPrefix(params.workspaceDir),
keyEndExclusive: eventKeyRangeEnd(params.workspaceDir),
limit,
order: "desc",
...(params.env ? { env: params.env } : {}),
}).flatMap((entry): PersistedMemoryHostEvent[] => {
const value = entry.value as StoredMemoryHostEvent;
return value.kind === "event" ? [{ ...entry, value }] : [];
});
return entries.toReversed();
}
/** Test-only retention override; production keeps a 10,000-event namespace budget. */
export function setMaxMemoryHostEventsForTests(maxEntries?: number): void {
maxMemoryHostEventsForTests = maxEntries;
}
+80
View File
@@ -0,0 +1,80 @@
import type { MemoryDreamingPhaseName } from "./dreaming.js";
type MemoryHostEventStorageMetadata = {
/** True when diagnostic detail was bounded before persistence. Aggregate counts stay exact. */
storageTruncated?: true;
};
/** Event emitted when a recall query records the selected memory snippets. */
export type MemoryHostRecallRecordedEvent = MemoryHostEventStorageMetadata & {
type: "memory.recall.recorded";
timestamp: string;
query: string;
resultCount: number;
results: Array<{
path: string;
startLine: number;
endLine: number;
score: number;
}>;
};
/** Event emitted when recall hits are visible but excluded from short-term promotion. */
export type MemoryHostRecallSkippedEvent = MemoryHostEventStorageMetadata & {
type: "memory.recall.skipped";
timestamp: string;
query: string;
reason: "non-short-term-memory-path";
eligibleResultCount: number;
skippedResultCount: number;
results: Array<{
path: string;
startLine: number;
endLine: number;
score: number;
reason: "non-short-term-memory-path";
}>;
};
/** Event emitted when deep-dream candidates are promoted into durable memory. */
export type MemoryHostPromotionAppliedEvent = MemoryHostEventStorageMetadata & {
type: "memory.promotion.applied";
timestamp: string;
memoryPath: string;
applied: number;
candidates: Array<{
key: string;
path: string;
startLine: number;
endLine: number;
score: number;
recallCount: number;
}>;
};
/** Normalized outcome for a dreaming phase run. */
export type MemoryDreamOutcome = "completed" | "failed";
/** Event emitted after a dreaming phase writes inline memory and/or reports. */
export type MemoryHostDreamCompletedEvent = MemoryHostEventStorageMetadata & {
type: "memory.dream.completed";
timestamp: string;
phase: MemoryDreamingPhaseName;
/** Missing on older event logs; readers should treat absent as "completed". */
outcome?: MemoryDreamOutcome;
/** Error detail when outcome is "failed". */
error?: string;
inlinePath?: string;
reportPath?: string;
lineCount: number;
storageMode: "inline" | "separate" | "both";
};
/** Durable memory host events consumed by status and public-artifact readers. */
export type MemoryHostEvent =
| MemoryHostRecallRecordedEvent
| MemoryHostPromotionAppliedEvent
| MemoryHostDreamCompletedEvent;
/** Full event record schema, including opt-in diagnostic variants. */
export type MemoryHostEventRecord = MemoryHostEvent | MemoryHostRecallSkippedEvent;
+37 -130
View File
@@ -1,150 +1,52 @@
// Memory host event helpers append and read memory host event logs.
import fs from "node:fs/promises";
// Memory host event helpers append and read persisted memory host events.
import path from "node:path";
import { appendRegularFile } from "../infra/fs-safe.js";
import type { MemoryDreamingPhaseName } from "./dreaming.js";
import {
listStoredMemoryHostEvents,
normalizeMemoryHostEventRecordForStorage,
registerMemoryHostEvent,
} from "./event-store.js";
import type { MemoryHostEvent, MemoryHostEventRecord } from "./event-types.js";
/** Workspace-relative JSONL audit log for memory recall, promotion, and dream events. */
export type {
MemoryDreamOutcome,
MemoryHostDreamCompletedEvent,
MemoryHostEvent,
MemoryHostEventRecord,
MemoryHostPromotionAppliedEvent,
MemoryHostRecallRecordedEvent,
MemoryHostRecallSkippedEvent,
} from "./event-types.js";
export { normalizeMemoryHostEventRecordForStorage };
/** Legacy workspace JSONL path retained only for doctor migration discovery. */
export const MEMORY_HOST_EVENT_LOG_RELATIVE_PATH = path.join("memory", ".dreams", "events.jsonl");
/** Event emitted when a recall query records the selected memory snippets. */
export type MemoryHostRecallRecordedEvent = {
type: "memory.recall.recorded";
timestamp: string;
query: string;
resultCount: number;
results: Array<{
path: string;
startLine: number;
endLine: number;
score: number;
}>;
};
/** Event emitted when recall hits are visible but excluded from short-term promotion. */
export type MemoryHostRecallSkippedEvent = {
type: "memory.recall.skipped";
timestamp: string;
query: string;
reason: "non-short-term-memory-path";
eligibleResultCount: number;
skippedResultCount: number;
results: Array<{
path: string;
startLine: number;
endLine: number;
score: number;
reason: "non-short-term-memory-path";
}>;
};
/** Event emitted when deep-dream candidates are promoted into durable memory. */
export type MemoryHostPromotionAppliedEvent = {
type: "memory.promotion.applied";
timestamp: string;
memoryPath: string;
applied: number;
candidates: Array<{
key: string;
path: string;
startLine: number;
endLine: number;
score: number;
recallCount: number;
}>;
};
/** Normalized outcome for a dreaming phase run. */
export type MemoryDreamOutcome = "completed" | "failed";
/** Event emitted after a dreaming phase writes inline memory and/or reports. */
export type MemoryHostDreamCompletedEvent = {
type: "memory.dream.completed";
timestamp: string;
phase: MemoryDreamingPhaseName;
/** Missing on older event logs; readers should treat absent as "completed". */
outcome?: MemoryDreamOutcome;
/** Error detail when outcome is "failed". */
error?: string;
inlinePath?: string;
reportPath?: string;
lineCount: number;
storageMode: "inline" | "separate" | "both";
};
/** Append-only memory host event schema stored as JSONL. */
export type MemoryHostEvent =
| MemoryHostRecallRecordedEvent
| MemoryHostPromotionAppliedEvent
| MemoryHostDreamCompletedEvent;
/** Full event-log record schema, including opt-in diagnostic variants. */
export type MemoryHostEventRecord = MemoryHostEvent | MemoryHostRecallSkippedEvent;
/** Resolve the event log path inside a workspace without touching the filesystem. */
/** Resolve the retired JSONL source path without reading it at runtime. */
export function resolveMemoryHostEventLogPath(workspaceDir: string): string {
return path.join(workspaceDir, MEMORY_HOST_EVENT_LOG_RELATIVE_PATH);
}
/** Append one memory host event, creating the dreams directory with symlink-safe writes. */
/** Append one memory host event to shared SQLite plugin state. */
export async function appendMemoryHostEvent(
workspaceDir: string,
event: MemoryHostEventRecord,
options: { env?: NodeJS.ProcessEnv } = {},
): Promise<void> {
const eventLogPath = resolveMemoryHostEventLogPath(workspaceDir);
await fs.mkdir(path.dirname(eventLogPath), { recursive: true });
await appendRegularFile({
filePath: eventLogPath,
content: `${JSON.stringify(event)}\n`,
rejectSymlinkParents: true,
registerMemoryHostEvent({
workspaceDir,
event,
...(options.env ? { env: options.env } : {}),
});
}
function parseMemoryHostEventRecord(line: string): MemoryHostEventRecord | null {
try {
const record = JSON.parse(line) as MemoryHostEventRecord;
if (
record.type === "memory.recall.recorded" ||
record.type === "memory.recall.skipped" ||
record.type === "memory.promotion.applied" ||
record.type === "memory.dream.completed"
) {
return record;
}
} catch {
// The log is best-effort diagnostics; one malformed line must not hide
// later valid events or break memory status rendering.
}
return null;
}
async function readMemoryHostEventRecordsRaw(params: {
workspaceDir: string;
limit?: number;
env?: NodeJS.ProcessEnv;
}): Promise<MemoryHostEventRecord[]> {
const eventLogPath = resolveMemoryHostEventLogPath(params.workspaceDir);
const raw = await fs.readFile(eventLogPath, "utf8").catch((err: unknown) => {
if ((err as NodeJS.ErrnoException)?.code === "ENOENT") {
return "";
}
throw err;
});
if (!raw.trim()) {
return [];
}
const events = raw
.split("\n")
.map((line) => line.trim())
.filter(Boolean)
.flatMap((line) => {
const record = parseMemoryHostEventRecord(line);
return record ? [record] : [];
});
if (!Number.isFinite(params.limit)) {
return events;
}
const limit = Math.max(0, Math.floor(params.limit as number));
return limit === 0 ? [] : events.slice(-limit);
const events = listStoredMemoryHostEvents(params).map((entry) => entry.value.event);
return applyMemoryHostEventLimit(events, params.limit);
}
function applyMemoryHostEventLimit<T>(events: T[], limit: number | undefined): T[] {
@@ -155,12 +57,16 @@ function applyMemoryHostEventLimit<T>(events: T[], limit: number | undefined): T
return normalizedLimit === 0 ? [] : events.slice(-normalizedLimit);
}
/** Read recent memory host events, ignoring corrupt JSONL lines left by partial writes. */
/** Read recent memory host events, excluding opt-in diagnostic variants. */
export async function readMemoryHostEvents(params: {
workspaceDir: string;
limit?: number;
env?: NodeJS.ProcessEnv;
}): Promise<MemoryHostEvent[]> {
const events = await readMemoryHostEventRecordsRaw({ workspaceDir: params.workspaceDir });
const events = await readMemoryHostEventRecordsRaw({
workspaceDir: params.workspaceDir,
...(params.env ? { env: params.env } : {}),
});
const legacyEvents = events.filter(
(event): event is MemoryHostEvent => event.type !== "memory.recall.skipped",
);
@@ -171,6 +77,7 @@ export async function readMemoryHostEvents(params: {
export async function readMemoryHostEventRecords(params: {
workspaceDir: string;
limit?: number;
env?: NodeJS.ProcessEnv;
}): Promise<MemoryHostEventRecord[]> {
return await readMemoryHostEventRecordsRaw(params);
}
+16 -1
View File
@@ -3,4 +3,19 @@
* `openclaw/plugin-sdk/memory-host-events` instead.
*/
export * from "../memory-host-sdk/events.js";
export {
appendMemoryHostEvent,
MEMORY_HOST_EVENT_LOG_RELATIVE_PATH,
readMemoryHostEventRecords,
readMemoryHostEvents,
resolveMemoryHostEventLogPath,
} from "../memory-host-sdk/events.js";
export type {
MemoryDreamOutcome,
MemoryHostDreamCompletedEvent,
MemoryHostEvent,
MemoryHostEventRecord,
MemoryHostPromotionAppliedEvent,
MemoryHostRecallRecordedEvent,
MemoryHostRecallSkippedEvent,
} from "../memory-host-sdk/events.js";
+873 -18
View File
@@ -1,10 +1,15 @@
/**
* Tests memory host core public artifact discovery and workspace handling.
*/
import { createHash } from "node:crypto";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
createPluginStateKeyedStore,
resetPluginStateStoreForTests,
} from "../plugin-state/plugin-state-store.js";
import {
clearMemoryPluginState,
registerMemoryCapability,
@@ -16,11 +21,14 @@ import {
listMemoryHostPublicArtifacts,
listActiveMemoryPublicArtifacts,
} from "./memory-host-core.js";
import { appendMemoryHostEvent, resolveMemoryHostEventLogPath } from "./memory-host-events.js";
import { appendMemoryHostEvent } from "./memory-host-events.js";
describe("memory-host-core helpers", () => {
afterEach(() => {
clearMemoryPluginState();
resetPluginStateStoreForTests();
vi.unstubAllEnvs();
vi.restoreAllMocks();
});
it("exposes the active memory prompt guidance builder for context engines", () => {
@@ -68,9 +76,429 @@ describe("memory-host-core helpers", () => {
]);
});
it("propagates workspace inspection failures", async () => {
vi.spyOn(fs, "stat").mockRejectedValueOnce(
Object.assign(new Error("permission denied"), { code: "EACCES" }),
);
await expect(
listMemoryHostPublicArtifacts({
cfg: {
agents: {
list: [{ id: "main", default: true, workspace: "/protected/workspace" }],
},
},
}),
).rejects.toMatchObject({ code: "EACCES" });
});
it("lists readable workspaces without requiring workspace-root writes", async () => {
const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-readonly-workspace-"));
const workspaceDir = path.join(fixtureRoot, "workspace");
try {
vi.stubEnv("OPENCLAW_STATE_DIR", path.join(fixtureRoot, "state"));
await fs.mkdir(workspaceDir);
await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), "# Read-only memory\n", "utf8");
await appendMemoryHostEvent(workspaceDir, {
type: "memory.recall.recorded",
timestamp: "2026-05-18T12:00:00.000Z",
query: "read-only",
resultCount: 0,
results: [],
});
await fs.chmod(workspaceDir, 0o500);
await expect(
listMemoryHostPublicArtifacts({
cfg: {
agents: {
list: [{ id: "main", default: true, workspace: workspaceDir }],
},
},
}),
).resolves.toMatchObject([
{
kind: "memory-root",
workspaceDir,
relativePath: "MEMORY.md",
},
]);
} finally {
await fs.chmod(workspaceDir, 0o700).catch(() => undefined);
await fs.rm(fixtureRoot, { recursive: true, force: true });
}
});
it.runIf(process.platform !== "win32")(
"propagates failures reading an existing event export",
async () => {
const fixtureRoot = await fs.mkdtemp(
path.join(os.tmpdir(), "memory-host-unreadable-export-"),
);
const workspaceDir = path.join(fixtureRoot, "workspace");
let eventExportPath: string | undefined;
try {
vi.stubEnv("OPENCLAW_STATE_DIR", path.join(fixtureRoot, "state"));
await fs.mkdir(workspaceDir);
await appendMemoryHostEvent(workspaceDir, {
type: "memory.recall.recorded",
timestamp: "2026-05-18T12:00:00.000Z",
query: "unreadable export",
resultCount: 0,
results: [],
});
const firstListing = await listMemoryHostPublicArtifacts({
cfg: {
agents: {
list: [{ id: "main", default: true, workspace: workspaceDir }],
},
},
});
eventExportPath = firstListing.find(
(artifact) => artifact.kind === "event-log",
)?.absolutePath;
if (!eventExportPath) {
throw new Error("expected memory event export");
}
await fs.chmod(eventExportPath, 0o000);
await expect(
listMemoryHostPublicArtifacts({
cfg: {
agents: {
list: [{ id: "main", default: true, workspace: workspaceDir }],
},
},
}),
).rejects.toThrow();
} finally {
if (eventExportPath) {
await fs.chmod(eventExportPath, 0o600).catch(() => undefined);
}
await fs.rm(fixtureRoot, { recursive: true, force: true });
}
},
);
it.runIf(process.platform !== "win32")(
"does not delete an event export through a symlinked parent",
async () => {
const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-export-symlink-"));
const workspaceDir = path.join(fixtureRoot, "workspace");
const externalMemoryDir = path.join(fixtureRoot, "external-memory");
const stateDir = path.join(fixtureRoot, "state");
try {
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
await fs.mkdir(workspaceDir);
await fs.mkdir(stateDir);
const stateHash = createHash("sha256")
.update(await fs.realpath(stateDir))
.digest("hex")
.slice(0, 32);
const externalExport = path.join(
externalMemoryDir,
"events",
stateHash,
"memory-host-events.jsonl",
);
const externalOwner = path.join(
path.dirname(externalExport),
".openclaw-memory-host-events-owner.json",
);
await fs.mkdir(path.dirname(externalExport), { recursive: true });
await fs.writeFile(externalExport, '{"type":"external"}\n', "utf8");
await fs.writeFile(externalOwner, '{"kind":"external"}\n', "utf8");
await fs.symlink(externalMemoryDir, path.join(workspaceDir, "memory"));
await expect(
listMemoryHostPublicArtifacts({
cfg: {
agents: {
list: [{ id: "main", default: true, workspace: workspaceDir }],
},
},
}),
).resolves.toEqual([]);
await expect(fs.readFile(externalExport, "utf8")).resolves.toBe('{"type":"external"}\n');
await expect(fs.readFile(externalOwner, "utf8")).resolves.toBe('{"kind":"external"}\n');
} finally {
await fs.rm(fixtureRoot, { recursive: true, force: true });
}
},
);
it("does not replace or delete an unowned event export path", async () => {
const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-unowned-export-"));
const workspaceDir = path.join(fixtureRoot, "workspace");
try {
vi.stubEnv("OPENCLAW_STATE_DIR", fixtureRoot);
await fs.mkdir(workspaceDir);
await appendMemoryHostEvent(workspaceDir, {
type: "memory.recall.recorded",
timestamp: "2026-05-18T12:00:00.000Z",
query: "must not replace user file",
resultCount: 0,
results: [],
});
const stateHash = createHash("sha256")
.update(await fs.realpath(fixtureRoot))
.digest("hex")
.slice(0, 32);
const exportPath = path.join(
workspaceDir,
"memory",
"events",
stateHash,
"memory-host-events.jsonl",
);
await fs.mkdir(path.dirname(exportPath), { recursive: true });
await fs.writeFile(exportPath, '{"owner":"user"}\n', "utf8");
const listed = await listMemoryHostPublicArtifacts({
cfg: { agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] } },
});
expect(listed.some((artifact) => artifact.kind === "event-log")).toBe(false);
await expect(fs.readFile(exportPath, "utf8")).resolves.toBe('{"owner":"user"}\n');
await createPluginStateKeyedStore("memory-core", {
namespace: "memory-host.events",
maxEntries: 10_000,
env: { ...process.env, OPENCLAW_STATE_DIR: fixtureRoot },
}).clear();
await listMemoryHostPublicArtifacts({
cfg: { agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] } },
});
await expect(fs.readFile(exportPath, "utf8")).resolves.toBe('{"owner":"user"}\n');
} finally {
await fs.rm(fixtureRoot, { recursive: true, force: true });
}
});
it("does not let an orphaned owner marker authorize a later workspace file", async () => {
const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-orphan-owner-"));
const stateDir = path.join(fixtureRoot, "state");
const workspaceDir = path.join(fixtureRoot, "workspace");
try {
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
await fs.mkdir(workspaceDir);
await appendMemoryHostEvent(workspaceDir, {
type: "memory.recall.recorded",
timestamp: "2026-05-18T12:00:00.000Z",
query: "expected export",
resultCount: 0,
results: [],
});
const stateHash = createHash("sha256")
.update(await fs.realpath(stateDir))
.digest("hex")
.slice(0, 32);
const workspaceHash = createHash("sha256")
.update(await fs.realpath(workspaceDir))
.digest("hex")
.slice(0, 32);
const exportDir = path.join(workspaceDir, "memory", "events", stateHash);
const exportPath = path.join(exportDir, "memory-host-events.jsonl");
const ownerPath = path.join(exportDir, ".openclaw-memory-host-events-owner.json");
const expectedContent = `${JSON.stringify({
type: "memory.recall.recorded",
timestamp: "2026-05-18T12:00:00.000Z",
query: "expected export",
resultCount: 0,
results: [],
})}\n`;
await fs.mkdir(exportDir, { recursive: true });
await fs.writeFile(
ownerPath,
`${JSON.stringify({
schemaVersion: 3,
kind: "openclaw-memory-host-events-export",
stateHash,
workspaceHash,
pendingContentSha256: createHash("sha256").update(expectedContent).digest("hex"),
})}\n`,
"utf8",
);
await fs.writeFile(exportPath, '{"owner":"user after crash"}\n', "utf8");
const listed = await listMemoryHostPublicArtifacts({
cfg: { agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] } },
});
expect(listed.some((artifact) => artifact.kind === "event-log")).toBe(false);
await expect(fs.readFile(exportPath, "utf8")).resolves.toBe('{"owner":"user after crash"}\n');
} finally {
await fs.rm(fixtureRoot, { recursive: true, force: true });
}
});
it("does not claim a same-content inode that replaces an exclusive export", async () => {
const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-create-replace-"));
const workspaceDir = path.join(fixtureRoot, "workspace");
const cfg = {
agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] },
};
const event = {
type: "memory.recall.recorded" as const,
timestamp: "2026-05-18T12:00:00.000Z",
query: "exclusive create",
resultCount: 0,
results: [],
};
const expectedContent = `${JSON.stringify(event)}\n`;
const originalOpen = fs.open.bind(fs);
let exportOpenCount = 0;
try {
vi.stubEnv("OPENCLAW_STATE_DIR", fixtureRoot);
await fs.mkdir(workspaceDir, { recursive: true });
await appendMemoryHostEvent(workspaceDir, event);
const stateHash = createHash("sha256")
.update(await fs.realpath(fixtureRoot))
.digest("hex")
.slice(0, 32);
const exportPath = path.join(
workspaceDir,
"memory",
"events",
stateHash,
"memory-host-events.jsonl",
);
vi.spyOn(fs, "open").mockImplementation(async (...args: Parameters<typeof fs.open>) => {
const target = args[0];
if (typeof target === "string" && path.resolve(target) === path.resolve(exportPath)) {
exportOpenCount += 1;
if (exportOpenCount === 4) {
await fs.rename(exportPath, `${exportPath}.openclaw-created`);
const replacement = await originalOpen(exportPath, "wx", 0o600);
try {
await replacement.writeFile(expectedContent, "utf8");
} finally {
await replacement.close();
}
}
}
return await originalOpen(...args);
});
const racedArtifacts = await listMemoryHostPublicArtifacts({ cfg });
expect(racedArtifacts.some((artifact) => artifact.kind === "event-log")).toBe(false);
await expect(fs.readFile(exportPath, "utf8")).resolves.toBe(expectedContent);
vi.restoreAllMocks();
await appendMemoryHostEvent(workspaceDir, { ...event, query: "must stay foreign" });
const retriedArtifacts = await listMemoryHostPublicArtifacts({ cfg });
expect(retriedArtifacts.some((artifact) => artifact.kind === "event-log")).toBe(false);
await expect(fs.readFile(exportPath, "utf8")).resolves.toBe(expectedContent);
} finally {
await fs.rm(fixtureRoot, { recursive: true, force: true });
}
});
it("does not claim a hash-only pending event export after interruption", async () => {
const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-pending-owner-"));
const stateDir = path.join(fixtureRoot, "state");
const workspaceDir = path.join(fixtureRoot, "workspace");
const firstEvent = {
type: "memory.recall.recorded" as const,
timestamp: "2026-05-18T12:00:00.000Z",
query: "pending export",
resultCount: 0,
results: [],
};
try {
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
await fs.mkdir(workspaceDir);
await appendMemoryHostEvent(workspaceDir, firstEvent);
const stateHash = createHash("sha256")
.update(await fs.realpath(stateDir))
.digest("hex")
.slice(0, 32);
const workspaceHash = createHash("sha256")
.update(await fs.realpath(workspaceDir))
.digest("hex")
.slice(0, 32);
const exportDir = path.join(workspaceDir, "memory", "events", stateHash);
const exportPath = path.join(exportDir, "memory-host-events.jsonl");
const ownerPath = path.join(exportDir, ".openclaw-memory-host-events-owner.json");
const pendingContent = `${JSON.stringify(firstEvent)}\n`;
await fs.mkdir(exportDir, { recursive: true });
await fs.writeFile(exportPath, pendingContent, "utf8");
await fs.writeFile(
ownerPath,
`${JSON.stringify({
schemaVersion: 3,
kind: "openclaw-memory-host-events-export",
stateHash,
workspaceHash,
pendingContentSha256: createHash("sha256").update(pendingContent).digest("hex"),
})}\n`,
"utf8",
);
await appendMemoryHostEvent(workspaceDir, {
...firstEvent,
timestamp: "2026-05-18T12:01:00.000Z",
query: "after interruption",
});
const listed = await listMemoryHostPublicArtifacts({
cfg: { agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] } },
});
expect(listed.some((artifact) => artifact.kind === "event-log")).toBe(false);
await expect(fs.readFile(exportPath, "utf8")).resolves.toBe(pendingContent);
const owner = JSON.parse(await fs.readFile(ownerPath, "utf8")) as {
contentSha256?: string;
fileDev?: string;
fileIno?: string;
pendingContentSha256?: string;
};
expect(owner.pendingContentSha256).toBe(
createHash("sha256")
.update(await fs.readFile(exportPath))
.digest("hex"),
);
expect(owner.contentSha256).toBeUndefined();
expect(owner.fileDev).toBeUndefined();
expect(owner.fileIno).toBeUndefined();
} finally {
await fs.rm(fixtureRoot, { recursive: true, force: true });
}
});
it("omits the event export when a workspace path component is a file", async () => {
const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-blocked-export-"));
const workspaceDir = path.join(fixtureRoot, "workspace");
try {
vi.stubEnv("OPENCLAW_STATE_DIR", path.join(fixtureRoot, "state"));
await fs.mkdir(workspaceDir);
await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), "# Still visible\n", "utf8");
await fs.writeFile(path.join(workspaceDir, "memory"), "user file\n", "utf8");
await appendMemoryHostEvent(workspaceDir, {
type: "memory.recall.recorded",
timestamp: "2026-05-18T12:00:00.000Z",
query: "blocked export",
resultCount: 0,
results: [],
});
await expect(
listMemoryHostPublicArtifacts({
cfg: { agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] } },
}),
).resolves.toEqual([
expect.objectContaining({ kind: "memory-root", relativePath: "MEMORY.md" }),
]);
await expect(fs.readFile(path.join(workspaceDir, "memory"), "utf8")).resolves.toBe(
"user file\n",
);
} finally {
await fs.rm(fixtureRoot, { recursive: true, force: true });
}
});
it("lists shared public artifacts from memory workspaces", async () => {
const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-public-artifacts-"));
try {
vi.stubEnv("OPENCLAW_STATE_DIR", fixtureRoot);
const workspaceDir = path.join(fixtureRoot, "workspace");
await fs.mkdir(path.join(workspaceDir, "memory", "dreaming"), { recursive: true });
await fs.writeFile(path.join(workspaceDir, "MEMORY.md"), "# Durable Memory\n", "utf8");
@@ -84,6 +512,8 @@ describe("memory-host-core helpers", () => {
"# Dream Report\n",
"utf8",
);
const eventStoredAt = Date.parse("2026-05-19T09:30:00.000Z");
vi.spyOn(Date, "now").mockReturnValue(eventStoredAt);
await appendMemoryHostEvent(workspaceDir, {
type: "memory.recall.recorded",
timestamp: "2026-05-18T12:00:00.000Z",
@@ -92,15 +522,18 @@ describe("memory-host-core helpers", () => {
results: [],
});
await expect(
listMemoryHostPublicArtifacts({
cfg: {
agents: {
list: [{ id: "main", default: true, workspace: workspaceDir }],
},
const artifacts = await listMemoryHostPublicArtifacts({
cfg: {
agents: {
list: [{ id: "main", default: true, workspace: workspaceDir }],
},
}),
).resolves.toEqual([
},
});
const eventArtifact = artifacts.find((artifact) => artifact.kind === "event-log");
if (!eventArtifact) {
throw new Error("expected memory event export");
}
expect(artifacts.filter((artifact) => artifact.kind !== "event-log")).toEqual([
{
kind: "memory-root",
workspaceDir,
@@ -125,15 +558,437 @@ describe("memory-host-core helpers", () => {
agentIds: ["main"],
contentType: "markdown",
},
{
kind: "event-log",
workspaceDir,
relativePath: "memory/.dreams/events.jsonl",
absolutePath: resolveMemoryHostEventLogPath(workspaceDir),
agentIds: ["main"],
contentType: "json",
},
]);
expect(eventArtifact).toMatchObject({
kind: "event-log",
workspaceDir,
agentIds: ["main"],
contentType: "json",
});
expect(eventArtifact.relativePath).toMatch(
/^memory\/events\/[a-f0-9]{32}\/memory-host-events\.jsonl$/u,
);
expect(eventArtifact.absolutePath).toBe(
path.join(workspaceDir, ...eventArtifact.relativePath.split("/")),
);
const eventExportPath = eventArtifact.absolutePath;
await expect(fs.readFile(eventExportPath, "utf8")).resolves.toBe(
`${JSON.stringify({
type: "memory.recall.recorded",
timestamp: "2026-05-18T12:00:00.000Z",
query: "bridge",
resultCount: 0,
results: [],
})}\n`,
);
const exportStat = await fs.stat(eventExportPath);
const exportOwner = JSON.parse(
await fs.readFile(
path.join(path.dirname(eventExportPath), ".openclaw-memory-host-events-owner.json"),
"utf8",
),
) as { fileDev?: string; fileIno?: string };
expect(exportOwner.fileDev).toBe(String(exportStat.dev));
expect(exportOwner.fileIno).toBe(String(exportStat.ino));
await expect(
fs.access(path.join(workspaceDir, ".memory-host-events-export.lock")),
).rejects.toMatchObject({ code: "ENOENT" });
expect(
(await fs.readdir(fixtureRoot)).some((entry) =>
entry.startsWith(".memory-host-events-export-"),
),
).toBe(false);
await createPluginStateKeyedStore("memory-core", {
namespace: "memory-host.events",
maxEntries: 10_000,
env: { ...process.env, OPENCLAW_STATE_DIR: fixtureRoot },
}).clear();
const afterRetention = await listMemoryHostPublicArtifacts({
cfg: {
agents: {
list: [{ id: "main", default: true, workspace: workspaceDir }],
},
},
});
expect(afterRetention.some((artifact) => artifact.kind === "event-log")).toBe(false);
await expect(fs.readFile(eventExportPath, "utf8")).resolves.toBe("");
} finally {
await fs.rm(fixtureRoot, { recursive: true, force: true });
}
});
it.runIf(process.platform !== "win32")(
"does not let a workspace alias overwrite a newer event export",
async () => {
const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-export-race-"));
const workspaceDir = path.join(fixtureRoot, "workspace");
const workspaceAlias = path.join(fixtureRoot, "workspace-alias");
const cfg = {
agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] },
};
const aliasCfg = {
agents: { list: [{ id: "main", default: true, workspace: workspaceAlias }] },
};
const originalOpen = fs.open.bind(fs);
let releaseFirstRead: (() => void) | undefined;
let signalFirstRead: (() => void) | undefined;
let shouldBlockFirstRead = true;
const firstReadStarted = new Promise<void>((resolve) => {
signalFirstRead = resolve;
});
const openSpy = vi
.spyOn(fs, "open")
.mockImplementation(async (...args: Parameters<typeof fs.open>) => {
const target = args[0];
if (
shouldBlockFirstRead &&
typeof target === "string" &&
path.basename(target) === "memory-host-events.jsonl" &&
path.resolve(target).startsWith(`${path.resolve(workspaceDir)}${path.sep}`)
) {
shouldBlockFirstRead = false;
signalFirstRead?.();
await new Promise<void>((resolve) => {
releaseFirstRead = resolve;
});
}
return await originalOpen(...args);
});
try {
vi.stubEnv("OPENCLAW_STATE_DIR", fixtureRoot);
await fs.mkdir(workspaceDir, { recursive: true });
await fs.symlink(workspaceDir, workspaceAlias);
await appendMemoryHostEvent(workspaceAlias, {
type: "memory.recall.recorded",
timestamp: "2026-05-18T12:00:00.000Z",
query: "older",
resultCount: 0,
results: [],
});
const olderListing = listMemoryHostPublicArtifacts({ cfg });
await firstReadStarted;
await appendMemoryHostEvent(workspaceDir, {
type: "memory.recall.recorded",
timestamp: "2026-05-18T12:01:00.000Z",
query: "newer",
resultCount: 0,
results: [],
});
const newerListing = listMemoryHostPublicArtifacts({ cfg: aliasCfg });
await Promise.race([
newerListing,
new Promise<void>((resolve) => {
setTimeout(resolve, 250);
}),
]);
releaseFirstRead?.();
const [olderArtifacts, newerArtifacts] = await Promise.all([olderListing, newerListing]);
const olderExport = olderArtifacts.find((artifact) => artifact.kind === "event-log");
const newerExport = newerArtifacts.find((artifact) => artifact.kind === "event-log");
expect(olderExport?.absolutePath).toBe(newerExport?.absolutePath);
const eventExportPath = newerExport?.absolutePath;
if (!eventExportPath) {
throw new Error("expected memory event export");
}
const exported = (await fs.readFile(eventExportPath, "utf8"))
.trim()
.split("\n")
.map((line) => JSON.parse(line) as { query: string });
expect(exported.map((event) => event.query)).toEqual(["older", "newer"]);
} finally {
releaseFirstRead?.();
openSpy.mockRestore();
await fs.rm(fixtureRoot, { recursive: true, force: true });
}
},
);
it.each([
{ name: "refresh", clearEvents: false },
{ name: "retirement", clearEvents: true },
])("preserves a replacement installed during event export $name", async ({ clearEvents }) => {
const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-export-replace-"));
const workspaceDir = path.join(fixtureRoot, "workspace");
const cfg = {
agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] },
};
const replacement = '{"owner":"workspace"}\n';
const originalOpen = fs.open.bind(fs);
let exportOpenCount = 0;
let eventExportPath: string | undefined;
try {
vi.stubEnv("OPENCLAW_STATE_DIR", fixtureRoot);
await fs.mkdir(workspaceDir, { recursive: true });
await appendMemoryHostEvent(workspaceDir, {
type: "memory.recall.recorded",
timestamp: "2026-05-18T12:00:00.000Z",
query: "first",
resultCount: 0,
results: [],
});
eventExportPath = (await listMemoryHostPublicArtifacts({ cfg })).find(
(artifact) => artifact.kind === "event-log",
)?.absolutePath;
if (!eventExportPath) {
throw new Error("expected memory event export");
}
if (clearEvents) {
await createPluginStateKeyedStore("memory-core", {
namespace: "memory-host.events",
maxEntries: 10_000,
env: { ...process.env, OPENCLAW_STATE_DIR: fixtureRoot },
}).clear();
} else {
await appendMemoryHostEvent(workspaceDir, {
type: "memory.recall.recorded",
timestamp: "2026-05-18T12:01:00.000Z",
query: "second",
resultCount: 0,
results: [],
});
}
const expectedExportPath = path.resolve(eventExportPath);
vi.spyOn(fs, "open").mockImplementation(async (...args: Parameters<typeof fs.open>) => {
const target = args[0];
if (typeof target === "string" && path.resolve(target) === expectedExportPath) {
exportOpenCount += 1;
if (exportOpenCount === 2) {
await fs.rename(expectedExportPath, `${expectedExportPath}.openclaw-owned`);
await fs.writeFile(expectedExportPath, replacement, "utf8");
}
}
return await originalOpen(...args);
});
const artifacts = await listMemoryHostPublicArtifacts({ cfg });
expect(artifacts.some((artifact) => artifact.kind === "event-log")).toBe(false);
await expect(fs.readFile(expectedExportPath, "utf8")).resolves.toBe(replacement);
} finally {
await fs.rm(fixtureRoot, { recursive: true, force: true });
}
});
it.each([
{ name: "refresh", clearEvents: false },
{ name: "retirement", clearEvents: true },
])("keeps ownership through same-inode event export $name", async ({ clearEvents }) => {
const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-export-inode-"));
const workspaceDir = path.join(fixtureRoot, "workspace");
const cfg = {
agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] },
};
const originalOpen = fs.open.bind(fs);
let exportOpenCount = 0;
try {
vi.stubEnv("OPENCLAW_STATE_DIR", fixtureRoot);
await fs.mkdir(workspaceDir, { recursive: true });
await appendMemoryHostEvent(workspaceDir, {
type: "memory.recall.recorded",
timestamp: "2026-05-18T12:00:00.000Z",
query: "first",
resultCount: 0,
results: [],
});
const eventExportPath = (await listMemoryHostPublicArtifacts({ cfg })).find(
(artifact) => artifact.kind === "event-log",
)?.absolutePath;
if (!eventExportPath) {
throw new Error("expected memory event export");
}
if (clearEvents) {
await createPluginStateKeyedStore("memory-core", {
namespace: "memory-host.events",
maxEntries: 10_000,
env: { ...process.env, OPENCLAW_STATE_DIR: fixtureRoot },
}).clear();
} else {
await appendMemoryHostEvent(workspaceDir, {
type: "memory.recall.recorded",
timestamp: "2026-05-18T12:01:00.000Z",
query: "second",
resultCount: 0,
results: [],
});
}
const expectedExportPath = path.resolve(eventExportPath);
const originalStat = await fs.stat(expectedExportPath);
vi.spyOn(fs, "open").mockImplementation(async (...args: Parameters<typeof fs.open>) => {
const target = args[0];
if (typeof target === "string" && path.resolve(target) === expectedExportPath) {
exportOpenCount += 1;
if (exportOpenCount === 2) {
const writer = await originalOpen(expectedExportPath, "w");
try {
await writer.writeFile('{"owner":"same inode"}\n', "utf8");
} finally {
await writer.close();
}
}
}
return await originalOpen(...args);
});
const artifacts = await listMemoryHostPublicArtifacts({ cfg });
expect((await fs.stat(expectedExportPath)).ino).toBe(originalStat.ino);
if (clearEvents) {
expect(artifacts.some((artifact) => artifact.kind === "event-log")).toBe(false);
await expect(fs.readFile(expectedExportPath, "utf8")).resolves.toBe("");
} else {
expect(artifacts.some((artifact) => artifact.kind === "event-log")).toBe(true);
await expect(fs.readFile(expectedExportPath, "utf8")).resolves.toContain("second");
}
} finally {
await fs.rm(fixtureRoot, { recursive: true, force: true });
}
});
it("retries an owned event export after a same-inode post-write race", async () => {
const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-export-retry-"));
const workspaceDir = path.join(fixtureRoot, "workspace");
const cfg = {
agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] },
};
const originalOpen = fs.open.bind(fs);
let exportOpenCount = 0;
try {
vi.stubEnv("OPENCLAW_STATE_DIR", fixtureRoot);
await fs.mkdir(workspaceDir, { recursive: true });
await appendMemoryHostEvent(workspaceDir, {
type: "memory.recall.recorded",
timestamp: "2026-05-18T12:00:00.000Z",
query: "first",
resultCount: 0,
results: [],
});
const eventExportPath = (await listMemoryHostPublicArtifacts({ cfg })).find(
(artifact) => artifact.kind === "event-log",
)?.absolutePath;
if (!eventExportPath) {
throw new Error("expected memory event export");
}
await appendMemoryHostEvent(workspaceDir, {
type: "memory.recall.recorded",
timestamp: "2026-05-18T12:01:00.000Z",
query: "second",
resultCount: 0,
results: [],
});
const expectedExportPath = path.resolve(eventExportPath);
const openSpy = vi
.spyOn(fs, "open")
.mockImplementation(async (...args: Parameters<typeof fs.open>) => {
const target = args[0];
if (typeof target === "string" && path.resolve(target) === expectedExportPath) {
exportOpenCount += 1;
if (exportOpenCount === 4) {
const writer = await originalOpen(expectedExportPath, "w");
try {
await writer.writeFile('{"owner":"post-write race"}\n', "utf8");
} finally {
await writer.close();
}
}
}
return await originalOpen(...args);
});
const racedArtifacts = await listMemoryHostPublicArtifacts({ cfg });
expect(racedArtifacts.some((artifact) => artifact.kind === "event-log")).toBe(false);
openSpy.mockRestore();
const retriedArtifacts = await listMemoryHostPublicArtifacts({ cfg });
expect(retriedArtifacts.some((artifact) => artifact.kind === "event-log")).toBe(true);
await expect(fs.readFile(expectedExportPath, "utf8")).resolves.toContain("second");
} finally {
await fs.rm(fixtureRoot, { recursive: true, force: true });
}
});
it("repairs an oversized owned event export by inode", async () => {
const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-export-oversized-"));
const workspaceDir = path.join(fixtureRoot, "workspace");
const cfg = {
agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] },
};
try {
vi.stubEnv("OPENCLAW_STATE_DIR", fixtureRoot);
await fs.mkdir(workspaceDir, { recursive: true });
await appendMemoryHostEvent(workspaceDir, {
type: "memory.recall.recorded",
timestamp: "2026-05-18T12:00:00.000Z",
query: "first",
resultCount: 0,
results: [],
});
const eventExportPath = (await listMemoryHostPublicArtifacts({ cfg })).find(
(artifact) => artifact.kind === "event-log",
)?.absolutePath;
if (!eventExportPath) {
throw new Error("expected memory event export");
}
const originalStat = await fs.stat(eventExportPath);
await fs.appendFile(eventExportPath, "x".repeat(1024 * 1024 + 1), "utf8");
await appendMemoryHostEvent(workspaceDir, {
type: "memory.recall.recorded",
timestamp: "2026-05-18T12:01:00.000Z",
query: "second",
resultCount: 0,
results: [],
});
const artifacts = await listMemoryHostPublicArtifacts({ cfg });
expect(artifacts.some((artifact) => artifact.kind === "event-log")).toBe(true);
expect((await fs.stat(eventExportPath)).ino).toBe(originalStat.ino);
expect((await fs.stat(eventExportPath)).size).toBeLessThan(1024 * 1024);
await expect(fs.readFile(eventExportPath, "utf8")).resolves.toContain("second");
} finally {
await fs.rm(fixtureRoot, { recursive: true, force: true });
}
});
it("keeps public event exports isolated across state directories", async () => {
const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-export-owner-"));
const workspaceDir = path.join(fixtureRoot, "workspace");
const cfg = {
agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] },
};
try {
await fs.mkdir(workspaceDir, { recursive: true });
const exports: string[] = [];
for (const profile of ["profile-a", "profile-b"]) {
vi.stubEnv("OPENCLAW_STATE_DIR", path.join(fixtureRoot, profile));
await appendMemoryHostEvent(workspaceDir, {
type: "memory.recall.recorded",
timestamp: "2026-05-18T12:00:00.000Z",
query: profile,
resultCount: 0,
results: [],
});
const artifact = (await listMemoryHostPublicArtifacts({ cfg })).find(
(candidate) => candidate.kind === "event-log",
);
if (!artifact) {
throw new Error("expected memory event export");
}
exports.push(artifact.absolutePath);
await expect(fs.readFile(artifact.absolutePath, "utf8")).resolves.toContain(profile);
}
expect(new Set(exports).size).toBe(2);
const [profileAExport, profileBExport] = exports;
if (!profileAExport || !profileBExport) {
throw new Error("expected state-qualified memory event exports");
}
await expect(fs.readFile(profileAExport, "utf8")).resolves.toContain("profile-a");
await expect(fs.readFile(profileBExport, "utf8")).resolves.toContain("profile-b");
} finally {
await fs.rm(fixtureRoot, { recursive: true, force: true });
}
+415 -14
View File
@@ -4,9 +4,208 @@
import fs from "node:fs/promises";
import path from "node:path";
import type { OpenClawConfig } from "../config/config.js";
import { resolveStateDir } from "../config/paths.js";
import { sha256Hex, sha256HexPrefix } from "../infra/crypto-digest.js";
import { withFileLock } from "../infra/file-lock.js";
import { sameFileIdentity, type FileIdentityStat } from "../infra/fs-safe-advanced.js";
import { FsSafeError, root as createFsSafeRoot } from "../infra/fs-safe.js";
import { syncDirectoryBestEffort } from "../infra/sqlite-snapshot.js";
import {
MAX_MEMORY_HOST_PUBLIC_EXPORT_BYTES,
serializeMemoryHostEventExport,
} from "../memory-host-sdk/event-export.js";
import { listStoredMemoryHostEvents } from "../memory-host-sdk/event-store.js";
import type { MemoryPluginPublicArtifact } from "../plugins/memory-state.js";
import { KeyedAsyncQueue } from "./keyed-async-queue.js";
import { resolveMemoryDreamingWorkspaces } from "./memory-core-host-status.js";
import { resolveMemoryHostEventLogPath } from "./memory-host-events.js";
import {
isMemoryHostEventArtifactAtIdentity,
isMissingPathError,
isRejectedWorkspaceArtifactPath,
memoryHostEventExportOwnerContent,
publishMemoryHostEventArtifact,
rewriteMemoryHostEventArtifactIfUnchanged,
} from "./memory-host-event-export.js";
const MEMORY_HOST_EVENTS_FILENAME = "memory-host-events.jsonl";
const MEMORY_HOST_EVENTS_OWNER_FILENAME = ".openclaw-memory-host-events-owner.json";
const MAX_MEMORY_HOST_PUBLIC_EXPORT_EVENTS = 1_000;
const MEMORY_HOST_EVENT_EXPORT_LOCK_OPTIONS = {
retries: { retries: 20, factor: 1.3, minTimeout: 25, maxTimeout: 250, randomize: true },
stale: 30_000,
} as const;
const memoryHostEventExportQueue = new KeyedAsyncQueue();
function isWorkspaceWriteUnavailable(error: unknown, seen = new Set<unknown>()): boolean {
if (!error || typeof error !== "object" || seen.has(error)) {
return false;
}
seen.add(error);
const code = (error as { code?: unknown }).code;
if (
code === "EACCES" ||
code === "EEXIST" ||
code === "ENOTDIR" ||
code === "EPERM" ||
code === "EROFS" ||
(error instanceof FsSafeError && (code === "not-file" || code === "not-removable"))
) {
return true;
}
if (error instanceof FsSafeError && error.category === "policy" && code !== "invalid-path") {
return false;
}
return isWorkspaceWriteUnavailable((error as { cause?: unknown }).cause, seen);
}
async function resolveMemoryHostEventExportOwner(workspaceDir: string): Promise<{
queueKey: string;
lockTarget: string;
relativePath: string;
ownerRelativePath: string;
stateHash: string;
workspaceHash: string;
}> {
const requestedStateDir = path.resolve(resolveStateDir());
await fs.mkdir(requestedStateDir, { recursive: true, mode: 0o700 });
const stateDir = await fs.realpath(requestedStateDir);
const stateHash = sha256HexPrefix(stateDir, 32);
const workspaceHash = sha256HexPrefix(path.resolve(workspaceDir), 32);
const exportDirectory = path.posix.join("memory", "events", stateHash);
return {
queueKey: `${stateHash}\0${workspaceHash}`,
lockTarget: path.join(stateDir, `.memory-host-events-export-${workspaceHash}`),
relativePath: path.posix.join(exportDirectory, MEMORY_HOST_EVENTS_FILENAME),
ownerRelativePath: path.posix.join(exportDirectory, MEMORY_HOST_EVENTS_OWNER_FILENAME),
stateHash,
workspaceHash,
};
}
async function readMemoryHostEventExportOwnership(
workspaceRoot: Awaited<ReturnType<typeof createFsSafeRoot>>,
owner: Awaited<ReturnType<typeof resolveMemoryHostEventExportOwner>>,
): Promise<
| {
kind: "owned";
content: string | undefined;
identity: FileIdentityStat;
ownerContent: string;
needsFinalize: boolean;
}
| { kind: "missing" }
| { kind: "orphan"; ownerContent: string }
| { kind: "pending-missing"; ownerContent: string }
| { kind: "foreign" }
> {
const content = await workspaceRoot.readText(owner.ownerRelativePath).catch((error: unknown) => {
if (isMissingPathError(error)) {
return undefined;
}
if (isRejectedWorkspaceArtifactPath(error)) {
return null;
}
throw error;
});
if (content === null) {
return { kind: "foreign" };
}
if (content === undefined) {
return { kind: "missing" };
}
let parsed: unknown;
try {
parsed = JSON.parse(content) as unknown;
} catch {
return { kind: "foreign" };
}
if (
!parsed ||
typeof parsed !== "object" ||
Array.isArray(parsed) ||
(parsed as { schemaVersion?: unknown }).schemaVersion !== 3 ||
(parsed as { kind?: unknown }).kind !== "openclaw-memory-host-events-export" ||
(parsed as { stateHash?: unknown }).stateHash !== owner.stateHash ||
(parsed as { workspaceHash?: unknown }).workspaceHash !== owner.workspaceHash ||
((parsed as { contentSha256?: unknown }).contentSha256 !== undefined &&
typeof (parsed as { contentSha256?: unknown }).contentSha256 !== "string") ||
((parsed as { pendingContentSha256?: unknown }).pendingContentSha256 !== undefined &&
typeof (parsed as { pendingContentSha256?: unknown }).pendingContentSha256 !== "string") ||
((parsed as { contentSha256?: unknown }).contentSha256 === undefined &&
(parsed as { pendingContentSha256?: unknown }).pendingContentSha256 === undefined) ||
((parsed as { fileDev?: unknown }).fileDev === undefined) !==
((parsed as { fileIno?: unknown }).fileIno === undefined) ||
((parsed as { fileDev?: unknown }).fileDev !== undefined &&
(typeof (parsed as { fileDev?: unknown }).fileDev !== "string" ||
!/^\d+$/u.test((parsed as { fileDev: string }).fileDev) ||
typeof (parsed as { fileIno?: unknown }).fileIno !== "string" ||
!/^\d+$/u.test((parsed as { fileIno: string }).fileIno)))
) {
return { kind: "foreign" };
}
const storedIdentity =
typeof (parsed as { fileDev?: unknown }).fileDev === "string" &&
typeof (parsed as { fileIno?: unknown }).fileIno === "string"
? {
dev: BigInt((parsed as { fileDev: string }).fileDev),
ino: BigInt((parsed as { fileIno: string }).fileIno),
}
: undefined;
let openedExport: Awaited<ReturnType<typeof workspaceRoot.open>> | undefined;
try {
openedExport = await workspaceRoot.open(owner.relativePath);
} catch (error) {
if (isMissingPathError(error)) {
openedExport = undefined;
} else if (isRejectedWorkspaceArtifactPath(error)) {
return { kind: "foreign" };
} else {
throw error;
}
}
if (!openedExport) {
return typeof (parsed as { pendingContentSha256?: unknown }).pendingContentSha256 === "string"
? { kind: "pending-missing", ownerContent: content }
: { kind: "orphan", ownerContent: content };
}
let exportContent: string;
const exportIdentity: FileIdentityStat = {
dev: openedExport.stat.dev,
ino: openedExport.stat.ino,
};
const identityOwned =
storedIdentity !== undefined && sameFileIdentity(storedIdentity, exportIdentity);
try {
if (openedExport.stat.size > MAX_MEMORY_HOST_PUBLIC_EXPORT_BYTES) {
return identityOwned
? {
kind: "owned",
content: undefined,
identity: exportIdentity,
ownerContent: content,
needsFinalize: true,
}
: { kind: "foreign" };
}
exportContent = await openedExport.handle.readFile({ encoding: "utf8" });
} finally {
await openedExport.handle.close().catch(() => undefined);
}
const exportSha256 = sha256Hex(exportContent);
const currentSha256 = (parsed as { contentSha256?: string }).contentSha256;
const pendingSha256 = (parsed as { pendingContentSha256?: string }).pendingContentSha256;
// Hash-only markers never reach the owned branch; only the persisted inode
// identity authorizes later mutation of this workspace artifact.
return identityOwned
? {
kind: "owned",
content: exportContent,
identity: exportIdentity,
ownerContent: content,
needsFinalize: exportSha256 !== currentSha256 || pendingSha256 !== undefined,
}
: { kind: "foreign" };
}
export {
buildMemoryPromptSection as buildActiveMemoryPromptSection,
@@ -25,15 +224,6 @@ export { resolveDefaultAgentId } from "../agents/agent-scope-config.js";
export { resolveSessionAgentId } from "../agents/agent-scope.js";
export { resolveSessionTranscriptsDirForAgent } from "../config/sessions/paths.js";
async function pathExists(filePath: string): Promise<boolean> {
try {
await fs.access(filePath);
return true;
} catch {
return false;
}
}
async function listMarkdownFilesRecursive(rootDir: string): Promise<string[]> {
const entries = await fs.readdir(rootDir, { withFileTypes: true }).catch(() => []);
const files: string[] = [];
@@ -50,6 +240,215 @@ async function listMarkdownFilesRecursive(rootDir: string): Promise<string[]> {
return files.toSorted((left, right) => left.localeCompare(right));
}
async function materializeMemoryHostEventExport(params: {
workspaceDir: string;
}): Promise<{ absolutePath: string; relativePath: string } | undefined> {
const requestedWorkspace = path.resolve(params.workspaceDir);
const workspace = await fs.stat(requestedWorkspace).catch((error: unknown) => {
if (isMissingPathError(error)) {
return undefined;
}
throw error;
});
if (!workspace?.isDirectory()) {
return undefined;
}
const workspaceRoot = await createFsSafeRoot(requestedWorkspace, {
hardlinks: "reject",
mkdir: true,
mode: 0o600,
symlinks: "reject",
});
const workspaceKey = workspaceRoot.rootReal;
const owner = await resolveMemoryHostEventExportOwner(workspaceKey);
// The queue handles re-entrant calls in this process; the sidecar lock makes
// snapshot, cleanup, and replacement one ordered operation across processes.
// State-qualified paths keep different profiles from replacing each other's export.
return memoryHostEventExportQueue.enqueue(owner.queueKey, async () => {
const absolutePath = path.join(workspaceKey, ...owner.relativePath.split("/"));
return await withFileLock(owner.lockTarget, MEMORY_HOST_EVENT_EXPORT_LOCK_OPTIONS, async () => {
const storedEvents = listStoredMemoryHostEvents({
workspaceDir: workspaceKey,
limit: MAX_MEMORY_HOST_PUBLIC_EXPORT_EVENTS,
});
const ownership = await readMemoryHostEventExportOwnership(workspaceRoot, owner);
if (ownership.kind === "foreign") {
return undefined;
}
if (storedEvents.length === 0 && ownership.kind !== "owned") {
return undefined;
}
const content = storedEvents.length > 0 ? serializeMemoryHostEventExport(storedEvents) : "";
const contentSha256 = sha256Hex(content);
let publishedIdentity: FileIdentityStat | undefined;
if (ownership.kind === "missing") {
const existing = await workspaceRoot
.readText(owner.relativePath)
.catch((error: unknown) => {
if (isMissingPathError(error)) {
return undefined;
}
if (isRejectedWorkspaceArtifactPath(error)) {
return null;
}
throw error;
});
if (existing !== undefined) {
return undefined;
}
try {
const pendingOwnerContent = memoryHostEventExportOwnerContent(owner, {
pendingSha256: contentSha256,
});
await workspaceRoot.create(owner.ownerRelativePath, pendingOwnerContent, {
mkdir: true,
mode: 0o600,
});
await syncDirectoryBestEffort(path.dirname(absolutePath));
publishedIdentity = await publishMemoryHostEventArtifact({
workspaceRoot,
owner,
absolutePath,
expectedOwnerContent: pendingOwnerContent,
content,
contentSha256,
});
if (!publishedIdentity) {
return undefined;
}
} catch (error) {
if (isWorkspaceWriteUnavailable(error)) {
return undefined;
}
throw error;
}
} else if (ownership.kind === "pending-missing" || ownership.kind === "orphan") {
try {
const pendingOwnerContent = memoryHostEventExportOwnerContent(owner, {
pendingSha256: contentSha256,
});
if (
!(await rewriteMemoryHostEventArtifactIfUnchanged({
workspaceRoot,
relativePath: owner.ownerRelativePath,
expectedContent: ownership.ownerContent,
nextContent: pendingOwnerContent,
}))
) {
return undefined;
}
await syncDirectoryBestEffort(path.dirname(absolutePath));
publishedIdentity = await publishMemoryHostEventArtifact({
workspaceRoot,
owner,
absolutePath,
expectedOwnerContent: pendingOwnerContent,
content,
contentSha256,
});
if (!publishedIdentity) {
return undefined;
}
} catch (error) {
if (isWorkspaceWriteUnavailable(error)) {
return undefined;
}
throw error;
}
} else if (ownership.content !== content) {
publishedIdentity = ownership.identity;
try {
const updateOwnerContent = memoryHostEventExportOwnerContent(owner, {
pendingSha256: contentSha256,
identity: ownership.identity,
...(ownership.content === undefined
? {}
: { currentSha256: sha256Hex(ownership.content) }),
});
const currentOwnerContent = memoryHostEventExportOwnerContent(owner, {
currentSha256: contentSha256,
identity: ownership.identity,
});
if (
!(await rewriteMemoryHostEventArtifactIfUnchanged({
workspaceRoot,
relativePath: owner.ownerRelativePath,
expectedContent: ownership.ownerContent,
nextContent: updateOwnerContent,
}))
) {
return undefined;
}
await syncDirectoryBestEffort(path.dirname(absolutePath));
if (
!(await rewriteMemoryHostEventArtifactIfUnchanged({
workspaceRoot,
relativePath: owner.relativePath,
expectedIdentity: ownership.identity,
nextContent: content,
}))
) {
return undefined;
}
await syncDirectoryBestEffort(path.dirname(absolutePath));
if (
!(await rewriteMemoryHostEventArtifactIfUnchanged({
workspaceRoot,
relativePath: owner.ownerRelativePath,
expectedContent: updateOwnerContent,
nextContent: currentOwnerContent,
}))
) {
return undefined;
}
await syncDirectoryBestEffort(path.dirname(absolutePath));
} catch (error) {
if (isWorkspaceWriteUnavailable(error)) {
return undefined;
}
throw error;
}
} else if (ownership.needsFinalize) {
publishedIdentity = ownership.identity;
try {
if (
!(await rewriteMemoryHostEventArtifactIfUnchanged({
workspaceRoot,
relativePath: owner.ownerRelativePath,
expectedContent: ownership.ownerContent,
nextContent: memoryHostEventExportOwnerContent(owner, {
currentSha256: contentSha256,
identity: ownership.identity,
}),
}))
) {
return undefined;
}
await syncDirectoryBestEffort(path.dirname(absolutePath));
} catch (error) {
if (isWorkspaceWriteUnavailable(error)) {
return undefined;
}
throw error;
}
} else {
publishedIdentity = ownership.identity;
}
if (storedEvents.length === 0 || !publishedIdentity) {
return undefined;
}
return (await isMemoryHostEventArtifactAtIdentity({
workspaceRoot,
relativePath: owner.relativePath,
expectedIdentity: publishedIdentity,
expectedContent: content,
}))
? { absolutePath, relativePath: owner.relativePath }
: undefined;
});
});
}
/** Lists public memory artifacts for one workspace, including notes and event logs. */
async function listMemoryWorkspacePublicArtifacts(params: {
workspaceDir: string;
@@ -87,13 +486,15 @@ async function listMemoryWorkspacePublicArtifacts(params: {
});
}
const eventLogPath = resolveMemoryHostEventLogPath(params.workspaceDir);
if (await pathExists(eventLogPath)) {
const eventExport = await materializeMemoryHostEventExport({
workspaceDir: params.workspaceDir,
});
if (eventExport) {
artifacts.push({
kind: "event-log",
workspaceDir: params.workspaceDir,
relativePath: path.relative(params.workspaceDir, eventLogPath).replace(/\\/g, "/"),
absolutePath: eventLogPath,
relativePath: eventExport.relativePath,
absolutePath: eventExport.absolutePath,
agentIds: [...params.agentIds],
contentType: "json",
});
@@ -0,0 +1,223 @@
import { createHash } from "node:crypto";
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { root as createFsSafeRoot } from "../infra/fs-safe.js";
import { resetPluginStateStoreForTests } from "../plugin-state/plugin-state-store.js";
import { clearMemoryPluginState } from "../plugins/memory-state.test-fixtures.js";
import { listMemoryHostPublicArtifacts } from "./memory-host-core.js";
import {
memoryHostEventExportOwnerContent,
publishMemoryHostEventArtifact,
} from "./memory-host-event-export.js";
import { appendMemoryHostEvent } from "./memory-host-events.js";
describe("memory host event export recovery", () => {
afterEach(() => {
clearMemoryPluginState();
resetPluginStateStoreForTests();
vi.unstubAllEnvs();
vi.restoreAllMocks();
});
it("does not finalize an initial export changed through the published inode", async () => {
const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-publish-race-"));
const workspaceDir = path.join(fixtureRoot, "workspace");
const relativePath = "memory/events/state/memory-host-events.jsonl";
const ownerRelativePath = "memory/events/state/.openclaw-memory-host-events-owner.json";
const absolutePath = path.join(workspaceDir, relativePath);
const owner = {
queueKey: "state\0workspace",
lockTarget: path.join(fixtureRoot, "lock"),
relativePath,
ownerRelativePath,
stateHash: "state",
workspaceHash: "workspace",
};
const content = '{"type":"memory.recall.recorded","query":""}\n';
const expectedBytes = Buffer.from(content, "utf8");
const replacementBytes = Buffer.from("", "utf8");
const replacementOffset = expectedBytes.indexOf(replacementBytes);
const foreignBytes = Buffer.concat([
expectedBytes.subarray(0, replacementOffset),
Buffer.from([0x80]),
expectedBytes.subarray(replacementOffset + replacementBytes.length),
]);
const contentSha256 = createHash("sha256").update(content).digest("hex");
const expectedOwnerContent = memoryHostEventExportOwnerContent(owner, {
pendingSha256: contentSha256,
});
try {
await fs.mkdir(path.dirname(absolutePath), { recursive: true });
await fs.writeFile(path.join(workspaceDir, ownerRelativePath), expectedOwnerContent, "utf8");
const workspaceRoot = await createFsSafeRoot(workspaceDir, {
hardlinks: "reject",
mkdir: true,
mode: 0o600,
symlinks: "reject",
});
const originalOpen = workspaceRoot.open.bind(workspaceRoot);
let replacedContent = false;
const interceptedWorkspaceRoot = new Proxy(workspaceRoot, {
get(target, property, receiver) {
if (property === "open") {
return async (
openedRelativePath: string,
options?: Parameters<typeof workspaceRoot.open>[1],
) => {
if (openedRelativePath === relativePath && !replacedContent) {
replacedContent = true;
await fs.writeFile(absolutePath, foreignBytes);
}
return await originalOpen(openedRelativePath, options);
};
}
const value = Reflect.get(target, property, receiver) as unknown;
return typeof value === "function" ? value.bind(target) : value;
},
});
await expect(
publishMemoryHostEventArtifact({
workspaceRoot: interceptedWorkspaceRoot,
owner,
absolutePath,
expectedOwnerContent,
content,
contentSha256,
}),
).resolves.toBeUndefined();
expect(foreignBytes.toString("utf8")).toBe(content);
await expect(fs.readFile(absolutePath)).resolves.toEqual(foreignBytes);
expect(
JSON.parse(await fs.readFile(path.join(workspaceDir, ownerRelativePath), "utf8")),
).toMatchObject({
pendingContentSha256: contentSha256,
fileDev: expect.any(String),
fileIno: expect.any(String),
});
} finally {
await fs.rm(fixtureRoot, { recursive: true, force: true });
}
});
it("finishes an inode-owned empty event export after interruption", async () => {
const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-inode-owner-"));
const stateDir = path.join(fixtureRoot, "state");
const workspaceDir = path.join(fixtureRoot, "workspace");
const event = {
type: "memory.recall.recorded" as const,
timestamp: "2026-05-18T12:00:00.000Z",
query: "recover inode-owned export",
resultCount: 0,
results: [],
};
try {
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
await fs.mkdir(workspaceDir);
await appendMemoryHostEvent(workspaceDir, event);
const stateHash = createHash("sha256")
.update(await fs.realpath(stateDir))
.digest("hex")
.slice(0, 32);
const workspaceHash = createHash("sha256")
.update(await fs.realpath(workspaceDir))
.digest("hex")
.slice(0, 32);
const exportDir = path.join(workspaceDir, "memory", "events", stateHash);
const exportPath = path.join(exportDir, "memory-host-events.jsonl");
const ownerPath = path.join(exportDir, ".openclaw-memory-host-events-owner.json");
const expectedContent = `${JSON.stringify(event)}\n`;
await fs.mkdir(exportDir, { recursive: true });
await fs.writeFile(exportPath, "", { mode: 0o600 });
const exportStat = await fs.stat(exportPath, { bigint: true });
await fs.writeFile(
ownerPath,
`${JSON.stringify({
schemaVersion: 3,
kind: "openclaw-memory-host-events-export",
stateHash,
workspaceHash,
pendingContentSha256: createHash("sha256").update(expectedContent).digest("hex"),
fileDev: String(exportStat.dev),
fileIno: String(exportStat.ino),
})}\n`,
"utf8",
);
const listed = await listMemoryHostPublicArtifacts({
cfg: { agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] } },
});
expect(listed.some((artifact) => artifact.kind === "event-log")).toBe(true);
await expect(fs.readFile(exportPath, "utf8")).resolves.toBe(expectedContent);
const owner = JSON.parse(await fs.readFile(ownerPath, "utf8")) as {
contentSha256?: string;
fileDev?: string;
fileIno?: string;
pendingContentSha256?: string;
};
expect(owner).toMatchObject({
contentSha256: createHash("sha256").update(expectedContent).digest("hex"),
fileDev: expect.any(String),
fileIno: expect.any(String),
});
expect(owner.pendingContentSha256).toBeUndefined();
} finally {
await fs.rm(fixtureRoot, { recursive: true, force: true });
}
});
it("does not claim an empty export after exclusive-create interruption", async () => {
const fixtureRoot = await fs.mkdtemp(path.join(os.tmpdir(), "memory-host-empty-export-"));
const stateDir = path.join(fixtureRoot, "state");
const workspaceDir = path.join(fixtureRoot, "workspace");
const event = {
type: "memory.recall.recorded" as const,
timestamp: "2026-05-18T12:00:00.000Z",
query: "leave empty export untouched",
resultCount: 0,
results: [],
};
try {
vi.stubEnv("OPENCLAW_STATE_DIR", stateDir);
await fs.mkdir(workspaceDir);
await appendMemoryHostEvent(workspaceDir, event);
const stateHash = createHash("sha256")
.update(await fs.realpath(stateDir))
.digest("hex")
.slice(0, 32);
const workspaceHash = createHash("sha256")
.update(await fs.realpath(workspaceDir))
.digest("hex")
.slice(0, 32);
const exportDir = path.join(workspaceDir, "memory", "events", stateHash);
const exportPath = path.join(exportDir, "memory-host-events.jsonl");
const ownerPath = path.join(exportDir, ".openclaw-memory-host-events-owner.json");
const expectedContent = `${JSON.stringify(event)}\n`;
await fs.mkdir(exportDir, { recursive: true });
await fs.writeFile(exportPath, "", { mode: 0o600 });
await fs.writeFile(
ownerPath,
`${JSON.stringify({
schemaVersion: 3,
kind: "openclaw-memory-host-events-export",
stateHash,
workspaceHash,
pendingContentSha256: createHash("sha256").update(expectedContent).digest("hex"),
})}\n`,
"utf8",
);
const listed = await listMemoryHostPublicArtifacts({
cfg: { agents: { list: [{ id: "main", default: true, workspace: workspaceDir }] } },
});
expect(listed.some((artifact) => artifact.kind === "event-log")).toBe(false);
await expect(fs.readFile(exportPath, "utf8")).resolves.toBe("");
} finally {
await fs.rm(fixtureRoot, { recursive: true, force: true });
}
});
});
+268
View File
@@ -0,0 +1,268 @@
import type { FileHandle } from "node:fs/promises";
import path from "node:path";
import { sameFileIdentity, type FileIdentityStat } from "../infra/fs-safe-advanced.js";
import { FsSafeError, root as createFsSafeRoot } from "../infra/fs-safe.js";
import { syncDirectoryBestEffort } from "../infra/sqlite-snapshot.js";
export type MemoryHostEventExportOwner = {
queueKey: string;
lockTarget: string;
relativePath: string;
ownerRelativePath: string;
stateHash: string;
workspaceHash: string;
};
type MemoryHostWorkspaceRoot = Awaited<ReturnType<typeof createFsSafeRoot>>;
export function isMissingPathError(error: unknown): boolean {
const code = (error as { code?: unknown }).code;
return (
code === "ENOENT" ||
code === "ENOTDIR" ||
(error instanceof FsSafeError && code === "not-found")
);
}
export function isRejectedWorkspaceArtifactPath(error: unknown): boolean {
if (!(error instanceof FsSafeError)) {
return false;
}
return (
error.code === "hardlink" ||
error.code === "not-file" ||
error.code === "outside-workspace" ||
error.code === "path-alias" ||
error.code === "path-mismatch" ||
error.code === "symlink"
);
}
export function memoryHostEventExportOwnerContent(
owner: MemoryHostEventExportOwner,
content: {
currentSha256?: string;
pendingSha256?: string;
identity?: FileIdentityStat;
},
): string {
return `${JSON.stringify({
schemaVersion: 3,
kind: "openclaw-memory-host-events-export",
stateHash: owner.stateHash,
workspaceHash: owner.workspaceHash,
...(content.identity
? { fileDev: String(content.identity.dev), fileIno: String(content.identity.ino) }
: {}),
...(content.currentSha256 ? { contentSha256: content.currentSha256 } : {}),
...(content.pendingSha256 ? { pendingContentSha256: content.pendingSha256 } : {}),
})}\n`;
}
async function writePinnedMemoryHostEventArtifact(
handle: FileHandle,
content: string,
): Promise<void> {
const bytes = Buffer.from(content, "utf8");
let offset = 0;
while (offset < bytes.length) {
const result = await handle.write(bytes, offset, bytes.length - offset, offset);
if (result.bytesWritten === 0) {
throw new Error("event export write made no progress");
}
offset += result.bytesWritten;
}
await handle.truncate(bytes.length);
await handle.chmod(0o600);
await handle.sync();
}
export async function rewriteMemoryHostEventArtifactIfUnchanged(params: {
workspaceRoot: MemoryHostWorkspaceRoot;
relativePath: string;
expectedContent?: string;
expectedIdentity?: FileIdentityStat;
nextContent: string;
}): Promise<boolean> {
let observed: Awaited<ReturnType<typeof params.workspaceRoot.open>>;
try {
observed = await params.workspaceRoot.open(params.relativePath);
} catch (error) {
if (isMissingPathError(error) || isRejectedWorkspaceArtifactPath(error)) {
return false;
}
throw error;
}
try {
if (
params.expectedIdentity
? !sameFileIdentity(params.expectedIdentity, observed.stat)
: (await observed.handle.readFile({ encoding: "utf8" })) !== params.expectedContent
) {
return false;
}
let writable: Awaited<ReturnType<typeof params.workspaceRoot.openWritable>>;
try {
writable = await params.workspaceRoot.openWritable(params.relativePath, {
mode: 0o600,
writeMode: "update",
});
} catch (error) {
if (isMissingPathError(error) || isRejectedWorkspaceArtifactPath(error)) {
return false;
}
throw error;
}
try {
// The matching marker/content snapshot owns this generated artifact inode for
// the locked update. A distinct workspace file can take the path only by
// replacing that inode; direct writes still target the owned export itself.
if (!sameFileIdentity(observed.stat, writable.stat)) {
return false;
}
await writable.handle.writeFile(params.nextContent, { encoding: "utf8" });
await writable.handle.truncate(Buffer.byteLength(params.nextContent, "utf8"));
await writable.handle.chmod(0o600);
await writable.handle.sync();
} finally {
await writable.handle.close().catch(() => undefined);
}
let verified: Awaited<ReturnType<typeof params.workspaceRoot.open>>;
try {
verified = await params.workspaceRoot.open(params.relativePath);
} catch (error) {
if (isMissingPathError(error) || isRejectedWorkspaceArtifactPath(error)) {
return false;
}
throw error;
}
try {
return (
sameFileIdentity(observed.stat, verified.stat) &&
(await verified.handle.readFile({ encoding: "utf8" })) === params.nextContent
);
} finally {
await verified.handle.close().catch(() => undefined);
}
} finally {
await observed.handle.close().catch(() => undefined);
}
}
export async function isMemoryHostEventArtifactAtIdentity(params: {
workspaceRoot: MemoryHostWorkspaceRoot;
relativePath: string;
expectedIdentity: FileIdentityStat;
expectedContent?: string;
}): Promise<boolean> {
let opened: Awaited<ReturnType<typeof params.workspaceRoot.open>>;
try {
opened = await params.workspaceRoot.open(params.relativePath);
} catch (error) {
if (isMissingPathError(error) || isRejectedWorkspaceArtifactPath(error)) {
return false;
}
throw error;
}
try {
if (!sameFileIdentity(params.expectedIdentity, opened.stat)) {
return false;
}
return (
params.expectedContent === undefined ||
(await opened.handle.readFile()).equals(Buffer.from(params.expectedContent, "utf8"))
);
} finally {
await opened.handle.close().catch(() => undefined);
}
}
export async function publishMemoryHostEventArtifact(params: {
workspaceRoot: MemoryHostWorkspaceRoot;
owner: MemoryHostEventExportOwner;
absolutePath: string;
expectedOwnerContent: string;
content: string;
contentSha256: string;
}): Promise<FileIdentityStat | undefined> {
let writable: Awaited<ReturnType<typeof params.workspaceRoot.openWritable>>;
try {
writable = await params.workspaceRoot.openWritable(params.owner.relativePath, {
mode: 0o600,
writeMode: "replace",
});
} catch (error) {
if (isMissingPathError(error) || isRejectedWorkspaceArtifactPath(error)) {
return undefined;
}
throw error;
}
try {
// `createdForWrite` is the exclusive-create proof. Keep its handle pinned
// through publication so a replacement path is never opened for mutation.
if (!writable.createdForWrite) {
return undefined;
}
const publishedIdentity = { dev: writable.stat.dev, ino: writable.stat.ino };
await syncDirectoryBestEffort(path.dirname(params.absolutePath));
const identityPendingOwnerContent = memoryHostEventExportOwnerContent(params.owner, {
pendingSha256: params.contentSha256,
identity: publishedIdentity,
});
if (
!(await rewriteMemoryHostEventArtifactIfUnchanged({
workspaceRoot: params.workspaceRoot,
relativePath: params.owner.ownerRelativePath,
expectedContent: params.expectedOwnerContent,
nextContent: identityPendingOwnerContent,
}))
) {
return undefined;
}
await syncDirectoryBestEffort(path.dirname(params.absolutePath));
await writePinnedMemoryHostEventArtifact(writable.handle, params.content);
// Workspace actors can mutate this inode without replacing the path. Verify
// bytes before finalizing the marker so foreign content never gains ownership.
if (
!(await isMemoryHostEventArtifactAtIdentity({
workspaceRoot: params.workspaceRoot,
relativePath: params.owner.relativePath,
expectedIdentity: publishedIdentity,
expectedContent: params.content,
}))
) {
return undefined;
}
await syncDirectoryBestEffort(path.dirname(params.absolutePath));
if (
!(await rewriteMemoryHostEventArtifactIfUnchanged({
workspaceRoot: params.workspaceRoot,
relativePath: params.owner.ownerRelativePath,
expectedContent: identityPendingOwnerContent,
nextContent: memoryHostEventExportOwnerContent(params.owner, {
currentSha256: params.contentSha256,
identity: publishedIdentity,
}),
}))
) {
return undefined;
}
await syncDirectoryBestEffort(path.dirname(params.absolutePath));
if (
!(await isMemoryHostEventArtifactAtIdentity({
workspaceRoot: params.workspaceRoot,
relativePath: params.owner.relativePath,
expectedIdentity: publishedIdentity,
expectedContent: params.content,
}))
) {
return undefined;
}
return publishedIdentity;
} finally {
await writable.handle.close().catch(() => undefined);
}
}
+230 -83
View File
@@ -3,13 +3,16 @@
*/
import fs from "node:fs/promises";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
listStoredMemoryHostEvents,
setMaxMemoryHostEventsForTests,
} from "../memory-host-sdk/event-store.js";
import { resetPluginStateStoreForTests } from "../plugin-state/plugin-state-store.js";
import {
appendMemoryHostEvent,
readMemoryHostEventRecords,
readMemoryHostEvents,
resolveMemoryHostEventLogPath,
} from "./memory-host-events.js";
import {
createClaimableDedupe,
@@ -32,45 +35,51 @@ function createDedupe(root: string, overrides?: { ttlMs?: number }) {
}
afterEach(() => {
vi.restoreAllMocks();
setMaxMemoryHostEventsForTests(undefined);
resetPluginStateStoreForTests();
});
describe("memory host event journal helpers", () => {
it("appends and reads typed workspace events", async () => {
const workspaceDir = await createTempDir("memory-host-events-");
const env = { ...process.env, OPENCLAW_STATE_DIR: workspaceDir };
await appendMemoryHostEvent(workspaceDir, {
type: "memory.recall.recorded",
timestamp: "2026-04-05T12:00:00.000Z",
query: "glacier backup",
resultCount: 1,
results: [
{
path: "memory/2026-04-05.md",
startLine: 1,
endLine: 3,
score: 0.9,
},
],
});
await appendMemoryHostEvent(workspaceDir, {
type: "memory.dream.completed",
timestamp: "2026-04-05T13:00:00.000Z",
phase: "light",
outcome: "completed",
lineCount: 4,
storageMode: "both",
inlinePath: path.join(workspaceDir, "memory", "2026-04-05.md"),
reportPath: path.join(workspaceDir, "memory", "dreaming", "light", "2026-04-05.md"),
});
const eventLogPath = resolveMemoryHostEventLogPath(workspaceDir);
await expect(fs.readFile(eventLogPath, "utf8")).resolves.toContain(
'"type":"memory.recall.recorded"',
await appendMemoryHostEvent(
workspaceDir,
{
type: "memory.recall.recorded",
timestamp: "2026-04-05T12:00:00.000Z",
query: "glacier backup",
resultCount: 1,
results: [
{
path: "memory/2026-04-05.md",
startLine: 1,
endLine: 3,
score: 0.9,
},
],
},
{ env },
);
await appendMemoryHostEvent(
workspaceDir,
{
type: "memory.dream.completed",
timestamp: "2026-04-05T11:00:00.000Z",
phase: "light",
outcome: "completed",
lineCount: 4,
storageMode: "both",
inlinePath: path.join(workspaceDir, "memory", "2026-04-05.md"),
reportPath: path.join(workspaceDir, "memory", "dreaming", "light", "2026-04-05.md"),
},
{ env },
);
const events = await readMemoryHostEvents({ workspaceDir });
const tail = await readMemoryHostEvents({ workspaceDir, limit: 1 });
const events = await readMemoryHostEvents({ workspaceDir, env });
const tail = await readMemoryHostEvents({ workspaceDir, limit: 1, env });
expect(events).toHaveLength(2);
expect(events[0]?.type).toBe("memory.recall.recorded");
@@ -83,62 +92,101 @@ describe("memory host event journal helpers", () => {
expect(tail[0]?.type).toBe("memory.dream.completed");
});
it("keeps journal retention timestamps in the current wall-clock domain", async () => {
const workspaceDir = await createTempDir("memory-host-events-created-at-");
const env = { ...process.env, OPENCLAW_STATE_DIR: workspaceDir };
const now = Date.parse("2026-07-16T12:00:00.000Z");
vi.useFakeTimers();
vi.setSystemTime(now);
for (const query of ["first", "second"]) {
await appendMemoryHostEvent(
workspaceDir,
{
type: "memory.recall.recorded",
timestamp: "2026-07-16T12:00:00.000Z",
query,
resultCount: 0,
results: [],
},
{ env },
);
}
expect(
listStoredMemoryHostEvents({ workspaceDir, env }).map((entry) => entry.createdAt),
).toEqual([now, now + 1]);
});
it("keeps legacy event readers stable when diagnostic records are present", async () => {
const workspaceDir = await createTempDir("memory-host-events-diagnostics-");
const env = { ...process.env, OPENCLAW_STATE_DIR: workspaceDir };
await appendMemoryHostEvent(workspaceDir, {
type: "memory.recall.skipped",
timestamp: "2026-04-05T12:00:00.000Z",
query: "durable memory",
reason: "non-short-term-memory-path",
eligibleResultCount: 0,
skippedResultCount: 1,
results: [
{
path: "MEMORY.md",
startLine: 3,
endLine: 3,
score: 0.9,
reason: "non-short-term-memory-path",
},
],
});
await appendMemoryHostEvent(
workspaceDir,
{
type: "memory.recall.skipped",
timestamp: "2026-04-05T12:00:00.000Z",
query: "durable memory",
reason: "non-short-term-memory-path",
eligibleResultCount: 0,
skippedResultCount: 1,
results: [
{
path: "MEMORY.md",
startLine: 3,
endLine: 3,
score: 0.9,
reason: "non-short-term-memory-path",
},
],
},
{ env },
);
await appendMemoryHostEvent(workspaceDir, {
type: "memory.recall.recorded",
timestamp: "2026-04-05T12:05:00.000Z",
query: "daily memory",
resultCount: 1,
results: [
{
path: "memory/2026-04-05.md",
startLine: 1,
endLine: 3,
score: 0.95,
},
],
});
await appendMemoryHostEvent(workspaceDir, {
type: "memory.recall.skipped",
timestamp: "2026-04-05T12:10:00.000Z",
query: "durable memory again",
reason: "non-short-term-memory-path",
eligibleResultCount: 1,
skippedResultCount: 1,
results: [
{
path: "MEMORY.md",
startLine: 4,
endLine: 4,
score: 0.8,
reason: "non-short-term-memory-path",
},
],
});
await appendMemoryHostEvent(
workspaceDir,
{
type: "memory.recall.recorded",
timestamp: "2026-04-05T12:05:00.000Z",
query: "daily memory",
resultCount: 1,
results: [
{
path: "memory/2026-04-05.md",
startLine: 1,
endLine: 3,
score: 0.95,
},
],
},
{ env },
);
await appendMemoryHostEvent(
workspaceDir,
{
type: "memory.recall.skipped",
timestamp: "2026-04-05T12:10:00.000Z",
query: "durable memory again",
reason: "non-short-term-memory-path",
eligibleResultCount: 1,
skippedResultCount: 1,
results: [
{
path: "MEMORY.md",
startLine: 4,
endLine: 4,
score: 0.8,
reason: "non-short-term-memory-path",
},
],
},
{ env },
);
const legacyEvents = await readMemoryHostEvents({ workspaceDir });
const legacyTail = await readMemoryHostEvents({ workspaceDir, limit: 1 });
const records = await readMemoryHostEventRecords({ workspaceDir });
const legacyEvents = await readMemoryHostEvents({ workspaceDir, env });
const legacyTail = await readMemoryHostEvents({ workspaceDir, limit: 1, env });
const records = await readMemoryHostEventRecords({ workspaceDir, env });
expect(legacyEvents.map((event) => event.type)).toEqual(["memory.recall.recorded"]);
expect(legacyTail.map((event) => event.type)).toEqual(["memory.recall.recorded"]);
@@ -148,6 +196,105 @@ describe("memory host event journal helpers", () => {
"memory.recall.skipped",
]);
});
it("bounds oversized diagnostic detail without failing the parent operation", async () => {
const workspaceDir = await createTempDir("memory-host-events-bounded-");
const env = { ...process.env, OPENCLAW_STATE_DIR: workspaceDir };
const results = Array.from({ length: 100 }, (_, index) => ({
path: `memory/${"wide-path-".repeat(100)}${index}.md`,
startLine: index + 1,
endLine: index + 2,
score: 0.9,
}));
await expect(
appendMemoryHostEvent(
workspaceDir,
{
type: "memory.recall.recorded",
timestamp: "2026-04-05T12:00:00.000Z",
query: "🔥".repeat(20_000),
resultCount: results.length,
results,
},
{ env },
),
).resolves.toBeUndefined();
const [event] = await readMemoryHostEventRecords({ workspaceDir, env });
expect(event).toMatchObject({
type: "memory.recall.recorded",
resultCount: 100,
storageTruncated: true,
});
if (event?.type !== "memory.recall.recorded") {
throw new Error("expected bounded recall event");
}
expect(event.results).toHaveLength(10);
expect(Buffer.byteLength(JSON.stringify(event), "utf8")).toBeLessThanOrEqual(8 * 1024);
});
it("rotates old events without evicting the workspace sequence cursor", async () => {
const workspaceDir = await createTempDir("memory-host-events-rotation-");
const env = { ...process.env, OPENCLAW_STATE_DIR: workspaceDir };
setMaxMemoryHostEventsForTests(3);
let clock = 1_000;
vi.spyOn(Date, "now").mockImplementation(() => clock--);
for (let index = 1; index <= 5; index += 1) {
await appendMemoryHostEvent(
workspaceDir,
{
type: "memory.recall.recorded",
timestamp: `2026-04-05T12:00:0${index}.000Z`,
query: `event-${index}`,
resultCount: 0,
results: [],
},
{ env },
);
}
const events = await readMemoryHostEventRecords({ workspaceDir, env });
expect(
events.map((event) => (event.type === "memory.recall.recorded" ? event.query : "")),
).toEqual(["event-3", "event-4", "event-5"]);
});
it("rotates events by namespace append order across workspaces", async () => {
const stateDir = await createTempDir("memory-host-events-shared-retention-");
const workspaceA = path.join(stateDir, "workspace-a");
const workspaceB = path.join(stateDir, "workspace-b");
const env = { ...process.env, OPENCLAW_STATE_DIR: stateDir };
setMaxMemoryHostEventsForTests(3);
const appendRecall = async (workspaceDir: string, query: string) => {
await appendMemoryHostEvent(
workspaceDir,
{
type: "memory.recall.recorded",
timestamp: "2026-04-05T12:00:00.000Z",
query,
resultCount: 0,
results: [],
},
{ env },
);
};
await appendRecall(workspaceA, "a-1");
await appendRecall(workspaceA, "a-2");
await appendRecall(workspaceA, "a-3");
await appendRecall(workspaceB, "b-1");
await appendRecall(workspaceA, "a-4");
const workspaceBEvents = await readMemoryHostEventRecords({
workspaceDir: workspaceB,
env,
});
expect(
workspaceBEvents.map((event) => (event.type === "memory.recall.recorded" ? event.query : "")),
).toEqual(["b-1"]);
});
});
describe("createPersistentDedupe", () => {
+2 -1
View File
@@ -1,8 +1,9 @@
/**
* Public SDK subpath for memory host event log types and helpers.
* Public SDK subpath for memory host event types and helpers.
*/
export {
appendMemoryHostEvent,
normalizeMemoryHostEventRecordForStorage,
readMemoryHostEventRecords,
readMemoryHostEvents,
resolveMemoryHostEventLogPath,
@@ -4,8 +4,11 @@
export {
createPluginStateKeyedStore as createPluginStateKeyedStoreForTests,
createPluginStateSyncKeyedStore as createPluginStateSyncKeyedStoreForTests,
getPluginStateCapacity as getPluginStateCapacityForTests,
importPluginStateEntriesForDoctor as importPluginStateEntriesForDoctorForTests,
resetPluginStateStoreForTests,
} from "../plugin-state/plugin-state-store.js";
export { setMaxMemoryHostEventsForTests } from "../memory-host-sdk/event-store.js";
export {
createPluginBlobStoreForTests,
resetPluginBlobStoreForTests,
+247 -3
View File
@@ -222,6 +222,33 @@ function selectPluginStateEntries(
).rows;
}
function selectPluginStateEntriesInKeyRange(
db: DatabaseSync,
params: {
pluginId: string;
namespace: string;
keyStartInclusive: string;
keyEndExclusive: string;
limit: number;
order: "asc" | "desc";
now: number;
},
): PluginStateRow[] {
return executeSqliteQuerySync(
db,
getPluginStateKysely(db)
.selectFrom("plugin_state_entries")
.select(["plugin_id", "namespace", "entry_key", "value_json", "created_at", "expires_at"])
.where("plugin_id", "=", params.pluginId)
.where("namespace", "=", params.namespace)
.where("entry_key", ">=", params.keyStartInclusive)
.where("entry_key", "<", params.keyEndExclusive)
.where((eb) => eb.or([eb("expires_at", "is", null), eb("expires_at", ">", params.now)]))
.orderBy("entry_key", params.order)
.limit(params.limit),
).rows;
}
function deletePluginStateEntry(
db: DatabaseSync,
params: { pluginId: string; namespace: string; key: string },
@@ -268,6 +295,26 @@ function countLivePluginStateNamespaceEntries(
return countRow(row);
}
function allocatePluginStateNamespaceCreatedAt(
db: DatabaseSync,
params: { pluginId: string; namespace: string; now: number },
): number {
const row = executeSqliteQueryTakeFirstSync(
db,
getPluginStateKysely(db)
.selectFrom("plugin_state_entries")
.select((eb) => eb.fn.max<number | bigint>("created_at").as("max_created_at"))
.where("plugin_id", "=", params.pluginId)
.where("namespace", "=", params.namespace),
);
const previous = normalizeSqliteNumber(row?.max_created_at ?? null);
const next = previous === undefined ? params.now : Math.max(params.now, previous + 1);
if (!Number.isSafeInteger(next)) {
throw new RangeError("Plugin state namespace append order exhausted safe integer range");
}
return next;
}
function countLivePluginStateEntries(
db: DatabaseSync,
params: { pluginId: string; now: number },
@@ -380,6 +427,7 @@ function enforcePostRegisterLimits(params: {
overflowPolicy: PluginStateOverflowPolicy;
now: number;
protectedKey: string;
enforcePluginLimit?: boolean;
}): void {
if (params.overflowPolicy === "reject-new") {
return;
@@ -399,6 +447,10 @@ function enforcePostRegisterLimits(params: {
});
}
if (params.enforcePluginLimit === false) {
return;
}
const pluginCount = countLivePluginStateEntries(params.store.db, {
pluginId: params.pluginId,
now: params.now,
@@ -408,7 +460,9 @@ function enforcePostRegisterLimits(params: {
return;
}
// Shed rows from the namespace that grew before failing the plugin write.
// Shed only rows from the namespace that grew. Sibling namespaces can hold
// durable state; if this namespace cannot cover the overflow, fail so the
// surrounding transaction rolls every insertion and deletion back.
deleteOldestPluginStateNamespaceEntries(params.store.db, {
pluginId: params.pluginId,
namespace: params.namespace,
@@ -551,6 +605,141 @@ export function pluginStateRegister(params: {
}
}
export function pluginStateRegisterSequencedJournalEntry(params: {
pluginId: string;
cursorNamespace: string;
cursorKey: string;
cursorMaxEntries: number;
journalNamespace: string;
journalMaxEntries: number;
initialSequence: number;
readCursorSequence: (valueJson: string) => number | undefined;
prepareEntry: (sequence: number) => {
cursorValueJson: string;
journalKey: string;
journalValueJson: string;
};
env?: NodeJS.ProcessEnv;
}): number {
try {
return runWriteTransaction(
"register",
(store) => {
const now = Date.now();
deleteExpiredPluginStateNamespaceEntries(store.db, {
pluginId: params.pluginId,
namespace: params.cursorNamespace,
now,
});
deleteExpiredPluginStateNamespaceEntries(store.db, {
pluginId: params.pluginId,
namespace: params.journalNamespace,
now,
});
const cursor = selectPluginStateEntry(store.db, {
pluginId: params.pluginId,
namespace: params.cursorNamespace,
key: params.cursorKey,
now,
});
const cursorSequence = cursor ? params.readCursorSequence(cursor.value_json) : undefined;
const lastSequence = Math.max(params.initialSequence, cursorSequence ?? 0);
const sequence = lastSequence + 1;
if (!Number.isSafeInteger(sequence)) {
throw new RangeError("Plugin state journal sequence exhausted safe integer range");
}
const prepared = params.prepareEntry(sequence);
const existingJournalEntry = selectPluginStateEntry(store.db, {
pluginId: params.pluginId,
namespace: params.journalNamespace,
key: prepared.journalKey,
now,
});
if (existingJournalEntry) {
throw createPluginStateError({
code: "PLUGIN_STATE_WRITE_FAILED",
operation: "register",
message: "Plugin state journal sequence already exists.",
path: store.path,
});
}
if (!cursor) {
assertCanInsertPluginStateEntry({
store,
pluginId: params.pluginId,
namespace: params.cursorNamespace,
maxEntries: params.cursorMaxEntries,
overflowPolicy: "evict-oldest",
now,
});
}
assertCanInsertPluginStateEntry({
store,
pluginId: params.pluginId,
namespace: params.journalNamespace,
maxEntries: params.journalMaxEntries,
overflowPolicy: "evict-oldest",
now,
});
upsertPluginStateEntry(
store.db,
bindPluginStateEntry({
pluginId: params.pluginId,
namespace: params.cursorNamespace,
key: params.cursorKey,
valueJson: prepared.cursorValueJson,
createdAt: now,
expiresAt: null,
}),
);
enforcePostRegisterLimits({
store,
pluginId: params.pluginId,
namespace: params.cursorNamespace,
maxEntries: params.cursorMaxEntries,
overflowPolicy: "evict-oldest",
now,
protectedKey: params.cursorKey,
enforcePluginLimit: false,
});
upsertPluginStateEntry(
store.db,
bindPluginStateEntry({
pluginId: params.pluginId,
namespace: params.journalNamespace,
key: prepared.journalKey,
valueJson: prepared.journalValueJson,
createdAt: allocatePluginStateNamespaceCreatedAt(store.db, {
pluginId: params.pluginId,
namespace: params.journalNamespace,
now,
}),
expiresAt: null,
}),
);
enforcePostRegisterLimits({
store,
pluginId: params.pluginId,
namespace: params.journalNamespace,
maxEntries: params.journalMaxEntries,
overflowPolicy: "evict-oldest",
now,
protectedKey: prepared.journalKey,
});
return sequence;
},
envOptions(params.env),
);
} catch (error) {
throw wrapPluginStateError(
error,
"register",
"PLUGIN_STATE_WRITE_FAILED",
"Failed to register sequenced plugin state journal entry.",
);
}
}
export function pluginStateRegisterIfAbsent(params: {
pluginId: string;
namespace: string;
@@ -852,6 +1041,51 @@ export function pluginStateEntries(params: {
}
}
/** Internal bounded key-range read for core owners with sortable plugin-state keys. */
export function pluginStateEntriesInKeyRange(params: {
pluginId: string;
namespace: string;
keyStartInclusive: string;
keyEndExclusive: string;
limit: number;
order?: "asc" | "desc";
env?: NodeJS.ProcessEnv;
}): PluginStateEntry<unknown>[] {
if (!Number.isSafeInteger(params.limit) || params.limit < 1) {
throw createPluginStateError({
code: "PLUGIN_STATE_INVALID_INPUT",
operation: "entries",
message: "Plugin state key-range limit must be a positive safe integer.",
});
}
if (params.keyStartInclusive >= params.keyEndExclusive) {
throw createPluginStateError({
code: "PLUGIN_STATE_INVALID_INPUT",
operation: "entries",
message: "Plugin state key range must have an increasing exclusive upper bound.",
});
}
try {
const { db } = openPluginStateDatabase("entries", envOptions(params.env));
return selectPluginStateEntriesInKeyRange(db, {
pluginId: params.pluginId,
namespace: params.namespace,
keyStartInclusive: params.keyStartInclusive,
keyEndExclusive: params.keyEndExclusive,
limit: params.limit,
order: params.order ?? "asc",
now: Date.now(),
}).map((row) => rowToEntry(row, "entries"));
} catch (error) {
throw wrapPluginStateError(
error,
"entries",
"PLUGIN_STATE_READ_FAILED",
"Failed to list plugin state entries by key range.",
);
}
}
export function pluginStateClear(params: {
pluginId: string;
namespace: string;
@@ -912,9 +1146,9 @@ function setMaxPluginStateEntriesPerPluginForTests(value?: number): void {
maxPluginStateEntriesPerPluginForTests = value;
}
export function countPluginStateLiveEntries(pluginId: string): number {
export function countPluginStateLiveEntries(pluginId: string, env?: NodeJS.ProcessEnv): number {
try {
const { db } = openPluginStateDatabase("entries");
const { db } = openPluginStateDatabase("entries", envOptions(env));
return countLivePluginStateEntries(db, { pluginId, now: Date.now() });
} catch (error) {
throw wrapPluginStateError(
@@ -926,6 +1160,16 @@ export function countPluginStateLiveEntries(pluginId: string): number {
}
}
export function getPluginStateCapacity(
pluginId: string,
env?: NodeJS.ProcessEnv,
): { liveEntries: number; maxEntries: number } {
return {
liveEntries: countPluginStateLiveEntries(pluginId, env),
maxEntries: resolveMaxPluginStateEntriesPerPlugin(),
};
}
function seedPluginStateDatabaseEntriesForTests(
entries: readonly PluginStateSeedEntryForTests[],
): void {
@@ -15,6 +15,8 @@ import {
createCorePluginStateSyncKeyedStore,
createPluginStateKeyedStore,
createPluginStateSyncKeyedStore,
pluginStateEntriesInKeyRange,
registerPluginStateSyncSequencedJournalEntry,
resetPluginStateStoreForTests,
sweepExpiredPluginStateEntries,
} from "./plugin-state-store.js";
@@ -103,6 +105,29 @@ describe("plugin state keyed store", () => {
});
});
it("reads a bounded sortable key range without scanning sibling keys", async () => {
await withPluginStateTestState(async () => {
const store = createPluginStateSyncKeyedStore<{ count: number }>("memory-core", {
namespace: "events",
maxEntries: 10,
});
store.register("workspace:event:0001", { count: 1 });
store.register("workspace:event:0002", { count: 2 });
store.register("workspace:other:0003", { count: 3 });
expect(
pluginStateEntriesInKeyRange({
pluginId: "memory-core",
namespace: "events",
keyStartInclusive: "workspace:event:",
keyEndExclusive: "workspace:event;",
limit: 1,
order: "desc",
}),
).toMatchObject([{ key: "workspace:event:0002", value: { count: 2 } }]);
});
});
it("updates a key from the current stored value", async () => {
await withPluginStateTestState(async () => {
const store = createPluginStateSyncKeyedStore<{ count: number }>("discord", {
@@ -537,6 +562,74 @@ describe("plugin state keyed store", () => {
});
});
it("sheds sequenced journal rows without evicting durable sibling state", async () => {
await withPluginStateTestState(async () => {
const maxPluginEntries = 40;
setMaxPluginStateEntriesPerPluginForTests(maxPluginEntries);
seedPluginStateEntriesForTests([
...Array.from({ length: maxPluginEntries - 3 }, (_, entryIndex) => ({
pluginId: "memory-core",
namespace: "durable-state",
key: `durable-${entryIndex}`,
value: { entryIndex },
})),
{
pluginId: "memory-core",
namespace: "memory-host.event-migration-checkpoints",
key: "generation",
value: { kind: "raw-checkpoint" },
},
{
pluginId: "memory-core",
namespace: "memory-host.event-cursors",
key: "workspace",
value: { kind: "cursor", lastSequence: 1 },
},
{
pluginId: "memory-core",
namespace: "memory-host.events",
key: "event-1",
value: { sequence: 1 },
},
]);
expect(
registerPluginStateSyncSequencedJournalEntry({
pluginId: "memory-core",
cursorOptions: {
namespace: "memory-host.event-cursors",
maxEntries: 1_000,
},
cursorKey: "workspace",
journalOptions: { namespace: "memory-host.events", maxEntries: 10_000 },
initialSequence: 0,
journalKey: (sequence) => `event-${sequence}`,
journalValue: (sequence) => ({ sequence }),
}),
).toBe(2);
const durable = createPluginStateKeyedStore("memory-core", {
namespace: "durable-state",
maxEntries: maxPluginEntries,
});
const checkpoints = createPluginStateKeyedStore("memory-core", {
namespace: "memory-host.event-migration-checkpoints",
maxEntries: 10_000,
overflowPolicy: "reject-new",
});
const journal = createPluginStateKeyedStore("memory-core", {
namespace: "memory-host.events",
maxEntries: 10_000,
});
await expect(durable.lookup("durable-0")).resolves.toEqual({ entryIndex: 0 });
await expect(checkpoints.lookup("generation")).resolves.toEqual({
kind: "raw-checkpoint",
});
await expect(journal.lookup("event-1")).resolves.toBeUndefined();
await expect(journal.lookup("event-2")).resolves.toEqual({ sequence: 2 });
});
});
it("leaves room for Telegram sibling namespaces at their persistent budgets", async () => {
await withPluginStateTestState(async () => {
seedPluginStateEntriesForTests([
+124
View File
@@ -12,6 +12,7 @@ import {
pluginStateLookup,
pluginStateRegister,
pluginStateRegisterIfAbsent,
pluginStateRegisterSequencedJournalEntry,
pluginStateUpdate,
} from "./plugin-state-store.sqlite.js";
import type {
@@ -44,8 +45,10 @@ export type {
export {
closePluginStateDatabase,
countPluginStateLiveEntries,
getPluginStateCapacity,
isPluginStateDatabaseOpen,
MAX_PLUGIN_STATE_ENTRIES_PER_PLUGIN,
pluginStateEntriesInKeyRange,
resolveMaxPluginStateEntriesPerPlugin,
sweepExpiredPluginStateEntries,
} from "./plugin-state-store.sqlite.js";
@@ -62,6 +65,12 @@ type PreparedRegisterParams = {
ttlMs?: number;
};
type PluginStateImportEntry = {
key: string;
value: unknown;
createdAt: number;
};
const namespaceOptionSignatures = new Map<string, StoreOptionSignature>();
function invalidInput(
message: string,
@@ -466,6 +475,121 @@ export function createPluginStateSyncKeyedStore<T>(
return createSyncKeyedStoreForPluginId<T>(pluginId, options);
}
/** Atomically allocates a workspace sequence and appends one journal entry. */
export function registerPluginStateSyncSequencedJournalEntry(params: {
pluginId: string;
cursorOptions: OpenKeyedStoreOptions;
cursorKey: string;
journalOptions: OpenKeyedStoreOptions;
initialSequence: number;
journalKey: (sequence: number) => string;
journalValue: (sequence: number) => unknown;
}): number {
if (params.pluginId.startsWith("core:")) {
throw invalidInput("Plugin ids starting with 'core:' are reserved for core consumers.", "open");
}
if (!Number.isSafeInteger(params.initialSequence) || params.initialSequence < 0) {
throw invalidInput("plugin state initial journal sequence must be a safe non-negative integer");
}
const cursorNamespace = validateNamespace(params.cursorOptions.namespace);
const cursorMaxEntries = validateMaxEntries(params.cursorOptions.maxEntries);
const cursorOverflowPolicy = validateOverflowPolicy(params.cursorOptions.overflowPolicy);
const cursorDefaultTtlMs = validateOptionalTtlMs(params.cursorOptions.defaultTtlMs);
const journalNamespace = validateNamespace(params.journalOptions.namespace);
const journalMaxEntries = validateMaxEntries(params.journalOptions.maxEntries);
const journalOverflowPolicy = validateOverflowPolicy(params.journalOptions.overflowPolicy);
const journalDefaultTtlMs = validateOptionalTtlMs(params.journalOptions.defaultTtlMs);
if (
cursorOverflowPolicy !== "evict-oldest" ||
journalOverflowPolicy !== "evict-oldest" ||
cursorDefaultTtlMs !== undefined ||
journalDefaultTtlMs !== undefined
) {
throw invalidInput("sequenced plugin state journals require non-expiring evict-oldest stores");
}
if (params.cursorOptions.env !== params.journalOptions.env) {
throw invalidInput("sequenced plugin state journal stores must share one environment");
}
const cursorKey = validateKey(params.cursorKey);
assertConsistentOptions(params.pluginId, cursorNamespace, {
maxEntries: cursorMaxEntries,
overflowPolicy: cursorOverflowPolicy,
defaultTtlMs: cursorDefaultTtlMs,
});
assertConsistentOptions(params.pluginId, journalNamespace, {
maxEntries: journalMaxEntries,
overflowPolicy: journalOverflowPolicy,
defaultTtlMs: journalDefaultTtlMs,
});
return pluginStateRegisterSequencedJournalEntry({
pluginId: params.pluginId,
cursorNamespace,
cursorKey,
cursorMaxEntries,
journalNamespace,
journalMaxEntries,
initialSequence: params.initialSequence,
readCursorSequence(valueJson) {
try {
const value = JSON.parse(valueJson) as { kind?: unknown; lastSequence?: unknown };
return value.kind === "cursor" && Number.isSafeInteger(value.lastSequence)
? (value.lastSequence as number)
: undefined;
} catch {
return undefined;
}
},
prepareEntry(sequence) {
const cursor = prepareRegisterParams(cursorKey, { kind: "cursor", lastSequence: sequence });
const journal = prepareRegisterParams(
params.journalKey(sequence),
params.journalValue(sequence),
);
return {
cursorValueJson: cursor.valueJson,
journalKey: journal.key,
journalValueJson: journal.valueJson,
};
},
...(params.cursorOptions.env ? { env: params.cursorOptions.env } : {}),
});
}
/** Doctor-only import that preserves source age for retention ordering. */
export function importPluginStateEntriesForDoctor(
pluginId: string,
options: OpenKeyedStoreOptions,
entries: readonly PluginStateImportEntry[],
): void {
if (pluginId.startsWith("core:")) {
throw invalidInput("Plugin ids starting with 'core:' are reserved for core consumers.", "open");
}
const namespace = validateNamespace(options.namespace);
const maxEntries = validateMaxEntries(options.maxEntries);
const overflowPolicy = validateOverflowPolicy(options.overflowPolicy);
const defaultTtlMs = validateOptionalTtlMs(options.defaultTtlMs);
const env = options.env;
assertConsistentOptions(pluginId, namespace, { maxEntries, overflowPolicy, defaultTtlMs });
for (const entry of entries) {
if (!Number.isSafeInteger(entry.createdAt)) {
throw invalidInput("plugin state import createdAt must be a safe integer", "register");
}
const prepared = prepareRegisterParams(entry.key, entry.value, defaultTtlMs);
pluginStateRegister({
pluginId,
namespace,
key: prepared.key,
valueJson: prepared.valueJson,
maxEntries,
overflowPolicy,
createdAtMs: entry.createdAt,
...(env ? { env } : {}),
...(prepared.ttlMs != null ? { ttlMs: prepared.ttlMs } : {}),
});
}
}
/** Opens a sync plugin-state namespace for a trusted core owner id. */
export function createCorePluginStateSyncKeyedStore<T>(
options: OpenKeyedStoreOptions & { ownerId: `core:${string}` },
+10
View File
@@ -59,11 +59,20 @@ export type PluginDoctorStateMigrationDetection = {
export type PluginDoctorStateMigrationContext = {
openPluginStateKeyedStore: <T>(options: OpenKeyedStoreOptions) => PluginStateKeyedStore<T>;
/** Doctor-only batch import preserving source age for retention ordering. */
importPluginStateEntries?: (
options: OpenKeyedStoreOptions,
entries: readonly { key: string; value: unknown; createdAt: number }[],
) => void;
/** Plugin-wide live-row capacity for import preflight. Older test hosts may omit it. */
getPluginStateCapacity?: () => { liveEntries: number; maxEntries: number };
};
export type PluginDoctorStateMigration = {
id: string;
label: string;
/** Import retired file state only during explicit `doctor --fix` repair. */
doctorOnly?: boolean;
detectLegacyState: (params: {
config: OpenClawConfig;
env: NodeJS.ProcessEnv;
@@ -216,6 +225,7 @@ function coercePluginDoctorStateMigrations(value: unknown): PluginDoctorStateMig
return value.filter(isPluginDoctorStateMigration).map((migration) => ({
id: migration.id.trim(),
label: migration.label.trim(),
doctorOnly: migration.doctorOnly === true ? true : undefined,
detectLegacyState: migration.detectLegacyState,
migrateLegacyState: migration.migrateLegacyState,
}));
+9
View File
@@ -10,6 +10,14 @@ export type Generated<T> =
? ColumnType<S, I | undefined, U>
: ColumnType<T, T | undefined, T>;
export interface AcpParentStreamEvents {
created_at: number;
event_json: string;
run_id: string;
seq: number;
session_id: string;
}
export interface AuthProfileState {
state_json: string;
state_key: string;
@@ -278,6 +286,7 @@ export interface TranscriptEvents {
}
export interface DB {
acp_parent_stream_events: AcpParentStreamEvents;
auth_profile_state: AuthProfileState;
auth_profile_store: AuthProfileStore;
cache_entries: CacheEntries;
+45 -4
View File
@@ -513,8 +513,8 @@ describe("openclaw agent database", () => {
expect(registered?.sizeBytes).toBeGreaterThan(0);
});
it("folds additive heartbeat storage into schema version 11", () => {
expect(OPENCLAW_AGENT_SCHEMA_VERSION).toBe(11);
it("keeps additive heartbeat repair while upgrading schema version 11", () => {
expect(OPENCLAW_AGENT_SCHEMA_VERSION).toBe(12);
const stateDir = createTempStateDir();
const env = { OPENCLAW_STATE_DIR: stateDir };
const opened = openOpenClawAgentDatabase({ agentId: "worker-1", env });
@@ -536,12 +536,12 @@ describe("openclaw agent database", () => {
.prepare("SELECT name FROM sqlite_schema WHERE type = 'table' AND name = ?")
.get("heartbeat_outcomes"),
).toEqual({ name: "heartbeat_outcomes" });
expect(readSqliteNumberPragma(reopened.db, "user_version")).toBe(11);
expect(readSqliteNumberPragma(reopened.db, "user_version")).toBe(OPENCLAW_AGENT_SCHEMA_VERSION);
expect(
reopened.db
.prepare("SELECT schema_version FROM schema_meta WHERE meta_key = 'primary'")
.get(),
).toEqual({ schema_version: 11 });
).toEqual({ schema_version: OPENCLAW_AGENT_SCHEMA_VERSION });
});
it("upgrades version 10 with agent state intact and adds lease storage", () => {
@@ -589,6 +589,47 @@ describe("openclaw agent database", () => {
).toEqual({ name: "state_leases" });
});
it("upgrades version 11 with agent state intact and adds ACP parent-stream storage", () => {
const stateDir = createTempStateDir();
const env = { OPENCLAW_STATE_DIR: stateDir };
const opened = openOpenClawAgentDatabase({ agentId: "worker-1", env });
const databasePath = opened.path;
opened.db
.prepare(
"INSERT INTO auth_profile_state (state_key, state_json, updated_at) VALUES (?, ?, ?)",
)
.run("last-good", '{"profile":"primary"}', 10);
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
const { DatabaseSync } = requireNodeSqlite();
const legacy = new DatabaseSync(databasePath);
legacy.exec(`
DROP INDEX idx_agent_acp_parent_stream_run;
DROP TABLE acp_parent_stream_events;
PRAGMA user_version = 11;
UPDATE schema_meta SET schema_version = 11 WHERE meta_key = 'primary';
`);
legacy.close();
const migrated = openOpenClawAgentDatabase({ agentId: "worker-1", env });
expect(migrated.db.prepare("PRAGMA user_version").get()).toEqual({
user_version: OPENCLAW_AGENT_SCHEMA_VERSION,
});
expect(
migrated.db
.prepare("SELECT state_json FROM auth_profile_state WHERE state_key = ?")
.get("last-good"),
).toEqual({ state_json: '{"profile":"primary"}' });
expect(
migrated.db
.prepare(
"SELECT name FROM sqlite_schema WHERE type = 'table' AND name = 'acp_parent_stream_events'",
)
.get(),
).toEqual({ name: "acp_parent_stream_events" });
});
it("migrates version 8 tables to STRICT without losing agent state", () => {
const stateDir = createTempStateDir();
const env = { OPENCLAW_STATE_DIR: stateDir };
+2 -1
View File
@@ -83,6 +83,7 @@ export { resolveOpenClawAgentSqlitePath } from "./openclaw-agent-db.paths.js";
* per pathname, protected with private file modes, and registered in the shared
* OpenClaw state database for discovery and maintenance.
*/
// v12 = session-owned ACP parent-stream events.
// v11 = agent-scoped runtime leases, durable delivery operations, canonical
// external conversation addresses, and bounded per-session heartbeat outcome context.
// v10 = materialized active transcript paths.
@@ -93,7 +94,7 @@ export { resolveOpenClawAgentSqlitePath } from "./openclaw-agent-db.paths.js";
// The v4 session/transcript flip and main's v2 memory-identity
// change is folded in structure-gated (migrateMemoryIndexSourcesIdentity), so
// v2 main DBs and pre-merge v4 flip DBs both converge on this schema.
export const OPENCLAW_AGENT_SCHEMA_VERSION = 11;
export const OPENCLAW_AGENT_SCHEMA_VERSION = 12;
const OPENCLAW_AGENT_DB_DIR_MODE = 0o700;
const OPENCLAW_AGENT_DB_FILE_MODE = 0o600;
const OPENCLAW_AGENT_DB_SLOW_OPEN_MS = 1_000;
@@ -222,6 +222,19 @@ CREATE INDEX IF NOT EXISTS idx_agent_trajectory_runtime_run
ON trajectory_runtime_events(session_id, run_id, seq)
WHERE run_id IS NOT NULL;
CREATE TABLE IF NOT EXISTS acp_parent_stream_events (
session_id TEXT NOT NULL,
run_id TEXT NOT NULL,
seq INTEGER NOT NULL,
event_json TEXT NOT NULL,
created_at INTEGER NOT NULL,
PRIMARY KEY (session_id, run_id, seq),
FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE
) STRICT;
CREATE INDEX IF NOT EXISTS idx_agent_acp_parent_stream_run
ON acp_parent_stream_events(run_id, seq);
CREATE TABLE IF NOT EXISTS transcript_event_identities (
session_id TEXT NOT NULL,
event_id TEXT NOT NULL,
+13
View File
@@ -217,6 +217,19 @@ CREATE INDEX IF NOT EXISTS idx_agent_trajectory_runtime_run
ON trajectory_runtime_events(session_id, run_id, seq)
WHERE run_id IS NOT NULL;
CREATE TABLE IF NOT EXISTS acp_parent_stream_events (
session_id TEXT NOT NULL,
run_id TEXT NOT NULL,
seq INTEGER NOT NULL,
event_json TEXT NOT NULL,
created_at INTEGER NOT NULL,
PRIMARY KEY (session_id, run_id, seq),
FOREIGN KEY (session_id) REFERENCES sessions(session_id) ON DELETE CASCADE
) STRICT;
CREATE INDEX IF NOT EXISTS idx_agent_acp_parent_stream_run
ON acp_parent_stream_events(run_id, seq);
CREATE TABLE IF NOT EXISTS transcript_event_identities (
session_id TEXT NOT NULL,
event_id TEXT NOT NULL,
+1
View File
@@ -501,6 +501,7 @@ export interface DiagnosticEvents {
event_key: string;
payload_json: string;
scope: string;
sequence: Generated<number>;
}
export interface DiagnosticStabilityBundles {
+50
View File
@@ -2336,6 +2336,56 @@ describe("openclaw state database", () => {
);
});
it("backfills diagnostic event sequences in legacy creation order", () => {
const stateDir = createTempStateDir();
const options = { env: { OPENCLAW_STATE_DIR: stateDir } };
const databasePath = openOpenClawStateDatabase(options).path;
closeOpenClawStateDatabaseForTest();
const { DatabaseSync } = requireNodeSqlite();
const legacyDb = new DatabaseSync(databasePath);
legacyDb.exec(`
DROP INDEX idx_diagnostic_events_scope_sequence;
ALTER TABLE diagnostic_events DROP COLUMN sequence;
CREATE INDEX idx_diagnostic_events_scope_created
ON diagnostic_events(scope, created_at, event_key);
INSERT INTO diagnostic_events (scope, event_key, payload_json, created_at) VALUES
('alpha', 'late', '{}', 20),
('alpha', 'tie-first', '{}', 10),
('alpha', 'tie-second', '{}', 10),
('beta', 'only', '{}', 30);
`);
legacyDb.close();
const reopened = openOpenClawStateDatabase(options);
const rows = reopened.db
.prepare(
`SELECT scope, event_key, sequence
FROM diagnostic_events
ORDER BY scope, sequence`,
)
.all();
expect(rows).toEqual([
{ scope: "alpha", event_key: "tie-first", sequence: 1 },
{ scope: "alpha", event_key: "tie-second", sequence: 2 },
{ scope: "alpha", event_key: "late", sequence: 3 },
{ scope: "beta", event_key: "only", sequence: 1 },
]);
const indexes = reopened.db.prepare("PRAGMA index_list(diagnostic_events)").all() as Array<{
name?: unknown;
}>;
expect(indexes).toEqual(
expect.arrayContaining([
expect.objectContaining({ name: "idx_diagnostic_events_scope_sequence" }),
]),
);
expect(indexes).not.toEqual(
expect.arrayContaining([
expect.objectContaining({ name: "idx_diagnostic_events_scope_created" }),
]),
);
});
it("adds relay origins to existing APNs registration tables", () => {
const stateDir = createTempStateDir();
const options = { env: { OPENCLAW_STATE_DIR: stateDir } };
+28
View File
@@ -94,6 +94,7 @@ const OPENCLAW_STATE_CANONICAL_UNIQUE_INDEXES = [
] as const satisfies readonly CanonicalSqliteUniqueIndex[];
const OPENCLAW_STATE_MAINTENANCE_SCHEMA_COMPATIBILITY = {
allowedColumnDefinitions: {
"diagnostic_events.sequence": ["sequence INTEGER NOT NULL DEFAULT 0"],
"commitments.attempts": ["attempts INTEGER NOT NULL DEFAULT 0"],
"commitments.confidence": ["confidence REAL NOT NULL DEFAULT 0"],
"commitments.created_at_ms": ["created_at_ms INTEGER NOT NULL DEFAULT 0"],
@@ -1446,6 +1447,33 @@ function backfillDeliveryQueueEntriesFromEntryJson(db: DatabaseSync): void {
// The caller owns the state.schema.ensure transaction so every probe, DDL
// change, and backfill observes one authoritative schema across processes.
function ensureAdditiveStateColumns(db: DatabaseSync): void {
const addedDiagnosticEventSequence = ensureColumn(
db,
"diagnostic_events",
"sequence INTEGER NOT NULL DEFAULT 0",
);
if (addedDiagnosticEventSequence) {
// Preserve the legacy (created_at, rowid) order before the new sequence
// index becomes authoritative, including stable ties within each scope.
db.exec(`
WITH ranked AS (
SELECT
rowid AS event_rowid,
ROW_NUMBER() OVER (
PARTITION BY scope
ORDER BY created_at ASC, rowid ASC
) AS sequence
FROM diagnostic_events
)
UPDATE diagnostic_events
SET sequence = (
SELECT ranked.sequence
FROM ranked
WHERE ranked.event_rowid = diagnostic_events.rowid
);
`);
}
db.exec("DROP INDEX IF EXISTS idx_diagnostic_events_scope_created;");
ensureColumn(db, "worktrees", "provisioned_paths_json TEXT");
ensureColumn(db, "node_host_config", "gateway_context_path TEXT");
ensureColumn(db, "node_host_config", "installed_apps_sharing INTEGER NOT NULL DEFAULT 0");
+7 -4
View File
@@ -18,13 +18,16 @@ import type { DB as OpenClawStateKyselyDatabase } from "./openclaw-state-db.gene
import {
openOpenClawStateDatabase,
runOpenClawStateWriteTransaction,
type OpenClawStateDatabaseOptions,
} from "./openclaw-state-db.js";
type LeaseDatabase = Pick<OpenClawStateKyselyDatabase, "state_leases">;
type AgentLeaseDatabase = Pick<OpenClawAgentKyselyDatabase, "state_leases">;
type LeaseKysely = ReturnType<typeof getNodeSqliteKysely<LeaseDatabase>>;
type OpenClawStateLeaseDatabase = { scope: "shared" } | { scope: "agent"; agentId: string };
type OpenClawStateLeaseDatabase =
| { scope: "shared"; options?: OpenClawStateDatabaseOptions }
| { scope: "agent"; agentId: string };
type OpenClawStateLeaseOptions = {
scope: string;
@@ -175,11 +178,11 @@ function withLeaseWriteTransaction<T>(
busyTimeoutMs = LEASE_DB_BUSY_TIMEOUT_MS,
): T {
if (database.scope === "shared") {
const stateDatabase = openOpenClawStateDatabase();
const stateDatabase = openOpenClawStateDatabase(database.options);
const run = () =>
runOpenClawStateWriteTransaction(
({ db }) => operation(db, getNodeSqliteKysely<LeaseDatabase>(db)),
{},
database.options,
{ operationLabel, busyTimeoutMs },
);
return withBusyTimeout(stateDatabase.db, busyTimeoutMs, run);
@@ -200,7 +203,7 @@ function withLeaseRead<T>(
): T {
const sqlite =
database.scope === "shared"
? openOpenClawStateDatabase().db
? openOpenClawStateDatabase(database.options).db
: openOpenClawAgentDatabase({ agentId: database.agentId }).db;
return operation(sqlite, getNodeSqliteKysely<LeaseDatabase>(sqlite));
}
+3 -2
View File
@@ -27,11 +27,12 @@ CREATE TABLE IF NOT EXISTS diagnostic_events (
event_key TEXT NOT NULL,
payload_json TEXT NOT NULL,
created_at INTEGER NOT NULL,
sequence INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (scope, event_key)
) STRICT;
CREATE INDEX IF NOT EXISTS idx_diagnostic_events_scope_created
ON diagnostic_events(scope, created_at, event_key);
CREATE INDEX IF NOT EXISTS idx_diagnostic_events_scope_sequence
ON diagnostic_events(scope, sequence, event_key);
CREATE TABLE IF NOT EXISTS skill_usage (
skill_file TEXT NOT NULL PRIMARY KEY,
+3 -2
View File
@@ -22,11 +22,12 @@ CREATE TABLE IF NOT EXISTS diagnostic_events (
event_key TEXT NOT NULL,
payload_json TEXT NOT NULL,
created_at INTEGER NOT NULL,
sequence INTEGER NOT NULL DEFAULT 0,
PRIMARY KEY (scope, event_key)
) STRICT;
CREATE INDEX IF NOT EXISTS idx_diagnostic_events_scope_created
ON diagnostic_events(scope, created_at, event_key);
CREATE INDEX IF NOT EXISTS idx_diagnostic_events_scope_sequence
ON diagnostic_events(scope, sequence, event_key);
CREATE TABLE IF NOT EXISTS skill_usage (
skill_file TEXT NOT NULL PRIMARY KEY,
+14
View File
@@ -0,0 +1,14 @@
import { createSqliteAuditRecordStore } from "../infra/sqlite-audit-record-store.js";
import {
SYSTEM_AGENT_AUDIT_MAX_ENTRIES,
SYSTEM_AGENT_AUDIT_SCOPE,
type SystemAgentAuditEntry,
} from "./audit.js";
export function listSystemAgentAuditEntriesForTests(params?: { env?: NodeJS.ProcessEnv }) {
return createSqliteAuditRecordStore<SystemAgentAuditEntry>({
scope: SYSTEM_AGENT_AUDIT_SCOPE,
maxEntries: SYSTEM_AGENT_AUDIT_MAX_ENTRIES,
...(params?.env ? { env: params.env } : {}),
}).entries();
}
+15 -9
View File
@@ -1,13 +1,15 @@
// OpenClaw audit tests cover filesystem-backed rescue audit scenarios.
import fs from "node:fs/promises";
// OpenClaw audit tests cover SQLite-backed rescue audit scenarios.
import { afterEach, describe, expect, it, vi } from "vitest";
import { resetPluginStateStoreForTests } from "../plugin-state/plugin-state-store.js";
import { withTempDir } from "../test-helpers/temp-dir.js";
import { appendSystemAgentAuditEntry, resolveSystemAgentAuditPath } from "./audit.js";
import { appendSystemAgentAuditEntry, SYSTEM_AGENT_AUDIT_STORE_LABEL } from "./audit.js";
import { listSystemAgentAuditEntriesForTests } from "./audit.test-support.js";
describe("OpenClaw audit log", () => {
const previousStateDir = process.env.OPENCLAW_STATE_DIR;
afterEach(() => {
resetPluginStateStoreForTests();
if (previousStateDir === undefined) {
delete process.env.OPENCLAW_STATE_DIR;
} else {
@@ -15,21 +17,25 @@ describe("OpenClaw audit log", () => {
}
});
it("writes jsonl records under the OpenClaw audit dir", async () => {
it("writes records into shared SQLite state", async () => {
await withTempDir({ prefix: "openclaw-audit-" }, async (tempDir) => {
vi.stubEnv("OPENCLAW_STATE_DIR", tempDir);
const auditPath = await appendSystemAgentAuditEntry({
const auditStore = await appendSystemAgentAuditEntry({
operation: "config.setDefaultModel",
summary: "Set default model to openai/gpt-5.2",
configHashBefore: "before",
configHashAfter: "after",
});
expect(auditPath).toBe(resolveSystemAgentAuditPath());
const lines = (await fs.readFile(auditPath, "utf8")).trim().split("\n");
expect(lines).toHaveLength(1);
const entry = JSON.parse(lines[0] ?? "{}") as Record<string, unknown>;
expect(auditStore).toBe(SYSTEM_AGENT_AUDIT_STORE_LABEL);
const records = listSystemAgentAuditEntriesForTests();
expect(records).toHaveLength(1);
const entry = records[0]?.value;
expect(entry).toBeDefined();
if (!entry) {
throw new Error("expected persisted system-agent audit entry");
}
expect(entry.operation).toBe("config.setDefaultModel");
expect(entry.summary).toBe("Set default model to openai/gpt-5.2");
expect(entry.configHashBefore).toBe("before");
+26 -25
View File
@@ -1,16 +1,15 @@
// OpenClaw audit helpers append JSONL records for approved local-state changes.
import fs from "node:fs/promises";
import path from "node:path";
import { resolveStateDir } from "../config/paths.js";
import { appendRegularFile } from "../infra/fs-safe.js";
// OpenClaw audit helpers persist approved local-state changes.
import { randomUUID } from "node:crypto";
import { createSqliteAuditRecordStore } from "../infra/sqlite-audit-record-store.js";
import { redactSecrets } from "../logging/redact.js";
/**
* Append-only audit log helpers for OpenClaw writes.
*
* Discovery and read-only commands stay quiet; persistent operations append a
* JSONL entry under the state directory with config hashes and redacted details.
* SQLite entry under the shared state directory with config hashes and redacted details.
*/
type SystemAgentAuditEntry = {
export type SystemAgentAuditEntry = {
timestamp: string;
operation: string;
summary: string;
@@ -20,30 +19,32 @@ type SystemAgentAuditEntry = {
details?: Record<string, unknown>;
};
/** Resolve the JSONL audit path for OpenClaw persistent operations. */
export function resolveSystemAgentAuditPath(
env: NodeJS.ProcessEnv = process.env,
stateDir = resolveStateDir(env),
): string {
return path.join(stateDir, "audit", "system-agent.jsonl");
export const SYSTEM_AGENT_AUDIT_SCOPE = "system-agent-audit";
export const SYSTEM_AGENT_AUDIT_MAX_ENTRIES = 50_000;
export const SYSTEM_AGENT_AUDIT_STORE_LABEL =
"SQLite diagnostic_events/system-agent-audit state (latest 50000 rows)";
function openSystemAgentAuditStore(env?: NodeJS.ProcessEnv) {
return createSqliteAuditRecordStore<SystemAgentAuditEntry>({
scope: SYSTEM_AGENT_AUDIT_SCOPE,
maxEntries: SYSTEM_AGENT_AUDIT_MAX_ENTRIES,
...(env ? { env } : {}),
});
}
/** Append one OpenClaw audit entry and return the file path written. */
/** Append one OpenClaw audit entry and return its SQLite owner label. */
export async function appendSystemAgentAuditEntry(
entry: Omit<SystemAgentAuditEntry, "timestamp">,
opts: { env?: NodeJS.ProcessEnv; auditPath?: string } = {},
opts: { env?: NodeJS.ProcessEnv } = {},
): Promise<string> {
const auditPath = opts.auditPath ?? resolveSystemAgentAuditPath(opts.env);
await fs.mkdir(path.dirname(auditPath), { recursive: true });
const line = JSON.stringify({
const record = redactSecrets({
timestamp: new Date().toISOString(),
...entry,
} satisfies SystemAgentAuditEntry);
// Audit writes reject symlinked parents so approval records cannot be redirected silently.
await appendRegularFile({
filePath: auditPath,
content: `${line}\n`,
rejectSymlinkParents: true,
});
return auditPath;
openSystemAgentAuditStore(opts.env).register(
`${record.timestamp}:${randomUUID()}`,
record,
Date.parse(record.timestamp),
);
return SYSTEM_AGENT_AUDIT_STORE_LABEL;
}
+2 -2
View File
@@ -5,7 +5,7 @@ import type { RuntimeEnv } from "../runtime.js";
import { resolveUserPath, shortenHomePath } from "../utils.js";
import { t } from "../wizard/i18n/index.js";
import { isReservedSystemAgentId } from "./agent-id.js";
import { resolveSystemAgentAuditPath } from "./audit.js";
import { SYSTEM_AGENT_AUDIT_STORE_LABEL } from "./audit.js";
import {
CONFIG_GET_OUTPUT_MAX_CHARS,
CONFIG_SCHEMA_CHILDREN_MAX,
@@ -108,7 +108,7 @@ export async function executeSystemAgentOperation(
return { applied: false };
}
case "audit":
runtime.log(`Audit log: ${resolveSystemAgentAuditPath()}`);
runtime.log(`Audit state: ${SYSTEM_AGENT_AUDIT_STORE_LABEL}`);
runtime.log("Only applied writes/actions are recorded; discovery stays quiet.");
return { applied: false };
case "config-validate": {
+7 -11
View File
@@ -3,19 +3,17 @@ import fs from "node:fs/promises";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import { resetPluginStateStoreForTests } from "../plugin-state/plugin-state-store.js";
import { captureEnv, setTestEnvValue } from "../test-utils/env.js";
import { listSystemAgentAuditEntriesForTests } from "./audit.test-support.js";
import { SystemAgentInferenceUnavailableError } from "./inference-error.js";
import { executeSystemAgentOperation, isPersistentSystemAgentOperation } from "./operations.js";
import { createSystemAgentTestRuntime } from "./system-agent.test-helpers.js";
type TestConfig = Record<string, unknown>;
function parseLastJsonLine(raw: string): unknown {
const lastLine = raw.trim().split("\n").at(-1);
if (!lastLine) {
throw new Error("Expected audit log to contain at least one JSON line");
}
return JSON.parse(lastLine) as unknown;
function readLastAuditEntry(): unknown {
return listSystemAgentAuditEntriesForTests().at(-1)?.value;
}
function requireRecord(value: unknown, label: string): Record<string, unknown> {
@@ -172,6 +170,7 @@ describe("parseSystemAgentOperation", () => {
});
afterEach(() => {
resetPluginStateStoreForTests();
stateDirSnapshot?.restore();
vi.unstubAllEnvs();
});
@@ -232,8 +231,7 @@ describe("parseSystemAgentOperation", () => {
{ commit: expect.any(Function) },
);
expect(lines.join("\n")).toContain("Default model: openai/gpt-5.5 (verified and kept)");
const auditPath = path.join(tempDir, "audit", "system-agent.jsonl");
const audit = JSON.parse((await fs.readFile(auditPath, "utf8")).trim());
const audit = readLastAuditEntry();
expectAuditRecord(
audit,
{
@@ -556,9 +554,7 @@ describe("parseSystemAgentOperation", () => {
expect(persisted.channels).toEqual({ telegram: { enabled: true } });
expect(lines.join("\n")).toContain("Default model: openai/gpt-5.5");
const audit = parseLastJsonLine(
await fs.readFile(path.join(tempDir, "audit", "system-agent.jsonl"), "utf8"),
);
const audit = readLastAuditEntry();
expectAuditRecord(
audit,
{
+12 -10
View File
@@ -3,8 +3,10 @@ import fs from "node:fs/promises";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import { resetPluginStateStoreForTests } from "../plugin-state/plugin-state-store.js";
import type { RuntimeEnv } from "../runtime.js";
import { captureEnv, setTestEnvValue } from "../test-utils/env.js";
import { listSystemAgentAuditEntriesForTests } from "./audit.test-support.js";
import {
describeSystemAgentPersistentOperation,
executeSystemAgentOperation,
@@ -38,6 +40,10 @@ function expectAuditRecord(
expectRecordFields(requireRecord(auditRecord.details, "audit details"), detailFields);
}
function readLastAuditEntry(): unknown {
return listSystemAgentAuditEntriesForTests().at(-1)?.value;
}
function requireFirstMockCall(mock: unknown, label: string): unknown[] {
const call = (mock as { mock?: { calls?: unknown[][] } }).mock?.calls?.[0];
if (!call) {
@@ -181,6 +187,7 @@ describe("parseSystemAgentOperation", () => {
});
afterEach(() => {
resetPluginStateStoreForTests();
stateDirSnapshot?.restore();
vi.unstubAllEnvs();
});
@@ -624,8 +631,7 @@ describe("parseSystemAgentOperation", () => {
cliOptions: {},
});
expect(lines.join("\n")).toContain("[openclaw] done: config.set");
const auditPath = path.join(tempDir, "audit", "system-agent.jsonl");
const audit = JSON.parse((await fs.readFile(auditPath, "utf8")).trim());
const audit = readLastAuditEntry();
expectAuditRecord(
audit,
{ operation: "config.set", summary: "Set config gateway.port" },
@@ -637,7 +643,7 @@ describe("parseSystemAgentOperation", () => {
);
});
it("reports an audit failure without claiming the committed operation failed", async () => {
it("records SQLite audit state despite a retired audit-directory symlink", async () => {
const tempDir = opTempDirs.make("openclaw-audit-warning-");
setTestEnvValue("OPENCLAW_STATE_DIR", tempDir);
const redirectedAuditDir = path.join(tempDir, "redirected-audit");
@@ -654,9 +660,7 @@ describe("parseSystemAgentOperation", () => {
expect(result.applied).toBe(true);
expect(runConfigSet).toHaveBeenCalledOnce();
expect(lines.join("\n")).toContain(
"Set config gateway.port, but OpenClaw could not record its audit entry:",
);
expect(readLastAuditEntry()).toMatchObject({ operation: "config.set" });
expect(lines.join("\n")).toContain("[openclaw] done: config.set");
});
@@ -691,8 +695,7 @@ describe("parseSystemAgentOperation", () => {
},
});
expect(lines.join("\n")).toContain("[openclaw] done: config.setRef");
const auditPath = path.join(tempDir, "audit", "system-agent.jsonl");
const audit = JSON.parse((await fs.readFile(auditPath, "utf8")).trim());
const audit = readLastAuditEntry();
expectAuditRecord(
audit,
{
@@ -938,8 +941,7 @@ describe("parseSystemAgentOperation", () => {
expect(installCall[0]).toBe("clawhub:openclaw-demo");
expectRuntimeArg(installCall[1]);
expect(lines.join("\n")).toContain("[openclaw] done: plugin.install");
const auditPath = path.join(tempDir, "audit", "system-agent.jsonl");
const audit = JSON.parse((await fs.readFile(auditPath, "utf8")).trim());
const audit = readLastAuditEntry();
expectAuditRecord(
audit,
{
+8 -5
View File
@@ -5,8 +5,10 @@ import { afterEach, describe, expect, it } from "vitest";
import type { CommandContext } from "../auto-reply/reply/commands-types.js";
import { clearConfigCache } from "../config/config.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { resetPluginStateStoreForTests } from "../plugin-state/plugin-state-store.js";
import { withTempDir } from "../test-helpers/temp-dir.js";
import { deleteTestEnvValue, setTestEnvValue } from "../test-utils/env.js";
import { listSystemAgentAuditEntriesForTests } from "./audit.test-support.js";
import { runSystemAgentRescueMessage } from "./rescue-message.js";
const originalStateDir = process.env.OPENCLAW_STATE_DIR;
@@ -53,6 +55,7 @@ async function runRescue(params: {
describeLive("OpenClaw live rescue channel smoke", () => {
afterEach(() => {
resetPluginStateStoreForTests();
clearConfigCache();
if (originalStateDir === undefined) {
deleteTestEnvValue("OPENCLAW_STATE_DIR");
@@ -105,11 +108,11 @@ describeLive("OpenClaw live rescue channel smoke", () => {
throw new Error("expected default model object");
}
expect(defaultModel.primary).toBe("openai/gpt-5.5");
const auditPath = path.join(tempDir, "audit", "system-agent.jsonl");
const auditLines = (await fs.readFile(auditPath, "utf8")).trim().split("\n");
expect(auditLines.some((line) => line.includes('"operation":"config.setDefaultModel"'))).toBe(
true,
);
expect(
listSystemAgentAuditEntriesForTests().some(
(entry) => entry.value.operation === "config.setDefaultModel",
),
).toBe(true);
});
});
});
+11 -9
View File
@@ -11,6 +11,7 @@ import {
} from "../plugin-state/plugin-state-store.js";
import type { RuntimeEnv } from "../runtime.js";
import { withEnvAsync } from "../test-utils/env.js";
import { listSystemAgentAuditEntriesForTests } from "./audit.test-support.js";
import { extractSystemAgentRescueMessage, runSystemAgentRescueMessage } from "./rescue-message.js";
let tempRoot = "";
@@ -18,6 +19,10 @@ let tempDirId = 0;
type TestConfig = Record<string, unknown>;
function readLastAuditEntry(): Record<string, unknown> {
return (listSystemAgentAuditEntriesForTests().at(-1)?.value ?? {}) as Record<string, unknown>;
}
const mockConfig = vi.hoisted(() => {
const state = {
path: "/tmp/openclaw.json",
@@ -531,7 +536,7 @@ describe("OpenClaw rescue message", () => {
});
it("queues and applies persistent writes through conversational approval", async () => {
await withRescueStateDir("models-", async (tempDir) => {
await withRescueStateDir("models-", async () => {
const cfg: OpenClawConfig = { systemAgent: { rescue: { enabled: true } } };
const deps = {
verifyInferenceConfig: vi.fn(async () => ({
@@ -552,8 +557,7 @@ describe("OpenClaw rescue message", () => {
};
const model = currentConfig.agents?.defaults?.model;
expect(typeof model === "string" ? model : model?.primary).toBe("openai/gpt-5.2");
const auditPath = path.join(tempDir, "audit", "system-agent.jsonl");
const audit = JSON.parse((await fs.readFile(auditPath, "utf8")).trim()) as {
const audit = readLastAuditEntry() as {
details?: { rescue?: boolean; channel?: string; accountId?: string; senderId?: string };
};
expect(audit.details?.rescue).toBe(true);
@@ -564,7 +568,7 @@ describe("OpenClaw rescue message", () => {
});
it("queues and applies gateway restart through conversational approval", async () => {
await withRescueStateDir("gateway-", async (tempDir) => {
await withRescueStateDir("gateway-", async () => {
const cfg: OpenClawConfig = { systemAgent: { rescue: { enabled: true } } };
const deps = { runGatewayRestart: vi.fn(async () => {}) };
@@ -576,8 +580,7 @@ describe("OpenClaw rescue message", () => {
);
expect(deps.runGatewayRestart).toHaveBeenCalledTimes(1);
const auditPath = path.join(tempDir, "audit", "system-agent.jsonl");
const audit = JSON.parse((await fs.readFile(auditPath, "utf8")).trim()) as {
const audit = readLastAuditEntry() as {
operation?: string;
details?: { rescue?: boolean; channel?: string; senderId?: string };
};
@@ -658,7 +661,7 @@ describe("OpenClaw rescue message", () => {
});
it("queues and applies agent creation through conversational approval", async () => {
await withRescueStateDir("agent-", async (tempDir) => {
await withRescueStateDir("agent-", async () => {
const cfg: OpenClawConfig = { systemAgent: { rescue: { enabled: true } } };
const deps = { runAgentsAdd: vi.fn(async () => {}) };
@@ -687,8 +690,7 @@ describe("OpenClaw rescue message", () => {
});
expect(agentRuntime).toBeTypeOf("object");
expect(agentOptions).toEqual({ hasFlags: true });
const auditPath = path.join(tempDir, "audit", "system-agent.jsonl");
const audit = JSON.parse((await fs.readFile(auditPath, "utf8")).trim()) as {
const audit = readLastAuditEntry() as {
operation?: string;
details?: {
rescue?: boolean;
@@ -7,7 +7,13 @@ import path from "node:path";
import { shouldStartOnboardingForFreshInstall } from "../../../../dist/cli/run-main.js";
import { clearConfigCache } from "../../../../dist/config/config.js";
import type { OpenClawConfig } from "../../../../dist/config/types.openclaw.js";
import { createSqliteAuditRecordStore } from "../../../../dist/infra/sqlite-audit-record-store.js";
import type { RuntimeEnv } from "../../../../dist/runtime.js";
import {
SYSTEM_AGENT_AUDIT_MAX_ENTRIES,
SYSTEM_AGENT_AUDIT_SCOPE,
type SystemAgentAuditEntry,
} from "../../../../dist/system-agent/audit.js";
import {
activateSetupInference,
verifySetupInference,
@@ -367,10 +373,15 @@ async function main() {
"OpenClaw persisted an unrelated ambient credential",
);
const auditPath = path.join(stateDir, "audit", "system-agent.jsonl");
const audit = (await fs.readFile(auditPath, "utf8")).trim();
const audit = createSqliteAuditRecordStore<SystemAgentAuditEntry>({
scope: SYSTEM_AGENT_AUDIT_SCOPE,
maxEntries: SYSTEM_AGENT_AUDIT_MAX_ENTRIES,
}).entries();
for (const operation of spec.auditOperations) {
assert(audit.includes(`"operation":"${operation}"`), `${operation} audit entry missing`);
assert(
audit.some((entry) => entry.value.operation === operation),
`${operation} audit entry missing`,
);
}
console.log("OpenClaw first-run Docker E2E passed");
@@ -315,6 +315,23 @@ describe("check-database-first-legacy-stores", () => {
expect(violations).toEqual([{ kind: "legacy store filesystem write", line: 4 }]);
});
it("flags runtime writes to retired core audit JSONL stores", () => {
const violations = collectDatabaseFirstLegacyStoreViolations(
`
import { promises as fs } from "node:fs";
import path from "node:path";
await fs.appendFile(path.join(stateDir, "logs", "config-audit.jsonl"), "{}\\n");
await fs.appendFile(path.join(stateDir, "audit", "system-agent.jsonl"), "{}\\n");
`,
"src/infra/audit-writer.ts",
);
expect(violations).toEqual([
{ kind: "legacy store filesystem write", line: 4 },
{ kind: "legacy store filesystem write", line: 5 },
]);
});
it("flags runtime writes to retired managed-image record JSON", () => {
const violations = collectDatabaseFirstLegacyStoreViolations(
`
+1 -1
View File
@@ -2868,7 +2868,7 @@ describe("scripts/test-projects changed-target routing", () => {
includePatterns: expect.arrayContaining(["src/plugin-sdk/access-groups.test.ts"]),
}),
expect.objectContaining({
config: "test/vitest/vitest.unit-fast-isolated.config.ts",
config: "test/vitest/vitest.unit-fast-fake-timers.config.ts",
includePatterns: ["src/plugin-sdk/memory-host-events.test.ts"],
}),
expect.objectContaining({
+2
View File
@@ -334,6 +334,8 @@ function buildDockerE2eHarnessEntries(): Record<string, string> {
"commitments/runtime.test-support": "src/commitments/runtime.test-support.ts",
"commitments/store": "src/commitments/store.ts",
"config/config": "src/config/config.ts",
"infra/sqlite-audit-record-store": "src/infra/sqlite-audit-record-store.ts",
"system-agent/audit": "src/system-agent/audit.ts",
"system-agent/system-agent": "src/system-agent/system-agent.ts",
"system-agent/rescue-message": "src/system-agent/rescue-message.ts",
"system-agent/setup-inference": "src/system-agent/setup-inference.ts",