fix(sessions): restore SQLite terminal freshness

This commit is contained in:
Vincent Koc
2026-07-12 04:43:33 +02:00
parent 2236246a43
commit 2f946506bb
16 changed files with 559 additions and 73 deletions
+1
View File
@@ -30,6 +30,7 @@ Docs: https://docs.openclaw.ai
### Fixes
- **SQLite terminal session recovery:** track physical transcript mutation time in the agent database so killed or timed-out main sessions rotate when transcript writes outlive the registry update, while preserving legacy transcript mtimes during doctor import.
- **Gateway chat typecheck:** import chat event types from their owning protocol schema after the retired aggregate type module was removed, restoring full project typechecks.
- **Packaged Crabbox commands:** include the lease-freshness helper imported by the published wrapper so `crabbox:*` commands do not fail with `ERR_MODULE_NOT_FOUND` in npm installs.
- **Plugin session catalogs:** reject unknown catalog filters, report catalogs as plugin capabilities, and preserve them in SDK registration captures instead of silently returning empty results or classifying catalog-only plugins as capability-free.
@@ -1 +1 @@
4bb5f50a60c3664656d56f2be8d9884d48287f084c03959120b2825991b23f21 sqlite-session-transcript-schema-baseline.sql
5afc3eb779d73bf3a38ab2924d3d3a2095ce6e928a791d47fef7c27747cfb0f7 sqlite-session-transcript-schema-baseline.sql
+8 -8
View File
@@ -51,14 +51,14 @@ function columnBaseType(columnTypeLocal) {
function columnType(column, primaryKeyColumnCount) {
const baseType = columnBaseType(String(column.type ?? ""));
const generated =
column.dflt_value != null ||
(primaryKeyColumnCount === 1 &&
Number(column.pk) > 0 &&
String(column.type ?? "")
.toUpperCase()
.includes("INT"));
const nullable = Number(column.notnull) !== 1 && !generated;
const generatedPrimaryKey =
primaryKeyColumnCount === 1 &&
Number(column.pk) > 0 &&
String(column.type ?? "")
.toUpperCase()
.includes("INT");
const generated = column.dflt_value != null || generatedPrimaryKey;
const nullable = Number(column.notnull) !== 1 && !generatedPrimaryKey;
const valueType = nullable ? `${baseType} | null` : baseType;
return generated ? `Generated<${valueType}>` : valueType;
}
@@ -6,6 +6,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
loadExactSqliteSessionEntry,
loadSqliteTranscriptEventsSync,
readSqliteTranscriptStatsSync,
upsertSqliteSessionEntry,
} from "../config/sessions/session-accessor.sqlite.js";
import { requireNodeSqlite } from "../infra/node-sqlite.js";
@@ -156,6 +157,29 @@ describe("runDoctorSessionSqlite", () => {
expect(message?.message?.content).toEqual([{ type: "text", text: "legacy string" }]);
});
it("preserves the legacy transcript mtime as the SQLite mutation watermark", async () => {
const store = createLegacyStore();
const transcriptMtimeMs = 1_700_000_000_000;
const transcriptMtime = new Date(transcriptMtimeMs);
fs.utimesSync(store.transcriptPath, transcriptMtime, transcriptMtime);
const report = await runDoctorSessionSqlite({
env: store.env,
mode: "import",
store: store.storePath,
});
expect(report.totals).toMatchObject({ importedEntries: 1, issues: 0 });
expect(
readSqliteTranscriptStatsSync({
agentId: "main",
sessionId: "session-1",
sessionKey: "agent:main:main",
storePath: store.storePath,
}).lastMutationAtMs,
).toBe(transcriptMtimeMs);
});
it("preserves a same-generation canonical harness owner during legacy import", async () => {
const store = createLegacyStore({
entryOverrides: { lifecycleRevision: "rev-1" },
+15
View File
@@ -446,6 +446,7 @@ async function importLegacySessionRecord(
report: DoctorSessionSqliteTargetReport,
): Promise<void> {
const result = countTranscriptEvents(record);
const transcriptMtimeMs = readLegacyTranscriptMtimeMs(record);
if (result.status === "missing") {
if (markAlreadyMigratedTranscript(target, record, report)) {
return;
@@ -473,6 +474,7 @@ async function importLegacySessionRecord(
...(record.transcriptPath
? { readTranscriptEvents: createTranscriptEventPrefixReader(record.transcriptPath) }
: {}),
...(transcriptMtimeMs !== undefined ? { transcriptMtimeMs } : {}),
});
report.importedEntries += 1;
report.importedTranscriptEvents += imported.transcriptEvents;
@@ -491,6 +493,7 @@ async function importLegacySessionRecord(
...(record.transcriptPath && result.status === "ok"
? { readTranscriptEvents: createTranscriptEventReader(record.transcriptPath) }
: {}),
...(transcriptMtimeMs !== undefined ? { transcriptMtimeMs } : {}),
});
report.importedEntries += 1;
report.importedTranscriptEvents += imported.transcriptEvents;
@@ -878,6 +881,18 @@ function countTranscriptEvents(
return countTranscriptEventsForPath(record.transcriptPath);
}
function readLegacyTranscriptMtimeMs(record: LegacySessionRecord): number | undefined {
if (!record.transcriptPath) {
return undefined;
}
try {
const mtimeMs = Math.floor(fs.statSync(record.transcriptPath).mtimeMs);
return Number.isFinite(mtimeMs) && mtimeMs >= 0 ? mtimeMs : undefined;
} catch {
return undefined;
}
}
function listUnreferencedJsonlFiles(
storePath: string,
referencedPaths: readonly string[],
+166
View File
@@ -0,0 +1,166 @@
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
closeOpenClawAgentDatabasesForTest,
resolveOpenClawAgentSqlitePath,
} from "../../state/openclaw-agent-db.js";
import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js";
import {
hasTerminalMainSessionTranscriptNewerThanRegistry,
hasTerminalMainSessionTranscriptNewerThanRegistrySync,
} from "./lifecycle.js";
import { appendTranscriptEvent, loadSessionEntry, upsertSessionEntry } from "./session-accessor.js";
import type { SessionEntry } from "./types.js";
describe("terminal main session transcript freshness", () => {
let stateDir: string;
let storePath: string;
beforeEach(() => {
stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "openclaw-session-lifecycle-"));
storePath = path.join(stateDir, "agents", "main", "sessions", "sessions.json");
});
afterEach(() => {
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
fs.rmSync(stateDir, { recursive: true, force: true });
});
async function createEntry(params: {
sessionFile?: string;
sessionKey?: string;
status: SessionEntry["status"];
updatedAt: number;
}): Promise<{ entry: SessionEntry; sessionKey: string }> {
const sessionKey = params.sessionKey ?? "agent:main:main";
const sessionId = `session-${params.status ?? "ended"}-${sessionKey.replaceAll(":", "-")}`;
const sessionEntry = {
sessionFile:
params.sessionFile ??
`sqlite:main:${sessionId}:${resolveOpenClawAgentSqlitePath({
agentId: "main",
env: { OPENCLAW_STATE_DIR: stateDir },
})}`,
sessionId,
status: params.status,
updatedAt: params.updatedAt,
};
await upsertSessionEntry({ agentId: "main", sessionKey, storePath }, sessionEntry);
await appendTranscriptEvent(
{ agentId: "main", sessionId, sessionKey, storePath },
{
type: "custom",
timestamp: "1970-01-01T00:00:00.001Z",
},
);
const storedEntry = loadSessionEntry({ agentId: "main", sessionKey, storePath });
if (!storedEntry) {
throw new Error("expected session entry");
}
return {
entry: {
...storedEntry,
sessionFile: sessionEntry.sessionFile,
status: params.status,
updatedAt: params.updatedAt,
},
sessionKey,
};
}
function check(entry: SessionEntry, sessionKey: string): boolean {
return hasTerminalMainSessionTranscriptNewerThanRegistrySync({
agentId: "main",
entry,
sessionKey,
storePath,
});
}
it("uses the physical SQLite mutation watermark instead of event timestamps", async () => {
const registryTimestampMs = Date.now() - 10_000;
const { entry, sessionKey } = await createEntry({
status: "killed",
updatedAt: registryTimestampMs,
});
expect(entry.updatedAt).toBe(registryTimestampMs);
expect(check(entry, sessionKey)).toBe(true);
await expect(
hasTerminalMainSessionTranscriptNewerThanRegistry({
agentId: "main",
entry,
sessionKey,
storePath,
}),
).resolves.toBe(true);
});
it.each(["done", "failed"] as const)("keeps %s terminal sessions reusable", async (status) => {
const { entry, sessionKey } = await createEntry({
status,
updatedAt: Date.now() - 10_000,
});
expect(check(entry, sessionKey)).toBe(false);
});
it("uses SQLite freshness for entries that still contain legacy transcript paths", async () => {
const { entry, sessionKey } = await createEntry({
sessionFile: path.join(stateDir, "legacy-session.jsonl"),
status: "killed",
updatedAt: Date.now() - 10_000,
});
expect(check(entry, sessionKey)).toBe(true);
});
it("does not rotate after a same-millisecond registry write observes the mutation", async () => {
const now = 1_700_000_000_000;
const dateNow = vi.spyOn(Date, "now").mockReturnValue(now);
const { entry, sessionKey } = await createEntry({
status: "killed",
updatedAt: now,
});
expect(check(entry, sessionKey)).toBe(true);
await upsertSessionEntry({ agentId: "main", sessionKey, storePath }, entry);
const refreshed = loadSessionEntry({ agentId: "main", sessionKey, storePath });
dateNow.mockRestore();
if (!refreshed) {
throw new Error("expected refreshed session entry");
}
expect(check(refreshed, sessionKey)).toBe(false);
});
it("does not rotate non-main sessions or rows newer than the transcript", async () => {
const nonMain = await createEntry({
sessionKey: "agent:main:other",
status: "killed",
updatedAt: Date.now() - 10_000,
});
const newerRegistry = await createEntry({
status: "timeout",
updatedAt: Date.now() + 10_000,
});
await upsertSessionEntry(
{ agentId: "main", sessionKey: newerRegistry.sessionKey, storePath },
newerRegistry.entry,
);
const refreshedRegistry = loadSessionEntry({
agentId: "main",
sessionKey: newerRegistry.sessionKey,
storePath,
});
if (!refreshedRegistry) {
throw new Error("expected refreshed registry entry");
}
expect(check(nonMain.entry, nonMain.sessionKey)).toBe(false);
expect(check(refreshedRegistry, newerRegistry.sessionKey)).toBe(false);
});
});
+19 -33
View File
@@ -1,6 +1,5 @@
// Session lifecycle timestamps prefer store metadata and fall back to transcript headers.
import fs from "node:fs";
import fsp from "node:fs/promises";
import { asDateTimestampMs } from "../../shared/number-coercion.js";
import { canonicalizeMainSessionAlias } from "./main-session.js";
import {
@@ -8,6 +7,7 @@ import {
resolveSessionFilePathOptions,
type SessionFilePathOptions,
} from "./paths.js";
import { readTranscriptStatsSync } from "./session-accessor.js";
import { isTerminalSessionStatus, type SessionEntry, type SessionScope } from "./types.js";
type SessionLifecycleEntry = Pick<
@@ -242,13 +242,13 @@ export function resolveTerminalMainSessionTranscriptRegistryCheck(
return { sessionId, registryTimestampMs };
}
function isTranscriptMtimeNewerThanRegistry(params: {
transcriptMtimeMs: number;
function isTranscriptMutationNewerThanRegistry(params: {
transcriptMutationAtMs: number;
registryTimestampMs: number;
}): boolean {
const transcriptMtimeMs = Math.floor(params.transcriptMtimeMs);
const transcriptMutationAtMs = Math.floor(params.transcriptMutationAtMs);
const registryTimestampMs = Math.floor(params.registryTimestampMs);
return Number.isFinite(transcriptMtimeMs) && transcriptMtimeMs > registryTimestampMs;
return Number.isFinite(transcriptMutationAtMs) && transcriptMutationAtMs > registryTimestampMs;
}
export function hasTerminalMainSessionTranscriptNewerThanRegistrySync(
@@ -258,16 +258,20 @@ export function hasTerminalMainSessionTranscriptNewerThanRegistrySync(
if (!check) {
return false;
}
const pathOptions = resolveSessionFilePathOptions({
agentId: params.agentId,
storePath: params.storePath,
});
try {
const sessionFile = resolveSessionFilePath(check.sessionId, params.entry, pathOptions);
const stats = fs.statSync(sessionFile);
return isTranscriptMtimeNewerThanRegistry({
transcriptMtimeMs: stats.mtimeMs,
registryTimestampMs: check.registryTimestampMs,
// Runtime transcripts are SQLite-only. Legacy-looking sessionFile values still
// resolve through agent/session/store scope, so a file stat would read stale state.
const stats = readTranscriptStatsSync({
agentId: params.agentId,
sessionId: check.sessionId,
storePath: params.storePath,
});
if (stats.lastMutationAtMs === undefined) {
return false;
}
return isTranscriptMutationNewerThanRegistry({
transcriptMutationAtMs: stats.lastMutationAtMs,
registryTimestampMs: stats.lastObservedMutationAtMs ?? check.registryTimestampMs,
});
} catch {
return false;
@@ -277,23 +281,5 @@ export function hasTerminalMainSessionTranscriptNewerThanRegistrySync(
export async function hasTerminalMainSessionTranscriptNewerThanRegistry(
params: TerminalMainSessionTranscriptRegistryParams,
): Promise<boolean> {
const check = resolveTerminalMainSessionTranscriptRegistryCheck(params);
if (!check) {
return false;
}
const pathOptions = resolveSessionFilePathOptions({
agentId: params.agentId,
storePath: params.storePath,
});
try {
// Session admission owns this bounded stat as the terminal-main reconciliation gate.
const sessionFile = resolveSessionFilePath(check.sessionId, params.entry, pathOptions);
const stats = await fsp.stat(sessionFile);
return isTranscriptMtimeNewerThanRegistry({
transcriptMtimeMs: stats.mtimeMs,
registryTimestampMs: check.registryTimestampMs,
});
} catch {
return false;
}
return hasTerminalMainSessionTranscriptNewerThanRegistrySync(params);
}
@@ -63,6 +63,8 @@ export type TranscriptEvent = unknown;
export type SessionTranscriptStats = {
eventCount: number;
lastMutationAtMs?: number;
lastObservedMutationAtMs?: number;
maxSeq: number;
sizeBytes: number;
};
+150 -27
View File
@@ -297,6 +297,7 @@ export type SqliteSessionImportRowsParams = {
sessionKey: string;
entry: SessionEntry;
readTranscriptEvents?: (append: (event: TranscriptEvent) => void) => void;
transcriptMtimeMs?: number;
};
/** Summary of rows written by an internal doctor/migration import. */
@@ -1393,8 +1394,21 @@ export function readSqliteTranscriptStatsSync(
])
.where("session_id", "=", resolved.sessionId),
);
const session = executeSqliteQueryTakeFirstSync(
database.db,
db
.selectFrom("sessions")
.select(["transcript_observed_at", "transcript_updated_at"])
.where("session_id", "=", resolved.sessionId),
);
return {
eventCount: row?.event_count ?? 0,
...(session?.transcript_updated_at !== null && session?.transcript_updated_at !== undefined
? { lastMutationAtMs: session.transcript_updated_at }
: {}),
...(session?.transcript_observed_at !== null && session?.transcript_observed_at !== undefined
? { lastObservedMutationAtMs: session.transcript_observed_at }
: {}),
maxSeq: row?.max_seq ?? 0,
sizeBytes: row?.size_bytes ?? 0,
};
@@ -1608,6 +1622,9 @@ export async function deleteSqliteTranscript(scope: SessionTranscriptReadScope):
let deleted = false;
runOpenClawAgentWriteTransaction((database) => {
deleted = deleteSqliteTranscriptEventsInTransaction(database, resolved.sessionId);
if (deleted) {
touchTranscriptMutationInTransaction(database, resolved.sessionId);
}
}, toDatabaseOptions(resolved));
return deleted;
});
@@ -1689,12 +1706,25 @@ export async function importSqliteSessionRows(
if (existingEventJson.has(eventJson)) {
return;
}
if (appendTranscriptEventInTransaction(database, transcriptScope, event)) {
if (
appendTranscriptEventInTransaction(database, transcriptScope, event, {
touchMutation: false,
})
) {
existingEventJson.add(eventJson);
transcriptEvents += 1;
}
});
}
if (params.transcriptMtimeMs !== undefined) {
advanceTranscriptMutationAtInTransaction(
database,
params.entry.sessionId,
params.transcriptMtimeMs,
);
} else if (transcriptEvents > 0) {
touchTranscriptMutationInTransaction(database, params.entry.sessionId);
}
}, toDatabaseOptions(resolved));
return {
sessionId: params.entry.sessionId,
@@ -1745,9 +1775,7 @@ export async function appendSqliteTranscriptEvents(
const resolved = resolveSqliteTranscriptScope(scope);
await runExclusiveSqliteSessionWrite(resolved, async () => {
runOpenClawAgentWriteTransaction((database) => {
for (const event of events) {
appendTranscriptEventInTransaction(database, resolved, event);
}
appendTranscriptEventsInTransaction(database, resolved, events);
}, toDatabaseOptions(resolved));
});
}
@@ -3897,7 +3925,15 @@ function writeSessionEntry(
const db = getSessionKysely(database.db);
const normalizedEntry = normalizeSqliteSessionEntryTimestamp(entry);
const updatedAt = normalizedEntry.updatedAt;
const sessionRow = bindSqliteSessionRoot({ entry: normalizedEntry, sessionKey, updatedAt });
// Registry writes snapshot the current transcript watermark so recovery can
// distinguish same-millisecond transcript writes before and after this row.
const transcriptObservedAt =
readTranscriptMutationStateInTransaction(database, normalizedEntry.sessionId).updatedAt ??
updatedAt;
const sessionRow = {
...bindSqliteSessionRoot({ entry: normalizedEntry, sessionKey, updatedAt }),
transcript_observed_at: transcriptObservedAt,
};
executeSqliteQuerySync(
database.db,
db
@@ -3907,6 +3943,7 @@ function writeSessionEntry(
conflict.column("session_id").doUpdateSet({
session_key: sessionKey,
session_scope: sessionRow.session_scope,
transcript_observed_at: transcriptObservedAt,
updated_at: updatedAt,
started_at: sessionRow.started_at,
ended_at: sessionRow.ended_at,
@@ -4134,6 +4171,67 @@ function readNextTranscriptSeq(database: OpenClawAgentDatabase, sessionId: strin
return maxSeq + 1;
}
function normalizeTranscriptMutationAtMs(value: number): number | undefined {
const timestamp = Math.floor(value);
return Number.isFinite(timestamp) && timestamp >= 0 ? timestamp : undefined;
}
function readTranscriptMutationStateInTransaction(
database: OpenClawAgentDatabase,
sessionId: string,
): { observedAt: number | null; updatedAt: number | null } {
const db = getSessionKysely(database.db);
const row = executeSqliteQueryTakeFirstSync(
database.db,
db
.selectFrom("sessions")
.select(["transcript_observed_at", "transcript_updated_at"])
.where("session_id", "=", sessionId),
);
return {
observedAt: row?.transcript_observed_at ?? null,
updatedAt: row?.transcript_updated_at ?? null,
};
}
function advanceTranscriptMutationAtInTransaction(
database: OpenClawAgentDatabase,
sessionId: string,
value: number,
options: { strictly?: boolean } = {},
): void {
const transcriptUpdatedAt = normalizeTranscriptMutationAtMs(value);
if (transcriptUpdatedAt === undefined) {
return;
}
const state = readTranscriptMutationStateInTransaction(database, sessionId);
const next = options.strictly
? Math.max(transcriptUpdatedAt, (state.updatedAt ?? -1) + 1, (state.observedAt ?? -1) + 1)
: Math.max(transcriptUpdatedAt, state.updatedAt ?? 0);
if (state.updatedAt !== null && state.updatedAt >= next) {
return;
}
const db = getSessionKysely(database.db);
executeSqliteQuerySync(
database.db,
db
.updateTable("sessions")
.set({ transcript_updated_at: next })
.where("session_id", "=", sessionId),
);
}
function touchTranscriptMutationInTransaction(
database: OpenClawAgentDatabase,
sessionId: string,
): void {
const now = normalizeTranscriptMutationAtMs(Date.now());
if (now === undefined) {
return;
}
advanceTranscriptMutationAtInTransaction(database, sessionId, now, { strictly: true });
}
function deleteSqliteTranscriptEventsInTransaction(
database: OpenClawAgentDatabase,
sessionId: string,
@@ -4490,16 +4588,18 @@ function writeSqliteParentForkTranscriptInTransaction(
...(params.source.appendMode ? { appendMode: params.source.appendMode } : {}),
}
: null;
appendTranscriptEventInTransaction(database, targetScope, {
...createSessionTranscriptHeader({
cwd: params.source.cwd,
sessionId: targetScope.sessionId,
}),
parentSession: params.parentSessionFile,
});
for (const event of [...pathEntries, ...labelEntries, ...(leafEntry ? [leafEntry] : [])]) {
appendTranscriptEventInTransaction(database, targetScope, event);
}
appendTranscriptEventsInTransaction(database, targetScope, [
{
...createSessionTranscriptHeader({
cwd: params.source.cwd,
sessionId: targetScope.sessionId,
}),
parentSession: params.parentSessionFile,
},
...pathEntries,
...labelEntries,
...(leafEntry ? [leafEntry] : []),
]);
}
function forkSqliteParentTranscriptInTransaction(
@@ -4679,20 +4779,13 @@ function forkSqliteCheckpointTranscriptInTransaction(
sessionKey: params.targetSessionKey,
};
const sessionFile = formatSqliteSessionMarkerForScope(targetScope);
appendTranscriptEventInTransaction(
database,
targetScope,
appendTranscriptEventsInTransaction(database, targetScope, [
createSessionTranscriptHeader({
cwd: readTranscriptHeaderCwd(selected.rows),
sessionId,
}),
);
for (const event of selected.rows) {
if (isSessionTranscriptHeader(event)) {
continue;
}
appendTranscriptEventInTransaction(database, targetScope, event);
}
...selected.rows.filter((event) => !isSessionTranscriptHeader(event)),
]);
return {
status: "created",
sessionId,
@@ -4834,7 +4927,7 @@ function appendTranscriptEventInTransaction(
database: OpenClawAgentDatabase,
scope: ResolvedTranscriptScope,
event: TranscriptEvent,
options: { dedupeByMessageIdempotency?: boolean } = {},
options: { dedupeByMessageIdempotency?: boolean; touchMutation?: boolean } = {},
): boolean {
const db = getSessionKysely(database.db);
const createdAt = readEventTimestamp(event) ?? Date.now();
@@ -4864,6 +4957,9 @@ function appendTranscriptEventInTransaction(
created_at: createdAt,
}),
);
if (options.touchMutation !== false) {
touchTranscriptMutationInTransaction(database, scope.sessionId);
}
if (!identity) {
return true;
}
@@ -4897,6 +4993,27 @@ function appendTranscriptEventInTransaction(
return true;
}
function appendTranscriptEventsInTransaction(
database: OpenClawAgentDatabase,
scope: ResolvedTranscriptScope,
events: readonly TranscriptEvent[],
): number {
let appended = 0;
for (const event of events) {
if (
appendTranscriptEventInTransaction(database, scope, event, {
touchMutation: false,
})
) {
appended += 1;
}
}
if (appended > 0) {
touchTranscriptMutationInTransaction(database, scope.sessionId);
}
return appended;
}
function appendTranscriptEventRowInTransaction(
database: OpenClawAgentDatabase,
scope: ResolvedTranscriptScope,
@@ -5041,8 +5158,11 @@ function replaceSqliteTranscriptEventsInTransaction(
resolved: ResolvedTranscriptScope,
events: readonly TranscriptEvent[],
): void {
deleteSqliteTranscriptEventsInTransaction(database, resolved.sessionId);
const deleted = deleteSqliteTranscriptEventsInTransaction(database, resolved.sessionId);
if (events.length === 0) {
if (deleted) {
touchTranscriptMutationInTransaction(database, resolved.sessionId);
}
return;
}
ensureTranscriptSessionRoot(database, resolved, readEventTimestamp(events[0]) ?? Date.now());
@@ -5058,6 +5178,9 @@ function replaceSqliteTranscriptEventsInTransaction(
seq += 1;
}
}
if (deleted || seq > 0) {
touchTranscriptMutationInTransaction(database, resolved.sessionId);
}
}
function readTranscriptIdentityByEventId(
@@ -24,6 +24,7 @@ import {
patchSessionEntryTarget,
persistSessionResetLifecycle,
persistSessionTranscriptTurn,
readTranscriptStatsSync,
readSessionUpdatedAt,
recordInboundSessionMeta,
replaceSessionEntry,
@@ -2424,6 +2425,55 @@ describe("session accessor seam", () => {
).toBe(`sqlite:main:session-1:${path.join(tempDir, "openclaw-agent.sqlite")}`);
});
it("tracks replacement and deletion transcript mutations", async () => {
const dateNow = vi.spyOn(Date, "now").mockReturnValue(1_700_000_000_000);
const scope = {
agentId: "main",
sessionId: "session-1",
sessionKey: "agent:main:main",
storePath,
};
await upsertSessionEntry(scope, {
sessionId: scope.sessionId,
updatedAt: 10,
});
await replaceSqliteTranscriptEvents(scope, [
{ sessionId: scope.sessionId, type: "session" },
{ timestamp: "1970-01-01T00:00:00.001Z", type: "custom" },
]);
const replaced = readTranscriptStatsSync(scope);
expect(replaced).toMatchObject({
eventCount: 2,
lastMutationAtMs: expect.any(Number),
});
expect(replaced.lastMutationAtMs).toBeGreaterThanOrEqual(1_700_000_000_000);
await importSqliteSessionRows({
agentId: scope.agentId,
entry: {
sessionId: scope.sessionId,
updatedAt: 10,
},
sessionKey: scope.sessionKey,
storePath: scope.storePath,
transcriptMtimeMs: 1_600_000_000_000,
});
const imported = readTranscriptStatsSync(scope);
expect(imported.lastMutationAtMs).toBe(replaced.lastMutationAtMs);
expect(imported.lastObservedMutationAtMs).toBe(replaced.lastMutationAtMs);
await replaceSqliteTranscriptEvents(scope, []);
const cleared = readTranscriptStatsSync(scope);
dateNow.mockRestore();
expect(cleared).toMatchObject({
eventCount: 0,
lastMutationAtMs: expect.any(Number),
});
expect(cleared.lastMutationAtMs).toBeGreaterThan(imported.lastMutationAtMs ?? 0);
});
it("resolves an explicit read transcript file without agent identity", () => {
const explicitSessionFile = path.join(tempDir, "explicit-read-session.jsonl");
+2
View File
@@ -289,6 +289,8 @@ export type TranscriptEvent = unknown;
export type SessionTranscriptStats = {
eventCount: number;
lastMutationAtMs?: number;
lastObservedMutationAtMs?: number;
maxSeq: number;
sizeBytes: number;
};
+2
View File
@@ -138,6 +138,8 @@ export interface Sessions {
spawned_by: string | null;
started_at: number | null;
status: string | null;
transcript_observed_at: Generated<number | null>;
transcript_updated_at: Generated<number | null>;
updated_at: number;
}
+77 -1
View File
@@ -1027,6 +1027,82 @@ describe("openclaw agent database", () => {
});
});
it("adds transcript mutation watermarks to v4 session tables", () => {
const stateDir = createTempStateDir();
const databasePath = path.join(
stateDir,
"agents",
"worker-1",
"agent",
"openclaw-agent.sqlite",
);
fs.mkdirSync(path.dirname(databasePath), { recursive: true });
const currentSchema = fs.readFileSync(
new URL("./openclaw-agent-schema.sql", import.meta.url),
"utf8",
);
const previousSchema = currentSchema.replace(
[
" transcript_updated_at INTEGER DEFAULT NULL,\n",
" transcript_observed_at INTEGER DEFAULT NULL,\n",
].join(""),
"",
);
expect(previousSchema).not.toBe(currentSchema);
const { DatabaseSync } = requireNodeSqlite();
const db = new DatabaseSync(databasePath);
db.exec(previousSchema);
db.exec(`
INSERT INTO schema_meta
(meta_key, role, schema_version, agent_id, app_version, created_at, updated_at)
VALUES ('primary', 'agent', 4, 'worker-1', NULL, 1, 1);
INSERT INTO sessions
(session_id, session_key, created_at, updated_at)
VALUES ('session-1', 'agent:worker-1:main', 10, 20);
INSERT INTO sessions
(session_id, session_key, created_at, updated_at)
VALUES ('session-2', 'agent:worker-1:other', 10, 20);
INSERT INTO transcript_events
(session_id, seq, event_json, created_at)
VALUES ('session-1', 0, '{"type":"custom"}', 1);
PRAGMA user_version = 4;
`);
db.close();
const database = openOpenClawAgentDatabase({
agentId: "worker-1",
env: { OPENCLAW_STATE_DIR: stateDir },
});
const columns = database.db.prepare("PRAGMA table_info(sessions)").all() as Array<{
name?: unknown;
}>;
expect(columns.map((column) => column.name)).toEqual(
expect.arrayContaining(["transcript_observed_at", "transcript_updated_at"]),
);
expect(
database.db
.prepare(
"SELECT transcript_observed_at, transcript_updated_at FROM sessions WHERE session_id = ?",
)
.get("session-1"),
).toEqual({
transcript_observed_at: 20,
transcript_updated_at: expect.any(Number),
});
expect(
database.db
.prepare(
"SELECT transcript_observed_at, transcript_updated_at FROM sessions WHERE session_id = ?",
)
.get("session-2"),
).toEqual({
transcript_observed_at: null,
transcript_updated_at: null,
});
expect(readSqliteNumberPragma(database.db, "user_version")).toBe(OPENCLAW_AGENT_SCHEMA_VERSION);
});
it("inspects registered database ownership without mutating the database", () => {
const stateDir = createTempStateDir();
const database = openOpenClawAgentDatabase({
@@ -1132,7 +1208,7 @@ describe("openclaw agent database", () => {
env: { OPENCLAW_STATE_DIR: stateDir },
});
expect(readSqliteNumberPragma(database.db, "user_version")).toBe(4);
expect(readSqliteNumberPragma(database.db, "user_version")).toBe(OPENCLAW_AGENT_SCHEMA_VERSION);
const session = database.db
.prepare(
`
+38 -3
View File
@@ -46,10 +46,10 @@ export { resolveOpenClawAgentSqlitePath } from "./openclaw-agent-db.paths.js";
* per pathname, protected with private file modes, and registered in the shared
* OpenClaw state database for discovery and maintenance.
*/
// v4 = session/transcript flip (branch lineage). Main's v2 memory-identity
// v5 = transcript mutation watermark. The v4 session/transcript flip and main's v2 memory-identity
// change is folded in structure-gated (migrateMemoryIndexSourcesIdentity), so
// v2 main DBs and pre-merge v4 flip DBs both converge on this schema.
export const OPENCLAW_AGENT_SCHEMA_VERSION = 4;
export const OPENCLAW_AGENT_SCHEMA_VERSION = 5;
const OPENCLAW_AGENT_DB_DIR_MODE = 0o700;
const OPENCLAW_AGENT_DB_FILE_MODE = 0o600;
const OPENCLAW_AGENT_DB_SLOW_OPEN_MS = 1_000;
@@ -155,6 +155,28 @@ function dropLegacyMemoryIndexSchema(db: DatabaseSync): void {
`);
}
function backfillTranscriptMutationWatermarks(db: DatabaseSync): void {
const transcriptTable = db
.prepare("SELECT 1 AS ok FROM sqlite_master WHERE type = 'table' AND name = ?")
.get("transcript_events") as { ok?: unknown } | undefined;
if (transcriptTable?.ok !== 1) {
return;
}
db.prepare(
`
UPDATE sessions
SET
transcript_updated_at = COALESCE(transcript_updated_at, ?),
transcript_observed_at = COALESCE(transcript_observed_at, updated_at)
WHERE EXISTS (
SELECT 1
FROM transcript_events
WHERE transcript_events.session_id = sessions.session_id
)
`,
).run(Date.now());
}
function migrateOpenClawAgentSchema(db: DatabaseSync): void {
const userVersion = readSqliteUserVersion(db);
if (userVersion >= OPENCLAW_AGENT_SCHEMA_VERSION) {
@@ -164,7 +186,17 @@ function migrateOpenClawAgentSchema(db: DatabaseSync): void {
db.exec("DROP INDEX IF EXISTS idx_agent_transcript_events_session;");
}
const columns = readSqliteSessionColumns(db);
if (userVersion > 1 || !columns) {
if (columns && !columns.has("transcript_updated_at")) {
db.exec("ALTER TABLE sessions ADD COLUMN transcript_updated_at INTEGER DEFAULT NULL;");
}
if (columns && !columns.has("transcript_observed_at")) {
db.exec("ALTER TABLE sessions ADD COLUMN transcript_observed_at INTEGER DEFAULT NULL;");
}
if (!columns) {
return;
}
if (userVersion > 1) {
backfillTranscriptMutationWatermarks(db);
return;
}
const copyColumns = [
@@ -232,6 +264,8 @@ function migrateOpenClawAgentSchema(db: DatabaseSync): void {
session_scope TEXT NOT NULL DEFAULT 'conversation' CHECK (session_scope IN ('conversation', 'shared-main', 'group', 'channel')),
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
transcript_updated_at INTEGER DEFAULT NULL,
transcript_observed_at INTEGER DEFAULT NULL,
started_at INTEGER,
ended_at INTEGER,
status TEXT CHECK (status IS NULL OR status IN ('running', 'done', 'failed', 'killed', 'timeout')),
@@ -252,6 +286,7 @@ function migrateOpenClawAgentSchema(db: DatabaseSync): void {
DROP TABLE sessions;
ALTER TABLE sessions_new RENAME TO sessions;
`);
backfillTranscriptMutationWatermarks(db);
}
function parseMigratedSessionEntry(value: unknown): MigratedSessionEntry | null {
@@ -19,6 +19,8 @@ CREATE TABLE IF NOT EXISTS sessions (
session_scope TEXT NOT NULL DEFAULT 'conversation' CHECK (session_scope IN ('conversation', 'shared-main', 'group', 'channel')),
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
transcript_updated_at INTEGER DEFAULT NULL,
transcript_observed_at INTEGER DEFAULT NULL,
started_at INTEGER,
ended_at INTEGER,
status TEXT CHECK (status IS NULL OR status IN ('running', 'done', 'failed', 'killed', 'timeout')),
+2
View File
@@ -14,6 +14,8 @@ CREATE TABLE IF NOT EXISTS sessions (
session_scope TEXT NOT NULL DEFAULT 'conversation' CHECK (session_scope IN ('conversation', 'shared-main', 'group', 'channel')),
created_at INTEGER NOT NULL,
updated_at INTEGER NOT NULL,
transcript_updated_at INTEGER DEFAULT NULL,
transcript_observed_at INTEGER DEFAULT NULL,
started_at INTEGER,
ended_at INTEGER,
status TEXT CHECK (status IS NULL OR status IN ('running', 'done', 'failed', 'killed', 'timeout')),