furyhawk/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-07-21 10:16:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(sqlite): compaction works with memory path search enabled (#107809)

Browse Source 
* fix(sqlite): accept canonical memory path FTS triggers

* test(sqlite): reject drifted path FTS trigger

---------

Co-authored-by: Josh Lehman <josh@martian.engineering>
This commit is contained in:
Jason (Json)
2026-07-14 16:25:37 -07:00
committed by GitHub
co-authored by Josh Lehman
parent 0b03d17ef2
commit 0644956830
4 changed files with 184 additions and 39 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/memory-host-sdk/src/host/memory-schema.ts
+44 -30
View File
@@ -13,10 +13,43 @@ export const MEMORY_INDEX_FTS_TABLE = "memory_index_chunks_fts";
export const MEMORY_INDEX_PATHS_FTS_TABLE = "memory_index_paths_fts";
export const MEMORY_INDEX_VECTOR_TABLE = "memory_index_chunks_vec";
const MEMORY_PATH_FTS_TRIGGER_NAMES = [
"memory_index_paths_fts_after_insert",
"memory_index_paths_fts_after_update",
"memory_index_paths_fts_after_delete",
/** Optional canonical triggers owned by the derived path FTS index. */
export const MEMORY_PATH_FTS_TRIGGER_DEFINITIONS = [
{
name: "memory_index_paths_fts_after_insert",
sql: `
CREATE TRIGGER IF NOT EXISTS main.memory_index_paths_fts_after_insert
AFTER INSERT ON ${MEMORY_INDEX_SOURCES_TABLE}
BEGIN
INSERT INTO ${MEMORY_INDEX_PATHS_FTS_TABLE} (rowid, path, source)
VALUES (NEW.id, NEW.path, NEW.source);
END;
`,
},
{
name: "memory_index_paths_fts_after_update",
sql: `
CREATE TRIGGER IF NOT EXISTS main.memory_index_paths_fts_after_update
AFTER UPDATE OF id, path, source ON ${MEMORY_INDEX_SOURCES_TABLE}
BEGIN
DELETE FROM ${MEMORY_INDEX_PATHS_FTS_TABLE}
WHERE rowid = OLD.id;
INSERT INTO ${MEMORY_INDEX_PATHS_FTS_TABLE} (rowid, path, source)
VALUES (NEW.id, NEW.path, NEW.source);
END;
`,
},
{
name: "memory_index_paths_fts_after_delete",
sql: `
CREATE TRIGGER IF NOT EXISTS main.memory_index_paths_fts_after_delete
AFTER DELETE ON ${MEMORY_INDEX_SOURCES_TABLE}
BEGIN
DELETE FROM ${MEMORY_INDEX_PATHS_FTS_TABLE}
WHERE rowid = OLD.id;
END;
`,
},
] as const;
const LEGACY_MEMORY_INDEX_TRIGGERS = [
@@ -454,37 +487,18 @@ function migrateLegacyMemoryIndexTables(
/** Drop the canonical source-to-path-FTS maintenance triggers. */
export function dropMemoryPathFtsTriggers(db: DatabaseSync): void {
for (const triggerName of MEMORY_PATH_FTS_TRIGGER_NAMES) {
db.exec(`DROP TRIGGER IF EXISTS main.${triggerName}`);
for (const trigger of MEMORY_PATH_FTS_TRIGGER_DEFINITIONS) {
db.exec(`DROP TRIGGER IF EXISTS main.${trigger.name}`);
}
}
/** Install the canonical source-to-path-FTS maintenance triggers. */
export function ensureMemoryPathFtsTriggers(db: DatabaseSync): void {
db.exec(`
-- The named integer source identity survives VACUUM and gives every
-- FTS update/delete a direct rowid lookup instead of a virtual-table scan.
CREATE TRIGGER IF NOT EXISTS main.memory_index_paths_fts_after_insert
AFTER INSERT ON ${MEMORY_INDEX_SOURCES_TABLE}
BEGIN
INSERT INTO ${MEMORY_INDEX_PATHS_FTS_TABLE} (rowid, path, source)
VALUES (NEW.id, NEW.path, NEW.source);
END;
CREATE TRIGGER IF NOT EXISTS main.memory_index_paths_fts_after_update
AFTER UPDATE OF id, path, source ON ${MEMORY_INDEX_SOURCES_TABLE}
BEGIN
DELETE FROM ${MEMORY_INDEX_PATHS_FTS_TABLE}
WHERE rowid = OLD.id;
INSERT INTO ${MEMORY_INDEX_PATHS_FTS_TABLE} (rowid, path, source)
VALUES (NEW.id, NEW.path, NEW.source);
END;
CREATE TRIGGER IF NOT EXISTS main.memory_index_paths_fts_after_delete
AFTER DELETE ON ${MEMORY_INDEX_SOURCES_TABLE}
BEGIN
DELETE FROM ${MEMORY_INDEX_PATHS_FTS_TABLE}
WHERE rowid = OLD.id;
END;
`);
// The named integer source identity survives VACUUM and gives every
// FTS update/delete a direct rowid lookup instead of a virtual-table scan.
for (const trigger of MEMORY_PATH_FTS_TRIGGER_DEFINITIONS) {
db.exec(trigger.sql);
}
}
function ensureMemoryPathFtsSchema(params: { db: DatabaseSync; tokenizeClause: string }): void {
src/infra/sqlite-schema-contract.ts
+56 -1
View File
@@ -63,6 +63,17 @@ export type SqliteSchemaCompatibility = {
* requires a temporary default that the clean schema does not retain.
*/
allowedColumnDefinitions?: Readonly<Record<string, readonly string[]>>;
/**
* Exact owner-defined trigger groups that may be absent when their derived
* schema is disabled, but must be complete and canonical when present.
*/
optionalCanonicalTriggerGroups?: readonly {
tableName: string;
triggers: readonly {
name: string;
sql: string;
}[];
}[];
};
const schemaContractCache = new Map<string, SqliteSchemaContract>();
@@ -119,9 +130,34 @@ export function assertSqliteSchemaContains(
mismatches.push(`missing or drifted trigger ${expectedTrigger.name}`);
}
}
const optionalCanonicalTriggerGroups = collectOptionalCanonicalTriggerGroups(
compatibility,
tableName,
);
for (const triggerGroup of optionalCanonicalTriggerGroups) {
const isPresent = actualTable.triggers.some((actualTrigger) =>
triggerGroup.some((canonicalTrigger) => actualTrigger.name === canonicalTrigger.name),
);
if (!isPresent) {
continue;
}
for (const canonicalTrigger of triggerGroup) {
if (
!actualTable.triggers.some((actualTrigger) => isEqual(actualTrigger, canonicalTrigger))
) {
mismatches.push(`missing or drifted trigger ${canonicalTrigger.name}`);
}
}
}
const optionalCanonicalTriggers = optionalCanonicalTriggerGroups.flat();
for (const actualTrigger of actualTable.triggers) {
if (
!expectedTable.triggers.some((expectedTrigger) => isEqual(actualTrigger, expectedTrigger))
!expectedTable.triggers.some((expectedTrigger) =>
isEqual(actualTrigger, expectedTrigger),
) &&
!optionalCanonicalTriggers.some((canonicalTrigger) =>
isEqual(actualTrigger, canonicalTrigger),
)
) {
mismatches.push(`unexpected trigger ${actualTrigger.name}`);
}
@@ -148,6 +184,25 @@ export function assertSqliteSchemaContains(
}
}
function collectOptionalCanonicalTriggerGroups(
compatibility: SqliteSchemaCompatibility,
tableName: string,
): Array<Array<{ name: string; sql: string | null }>> {
return (compatibility.optionalCanonicalTriggerGroups ?? [])
.filter((group) => group.tableName === tableName)
.map((group) =>
group.triggers.map((trigger) => ({
name: trigger.name,
sql: normalizeOptionalCanonicalTriggerSql(trigger.sql),
})),
);
}
function normalizeOptionalCanonicalTriggerSql(sql: string): string | null {
// sqlite_schema stores main-schema trigger names without the schema qualifier.
return normalizeSchemaSql(sql)?.replace(/^(CREATE TRIGGER) main\./iu, "$1 ") ?? null;
}
function buildSqliteSchemaContract(schemaSql: string): SqliteSchemaContract {
const sqlite = requireNodeSqlite();
const database = new sqlite.DatabaseSync(":memory:");
src/state/openclaw-agent-db.ts
+23 -3
View File
@@ -2,7 +2,11 @@
import { chmodSync, existsSync, lstatSync, mkdirSync, statSync } from "node:fs";
import path from "node:path";
import type { DatabaseSync } from "node:sqlite";
import { migrateMemoryIndexSourcesIdentity } from "../../packages/memory-host-sdk/src/host/memory-schema.js";
import {
MEMORY_INDEX_SOURCES_TABLE,
MEMORY_PATH_FTS_TRIGGER_DEFINITIONS,
migrateMemoryIndexSourcesIdentity,
} from "../../packages/memory-host-sdk/src/host/memory-schema.js";
import {
clearNodeSqliteKyselyCacheForDatabase,
executeSqliteQuerySync,
@@ -15,7 +19,10 @@ import {
type CanonicalSqliteUniqueIndex,
} from "../infra/sqlite-index-schema.js";
import { assertSqliteIntegrity } from "../infra/sqlite-integrity.js";
import { assertSqliteSchemaContains } from "../infra/sqlite-schema-contract.js";
import {
assertSqliteSchemaContains,
type SqliteSchemaCompatibility,
} from "../infra/sqlite-schema-contract.js";
import {
runSqliteImmediateTransactionSync,
type SqliteTransactionOptions,
@@ -101,6 +108,14 @@ const OPENCLAW_AGENT_CANONICAL_UNIQUE_INDEXES = [
`,
},
] as const satisfies readonly CanonicalSqliteUniqueIndex[];
const OPENCLAW_AGENT_MAINTENANCE_SCHEMA_COMPATIBILITY = {
optionalCanonicalTriggerGroups: [
{
tableName: MEMORY_INDEX_SOURCES_TABLE,
triggers: MEMORY_PATH_FTS_TRIGGER_DEFINITIONS,
},
],
} satisfies SqliteSchemaCompatibility;
const agentDbLog = createSubsystemLogger("state/agent-db");
/** Open per-agent SQLite database handle plus lifecycle maintenance. */
@@ -607,7 +622,12 @@ export function assertOpenClawAgentDatabaseForMaintenance(
`OpenClaw agent database ${options.pathname} metadata schema version ${metadata.schemaVersion ?? "invalid"} does not match ${OPENCLAW_AGENT_SCHEMA_VERSION}; run openclaw doctor --fix before compacting it.`,
);
}
assertSqliteSchemaContains(database, options.pathname, OPENCLAW_AGENT_SCHEMA_SQL);
assertSqliteSchemaContains(
database,
options.pathname,
OPENCLAW_AGENT_SCHEMA_SQL,
OPENCLAW_AGENT_MAINTENANCE_SCHEMA_COMPATIBILITY,
);
}
/** Upgrade a supported older owned schema before strict offline maintenance. */
src/state/openclaw-database-maintenance.test.ts
+61 -5
View File
@@ -1,5 +1,6 @@
import { DatabaseSync } from "node:sqlite";
import { describe, expect, it } from "vitest";
import { ensureMemoryIndexSchema } from "../../packages/memory-host-sdk/src/host/memory-schema.js";
import {
assertOpenClawAgentDatabaseForMaintenance,
OPENCLAW_AGENT_SCHEMA_VERSION,
@@ -164,14 +165,40 @@ describe("OpenClaw database maintenance schema validation", () => {
}
});
it("rejects a current agent database with an unexpected trigger", () => {
it("accepts only canonical memory path FTS triggers", () => {
const database = createAgentDatabase();
try {
ensureMemoryIndexSchema({
db: database,
cacheEnabled: true,
ftsEnabled: true,
});
expect(() =>
assertOpenClawAgentDatabaseForMaintenance(database, {
agentId: "worker-1",
pathname: "agent.sqlite",
}),
).not.toThrow();
database.exec("DROP TRIGGER memory_index_paths_fts_after_delete;");
expect(() =>
assertOpenClawAgentDatabaseForMaintenance(database, {
agentId: "worker-1",
pathname: "agent.sqlite",
}),
).toThrow("missing or drifted trigger memory_index_paths_fts_after_delete");
ensureMemoryIndexSchema({
db: database,
cacheEnabled: true,
ftsEnabled: true,
});
database.exec(`
CREATE TRIGGER sessions_unexpected_delete_after_insert
AFTER INSERT ON sessions
CREATE TRIGGER memory_index_sources_unexpected_after_insert
AFTER INSERT ON memory_index_sources
BEGIN
DELETE FROM sessions WHERE session_key = NEW.session_key;
UPDATE memory_index_state SET revision = revision + 100 WHERE id = 1;
END;
`);
@@ -180,7 +207,36 @@ describe("OpenClaw database maintenance schema validation", () => {
agentId: "worker-1",
pathname: "agent.sqlite",
}),
).toThrow("unexpected trigger sessions_unexpected_delete_after_insert");
).toThrow("unexpected trigger memory_index_sources_unexpected_after_insert");
} finally {
database.close();
}
});
it("rejects a drifted canonical memory path FTS trigger", () => {
const database = createAgentDatabase();
try {
ensureMemoryIndexSchema({
db: database,
cacheEnabled: true,
ftsEnabled: true,
});
database.exec(`
DROP TRIGGER memory_index_paths_fts_after_insert;
CREATE TRIGGER memory_index_paths_fts_after_insert
AFTER INSERT ON memory_index_sources
BEGIN
INSERT INTO memory_index_paths_fts (rowid, path, source)
VALUES (NEW.id, NEW.path || '-drifted', NEW.source);
END;
`);
expect(() =>
assertOpenClawAgentDatabaseForMaintenance(database, {
agentId: "worker-1",
pathname: "agent.sqlite",
}),
).toThrow("missing or drifted trigger memory_index_paths_fts_after_insert");
} finally {
database.close();
}