fix(session): stop repeated restart recovery after retry budget (#96230)

* fix(session): bound restart recovery across gateway restarts

Co-authored-by: Mason Huang <masonxhuang@proton.me>

* style(session): format restart recovery changes

* fix(session): fence recovery notice state

* test(session): align internal recovery types

* test(session): type recovery completion fixture

* style(session): satisfy recovery lint rules

* test(plugin-sdk): split recovery boundary coverage

* test(plugin-sdk): use tracked recovery temp dirs

* fix(session): preserve stale admission errors

* fix(session): close recovery lifecycle races

* fix(session): rotate rejected recovery dispatch ids

* fix(session): enforce recovery quarantine boundaries

* fix(session): close restart recovery exhaustion gaps

* fix(session): preserve recovery command narrowing

* test(session): type restart recovery fixtures

* fix(session): enforce restart recovery quarantine

* fix(session): close restart recovery races

* fix(session): type recovery lifecycle state

* fix(session): enforce recovery ownership fences

* fix(session): fence recovery lifecycle completion

* fix(session): quarantine cross-process recovery

* fix(session): retire stale recovery ownership

* fix(session): restore rejected recovery admission

* fix(session): preserve restored recovery target

* fix(session): retry pending recovery races

* fix(session): preserve concurrent recovery aborts

* fix(session): quarantine recovery lifecycle state

* fix(session): settle exact recovery delivery fences

* fix(session): keep recovery restoration internal

* fix(session): close recovery admission races

* fix(session): retry durable recovery rollback

* fix(session): guard cached recovery settlement

* fix(session): close recovery ownership gaps

* fix(session): resume after delayed recovery rollback

* fix(session): finalize unrecoverable restart state

* fix(session): retry admitted recovery restoration

* fix(session): preserve recovery cleanup guarantees

* fix(session): restore failed terminal settlement

* chore(plugin-sdk): refresh API baseline

* fix(session): close recovery owner retry races

* test(session): type recovery owner fixture

* refactor(session): remove obsolete owner validator

* fix(session): preserve explicit abort recovery

---------

Co-authored-by: Mason Huang <masonxhuang@proton.me>
Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
xingzhou
2026-07-17 15:36:09 +01:00
committed by GitHub
co-authored by Mason Huang Peter Steinberger
parent 90ab572582
commit 0a868179e4
89 changed files with 8707 additions and 506 deletions
@@ -1,2 +1,2 @@
dc2bf410e15972389b8e60d772f92731130148845dff24294fae4786d3b9f41a plugin-sdk-api-baseline.json
ce076f7e71bb1ffbd0eead42a178dd069e15693348c4fd69b2ed291d51a8fcd7 plugin-sdk-api-baseline.jsonl
16f7664192e83f3e689685ecc917fa0578db5686646d3e7277ba866f07e1002d plugin-sdk-api-baseline.json
f8c7d2a77b258f347bd6b741461990c57c4f70d85d91d05b2526829e2980df60 plugin-sdk-api-baseline.jsonl
+1
View File
@@ -53,6 +53,7 @@ openclaw agent --agent ops --message "Run locally" --local
- Pass exactly one of `--message` or `--message-file`. `--message-file` strips a leading UTF-8 BOM and preserves multiline content; it rejects files that are not valid UTF-8.
- Slash commands (for example `/compact`) cannot run through `--message`. The CLI rejects them and points you at the first-class command instead (`openclaw sessions compact <key>` for compaction).
- `--local` and embedded fallback runs are one-shot: bundled MCP loopback resources and warm Claude stdio sessions opened for the run are retired after the reply, so scripted invocations do not leave local child processes running. Gateway-backed runs keep Gateway-owned MCP loopback resources under the running Gateway process instead.
- Standalone embedded execution (`--local` and transport fallback) refuses to reuse an existing main session while restart recovery is pending. Run the turn through a healthy Gateway, or reset it there with `/new` or `/reset`; an independent embedded process cannot safely coordinate that recovery owner with the Gateway scanner.
- With `--agent`, `--channel` and `--to` together, session routing follows the channel's canonical recipient and `session.dmScope`. Channels with a stable outbound-only recipient identity use a provider-owned session isolated from the agent's main session. `--reply-channel` and `--reply-account` affect delivery only.
- `--session-key` selects an explicit session key. Agent-prefixed keys must use `agent:<agent-id>:<session-key>`, and `--agent` must match the key's agent id when both are given. Bare non-sentinel keys scope to `--agent` when supplied, or to the configured default agent otherwise; for example `--agent ops --session-key incident-42` routes to `agent:ops:incident-42`. The literal keys `global` and `unknown` stay unscoped only when no `--agent` is supplied.
- `--json` reserves stdout for the JSON response; Gateway, plugin, and embedded-fallback diagnostics go to stderr so scripts can parse stdout directly.
+1 -1
View File
@@ -170,7 +170,7 @@ Flags:
<Accordion title="State and integrity">
- Session lock file inspection and stale lock cleanup.
- Session transcript repair for duplicated prompt-rewrite branches created by affected 2026.4.24 builds.
- Wedged subagent restart-recovery tombstone detection, with `--fix` support for clearing stale aborted recovery flags so startup does not keep treating the child as restart-aborted.
- Wedged main-session and subagent restart-recovery tombstone detection. Doctor reports the blocked sessions and only repairs stale aborted flags that conflict with an existing tombstone; it does not re-enable automatic recovery.
- State integrity and permissions checks (sessions, transcripts, state dir).
- Config file permission checks (chmod 600) when running locally.
- Model auth health: checks OAuth expiry, can refresh expiring tokens, and reports auth-profile cooldown/disabled states.
+29 -8
View File
@@ -10,8 +10,9 @@ title: "Restart recovery"
Restarting the gateway does not lose agent state. Conversations, transcripts,
scheduled jobs, background task records, and queued outbound messages all live
on disk, and work that was interrupted mid-turn is detected and resumed
automatically after the gateway comes back up. No manual intervention is
required, and there is nothing to configure: recovery is always on.
automatically after the gateway comes back up. Recovery is always on and
normally needs no manual intervention. Repeatedly failing recovery is bounded
and may quarantine one session until you inspect or replace it.
This page describes what survives a restart, how interrupted work is detected,
and what the automatic resume looks like.
@@ -75,12 +76,26 @@ A few seconds after startup, the gateway re-dispatches each marked session
with a synthetic system message telling the agent its previous turn was
interrupted by a restart and to continue from the existing transcript. If a
final reply had already been produced but not delivered, its text is included
so the agent can deliver it instead of redoing the work. Recovery retries up
to 3 times with exponential backoff. Every retry reuses one durable dispatch
identifier, so an ambiguous connection failure cannot start the same recovery
twice. Completed and unresumable Control UI turns also retain bounded durable
idempotency tombstones, allowing a reconnecting outbox to retire them without
re-executing the request.
so the agent can deliver it instead of redoing the work.
Startup reconciliation retries transient failures up to three times with
exponential backoff. Separately, each interrupted main-session cycle has a
durable budget of three charged automatic dispatch attempts, retained across
gateway restarts. OpenClaw charges an attempt before dispatch, refunds it when
the gateway explicitly rejects the request before acceptance, and retains the
charge when a post-dispatch result is uncertain to avoid replaying work.
Foreground work that already owns the session keeps automatic recovery out
until that work settles.
After the durable budget is exhausted, the session is tombstoned instead of
looping forever. Inspect the failed session and use `/new` or `/reset` to start a
replacement. `openclaw doctor --fix` can repair a stale aborted flag that
conflicts with a tombstone, but it does not re-enable that recovery cycle.
Every retry reuses one durable dispatch identifier, so an ambiguous connection
failure cannot start the same recovery twice. Completed and unresumable Control
UI turns also retain bounded durable idempotency tombstones, allowing a
reconnecting outbox to retire them without re-executing the request.
Message-tool-only replies use a second durable correlation. Before a terminal
same-conversation send reaches the channel, the gateway records an unresolved
@@ -149,6 +164,9 @@ agent picks up exactly where it left off, on the same channel and thread.
- **Crash-loop breaker:** 3 unclean boots within 5 minutes trip a breaker that
suppresses auto-start side services on the next boot, so a crashing gateway
does not amplify itself. It recovers once the unclean-boot window drains.
- **Main-session attempt budget:** three charged automatic dispatch attempts
per interrupted cycle; exhaustion tombstones that session until it is
inspected and replaced.
- **Metrics:** recovery activity is exported via
[Prometheus](/gateway/prometheus) as `openclaw_session_recovery_total` and
`openclaw_session_recovery_age_seconds`.
@@ -167,3 +185,6 @@ agent picks up exactly where it left off, on the same channel and thread.
- Work that was never admitted: messages arriving during the drain window are
rejected with an explicit restart error rather than silently queued into a
dying process.
- Standalone embedded turns cannot take over a main session with pending
restart recovery because they do not share the gateway's lifecycle owner.
Run the turn through the gateway or reset it there with `/new` or `/reset`.
@@ -0,0 +1,615 @@
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import type { InternalSessionEntry as SessionEntry } from "../config/sessions.js";
import {
applySessionEntryLifecycleMutation,
loadSessionEntry,
replaceSessionEntry,
} from "../config/sessions/session-accessor.js";
import { getAgentEventLifecycleGeneration } from "../infra/agent-events.js";
import { runWithAgentCommandRecoveryOwner } from "./agent-command-recovery-owner.js";
import type { AgentCommandOpts } from "./command/types.js";
import { claimMainSessionRecoveryOwner } from "./main-session-recovery-store.js";
const recoveryOwnerMocks = vi.hoisted(() => ({
scheduleMainSessionRecoveryPendingTarget: vi.fn(),
}));
vi.mock("./main-session-recovery-owner-release.js", () => ({
scheduleMainSessionRecoveryPendingTarget:
recoveryOwnerMocks.scheduleMainSessionRecoveryPendingTarget,
}));
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
const sessionKey = "agent:main:main";
afterEach(() => {
vi.clearAllMocks();
});
describe("agent command restart recovery ownership", () => {
function createTarget() {
const storePath = path.join(tempDirs.make("openclaw-agent-command-owner-"), "sessions.json");
return {
isNewSession: false,
sessionId: "session-1",
sessionKey,
storePath,
};
}
async function write(target: ReturnType<typeof createTarget>, entry: SessionEntry) {
await replaceSessionEntry({ sessionKey, storePath: target.storePath }, entry);
}
it("rejects standalone work when interruption appears during preparation", async () => {
const target = createTarget();
await write(target, { sessionId: target.sessionId, updatedAt: 100 });
const run = vi.fn();
await expect(
runWithAgentCommandRecoveryOwner({
lifecycleGeneration: getAgentEventLifecycleGeneration(),
mode: "reject_uncoordinated",
opts: {} as AgentCommandOpts,
prepare: async () => {
await write(target, {
sessionId: target.sessionId,
updatedAt: 200,
status: "running",
abortedLastRun: true,
});
return target;
},
run,
}),
).rejects.toThrow("interrupted work pending restart recovery");
expect(run).not.toHaveBeenCalled();
expect(
(loadSessionEntry({ sessionKey, storePath: target.storePath }) as SessionEntry | undefined)
?.mainRestartRecovery?.foregroundClaims,
).toBeUndefined();
});
it("refreshes the prepared working copy after claiming a recovery owner", async () => {
const base = createTarget();
const staleEntry: SessionEntry = {
sessionId: base.sessionId,
updatedAt: 100,
status: "running",
abortedLastRun: true,
};
const target = {
...base,
sessionEntry: { ...staleEntry },
sessionStore: { [sessionKey]: { ...staleEntry } },
};
await write(target, staleEntry);
await runWithAgentCommandRecoveryOwner({
lifecycleGeneration: getAgentEventLifecycleGeneration(),
mode: "claim",
opts: { runId: "foreground-run" } as AgentCommandOpts,
prepare: async () => target,
run: async (prepared) => {
const claims = prepared.sessionEntry.mainRestartRecovery?.foregroundClaims;
expect(claims?.tokens).toEqual([expect.any(String)]);
expect(Object.values(claims?.runIdsByClaimId ?? {})).toContain("foreground-run");
expect(prepared.sessionStore[sessionKey]).toEqual(prepared.sessionEntry);
const completed = { ...prepared.sessionEntry, abortedLastRun: false };
await write(target, completed);
},
});
const completed = loadSessionEntry({
sessionKey,
storePath: target.storePath,
}) as SessionEntry | undefined;
expect(completed?.abortedLastRun).toBe(false);
expect(completed?.mainRestartRecovery).toBeUndefined();
expect(recoveryOwnerMocks.scheduleMainSessionRecoveryPendingTarget).toHaveBeenLastCalledWith(
undefined,
);
});
it("rejects standalone work owned by a legacy session-key alias", async () => {
const target = createTarget();
await applySessionEntryLifecycleMutation({
storePath: target.storePath,
upserts: [
{
sessionKey: "main",
entry: {
sessionId: target.sessionId,
updatedAt: 100,
status: "running",
abortedLastRun: true,
},
},
],
skipMaintenance: true,
});
const run = vi.fn();
await expect(
runWithAgentCommandRecoveryOwner({
lifecycleGeneration: getAgentEventLifecycleGeneration(),
mode: "reject_uncoordinated",
opts: {} as AgentCommandOpts,
prepare: async () => target,
run,
}),
).rejects.toThrow("interrupted work pending restart recovery");
expect(run).not.toHaveBeenCalled();
});
it("rejects a legacy interrupted predecessor after the canonical key is reused", async () => {
const base = createTarget();
const target = {
...base,
isNewSession: true,
previousSessionId: base.sessionId,
sessionId: "replacement-session",
};
await applySessionEntryLifecycleMutation({
storePath: target.storePath,
upserts: [
{
sessionKey,
entry: { sessionId: target.sessionId, updatedAt: 200 },
},
{
sessionKey: "main",
entry: {
sessionId: target.previousSessionId,
updatedAt: 100,
status: "running",
abortedLastRun: true,
},
},
],
skipMaintenance: true,
});
const run = vi.fn();
await expect(
runWithAgentCommandRecoveryOwner({
lifecycleGeneration: getAgentEventLifecycleGeneration(),
mode: "reject_uncoordinated",
opts: {} as AgentCommandOpts,
prepare: async () => target,
run,
}),
).rejects.toThrow("interrupted work pending restart recovery");
expect(run).not.toHaveBeenCalled();
});
it("allows standalone work when interruption clears during preparation", async () => {
const target = createTarget();
await write(target, {
sessionId: target.sessionId,
updatedAt: 100,
status: "running",
abortedLastRun: true,
});
const run = vi.fn(async () => "ran");
await expect(
runWithAgentCommandRecoveryOwner({
lifecycleGeneration: getAgentEventLifecycleGeneration(),
mode: "reject_uncoordinated",
opts: {} as AgentCommandOpts,
prepare: async () => {
await write(target, { sessionId: target.sessionId, updatedAt: 200 });
return target;
},
run,
}),
).resolves.toBe("ran");
expect(run).toHaveBeenCalledOnce();
});
it("runs a Gateway-admitted recovery without acquiring a foreground owner", async () => {
const target = createTarget();
await write(target, {
sessionId: target.sessionId,
updatedAt: 200,
status: "running",
abortedLastRun: false,
restartRecoveryRuns: [{ runId: "recovery-run", lifecycleGeneration: "previous" }],
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 3,
chargedAttempts: 1,
},
});
const run = vi.fn(async () => "recovered");
await expect(
runWithAgentCommandRecoveryOwner({
lifecycleGeneration: getAgentEventLifecycleGeneration(),
mode: "claim",
opts: { mainRestartRecoveryAdmitted: true } as AgentCommandOpts,
prepare: async () => target,
run,
}),
).resolves.toBe("recovered");
expect(run).toHaveBeenCalledOnce();
});
it("restores a Gateway-admitted recovery when command preparation fails", async () => {
const target = createTarget();
const lifecycleGeneration = getAgentEventLifecycleGeneration();
await write(target, {
sessionId: target.sessionId,
updatedAt: 200,
status: "running",
abortedLastRun: false,
restartRecoveryRuns: [{ runId: "recovery-run", lifecycleGeneration }],
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 3,
chargedAttempts: 1,
},
});
const restoredTarget = {
sessionId: target.sessionId,
sessionKey,
storePath: target.storePath,
};
const restoreAdmittedRecovery = vi.fn(async () => {
const entry = loadSessionEntry({ sessionKey, storePath: target.storePath }) as SessionEntry;
entry.abortedLastRun = true;
await write(target, entry);
return restoredTarget;
});
const run = vi.fn();
await expect(
runWithAgentCommandRecoveryOwner({
lifecycleGeneration,
mode: "claim",
opts: { mainRestartRecoveryAdmitted: true } as AgentCommandOpts,
prepare: async () => {
throw new Error("model preparation failed");
},
restoreAdmittedRecovery,
run,
}),
).rejects.toThrow("model preparation failed");
expect(restoreAdmittedRecovery).toHaveBeenCalledOnce();
expect(recoveryOwnerMocks.scheduleMainSessionRecoveryPendingTarget).toHaveBeenCalledWith(
restoredTarget,
);
expect(run).not.toHaveBeenCalled();
expect(loadSessionEntry({ sessionKey, storePath: target.storePath })).toMatchObject({
abortedLastRun: true,
});
});
it("keeps retrying admitted recovery restoration after immediate store failures", async () => {
vi.useFakeTimers();
try {
const target = createTarget();
const restoredTarget = {
sessionId: target.sessionId,
sessionKey,
storePath: target.storePath,
};
let failures = 0;
const restoreAdmittedRecovery = vi.fn(async () => {
if (failures < 3) {
failures += 1;
throw new Error("transient session-store failure");
}
return restoredTarget;
});
const recovery = runWithAgentCommandRecoveryOwner({
lifecycleGeneration: getAgentEventLifecycleGeneration(),
mode: "claim",
opts: { mainRestartRecoveryAdmitted: true } as AgentCommandOpts,
prepare: async () => {
throw new Error("model preparation failed");
},
restoreAdmittedRecovery,
run: vi.fn(),
});
const rejected = expect(recovery).rejects.toThrow("model preparation failed");
await vi.advanceTimersByTimeAsync(100);
await rejected;
expect(restoreAdmittedRecovery).toHaveBeenCalledTimes(3);
expect(recoveryOwnerMocks.scheduleMainSessionRecoveryPendingTarget).toHaveBeenCalledWith(
undefined,
);
await vi.advanceTimersByTimeAsync(1_000);
expect(restoreAdmittedRecovery).toHaveBeenCalledTimes(4);
expect(recoveryOwnerMocks.scheduleMainSessionRecoveryPendingTarget).toHaveBeenCalledWith(
restoredTarget,
);
} finally {
vi.useRealTimers();
}
});
it("rejects ordinary work while an admitted recovery is still running", async () => {
const target = createTarget();
const lifecycleGeneration = getAgentEventLifecycleGeneration();
await write(target, {
sessionId: target.sessionId,
updatedAt: 200,
status: "running",
abortedLastRun: false,
restartRecoveryRuns: [{ runId: "recovery-run", lifecycleGeneration: "gateway-generation" }],
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 3,
chargedAttempts: 1,
},
});
const run = vi.fn();
await expect(
runWithAgentCommandRecoveryOwner({
lifecycleGeneration,
mode: "reject_uncoordinated",
opts: {} as AgentCommandOpts,
prepare: async () => target,
run,
}),
).rejects.toThrow("interrupted work pending restart recovery");
expect(run).not.toHaveBeenCalled();
});
it("fences the durable predecessor during an automatic freshness rollover", async () => {
const base = createTarget();
const target = {
...base,
isNewSession: true,
previousSessionId: "session-1",
sessionId: "session-2",
};
await write(base, {
sessionId: target.previousSessionId,
updatedAt: 100,
status: "running",
abortedLastRun: true,
});
const run = vi.fn();
await expect(
runWithAgentCommandRecoveryOwner({
lifecycleGeneration: getAgentEventLifecycleGeneration(),
mode: "reject_uncoordinated",
opts: {} as AgentCommandOpts,
prepare: async () => target,
run,
}),
).rejects.toThrow("interrupted work pending restart recovery");
expect(run).not.toHaveBeenCalled();
});
it("allows a freshness successor after its clean replacement commits", async () => {
const base = createTarget();
const target = {
...base,
isNewSession: true,
previousSessionId: "session-1",
sessionId: "session-2",
};
await write(base, { sessionId: target.sessionId, updatedAt: 200 });
const run = vi.fn(async () => "successor");
await expect(
runWithAgentCommandRecoveryOwner({
lifecycleGeneration: getAgentEventLifecycleGeneration(),
mode: "claim",
opts: {} as AgentCommandOpts,
prepare: async () => target,
run,
}),
).resolves.toBe("successor");
expect(run).toHaveBeenCalledOnce();
});
it("binds a transferred rollover lease to its exact predecessor", async () => {
const base = createTarget();
await write(base, {
sessionId: base.sessionId,
updatedAt: 100,
status: "running",
abortedLastRun: true,
});
const lifecycleGeneration = getAgentEventLifecycleGeneration();
const claim = await claimMainSessionRecoveryOwner({
lifecycleGeneration,
sessionId: base.sessionId,
target: { sessionKey, storePath: base.storePath },
});
if (claim.kind !== "claimed") {
throw new Error("expected recovery owner claim");
}
const target = {
...base,
isNewSession: true,
previousSessionId: "different-predecessor",
sessionId: "successor-session",
};
const run = vi.fn();
await expect(
runWithAgentCommandRecoveryOwner({
lifecycleGeneration,
mode: "claim",
opts: { mainRestartRecoveryOwnerLease: claim.lease } as AgentCommandOpts,
prepare: async () => target,
run,
}),
).rejects.toThrow("recovery owner changed during ingress preparation");
expect(run).not.toHaveBeenCalled();
});
it("binds a transferred recovery owner to the actual agent run", async () => {
const target = createTarget();
await write(target, {
sessionId: target.sessionId,
updatedAt: 100,
status: "running",
abortedLastRun: true,
});
const lifecycleGeneration = getAgentEventLifecycleGeneration();
const claim = await claimMainSessionRecoveryOwner({
lifecycleGeneration,
sessionId: target.sessionId,
target: { sessionKey, storePath: target.storePath },
});
if (claim.kind !== "claimed") {
throw new Error("expected recovery owner claim");
}
const run = vi.fn(async () => {
const entry = loadSessionEntry({ sessionKey, storePath: target.storePath }) as SessionEntry;
expect(entry.restartRecoveryRuns).toContainEqual({
lifecycleGeneration,
runId: "foreground-run",
});
expect(entry.mainRestartRecovery?.foregroundClaims?.runIdsByClaimId).toEqual({
[claim.lease.claimId]: "foreground-run",
});
return "ran";
});
await expect(
runWithAgentCommandRecoveryOwner({
lifecycleGeneration,
mode: "claim",
opts: {
mainRestartRecoveryOwnerLease: claim.lease,
runId: "foreground-run",
} as AgentCommandOpts,
prepare: async () => target,
run,
}),
).resolves.toBe("ran");
expect(run).toHaveBeenCalledOnce();
});
it("allows an explicitly requested fresh session without a predecessor", async () => {
const target = { ...createTarget(), sessionId: "fresh-session" };
const run = vi.fn(async () => "fresh");
await expect(
runWithAgentCommandRecoveryOwner({
lifecycleGeneration: getAgentEventLifecycleGeneration(),
mode: "reject_uncoordinated",
opts: { sessionId: target.sessionId } as AgentCommandOpts,
prepare: async () => target,
run,
}),
).resolves.toBe("fresh");
expect(run).toHaveBeenCalledOnce();
});
it("invalidates an explicit session replaced during preparation", async () => {
const target = createTarget();
await write(target, { sessionId: target.sessionId, updatedAt: 100 });
const run = vi.fn();
await expect(
runWithAgentCommandRecoveryOwner({
lifecycleGeneration: getAgentEventLifecycleGeneration(),
mode: "reject_uncoordinated",
opts: { sessionId: target.sessionId } as AgentCommandOpts,
prepare: async () => {
await write(target, { sessionId: "replacement-session", updatedAt: 200 });
return target;
},
run,
}),
).rejects.toThrow("changed while starting work");
expect(run).not.toHaveBeenCalled();
});
it("rejects a synthetic explicit replacement from a standalone process", async () => {
const base = createTarget();
const target = {
...base,
isNewSession: true,
previousSessionId: base.sessionId,
sessionId: "fresh-session",
};
await write(base, {
sessionId: base.sessionId,
updatedAt: 100,
status: "running",
abortedLastRun: true,
});
const run = vi.fn(async () => "fresh");
await expect(
runWithAgentCommandRecoveryOwner({
lifecycleGeneration: getAgentEventLifecycleGeneration(),
mode: "reject_uncoordinated",
opts: { sessionId: target.sessionId } as AgentCommandOpts,
prepare: async () => target,
run,
}),
).rejects.toThrow("interrupted work pending restart recovery");
expect(run).not.toHaveBeenCalled();
});
it("rejects standalone reuse of a tombstoned session", async () => {
const target = createTarget();
await write(target, {
sessionId: target.sessionId,
updatedAt: 100,
status: "failed",
abortedLastRun: false,
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 4,
chargedAttempts: 3,
tombstone: { reason: "automatic recovery exhausted" },
},
});
const run = vi.fn(async () => "reused");
await expect(
runWithAgentCommandRecoveryOwner({
lifecycleGeneration: getAgentEventLifecycleGeneration(),
mode: "reject_uncoordinated",
opts: { sessionId: target.sessionId } as AgentCommandOpts,
prepare: async () => target,
run,
}),
).rejects.toThrow("interrupted work pending restart recovery");
expect(run).not.toHaveBeenCalled();
});
it("revalidates a fresh key when interruption appears during preparation", async () => {
const base = createTarget();
const target = { ...base, isNewSession: true, sessionId: "fresh-session" };
const run = vi.fn();
await expect(
runWithAgentCommandRecoveryOwner({
lifecycleGeneration: getAgentEventLifecycleGeneration(),
mode: "reject_uncoordinated",
opts: {} as AgentCommandOpts,
prepare: async () => {
await write(base, {
sessionId: target.sessionId,
updatedAt: 200,
status: "running",
abortedLastRun: true,
});
return target;
},
run,
}),
).rejects.toThrow("interrupted work pending restart recovery");
expect(run).not.toHaveBeenCalled();
});
});
+221
View File
@@ -0,0 +1,221 @@
import path from "node:path";
import type { InternalSessionEntry } from "../config/sessions.js";
import { formatErrorMessage } from "../infra/errors.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import type { AgentCommandOpts } from "./command/types.js";
import { scheduleMainSessionRecoveryPendingTarget } from "./main-session-recovery-owner-release.js";
import {
restoreAdmittedRecoveryWithRetries,
scheduleAdmittedRecoveryRestore,
} from "./main-session-recovery-restore.js";
import {
bindMainSessionRecoveryOwnerRun,
claimMainSessionRecoveryOwner,
inspectMainSessionRecoveryRequired,
readMainSessionRecoveryOwner,
releaseMainSessionRecoveryOwner,
type MainSessionRecoveryOwnerLease,
type MainSessionRecoveryPendingTarget,
} from "./main-session-recovery-store.js";
const log = createSubsystemLogger("agents/agent-command");
type PreparedRecoveryOwnerTarget = object & {
isNewSession: boolean;
previousSessionId?: string;
sessionId: string;
sessionKey?: string;
sessionEntry?: InternalSessionEntry;
sessionStore?: Record<string, InternalSessionEntry>;
storePath: string;
runLease?: { release: () => Promise<void> };
};
type AcquiredRecoveryOwner = {
lease: MainSessionRecoveryOwnerLease;
entry: InternalSessionEntry;
sessionKey: string;
};
function cloneRecoveryOwnerEntry(entry: InternalSessionEntry): InternalSessionEntry {
const state = entry.mainRestartRecovery;
return {
...entry,
...(entry.restartRecoveryRuns
? { restartRecoveryRuns: entry.restartRecoveryRuns.map((run) => ({ ...run })) }
: {}),
...(state
? {
mainRestartRecovery: {
...state,
...(state.reservation ? { reservation: { ...state.reservation } } : {}),
...(state.foregroundClaims
? {
foregroundClaims: {
...state.foregroundClaims,
tokens: [...state.foregroundClaims.tokens],
...(state.foregroundClaims.runIdsByClaimId
? { runIdsByClaimId: { ...state.foregroundClaims.runIdsByClaimId } }
: {}),
},
}
: {}),
...(state.tombstone ? { tombstone: { ...state.tombstone } } : {}),
},
}
: {}),
};
}
function refreshPreparedRecoveryOwnerTarget(
prepared: PreparedRecoveryOwnerTarget,
acquired: AcquiredRecoveryOwner | undefined,
): void {
if (!acquired || acquired.entry.sessionId !== prepared.sessionId) {
return;
}
const entry = cloneRecoveryOwnerEntry(acquired.entry);
prepared.sessionEntry = entry;
if (prepared.sessionStore && prepared.sessionKey) {
prepared.sessionStore[prepared.sessionKey] = entry;
}
}
async function claimAgentCommandRecoveryOwner(params: {
lifecycleGeneration: string;
mode: "claim" | "reject_uncoordinated";
opts: AgentCommandOpts;
prepared: PreparedRecoveryOwnerTarget;
}): Promise<AcquiredRecoveryOwner | undefined> {
const transferredLease = params.opts.mainRestartRecoveryOwnerLease;
if (transferredLease) {
const expectedLeaseSessionId = params.prepared.isNewSession
? params.prepared.previousSessionId
: params.prepared.sessionId;
const matchesPreparedTarget =
expectedLeaseSessionId !== undefined &&
transferredLease.lifecycleGeneration === params.lifecycleGeneration &&
transferredLease.sessionId === expectedLeaseSessionId &&
transferredLease.sessionKey === params.prepared.sessionKey &&
path.resolve(transferredLease.storePath) === path.resolve(params.prepared.storePath);
if (!matchesPreparedTarget) {
// Gateway transfers a persisted fence before preparation; bind it again after
// session resolution so rollover or rerouting cannot execute under another row's lease.
throw new Error("main-session recovery owner changed during ingress preparation; retry");
}
if (params.opts.runId) {
return await bindMainSessionRecoveryOwnerRun(transferredLease, params.opts.runId);
}
const snapshot = await readMainSessionRecoveryOwner(transferredLease);
if (!snapshot) {
throw new Error("main-session recovery owner changed during ingress preparation; retry");
}
return { ...snapshot, lease: transferredLease };
}
if (params.opts.sessionEffects === "internal") {
return undefined;
}
if (params.opts.mainRestartRecoveryAdmitted === true) {
return undefined;
}
const sessionKey = params.prepared.sessionKey;
if (!sessionKey) {
return undefined;
}
if (params.mode === "reject_uncoordinated") {
const recoveryInspection = await inspectMainSessionRecoveryRequired({
allowMissingSession:
(params.prepared.isNewSession && !params.prepared.previousSessionId) ||
params.opts.sessionId?.trim() === params.prepared.sessionId,
expectedSessionId: params.prepared.previousSessionId ?? params.prepared.sessionId,
lifecycleGeneration: params.lifecycleGeneration,
target: { sessionKey, storePath: params.prepared.storePath },
});
if (recoveryInspection.kind === "invalidated") {
throw new Error(`Session "${sessionKey}" changed while starting work. Retry.`);
}
if (recoveryInspection.kind === "required") {
throw new Error(
`Session "${sessionKey}" has interrupted work pending restart recovery; retry through a healthy Gateway or reset it there with /new or /reset.`,
);
}
return undefined;
}
// Claim against the latest durable row instead of the preparation snapshot.
// A restart marker may appear or clear while preparation reads the session.
const claim = await claimMainSessionRecoveryOwner({
allowMissingSession:
(params.prepared.isNewSession && !params.prepared.previousSessionId) ||
params.opts.sessionId?.trim() === params.prepared.sessionId,
lifecycleGeneration: params.lifecycleGeneration,
sessionId: params.prepared.previousSessionId ?? params.prepared.sessionId,
replacementSessionId: params.prepared.isNewSession ? params.prepared.sessionId : undefined,
runId: params.opts.runId,
target: { sessionKey, storePath: params.prepared.storePath },
});
if (claim.kind === "invalidated") {
throw new Error(`Session "${sessionKey}" changed while starting work. Retry.`);
}
if (claim.kind === "not_required") {
return undefined;
}
// Explicit replacements keep this token through successor persistence so
// recovery cannot race the replacement; Gateway claims follow the same lease path.
return { lease: claim.lease, entry: claim.entry, sessionKey: claim.sessionKey };
}
export async function runWithAgentCommandRecoveryOwner<
TPrepared extends PreparedRecoveryOwnerTarget,
TResult,
>(params: {
lifecycleGeneration: string;
mode: "claim" | "reject_uncoordinated";
opts: AgentCommandOpts;
prepare: (opts: AgentCommandOpts) => Promise<TPrepared>;
restoreAdmittedRecovery?: () => Promise<MainSessionRecoveryPendingTarget | undefined>;
run: (prepared: TPrepared) => Promise<TResult>;
}): Promise<TResult> {
// Gateway may preclaim before dispatch, so every preparation outcome must release ownership.
let lease = params.opts.mainRestartRecoveryOwnerLease;
let pendingRecovery: Awaited<ReturnType<typeof releaseMainSessionRecoveryOwner>> = undefined;
let prepared: TPrepared | undefined;
try {
try {
prepared = await params.prepare(params.opts);
} catch (error) {
// Gateway admission consumes the durable reservation before command
// preparation. Restore it when preparation fails before a run exists.
if (params.restoreAdmittedRecovery) {
try {
pendingRecovery = await restoreAdmittedRecoveryWithRetries(
params.restoreAdmittedRecovery,
);
} catch (restoreError) {
log.warn(
`failed to restore admitted recovery after command preparation: ${formatErrorMessage(restoreError)}`,
);
scheduleAdmittedRecoveryRestore(params.restoreAdmittedRecovery);
}
}
throw error;
}
const acquired = await claimAgentCommandRecoveryOwner({ ...params, prepared });
lease = acquired?.lease;
// Preparation uses a detached working copy. Carry the owner transaction's
// exact row forward so successful settlement can consume the same recovery cycle.
refreshPreparedRecoveryOwnerTarget(prepared, acquired);
return await params.run(prepared);
} finally {
try {
const releasedRecovery = await releaseMainSessionRecoveryOwner(lease);
pendingRecovery ??= releasedRecovery;
} catch (error) {
log.warn(`failed to release main-session recovery owner: ${formatErrorMessage(error)}`);
}
try {
await prepared?.runLease?.release();
} finally {
scheduleMainSessionRecoveryPendingTarget(pendingRecovery);
}
}
}
@@ -199,6 +199,15 @@ vi.mock("./command/session.js", () => ({
vi.mock("./command/types.js", () => ({}));
// Recovery ownership has dedicated store-backed coverage. This command suite
// uses an intentionally synthetic session resolver with no durable store path.
vi.mock("./main-session-recovery-store.js", () => ({
claimMainSessionRecoveryOwner: vi.fn(async () => ({ kind: "not_required" })),
inspectMainSessionRecoveryRequired: vi.fn(async () => ({ kind: "not_required" })),
releaseMainSessionRecoveryOwner: vi.fn(async () => undefined),
validateMainSessionRecoveryOwner: vi.fn(async () => true),
}));
vi.mock("./harness/runtime-plugin.js", () => ({
ensureSelectedAgentHarnessPlugin: vi.fn(async () => undefined),
}));
@@ -2980,7 +2989,7 @@ describe("agentCommand LiveSessionModelSwitchError retry", () => {
expect(stored?.restartRecoveryDeliveryContext).toBeUndefined();
expect(stored?.restartRecoveryDeliveryRunId).toBeUndefined();
expect(stored?.restartRecoveryDeliverySourceRunId).toBeUndefined();
expect(stored?.restartRecoveryTerminalRunIds).toEqual(["control-ui-run"]);
expect(stored?.restartRecoveryTerminalRunIds).toEqual(["control-ui-run", "recovery-run"]);
});
it("refreshes delivery session entries through the session accessor", async () => {
+60 -31
View File
@@ -22,6 +22,7 @@ import { resolveSendPolicy } from "../sessions/send-policy.js";
import { beginSessionWorkAdmission } from "../sessions/session-lifecycle-admission.js";
import { classifySessionStateActor } from "../sessions/session-state-events.js";
import type { DeliveryContext } from "../utils/delivery-context.shared.js";
import { runWithAgentCommandRecoveryOwner } from "./agent-command-recovery-owner.js";
import {
buildCurrentRunRestartRecoveryClaim,
shouldPersistRestartRecoveryCleanup,
@@ -51,12 +52,14 @@ import {
resolveInternalSessionEffectsTarget,
} from "./internal-session-effects.js";
import { AGENT_LANE_SUBAGENT } from "./lanes.js";
import type { MainSessionRecoveryPendingTarget } from "./main-session-recovery-store.js";
import type { AgentRunSessionTarget } from "./run-session-target.js";
import { createAgentRunRestartAbortError } from "./run-termination.js";
const log = createSubsystemLogger("agents/agent-command");
async function agentCommandInternal(
prepared: Awaited<ReturnType<typeof prepareAgentCommandExecution>>,
initialOpts: AgentCommandOpts,
runtime: RuntimeEnv = defaultRuntime,
deps?: CliDeps,
@@ -66,7 +69,6 @@ async function agentCommandInternal(
const suppressVisibleSessionEffects = initialOpts.sessionEffects === "internal";
const preserveUserFacingSessionModelState =
initialOpts.preserveUserFacingSessionModelState === true;
const prepared = await prepareAgentCommandExecution(initialOpts, runtime);
const lifecycleAbortController = new AbortController();
const storedDeliveryMediaUrls =
prepared.sessionEntry?.restartRecoveryDeliveryRunId === prepared.runId &&
@@ -129,7 +131,6 @@ async function agentCommandInternal(
pluginsEnabled,
manifestMetadataSnapshot,
modelManifestContext,
runLease,
} = prepared;
let lifecycleGeneration = opts.lifecycleGeneration ?? captureAgentRunLifecycleGeneration(runId);
let sessionEntry = prepared.sessionEntry,
@@ -454,9 +455,6 @@ async function agentCommandInternal(
return finalized.deliveryResult;
});
} finally {
if (runLease) {
await runLease.release();
}
sessionWorkAdmission?.release();
if (internalModelRunTargets) {
// Compaction may rotate a private session identity. Remove every owned
@@ -489,6 +487,7 @@ async function agentCommandInternal(
...buildRestartRecoveryClaimCleanupPatch({
entry,
recordTerminalSource: true,
terminalRunId: runId,
terminalDeliveryEvidence: restartRecoveryTerminalDeliveryEvidence,
}),
updatedAt: Date.now(),
@@ -532,8 +531,10 @@ export async function agentCommand(
getRuntimeConfig,
},
async () =>
await agentCommandInternal(
{
await runWithAgentCommandRecoveryOwner({
lifecycleGeneration,
mode: "reject_uncoordinated",
opts: {
...opts,
lifecycleGeneration,
// agentCommand is the trusted-operator entrypoint used by CLI/local flows.
@@ -543,41 +544,69 @@ export async function agentCommand(
// Local/CLI callers are trusted by default for per-run model overrides.
allowModelOverride: opts.allowModelOverride ?? true,
},
runtime,
resolvedDeps,
),
prepare: async (preparedOpts) =>
await prepareAgentCommandExecution(preparedOpts, runtime),
run: async (prepared) =>
await agentCommandInternal(prepared, prepared.opts, runtime, resolvedDeps),
}),
),
);
}
async function agentCommandFromIngressInternal(
opts: AgentCommandIngressOpts,
runtime: RuntimeEnv = defaultRuntime,
deps?: CliDeps,
recovery?: {
restoreAdmittedRecovery?: () => Promise<MainSessionRecoveryPendingTarget | undefined>;
},
) {
if (typeof opts.allowModelOverride !== "boolean") {
throw new Error("allowModelOverride must be explicitly set for ingress agent runs.");
}
const lifecycleGeneration =
opts.lifecycleGeneration ?? captureAgentRunLifecycleGeneration(opts.runId ?? "");
return await withAgentRunLifecycleGeneration(lifecycleGeneration, async () => {
const result = await runWithAgentCommandRecoveryOwner({
lifecycleGeneration,
mode: "claim",
opts: {
...opts,
lifecycleGeneration,
senderIsOwner: opts.senderIsOwner === true,
},
prepare: async (preparedOpts) => await prepareAgentCommandExecution(preparedOpts, runtime),
restoreAdmittedRecovery: recovery?.restoreAdmittedRecovery,
run: async (prepared) => await agentCommandInternal(prepared, prepared.opts, runtime, deps),
});
if (result) {
emitIngressModelUsageDiagnostic(result, opts);
}
return result;
});
}
/** Runs an agent turn from an inbound channel/gateway ingress context. */
export async function agentCommandFromIngress(
opts: AgentCommandIngressOpts,
runtime: RuntimeEnv = defaultRuntime,
deps?: CliDeps,
) {
if (typeof opts.allowModelOverride !== "boolean") {
throw new Error("allowModelOverride must be explicitly set for ingress agent runs.");
}
const lifecycleGeneration =
opts.lifecycleGeneration ?? captureAgentRunLifecycleGeneration(opts.runId ?? "");
return await withAgentRunLifecycleGeneration(lifecycleGeneration, async () => {
const result = await agentCommandInternal(
{
...opts,
lifecycleGeneration,
senderIsOwner: opts.senderIsOwner === true,
},
runtime,
deps,
);
return await agentCommandFromIngressInternal(opts, runtime, deps);
}
if (result) {
emitIngressModelUsageDiagnostic(result, opts);
}
return result;
});
/** Internal Gateway entrypoint that restores a rejected restart-recovery admission. */
export async function agentCommandFromGatewayIngress(
opts: AgentCommandIngressOpts,
runtime: RuntimeEnv,
deps: CliDeps | undefined,
recovery: {
restoreAdmittedRecovery?: () => Promise<MainSessionRecoveryPendingTarget | undefined>;
},
) {
return await agentCommandFromIngressInternal(opts, runtime, deps, recovery);
}
export const testing = {
+12 -4
View File
@@ -247,9 +247,15 @@ export async function prepareAgentCommandExecution(opts: AgentCommandOpts, runti
agentId: agentIdOverride,
clone: false,
});
const { sessionId, sessionKey, storePath, isNewSession, persistedThinking, persistedVerbose } =
sessionResolution;
const {
sessionId,
sessionKey,
storePath,
isNewSession,
previousSessionId,
persistedThinking,
persistedVerbose,
} = sessionResolution;
const harnessSessionError = sessionKey
? resolveAgentHarnessSessionContextError(sessionKey, sessionResolution.sessionEntry)
: undefined;
@@ -348,7 +354,7 @@ export async function prepareAgentCommandExecution(opts: AgentCommandOpts, runti
const transcriptBody =
opts.transcriptMessage ?? resolveInternalEventTranscriptBody(message, opts.internalEvents);
return {
const prepared = {
opts: commandOpts,
body,
transcriptBody,
@@ -367,6 +373,7 @@ export async function prepareAgentCommandExecution(opts: AgentCommandOpts, runti
sessionStore,
storePath,
isNewSession,
previousSessionId,
persistedThinking,
persistedVerbose,
sessionAgentId,
@@ -383,6 +390,7 @@ export async function prepareAgentCommandExecution(opts: AgentCommandOpts, runti
acpResolution,
runLease,
};
return prepared;
} catch (error) {
await runLease?.release();
throw error;
+200 -1
View File
@@ -5,7 +5,7 @@ import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it, vi } from "vitest";
import type { OpenClawConfig } from "../../config/config.js";
import type { SessionEntry } from "../../config/sessions.js";
import type { InternalSessionEntry as SessionEntry } from "../../config/sessions.js";
import {
listSessionEntries,
loadSessionEntry,
@@ -1849,6 +1849,205 @@ describe("updateSessionStoreAfterAgentRun", () => {
});
});
it("clears main recovery markers after settled background progress", async () => {
await withTempSessionStore(async ({ storePath }) => {
const cfg = {} as OpenClawConfig;
const sessionKey = "agent:main:explicit:test-clear-recovery-state";
const sessionId = "test-clear-recovery-state-session";
const sessionStore: Record<string, SessionEntry> = {
[sessionKey]: {
sessionId,
updatedAt: 1,
abortedLastRun: true,
restartRecoveryRuns: [
{ runId: "initial-wedged-run", lifecycleGeneration: "gen-1" },
{ runId: "recovery-run-1", lifecycleGeneration: "gen-2" },
{ runId: "recovery-run-2", lifecycleGeneration: "gen-3" },
{ runId: "recovery-run-3", lifecycleGeneration: "gen-4" },
],
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 3,
chargedAttempts: 2,
},
subagentRecovery: {
automaticAttempts: 2,
lastAttemptAt: 3,
wedgedAt: 4,
wedgedReason: "automatic_attempt_budget_exceeded",
},
},
};
await seedSessionStore(storePath, sessionStore);
await updateSessionStoreAfterAgentRun({
cfg,
sessionId,
sessionKey,
storePath,
sessionStore,
defaultProvider: "openai",
defaultModel: "gpt-5.5",
touchInteraction: false,
touchActivity: false,
preserveRuntimeModel: true,
result: {
meta: {
durationMs: 1,
aborted: false,
agentMeta: {
sessionId,
provider: "openai",
model: "gpt-5.5",
},
},
} as EmbeddedAgentRunResult,
});
expect(sessionStore[sessionKey]?.abortedLastRun).toBe(false);
expect(sessionStore[sessionKey]?.restartRecoveryRuns).toBeUndefined();
expect(sessionStore[sessionKey]?.mainRestartRecovery).toBeUndefined();
expect(sessionStore[sessionKey]?.subagentRecovery).toEqual({
automaticAttempts: 2,
lastAttemptAt: 3,
wedgedAt: 4,
wedgedReason: "automatic_attempt_budget_exceeded",
});
const persisted = loadPersistedSessionEntry(storePath, sessionKey);
expect(persisted?.abortedLastRun).toBe(false);
expect(persisted?.restartRecoveryRuns).toBeUndefined();
expect(persisted).not.toHaveProperty("mainRestartRecovery");
expect(persisted?.subagentRecovery).toEqual({
automaticAttempts: 2,
lastAttemptAt: 3,
wedgedAt: 4,
wedgedReason: "automatic_attempt_budget_exceeded",
});
});
});
it("preserves a replacement recovery cycle from an older healthy finalizer", async () => {
await withTempSessionStore(async ({ storePath }) => {
const sessionKey = "agent:main:explicit:test-recovery-finalizer-aba";
const sessionId = "test-recovery-finalizer-aba-session";
const staleEntry: SessionEntry = {
sessionId,
updatedAt: 1,
status: "running",
abortedLastRun: true,
mainRestartRecovery: {
cycleId: "cycle-old",
revision: 2,
chargedAttempts: 1,
},
};
const sessionStore = { [sessionKey]: staleEntry };
const replacementEntry: SessionEntry = {
...staleEntry,
updatedAt: 2,
restartRecoveryRuns: [{ runId: "replacement-run", lifecycleGeneration: "gen-new" }],
mainRestartRecovery: {
cycleId: "cycle-new",
revision: 1,
chargedAttempts: 0,
},
};
await seedSessionStore(storePath, { [sessionKey]: replacementEntry });
await updateSessionStoreAfterAgentRun({
cfg: {} as OpenClawConfig,
sessionId,
sessionKey,
storePath,
sessionStore,
defaultProvider: "openai",
defaultModel: "gpt-5.5",
result: {
meta: {
durationMs: 1,
aborted: false,
agentMeta: { sessionId, provider: "openai", model: "gpt-5.5" },
},
} as EmbeddedAgentRunResult,
});
const persisted = loadPersistedSessionEntry(storePath, sessionKey);
expect(persisted).toMatchObject({
abortedLastRun: true,
restartRecoveryRuns: [{ runId: "replacement-run", lifecycleGeneration: "gen-new" }],
mainRestartRecovery: {
cycleId: "cycle-new",
revision: 1,
chargedAttempts: 0,
},
});
expect(sessionStore[sessionKey]).toEqual(persisted);
});
});
it("preserves a concurrent restart marker when a stale run settles healthy", async () => {
await withTempSessionStore(async ({ storePath }) => {
const sessionKey = "agent:main:explicit:test-restart-finalizer-race";
const sessionId = "test-restart-finalizer-race-session";
const initialEntry: SessionEntry = {
sessionId,
updatedAt: 1,
status: "running",
abortedLastRun: true,
restartRecoveryRuns: [{ runId: "run-1", lifecycleGeneration: "generation-1" }],
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 2,
chargedAttempts: 0,
foregroundClaims: {
lifecycleGeneration: "generation-1",
tokens: ["owner-1"],
},
},
};
const sessionStore = { [sessionKey]: initialEntry };
const concurrentEntry: SessionEntry = {
...structuredClone(initialEntry),
updatedAt: 2,
restartRecoveryRuns: [
...(initialEntry.restartRecoveryRuns ?? []),
{ runId: "run-1", lifecycleGeneration: "generation-2" },
],
mainRestartRecovery: {
...initialEntry.mainRestartRecovery!,
revision: 3,
},
};
await seedSessionStore(storePath, { [sessionKey]: concurrentEntry });
await updateSessionStoreAfterAgentRun({
cfg: {} as OpenClawConfig,
sessionId,
sessionKey,
storePath,
sessionStore,
defaultProvider: "openai",
defaultModel: "gpt-5.5",
result: {
meta: {
durationMs: 1,
aborted: false,
agentMeta: { sessionId, provider: "openai", model: "gpt-5.5" },
},
} as EmbeddedAgentRunResult,
});
for (const entry of [
sessionStore[sessionKey],
loadPersistedSessionEntry(storePath, sessionKey),
]) {
expect(entry?.abortedLastRun).toBe(true);
expect(entry?.restartRecoveryRuns).toEqual(concurrentEntry.restartRecoveryRuns);
expect(entry?.mainRestartRecovery).toEqual(concurrentEntry.mainRestartRecovery);
}
});
});
it("preserves runtime model and contextTokens when preserveRuntimeModel is true (heartbeat bleed fix)", async () => {
await withTempSessionStore(async ({ storePath }) => {
const cfg = {} as OpenClawConfig;
+8 -4
View File
@@ -15,6 +15,7 @@ import { createLazyImportLoader } from "../../shared/lazy-promise.js";
import { resolveNonNegativeNumber } from "../../shared/number-coercion.js";
import { clearCliSession, setCliSessionBinding, setCliSessionId } from "../cli-session.js";
import { DEFAULT_CONTEXT_TOKENS } from "../defaults.js";
import { clearMainSessionRecoveryAfterAgentRun } from "../main-session-recovery-clear.js";
import { isCliProvider } from "../model-selection.js";
import { deriveSessionTotalTokens, hasNonzeroUsage } from "../usage.js";
@@ -199,9 +200,7 @@ export async function updateSessionStoreAfterAgentRun(params: {
}
}
next.abortedLastRun = result.meta.aborted ?? false;
if (params.clearRestartRecoveryForceSafeTools && result.meta.aborted !== true) {
next.restartRecoveryForceSafeTools = undefined;
}
clearMainSessionRecoveryAfterAgentRun(next, params.clearRestartRecoveryForceSafeTools);
if (result.meta.systemPromptReport) {
next.systemPromptReport = result.meta.systemPromptReport;
}
@@ -310,7 +309,12 @@ export async function updateSessionStoreAfterAgentRun(params: {
}
return preserveUserFacingRunState
? metadataPatch
: projectSessionSnapshotChanges({ initial: entry, next, current: currentEntry });
: projectSessionSnapshotChanges({
initial: entry,
next,
current: currentEntry,
reassertAbortedLastRun: result.meta.aborted === true,
});
},
{
...(preserveUserFacingRunState ? {} : { fallbackEntry: entry }),
+4
View File
@@ -41,6 +41,7 @@ import { resolveSessionIdMatchSelection } from "../../sessions/session-id-resolu
import { listAgentIds, resolveDefaultAgentId } from "../agent-scope.js";
import { clearBootstrapSnapshotOnSessionRollover } from "../bootstrap-cache.js";
import { clearAllCliSessions } from "../cli-session.js";
import { transitionMainSessionRecovery } from "../main-session-recovery-state.js";
/** Resolved command session identity plus backing store metadata. */
type SessionResolution = {
@@ -50,6 +51,7 @@ type SessionResolution = {
sessionStore?: Record<string, SessionEntry>;
storePath: string;
isNewSession: boolean;
previousSessionId?: string;
persistedThinking?: ThinkLevel;
persistedVerbose?: VerboseLevel;
};
@@ -90,6 +92,7 @@ export function clearRotatedSessionMetadata(entry: SessionEntry): SessionEntry {
sessionStartedAt: undefined,
lastInteractionAt: undefined,
};
transitionMainSessionRecovery(next, { kind: "clear" });
clearAllCliSessions(next);
return next;
}
@@ -472,6 +475,7 @@ export function resolveSession(opts: {
sessionStore,
storePath,
isNewSession,
previousSessionId: isNewSession ? sessionEntry?.sessionId : undefined,
persistedThinking,
persistedVerbose,
};
+5
View File
@@ -18,6 +18,7 @@ import type {
import type { ExecElevatedDefaults } from "../bash-tools.exec-types.js";
import type { BootstrapContextRunKind } from "../bootstrap-mode.js";
import type { CliSessionBindingFacts } from "../cli-runner/types.js";
import type { MainSessionRecoveryOwnerLease } from "../main-session-recovery-store.js";
import type { AgentStreamParams, ClientToolDefinition } from "./shared-types.js";
/** Image content block for Claude API multimodal messages. */
@@ -173,6 +174,10 @@ export type AgentCommandOpts = {
oneShotCliRun?: boolean;
/** Gateway-owned runs can late-bind plugin subagent and node runtime helpers. */
allowGatewaySubagentBinding?: boolean;
/** Opaque foreground fence transferred by Gateway after atomic session admission. */
mainRestartRecoveryOwnerLease?: MainSessionRecoveryOwnerLease;
/** Gateway already consumed this automatic recovery run's durable reservation. */
mainRestartRecoveryAdmitted?: boolean;
/** Internal local CLI callers can annotate result metadata before JSON/text output. */
resultMetaOverrides?: AgentCommandResultMetaOverrides;
/** Called when the actual run model is selected, including fallback retries. */
+40
View File
@@ -0,0 +1,40 @@
import type { InternalSessionEntry as SessionEntry } from "../config/sessions.js";
type MainRecoveryStateFields = Pick<
SessionEntry,
"abortedLastRun" | "restartRecoveryRuns" | "mainRestartRecovery"
>;
export const MAIN_SESSION_RECOVERY_CLEAR_PATCH: Partial<MainRecoveryStateFields> = {
abortedLastRun: false,
restartRecoveryRuns: undefined,
mainRestartRecovery: undefined,
};
export function buildMainSessionRecoveryClearPatch(
entry?: Partial<MainRecoveryStateFields> | null,
): Partial<MainRecoveryStateFields> {
if (
entry?.abortedLastRun !== true &&
entry?.restartRecoveryRuns === undefined &&
entry?.mainRestartRecovery === undefined
) {
return {};
}
return MAIN_SESSION_RECOVERY_CLEAR_PATCH;
}
export function clearMainSessionRecoveryAfterAgentRun(
entry: SessionEntry,
clearForceSafeTools: boolean | undefined,
): void {
const aborted = entry.abortedLastRun === true;
if (clearForceSafeTools && !aborted) {
entry.restartRecoveryForceSafeTools = undefined;
}
if (!aborted) {
Object.assign(entry, buildMainSessionRecoveryClearPatch(entry));
}
}
export type { MainRecoveryStateFields };
@@ -0,0 +1,215 @@
import type { InternalSessionEntry as SessionEntry } from "../config/sessions.js";
import { mergeRestartRecoveryTerminalRunIds } from "../config/sessions/restart-recovery-state.js";
import {
buildMainSessionRecoveryClearPatch,
type MainRecoveryStateFields,
} from "./main-session-recovery-clear.js";
const MAIN_RESTART_RECOVERY_WEDGED_FALLBACK_REASON =
"main-session restart recovery is tombstoned for this session";
type MainRecoveryLifecycleEvent = {
runId?: string;
lifecycleGeneration?: string;
data?: { error?: unknown; phase?: unknown; stopReason?: unknown };
};
export function inspectMainSessionRecoveryHealth(entry: SessionEntry):
| { status: "none" }
| { status: "active" }
| {
status: "tombstoned";
reason: string;
repair: "clear_stale_abort" | null;
} {
const state = entry.mainRestartRecovery;
if (!state) {
return { status: "none" };
}
if (!state.tombstone) {
return { status: "active" };
}
return {
status: "tombstoned",
reason: state.tombstone.reason.trim() || MAIN_RESTART_RECOVERY_WEDGED_FALLBACK_REASON,
repair: entry.abortedLastRun === true ? "clear_stale_abort" : null,
};
}
function lifecyclePhase(event: MainRecoveryLifecycleEvent): "start" | "end" | "error" | null {
const phase = event.data?.phase;
return phase === "start" || phase === "end" || phase === "error" ? phase : null;
}
export function isMainSessionRecoveryLifecycleEvent(params: {
entry?: Partial<Pick<SessionEntry, "restartRecoveryRuns">> | null;
event: MainRecoveryLifecycleEvent;
}): boolean {
const runId = params.event.runId?.trim();
const lifecycleGeneration = params.event.lifecycleGeneration?.trim();
const phase = lifecyclePhase(params.event);
const interrupted = params.event.data?.stopReason === "restart";
const matchesFence = Boolean(
runId &&
lifecycleGeneration &&
params.entry?.restartRecoveryRuns?.some(
(run) => run.runId === runId && run.lifecycleGeneration === lifecycleGeneration,
),
);
return (
matchesFence && (phase === "start" || ((phase === "end" || phase === "error") && interrupted))
);
}
export function projectMainSessionRecoveryLifecycle(params: {
currentLifecycleGeneration: string;
entry?:
| (Partial<MainRecoveryStateFields> &
Pick<
Partial<SessionEntry>,
"restartRecoveryDeliveryRunId" | "restartRecoveryTerminalRunIds"
>)
| null;
event: MainRecoveryLifecycleEvent;
snapshotPatch: Partial<SessionEntry>;
}): { action: "suppress" } | { action: "apply"; patch: Partial<SessionEntry> } {
if (params.entry?.mainRestartRecovery?.tombstone) {
// Keep the operator boundary while allowing unrelated lifecycle status to settle.
return isMainSessionRecoveryLifecycleEvent(params)
? { action: "suppress" }
: {
action: "apply",
patch: {
...params.snapshotPatch,
abortedLastRun: params.entry.abortedLastRun,
restartRecoveryRuns: params.entry.restartRecoveryRuns,
mainRestartRecovery: params.entry.mainRestartRecovery,
},
};
}
if (isMainSessionRecoveryLifecycleEvent(params)) {
return { action: "suppress" };
}
const phase = lifecyclePhase(params.event);
const settlesRecovery =
(phase === "end" || phase === "error") && params.event.data?.stopReason !== "restart";
const patch = { ...params.snapshotPatch };
const runId = params.event.runId?.trim();
const lifecycleGeneration = params.event.lifecycleGeneration?.trim();
const runs = params.entry?.restartRecoveryRuns;
const matchesFence = Boolean(
runId &&
lifecycleGeneration &&
runs?.some((run) => run.runId === runId && run.lifecycleGeneration === lifecycleGeneration),
);
const remaining = matchesFence
? runs?.filter((run) => run.runId !== runId || run.lifecycleGeneration !== lifecycleGeneration)
: runs;
if (settlesRecovery) {
const foregroundClaims = params.entry?.mainRestartRecovery?.foregroundClaims;
const foregroundOwnerClaimId =
runId &&
lifecycleGeneration &&
lifecycleGeneration === params.currentLifecycleGeneration &&
foregroundClaims?.lifecycleGeneration === lifecycleGeneration
? foregroundClaims.tokens.find(
(claimId) => foregroundClaims.runIdsByClaimId?.[claimId] === runId,
)
: undefined;
const remainingForegroundClaimIds = foregroundOwnerClaimId
? foregroundClaims!.tokens.filter((claimId) => claimId !== foregroundOwnerClaimId)
: foregroundClaims?.tokens;
const remainingForegroundRunIds = foregroundOwnerClaimId
? Object.fromEntries(
Object.entries(foregroundClaims?.runIdsByClaimId ?? {}).filter(
([claimId]) => claimId !== foregroundOwnerClaimId,
),
)
: foregroundClaims?.runIdsByClaimId;
const remainingForegroundClaims = remainingForegroundClaimIds?.length
? {
lifecycleGeneration: foregroundClaims!.lifecycleGeneration,
tokens: remainingForegroundClaimIds,
...(remainingForegroundRunIds && Object.keys(remainingForegroundRunIds).length > 0
? { runIdsByClaimId: remainingForegroundRunIds }
: {}),
}
: undefined;
const recoveryStateAfterForegroundSettlement = foregroundOwnerClaimId
? {
...params.entry!.mainRestartRecovery!,
revision: params.entry!.mainRestartRecovery!.revision + 1,
foregroundClaims: remainingForegroundClaims,
}
: params.entry?.mainRestartRecovery;
const hasForegroundOwners = Boolean(
remainingForegroundClaims?.lifecycleGeneration === params.currentLifecycleGeneration &&
remainingForegroundClaims.tokens.length,
);
const reservation = params.entry?.mainRestartRecovery?.reservation;
const hasCurrentReservation =
reservation?.lifecycleGeneration === params.currentLifecycleGeneration;
const hasCurrentOwner = hasForegroundOwners || hasCurrentReservation;
if (!matchesFence) {
// No terminal snapshot may settle a recovery row it cannot identify.
return params.entry?.mainRestartRecovery || runs?.length
? { action: "suppress" }
: { action: "apply", patch };
}
if (hasCurrentOwner) {
// A terminal event may consume its own claim. Another owner still keeps
// the aggregate live until that owner's terminal event or release.
return {
action: "apply",
patch: {
restartRecoveryRuns: remaining?.length ? remaining : undefined,
restartRecoveryTerminalRunIds: mergeRestartRecoveryTerminalRunIds(
params.entry?.restartRecoveryTerminalRunIds,
[runId],
),
...(foregroundOwnerClaimId
? { mainRestartRecovery: recoveryStateAfterForegroundSettlement }
: {}),
},
};
}
if (foregroundOwnerClaimId) {
// This exact foreground run completed while its release lease was still
// active. Its terminal snapshot is authoritative and consumes the cycle.
Object.assign(patch, buildMainSessionRecoveryClearPatch(params.entry));
return { action: "apply", patch };
}
if (
!hasForegroundOwners &&
!hasCurrentReservation &&
params.entry?.abortedLastRun === true &&
(remaining?.length ?? 0) > 0
) {
return { action: "apply", patch: { restartRecoveryRuns: remaining } };
}
const recoveryDeliveryRunId =
typeof params.entry?.restartRecoveryDeliveryRunId === "string"
? params.entry.restartRecoveryDeliveryRunId.trim()
: undefined;
if ((remaining?.length ?? 0) > 0 && recoveryDeliveryRunId !== runId) {
// A different terminal run may consume only its own fence. Another
// admitted recovery remains the durable owner of the aggregate.
patch.abortedLastRun = false;
patch.restartRecoveryRuns = remaining;
patch.mainRestartRecovery = params.entry?.mainRestartRecovery;
return { action: "apply", patch };
}
// An admitted recovery clears the interruption flag before it runs. With
// no live owner left, that exact delivery run is the durable cleanup boundary.
Object.assign(patch, buildMainSessionRecoveryClearPatch(params.entry));
return { action: "apply", patch };
}
if (phase === "start" || !matchesFence || !remaining) {
return { action: "apply", patch };
}
if (params.entry?.abortedLastRun === true && remaining.length > 0) {
return { action: "apply", patch: { restartRecoveryRuns: remaining } };
}
patch.restartRecoveryRuns = remaining.length > 0 ? remaining : undefined;
return { action: "apply", patch };
}
@@ -0,0 +1,26 @@
import { getRuntimeConfig } from "../config/io.js";
import { getGatewayRecoveryRuntime } from "../gateway/server-recovery-runtime-context.js";
import type { MainSessionRecoveryPendingTarget } from "./main-session-recovery-store.js";
/** Schedules exact-row recovery only after the caller releases its lifecycle admission. */
export function scheduleMainSessionRecoveryPendingTarget(
target: MainSessionRecoveryPendingTarget | undefined,
): void {
if (!target) {
return;
}
void import("./main-session-restart-recovery.js").then(
({ scheduleRestartAbortedMainSessionRecoveryAfterOwnerRelease }) => {
scheduleRestartAbortedMainSessionRecoveryAfterOwnerRelease({
expectedSessionId: target.sessionId,
getConfig: getRuntimeConfig,
getGatewayRuntime: getGatewayRecoveryRuntime,
sessionKey: target.sessionKey,
storePath: target.storePath,
});
},
() => {
// Startup recovery remains the fallback if the optional recovery module cannot load.
},
);
}
@@ -0,0 +1,36 @@
import { formatErrorMessage } from "../infra/errors.js";
import { retryAsync } from "../infra/retry.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import { scheduleMainSessionRecoveryPendingTarget } from "./main-session-recovery-owner-release.js";
import type { MainSessionRecoveryPendingTarget } from "./main-session-recovery-store.js";
const log = createSubsystemLogger("main-session-recovery");
const RESTORE_RETRY_DELAY_MS = 1_000;
const RESTORE_RETRY_MAX_DELAY_MS = 30_000;
export type RestoreAdmittedRecovery = () => Promise<MainSessionRecoveryPendingTarget | undefined>;
export async function restoreAdmittedRecoveryWithRetries(
restore: RestoreAdmittedRecovery,
): Promise<MainSessionRecoveryPendingTarget | undefined> {
return await retryAsync(restore, 3, 25);
}
export function scheduleAdmittedRecoveryRestore(
restore: RestoreAdmittedRecovery,
delayMs = RESTORE_RETRY_DELAY_MS,
): void {
// Gateway admission consumed the reservation already. Keep restoration
// alive until this exact idempotent callback repairs or rejects its fence.
setTimeout(() => {
void restoreAdmittedRecoveryWithRetries(restore).then(
(pendingRecovery) => {
scheduleMainSessionRecoveryPendingTarget(pendingRecovery);
},
(error: unknown) => {
log.warn(`failed delayed admitted recovery restoration: ${formatErrorMessage(error)}`);
scheduleAdmittedRecoveryRestore(restore, Math.min(delayMs * 2, RESTORE_RETRY_MAX_DELAY_MS));
},
);
}, delayMs).unref?.();
}
@@ -0,0 +1,916 @@
import { describe, expect, it } from "vitest";
import type {
InternalSessionEntry as SessionEntry,
MainRestartRecoveryState,
} from "../config/sessions.js";
import { buildMainSessionRecoveryClearPatch } from "./main-session-recovery-clear.js";
import {
inspectMainSessionRecoveryHealth,
projectMainSessionRecoveryLifecycle,
} from "./main-session-recovery-lifecycle.js";
import { transitionMainSessionRecovery } from "./main-session-recovery-state.js";
const sessionKey = "agent:main:main";
function recoveryState(
overrides: Partial<MainRestartRecoveryState> = {},
): MainRestartRecoveryState {
return {
cycleId: "cycle-1",
revision: 1,
chargedAttempts: 0,
...overrides,
};
}
function interruptedEntry(overrides: Partial<SessionEntry> = {}): SessionEntry {
return {
sessionId: "session-1",
updatedAt: 100,
status: "running",
abortedLastRun: true,
mainRestartRecovery: recoveryState(),
...overrides,
};
}
function observe(entry: SessionEntry, lifecycleGeneration: string) {
const result = transitionMainSessionRecovery(entry, {
kind: "observe",
cycleId: "unused-cycle",
lifecycleGeneration,
sessionKey,
});
if (result.kind !== "observed") {
throw new Error("expected recovery observation");
}
return result.view;
}
describe("main session recovery state", () => {
it("gives a legacy interrupted row a stable cycle before exposing it to a scan", () => {
const entry = interruptedEntry({ mainRestartRecovery: undefined });
const observed = transitionMainSessionRecovery(entry, {
kind: "observe",
cycleId: "legacy-cycle",
lifecycleGeneration: "generation-1",
sessionKey,
});
expect(observed).toEqual({
kind: "observed",
view: {
status: "recoverable",
observation: { sessionId: "session-1", cycleId: "legacy-cycle", revision: 1 },
nextAttempt: 1,
},
});
expect(entry.mainRestartRecovery).toEqual(recoveryState({ cycleId: "legacy-cycle" }));
});
it("inspects a live reservation without adopting or releasing it", () => {
const entry = interruptedEntry({
mainRestartRecovery: recoveryState({
chargedAttempts: 1,
reservation: {
attempt: 1,
lifecycleGeneration: "gateway-generation",
runId: "recovery-1",
},
}),
});
const before = structuredClone(entry);
expect(
transitionMainSessionRecovery(entry, {
kind: "inspect",
lifecycleGeneration: "standalone-generation",
sessionKey,
}),
).toEqual({ kind: "observed", view: { status: "blocked" } });
expect(entry).toEqual(before);
});
it("marks without charging and preserves generation-scoped lifecycle fences", () => {
const entry = interruptedEntry({
restartRecoveryRuns: [
{ runId: "older-run", lifecycleGeneration: "generation-old" },
{ runId: "shared-run", lifecycleGeneration: "generation-1" },
],
mainRestartRecovery: recoveryState({
revision: 4,
chargedAttempts: 2,
}),
});
transitionMainSessionRecovery(entry, {
kind: "mark_interrupted",
cycleId: "unused-cycle",
now: 200,
runs: [
{ runId: "shared-run", lifecycleGeneration: "generation-2" },
{ runId: "new-run", lifecycleGeneration: "generation-2" },
],
});
expect(entry.mainRestartRecovery).toEqual(
recoveryState({
revision: 4,
chargedAttempts: 2,
}),
);
expect(entry.restartRecoveryRuns).toEqual([
{ runId: "new-run", lifecycleGeneration: "generation-2" },
{ runId: "older-run", lifecycleGeneration: "generation-old" },
{ runId: "shared-run", lifecycleGeneration: "generation-1" },
{ runId: "shared-run", lifecycleGeneration: "generation-2" },
]);
});
it("charges at reservation and refunds only the matching reservation", () => {
const entry = interruptedEntry();
const prepared = transitionMainSessionRecovery(entry, {
kind: "prepare_attempt",
attempt: 1,
lifecycleGeneration: "generation-1",
now: 200,
observation: { sessionId: "session-1", cycleId: "cycle-1", revision: 1 },
runId: "recovery-1",
});
expect(prepared.kind).toBe("reserved");
if (prepared.kind !== "reserved") {
throw new Error("expected reservation");
}
expect(
transitionMainSessionRecovery(entry, {
kind: "prepare_attempt",
attempt: 1,
lifecycleGeneration: "generation-1",
now: 201,
observation: { sessionId: "session-1", cycleId: "cycle-1", revision: 1 },
runId: "recovery-2",
}),
).toEqual({ kind: "rejected", reason: "stale_revision" });
expect(entry.mainRestartRecovery?.reservation).toMatchObject({
runId: "recovery-1",
attempt: 1,
});
const claim = transitionMainSessionRecovery(entry, {
kind: "claim_foreground",
cycleId: "unused",
lifecycleGeneration: "generation-1",
sessionId: "session-1",
sessionKey,
claimId: "foreground-1",
});
expect(claim.kind).toBe("foreground_claimed");
expect(
transitionMainSessionRecovery(entry, {
kind: "cancel_reservation",
reservation: prepared.reservation,
}),
).toEqual({ kind: "applied" });
expect(entry.mainRestartRecovery).toMatchObject({
chargedAttempts: 0,
foregroundClaims: {
lifecycleGeneration: "generation-1",
tokens: ["foreground-1"],
},
});
expect(entry.mainRestartRecovery?.reservation).toBeUndefined();
expect(observe(entry, "generation-1")).toEqual({ status: "blocked" });
});
it("rejects foreground work after the automatic recovery budget is exhausted", () => {
const entry = interruptedEntry({
mainRestartRecovery: recoveryState({ chargedAttempts: 3 }),
});
const before = structuredClone(entry);
expect(
transitionMainSessionRecovery(entry, {
kind: "claim_foreground",
cycleId: "unused",
lifecycleGeneration: "generation-1",
sessionId: "session-1",
sessionKey,
claimId: "foreground-1",
}),
).toEqual({ kind: "rejected", reason: "recovery_exhausted" });
expect(entry).toEqual(before);
});
it("retires a stale reservation before granting foreground ownership", () => {
const entry = interruptedEntry({
mainRestartRecovery: recoveryState({
revision: 3,
chargedAttempts: 1,
reservation: {
attempt: 1,
lifecycleGeneration: "previous-generation",
runId: "stale-recovery",
},
}),
});
expect(
transitionMainSessionRecovery(entry, {
kind: "claim_foreground",
cycleId: "unused",
lifecycleGeneration: "generation-1",
sessionId: "session-1",
sessionKey,
claimId: "foreground-1",
}),
).toMatchObject({ kind: "foreground_claimed" });
expect(entry.mainRestartRecovery).toMatchObject({
chargedAttempts: 1,
foregroundClaims: {
lifecycleGeneration: "generation-1",
tokens: ["foreground-1"],
},
});
expect(entry.mainRestartRecovery?.reservation).toBeUndefined();
});
it("releases an ambiguous dispatch reservation without refunding its charge", () => {
const entry = interruptedEntry();
const prepared = transitionMainSessionRecovery(entry, {
kind: "prepare_attempt",
attempt: 1,
lifecycleGeneration: "generation-1",
now: 200,
observation: { sessionId: "session-1", cycleId: "cycle-1", revision: 1 },
runId: "recovery-1",
});
if (prepared.kind !== "reserved") {
throw new Error("expected reservation");
}
expect(
transitionMainSessionRecovery(entry, {
kind: "abandon_reservation",
reservation: prepared.reservation,
}),
).toEqual({ kind: "applied" });
expect(entry.mainRestartRecovery).toMatchObject({ chargedAttempts: 1 });
expect(entry.mainRestartRecovery?.reservation).toBeUndefined();
expect(observe(entry, "generation-1")).toMatchObject({
status: "recoverable",
nextAttempt: 2,
});
});
it("moves a reservation into the lifecycle fence during Gateway admission", () => {
const entry = interruptedEntry({
pendingFinalDelivery: true,
pendingFinalDeliveryText: " captured reply ",
restartRecoveryDeliveryRunId: "recovery-1",
restartRecoveryDeliverySourceRunId: "source-1",
mainRestartRecovery: recoveryState({
revision: 2,
chargedAttempts: 1,
reservation: {
runId: "recovery-1",
attempt: 1,
lifecycleGeneration: "generation-1",
},
}),
});
expect(
transitionMainSessionRecovery(entry, {
kind: "validate_recovery",
lifecycleGeneration: "generation-1",
runId: "recovery-1",
sessionId: "session-1",
}),
).toEqual({ kind: "recovery_validated" });
expect(entry.mainRestartRecovery?.reservation).toBeDefined();
expect(entry.abortedLastRun).toBe(true);
expect(
transitionMainSessionRecovery(entry, {
kind: "admit_recovery",
lifecycleGeneration: "generation-1",
now: 300,
runId: "recovery-1",
sessionId: "session-1",
}),
).toEqual({ kind: "admitted_recovery" });
expect(entry).toMatchObject({
abortedLastRun: false,
pendingFinalDeliveryAttemptCount: 1,
pendingFinalDeliveryLastAttemptAt: 300,
restartRecoveryRuns: [{ runId: "recovery-1", lifecycleGeneration: "generation-1" }],
mainRestartRecovery: {
revision: 3,
chargedAttempts: 1,
},
});
expect(entry.mainRestartRecovery?.reservation).toBeUndefined();
expect(
transitionMainSessionRecovery(entry, {
kind: "mark_admitted_recovery_interrupted",
lifecycleGeneration: "generation-1",
now: 400,
runId: "recovery-1",
sessionId: "session-1",
}),
).toMatchObject({ kind: "applied" });
expect(entry.mainRestartRecovery?.chargedAttempts).toBe(1);
expect(entry.mainRestartRecovery?.reservation).toBeUndefined();
expect(entry.abortedLastRun).toBe(true);
expect(entry.restartRecoveryDeliveryRunId).toBeUndefined();
expect(entry.restartRecoveryDeliverySourceRunId).toBe("source-1");
});
it("rejects a reservation created by an older lifecycle generation", () => {
const entry = interruptedEntry({
mainRestartRecovery: recoveryState({
revision: 2,
chargedAttempts: 1,
reservation: {
runId: "recovery-1",
attempt: 1,
lifecycleGeneration: "generation-old",
},
}),
});
expect(
transitionMainSessionRecovery(entry, {
kind: "admit_recovery",
lifecycleGeneration: "generation-new",
now: 300,
runId: "recovery-1",
sessionId: "session-1",
}),
).toEqual({ kind: "rejected", reason: "stale_reservation" });
expect(entry.mainRestartRecovery?.reservation).toBeDefined();
expect(entry.abortedLastRun).toBe(true);
});
it("rejects recovery admission while any current-generation foreground claim remains", () => {
const entry = interruptedEntry({
mainRestartRecovery: recoveryState({
revision: 4,
chargedAttempts: 1,
reservation: {
runId: "recovery-1",
attempt: 1,
lifecycleGeneration: "generation-1",
},
foregroundClaims: {
lifecycleGeneration: "generation-1",
tokens: ["foreground-1", "foreground-2"],
},
}),
});
expect(
transitionMainSessionRecovery(entry, {
kind: "admit_recovery",
lifecycleGeneration: "generation-1",
now: 300,
runId: "recovery-1",
sessionId: "session-1",
}),
).toEqual({ kind: "rejected", reason: "foreground_active" });
expect(observe(entry, "generation-1")).toEqual({ status: "blocked" });
});
it("clears a healthy recovery aggregate when its final foreground owner releases", () => {
const entry = interruptedEntry({
abortedLastRun: false,
restartRecoveryRuns: [{ runId: "recovery-1", lifecycleGeneration: "generation-1" }],
mainRestartRecovery: recoveryState({
revision: 3,
chargedAttempts: 1,
foregroundClaims: {
lifecycleGeneration: "generation-1",
tokens: ["foreground-1"],
},
}),
});
expect(observe(entry, "generation-1")).toEqual({ status: "blocked" });
expect(
transitionMainSessionRecovery(entry, {
kind: "release_foreground",
claim: {
cycleId: "cycle-1",
lifecycleGeneration: "generation-1",
claimId: "foreground-1",
sessionId: "session-1",
sessionKey,
},
}),
).toEqual({ kind: "applied" });
expect(entry).toMatchObject({ abortedLastRun: false });
expect(entry.restartRecoveryRuns).toBeUndefined();
expect(entry.mainRestartRecovery).toBeUndefined();
});
it("expires foreground claims from an older lifecycle generation", () => {
const entry = interruptedEntry({
mainRestartRecovery: recoveryState({
foregroundClaims: {
lifecycleGeneration: "generation-old",
tokens: ["old-owner"],
},
}),
});
const view = observe(entry, "generation-new");
expect(view).toMatchObject({ status: "recoverable", nextAttempt: 1 });
expect(entry.mainRestartRecovery).toMatchObject({ revision: 2 });
expect(entry.mainRestartRecovery?.foregroundClaims).toBeUndefined();
});
it("clears a healthy aggregate owned by an older lifecycle generation", () => {
const entry = interruptedEntry({
abortedLastRun: false,
restartRecoveryRuns: [{ runId: "completed-run", lifecycleGeneration: "generation-old" }],
mainRestartRecovery: recoveryState({
foregroundClaims: {
lifecycleGeneration: "generation-old",
tokens: ["old-owner"],
},
}),
});
expect(observe(entry, "generation-new")).toEqual({ status: "inactive" });
expect(entry.restartRecoveryRuns).toBeUndefined();
expect(entry.mainRestartRecovery).toBeUndefined();
});
it("retains the charge but releases a reservation orphaned by process restart", () => {
const entry = interruptedEntry({
mainRestartRecovery: recoveryState({
revision: 2,
chargedAttempts: 1,
reservation: {
runId: "recovery-1",
attempt: 1,
lifecycleGeneration: "generation-old",
},
}),
});
expect(
transitionMainSessionRecovery(entry, {
kind: "observe",
cycleId: "unused",
lifecycleGeneration: "generation-new",
sessionKey,
}),
).toEqual({
kind: "observed",
view: {
status: "recoverable",
observation: { sessionId: "session-1", cycleId: "cycle-1", revision: 3 },
nextAttempt: 2,
},
});
expect(entry.mainRestartRecovery).toMatchObject({
revision: 3,
chargedAttempts: 1,
});
expect(entry.mainRestartRecovery?.reservation).toBeUndefined();
});
it("rejects an old observation after a healthy clear and a new interrupted cycle", () => {
const entry = interruptedEntry();
const oldObservation = { sessionId: "session-1", cycleId: "cycle-1", revision: 1 };
transitionMainSessionRecovery(entry, { kind: "clear" });
transitionMainSessionRecovery(entry, {
kind: "mark_interrupted",
cycleId: "cycle-2",
now: 400,
});
expect(
transitionMainSessionRecovery(entry, {
kind: "prepare_attempt",
attempt: 1,
lifecycleGeneration: "generation-1",
now: 500,
observation: oldObservation,
runId: "stale-run",
}),
).toEqual({ kind: "rejected", reason: "stale_cycle" });
});
it("preserves the charged cycle when an accepted recovery is interrupted again", () => {
const entry = interruptedEntry({
abortedLastRun: false,
mainRestartRecovery: recoveryState({
revision: 3,
chargedAttempts: 1,
}),
});
transitionMainSessionRecovery(entry, {
kind: "mark_interrupted",
cycleId: "replacement-cycle",
now: 300,
});
expect(entry.abortedLastRun).toBe(true);
expect(entry.mainRestartRecovery).toEqual(
recoveryState({
revision: 3,
chargedAttempts: 1,
}),
);
});
it("tombstones an exhausted cycle and exposes only the Doctor repair action", () => {
const entry = interruptedEntry({
mainRestartRecovery: recoveryState({
chargedAttempts: 3,
}),
});
const view = observe(entry, "generation-1");
expect(view.status).toBe("exhausted");
if (view.status !== "exhausted") {
throw new Error("expected exhausted cycle");
}
expect(
transitionMainSessionRecovery(entry, {
kind: "tombstone",
now: 400,
observation: view.observation,
reason: view.reason,
}),
).toEqual({ kind: "tombstoned" });
expect(inspectMainSessionRecoveryHealth(entry)).toEqual({
status: "tombstoned",
reason: view.reason,
repair: null,
});
expect(observe(entry, "generation-1")).toEqual({ status: "tombstoned" });
entry.abortedLastRun = true;
expect(inspectMainSessionRecoveryHealth(entry)).toMatchObject({
status: "tombstoned",
repair: "clear_stale_abort",
});
expect(transitionMainSessionRecovery(entry, { kind: "doctor_repair", now: 500 })).toEqual({
kind: "doctor_repaired",
});
expect(entry.abortedLastRun).toBe(false);
});
it("owns lifecycle fence suppression, consumption, and healthy clearing", () => {
const entry = interruptedEntry({
abortedLastRun: false,
restartRecoveryDeliveryRunId: "recovery",
restartRecoveryRuns: [
{ runId: "interrupted", lifecycleGeneration: "generation-1" },
{ runId: "recovery", lifecycleGeneration: "generation-1" },
],
});
expect(
projectMainSessionRecoveryLifecycle({
currentLifecycleGeneration: "generation-1",
entry,
event: {
runId: "interrupted",
lifecycleGeneration: "generation-1",
data: { phase: "error", stopReason: "restart" },
},
snapshotPatch: { status: "failed" },
}),
).toEqual({ action: "suppress" });
expect(
projectMainSessionRecoveryLifecycle({
currentLifecycleGeneration: "generation-1",
entry,
event: {
runId: "recovery",
lifecycleGeneration: "generation-1",
data: { phase: "end" },
},
snapshotPatch: { status: "done", abortedLastRun: false },
}),
).toEqual({
action: "apply",
patch: {
status: "done",
abortedLastRun: false,
restartRecoveryRuns: undefined,
mainRestartRecovery: undefined,
},
});
expect(
projectMainSessionRecoveryLifecycle({
currentLifecycleGeneration: "generation-1",
entry,
event: {
runId: "recovery",
lifecycleGeneration: "generation-1",
data: { phase: "error", error: "provider failed" },
},
snapshotPatch: { status: "failed", abortedLastRun: false },
}),
).toEqual({
action: "apply",
patch: {
status: "failed",
abortedLastRun: false,
restartRecoveryRuns: undefined,
mainRestartRecovery: undefined,
},
});
});
it("does not let a delayed lifecycle event clear current-generation owners", () => {
const entry = interruptedEntry({
abortedLastRun: false,
restartRecoveryRuns: [{ runId: "old-run", lifecycleGeneration: "generation-1" }],
restartRecoveryTerminalRunIds: ["prior-run"],
mainRestartRecovery: recoveryState({
foregroundClaims: {
lifecycleGeneration: "generation-2",
tokens: ["current-owner"],
},
}),
});
expect(
projectMainSessionRecoveryLifecycle({
currentLifecycleGeneration: "generation-2",
entry,
event: {
runId: "old-run",
lifecycleGeneration: "generation-1",
data: { phase: "end" },
},
snapshotPatch: { status: "done", abortedLastRun: false },
}),
).toEqual({
action: "apply",
patch: {
restartRecoveryRuns: undefined,
restartRecoveryTerminalRunIds: ["prior-run", "old-run"],
},
});
expect(entry.mainRestartRecovery?.foregroundClaims?.tokens).toEqual(["current-owner"]);
});
it("does not let a delayed lifecycle event clear a current reservation", () => {
const entry = interruptedEntry({
restartRecoveryRuns: [{ runId: "old-run", lifecycleGeneration: "generation-1" }],
mainRestartRecovery: recoveryState({
chargedAttempts: 1,
reservation: {
runId: "current-run",
attempt: 1,
lifecycleGeneration: "generation-2",
},
}),
});
expect(
projectMainSessionRecoveryLifecycle({
currentLifecycleGeneration: "generation-2",
entry,
event: {
runId: "old-run",
lifecycleGeneration: "generation-1",
data: { phase: "end" },
},
snapshotPatch: { status: "done", abortedLastRun: false },
}),
).toEqual({
action: "apply",
patch: {
restartRecoveryRuns: undefined,
restartRecoveryTerminalRunIds: ["old-run"],
},
});
expect(entry.mainRestartRecovery?.reservation?.runId).toBe("current-run");
});
it("preserves concurrent owners and fences when one lifecycle run completes", () => {
const entry = interruptedEntry({
abortedLastRun: false,
restartRecoveryRuns: [
{ runId: "recovery-1", lifecycleGeneration: "generation-1" },
{ runId: "recovery-2", lifecycleGeneration: "generation-1" },
],
mainRestartRecovery: recoveryState({
revision: 4,
foregroundClaims: {
lifecycleGeneration: "generation-1",
tokens: ["owner-1", "owner-2"],
},
}),
});
expect(
projectMainSessionRecoveryLifecycle({
currentLifecycleGeneration: "generation-1",
entry,
event: {
runId: "recovery-1",
lifecycleGeneration: "generation-1",
data: { phase: "end" },
},
snapshotPatch: {
status: "done",
abortedLastRun: false,
restartRecoveryRuns: undefined,
mainRestartRecovery: undefined,
},
}),
).toEqual({
action: "apply",
patch: {
restartRecoveryRuns: [{ runId: "recovery-2", lifecycleGeneration: "generation-1" }],
restartRecoveryTerminalRunIds: ["recovery-1"],
},
});
});
it("does not let another terminal run clear the active recovery delivery", () => {
const entry = interruptedEntry({
abortedLastRun: false,
restartRecoveryDeliveryRunId: "recovery-2",
restartRecoveryRuns: [
{ runId: "recovery-1", lifecycleGeneration: "generation-1" },
{ runId: "recovery-2", lifecycleGeneration: "generation-1" },
],
});
expect(
projectMainSessionRecoveryLifecycle({
currentLifecycleGeneration: "generation-1",
entry,
event: {
runId: "recovery-1",
lifecycleGeneration: "generation-1",
data: { phase: "end" },
},
snapshotPatch: { status: "done", abortedLastRun: false },
}),
).toEqual({
action: "apply",
patch: {
status: "done",
abortedLastRun: false,
restartRecoveryRuns: [{ runId: "recovery-2", lifecycleGeneration: "generation-1" }],
mainRestartRecovery: entry.mainRestartRecovery,
},
});
});
it("does not let an unrelated lifecycle completion clear pending recovery", () => {
const entry = interruptedEntry({
restartRecoveryRuns: [{ runId: "recovery", lifecycleGeneration: "generation-1" }],
});
expect(
projectMainSessionRecoveryLifecycle({
currentLifecycleGeneration: "generation-1",
entry,
event: {
runId: "ordinary-run",
lifecycleGeneration: "generation-1",
data: { phase: "end" },
},
snapshotPatch: {
status: "done",
abortedLastRun: false,
restartRecoveryRuns: undefined,
mainRestartRecovery: undefined,
},
}),
).toEqual({ action: "suppress" });
});
it("suppresses an unmatched completion while a current foreground owner is active", () => {
const entry = interruptedEntry({
abortedLastRun: false,
mainRestartRecovery: recoveryState({
foregroundClaims: {
lifecycleGeneration: "generation-1",
tokens: ["current-owner"],
},
}),
});
expect(
projectMainSessionRecoveryLifecycle({
currentLifecycleGeneration: "generation-1",
entry,
event: {
runId: "unrelated-run",
lifecycleGeneration: "generation-1",
data: { phase: "end" },
},
snapshotPatch: { status: "done", abortedLastRun: false },
}),
).toEqual({ action: "suppress" });
});
it("applies ordinary lifecycle completion without recovery metadata", () => {
expect(
projectMainSessionRecoveryLifecycle({
currentLifecycleGeneration: "generation-1",
entry: { abortedLastRun: false },
event: {
runId: "ordinary-run",
lifecycleGeneration: "generation-1",
data: { phase: "end" },
},
snapshotPatch: { status: "done", abortedLastRun: false },
}),
).toEqual({
action: "apply",
patch: { status: "done", abortedLastRun: false },
});
});
it("does not preserve foreground claims from an older lifecycle generation", () => {
const entry = interruptedEntry({
abortedLastRun: false,
restartRecoveryRuns: [{ runId: "recovery", lifecycleGeneration: "generation-2" }],
mainRestartRecovery: recoveryState({
foregroundClaims: {
lifecycleGeneration: "generation-1",
tokens: ["stale-owner"],
},
}),
});
expect(
projectMainSessionRecoveryLifecycle({
currentLifecycleGeneration: "generation-2",
entry,
event: {
runId: "recovery",
lifecycleGeneration: "generation-2",
data: { phase: "end" },
},
snapshotPatch: { status: "done", abortedLastRun: false },
}),
).toEqual({
action: "apply",
patch: {
status: "done",
abortedLastRun: false,
restartRecoveryRuns: undefined,
mainRestartRecovery: undefined,
},
});
});
it("does not let a delayed lifecycle event erase a recovery tombstone", () => {
const entry = interruptedEntry({
abortedLastRun: false,
status: "failed",
restartRecoveryRuns: [{ runId: "old-run", lifecycleGeneration: "generation-1" }],
mainRestartRecovery: recoveryState({
revision: 4,
chargedAttempts: 3,
tombstone: { reason: "automatic recovery exhausted" },
}),
});
expect(
projectMainSessionRecoveryLifecycle({
currentLifecycleGeneration: "generation-1",
entry,
event: {
runId: "old-run",
lifecycleGeneration: "generation-1",
data: { phase: "end" },
},
snapshotPatch: { status: "done", abortedLastRun: false },
}),
).toEqual({
action: "apply",
patch: {
status: "done",
abortedLastRun: false,
restartRecoveryRuns: entry.restartRecoveryRuns,
mainRestartRecovery: entry.mainRestartRecovery,
},
});
});
it("builds an empty clear patch when no main recovery state exists", () => {
expect(buildMainSessionRecoveryClearPatch({ abortedLastRun: false })).toEqual({});
});
});
+659
View File
@@ -0,0 +1,659 @@
import {
PENDING_FINAL_DELIVERY_CLEAR_PATCH,
sanitizePendingFinalDeliveryText,
} from "../auto-reply/reply/pending-final-delivery.js";
import type {
InternalSessionEntry as SessionEntry,
MainRestartRecoveryState,
RestartRecoveryRun,
} from "../config/sessions.js";
import { buildRestartRecoveryClaimCleanupPatch } from "../config/sessions/restart-recovery-state.js";
import { isAcpSessionKey, isCronSessionKey, isSubagentSessionKey } from "../routing/session-key.js";
import { buildMainSessionRecoveryClearPatch } from "./main-session-recovery-clear.js";
import type {
MainSessionRecoveryCommand,
MainSessionRecoveryConflict,
MainSessionRecoveryObservation,
MainSessionRecoveryTransitionResult,
MainSessionRecoveryView,
} from "./main-session-recovery-types.js";
export type {
MainSessionRecoveryCommand,
MainSessionRecoveryObservation,
MainSessionRecoveryOwnerClaim,
MainSessionRecoveryReservation,
MainSessionRecoveryTransitionResult,
} from "./main-session-recovery-types.js";
const MAIN_RESTART_RECOVERY_MAX_AUTOMATIC_ATTEMPTS = 3;
const MAIN_RESTART_RECOVERY_REMEDIATION_HINT =
"inspect the failed main session and use /new or reset to start a replacement session";
function nextRevision(state: MainRestartRecoveryState): number {
return state.revision + 1;
}
function createCycle(cycleId: string): MainRestartRecoveryState {
return {
cycleId,
revision: 1,
chargedAttempts: 0,
};
}
function observationFor(entry: SessionEntry): MainSessionRecoveryObservation | undefined {
const state = entry.mainRestartRecovery;
if (!state) {
return undefined;
}
return {
sessionId: entry.sessionId,
cycleId: state.cycleId,
revision: state.revision,
};
}
function matchesObservation(
entry: SessionEntry,
observation: MainSessionRecoveryObservation,
): MainSessionRecoveryConflict | null {
if (entry.sessionId !== observation.sessionId) {
return "session_replaced";
}
if (entry.mainRestartRecovery?.cycleId !== observation.cycleId) {
return "stale_cycle";
}
return entry.mainRestartRecovery.revision === observation.revision ? null : "stale_revision";
}
function hasCurrentForegroundClaim(
state: MainRestartRecoveryState,
lifecycleGeneration: string,
): boolean {
return (
state.foregroundClaims?.lifecycleGeneration === lifecycleGeneration &&
state.foregroundClaims.tokens.length > 0
);
}
function validateRecoveryAdmission(
entry: SessionEntry,
command: {
lifecycleGeneration: string;
runId: string;
sessionId: string;
},
): MainSessionRecoveryConflict | null {
const state = entry.mainRestartRecovery;
if (entry.sessionId !== command.sessionId) {
return "session_replaced";
}
if (entry.status !== "running" || entry.abortedLastRun !== true || !state) {
return "not_interrupted";
}
if (
state.reservation?.runId !== command.runId ||
state.reservation.lifecycleGeneration !== command.lifecycleGeneration
) {
return "stale_reservation";
}
return hasCurrentForegroundClaim(state, command.lifecycleGeneration) ? "foreground_active" : null;
}
function recordLifecycleFence(entry: SessionEntry, run: RestartRecoveryRun): void {
// Lifecycle fences can overlap and are consumed independently by their matching events.
const runs = new Map<string, RestartRecoveryRun>();
for (const existing of entry.restartRecoveryRuns ?? []) {
runs.set(`${existing.runId}\u0000${existing.lifecycleGeneration}`, existing);
}
runs.set(`${run.runId}\u0000${run.lifecycleGeneration}`, run);
entry.restartRecoveryRuns = [...runs.values()].toSorted((a, b) =>
a.runId === b.runId
? a.lifecycleGeneration.localeCompare(b.lifecycleGeneration)
: a.runId.localeCompare(b.runId),
);
}
function hasLifecycleFence(entry: SessionEntry, run: RestartRecoveryRun): boolean {
return Boolean(
entry.restartRecoveryRuns?.some(
(candidate) =>
candidate.runId === run.runId && candidate.lifecycleGeneration === run.lifecycleGeneration,
),
);
}
function formatAttemptBudgetReason(attempts: number): string {
return (
`main-session restart recovery blocked after ${attempts} charged automatic resume attempts; ` +
MAIN_RESTART_RECOVERY_REMEDIATION_HINT
);
}
export function isMainSessionRecoveryExhausted(entry: SessionEntry): boolean {
return (
entry.status === "running" &&
entry.abortedLastRun === true &&
(entry.mainRestartRecovery?.chargedAttempts ?? 0) >=
MAIN_RESTART_RECOVERY_MAX_AUTOMATIC_ATTEMPTS
);
}
export function isMainRestartRecoveryCandidate(entry: SessionEntry, sessionKey: string): boolean {
if (typeof entry.spawnDepth === "number" && entry.spawnDepth > 0) {
return false;
}
if (entry.subagentRole != null) {
return false;
}
return (
!isSubagentSessionKey(sessionKey) &&
!isCronSessionKey(sessionKey) &&
!isAcpSessionKey(sessionKey)
);
}
function inspectMainSessionRecovery(params: {
entry: SessionEntry;
lifecycleGeneration: string;
sessionKey: string;
}): MainSessionRecoveryView {
const { entry } = params;
const state = entry.mainRestartRecovery;
if (state?.tombstone) {
return { status: "tombstoned" };
}
if (state && hasCurrentForegroundClaim(state, params.lifecycleGeneration)) {
return { status: "blocked" };
}
if (
entry.status === "running" &&
entry.abortedLastRun !== true &&
state &&
entry.restartRecoveryRuns?.some((run) => run.lifecycleGeneration === params.lifecycleGeneration)
) {
// Admission clears the interruption flag before the recovery run settles.
// Keep ordinary work fenced until that run clears its lifecycle metadata.
return { status: "blocked" };
}
if (
entry.status !== "running" ||
entry.abortedLastRun !== true ||
!isMainRestartRecoveryCandidate(entry, params.sessionKey)
) {
return { status: "inactive" };
}
const observation = observationFor(entry);
if (!state || !observation) {
return { status: "inactive" };
}
if (state.reservation) {
return { status: "blocked" };
}
if (state.chargedAttempts >= MAIN_RESTART_RECOVERY_MAX_AUTOMATIC_ATTEMPTS) {
return {
status: "exhausted",
observation,
reason: formatAttemptBudgetReason(state.chargedAttempts),
};
}
return {
status: "recoverable",
observation,
nextAttempt: state.chargedAttempts + 1,
};
}
function inspectMainSessionRecoveryForAdmission(params: {
entry: SessionEntry;
lifecycleGeneration: string;
sessionKey: string;
}): MainSessionRecoveryView {
if (
params.entry.status === "running" &&
params.entry.abortedLastRun !== true &&
params.entry.mainRestartRecovery &&
params.entry.restartRecoveryRuns?.length
) {
// Standalone callers may use another process generation. Any admitted
// recovery fence remains authoritative until Gateway lifecycle settlement.
return { status: "blocked" };
}
if (
params.entry.status === "running" &&
params.entry.abortedLastRun === true &&
isMainRestartRecoveryCandidate(params.entry, params.sessionKey) &&
!params.entry.mainRestartRecovery
) {
// Older interrupted rows still quarantine foreground work, but only the
// Gateway startup owner may assign their durable recovery cycle.
return { status: "blocked" };
}
return inspectMainSessionRecovery(params);
}
export function transitionMainSessionRecovery(
entry: SessionEntry,
command: MainSessionRecoveryCommand,
): MainSessionRecoveryTransitionResult {
switch (command.kind) {
case "mark_interrupted": {
if (!entry.mainRestartRecovery) {
entry.mainRestartRecovery = createCycle(command.cycleId);
}
entry.status = "running";
entry.abortedLastRun = true;
if (command.resetRuntime) {
entry.startedAt = undefined;
entry.endedAt = undefined;
entry.runtimeMs = undefined;
}
for (const run of command.runs ?? []) {
recordLifecycleFence(entry, run);
}
entry.updatedAt = command.now;
return { kind: "applied" };
}
case "inspect": {
return {
kind: "observed",
view: inspectMainSessionRecoveryForAdmission({
entry,
lifecycleGeneration: command.lifecycleGeneration,
sessionKey: command.sessionKey,
}),
};
}
case "observe": {
if (
entry.status === "running" &&
entry.abortedLastRun === true &&
isMainRestartRecoveryCandidate(entry, command.sessionKey) &&
!entry.mainRestartRecovery
) {
// Rows interrupted by an older shipped version acquire identity before scanning.
entry.mainRestartRecovery = createCycle(command.cycleId);
}
let state = entry.mainRestartRecovery;
if (
state?.foregroundClaims &&
state.foregroundClaims.lifecycleGeneration !== command.lifecycleGeneration
) {
// Process-local owners cannot survive a Gateway generation. Retire their
// durable lease before the new process decides whether recovery is needed.
if (entry.abortedLastRun !== true) {
Object.assign(entry, buildMainSessionRecoveryClearPatch(entry));
state = undefined;
} else {
entry.mainRestartRecovery = state = {
...state,
revision: nextRevision(state),
foregroundClaims: undefined,
};
}
}
if (
state?.reservation &&
state.reservation.lifecycleGeneration !== command.lifecycleGeneration
) {
// A process restart makes dispatch outcome unknowable: retain the charge,
// but release the stale slot so the next bounded attempt can proceed.
entry.mainRestartRecovery = {
...state,
revision: nextRevision(state),
reservation: undefined,
};
}
return {
kind: "observed",
view: inspectMainSessionRecovery({
entry,
lifecycleGeneration: command.lifecycleGeneration,
sessionKey: command.sessionKey,
}),
};
}
case "prepare_attempt": {
const conflict = matchesObservation(entry, command.observation);
if (conflict) {
return { kind: "rejected", reason: conflict };
}
const state = entry.mainRestartRecovery!;
if (entry.status !== "running" || entry.abortedLastRun !== true) {
return { kind: "rejected", reason: "not_interrupted" };
}
if (state.tombstone) {
return { kind: "rejected", reason: "already_tombstoned" };
}
if (state.reservation) {
return { kind: "rejected", reason: "reservation_active" };
}
if (command.attempt !== state.chargedAttempts + 1) {
return { kind: "rejected", reason: "stale_revision" };
}
entry.mainRestartRecovery = {
...state,
revision: nextRevision(state),
chargedAttempts: command.attempt,
reservation: {
runId: command.runId,
attempt: command.attempt,
lifecycleGeneration: command.lifecycleGeneration,
},
};
entry.updatedAt = command.now;
return {
kind: "reserved",
reservation: {
sessionId: entry.sessionId,
cycleId: state.cycleId,
lifecycleGeneration: command.lifecycleGeneration,
runId: command.runId,
attempt: command.attempt,
},
};
}
case "cancel_reservation": {
const state = entry.mainRestartRecovery;
const reserved = state?.reservation;
if (
!state ||
entry.sessionId !== command.reservation.sessionId ||
state.cycleId !== command.reservation.cycleId ||
reserved?.runId !== command.reservation.runId ||
reserved.attempt !== command.reservation.attempt ||
reserved.lifecycleGeneration !== command.reservation.lifecycleGeneration
) {
return { kind: "rejected", reason: "stale_reservation" };
}
entry.mainRestartRecovery = {
...state,
revision: nextRevision(state),
chargedAttempts: Math.max(0, command.reservation.attempt - 1),
reservation: undefined,
};
return { kind: "applied" };
}
case "abandon_reservation": {
const state = entry.mainRestartRecovery;
const reserved = state?.reservation;
if (
!state ||
entry.sessionId !== command.reservation.sessionId ||
state.cycleId !== command.reservation.cycleId ||
reserved?.runId !== command.reservation.runId ||
reserved.attempt !== command.reservation.attempt ||
reserved.lifecycleGeneration !== command.reservation.lifecycleGeneration
) {
return { kind: "rejected", reason: "stale_reservation" };
}
entry.mainRestartRecovery = {
...state,
revision: nextRevision(state),
reservation: undefined,
};
return { kind: "applied" };
}
case "validate_recovery": {
const conflict = validateRecoveryAdmission(entry, command);
return conflict ? { kind: "rejected", reason: conflict } : { kind: "recovery_validated" };
}
case "admit_recovery": {
const conflict = validateRecoveryAdmission(entry, command);
if (conflict) {
return { kind: "rejected", reason: conflict };
}
const state = entry.mainRestartRecovery!;
entry.mainRestartRecovery = {
...state,
revision: nextRevision(state),
reservation: undefined,
foregroundClaims: undefined,
};
entry.abortedLastRun = false;
recordLifecycleFence(entry, {
runId: command.runId,
lifecycleGeneration: command.lifecycleGeneration,
});
if (entry.pendingFinalDelivery || entry.pendingFinalDeliveryText) {
const pendingText = sanitizePendingFinalDeliveryText(entry.pendingFinalDeliveryText ?? "");
if (pendingText) {
entry.pendingFinalDeliveryLastAttemptAt = command.now;
entry.pendingFinalDeliveryAttemptCount =
(entry.pendingFinalDeliveryAttemptCount ?? 0) + 1;
entry.pendingFinalDeliveryLastError = null;
entry.pendingFinalDeliveryText = pendingText;
} else {
Object.assign(entry, PENDING_FINAL_DELIVERY_CLEAR_PATCH);
}
}
return { kind: "admitted_recovery" };
}
case "mark_admitted_recovery_interrupted": {
const state = entry.mainRestartRecovery;
if (entry.sessionId !== command.sessionId) {
return { kind: "rejected", reason: "session_replaced" };
}
if (
!state ||
state.reservation ||
!hasLifecycleFence(entry, {
runId: command.runId,
lifecycleGeneration: command.lifecycleGeneration,
})
) {
return { kind: "rejected", reason: "stale_reservation" };
}
entry.status = "running";
entry.abortedLastRun = true;
entry.startedAt = undefined;
entry.endedAt = undefined;
entry.runtimeMs = undefined;
if (entry.restartRecoveryDeliveryRunId === command.runId) {
// Gateway accepted this RPC id before setup failed. Rotate it on retry
// or the dedupe cache replays that terminal pre-dispatch failure.
entry.restartRecoveryDeliveryRunId = undefined;
}
entry.updatedAt = command.now;
return { kind: "applied" };
}
case "claim_foreground": {
if (
entry.sessionId !== command.sessionId ||
entry.status !== "running" ||
entry.abortedLastRun !== true ||
!isMainRestartRecoveryCandidate(entry, command.sessionKey)
) {
return { kind: "no_change" };
}
const state = entry.mainRestartRecovery ?? createCycle(command.cycleId);
if (state.tombstone) {
return { kind: "rejected", reason: "already_tombstoned" };
}
if (state.chargedAttempts >= MAIN_RESTART_RECOVERY_MAX_AUTOMATIC_ATTEMPTS) {
// The final charge fences foreground work until the scheduler commits
// the matching tombstone. Admitting here can race that reconciliation.
return { kind: "rejected", reason: "recovery_exhausted" };
}
const currentTokens =
state.foregroundClaims?.lifecycleGeneration === command.lifecycleGeneration
? state.foregroundClaims.tokens
: [];
const tokens = [...new Set([...currentTokens, command.claimId])].toSorted();
const currentRunIds =
state.foregroundClaims?.lifecycleGeneration === command.lifecycleGeneration
? state.foregroundClaims.runIdsByClaimId
: undefined;
const runIdsByClaimId = command.runId
? { ...currentRunIds, [command.claimId]: command.runId }
: currentRunIds;
if (command.runId) {
recordLifecycleFence(entry, {
lifecycleGeneration: command.lifecycleGeneration,
runId: command.runId,
});
}
entry.mainRestartRecovery = {
...state,
revision: nextRevision(state),
reservation:
state.reservation?.lifecycleGeneration === command.lifecycleGeneration
? state.reservation
: undefined,
foregroundClaims: {
lifecycleGeneration: command.lifecycleGeneration,
tokens,
...(runIdsByClaimId ? { runIdsByClaimId } : {}),
},
};
return {
kind: "foreground_claimed",
claim: {
cycleId: state.cycleId,
lifecycleGeneration: command.lifecycleGeneration,
claimId: command.claimId,
sessionId: entry.sessionId,
sessionKey: command.sessionKey,
...(command.runId ? { runId: command.runId } : {}),
},
};
}
case "bind_foreground_run": {
const state = entry.mainRestartRecovery;
const claims = state?.foregroundClaims;
if (
!state ||
state.cycleId !== command.claim.cycleId ||
claims?.lifecycleGeneration !== command.claim.lifecycleGeneration ||
!claims.tokens.includes(command.claim.claimId)
) {
return { kind: "no_change" };
}
recordLifecycleFence(entry, {
lifecycleGeneration: command.claim.lifecycleGeneration,
runId: command.runId,
});
entry.mainRestartRecovery = {
...state,
revision: nextRevision(state),
foregroundClaims: {
...claims,
runIdsByClaimId: { ...claims.runIdsByClaimId, [command.claim.claimId]: command.runId },
},
};
return { kind: "applied" };
}
case "validate_foreground": {
const state = entry.mainRestartRecovery;
const claims = state?.foregroundClaims;
return entry.sessionId === command.claim.sessionId &&
state?.cycleId === command.claim.cycleId &&
claims?.lifecycleGeneration === command.claim.lifecycleGeneration &&
claims.tokens.includes(command.claim.claimId)
? { kind: "foreground_validated" }
: { kind: "no_change" };
}
case "release_foreground": {
const state = entry.mainRestartRecovery;
const claims = state?.foregroundClaims;
if (
!state ||
state.cycleId !== command.claim.cycleId ||
claims?.lifecycleGeneration !== command.claim.lifecycleGeneration ||
!claims.tokens.includes(command.claim.claimId)
) {
return { kind: "no_change" };
}
const tokens = claims.tokens.filter((token) => token !== command.claim.claimId);
const runIdsByClaimId = Object.fromEntries(
Object.entries(claims.runIdsByClaimId ?? {}).filter(
([token]) => token !== command.claim.claimId,
),
);
if (tokens.length === 0 && entry.abortedLastRun !== true) {
Object.assign(entry, buildMainSessionRecoveryClearPatch(entry));
return { kind: "applied" };
}
entry.mainRestartRecovery = {
...state,
revision: nextRevision(state),
foregroundClaims:
tokens.length > 0
? {
lifecycleGeneration: command.claim.lifecycleGeneration,
tokens,
...(Object.keys(runIdsByClaimId).length > 0 ? { runIdsByClaimId } : {}),
}
: undefined,
};
return { kind: "applied" };
}
case "tombstone": {
const conflict = matchesObservation(entry, command.observation);
if (conflict) {
return { kind: "rejected", reason: conflict };
}
const state = entry.mainRestartRecovery!;
if (state.reservation) {
return { kind: "rejected", reason: "reservation_active" };
}
if (state.tombstone) {
return { kind: "rejected", reason: "already_tombstoned" };
}
entry.mainRestartRecovery = {
...state,
revision: nextRevision(state),
tombstone: {
reason: command.reason,
},
};
entry.abortedLastRun = false;
entry.status = "failed";
entry.endedAt = command.now;
entry.runtimeMs = Math.max(0, command.now - (entry.startedAt ?? command.now));
entry.updatedAt = command.now;
return { kind: "tombstoned" };
}
case "fail_recovery": {
const conflict = matchesObservation(entry, command.observation);
if (conflict) {
return { kind: "rejected", reason: conflict };
}
const noticeEntry = structuredClone(entry);
entry.status = "failed";
entry.abortedLastRun = true;
entry.endedAt = command.now;
entry.updatedAt = command.now;
Object.assign(entry, PENDING_FINAL_DELIVERY_CLEAR_PATCH);
Object.assign(
entry,
buildRestartRecoveryClaimCleanupPatch({
entry,
recordTerminalSource: true,
}),
);
entry.mainRestartRecovery = undefined;
return { kind: "failed", noticeEntry };
}
case "doctor_repair": {
if (!entry.mainRestartRecovery?.tombstone || entry.abortedLastRun !== true) {
return { kind: "no_change" };
}
entry.abortedLastRun = false;
entry.updatedAt = command.now;
return { kind: "doctor_repaired" };
}
case "clear": {
const patch = buildMainSessionRecoveryClearPatch(entry);
if (Object.keys(patch).length === 0) {
return { kind: "no_change" };
}
Object.assign(entry, patch);
return { kind: "applied" };
}
default:
return command satisfies never;
}
}
@@ -0,0 +1,768 @@
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import type { InternalSessionEntry as SessionEntry } from "../config/sessions.js";
import * as sessionAccessor from "../config/sessions/session-accessor.js";
import {
applySessionEntryLifecycleMutation,
listSessionEntries,
} from "../config/sessions/session-accessor.js";
import {
getAgentEventLifecycleGeneration,
rotateAgentEventLifecycleGeneration,
} from "../infra/agent-events.js";
import {
claimMainSessionRecoveryOwner,
commitMainSessionRecovery,
inspectMainSessionRecoveryRequired,
readMainSessionRecoveryOwner,
releaseMainSessionRecoveryOwner,
} from "./main-session-recovery-store.js";
const sessionKey = "agent:main:main";
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
describe("main session recovery store", () => {
let dir: string;
let lifecycleGeneration: string;
let storePath: string;
beforeEach(() => {
dir = tempDirs.make("openclaw-main-recovery-store-");
lifecycleGeneration = getAgentEventLifecycleGeneration();
storePath = path.join(dir, "sessions.json");
});
afterEach(() => {
vi.restoreAllMocks();
});
async function write(entry: SessionEntry): Promise<void> {
await sessionAccessor.replaceSessionEntry({ sessionKey, storePath }, entry);
}
function read(): SessionEntry {
return sessionAccessor.loadSessionEntry({ sessionKey, storePath })!;
}
function readStore(): Record<string, SessionEntry> {
return Object.fromEntries(
listSessionEntries({ storePath }).map(({ sessionKey: key, entry }) => [key, entry]),
);
}
async function seedExact(entries: Record<string, SessionEntry>): Promise<void> {
await applySessionEntryLifecycleMutation({
storePath,
upserts: Object.entries(entries).map(([key, entry]) => ({ sessionKey: key, entry })),
skipMaintenance: true,
});
}
function interruptedEntry(overrides: Partial<SessionEntry> = {}): SessionEntry {
return {
sessionId: "session-1",
updatedAt: 100,
status: "running",
abortedLastRun: true,
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 1,
chargedAttempts: 0,
},
...overrides,
};
}
async function reserve(targetSessionKey = sessionKey) {
const result = await commitMainSessionRecovery({
command: {
kind: "prepare_attempt",
attempt: 1,
lifecycleGeneration,
now: 200,
observation: { sessionId: "session-1", cycleId: "cycle-1", revision: 1 },
runId: "recovery-1",
},
target: { sessionKey: targetSessionKey, storePath },
});
if (result.transition.kind !== "reserved") {
throw new Error("expected reservation");
}
return result.transition.reservation;
}
it("persists a cycle before returning a legacy interrupted observation", async () => {
await write({
sessionId: "session-1",
updatedAt: 100,
status: "running",
abortedLastRun: true,
});
const result = await commitMainSessionRecovery({
command: {
kind: "observe",
cycleId: "cycle-1",
lifecycleGeneration,
sessionKey,
},
requireWriteSuccess: true,
target: { sessionKey, storePath },
});
expect(result.transition).toMatchObject({
kind: "observed",
view: { status: "recoverable" },
});
expect(read().mainRestartRecovery).toMatchObject({
cycleId: "cycle-1",
revision: 1,
});
});
it("preserves a concurrent foreground claim while cancelling its reservation", async () => {
await write(interruptedEntry());
const reservation = await reserve();
await commitMainSessionRecovery({
command: {
kind: "claim_foreground",
cycleId: "unused",
lifecycleGeneration,
sessionId: "session-1",
sessionKey,
claimId: "foreground-1",
},
target: { sessionKey, storePath },
});
await commitMainSessionRecovery({
command: { kind: "cancel_reservation", reservation },
target: { sessionKey, storePath },
});
expect(read().mainRestartRecovery).toMatchObject({
chargedAttempts: 0,
foregroundClaims: {
lifecycleGeneration,
tokens: ["foreground-1"],
},
});
});
it("cancels a reservation after Gateway migrates its legacy session key", async () => {
const legacyKey = "main";
await seedExact({ [legacyKey]: interruptedEntry() });
const reservation = await reserve(legacyKey);
const legacyEntry = readStore()[legacyKey]!;
await applySessionEntryLifecycleMutation({
storePath,
removals: [{ sessionKey: legacyKey }],
upserts: [{ sessionKey, entry: legacyEntry }],
skipMaintenance: true,
});
const cancelled = await commitMainSessionRecovery({
command: { kind: "cancel_reservation", reservation },
target: { sessionKey: legacyKey, storePath },
});
expect(cancelled.transition).toEqual({ kind: "applied" });
expect(read().mainRestartRecovery).toMatchObject({ chargedAttempts: 0 });
expect(read().mainRestartRecovery?.reservation).toBeUndefined();
});
it("abandons a reservation after Gateway migrates its legacy session key", async () => {
const legacyKey = "main";
await seedExact({ [legacyKey]: interruptedEntry() });
const reservation = await reserve(legacyKey);
const legacyEntry = readStore()[legacyKey]!;
await applySessionEntryLifecycleMutation({
storePath,
removals: [{ sessionKey: legacyKey }],
upserts: [{ sessionKey, entry: legacyEntry }],
skipMaintenance: true,
});
const abandoned = await commitMainSessionRecovery({
command: { kind: "abandon_reservation", reservation },
target: { sessionKey: legacyKey, storePath },
});
expect(abandoned.transition).toEqual({ kind: "applied" });
expect(read().mainRestartRecovery).toMatchObject({ chargedAttempts: 1 });
expect(read().mainRestartRecovery?.reservation).toBeUndefined();
});
it("admits a legacy-key reservation through its canonical Gateway key", async () => {
const legacyKey = "main";
await seedExact({ [legacyKey]: interruptedEntry() });
await reserve(legacyKey);
const validated = await commitMainSessionRecovery({
command: {
kind: "validate_recovery",
lifecycleGeneration,
runId: "recovery-1",
sessionId: "session-1",
},
target: { sessionKey, storePath },
});
expect(validated.transition).toEqual({ kind: "recovery_validated" });
const admitted = await commitMainSessionRecovery({
command: {
kind: "admit_recovery",
lifecycleGeneration,
now: 300,
runId: "recovery-1",
sessionId: "session-1",
},
target: { sessionKey, storePath },
});
expect(admitted.transition).toEqual({ kind: "admitted_recovery" });
expect(readStore()[legacyKey]).toMatchObject({
abortedLastRun: false,
mainRestartRecovery: { chargedAttempts: 1 },
});
expect(readStore()[legacyKey]?.mainRestartRecovery?.reservation).toBeUndefined();
const restored = await commitMainSessionRecovery({
command: {
kind: "mark_admitted_recovery_interrupted",
lifecycleGeneration,
now: 400,
runId: "recovery-1",
sessionId: "session-1",
},
target: { sessionKey: admitted.sessionKey!, storePath },
});
expect(restored.transition).toEqual({ kind: "applied" });
expect(readStore()[legacyKey]).toMatchObject({ abortedLastRun: true });
});
it("rejects an observation after the session is replaced", async () => {
await write({
sessionId: "session-2",
updatedAt: 300,
status: "running",
abortedLastRun: true,
mainRestartRecovery: {
cycleId: "cycle-2",
revision: 1,
chargedAttempts: 0,
},
});
const result = await commitMainSessionRecovery({
command: {
kind: "prepare_attempt",
attempt: 1,
lifecycleGeneration,
now: 400,
observation: { sessionId: "session-1", cycleId: "cycle-1", revision: 1 },
runId: "stale-recovery",
},
target: { sessionKey, storePath },
});
expect(result.transition).toEqual({ kind: "rejected", reason: "session_replaced" });
expect(read().mainRestartRecovery?.reservation).toBeUndefined();
});
it("does not cancel a reservation after its session is replaced", async () => {
await write(interruptedEntry());
const reservation = await reserve();
await write({
sessionId: "session-2",
updatedAt: 300,
status: "running",
abortedLastRun: true,
mainRestartRecovery: {
cycleId: "cycle-2",
revision: 4,
chargedAttempts: 2,
},
});
const cancelled = await commitMainSessionRecovery({
command: { kind: "cancel_reservation", reservation },
target: { sessionKey, storePath },
});
expect(cancelled.transition).toEqual({ kind: "rejected", reason: "stale_reservation" });
expect(read()).toMatchObject({
sessionId: "session-2",
mainRestartRecovery: {
cycleId: "cycle-2",
revision: 4,
chargedAttempts: 2,
},
});
});
it("does not let an old reservation survive healthy clear and immediate re-wedge", async () => {
await write(interruptedEntry());
const reservation = await reserve();
await commitMainSessionRecovery({
command: { kind: "clear" },
target: { sessionKey, storePath },
});
await commitMainSessionRecovery({
command: { kind: "mark_interrupted", cycleId: "cycle-2", now: 300 },
target: { sessionKey, storePath },
});
const cancelled = await commitMainSessionRecovery({
command: { kind: "cancel_reservation", reservation },
target: { sessionKey, storePath },
});
expect(cancelled.transition).toEqual({ kind: "rejected", reason: "stale_reservation" });
expect(read().mainRestartRecovery).toMatchObject({
cycleId: "cycle-2",
chargedAttempts: 0,
});
});
it("claims an interrupted row through its pre-migration alias", async () => {
const legacyKey = "main";
await seedExact({ [legacyKey]: interruptedEntry() });
const claim = await claimMainSessionRecoveryOwner({
lifecycleGeneration,
sessionId: "session-1",
target: { sessionKey, storePath },
});
expect(claim.kind).toBe("claimed");
if (claim.kind !== "claimed") {
return;
}
expect(claim.lease.sessionKey).toBe(legacyKey);
expect(readStore()[legacyKey]).toMatchObject({
mainRestartRecovery: {
foregroundClaims: {
lifecycleGeneration,
tokens: [claim.lease.claimId],
},
},
});
});
it("claims the exact foreground row without scanning aliases", async () => {
await write(interruptedEntry());
const accessorSpy = vi.spyOn(sessionAccessor, "applySessionEntryReplacements");
const claim = await claimMainSessionRecoveryOwner({
lifecycleGeneration,
sessionId: "session-1",
target: { sessionKey, storePath },
});
expect(claim.kind).toBe("claimed");
expect(accessorSpy).toHaveBeenCalledOnce();
expect(accessorSpy.mock.calls[0]?.[0]).toMatchObject({ sessionKeys: [sessionKey] });
});
it("binds a foreground claim to its lifecycle run", async () => {
await write(interruptedEntry());
const claim = await claimMainSessionRecoveryOwner({
lifecycleGeneration,
runId: "foreground-run",
sessionId: "session-1",
target: { sessionKey, storePath },
});
if (claim.kind !== "claimed") {
throw new Error("expected foreground owner claim");
}
expect(read()).toMatchObject({
restartRecoveryRuns: [{ lifecycleGeneration, runId: "foreground-run" }],
mainRestartRecovery: {
foregroundClaims: {
lifecycleGeneration,
runIdsByClaimId: { [claim.lease.claimId]: "foreground-run" },
},
},
});
});
it("releases an owner after the durable row session id rotates", async () => {
await write(interruptedEntry());
const claim = await claimMainSessionRecoveryOwner({
lifecycleGeneration,
sessionId: "session-1",
target: { sessionKey, storePath },
});
if (claim.kind !== "claimed") {
throw new Error("expected foreground owner claim");
}
const current = read();
await write({ ...current, sessionId: "session-2" });
await expect(releaseMainSessionRecoveryOwner(claim.lease)).resolves.toBeUndefined();
expect(read().mainRestartRecovery?.foregroundClaims).toBeUndefined();
});
it("keeps retrying an exact owner release after immediate store retries fail", async () => {
vi.useFakeTimers();
try {
await write(interruptedEntry());
const claim = await claimMainSessionRecoveryOwner({
lifecycleGeneration,
sessionId: "session-1",
target: { sessionKey, storePath },
});
if (claim.kind !== "claimed") {
throw new Error("expected foreground owner claim");
}
const applySessionEntryReplacements = sessionAccessor.applySessionEntryReplacements;
let failures = 0;
vi.spyOn(sessionAccessor, "applySessionEntryReplacements").mockImplementation(
async (params) => {
if (failures < 3) {
failures += 1;
throw new Error("transient session-store failure");
}
return await applySessionEntryReplacements(params);
},
);
const immediateRelease = releaseMainSessionRecoveryOwner(claim.lease);
const immediateReleaseRejected = expect(immediateRelease).rejects.toThrow(
"transient session-store failure",
);
await vi.advanceTimersByTimeAsync(100);
await immediateReleaseRejected;
expect(read().mainRestartRecovery?.foregroundClaims?.tokens).toEqual([claim.lease.claimId]);
await vi.advanceTimersByTimeAsync(1_000);
await vi.waitFor(() => {
expect(read().mainRestartRecovery?.foregroundClaims).toBeUndefined();
});
} finally {
vi.useRealTimers();
}
});
it("leaves interrupted non-main rows to their specialized recovery owner", async () => {
const subagentKey = "agent:main:subagent:child";
await seedExact({ [subagentKey]: interruptedEntry({ spawnDepth: 1 }) });
const claim = await claimMainSessionRecoveryOwner({
lifecycleGeneration,
sessionId: "session-1",
target: { sessionKey: subagentKey, storePath },
});
expect(claim).toEqual({ kind: "not_required" });
expect(readStore()[subagentKey]?.mainRestartRecovery?.foregroundClaims).toBeUndefined();
});
it("does not let a replacement bypass a tombstoned predecessor", async () => {
await write(
interruptedEntry({
status: "failed",
abortedLastRun: false,
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 4,
chargedAttempts: 3,
tombstone: { reason: "automatic recovery exhausted" },
},
}),
);
await expect(
claimMainSessionRecoveryOwner({
lifecycleGeneration,
replacementSessionId: "session-2",
sessionId: "session-1",
target: { sessionKey, storePath },
}),
).resolves.toEqual({ kind: "invalidated", reason: "state_changed" });
});
it("does not let foreground work bypass an exhausted predecessor", async () => {
await write(
interruptedEntry({
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 4,
chargedAttempts: 3,
},
}),
);
await expect(
claimMainSessionRecoveryOwner({
lifecycleGeneration,
sessionId: "session-1",
target: { sessionKey, storePath },
}),
).resolves.toEqual({ kind: "invalidated", reason: "recovery_exhausted" });
expect(read().mainRestartRecovery?.foregroundClaims).toBeUndefined();
});
it("claims an interrupted legacy predecessor after its canonical key is reused", async () => {
const legacyKey = "main";
await seedExact({
[sessionKey]: { sessionId: "session-2", updatedAt: 200 },
[legacyKey]: interruptedEntry(),
});
const claim = await claimMainSessionRecoveryOwner({
lifecycleGeneration,
replacementSessionId: "session-2",
sessionId: "session-1",
target: { sessionKey, storePath },
});
expect(claim.kind).toBe("claimed");
if (claim.kind !== "claimed") {
throw new Error("expected foreground owner claim");
}
expect(claim.lease.sessionKey).toBe(legacyKey);
expect(readStore()[legacyKey]?.mainRestartRecovery?.foregroundClaims?.tokens).toEqual([
claim.lease.claimId,
]);
expect(readStore()[sessionKey]?.mainRestartRecovery).toBeUndefined();
});
it("validates a transferred owner against the latest durable row", async () => {
await write(interruptedEntry());
const claim = await claimMainSessionRecoveryOwner({
lifecycleGeneration,
sessionId: "session-1",
target: { sessionKey, storePath },
});
if (claim.kind !== "claimed") {
throw new Error("expected foreground owner claim");
}
await expect(readMainSessionRecoveryOwner(claim.lease)).resolves.toBeDefined();
await releaseMainSessionRecoveryOwner(claim.lease);
await expect(readMainSessionRecoveryOwner(claim.lease)).resolves.toBeUndefined();
});
it("returns a retry target only when the final foreground owner releases", async () => {
await write(interruptedEntry());
const first = await claimMainSessionRecoveryOwner({
lifecycleGeneration,
sessionId: "session-1",
target: { sessionKey, storePath },
});
const second = await claimMainSessionRecoveryOwner({
lifecycleGeneration,
sessionId: "session-1",
target: { sessionKey, storePath },
});
if (first.kind !== "claimed" || second.kind !== "claimed") {
throw new Error("expected foreground owner claims");
}
await expect(releaseMainSessionRecoveryOwner(first.lease)).resolves.toBeUndefined();
await expect(releaseMainSessionRecoveryOwner(second.lease)).resolves.toEqual({
sessionId: "session-1",
sessionKey,
storePath,
});
await expect(releaseMainSessionRecoveryOwner(second.lease)).resolves.toEqual({
sessionId: "session-1",
sessionKey,
storePath,
});
});
it("does not let an old lease release a same-token claim from a new cycle", async () => {
await write(interruptedEntry());
const oldClaim = await claimMainSessionRecoveryOwner({
lifecycleGeneration,
sessionId: "session-1",
target: { sessionKey, storePath },
});
if (oldClaim.kind !== "claimed") {
throw new Error("expected foreground owner claim");
}
await commitMainSessionRecovery({
command: { kind: "clear" },
target: { sessionKey, storePath },
});
await commitMainSessionRecovery({
command: { kind: "mark_interrupted", cycleId: "cycle-2", now: 300 },
target: { sessionKey, storePath },
});
await commitMainSessionRecovery({
command: {
kind: "claim_foreground",
cycleId: "unused",
lifecycleGeneration,
sessionId: "session-1",
sessionKey,
claimId: oldClaim.lease.claimId,
},
target: { sessionKey, storePath },
});
await releaseMainSessionRecoveryOwner(oldClaim.lease);
expect(read().mainRestartRecovery).toMatchObject({
cycleId: "cycle-2",
foregroundClaims: {
lifecycleGeneration,
tokens: [oldClaim.lease.claimId],
},
});
});
it("retries a transient owner release write failure", async () => {
await write(interruptedEntry());
const claim = await claimMainSessionRecoveryOwner({
lifecycleGeneration,
sessionId: "session-1",
target: { sessionKey, storePath },
});
if (claim.kind !== "claimed") {
throw new Error("expected foreground owner claim");
}
const applySessionEntryReplacements = sessionAccessor.applySessionEntryReplacements;
const accessorSpy = vi
.spyOn(sessionAccessor, "applySessionEntryReplacements")
.mockRejectedValueOnce(new Error("transient writer failure"))
.mockImplementation(async (params) => await applySessionEntryReplacements(params));
await releaseMainSessionRecoveryOwner(claim.lease);
expect(accessorSpy).toHaveBeenCalledTimes(2);
expect(read().mainRestartRecovery?.foregroundClaims).toBeUndefined();
});
it("rejects an old claimant queued ahead of the current lifecycle generation", async () => {
await write(interruptedEntry());
let enterWriter = () => {};
const writerEntered = new Promise<void>((resolve) => {
enterWriter = resolve;
});
let releaseWriter = () => {};
const writerGate = new Promise<void>((resolve) => {
releaseWriter = resolve;
});
const blocker = sessionAccessor.applySessionEntryReplacements({
storePath,
update: async () => {
enterWriter();
await writerGate;
return { result: undefined };
},
});
await writerEntered;
const staleClaim = commitMainSessionRecovery({
command: {
kind: "claim_foreground",
cycleId: "unused",
lifecycleGeneration,
sessionId: "session-1",
sessionKey,
claimId: "stale-owner",
},
target: { sessionKey, storePath },
});
const staleOwnerClaim = claimMainSessionRecoveryOwner({
allowMissingSession: true,
lifecycleGeneration,
replacementSessionId: "session-2",
sessionId: "session-1",
target: { sessionKey, storePath },
});
const staleInspection = inspectMainSessionRecoveryRequired({
expectedSessionId: "session-1",
lifecycleGeneration,
target: { sessionKey, storePath },
});
await Promise.resolve();
const currentGeneration = rotateAgentEventLifecycleGeneration();
const currentClaim = commitMainSessionRecovery({
command: {
kind: "claim_foreground",
cycleId: "unused",
lifecycleGeneration: currentGeneration,
sessionId: "session-1",
sessionKey,
claimId: "current-owner",
},
target: { sessionKey, storePath },
});
releaseWriter();
await blocker;
expect((await staleClaim).transition).toEqual({
kind: "rejected",
reason: "stale_generation",
});
await expect(staleOwnerClaim).resolves.toEqual({
kind: "invalidated",
reason: "stale_generation",
});
await expect(staleInspection).resolves.toEqual({
kind: "invalidated",
reason: "stale_generation",
});
expect((await currentClaim).transition).toMatchObject({
kind: "foreground_claimed",
claim: { claimId: "current-owner" },
});
expect(read().mainRestartRecovery?.foregroundClaims).toEqual({
lifecycleGeneration: currentGeneration,
tokens: ["current-owner"],
});
});
it("rejects a delayed admitted-interruption callback after lifecycle rotation", async () => {
await write(
interruptedEntry({
abortedLastRun: false,
restartRecoveryRuns: [{ runId: "recovery-1", lifecycleGeneration }],
}),
);
rotateAgentEventLifecycleGeneration();
const result = await commitMainSessionRecovery({
command: {
kind: "mark_admitted_recovery_interrupted",
lifecycleGeneration,
now: 300,
runId: "recovery-1",
sessionId: "session-1",
},
target: { sessionKey, storePath },
});
expect(result.transition).toEqual({ kind: "rejected", reason: "stale_generation" });
expect(read()).toMatchObject({
sessionId: "session-1",
status: "running",
abortedLastRun: false,
});
});
it("rejects a transferred foreground lease after lifecycle rotation", async () => {
await write(interruptedEntry());
const claim = await claimMainSessionRecoveryOwner({
lifecycleGeneration,
sessionId: "session-1",
target: { sessionKey, storePath },
});
if (claim.kind !== "claimed") {
throw new Error("expected foreground owner claim");
}
rotateAgentEventLifecycleGeneration();
await expect(readMainSessionRecoveryOwner(claim.lease)).resolves.toBeUndefined();
});
});
+436
View File
@@ -0,0 +1,436 @@
import { randomUUID } from "node:crypto";
import type { InternalSessionEntry as SessionEntry } from "../config/sessions.js";
import { applySessionEntryReplacements } from "../config/sessions/session-accessor.js";
import { getAgentEventLifecycleGeneration } from "../infra/agent-events.js";
import { retryAsync } from "../infra/retry.js";
import {
isMainRestartRecoveryCandidate,
transitionMainSessionRecovery,
type MainSessionRecoveryCommand,
type MainSessionRecoveryOwnerClaim,
type MainSessionRecoveryReservation,
type MainSessionRecoveryTransitionResult,
} from "./main-session-recovery-state.js";
type MainSessionRecoveryStoreTarget = {
sessionKey: string;
storePath: string;
};
const OWNER_RELEASE_RETRY_DELAY_MS = 1_000;
const OWNER_RELEASE_RETRY_MAX_DELAY_MS = 30_000;
export type MainSessionRecoveryOwnerLease = MainSessionRecoveryOwnerClaim &
MainSessionRecoveryStoreTarget;
type MainSessionRecoveryStoreResult = {
entry?: SessionEntry;
sessionKey?: string;
transition: MainSessionRecoveryTransitionResult;
};
export type MainSessionRecoveryPendingTarget = MainSessionRecoveryStoreTarget & {
sessionId: string;
};
type MainSessionRecoveryOwnerClaimResult =
| {
kind: "claimed";
lease: MainSessionRecoveryOwnerLease;
entry: SessionEntry;
sessionKey: string;
}
| { kind: "invalidated"; reason: string }
| { kind: "not_required" };
type MainSessionRecoveryInspectionResult =
| { kind: "invalidated"; reason: string }
| { kind: "not_required" }
| { kind: "required" };
function transitionChanged(result: MainSessionRecoveryTransitionResult): boolean {
return (
result.kind !== "foreground_validated" &&
result.kind !== "no_change" &&
result.kind !== "observed" &&
result.kind !== "rejected"
);
}
function matchesReservation(entry: SessionEntry, reservation: MainSessionRecoveryReservation) {
const state = entry.mainRestartRecovery;
return (
entry.sessionId === reservation.sessionId &&
state?.cycleId === reservation.cycleId &&
state.reservation?.runId === reservation.runId &&
state.reservation.lifecycleGeneration === reservation.lifecycleGeneration
);
}
function matchesRecoveryAdmission(
entry: SessionEntry,
command: Extract<MainSessionRecoveryCommand, { kind: "admit_recovery" | "validate_recovery" }>,
): boolean {
const reservation = entry.mainRestartRecovery?.reservation;
return (
entry.sessionId === command.sessionId &&
reservation?.runId === command.runId &&
reservation.lifecycleGeneration === command.lifecycleGeneration
);
}
function matchesOwnerClaim(entry: SessionEntry, claim: MainSessionRecoveryOwnerClaim): boolean {
const state = entry.mainRestartRecovery;
return (
state?.cycleId === claim.cycleId &&
state.foregroundClaims?.lifecycleGeneration === claim.lifecycleGeneration &&
state.foregroundClaims.tokens.includes(claim.claimId)
);
}
function currentGenerationRequiredBy(command: MainSessionRecoveryCommand): string | undefined {
// Generation gates new decisions. Exact reservation/token cleanup must remain
// valid after a restart so the old owner cannot leak its slot or claim.
switch (command.kind) {
case "admit_recovery":
case "claim_foreground":
case "inspect":
case "mark_admitted_recovery_interrupted":
case "observe":
case "prepare_attempt":
case "validate_recovery":
return command.lifecycleGeneration;
case "validate_foreground":
case "bind_foreground_run":
return command.claim.lifecycleGeneration;
default:
return undefined;
}
}
export async function commitMainSessionRecovery(params: {
command: MainSessionRecoveryCommand;
expectedSessionId?: string;
requireWriteSuccess?: boolean;
scanAliases?: boolean;
target: MainSessionRecoveryStoreTarget;
}): Promise<MainSessionRecoveryStoreResult> {
const cancellation =
params.command.kind === "cancel_reservation" ? params.command.reservation : undefined;
const abandonment =
params.command.kind === "abandon_reservation" ? params.command.reservation : undefined;
const recoveryAdmission =
params.command.kind === "admit_recovery" || params.command.kind === "validate_recovery"
? params.command
: undefined;
const ownerClaim = params.command.kind === "claim_foreground" ? params.command : undefined;
const ownerValidation =
params.command.kind === "validate_foreground" ? params.command.claim : undefined;
const ownerRelease =
params.command.kind === "release_foreground" ? params.command.claim : undefined;
const reservationCleanup = cancellation ?? abandonment;
const scansAliases = Boolean(
params.scanAliases ||
reservationCleanup ||
recoveryAdmission ||
ownerValidation ||
ownerRelease,
);
return await applySessionEntryReplacements<MainSessionRecoveryStoreResult>({
requireWriteSuccess: params.requireWriteSuccess,
...(scansAliases ? {} : { sessionKeys: [params.target.sessionKey] }),
storePath: params.target.storePath,
update: (entries) => {
const expectedGeneration = currentGenerationRequiredBy(params.command);
if (expectedGeneration && expectedGeneration !== getAgentEventLifecycleGeneration()) {
return {
result: {
transition: { kind: "rejected", reason: "stale_generation" },
},
};
}
const selected = entries.find(({ sessionKey }) => sessionKey === params.target.sessionKey);
let candidate =
(params.expectedSessionId && selected?.entry.sessionId !== params.expectedSessionId) ||
(ownerClaim && selected?.entry.sessionId !== ownerClaim.sessionId)
? undefined
: selected;
if (reservationCleanup) {
candidate =
entries.find(({ entry }) => matchesReservation(entry, reservationCleanup)) ?? selected;
} else if (recoveryAdmission) {
// Canonical session-key migration may happen between reservation and
// Gateway admission; the reservation identity remains authoritative.
candidate =
entries.find(({ entry }) => matchesRecoveryAdmission(entry, recoveryAdmission)) ??
selected;
} else if (ownerValidation || ownerRelease) {
const exactClaim = ownerValidation ?? ownerRelease!;
candidate = entries.find(({ entry }) => matchesOwnerClaim(entry, exactClaim)) ?? selected;
} else if (ownerClaim && (!selected || selected.entry.sessionId !== ownerClaim.sessionId)) {
candidate = entries.find(({ entry }) => entry.sessionId === ownerClaim.sessionId);
} else if (params.scanAliases && params.expectedSessionId) {
candidate = entries.find(({ entry }) => entry.sessionId === params.expectedSessionId);
}
if (!candidate) {
return {
result: {
entry: selected?.entry,
sessionKey: selected?.sessionKey,
transition: { kind: "rejected", reason: "session_replaced" },
},
};
}
const entry = candidate.entry as SessionEntry;
const previousRecoveryState = entry.mainRestartRecovery;
let command: MainSessionRecoveryCommand;
if (ownerClaim) {
command =
ownerClaim.sessionKey === candidate.sessionKey
? ownerClaim
: { ...ownerClaim, sessionKey: candidate.sessionKey };
} else if (
(params.command.kind === "observe" || params.command.kind === "inspect") &&
params.command.sessionKey !== candidate.sessionKey
) {
command = { ...params.command, sessionKey: candidate.sessionKey };
} else {
command = params.command;
}
const transition = transitionMainSessionRecovery(entry, command);
const changed =
transitionChanged(transition) || previousRecoveryState !== entry.mainRestartRecovery;
return {
result: { entry, sessionKey: candidate.sessionKey, transition },
...(changed ? { replacements: [{ sessionKey: candidate.sessionKey, entry }] } : {}),
};
},
});
}
export async function readMainSessionRecoveryOwner(
lease: MainSessionRecoveryOwnerLease,
): Promise<{ entry: SessionEntry; sessionKey: string } | undefined> {
const result = await commitMainSessionRecovery({
command: { kind: "validate_foreground", claim: lease },
requireWriteSuccess: true,
target: lease,
});
return result.transition.kind === "foreground_validated" && result.entry && result.sessionKey
? { entry: result.entry, sessionKey: result.sessionKey }
: undefined;
}
export async function claimMainSessionRecoveryOwner(params: {
allowMissingSession?: boolean;
lifecycleGeneration: string;
replacementSessionId?: string;
sessionId: string;
runId?: string;
target: MainSessionRecoveryStoreTarget;
}): Promise<MainSessionRecoveryOwnerClaimResult> {
const command = {
kind: "claim_foreground" as const,
cycleId: randomUUID(),
lifecycleGeneration: params.lifecycleGeneration,
sessionId: params.sessionId,
sessionKey: params.target.sessionKey,
claimId: randomUUID(),
...(params.runId ? { runId: params.runId } : {}),
};
let claim = await commitMainSessionRecovery({
command,
requireWriteSuccess: true,
target: params.target,
});
if (claim.transition.kind === "rejected" && claim.transition.reason === "session_replaced") {
claim = await commitMainSessionRecovery({
command,
requireWriteSuccess: true,
scanAliases: true,
target: params.target,
});
}
if (claim.transition.kind === "foreground_claimed") {
if (!claim.entry || !claim.sessionKey) {
return { kind: "invalidated", reason: "state_changed" };
}
return {
kind: "claimed",
lease: { ...claim.transition.claim, storePath: params.target.storePath },
entry: claim.entry,
sessionKey: claim.sessionKey,
};
}
if (claim.transition.kind === "rejected" && claim.transition.reason === "stale_generation") {
return { kind: "invalidated", reason: claim.transition.reason };
}
if (!claim.entry && (params.allowMissingSession || params.replacementSessionId)) {
// A fresh explicit session has no predecessor. An automatic rollover can
// also lose its predecessor before admission. Either way, no row remains to fence.
return { kind: "not_required" };
}
if (
params.replacementSessionId &&
claim.entry?.sessionId === params.replacementSessionId &&
claim.entry.abortedLastRun !== true &&
claim.entry.restartRecoveryRuns === undefined &&
claim.entry.mainRestartRecovery === undefined
) {
return { kind: "not_required" };
}
if (
claim.entry?.sessionId === params.sessionId &&
claim.sessionKey &&
!isMainRestartRecoveryCandidate(claim.entry, claim.sessionKey)
) {
return { kind: "not_required" };
}
if (
claim.entry?.sessionId === params.sessionId &&
claim.entry.abortedLastRun !== true &&
claim.entry.restartRecoveryRuns === undefined &&
claim.entry.mainRestartRecovery === undefined
) {
// A healthy completion may clear recovery between the caller's read and this
// transaction. Only that fully clean same-session state can proceed unclaimed.
return { kind: "not_required" };
}
const reason = claim.transition.kind === "rejected" ? claim.transition.reason : "state_changed";
return { kind: "invalidated", reason };
}
export async function bindMainSessionRecoveryOwnerRun(
lease: MainSessionRecoveryOwnerLease,
runId: string,
): Promise<{
lease: MainSessionRecoveryOwnerLease;
entry: SessionEntry;
sessionKey: string;
}> {
const result = await commitMainSessionRecovery({
command: { kind: "bind_foreground_run", claim: lease, runId },
requireWriteSuccess: true,
target: lease,
});
if (result.transition.kind !== "applied" || !result.entry || !result.sessionKey) {
throw new Error("main-session recovery owner changed before run binding");
}
return { lease: { ...lease, runId }, entry: result.entry, sessionKey: result.sessionKey };
}
export async function inspectMainSessionRecoveryRequired(params: {
allowMissingSession?: boolean;
expectedSessionId: string;
lifecycleGeneration: string;
target: MainSessionRecoveryStoreTarget;
}): Promise<MainSessionRecoveryInspectionResult> {
const command = {
kind: "inspect" as const,
lifecycleGeneration: params.lifecycleGeneration,
sessionKey: params.target.sessionKey,
};
let result = await commitMainSessionRecovery({
command,
expectedSessionId: params.expectedSessionId,
requireWriteSuccess: true,
target: params.target,
});
if (result.transition.kind === "rejected" && result.transition.reason === "session_replaced") {
result = await commitMainSessionRecovery({
command,
expectedSessionId: params.expectedSessionId,
requireWriteSuccess: true,
scanAliases: true,
target: params.target,
});
}
if (result.transition.kind === "observed") {
return result.transition.view.status === "inactive"
? { kind: "not_required" }
: { kind: "required" };
}
if (result.transition.kind === "rejected" && result.transition.reason === "session_replaced") {
return !result.entry && params.allowMissingSession
? { kind: "not_required" }
: { kind: "invalidated", reason: result.transition.reason };
}
return {
kind: "invalidated",
reason: result.transition.kind === "rejected" ? result.transition.reason : "state_changed",
};
}
async function releaseMainSessionRecoveryOwnerWithRetries(
lease: MainSessionRecoveryOwnerLease,
): Promise<MainSessionRecoveryPendingTarget | undefined> {
// A leaked current-generation token blocks automatic recovery until restart.
// Token-scoped release is idempotent, so transient writer failures are safe to retry.
const released = await retryAsync(
async () =>
await commitMainSessionRecovery({
command: { kind: "release_foreground", claim: lease },
requireWriteSuccess: true,
target: lease,
}),
3,
25,
);
const { entry, sessionKey } = released;
const state = entry?.mainRestartRecovery;
if (
(released.transition.kind !== "applied" && released.transition.kind !== "no_change") ||
!entry ||
!sessionKey ||
entry.sessionId !== lease.sessionId ||
entry.status !== "running" ||
entry.abortedLastRun !== true ||
!isMainRestartRecoveryCandidate(entry, sessionKey) ||
state?.foregroundClaims ||
state?.reservation ||
state?.tombstone
) {
return undefined;
}
return { sessionId: entry.sessionId, sessionKey, storePath: lease.storePath };
}
function scheduleMainSessionRecoveryOwnerRelease(
lease: MainSessionRecoveryOwnerLease,
delayMs = OWNER_RELEASE_RETRY_DELAY_MS,
): void {
// A token is process-owned but durably blocks recovery. Keep exact-token
// cleanup alive through transient writer outages until release or restart.
setTimeout(() => {
void releaseMainSessionRecoveryOwnerWithRetries(lease).then(
async (pending) => {
if (!pending) {
return;
}
const { scheduleMainSessionRecoveryPendingTarget } =
await import("./main-session-recovery-owner-release.js");
scheduleMainSessionRecoveryPendingTarget(pending);
},
() => {
scheduleMainSessionRecoveryOwnerRelease(
lease,
Math.min(delayMs * 2, OWNER_RELEASE_RETRY_MAX_DELAY_MS),
);
},
);
}, delayMs).unref?.();
}
export async function releaseMainSessionRecoveryOwner(
lease: MainSessionRecoveryOwnerLease | undefined,
): Promise<MainSessionRecoveryPendingTarget | undefined> {
if (!lease) {
return undefined;
}
try {
return await releaseMainSessionRecoveryOwnerWithRetries(lease);
} catch (error) {
scheduleMainSessionRecoveryOwnerRelease(lease);
throw error;
}
}
+143
View File
@@ -0,0 +1,143 @@
import type {
InternalSessionEntry as SessionEntry,
RestartRecoveryRun,
} from "../config/sessions.js";
export type MainSessionRecoveryObservation = {
sessionId: string;
cycleId: string;
revision: number;
};
export type MainSessionRecoveryReservation = {
sessionId: string;
cycleId: string;
lifecycleGeneration: string;
runId: string;
attempt: number;
};
export type MainSessionRecoveryOwnerClaim = {
cycleId: string;
lifecycleGeneration: string;
claimId: string;
sessionId: string;
sessionKey: string;
runId?: string;
};
export type MainSessionRecoveryView =
| { status: "inactive" }
| { status: "blocked" }
| {
status: "recoverable";
observation: MainSessionRecoveryObservation;
nextAttempt: number;
}
| {
status: "exhausted";
observation: MainSessionRecoveryObservation;
reason: string;
}
| { status: "tombstoned" };
export type MainSessionRecoveryConflict =
| "already_tombstoned"
| "foreground_active"
| "not_interrupted"
| "recovery_exhausted"
| "reservation_active"
| "session_replaced"
| "stale_cycle"
| "stale_generation"
| "stale_reservation"
| "stale_revision";
export type MainSessionRecoveryCommand =
| {
kind: "mark_interrupted";
cycleId: string;
now: number;
runs?: RestartRecoveryRun[];
resetRuntime?: boolean;
}
| {
kind: "observe";
cycleId: string;
lifecycleGeneration: string;
sessionKey: string;
}
| {
kind: "inspect";
lifecycleGeneration: string;
sessionKey: string;
}
| {
kind: "prepare_attempt";
attempt: number;
lifecycleGeneration: string;
now: number;
observation: MainSessionRecoveryObservation;
runId: string;
}
| { kind: "cancel_reservation"; reservation: MainSessionRecoveryReservation }
| { kind: "abandon_reservation"; reservation: MainSessionRecoveryReservation }
| {
kind: "validate_recovery";
lifecycleGeneration: string;
runId: string;
sessionId: string;
}
| {
kind: "admit_recovery";
lifecycleGeneration: string;
now: number;
runId: string;
sessionId: string;
}
| {
kind: "mark_admitted_recovery_interrupted";
lifecycleGeneration: string;
now: number;
runId: string;
sessionId: string;
}
| {
kind: "claim_foreground";
cycleId: string;
lifecycleGeneration: string;
sessionId: string;
sessionKey: string;
claimId: string;
runId?: string;
}
| { kind: "bind_foreground_run"; claim: MainSessionRecoveryOwnerClaim; runId: string }
| { kind: "validate_foreground"; claim: MainSessionRecoveryOwnerClaim }
| { kind: "release_foreground"; claim: MainSessionRecoveryOwnerClaim }
| {
kind: "tombstone";
now: number;
observation: MainSessionRecoveryObservation;
reason: string;
}
| {
kind: "fail_recovery";
now: number;
observation: MainSessionRecoveryObservation;
}
| { kind: "doctor_repair"; now: number }
| { kind: "clear" };
export type MainSessionRecoveryTransitionResult =
| { kind: "admitted_recovery" }
| { kind: "applied" }
| { kind: "doctor_repaired" }
| { kind: "failed"; noticeEntry: SessionEntry }
| { kind: "foreground_claimed"; claim: MainSessionRecoveryOwnerClaim }
| { kind: "foreground_validated" }
| { kind: "no_change" }
| { kind: "observed"; view: MainSessionRecoveryView }
| { kind: "rejected"; reason: MainSessionRecoveryConflict }
| { kind: "recovery_validated" }
| { kind: "reserved"; reservation: MainSessionRecoveryReservation }
| { kind: "tombstoned" };
+280 -45
View File
@@ -1,19 +1,24 @@
import { randomUUID } from "node:crypto";
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import { GatewayClientRequestError } from "../../packages/gateway-client/src/index.js";
import { sanitizePendingFinalDeliveryText } from "../auto-reply/reply/pending-final-delivery.js";
import type { SessionEntry } from "../config/sessions.js";
import {
buildRestartRecoveryClaimCleanupPatch,
hasRestartRecoveryTerminalRun,
resolveRestartRecoveryChannelAuthority,
} from "../config/sessions/restart-recovery-state.js";
import { applySessionEntryReplacements } from "../config/sessions/session-accessor.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { isTrustedMessageActionTurnIngress } from "../gateway/message-action-turn-capability.js";
import type { GatewayRecoveryRuntime } from "../gateway/server-instance-runtime.types.js";
import { getAgentEventLifecycleGeneration } from "../infra/agent-events.js";
import { retryAsync } from "../infra/retry.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import { findRestartRecoveryUnsafeReplyHook } from "../plugins/restart-recovery-hook-safety.js";
import { CommandLane } from "../process/lanes.js";
import { resolveAgentIdFromSessionKey } from "../routing/session-key.js";
import { MAIN_SESSION_RESTART_RECOVERY_SOURCE_TOOL } from "../sessions/input-provenance.js";
import { resolveSendPolicy } from "../sessions/send-policy.js";
import {
deliveryContextFromSession,
@@ -22,9 +27,24 @@ import {
} from "../utils/delivery-context.shared.js";
import { isDeliverableMessageChannel } from "../utils/message-channel.js";
import { resolveAgentWorkspaceDir } from "./agent-scope.js";
import { buildMainSessionRecoveryClearPatch } from "./main-session-recovery-clear.js";
import { scheduleMainSessionRecoveryPendingTarget } from "./main-session-recovery-owner-release.js";
import {
restoreAdmittedRecoveryWithRetries,
scheduleAdmittedRecoveryRestore,
type RestoreAdmittedRecovery,
} from "./main-session-recovery-restore.js";
import {
isMainRestartRecoveryCandidate,
type MainSessionRecoveryObservation,
type MainSessionRecoveryReservation,
} from "./main-session-recovery-state.js";
import { commitMainSessionRecovery } from "./main-session-recovery-store.js";
import { ensureRuntimePluginsLoaded } from "./runtime-plugins.js";
const log = createSubsystemLogger("main-session-restart-recovery");
const RESERVATION_ROLLBACK_RETRY_DELAY_MS = 1_000;
const RESERVATION_ROLLBACK_RETRY_MAX_DELAY_MS = 30_000;
const RESTART_RECOVERY_RESUME_MESSAGE =
"[System] Your previous turn was interrupted by a gateway restart while " +
"OpenClaw was waiting on tool/model work. Continue from the existing " +
@@ -172,7 +192,6 @@ async function settleRestartRecoveryDispatch(params: {
expectedRecoveryRunId: string;
expectedRecoverySourceRunId?: string;
expectedSessionId: string;
pendingFinalDeliveryText: string;
sessionKeys: readonly string[];
storePath: string;
terminalStatus?: RestartRecoveryTerminalStatus;
@@ -215,30 +234,15 @@ async function settleRestartRecoveryDispatch(params: {
buildRestartRecoveryClaimCleanupPatch({
entry,
recordTerminalSource: true,
terminalRunId: params.expectedRecoveryRunId,
terminalSourceRunId: params.expectedRecoverySourceRunId,
}),
buildMainSessionRecoveryClearPatch(entry),
);
} else {
entry.abortedLastRun = false;
}
entry.updatedAt = now;
if (entry.pendingFinalDelivery || entry.pendingFinalDeliveryText) {
if (params.pendingFinalDeliveryText) {
entry.pendingFinalDeliveryLastAttemptAt = now;
entry.pendingFinalDeliveryAttemptCount =
(entry.pendingFinalDeliveryAttemptCount ?? 0) + 1;
entry.pendingFinalDeliveryLastError = null;
entry.pendingFinalDeliveryText = params.pendingFinalDeliveryText;
} else {
entry.pendingFinalDelivery = undefined;
entry.pendingFinalDeliveryText = undefined;
entry.pendingFinalDeliveryCreatedAt = undefined;
entry.pendingFinalDeliveryLastAttemptAt = undefined;
entry.pendingFinalDeliveryAttemptCount = undefined;
entry.pendingFinalDeliveryLastError = undefined;
entry.pendingFinalDeliveryContext = undefined;
}
}
return {
result: undefined,
replacements: [{ sessionKey: current.sessionKey, entry }],
@@ -247,17 +251,104 @@ async function settleRestartRecoveryDispatch(params: {
});
}
function isExactRestartRecoveryDispatchAdmission(params: {
admission: Awaited<ReturnType<typeof commitMainSessionRecovery>>;
lifecycleGeneration: string;
recoveryRunId: string;
sessionId: string;
terminalStatus?: RestartRecoveryTerminalStatus;
}): boolean {
const entry = params.admission.entry;
return (
entry?.sessionId === params.sessionId &&
((entry.abortedLastRun === false &&
normalizeOptionalString(entry.restartRecoveryDeliveryRunId) === params.recoveryRunId &&
entry.restartRecoveryRuns?.some(
(run) =>
run.runId === params.recoveryRunId &&
run.lifecycleGeneration === params.lifecycleGeneration,
) === true) ||
(hasRestartRecoveryTerminalRun(entry, params.recoveryRunId) &&
((params.terminalStatus === "ok" && entry.status === "done") ||
(params.terminalStatus === "error" && entry.status === "failed") ||
(params.terminalStatus === "timeout" && entry.status === "timeout"))))
);
}
type MainSessionResumeResult = "resumed" | "skipped" | "failed";
async function rollbackRestartRecoveryReservation(params: {
kind: "abandon_reservation" | "cancel_reservation";
reservation: MainSessionRecoveryReservation;
sessionKey: string;
storePath: string;
}) {
return await retryAsync(
async () =>
await commitMainSessionRecovery({
command: { kind: params.kind, reservation: params.reservation },
requireWriteSuccess: true,
target: { sessionKey: params.sessionKey, storePath: params.storePath },
}),
3,
25,
);
}
function scheduleRestartRecoveryReservationRollback(
params: Parameters<typeof rollbackRestartRecoveryReservation>[0],
delayMs = RESERVATION_ROLLBACK_RETRY_DELAY_MS,
): void {
// Keep the exact reservation token alive after transient store outages.
// A Gateway restart safely retires the timer and its stale-generation slot.
setTimeout(() => {
void rollbackRestartRecoveryReservation(params).then(
({ entry, sessionKey }) => {
const state = entry?.mainRestartRecovery;
if (
entry?.sessionId === params.reservation.sessionId &&
sessionKey &&
entry.status === "running" &&
entry.abortedLastRun === true &&
isMainRestartRecoveryCandidate(entry, sessionKey) &&
state &&
!state.foregroundClaims &&
!state.reservation &&
!state.tombstone
) {
scheduleMainSessionRecoveryPendingTarget({
sessionId: entry.sessionId,
sessionKey,
storePath: params.storePath,
});
}
},
(error: unknown) => {
log.warn(
`failed delayed restart recovery reservation rollback ${params.sessionKey}: ${String(error)}`,
);
scheduleRestartRecoveryReservationRollback(
params,
Math.min(delayMs * 2, RESERVATION_ROLLBACK_RETRY_MAX_DELAY_MS),
);
},
);
}, delayMs).unref?.();
}
export async function resumeMainSession(params: {
canonicalSessionKey?: string;
cfg?: OpenClawConfig;
entry: SessionEntry;
observation: MainSessionRecoveryObservation;
recoveryAttempt: number;
storePath: string;
sessionKey: string;
pendingFinalDeliveryText?: string | null;
forceRestartSafeTools?: boolean;
sessionWorkAdmissionHandoffId?: string;
gatewayRuntime: GatewayRecoveryRuntime;
}): Promise<boolean> {
}): Promise<MainSessionResumeResult> {
const sanitizedPendingText =
typeof params.pendingFinalDeliveryText === "string"
? sanitizePendingFinalDeliveryText(params.pendingFinalDeliveryText)
@@ -274,14 +365,31 @@ export async function resumeMainSession(params: {
!hasRestartRecoveryMessageActionAuthority(params.entry)
) {
log.warn(`refusing message-tool-only recovery without channel authority: ${params.sessionKey}`);
return false;
return "failed";
}
const recoveryRunId = claimedRunId && claimedRunId !== sourceRunId ? claimedRunId : randomUUID();
const reusingRecoveryRunId = recoveryRunId === claimedRunId;
const dispatchSessionKey = params.canonicalSessionKey ?? params.sessionKey;
const recoverySessionKeys = Array.from(new Set([dispatchSessionKey, params.sessionKey]));
let dispatchOutcomeUnknown = false;
let reservation: MainSessionRecoveryReservation | undefined;
let dispatchStarted = false;
try {
const reserved = await commitMainSessionRecovery({
command: {
kind: "prepare_attempt",
attempt: params.recoveryAttempt,
lifecycleGeneration: getAgentEventLifecycleGeneration(),
now: Date.now(),
observation: params.observation,
runId: recoveryRunId,
},
requireWriteSuccess: true,
target: { sessionKey: params.sessionKey, storePath: params.storePath },
});
if (reserved.transition.kind !== "reserved") {
return "skipped";
}
reservation = reserved.transition.reservation;
// Persist one stable RPC id before dispatch. A transport rejection is
// ambiguous; retries must reuse this id so accepted work cannot duplicate.
const recoveryStatePrepared = await applySessionEntryReplacements({
@@ -312,7 +420,21 @@ export async function resumeMainSession(params: {
},
});
if (!recoveryStatePrepared) {
throw new Error("restart recovery session ownership changed before dispatch");
const rollback = await rollbackRestartRecoveryReservation({
kind: "cancel_reservation",
reservation,
sessionKey: params.sessionKey,
storePath: params.storePath,
});
reservation = undefined;
const current = rollback.entry;
return current?.sessionId === params.entry.sessionId &&
current.status === "running" &&
current.abortedLastRun === true &&
!current.mainRestartRecovery?.reservation &&
!current.mainRestartRecovery?.tombstone
? "failed"
: "skipped";
}
const agentParams: Record<string, unknown> = {
message: buildResumeMessage(sanitizedPendingText),
@@ -330,6 +452,11 @@ export async function resumeMainSession(params: {
? { sourceReplyDeliveryMode: params.entry.restartRecoverySourceReplyDeliveryMode }
: {}),
...(params.forceRestartSafeTools ? { forceRestartSafeTools: true } : {}),
inputProvenance: {
kind: "internal_system",
sourceSessionKey: dispatchSessionKey,
sourceTool: MAIN_SESSION_RESTART_RECOVERY_SOURCE_TOOL,
},
};
if (deliveryContext) {
agentParams.channel = deliveryContext.channel;
@@ -345,14 +472,14 @@ export async function resumeMainSession(params: {
if (params.forceRestartSafeTools) {
log.info(`dispatching restart-safe recovery for ${params.sessionKey}`);
}
// Once dispatch starts, any rejection is ambiguous because the stable RPC
// may still have been accepted; a successful return resolves that ambiguity.
dispatchOutcomeUnknown = true;
dispatchStarted = true;
const dispatchResult = await params.gatewayRuntime.dispatchAgent<{
runId: string;
status?: unknown;
}>(agentParams, 10_000);
dispatchOutcomeUnknown = false;
// Real Gateway admission consumes the reservation before returning accepted.
// Recovery-runtime fakes may return directly, so keep this idempotent fallback
// to make the durable acceptance boundary explicit in focused tests too.
let terminalStatus = normalizeRestartRecoveryTerminalStatus(dispatchResult.status);
if (!terminalStatus && reusingRecoveryRunId && dispatchResult.status === "accepted") {
terminalStatus = await probeRestartRecoveryTerminalStatus(
@@ -360,11 +487,33 @@ export async function resumeMainSession(params: {
params.gatewayRuntime,
);
}
const lifecycleGeneration = getAgentEventLifecycleGeneration();
const admission = await commitMainSessionRecovery({
command: {
kind: "admit_recovery",
lifecycleGeneration,
now: Date.now(),
runId: recoveryRunId,
sessionId: params.entry.sessionId,
},
target: { sessionKey: params.sessionKey, storePath: params.storePath },
});
if (
admission.transition.kind !== "admitted_recovery" &&
!isExactRestartRecoveryDispatchAdmission({
admission,
lifecycleGeneration,
recoveryRunId,
sessionId: params.entry.sessionId,
terminalStatus,
})
) {
throw new Error(`restart recovery admission changed before settlement: ${params.sessionKey}`);
}
await settleRestartRecoveryDispatch({
expectedRecoveryRunId: recoveryRunId,
expectedRecoverySourceRunId: sourceRunId,
expectedSessionId: params.entry.sessionId,
pendingFinalDeliveryText: sanitizedPendingText,
sessionKeys: recoverySessionKeys,
storePath: params.storePath,
terminalStatus,
@@ -374,28 +523,114 @@ export async function resumeMainSession(params: {
sanitizedPendingText ? " (with pending payload)" : ""
}`,
);
return true;
return "resumed";
} catch (error) {
if (reusingRecoveryRunId && dispatchOutcomeUnknown) {
const terminalStatus = await probeRestartRecoveryTerminalStatus(
recoveryRunId,
params.gatewayRuntime,
const explicitlyRejected = error instanceof GatewayClientRequestError;
try {
if (dispatchStarted && !explicitlyRejected) {
const terminalStatus = await probeRestartRecoveryTerminalStatus(
recoveryRunId,
params.gatewayRuntime,
);
if (terminalStatus) {
const lifecycleGeneration = getAgentEventLifecycleGeneration();
const admission = await commitMainSessionRecovery({
command: {
kind: "admit_recovery",
lifecycleGeneration,
now: Date.now(),
runId: recoveryRunId,
sessionId: params.entry.sessionId,
},
target: { sessionKey: params.sessionKey, storePath: params.storePath },
});
const exactRunAlreadyAdmitted = isExactRestartRecoveryDispatchAdmission({
admission,
lifecycleGeneration,
recoveryRunId,
sessionId: params.entry.sessionId,
terminalStatus,
});
if (admission.transition.kind !== "admitted_recovery" && !exactRunAlreadyAdmitted) {
log.warn(`restart recovery admission changed before settlement: ${params.sessionKey}`);
} else {
if (reservation) {
await commitMainSessionRecovery({
command: { kind: "abandon_reservation", reservation },
target: { sessionKey: params.sessionKey, storePath: params.storePath },
});
}
await settleRestartRecoveryDispatch({
expectedRecoveryRunId: recoveryRunId,
expectedRecoverySourceRunId: sourceRunId,
expectedSessionId: params.entry.sessionId,
sessionKeys: recoverySessionKeys,
storePath: params.storePath,
terminalStatus,
});
log.info(`settled completed restart recovery for ${params.sessionKey}`);
return "resumed";
}
}
}
} catch (settlementError) {
log.warn(
`failed to settle ambiguous restart recovery ${params.sessionKey}: ${String(settlementError)}`,
);
if (terminalStatus) {
await settleRestartRecoveryDispatch({
expectedRecoveryRunId: recoveryRunId,
expectedRecoverySourceRunId: sourceRunId,
expectedSessionId: params.entry.sessionId,
pendingFinalDeliveryText: sanitizedPendingText,
sessionKeys: recoverySessionKeys,
storePath: params.storePath,
terminalStatus,
const restoreAdmittedRecovery: RestoreAdmittedRecovery = async () => {
const restored = await commitMainSessionRecovery({
command: {
kind: "mark_admitted_recovery_interrupted",
lifecycleGeneration: getAgentEventLifecycleGeneration(),
now: Date.now(),
runId: recoveryRunId,
sessionId: params.entry.sessionId,
},
requireWriteSuccess: true,
target: { sessionKey: params.sessionKey, storePath: params.storePath },
});
log.info(`settled completed restart recovery for ${params.sessionKey}`);
return true;
return restored.transition.kind === "applied" && restored.entry && restored.sessionKey
? {
sessionId: restored.entry.sessionId,
sessionKey: restored.sessionKey,
storePath: params.storePath,
}
: undefined;
};
try {
scheduleMainSessionRecoveryPendingTarget(
await restoreAdmittedRecoveryWithRetries(restoreAdmittedRecovery),
);
} catch (restoreError) {
log.warn(
`failed to restore ambiguous restart recovery ${params.sessionKey}: ${String(restoreError)}`,
);
scheduleAdmittedRecoveryRestore(restoreAdmittedRecovery);
}
}
log.warn(`failed to resume interrupted main session ${params.sessionKey}: ${String(error)}`);
return false;
if (reservation) {
const rollbackReservation = reservation;
await rollbackRestartRecoveryReservation({
kind: dispatchStarted && !explicitlyRejected ? "abandon_reservation" : "cancel_reservation",
reservation: rollbackReservation,
sessionKey: params.sessionKey,
storePath: params.storePath,
}).catch((rollbackError: unknown) => {
log.warn(
`failed to roll back interrupted main session recovery attempt ${params.sessionKey}: ${String(rollbackError)}`,
);
scheduleRestartRecoveryReservationRollback({
kind:
dispatchStarted && !explicitlyRejected ? "abandon_reservation" : "cancel_reservation",
reservation: rollbackReservation,
sessionKey: params.sessionKey,
storePath: params.storePath,
});
});
}
log.warn(
`failed to resume interrupted main session ${params.sessionKey}: ${error instanceof Error ? (error.stack ?? error.message) : String(error)}`,
);
return "failed";
}
}
@@ -0,0 +1,228 @@
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import type { InternalSessionEntry as SessionEntry } from "../config/sessions.js";
import { loadSessionEntry } from "../config/sessions/session-accessor.js";
import { appendAssistantMessageToSessionTranscript } from "../config/sessions/transcript.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import type { GatewayRecoveryRuntime } from "../gateway/server-instance-runtime.types.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import { resolveAgentIdFromSessionKey } from "../routing/session-key.js";
import type { DeliveryContext } from "../utils/delivery-context.shared.js";
import {
isMainSessionRecoveryExhausted,
type MainSessionRecoveryObservation,
} from "./main-session-recovery-state.js";
import { commitMainSessionRecovery } from "./main-session-recovery-store.js";
import { buildUnresumableSessionNoticeIdempotencyKey } from "./main-session-restart-claim.js";
import { resolveRestartRecoveryDeliveryContext } from "./main-session-restart-dispatch.js";
const log = createSubsystemLogger("main-session-restart-recovery");
const TOMBSTONED_SESSION_NOTICE =
"I couldn't recover this session after repeated gateway restarts. " +
"Use /new or /reset to start a replacement session.";
async function claimMainRestartRecoveryTombstone(params: {
observation: MainSessionRecoveryObservation;
reason: string;
storePath: string;
sessionKey: string;
}): Promise<SessionEntry | null> {
const claim = await commitMainSessionRecovery({
command: {
kind: "tombstone",
now: Date.now(),
observation: params.observation,
reason: params.reason,
},
requireWriteSuccess: true,
target: { sessionKey: params.sessionKey, storePath: params.storePath },
});
if (claim.transition.kind !== "tombstoned" || !claim.entry) {
return null;
}
log.warn(`tombstoned main-session restart recovery: ${params.sessionKey} (${params.reason})`);
return claim.entry;
}
export async function tombstoneMainRestartRecoveryWithNotice(params: {
cfg?: OpenClawConfig;
entry: SessionEntry;
gatewayRuntime: GatewayRecoveryRuntime;
observation: MainSessionRecoveryObservation;
reason: string;
sessionKey: string;
storePath: string;
}): Promise<"notice_failed" | "skipped" | "tombstoned"> {
const deliveryContext = resolveRestartRecoveryDeliveryContext({
cfg: params.cfg,
entry: params.entry,
includeSessionDeliveryFallback: true,
sessionKey: params.sessionKey,
});
if (!deliveryContext) {
// The transcript notice and tombstone share one SQLite transaction so a
// foreground takeover cannot leave behind a false terminal notice.
let entry = params.entry;
let observation = params.observation;
for (let attempt = 0; attempt < 3; attempt += 1) {
const notice = await writeUnresumableSessionNotice({
...params,
entry,
observation,
text: TOMBSTONED_SESSION_NOTICE,
});
if (notice === "written") {
return "tombstoned";
}
if (notice === "failed") {
return "notice_failed";
}
const current = loadSessionEntry({
sessionKey: params.sessionKey,
storePath: params.storePath,
readConsistency: "latest",
}) as SessionEntry | undefined;
const state = current?.mainRestartRecovery;
if (
!current ||
current.sessionId !== params.entry.sessionId ||
state?.cycleId !== params.observation.cycleId ||
state.tombstone ||
!isMainSessionRecoveryExhausted(current)
) {
return "skipped";
}
entry = current;
observation = {
sessionId: current.sessionId,
cycleId: state.cycleId,
revision: state.revision,
};
}
return "notice_failed";
}
const tombstonedEntry = await claimMainRestartRecoveryTombstone(params);
if (!tombstonedEntry) {
return "skipped";
}
if (deliveryContext) {
await sendUnresumableSessionNotice({
deliveryContext,
entry: tombstonedEntry,
gatewayRuntime: params.gatewayRuntime,
reason: params.reason,
sessionKey: params.sessionKey,
text: TOMBSTONED_SESSION_NOTICE,
});
}
return "tombstoned";
}
async function sendUnresumableSessionNotice(params: {
deliveryContext: DeliveryContext;
entry: SessionEntry;
gatewayRuntime: GatewayRecoveryRuntime;
reason: string;
sessionKey: string;
text: string;
}): Promise<void> {
const messageParams: Record<string, unknown> = {
to: params.deliveryContext.to,
message: params.text,
bestEffort: true,
...(params.deliveryContext.threadId != null
? { threadId: params.deliveryContext.threadId }
: {}),
};
const actionParams: Record<string, unknown> = {
channel: params.deliveryContext.channel,
action: "send",
sessionKey: params.sessionKey,
sessionId: params.entry.sessionId,
idempotencyKey: buildUnresumableSessionNoticeIdempotencyKey(params.entry),
params: messageParams,
};
const accountId = normalizeOptionalString(params.deliveryContext.accountId);
if (accountId) {
actionParams.accountId = accountId;
}
try {
await params.gatewayRuntime.sendRecoveryNotice(actionParams, 10_000);
log.info(
`sent interrupted main session recovery notice: ${params.sessionKey} (${params.reason})`,
);
} catch (error) {
log.warn(
`failed to send interrupted main session recovery notice ${params.sessionKey}: ${String(error)}`,
);
}
}
async function writeUnresumableSessionNotice(params: {
entry: SessionEntry;
observation: MainSessionRecoveryObservation;
reason: string;
sessionKey: string;
storePath: string;
text: string;
}): Promise<"failed" | "stale" | "written"> {
const recoveryState = params.entry.mainRestartRecovery;
if (
!recoveryState ||
recoveryState.cycleId !== params.observation.cycleId ||
recoveryState.revision !== params.observation.revision
) {
return "stale";
}
const now = Date.now();
const result = await appendAssistantMessageToSessionTranscript({
agentId: resolveAgentIdFromSessionKey(params.sessionKey),
sessionKey: params.sessionKey,
expectedSessionId: params.entry.sessionId,
expectedSessionState: {
abortedLastRun: params.entry.abortedLastRun,
mainRestartRecoveryCycleId: params.observation.cycleId,
mainRestartRecoveryRevision: params.observation.revision,
restartRecoveryBeforeAgentReplyState: params.entry.restartRecoveryBeforeAgentReplyState,
restartRecoveryDeliveryReceiptState: params.entry.restartRecoveryDeliveryReceiptState,
restartRecoveryDeliveryToolCallId: params.entry.restartRecoveryDeliveryToolCallId,
restartRecoveryDeliveryRequestFingerprint:
params.entry.restartRecoveryDeliveryRequestFingerprint,
restartRecoveryDeliveryRunId: params.entry.restartRecoveryDeliveryRunId,
restartRecoveryDeliverySourceRunId: params.entry.restartRecoveryDeliverySourceRunId,
restartRecoveryRequesterAccountId: params.entry.restartRecoveryRequesterAccountId,
restartRecoveryRequesterSenderId: params.entry.restartRecoveryRequesterSenderId,
restartRecoverySameChannelThreadRequired:
params.entry.restartRecoverySameChannelThreadRequired,
restartRecoverySourceIngress: params.entry.restartRecoverySourceIngress,
restartRecoverySourceReplyDeliveryMode: params.entry.restartRecoverySourceReplyDeliveryMode,
restartRecoveryTerminalRunIds: params.entry.restartRecoveryTerminalRunIds,
status: params.entry.status,
updatedAt: params.entry.updatedAt,
},
sessionLifecyclePatch: {
abortedLastRun: false,
endedAt: now,
mainRestartRecovery: {
...recoveryState,
revision: recoveryState.revision + 1,
tombstone: { reason: params.reason },
},
runtimeMs: Math.max(0, now - (params.entry.startedAt ?? now)),
status: "failed",
updatedAt: now,
},
storePath: params.storePath,
text: params.text,
idempotencyKey: buildUnresumableSessionNoticeIdempotencyKey(params.entry),
}).catch((error: unknown) => ({ ok: false as const, reason: String(error) }));
if (!result.ok) {
log.warn(
`failed to write interrupted main session notice ${params.sessionKey}: ${result.reason}`,
);
}
return result.ok
? "written"
: "code" in result && result.code === "session-rebound"
? "stale"
: "failed";
}
File diff suppressed because it is too large Load Diff
+363 -93
View File
@@ -2,14 +2,15 @@
* Post-restart recovery for main sessions interrupted while holding a transcript lock.
*/
import { randomUUID } from "node:crypto";
import fs from "node:fs";
import path from "node:path";
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import { isSilentReplyPayloadText, SILENT_REPLY_TOKEN } from "../auto-reply/tokens.js";
import { resolveStateDir } from "../config/paths.js";
import {
type InternalSessionEntry as SessionEntry,
type RestartRecoveryRun,
type SessionEntry,
resolveSessionWorkStartError,
resolveAllAgentSessionStoreTargetsSync,
resolveSessionFilePath,
@@ -18,6 +19,7 @@ import {
import { buildRestartRecoveryClaimCleanupPatch } from "../config/sessions/restart-recovery-state.js";
import {
applySessionEntryReplacements,
loadExactSessionEntry,
listSessionEntriesByStatus,
persistSessionTranscriptTurn,
type SessionTranscriptTurnExpectedState,
@@ -51,6 +53,12 @@ import {
listActiveEmbeddedRunSessionIds,
listActiveEmbeddedRunSessionKeys,
} from "./embedded-agent-runner/run-state.js";
import {
isMainRestartRecoveryCandidate,
transitionMainSessionRecovery,
type MainSessionRecoveryObservation,
} from "./main-session-recovery-state.js";
import { commitMainSessionRecovery } from "./main-session-recovery-store.js";
import {
buildUnresumableSessionNoticeIdempotencyKey,
loadExpectedRestartRecoveryClaim,
@@ -63,6 +71,7 @@ import {
resolveRestartRecoveryDeliveryContext,
resumeMainSession,
} from "./main-session-restart-dispatch.js";
import { tombstoneMainRestartRecoveryWithNotice } from "./main-session-restart-recovery-failure.js";
import { resolveAgentSessionDirs } from "./session-dirs.js";
import type { SessionLockInspection } from "./session-write-lock.js";
@@ -74,6 +83,34 @@ const UNRESUMABLE_SESSION_NOTICE =
"I was interrupted by a gateway restart and couldn't safely resume the previous turn. " +
"Please send that last request again and I'll pick it up cleanly.";
type ExpectedRestartRecoveryTarget = {
canonicalSessionKey?: string;
sessionId: string;
sessionKey: string;
};
type ExhaustedRestartRecoveryTarget = ExpectedRestartRecoveryTarget & {
storePath: string;
};
function loadExpectedRestartRecoveryTarget(params: {
expected: ExpectedRestartRecoveryTarget;
storePath: string;
}): SessionEntry | undefined {
const exact = loadExactSessionEntry({
sessionKey: params.expected.sessionKey,
storePath: params.storePath,
readConsistency: "latest",
});
const entry = exact?.sessionKey === params.expected.sessionKey ? exact.entry : undefined;
return entry?.sessionId === params.expected.sessionId &&
entry.status === "running" &&
entry.abortedLastRun === true &&
isMainRestartRecoveryCandidate(entry, params.expected.sessionKey)
? entry
: undefined;
}
function shouldSkipMainRecovery(entry: SessionEntry, sessionKey: string): boolean {
if (typeof entry.spawnDepth === "number" && entry.spawnDepth > 0) {
return true;
@@ -270,13 +307,6 @@ export async function markRestartAbortedMainSessions(params: {
continue;
}
const wasRunning = entry.status === "running";
entry.status = "running";
entry.abortedLastRun = true;
if (!wasRunning) {
entry.startedAt = undefined;
entry.endedAt = undefined;
entry.runtimeMs = undefined;
}
const recoveryRuns = new Map<string, RestartRecoveryRun>();
for (const run of entry.restartRecoveryRuns ?? []) {
if (run.lifecycleGeneration === currentLifecycleGeneration) {
@@ -305,7 +335,13 @@ export async function markRestartAbortedMainSessions(params: {
? a.lifecycleGeneration.localeCompare(b.lifecycleGeneration)
: a.runId.localeCompare(b.runId),
);
entry.updatedAt = Date.now();
transitionMainSessionRecovery(entry, {
kind: "mark_interrupted",
cycleId: randomUUID(),
now: Date.now(),
resetRuntime: !wasRunning,
runs: entry.restartRecoveryRuns,
});
replacements.push({ sessionKey, entry });
counts.marked++;
}
@@ -379,8 +415,11 @@ export async function markStartupOrphanedMainSessionsForRecovery(params: {
) {
continue;
}
entry.abortedLastRun = true;
entry.updatedAt = Date.now();
transitionMainSessionRecovery(entry, {
kind: "mark_interrupted",
cycleId: randomUUID(),
now: Date.now(),
});
replacements.push({ sessionKey, entry });
counts.marked++;
}
@@ -1018,60 +1057,25 @@ function resolveMainSessionResumePolicy(
}
async function markSessionFailed(params: {
expectedRecoveryRunId?: string;
expectedRecoverySourceRunId?: string;
expectedSessionId: string;
observation: MainSessionRecoveryObservation;
storePath: string;
sessionKey: string;
reason: string;
}): Promise<boolean> {
const marked = await applySessionEntryReplacements({
sessionKeys: [params.sessionKey],
storePath: params.storePath,
update: (entries) => {
const current = entries.find((entry) => entry.sessionKey === params.sessionKey);
const entry = current?.entry;
if (
!entry ||
entry.sessionId !== params.expectedSessionId ||
entry.status !== "running" ||
entry.abortedLastRun !== true ||
normalizeOptionalString(entry.restartRecoveryDeliveryRunId) !==
params.expectedRecoveryRunId ||
normalizeOptionalString(entry.restartRecoveryDeliverySourceRunId) !==
params.expectedRecoverySourceRunId
) {
return { result: false };
}
entry.status = "failed";
entry.abortedLastRun = true;
entry.endedAt = Date.now();
entry.updatedAt = entry.endedAt;
entry.pendingFinalDelivery = undefined;
entry.pendingFinalDeliveryText = undefined;
entry.pendingFinalDeliveryCreatedAt = undefined;
entry.pendingFinalDeliveryLastAttemptAt = undefined;
entry.pendingFinalDeliveryAttemptCount = undefined;
entry.pendingFinalDeliveryLastError = undefined;
entry.pendingFinalDeliveryContext = undefined;
entry.pendingFinalDeliveryIntentId = undefined;
Object.assign(
entry,
buildRestartRecoveryClaimCleanupPatch({
entry,
recordTerminalSource: true,
}),
);
return {
result: true,
replacements: [{ sessionKey: params.sessionKey, entry }],
};
const marked = await commitMainSessionRecovery({
command: {
kind: "fail_recovery",
now: Date.now(),
observation: params.observation,
},
requireWriteSuccess: true,
target: { sessionKey: params.sessionKey, storePath: params.storePath },
});
if (marked) {
if (marked.transition.kind === "failed") {
log.warn(`marked interrupted main session failed: ${params.sessionKey} (${params.reason})`);
return true;
}
return marked;
return false;
}
type RecoveryCheckpointCompletion =
@@ -1389,6 +1393,7 @@ async function failUnresumableMainSession(params: {
entry: SessionEntry;
expectedRecoverySourceRunId?: string;
gatewayRuntime: GatewayRecoveryRuntime;
observation: MainSessionRecoveryObservation;
reason: string;
sessionKey: string;
storePath: string;
@@ -1411,9 +1416,7 @@ async function failUnresumableMainSession(params: {
return "failed";
}
const marked = await markSessionFailed({
expectedRecoveryRunId: normalizeOptionalString(params.entry.restartRecoveryDeliveryRunId),
expectedRecoverySourceRunId: params.expectedRecoverySourceRunId,
expectedSessionId: params.entry.sessionId,
observation: params.observation,
storePath: params.storePath,
sessionKey: params.sessionKey,
reason: params.reason,
@@ -1467,7 +1470,11 @@ export async function markRestartAbortedMainSessionsFromLocks(params: {
if (!entryLockPaths.some((lockPath) => interruptedLockPaths.has(lockPath))) {
continue;
}
entry.abortedLastRun = true;
transitionMainSessionRecovery(entry, {
kind: "mark_interrupted",
cycleId: randomUUID(),
now: Date.now(),
});
replacements.push({ sessionKey, entry });
counts.marked++;
}
@@ -1508,9 +1515,12 @@ function resolveRecoveryDispatchSessionKey(params: {
async function recoverStore(params: {
cfg?: OpenClawConfig;
observationOnly?: boolean;
onExhaustedTarget?: (target: ExhaustedRestartRecoveryTarget) => void;
storePath: string;
resumedSessionKeys: Set<string>;
expectedClaim?: ExpectedRestartRecoveryClaim;
expectedTarget?: ExpectedRestartRecoveryTarget;
sessionWorkAdmissionHandoffId?: string;
activeSessionIds?: Iterable<string>;
activeSessionKeys?: Iterable<string>;
@@ -1535,6 +1545,12 @@ async function recoverStore(params: {
storePath: params.storePath,
});
entries = entry ? [{ sessionKey: params.expectedClaim.sessionKey, entry }] : [];
} else if (params.expectedTarget) {
const entry = loadExpectedRestartRecoveryTarget({
expected: params.expectedTarget,
storePath: params.storePath,
});
entries = entry ? [{ sessionKey: params.expectedTarget.sessionKey, entry }] : [];
} else {
entries = listSessionEntriesByStatus({ storePath: params.storePath }, ["running"]);
}
@@ -1544,9 +1560,10 @@ async function recoverStore(params: {
return result;
}
for (const { sessionKey, entry } of entries.toSorted((a, b) =>
for (const { sessionKey, entry: loadedEntry } of entries.toSorted((a, b) =>
a.sessionKey.localeCompare(b.sessionKey),
)) {
let entry = loadedEntry;
if (!entry || entry.status !== "running" || entry.abortedLastRun !== true) {
continue;
}
@@ -1568,7 +1585,9 @@ async function recoverStore(params: {
continue;
}
const dispatchSessionKey =
params.expectedClaim?.canonicalSessionKey ?? resolvedDispatchSessionKey;
params.expectedClaim?.canonicalSessionKey ??
params.expectedTarget?.canonicalSessionKey ??
resolvedDispatchSessionKey;
if (
hasCurrentProcessOwner({
activeSessionIds: resolveActiveSessionIds(),
@@ -1586,6 +1605,77 @@ async function recoverStore(params: {
continue;
}
const observed = await commitMainSessionRecovery({
command: {
kind: "observe",
cycleId: randomUUID(),
lifecycleGeneration: getAgentEventLifecycleGeneration(),
sessionKey,
},
requireWriteSuccess: true,
target: { sessionKey, storePath: params.storePath },
});
if (!observed.entry || observed.transition.kind !== "observed") {
result.skipped++;
continue;
}
entry = observed.entry;
const recoveryView = observed.transition.view;
if (
recoveryView.status === "inactive" ||
recoveryView.status === "blocked" ||
recoveryView.status === "tombstoned"
) {
result.skipped++;
continue;
}
if (recoveryView.status === "exhausted") {
const tombstone = await tombstoneMainRestartRecoveryWithNotice({
cfg: params.cfg,
entry,
gatewayRuntime: params.gatewayRuntime,
observation: recoveryView.observation,
reason: recoveryView.reason,
sessionKey,
storePath: params.storePath,
});
if (tombstone === "notice_failed") {
result.failed++;
} else {
result.skipped++;
}
continue;
}
if (params.observationOnly) {
result.skipped++;
continue;
}
const recordResumeResult = (resumeResult: Awaited<ReturnType<typeof resumeMainSession>>) => {
if (resumeResult === "resumed") {
params.resumedSessionKeys.add(resumeDedupeKey);
result.recovered++;
} else if (resumeResult === "skipped") {
result.skipped++;
} else {
result.failed++;
const current = loadExpectedRestartRecoveryTarget({
expected: { sessionId: entry.sessionId, sessionKey },
storePath: params.storePath,
});
if (
current?.mainRestartRecovery?.chargedAttempts === MAX_RECOVERY_RETRIES &&
!current.mainRestartRecovery.reservation
) {
params.onExhaustedTarget?.({
canonicalSessionKey: dispatchSessionKey,
sessionId: entry.sessionId,
sessionKey,
storePath: params.storePath,
});
}
}
};
if (
requiresRestartRecoveryMessageActionAuthority(entry) &&
!hasRestartRecoveryMessageActionAuthority(entry)
@@ -1597,6 +1687,7 @@ async function recoverStore(params: {
entry.restartRecoveryDeliverySourceRunId,
),
gatewayRuntime: params.gatewayRuntime,
observation: recoveryView.observation,
reason: "message-tool-only recovery authority is unavailable",
sessionKey,
storePath: params.storePath,
@@ -1627,6 +1718,7 @@ async function recoverStore(params: {
entry,
expectedRecoverySourceRunId,
gatewayRuntime: params.gatewayRuntime,
observation: recoveryView.observation,
reason: resumeBlockReason,
sessionKey,
storePath: params.storePath,
@@ -1647,6 +1739,8 @@ async function recoverStore(params: {
canonicalSessionKey: dispatchSessionKey,
cfg: params.cfg,
entry,
observation: recoveryView.observation,
recoveryAttempt: recoveryView.nextAttempt,
storePath: params.storePath,
sessionKey,
pendingFinalDeliveryText: entry.pendingFinalDeliveryText,
@@ -1654,12 +1748,7 @@ async function recoverStore(params: {
sessionWorkAdmissionHandoffId: params.sessionWorkAdmissionHandoffId,
gatewayRuntime: params.gatewayRuntime,
});
if (resumed) {
params.resumedSessionKeys.add(resumeDedupeKey);
result.recovered++;
} else {
result.failed++;
}
recordResumeResult(resumed);
continue;
}
@@ -1691,18 +1780,15 @@ async function recoverStore(params: {
canonicalSessionKey: dispatchSessionKey,
cfg: params.cfg,
entry,
observation: recoveryView.observation,
recoveryAttempt: recoveryView.nextAttempt,
storePath: params.storePath,
sessionKey,
pendingFinalDeliveryText: entry.pendingFinalDeliveryText,
sessionWorkAdmissionHandoffId: params.sessionWorkAdmissionHandoffId,
gatewayRuntime: params.gatewayRuntime,
});
if (resumed) {
params.resumedSessionKeys.add(resumeDedupeKey);
result.recovered++;
} else {
result.failed++;
}
recordResumeResult(resumed);
continue;
}
log.warn(`failed to read transcript for ${sessionKey}: ${String(err)}`);
@@ -1718,6 +1804,8 @@ async function recoverStore(params: {
canonicalSessionKey: dispatchSessionKey,
cfg: params.cfg,
entry,
observation: recoveryView.observation,
recoveryAttempt: recoveryView.nextAttempt,
storePath: params.storePath,
sessionKey,
pendingFinalDeliveryText: entry.pendingFinalDeliveryText,
@@ -1725,12 +1813,7 @@ async function recoverStore(params: {
sessionWorkAdmissionHandoffId: params.sessionWorkAdmissionHandoffId,
gatewayRuntime: params.gatewayRuntime,
});
if (resumed) {
params.resumedSessionKeys.add(resumeDedupeKey);
result.recovered++;
} else {
result.failed++;
}
recordResumeResult(resumed);
continue;
}
@@ -1767,6 +1850,7 @@ async function recoverStore(params: {
entry,
expectedRecoverySourceRunId,
gatewayRuntime: params.gatewayRuntime,
observation: recoveryView.observation,
reason: completion.reason,
sessionKey,
storePath: params.storePath,
@@ -1781,6 +1865,7 @@ async function recoverStore(params: {
entry,
expectedRecoverySourceRunId,
gatewayRuntime: params.gatewayRuntime,
observation: recoveryView.observation,
reason: resumePolicy.reason,
sessionKey,
storePath: params.storePath,
@@ -1796,6 +1881,8 @@ async function recoverStore(params: {
canonicalSessionKey: dispatchSessionKey,
cfg: params.cfg,
entry,
observation: recoveryView.observation,
recoveryAttempt: recoveryView.nextAttempt,
storePath: params.storePath,
sessionKey,
pendingFinalDeliveryText: entry.pendingFinalDeliveryText,
@@ -1804,12 +1891,7 @@ async function recoverStore(params: {
sessionWorkAdmissionHandoffId: params.sessionWorkAdmissionHandoffId,
gatewayRuntime: params.gatewayRuntime,
});
if (resumed) {
params.resumedSessionKeys.add(resumeDedupeKey);
result.recovered++;
} else {
result.failed++;
}
recordResumeResult(resumed);
}
return result;
@@ -1833,8 +1915,9 @@ async function resolveRestartRecoveryStorePaths(params: {
return [...storePaths].toSorted((a, b) => a.localeCompare(b));
}
export async function recoverRestartAbortedMainSessions(params: {
async function recoverRestartAbortedMainSessionsWithOptions(params: {
cfg?: OpenClawConfig;
onExhaustedTarget?: (target: ExhaustedRestartRecoveryTarget) => void;
stateDir?: string;
resumedSessionKeys?: Set<string>;
activeSessionIds?: Iterable<string>;
@@ -1847,6 +1930,7 @@ export async function recoverRestartAbortedMainSessions(params: {
for (const storePath of await resolveRestartRecoveryStorePaths(params)) {
const storeResult = await recoverStore({
cfg: params.cfg,
onExhaustedTarget: params.onExhaustedTarget,
storePath,
resumedSessionKeys,
activeSessionIds: params.activeSessionIds,
@@ -1866,6 +1950,17 @@ export async function recoverRestartAbortedMainSessions(params: {
return result;
}
export async function recoverRestartAbortedMainSessions(params: {
cfg?: OpenClawConfig;
stateDir?: string;
resumedSessionKeys?: Set<string>;
activeSessionIds?: Iterable<string>;
activeSessionKeys?: Iterable<string>;
gatewayRuntime: GatewayRecoveryRuntime;
}): Promise<{ recovered: number; failed: number; skipped: number }> {
return await recoverRestartAbortedMainSessionsWithOptions(params);
}
/** Retries one exact durable Control UI row from its owning per-agent SQLite store. */
export async function retryRestartAbortedMainSessionRecovery(params: {
canonicalSessionKey?: string;
@@ -1918,16 +2013,147 @@ export async function retryRestartAbortedMainSessionRecovery(params: {
);
} finally {
cancelSessionWorkAdmissionHandoff(handoffId);
admission.release();
}
}
export async function recoverStartupOrphanedMainSessions(params: {
/** Reconciles one interrupted row after its final foreground owner releases. */
export async function retryRestartAbortedMainSessionRecoveryAfterOwnerRelease(params: {
cfg?: OpenClawConfig;
expectedSessionId: string;
sessionKey: string;
storePath: string;
gatewayRuntime: GatewayRecoveryRuntime;
}): Promise<{ recovered: number; failed: number; skipped: number }> {
return await recoverExpectedRestartRecoveryTarget(params);
}
async function recoverExpectedRestartRecoveryTarget(params: {
canonicalSessionKey?: string;
cfg?: OpenClawConfig;
expectedSessionId: string;
observationOnly?: boolean;
sessionKey: string;
storePath: string;
gatewayRuntime: GatewayRecoveryRuntime;
}): Promise<{ recovered: number; failed: number; skipped: number }> {
const expectedTarget: ExpectedRestartRecoveryTarget = {
canonicalSessionKey: params.canonicalSessionKey,
sessionId: params.expectedSessionId,
sessionKey: params.sessionKey,
};
const assertTargetCurrent = () => {
if (
!loadExpectedRestartRecoveryTarget({ expected: expectedTarget, storePath: params.storePath })
) {
throw new Error("restart recovery session ownership changed before owner-release retry");
}
};
if (
!loadExpectedRestartRecoveryTarget({ expected: expectedTarget, storePath: params.storePath })
) {
return { recovered: 0, failed: 0, skipped: 0 };
}
const admission = await beginSessionWorkAdmission({
scope: params.storePath,
identities: [params.sessionKey, params.expectedSessionId],
assertAllowed: assertTargetCurrent,
revalidateAllowed: assertTargetCurrent,
});
const handoffId = admission.createHandoff();
try {
return await admission.run(
async () =>
await recoverStore({
cfg: params.cfg,
observationOnly: params.observationOnly,
storePath: params.storePath,
resumedSessionKeys: new Set<string>(),
expectedTarget,
sessionWorkAdmissionHandoffId: handoffId,
gatewayRuntime: params.gatewayRuntime,
}),
);
} finally {
cancelSessionWorkAdmissionHandoff(handoffId);
admission.release();
}
}
export function scheduleRestartAbortedMainSessionRecoveryAfterOwnerRelease(params: {
delayMs?: number;
expectedSessionId: string;
getConfig: () => OpenClawConfig;
getGatewayRuntime: () => GatewayRecoveryRuntime | undefined;
maxRetries?: number;
sessionKey: string;
storePath: string;
}): void {
const retryDelayMs = params.delayMs ?? DEFAULT_RECOVERY_DELAY_MS;
const maxRetries = params.maxRetries ?? MAX_RECOVERY_RETRIES;
const scheduleAttempt = (attempt: number, delayMs: number) => {
const run = () => {
void runWithGatewayIndependentRootWorkAdmission(async () => {
const gatewayRuntime = params.getGatewayRuntime();
if (!gatewayRuntime) {
throw new Error("Gateway recovery runtime is unavailable");
}
return await retryRestartAbortedMainSessionRecoveryAfterOwnerRelease({
cfg: params.getConfig(),
expectedSessionId: params.expectedSessionId,
sessionKey: params.sessionKey,
storePath: params.storePath,
gatewayRuntime,
});
})
.then((result) => {
const stillPending = loadExpectedRestartRecoveryTarget({
expected: {
sessionId: params.expectedSessionId,
sessionKey: params.sessionKey,
},
storePath: params.storePath,
});
if (
(result.failed > 0 || (result.recovered === 0 && stillPending)) &&
attempt < maxRetries
) {
scheduleAttempt(attempt + 1, retryDelayMs * 2 ** (attempt - 1));
} else if (
attempt === maxRetries &&
stillPending?.mainRestartRecovery?.chargedAttempts === MAX_RECOVERY_RETRIES &&
!stillPending.mainRestartRecovery.reservation
) {
// The last ambiguous dispatch consumed the final durable charge.
// One exact observation tombstones exhaustion without dispatching again.
scheduleAttempt(attempt + 1, 0);
}
})
.catch((error: unknown) => {
if (attempt < maxRetries) {
scheduleAttempt(attempt + 1, retryDelayMs * 2 ** (attempt - 1));
} else {
log.warn(`main-session owner-release recovery failed: ${String(error)}`);
}
});
};
if (delayMs <= 0) {
run();
} else {
setTimeout(run, delayMs).unref?.();
}
};
scheduleAttempt(1, 0);
}
async function recoverStartupOrphanedMainSessionsWithOptions(params: {
cfg?: OpenClawConfig;
stateDir?: string;
activeSessionIds?: Iterable<string>;
activeSessionKeys?: Iterable<string>;
updatedBeforeMs?: number;
resumedSessionKeys?: Set<string>;
onExhaustedTarget?: (target: ExhaustedRestartRecoveryTarget) => void;
gatewayRuntime: GatewayRecoveryRuntime;
}): Promise<{ marked: number; recovered: number; failed: number; skipped: number }> {
const startupRecoveryCutoffMs = params.updatedBeforeMs ?? Date.now();
@@ -1938,8 +2164,9 @@ export async function recoverStartupOrphanedMainSessions(params: {
activeSessionKeys: params.activeSessionKeys,
updatedBeforeMs: startupRecoveryCutoffMs,
});
const recovered = await recoverRestartAbortedMainSessions({
const recovered = await recoverRestartAbortedMainSessionsWithOptions({
cfg: params.cfg,
onExhaustedTarget: params.onExhaustedTarget,
stateDir: params.stateDir,
resumedSessionKeys: params.resumedSessionKeys,
activeSessionIds: params.activeSessionIds,
@@ -1954,6 +2181,18 @@ export async function recoverStartupOrphanedMainSessions(params: {
};
}
export async function recoverStartupOrphanedMainSessions(params: {
cfg?: OpenClawConfig;
stateDir?: string;
activeSessionIds?: Iterable<string>;
activeSessionKeys?: Iterable<string>;
updatedBeforeMs?: number;
resumedSessionKeys?: Set<string>;
gatewayRuntime: GatewayRecoveryRuntime;
}): Promise<{ marked: number; recovered: number; failed: number; skipped: number }> {
return await recoverStartupOrphanedMainSessionsWithOptions(params);
}
export function scheduleRestartAbortedMainSessionRecovery(params: {
cfg?: OpenClawConfig;
delayMs?: number;
@@ -1969,29 +2208,60 @@ export function scheduleRestartAbortedMainSessionRecovery(params: {
const startupRecoveryCutoffMs = Date.now();
const runRecoveryAttempt = (attempt: number, delay: number) => {
const exhaustedTargets = new Map<string, ExhaustedRestartRecoveryTarget>();
const reconcileExhaustedTargets = async () => {
const outcomes = await Promise.allSettled(
[...exhaustedTargets.values()].map((target) =>
runWithGatewayIndependentRootWorkAdmission(
async () =>
await recoverExpectedRestartRecoveryTarget({
canonicalSessionKey: target.canonicalSessionKey,
cfg: params.cfg,
expectedSessionId: target.sessionId,
observationOnly: true,
sessionKey: target.sessionKey,
storePath: target.storePath,
gatewayRuntime: params.gatewayRuntime,
}),
),
),
);
for (const outcome of outcomes) {
if (outcome.status === "rejected") {
log.warn(`main-session exhaustion reconciliation failed: ${String(outcome.reason)}`);
}
}
};
// Delayed retries outlive startup; each attempt must independently block
// host suspension while it reads and rewrites recovery session state.
void runWithGatewayIndependentRootWorkAdmission(
async () =>
await recoverStartupOrphanedMainSessions({
await recoverStartupOrphanedMainSessionsWithOptions({
cfg: params.cfg,
onExhaustedTarget: (target) => {
exhaustedTargets.set(`${target.storePath}\u0000${target.sessionKey}`, target);
},
stateDir: params.stateDir,
resumedSessionKeys,
updatedBeforeMs: startupRecoveryCutoffMs,
gatewayRuntime: params.gatewayRuntime,
}),
)
.then((result) => {
.then(async (result) => {
if (result.failed > 0 && attempt < maxRetries) {
scheduleAttempt(attempt + 1, delay * RETRY_BACKOFF_MULTIPLIER);
} else if (result.failed > 0 && attempt === maxRetries && exhaustedTargets.size > 0) {
// Reconcile only exact rows whose final dispatch retained its durable charge.
await reconcileExhaustedTargets();
}
})
.catch((err: unknown) => {
.catch(async (err: unknown) => {
if (attempt < maxRetries) {
log.warn(`main-session restart recovery failed: ${String(err)}`);
scheduleAttempt(attempt + 1, delay * RETRY_BACKOFF_MULTIPLIER);
} else {
log.warn(`main-session restart recovery gave up: ${String(err)}`);
await reconcileExhaustedTargets();
}
});
};
+26 -1
View File
@@ -678,7 +678,7 @@ describe("subagent-orphan-recovery", () => {
const store = mockSingleAbortedSession({
subagentRecovery: {
automaticAttempts: 2,
lastAttemptAt: now - 30_000,
lastAttemptAt: now - 2 * 60_000,
lastRunId: "previous-run",
},
});
@@ -710,6 +710,31 @@ describe("subagent-orphan-recovery", () => {
expect(recovery.wedgedReason).toContain("recovery blocked");
});
it("starts a new attempt burst after the two-minute re-wedge window", async () => {
const now = Date.now();
const expiredRecovery = {
automaticAttempts: 2,
lastAttemptAt: now - 2 * 60_000 - 1,
lastRunId: "previous-run",
};
const store = mockSingleAbortedSession({ subagentRecovery: expiredRecovery });
const result = await recoverOrphanedSubagentSessions({
getActiveRuns: () => createActiveRuns(createTestRunRecord()),
});
expect(result.recovered).toBe(1);
expect(dispatchAgent).toHaveBeenCalledOnce();
expect(sessionAccessor.patchSessionEntry).toHaveBeenCalledOnce();
const sessionEntry = requireRecord(
store["agent:main:subagent:test-session-1"],
"updated session entry",
);
const recovery = requireRecord(sessionEntry.subagentRecovery, "subagent recovery");
expect(recovery.automaticAttempts).toBe(1);
expect(recovery.lastRunId).toBe("run-1");
});
it("skips already tombstoned wedged sessions without rewriting them", async () => {
mockSingleAbortedSession({
subagentRecovery: {
+7 -10
View File
@@ -8,7 +8,10 @@ import type { SessionEntry } from "../config/sessions.js";
const SUBAGENT_RECOVERY_MAX_AUTOMATIC_ATTEMPTS = 2;
const SUBAGENT_RECOVERY_REWEDGE_WINDOW_MS = 2 * 60_000;
/** Decision returned before attempting automatic subagent orphan recovery. */
function normalizeRecoveryAttempts(value: unknown): number {
return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : 0;
}
type SubagentRecoveryGate =
| {
allowed: true;
@@ -31,7 +34,6 @@ function isRecentRecoveryAttempt(entry: SessionEntry, now: number): boolean {
);
}
/** Returns true when recovery has been tombstoned for a session entry. */
export function isSubagentRecoveryWedgedEntry(entry: unknown): boolean {
if (!entry || typeof entry !== "object") {
return false;
@@ -44,7 +46,6 @@ export function isSubagentRecoveryWedgedEntry(entry: unknown): boolean {
);
}
/** Formats the operator-facing reason for a wedged recovery entry. */
export function formatSubagentRecoveryWedgedReason(entry: SessionEntry): string {
return (
entry.subagentRecovery?.wedgedReason?.trim() ||
@@ -52,7 +53,6 @@ export function formatSubagentRecoveryWedgedReason(entry: SessionEntry): string
);
}
/** Checks whether automatic orphan recovery may run for this session entry. */
export function evaluateSubagentRecoveryGate(
entry: SessionEntry,
now: number,
@@ -66,7 +66,7 @@ export function evaluateSubagentRecoveryGate(
}
const previousAttempts = isRecentRecoveryAttempt(entry, now)
? Math.max(0, entry.subagentRecovery?.automaticAttempts ?? 0)
? normalizeRecoveryAttempts(entry.subagentRecovery?.automaticAttempts)
: 0;
if (previousAttempts >= SUBAGENT_RECOVERY_MAX_AUTOMATIC_ATTEMPTS) {
return {
@@ -84,7 +84,6 @@ export function evaluateSubagentRecoveryGate(
};
}
/** Records one accepted automatic orphan-recovery attempt. */
export function markSubagentRecoveryAttempt(params: {
entry: SessionEntry;
now: number;
@@ -98,11 +97,10 @@ export function markSubagentRecoveryAttempt(params: {
};
}
/** Tombstones automatic recovery until maintenance or doctor clears the state. */
export function markSubagentRecoveryWedged(params: {
entry: SessionEntry;
now: number;
runId?: string;
runId: string;
reason: string;
}): void {
params.entry.abortedLastRun = false;
@@ -113,14 +111,13 @@ export function markSubagentRecoveryWedged(params: {
SUBAGENT_RECOVERY_MAX_AUTOMATIC_ATTEMPTS,
),
lastAttemptAt: params.entry.subagentRecovery?.lastAttemptAt ?? params.now,
...(params.runId ? { lastRunId: params.runId } : {}),
lastRunId: params.runId,
wedgedAt: params.now,
wedgedReason: params.reason,
};
params.entry.updatedAt = params.now;
}
/** Clears stale abort state when a wedged entry should no longer look runnable. */
export function clearWedgedSubagentRecoveryAbort(entry: SessionEntry, now: number): boolean {
if (!isSubagentRecoveryWedgedEntry(entry) || entry.abortedLastRun !== true) {
return false;
@@ -1,4 +1,5 @@
// Handles session reset requests produced during agent runner execution.
import { transitionMainSessionRecovery } from "../../agents/main-session-recovery-state.js";
import type { SessionEntry } from "../../config/sessions.js";
import { resolveAgentIdFromSessionKey } from "../../config/sessions.js";
import { persistSessionResetLifecycle } from "../../config/sessions/session-accessor.js";
@@ -101,6 +102,7 @@ export async function resetReplyRunSession(params: {
memoryFlushLastFailedAt: undefined,
memoryFlushLastFailureError: undefined,
};
transitionMainSessionRecovery(nextEntry, { kind: "clear" });
const agentId = resolveAgentIdFromSessionKey(params.sessionKey);
const nextSessionFile = formatSqliteSessionFileMarker({
agentId,
@@ -340,6 +340,7 @@ describe("getReplyFromConfig fast test bootstrap", () => {
pendingFinalDeliveryCreatedAt: 1,
pendingFinalDeliveryAttemptCount: 4,
pendingFinalDeliveryLastError: null,
pendingFinalDeliveryIntentId: "stale-heartbeat-intent",
},
});
const cfg = withFastReplyConfig({
@@ -361,6 +362,7 @@ describe("getReplyFromConfig fast test bootstrap", () => {
expect(stored.pendingFinalDelivery).toBeUndefined();
expect(stored.pendingFinalDeliveryText).toBeUndefined();
expect(stored.pendingFinalDeliveryAttemptCount).toBeUndefined();
expect(stored.pendingFinalDeliveryIntentId).toBeUndefined();
});
it("keeps non-ack heartbeat pending delivery without direct replay", async () => {
+6 -17
View File
@@ -62,7 +62,10 @@ import {
} from "./inbound-media.js";
import { emitPreAgentMessageHooks } from "./message-preprocess-hooks.js";
import { createFastTestModelSelectionState, createModelSelectionState } from "./model-selection.js";
import { sanitizePendingFinalDeliveryText } from "./pending-final-delivery.js";
import {
PENDING_FINAL_DELIVERY_CLEAR_PATCH,
sanitizePendingFinalDeliveryText,
} from "./pending-final-delivery.js";
import { attachProgressNarratorToReplyOptions } from "./progress-narrator.js";
import { createReplyTimingTracker } from "./reply-timing-tracker.js";
import { initSessionState, resolveReplySessionPreprocessingState } from "./session.js";
@@ -553,13 +556,7 @@ export async function getReplyFromConfig(
resolveHeartbeatAckMaxChars(cfg, agentId),
);
if (heartbeatPending.shouldClear) {
sessionEntry.pendingFinalDelivery = undefined;
sessionEntry.pendingFinalDeliveryText = undefined;
sessionEntry.pendingFinalDeliveryCreatedAt = undefined;
sessionEntry.pendingFinalDeliveryLastAttemptAt = undefined;
sessionEntry.pendingFinalDeliveryAttemptCount = undefined;
sessionEntry.pendingFinalDeliveryLastError = undefined;
sessionEntry.pendingFinalDeliveryContext = undefined;
Object.assign(sessionEntry, PENDING_FINAL_DELIVERY_CLEAR_PATCH);
sessionEntryHandle.replaceCurrent(sessionEntry);
if (sessionKey && sessionStore) {
sessionStore[sessionKey] = sessionEntry;
@@ -568,15 +565,7 @@ export async function getReplyFromConfig(
const { updateSessionEntry } = await import("../../config/sessions/session-accessor.js");
await updateSessionEntry(
{ storePath, sessionKey },
() => ({
pendingFinalDelivery: undefined,
pendingFinalDeliveryText: undefined,
pendingFinalDeliveryCreatedAt: undefined,
pendingFinalDeliveryLastAttemptAt: undefined,
pendingFinalDeliveryAttemptCount: undefined,
pendingFinalDeliveryLastError: undefined,
pendingFinalDeliveryContext: undefined,
}),
() => ({ ...PENDING_FINAL_DELIVERY_CLEAR_PATCH }),
{
skipMaintenance: true,
takeCacheOwnership: true,
+15 -2
View File
@@ -1,3 +1,4 @@
import type { SessionEntry } from "../../config/sessions/types.js";
import { normalizeReplyPayloadsForDelivery } from "../../infra/outbound/payloads.js";
import {
isSilentReplyPayloadText,
@@ -7,7 +8,6 @@ import {
stripLeadingSilentToken,
stripSilentToken,
} from "../tokens.js";
/** Sanitizes pending final delivery text before channel-visible output. */
import type { ReplyPayload } from "../types.js";
import { stripInternalMetadataForDisplay } from "./display-text-sanitize.js";
import { normalizeReplyPayload } from "./normalize-reply.js";
@@ -87,6 +87,19 @@ export function buildPendingFinalDeliveryText(payloads: ReplyPayload[]): string
return sanitizePendingFinalDeliveryText(text);
}
// A delivered or discarded final must lose the whole record. Keeping this list
// centralized prevents new ownership fields from leaving a phantom pending delivery.
export const PENDING_FINAL_DELIVERY_CLEAR_PATCH = {
pendingFinalDelivery: undefined,
pendingFinalDeliveryText: undefined,
pendingFinalDeliveryCreatedAt: undefined,
pendingFinalDeliveryLastAttemptAt: undefined,
pendingFinalDeliveryAttemptCount: undefined,
pendingFinalDeliveryLastError: undefined,
pendingFinalDeliveryContext: undefined,
pendingFinalDeliveryIntentId: undefined,
} as const satisfies Partial<SessionEntry>;
function collectDurableMediaDirectives(payload: ReplyPayload): string[] {
if (payload.sensitiveMedia === true) {
return [];
@@ -144,7 +157,7 @@ function hasUnrecoverableNormalizedDeliveryShape(payload: ReplyPayload): boolean
);
}
/** Sanitizes final pending-delivery text and removes silent control tokens. */
/** Sanitizes pending final delivery text before channel-visible output. */
export function sanitizePendingFinalDeliveryText(text: string): string {
let stripped = stripInternalMetadataForDisplay(text).trim();
if (isSilentReplyPayloadText(stripped, SILENT_REPLY_TOKEN)) {
@@ -4,10 +4,11 @@ import { afterEach, describe, expect, it, vi } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../../test/helpers/temp-dir.js";
import {
deleteSessionEntryLifecycle,
loadSessionEntry,
replaceSessionEntry,
replaceSessionEntrySync,
} from "../../config/sessions/session-accessor.js";
import type { SessionEntry } from "../../config/sessions/types.js";
import type { InternalSessionEntry as SessionEntry } from "../../config/sessions/types.js";
import {
resetDiagnosticRunActivityForTest,
RUN_STALE_TAKEOVER_MS,
@@ -28,6 +29,15 @@ import {
import { testing } from "./reply-run-registry.test-support.js";
import { admitReplyTurn, runWithReplyOperationLifecycleAdmission } from "./reply-turn-admission.js";
const recoveryOwnerReleaseMocks = vi.hoisted(() => ({
schedulePendingTarget: vi.fn(),
}));
vi.mock("../../agents/main-session-recovery-owner-release.js", async (importOriginal) => ({
...(await importOriginal<typeof import("../../agents/main-session-recovery-owner-release.js")>()),
scheduleMainSessionRecoveryPendingTarget: recoveryOwnerReleaseMocks.schedulePendingTarget,
}));
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
function createDeferred() {
@@ -49,10 +59,18 @@ function createSessionStore(entries: Record<string, object>): string {
return storePath;
}
async function readSessionEntry(
storePath: string,
sessionKey: string,
): Promise<SessionEntry | undefined> {
return loadSessionEntry({ sessionKey, storePath });
}
describe("reply turn admission", () => {
afterEach(() => {
testing.resetReplyRunRegistry();
resetDiagnosticRunActivityForTest();
recoveryOwnerReleaseMocks.schedulePendingTarget.mockClear();
});
it("rejects a reply when an archive commits before admission", async () => {
@@ -324,6 +342,210 @@ describe("reply turn admission", () => {
expect(mutationRan).toBe(true);
});
it.each(["visible", "heartbeat", "queued_followup"] as const)(
"fences restart recovery from %s reply admission until the operation clears",
async (kind) => {
const sessionKey = `agent:main:telegram:topic:recovery-race:${kind}`;
const sessionId = "interrupted-session";
const storePath = createSessionStore({
[sessionKey]: {
sessionId,
updatedAt: 100,
status: "running",
abortedLastRun: true,
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 1,
chargedAttempts: 2,
},
},
});
const admission = await admitReplyTurn({
sessionKey,
sessionId,
expectedSessionId: sessionId,
storePath,
kind,
resetTriggered: false,
});
expect(admission.status).toBe("owned");
if (admission.status !== "owned") {
return;
}
const claimedEntry = await readSessionEntry(storePath, sessionKey);
admission.operation.complete();
await vi.waitFor(async () => {
const entry = await readSessionEntry(storePath, sessionKey);
expect(entry?.mainRestartRecovery?.foregroundClaims).toBeUndefined();
});
expect(claimedEntry?.mainRestartRecovery).toMatchObject({
foregroundClaims: {
tokens: [expect.any(String)],
},
});
await expect(readSessionEntry(storePath, sessionKey)).resolves.toMatchObject({
sessionId,
status: "running",
});
},
);
it.each(["visible", "heartbeat"] as const)(
"rejects %s reply admission for a tombstoned recovery session",
async (kind) => {
const sessionKey = `agent:main:telegram:topic:recovery-tombstone:${kind}`;
const sessionId = "tombstoned-session";
const storePath = createSessionStore({
[sessionKey]: {
sessionId,
updatedAt: 100,
status: "failed",
abortedLastRun: false,
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 4,
chargedAttempts: 3,
tombstone: { reason: "automatic recovery exhausted" },
},
},
});
await expect(
admitReplyTurn({
sessionKey,
sessionId,
expectedSessionId: sessionId,
storePath,
kind,
resetTriggered: false,
}),
).rejects.toThrow(/changed while starting work/i);
},
);
it("drops a queued followup for an admitted recovery fence", async () => {
const sessionKey = "agent:main:telegram:topic:admitted-recovery";
const sessionId = "admitted-recovery-session";
const storePath = createSessionStore({
[sessionKey]: {
sessionId,
updatedAt: 100,
status: "running",
abortedLastRun: false,
restartRecoveryRuns: [{ runId: "recovery-run", lifecycleGeneration: "generation-1" }],
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 3,
chargedAttempts: 1,
},
},
});
await expect(
admitReplyTurn({
sessionKey,
sessionId,
expectedSessionId: sessionId,
storePath,
kind: "queued_followup",
resetTriggered: false,
}),
).resolves.toEqual({ status: "skipped", reason: "lifecycle-invalidated" });
});
it("schedules released recovery only after retained admission exits", async () => {
const sourceSessionKey = "agent:main:telegram:slash:recovery-adoption";
const sessionKey = "agent:main:telegram:topic:recovery-adoption";
const sessionId = "interrupted-session";
const storePath = createSessionStore({
[sessionKey]: {
sessionId,
updatedAt: 100,
status: "running",
abortedLastRun: true,
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 1,
chargedAttempts: 0,
},
},
});
const blocker = createReplyOperation({
sessionKey,
sessionId,
resetTriggered: false,
});
const reservation = createReplyOperation({
sessionKey: sourceSessionKey,
sessionId: "source-session",
resetTriggered: false,
});
const result = await admitReplyTurn({
sessionKey,
sessionId: reservation.sessionId,
expectedSessionId: sessionId,
storePath,
kind: "visible",
resetTriggered: false,
waitForActive: false,
retainLifecycleAdmissionOnActive: true,
adoptOperation: reservation,
});
expect(result).toMatchObject({ status: "skipped", reason: "active-run" });
expect(recoveryOwnerReleaseMocks.schedulePendingTarget).not.toHaveBeenCalled();
await expect(readSessionEntry(storePath, sessionKey)).resolves.not.toHaveProperty(
"mainRestartRecovery.foregroundClaims",
);
if (result.status === "skipped") {
result.lifecycleAdmission?.release();
}
await vi.waitFor(() => {
expect(recoveryOwnerReleaseMocks.schedulePendingTarget).toHaveBeenCalledWith({
sessionId,
sessionKey,
storePath,
});
});
blocker.complete();
reservation.complete();
});
it("leaves interrupted subagent sessions to the subagent recovery owner", async () => {
const sessionKey = "agent:main:subagent:child-1";
const sessionId = "subagent-session";
const storePath = createSessionStore({
[sessionKey]: {
sessionId,
updatedAt: 100,
status: "running",
abortedLastRun: true,
spawnDepth: 1,
},
});
const admission = await admitReplyTurn({
sessionKey,
sessionId,
expectedSessionId: sessionId,
storePath,
kind: "visible",
resetTriggered: false,
});
expect(admission.status).toBe("owned");
if (admission.status === "owned") {
admission.operation.complete();
}
await expect(readSessionEntry(storePath, sessionKey)).resolves.not.toHaveProperty(
"mainRestartRecovery",
);
});
it("holds interrupted queued reply work until its owner exits", async () => {
const sessionKey = "agent:main:telegram:topic:queued-delete";
const sessionId = "session-before-delete";
+71 -13
View File
@@ -1,11 +1,22 @@
import { scheduleMainSessionRecoveryPendingTarget } from "../../agents/main-session-recovery-owner-release.js";
import { isMainRestartRecoveryCandidate } from "../../agents/main-session-recovery-state.js";
import {
claimMainSessionRecoveryOwner,
releaseMainSessionRecoveryOwner,
type MainSessionRecoveryPendingTarget,
type MainSessionRecoveryOwnerLease,
} from "../../agents/main-session-recovery-store.js";
// Decides whether an inbound turn may start, queue, or abort a reply run.
import { resolveSessionWorkStartError } from "../../config/sessions/lifecycle.js";
import { loadSessionEntry } from "../../config/sessions/session-accessor.js";
import type { SessionEntry } from "../../config/sessions/types.js";
import type { InternalSessionEntry, SessionEntry } from "../../config/sessions/types.js";
import { getAgentEventLifecycleGeneration } from "../../infra/agent-events.js";
import { formatErrorMessage } from "../../infra/errors.js";
import {
getDiagnosticSessionActivitySnapshot,
resolveRunStaleThresholdMs,
} from "../../logging/diagnostic-run-activity.js";
import { createSubsystemLogger } from "../../logging/subsystem.js";
import {
beginSessionWorkAdmission,
type SessionWorkAdmissionLease,
@@ -40,14 +51,26 @@ type ReplyTurnAdmission =
class QueuedFollowupLifecycleInvalidatedError extends Error {}
const log = createSubsystemLogger("auto-reply/reply-turn-admission");
const lifecycleAdmissionByOperation = new WeakMap<ReplyOperation, SessionWorkAdmissionLease>();
async function releaseReplyRecoveryOwner(
lease: MainSessionRecoveryOwnerLease | undefined,
): Promise<MainSessionRecoveryPendingTarget | undefined> {
try {
return await releaseMainSessionRecoveryOwner(lease);
} catch (error) {
log.warn(`failed to release main-session recovery reply owner: ${formatErrorMessage(error)}`);
return undefined;
}
}
/** Runs owner work with its admission marked as the initiating lifecycle context. */
export async function runWithReplyOperationLifecycleAdmission<T>(
operation: ReplyOperation | undefined,
operation: ReplyOperation,
run: () => Promise<T>,
): Promise<T> {
const admission = operation ? lifecycleAdmissionByOperation.get(operation) : undefined;
const admission = lifecycleAdmissionByOperation.get(operation);
return admission ? await admission.run(run) : await run();
}
@@ -154,7 +177,8 @@ async function admitReplyTurnWithWaitSignal(
try {
const storePath = params.storePath;
let operation: ReplyOperation | undefined;
let admittedSessionEntry: SessionEntry | undefined;
let admittedSessionEntry: InternalSessionEntry | undefined;
let recoveryOwnerLease: MainSessionRecoveryOwnerLease | undefined;
let interruptedBeforeOperation = false;
const admission = storePath
? await beginSessionWorkAdmission({
@@ -172,7 +196,7 @@ async function admitReplyTurnWithWaitSignal(
sessionKey: params.sessionKey,
readConsistency: "latest",
});
admittedSessionEntry = currentEntry;
admittedSessionEntry = currentEntry as InternalSessionEntry | undefined;
if (expectedSessionId && !currentEntry) {
rejectLifecycleInvalidatedWork({
kind: params.kind,
@@ -229,14 +253,37 @@ async function admitReplyTurnWithWaitSignal(
},
})
: undefined;
if (interruptedBeforeOperation) {
admission?.release();
rejectLifecycleInvalidatedWork({
kind: params.kind,
message: `Session "${params.sessionKey}" changed while starting work. Retry.`,
});
}
try {
if (
storePath &&
!params.resetTriggered &&
admittedSessionEntry &&
((admittedSessionEntry.status === "running" &&
(admittedSessionEntry.abortedLastRun === true ||
admittedSessionEntry.restartRecoveryRuns !== undefined ||
admittedSessionEntry.mainRestartRecovery !== undefined)) ||
admittedSessionEntry.mainRestartRecovery?.tombstone !== undefined) &&
isMainRestartRecoveryCandidate(admittedSessionEntry, params.sessionKey)
) {
const ownerClaim = await claimMainSessionRecoveryOwner({
lifecycleGeneration: getAgentEventLifecycleGeneration(),
sessionId,
target: { sessionKey: params.sessionKey, storePath },
});
if (ownerClaim.kind === "invalidated") {
rejectLifecycleInvalidatedWork({
kind: params.kind,
message: `Session "${params.sessionKey}" changed while starting work. Retry.`,
});
}
recoveryOwnerLease = ownerClaim.kind === "claimed" ? ownerClaim.lease : undefined;
}
if (interruptedBeforeOperation || isAbortSignalAborted(params.upstreamAbortSignal)) {
rejectLifecycleInvalidatedWork({
kind: params.kind,
message: `Session "${params.sessionKey}" changed while starting work. Retry.`,
});
}
if (params.adoptOperation) {
// The dispatch closures own this object's abort/delivery lifecycle,
// so the reservation must move rather than be recreated. Throws
@@ -255,11 +302,17 @@ async function admitReplyTurnWithWaitSignal(
});
}
} catch (error) {
const pendingRecovery = recoveryOwnerLease
? await releaseReplyRecoveryOwner(recoveryOwnerLease)
: undefined;
if (
error instanceof ReplyRunAlreadyActiveError &&
admission &&
params.retainLifecycleAdmissionOnActive
) {
void admission.released.then(() => {
scheduleMainSessionRecoveryPendingTarget(pendingRecovery);
});
return {
status: "skipped",
reason: "active-run",
@@ -268,6 +321,7 @@ async function admitReplyTurnWithWaitSignal(
};
}
admission?.release();
scheduleMainSessionRecoveryPendingTarget(pendingRecovery);
throw error;
}
if (admission) {
@@ -281,7 +335,11 @@ async function admitReplyTurnWithWaitSignal(
lifecycleAdmissionByOperation.set(operation, admission);
runAfterReplyOperationClear(operation, () => {
lifecycleAdmissionByOperation.delete(operation);
admission.release();
// Keep reset/delete behind durable owner release and its writer lock.
void releaseReplyRecoveryOwner(recoveryOwnerLease).then((pendingTarget) => {
admission.release();
scheduleMainSessionRecoveryPendingTarget(pendingTarget);
});
});
}
return {
@@ -1,4 +1,5 @@
// Prepares parent-context fork metadata for guarded reply session initialization.
import { buildMainSessionRecoveryClearPatch } from "../../agents/main-session-recovery-clear.js";
import type { SessionEntry } from "../../config/sessions.js";
import { forkSessionFromParent, resolveParentForkDecision } from "./session-fork.js";
@@ -52,8 +53,11 @@ export async function prepareReplySessionParentFork(params: {
`parentTokens=${decision.parentTokens ?? "unknown"}`,
);
params.warn(`forked session created: file=${fork.sessionFile}`);
// The fork replaces this thread's transcript identity; recovery state from
// the preseed row must not govern a later interruption of the fork.
return {
...params.sessionEntry,
...buildMainSessionRecoveryClearPatch(params.sessionEntry),
sessionId: fork.sessionId,
sessionFile: fork.sessionFile,
forkedFromParent: true,
+14 -1
View File
@@ -8,7 +8,7 @@ import { testing as sessionMcpTesting } from "../../agents/agent-bundle-mcp-runt
import { getOrCreateSessionMcpRuntime } from "../../agents/agent-bundle-mcp-tools.js";
import * as bootstrapCache from "../../agents/bootstrap-cache.js";
import type { OpenClawConfig } from "../../config/config.js";
import type { SessionEntry } from "../../config/sessions.js";
import type { InternalSessionEntry as SessionEntry } from "../../config/sessions.js";
import {
appendTranscriptEvent,
listSessionEntries,
@@ -784,6 +784,14 @@ describe("initSessionState thread forking", () => {
updatedAt: Date.now(),
totalTokens: 0,
totalTokensFresh: true,
abortedLastRun: true,
restartRecoveryRuns: [{ runId: "old-run", lifecycleGeneration: "old-generation" }],
mainRestartRecovery: {
cycleId: "old-cycle",
revision: 4,
chargedAttempts: 3,
tombstone: { reason: "old transcript exhausted" },
},
},
});
@@ -805,6 +813,9 @@ describe("initSessionState thread forking", () => {
expect(first.sessionEntry.forkedFromParent).toBe(true);
expect(first.sessionEntry.totalTokens).toBeUndefined();
expect(first.sessionEntry.totalTokensFresh).toBe(false);
expect(first.sessionEntry.abortedLastRun).toBe(false);
expect(first.sessionEntry.restartRecoveryRuns).toBeUndefined();
expect((first.sessionEntry as SessionEntry).mainRestartRecovery).toBeUndefined();
const second = await initSessionState({
ctx: {
@@ -820,6 +831,8 @@ describe("initSessionState thread forking", () => {
expect(second.sessionEntry.forkedFromParent).toBe(true);
expect(second.sessionEntry.totalTokens).toBeUndefined();
expect(second.sessionEntry.totalTokensFresh).toBe(false);
expect(second.sessionEntry.restartRecoveryRuns).toBeUndefined();
expect((second.sessionEntry as SessionEntry).mainRestartRecovery).toBeUndefined();
warn.mockRestore();
});
+2 -1
View File
@@ -25,7 +25,7 @@ import {
} from "../config/sessions/session-accessor.js";
import { parseSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js";
import { clearSessionStoreCacheForTest } from "../config/sessions/store.js";
import type { SessionEntry } from "../config/sessions/types.js";
import type { InternalSessionEntry as SessionEntry } from "../config/sessions/types.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { emitAgentEvent, onAgentEvent, resetAgentEventsForTest } from "../infra/agent-events.js";
import type { PluginProviderRegistration } from "../plugins/registry.test-fixtures.js";
@@ -1100,6 +1100,7 @@ describe("agentCommand", () => {
expect(prepared.sessionStore).not.toBe(cached);
expect(prepared.sessionEntry).not.toBe(cached);
expect(prepared).not.toHaveProperty("recoveryCandidateEntry");
expect(prepared.sessionStore?.[sessionKey]).toBe(prepared.sessionEntry);
expect(prepared.sessionStore?.["agent:main:other"]).toBeUndefined();
});
@@ -0,0 +1,83 @@
import { uniqueStrings } from "@openclaw/normalization-core/string-normalization";
import { inspectMainSessionRecoveryHealth } from "../agents/main-session-recovery-lifecycle.js";
import { transitionMainSessionRecovery } from "../agents/main-session-recovery-state.js";
import {
applySessionEntryReplacements,
listSessionEntries,
} from "../config/sessions/session-accessor.js";
type MainSessionRecoveryDoctorParams = {
agentId: string;
storePath: string;
warnings: string[];
changes: string[];
confirmRepair: (params: { message: string; initialValue?: boolean }) => Promise<boolean>;
countLabel: (count: number, singular: string, plural?: string) => string;
};
export async function noteMainSessionRecoveryIntegrity(
params: MainSessionRecoveryDoctorParams,
): Promise<number> {
const entries = listSessionEntries({ agentId: params.agentId, storePath: params.storePath });
const wedged = entries.flatMap(({ entry, sessionKey }) => {
const health = inspectMainSessionRecoveryHealth(entry);
return health.status === "tombstoned" ? [{ key: sessionKey, health }] : [];
});
if (wedged.length === 0) {
return entries.length;
}
const wedgedCount = params.countLabel(wedged.length, "wedged main session");
params.warnings.push(
[
`- Found ${wedgedCount} with automatic restart recovery tombstoned.`,
" OpenClaw will not auto-resume these sessions again; inspect the failed turn, then use /new or reset to replace the session.",
` Examples: ${wedged
.slice(0, 3)
.map(({ key }) => key)
.join(", ")}`,
].join("\n"),
);
const visibleReasons = uniqueStrings(wedged.map(({ health }) => health.reason)).slice(0, 2);
if (visibleReasons.length > 0) {
params.warnings.push(visibleReasons.map((reason) => ` Reason: ${reason}`).join("\n"));
}
const staleAborted = wedged.filter(({ health }) => health.repair === "clear_stale_abort");
if (staleAborted.length === 0) {
return entries.length;
}
const staleCount = params.countLabel(staleAborted.length, "wedged main session");
if (
!(await params.confirmRepair({
message: `Clear stale aborted recovery flags for ${staleCount}?`,
initialValue: true,
}))
) {
return entries.length;
}
const repairedAt = Date.now();
// Revalidate under the writer lock because session state can change while Doctor prompts.
const repaired = await applySessionEntryReplacements<number>({
sessionKeys: staleAborted.map(({ key }) => key),
storePath: params.storePath,
update: (currentEntries) => {
const replacements = currentEntries.flatMap(({ sessionKey, entry }) => {
const transition = transitionMainSessionRecovery(entry, {
kind: "doctor_repair",
now: repairedAt,
});
return transition.kind === "doctor_repaired" ? [{ sessionKey, entry }] : [];
});
return { replacements, result: replacements.length };
},
});
if (repaired > 0) {
params.changes.push(
`- Cleared aborted restart-recovery flags for ${params.countLabel(repaired, "wedged main session")}.`,
);
}
return entries.length;
}
@@ -0,0 +1,11 @@
import path from "node:path";
export function countLabel(count: number, singular: string, plural = `${singular}s`): string {
return `${count} ${count === 1 ? singular : plural}`;
}
export function formatFilePreview(paths: string[], limit = 3): string {
const names = paths.slice(0, limit).map((filePath) => path.basename(filePath));
const remaining = paths.length - names.length;
return remaining > 0 ? `${names.join(", ")}, and ${remaining} more` : names.join(", ");
}
+11 -14
View File
@@ -39,7 +39,9 @@ import { parseAgentSessionKey } from "../sessions/session-key-utils.js";
import { shortenHomePath } from "../utils.js";
import { repairHeartbeatPoisonedMainSession } from "./doctor-heartbeat-main-session-repair.js";
import { describeHeartbeatSessionTargetIssues } from "./doctor-heartbeat-session-target.js";
import { noteMainSessionRecoveryIntegrity } from "./doctor-main-session-recovery.js";
import { runPluginSessionStateDoctorRepairs } from "./doctor-session-state-providers.js";
import { countLabel, formatFilePreview } from "./doctor-state-integrity-format.js";
const STATE_INTEGRITY_CHECK_ID = "core/doctor/state-integrity";
@@ -52,19 +54,6 @@ type DoctorPrompterLike = {
note?: typeof note;
};
function countLabel(count: number, singular: string, plural = `${singular}s`): string {
return `${count} ${count === 1 ? singular : plural}`;
}
function formatFilePreview(paths: string[], limit = 3): string {
const names = paths.slice(0, limit).map((filePath) => path.basename(filePath));
const remaining = paths.length - names.length;
if (remaining > 0) {
return `${names.join(", ")}, and ${remaining} more`;
}
return names.join(", ");
}
function existsDir(dir: string): boolean {
try {
return fs.existsSync(dir) && fs.statSync(dir).isDirectory();
@@ -1276,7 +1265,15 @@ export async function noteStateIntegrity(
const store = loadSessionStore(storePath, { skipCache: true, clone: false });
const sessionPathOpts = resolveSessionFilePathOptions({ agentId, storePath });
const entries = Object.entries(store).filter(([, entry]) => entry && typeof entry === "object");
if (entries.length > 0) {
const canonicalEntryCount = await noteMainSessionRecoveryIntegrity({
agentId,
storePath: absoluteStorePath,
warnings,
changes,
confirmRepair: (params) => prompter.confirmRuntimeRepair(params),
countLabel,
});
if (entries.length > 0 || canonicalEntryCount > 0) {
const recent = entries
.slice()
.toSorted((a, b) => {
+6 -4
View File
@@ -150,17 +150,19 @@ export function createExpiringMapCache<TKey, TValue>(options: {
}
type FileStatSnapshot = {
mtimeMs: number;
ctimeNs: bigint;
mtimeNs: bigint;
sizeBytes: number;
};
/** Captures the file attributes used by cache invalidation without exposing fs.Stats. */
export function getFileStatSnapshot(filePath: string): FileStatSnapshot | undefined {
try {
const stats = fs.statSync(filePath);
const stats = fs.statSync(filePath, { bigint: true });
return {
mtimeMs: stats.mtimeMs,
sizeBytes: stats.size,
ctimeNs: stats.ctimeNs,
mtimeNs: stats.mtimeNs,
sizeBytes: Number(stats.size),
};
} catch {
return undefined;
+1
View File
@@ -6,6 +6,7 @@ export * from "./sessions/goals.js";
export * from "./sessions/artifacts.js";
export * from "./sessions/metadata.js";
export * from "./sessions/main-session.js";
export type { MainRestartRecoveryState } from "./sessions/main-session-recovery.types.js";
export * from "./sessions/main-session.runtime.js";
export * from "./sessions/lifecycle.js";
export * from "./sessions/paths.js";
@@ -0,0 +1,20 @@
export type MainRestartRecoveryState = {
/** Stable identity for one interrupted episode; prevents clear-and-rewedge ABA matches. */
cycleId: string;
/** Monotonic identity for observations within the current recovery cycle. */
revision: number;
/** Attempts charged when their reservation is persisted, before dispatch. */
chargedAttempts: number;
reservation?: {
runId: string;
attempt: number;
lifecycleGeneration: string;
};
foregroundClaims?: {
lifecycleGeneration: string;
tokens: string[];
/** Run identity for claims that have crossed the actual agent-run boundary. */
runIdsByClaimId?: Record<string, string>;
};
tombstone?: { reason: string };
};
@@ -470,15 +470,17 @@ export function buildRestartRecoveryClaimCleanupPatch(params: {
entry: SessionEntry;
recordTerminalSource: boolean;
terminalDeliveryEvidence?: RestartRecoveryTerminalDeliveryEvidenceResult;
terminalRunId?: string;
terminalSourceRunId?: string;
}): Partial<SessionEntry> {
const sourceRunId =
normalizeRunId(params.terminalSourceRunId) ??
normalizeRunId(params.entry.restartRecoveryDeliverySourceRunId);
const terminalRunIds =
params.recordTerminalSource && sourceRunId
params.recordTerminalSource && (sourceRunId || params.terminalRunId)
? mergeRestartRecoveryTerminalRunIds(params.entry.restartRecoveryTerminalRunIds, [
sourceRunId,
...(sourceRunId ? [sourceRunId] : []),
...(params.terminalRunId ? [params.terminalRunId] : []),
])
: undefined;
const terminalDeliveryEvidence =
@@ -4,7 +4,7 @@ import {
projectSessionSnapshotChanges,
sessionSnapshotTouchedFieldsConflict,
} from "./session-snapshot-merge.js";
import { mergeSessionEntry, type SessionEntry } from "./types.js";
import { mergeSessionEntry, type InternalSessionEntry as SessionEntry } from "./types.js";
const initial: SessionEntry = {
sessionId: "session-1",
@@ -13,7 +13,7 @@ const initial: SessionEntry = {
model: "claude-opus-4-6",
};
describe("session snapshot model merge", () => {
describe("session snapshot merge", () => {
it("projects same-provider model changes as an atomic pair", () => {
const next = { ...initial, model: "claude-sonnet-4-6", updatedAt: 2 };
const patch = projectSessionSnapshotChanges({ initial, next, current: initial });
@@ -340,4 +340,274 @@ describe("session snapshot model merge", () => {
expect(mergeSessionSnapshotChanges({ initial, next, current })).toEqual(current);
});
it("preserves the recovery aggregate when a restart marker wins a stale healthy clear", () => {
const initialRecovery: SessionEntry = {
...initial,
abortedLastRun: true,
restartRecoveryRuns: [{ runId: "run-1", lifecycleGeneration: "generation-1" }],
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 2,
chargedAttempts: 0,
foregroundClaims: {
lifecycleGeneration: "generation-1",
tokens: ["owner-1"],
},
},
};
const next: SessionEntry = {
...initialRecovery,
updatedAt: 2,
abortedLastRun: false,
restartRecoveryRuns: undefined,
mainRestartRecovery: undefined,
};
const current: SessionEntry = {
...initialRecovery,
updatedAt: 3,
restartRecoveryRuns: [
...(initialRecovery.restartRecoveryRuns ?? []),
{ runId: "run-1", lifecycleGeneration: "generation-2" },
],
};
const merged = mergeSessionSnapshotChanges({ initial: initialRecovery, next, current });
expect(merged.abortedLastRun).toBe(true);
expect(merged.restartRecoveryRuns).toEqual(current.restartRecoveryRuns);
expect(merged.mainRestartRecovery).toEqual(current.mainRestartRecovery);
});
it("marks a claimed recovery healthy without erasing its owner aggregate", () => {
const initialRecovery: SessionEntry = {
...initial,
abortedLastRun: true,
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 1,
chargedAttempts: 1,
},
};
const next: SessionEntry = {
...initialRecovery,
updatedAt: 2,
abortedLastRun: false,
mainRestartRecovery: undefined,
};
const current: SessionEntry = {
...initialRecovery,
updatedAt: 3,
mainRestartRecovery: {
...initialRecovery.mainRestartRecovery!,
revision: 2,
foregroundClaims: {
lifecycleGeneration: "generation-1",
tokens: ["owner-1"],
},
},
};
const merged = mergeSessionSnapshotChanges({ initial: initialRecovery, next, current });
expect(merged.abortedLastRun).toBe(false);
expect(merged.restartRecoveryRuns).toBeUndefined();
expect(merged.mainRestartRecovery).toEqual(current.mainRestartRecovery);
});
it("clears recovery after lifecycle settlement consumes its run fence", () => {
const initialRecovery: SessionEntry = {
...initial,
abortedLastRun: true,
restartRecoveryRuns: [
{ runId: "interrupted-run", lifecycleGeneration: "generation-1" },
{ runId: "recovery-run", lifecycleGeneration: "generation-1" },
],
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 3,
chargedAttempts: 1,
},
};
const next: SessionEntry = {
...initialRecovery,
updatedAt: 2,
abortedLastRun: false,
restartRecoveryRuns: undefined,
mainRestartRecovery: undefined,
};
const current: SessionEntry = {
...structuredClone(initialRecovery),
updatedAt: 3,
abortedLastRun: false,
restartRecoveryRuns: [{ runId: "interrupted-run", lifecycleGeneration: "generation-1" }],
};
const merged = mergeSessionSnapshotChanges({ initial: initialRecovery, next, current });
expect(merged.abortedLastRun).toBe(false);
expect(merged.restartRecoveryRuns).toBeUndefined();
expect(merged.mainRestartRecovery).toBeUndefined();
});
it("preserves every concurrent owner while marking a recovered session healthy", () => {
const initialRecovery: SessionEntry = {
...initial,
abortedLastRun: true,
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 1,
chargedAttempts: 1,
},
};
const next: SessionEntry = {
...initialRecovery,
updatedAt: 2,
abortedLastRun: false,
mainRestartRecovery: undefined,
};
const current: SessionEntry = {
...initialRecovery,
updatedAt: 3,
mainRestartRecovery: {
...initialRecovery.mainRestartRecovery!,
revision: 3,
foregroundClaims: {
lifecycleGeneration: "generation-1",
tokens: ["owner-1", "owner-2"],
},
},
};
const merged = mergeSessionSnapshotChanges({ initial: initialRecovery, next, current });
expect(merged.abortedLastRun).toBe(false);
expect(merged.mainRestartRecovery?.foregroundClaims?.tokens).toEqual(["owner-1", "owner-2"]);
});
it("reasserts a terminal abort after another owner marks the session healthy", () => {
const initialRecovery: SessionEntry = {
...initial,
abortedLastRun: true,
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 3,
chargedAttempts: 1,
foregroundClaims: {
lifecycleGeneration: "generation-1",
tokens: ["owner-1", "owner-2"],
},
},
};
const next: SessionEntry = {
...structuredClone(initialRecovery),
updatedAt: 2,
};
const current: SessionEntry = {
...structuredClone(initialRecovery),
updatedAt: 3,
abortedLastRun: false,
mainRestartRecovery: {
...initialRecovery.mainRestartRecovery!,
revision: 4,
foregroundClaims: {
lifecycleGeneration: "generation-1",
tokens: ["owner-2"],
},
},
};
const merged = mergeSessionSnapshotChanges({
initial: initialRecovery,
next,
current,
reassertAbortedLastRun: true,
});
expect(merged.abortedLastRun).toBe(true);
expect(merged.mainRestartRecovery).toEqual(current.mainRestartRecovery);
});
it("does not attach a stale terminal abort to a replacement recovery cycle", () => {
const initialRecovery: SessionEntry = {
...initial,
abortedLastRun: true,
mainRestartRecovery: {
cycleId: "cycle-old",
revision: 2,
chargedAttempts: 1,
foregroundClaims: {
lifecycleGeneration: "generation-old",
tokens: ["old-owner"],
},
},
};
const current: SessionEntry = {
...structuredClone(initialRecovery),
updatedAt: 3,
abortedLastRun: false,
mainRestartRecovery: {
cycleId: "cycle-new",
revision: 1,
chargedAttempts: 0,
foregroundClaims: {
lifecycleGeneration: "generation-new",
tokens: ["new-owner"],
},
},
};
const merged = mergeSessionSnapshotChanges({
initial: initialRecovery,
next: { ...structuredClone(initialRecovery), updatedAt: 2 },
current,
reassertAbortedLastRun: true,
});
expect(merged.abortedLastRun).toBe(false);
expect(merged.mainRestartRecovery).toEqual(current.mainRestartRecovery);
});
it("preserves a concurrent foreground claim over a partial recovery update", () => {
const initialRecovery: SessionEntry = {
...initial,
abortedLastRun: true,
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 1,
chargedAttempts: 0,
},
};
const next: SessionEntry = {
...initialRecovery,
updatedAt: 2,
mainRestartRecovery: {
...initialRecovery.mainRestartRecovery!,
revision: 2,
chargedAttempts: 1,
reservation: {
runId: "recovery-1",
attempt: 1,
lifecycleGeneration: "generation-1",
},
},
};
const current: SessionEntry = {
...initialRecovery,
updatedAt: 3,
mainRestartRecovery: {
...initialRecovery.mainRestartRecovery!,
revision: 2,
foregroundClaims: {
lifecycleGeneration: "generation-1",
tokens: ["owner-1"],
},
},
};
const merged = mergeSessionSnapshotChanges({ initial: initialRecovery, next, current });
expect(merged.mainRestartRecovery).toEqual(current.mainRestartRecovery);
expect(merged.mainRestartRecovery?.reservation).toBeUndefined();
});
});
+116 -1
View File
@@ -1,5 +1,5 @@
import { isDeepStrictEqual } from "node:util";
import type { SessionEntry } from "./types.js";
import type { InternalSessionEntry as SessionEntry } from "./types.js";
type SessionEntryRecord = Partial<Record<keyof SessionEntry, unknown>>;
@@ -42,6 +42,12 @@ const MODEL_OVERRIDE_CONFLICT_DEPENDENT_FIELDS = ["thinkingLevel"] as const sati
keyof SessionEntry
>;
const MAIN_SESSION_RECOVERY_TRANSACTION_FIELDS = [
"abortedLastRun",
"restartRecoveryRuns",
"mainRestartRecovery",
] as const satisfies ReadonlyArray<keyof SessionEntry>;
function anySessionFieldChanged(
before: SessionEntryRecord,
after: SessionEntryRecord,
@@ -50,11 +56,69 @@ function anySessionFieldChanged(
return fields.some((field) => !isDeepStrictEqual(before[field], after[field]));
}
function mainSessionRecoveryTransactionChanged(before: SessionEntry, after: SessionEntry): boolean {
const beforeState = before.mainRestartRecovery;
const afterState = after.mainRestartRecovery;
return (
before.abortedLastRun !== after.abortedLastRun ||
!isDeepStrictEqual(before.restartRecoveryRuns, after.restartRecoveryRuns) ||
beforeState?.cycleId !== afterState?.cycleId ||
beforeState?.chargedAttempts !== afterState?.chargedAttempts ||
!isDeepStrictEqual(beforeState?.reservation, afterState?.reservation) ||
!isDeepStrictEqual(beforeState?.tombstone, afterState?.tombstone)
);
}
function mainSessionRecoveryOwnershipChanged(before: SessionEntry, after: SessionEntry): boolean {
return (
before.mainRestartRecovery?.revision !== after.mainRestartRecovery?.revision ||
!isDeepStrictEqual(
before.mainRestartRecovery?.foregroundClaims,
after.mainRestartRecovery?.foregroundClaims,
)
);
}
function mainSessionRecoveryCycleStateUnchanged(
before: SessionEntry,
after: SessionEntry,
): boolean {
const beforeState = before.mainRestartRecovery;
const afterState = after.mainRestartRecovery;
if (!beforeState || !afterState) {
return false;
}
return (
beforeState.cycleId === afterState.cycleId &&
beforeState.chargedAttempts === afterState.chargedAttempts &&
isDeepStrictEqual(beforeState.reservation, afterState.reservation) &&
isDeepStrictEqual(beforeState.tombstone, afterState.tombstone)
);
}
function restartRecoveryRunsOnlyConsumed(before: SessionEntry, after: SessionEntry): boolean {
const initialRuns = new Set(
(before.restartRecoveryRuns ?? []).map((run) => `${run.runId}\u0000${run.lifecycleGeneration}`),
);
return (after.restartRecoveryRuns ?? []).every((run) =>
initialRuns.has(`${run.runId}\u0000${run.lifecycleGeneration}`),
);
}
function isCanonicalMainSessionRecoveryClear(entry: SessionEntry): boolean {
return (
entry.abortedLastRun === false &&
entry.restartRecoveryRuns === undefined &&
entry.mainRestartRecovery === undefined
);
}
/** Projects run-local snapshot changes without restoring concurrently changed fields. */
export function projectSessionSnapshotChanges(params: {
initial: SessionEntry;
next: SessionEntry;
current: SessionEntry;
reassertAbortedLastRun?: boolean;
reassertLiveModelSwitchPending?: boolean;
}): Partial<SessionEntry> {
if (params.current.sessionId !== params.initial.sessionId) {
@@ -131,10 +195,60 @@ export function projectSessionSnapshotChanges(params: {
patch.modelProvider = params.next.modelProvider;
}
const mainRecoveryChanged = mainSessionRecoveryTransactionChanged(params.initial, params.next);
const mainRecoveryChangedConcurrently = mainSessionRecoveryTransactionChanged(
params.initial,
params.current,
);
const mainRecoveryOwnershipChangedConcurrently = mainSessionRecoveryOwnershipChanged(
params.initial,
params.current,
);
const initialForegroundClaims = params.initial.mainRestartRecovery?.foregroundClaims;
const currentForegroundClaims = params.current.mainRestartRecovery?.foregroundClaims;
if (
params.reassertAbortedLastRun &&
params.next.abortedLastRun === true &&
initialForegroundClaims &&
currentForegroundClaims &&
params.initial.mainRestartRecovery?.cycleId === params.current.mainRestartRecovery?.cycleId &&
initialForegroundClaims.lifecycleGeneration === currentForegroundClaims.lifecycleGeneration
) {
// A terminal abort is authoritative even when true matches this owner's
// initial snapshot. Preserve the concurrently narrowed owner aggregate.
patch.abortedLastRun = true;
}
const currentOnlyConsumedLifecycleFences =
isCanonicalMainSessionRecoveryClear(params.next) &&
!mainRecoveryOwnershipChangedConcurrently &&
mainSessionRecoveryCycleStateUnchanged(params.initial, params.current) &&
restartRecoveryRunsOnlyConsumed(params.initial, params.current);
if (
mainRecoveryChanged &&
(!mainRecoveryChangedConcurrently || currentOnlyConsumedLifecycleFences)
) {
// Apply all three fields together: a stale healthy flag can otherwise hide a newer marker.
// A healthy run first marks its claim non-interrupted; token-scoped release
// removes the aggregate only after the final concurrent owner exits.
if (
mainRecoveryOwnershipChangedConcurrently &&
isCanonicalMainSessionRecoveryClear(params.next)
) {
patch.abortedLastRun = false;
} else if (!mainRecoveryOwnershipChangedConcurrently) {
for (const field of MAIN_SESSION_RECOVERY_TRANSACTION_FIELDS) {
patchRecord[field] = Object.hasOwn(params.next, field) ? next[field] : undefined;
}
}
}
for (const field of fields) {
if (
field === "model" ||
field === "modelProvider" ||
MAIN_SESSION_RECOVERY_TRANSACTION_FIELDS.includes(
field as (typeof MAIN_SESSION_RECOVERY_TRANSACTION_FIELDS)[number],
) ||
SESSION_MODEL_OVERRIDE_TRANSACTION_FIELDS.includes(
field as (typeof SESSION_MODEL_OVERRIDE_TRANSACTION_FIELDS)[number],
)
@@ -274,6 +388,7 @@ export function mergeSessionSnapshotChanges(params: {
initial: SessionEntry;
next: SessionEntry;
current: SessionEntry;
reassertAbortedLastRun?: boolean;
reassertLiveModelSwitchPending?: boolean;
}): SessionEntry {
const merged = { ...params.current };
@@ -1,11 +1,14 @@
import type { SessionRestartRecoveryState } from "./restart-recovery-types.js";
import type { SessionEntry } from "./types.js";
import type { InternalSessionEntry as SessionEntry } from "./types.js";
type SessionRunStatus = "running" | "done" | "failed" | "killed" | "timeout";
/** Authoritative lifecycle snapshot required for an atomic transcript admission. */
export type SessionTranscriptTurnExpectedState = {
abortedLastRun: boolean | undefined;
/** Fences recovery-only transcript writes against concurrent ownership changes. */
mainRestartRecoveryCycleId?: string;
mainRestartRecoveryRevision?: number;
restartRecoveryBeforeAgentReplyState: SessionRestartRecoveryState["restartRecoveryBeforeAgentReplyState"];
restartRecoveryDeliveryReceiptState: SessionRestartRecoveryState["restartRecoveryDeliveryReceiptState"];
restartRecoveryDeliveryToolCallId: SessionRestartRecoveryState["restartRecoveryDeliveryToolCallId"];
@@ -34,6 +37,7 @@ export type SessionTranscriptTurnLifecyclePatch = {
pendingFinalDeliveryLastAttemptAt?: SessionEntry["pendingFinalDeliveryLastAttemptAt"];
pendingFinalDeliveryLastError?: SessionEntry["pendingFinalDeliveryLastError"];
pendingFinalDeliveryText?: SessionEntry["pendingFinalDeliveryText"];
mainRestartRecovery?: SessionEntry["mainRestartRecovery"];
restartRecoveryBeforeAgentReplyState?: SessionRestartRecoveryState["restartRecoveryBeforeAgentReplyState"];
restartRecoveryDeliveryReceiptState?: SessionRestartRecoveryState["restartRecoveryDeliveryReceiptState"];
restartRecoveryDeliveryToolCallId?: SessionRestartRecoveryState["restartRecoveryDeliveryToolCallId"];
@@ -6,7 +6,7 @@ import type {
SessionTranscriptTurnExpectedState,
SessionTranscriptTurnLifecyclePatch,
} from "./session-transcript-turn-lifecycle.types.js";
import type { SessionEntry } from "./types.js";
import type { InternalSessionEntry as SessionEntry } from "./types.js";
export function sessionMatchesExpectedTranscriptTurn<T extends { entry: SessionEntry }>(
selected: T | undefined,
@@ -24,6 +24,12 @@ export function sessionMatchesExpectedTranscriptTurn<T extends { entry: SessionE
selected.entry.lifecycleRevision === expected.expectedLifecycleRevision) &&
(expectedState === undefined ||
(selected.entry.abortedLastRun === expectedState.abortedLastRun &&
(expectedState.mainRestartRecoveryCycleId === undefined ||
selected.entry.mainRestartRecovery?.cycleId ===
expectedState.mainRestartRecoveryCycleId) &&
(expectedState.mainRestartRecoveryRevision === undefined ||
selected.entry.mainRestartRecovery?.revision ===
expectedState.mainRestartRecoveryRevision) &&
selected.entry.restartRecoveryBeforeAgentReplyState ===
expectedState.restartRecoveryBeforeAgentReplyState &&
selected.entry.restartRecoveryDeliveryReceiptState ===
+35 -2
View File
@@ -894,7 +894,8 @@ describe("session store writer queue", () => {
writeSessionStoreCache({
storePath,
store,
mtimeMs: 1,
ctimeNs: 1n,
mtimeNs: 1n,
sizeBytes: serialized.length,
serialized,
cloneSerialized: serialized,
@@ -904,13 +905,45 @@ describe("session store writer queue", () => {
const cached = readSessionStoreCache({
storePath,
mtimeMs: 1,
ctimeNs: 1n,
mtimeNs: 1n,
sizeBytes: serialized.length,
});
expect(cached?.[key]?.sessionId).toBe("s-serialized-cache");
});
it("invalidates session store cache when ctime nanoseconds change inside the same millisecond", () => {
const key = "agent:main:ctime-ns-cache";
const storePath = "/tmp/openclaw-ctime-ns-cache-test.json";
const store = {
[key]: {
sessionId: "s-ctime-ns-cache",
updatedAt: Date.now(),
},
} satisfies Record<string, SessionEntry>;
const serialized = JSON.stringify(store);
writeSessionStoreCache({
storePath,
store,
ctimeNs: 1_000_000n,
mtimeNs: 1_000_000n,
sizeBytes: serialized.length,
serialized,
cloneSerialized: serialized,
takeOwnership: true,
});
const cached = readSessionStoreCache({
storePath,
ctimeNs: 1_000_001n,
mtimeNs: 1_000_000n,
sizeBytes: serialized.length,
});
expect(cached).toBeNull();
});
it("returns an owned parsed store for fresh skip-cache loads without cloning again", async () => {
const key = "agent:main:owned-skip-cache";
const { storePath } = await makeTmpStore({
+20 -7
View File
@@ -17,7 +17,8 @@ export type SessionStoreSnapshotEntry = DeepReadonly<SessionEntry>;
type SessionStoreCacheEntry = {
store: Record<string, SessionEntry>;
mtimeMs?: number;
ctimeNs?: bigint;
mtimeNs?: bigint;
sizeBytes?: number;
serialized?: string;
};
@@ -298,7 +299,8 @@ export function dropSessionStoreObjectCache(storePath: string): void {
export function readSessionStoreCache(params: {
storePath: string;
mtimeMs?: number;
ctimeNs?: bigint;
mtimeNs?: bigint;
sizeBytes?: number;
clone?: boolean;
}): Record<string, SessionEntry> | null {
@@ -306,7 +308,11 @@ export function readSessionStoreCache(params: {
if (!cached) {
return null;
}
if (params.mtimeMs !== cached.mtimeMs || params.sizeBytes !== cached.sizeBytes) {
if (
params.ctimeNs !== cached.ctimeNs ||
params.mtimeNs !== cached.mtimeNs ||
params.sizeBytes !== cached.sizeBytes
) {
invalidateSessionStoreCache(params.storePath);
return null;
}
@@ -318,14 +324,19 @@ export function readSessionStoreCache(params: {
export function takeMutableSessionStoreCache(params: {
storePath: string;
mtimeMs?: number;
ctimeNs?: bigint;
mtimeNs?: bigint;
sizeBytes?: number;
}): Record<string, SessionEntry> | null {
const cached = SESSION_STORE_CACHE.get(params.storePath);
if (!cached) {
return null;
}
if (params.mtimeMs !== cached.mtimeMs || params.sizeBytes !== cached.sizeBytes) {
if (
params.ctimeNs !== cached.ctimeNs ||
params.mtimeNs !== cached.mtimeNs ||
params.sizeBytes !== cached.sizeBytes
) {
invalidateSessionStoreCache(params.storePath);
return null;
}
@@ -336,7 +347,8 @@ export function takeMutableSessionStoreCache(params: {
export function writeSessionStoreCache(params: {
storePath: string;
store: Record<string, SessionEntry>;
mtimeMs?: number;
ctimeNs?: bigint;
mtimeNs?: bigint;
sizeBytes?: number;
serialized?: string;
serializedPromptRefs?: ReadonlyMap<string, SessionSkillPromptRef>;
@@ -352,7 +364,8 @@ export function writeSessionStoreCache(params: {
}
SESSION_STORE_CACHE.set(params.storePath, {
store,
mtimeMs: params.mtimeMs,
ctimeNs: params.ctimeNs,
mtimeNs: params.mtimeNs,
sizeBytes: params.sizeBytes,
serialized: params.cloneSerialized,
});
+2 -5
View File
@@ -396,8 +396,7 @@ export function loadSessionStore(
const currentFileStat = getFileStatSnapshot(storePath);
const cached = readSessionStoreCache({
storePath,
mtimeMs: currentFileStat?.mtimeMs,
sizeBytes: currentFileStat?.sizeBytes,
...currentFileStat,
clone: opts.clone,
});
if (cached) {
@@ -409,7 +408,6 @@ export function loadSessionStore(
// transiently invalid content while another process is swapping the file.
let store: Record<string, SessionEntry> = {};
const fileStat = getFileStatSnapshot(storePath);
const mtimeMs = fileStat?.mtimeMs;
let serializedFromDisk: string | undefined;
const maxReadAttempts = 3;
const retryBuf = maxReadAttempts > 1 ? new Int32Array(new SharedArrayBuffer(4)) : undefined;
@@ -515,8 +513,7 @@ export function loadSessionStore(
writeSessionStoreCache({
storePath,
store,
mtimeMs,
sizeBytes: fileStat?.sizeBytes,
...fileStat,
serialized: serializedFromDisk,
cloneSerialized: serializedFromDisk,
takeOwnership: serializedFromDisk !== undefined,
@@ -9,7 +9,7 @@ import {
recordInboundSessionMeta,
updateSessionLastRoute,
} from "./session-accessor.js";
import type { SessionEntry } from "./types.js";
import type { InternalSessionEntry as SessionEntry } from "./types.js";
// Materializes the SQLite-backed session store as a keyed object so key
// normalization/migration assertions keep matching the old JSON-store shape.
@@ -60,6 +60,50 @@ function createSignalGroupContext(): MsgContext {
};
}
function createRecoveryEntry(sessionId: string): SessionEntry {
return {
sessionId,
updatedAt: 1,
abortedLastRun: true,
restartRecoveryRuns: [
{ runId: "initial-wedged-run", lifecycleGeneration: "gen-1" },
{ runId: "recovery-run-1", lifecycleGeneration: "gen-2" },
],
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 3,
chargedAttempts: 2,
},
subagentRecovery: {
automaticAttempts: 2,
lastAttemptAt: 3,
wedgedAt: 4,
wedgedReason: "automatic_attempt_budget_exceeded",
},
};
}
function expectRecoveryMarkers(entry: SessionEntry | undefined): void {
expect(entry).toMatchObject({
abortedLastRun: true,
restartRecoveryRuns: [
{ runId: "initial-wedged-run", lifecycleGeneration: "gen-1" },
{ runId: "recovery-run-1", lifecycleGeneration: "gen-2" },
],
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 3,
chargedAttempts: 2,
},
subagentRecovery: {
automaticAttempts: 2,
lastAttemptAt: 3,
wedgedAt: 4,
wedgedReason: "automatic_attempt_budget_exceeded",
},
});
}
describe("session store key normalization", () => {
const suiteRootTracker = createSuiteTempRootTracker({
prefix: "openclaw-session-key-normalize-",
@@ -203,6 +247,39 @@ describe("session store key normalization", () => {
expect(store[CANONICAL_KEY]?.origin?.provider).toBe("webchat");
});
it("preserves recovery markers when recording inbound metadata", async () => {
await seedStore(storePath, {
[CANONICAL_KEY]: createRecoveryEntry("recovered-session"),
});
await recordInboundSessionMeta({
storePath,
sessionKey: MIXED_CASE_KEY,
ctx: createInboundContext(),
});
const store = loadStore(storePath);
expectRecoveryMarkers(store[CANONICAL_KEY]);
expect(store[CANONICAL_KEY]?.origin?.provider).toBe("webchat");
});
it("preserves recovery markers when updating the last route", async () => {
await seedStore(storePath, {
[CANONICAL_KEY]: createRecoveryEntry("route-session"),
});
await updateSessionLastRoute({
storePath,
sessionKey: CANONICAL_KEY,
channel: "webchat",
to: "webchat:user-1",
});
const store = loadStore(storePath);
expectRecoveryMarkers(store[CANONICAL_KEY]);
expect(store[CANONICAL_KEY]?.lastTo).toBe("webchat:user-1");
});
it("records Signal group metadata under the mixed-case opaque group id", async () => {
await recordInboundSessionMeta({
storePath,
+5 -7
View File
@@ -367,8 +367,7 @@ function updateSessionStoreWriteCaches(params: {
writeSessionStoreCache({
storePath: params.storePath,
store: params.store,
mtimeMs: fileStat?.mtimeMs,
sizeBytes: fileStat?.sizeBytes,
...fileStat,
serialized: params.serialized,
serializedPromptRefs: params.serializedPromptRefs,
cloneSerialized: params.cloneSerialized,
@@ -386,7 +385,8 @@ function restoreUnchangedSessionStoreCache(
const loadedFileStat = writerStoreFileStats.get(store) ?? null;
const currentFileStat = getFileStatSnapshot(storePath) ?? null;
if (
loadedFileStat?.mtimeMs !== currentFileStat?.mtimeMs ||
loadedFileStat?.ctimeNs !== currentFileStat?.ctimeNs ||
loadedFileStat?.mtimeNs !== currentFileStat?.mtimeNs ||
loadedFileStat?.sizeBytes !== currentFileStat?.sizeBytes
) {
invalidateSessionStoreCache(storePath);
@@ -398,8 +398,7 @@ function restoreUnchangedSessionStoreCache(
writeSessionStoreCache({
storePath,
store,
mtimeMs: loadedFileStat?.mtimeMs,
sizeBytes: loadedFileStat?.sizeBytes,
...loadedFileStat,
serialized,
serializedPromptRefs,
takeOwnership: true,
@@ -598,8 +597,7 @@ function loadMutableSessionStoreForWriter(storePath: string): Record<string, Ses
if (isSessionStoreCacheEnabled()) {
const cached = takeMutableSessionStoreCache({
storePath,
mtimeMs: currentFileStat?.mtimeMs,
sizeBytes: currentFileStat?.sizeBytes,
...currentFileStat,
});
if (cached) {
writerStoreFileStats.set(cached, currentFileStat ?? null);
+9
View File
@@ -31,6 +31,7 @@ import {
type SessionTranscriptTurnWriteContext,
type SessionTranscriptTurnExpectedState,
} from "./session-accessor.js";
import type { SessionTranscriptTurnLifecyclePatch } from "./session-transcript-turn-lifecycle.types.js";
import { parseSqliteSessionFileMarker, type SqliteSessionFileMarker } from "./sqlite-marker.js";
import { resolveSessionStoreEntry } from "./store.js";
import {
@@ -435,6 +436,7 @@ export async function appendAssistantMessageToSessionTranscript(params: {
expectedSessionId?: string;
expectedLifecycleRevision?: string;
expectedSessionState?: SessionTranscriptTurnExpectedState;
sessionLifecyclePatch?: SessionTranscriptTurnLifecyclePatch;
text?: string;
mediaUrls?: string[];
idempotencyKey?: string;
@@ -466,6 +468,9 @@ export async function appendAssistantMessageToSessionTranscript(params: {
? { expectedLifecycleRevision: params.expectedLifecycleRevision }
: {}),
...(params.expectedSessionState ? { expectedSessionState: params.expectedSessionState } : {}),
...(params.sessionLifecyclePatch
? { sessionLifecyclePatch: params.sessionLifecyclePatch }
: {}),
storePath: params.storePath,
idempotencyKey: params.idempotencyKey,
updateMode: params.updateMode,
@@ -504,6 +509,7 @@ export async function appendExactAssistantMessageToSessionTranscript(params: {
expectedSessionId?: string;
expectedLifecycleRevision?: string;
expectedSessionState?: SessionTranscriptTurnExpectedState;
sessionLifecyclePatch?: SessionTranscriptTurnLifecyclePatch;
message: SessionTranscriptAssistantMessage;
idempotencyKey?: string;
storePath?: string;
@@ -602,6 +608,9 @@ export async function appendExactAssistantMessageToSessionTranscript(params: {
...(params.expectedSessionState
? { expectedSessionState: params.expectedSessionState }
: {}),
...(params.sessionLifecyclePatch
? { sessionLifecyclePatch: params.sessionLifecyclePatch }
: {}),
...(params.config ? { config: params.config } : {}),
updateMode: params.updateMode ?? "inline",
touchSessionEntry: true,
+6
View File
@@ -12,6 +12,7 @@ import type { ChannelRouteRef } from "../../plugin-sdk/channel-route.js";
import type { Skill } from "../../skills/loading/skill-contract.js";
import type { DeliveryContext } from "../../utils/delivery-context.types.js";
import type { TtsAutoMode } from "../types.tts.js";
import type { MainRestartRecoveryState } from "./main-session-recovery.types.js";
import type { SessionRestartRecoveryState } from "./restart-recovery-types.js";
import type { SessionEntryProvenance } from "./session-entry-provenance.js";
import { rewriteSessionFileForNewSessionId } from "./session-file-rotation.js";
@@ -494,6 +495,11 @@ export type SessionEntry = SessionRestartRecoveryState &
acp?: SessionAcpMeta;
};
/** Internal durable fields excluded from public/plugin session projections. */
export type InternalSessionEntry = SessionEntry & {
mainRestartRecovery?: MainRestartRecoveryState;
};
export function isTerminalSessionStatus(
status: unknown,
): status is Exclude<NonNullable<SessionEntry["status"]>, "running"> {
@@ -10,6 +10,7 @@ import { parseExecApprovalFollowupApprovalId } from "../../agents/bash-tools.exe
import { normalizeSpawnedRunMetadata } from "../../agents/spawned-context.js";
import { getAgentEventLifecycleGeneration } from "../../infra/agent-events.js";
import {
isMainSessionRestartRecoveryInputProvenance,
normalizeInputProvenance,
shouldPreserveUserFacingSessionStateForInputProvenance,
} from "../../sessions/input-provenance.js";
@@ -45,6 +46,7 @@ type AgentRequestPreflight = {
execApprovalFollowupApprovalId?: string;
normalizedSpawned: ReturnType<typeof normalizeSpawnedRunMetadata>;
inputProvenance: ReturnType<typeof normalizeInputProvenance>;
isRestartRecoveryResumeRun: boolean;
preserveUserFacingSessionModelState: boolean;
sessionEffects?: "visible" | "internal";
suppressVisibleSessionEffects: boolean;
@@ -215,6 +217,8 @@ export function prepareAgentRequestPreflight(
groupSpace: request.groupSpace,
}),
inputProvenance,
isRestartRecoveryResumeRun:
canUseInternalRuntimeHandoff && isMainSessionRestartRecoveryInputProvenance(inputProvenance),
preserveUserFacingSessionModelState:
canUseInternalRuntimeHandoff &&
shouldPreserveUserFacingSessionStateForInputProvenance(inputProvenance),
@@ -5,6 +5,10 @@ import {
isEmbeddedAgentRunAbortableForRunId,
retainEmbeddedAgentRunAbortabilityForRunId,
} from "../../agents/embedded-agent-runner/runs.js";
import {
commitMainSessionRecovery,
type MainSessionRecoveryPendingTarget,
} from "../../agents/main-session-recovery-store.js";
import { resolvePersistedOverrideModelRef } from "../../agents/model-selection.js";
import { resolveProviderIdForAuth } from "../../agents/provider-auth-aliases.js";
import { resolveEffectiveAgentRuntime } from "../../agents/thinking-runtime.js";
@@ -45,6 +49,9 @@ export type PreparedAgentRunDispatch = {
lifecycleStorePath: string;
resolvedThreadId?: string | number;
dispatchTaskTrackingMode: Exclude<GatewayAgentTaskTrackingMode, "plugin_subagent">;
restoreAdmittedRestartRecoveryInterrupted?: () => Promise<
MainSessionRecoveryPendingTarget | undefined
>;
};
export async function prepareAgentRunDispatch(params: {
@@ -73,6 +80,7 @@ export async function prepareAgentRunDispatch(params: {
pendingChatRun?: { sessionKey: string; agentId?: string };
inputProvenance?: InputProvenance;
isOneShotModelRun: boolean;
isRestartRecoveryResumeRun: boolean;
runId: string;
agentDedupeKeys: readonly string[];
context: GatewayRequestHandlerOptions["context"];
@@ -300,6 +308,75 @@ export async function prepareAgentRunDispatch(params: {
dispatchTaskTrackingMode = "cli";
}
}
let restoreAdmittedRestartRecoveryInterrupted:
| (() => Promise<MainSessionRecoveryPendingTarget | undefined>)
| undefined;
if (params.isRestartRecoveryResumeRun) {
const recoverySessionKey = params.resolvedSessionKey;
if (!recoverySessionKey) {
activeRunAbort.cleanup({ force: true });
activeGatewayWorkAdmission.release();
params.respond(
false,
undefined,
errorShape(ErrorCodes.UNAVAILABLE, "restart recovery session target is unavailable"),
);
return undefined;
}
try {
const recoveryAdmission = await commitMainSessionRecovery({
command: {
kind: "admit_recovery",
lifecycleGeneration: params.lifecycleGeneration,
now: Date.now(),
runId: params.runId,
sessionId: params.request.expectedExistingSessionId ?? params.getAdmittedSessionId(),
},
requireWriteSuccess: true,
target: { sessionKey: recoverySessionKey, storePath: lifecycleStorePath },
});
if (recoveryAdmission.transition.kind !== "admitted_recovery") {
throw new Error(
`Session "${recoverySessionKey}" restart recovery reservation is stale; recovery was skipped.`,
);
}
const admittedRecoverySessionKey = recoveryAdmission.sessionKey ?? recoverySessionKey;
let restored = false;
restoreAdmittedRestartRecoveryInterrupted = async () => {
if (restored) {
return undefined;
}
const recovery = await commitMainSessionRecovery({
command: {
kind: "mark_admitted_recovery_interrupted",
lifecycleGeneration: params.lifecycleGeneration,
now: Date.now(),
runId: params.runId,
sessionId: params.request.expectedExistingSessionId ?? params.getAdmittedSessionId(),
},
requireWriteSuccess: true,
target: { sessionKey: admittedRecoverySessionKey, storePath: lifecycleStorePath },
});
restored = true;
const expectedSessionId =
params.request.expectedExistingSessionId ?? params.getAdmittedSessionId();
return recovery.transition.kind === "applied" &&
recovery.entry?.sessionId === expectedSessionId &&
recovery.sessionKey
? {
sessionId: recovery.entry.sessionId,
sessionKey: recovery.sessionKey,
storePath: lifecycleStorePath,
}
: undefined;
};
} catch (err) {
activeRunAbort.cleanup({ force: true });
activeGatewayWorkAdmission.release();
params.respond(false, undefined, errorShape(ErrorCodes.UNAVAILABLE, formatForLog(err)));
return undefined;
}
}
const accepted = {
runId: params.runId,
sessionKey: params.resolvedSessionKey,
@@ -336,5 +413,6 @@ export async function prepareAgentRunDispatch(params: {
lifecycleStorePath,
resolvedThreadId,
dispatchTaskTrackingMode,
restoreAdmittedRestartRecoveryInterrupted,
};
}
@@ -3,12 +3,13 @@ import {
buildAgentRunTerminalOutcome,
type AgentRunTerminalOutcome,
} from "../../agents/agent-run-terminal-outcome.js";
import type { MainSessionRecoveryPendingTarget } from "../../agents/main-session-recovery-store.js";
import { isAgentRunRestartAbortReason } from "../../agents/run-termination.js";
import {
normalizeAgentRunTimeoutPhase,
normalizeProviderStarted,
} from "../../agents/run-timeout-attribution.js";
import { agentCommandFromIngress } from "../../commands/agent.js";
import { agentCommandFromGatewayIngress } from "../../commands/agent.js";
import { isAbortError } from "../../infra/abort-signal.js";
import { clearAgentRunContext } from "../../infra/agent-events.js";
import { readErrorName } from "../../infra/errors.js";
@@ -77,7 +78,7 @@ export function deleteGatewayDedupeEntries(params: {
}
export function dispatchAgentRunFromGateway(params: {
ingressOpts: Parameters<typeof agentCommandFromIngress>[0];
ingressOpts: Parameters<typeof agentCommandFromGatewayIngress>[0];
runId: string;
dedupeKeys: readonly string[];
/**
@@ -90,6 +91,7 @@ export function dispatchAgentRunFromGateway(params: {
respond: GatewayRequestHandlerOptions["respond"];
context: GatewayRequestHandlerOptions["context"];
taskTrackingMode: Exclude<GatewayAgentTaskTrackingMode, "plugin_subagent">;
restoreAdmittedRecovery?: () => Promise<MainSessionRecoveryPendingTarget | undefined>;
onSettled?: (outcome: {
terminalOutcome: AgentRunTerminalOutcome;
onRecovered?: () => void;
@@ -139,7 +141,9 @@ export function dispatchAgentRunFromGateway(params: {
return false;
}
};
void agentCommandFromIngress(params.ingressOpts, defaultRuntime, params.context.deps)
void agentCommandFromGatewayIngress(params.ingressOpts, defaultRuntime, params.context.deps, {
restoreAdmittedRecovery: params.restoreAdmittedRecovery,
})
.then(async (result) => {
const aborted = result?.meta?.aborted === true;
const timeoutAttribution = readAgentRunTimeoutAttribution(result?.meta);
@@ -2,6 +2,16 @@ import { ErrorCodes, errorShape } from "../../../packages/gateway-protocol/src/i
import type { AgentRunTerminalOutcome } from "../../agents/agent-run-terminal-outcome.js";
import { consumeExecApprovalFollowupRuntimeHandoff } from "../../agents/bash-tools.exec-approval-followup-state.js";
import { runAgentHarnessBeforeMessageWriteHook } from "../../agents/harness/hook-helpers.js";
import { scheduleMainSessionRecoveryPendingTarget } from "../../agents/main-session-recovery-owner-release.js";
import {
restoreAdmittedRecoveryWithRetries,
scheduleAdmittedRecoveryRestore,
} from "../../agents/main-session-recovery-restore.js";
import {
releaseMainSessionRecoveryOwner,
type MainSessionRecoveryPendingTarget,
type MainSessionRecoveryOwnerLease,
} from "../../agents/main-session-recovery-store.js";
import { resolveIngressWorkspaceOverrideForSessionRun } from "../../agents/spawned-context.js";
import {
setChannelSourceTurnId,
@@ -45,6 +55,7 @@ import type { GatewayRequestHandlerOptions } from "./types.js";
export function startAgentRunExecution(params: {
prepared: PreparedAgentRunDispatch;
mainRestartRecoveryOwnerLease?: MainSessionRecoveryOwnerLease;
request: AgentRunRequest;
cfg: OpenClawConfig;
cfgForAgent?: OpenClawConfig;
@@ -59,6 +70,7 @@ export function startAgentRunExecution(params: {
isNewSession: boolean;
isRawModelRun: boolean;
isOneShotModelRun: boolean;
isRestartRecoveryResumeRun: boolean;
suppressVisibleSessionEffects: boolean;
message: string;
images: Array<{ type: "image"; data: string; mimeType: string }>;
@@ -101,8 +113,10 @@ export function startAgentRunExecution(params: {
void prepared.activeGatewayWorkAdmission.run(async () => {
await yieldAfterAgentAcceptedAck();
let dispatched = false;
let pendingRecovery: MainSessionRecoveryPendingTarget | undefined;
try {
if (prepared.activeRunAbort.controller.signal.aborted) {
pendingRecovery = await prepared.restoreAdmittedRestartRecoveryInterrupted?.();
const stopReason = resolveAbortedAgentStopReason(prepared.activeRunAbort.entry);
setAbortedAgentDedupeEntries({
dedupe: params.context.dedupe,
@@ -382,6 +396,10 @@ export function startAgentRunExecution(params: {
sessionEntry: params.sessionEntry,
}),
allowGatewaySubagentBinding: true,
...(params.mainRestartRecoveryOwnerLease
? { mainRestartRecoveryOwnerLease: params.mainRestartRecoveryOwnerLease }
: {}),
...(params.isRestartRecoveryResumeRun ? { mainRestartRecoveryAdmitted: true } : {}),
allowModelOverride: prepared.effectiveAllowModelOverride,
},
runId: params.runId,
@@ -398,6 +416,7 @@ export function startAgentRunExecution(params: {
respond: params.respond,
context: params.context,
taskTrackingMode: prepared.dispatchTaskTrackingMode,
restoreAdmittedRecovery: prepared.restoreAdmittedRestartRecoveryInterrupted,
});
dispatched = true;
} catch (err) {
@@ -419,9 +438,38 @@ export function startAgentRunExecution(params: {
} finally {
if (!dispatched) {
try {
await params.releaseCronContinuationClaimWithRecovery();
if (prepared.restoreAdmittedRestartRecoveryInterrupted) {
try {
pendingRecovery ??= await restoreAdmittedRecoveryWithRetries(
prepared.restoreAdmittedRestartRecoveryInterrupted,
);
} catch (err) {
params.context.logGateway.warn(
`failed to restore undispatched restart recovery: ${formatForLog(err)}`,
);
scheduleAdmittedRecoveryRestore(prepared.restoreAdmittedRestartRecoveryInterrupted);
}
}
} finally {
cleanupAdmittedRun({ force: true });
try {
await params.releaseCronContinuationClaimWithRecovery();
} finally {
try {
pendingRecovery ??= await releaseMainSessionRecoveryOwner(
params.mainRestartRecoveryOwnerLease,
);
} catch (err) {
params.context.logGateway.warn(
`failed to release undispatched main restart recovery owner: ${formatForLog(err)}`,
);
} finally {
try {
cleanupAdmittedRun({ force: true });
} finally {
scheduleMainSessionRecoveryPendingTarget(pendingRecovery);
}
}
}
}
}
}
@@ -1,5 +1,10 @@
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import { ErrorCodes, errorShape } from "../../../packages/gateway-protocol/src/index.js";
import { scheduleMainSessionRecoveryPendingTarget } from "../../agents/main-session-recovery-owner-release.js";
import {
releaseMainSessionRecoveryOwner,
type MainSessionRecoveryOwnerLease,
} from "../../agents/main-session-recovery-store.js";
import { mergeSessionEntry, type SessionEntry } from "../../config/sessions.js";
import type { OpenClawConfig } from "../../config/types.openclaw.js";
import { normalizeDeliveryContext } from "../../utils/delivery-context.shared.js";
@@ -46,6 +51,7 @@ export const agentRunHandler: GatewayRequestHandlers["agent"] = async ({
execApprovalFollowupApprovalId,
normalizedSpawned,
inputProvenance,
isRestartRecoveryResumeRun,
preserveUserFacingSessionModelState,
sessionEffects,
suppressVisibleSessionEffects,
@@ -108,6 +114,7 @@ export const agentRunHandler: GatewayRequestHandlers["agent"] = async ({
let agentId = routing.agentId;
let requestedSessionKey = routing.requestedSessionKey;
let gatewayAdmissionTransferred = false;
let mainRestartRecoveryOwnerLease: MainSessionRecoveryOwnerLease | undefined;
let releaseGatewayAdmission = () => {};
const cronContinuation = createCronContinuationController({
runId,
@@ -329,6 +336,7 @@ export const agentRunHandler: GatewayRequestHandlers["agent"] = async ({
sessionAgentId,
mainSessionKey,
lifecycleGeneration,
isRestartRecoveryResumeRun,
runId,
agentId,
suppressVisibleSessionEffects,
@@ -355,6 +363,9 @@ export const agentRunHandler: GatewayRequestHandlers["agent"] = async ({
},
getAdmittedSessionId: () => admittedSessionId,
setCronContinuationClaim: cronContinuation.setClaim,
setMainRestartRecoveryOwnerLease: (lease) => {
mainRestartRecoveryOwnerLease = lease;
},
respond,
});
if (!persistedSession) {
@@ -425,6 +436,7 @@ export const agentRunHandler: GatewayRequestHandlers["agent"] = async ({
pendingChatRun,
inputProvenance,
isOneShotModelRun,
isRestartRecoveryResumeRun,
runId,
agentDedupeKeys,
context,
@@ -448,6 +460,7 @@ export const agentRunHandler: GatewayRequestHandlers["agent"] = async ({
gatewayAdmissionTransferred = true;
startAgentRunExecution({
prepared: preparedDispatch,
mainRestartRecoveryOwnerLease,
request,
cfg,
cfgForAgent,
@@ -462,6 +475,7 @@ export const agentRunHandler: GatewayRequestHandlers["agent"] = async ({
isNewSession,
isRawModelRun,
isOneShotModelRun,
isRestartRecoveryResumeRun,
suppressVisibleSessionEffects,
message,
images,
@@ -490,11 +504,28 @@ export const agentRunHandler: GatewayRequestHandlers["agent"] = async ({
respond,
releaseCronContinuationClaimWithRecovery,
});
mainRestartRecoveryOwnerLease = undefined;
} finally {
if (!gatewayAdmissionTransferred) {
releaseGatewayAdmission();
await releaseCronContinuationClaimWithRecovery();
try {
if (!gatewayAdmissionTransferred) {
let pendingRecovery: Awaited<ReturnType<typeof releaseMainSessionRecoveryOwner>> =
undefined;
try {
pendingRecovery = await releaseMainSessionRecoveryOwner(mainRestartRecoveryOwnerLease);
} finally {
try {
releaseGatewayAdmission();
} finally {
try {
await releaseCronContinuationClaimWithRecovery();
} finally {
scheduleMainSessionRecoveryPendingTarget(pendingRecovery);
}
}
}
}
} finally {
clearUnacceptedAgentDedupe();
}
clearUnacceptedAgentDedupe();
}
};
@@ -1,6 +1,7 @@
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import { resolveTrustedGroupId } from "../../agents/agent-tools.policy.js";
import { clearAllCliSessions } from "../../agents/cli-session.js";
import { buildMainSessionRecoveryClearPatch } from "../../agents/main-session-recovery-clear.js";
import {
evaluateSessionFreshness,
hasTerminalMainSessionTranscriptNewerThanRegistrySync,
@@ -201,11 +202,15 @@ export function buildAgentSessionPatch(params: {
freshRecoverableTerminalSession &&
!freshSessionRotatedSinceLoad &&
patchSessionId === params.freshEntry?.sessionId;
const automaticRecoveryClearPatch = shouldClearRotatedState
? buildMainSessionRecoveryClearPatch(params.freshEntry)
: {};
const patch: Partial<SessionEntry> = {
sessionId: patchSessionId,
updatedAt: params.now,
...(freshIsNewSession && !freshSessionRotatedSinceLoad ? { sessionStartedAt: params.now } : {}),
...(params.touchInteraction ? { lastInteractionAt: params.now } : {}),
...automaticRecoveryClearPatch,
...(effectiveDeliveryFields.route ? { route: effectiveDeliveryFields.route } : {}),
...(effectiveDeliveryFields.deliveryContext
? { deliveryContext: effectiveDeliveryFields.deliveryContext }
@@ -1,10 +1,17 @@
import { randomUUID } from "node:crypto";
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
import { ErrorCodes, errorShape } from "../../../packages/gateway-protocol/src/index.js";
import {
isMainSessionRecoveryExhausted,
transitionMainSessionRecovery,
} from "../../agents/main-session-recovery-state.js";
import type { MainSessionRecoveryOwnerLease } from "../../agents/main-session-recovery-store.js";
import {
mergeSessionEntry,
resolveSessionLifecycleTimestamps,
resolveSessionWorkStartError,
type SessionEntry,
type InternalSessionEntry,
type SessionFreshness,
} from "../../config/sessions.js";
import {
@@ -69,6 +76,7 @@ export async function persistAgentSessionPhase(params: {
sessionAgentId: string;
mainSessionKey: string;
lifecycleGeneration: string;
isRestartRecoveryResumeRun: boolean;
runId: string;
agentId?: string;
suppressVisibleSessionEffects: boolean;
@@ -98,6 +106,7 @@ export async function persistAgentSessionPhase(params: {
}) => void;
getAdmittedSessionId: () => string;
setCronContinuationClaim: (claim: CronContinuationClaim) => void;
setMainRestartRecoveryOwnerLease: (lease: MainSessionRecoveryOwnerLease) => void;
respond: GatewayRequestHandlerOptions["respond"];
}): Promise<AgentSessionPersistResult | undefined> {
let patchBuild = params.initialPatchBuild;
@@ -106,6 +115,7 @@ export async function persistAgentSessionPhase(params: {
let sessionPersistedBeforeGatewayAdmission = params.initialSessionPersistedBeforeGatewayAdmission;
let supersededSessionId = params.initialSupersededSessionId;
let restoredCronContinuation: RestoredCronContinuation | undefined;
let mainRestartRecoveryOwnerLease: MainSessionRecoveryOwnerLease | undefined;
let skipAgentInitialSessionTouch = false;
const recoveredSessionStartedAt =
!patchBuild.isNewSession &&
@@ -133,6 +143,7 @@ export async function persistAgentSessionPhase(params: {
let archivedDuringStoreUpdateError: string | undefined;
let deletedDuringStoreUpdateError: string | undefined;
let restoredCronContinuationError: string | undefined;
let restartRecoveryReservationConflict: string | undefined;
try {
persisted =
(await patchSessionEntryTarget(
@@ -164,6 +175,18 @@ export async function persistAgentSessionPhase(params: {
archivedDuringStoreUpdateError = archivedError;
throw new Error(archivedError);
}
const internalFreshEntry = freshEntry as InternalSessionEntry | undefined;
if (
!params.isRestartRecoveryResumeRun &&
internalFreshEntry &&
(internalFreshEntry.mainRestartRecovery?.tombstone ||
isMainSessionRecoveryExhausted(internalFreshEntry))
) {
restartRecoveryReservationConflict =
`Session "${params.canonicalSessionKey}" is quarantined after restart recovery ` +
"exhaustion; use /new or /reset before starting new work.";
throw new Error(restartRecoveryReservationConflict);
}
let entryForPatch = freshEntry;
if (params.restoredCronContinuationIdentity) {
const marker = freshEntry?.cronRunContinuation;
@@ -235,6 +258,38 @@ export async function persistAgentSessionPhase(params: {
sessionKey: params.canonicalSessionKey,
storePath: params.storePath,
});
const recoveryTransition = params.isRestartRecoveryResumeRun
? transitionMainSessionRecovery(merged as InternalSessionEntry, {
kind: "validate_recovery",
lifecycleGeneration: params.lifecycleGeneration,
runId: params.runId,
sessionId: params.request.expectedExistingSessionId ?? merged.sessionId,
})
: transitionMainSessionRecovery(merged as InternalSessionEntry, {
kind: "claim_foreground",
cycleId: randomUUID(),
lifecycleGeneration: params.lifecycleGeneration,
sessionId: merged.sessionId,
sessionKey: params.canonicalSessionKey,
claimId: mainRestartRecoveryOwnerLease?.claimId ?? randomUUID(),
runId: params.runId,
});
if (
params.isRestartRecoveryResumeRun &&
recoveryTransition.kind !== "recovery_validated"
) {
restartRecoveryReservationConflict =
`Session "${params.canonicalSessionKey}" restart recovery reservation is stale; ` +
"recovery was skipped.";
throw new Error(restartRecoveryReservationConflict);
}
if (recoveryTransition.kind === "foreground_claimed") {
mainRestartRecoveryOwnerLease = {
...recoveryTransition.claim,
storePath: params.storePath,
};
params.setMainRestartRecoveryOwnerLease(mainRestartRecoveryOwnerLease);
}
if (
params.request.deliver === true &&
resolveSendPolicy({
@@ -291,6 +346,14 @@ export async function persistAgentSessionPhase(params: {
);
return undefined;
}
if (restartRecoveryReservationConflict) {
params.respond(
false,
undefined,
errorShape(ErrorCodes.UNAVAILABLE, restartRecoveryReservationConflict),
);
return undefined;
}
throw err;
}
if (
@@ -2,6 +2,8 @@
import { expectDefined } from "@openclaw/normalization-core";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { registerExecApprovalFollowupRuntimeHandoff } from "../../agents/bash-tools.exec-approval-followup-state.js";
import type { InternalSessionEntry as SessionEntry } from "../../config/sessions.js";
import { runExclusiveSessionLifecycleMutation } from "../../sessions/session-lifecycle-admission.js";
import { setGatewayDedupeEntry } from "./agent-job.js";
import {
getAgentTestMocks,
@@ -1998,6 +2000,201 @@ describe("gateway agent handler chat.abort integration", () => {
expectRecordFields(errorArgs[3], { runId });
});
it("restores admitted restart recovery if pre-dispatch reactivation fails", async () => {
const sessionKey = "agent:main:main";
const sessionId = "recovery-session";
const runId = "recovery-reactivation-fails";
const storePath = "/tmp/sessions.json";
const store: Record<string, SessionEntry> = {
[sessionKey]: {
sessionId,
updatedAt: Date.now() - 10_000,
status: "running",
abortedLastRun: true,
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 1,
chargedAttempts: 1,
reservation: {
runId,
attempt: 1,
lifecycleGeneration: "test-generation",
},
},
},
};
mocks.loadSessionEntry.mockImplementation(() => ({
cfg: {},
storePath,
entry: structuredClone(store[sessionKey]),
canonicalKey: sessionKey,
}));
mocks.updateSessionStore.mockImplementation(async (_path, updater) => await updater(store));
mocks.getLatestSubagentRunByChildSessionKey.mockReturnValueOnce({
runId: "previous-run",
childSessionKey: sessionKey,
controllerSessionKey: sessionKey,
ownerKey: sessionKey,
scopeKind: "session",
requesterDisplayKey: "main",
task: "old task",
cleanup: "keep",
createdAt: 1,
startedAt: 2,
endedAt: 3,
outcome: { status: "ok" },
});
mocks.replaceSubagentRunAfterSteer.mockRejectedValueOnce(new Error("reactivate boom"));
const respond = vi.fn();
await invokeAgent(
{
message: "resume after restart",
agentId: "main",
sessionKey,
sessionId,
expectedExistingSessionId: sessionId,
idempotencyKey: runId,
inputProvenance: {
kind: "internal_system",
sourceSessionKey: sessionKey,
sourceTool: "main_session_restart_recovery",
},
},
{ client: backendGatewayClient(), reqId: runId, respond },
);
expect(mocks.agentCommand).not.toHaveBeenCalled();
expect(store[sessionKey]).toMatchObject({
sessionId,
status: "running",
abortedLastRun: true,
mainRestartRecovery: {
chargedAttempts: 1,
},
});
expect(store[sessionKey]?.mainRestartRecovery?.reservation).toBeUndefined();
expect(
respond.mock.calls.some(
([ok, payload]) =>
ok === false && (payload as { runId?: string; status?: string })?.runId === runId,
),
).toBe(true);
});
it("releases a foreground recovery owner if pre-dispatch reactivation fails", async () => {
const sessionKey = "agent:main:main";
const sessionId = "interrupted-session";
const runId = "foreground-reactivation-fails";
const storePath = "/tmp/sessions.json";
const store: Record<string, SessionEntry> = {
[sessionKey]: {
sessionId,
updatedAt: Date.now() - 10_000,
status: "running",
abortedLastRun: true,
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 1,
chargedAttempts: 1,
},
},
};
mocks.loadSessionEntry.mockImplementation(() => ({
cfg: {},
storePath,
entry: structuredClone(store[sessionKey]),
canonicalKey: sessionKey,
}));
mocks.updateSessionStore.mockImplementation(async (_path, updater) => await updater(store));
mocks.getLatestSubagentRunByChildSessionKey.mockReturnValueOnce({
runId: "previous-run",
childSessionKey: sessionKey,
controllerSessionKey: sessionKey,
ownerKey: sessionKey,
scopeKind: "session",
requesterDisplayKey: "main",
task: "old task",
cleanup: "keep",
createdAt: 1,
startedAt: 2,
endedAt: 3,
outcome: { status: "ok" },
});
mocks.replaceSubagentRunAfterSteer.mockRejectedValueOnce(new Error("reactivate boom"));
await invokeAgent(
{
message: "new foreground turn",
agentId: "main",
sessionKey,
sessionId,
idempotencyKey: runId,
},
{ client: backendGatewayClient(), reqId: runId, respond: vi.fn() },
);
expect(mocks.agentCommand).not.toHaveBeenCalled();
expect(store[sessionKey]?.mainRestartRecovery?.foregroundClaims).toBeUndefined();
});
it("releases gateway admission when foreground owner cleanup exhausts retries", async () => {
const sessionKey = "agent:main:main";
const sessionId = "interrupted-session";
const runId = "foreground-release-fails";
const storePath = "/tmp/sessions.json";
const store: Record<string, SessionEntry> = {
[sessionKey]: {
sessionId,
updatedAt: Date.now() - 10_000,
status: "running",
abortedLastRun: true,
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 1,
chargedAttempts: 1,
},
},
};
mocks.loadSessionEntry.mockImplementation(() => ({
cfg: {},
storePath,
entry: structuredClone(store[sessionKey]),
canonicalKey: sessionKey,
}));
mocks.updateSessionStore.mockImplementation(async (_path, updater) => await updater(store));
mocks.applySessionEntryReplacements.mockRejectedValue(new Error("owner release write failed"));
await expect(
invokeAgent(
{
message: "new foreground turn",
agentId: "main",
sessionKey,
sessionId,
deliver: true,
replyChannel: "telegram",
bestEffortDeliver: false,
idempotencyKey: runId,
},
{
client: backendGatewayClient(),
reqId: runId,
respond: vi.fn(),
flushDispatch: false,
},
),
).rejects.toThrow("owner release write failed");
await expect(
runExclusiveSessionLifecycleMutation({
scope: storePath,
identities: [sessionKey, sessionId],
signal: AbortSignal.timeout(100),
run: async () => "released",
}),
).resolves.toBe("released");
});
it("does not dispatch a duplicate agent run when dedupe was evicted but the run is active", async () => {
prime();
mocks.agentCommand.mockResolvedValueOnce({
@@ -1254,6 +1254,48 @@ describe("gateway agent handler", () => {
);
});
it("clears automatic recovery quarantine state when a user turn rotates the session id", async () => {
vi.useFakeTimers({ toFake: ["Date"] });
setDateOnlyFakeClockActive(true);
vi.setSystemTime(new Date("2026-05-07T12:00:00.000Z"));
const staleEntry = {
sessionId: "quarantined-session-id",
updatedAt: 0,
sessionStartedAt: 0,
lastInteractionAt: 0,
abortedLastRun: true,
restartRecoveryRuns: [
{ runId: "initial-wedged-run", lifecycleGeneration: "gen-1" },
{ runId: "recovery-run-1", lifecycleGeneration: "gen-2" },
],
mainRestartRecovery: {
automaticAttempts: 2,
lastAttemptAt: 3,
lastRunId: "recovery-run-1",
},
subagentRecovery: {
automaticAttempts: 2,
lastAttemptAt: 3,
wedgedAt: 4,
wedgedReason: "automatic_attempt_budget_exceeded",
},
};
mockMainSessionEntry(staleEntry);
const capturedEntry = await runMainAgentAndCaptureEntry("test-idem-rotated-recovery-clear");
expect(capturedEntry.sessionId).not.toBe("quarantined-session-id");
expect(capturedEntry.abortedLastRun).toBeUndefined();
expect(capturedEntry.restartRecoveryRuns).toBeUndefined();
expect(capturedEntry.mainRestartRecovery).toBeUndefined();
expect(capturedEntry.subagentRecovery).toEqual({
automaticAttempts: 2,
lastAttemptAt: 3,
wedgedAt: 4,
wedgedReason: "automatic_attempt_budget_exceeded",
});
});
it("drops a stale transcript path when a stale session rotates ids", async () => {
vi.useFakeTimers({ toFake: ["Date"] });
setDateOnlyFakeClockActive(true);
@@ -37,6 +37,7 @@ vi.mock("../../config/config.js", () => ({
}));
vi.mock("../../commands/agent.js", () => ({
agentCommandFromGatewayIngress: agentIngressMocks.agentCommandFromIngress,
agentCommandFromIngress: agentIngressMocks.agentCommandFromIngress,
}));
@@ -13,6 +13,7 @@ const performGatewaySessionResetMock = vi.hoisted(() => vi.fn());
const parseMessageWithAttachmentsMock = vi.hoisted(() => vi.fn());
vi.mock("../../commands/agent.js", () => ({
agentCommandFromGatewayIngress: agentCommandFromIngressMock,
agentCommandFromIngress: agentCommandFromIngressMock,
}));
@@ -46,6 +46,73 @@ const mocks = getAgentTestMocks();
describe("gateway agent handler", () => {
afterEach(describe0AfterEach0);
it("rejects ordinary work on a restart-recovery tombstone", async () => {
const entry = {
sessionId: "tombstoned-session",
updatedAt: Date.now(),
status: "failed",
abortedLastRun: false,
mainRestartRecovery: {
cycleId: "cycle-exhausted",
revision: 4,
chargedAttempts: 3,
tombstone: { reason: "automatic recovery exhausted" },
},
};
mockMainSessionEntry(entry);
mocks.updateSessionStore.mockImplementation(
async (_path, updater) => await updater({ "agent:main:main": structuredClone(entry) }),
);
const commandCallCount = mocks.agentCommand.mock.calls.length;
const respond = vi.fn();
await invokeAgent(
{
message: "continue old work",
sessionKey: "agent:main:main",
idempotencyKey: "tombstone-reuse",
},
{ reqId: "tombstone-reuse", respond },
);
expect(mocks.agentCommand).toHaveBeenCalledTimes(commandCallCount);
const error = expectRespondError(respond, { code: ErrorCodes.UNAVAILABLE });
expectStringFieldContains(error, "message", "quarantined after restart recovery exhaustion");
});
it("rejects ordinary work while restart recovery exhaustion is being tombstoned", async () => {
const entry = {
sessionId: "exhausted-session",
updatedAt: Date.now(),
status: "running",
abortedLastRun: true,
mainRestartRecovery: {
cycleId: "cycle-exhausted",
revision: 4,
chargedAttempts: 3,
},
};
mockMainSessionEntry(entry);
mocks.updateSessionStore.mockImplementation(
async (_path, updater) => await updater({ "agent:main:main": structuredClone(entry) }),
);
const commandCallCount = mocks.agentCommand.mock.calls.length;
const respond = vi.fn();
await invokeAgent(
{
message: "continue old work",
sessionKey: "agent:main:main",
idempotencyKey: "exhausted-reuse",
},
{ reqId: "exhausted-reuse", respond },
);
expect(mocks.agentCommand).toHaveBeenCalledTimes(commandCallCount);
const error = expectRespondError(respond, { code: ErrorCodes.UNAVAILABLE });
expectStringFieldContains(error, "message", "quarantined after restart recovery exhaustion");
});
it("does not restore elevated defaults from idempotency key suffixes", async () => {
const bashElevated = {
enabled: true,
@@ -130,6 +130,7 @@ vi.mock("../../config/sessions/session-accessor.js", async () => {
vi.mock("../../commands/agent.js", () => ({
agentCommand: mocks.agentCommand,
agentCommandFromGatewayIngress: mocks.agentCommand,
agentCommandFromIngress: mocks.agentCommand,
}));
@@ -677,7 +678,7 @@ export async function runMainAgentAndCaptureEntry(idempotencyKey: string) {
[canonicalKey]: existingEntry,
};
const result = await updater(store);
capturedEntry = result as Record<string, unknown>;
capturedEntry = structuredClone(store[canonicalKey]) as Record<string, unknown>;
return result;
});
mocks.agentCommand.mockResolvedValue({
@@ -37,12 +37,6 @@ const hoisted = vi.hoisted(() => {
marked: 0,
skipped: 0,
}));
const recoverStartupOrphanedMainSessions = vi.fn(async () => ({
marked: 0,
recovered: 0,
failed: 0,
skipped: 0,
}));
const scheduleRestartAbortedMainSessionRecovery = vi.fn();
const scheduleRestartSentinelWake =
vi.fn<typeof import("./server-restart-sentinel.js").scheduleRestartSentinelWake>();
@@ -96,7 +90,6 @@ const hoisted = vi.hoisted(() => {
scheduleSubagentOrphanRecovery,
markRestartAbortedMainSessionsFromLocks,
markStartupOrphanedMainSessionsForRecovery,
recoverStartupOrphanedMainSessions,
scheduleRestartAbortedMainSessionRecovery,
scheduleRestartSentinelWake,
refreshLatestUpdateRestartSentinel,
@@ -133,7 +126,6 @@ vi.mock("../agents/subagent-registry.js", () => ({
vi.mock("../agents/main-session-restart-recovery.js", () => ({
markRestartAbortedMainSessionsFromLocks: hoisted.markRestartAbortedMainSessionsFromLocks,
markStartupOrphanedMainSessionsForRecovery: hoisted.markStartupOrphanedMainSessionsForRecovery,
recoverStartupOrphanedMainSessions: hoisted.recoverStartupOrphanedMainSessions,
scheduleRestartAbortedMainSessionRecovery: hoisted.scheduleRestartAbortedMainSessionRecovery,
}));
@@ -343,13 +335,6 @@ describe("startGatewayPostAttachRuntime", () => {
marked: 0,
skipped: 0,
});
hoisted.recoverStartupOrphanedMainSessions.mockReset();
hoisted.recoverStartupOrphanedMainSessions.mockResolvedValue({
marked: 0,
recovered: 0,
failed: 0,
skipped: 0,
});
hoisted.scheduleRestartAbortedMainSessionRecovery.mockClear();
hoisted.scheduleRestartSentinelWake.mockClear();
hoisted.refreshLatestUpdateRestartSentinel.mockReset();
@@ -4,6 +4,7 @@ import fs from "node:fs/promises";
import path from "node:path";
import { expect, test, vi } from "vitest";
import { listSessionEntries, loadSessionEntry } from "../config/sessions/session-accessor.js";
import type { InternalSessionEntry } from "../config/sessions/types.js";
import { beginSessionWorkAdmission } from "../sessions/session-lifecycle-admission.js";
import { embeddedRunMock, testState, writeSessionStore } from "./test-helpers.js";
import {
@@ -350,6 +351,44 @@ test("sessions.reset emits internal command hook with reason", async () => {
expect(event.context?.previousSessionEntry?.sessionId).toBe("sess-main");
});
test("sessions.reset removes automatic recovery state from the replacement session", async () => {
const { dir } = await createSessionStoreDir();
await writeSingleLineSession(dir, "sess-recovery", "hello");
await writeMainSessionEntry("sess-recovery", {
abortedLastRun: true,
restartRecoveryRuns: [{ runId: "recovery-run", lifecycleGeneration: "generation-1" }],
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 5,
chargedAttempts: 3,
foregroundClaims: {
lifecycleGeneration: "generation-1",
tokens: ["foreground-owner"],
},
tombstone: {
reason: "exhausted",
},
},
subagentRecovery: {
automaticAttempts: 2,
lastAttemptAt: 10,
lastRunId: "child-recovery-run",
wedgedAt: 20,
wedgedReason: "child exhausted",
},
});
await resetMainSession();
const store = await loadGatewaySessionStoreForKey("main");
const replacement = store["agent:main:main"];
expect(replacement?.sessionId).not.toBe("sess-recovery");
expect(replacement?.abortedLastRun).toBe(false);
expect(replacement?.restartRecoveryRuns).toBeUndefined();
expect((replacement as InternalSessionEntry | undefined)?.mainRestartRecovery).toBeUndefined();
expect(replacement?.subagentRecovery).toBeUndefined();
});
test("sessions.reset does not begin cleanup after losing lifecycle ownership", async () => {
const { dir } = await createSessionStoreDir();
await writeSingleLineSession(dir, "sess-main", "hello");
@@ -0,0 +1,22 @@
import { buildMainSessionRecoveryClearPatch } from "../agents/main-session-recovery-clear.js";
import type { SessionEntry } from "../config/sessions.js";
export function buildCheckpointSessionResetPatch(params: {
entry: SessionEntry;
sessionId: string;
sessionFile: string;
}): Partial<SessionEntry> {
// A checkpoint clone has a new transcript identity, so prior recovery ownership cannot follow it.
return {
...buildMainSessionRecoveryClearPatch(params.entry),
sessionId: params.sessionId,
sessionFile: params.sessionFile,
updatedAt: Date.now(),
systemSent: false,
abortedLastRun: false,
startedAt: undefined,
endedAt: undefined,
runtimeMs: undefined,
status: undefined,
};
}
@@ -33,6 +33,7 @@ import { scanSessionTranscriptTree } from "../config/sessions/transcript-tree.js
import { CURRENT_SESSION_VERSION } from "../config/sessions/version.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import { createSubsystemLogger } from "../logging/subsystem.js";
import { buildCheckpointSessionResetPatch } from "./session-compaction-checkpoint-entry.js";
import { resolveGatewaySessionStoreTarget } from "./session-utils.js";
const log = createSubsystemLogger("gateway/session-compaction-checkpoints");
@@ -587,15 +588,11 @@ function cloneCheckpointSessionEntry(params: {
}): SessionEntry {
return {
...params.currentEntry,
sessionId: params.nextSessionId,
sessionFile: params.nextSessionFile,
updatedAt: Date.now(),
systemSent: false,
abortedLastRun: false,
startedAt: undefined,
endedAt: undefined,
runtimeMs: undefined,
status: undefined,
...buildCheckpointSessionResetPatch({
entry: params.currentEntry,
sessionId: params.nextSessionId,
sessionFile: params.nextSessionFile,
}),
inputTokens: undefined,
outputTokens: undefined,
cacheRead: undefined,
+18
View File
@@ -0,0 +1,18 @@
import { buildMainSessionRecoveryClearPatch } from "../agents/main-session-recovery-clear.js";
import type { SessionEntry } from "../config/sessions.js";
export function buildForkedGatewaySessionEntry(
entry: SessionEntry,
fork: { sessionId: string; sessionFile: string },
): SessionEntry {
// Replacing the transcript identity also replaces the recovery episode owned by the old row.
return {
...entry,
...buildMainSessionRecoveryClearPatch(entry),
sessionId: fork.sessionId,
sessionFile: fork.sessionFile,
forkedFromParent: true,
totalTokens: undefined,
totalTokensFresh: false,
};
}
+2 -8
View File
@@ -51,6 +51,7 @@ import {
runExclusiveSessionLifecycleMutation,
} from "../sessions/session-lifecycle-admission.js";
import { createLazyRuntimeModule } from "../shared/lazy-runtime.js";
import { buildForkedGatewaySessionEntry } from "./session-create-fork-entry.js";
import { resolveSessionStoreAgentId, resolveSessionStoreKey } from "./session-store-key.js";
import { loadSessionEntry, resolveGatewaySessionStoreTarget } from "./session-utils.js";
import { applySessionsPatchToStore } from "./sessions-patch.js";
@@ -700,14 +701,7 @@ export async function createGatewaySession(params: {
}
return {
...initialized,
entry: {
...entry,
sessionId: fork.sessionId,
sessionFile: fork.sessionFile,
forkedFromParent: true,
totalTokens: undefined,
totalTokensFresh: false,
},
entry: buildForkedGatewaySessionEntry(entry, fork),
};
},
params.initialEntry
+125 -1
View File
@@ -2,7 +2,8 @@
* Session lifecycle state derivation tests.
*/
import { describe, expect, it, vi } from "vitest";
import type { SessionEntry } from "../config/sessions.js";
import type { InternalSessionEntry as SessionEntry } from "../config/sessions.js";
import { getAgentEventLifecycleGeneration } from "../infra/agent-events.js";
const persistenceMocks = vi.hoisted(() => ({
loadSessionEntry: vi.fn(),
@@ -258,6 +259,11 @@ describe("session lifecycle state", () => {
});
it("preserves recovery state for a late interrupted-run event", async () => {
const mainRestartRecovery = {
cycleId: "cycle-1",
revision: 2,
chargedAttempts: 2,
};
const persisted = await persistLifecycle(
{
sessionId: "session-id",
@@ -265,6 +271,7 @@ describe("session lifecycle state", () => {
status: "running",
abortedLastRun: true,
restartRecoveryRuns: [{ runId: "restart-run", lifecycleGeneration: "pre-restart" }],
mainRestartRecovery,
},
{
ts: 2_000,
@@ -279,6 +286,123 @@ describe("session lifecycle state", () => {
status: "running",
abortedLastRun: true,
restartRecoveryRuns: [{ runId: "restart-run", lifecycleGeneration: "pre-restart" }],
mainRestartRecovery,
});
});
it("ignores an unidentified completion while recovery remains pending", async () => {
const persisted = await persistLifecycle(
{
sessionId: "session-id",
updatedAt: 1_000,
startedAt: 1_050,
status: "running",
abortedLastRun: true,
restartRecoveryRuns: [{ runId: "restart-run", lifecycleGeneration: "pre-restart" }],
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 2,
chargedAttempts: 2,
},
},
{
ts: 2_000,
sessionId: "session-id",
data: { phase: "end", endedAt: 1_800 },
},
);
expect(persisted).toMatchObject({
status: "running",
abortedLastRun: true,
});
expect(persisted.restartRecoveryRuns).toEqual([
{ runId: "restart-run", lifecycleGeneration: "pre-restart" },
]);
expect(persisted.mainRestartRecovery).toMatchObject({ cycleId: "cycle-1" });
});
it("applies the terminal snapshot for the foreground owner run", async () => {
const lifecycleGeneration = getAgentEventLifecycleGeneration();
const persisted = await persistLifecycle(
{
sessionId: "session-id",
updatedAt: 1_000,
startedAt: 1_050,
status: "running",
abortedLastRun: true,
restartRecoveryRuns: [
{ runId: "interrupted-run", lifecycleGeneration: "pre-restart" },
{ runId: "foreground-run", lifecycleGeneration },
],
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 2,
chargedAttempts: 2,
foregroundClaims: {
lifecycleGeneration,
tokens: ["owner-claim"],
runIdsByClaimId: { "owner-claim": "foreground-run" },
},
},
},
{
ts: 2_000,
sessionId: "session-id",
runId: "foreground-run",
lifecycleGeneration,
data: { phase: "end", endedAt: 1_800 },
},
);
expect(persisted).toMatchObject({
status: "done",
endedAt: 1_800,
abortedLastRun: false,
});
expect(persisted.restartRecoveryRuns).toBeUndefined();
expect(persisted.mainRestartRecovery).toBeUndefined();
});
it("does not settle a foreground owner from a stale lifecycle generation", async () => {
const persisted = await persistLifecycle(
{
sessionId: "session-id",
updatedAt: 1_000,
startedAt: 1_050,
status: "running",
abortedLastRun: true,
restartRecoveryRuns: [
{ runId: "interrupted-run", lifecycleGeneration: "pre-restart" },
{ runId: "foreground-run", lifecycleGeneration: "pre-restart" },
],
mainRestartRecovery: {
cycleId: "cycle-1",
revision: 2,
chargedAttempts: 2,
foregroundClaims: {
lifecycleGeneration: "pre-restart",
tokens: ["owner-claim"],
runIdsByClaimId: { "owner-claim": "foreground-run" },
},
},
},
{
ts: 2_000,
sessionId: "session-id",
runId: "foreground-run",
lifecycleGeneration: "pre-restart",
data: { phase: "end", endedAt: 1_800 },
},
);
expect(persisted).toMatchObject({
status: "running",
abortedLastRun: true,
restartRecoveryRuns: [{ runId: "interrupted-run", lifecycleGeneration: "pre-restart" }],
mainRestartRecovery: {
foregroundClaims: { tokens: ["owner-claim"] },
},
});
});
+16 -41
View File
@@ -5,9 +5,13 @@ import {
buildAgentRunTerminalOutcome,
type AgentRunTerminalOutcome,
} from "../agents/agent-run-terminal-outcome.js";
import type { SessionEntry } from "../config/sessions.js";
import {
isMainSessionRecoveryLifecycleEvent,
projectMainSessionRecoveryLifecycle,
} from "../agents/main-session-recovery-lifecycle.js";
import type { InternalSessionEntry as SessionEntry } from "../config/sessions.js";
import { updateSessionEntry } from "../config/sessions/session-accessor.js";
import type { AgentEventPayload } from "../infra/agent-events.js";
import { getAgentEventLifecycleGeneration, type AgentEventPayload } from "../infra/agent-events.js";
import { parseCronRunScopeSuffix } from "../sessions/session-key-utils.js";
import { loadSessionEntry } from "./session-utils.js";
import type { GatewaySessionRow, SessionRunStatus } from "./session-utils.types.js";
@@ -46,6 +50,7 @@ type PersistedLifecycleSessionShape = Pick<
| "runtimeMs"
| "abortedLastRun"
| "restartRecoveryRuns"
| "mainRestartRecovery"
>;
type GatewaySessionLifecycleSnapshot = Partial<LifecycleSessionShape>;
@@ -192,37 +197,21 @@ function derivePersistedSessionLifecyclePatch(params: {
entry?: Partial<PersistedLifecycleSessionShape> | null;
event: LifecycleEventLike;
}): Partial<PersistedLifecycleSessionShape> {
if (isRestartRecoveryLifecycleEvent(params)) {
return {};
}
const snapshot = deriveGatewaySessionLifecycleSnapshot({
session: params.entry ?? undefined,
event: params.event,
});
const patch: Partial<PersistedLifecycleSessionShape> = {
const snapshotPatch: Partial<PersistedLifecycleSessionShape> = {
...snapshot,
updatedAt: typeof snapshot.updatedAt === "number" ? snapshot.updatedAt : undefined,
};
const runId = params.event.runId?.trim();
const lifecycleGeneration = params.event.lifecycleGeneration?.trim();
const restartRecoveryRuns = params.entry?.restartRecoveryRuns;
if (
resolveLifecyclePhase(params.event) !== "start" &&
runId &&
lifecycleGeneration &&
restartRecoveryRuns?.some(
(run) => run.runId === runId && run.lifecycleGeneration === lifecycleGeneration,
)
) {
const remainingRuns = restartRecoveryRuns.filter(
(run) => run.runId !== runId || run.lifecycleGeneration !== lifecycleGeneration,
);
if (remainingRuns.length > 0) {
return { restartRecoveryRuns: remainingRuns };
}
patch.restartRecoveryRuns = undefined;
}
return patch;
const projection = projectMainSessionRecoveryLifecycle({
currentLifecycleGeneration: getAgentEventLifecycleGeneration(),
entry: params.entry,
event: params.event,
snapshotPatch,
});
return projection.action === "suppress" ? {} : projection.patch;
}
export function deriveGatewaySessionLifecycleProjectionPatch(params: {
@@ -238,21 +227,7 @@ export function isRestartRecoveryLifecycleEvent(params: {
entry?: Pick<SessionEntry, "restartRecoveryRuns"> | null;
event: Pick<LifecycleEventLike, "runId" | "lifecycleGeneration" | "data">;
}): boolean {
const runId = params.event.runId?.trim();
const lifecycleGeneration = params.event.lifecycleGeneration?.trim();
const phase = resolveLifecyclePhase(params.event);
const interrupted = params.event.data?.stopReason === "restart";
const matchesRecoveryRun = Boolean(
runId &&
lifecycleGeneration &&
params.entry?.restartRecoveryRuns?.some(
(run) => run.runId === runId && run.lifecycleGeneration === lifecycleGeneration,
),
);
return (
matchesRecoveryRun &&
(phase === "start" || ((phase === "end" || phase === "error") && interrupted))
);
return isMainSessionRecoveryLifecycleEvent(params);
}
/**
+1
View File
@@ -251,6 +251,7 @@ vi.mock("../commands/status.js", () => ({
}));
vi.mock("../commands/agent.js", () => ({
agentCommand,
agentCommandFromGatewayIngress: agentCommand,
agentCommandFromIngress: agentCommand,
}));
vi.mock("../agents/btw.js", () => ({
@@ -8,7 +8,7 @@ import path from "node:path";
import { expectDefined } from "@openclaw/normalization-core";
import type { AssistantMessage, UserMessage } from "openclaw/plugin-sdk/llm";
import { afterAll, beforeAll, beforeEach, expect, vi } from "vitest";
import type { SessionEntry } from "../../config/sessions.js";
import type { InternalSessionEntry as SessionEntry } from "../../config/sessions.js";
import {
loadTranscriptEvents,
persistSessionTranscriptTurn,
+2 -1
View File
@@ -7,4 +7,5 @@ export type * from "../config/types.js";
export type { ConfigWriteAfterWrite } from "../config/runtime-snapshot.js";
export type { ChannelGroupPolicy } from "../config/group-policy.js";
export type { SessionResetMode } from "../config/sessions/reset.js";
export type { SessionEntry, SessionScope } from "../config/sessions/types.js";
export type { SessionEntry } from "../config/sessions/types.js";
export type { SessionScope } from "../config/sessions/types.js";
@@ -0,0 +1,107 @@
import { MAIN_SESSION_RECOVERY_CLEAR_PATCH } from "../agents/main-session-recovery-clear.js";
import type { SessionAccessScope } from "../config/sessions/session-accessor.js";
import type { InternalSessionEntry, SessionEntry } from "../config/sessions/types.js";
export type SessionStoreReadParams = {
agentId?: string;
env?: NodeJS.ProcessEnv;
hydrateSkillPromptRefs?: boolean;
readConsistency?: "latest";
sessionKey: string;
storePath?: string;
};
export function toSessionAccessScope(params: SessionStoreReadParams): SessionAccessScope {
// Keep plugin-facing options separate from internal accessor-only controls.
return {
sessionKey: params.sessionKey,
...(params.agentId !== undefined ? { agentId: params.agentId } : {}),
...(params.env !== undefined ? { env: params.env } : {}),
...(params.hydrateSkillPromptRefs !== undefined
? { hydrateSkillPromptRefs: params.hydrateSkillPromptRefs }
: {}),
...(params.readConsistency !== undefined ? { readConsistency: params.readConsistency } : {}),
...(params.storePath !== undefined ? { storePath: params.storePath } : {}),
};
}
export function projectPluginSessionEntry(entry: InternalSessionEntry): SessionEntry {
const { mainRestartRecovery: _mainRestartRecovery, ...publicEntry } = entry;
return {
...publicEntry,
...(entry.restartRecoveryRuns
? { restartRecoveryRuns: entry.restartRecoveryRuns.map((run) => ({ ...run })) }
: {}),
};
}
export function projectPluginSessionEntryPatch(
patch: Partial<InternalSessionEntry>,
): Partial<SessionEntry> {
const { mainRestartRecovery: _mainRestartRecovery, ...publicPatch } = patch;
return publicPatch;
}
export function projectPluginSessionStore(
store: Record<string, InternalSessionEntry>,
): Record<string, SessionEntry> {
return Object.fromEntries(
Object.entries(store).map(([sessionKey, entry]) => [
sessionKey,
projectPluginSessionEntry(entry),
]),
);
}
export function activeRecoveryFieldsForSameSession(
existingEntry: InternalSessionEntry | undefined,
nextSessionId: string | undefined,
): Partial<InternalSessionEntry> | undefined {
if (
!existingEntry ||
existingEntry.sessionId !== nextSessionId ||
existingEntry.mainRestartRecovery === undefined
) {
return undefined;
}
return {
abortedLastRun: existingEntry.abortedLastRun,
restartRecoveryRuns: existingEntry.restartRecoveryRuns,
mainRestartRecovery: existingEntry.mainRestartRecovery,
};
}
export function clearRecoveryStateForRotatedSessionPatch(
existingEntry: InternalSessionEntry,
publicPatch: Partial<SessionEntry>,
): Partial<InternalSessionEntry> {
return Object.hasOwn(publicPatch, "sessionId") &&
publicPatch.sessionId !== existingEntry.sessionId
? { ...publicPatch, ...MAIN_SESSION_RECOVERY_CLEAR_PATCH }
: publicPatch;
}
export function reconcilePluginSessionStore(params: {
internalStore: Record<string, InternalSessionEntry>;
publicStore: Record<string, SessionEntry>;
}): void {
for (const sessionKey of Object.keys(params.internalStore)) {
if (!Object.hasOwn(params.publicStore, sessionKey)) {
delete params.internalStore[sessionKey];
}
}
for (const [sessionKey, publicEntry] of Object.entries(params.publicStore)) {
const projectedEntry = projectPluginSessionEntry(publicEntry as InternalSessionEntry);
const existingRecovery = activeRecoveryFieldsForSameSession(
params.internalStore[sessionKey],
projectedEntry.sessionId,
);
const existingEntry = params.internalStore[sessionKey];
params.internalStore[sessionKey] =
existingEntry && existingEntry.sessionId !== projectedEntry.sessionId
? { ...projectedEntry, ...MAIN_SESSION_RECOVERY_CLEAR_PATCH }
: existingRecovery
? { ...projectedEntry, ...existingRecovery }
: projectedEntry;
}
}
@@ -0,0 +1,140 @@
import {
patchSessionEntry as patchAccessorSessionEntry,
updateSessionEntry,
} from "../config/sessions/session-accessor.js";
import { normalizeResolvedMaintenanceConfigInput } from "../config/sessions/store-maintenance.js";
import type { ResolvedSessionMaintenanceConfigInput } from "../config/sessions/store.js";
import type { InternalSessionEntry, SessionEntry } from "../config/sessions/types.js";
import {
clearRecoveryStateForRotatedSessionPatch,
projectPluginSessionEntry,
projectPluginSessionEntryPatch,
activeRecoveryFieldsForSameSession,
type SessionStoreReadParams,
toSessionAccessScope,
} from "./session-store-runtime-internal.js";
type SessionStoreEntryUpdate = (
entry: SessionEntry,
) => Promise<Partial<SessionEntry> | null> | Partial<SessionEntry> | null;
type SessionStoreEntryPatch = (
entry: SessionEntry,
context: { existingEntry?: SessionEntry },
) => Promise<Partial<SessionEntry> | null> | Partial<SessionEntry> | null;
type PatchSessionEntryParams = SessionStoreReadParams & {
fallbackEntry?: SessionEntry;
maintenanceConfig?: ResolvedSessionMaintenanceConfigInput;
preserveActivity?: boolean;
requireWriteSuccess?: boolean;
replaceEntry?: boolean;
skipMaintenance?: boolean;
update: SessionStoreEntryPatch;
};
type UpdateSessionStoreEntryParams = {
storePath: string;
sessionKey: string;
update: SessionStoreEntryUpdate;
skipMaintenance?: boolean;
takeCacheOwnership?: boolean;
requireWriteSuccess?: boolean;
};
type UpsertSessionEntryParams = SessionStoreReadParams & { entry: SessionEntry };
/** Patches one public plugin session entry without exposing core recovery coordination. */
export async function patchPluginSessionEntry(
params: PatchSessionEntryParams,
): Promise<SessionEntry | null> {
const entry = await patchAccessorSessionEntry(
toSessionAccessScope(params),
async (internalEntry, context) => {
const persistedEntry = internalEntry as InternalSessionEntry;
const patch = await params.update(projectPluginSessionEntry(internalEntry), {
existingEntry: context.existingEntry
? projectPluginSessionEntry(context.existingEntry)
: undefined,
});
if (!patch) {
return null;
}
const publicPatch = projectPluginSessionEntryPatch(patch);
const nextSessionId = Object.hasOwn(publicPatch, "sessionId")
? publicPatch.sessionId
: persistedEntry.sessionId;
const existingRecovery = activeRecoveryFieldsForSameSession(persistedEntry, nextSessionId);
return params.replaceEntry
? existingRecovery
? { ...publicPatch, ...existingRecovery }
: clearRecoveryStateForRotatedSessionPatch(persistedEntry, publicPatch)
: existingRecovery
? { ...publicPatch, ...existingRecovery }
: clearRecoveryStateForRotatedSessionPatch(persistedEntry, publicPatch);
},
{
fallbackEntry: params.fallbackEntry
? projectPluginSessionEntry(params.fallbackEntry)
: undefined,
maintenanceConfig:
params.maintenanceConfig !== undefined
? normalizeResolvedMaintenanceConfigInput(params.maintenanceConfig)
: undefined,
preserveActivity: params.preserveActivity,
requireWriteSuccess: params.requireWriteSuccess,
replaceEntry: params.replaceEntry,
skipMaintenance: params.skipMaintenance,
},
);
return entry ? projectPluginSessionEntry(entry) : null;
}
/** Updates one public plugin session entry by store path and session key. */
export async function updatePluginSessionStoreEntry(
params: UpdateSessionStoreEntryParams,
): Promise<SessionEntry | null> {
const entry = await updateSessionEntry(
{ sessionKey: params.sessionKey, storePath: params.storePath },
async (internalEntry) => {
const patch = await params.update(projectPluginSessionEntry(internalEntry));
if (!patch) {
return null;
}
const persistedEntry = internalEntry as InternalSessionEntry;
const publicPatch = projectPluginSessionEntryPatch(patch);
const nextSessionId = Object.hasOwn(publicPatch, "sessionId")
? publicPatch.sessionId
: persistedEntry.sessionId;
const existingRecovery = activeRecoveryFieldsForSameSession(persistedEntry, nextSessionId);
return existingRecovery
? { ...publicPatch, ...existingRecovery }
: clearRecoveryStateForRotatedSessionPatch(persistedEntry, publicPatch);
},
{
skipMaintenance: params.skipMaintenance,
takeCacheOwnership: params.takeCacheOwnership,
requireWriteSuccess: params.requireWriteSuccess,
},
);
return entry ? projectPluginSessionEntry(entry) : null;
}
/** Replaces or creates one public plugin session entry. */
export async function upsertPluginSessionEntry(params: UpsertSessionEntryParams): Promise<void> {
const publicEntry = projectPluginSessionEntry(params.entry);
await patchAccessorSessionEntry(
toSessionAccessScope(params),
(internalEntry) => {
const persistedEntry = internalEntry as InternalSessionEntry;
const existingRecovery = activeRecoveryFieldsForSameSession(
persistedEntry,
publicEntry.sessionId,
);
return existingRecovery
? { ...publicEntry, ...existingRecovery }
: clearRecoveryStateForRotatedSessionPatch(persistedEntry, publicEntry);
},
{ fallbackEntry: publicEntry, replaceEntry: true },
);
}
@@ -0,0 +1,85 @@
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it } from "vitest";
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
import { loadSessionEntry as loadInternalSessionEntry } from "../config/sessions/session-accessor.js";
import {
patchSessionEntry,
upsertSessionEntry,
type SessionEntry,
} from "./session-store-runtime.js";
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
describe("session-store-runtime recovery boundary", () => {
let tempDir: string;
let storePath: string;
beforeEach(() => {
tempDir = tempDirs.make("openclaw-sdk-session-recovery-");
storePath = path.join(tempDir, "sessions.json");
});
it("allows public recovery fields to change without an active core transaction", async () => {
const sessionKey = "agent:main:healthy-public-recovery";
await upsertSessionEntry({
agentId: "main",
sessionKey,
storePath,
entry: {
sessionId: "healthy-session",
updatedAt: 10,
},
});
await patchSessionEntry({
sessionKey,
storePath,
update: () => ({
abortedLastRun: true,
restartRecoveryRuns: [{ lifecycleGeneration: "generation-1", runId: "run-1" }],
}),
});
expect(loadInternalSessionEntry({ sessionKey, storePath })).toMatchObject({
abortedLastRun: true,
restartRecoveryRuns: [{ lifecycleGeneration: "generation-1", runId: "run-1" }],
sessionId: "healthy-session",
});
});
it("rejects core recovery state from runtime-escaped creation inputs", async () => {
const mainRestartRecovery = {
chargedAttempts: 1,
cycleId: "cycle-injected",
revision: 1,
};
const patchSessionKey = "agent:main:patch-created";
await patchSessionEntry({
fallbackEntry: {
mainRestartRecovery,
sessionId: "patch-created",
updatedAt: 10,
} as unknown as SessionEntry,
sessionKey: patchSessionKey,
storePath,
update: () => ({ updatedAt: 20 }),
});
expect(loadInternalSessionEntry({ sessionKey: patchSessionKey, storePath })).not.toHaveProperty(
"mainRestartRecovery",
);
const upsertSessionKey = "agent:main:upsert-created";
await upsertSessionEntry({
entry: {
mainRestartRecovery,
sessionId: "upsert-created",
updatedAt: 10,
} as unknown as SessionEntry,
sessionKey: upsertSessionKey,
storePath,
});
expect(
loadInternalSessionEntry({ sessionKey: upsertSessionKey, storePath }),
).not.toHaveProperty("mainRestartRecovery");
});
});
+351 -2
View File
@@ -5,7 +5,12 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest";
import {
appendTranscriptEvent,
appendTranscriptMessage,
loadSessionEntry as loadInternalSessionEntry,
patchSessionEntry as patchInternalSessionEntry,
replaceSessionEntry as replaceInternalSessionEntry,
} from "../config/sessions/session-accessor.js";
import type { InternalSessionEntry } from "../config/sessions/types.js";
import type { SessionEntry as ConfigSessionEntry } from "./config-types.js";
import {
cleanupSessionLifecycleArtifacts,
deleteSessionEntry,
@@ -26,6 +31,14 @@ import {
const DAY_MS = 24 * 60 * 60 * 1000;
const LEGACY_TRANSCRIPT_INSPECTION_MAX_BYTES = 16 * 1024 * 1024;
const sessionEntryKeepsRecoveryPrivate: "mainRestartRecovery" extends keyof SessionEntry
? false
: true = true;
const configSessionEntryKeepsRecoveryPrivate: "mainRestartRecovery" extends keyof ConfigSessionEntry
? false
: true = true;
void sessionEntryKeepsRecoveryPrivate;
void configSessionEntryKeepsRecoveryPrivate;
describe("session-store-runtime compatibility surface", () => {
let tempDir: string;
@@ -145,7 +158,7 @@ describe("session-store-runtime compatibility surface", () => {
agentId: "main",
sessionsDir: path.dirname(storePath),
});
expect(transcriptPath).toBe(path.join(tempDir, `${sessionId}.jsonl`));
expect(transcriptPath).toBe(path.join(fs.realpathSync(tempDir), `${sessionId}.jsonl`));
expect(fs.statSync(transcriptPath).mode & 0o777).toBe(0o600);
expect(
fs
@@ -162,7 +175,9 @@ describe("session-store-runtime compatibility surface", () => {
]);
const codexSidecarPath = `${transcriptPath}.codex-app-server.json`;
fs.writeFileSync(codexSidecarPath, "{}\n", { mode: 0o600 });
expect(codexSidecarPath).toBe(path.join(tempDir, `${sessionId}.jsonl.codex-app-server.json`));
expect(codexSidecarPath).toBe(
path.join(fs.realpathSync(tempDir), `${sessionId}.jsonl.codex-app-server.json`),
);
});
it("preserves beta.5 doctor archives after deleting the compatibility row", async () => {
@@ -320,6 +335,104 @@ describe("session-store-runtime compatibility surface", () => {
expect(fs.existsSync(storePath)).toBe(false);
});
it("keeps recovery state private across deprecated whole-store mutations", async () => {
const keptKey = "agent:main:telegram:direct:compat-recovery";
const removedKey = "agent:main:telegram:direct:compat-removed";
const mainRestartRecovery = {
chargedAttempts: 2,
cycleId: "compat-cycle",
revision: 4,
};
await seedSessionEntry(keptKey, {
abortedLastRun: true,
model: "gpt-5.5",
restartRecoveryRuns: [{ lifecycleGeneration: "compat-generation", runId: "compat-run" }],
sessionId: "compat-recovery-session",
updatedAt: 10,
});
await patchInternalSessionEntry(
{ agentId: "main", sessionKey: keptKey, storePath },
() => ({ mainRestartRecovery }) as Partial<InternalSessionEntry>,
);
await seedSessionEntry(removedKey, {
sessionId: "compat-removed-session",
updatedAt: 10,
});
const compatibilityStore = loadSessionStore(storePath);
expect(Object.keys(compatibilityStore)).toContain(keptKey);
expect(compatibilityStore[keptKey]).not.toHaveProperty("mainRestartRecovery");
await updateSessionStore(
storePath,
(store) => {
expect(store[keptKey]).not.toHaveProperty("mainRestartRecovery");
const escapedStore = store as unknown as Record<string, InternalSessionEntry>;
escapedStore[keptKey] = {
mainRestartRecovery: {
chargedAttempts: 99,
cycleId: "injected-cycle",
revision: 99,
},
model: "gpt-5.6",
sessionId: "compat-recovery-session",
updatedAt: 20,
};
delete escapedStore[removedKey];
escapedStore["agent:main:telegram:direct:compat-created"] = {
mainRestartRecovery: {
chargedAttempts: 1,
cycleId: "created-injection",
revision: 1,
},
sessionId: "compat-created-session",
updatedAt: 20,
};
},
{ skipMaintenance: true },
);
expect(loadInternalSessionEntry({ sessionKey: keptKey, storePath })).toMatchObject({
abortedLastRun: true,
mainRestartRecovery,
model: "gpt-5.6",
restartRecoveryRuns: [{ lifecycleGeneration: "compat-generation", runId: "compat-run" }],
});
expect(loadInternalSessionEntry({ sessionKey: removedKey, storePath })).toBeUndefined();
expect(
loadInternalSessionEntry({
sessionKey: "agent:main:telegram:direct:compat-created",
storePath,
}),
).not.toHaveProperty("mainRestartRecovery");
await updateSessionStore(
storePath,
(store) => {
const escapedStore = store as unknown as Record<string, InternalSessionEntry>;
escapedStore[keptKey] = {
...escapedStore[keptKey]!,
mainRestartRecovery: {
chargedAttempts: 99,
cycleId: "replacement-injection",
revision: 99,
},
sessionId: "compat-replacement-session",
};
},
{ skipMaintenance: true },
);
expect(loadInternalSessionEntry({ sessionKey: keptKey, storePath })).toMatchObject({
abortedLastRun: false,
sessionId: "compat-replacement-session",
});
const replacedEntry = loadInternalSessionEntry({
sessionKey: keptKey,
storePath,
}) as InternalSessionEntry | undefined;
expect(replacedEntry?.mainRestartRecovery).toBeUndefined();
expect(replacedEntry?.restartRecoveryRuns).toBeUndefined();
});
it("serializes compatibility callbacks with concurrent row writes", async () => {
const sessionKey = "agent:main:serialized";
await seedSessionEntry(sessionKey, {
@@ -483,6 +596,242 @@ describe("session-store-runtime compatibility surface", () => {
});
});
it("hides core recovery state and preserves it across public mutations", async () => {
const sessionKey = "agent:main:recovery-owned";
const mainRestartRecovery = {
chargedAttempts: 1,
cycleId: "cycle-1",
reservation: {
attempt: 1,
lifecycleGeneration: "generation-1",
runId: "run-1",
},
revision: 1,
};
await replaceInternalSessionEntry({ sessionKey, storePath }, {
abortedLastRun: true,
mainRestartRecovery,
model: "gpt-5.5",
restartRecoveryRuns: [{ lifecycleGeneration: "generation-1", runId: "run-1" }],
sessionId: "session-recovery",
updatedAt: 10,
} as InternalSessionEntry);
expect(getSessionEntry({ sessionKey, storePath })).not.toHaveProperty("mainRestartRecovery");
expect(listSessionEntries({ storePath })[0]?.entry).not.toHaveProperty("mainRestartRecovery");
await patchSessionEntry({
sessionKey,
storePath,
update: (entry) => {
entry.restartRecoveryRuns?.splice(0);
return {
abortedLastRun: false,
mainRestartRecovery: undefined,
model: "gpt-5.6",
restartRecoveryRuns: undefined,
} as unknown as Partial<SessionEntry>;
},
});
expect(loadInternalSessionEntry({ sessionKey, storePath })).toMatchObject({
abortedLastRun: true,
mainRestartRecovery,
model: "gpt-5.6",
restartRecoveryRuns: [{ lifecycleGeneration: "generation-1", runId: "run-1" }],
});
await updateSessionStoreEntry({
sessionKey,
storePath,
update: () => ({ abortedLastRun: false, restartRecoveryRuns: undefined }),
});
expect(loadInternalSessionEntry({ sessionKey, storePath })).toMatchObject({
abortedLastRun: true,
mainRestartRecovery,
restartRecoveryRuns: [{ lifecycleGeneration: "generation-1", runId: "run-1" }],
});
await upsertSessionEntry({
sessionKey,
storePath,
entry: {
sessionId: "session-recovery",
updatedAt: 20,
},
});
expect(loadInternalSessionEntry({ sessionKey, storePath })).toMatchObject({
abortedLastRun: true,
mainRestartRecovery,
restartRecoveryRuns: [{ lifecycleGeneration: "generation-1", runId: "run-1" }],
sessionId: "session-recovery",
updatedAt: 20,
});
expect(loadInternalSessionEntry({ sessionKey, storePath })?.model).toBeUndefined();
});
it("clears core recovery state when public replacements change session identity", async () => {
const patchKey = "agent:main:telegram:direct:patch-rotation";
const upsertKey = "agent:main:telegram:direct:upsert-rotation";
const upsertStorePath = path.join(tempDir, "upsert-sessions.json");
const mainRestartRecovery = {
chargedAttempts: 1,
cycleId: "rotation-cycle",
revision: 1,
};
await seedSessionEntry(patchKey, {
abortedLastRun: true,
restartRecoveryRuns: [{ lifecycleGeneration: "patch-generation", runId: "patch-run" }],
sessionId: "patch-before",
updatedAt: 10,
});
await patchInternalSessionEntry(
{ agentId: "main", sessionKey: patchKey, storePath },
() =>
({
abortedLastRun: true,
mainRestartRecovery,
restartRecoveryRuns: [{ lifecycleGeneration: "patch-generation", runId: "patch-run" }],
}) as Partial<InternalSessionEntry>,
);
await upsertSessionEntry({
agentId: "main",
entry: { sessionId: "upsert-before", updatedAt: 10 },
sessionKey: upsertKey,
storePath: upsertStorePath,
});
await patchInternalSessionEntry(
{ agentId: "main", sessionKey: upsertKey, storePath: upsertStorePath },
() =>
({
abortedLastRun: true,
mainRestartRecovery,
restartRecoveryRuns: [{ lifecycleGeneration: "upsert-generation", runId: "upsert-run" }],
}) as Partial<InternalSessionEntry>,
);
await patchSessionEntry({
replaceEntry: true,
sessionKey: patchKey,
storePath,
update: () => ({ sessionId: "patch-after", updatedAt: 20 }),
});
await upsertSessionEntry({
entry: {
abortedLastRun: true,
restartRecoveryRuns: [{ lifecycleGeneration: "upsert-generation", runId: "upsert-run" }],
sessionId: "upsert-after",
updatedAt: 20,
},
sessionKey: upsertKey,
storePath: upsertStorePath,
});
expect(loadInternalSessionEntry({ sessionKey: patchKey, storePath })).toMatchObject({
abortedLastRun: false,
sessionId: "patch-after",
});
expect(
loadInternalSessionEntry({ sessionKey: patchKey, storePath })?.restartRecoveryRuns,
).toBeUndefined();
expect(loadInternalSessionEntry({ sessionKey: patchKey, storePath })).not.toHaveProperty(
"mainRestartRecovery",
);
expect(
loadInternalSessionEntry({ sessionKey: upsertKey, storePath: upsertStorePath }),
).toMatchObject({
sessionId: "upsert-after",
});
expect(
loadInternalSessionEntry({ sessionKey: upsertKey, storePath: upsertStorePath })
?.abortedLastRun,
).not.toBe(true);
expect(
loadInternalSessionEntry({ sessionKey: upsertKey, storePath: upsertStorePath })
?.restartRecoveryRuns,
).toBeUndefined();
expect(
loadInternalSessionEntry({ sessionKey: upsertKey, storePath: upsertStorePath }),
).not.toHaveProperty("mainRestartRecovery");
});
it("clears core recovery state when public patches change session identity", async () => {
const patchKey = "agent:main:telegram:direct:patch-rotation";
const updateKey = "agent:main:telegram:direct:update-rotation";
const updateStorePath = path.join(tempDir, "update-patch-sessions.json");
const mainRestartRecovery = {
chargedAttempts: 1,
cycleId: "rotation-cycle",
revision: 1,
};
await seedSessionEntry(patchKey, {
abortedLastRun: true,
restartRecoveryRuns: [{ lifecycleGeneration: "patch-generation", runId: "patch-run" }],
sessionId: "patch-before",
updatedAt: 10,
});
await upsertSessionEntry({
agentId: "main",
entry: { sessionId: "update-before", updatedAt: 10 },
sessionKey: updateKey,
storePath: updateStorePath,
});
await patchInternalSessionEntry(
{ agentId: "main", sessionKey: patchKey, storePath },
() =>
({
abortedLastRun: true,
mainRestartRecovery,
restartRecoveryRuns: [{ lifecycleGeneration: "patch-generation", runId: "patch-run" }],
}) as Partial<InternalSessionEntry>,
);
await patchInternalSessionEntry(
{ agentId: "main", sessionKey: updateKey, storePath: updateStorePath },
() =>
({
abortedLastRun: true,
mainRestartRecovery,
restartRecoveryRuns: [{ lifecycleGeneration: "update-generation", runId: "update-run" }],
}) as Partial<InternalSessionEntry>,
);
await patchSessionEntry({
sessionKey: patchKey,
skipMaintenance: true,
storePath,
update: () => ({ sessionId: "patch-after", updatedAt: 20 }),
});
await updateSessionStoreEntry({
sessionKey: updateKey,
skipMaintenance: true,
storePath: updateStorePath,
update: () => ({ sessionId: "update-after", updatedAt: 20 }),
});
expect(loadInternalSessionEntry({ sessionKey: patchKey, storePath })).toMatchObject({
abortedLastRun: false,
sessionId: "patch-after",
});
expect(
loadInternalSessionEntry({ sessionKey: patchKey, storePath })?.restartRecoveryRuns,
).toBeUndefined();
expect(loadInternalSessionEntry({ sessionKey: patchKey, storePath })).not.toHaveProperty(
"mainRestartRecovery",
);
expect(
loadInternalSessionEntry({ sessionKey: updateKey, storePath: updateStorePath }),
).toMatchObject({
abortedLastRun: false,
sessionId: "update-after",
});
expect(
loadInternalSessionEntry({ sessionKey: updateKey, storePath: updateStorePath })
?.restartRecoveryRuns,
).toBeUndefined();
expect(
loadInternalSessionEntry({ sessionKey: updateKey, storePath: updateStorePath }),
).not.toHaveProperty("mainRestartRecovery");
});
it("preserves resolved maintenance settings through entry patches", async () => {
const staleSessionKey = "agent:main:stale";
const activeSessionKey = "agent:main:active";
+46 -68
View File
@@ -17,13 +17,9 @@ import {
loadTranscriptEventsSync as loadAccessorTranscriptEventsSync,
listSessionEntries as listAccessorSessionEntries,
loadSessionEntry,
patchSessionEntry as patchAccessorSessionEntry,
readSessionUpdatedAt as readAccessorSessionUpdatedAt,
readTranscriptStatsSync as readAccessorTranscriptStatsSync,
replaceSessionEntry,
resolveTranscriptSessionKeyBySessionId as resolveAccessorTranscriptSessionKeyBySessionId,
type SessionAccessScope,
updateSessionEntry,
} from "../config/sessions/session-accessor.js";
import { resolveSqliteTargetFromSessionStorePath } from "../config/sessions/session-sqlite-target.js";
import {
@@ -31,11 +27,26 @@ import {
parseSqliteSessionFileMarker,
} from "../config/sessions/sqlite-marker.js";
import { resolveSessionStoreEntry as resolveSessionStoreEntryFromStore } from "../config/sessions/store-entry.js";
import { normalizeResolvedMaintenanceConfigInput } from "../config/sessions/store-maintenance.js";
import type { ResolvedSessionMaintenanceConfigInput } from "../config/sessions/store.js";
import type { AmbientTranscriptWatermark, SessionEntry } from "../config/sessions/types.js";
import type {
AmbientTranscriptWatermark,
InternalSessionEntry,
SessionEntry,
} from "../config/sessions/types.js";
import { replaceFileAtomicSync } from "../infra/replace-file.js";
import { resolveAgentIdFromSessionKey } from "../routing/session-key.js";
import {
projectPluginSessionEntry,
projectPluginSessionStore,
reconcilePluginSessionStore,
type SessionStoreReadParams,
toSessionAccessScope,
} from "./session-store-runtime-internal.js";
import {
patchPluginSessionEntry,
updatePluginSessionStoreEntry,
upsertPluginSessionEntry,
} from "./session-store-runtime-mutations.js";
import type { SessionTranscriptEvent } from "./session-transcript-runtime.js";
const SQLITE_SESSION_STORE_BACKUP_SUFFIXES = ["", "-wal", "-shm", "-journal"] as const;
@@ -45,15 +56,6 @@ const LEGACY_TRANSCRIPT_INSPECTION_MAX_BYTES = 16 * 1024 * 1024;
// caller. This map is not a general replacement for target-aware SDK methods.
const legacyStoreAgentIds = new Map<string, string>();
type SessionStoreReadParams = {
agentId?: string;
env?: NodeJS.ProcessEnv;
hydrateSkillPromptRefs?: boolean;
readConsistency?: "latest";
sessionKey: string;
storePath?: string;
};
type SessionStoreListParams = Partial<Omit<SessionStoreReadParams, "sessionKey">>;
type SessionStoreEntrySummary = {
@@ -93,10 +95,6 @@ type PatchSessionEntryParams = SessionStoreReadParams & {
update: SessionStoreEntryPatch;
};
type ReadAmbientTranscriptWatermarkParams = SessionStoreReadParams & {
key: string;
};
type UpdateSessionStoreEntryParams = {
storePath: string;
sessionKey: string;
@@ -106,8 +104,10 @@ type UpdateSessionStoreEntryParams = {
requireWriteSuccess?: boolean;
};
type UpsertSessionEntryParams = SessionStoreReadParams & {
entry: SessionEntry;
type UpsertSessionEntryParams = SessionStoreReadParams & { entry: SessionEntry };
type ReadAmbientTranscriptWatermarkParams = SessionStoreReadParams & {
key: string;
};
type DeleteSessionEntryParams = SessionStoreReadParams & {
@@ -131,21 +131,6 @@ type SessionLifecycleArtifactsCleanupResult = {
removedEntries: number;
};
function toSessionAccessScope(params: SessionStoreReadParams): SessionAccessScope {
// Maintainer note: keep this adapter narrow so plugin callers retain the
// object-parameter API while internal accessor-only options stay private.
return {
sessionKey: params.sessionKey,
...(params.agentId !== undefined ? { agentId: params.agentId } : {}),
...(params.env !== undefined ? { env: params.env } : {}),
...(params.hydrateSkillPromptRefs !== undefined
? { hydrateSkillPromptRefs: params.hydrateSkillPromptRefs }
: {}),
...(params.readConsistency !== undefined ? { readConsistency: params.readConsistency } : {}),
...(params.storePath !== undefined ? { storePath: params.storePath } : {}),
};
}
function resolveLegacySessionStoreTarget(storePath: string): {
agentId?: string;
storePath: string;
@@ -244,13 +229,14 @@ export function loadSessionStore(
hydrateSkillPromptRefs: options.hydrateSkillPromptRefs,
}).map(({ sessionKey, entry }) => {
const sessionId = entry.sessionId?.trim();
if (entry.sessionFile || !sessionId) {
return [sessionKey, entry];
const projectedEntry = projectPluginSessionEntry(entry as InternalSessionEntry);
if (projectedEntry.sessionFile || !sessionId) {
return [sessionKey, projectedEntry];
}
return [
sessionKey,
{
...entry,
...projectedEntry,
// SQLite does not persist sessionFile. Beta.5 needs a locator only in
// this detached projection so its file-based doctor reaches the bridge.
sessionFile: formatSqliteSessionFileMarker({
@@ -285,9 +271,17 @@ export async function updateSessionStore<T>(
storePath: target.storePath,
skipMaintenance: options.skipMaintenance,
update: async (store) => {
const result = await mutator(store);
const internalStore = store as Record<string, InternalSessionEntry>;
const publicStore = projectPluginSessionStore(internalStore);
const result = await mutator(publicStore);
const persist = !options.skipSaveWhenResult?.(result);
if (persist) {
// The deprecated callback owns public row changes and deletions, but
// core recovery coordination remains invisible and non-overwritable.
reconcilePluginSessionStore({ internalStore, publicStore });
}
return {
persist: !options.skipSaveWhenResult?.(result),
persist,
result,
};
},
@@ -346,7 +340,8 @@ export function resolveSessionStoreEntry(params: {
/** Loads one session entry by agent/session identity. */
export function getSessionEntry(params: SessionStoreReadParams): SessionEntry | undefined {
return loadSessionEntry(toSessionAccessScope(params));
const entry = loadSessionEntry(toSessionAccessScope(params));
return entry ? projectPluginSessionEntry(entry) : undefined;
}
/** Lists session entries for one agent. */
@@ -360,7 +355,10 @@ export function listSessionEntries(
? { hydrateSkillPromptRefs: params.hydrateSkillPromptRefs }
: {}),
...(params.storePath !== undefined ? { storePath: params.storePath } : {}),
});
}).map(({ sessionKey, entry }) => ({
sessionKey,
entry: projectPluginSessionEntry(entry),
}));
}
/** Reads transcript events for a live SQLite-backed session identity. */
@@ -399,17 +397,7 @@ export function resolveTranscriptSessionKeyBySessionId(params: {
export async function patchSessionEntry(
params: PatchSessionEntryParams,
): Promise<SessionEntry | null> {
return await patchAccessorSessionEntry(toSessionAccessScope(params), params.update, {
fallbackEntry: params.fallbackEntry,
maintenanceConfig:
params.maintenanceConfig !== undefined
? normalizeResolvedMaintenanceConfigInput(params.maintenanceConfig)
: undefined,
preserveActivity: params.preserveActivity,
requireWriteSuccess: params.requireWriteSuccess,
replaceEntry: params.replaceEntry,
skipMaintenance: params.skipMaintenance,
});
return await patchPluginSessionEntry(params);
}
/** Reads the last activity timestamp for one session entry. */
@@ -430,23 +418,12 @@ export function readAmbientTranscriptWatermark(
export async function updateSessionStoreEntry(
params: UpdateSessionStoreEntryParams,
): Promise<SessionEntry | null> {
return await updateSessionEntry(
{
sessionKey: params.sessionKey,
storePath: params.storePath,
},
params.update,
{
skipMaintenance: params.skipMaintenance,
takeCacheOwnership: params.takeCacheOwnership,
requireWriteSuccess: params.requireWriteSuccess,
},
);
return await updatePluginSessionStoreEntry(params);
}
/** Replaces or creates one session entry by agent/session identity. */
export async function upsertSessionEntry(params: UpsertSessionEntryParams): Promise<void> {
await replaceSessionEntry(toSessionAccessScope(params), params.entry);
await upsertPluginSessionEntry(params);
}
/** Deletes one session entry by agent/session identity. */
@@ -539,4 +516,5 @@ export {
resolveThreadFlag,
} from "../config/sessions/reset.js";
export { resolveSendPolicy } from "../sessions/send-policy.js";
export type { SessionEntry, SessionScope } from "../config/sessions/types.js";
export type { SessionEntry } from "../config/sessions/types.js";
export type { SessionScope } from "../config/sessions/types.js";
@@ -289,6 +289,11 @@ describe("plugin session extension SessionEntry projection", () => {
description: "bad slot",
sessionEntrySlotKey: "updatedAt",
});
api.registerSessionExtension({
namespace: "main-recovery",
description: "bad main recovery slot",
sessionEntrySlotKey: "mainRestartRecovery",
});
api.registerSessionExtension({
namespace: "recovery",
description: "bad fresh-main slot",
@@ -305,6 +310,10 @@ describe("plugin session extension SessionEntry projection", () => {
pluginId: "slot-collision",
message: "sessionEntrySlotKey is reserved by SessionEntry: updatedAt",
},
{
pluginId: "slot-collision",
message: "sessionEntrySlotKey is reserved by SessionEntry: mainRestartRecovery",
},
{
pluginId: "slot-collision",
message: "sessionEntrySlotKey is reserved by SessionEntry: subagentRecovery",
+2 -1
View File
@@ -1,5 +1,5 @@
/** Reserves session-entry keys so plugin extension slots cannot collide with core session state. */
import type { SessionEntry } from "../config/sessions/types.js";
import type { InternalSessionEntry as SessionEntry } from "../config/sessions/types.js";
const SESSION_ENTRY_RESERVED_SLOT_KEY_LIST = [
"__proto__",
@@ -33,6 +33,7 @@ const SESSION_ENTRY_RESERVED_SLOT_KEY_LIST = [
"subagentControlScope",
"inheritedToolDeny",
"inheritedToolAllow",
"mainRestartRecovery",
"subagentRecovery",
"pluginOwnerId",
"systemSent",
+11
View File
@@ -17,6 +17,8 @@ export type InputProvenance = {
sourceTool?: string;
};
export const MAIN_SESSION_RESTART_RECOVERY_SOURCE_TOOL = "main_session_restart_recovery" as const;
export const INTER_SESSION_PROMPT_PREFIX_BASE = "[Inter-session message]";
const AGENT_MEDIATED_COMPLETION_SOURCE_TOOLS = [
"agent_harness_task",
@@ -74,6 +76,15 @@ export function isInterSessionInputProvenance(value: unknown): boolean {
return normalizeInputProvenance(value)?.kind === "inter_session";
}
export function isMainSessionRestartRecoveryInputProvenance(value: unknown): boolean {
const provenance = normalizeInputProvenance(value);
return (
provenance?.kind === "internal_system" &&
normalizeOptionalString(provenance.sourceTool)?.toLowerCase() ===
MAIN_SESSION_RESTART_RECOVERY_SOURCE_TOOL
);
}
const AGENT_MEDIATED_COMPLETION_SOURCE_TOOL_SET: ReadonlySet<string> = new Set(
AGENT_MEDIATED_COMPLETION_SOURCE_TOOLS,
);
@@ -428,6 +428,7 @@ export async function beginSessionWorkAdmission(params: {
return createSessionWorkAdmissionHandoff(admission, lease);
},
release,
released: admission.released,
run: async <T>(run: () => Promise<T>) => {
const current = new Set(CURRENT_SESSION_WORK_ADMISSIONS.getStore());
current.add(admission);
@@ -5,6 +5,7 @@ import { normalizeSessionIdentities } from "./session-lifecycle-identity.js";
export type SessionWorkAdmissionLease = {
createHandoff: () => string;
release: () => void;
released: Promise<void>;
run: <T>(run: () => Promise<T>) => Promise<T>;
};