mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 10:16:44 +00:00
fix(session): stop repeated restart recovery after retry budget (#96230)
* fix(session): bound restart recovery across gateway restarts Co-authored-by: Mason Huang <masonxhuang@proton.me> * style(session): format restart recovery changes * fix(session): fence recovery notice state * test(session): align internal recovery types * test(session): type recovery completion fixture * style(session): satisfy recovery lint rules * test(plugin-sdk): split recovery boundary coverage * test(plugin-sdk): use tracked recovery temp dirs * fix(session): preserve stale admission errors * fix(session): close recovery lifecycle races * fix(session): rotate rejected recovery dispatch ids * fix(session): enforce recovery quarantine boundaries * fix(session): close restart recovery exhaustion gaps * fix(session): preserve recovery command narrowing * test(session): type restart recovery fixtures * fix(session): enforce restart recovery quarantine * fix(session): close restart recovery races * fix(session): type recovery lifecycle state * fix(session): enforce recovery ownership fences * fix(session): fence recovery lifecycle completion * fix(session): quarantine cross-process recovery * fix(session): retire stale recovery ownership * fix(session): restore rejected recovery admission * fix(session): preserve restored recovery target * fix(session): retry pending recovery races * fix(session): preserve concurrent recovery aborts * fix(session): quarantine recovery lifecycle state * fix(session): settle exact recovery delivery fences * fix(session): keep recovery restoration internal * fix(session): close recovery admission races * fix(session): retry durable recovery rollback * fix(session): guard cached recovery settlement * fix(session): close recovery ownership gaps * fix(session): resume after delayed recovery rollback * fix(session): finalize unrecoverable restart state * fix(session): retry admitted recovery restoration * fix(session): preserve recovery cleanup guarantees * fix(session): restore failed terminal settlement * chore(plugin-sdk): refresh API baseline * fix(session): close recovery owner retry races * test(session): type recovery owner fixture * refactor(session): remove obsolete owner validator * fix(session): preserve explicit abort recovery --------- Co-authored-by: Mason Huang <masonxhuang@proton.me> Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
co-authored by
Mason Huang
Peter Steinberger
parent
90ab572582
commit
0a868179e4
@@ -1,2 +1,2 @@
|
||||
dc2bf410e15972389b8e60d772f92731130148845dff24294fae4786d3b9f41a plugin-sdk-api-baseline.json
|
||||
ce076f7e71bb1ffbd0eead42a178dd069e15693348c4fd69b2ed291d51a8fcd7 plugin-sdk-api-baseline.jsonl
|
||||
16f7664192e83f3e689685ecc917fa0578db5686646d3e7277ba866f07e1002d plugin-sdk-api-baseline.json
|
||||
f8c7d2a77b258f347bd6b741461990c57c4f70d85d91d05b2526829e2980df60 plugin-sdk-api-baseline.jsonl
|
||||
|
||||
@@ -53,6 +53,7 @@ openclaw agent --agent ops --message "Run locally" --local
|
||||
- Pass exactly one of `--message` or `--message-file`. `--message-file` strips a leading UTF-8 BOM and preserves multiline content; it rejects files that are not valid UTF-8.
|
||||
- Slash commands (for example `/compact`) cannot run through `--message`. The CLI rejects them and points you at the first-class command instead (`openclaw sessions compact <key>` for compaction).
|
||||
- `--local` and embedded fallback runs are one-shot: bundled MCP loopback resources and warm Claude stdio sessions opened for the run are retired after the reply, so scripted invocations do not leave local child processes running. Gateway-backed runs keep Gateway-owned MCP loopback resources under the running Gateway process instead.
|
||||
- Standalone embedded execution (`--local` and transport fallback) refuses to reuse an existing main session while restart recovery is pending. Run the turn through a healthy Gateway, or reset it there with `/new` or `/reset`; an independent embedded process cannot safely coordinate that recovery owner with the Gateway scanner.
|
||||
- With `--agent`, `--channel` and `--to` together, session routing follows the channel's canonical recipient and `session.dmScope`. Channels with a stable outbound-only recipient identity use a provider-owned session isolated from the agent's main session. `--reply-channel` and `--reply-account` affect delivery only.
|
||||
- `--session-key` selects an explicit session key. Agent-prefixed keys must use `agent:<agent-id>:<session-key>`, and `--agent` must match the key's agent id when both are given. Bare non-sentinel keys scope to `--agent` when supplied, or to the configured default agent otherwise; for example `--agent ops --session-key incident-42` routes to `agent:ops:incident-42`. The literal keys `global` and `unknown` stay unscoped only when no `--agent` is supplied.
|
||||
- `--json` reserves stdout for the JSON response; Gateway, plugin, and embedded-fallback diagnostics go to stderr so scripts can parse stdout directly.
|
||||
|
||||
@@ -170,7 +170,7 @@ Flags:
|
||||
<Accordion title="State and integrity">
|
||||
- Session lock file inspection and stale lock cleanup.
|
||||
- Session transcript repair for duplicated prompt-rewrite branches created by affected 2026.4.24 builds.
|
||||
- Wedged subagent restart-recovery tombstone detection, with `--fix` support for clearing stale aborted recovery flags so startup does not keep treating the child as restart-aborted.
|
||||
- Wedged main-session and subagent restart-recovery tombstone detection. Doctor reports the blocked sessions and only repairs stale aborted flags that conflict with an existing tombstone; it does not re-enable automatic recovery.
|
||||
- State integrity and permissions checks (sessions, transcripts, state dir).
|
||||
- Config file permission checks (chmod 600) when running locally.
|
||||
- Model auth health: checks OAuth expiry, can refresh expiring tokens, and reports auth-profile cooldown/disabled states.
|
||||
|
||||
@@ -10,8 +10,9 @@ title: "Restart recovery"
|
||||
Restarting the gateway does not lose agent state. Conversations, transcripts,
|
||||
scheduled jobs, background task records, and queued outbound messages all live
|
||||
on disk, and work that was interrupted mid-turn is detected and resumed
|
||||
automatically after the gateway comes back up. No manual intervention is
|
||||
required, and there is nothing to configure: recovery is always on.
|
||||
automatically after the gateway comes back up. Recovery is always on and
|
||||
normally needs no manual intervention. Repeatedly failing recovery is bounded
|
||||
and may quarantine one session until you inspect or replace it.
|
||||
|
||||
This page describes what survives a restart, how interrupted work is detected,
|
||||
and what the automatic resume looks like.
|
||||
@@ -75,12 +76,26 @@ A few seconds after startup, the gateway re-dispatches each marked session
|
||||
with a synthetic system message telling the agent its previous turn was
|
||||
interrupted by a restart and to continue from the existing transcript. If a
|
||||
final reply had already been produced but not delivered, its text is included
|
||||
so the agent can deliver it instead of redoing the work. Recovery retries up
|
||||
to 3 times with exponential backoff. Every retry reuses one durable dispatch
|
||||
identifier, so an ambiguous connection failure cannot start the same recovery
|
||||
twice. Completed and unresumable Control UI turns also retain bounded durable
|
||||
idempotency tombstones, allowing a reconnecting outbox to retire them without
|
||||
re-executing the request.
|
||||
so the agent can deliver it instead of redoing the work.
|
||||
|
||||
Startup reconciliation retries transient failures up to three times with
|
||||
exponential backoff. Separately, each interrupted main-session cycle has a
|
||||
durable budget of three charged automatic dispatch attempts, retained across
|
||||
gateway restarts. OpenClaw charges an attempt before dispatch, refunds it when
|
||||
the gateway explicitly rejects the request before acceptance, and retains the
|
||||
charge when a post-dispatch result is uncertain to avoid replaying work.
|
||||
Foreground work that already owns the session keeps automatic recovery out
|
||||
until that work settles.
|
||||
|
||||
After the durable budget is exhausted, the session is tombstoned instead of
|
||||
looping forever. Inspect the failed session and use `/new` or `/reset` to start a
|
||||
replacement. `openclaw doctor --fix` can repair a stale aborted flag that
|
||||
conflicts with a tombstone, but it does not re-enable that recovery cycle.
|
||||
|
||||
Every retry reuses one durable dispatch identifier, so an ambiguous connection
|
||||
failure cannot start the same recovery twice. Completed and unresumable Control
|
||||
UI turns also retain bounded durable idempotency tombstones, allowing a
|
||||
reconnecting outbox to retire them without re-executing the request.
|
||||
|
||||
Message-tool-only replies use a second durable correlation. Before a terminal
|
||||
same-conversation send reaches the channel, the gateway records an unresolved
|
||||
@@ -149,6 +164,9 @@ agent picks up exactly where it left off, on the same channel and thread.
|
||||
- **Crash-loop breaker:** 3 unclean boots within 5 minutes trip a breaker that
|
||||
suppresses auto-start side services on the next boot, so a crashing gateway
|
||||
does not amplify itself. It recovers once the unclean-boot window drains.
|
||||
- **Main-session attempt budget:** three charged automatic dispatch attempts
|
||||
per interrupted cycle; exhaustion tombstones that session until it is
|
||||
inspected and replaced.
|
||||
- **Metrics:** recovery activity is exported via
|
||||
[Prometheus](/gateway/prometheus) as `openclaw_session_recovery_total` and
|
||||
`openclaw_session_recovery_age_seconds`.
|
||||
@@ -167,3 +185,6 @@ agent picks up exactly where it left off, on the same channel and thread.
|
||||
- Work that was never admitted: messages arriving during the drain window are
|
||||
rejected with an explicit restart error rather than silently queued into a
|
||||
dying process.
|
||||
- Standalone embedded turns cannot take over a main session with pending
|
||||
restart recovery because they do not share the gateway's lifecycle owner.
|
||||
Run the turn through the gateway or reset it there with `/new` or `/reset`.
|
||||
|
||||
@@ -0,0 +1,615 @@
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import type { InternalSessionEntry as SessionEntry } from "../config/sessions.js";
|
||||
import {
|
||||
applySessionEntryLifecycleMutation,
|
||||
loadSessionEntry,
|
||||
replaceSessionEntry,
|
||||
} from "../config/sessions/session-accessor.js";
|
||||
import { getAgentEventLifecycleGeneration } from "../infra/agent-events.js";
|
||||
import { runWithAgentCommandRecoveryOwner } from "./agent-command-recovery-owner.js";
|
||||
import type { AgentCommandOpts } from "./command/types.js";
|
||||
import { claimMainSessionRecoveryOwner } from "./main-session-recovery-store.js";
|
||||
|
||||
const recoveryOwnerMocks = vi.hoisted(() => ({
|
||||
scheduleMainSessionRecoveryPendingTarget: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("./main-session-recovery-owner-release.js", () => ({
|
||||
scheduleMainSessionRecoveryPendingTarget:
|
||||
recoveryOwnerMocks.scheduleMainSessionRecoveryPendingTarget,
|
||||
}));
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
const sessionKey = "agent:main:main";
|
||||
|
||||
afterEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe("agent command restart recovery ownership", () => {
|
||||
function createTarget() {
|
||||
const storePath = path.join(tempDirs.make("openclaw-agent-command-owner-"), "sessions.json");
|
||||
return {
|
||||
isNewSession: false,
|
||||
sessionId: "session-1",
|
||||
sessionKey,
|
||||
storePath,
|
||||
};
|
||||
}
|
||||
|
||||
async function write(target: ReturnType<typeof createTarget>, entry: SessionEntry) {
|
||||
await replaceSessionEntry({ sessionKey, storePath: target.storePath }, entry);
|
||||
}
|
||||
|
||||
it("rejects standalone work when interruption appears during preparation", async () => {
|
||||
const target = createTarget();
|
||||
await write(target, { sessionId: target.sessionId, updatedAt: 100 });
|
||||
const run = vi.fn();
|
||||
|
||||
await expect(
|
||||
runWithAgentCommandRecoveryOwner({
|
||||
lifecycleGeneration: getAgentEventLifecycleGeneration(),
|
||||
mode: "reject_uncoordinated",
|
||||
opts: {} as AgentCommandOpts,
|
||||
prepare: async () => {
|
||||
await write(target, {
|
||||
sessionId: target.sessionId,
|
||||
updatedAt: 200,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
});
|
||||
return target;
|
||||
},
|
||||
run,
|
||||
}),
|
||||
).rejects.toThrow("interrupted work pending restart recovery");
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
expect(
|
||||
(loadSessionEntry({ sessionKey, storePath: target.storePath }) as SessionEntry | undefined)
|
||||
?.mainRestartRecovery?.foregroundClaims,
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("refreshes the prepared working copy after claiming a recovery owner", async () => {
|
||||
const base = createTarget();
|
||||
const staleEntry: SessionEntry = {
|
||||
sessionId: base.sessionId,
|
||||
updatedAt: 100,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
};
|
||||
const target = {
|
||||
...base,
|
||||
sessionEntry: { ...staleEntry },
|
||||
sessionStore: { [sessionKey]: { ...staleEntry } },
|
||||
};
|
||||
await write(target, staleEntry);
|
||||
|
||||
await runWithAgentCommandRecoveryOwner({
|
||||
lifecycleGeneration: getAgentEventLifecycleGeneration(),
|
||||
mode: "claim",
|
||||
opts: { runId: "foreground-run" } as AgentCommandOpts,
|
||||
prepare: async () => target,
|
||||
run: async (prepared) => {
|
||||
const claims = prepared.sessionEntry.mainRestartRecovery?.foregroundClaims;
|
||||
expect(claims?.tokens).toEqual([expect.any(String)]);
|
||||
expect(Object.values(claims?.runIdsByClaimId ?? {})).toContain("foreground-run");
|
||||
expect(prepared.sessionStore[sessionKey]).toEqual(prepared.sessionEntry);
|
||||
const completed = { ...prepared.sessionEntry, abortedLastRun: false };
|
||||
await write(target, completed);
|
||||
},
|
||||
});
|
||||
|
||||
const completed = loadSessionEntry({
|
||||
sessionKey,
|
||||
storePath: target.storePath,
|
||||
}) as SessionEntry | undefined;
|
||||
expect(completed?.abortedLastRun).toBe(false);
|
||||
expect(completed?.mainRestartRecovery).toBeUndefined();
|
||||
expect(recoveryOwnerMocks.scheduleMainSessionRecoveryPendingTarget).toHaveBeenLastCalledWith(
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects standalone work owned by a legacy session-key alias", async () => {
|
||||
const target = createTarget();
|
||||
await applySessionEntryLifecycleMutation({
|
||||
storePath: target.storePath,
|
||||
upserts: [
|
||||
{
|
||||
sessionKey: "main",
|
||||
entry: {
|
||||
sessionId: target.sessionId,
|
||||
updatedAt: 100,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
},
|
||||
},
|
||||
],
|
||||
skipMaintenance: true,
|
||||
});
|
||||
const run = vi.fn();
|
||||
|
||||
await expect(
|
||||
runWithAgentCommandRecoveryOwner({
|
||||
lifecycleGeneration: getAgentEventLifecycleGeneration(),
|
||||
mode: "reject_uncoordinated",
|
||||
opts: {} as AgentCommandOpts,
|
||||
prepare: async () => target,
|
||||
run,
|
||||
}),
|
||||
).rejects.toThrow("interrupted work pending restart recovery");
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects a legacy interrupted predecessor after the canonical key is reused", async () => {
|
||||
const base = createTarget();
|
||||
const target = {
|
||||
...base,
|
||||
isNewSession: true,
|
||||
previousSessionId: base.sessionId,
|
||||
sessionId: "replacement-session",
|
||||
};
|
||||
await applySessionEntryLifecycleMutation({
|
||||
storePath: target.storePath,
|
||||
upserts: [
|
||||
{
|
||||
sessionKey,
|
||||
entry: { sessionId: target.sessionId, updatedAt: 200 },
|
||||
},
|
||||
{
|
||||
sessionKey: "main",
|
||||
entry: {
|
||||
sessionId: target.previousSessionId,
|
||||
updatedAt: 100,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
},
|
||||
},
|
||||
],
|
||||
skipMaintenance: true,
|
||||
});
|
||||
const run = vi.fn();
|
||||
|
||||
await expect(
|
||||
runWithAgentCommandRecoveryOwner({
|
||||
lifecycleGeneration: getAgentEventLifecycleGeneration(),
|
||||
mode: "reject_uncoordinated",
|
||||
opts: {} as AgentCommandOpts,
|
||||
prepare: async () => target,
|
||||
run,
|
||||
}),
|
||||
).rejects.toThrow("interrupted work pending restart recovery");
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("allows standalone work when interruption clears during preparation", async () => {
|
||||
const target = createTarget();
|
||||
await write(target, {
|
||||
sessionId: target.sessionId,
|
||||
updatedAt: 100,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
});
|
||||
const run = vi.fn(async () => "ran");
|
||||
|
||||
await expect(
|
||||
runWithAgentCommandRecoveryOwner({
|
||||
lifecycleGeneration: getAgentEventLifecycleGeneration(),
|
||||
mode: "reject_uncoordinated",
|
||||
opts: {} as AgentCommandOpts,
|
||||
prepare: async () => {
|
||||
await write(target, { sessionId: target.sessionId, updatedAt: 200 });
|
||||
return target;
|
||||
},
|
||||
run,
|
||||
}),
|
||||
).resolves.toBe("ran");
|
||||
expect(run).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("runs a Gateway-admitted recovery without acquiring a foreground owner", async () => {
|
||||
const target = createTarget();
|
||||
await write(target, {
|
||||
sessionId: target.sessionId,
|
||||
updatedAt: 200,
|
||||
status: "running",
|
||||
abortedLastRun: false,
|
||||
restartRecoveryRuns: [{ runId: "recovery-run", lifecycleGeneration: "previous" }],
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 3,
|
||||
chargedAttempts: 1,
|
||||
},
|
||||
});
|
||||
const run = vi.fn(async () => "recovered");
|
||||
|
||||
await expect(
|
||||
runWithAgentCommandRecoveryOwner({
|
||||
lifecycleGeneration: getAgentEventLifecycleGeneration(),
|
||||
mode: "claim",
|
||||
opts: { mainRestartRecoveryAdmitted: true } as AgentCommandOpts,
|
||||
prepare: async () => target,
|
||||
run,
|
||||
}),
|
||||
).resolves.toBe("recovered");
|
||||
expect(run).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("restores a Gateway-admitted recovery when command preparation fails", async () => {
|
||||
const target = createTarget();
|
||||
const lifecycleGeneration = getAgentEventLifecycleGeneration();
|
||||
await write(target, {
|
||||
sessionId: target.sessionId,
|
||||
updatedAt: 200,
|
||||
status: "running",
|
||||
abortedLastRun: false,
|
||||
restartRecoveryRuns: [{ runId: "recovery-run", lifecycleGeneration }],
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 3,
|
||||
chargedAttempts: 1,
|
||||
},
|
||||
});
|
||||
const restoredTarget = {
|
||||
sessionId: target.sessionId,
|
||||
sessionKey,
|
||||
storePath: target.storePath,
|
||||
};
|
||||
const restoreAdmittedRecovery = vi.fn(async () => {
|
||||
const entry = loadSessionEntry({ sessionKey, storePath: target.storePath }) as SessionEntry;
|
||||
entry.abortedLastRun = true;
|
||||
await write(target, entry);
|
||||
return restoredTarget;
|
||||
});
|
||||
const run = vi.fn();
|
||||
|
||||
await expect(
|
||||
runWithAgentCommandRecoveryOwner({
|
||||
lifecycleGeneration,
|
||||
mode: "claim",
|
||||
opts: { mainRestartRecoveryAdmitted: true } as AgentCommandOpts,
|
||||
prepare: async () => {
|
||||
throw new Error("model preparation failed");
|
||||
},
|
||||
restoreAdmittedRecovery,
|
||||
run,
|
||||
}),
|
||||
).rejects.toThrow("model preparation failed");
|
||||
|
||||
expect(restoreAdmittedRecovery).toHaveBeenCalledOnce();
|
||||
expect(recoveryOwnerMocks.scheduleMainSessionRecoveryPendingTarget).toHaveBeenCalledWith(
|
||||
restoredTarget,
|
||||
);
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
expect(loadSessionEntry({ sessionKey, storePath: target.storePath })).toMatchObject({
|
||||
abortedLastRun: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps retrying admitted recovery restoration after immediate store failures", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const target = createTarget();
|
||||
const restoredTarget = {
|
||||
sessionId: target.sessionId,
|
||||
sessionKey,
|
||||
storePath: target.storePath,
|
||||
};
|
||||
let failures = 0;
|
||||
const restoreAdmittedRecovery = vi.fn(async () => {
|
||||
if (failures < 3) {
|
||||
failures += 1;
|
||||
throw new Error("transient session-store failure");
|
||||
}
|
||||
return restoredTarget;
|
||||
});
|
||||
const recovery = runWithAgentCommandRecoveryOwner({
|
||||
lifecycleGeneration: getAgentEventLifecycleGeneration(),
|
||||
mode: "claim",
|
||||
opts: { mainRestartRecoveryAdmitted: true } as AgentCommandOpts,
|
||||
prepare: async () => {
|
||||
throw new Error("model preparation failed");
|
||||
},
|
||||
restoreAdmittedRecovery,
|
||||
run: vi.fn(),
|
||||
});
|
||||
const rejected = expect(recovery).rejects.toThrow("model preparation failed");
|
||||
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
await rejected;
|
||||
expect(restoreAdmittedRecovery).toHaveBeenCalledTimes(3);
|
||||
expect(recoveryOwnerMocks.scheduleMainSessionRecoveryPendingTarget).toHaveBeenCalledWith(
|
||||
undefined,
|
||||
);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
expect(restoreAdmittedRecovery).toHaveBeenCalledTimes(4);
|
||||
expect(recoveryOwnerMocks.scheduleMainSessionRecoveryPendingTarget).toHaveBeenCalledWith(
|
||||
restoredTarget,
|
||||
);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects ordinary work while an admitted recovery is still running", async () => {
|
||||
const target = createTarget();
|
||||
const lifecycleGeneration = getAgentEventLifecycleGeneration();
|
||||
await write(target, {
|
||||
sessionId: target.sessionId,
|
||||
updatedAt: 200,
|
||||
status: "running",
|
||||
abortedLastRun: false,
|
||||
restartRecoveryRuns: [{ runId: "recovery-run", lifecycleGeneration: "gateway-generation" }],
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 3,
|
||||
chargedAttempts: 1,
|
||||
},
|
||||
});
|
||||
const run = vi.fn();
|
||||
|
||||
await expect(
|
||||
runWithAgentCommandRecoveryOwner({
|
||||
lifecycleGeneration,
|
||||
mode: "reject_uncoordinated",
|
||||
opts: {} as AgentCommandOpts,
|
||||
prepare: async () => target,
|
||||
run,
|
||||
}),
|
||||
).rejects.toThrow("interrupted work pending restart recovery");
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("fences the durable predecessor during an automatic freshness rollover", async () => {
|
||||
const base = createTarget();
|
||||
const target = {
|
||||
...base,
|
||||
isNewSession: true,
|
||||
previousSessionId: "session-1",
|
||||
sessionId: "session-2",
|
||||
};
|
||||
await write(base, {
|
||||
sessionId: target.previousSessionId,
|
||||
updatedAt: 100,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
});
|
||||
const run = vi.fn();
|
||||
|
||||
await expect(
|
||||
runWithAgentCommandRecoveryOwner({
|
||||
lifecycleGeneration: getAgentEventLifecycleGeneration(),
|
||||
mode: "reject_uncoordinated",
|
||||
opts: {} as AgentCommandOpts,
|
||||
prepare: async () => target,
|
||||
run,
|
||||
}),
|
||||
).rejects.toThrow("interrupted work pending restart recovery");
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("allows a freshness successor after its clean replacement commits", async () => {
|
||||
const base = createTarget();
|
||||
const target = {
|
||||
...base,
|
||||
isNewSession: true,
|
||||
previousSessionId: "session-1",
|
||||
sessionId: "session-2",
|
||||
};
|
||||
await write(base, { sessionId: target.sessionId, updatedAt: 200 });
|
||||
const run = vi.fn(async () => "successor");
|
||||
|
||||
await expect(
|
||||
runWithAgentCommandRecoveryOwner({
|
||||
lifecycleGeneration: getAgentEventLifecycleGeneration(),
|
||||
mode: "claim",
|
||||
opts: {} as AgentCommandOpts,
|
||||
prepare: async () => target,
|
||||
run,
|
||||
}),
|
||||
).resolves.toBe("successor");
|
||||
expect(run).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("binds a transferred rollover lease to its exact predecessor", async () => {
|
||||
const base = createTarget();
|
||||
await write(base, {
|
||||
sessionId: base.sessionId,
|
||||
updatedAt: 100,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
});
|
||||
const lifecycleGeneration = getAgentEventLifecycleGeneration();
|
||||
const claim = await claimMainSessionRecoveryOwner({
|
||||
lifecycleGeneration,
|
||||
sessionId: base.sessionId,
|
||||
target: { sessionKey, storePath: base.storePath },
|
||||
});
|
||||
if (claim.kind !== "claimed") {
|
||||
throw new Error("expected recovery owner claim");
|
||||
}
|
||||
const target = {
|
||||
...base,
|
||||
isNewSession: true,
|
||||
previousSessionId: "different-predecessor",
|
||||
sessionId: "successor-session",
|
||||
};
|
||||
const run = vi.fn();
|
||||
|
||||
await expect(
|
||||
runWithAgentCommandRecoveryOwner({
|
||||
lifecycleGeneration,
|
||||
mode: "claim",
|
||||
opts: { mainRestartRecoveryOwnerLease: claim.lease } as AgentCommandOpts,
|
||||
prepare: async () => target,
|
||||
run,
|
||||
}),
|
||||
).rejects.toThrow("recovery owner changed during ingress preparation");
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("binds a transferred recovery owner to the actual agent run", async () => {
|
||||
const target = createTarget();
|
||||
await write(target, {
|
||||
sessionId: target.sessionId,
|
||||
updatedAt: 100,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
});
|
||||
const lifecycleGeneration = getAgentEventLifecycleGeneration();
|
||||
const claim = await claimMainSessionRecoveryOwner({
|
||||
lifecycleGeneration,
|
||||
sessionId: target.sessionId,
|
||||
target: { sessionKey, storePath: target.storePath },
|
||||
});
|
||||
if (claim.kind !== "claimed") {
|
||||
throw new Error("expected recovery owner claim");
|
||||
}
|
||||
const run = vi.fn(async () => {
|
||||
const entry = loadSessionEntry({ sessionKey, storePath: target.storePath }) as SessionEntry;
|
||||
expect(entry.restartRecoveryRuns).toContainEqual({
|
||||
lifecycleGeneration,
|
||||
runId: "foreground-run",
|
||||
});
|
||||
expect(entry.mainRestartRecovery?.foregroundClaims?.runIdsByClaimId).toEqual({
|
||||
[claim.lease.claimId]: "foreground-run",
|
||||
});
|
||||
return "ran";
|
||||
});
|
||||
|
||||
await expect(
|
||||
runWithAgentCommandRecoveryOwner({
|
||||
lifecycleGeneration,
|
||||
mode: "claim",
|
||||
opts: {
|
||||
mainRestartRecoveryOwnerLease: claim.lease,
|
||||
runId: "foreground-run",
|
||||
} as AgentCommandOpts,
|
||||
prepare: async () => target,
|
||||
run,
|
||||
}),
|
||||
).resolves.toBe("ran");
|
||||
expect(run).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("allows an explicitly requested fresh session without a predecessor", async () => {
|
||||
const target = { ...createTarget(), sessionId: "fresh-session" };
|
||||
const run = vi.fn(async () => "fresh");
|
||||
|
||||
await expect(
|
||||
runWithAgentCommandRecoveryOwner({
|
||||
lifecycleGeneration: getAgentEventLifecycleGeneration(),
|
||||
mode: "reject_uncoordinated",
|
||||
opts: { sessionId: target.sessionId } as AgentCommandOpts,
|
||||
prepare: async () => target,
|
||||
run,
|
||||
}),
|
||||
).resolves.toBe("fresh");
|
||||
expect(run).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("invalidates an explicit session replaced during preparation", async () => {
|
||||
const target = createTarget();
|
||||
await write(target, { sessionId: target.sessionId, updatedAt: 100 });
|
||||
const run = vi.fn();
|
||||
|
||||
await expect(
|
||||
runWithAgentCommandRecoveryOwner({
|
||||
lifecycleGeneration: getAgentEventLifecycleGeneration(),
|
||||
mode: "reject_uncoordinated",
|
||||
opts: { sessionId: target.sessionId } as AgentCommandOpts,
|
||||
prepare: async () => {
|
||||
await write(target, { sessionId: "replacement-session", updatedAt: 200 });
|
||||
return target;
|
||||
},
|
||||
run,
|
||||
}),
|
||||
).rejects.toThrow("changed while starting work");
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects a synthetic explicit replacement from a standalone process", async () => {
|
||||
const base = createTarget();
|
||||
const target = {
|
||||
...base,
|
||||
isNewSession: true,
|
||||
previousSessionId: base.sessionId,
|
||||
sessionId: "fresh-session",
|
||||
};
|
||||
await write(base, {
|
||||
sessionId: base.sessionId,
|
||||
updatedAt: 100,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
});
|
||||
const run = vi.fn(async () => "fresh");
|
||||
|
||||
await expect(
|
||||
runWithAgentCommandRecoveryOwner({
|
||||
lifecycleGeneration: getAgentEventLifecycleGeneration(),
|
||||
mode: "reject_uncoordinated",
|
||||
opts: { sessionId: target.sessionId } as AgentCommandOpts,
|
||||
prepare: async () => target,
|
||||
run,
|
||||
}),
|
||||
).rejects.toThrow("interrupted work pending restart recovery");
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("rejects standalone reuse of a tombstoned session", async () => {
|
||||
const target = createTarget();
|
||||
await write(target, {
|
||||
sessionId: target.sessionId,
|
||||
updatedAt: 100,
|
||||
status: "failed",
|
||||
abortedLastRun: false,
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 4,
|
||||
chargedAttempts: 3,
|
||||
tombstone: { reason: "automatic recovery exhausted" },
|
||||
},
|
||||
});
|
||||
const run = vi.fn(async () => "reused");
|
||||
|
||||
await expect(
|
||||
runWithAgentCommandRecoveryOwner({
|
||||
lifecycleGeneration: getAgentEventLifecycleGeneration(),
|
||||
mode: "reject_uncoordinated",
|
||||
opts: { sessionId: target.sessionId } as AgentCommandOpts,
|
||||
prepare: async () => target,
|
||||
run,
|
||||
}),
|
||||
).rejects.toThrow("interrupted work pending restart recovery");
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("revalidates a fresh key when interruption appears during preparation", async () => {
|
||||
const base = createTarget();
|
||||
const target = { ...base, isNewSession: true, sessionId: "fresh-session" };
|
||||
const run = vi.fn();
|
||||
|
||||
await expect(
|
||||
runWithAgentCommandRecoveryOwner({
|
||||
lifecycleGeneration: getAgentEventLifecycleGeneration(),
|
||||
mode: "reject_uncoordinated",
|
||||
opts: {} as AgentCommandOpts,
|
||||
prepare: async () => {
|
||||
await write(base, {
|
||||
sessionId: target.sessionId,
|
||||
updatedAt: 200,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
});
|
||||
return target;
|
||||
},
|
||||
run,
|
||||
}),
|
||||
).rejects.toThrow("interrupted work pending restart recovery");
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,221 @@
|
||||
import path from "node:path";
|
||||
import type { InternalSessionEntry } from "../config/sessions.js";
|
||||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import type { AgentCommandOpts } from "./command/types.js";
|
||||
import { scheduleMainSessionRecoveryPendingTarget } from "./main-session-recovery-owner-release.js";
|
||||
import {
|
||||
restoreAdmittedRecoveryWithRetries,
|
||||
scheduleAdmittedRecoveryRestore,
|
||||
} from "./main-session-recovery-restore.js";
|
||||
import {
|
||||
bindMainSessionRecoveryOwnerRun,
|
||||
claimMainSessionRecoveryOwner,
|
||||
inspectMainSessionRecoveryRequired,
|
||||
readMainSessionRecoveryOwner,
|
||||
releaseMainSessionRecoveryOwner,
|
||||
type MainSessionRecoveryOwnerLease,
|
||||
type MainSessionRecoveryPendingTarget,
|
||||
} from "./main-session-recovery-store.js";
|
||||
|
||||
const log = createSubsystemLogger("agents/agent-command");
|
||||
|
||||
type PreparedRecoveryOwnerTarget = object & {
|
||||
isNewSession: boolean;
|
||||
previousSessionId?: string;
|
||||
sessionId: string;
|
||||
sessionKey?: string;
|
||||
sessionEntry?: InternalSessionEntry;
|
||||
sessionStore?: Record<string, InternalSessionEntry>;
|
||||
storePath: string;
|
||||
runLease?: { release: () => Promise<void> };
|
||||
};
|
||||
|
||||
type AcquiredRecoveryOwner = {
|
||||
lease: MainSessionRecoveryOwnerLease;
|
||||
entry: InternalSessionEntry;
|
||||
sessionKey: string;
|
||||
};
|
||||
|
||||
function cloneRecoveryOwnerEntry(entry: InternalSessionEntry): InternalSessionEntry {
|
||||
const state = entry.mainRestartRecovery;
|
||||
return {
|
||||
...entry,
|
||||
...(entry.restartRecoveryRuns
|
||||
? { restartRecoveryRuns: entry.restartRecoveryRuns.map((run) => ({ ...run })) }
|
||||
: {}),
|
||||
...(state
|
||||
? {
|
||||
mainRestartRecovery: {
|
||||
...state,
|
||||
...(state.reservation ? { reservation: { ...state.reservation } } : {}),
|
||||
...(state.foregroundClaims
|
||||
? {
|
||||
foregroundClaims: {
|
||||
...state.foregroundClaims,
|
||||
tokens: [...state.foregroundClaims.tokens],
|
||||
...(state.foregroundClaims.runIdsByClaimId
|
||||
? { runIdsByClaimId: { ...state.foregroundClaims.runIdsByClaimId } }
|
||||
: {}),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
...(state.tombstone ? { tombstone: { ...state.tombstone } } : {}),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
function refreshPreparedRecoveryOwnerTarget(
|
||||
prepared: PreparedRecoveryOwnerTarget,
|
||||
acquired: AcquiredRecoveryOwner | undefined,
|
||||
): void {
|
||||
if (!acquired || acquired.entry.sessionId !== prepared.sessionId) {
|
||||
return;
|
||||
}
|
||||
const entry = cloneRecoveryOwnerEntry(acquired.entry);
|
||||
prepared.sessionEntry = entry;
|
||||
if (prepared.sessionStore && prepared.sessionKey) {
|
||||
prepared.sessionStore[prepared.sessionKey] = entry;
|
||||
}
|
||||
}
|
||||
|
||||
async function claimAgentCommandRecoveryOwner(params: {
|
||||
lifecycleGeneration: string;
|
||||
mode: "claim" | "reject_uncoordinated";
|
||||
opts: AgentCommandOpts;
|
||||
prepared: PreparedRecoveryOwnerTarget;
|
||||
}): Promise<AcquiredRecoveryOwner | undefined> {
|
||||
const transferredLease = params.opts.mainRestartRecoveryOwnerLease;
|
||||
if (transferredLease) {
|
||||
const expectedLeaseSessionId = params.prepared.isNewSession
|
||||
? params.prepared.previousSessionId
|
||||
: params.prepared.sessionId;
|
||||
const matchesPreparedTarget =
|
||||
expectedLeaseSessionId !== undefined &&
|
||||
transferredLease.lifecycleGeneration === params.lifecycleGeneration &&
|
||||
transferredLease.sessionId === expectedLeaseSessionId &&
|
||||
transferredLease.sessionKey === params.prepared.sessionKey &&
|
||||
path.resolve(transferredLease.storePath) === path.resolve(params.prepared.storePath);
|
||||
if (!matchesPreparedTarget) {
|
||||
// Gateway transfers a persisted fence before preparation; bind it again after
|
||||
// session resolution so rollover or rerouting cannot execute under another row's lease.
|
||||
throw new Error("main-session recovery owner changed during ingress preparation; retry");
|
||||
}
|
||||
if (params.opts.runId) {
|
||||
return await bindMainSessionRecoveryOwnerRun(transferredLease, params.opts.runId);
|
||||
}
|
||||
const snapshot = await readMainSessionRecoveryOwner(transferredLease);
|
||||
if (!snapshot) {
|
||||
throw new Error("main-session recovery owner changed during ingress preparation; retry");
|
||||
}
|
||||
return { ...snapshot, lease: transferredLease };
|
||||
}
|
||||
if (params.opts.sessionEffects === "internal") {
|
||||
return undefined;
|
||||
}
|
||||
if (params.opts.mainRestartRecoveryAdmitted === true) {
|
||||
return undefined;
|
||||
}
|
||||
const sessionKey = params.prepared.sessionKey;
|
||||
if (!sessionKey) {
|
||||
return undefined;
|
||||
}
|
||||
if (params.mode === "reject_uncoordinated") {
|
||||
const recoveryInspection = await inspectMainSessionRecoveryRequired({
|
||||
allowMissingSession:
|
||||
(params.prepared.isNewSession && !params.prepared.previousSessionId) ||
|
||||
params.opts.sessionId?.trim() === params.prepared.sessionId,
|
||||
expectedSessionId: params.prepared.previousSessionId ?? params.prepared.sessionId,
|
||||
lifecycleGeneration: params.lifecycleGeneration,
|
||||
target: { sessionKey, storePath: params.prepared.storePath },
|
||||
});
|
||||
if (recoveryInspection.kind === "invalidated") {
|
||||
throw new Error(`Session "${sessionKey}" changed while starting work. Retry.`);
|
||||
}
|
||||
if (recoveryInspection.kind === "required") {
|
||||
throw new Error(
|
||||
`Session "${sessionKey}" has interrupted work pending restart recovery; retry through a healthy Gateway or reset it there with /new or /reset.`,
|
||||
);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
// Claim against the latest durable row instead of the preparation snapshot.
|
||||
// A restart marker may appear or clear while preparation reads the session.
|
||||
const claim = await claimMainSessionRecoveryOwner({
|
||||
allowMissingSession:
|
||||
(params.prepared.isNewSession && !params.prepared.previousSessionId) ||
|
||||
params.opts.sessionId?.trim() === params.prepared.sessionId,
|
||||
lifecycleGeneration: params.lifecycleGeneration,
|
||||
sessionId: params.prepared.previousSessionId ?? params.prepared.sessionId,
|
||||
replacementSessionId: params.prepared.isNewSession ? params.prepared.sessionId : undefined,
|
||||
runId: params.opts.runId,
|
||||
target: { sessionKey, storePath: params.prepared.storePath },
|
||||
});
|
||||
if (claim.kind === "invalidated") {
|
||||
throw new Error(`Session "${sessionKey}" changed while starting work. Retry.`);
|
||||
}
|
||||
if (claim.kind === "not_required") {
|
||||
return undefined;
|
||||
}
|
||||
// Explicit replacements keep this token through successor persistence so
|
||||
// recovery cannot race the replacement; Gateway claims follow the same lease path.
|
||||
return { lease: claim.lease, entry: claim.entry, sessionKey: claim.sessionKey };
|
||||
}
|
||||
|
||||
export async function runWithAgentCommandRecoveryOwner<
|
||||
TPrepared extends PreparedRecoveryOwnerTarget,
|
||||
TResult,
|
||||
>(params: {
|
||||
lifecycleGeneration: string;
|
||||
mode: "claim" | "reject_uncoordinated";
|
||||
opts: AgentCommandOpts;
|
||||
prepare: (opts: AgentCommandOpts) => Promise<TPrepared>;
|
||||
restoreAdmittedRecovery?: () => Promise<MainSessionRecoveryPendingTarget | undefined>;
|
||||
run: (prepared: TPrepared) => Promise<TResult>;
|
||||
}): Promise<TResult> {
|
||||
// Gateway may preclaim before dispatch, so every preparation outcome must release ownership.
|
||||
let lease = params.opts.mainRestartRecoveryOwnerLease;
|
||||
let pendingRecovery: Awaited<ReturnType<typeof releaseMainSessionRecoveryOwner>> = undefined;
|
||||
let prepared: TPrepared | undefined;
|
||||
try {
|
||||
try {
|
||||
prepared = await params.prepare(params.opts);
|
||||
} catch (error) {
|
||||
// Gateway admission consumes the durable reservation before command
|
||||
// preparation. Restore it when preparation fails before a run exists.
|
||||
if (params.restoreAdmittedRecovery) {
|
||||
try {
|
||||
pendingRecovery = await restoreAdmittedRecoveryWithRetries(
|
||||
params.restoreAdmittedRecovery,
|
||||
);
|
||||
} catch (restoreError) {
|
||||
log.warn(
|
||||
`failed to restore admitted recovery after command preparation: ${formatErrorMessage(restoreError)}`,
|
||||
);
|
||||
scheduleAdmittedRecoveryRestore(params.restoreAdmittedRecovery);
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const acquired = await claimAgentCommandRecoveryOwner({ ...params, prepared });
|
||||
lease = acquired?.lease;
|
||||
// Preparation uses a detached working copy. Carry the owner transaction's
|
||||
// exact row forward so successful settlement can consume the same recovery cycle.
|
||||
refreshPreparedRecoveryOwnerTarget(prepared, acquired);
|
||||
return await params.run(prepared);
|
||||
} finally {
|
||||
try {
|
||||
const releasedRecovery = await releaseMainSessionRecoveryOwner(lease);
|
||||
pendingRecovery ??= releasedRecovery;
|
||||
} catch (error) {
|
||||
log.warn(`failed to release main-session recovery owner: ${formatErrorMessage(error)}`);
|
||||
}
|
||||
try {
|
||||
await prepared?.runLease?.release();
|
||||
} finally {
|
||||
scheduleMainSessionRecoveryPendingTarget(pendingRecovery);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -199,6 +199,15 @@ vi.mock("./command/session.js", () => ({
|
||||
|
||||
vi.mock("./command/types.js", () => ({}));
|
||||
|
||||
// Recovery ownership has dedicated store-backed coverage. This command suite
|
||||
// uses an intentionally synthetic session resolver with no durable store path.
|
||||
vi.mock("./main-session-recovery-store.js", () => ({
|
||||
claimMainSessionRecoveryOwner: vi.fn(async () => ({ kind: "not_required" })),
|
||||
inspectMainSessionRecoveryRequired: vi.fn(async () => ({ kind: "not_required" })),
|
||||
releaseMainSessionRecoveryOwner: vi.fn(async () => undefined),
|
||||
validateMainSessionRecoveryOwner: vi.fn(async () => true),
|
||||
}));
|
||||
|
||||
vi.mock("./harness/runtime-plugin.js", () => ({
|
||||
ensureSelectedAgentHarnessPlugin: vi.fn(async () => undefined),
|
||||
}));
|
||||
@@ -2980,7 +2989,7 @@ describe("agentCommand – LiveSessionModelSwitchError retry", () => {
|
||||
expect(stored?.restartRecoveryDeliveryContext).toBeUndefined();
|
||||
expect(stored?.restartRecoveryDeliveryRunId).toBeUndefined();
|
||||
expect(stored?.restartRecoveryDeliverySourceRunId).toBeUndefined();
|
||||
expect(stored?.restartRecoveryTerminalRunIds).toEqual(["control-ui-run"]);
|
||||
expect(stored?.restartRecoveryTerminalRunIds).toEqual(["control-ui-run", "recovery-run"]);
|
||||
});
|
||||
|
||||
it("refreshes delivery session entries through the session accessor", async () => {
|
||||
|
||||
+60
-31
@@ -22,6 +22,7 @@ import { resolveSendPolicy } from "../sessions/send-policy.js";
|
||||
import { beginSessionWorkAdmission } from "../sessions/session-lifecycle-admission.js";
|
||||
import { classifySessionStateActor } from "../sessions/session-state-events.js";
|
||||
import type { DeliveryContext } from "../utils/delivery-context.shared.js";
|
||||
import { runWithAgentCommandRecoveryOwner } from "./agent-command-recovery-owner.js";
|
||||
import {
|
||||
buildCurrentRunRestartRecoveryClaim,
|
||||
shouldPersistRestartRecoveryCleanup,
|
||||
@@ -51,12 +52,14 @@ import {
|
||||
resolveInternalSessionEffectsTarget,
|
||||
} from "./internal-session-effects.js";
|
||||
import { AGENT_LANE_SUBAGENT } from "./lanes.js";
|
||||
import type { MainSessionRecoveryPendingTarget } from "./main-session-recovery-store.js";
|
||||
import type { AgentRunSessionTarget } from "./run-session-target.js";
|
||||
import { createAgentRunRestartAbortError } from "./run-termination.js";
|
||||
|
||||
const log = createSubsystemLogger("agents/agent-command");
|
||||
|
||||
async function agentCommandInternal(
|
||||
prepared: Awaited<ReturnType<typeof prepareAgentCommandExecution>>,
|
||||
initialOpts: AgentCommandOpts,
|
||||
runtime: RuntimeEnv = defaultRuntime,
|
||||
deps?: CliDeps,
|
||||
@@ -66,7 +69,6 @@ async function agentCommandInternal(
|
||||
const suppressVisibleSessionEffects = initialOpts.sessionEffects === "internal";
|
||||
const preserveUserFacingSessionModelState =
|
||||
initialOpts.preserveUserFacingSessionModelState === true;
|
||||
const prepared = await prepareAgentCommandExecution(initialOpts, runtime);
|
||||
const lifecycleAbortController = new AbortController();
|
||||
const storedDeliveryMediaUrls =
|
||||
prepared.sessionEntry?.restartRecoveryDeliveryRunId === prepared.runId &&
|
||||
@@ -129,7 +131,6 @@ async function agentCommandInternal(
|
||||
pluginsEnabled,
|
||||
manifestMetadataSnapshot,
|
||||
modelManifestContext,
|
||||
runLease,
|
||||
} = prepared;
|
||||
let lifecycleGeneration = opts.lifecycleGeneration ?? captureAgentRunLifecycleGeneration(runId);
|
||||
let sessionEntry = prepared.sessionEntry,
|
||||
@@ -454,9 +455,6 @@ async function agentCommandInternal(
|
||||
return finalized.deliveryResult;
|
||||
});
|
||||
} finally {
|
||||
if (runLease) {
|
||||
await runLease.release();
|
||||
}
|
||||
sessionWorkAdmission?.release();
|
||||
if (internalModelRunTargets) {
|
||||
// Compaction may rotate a private session identity. Remove every owned
|
||||
@@ -489,6 +487,7 @@ async function agentCommandInternal(
|
||||
...buildRestartRecoveryClaimCleanupPatch({
|
||||
entry,
|
||||
recordTerminalSource: true,
|
||||
terminalRunId: runId,
|
||||
terminalDeliveryEvidence: restartRecoveryTerminalDeliveryEvidence,
|
||||
}),
|
||||
updatedAt: Date.now(),
|
||||
@@ -532,8 +531,10 @@ export async function agentCommand(
|
||||
getRuntimeConfig,
|
||||
},
|
||||
async () =>
|
||||
await agentCommandInternal(
|
||||
{
|
||||
await runWithAgentCommandRecoveryOwner({
|
||||
lifecycleGeneration,
|
||||
mode: "reject_uncoordinated",
|
||||
opts: {
|
||||
...opts,
|
||||
lifecycleGeneration,
|
||||
// agentCommand is the trusted-operator entrypoint used by CLI/local flows.
|
||||
@@ -543,41 +544,69 @@ export async function agentCommand(
|
||||
// Local/CLI callers are trusted by default for per-run model overrides.
|
||||
allowModelOverride: opts.allowModelOverride ?? true,
|
||||
},
|
||||
runtime,
|
||||
resolvedDeps,
|
||||
),
|
||||
prepare: async (preparedOpts) =>
|
||||
await prepareAgentCommandExecution(preparedOpts, runtime),
|
||||
run: async (prepared) =>
|
||||
await agentCommandInternal(prepared, prepared.opts, runtime, resolvedDeps),
|
||||
}),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
async function agentCommandFromIngressInternal(
|
||||
opts: AgentCommandIngressOpts,
|
||||
runtime: RuntimeEnv = defaultRuntime,
|
||||
deps?: CliDeps,
|
||||
recovery?: {
|
||||
restoreAdmittedRecovery?: () => Promise<MainSessionRecoveryPendingTarget | undefined>;
|
||||
},
|
||||
) {
|
||||
if (typeof opts.allowModelOverride !== "boolean") {
|
||||
throw new Error("allowModelOverride must be explicitly set for ingress agent runs.");
|
||||
}
|
||||
const lifecycleGeneration =
|
||||
opts.lifecycleGeneration ?? captureAgentRunLifecycleGeneration(opts.runId ?? "");
|
||||
return await withAgentRunLifecycleGeneration(lifecycleGeneration, async () => {
|
||||
const result = await runWithAgentCommandRecoveryOwner({
|
||||
lifecycleGeneration,
|
||||
mode: "claim",
|
||||
opts: {
|
||||
...opts,
|
||||
lifecycleGeneration,
|
||||
senderIsOwner: opts.senderIsOwner === true,
|
||||
},
|
||||
prepare: async (preparedOpts) => await prepareAgentCommandExecution(preparedOpts, runtime),
|
||||
restoreAdmittedRecovery: recovery?.restoreAdmittedRecovery,
|
||||
run: async (prepared) => await agentCommandInternal(prepared, prepared.opts, runtime, deps),
|
||||
});
|
||||
|
||||
if (result) {
|
||||
emitIngressModelUsageDiagnostic(result, opts);
|
||||
}
|
||||
|
||||
return result;
|
||||
});
|
||||
}
|
||||
|
||||
/** Runs an agent turn from an inbound channel/gateway ingress context. */
|
||||
export async function agentCommandFromIngress(
|
||||
opts: AgentCommandIngressOpts,
|
||||
runtime: RuntimeEnv = defaultRuntime,
|
||||
deps?: CliDeps,
|
||||
) {
|
||||
if (typeof opts.allowModelOverride !== "boolean") {
|
||||
throw new Error("allowModelOverride must be explicitly set for ingress agent runs.");
|
||||
}
|
||||
const lifecycleGeneration =
|
||||
opts.lifecycleGeneration ?? captureAgentRunLifecycleGeneration(opts.runId ?? "");
|
||||
return await withAgentRunLifecycleGeneration(lifecycleGeneration, async () => {
|
||||
const result = await agentCommandInternal(
|
||||
{
|
||||
...opts,
|
||||
lifecycleGeneration,
|
||||
senderIsOwner: opts.senderIsOwner === true,
|
||||
},
|
||||
runtime,
|
||||
deps,
|
||||
);
|
||||
return await agentCommandFromIngressInternal(opts, runtime, deps);
|
||||
}
|
||||
|
||||
if (result) {
|
||||
emitIngressModelUsageDiagnostic(result, opts);
|
||||
}
|
||||
|
||||
return result;
|
||||
});
|
||||
/** Internal Gateway entrypoint that restores a rejected restart-recovery admission. */
|
||||
export async function agentCommandFromGatewayIngress(
|
||||
opts: AgentCommandIngressOpts,
|
||||
runtime: RuntimeEnv,
|
||||
deps: CliDeps | undefined,
|
||||
recovery: {
|
||||
restoreAdmittedRecovery?: () => Promise<MainSessionRecoveryPendingTarget | undefined>;
|
||||
},
|
||||
) {
|
||||
return await agentCommandFromIngressInternal(opts, runtime, deps, recovery);
|
||||
}
|
||||
|
||||
export const testing = {
|
||||
|
||||
@@ -247,9 +247,15 @@ export async function prepareAgentCommandExecution(opts: AgentCommandOpts, runti
|
||||
agentId: agentIdOverride,
|
||||
clone: false,
|
||||
});
|
||||
|
||||
const { sessionId, sessionKey, storePath, isNewSession, persistedThinking, persistedVerbose } =
|
||||
sessionResolution;
|
||||
const {
|
||||
sessionId,
|
||||
sessionKey,
|
||||
storePath,
|
||||
isNewSession,
|
||||
previousSessionId,
|
||||
persistedThinking,
|
||||
persistedVerbose,
|
||||
} = sessionResolution;
|
||||
const harnessSessionError = sessionKey
|
||||
? resolveAgentHarnessSessionContextError(sessionKey, sessionResolution.sessionEntry)
|
||||
: undefined;
|
||||
@@ -348,7 +354,7 @@ export async function prepareAgentCommandExecution(opts: AgentCommandOpts, runti
|
||||
const transcriptBody =
|
||||
opts.transcriptMessage ?? resolveInternalEventTranscriptBody(message, opts.internalEvents);
|
||||
|
||||
return {
|
||||
const prepared = {
|
||||
opts: commandOpts,
|
||||
body,
|
||||
transcriptBody,
|
||||
@@ -367,6 +373,7 @@ export async function prepareAgentCommandExecution(opts: AgentCommandOpts, runti
|
||||
sessionStore,
|
||||
storePath,
|
||||
isNewSession,
|
||||
previousSessionId,
|
||||
persistedThinking,
|
||||
persistedVerbose,
|
||||
sessionAgentId,
|
||||
@@ -383,6 +390,7 @@ export async function prepareAgentCommandExecution(opts: AgentCommandOpts, runti
|
||||
acpResolution,
|
||||
runLease,
|
||||
};
|
||||
return prepared;
|
||||
} catch (error) {
|
||||
await runLease?.release();
|
||||
throw error;
|
||||
|
||||
@@ -5,7 +5,7 @@ import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { OpenClawConfig } from "../../config/config.js";
|
||||
import type { SessionEntry } from "../../config/sessions.js";
|
||||
import type { InternalSessionEntry as SessionEntry } from "../../config/sessions.js";
|
||||
import {
|
||||
listSessionEntries,
|
||||
loadSessionEntry,
|
||||
@@ -1849,6 +1849,205 @@ describe("updateSessionStoreAfterAgentRun", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("clears main recovery markers after settled background progress", async () => {
|
||||
await withTempSessionStore(async ({ storePath }) => {
|
||||
const cfg = {} as OpenClawConfig;
|
||||
const sessionKey = "agent:main:explicit:test-clear-recovery-state";
|
||||
const sessionId = "test-clear-recovery-state-session";
|
||||
const sessionStore: Record<string, SessionEntry> = {
|
||||
[sessionKey]: {
|
||||
sessionId,
|
||||
updatedAt: 1,
|
||||
abortedLastRun: true,
|
||||
restartRecoveryRuns: [
|
||||
{ runId: "initial-wedged-run", lifecycleGeneration: "gen-1" },
|
||||
{ runId: "recovery-run-1", lifecycleGeneration: "gen-2" },
|
||||
{ runId: "recovery-run-2", lifecycleGeneration: "gen-3" },
|
||||
{ runId: "recovery-run-3", lifecycleGeneration: "gen-4" },
|
||||
],
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 3,
|
||||
chargedAttempts: 2,
|
||||
},
|
||||
subagentRecovery: {
|
||||
automaticAttempts: 2,
|
||||
lastAttemptAt: 3,
|
||||
wedgedAt: 4,
|
||||
wedgedReason: "automatic_attempt_budget_exceeded",
|
||||
},
|
||||
},
|
||||
};
|
||||
await seedSessionStore(storePath, sessionStore);
|
||||
|
||||
await updateSessionStoreAfterAgentRun({
|
||||
cfg,
|
||||
sessionId,
|
||||
sessionKey,
|
||||
storePath,
|
||||
sessionStore,
|
||||
defaultProvider: "openai",
|
||||
defaultModel: "gpt-5.5",
|
||||
touchInteraction: false,
|
||||
touchActivity: false,
|
||||
preserveRuntimeModel: true,
|
||||
result: {
|
||||
meta: {
|
||||
durationMs: 1,
|
||||
aborted: false,
|
||||
agentMeta: {
|
||||
sessionId,
|
||||
provider: "openai",
|
||||
model: "gpt-5.5",
|
||||
},
|
||||
},
|
||||
} as EmbeddedAgentRunResult,
|
||||
});
|
||||
|
||||
expect(sessionStore[sessionKey]?.abortedLastRun).toBe(false);
|
||||
expect(sessionStore[sessionKey]?.restartRecoveryRuns).toBeUndefined();
|
||||
expect(sessionStore[sessionKey]?.mainRestartRecovery).toBeUndefined();
|
||||
expect(sessionStore[sessionKey]?.subagentRecovery).toEqual({
|
||||
automaticAttempts: 2,
|
||||
lastAttemptAt: 3,
|
||||
wedgedAt: 4,
|
||||
wedgedReason: "automatic_attempt_budget_exceeded",
|
||||
});
|
||||
const persisted = loadPersistedSessionEntry(storePath, sessionKey);
|
||||
expect(persisted?.abortedLastRun).toBe(false);
|
||||
expect(persisted?.restartRecoveryRuns).toBeUndefined();
|
||||
expect(persisted).not.toHaveProperty("mainRestartRecovery");
|
||||
expect(persisted?.subagentRecovery).toEqual({
|
||||
automaticAttempts: 2,
|
||||
lastAttemptAt: 3,
|
||||
wedgedAt: 4,
|
||||
wedgedReason: "automatic_attempt_budget_exceeded",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves a replacement recovery cycle from an older healthy finalizer", async () => {
|
||||
await withTempSessionStore(async ({ storePath }) => {
|
||||
const sessionKey = "agent:main:explicit:test-recovery-finalizer-aba";
|
||||
const sessionId = "test-recovery-finalizer-aba-session";
|
||||
const staleEntry: SessionEntry = {
|
||||
sessionId,
|
||||
updatedAt: 1,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-old",
|
||||
revision: 2,
|
||||
chargedAttempts: 1,
|
||||
},
|
||||
};
|
||||
const sessionStore = { [sessionKey]: staleEntry };
|
||||
const replacementEntry: SessionEntry = {
|
||||
...staleEntry,
|
||||
updatedAt: 2,
|
||||
restartRecoveryRuns: [{ runId: "replacement-run", lifecycleGeneration: "gen-new" }],
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-new",
|
||||
revision: 1,
|
||||
chargedAttempts: 0,
|
||||
},
|
||||
};
|
||||
await seedSessionStore(storePath, { [sessionKey]: replacementEntry });
|
||||
|
||||
await updateSessionStoreAfterAgentRun({
|
||||
cfg: {} as OpenClawConfig,
|
||||
sessionId,
|
||||
sessionKey,
|
||||
storePath,
|
||||
sessionStore,
|
||||
defaultProvider: "openai",
|
||||
defaultModel: "gpt-5.5",
|
||||
result: {
|
||||
meta: {
|
||||
durationMs: 1,
|
||||
aborted: false,
|
||||
agentMeta: { sessionId, provider: "openai", model: "gpt-5.5" },
|
||||
},
|
||||
} as EmbeddedAgentRunResult,
|
||||
});
|
||||
|
||||
const persisted = loadPersistedSessionEntry(storePath, sessionKey);
|
||||
expect(persisted).toMatchObject({
|
||||
abortedLastRun: true,
|
||||
restartRecoveryRuns: [{ runId: "replacement-run", lifecycleGeneration: "gen-new" }],
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-new",
|
||||
revision: 1,
|
||||
chargedAttempts: 0,
|
||||
},
|
||||
});
|
||||
expect(sessionStore[sessionKey]).toEqual(persisted);
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves a concurrent restart marker when a stale run settles healthy", async () => {
|
||||
await withTempSessionStore(async ({ storePath }) => {
|
||||
const sessionKey = "agent:main:explicit:test-restart-finalizer-race";
|
||||
const sessionId = "test-restart-finalizer-race-session";
|
||||
const initialEntry: SessionEntry = {
|
||||
sessionId,
|
||||
updatedAt: 1,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
restartRecoveryRuns: [{ runId: "run-1", lifecycleGeneration: "generation-1" }],
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 2,
|
||||
chargedAttempts: 0,
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration: "generation-1",
|
||||
tokens: ["owner-1"],
|
||||
},
|
||||
},
|
||||
};
|
||||
const sessionStore = { [sessionKey]: initialEntry };
|
||||
const concurrentEntry: SessionEntry = {
|
||||
...structuredClone(initialEntry),
|
||||
updatedAt: 2,
|
||||
restartRecoveryRuns: [
|
||||
...(initialEntry.restartRecoveryRuns ?? []),
|
||||
{ runId: "run-1", lifecycleGeneration: "generation-2" },
|
||||
],
|
||||
mainRestartRecovery: {
|
||||
...initialEntry.mainRestartRecovery!,
|
||||
revision: 3,
|
||||
},
|
||||
};
|
||||
await seedSessionStore(storePath, { [sessionKey]: concurrentEntry });
|
||||
|
||||
await updateSessionStoreAfterAgentRun({
|
||||
cfg: {} as OpenClawConfig,
|
||||
sessionId,
|
||||
sessionKey,
|
||||
storePath,
|
||||
sessionStore,
|
||||
defaultProvider: "openai",
|
||||
defaultModel: "gpt-5.5",
|
||||
result: {
|
||||
meta: {
|
||||
durationMs: 1,
|
||||
aborted: false,
|
||||
agentMeta: { sessionId, provider: "openai", model: "gpt-5.5" },
|
||||
},
|
||||
} as EmbeddedAgentRunResult,
|
||||
});
|
||||
|
||||
for (const entry of [
|
||||
sessionStore[sessionKey],
|
||||
loadPersistedSessionEntry(storePath, sessionKey),
|
||||
]) {
|
||||
expect(entry?.abortedLastRun).toBe(true);
|
||||
expect(entry?.restartRecoveryRuns).toEqual(concurrentEntry.restartRecoveryRuns);
|
||||
expect(entry?.mainRestartRecovery).toEqual(concurrentEntry.mainRestartRecovery);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves runtime model and contextTokens when preserveRuntimeModel is true (heartbeat bleed fix)", async () => {
|
||||
await withTempSessionStore(async ({ storePath }) => {
|
||||
const cfg = {} as OpenClawConfig;
|
||||
|
||||
@@ -15,6 +15,7 @@ import { createLazyImportLoader } from "../../shared/lazy-promise.js";
|
||||
import { resolveNonNegativeNumber } from "../../shared/number-coercion.js";
|
||||
import { clearCliSession, setCliSessionBinding, setCliSessionId } from "../cli-session.js";
|
||||
import { DEFAULT_CONTEXT_TOKENS } from "../defaults.js";
|
||||
import { clearMainSessionRecoveryAfterAgentRun } from "../main-session-recovery-clear.js";
|
||||
import { isCliProvider } from "../model-selection.js";
|
||||
import { deriveSessionTotalTokens, hasNonzeroUsage } from "../usage.js";
|
||||
|
||||
@@ -199,9 +200,7 @@ export async function updateSessionStoreAfterAgentRun(params: {
|
||||
}
|
||||
}
|
||||
next.abortedLastRun = result.meta.aborted ?? false;
|
||||
if (params.clearRestartRecoveryForceSafeTools && result.meta.aborted !== true) {
|
||||
next.restartRecoveryForceSafeTools = undefined;
|
||||
}
|
||||
clearMainSessionRecoveryAfterAgentRun(next, params.clearRestartRecoveryForceSafeTools);
|
||||
if (result.meta.systemPromptReport) {
|
||||
next.systemPromptReport = result.meta.systemPromptReport;
|
||||
}
|
||||
@@ -310,7 +309,12 @@ export async function updateSessionStoreAfterAgentRun(params: {
|
||||
}
|
||||
return preserveUserFacingRunState
|
||||
? metadataPatch
|
||||
: projectSessionSnapshotChanges({ initial: entry, next, current: currentEntry });
|
||||
: projectSessionSnapshotChanges({
|
||||
initial: entry,
|
||||
next,
|
||||
current: currentEntry,
|
||||
reassertAbortedLastRun: result.meta.aborted === true,
|
||||
});
|
||||
},
|
||||
{
|
||||
...(preserveUserFacingRunState ? {} : { fallbackEntry: entry }),
|
||||
|
||||
@@ -41,6 +41,7 @@ import { resolveSessionIdMatchSelection } from "../../sessions/session-id-resolu
|
||||
import { listAgentIds, resolveDefaultAgentId } from "../agent-scope.js";
|
||||
import { clearBootstrapSnapshotOnSessionRollover } from "../bootstrap-cache.js";
|
||||
import { clearAllCliSessions } from "../cli-session.js";
|
||||
import { transitionMainSessionRecovery } from "../main-session-recovery-state.js";
|
||||
|
||||
/** Resolved command session identity plus backing store metadata. */
|
||||
type SessionResolution = {
|
||||
@@ -50,6 +51,7 @@ type SessionResolution = {
|
||||
sessionStore?: Record<string, SessionEntry>;
|
||||
storePath: string;
|
||||
isNewSession: boolean;
|
||||
previousSessionId?: string;
|
||||
persistedThinking?: ThinkLevel;
|
||||
persistedVerbose?: VerboseLevel;
|
||||
};
|
||||
@@ -90,6 +92,7 @@ export function clearRotatedSessionMetadata(entry: SessionEntry): SessionEntry {
|
||||
sessionStartedAt: undefined,
|
||||
lastInteractionAt: undefined,
|
||||
};
|
||||
transitionMainSessionRecovery(next, { kind: "clear" });
|
||||
clearAllCliSessions(next);
|
||||
return next;
|
||||
}
|
||||
@@ -472,6 +475,7 @@ export function resolveSession(opts: {
|
||||
sessionStore,
|
||||
storePath,
|
||||
isNewSession,
|
||||
previousSessionId: isNewSession ? sessionEntry?.sessionId : undefined,
|
||||
persistedThinking,
|
||||
persistedVerbose,
|
||||
};
|
||||
|
||||
@@ -18,6 +18,7 @@ import type {
|
||||
import type { ExecElevatedDefaults } from "../bash-tools.exec-types.js";
|
||||
import type { BootstrapContextRunKind } from "../bootstrap-mode.js";
|
||||
import type { CliSessionBindingFacts } from "../cli-runner/types.js";
|
||||
import type { MainSessionRecoveryOwnerLease } from "../main-session-recovery-store.js";
|
||||
import type { AgentStreamParams, ClientToolDefinition } from "./shared-types.js";
|
||||
|
||||
/** Image content block for Claude API multimodal messages. */
|
||||
@@ -173,6 +174,10 @@ export type AgentCommandOpts = {
|
||||
oneShotCliRun?: boolean;
|
||||
/** Gateway-owned runs can late-bind plugin subagent and node runtime helpers. */
|
||||
allowGatewaySubagentBinding?: boolean;
|
||||
/** Opaque foreground fence transferred by Gateway after atomic session admission. */
|
||||
mainRestartRecoveryOwnerLease?: MainSessionRecoveryOwnerLease;
|
||||
/** Gateway already consumed this automatic recovery run's durable reservation. */
|
||||
mainRestartRecoveryAdmitted?: boolean;
|
||||
/** Internal local CLI callers can annotate result metadata before JSON/text output. */
|
||||
resultMetaOverrides?: AgentCommandResultMetaOverrides;
|
||||
/** Called when the actual run model is selected, including fallback retries. */
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import type { InternalSessionEntry as SessionEntry } from "../config/sessions.js";
|
||||
|
||||
type MainRecoveryStateFields = Pick<
|
||||
SessionEntry,
|
||||
"abortedLastRun" | "restartRecoveryRuns" | "mainRestartRecovery"
|
||||
>;
|
||||
|
||||
export const MAIN_SESSION_RECOVERY_CLEAR_PATCH: Partial<MainRecoveryStateFields> = {
|
||||
abortedLastRun: false,
|
||||
restartRecoveryRuns: undefined,
|
||||
mainRestartRecovery: undefined,
|
||||
};
|
||||
|
||||
export function buildMainSessionRecoveryClearPatch(
|
||||
entry?: Partial<MainRecoveryStateFields> | null,
|
||||
): Partial<MainRecoveryStateFields> {
|
||||
if (
|
||||
entry?.abortedLastRun !== true &&
|
||||
entry?.restartRecoveryRuns === undefined &&
|
||||
entry?.mainRestartRecovery === undefined
|
||||
) {
|
||||
return {};
|
||||
}
|
||||
return MAIN_SESSION_RECOVERY_CLEAR_PATCH;
|
||||
}
|
||||
|
||||
export function clearMainSessionRecoveryAfterAgentRun(
|
||||
entry: SessionEntry,
|
||||
clearForceSafeTools: boolean | undefined,
|
||||
): void {
|
||||
const aborted = entry.abortedLastRun === true;
|
||||
if (clearForceSafeTools && !aborted) {
|
||||
entry.restartRecoveryForceSafeTools = undefined;
|
||||
}
|
||||
if (!aborted) {
|
||||
Object.assign(entry, buildMainSessionRecoveryClearPatch(entry));
|
||||
}
|
||||
}
|
||||
|
||||
export type { MainRecoveryStateFields };
|
||||
@@ -0,0 +1,215 @@
|
||||
import type { InternalSessionEntry as SessionEntry } from "../config/sessions.js";
|
||||
import { mergeRestartRecoveryTerminalRunIds } from "../config/sessions/restart-recovery-state.js";
|
||||
import {
|
||||
buildMainSessionRecoveryClearPatch,
|
||||
type MainRecoveryStateFields,
|
||||
} from "./main-session-recovery-clear.js";
|
||||
|
||||
const MAIN_RESTART_RECOVERY_WEDGED_FALLBACK_REASON =
|
||||
"main-session restart recovery is tombstoned for this session";
|
||||
|
||||
type MainRecoveryLifecycleEvent = {
|
||||
runId?: string;
|
||||
lifecycleGeneration?: string;
|
||||
data?: { error?: unknown; phase?: unknown; stopReason?: unknown };
|
||||
};
|
||||
|
||||
export function inspectMainSessionRecoveryHealth(entry: SessionEntry):
|
||||
| { status: "none" }
|
||||
| { status: "active" }
|
||||
| {
|
||||
status: "tombstoned";
|
||||
reason: string;
|
||||
repair: "clear_stale_abort" | null;
|
||||
} {
|
||||
const state = entry.mainRestartRecovery;
|
||||
if (!state) {
|
||||
return { status: "none" };
|
||||
}
|
||||
if (!state.tombstone) {
|
||||
return { status: "active" };
|
||||
}
|
||||
return {
|
||||
status: "tombstoned",
|
||||
reason: state.tombstone.reason.trim() || MAIN_RESTART_RECOVERY_WEDGED_FALLBACK_REASON,
|
||||
repair: entry.abortedLastRun === true ? "clear_stale_abort" : null,
|
||||
};
|
||||
}
|
||||
|
||||
function lifecyclePhase(event: MainRecoveryLifecycleEvent): "start" | "end" | "error" | null {
|
||||
const phase = event.data?.phase;
|
||||
return phase === "start" || phase === "end" || phase === "error" ? phase : null;
|
||||
}
|
||||
|
||||
export function isMainSessionRecoveryLifecycleEvent(params: {
|
||||
entry?: Partial<Pick<SessionEntry, "restartRecoveryRuns">> | null;
|
||||
event: MainRecoveryLifecycleEvent;
|
||||
}): boolean {
|
||||
const runId = params.event.runId?.trim();
|
||||
const lifecycleGeneration = params.event.lifecycleGeneration?.trim();
|
||||
const phase = lifecyclePhase(params.event);
|
||||
const interrupted = params.event.data?.stopReason === "restart";
|
||||
const matchesFence = Boolean(
|
||||
runId &&
|
||||
lifecycleGeneration &&
|
||||
params.entry?.restartRecoveryRuns?.some(
|
||||
(run) => run.runId === runId && run.lifecycleGeneration === lifecycleGeneration,
|
||||
),
|
||||
);
|
||||
return (
|
||||
matchesFence && (phase === "start" || ((phase === "end" || phase === "error") && interrupted))
|
||||
);
|
||||
}
|
||||
|
||||
export function projectMainSessionRecoveryLifecycle(params: {
|
||||
currentLifecycleGeneration: string;
|
||||
entry?:
|
||||
| (Partial<MainRecoveryStateFields> &
|
||||
Pick<
|
||||
Partial<SessionEntry>,
|
||||
"restartRecoveryDeliveryRunId" | "restartRecoveryTerminalRunIds"
|
||||
>)
|
||||
| null;
|
||||
event: MainRecoveryLifecycleEvent;
|
||||
snapshotPatch: Partial<SessionEntry>;
|
||||
}): { action: "suppress" } | { action: "apply"; patch: Partial<SessionEntry> } {
|
||||
if (params.entry?.mainRestartRecovery?.tombstone) {
|
||||
// Keep the operator boundary while allowing unrelated lifecycle status to settle.
|
||||
return isMainSessionRecoveryLifecycleEvent(params)
|
||||
? { action: "suppress" }
|
||||
: {
|
||||
action: "apply",
|
||||
patch: {
|
||||
...params.snapshotPatch,
|
||||
abortedLastRun: params.entry.abortedLastRun,
|
||||
restartRecoveryRuns: params.entry.restartRecoveryRuns,
|
||||
mainRestartRecovery: params.entry.mainRestartRecovery,
|
||||
},
|
||||
};
|
||||
}
|
||||
if (isMainSessionRecoveryLifecycleEvent(params)) {
|
||||
return { action: "suppress" };
|
||||
}
|
||||
const phase = lifecyclePhase(params.event);
|
||||
const settlesRecovery =
|
||||
(phase === "end" || phase === "error") && params.event.data?.stopReason !== "restart";
|
||||
const patch = { ...params.snapshotPatch };
|
||||
const runId = params.event.runId?.trim();
|
||||
const lifecycleGeneration = params.event.lifecycleGeneration?.trim();
|
||||
const runs = params.entry?.restartRecoveryRuns;
|
||||
const matchesFence = Boolean(
|
||||
runId &&
|
||||
lifecycleGeneration &&
|
||||
runs?.some((run) => run.runId === runId && run.lifecycleGeneration === lifecycleGeneration),
|
||||
);
|
||||
const remaining = matchesFence
|
||||
? runs?.filter((run) => run.runId !== runId || run.lifecycleGeneration !== lifecycleGeneration)
|
||||
: runs;
|
||||
if (settlesRecovery) {
|
||||
const foregroundClaims = params.entry?.mainRestartRecovery?.foregroundClaims;
|
||||
const foregroundOwnerClaimId =
|
||||
runId &&
|
||||
lifecycleGeneration &&
|
||||
lifecycleGeneration === params.currentLifecycleGeneration &&
|
||||
foregroundClaims?.lifecycleGeneration === lifecycleGeneration
|
||||
? foregroundClaims.tokens.find(
|
||||
(claimId) => foregroundClaims.runIdsByClaimId?.[claimId] === runId,
|
||||
)
|
||||
: undefined;
|
||||
const remainingForegroundClaimIds = foregroundOwnerClaimId
|
||||
? foregroundClaims!.tokens.filter((claimId) => claimId !== foregroundOwnerClaimId)
|
||||
: foregroundClaims?.tokens;
|
||||
const remainingForegroundRunIds = foregroundOwnerClaimId
|
||||
? Object.fromEntries(
|
||||
Object.entries(foregroundClaims?.runIdsByClaimId ?? {}).filter(
|
||||
([claimId]) => claimId !== foregroundOwnerClaimId,
|
||||
),
|
||||
)
|
||||
: foregroundClaims?.runIdsByClaimId;
|
||||
const remainingForegroundClaims = remainingForegroundClaimIds?.length
|
||||
? {
|
||||
lifecycleGeneration: foregroundClaims!.lifecycleGeneration,
|
||||
tokens: remainingForegroundClaimIds,
|
||||
...(remainingForegroundRunIds && Object.keys(remainingForegroundRunIds).length > 0
|
||||
? { runIdsByClaimId: remainingForegroundRunIds }
|
||||
: {}),
|
||||
}
|
||||
: undefined;
|
||||
const recoveryStateAfterForegroundSettlement = foregroundOwnerClaimId
|
||||
? {
|
||||
...params.entry!.mainRestartRecovery!,
|
||||
revision: params.entry!.mainRestartRecovery!.revision + 1,
|
||||
foregroundClaims: remainingForegroundClaims,
|
||||
}
|
||||
: params.entry?.mainRestartRecovery;
|
||||
const hasForegroundOwners = Boolean(
|
||||
remainingForegroundClaims?.lifecycleGeneration === params.currentLifecycleGeneration &&
|
||||
remainingForegroundClaims.tokens.length,
|
||||
);
|
||||
const reservation = params.entry?.mainRestartRecovery?.reservation;
|
||||
const hasCurrentReservation =
|
||||
reservation?.lifecycleGeneration === params.currentLifecycleGeneration;
|
||||
const hasCurrentOwner = hasForegroundOwners || hasCurrentReservation;
|
||||
if (!matchesFence) {
|
||||
// No terminal snapshot may settle a recovery row it cannot identify.
|
||||
return params.entry?.mainRestartRecovery || runs?.length
|
||||
? { action: "suppress" }
|
||||
: { action: "apply", patch };
|
||||
}
|
||||
if (hasCurrentOwner) {
|
||||
// A terminal event may consume its own claim. Another owner still keeps
|
||||
// the aggregate live until that owner's terminal event or release.
|
||||
return {
|
||||
action: "apply",
|
||||
patch: {
|
||||
restartRecoveryRuns: remaining?.length ? remaining : undefined,
|
||||
restartRecoveryTerminalRunIds: mergeRestartRecoveryTerminalRunIds(
|
||||
params.entry?.restartRecoveryTerminalRunIds,
|
||||
[runId],
|
||||
),
|
||||
...(foregroundOwnerClaimId
|
||||
? { mainRestartRecovery: recoveryStateAfterForegroundSettlement }
|
||||
: {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
if (foregroundOwnerClaimId) {
|
||||
// This exact foreground run completed while its release lease was still
|
||||
// active. Its terminal snapshot is authoritative and consumes the cycle.
|
||||
Object.assign(patch, buildMainSessionRecoveryClearPatch(params.entry));
|
||||
return { action: "apply", patch };
|
||||
}
|
||||
if (
|
||||
!hasForegroundOwners &&
|
||||
!hasCurrentReservation &&
|
||||
params.entry?.abortedLastRun === true &&
|
||||
(remaining?.length ?? 0) > 0
|
||||
) {
|
||||
return { action: "apply", patch: { restartRecoveryRuns: remaining } };
|
||||
}
|
||||
const recoveryDeliveryRunId =
|
||||
typeof params.entry?.restartRecoveryDeliveryRunId === "string"
|
||||
? params.entry.restartRecoveryDeliveryRunId.trim()
|
||||
: undefined;
|
||||
if ((remaining?.length ?? 0) > 0 && recoveryDeliveryRunId !== runId) {
|
||||
// A different terminal run may consume only its own fence. Another
|
||||
// admitted recovery remains the durable owner of the aggregate.
|
||||
patch.abortedLastRun = false;
|
||||
patch.restartRecoveryRuns = remaining;
|
||||
patch.mainRestartRecovery = params.entry?.mainRestartRecovery;
|
||||
return { action: "apply", patch };
|
||||
}
|
||||
// An admitted recovery clears the interruption flag before it runs. With
|
||||
// no live owner left, that exact delivery run is the durable cleanup boundary.
|
||||
Object.assign(patch, buildMainSessionRecoveryClearPatch(params.entry));
|
||||
return { action: "apply", patch };
|
||||
}
|
||||
if (phase === "start" || !matchesFence || !remaining) {
|
||||
return { action: "apply", patch };
|
||||
}
|
||||
if (params.entry?.abortedLastRun === true && remaining.length > 0) {
|
||||
return { action: "apply", patch: { restartRecoveryRuns: remaining } };
|
||||
}
|
||||
patch.restartRecoveryRuns = remaining.length > 0 ? remaining : undefined;
|
||||
return { action: "apply", patch };
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
import { getRuntimeConfig } from "../config/io.js";
|
||||
import { getGatewayRecoveryRuntime } from "../gateway/server-recovery-runtime-context.js";
|
||||
import type { MainSessionRecoveryPendingTarget } from "./main-session-recovery-store.js";
|
||||
|
||||
/** Schedules exact-row recovery only after the caller releases its lifecycle admission. */
|
||||
export function scheduleMainSessionRecoveryPendingTarget(
|
||||
target: MainSessionRecoveryPendingTarget | undefined,
|
||||
): void {
|
||||
if (!target) {
|
||||
return;
|
||||
}
|
||||
void import("./main-session-restart-recovery.js").then(
|
||||
({ scheduleRestartAbortedMainSessionRecoveryAfterOwnerRelease }) => {
|
||||
scheduleRestartAbortedMainSessionRecoveryAfterOwnerRelease({
|
||||
expectedSessionId: target.sessionId,
|
||||
getConfig: getRuntimeConfig,
|
||||
getGatewayRuntime: getGatewayRecoveryRuntime,
|
||||
sessionKey: target.sessionKey,
|
||||
storePath: target.storePath,
|
||||
});
|
||||
},
|
||||
() => {
|
||||
// Startup recovery remains the fallback if the optional recovery module cannot load.
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
import { retryAsync } from "../infra/retry.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import { scheduleMainSessionRecoveryPendingTarget } from "./main-session-recovery-owner-release.js";
|
||||
import type { MainSessionRecoveryPendingTarget } from "./main-session-recovery-store.js";
|
||||
|
||||
const log = createSubsystemLogger("main-session-recovery");
|
||||
const RESTORE_RETRY_DELAY_MS = 1_000;
|
||||
const RESTORE_RETRY_MAX_DELAY_MS = 30_000;
|
||||
|
||||
export type RestoreAdmittedRecovery = () => Promise<MainSessionRecoveryPendingTarget | undefined>;
|
||||
|
||||
export async function restoreAdmittedRecoveryWithRetries(
|
||||
restore: RestoreAdmittedRecovery,
|
||||
): Promise<MainSessionRecoveryPendingTarget | undefined> {
|
||||
return await retryAsync(restore, 3, 25);
|
||||
}
|
||||
|
||||
export function scheduleAdmittedRecoveryRestore(
|
||||
restore: RestoreAdmittedRecovery,
|
||||
delayMs = RESTORE_RETRY_DELAY_MS,
|
||||
): void {
|
||||
// Gateway admission consumed the reservation already. Keep restoration
|
||||
// alive until this exact idempotent callback repairs or rejects its fence.
|
||||
setTimeout(() => {
|
||||
void restoreAdmittedRecoveryWithRetries(restore).then(
|
||||
(pendingRecovery) => {
|
||||
scheduleMainSessionRecoveryPendingTarget(pendingRecovery);
|
||||
},
|
||||
(error: unknown) => {
|
||||
log.warn(`failed delayed admitted recovery restoration: ${formatErrorMessage(error)}`);
|
||||
scheduleAdmittedRecoveryRestore(restore, Math.min(delayMs * 2, RESTORE_RETRY_MAX_DELAY_MS));
|
||||
},
|
||||
);
|
||||
}, delayMs).unref?.();
|
||||
}
|
||||
@@ -0,0 +1,916 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type {
|
||||
InternalSessionEntry as SessionEntry,
|
||||
MainRestartRecoveryState,
|
||||
} from "../config/sessions.js";
|
||||
import { buildMainSessionRecoveryClearPatch } from "./main-session-recovery-clear.js";
|
||||
import {
|
||||
inspectMainSessionRecoveryHealth,
|
||||
projectMainSessionRecoveryLifecycle,
|
||||
} from "./main-session-recovery-lifecycle.js";
|
||||
import { transitionMainSessionRecovery } from "./main-session-recovery-state.js";
|
||||
|
||||
const sessionKey = "agent:main:main";
|
||||
|
||||
function recoveryState(
|
||||
overrides: Partial<MainRestartRecoveryState> = {},
|
||||
): MainRestartRecoveryState {
|
||||
return {
|
||||
cycleId: "cycle-1",
|
||||
revision: 1,
|
||||
chargedAttempts: 0,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function interruptedEntry(overrides: Partial<SessionEntry> = {}): SessionEntry {
|
||||
return {
|
||||
sessionId: "session-1",
|
||||
updatedAt: 100,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery: recoveryState(),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function observe(entry: SessionEntry, lifecycleGeneration: string) {
|
||||
const result = transitionMainSessionRecovery(entry, {
|
||||
kind: "observe",
|
||||
cycleId: "unused-cycle",
|
||||
lifecycleGeneration,
|
||||
sessionKey,
|
||||
});
|
||||
if (result.kind !== "observed") {
|
||||
throw new Error("expected recovery observation");
|
||||
}
|
||||
return result.view;
|
||||
}
|
||||
|
||||
describe("main session recovery state", () => {
|
||||
it("gives a legacy interrupted row a stable cycle before exposing it to a scan", () => {
|
||||
const entry = interruptedEntry({ mainRestartRecovery: undefined });
|
||||
|
||||
const observed = transitionMainSessionRecovery(entry, {
|
||||
kind: "observe",
|
||||
cycleId: "legacy-cycle",
|
||||
lifecycleGeneration: "generation-1",
|
||||
sessionKey,
|
||||
});
|
||||
|
||||
expect(observed).toEqual({
|
||||
kind: "observed",
|
||||
view: {
|
||||
status: "recoverable",
|
||||
observation: { sessionId: "session-1", cycleId: "legacy-cycle", revision: 1 },
|
||||
nextAttempt: 1,
|
||||
},
|
||||
});
|
||||
expect(entry.mainRestartRecovery).toEqual(recoveryState({ cycleId: "legacy-cycle" }));
|
||||
});
|
||||
|
||||
it("inspects a live reservation without adopting or releasing it", () => {
|
||||
const entry = interruptedEntry({
|
||||
mainRestartRecovery: recoveryState({
|
||||
chargedAttempts: 1,
|
||||
reservation: {
|
||||
attempt: 1,
|
||||
lifecycleGeneration: "gateway-generation",
|
||||
runId: "recovery-1",
|
||||
},
|
||||
}),
|
||||
});
|
||||
const before = structuredClone(entry);
|
||||
|
||||
expect(
|
||||
transitionMainSessionRecovery(entry, {
|
||||
kind: "inspect",
|
||||
lifecycleGeneration: "standalone-generation",
|
||||
sessionKey,
|
||||
}),
|
||||
).toEqual({ kind: "observed", view: { status: "blocked" } });
|
||||
expect(entry).toEqual(before);
|
||||
});
|
||||
|
||||
it("marks without charging and preserves generation-scoped lifecycle fences", () => {
|
||||
const entry = interruptedEntry({
|
||||
restartRecoveryRuns: [
|
||||
{ runId: "older-run", lifecycleGeneration: "generation-old" },
|
||||
{ runId: "shared-run", lifecycleGeneration: "generation-1" },
|
||||
],
|
||||
mainRestartRecovery: recoveryState({
|
||||
revision: 4,
|
||||
chargedAttempts: 2,
|
||||
}),
|
||||
});
|
||||
|
||||
transitionMainSessionRecovery(entry, {
|
||||
kind: "mark_interrupted",
|
||||
cycleId: "unused-cycle",
|
||||
now: 200,
|
||||
runs: [
|
||||
{ runId: "shared-run", lifecycleGeneration: "generation-2" },
|
||||
{ runId: "new-run", lifecycleGeneration: "generation-2" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(entry.mainRestartRecovery).toEqual(
|
||||
recoveryState({
|
||||
revision: 4,
|
||||
chargedAttempts: 2,
|
||||
}),
|
||||
);
|
||||
expect(entry.restartRecoveryRuns).toEqual([
|
||||
{ runId: "new-run", lifecycleGeneration: "generation-2" },
|
||||
{ runId: "older-run", lifecycleGeneration: "generation-old" },
|
||||
{ runId: "shared-run", lifecycleGeneration: "generation-1" },
|
||||
{ runId: "shared-run", lifecycleGeneration: "generation-2" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("charges at reservation and refunds only the matching reservation", () => {
|
||||
const entry = interruptedEntry();
|
||||
const prepared = transitionMainSessionRecovery(entry, {
|
||||
kind: "prepare_attempt",
|
||||
attempt: 1,
|
||||
lifecycleGeneration: "generation-1",
|
||||
now: 200,
|
||||
observation: { sessionId: "session-1", cycleId: "cycle-1", revision: 1 },
|
||||
runId: "recovery-1",
|
||||
});
|
||||
expect(prepared.kind).toBe("reserved");
|
||||
if (prepared.kind !== "reserved") {
|
||||
throw new Error("expected reservation");
|
||||
}
|
||||
|
||||
expect(
|
||||
transitionMainSessionRecovery(entry, {
|
||||
kind: "prepare_attempt",
|
||||
attempt: 1,
|
||||
lifecycleGeneration: "generation-1",
|
||||
now: 201,
|
||||
observation: { sessionId: "session-1", cycleId: "cycle-1", revision: 1 },
|
||||
runId: "recovery-2",
|
||||
}),
|
||||
).toEqual({ kind: "rejected", reason: "stale_revision" });
|
||||
expect(entry.mainRestartRecovery?.reservation).toMatchObject({
|
||||
runId: "recovery-1",
|
||||
attempt: 1,
|
||||
});
|
||||
|
||||
const claim = transitionMainSessionRecovery(entry, {
|
||||
kind: "claim_foreground",
|
||||
cycleId: "unused",
|
||||
lifecycleGeneration: "generation-1",
|
||||
sessionId: "session-1",
|
||||
sessionKey,
|
||||
claimId: "foreground-1",
|
||||
});
|
||||
expect(claim.kind).toBe("foreground_claimed");
|
||||
|
||||
expect(
|
||||
transitionMainSessionRecovery(entry, {
|
||||
kind: "cancel_reservation",
|
||||
reservation: prepared.reservation,
|
||||
}),
|
||||
).toEqual({ kind: "applied" });
|
||||
expect(entry.mainRestartRecovery).toMatchObject({
|
||||
chargedAttempts: 0,
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration: "generation-1",
|
||||
tokens: ["foreground-1"],
|
||||
},
|
||||
});
|
||||
expect(entry.mainRestartRecovery?.reservation).toBeUndefined();
|
||||
expect(observe(entry, "generation-1")).toEqual({ status: "blocked" });
|
||||
});
|
||||
|
||||
it("rejects foreground work after the automatic recovery budget is exhausted", () => {
|
||||
const entry = interruptedEntry({
|
||||
mainRestartRecovery: recoveryState({ chargedAttempts: 3 }),
|
||||
});
|
||||
const before = structuredClone(entry);
|
||||
|
||||
expect(
|
||||
transitionMainSessionRecovery(entry, {
|
||||
kind: "claim_foreground",
|
||||
cycleId: "unused",
|
||||
lifecycleGeneration: "generation-1",
|
||||
sessionId: "session-1",
|
||||
sessionKey,
|
||||
claimId: "foreground-1",
|
||||
}),
|
||||
).toEqual({ kind: "rejected", reason: "recovery_exhausted" });
|
||||
expect(entry).toEqual(before);
|
||||
});
|
||||
|
||||
it("retires a stale reservation before granting foreground ownership", () => {
|
||||
const entry = interruptedEntry({
|
||||
mainRestartRecovery: recoveryState({
|
||||
revision: 3,
|
||||
chargedAttempts: 1,
|
||||
reservation: {
|
||||
attempt: 1,
|
||||
lifecycleGeneration: "previous-generation",
|
||||
runId: "stale-recovery",
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(
|
||||
transitionMainSessionRecovery(entry, {
|
||||
kind: "claim_foreground",
|
||||
cycleId: "unused",
|
||||
lifecycleGeneration: "generation-1",
|
||||
sessionId: "session-1",
|
||||
sessionKey,
|
||||
claimId: "foreground-1",
|
||||
}),
|
||||
).toMatchObject({ kind: "foreground_claimed" });
|
||||
expect(entry.mainRestartRecovery).toMatchObject({
|
||||
chargedAttempts: 1,
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration: "generation-1",
|
||||
tokens: ["foreground-1"],
|
||||
},
|
||||
});
|
||||
expect(entry.mainRestartRecovery?.reservation).toBeUndefined();
|
||||
});
|
||||
|
||||
it("releases an ambiguous dispatch reservation without refunding its charge", () => {
|
||||
const entry = interruptedEntry();
|
||||
const prepared = transitionMainSessionRecovery(entry, {
|
||||
kind: "prepare_attempt",
|
||||
attempt: 1,
|
||||
lifecycleGeneration: "generation-1",
|
||||
now: 200,
|
||||
observation: { sessionId: "session-1", cycleId: "cycle-1", revision: 1 },
|
||||
runId: "recovery-1",
|
||||
});
|
||||
if (prepared.kind !== "reserved") {
|
||||
throw new Error("expected reservation");
|
||||
}
|
||||
|
||||
expect(
|
||||
transitionMainSessionRecovery(entry, {
|
||||
kind: "abandon_reservation",
|
||||
reservation: prepared.reservation,
|
||||
}),
|
||||
).toEqual({ kind: "applied" });
|
||||
expect(entry.mainRestartRecovery).toMatchObject({ chargedAttempts: 1 });
|
||||
expect(entry.mainRestartRecovery?.reservation).toBeUndefined();
|
||||
expect(observe(entry, "generation-1")).toMatchObject({
|
||||
status: "recoverable",
|
||||
nextAttempt: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it("moves a reservation into the lifecycle fence during Gateway admission", () => {
|
||||
const entry = interruptedEntry({
|
||||
pendingFinalDelivery: true,
|
||||
pendingFinalDeliveryText: " captured reply ",
|
||||
restartRecoveryDeliveryRunId: "recovery-1",
|
||||
restartRecoveryDeliverySourceRunId: "source-1",
|
||||
mainRestartRecovery: recoveryState({
|
||||
revision: 2,
|
||||
chargedAttempts: 1,
|
||||
reservation: {
|
||||
runId: "recovery-1",
|
||||
attempt: 1,
|
||||
lifecycleGeneration: "generation-1",
|
||||
},
|
||||
}),
|
||||
});
|
||||
expect(
|
||||
transitionMainSessionRecovery(entry, {
|
||||
kind: "validate_recovery",
|
||||
lifecycleGeneration: "generation-1",
|
||||
runId: "recovery-1",
|
||||
sessionId: "session-1",
|
||||
}),
|
||||
).toEqual({ kind: "recovery_validated" });
|
||||
expect(entry.mainRestartRecovery?.reservation).toBeDefined();
|
||||
expect(entry.abortedLastRun).toBe(true);
|
||||
|
||||
expect(
|
||||
transitionMainSessionRecovery(entry, {
|
||||
kind: "admit_recovery",
|
||||
lifecycleGeneration: "generation-1",
|
||||
now: 300,
|
||||
runId: "recovery-1",
|
||||
sessionId: "session-1",
|
||||
}),
|
||||
).toEqual({ kind: "admitted_recovery" });
|
||||
expect(entry).toMatchObject({
|
||||
abortedLastRun: false,
|
||||
pendingFinalDeliveryAttemptCount: 1,
|
||||
pendingFinalDeliveryLastAttemptAt: 300,
|
||||
restartRecoveryRuns: [{ runId: "recovery-1", lifecycleGeneration: "generation-1" }],
|
||||
mainRestartRecovery: {
|
||||
revision: 3,
|
||||
chargedAttempts: 1,
|
||||
},
|
||||
});
|
||||
expect(entry.mainRestartRecovery?.reservation).toBeUndefined();
|
||||
|
||||
expect(
|
||||
transitionMainSessionRecovery(entry, {
|
||||
kind: "mark_admitted_recovery_interrupted",
|
||||
lifecycleGeneration: "generation-1",
|
||||
now: 400,
|
||||
runId: "recovery-1",
|
||||
sessionId: "session-1",
|
||||
}),
|
||||
).toMatchObject({ kind: "applied" });
|
||||
expect(entry.mainRestartRecovery?.chargedAttempts).toBe(1);
|
||||
expect(entry.mainRestartRecovery?.reservation).toBeUndefined();
|
||||
expect(entry.abortedLastRun).toBe(true);
|
||||
expect(entry.restartRecoveryDeliveryRunId).toBeUndefined();
|
||||
expect(entry.restartRecoveryDeliverySourceRunId).toBe("source-1");
|
||||
});
|
||||
|
||||
it("rejects a reservation created by an older lifecycle generation", () => {
|
||||
const entry = interruptedEntry({
|
||||
mainRestartRecovery: recoveryState({
|
||||
revision: 2,
|
||||
chargedAttempts: 1,
|
||||
reservation: {
|
||||
runId: "recovery-1",
|
||||
attempt: 1,
|
||||
lifecycleGeneration: "generation-old",
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(
|
||||
transitionMainSessionRecovery(entry, {
|
||||
kind: "admit_recovery",
|
||||
lifecycleGeneration: "generation-new",
|
||||
now: 300,
|
||||
runId: "recovery-1",
|
||||
sessionId: "session-1",
|
||||
}),
|
||||
).toEqual({ kind: "rejected", reason: "stale_reservation" });
|
||||
expect(entry.mainRestartRecovery?.reservation).toBeDefined();
|
||||
expect(entry.abortedLastRun).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects recovery admission while any current-generation foreground claim remains", () => {
|
||||
const entry = interruptedEntry({
|
||||
mainRestartRecovery: recoveryState({
|
||||
revision: 4,
|
||||
chargedAttempts: 1,
|
||||
reservation: {
|
||||
runId: "recovery-1",
|
||||
attempt: 1,
|
||||
lifecycleGeneration: "generation-1",
|
||||
},
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration: "generation-1",
|
||||
tokens: ["foreground-1", "foreground-2"],
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(
|
||||
transitionMainSessionRecovery(entry, {
|
||||
kind: "admit_recovery",
|
||||
lifecycleGeneration: "generation-1",
|
||||
now: 300,
|
||||
runId: "recovery-1",
|
||||
sessionId: "session-1",
|
||||
}),
|
||||
).toEqual({ kind: "rejected", reason: "foreground_active" });
|
||||
expect(observe(entry, "generation-1")).toEqual({ status: "blocked" });
|
||||
});
|
||||
|
||||
it("clears a healthy recovery aggregate when its final foreground owner releases", () => {
|
||||
const entry = interruptedEntry({
|
||||
abortedLastRun: false,
|
||||
restartRecoveryRuns: [{ runId: "recovery-1", lifecycleGeneration: "generation-1" }],
|
||||
mainRestartRecovery: recoveryState({
|
||||
revision: 3,
|
||||
chargedAttempts: 1,
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration: "generation-1",
|
||||
tokens: ["foreground-1"],
|
||||
},
|
||||
}),
|
||||
});
|
||||
expect(observe(entry, "generation-1")).toEqual({ status: "blocked" });
|
||||
|
||||
expect(
|
||||
transitionMainSessionRecovery(entry, {
|
||||
kind: "release_foreground",
|
||||
claim: {
|
||||
cycleId: "cycle-1",
|
||||
lifecycleGeneration: "generation-1",
|
||||
claimId: "foreground-1",
|
||||
sessionId: "session-1",
|
||||
sessionKey,
|
||||
},
|
||||
}),
|
||||
).toEqual({ kind: "applied" });
|
||||
expect(entry).toMatchObject({ abortedLastRun: false });
|
||||
expect(entry.restartRecoveryRuns).toBeUndefined();
|
||||
expect(entry.mainRestartRecovery).toBeUndefined();
|
||||
});
|
||||
|
||||
it("expires foreground claims from an older lifecycle generation", () => {
|
||||
const entry = interruptedEntry({
|
||||
mainRestartRecovery: recoveryState({
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration: "generation-old",
|
||||
tokens: ["old-owner"],
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
const view = observe(entry, "generation-new");
|
||||
|
||||
expect(view).toMatchObject({ status: "recoverable", nextAttempt: 1 });
|
||||
expect(entry.mainRestartRecovery).toMatchObject({ revision: 2 });
|
||||
expect(entry.mainRestartRecovery?.foregroundClaims).toBeUndefined();
|
||||
});
|
||||
|
||||
it("clears a healthy aggregate owned by an older lifecycle generation", () => {
|
||||
const entry = interruptedEntry({
|
||||
abortedLastRun: false,
|
||||
restartRecoveryRuns: [{ runId: "completed-run", lifecycleGeneration: "generation-old" }],
|
||||
mainRestartRecovery: recoveryState({
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration: "generation-old",
|
||||
tokens: ["old-owner"],
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(observe(entry, "generation-new")).toEqual({ status: "inactive" });
|
||||
expect(entry.restartRecoveryRuns).toBeUndefined();
|
||||
expect(entry.mainRestartRecovery).toBeUndefined();
|
||||
});
|
||||
|
||||
it("retains the charge but releases a reservation orphaned by process restart", () => {
|
||||
const entry = interruptedEntry({
|
||||
mainRestartRecovery: recoveryState({
|
||||
revision: 2,
|
||||
chargedAttempts: 1,
|
||||
reservation: {
|
||||
runId: "recovery-1",
|
||||
attempt: 1,
|
||||
lifecycleGeneration: "generation-old",
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(
|
||||
transitionMainSessionRecovery(entry, {
|
||||
kind: "observe",
|
||||
cycleId: "unused",
|
||||
lifecycleGeneration: "generation-new",
|
||||
sessionKey,
|
||||
}),
|
||||
).toEqual({
|
||||
kind: "observed",
|
||||
view: {
|
||||
status: "recoverable",
|
||||
observation: { sessionId: "session-1", cycleId: "cycle-1", revision: 3 },
|
||||
nextAttempt: 2,
|
||||
},
|
||||
});
|
||||
expect(entry.mainRestartRecovery).toMatchObject({
|
||||
revision: 3,
|
||||
chargedAttempts: 1,
|
||||
});
|
||||
expect(entry.mainRestartRecovery?.reservation).toBeUndefined();
|
||||
});
|
||||
|
||||
it("rejects an old observation after a healthy clear and a new interrupted cycle", () => {
|
||||
const entry = interruptedEntry();
|
||||
const oldObservation = { sessionId: "session-1", cycleId: "cycle-1", revision: 1 };
|
||||
transitionMainSessionRecovery(entry, { kind: "clear" });
|
||||
transitionMainSessionRecovery(entry, {
|
||||
kind: "mark_interrupted",
|
||||
cycleId: "cycle-2",
|
||||
now: 400,
|
||||
});
|
||||
|
||||
expect(
|
||||
transitionMainSessionRecovery(entry, {
|
||||
kind: "prepare_attempt",
|
||||
attempt: 1,
|
||||
lifecycleGeneration: "generation-1",
|
||||
now: 500,
|
||||
observation: oldObservation,
|
||||
runId: "stale-run",
|
||||
}),
|
||||
).toEqual({ kind: "rejected", reason: "stale_cycle" });
|
||||
});
|
||||
|
||||
it("preserves the charged cycle when an accepted recovery is interrupted again", () => {
|
||||
const entry = interruptedEntry({
|
||||
abortedLastRun: false,
|
||||
mainRestartRecovery: recoveryState({
|
||||
revision: 3,
|
||||
chargedAttempts: 1,
|
||||
}),
|
||||
});
|
||||
|
||||
transitionMainSessionRecovery(entry, {
|
||||
kind: "mark_interrupted",
|
||||
cycleId: "replacement-cycle",
|
||||
now: 300,
|
||||
});
|
||||
|
||||
expect(entry.abortedLastRun).toBe(true);
|
||||
expect(entry.mainRestartRecovery).toEqual(
|
||||
recoveryState({
|
||||
revision: 3,
|
||||
chargedAttempts: 1,
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("tombstones an exhausted cycle and exposes only the Doctor repair action", () => {
|
||||
const entry = interruptedEntry({
|
||||
mainRestartRecovery: recoveryState({
|
||||
chargedAttempts: 3,
|
||||
}),
|
||||
});
|
||||
const view = observe(entry, "generation-1");
|
||||
expect(view.status).toBe("exhausted");
|
||||
if (view.status !== "exhausted") {
|
||||
throw new Error("expected exhausted cycle");
|
||||
}
|
||||
|
||||
expect(
|
||||
transitionMainSessionRecovery(entry, {
|
||||
kind: "tombstone",
|
||||
now: 400,
|
||||
observation: view.observation,
|
||||
reason: view.reason,
|
||||
}),
|
||||
).toEqual({ kind: "tombstoned" });
|
||||
expect(inspectMainSessionRecoveryHealth(entry)).toEqual({
|
||||
status: "tombstoned",
|
||||
reason: view.reason,
|
||||
repair: null,
|
||||
});
|
||||
expect(observe(entry, "generation-1")).toEqual({ status: "tombstoned" });
|
||||
|
||||
entry.abortedLastRun = true;
|
||||
expect(inspectMainSessionRecoveryHealth(entry)).toMatchObject({
|
||||
status: "tombstoned",
|
||||
repair: "clear_stale_abort",
|
||||
});
|
||||
expect(transitionMainSessionRecovery(entry, { kind: "doctor_repair", now: 500 })).toEqual({
|
||||
kind: "doctor_repaired",
|
||||
});
|
||||
expect(entry.abortedLastRun).toBe(false);
|
||||
});
|
||||
|
||||
it("owns lifecycle fence suppression, consumption, and healthy clearing", () => {
|
||||
const entry = interruptedEntry({
|
||||
abortedLastRun: false,
|
||||
restartRecoveryDeliveryRunId: "recovery",
|
||||
restartRecoveryRuns: [
|
||||
{ runId: "interrupted", lifecycleGeneration: "generation-1" },
|
||||
{ runId: "recovery", lifecycleGeneration: "generation-1" },
|
||||
],
|
||||
});
|
||||
expect(
|
||||
projectMainSessionRecoveryLifecycle({
|
||||
currentLifecycleGeneration: "generation-1",
|
||||
entry,
|
||||
event: {
|
||||
runId: "interrupted",
|
||||
lifecycleGeneration: "generation-1",
|
||||
data: { phase: "error", stopReason: "restart" },
|
||||
},
|
||||
snapshotPatch: { status: "failed" },
|
||||
}),
|
||||
).toEqual({ action: "suppress" });
|
||||
|
||||
expect(
|
||||
projectMainSessionRecoveryLifecycle({
|
||||
currentLifecycleGeneration: "generation-1",
|
||||
entry,
|
||||
event: {
|
||||
runId: "recovery",
|
||||
lifecycleGeneration: "generation-1",
|
||||
data: { phase: "end" },
|
||||
},
|
||||
snapshotPatch: { status: "done", abortedLastRun: false },
|
||||
}),
|
||||
).toEqual({
|
||||
action: "apply",
|
||||
patch: {
|
||||
status: "done",
|
||||
abortedLastRun: false,
|
||||
restartRecoveryRuns: undefined,
|
||||
mainRestartRecovery: undefined,
|
||||
},
|
||||
});
|
||||
|
||||
expect(
|
||||
projectMainSessionRecoveryLifecycle({
|
||||
currentLifecycleGeneration: "generation-1",
|
||||
entry,
|
||||
event: {
|
||||
runId: "recovery",
|
||||
lifecycleGeneration: "generation-1",
|
||||
data: { phase: "error", error: "provider failed" },
|
||||
},
|
||||
snapshotPatch: { status: "failed", abortedLastRun: false },
|
||||
}),
|
||||
).toEqual({
|
||||
action: "apply",
|
||||
patch: {
|
||||
status: "failed",
|
||||
abortedLastRun: false,
|
||||
restartRecoveryRuns: undefined,
|
||||
mainRestartRecovery: undefined,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("does not let a delayed lifecycle event clear current-generation owners", () => {
|
||||
const entry = interruptedEntry({
|
||||
abortedLastRun: false,
|
||||
restartRecoveryRuns: [{ runId: "old-run", lifecycleGeneration: "generation-1" }],
|
||||
restartRecoveryTerminalRunIds: ["prior-run"],
|
||||
mainRestartRecovery: recoveryState({
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration: "generation-2",
|
||||
tokens: ["current-owner"],
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(
|
||||
projectMainSessionRecoveryLifecycle({
|
||||
currentLifecycleGeneration: "generation-2",
|
||||
entry,
|
||||
event: {
|
||||
runId: "old-run",
|
||||
lifecycleGeneration: "generation-1",
|
||||
data: { phase: "end" },
|
||||
},
|
||||
snapshotPatch: { status: "done", abortedLastRun: false },
|
||||
}),
|
||||
).toEqual({
|
||||
action: "apply",
|
||||
patch: {
|
||||
restartRecoveryRuns: undefined,
|
||||
restartRecoveryTerminalRunIds: ["prior-run", "old-run"],
|
||||
},
|
||||
});
|
||||
expect(entry.mainRestartRecovery?.foregroundClaims?.tokens).toEqual(["current-owner"]);
|
||||
});
|
||||
|
||||
it("does not let a delayed lifecycle event clear a current reservation", () => {
|
||||
const entry = interruptedEntry({
|
||||
restartRecoveryRuns: [{ runId: "old-run", lifecycleGeneration: "generation-1" }],
|
||||
mainRestartRecovery: recoveryState({
|
||||
chargedAttempts: 1,
|
||||
reservation: {
|
||||
runId: "current-run",
|
||||
attempt: 1,
|
||||
lifecycleGeneration: "generation-2",
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(
|
||||
projectMainSessionRecoveryLifecycle({
|
||||
currentLifecycleGeneration: "generation-2",
|
||||
entry,
|
||||
event: {
|
||||
runId: "old-run",
|
||||
lifecycleGeneration: "generation-1",
|
||||
data: { phase: "end" },
|
||||
},
|
||||
snapshotPatch: { status: "done", abortedLastRun: false },
|
||||
}),
|
||||
).toEqual({
|
||||
action: "apply",
|
||||
patch: {
|
||||
restartRecoveryRuns: undefined,
|
||||
restartRecoveryTerminalRunIds: ["old-run"],
|
||||
},
|
||||
});
|
||||
expect(entry.mainRestartRecovery?.reservation?.runId).toBe("current-run");
|
||||
});
|
||||
|
||||
it("preserves concurrent owners and fences when one lifecycle run completes", () => {
|
||||
const entry = interruptedEntry({
|
||||
abortedLastRun: false,
|
||||
restartRecoveryRuns: [
|
||||
{ runId: "recovery-1", lifecycleGeneration: "generation-1" },
|
||||
{ runId: "recovery-2", lifecycleGeneration: "generation-1" },
|
||||
],
|
||||
mainRestartRecovery: recoveryState({
|
||||
revision: 4,
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration: "generation-1",
|
||||
tokens: ["owner-1", "owner-2"],
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(
|
||||
projectMainSessionRecoveryLifecycle({
|
||||
currentLifecycleGeneration: "generation-1",
|
||||
entry,
|
||||
event: {
|
||||
runId: "recovery-1",
|
||||
lifecycleGeneration: "generation-1",
|
||||
data: { phase: "end" },
|
||||
},
|
||||
snapshotPatch: {
|
||||
status: "done",
|
||||
abortedLastRun: false,
|
||||
restartRecoveryRuns: undefined,
|
||||
mainRestartRecovery: undefined,
|
||||
},
|
||||
}),
|
||||
).toEqual({
|
||||
action: "apply",
|
||||
patch: {
|
||||
restartRecoveryRuns: [{ runId: "recovery-2", lifecycleGeneration: "generation-1" }],
|
||||
restartRecoveryTerminalRunIds: ["recovery-1"],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("does not let another terminal run clear the active recovery delivery", () => {
|
||||
const entry = interruptedEntry({
|
||||
abortedLastRun: false,
|
||||
restartRecoveryDeliveryRunId: "recovery-2",
|
||||
restartRecoveryRuns: [
|
||||
{ runId: "recovery-1", lifecycleGeneration: "generation-1" },
|
||||
{ runId: "recovery-2", lifecycleGeneration: "generation-1" },
|
||||
],
|
||||
});
|
||||
|
||||
expect(
|
||||
projectMainSessionRecoveryLifecycle({
|
||||
currentLifecycleGeneration: "generation-1",
|
||||
entry,
|
||||
event: {
|
||||
runId: "recovery-1",
|
||||
lifecycleGeneration: "generation-1",
|
||||
data: { phase: "end" },
|
||||
},
|
||||
snapshotPatch: { status: "done", abortedLastRun: false },
|
||||
}),
|
||||
).toEqual({
|
||||
action: "apply",
|
||||
patch: {
|
||||
status: "done",
|
||||
abortedLastRun: false,
|
||||
restartRecoveryRuns: [{ runId: "recovery-2", lifecycleGeneration: "generation-1" }],
|
||||
mainRestartRecovery: entry.mainRestartRecovery,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("does not let an unrelated lifecycle completion clear pending recovery", () => {
|
||||
const entry = interruptedEntry({
|
||||
restartRecoveryRuns: [{ runId: "recovery", lifecycleGeneration: "generation-1" }],
|
||||
});
|
||||
|
||||
expect(
|
||||
projectMainSessionRecoveryLifecycle({
|
||||
currentLifecycleGeneration: "generation-1",
|
||||
entry,
|
||||
event: {
|
||||
runId: "ordinary-run",
|
||||
lifecycleGeneration: "generation-1",
|
||||
data: { phase: "end" },
|
||||
},
|
||||
snapshotPatch: {
|
||||
status: "done",
|
||||
abortedLastRun: false,
|
||||
restartRecoveryRuns: undefined,
|
||||
mainRestartRecovery: undefined,
|
||||
},
|
||||
}),
|
||||
).toEqual({ action: "suppress" });
|
||||
});
|
||||
|
||||
it("suppresses an unmatched completion while a current foreground owner is active", () => {
|
||||
const entry = interruptedEntry({
|
||||
abortedLastRun: false,
|
||||
mainRestartRecovery: recoveryState({
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration: "generation-1",
|
||||
tokens: ["current-owner"],
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(
|
||||
projectMainSessionRecoveryLifecycle({
|
||||
currentLifecycleGeneration: "generation-1",
|
||||
entry,
|
||||
event: {
|
||||
runId: "unrelated-run",
|
||||
lifecycleGeneration: "generation-1",
|
||||
data: { phase: "end" },
|
||||
},
|
||||
snapshotPatch: { status: "done", abortedLastRun: false },
|
||||
}),
|
||||
).toEqual({ action: "suppress" });
|
||||
});
|
||||
|
||||
it("applies ordinary lifecycle completion without recovery metadata", () => {
|
||||
expect(
|
||||
projectMainSessionRecoveryLifecycle({
|
||||
currentLifecycleGeneration: "generation-1",
|
||||
entry: { abortedLastRun: false },
|
||||
event: {
|
||||
runId: "ordinary-run",
|
||||
lifecycleGeneration: "generation-1",
|
||||
data: { phase: "end" },
|
||||
},
|
||||
snapshotPatch: { status: "done", abortedLastRun: false },
|
||||
}),
|
||||
).toEqual({
|
||||
action: "apply",
|
||||
patch: { status: "done", abortedLastRun: false },
|
||||
});
|
||||
});
|
||||
|
||||
it("does not preserve foreground claims from an older lifecycle generation", () => {
|
||||
const entry = interruptedEntry({
|
||||
abortedLastRun: false,
|
||||
restartRecoveryRuns: [{ runId: "recovery", lifecycleGeneration: "generation-2" }],
|
||||
mainRestartRecovery: recoveryState({
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration: "generation-1",
|
||||
tokens: ["stale-owner"],
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
expect(
|
||||
projectMainSessionRecoveryLifecycle({
|
||||
currentLifecycleGeneration: "generation-2",
|
||||
entry,
|
||||
event: {
|
||||
runId: "recovery",
|
||||
lifecycleGeneration: "generation-2",
|
||||
data: { phase: "end" },
|
||||
},
|
||||
snapshotPatch: { status: "done", abortedLastRun: false },
|
||||
}),
|
||||
).toEqual({
|
||||
action: "apply",
|
||||
patch: {
|
||||
status: "done",
|
||||
abortedLastRun: false,
|
||||
restartRecoveryRuns: undefined,
|
||||
mainRestartRecovery: undefined,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("does not let a delayed lifecycle event erase a recovery tombstone", () => {
|
||||
const entry = interruptedEntry({
|
||||
abortedLastRun: false,
|
||||
status: "failed",
|
||||
restartRecoveryRuns: [{ runId: "old-run", lifecycleGeneration: "generation-1" }],
|
||||
mainRestartRecovery: recoveryState({
|
||||
revision: 4,
|
||||
chargedAttempts: 3,
|
||||
tombstone: { reason: "automatic recovery exhausted" },
|
||||
}),
|
||||
});
|
||||
|
||||
expect(
|
||||
projectMainSessionRecoveryLifecycle({
|
||||
currentLifecycleGeneration: "generation-1",
|
||||
entry,
|
||||
event: {
|
||||
runId: "old-run",
|
||||
lifecycleGeneration: "generation-1",
|
||||
data: { phase: "end" },
|
||||
},
|
||||
snapshotPatch: { status: "done", abortedLastRun: false },
|
||||
}),
|
||||
).toEqual({
|
||||
action: "apply",
|
||||
patch: {
|
||||
status: "done",
|
||||
abortedLastRun: false,
|
||||
restartRecoveryRuns: entry.restartRecoveryRuns,
|
||||
mainRestartRecovery: entry.mainRestartRecovery,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("builds an empty clear patch when no main recovery state exists", () => {
|
||||
expect(buildMainSessionRecoveryClearPatch({ abortedLastRun: false })).toEqual({});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,659 @@
|
||||
import {
|
||||
PENDING_FINAL_DELIVERY_CLEAR_PATCH,
|
||||
sanitizePendingFinalDeliveryText,
|
||||
} from "../auto-reply/reply/pending-final-delivery.js";
|
||||
import type {
|
||||
InternalSessionEntry as SessionEntry,
|
||||
MainRestartRecoveryState,
|
||||
RestartRecoveryRun,
|
||||
} from "../config/sessions.js";
|
||||
import { buildRestartRecoveryClaimCleanupPatch } from "../config/sessions/restart-recovery-state.js";
|
||||
import { isAcpSessionKey, isCronSessionKey, isSubagentSessionKey } from "../routing/session-key.js";
|
||||
import { buildMainSessionRecoveryClearPatch } from "./main-session-recovery-clear.js";
|
||||
import type {
|
||||
MainSessionRecoveryCommand,
|
||||
MainSessionRecoveryConflict,
|
||||
MainSessionRecoveryObservation,
|
||||
MainSessionRecoveryTransitionResult,
|
||||
MainSessionRecoveryView,
|
||||
} from "./main-session-recovery-types.js";
|
||||
|
||||
export type {
|
||||
MainSessionRecoveryCommand,
|
||||
MainSessionRecoveryObservation,
|
||||
MainSessionRecoveryOwnerClaim,
|
||||
MainSessionRecoveryReservation,
|
||||
MainSessionRecoveryTransitionResult,
|
||||
} from "./main-session-recovery-types.js";
|
||||
|
||||
const MAIN_RESTART_RECOVERY_MAX_AUTOMATIC_ATTEMPTS = 3;
|
||||
|
||||
const MAIN_RESTART_RECOVERY_REMEDIATION_HINT =
|
||||
"inspect the failed main session and use /new or reset to start a replacement session";
|
||||
|
||||
function nextRevision(state: MainRestartRecoveryState): number {
|
||||
return state.revision + 1;
|
||||
}
|
||||
|
||||
function createCycle(cycleId: string): MainRestartRecoveryState {
|
||||
return {
|
||||
cycleId,
|
||||
revision: 1,
|
||||
chargedAttempts: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function observationFor(entry: SessionEntry): MainSessionRecoveryObservation | undefined {
|
||||
const state = entry.mainRestartRecovery;
|
||||
if (!state) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
sessionId: entry.sessionId,
|
||||
cycleId: state.cycleId,
|
||||
revision: state.revision,
|
||||
};
|
||||
}
|
||||
|
||||
function matchesObservation(
|
||||
entry: SessionEntry,
|
||||
observation: MainSessionRecoveryObservation,
|
||||
): MainSessionRecoveryConflict | null {
|
||||
if (entry.sessionId !== observation.sessionId) {
|
||||
return "session_replaced";
|
||||
}
|
||||
if (entry.mainRestartRecovery?.cycleId !== observation.cycleId) {
|
||||
return "stale_cycle";
|
||||
}
|
||||
return entry.mainRestartRecovery.revision === observation.revision ? null : "stale_revision";
|
||||
}
|
||||
|
||||
function hasCurrentForegroundClaim(
|
||||
state: MainRestartRecoveryState,
|
||||
lifecycleGeneration: string,
|
||||
): boolean {
|
||||
return (
|
||||
state.foregroundClaims?.lifecycleGeneration === lifecycleGeneration &&
|
||||
state.foregroundClaims.tokens.length > 0
|
||||
);
|
||||
}
|
||||
|
||||
function validateRecoveryAdmission(
|
||||
entry: SessionEntry,
|
||||
command: {
|
||||
lifecycleGeneration: string;
|
||||
runId: string;
|
||||
sessionId: string;
|
||||
},
|
||||
): MainSessionRecoveryConflict | null {
|
||||
const state = entry.mainRestartRecovery;
|
||||
if (entry.sessionId !== command.sessionId) {
|
||||
return "session_replaced";
|
||||
}
|
||||
if (entry.status !== "running" || entry.abortedLastRun !== true || !state) {
|
||||
return "not_interrupted";
|
||||
}
|
||||
if (
|
||||
state.reservation?.runId !== command.runId ||
|
||||
state.reservation.lifecycleGeneration !== command.lifecycleGeneration
|
||||
) {
|
||||
return "stale_reservation";
|
||||
}
|
||||
return hasCurrentForegroundClaim(state, command.lifecycleGeneration) ? "foreground_active" : null;
|
||||
}
|
||||
|
||||
function recordLifecycleFence(entry: SessionEntry, run: RestartRecoveryRun): void {
|
||||
// Lifecycle fences can overlap and are consumed independently by their matching events.
|
||||
const runs = new Map<string, RestartRecoveryRun>();
|
||||
for (const existing of entry.restartRecoveryRuns ?? []) {
|
||||
runs.set(`${existing.runId}\u0000${existing.lifecycleGeneration}`, existing);
|
||||
}
|
||||
runs.set(`${run.runId}\u0000${run.lifecycleGeneration}`, run);
|
||||
entry.restartRecoveryRuns = [...runs.values()].toSorted((a, b) =>
|
||||
a.runId === b.runId
|
||||
? a.lifecycleGeneration.localeCompare(b.lifecycleGeneration)
|
||||
: a.runId.localeCompare(b.runId),
|
||||
);
|
||||
}
|
||||
|
||||
function hasLifecycleFence(entry: SessionEntry, run: RestartRecoveryRun): boolean {
|
||||
return Boolean(
|
||||
entry.restartRecoveryRuns?.some(
|
||||
(candidate) =>
|
||||
candidate.runId === run.runId && candidate.lifecycleGeneration === run.lifecycleGeneration,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
function formatAttemptBudgetReason(attempts: number): string {
|
||||
return (
|
||||
`main-session restart recovery blocked after ${attempts} charged automatic resume attempts; ` +
|
||||
MAIN_RESTART_RECOVERY_REMEDIATION_HINT
|
||||
);
|
||||
}
|
||||
|
||||
export function isMainSessionRecoveryExhausted(entry: SessionEntry): boolean {
|
||||
return (
|
||||
entry.status === "running" &&
|
||||
entry.abortedLastRun === true &&
|
||||
(entry.mainRestartRecovery?.chargedAttempts ?? 0) >=
|
||||
MAIN_RESTART_RECOVERY_MAX_AUTOMATIC_ATTEMPTS
|
||||
);
|
||||
}
|
||||
|
||||
export function isMainRestartRecoveryCandidate(entry: SessionEntry, sessionKey: string): boolean {
|
||||
if (typeof entry.spawnDepth === "number" && entry.spawnDepth > 0) {
|
||||
return false;
|
||||
}
|
||||
if (entry.subagentRole != null) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
!isSubagentSessionKey(sessionKey) &&
|
||||
!isCronSessionKey(sessionKey) &&
|
||||
!isAcpSessionKey(sessionKey)
|
||||
);
|
||||
}
|
||||
|
||||
function inspectMainSessionRecovery(params: {
|
||||
entry: SessionEntry;
|
||||
lifecycleGeneration: string;
|
||||
sessionKey: string;
|
||||
}): MainSessionRecoveryView {
|
||||
const { entry } = params;
|
||||
const state = entry.mainRestartRecovery;
|
||||
if (state?.tombstone) {
|
||||
return { status: "tombstoned" };
|
||||
}
|
||||
if (state && hasCurrentForegroundClaim(state, params.lifecycleGeneration)) {
|
||||
return { status: "blocked" };
|
||||
}
|
||||
if (
|
||||
entry.status === "running" &&
|
||||
entry.abortedLastRun !== true &&
|
||||
state &&
|
||||
entry.restartRecoveryRuns?.some((run) => run.lifecycleGeneration === params.lifecycleGeneration)
|
||||
) {
|
||||
// Admission clears the interruption flag before the recovery run settles.
|
||||
// Keep ordinary work fenced until that run clears its lifecycle metadata.
|
||||
return { status: "blocked" };
|
||||
}
|
||||
if (
|
||||
entry.status !== "running" ||
|
||||
entry.abortedLastRun !== true ||
|
||||
!isMainRestartRecoveryCandidate(entry, params.sessionKey)
|
||||
) {
|
||||
return { status: "inactive" };
|
||||
}
|
||||
const observation = observationFor(entry);
|
||||
if (!state || !observation) {
|
||||
return { status: "inactive" };
|
||||
}
|
||||
if (state.reservation) {
|
||||
return { status: "blocked" };
|
||||
}
|
||||
if (state.chargedAttempts >= MAIN_RESTART_RECOVERY_MAX_AUTOMATIC_ATTEMPTS) {
|
||||
return {
|
||||
status: "exhausted",
|
||||
observation,
|
||||
reason: formatAttemptBudgetReason(state.chargedAttempts),
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: "recoverable",
|
||||
observation,
|
||||
nextAttempt: state.chargedAttempts + 1,
|
||||
};
|
||||
}
|
||||
|
||||
function inspectMainSessionRecoveryForAdmission(params: {
|
||||
entry: SessionEntry;
|
||||
lifecycleGeneration: string;
|
||||
sessionKey: string;
|
||||
}): MainSessionRecoveryView {
|
||||
if (
|
||||
params.entry.status === "running" &&
|
||||
params.entry.abortedLastRun !== true &&
|
||||
params.entry.mainRestartRecovery &&
|
||||
params.entry.restartRecoveryRuns?.length
|
||||
) {
|
||||
// Standalone callers may use another process generation. Any admitted
|
||||
// recovery fence remains authoritative until Gateway lifecycle settlement.
|
||||
return { status: "blocked" };
|
||||
}
|
||||
if (
|
||||
params.entry.status === "running" &&
|
||||
params.entry.abortedLastRun === true &&
|
||||
isMainRestartRecoveryCandidate(params.entry, params.sessionKey) &&
|
||||
!params.entry.mainRestartRecovery
|
||||
) {
|
||||
// Older interrupted rows still quarantine foreground work, but only the
|
||||
// Gateway startup owner may assign their durable recovery cycle.
|
||||
return { status: "blocked" };
|
||||
}
|
||||
return inspectMainSessionRecovery(params);
|
||||
}
|
||||
|
||||
export function transitionMainSessionRecovery(
|
||||
entry: SessionEntry,
|
||||
command: MainSessionRecoveryCommand,
|
||||
): MainSessionRecoveryTransitionResult {
|
||||
switch (command.kind) {
|
||||
case "mark_interrupted": {
|
||||
if (!entry.mainRestartRecovery) {
|
||||
entry.mainRestartRecovery = createCycle(command.cycleId);
|
||||
}
|
||||
entry.status = "running";
|
||||
entry.abortedLastRun = true;
|
||||
if (command.resetRuntime) {
|
||||
entry.startedAt = undefined;
|
||||
entry.endedAt = undefined;
|
||||
entry.runtimeMs = undefined;
|
||||
}
|
||||
for (const run of command.runs ?? []) {
|
||||
recordLifecycleFence(entry, run);
|
||||
}
|
||||
entry.updatedAt = command.now;
|
||||
return { kind: "applied" };
|
||||
}
|
||||
case "inspect": {
|
||||
return {
|
||||
kind: "observed",
|
||||
view: inspectMainSessionRecoveryForAdmission({
|
||||
entry,
|
||||
lifecycleGeneration: command.lifecycleGeneration,
|
||||
sessionKey: command.sessionKey,
|
||||
}),
|
||||
};
|
||||
}
|
||||
case "observe": {
|
||||
if (
|
||||
entry.status === "running" &&
|
||||
entry.abortedLastRun === true &&
|
||||
isMainRestartRecoveryCandidate(entry, command.sessionKey) &&
|
||||
!entry.mainRestartRecovery
|
||||
) {
|
||||
// Rows interrupted by an older shipped version acquire identity before scanning.
|
||||
entry.mainRestartRecovery = createCycle(command.cycleId);
|
||||
}
|
||||
let state = entry.mainRestartRecovery;
|
||||
if (
|
||||
state?.foregroundClaims &&
|
||||
state.foregroundClaims.lifecycleGeneration !== command.lifecycleGeneration
|
||||
) {
|
||||
// Process-local owners cannot survive a Gateway generation. Retire their
|
||||
// durable lease before the new process decides whether recovery is needed.
|
||||
if (entry.abortedLastRun !== true) {
|
||||
Object.assign(entry, buildMainSessionRecoveryClearPatch(entry));
|
||||
state = undefined;
|
||||
} else {
|
||||
entry.mainRestartRecovery = state = {
|
||||
...state,
|
||||
revision: nextRevision(state),
|
||||
foregroundClaims: undefined,
|
||||
};
|
||||
}
|
||||
}
|
||||
if (
|
||||
state?.reservation &&
|
||||
state.reservation.lifecycleGeneration !== command.lifecycleGeneration
|
||||
) {
|
||||
// A process restart makes dispatch outcome unknowable: retain the charge,
|
||||
// but release the stale slot so the next bounded attempt can proceed.
|
||||
entry.mainRestartRecovery = {
|
||||
...state,
|
||||
revision: nextRevision(state),
|
||||
reservation: undefined,
|
||||
};
|
||||
}
|
||||
return {
|
||||
kind: "observed",
|
||||
view: inspectMainSessionRecovery({
|
||||
entry,
|
||||
lifecycleGeneration: command.lifecycleGeneration,
|
||||
sessionKey: command.sessionKey,
|
||||
}),
|
||||
};
|
||||
}
|
||||
case "prepare_attempt": {
|
||||
const conflict = matchesObservation(entry, command.observation);
|
||||
if (conflict) {
|
||||
return { kind: "rejected", reason: conflict };
|
||||
}
|
||||
const state = entry.mainRestartRecovery!;
|
||||
if (entry.status !== "running" || entry.abortedLastRun !== true) {
|
||||
return { kind: "rejected", reason: "not_interrupted" };
|
||||
}
|
||||
if (state.tombstone) {
|
||||
return { kind: "rejected", reason: "already_tombstoned" };
|
||||
}
|
||||
if (state.reservation) {
|
||||
return { kind: "rejected", reason: "reservation_active" };
|
||||
}
|
||||
if (command.attempt !== state.chargedAttempts + 1) {
|
||||
return { kind: "rejected", reason: "stale_revision" };
|
||||
}
|
||||
entry.mainRestartRecovery = {
|
||||
...state,
|
||||
revision: nextRevision(state),
|
||||
chargedAttempts: command.attempt,
|
||||
reservation: {
|
||||
runId: command.runId,
|
||||
attempt: command.attempt,
|
||||
lifecycleGeneration: command.lifecycleGeneration,
|
||||
},
|
||||
};
|
||||
entry.updatedAt = command.now;
|
||||
return {
|
||||
kind: "reserved",
|
||||
reservation: {
|
||||
sessionId: entry.sessionId,
|
||||
cycleId: state.cycleId,
|
||||
lifecycleGeneration: command.lifecycleGeneration,
|
||||
runId: command.runId,
|
||||
attempt: command.attempt,
|
||||
},
|
||||
};
|
||||
}
|
||||
case "cancel_reservation": {
|
||||
const state = entry.mainRestartRecovery;
|
||||
const reserved = state?.reservation;
|
||||
if (
|
||||
!state ||
|
||||
entry.sessionId !== command.reservation.sessionId ||
|
||||
state.cycleId !== command.reservation.cycleId ||
|
||||
reserved?.runId !== command.reservation.runId ||
|
||||
reserved.attempt !== command.reservation.attempt ||
|
||||
reserved.lifecycleGeneration !== command.reservation.lifecycleGeneration
|
||||
) {
|
||||
return { kind: "rejected", reason: "stale_reservation" };
|
||||
}
|
||||
entry.mainRestartRecovery = {
|
||||
...state,
|
||||
revision: nextRevision(state),
|
||||
chargedAttempts: Math.max(0, command.reservation.attempt - 1),
|
||||
reservation: undefined,
|
||||
};
|
||||
return { kind: "applied" };
|
||||
}
|
||||
case "abandon_reservation": {
|
||||
const state = entry.mainRestartRecovery;
|
||||
const reserved = state?.reservation;
|
||||
if (
|
||||
!state ||
|
||||
entry.sessionId !== command.reservation.sessionId ||
|
||||
state.cycleId !== command.reservation.cycleId ||
|
||||
reserved?.runId !== command.reservation.runId ||
|
||||
reserved.attempt !== command.reservation.attempt ||
|
||||
reserved.lifecycleGeneration !== command.reservation.lifecycleGeneration
|
||||
) {
|
||||
return { kind: "rejected", reason: "stale_reservation" };
|
||||
}
|
||||
entry.mainRestartRecovery = {
|
||||
...state,
|
||||
revision: nextRevision(state),
|
||||
reservation: undefined,
|
||||
};
|
||||
return { kind: "applied" };
|
||||
}
|
||||
case "validate_recovery": {
|
||||
const conflict = validateRecoveryAdmission(entry, command);
|
||||
return conflict ? { kind: "rejected", reason: conflict } : { kind: "recovery_validated" };
|
||||
}
|
||||
case "admit_recovery": {
|
||||
const conflict = validateRecoveryAdmission(entry, command);
|
||||
if (conflict) {
|
||||
return { kind: "rejected", reason: conflict };
|
||||
}
|
||||
const state = entry.mainRestartRecovery!;
|
||||
entry.mainRestartRecovery = {
|
||||
...state,
|
||||
revision: nextRevision(state),
|
||||
reservation: undefined,
|
||||
foregroundClaims: undefined,
|
||||
};
|
||||
entry.abortedLastRun = false;
|
||||
recordLifecycleFence(entry, {
|
||||
runId: command.runId,
|
||||
lifecycleGeneration: command.lifecycleGeneration,
|
||||
});
|
||||
if (entry.pendingFinalDelivery || entry.pendingFinalDeliveryText) {
|
||||
const pendingText = sanitizePendingFinalDeliveryText(entry.pendingFinalDeliveryText ?? "");
|
||||
if (pendingText) {
|
||||
entry.pendingFinalDeliveryLastAttemptAt = command.now;
|
||||
entry.pendingFinalDeliveryAttemptCount =
|
||||
(entry.pendingFinalDeliveryAttemptCount ?? 0) + 1;
|
||||
entry.pendingFinalDeliveryLastError = null;
|
||||
entry.pendingFinalDeliveryText = pendingText;
|
||||
} else {
|
||||
Object.assign(entry, PENDING_FINAL_DELIVERY_CLEAR_PATCH);
|
||||
}
|
||||
}
|
||||
return { kind: "admitted_recovery" };
|
||||
}
|
||||
case "mark_admitted_recovery_interrupted": {
|
||||
const state = entry.mainRestartRecovery;
|
||||
if (entry.sessionId !== command.sessionId) {
|
||||
return { kind: "rejected", reason: "session_replaced" };
|
||||
}
|
||||
if (
|
||||
!state ||
|
||||
state.reservation ||
|
||||
!hasLifecycleFence(entry, {
|
||||
runId: command.runId,
|
||||
lifecycleGeneration: command.lifecycleGeneration,
|
||||
})
|
||||
) {
|
||||
return { kind: "rejected", reason: "stale_reservation" };
|
||||
}
|
||||
entry.status = "running";
|
||||
entry.abortedLastRun = true;
|
||||
entry.startedAt = undefined;
|
||||
entry.endedAt = undefined;
|
||||
entry.runtimeMs = undefined;
|
||||
if (entry.restartRecoveryDeliveryRunId === command.runId) {
|
||||
// Gateway accepted this RPC id before setup failed. Rotate it on retry
|
||||
// or the dedupe cache replays that terminal pre-dispatch failure.
|
||||
entry.restartRecoveryDeliveryRunId = undefined;
|
||||
}
|
||||
entry.updatedAt = command.now;
|
||||
return { kind: "applied" };
|
||||
}
|
||||
case "claim_foreground": {
|
||||
if (
|
||||
entry.sessionId !== command.sessionId ||
|
||||
entry.status !== "running" ||
|
||||
entry.abortedLastRun !== true ||
|
||||
!isMainRestartRecoveryCandidate(entry, command.sessionKey)
|
||||
) {
|
||||
return { kind: "no_change" };
|
||||
}
|
||||
const state = entry.mainRestartRecovery ?? createCycle(command.cycleId);
|
||||
if (state.tombstone) {
|
||||
return { kind: "rejected", reason: "already_tombstoned" };
|
||||
}
|
||||
if (state.chargedAttempts >= MAIN_RESTART_RECOVERY_MAX_AUTOMATIC_ATTEMPTS) {
|
||||
// The final charge fences foreground work until the scheduler commits
|
||||
// the matching tombstone. Admitting here can race that reconciliation.
|
||||
return { kind: "rejected", reason: "recovery_exhausted" };
|
||||
}
|
||||
const currentTokens =
|
||||
state.foregroundClaims?.lifecycleGeneration === command.lifecycleGeneration
|
||||
? state.foregroundClaims.tokens
|
||||
: [];
|
||||
const tokens = [...new Set([...currentTokens, command.claimId])].toSorted();
|
||||
const currentRunIds =
|
||||
state.foregroundClaims?.lifecycleGeneration === command.lifecycleGeneration
|
||||
? state.foregroundClaims.runIdsByClaimId
|
||||
: undefined;
|
||||
const runIdsByClaimId = command.runId
|
||||
? { ...currentRunIds, [command.claimId]: command.runId }
|
||||
: currentRunIds;
|
||||
if (command.runId) {
|
||||
recordLifecycleFence(entry, {
|
||||
lifecycleGeneration: command.lifecycleGeneration,
|
||||
runId: command.runId,
|
||||
});
|
||||
}
|
||||
entry.mainRestartRecovery = {
|
||||
...state,
|
||||
revision: nextRevision(state),
|
||||
reservation:
|
||||
state.reservation?.lifecycleGeneration === command.lifecycleGeneration
|
||||
? state.reservation
|
||||
: undefined,
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration: command.lifecycleGeneration,
|
||||
tokens,
|
||||
...(runIdsByClaimId ? { runIdsByClaimId } : {}),
|
||||
},
|
||||
};
|
||||
return {
|
||||
kind: "foreground_claimed",
|
||||
claim: {
|
||||
cycleId: state.cycleId,
|
||||
lifecycleGeneration: command.lifecycleGeneration,
|
||||
claimId: command.claimId,
|
||||
sessionId: entry.sessionId,
|
||||
sessionKey: command.sessionKey,
|
||||
...(command.runId ? { runId: command.runId } : {}),
|
||||
},
|
||||
};
|
||||
}
|
||||
case "bind_foreground_run": {
|
||||
const state = entry.mainRestartRecovery;
|
||||
const claims = state?.foregroundClaims;
|
||||
if (
|
||||
!state ||
|
||||
state.cycleId !== command.claim.cycleId ||
|
||||
claims?.lifecycleGeneration !== command.claim.lifecycleGeneration ||
|
||||
!claims.tokens.includes(command.claim.claimId)
|
||||
) {
|
||||
return { kind: "no_change" };
|
||||
}
|
||||
recordLifecycleFence(entry, {
|
||||
lifecycleGeneration: command.claim.lifecycleGeneration,
|
||||
runId: command.runId,
|
||||
});
|
||||
entry.mainRestartRecovery = {
|
||||
...state,
|
||||
revision: nextRevision(state),
|
||||
foregroundClaims: {
|
||||
...claims,
|
||||
runIdsByClaimId: { ...claims.runIdsByClaimId, [command.claim.claimId]: command.runId },
|
||||
},
|
||||
};
|
||||
return { kind: "applied" };
|
||||
}
|
||||
case "validate_foreground": {
|
||||
const state = entry.mainRestartRecovery;
|
||||
const claims = state?.foregroundClaims;
|
||||
return entry.sessionId === command.claim.sessionId &&
|
||||
state?.cycleId === command.claim.cycleId &&
|
||||
claims?.lifecycleGeneration === command.claim.lifecycleGeneration &&
|
||||
claims.tokens.includes(command.claim.claimId)
|
||||
? { kind: "foreground_validated" }
|
||||
: { kind: "no_change" };
|
||||
}
|
||||
case "release_foreground": {
|
||||
const state = entry.mainRestartRecovery;
|
||||
const claims = state?.foregroundClaims;
|
||||
if (
|
||||
!state ||
|
||||
state.cycleId !== command.claim.cycleId ||
|
||||
claims?.lifecycleGeneration !== command.claim.lifecycleGeneration ||
|
||||
!claims.tokens.includes(command.claim.claimId)
|
||||
) {
|
||||
return { kind: "no_change" };
|
||||
}
|
||||
const tokens = claims.tokens.filter((token) => token !== command.claim.claimId);
|
||||
const runIdsByClaimId = Object.fromEntries(
|
||||
Object.entries(claims.runIdsByClaimId ?? {}).filter(
|
||||
([token]) => token !== command.claim.claimId,
|
||||
),
|
||||
);
|
||||
if (tokens.length === 0 && entry.abortedLastRun !== true) {
|
||||
Object.assign(entry, buildMainSessionRecoveryClearPatch(entry));
|
||||
return { kind: "applied" };
|
||||
}
|
||||
entry.mainRestartRecovery = {
|
||||
...state,
|
||||
revision: nextRevision(state),
|
||||
foregroundClaims:
|
||||
tokens.length > 0
|
||||
? {
|
||||
lifecycleGeneration: command.claim.lifecycleGeneration,
|
||||
tokens,
|
||||
...(Object.keys(runIdsByClaimId).length > 0 ? { runIdsByClaimId } : {}),
|
||||
}
|
||||
: undefined,
|
||||
};
|
||||
return { kind: "applied" };
|
||||
}
|
||||
case "tombstone": {
|
||||
const conflict = matchesObservation(entry, command.observation);
|
||||
if (conflict) {
|
||||
return { kind: "rejected", reason: conflict };
|
||||
}
|
||||
const state = entry.mainRestartRecovery!;
|
||||
if (state.reservation) {
|
||||
return { kind: "rejected", reason: "reservation_active" };
|
||||
}
|
||||
if (state.tombstone) {
|
||||
return { kind: "rejected", reason: "already_tombstoned" };
|
||||
}
|
||||
entry.mainRestartRecovery = {
|
||||
...state,
|
||||
revision: nextRevision(state),
|
||||
tombstone: {
|
||||
reason: command.reason,
|
||||
},
|
||||
};
|
||||
entry.abortedLastRun = false;
|
||||
entry.status = "failed";
|
||||
entry.endedAt = command.now;
|
||||
entry.runtimeMs = Math.max(0, command.now - (entry.startedAt ?? command.now));
|
||||
entry.updatedAt = command.now;
|
||||
return { kind: "tombstoned" };
|
||||
}
|
||||
case "fail_recovery": {
|
||||
const conflict = matchesObservation(entry, command.observation);
|
||||
if (conflict) {
|
||||
return { kind: "rejected", reason: conflict };
|
||||
}
|
||||
const noticeEntry = structuredClone(entry);
|
||||
entry.status = "failed";
|
||||
entry.abortedLastRun = true;
|
||||
entry.endedAt = command.now;
|
||||
entry.updatedAt = command.now;
|
||||
Object.assign(entry, PENDING_FINAL_DELIVERY_CLEAR_PATCH);
|
||||
Object.assign(
|
||||
entry,
|
||||
buildRestartRecoveryClaimCleanupPatch({
|
||||
entry,
|
||||
recordTerminalSource: true,
|
||||
}),
|
||||
);
|
||||
entry.mainRestartRecovery = undefined;
|
||||
return { kind: "failed", noticeEntry };
|
||||
}
|
||||
case "doctor_repair": {
|
||||
if (!entry.mainRestartRecovery?.tombstone || entry.abortedLastRun !== true) {
|
||||
return { kind: "no_change" };
|
||||
}
|
||||
entry.abortedLastRun = false;
|
||||
entry.updatedAt = command.now;
|
||||
return { kind: "doctor_repaired" };
|
||||
}
|
||||
case "clear": {
|
||||
const patch = buildMainSessionRecoveryClearPatch(entry);
|
||||
if (Object.keys(patch).length === 0) {
|
||||
return { kind: "no_change" };
|
||||
}
|
||||
Object.assign(entry, patch);
|
||||
return { kind: "applied" };
|
||||
}
|
||||
default:
|
||||
return command satisfies never;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,768 @@
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import type { InternalSessionEntry as SessionEntry } from "../config/sessions.js";
|
||||
import * as sessionAccessor from "../config/sessions/session-accessor.js";
|
||||
import {
|
||||
applySessionEntryLifecycleMutation,
|
||||
listSessionEntries,
|
||||
} from "../config/sessions/session-accessor.js";
|
||||
import {
|
||||
getAgentEventLifecycleGeneration,
|
||||
rotateAgentEventLifecycleGeneration,
|
||||
} from "../infra/agent-events.js";
|
||||
import {
|
||||
claimMainSessionRecoveryOwner,
|
||||
commitMainSessionRecovery,
|
||||
inspectMainSessionRecoveryRequired,
|
||||
readMainSessionRecoveryOwner,
|
||||
releaseMainSessionRecoveryOwner,
|
||||
} from "./main-session-recovery-store.js";
|
||||
|
||||
const sessionKey = "agent:main:main";
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
describe("main session recovery store", () => {
|
||||
let dir: string;
|
||||
let lifecycleGeneration: string;
|
||||
let storePath: string;
|
||||
|
||||
beforeEach(() => {
|
||||
dir = tempDirs.make("openclaw-main-recovery-store-");
|
||||
lifecycleGeneration = getAgentEventLifecycleGeneration();
|
||||
storePath = path.join(dir, "sessions.json");
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
async function write(entry: SessionEntry): Promise<void> {
|
||||
await sessionAccessor.replaceSessionEntry({ sessionKey, storePath }, entry);
|
||||
}
|
||||
|
||||
function read(): SessionEntry {
|
||||
return sessionAccessor.loadSessionEntry({ sessionKey, storePath })!;
|
||||
}
|
||||
|
||||
function readStore(): Record<string, SessionEntry> {
|
||||
return Object.fromEntries(
|
||||
listSessionEntries({ storePath }).map(({ sessionKey: key, entry }) => [key, entry]),
|
||||
);
|
||||
}
|
||||
|
||||
async function seedExact(entries: Record<string, SessionEntry>): Promise<void> {
|
||||
await applySessionEntryLifecycleMutation({
|
||||
storePath,
|
||||
upserts: Object.entries(entries).map(([key, entry]) => ({ sessionKey: key, entry })),
|
||||
skipMaintenance: true,
|
||||
});
|
||||
}
|
||||
|
||||
function interruptedEntry(overrides: Partial<SessionEntry> = {}): SessionEntry {
|
||||
return {
|
||||
sessionId: "session-1",
|
||||
updatedAt: 100,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 1,
|
||||
chargedAttempts: 0,
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function reserve(targetSessionKey = sessionKey) {
|
||||
const result = await commitMainSessionRecovery({
|
||||
command: {
|
||||
kind: "prepare_attempt",
|
||||
attempt: 1,
|
||||
lifecycleGeneration,
|
||||
now: 200,
|
||||
observation: { sessionId: "session-1", cycleId: "cycle-1", revision: 1 },
|
||||
runId: "recovery-1",
|
||||
},
|
||||
target: { sessionKey: targetSessionKey, storePath },
|
||||
});
|
||||
if (result.transition.kind !== "reserved") {
|
||||
throw new Error("expected reservation");
|
||||
}
|
||||
return result.transition.reservation;
|
||||
}
|
||||
|
||||
it("persists a cycle before returning a legacy interrupted observation", async () => {
|
||||
await write({
|
||||
sessionId: "session-1",
|
||||
updatedAt: 100,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
});
|
||||
|
||||
const result = await commitMainSessionRecovery({
|
||||
command: {
|
||||
kind: "observe",
|
||||
cycleId: "cycle-1",
|
||||
lifecycleGeneration,
|
||||
sessionKey,
|
||||
},
|
||||
requireWriteSuccess: true,
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
|
||||
expect(result.transition).toMatchObject({
|
||||
kind: "observed",
|
||||
view: { status: "recoverable" },
|
||||
});
|
||||
expect(read().mainRestartRecovery).toMatchObject({
|
||||
cycleId: "cycle-1",
|
||||
revision: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves a concurrent foreground claim while cancelling its reservation", async () => {
|
||||
await write(interruptedEntry());
|
||||
const reservation = await reserve();
|
||||
await commitMainSessionRecovery({
|
||||
command: {
|
||||
kind: "claim_foreground",
|
||||
cycleId: "unused",
|
||||
lifecycleGeneration,
|
||||
sessionId: "session-1",
|
||||
sessionKey,
|
||||
claimId: "foreground-1",
|
||||
},
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
|
||||
await commitMainSessionRecovery({
|
||||
command: { kind: "cancel_reservation", reservation },
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
|
||||
expect(read().mainRestartRecovery).toMatchObject({
|
||||
chargedAttempts: 0,
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration,
|
||||
tokens: ["foreground-1"],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("cancels a reservation after Gateway migrates its legacy session key", async () => {
|
||||
const legacyKey = "main";
|
||||
await seedExact({ [legacyKey]: interruptedEntry() });
|
||||
const reservation = await reserve(legacyKey);
|
||||
const legacyEntry = readStore()[legacyKey]!;
|
||||
await applySessionEntryLifecycleMutation({
|
||||
storePath,
|
||||
removals: [{ sessionKey: legacyKey }],
|
||||
upserts: [{ sessionKey, entry: legacyEntry }],
|
||||
skipMaintenance: true,
|
||||
});
|
||||
|
||||
const cancelled = await commitMainSessionRecovery({
|
||||
command: { kind: "cancel_reservation", reservation },
|
||||
target: { sessionKey: legacyKey, storePath },
|
||||
});
|
||||
|
||||
expect(cancelled.transition).toEqual({ kind: "applied" });
|
||||
expect(read().mainRestartRecovery).toMatchObject({ chargedAttempts: 0 });
|
||||
expect(read().mainRestartRecovery?.reservation).toBeUndefined();
|
||||
});
|
||||
|
||||
it("abandons a reservation after Gateway migrates its legacy session key", async () => {
|
||||
const legacyKey = "main";
|
||||
await seedExact({ [legacyKey]: interruptedEntry() });
|
||||
const reservation = await reserve(legacyKey);
|
||||
const legacyEntry = readStore()[legacyKey]!;
|
||||
await applySessionEntryLifecycleMutation({
|
||||
storePath,
|
||||
removals: [{ sessionKey: legacyKey }],
|
||||
upserts: [{ sessionKey, entry: legacyEntry }],
|
||||
skipMaintenance: true,
|
||||
});
|
||||
|
||||
const abandoned = await commitMainSessionRecovery({
|
||||
command: { kind: "abandon_reservation", reservation },
|
||||
target: { sessionKey: legacyKey, storePath },
|
||||
});
|
||||
|
||||
expect(abandoned.transition).toEqual({ kind: "applied" });
|
||||
expect(read().mainRestartRecovery).toMatchObject({ chargedAttempts: 1 });
|
||||
expect(read().mainRestartRecovery?.reservation).toBeUndefined();
|
||||
});
|
||||
|
||||
it("admits a legacy-key reservation through its canonical Gateway key", async () => {
|
||||
const legacyKey = "main";
|
||||
await seedExact({ [legacyKey]: interruptedEntry() });
|
||||
await reserve(legacyKey);
|
||||
|
||||
const validated = await commitMainSessionRecovery({
|
||||
command: {
|
||||
kind: "validate_recovery",
|
||||
lifecycleGeneration,
|
||||
runId: "recovery-1",
|
||||
sessionId: "session-1",
|
||||
},
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
expect(validated.transition).toEqual({ kind: "recovery_validated" });
|
||||
|
||||
const admitted = await commitMainSessionRecovery({
|
||||
command: {
|
||||
kind: "admit_recovery",
|
||||
lifecycleGeneration,
|
||||
now: 300,
|
||||
runId: "recovery-1",
|
||||
sessionId: "session-1",
|
||||
},
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
expect(admitted.transition).toEqual({ kind: "admitted_recovery" });
|
||||
expect(readStore()[legacyKey]).toMatchObject({
|
||||
abortedLastRun: false,
|
||||
mainRestartRecovery: { chargedAttempts: 1 },
|
||||
});
|
||||
expect(readStore()[legacyKey]?.mainRestartRecovery?.reservation).toBeUndefined();
|
||||
|
||||
const restored = await commitMainSessionRecovery({
|
||||
command: {
|
||||
kind: "mark_admitted_recovery_interrupted",
|
||||
lifecycleGeneration,
|
||||
now: 400,
|
||||
runId: "recovery-1",
|
||||
sessionId: "session-1",
|
||||
},
|
||||
target: { sessionKey: admitted.sessionKey!, storePath },
|
||||
});
|
||||
expect(restored.transition).toEqual({ kind: "applied" });
|
||||
expect(readStore()[legacyKey]).toMatchObject({ abortedLastRun: true });
|
||||
});
|
||||
|
||||
it("rejects an observation after the session is replaced", async () => {
|
||||
await write({
|
||||
sessionId: "session-2",
|
||||
updatedAt: 300,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-2",
|
||||
revision: 1,
|
||||
chargedAttempts: 0,
|
||||
},
|
||||
});
|
||||
|
||||
const result = await commitMainSessionRecovery({
|
||||
command: {
|
||||
kind: "prepare_attempt",
|
||||
attempt: 1,
|
||||
lifecycleGeneration,
|
||||
now: 400,
|
||||
observation: { sessionId: "session-1", cycleId: "cycle-1", revision: 1 },
|
||||
runId: "stale-recovery",
|
||||
},
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
|
||||
expect(result.transition).toEqual({ kind: "rejected", reason: "session_replaced" });
|
||||
expect(read().mainRestartRecovery?.reservation).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not cancel a reservation after its session is replaced", async () => {
|
||||
await write(interruptedEntry());
|
||||
const reservation = await reserve();
|
||||
|
||||
await write({
|
||||
sessionId: "session-2",
|
||||
updatedAt: 300,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-2",
|
||||
revision: 4,
|
||||
chargedAttempts: 2,
|
||||
},
|
||||
});
|
||||
|
||||
const cancelled = await commitMainSessionRecovery({
|
||||
command: { kind: "cancel_reservation", reservation },
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
|
||||
expect(cancelled.transition).toEqual({ kind: "rejected", reason: "stale_reservation" });
|
||||
expect(read()).toMatchObject({
|
||||
sessionId: "session-2",
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-2",
|
||||
revision: 4,
|
||||
chargedAttempts: 2,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("does not let an old reservation survive healthy clear and immediate re-wedge", async () => {
|
||||
await write(interruptedEntry());
|
||||
const reservation = await reserve();
|
||||
await commitMainSessionRecovery({
|
||||
command: { kind: "clear" },
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
await commitMainSessionRecovery({
|
||||
command: { kind: "mark_interrupted", cycleId: "cycle-2", now: 300 },
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
|
||||
const cancelled = await commitMainSessionRecovery({
|
||||
command: { kind: "cancel_reservation", reservation },
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
|
||||
expect(cancelled.transition).toEqual({ kind: "rejected", reason: "stale_reservation" });
|
||||
expect(read().mainRestartRecovery).toMatchObject({
|
||||
cycleId: "cycle-2",
|
||||
chargedAttempts: 0,
|
||||
});
|
||||
});
|
||||
|
||||
it("claims an interrupted row through its pre-migration alias", async () => {
|
||||
const legacyKey = "main";
|
||||
await seedExact({ [legacyKey]: interruptedEntry() });
|
||||
|
||||
const claim = await claimMainSessionRecoveryOwner({
|
||||
lifecycleGeneration,
|
||||
sessionId: "session-1",
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
|
||||
expect(claim.kind).toBe("claimed");
|
||||
if (claim.kind !== "claimed") {
|
||||
return;
|
||||
}
|
||||
expect(claim.lease.sessionKey).toBe(legacyKey);
|
||||
expect(readStore()[legacyKey]).toMatchObject({
|
||||
mainRestartRecovery: {
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration,
|
||||
tokens: [claim.lease.claimId],
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("claims the exact foreground row without scanning aliases", async () => {
|
||||
await write(interruptedEntry());
|
||||
const accessorSpy = vi.spyOn(sessionAccessor, "applySessionEntryReplacements");
|
||||
|
||||
const claim = await claimMainSessionRecoveryOwner({
|
||||
lifecycleGeneration,
|
||||
sessionId: "session-1",
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
|
||||
expect(claim.kind).toBe("claimed");
|
||||
expect(accessorSpy).toHaveBeenCalledOnce();
|
||||
expect(accessorSpy.mock.calls[0]?.[0]).toMatchObject({ sessionKeys: [sessionKey] });
|
||||
});
|
||||
|
||||
it("binds a foreground claim to its lifecycle run", async () => {
|
||||
await write(interruptedEntry());
|
||||
|
||||
const claim = await claimMainSessionRecoveryOwner({
|
||||
lifecycleGeneration,
|
||||
runId: "foreground-run",
|
||||
sessionId: "session-1",
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
|
||||
if (claim.kind !== "claimed") {
|
||||
throw new Error("expected foreground owner claim");
|
||||
}
|
||||
expect(read()).toMatchObject({
|
||||
restartRecoveryRuns: [{ lifecycleGeneration, runId: "foreground-run" }],
|
||||
mainRestartRecovery: {
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration,
|
||||
runIdsByClaimId: { [claim.lease.claimId]: "foreground-run" },
|
||||
},
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("releases an owner after the durable row session id rotates", async () => {
|
||||
await write(interruptedEntry());
|
||||
const claim = await claimMainSessionRecoveryOwner({
|
||||
lifecycleGeneration,
|
||||
sessionId: "session-1",
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
if (claim.kind !== "claimed") {
|
||||
throw new Error("expected foreground owner claim");
|
||||
}
|
||||
const current = read();
|
||||
await write({ ...current, sessionId: "session-2" });
|
||||
|
||||
await expect(releaseMainSessionRecoveryOwner(claim.lease)).resolves.toBeUndefined();
|
||||
|
||||
expect(read().mainRestartRecovery?.foregroundClaims).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps retrying an exact owner release after immediate store retries fail", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
await write(interruptedEntry());
|
||||
const claim = await claimMainSessionRecoveryOwner({
|
||||
lifecycleGeneration,
|
||||
sessionId: "session-1",
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
if (claim.kind !== "claimed") {
|
||||
throw new Error("expected foreground owner claim");
|
||||
}
|
||||
const applySessionEntryReplacements = sessionAccessor.applySessionEntryReplacements;
|
||||
let failures = 0;
|
||||
vi.spyOn(sessionAccessor, "applySessionEntryReplacements").mockImplementation(
|
||||
async (params) => {
|
||||
if (failures < 3) {
|
||||
failures += 1;
|
||||
throw new Error("transient session-store failure");
|
||||
}
|
||||
return await applySessionEntryReplacements(params);
|
||||
},
|
||||
);
|
||||
|
||||
const immediateRelease = releaseMainSessionRecoveryOwner(claim.lease);
|
||||
const immediateReleaseRejected = expect(immediateRelease).rejects.toThrow(
|
||||
"transient session-store failure",
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(100);
|
||||
await immediateReleaseRejected;
|
||||
expect(read().mainRestartRecovery?.foregroundClaims?.tokens).toEqual([claim.lease.claimId]);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
await vi.waitFor(() => {
|
||||
expect(read().mainRestartRecovery?.foregroundClaims).toBeUndefined();
|
||||
});
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("leaves interrupted non-main rows to their specialized recovery owner", async () => {
|
||||
const subagentKey = "agent:main:subagent:child";
|
||||
await seedExact({ [subagentKey]: interruptedEntry({ spawnDepth: 1 }) });
|
||||
|
||||
const claim = await claimMainSessionRecoveryOwner({
|
||||
lifecycleGeneration,
|
||||
sessionId: "session-1",
|
||||
target: { sessionKey: subagentKey, storePath },
|
||||
});
|
||||
|
||||
expect(claim).toEqual({ kind: "not_required" });
|
||||
expect(readStore()[subagentKey]?.mainRestartRecovery?.foregroundClaims).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not let a replacement bypass a tombstoned predecessor", async () => {
|
||||
await write(
|
||||
interruptedEntry({
|
||||
status: "failed",
|
||||
abortedLastRun: false,
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 4,
|
||||
chargedAttempts: 3,
|
||||
tombstone: { reason: "automatic recovery exhausted" },
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
claimMainSessionRecoveryOwner({
|
||||
lifecycleGeneration,
|
||||
replacementSessionId: "session-2",
|
||||
sessionId: "session-1",
|
||||
target: { sessionKey, storePath },
|
||||
}),
|
||||
).resolves.toEqual({ kind: "invalidated", reason: "state_changed" });
|
||||
});
|
||||
|
||||
it("does not let foreground work bypass an exhausted predecessor", async () => {
|
||||
await write(
|
||||
interruptedEntry({
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 4,
|
||||
chargedAttempts: 3,
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(
|
||||
claimMainSessionRecoveryOwner({
|
||||
lifecycleGeneration,
|
||||
sessionId: "session-1",
|
||||
target: { sessionKey, storePath },
|
||||
}),
|
||||
).resolves.toEqual({ kind: "invalidated", reason: "recovery_exhausted" });
|
||||
expect(read().mainRestartRecovery?.foregroundClaims).toBeUndefined();
|
||||
});
|
||||
|
||||
it("claims an interrupted legacy predecessor after its canonical key is reused", async () => {
|
||||
const legacyKey = "main";
|
||||
await seedExact({
|
||||
[sessionKey]: { sessionId: "session-2", updatedAt: 200 },
|
||||
[legacyKey]: interruptedEntry(),
|
||||
});
|
||||
|
||||
const claim = await claimMainSessionRecoveryOwner({
|
||||
lifecycleGeneration,
|
||||
replacementSessionId: "session-2",
|
||||
sessionId: "session-1",
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
|
||||
expect(claim.kind).toBe("claimed");
|
||||
if (claim.kind !== "claimed") {
|
||||
throw new Error("expected foreground owner claim");
|
||||
}
|
||||
expect(claim.lease.sessionKey).toBe(legacyKey);
|
||||
expect(readStore()[legacyKey]?.mainRestartRecovery?.foregroundClaims?.tokens).toEqual([
|
||||
claim.lease.claimId,
|
||||
]);
|
||||
expect(readStore()[sessionKey]?.mainRestartRecovery).toBeUndefined();
|
||||
});
|
||||
|
||||
it("validates a transferred owner against the latest durable row", async () => {
|
||||
await write(interruptedEntry());
|
||||
const claim = await claimMainSessionRecoveryOwner({
|
||||
lifecycleGeneration,
|
||||
sessionId: "session-1",
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
if (claim.kind !== "claimed") {
|
||||
throw new Error("expected foreground owner claim");
|
||||
}
|
||||
|
||||
await expect(readMainSessionRecoveryOwner(claim.lease)).resolves.toBeDefined();
|
||||
await releaseMainSessionRecoveryOwner(claim.lease);
|
||||
await expect(readMainSessionRecoveryOwner(claim.lease)).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("returns a retry target only when the final foreground owner releases", async () => {
|
||||
await write(interruptedEntry());
|
||||
const first = await claimMainSessionRecoveryOwner({
|
||||
lifecycleGeneration,
|
||||
sessionId: "session-1",
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
const second = await claimMainSessionRecoveryOwner({
|
||||
lifecycleGeneration,
|
||||
sessionId: "session-1",
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
if (first.kind !== "claimed" || second.kind !== "claimed") {
|
||||
throw new Error("expected foreground owner claims");
|
||||
}
|
||||
|
||||
await expect(releaseMainSessionRecoveryOwner(first.lease)).resolves.toBeUndefined();
|
||||
await expect(releaseMainSessionRecoveryOwner(second.lease)).resolves.toEqual({
|
||||
sessionId: "session-1",
|
||||
sessionKey,
|
||||
storePath,
|
||||
});
|
||||
await expect(releaseMainSessionRecoveryOwner(second.lease)).resolves.toEqual({
|
||||
sessionId: "session-1",
|
||||
sessionKey,
|
||||
storePath,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not let an old lease release a same-token claim from a new cycle", async () => {
|
||||
await write(interruptedEntry());
|
||||
const oldClaim = await claimMainSessionRecoveryOwner({
|
||||
lifecycleGeneration,
|
||||
sessionId: "session-1",
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
if (oldClaim.kind !== "claimed") {
|
||||
throw new Error("expected foreground owner claim");
|
||||
}
|
||||
await commitMainSessionRecovery({
|
||||
command: { kind: "clear" },
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
await commitMainSessionRecovery({
|
||||
command: { kind: "mark_interrupted", cycleId: "cycle-2", now: 300 },
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
await commitMainSessionRecovery({
|
||||
command: {
|
||||
kind: "claim_foreground",
|
||||
cycleId: "unused",
|
||||
lifecycleGeneration,
|
||||
sessionId: "session-1",
|
||||
sessionKey,
|
||||
claimId: oldClaim.lease.claimId,
|
||||
},
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
|
||||
await releaseMainSessionRecoveryOwner(oldClaim.lease);
|
||||
|
||||
expect(read().mainRestartRecovery).toMatchObject({
|
||||
cycleId: "cycle-2",
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration,
|
||||
tokens: [oldClaim.lease.claimId],
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("retries a transient owner release write failure", async () => {
|
||||
await write(interruptedEntry());
|
||||
const claim = await claimMainSessionRecoveryOwner({
|
||||
lifecycleGeneration,
|
||||
sessionId: "session-1",
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
if (claim.kind !== "claimed") {
|
||||
throw new Error("expected foreground owner claim");
|
||||
}
|
||||
const applySessionEntryReplacements = sessionAccessor.applySessionEntryReplacements;
|
||||
const accessorSpy = vi
|
||||
.spyOn(sessionAccessor, "applySessionEntryReplacements")
|
||||
.mockRejectedValueOnce(new Error("transient writer failure"))
|
||||
.mockImplementation(async (params) => await applySessionEntryReplacements(params));
|
||||
|
||||
await releaseMainSessionRecoveryOwner(claim.lease);
|
||||
|
||||
expect(accessorSpy).toHaveBeenCalledTimes(2);
|
||||
expect(read().mainRestartRecovery?.foregroundClaims).toBeUndefined();
|
||||
});
|
||||
|
||||
it("rejects an old claimant queued ahead of the current lifecycle generation", async () => {
|
||||
await write(interruptedEntry());
|
||||
|
||||
let enterWriter = () => {};
|
||||
const writerEntered = new Promise<void>((resolve) => {
|
||||
enterWriter = resolve;
|
||||
});
|
||||
let releaseWriter = () => {};
|
||||
const writerGate = new Promise<void>((resolve) => {
|
||||
releaseWriter = resolve;
|
||||
});
|
||||
const blocker = sessionAccessor.applySessionEntryReplacements({
|
||||
storePath,
|
||||
update: async () => {
|
||||
enterWriter();
|
||||
await writerGate;
|
||||
return { result: undefined };
|
||||
},
|
||||
});
|
||||
await writerEntered;
|
||||
|
||||
const staleClaim = commitMainSessionRecovery({
|
||||
command: {
|
||||
kind: "claim_foreground",
|
||||
cycleId: "unused",
|
||||
lifecycleGeneration,
|
||||
sessionId: "session-1",
|
||||
sessionKey,
|
||||
claimId: "stale-owner",
|
||||
},
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
const staleOwnerClaim = claimMainSessionRecoveryOwner({
|
||||
allowMissingSession: true,
|
||||
lifecycleGeneration,
|
||||
replacementSessionId: "session-2",
|
||||
sessionId: "session-1",
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
const staleInspection = inspectMainSessionRecoveryRequired({
|
||||
expectedSessionId: "session-1",
|
||||
lifecycleGeneration,
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
await Promise.resolve();
|
||||
const currentGeneration = rotateAgentEventLifecycleGeneration();
|
||||
const currentClaim = commitMainSessionRecovery({
|
||||
command: {
|
||||
kind: "claim_foreground",
|
||||
cycleId: "unused",
|
||||
lifecycleGeneration: currentGeneration,
|
||||
sessionId: "session-1",
|
||||
sessionKey,
|
||||
claimId: "current-owner",
|
||||
},
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
releaseWriter();
|
||||
|
||||
await blocker;
|
||||
expect((await staleClaim).transition).toEqual({
|
||||
kind: "rejected",
|
||||
reason: "stale_generation",
|
||||
});
|
||||
await expect(staleOwnerClaim).resolves.toEqual({
|
||||
kind: "invalidated",
|
||||
reason: "stale_generation",
|
||||
});
|
||||
await expect(staleInspection).resolves.toEqual({
|
||||
kind: "invalidated",
|
||||
reason: "stale_generation",
|
||||
});
|
||||
expect((await currentClaim).transition).toMatchObject({
|
||||
kind: "foreground_claimed",
|
||||
claim: { claimId: "current-owner" },
|
||||
});
|
||||
expect(read().mainRestartRecovery?.foregroundClaims).toEqual({
|
||||
lifecycleGeneration: currentGeneration,
|
||||
tokens: ["current-owner"],
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects a delayed admitted-interruption callback after lifecycle rotation", async () => {
|
||||
await write(
|
||||
interruptedEntry({
|
||||
abortedLastRun: false,
|
||||
restartRecoveryRuns: [{ runId: "recovery-1", lifecycleGeneration }],
|
||||
}),
|
||||
);
|
||||
rotateAgentEventLifecycleGeneration();
|
||||
|
||||
const result = await commitMainSessionRecovery({
|
||||
command: {
|
||||
kind: "mark_admitted_recovery_interrupted",
|
||||
lifecycleGeneration,
|
||||
now: 300,
|
||||
runId: "recovery-1",
|
||||
sessionId: "session-1",
|
||||
},
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
|
||||
expect(result.transition).toEqual({ kind: "rejected", reason: "stale_generation" });
|
||||
expect(read()).toMatchObject({
|
||||
sessionId: "session-1",
|
||||
status: "running",
|
||||
abortedLastRun: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects a transferred foreground lease after lifecycle rotation", async () => {
|
||||
await write(interruptedEntry());
|
||||
const claim = await claimMainSessionRecoveryOwner({
|
||||
lifecycleGeneration,
|
||||
sessionId: "session-1",
|
||||
target: { sessionKey, storePath },
|
||||
});
|
||||
if (claim.kind !== "claimed") {
|
||||
throw new Error("expected foreground owner claim");
|
||||
}
|
||||
rotateAgentEventLifecycleGeneration();
|
||||
|
||||
await expect(readMainSessionRecoveryOwner(claim.lease)).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,436 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import type { InternalSessionEntry as SessionEntry } from "../config/sessions.js";
|
||||
import { applySessionEntryReplacements } from "../config/sessions/session-accessor.js";
|
||||
import { getAgentEventLifecycleGeneration } from "../infra/agent-events.js";
|
||||
import { retryAsync } from "../infra/retry.js";
|
||||
import {
|
||||
isMainRestartRecoveryCandidate,
|
||||
transitionMainSessionRecovery,
|
||||
type MainSessionRecoveryCommand,
|
||||
type MainSessionRecoveryOwnerClaim,
|
||||
type MainSessionRecoveryReservation,
|
||||
type MainSessionRecoveryTransitionResult,
|
||||
} from "./main-session-recovery-state.js";
|
||||
|
||||
type MainSessionRecoveryStoreTarget = {
|
||||
sessionKey: string;
|
||||
storePath: string;
|
||||
};
|
||||
|
||||
const OWNER_RELEASE_RETRY_DELAY_MS = 1_000;
|
||||
const OWNER_RELEASE_RETRY_MAX_DELAY_MS = 30_000;
|
||||
|
||||
export type MainSessionRecoveryOwnerLease = MainSessionRecoveryOwnerClaim &
|
||||
MainSessionRecoveryStoreTarget;
|
||||
|
||||
type MainSessionRecoveryStoreResult = {
|
||||
entry?: SessionEntry;
|
||||
sessionKey?: string;
|
||||
transition: MainSessionRecoveryTransitionResult;
|
||||
};
|
||||
|
||||
export type MainSessionRecoveryPendingTarget = MainSessionRecoveryStoreTarget & {
|
||||
sessionId: string;
|
||||
};
|
||||
|
||||
type MainSessionRecoveryOwnerClaimResult =
|
||||
| {
|
||||
kind: "claimed";
|
||||
lease: MainSessionRecoveryOwnerLease;
|
||||
entry: SessionEntry;
|
||||
sessionKey: string;
|
||||
}
|
||||
| { kind: "invalidated"; reason: string }
|
||||
| { kind: "not_required" };
|
||||
|
||||
type MainSessionRecoveryInspectionResult =
|
||||
| { kind: "invalidated"; reason: string }
|
||||
| { kind: "not_required" }
|
||||
| { kind: "required" };
|
||||
|
||||
function transitionChanged(result: MainSessionRecoveryTransitionResult): boolean {
|
||||
return (
|
||||
result.kind !== "foreground_validated" &&
|
||||
result.kind !== "no_change" &&
|
||||
result.kind !== "observed" &&
|
||||
result.kind !== "rejected"
|
||||
);
|
||||
}
|
||||
|
||||
function matchesReservation(entry: SessionEntry, reservation: MainSessionRecoveryReservation) {
|
||||
const state = entry.mainRestartRecovery;
|
||||
return (
|
||||
entry.sessionId === reservation.sessionId &&
|
||||
state?.cycleId === reservation.cycleId &&
|
||||
state.reservation?.runId === reservation.runId &&
|
||||
state.reservation.lifecycleGeneration === reservation.lifecycleGeneration
|
||||
);
|
||||
}
|
||||
|
||||
function matchesRecoveryAdmission(
|
||||
entry: SessionEntry,
|
||||
command: Extract<MainSessionRecoveryCommand, { kind: "admit_recovery" | "validate_recovery" }>,
|
||||
): boolean {
|
||||
const reservation = entry.mainRestartRecovery?.reservation;
|
||||
return (
|
||||
entry.sessionId === command.sessionId &&
|
||||
reservation?.runId === command.runId &&
|
||||
reservation.lifecycleGeneration === command.lifecycleGeneration
|
||||
);
|
||||
}
|
||||
|
||||
function matchesOwnerClaim(entry: SessionEntry, claim: MainSessionRecoveryOwnerClaim): boolean {
|
||||
const state = entry.mainRestartRecovery;
|
||||
return (
|
||||
state?.cycleId === claim.cycleId &&
|
||||
state.foregroundClaims?.lifecycleGeneration === claim.lifecycleGeneration &&
|
||||
state.foregroundClaims.tokens.includes(claim.claimId)
|
||||
);
|
||||
}
|
||||
|
||||
function currentGenerationRequiredBy(command: MainSessionRecoveryCommand): string | undefined {
|
||||
// Generation gates new decisions. Exact reservation/token cleanup must remain
|
||||
// valid after a restart so the old owner cannot leak its slot or claim.
|
||||
switch (command.kind) {
|
||||
case "admit_recovery":
|
||||
case "claim_foreground":
|
||||
case "inspect":
|
||||
case "mark_admitted_recovery_interrupted":
|
||||
case "observe":
|
||||
case "prepare_attempt":
|
||||
case "validate_recovery":
|
||||
return command.lifecycleGeneration;
|
||||
case "validate_foreground":
|
||||
case "bind_foreground_run":
|
||||
return command.claim.lifecycleGeneration;
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export async function commitMainSessionRecovery(params: {
|
||||
command: MainSessionRecoveryCommand;
|
||||
expectedSessionId?: string;
|
||||
requireWriteSuccess?: boolean;
|
||||
scanAliases?: boolean;
|
||||
target: MainSessionRecoveryStoreTarget;
|
||||
}): Promise<MainSessionRecoveryStoreResult> {
|
||||
const cancellation =
|
||||
params.command.kind === "cancel_reservation" ? params.command.reservation : undefined;
|
||||
const abandonment =
|
||||
params.command.kind === "abandon_reservation" ? params.command.reservation : undefined;
|
||||
const recoveryAdmission =
|
||||
params.command.kind === "admit_recovery" || params.command.kind === "validate_recovery"
|
||||
? params.command
|
||||
: undefined;
|
||||
const ownerClaim = params.command.kind === "claim_foreground" ? params.command : undefined;
|
||||
const ownerValidation =
|
||||
params.command.kind === "validate_foreground" ? params.command.claim : undefined;
|
||||
const ownerRelease =
|
||||
params.command.kind === "release_foreground" ? params.command.claim : undefined;
|
||||
const reservationCleanup = cancellation ?? abandonment;
|
||||
const scansAliases = Boolean(
|
||||
params.scanAliases ||
|
||||
reservationCleanup ||
|
||||
recoveryAdmission ||
|
||||
ownerValidation ||
|
||||
ownerRelease,
|
||||
);
|
||||
return await applySessionEntryReplacements<MainSessionRecoveryStoreResult>({
|
||||
requireWriteSuccess: params.requireWriteSuccess,
|
||||
...(scansAliases ? {} : { sessionKeys: [params.target.sessionKey] }),
|
||||
storePath: params.target.storePath,
|
||||
update: (entries) => {
|
||||
const expectedGeneration = currentGenerationRequiredBy(params.command);
|
||||
if (expectedGeneration && expectedGeneration !== getAgentEventLifecycleGeneration()) {
|
||||
return {
|
||||
result: {
|
||||
transition: { kind: "rejected", reason: "stale_generation" },
|
||||
},
|
||||
};
|
||||
}
|
||||
const selected = entries.find(({ sessionKey }) => sessionKey === params.target.sessionKey);
|
||||
let candidate =
|
||||
(params.expectedSessionId && selected?.entry.sessionId !== params.expectedSessionId) ||
|
||||
(ownerClaim && selected?.entry.sessionId !== ownerClaim.sessionId)
|
||||
? undefined
|
||||
: selected;
|
||||
if (reservationCleanup) {
|
||||
candidate =
|
||||
entries.find(({ entry }) => matchesReservation(entry, reservationCleanup)) ?? selected;
|
||||
} else if (recoveryAdmission) {
|
||||
// Canonical session-key migration may happen between reservation and
|
||||
// Gateway admission; the reservation identity remains authoritative.
|
||||
candidate =
|
||||
entries.find(({ entry }) => matchesRecoveryAdmission(entry, recoveryAdmission)) ??
|
||||
selected;
|
||||
} else if (ownerValidation || ownerRelease) {
|
||||
const exactClaim = ownerValidation ?? ownerRelease!;
|
||||
candidate = entries.find(({ entry }) => matchesOwnerClaim(entry, exactClaim)) ?? selected;
|
||||
} else if (ownerClaim && (!selected || selected.entry.sessionId !== ownerClaim.sessionId)) {
|
||||
candidate = entries.find(({ entry }) => entry.sessionId === ownerClaim.sessionId);
|
||||
} else if (params.scanAliases && params.expectedSessionId) {
|
||||
candidate = entries.find(({ entry }) => entry.sessionId === params.expectedSessionId);
|
||||
}
|
||||
if (!candidate) {
|
||||
return {
|
||||
result: {
|
||||
entry: selected?.entry,
|
||||
sessionKey: selected?.sessionKey,
|
||||
transition: { kind: "rejected", reason: "session_replaced" },
|
||||
},
|
||||
};
|
||||
}
|
||||
const entry = candidate.entry as SessionEntry;
|
||||
const previousRecoveryState = entry.mainRestartRecovery;
|
||||
let command: MainSessionRecoveryCommand;
|
||||
if (ownerClaim) {
|
||||
command =
|
||||
ownerClaim.sessionKey === candidate.sessionKey
|
||||
? ownerClaim
|
||||
: { ...ownerClaim, sessionKey: candidate.sessionKey };
|
||||
} else if (
|
||||
(params.command.kind === "observe" || params.command.kind === "inspect") &&
|
||||
params.command.sessionKey !== candidate.sessionKey
|
||||
) {
|
||||
command = { ...params.command, sessionKey: candidate.sessionKey };
|
||||
} else {
|
||||
command = params.command;
|
||||
}
|
||||
const transition = transitionMainSessionRecovery(entry, command);
|
||||
const changed =
|
||||
transitionChanged(transition) || previousRecoveryState !== entry.mainRestartRecovery;
|
||||
return {
|
||||
result: { entry, sessionKey: candidate.sessionKey, transition },
|
||||
...(changed ? { replacements: [{ sessionKey: candidate.sessionKey, entry }] } : {}),
|
||||
};
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export async function readMainSessionRecoveryOwner(
|
||||
lease: MainSessionRecoveryOwnerLease,
|
||||
): Promise<{ entry: SessionEntry; sessionKey: string } | undefined> {
|
||||
const result = await commitMainSessionRecovery({
|
||||
command: { kind: "validate_foreground", claim: lease },
|
||||
requireWriteSuccess: true,
|
||||
target: lease,
|
||||
});
|
||||
return result.transition.kind === "foreground_validated" && result.entry && result.sessionKey
|
||||
? { entry: result.entry, sessionKey: result.sessionKey }
|
||||
: undefined;
|
||||
}
|
||||
|
||||
export async function claimMainSessionRecoveryOwner(params: {
|
||||
allowMissingSession?: boolean;
|
||||
lifecycleGeneration: string;
|
||||
replacementSessionId?: string;
|
||||
sessionId: string;
|
||||
runId?: string;
|
||||
target: MainSessionRecoveryStoreTarget;
|
||||
}): Promise<MainSessionRecoveryOwnerClaimResult> {
|
||||
const command = {
|
||||
kind: "claim_foreground" as const,
|
||||
cycleId: randomUUID(),
|
||||
lifecycleGeneration: params.lifecycleGeneration,
|
||||
sessionId: params.sessionId,
|
||||
sessionKey: params.target.sessionKey,
|
||||
claimId: randomUUID(),
|
||||
...(params.runId ? { runId: params.runId } : {}),
|
||||
};
|
||||
let claim = await commitMainSessionRecovery({
|
||||
command,
|
||||
requireWriteSuccess: true,
|
||||
target: params.target,
|
||||
});
|
||||
if (claim.transition.kind === "rejected" && claim.transition.reason === "session_replaced") {
|
||||
claim = await commitMainSessionRecovery({
|
||||
command,
|
||||
requireWriteSuccess: true,
|
||||
scanAliases: true,
|
||||
target: params.target,
|
||||
});
|
||||
}
|
||||
if (claim.transition.kind === "foreground_claimed") {
|
||||
if (!claim.entry || !claim.sessionKey) {
|
||||
return { kind: "invalidated", reason: "state_changed" };
|
||||
}
|
||||
return {
|
||||
kind: "claimed",
|
||||
lease: { ...claim.transition.claim, storePath: params.target.storePath },
|
||||
entry: claim.entry,
|
||||
sessionKey: claim.sessionKey,
|
||||
};
|
||||
}
|
||||
if (claim.transition.kind === "rejected" && claim.transition.reason === "stale_generation") {
|
||||
return { kind: "invalidated", reason: claim.transition.reason };
|
||||
}
|
||||
if (!claim.entry && (params.allowMissingSession || params.replacementSessionId)) {
|
||||
// A fresh explicit session has no predecessor. An automatic rollover can
|
||||
// also lose its predecessor before admission. Either way, no row remains to fence.
|
||||
return { kind: "not_required" };
|
||||
}
|
||||
if (
|
||||
params.replacementSessionId &&
|
||||
claim.entry?.sessionId === params.replacementSessionId &&
|
||||
claim.entry.abortedLastRun !== true &&
|
||||
claim.entry.restartRecoveryRuns === undefined &&
|
||||
claim.entry.mainRestartRecovery === undefined
|
||||
) {
|
||||
return { kind: "not_required" };
|
||||
}
|
||||
if (
|
||||
claim.entry?.sessionId === params.sessionId &&
|
||||
claim.sessionKey &&
|
||||
!isMainRestartRecoveryCandidate(claim.entry, claim.sessionKey)
|
||||
) {
|
||||
return { kind: "not_required" };
|
||||
}
|
||||
if (
|
||||
claim.entry?.sessionId === params.sessionId &&
|
||||
claim.entry.abortedLastRun !== true &&
|
||||
claim.entry.restartRecoveryRuns === undefined &&
|
||||
claim.entry.mainRestartRecovery === undefined
|
||||
) {
|
||||
// A healthy completion may clear recovery between the caller's read and this
|
||||
// transaction. Only that fully clean same-session state can proceed unclaimed.
|
||||
return { kind: "not_required" };
|
||||
}
|
||||
const reason = claim.transition.kind === "rejected" ? claim.transition.reason : "state_changed";
|
||||
return { kind: "invalidated", reason };
|
||||
}
|
||||
|
||||
export async function bindMainSessionRecoveryOwnerRun(
|
||||
lease: MainSessionRecoveryOwnerLease,
|
||||
runId: string,
|
||||
): Promise<{
|
||||
lease: MainSessionRecoveryOwnerLease;
|
||||
entry: SessionEntry;
|
||||
sessionKey: string;
|
||||
}> {
|
||||
const result = await commitMainSessionRecovery({
|
||||
command: { kind: "bind_foreground_run", claim: lease, runId },
|
||||
requireWriteSuccess: true,
|
||||
target: lease,
|
||||
});
|
||||
if (result.transition.kind !== "applied" || !result.entry || !result.sessionKey) {
|
||||
throw new Error("main-session recovery owner changed before run binding");
|
||||
}
|
||||
return { lease: { ...lease, runId }, entry: result.entry, sessionKey: result.sessionKey };
|
||||
}
|
||||
|
||||
export async function inspectMainSessionRecoveryRequired(params: {
|
||||
allowMissingSession?: boolean;
|
||||
expectedSessionId: string;
|
||||
lifecycleGeneration: string;
|
||||
target: MainSessionRecoveryStoreTarget;
|
||||
}): Promise<MainSessionRecoveryInspectionResult> {
|
||||
const command = {
|
||||
kind: "inspect" as const,
|
||||
lifecycleGeneration: params.lifecycleGeneration,
|
||||
sessionKey: params.target.sessionKey,
|
||||
};
|
||||
let result = await commitMainSessionRecovery({
|
||||
command,
|
||||
expectedSessionId: params.expectedSessionId,
|
||||
requireWriteSuccess: true,
|
||||
target: params.target,
|
||||
});
|
||||
if (result.transition.kind === "rejected" && result.transition.reason === "session_replaced") {
|
||||
result = await commitMainSessionRecovery({
|
||||
command,
|
||||
expectedSessionId: params.expectedSessionId,
|
||||
requireWriteSuccess: true,
|
||||
scanAliases: true,
|
||||
target: params.target,
|
||||
});
|
||||
}
|
||||
if (result.transition.kind === "observed") {
|
||||
return result.transition.view.status === "inactive"
|
||||
? { kind: "not_required" }
|
||||
: { kind: "required" };
|
||||
}
|
||||
if (result.transition.kind === "rejected" && result.transition.reason === "session_replaced") {
|
||||
return !result.entry && params.allowMissingSession
|
||||
? { kind: "not_required" }
|
||||
: { kind: "invalidated", reason: result.transition.reason };
|
||||
}
|
||||
return {
|
||||
kind: "invalidated",
|
||||
reason: result.transition.kind === "rejected" ? result.transition.reason : "state_changed",
|
||||
};
|
||||
}
|
||||
|
||||
async function releaseMainSessionRecoveryOwnerWithRetries(
|
||||
lease: MainSessionRecoveryOwnerLease,
|
||||
): Promise<MainSessionRecoveryPendingTarget | undefined> {
|
||||
// A leaked current-generation token blocks automatic recovery until restart.
|
||||
// Token-scoped release is idempotent, so transient writer failures are safe to retry.
|
||||
const released = await retryAsync(
|
||||
async () =>
|
||||
await commitMainSessionRecovery({
|
||||
command: { kind: "release_foreground", claim: lease },
|
||||
requireWriteSuccess: true,
|
||||
target: lease,
|
||||
}),
|
||||
3,
|
||||
25,
|
||||
);
|
||||
const { entry, sessionKey } = released;
|
||||
const state = entry?.mainRestartRecovery;
|
||||
if (
|
||||
(released.transition.kind !== "applied" && released.transition.kind !== "no_change") ||
|
||||
!entry ||
|
||||
!sessionKey ||
|
||||
entry.sessionId !== lease.sessionId ||
|
||||
entry.status !== "running" ||
|
||||
entry.abortedLastRun !== true ||
|
||||
!isMainRestartRecoveryCandidate(entry, sessionKey) ||
|
||||
state?.foregroundClaims ||
|
||||
state?.reservation ||
|
||||
state?.tombstone
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
return { sessionId: entry.sessionId, sessionKey, storePath: lease.storePath };
|
||||
}
|
||||
|
||||
function scheduleMainSessionRecoveryOwnerRelease(
|
||||
lease: MainSessionRecoveryOwnerLease,
|
||||
delayMs = OWNER_RELEASE_RETRY_DELAY_MS,
|
||||
): void {
|
||||
// A token is process-owned but durably blocks recovery. Keep exact-token
|
||||
// cleanup alive through transient writer outages until release or restart.
|
||||
setTimeout(() => {
|
||||
void releaseMainSessionRecoveryOwnerWithRetries(lease).then(
|
||||
async (pending) => {
|
||||
if (!pending) {
|
||||
return;
|
||||
}
|
||||
const { scheduleMainSessionRecoveryPendingTarget } =
|
||||
await import("./main-session-recovery-owner-release.js");
|
||||
scheduleMainSessionRecoveryPendingTarget(pending);
|
||||
},
|
||||
() => {
|
||||
scheduleMainSessionRecoveryOwnerRelease(
|
||||
lease,
|
||||
Math.min(delayMs * 2, OWNER_RELEASE_RETRY_MAX_DELAY_MS),
|
||||
);
|
||||
},
|
||||
);
|
||||
}, delayMs).unref?.();
|
||||
}
|
||||
|
||||
export async function releaseMainSessionRecoveryOwner(
|
||||
lease: MainSessionRecoveryOwnerLease | undefined,
|
||||
): Promise<MainSessionRecoveryPendingTarget | undefined> {
|
||||
if (!lease) {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
return await releaseMainSessionRecoveryOwnerWithRetries(lease);
|
||||
} catch (error) {
|
||||
scheduleMainSessionRecoveryOwnerRelease(lease);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
import type {
|
||||
InternalSessionEntry as SessionEntry,
|
||||
RestartRecoveryRun,
|
||||
} from "../config/sessions.js";
|
||||
|
||||
export type MainSessionRecoveryObservation = {
|
||||
sessionId: string;
|
||||
cycleId: string;
|
||||
revision: number;
|
||||
};
|
||||
|
||||
export type MainSessionRecoveryReservation = {
|
||||
sessionId: string;
|
||||
cycleId: string;
|
||||
lifecycleGeneration: string;
|
||||
runId: string;
|
||||
attempt: number;
|
||||
};
|
||||
|
||||
export type MainSessionRecoveryOwnerClaim = {
|
||||
cycleId: string;
|
||||
lifecycleGeneration: string;
|
||||
claimId: string;
|
||||
sessionId: string;
|
||||
sessionKey: string;
|
||||
runId?: string;
|
||||
};
|
||||
|
||||
export type MainSessionRecoveryView =
|
||||
| { status: "inactive" }
|
||||
| { status: "blocked" }
|
||||
| {
|
||||
status: "recoverable";
|
||||
observation: MainSessionRecoveryObservation;
|
||||
nextAttempt: number;
|
||||
}
|
||||
| {
|
||||
status: "exhausted";
|
||||
observation: MainSessionRecoveryObservation;
|
||||
reason: string;
|
||||
}
|
||||
| { status: "tombstoned" };
|
||||
|
||||
export type MainSessionRecoveryConflict =
|
||||
| "already_tombstoned"
|
||||
| "foreground_active"
|
||||
| "not_interrupted"
|
||||
| "recovery_exhausted"
|
||||
| "reservation_active"
|
||||
| "session_replaced"
|
||||
| "stale_cycle"
|
||||
| "stale_generation"
|
||||
| "stale_reservation"
|
||||
| "stale_revision";
|
||||
|
||||
export type MainSessionRecoveryCommand =
|
||||
| {
|
||||
kind: "mark_interrupted";
|
||||
cycleId: string;
|
||||
now: number;
|
||||
runs?: RestartRecoveryRun[];
|
||||
resetRuntime?: boolean;
|
||||
}
|
||||
| {
|
||||
kind: "observe";
|
||||
cycleId: string;
|
||||
lifecycleGeneration: string;
|
||||
sessionKey: string;
|
||||
}
|
||||
| {
|
||||
kind: "inspect";
|
||||
lifecycleGeneration: string;
|
||||
sessionKey: string;
|
||||
}
|
||||
| {
|
||||
kind: "prepare_attempt";
|
||||
attempt: number;
|
||||
lifecycleGeneration: string;
|
||||
now: number;
|
||||
observation: MainSessionRecoveryObservation;
|
||||
runId: string;
|
||||
}
|
||||
| { kind: "cancel_reservation"; reservation: MainSessionRecoveryReservation }
|
||||
| { kind: "abandon_reservation"; reservation: MainSessionRecoveryReservation }
|
||||
| {
|
||||
kind: "validate_recovery";
|
||||
lifecycleGeneration: string;
|
||||
runId: string;
|
||||
sessionId: string;
|
||||
}
|
||||
| {
|
||||
kind: "admit_recovery";
|
||||
lifecycleGeneration: string;
|
||||
now: number;
|
||||
runId: string;
|
||||
sessionId: string;
|
||||
}
|
||||
| {
|
||||
kind: "mark_admitted_recovery_interrupted";
|
||||
lifecycleGeneration: string;
|
||||
now: number;
|
||||
runId: string;
|
||||
sessionId: string;
|
||||
}
|
||||
| {
|
||||
kind: "claim_foreground";
|
||||
cycleId: string;
|
||||
lifecycleGeneration: string;
|
||||
sessionId: string;
|
||||
sessionKey: string;
|
||||
claimId: string;
|
||||
runId?: string;
|
||||
}
|
||||
| { kind: "bind_foreground_run"; claim: MainSessionRecoveryOwnerClaim; runId: string }
|
||||
| { kind: "validate_foreground"; claim: MainSessionRecoveryOwnerClaim }
|
||||
| { kind: "release_foreground"; claim: MainSessionRecoveryOwnerClaim }
|
||||
| {
|
||||
kind: "tombstone";
|
||||
now: number;
|
||||
observation: MainSessionRecoveryObservation;
|
||||
reason: string;
|
||||
}
|
||||
| {
|
||||
kind: "fail_recovery";
|
||||
now: number;
|
||||
observation: MainSessionRecoveryObservation;
|
||||
}
|
||||
| { kind: "doctor_repair"; now: number }
|
||||
| { kind: "clear" };
|
||||
|
||||
export type MainSessionRecoveryTransitionResult =
|
||||
| { kind: "admitted_recovery" }
|
||||
| { kind: "applied" }
|
||||
| { kind: "doctor_repaired" }
|
||||
| { kind: "failed"; noticeEntry: SessionEntry }
|
||||
| { kind: "foreground_claimed"; claim: MainSessionRecoveryOwnerClaim }
|
||||
| { kind: "foreground_validated" }
|
||||
| { kind: "no_change" }
|
||||
| { kind: "observed"; view: MainSessionRecoveryView }
|
||||
| { kind: "rejected"; reason: MainSessionRecoveryConflict }
|
||||
| { kind: "recovery_validated" }
|
||||
| { kind: "reserved"; reservation: MainSessionRecoveryReservation }
|
||||
| { kind: "tombstoned" };
|
||||
@@ -1,19 +1,24 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
|
||||
import { GatewayClientRequestError } from "../../packages/gateway-client/src/index.js";
|
||||
import { sanitizePendingFinalDeliveryText } from "../auto-reply/reply/pending-final-delivery.js";
|
||||
import type { SessionEntry } from "../config/sessions.js";
|
||||
import {
|
||||
buildRestartRecoveryClaimCleanupPatch,
|
||||
hasRestartRecoveryTerminalRun,
|
||||
resolveRestartRecoveryChannelAuthority,
|
||||
} from "../config/sessions/restart-recovery-state.js";
|
||||
import { applySessionEntryReplacements } from "../config/sessions/session-accessor.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { isTrustedMessageActionTurnIngress } from "../gateway/message-action-turn-capability.js";
|
||||
import type { GatewayRecoveryRuntime } from "../gateway/server-instance-runtime.types.js";
|
||||
import { getAgentEventLifecycleGeneration } from "../infra/agent-events.js";
|
||||
import { retryAsync } from "../infra/retry.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import { findRestartRecoveryUnsafeReplyHook } from "../plugins/restart-recovery-hook-safety.js";
|
||||
import { CommandLane } from "../process/lanes.js";
|
||||
import { resolveAgentIdFromSessionKey } from "../routing/session-key.js";
|
||||
import { MAIN_SESSION_RESTART_RECOVERY_SOURCE_TOOL } from "../sessions/input-provenance.js";
|
||||
import { resolveSendPolicy } from "../sessions/send-policy.js";
|
||||
import {
|
||||
deliveryContextFromSession,
|
||||
@@ -22,9 +27,24 @@ import {
|
||||
} from "../utils/delivery-context.shared.js";
|
||||
import { isDeliverableMessageChannel } from "../utils/message-channel.js";
|
||||
import { resolveAgentWorkspaceDir } from "./agent-scope.js";
|
||||
import { buildMainSessionRecoveryClearPatch } from "./main-session-recovery-clear.js";
|
||||
import { scheduleMainSessionRecoveryPendingTarget } from "./main-session-recovery-owner-release.js";
|
||||
import {
|
||||
restoreAdmittedRecoveryWithRetries,
|
||||
scheduleAdmittedRecoveryRestore,
|
||||
type RestoreAdmittedRecovery,
|
||||
} from "./main-session-recovery-restore.js";
|
||||
import {
|
||||
isMainRestartRecoveryCandidate,
|
||||
type MainSessionRecoveryObservation,
|
||||
type MainSessionRecoveryReservation,
|
||||
} from "./main-session-recovery-state.js";
|
||||
import { commitMainSessionRecovery } from "./main-session-recovery-store.js";
|
||||
import { ensureRuntimePluginsLoaded } from "./runtime-plugins.js";
|
||||
|
||||
const log = createSubsystemLogger("main-session-restart-recovery");
|
||||
const RESERVATION_ROLLBACK_RETRY_DELAY_MS = 1_000;
|
||||
const RESERVATION_ROLLBACK_RETRY_MAX_DELAY_MS = 30_000;
|
||||
const RESTART_RECOVERY_RESUME_MESSAGE =
|
||||
"[System] Your previous turn was interrupted by a gateway restart while " +
|
||||
"OpenClaw was waiting on tool/model work. Continue from the existing " +
|
||||
@@ -172,7 +192,6 @@ async function settleRestartRecoveryDispatch(params: {
|
||||
expectedRecoveryRunId: string;
|
||||
expectedRecoverySourceRunId?: string;
|
||||
expectedSessionId: string;
|
||||
pendingFinalDeliveryText: string;
|
||||
sessionKeys: readonly string[];
|
||||
storePath: string;
|
||||
terminalStatus?: RestartRecoveryTerminalStatus;
|
||||
@@ -215,30 +234,15 @@ async function settleRestartRecoveryDispatch(params: {
|
||||
buildRestartRecoveryClaimCleanupPatch({
|
||||
entry,
|
||||
recordTerminalSource: true,
|
||||
terminalRunId: params.expectedRecoveryRunId,
|
||||
terminalSourceRunId: params.expectedRecoverySourceRunId,
|
||||
}),
|
||||
buildMainSessionRecoveryClearPatch(entry),
|
||||
);
|
||||
} else {
|
||||
entry.abortedLastRun = false;
|
||||
}
|
||||
entry.updatedAt = now;
|
||||
if (entry.pendingFinalDelivery || entry.pendingFinalDeliveryText) {
|
||||
if (params.pendingFinalDeliveryText) {
|
||||
entry.pendingFinalDeliveryLastAttemptAt = now;
|
||||
entry.pendingFinalDeliveryAttemptCount =
|
||||
(entry.pendingFinalDeliveryAttemptCount ?? 0) + 1;
|
||||
entry.pendingFinalDeliveryLastError = null;
|
||||
entry.pendingFinalDeliveryText = params.pendingFinalDeliveryText;
|
||||
} else {
|
||||
entry.pendingFinalDelivery = undefined;
|
||||
entry.pendingFinalDeliveryText = undefined;
|
||||
entry.pendingFinalDeliveryCreatedAt = undefined;
|
||||
entry.pendingFinalDeliveryLastAttemptAt = undefined;
|
||||
entry.pendingFinalDeliveryAttemptCount = undefined;
|
||||
entry.pendingFinalDeliveryLastError = undefined;
|
||||
entry.pendingFinalDeliveryContext = undefined;
|
||||
}
|
||||
}
|
||||
return {
|
||||
result: undefined,
|
||||
replacements: [{ sessionKey: current.sessionKey, entry }],
|
||||
@@ -247,17 +251,104 @@ async function settleRestartRecoveryDispatch(params: {
|
||||
});
|
||||
}
|
||||
|
||||
function isExactRestartRecoveryDispatchAdmission(params: {
|
||||
admission: Awaited<ReturnType<typeof commitMainSessionRecovery>>;
|
||||
lifecycleGeneration: string;
|
||||
recoveryRunId: string;
|
||||
sessionId: string;
|
||||
terminalStatus?: RestartRecoveryTerminalStatus;
|
||||
}): boolean {
|
||||
const entry = params.admission.entry;
|
||||
return (
|
||||
entry?.sessionId === params.sessionId &&
|
||||
((entry.abortedLastRun === false &&
|
||||
normalizeOptionalString(entry.restartRecoveryDeliveryRunId) === params.recoveryRunId &&
|
||||
entry.restartRecoveryRuns?.some(
|
||||
(run) =>
|
||||
run.runId === params.recoveryRunId &&
|
||||
run.lifecycleGeneration === params.lifecycleGeneration,
|
||||
) === true) ||
|
||||
(hasRestartRecoveryTerminalRun(entry, params.recoveryRunId) &&
|
||||
((params.terminalStatus === "ok" && entry.status === "done") ||
|
||||
(params.terminalStatus === "error" && entry.status === "failed") ||
|
||||
(params.terminalStatus === "timeout" && entry.status === "timeout"))))
|
||||
);
|
||||
}
|
||||
|
||||
type MainSessionResumeResult = "resumed" | "skipped" | "failed";
|
||||
|
||||
async function rollbackRestartRecoveryReservation(params: {
|
||||
kind: "abandon_reservation" | "cancel_reservation";
|
||||
reservation: MainSessionRecoveryReservation;
|
||||
sessionKey: string;
|
||||
storePath: string;
|
||||
}) {
|
||||
return await retryAsync(
|
||||
async () =>
|
||||
await commitMainSessionRecovery({
|
||||
command: { kind: params.kind, reservation: params.reservation },
|
||||
requireWriteSuccess: true,
|
||||
target: { sessionKey: params.sessionKey, storePath: params.storePath },
|
||||
}),
|
||||
3,
|
||||
25,
|
||||
);
|
||||
}
|
||||
|
||||
function scheduleRestartRecoveryReservationRollback(
|
||||
params: Parameters<typeof rollbackRestartRecoveryReservation>[0],
|
||||
delayMs = RESERVATION_ROLLBACK_RETRY_DELAY_MS,
|
||||
): void {
|
||||
// Keep the exact reservation token alive after transient store outages.
|
||||
// A Gateway restart safely retires the timer and its stale-generation slot.
|
||||
setTimeout(() => {
|
||||
void rollbackRestartRecoveryReservation(params).then(
|
||||
({ entry, sessionKey }) => {
|
||||
const state = entry?.mainRestartRecovery;
|
||||
if (
|
||||
entry?.sessionId === params.reservation.sessionId &&
|
||||
sessionKey &&
|
||||
entry.status === "running" &&
|
||||
entry.abortedLastRun === true &&
|
||||
isMainRestartRecoveryCandidate(entry, sessionKey) &&
|
||||
state &&
|
||||
!state.foregroundClaims &&
|
||||
!state.reservation &&
|
||||
!state.tombstone
|
||||
) {
|
||||
scheduleMainSessionRecoveryPendingTarget({
|
||||
sessionId: entry.sessionId,
|
||||
sessionKey,
|
||||
storePath: params.storePath,
|
||||
});
|
||||
}
|
||||
},
|
||||
(error: unknown) => {
|
||||
log.warn(
|
||||
`failed delayed restart recovery reservation rollback ${params.sessionKey}: ${String(error)}`,
|
||||
);
|
||||
scheduleRestartRecoveryReservationRollback(
|
||||
params,
|
||||
Math.min(delayMs * 2, RESERVATION_ROLLBACK_RETRY_MAX_DELAY_MS),
|
||||
);
|
||||
},
|
||||
);
|
||||
}, delayMs).unref?.();
|
||||
}
|
||||
|
||||
export async function resumeMainSession(params: {
|
||||
canonicalSessionKey?: string;
|
||||
cfg?: OpenClawConfig;
|
||||
entry: SessionEntry;
|
||||
observation: MainSessionRecoveryObservation;
|
||||
recoveryAttempt: number;
|
||||
storePath: string;
|
||||
sessionKey: string;
|
||||
pendingFinalDeliveryText?: string | null;
|
||||
forceRestartSafeTools?: boolean;
|
||||
sessionWorkAdmissionHandoffId?: string;
|
||||
gatewayRuntime: GatewayRecoveryRuntime;
|
||||
}): Promise<boolean> {
|
||||
}): Promise<MainSessionResumeResult> {
|
||||
const sanitizedPendingText =
|
||||
typeof params.pendingFinalDeliveryText === "string"
|
||||
? sanitizePendingFinalDeliveryText(params.pendingFinalDeliveryText)
|
||||
@@ -274,14 +365,31 @@ export async function resumeMainSession(params: {
|
||||
!hasRestartRecoveryMessageActionAuthority(params.entry)
|
||||
) {
|
||||
log.warn(`refusing message-tool-only recovery without channel authority: ${params.sessionKey}`);
|
||||
return false;
|
||||
return "failed";
|
||||
}
|
||||
const recoveryRunId = claimedRunId && claimedRunId !== sourceRunId ? claimedRunId : randomUUID();
|
||||
const reusingRecoveryRunId = recoveryRunId === claimedRunId;
|
||||
const dispatchSessionKey = params.canonicalSessionKey ?? params.sessionKey;
|
||||
const recoverySessionKeys = Array.from(new Set([dispatchSessionKey, params.sessionKey]));
|
||||
let dispatchOutcomeUnknown = false;
|
||||
let reservation: MainSessionRecoveryReservation | undefined;
|
||||
let dispatchStarted = false;
|
||||
try {
|
||||
const reserved = await commitMainSessionRecovery({
|
||||
command: {
|
||||
kind: "prepare_attempt",
|
||||
attempt: params.recoveryAttempt,
|
||||
lifecycleGeneration: getAgentEventLifecycleGeneration(),
|
||||
now: Date.now(),
|
||||
observation: params.observation,
|
||||
runId: recoveryRunId,
|
||||
},
|
||||
requireWriteSuccess: true,
|
||||
target: { sessionKey: params.sessionKey, storePath: params.storePath },
|
||||
});
|
||||
if (reserved.transition.kind !== "reserved") {
|
||||
return "skipped";
|
||||
}
|
||||
reservation = reserved.transition.reservation;
|
||||
// Persist one stable RPC id before dispatch. A transport rejection is
|
||||
// ambiguous; retries must reuse this id so accepted work cannot duplicate.
|
||||
const recoveryStatePrepared = await applySessionEntryReplacements({
|
||||
@@ -312,7 +420,21 @@ export async function resumeMainSession(params: {
|
||||
},
|
||||
});
|
||||
if (!recoveryStatePrepared) {
|
||||
throw new Error("restart recovery session ownership changed before dispatch");
|
||||
const rollback = await rollbackRestartRecoveryReservation({
|
||||
kind: "cancel_reservation",
|
||||
reservation,
|
||||
sessionKey: params.sessionKey,
|
||||
storePath: params.storePath,
|
||||
});
|
||||
reservation = undefined;
|
||||
const current = rollback.entry;
|
||||
return current?.sessionId === params.entry.sessionId &&
|
||||
current.status === "running" &&
|
||||
current.abortedLastRun === true &&
|
||||
!current.mainRestartRecovery?.reservation &&
|
||||
!current.mainRestartRecovery?.tombstone
|
||||
? "failed"
|
||||
: "skipped";
|
||||
}
|
||||
const agentParams: Record<string, unknown> = {
|
||||
message: buildResumeMessage(sanitizedPendingText),
|
||||
@@ -330,6 +452,11 @@ export async function resumeMainSession(params: {
|
||||
? { sourceReplyDeliveryMode: params.entry.restartRecoverySourceReplyDeliveryMode }
|
||||
: {}),
|
||||
...(params.forceRestartSafeTools ? { forceRestartSafeTools: true } : {}),
|
||||
inputProvenance: {
|
||||
kind: "internal_system",
|
||||
sourceSessionKey: dispatchSessionKey,
|
||||
sourceTool: MAIN_SESSION_RESTART_RECOVERY_SOURCE_TOOL,
|
||||
},
|
||||
};
|
||||
if (deliveryContext) {
|
||||
agentParams.channel = deliveryContext.channel;
|
||||
@@ -345,14 +472,14 @@ export async function resumeMainSession(params: {
|
||||
if (params.forceRestartSafeTools) {
|
||||
log.info(`dispatching restart-safe recovery for ${params.sessionKey}`);
|
||||
}
|
||||
// Once dispatch starts, any rejection is ambiguous because the stable RPC
|
||||
// may still have been accepted; a successful return resolves that ambiguity.
|
||||
dispatchOutcomeUnknown = true;
|
||||
dispatchStarted = true;
|
||||
const dispatchResult = await params.gatewayRuntime.dispatchAgent<{
|
||||
runId: string;
|
||||
status?: unknown;
|
||||
}>(agentParams, 10_000);
|
||||
dispatchOutcomeUnknown = false;
|
||||
// Real Gateway admission consumes the reservation before returning accepted.
|
||||
// Recovery-runtime fakes may return directly, so keep this idempotent fallback
|
||||
// to make the durable acceptance boundary explicit in focused tests too.
|
||||
let terminalStatus = normalizeRestartRecoveryTerminalStatus(dispatchResult.status);
|
||||
if (!terminalStatus && reusingRecoveryRunId && dispatchResult.status === "accepted") {
|
||||
terminalStatus = await probeRestartRecoveryTerminalStatus(
|
||||
@@ -360,11 +487,33 @@ export async function resumeMainSession(params: {
|
||||
params.gatewayRuntime,
|
||||
);
|
||||
}
|
||||
const lifecycleGeneration = getAgentEventLifecycleGeneration();
|
||||
const admission = await commitMainSessionRecovery({
|
||||
command: {
|
||||
kind: "admit_recovery",
|
||||
lifecycleGeneration,
|
||||
now: Date.now(),
|
||||
runId: recoveryRunId,
|
||||
sessionId: params.entry.sessionId,
|
||||
},
|
||||
target: { sessionKey: params.sessionKey, storePath: params.storePath },
|
||||
});
|
||||
if (
|
||||
admission.transition.kind !== "admitted_recovery" &&
|
||||
!isExactRestartRecoveryDispatchAdmission({
|
||||
admission,
|
||||
lifecycleGeneration,
|
||||
recoveryRunId,
|
||||
sessionId: params.entry.sessionId,
|
||||
terminalStatus,
|
||||
})
|
||||
) {
|
||||
throw new Error(`restart recovery admission changed before settlement: ${params.sessionKey}`);
|
||||
}
|
||||
await settleRestartRecoveryDispatch({
|
||||
expectedRecoveryRunId: recoveryRunId,
|
||||
expectedRecoverySourceRunId: sourceRunId,
|
||||
expectedSessionId: params.entry.sessionId,
|
||||
pendingFinalDeliveryText: sanitizedPendingText,
|
||||
sessionKeys: recoverySessionKeys,
|
||||
storePath: params.storePath,
|
||||
terminalStatus,
|
||||
@@ -374,28 +523,114 @@ export async function resumeMainSession(params: {
|
||||
sanitizedPendingText ? " (with pending payload)" : ""
|
||||
}`,
|
||||
);
|
||||
return true;
|
||||
return "resumed";
|
||||
} catch (error) {
|
||||
if (reusingRecoveryRunId && dispatchOutcomeUnknown) {
|
||||
const terminalStatus = await probeRestartRecoveryTerminalStatus(
|
||||
recoveryRunId,
|
||||
params.gatewayRuntime,
|
||||
const explicitlyRejected = error instanceof GatewayClientRequestError;
|
||||
try {
|
||||
if (dispatchStarted && !explicitlyRejected) {
|
||||
const terminalStatus = await probeRestartRecoveryTerminalStatus(
|
||||
recoveryRunId,
|
||||
params.gatewayRuntime,
|
||||
);
|
||||
if (terminalStatus) {
|
||||
const lifecycleGeneration = getAgentEventLifecycleGeneration();
|
||||
const admission = await commitMainSessionRecovery({
|
||||
command: {
|
||||
kind: "admit_recovery",
|
||||
lifecycleGeneration,
|
||||
now: Date.now(),
|
||||
runId: recoveryRunId,
|
||||
sessionId: params.entry.sessionId,
|
||||
},
|
||||
target: { sessionKey: params.sessionKey, storePath: params.storePath },
|
||||
});
|
||||
const exactRunAlreadyAdmitted = isExactRestartRecoveryDispatchAdmission({
|
||||
admission,
|
||||
lifecycleGeneration,
|
||||
recoveryRunId,
|
||||
sessionId: params.entry.sessionId,
|
||||
terminalStatus,
|
||||
});
|
||||
if (admission.transition.kind !== "admitted_recovery" && !exactRunAlreadyAdmitted) {
|
||||
log.warn(`restart recovery admission changed before settlement: ${params.sessionKey}`);
|
||||
} else {
|
||||
if (reservation) {
|
||||
await commitMainSessionRecovery({
|
||||
command: { kind: "abandon_reservation", reservation },
|
||||
target: { sessionKey: params.sessionKey, storePath: params.storePath },
|
||||
});
|
||||
}
|
||||
await settleRestartRecoveryDispatch({
|
||||
expectedRecoveryRunId: recoveryRunId,
|
||||
expectedRecoverySourceRunId: sourceRunId,
|
||||
expectedSessionId: params.entry.sessionId,
|
||||
sessionKeys: recoverySessionKeys,
|
||||
storePath: params.storePath,
|
||||
terminalStatus,
|
||||
});
|
||||
log.info(`settled completed restart recovery for ${params.sessionKey}`);
|
||||
return "resumed";
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (settlementError) {
|
||||
log.warn(
|
||||
`failed to settle ambiguous restart recovery ${params.sessionKey}: ${String(settlementError)}`,
|
||||
);
|
||||
if (terminalStatus) {
|
||||
await settleRestartRecoveryDispatch({
|
||||
expectedRecoveryRunId: recoveryRunId,
|
||||
expectedRecoverySourceRunId: sourceRunId,
|
||||
expectedSessionId: params.entry.sessionId,
|
||||
pendingFinalDeliveryText: sanitizedPendingText,
|
||||
sessionKeys: recoverySessionKeys,
|
||||
storePath: params.storePath,
|
||||
terminalStatus,
|
||||
const restoreAdmittedRecovery: RestoreAdmittedRecovery = async () => {
|
||||
const restored = await commitMainSessionRecovery({
|
||||
command: {
|
||||
kind: "mark_admitted_recovery_interrupted",
|
||||
lifecycleGeneration: getAgentEventLifecycleGeneration(),
|
||||
now: Date.now(),
|
||||
runId: recoveryRunId,
|
||||
sessionId: params.entry.sessionId,
|
||||
},
|
||||
requireWriteSuccess: true,
|
||||
target: { sessionKey: params.sessionKey, storePath: params.storePath },
|
||||
});
|
||||
log.info(`settled completed restart recovery for ${params.sessionKey}`);
|
||||
return true;
|
||||
return restored.transition.kind === "applied" && restored.entry && restored.sessionKey
|
||||
? {
|
||||
sessionId: restored.entry.sessionId,
|
||||
sessionKey: restored.sessionKey,
|
||||
storePath: params.storePath,
|
||||
}
|
||||
: undefined;
|
||||
};
|
||||
try {
|
||||
scheduleMainSessionRecoveryPendingTarget(
|
||||
await restoreAdmittedRecoveryWithRetries(restoreAdmittedRecovery),
|
||||
);
|
||||
} catch (restoreError) {
|
||||
log.warn(
|
||||
`failed to restore ambiguous restart recovery ${params.sessionKey}: ${String(restoreError)}`,
|
||||
);
|
||||
scheduleAdmittedRecoveryRestore(restoreAdmittedRecovery);
|
||||
}
|
||||
}
|
||||
log.warn(`failed to resume interrupted main session ${params.sessionKey}: ${String(error)}`);
|
||||
return false;
|
||||
if (reservation) {
|
||||
const rollbackReservation = reservation;
|
||||
await rollbackRestartRecoveryReservation({
|
||||
kind: dispatchStarted && !explicitlyRejected ? "abandon_reservation" : "cancel_reservation",
|
||||
reservation: rollbackReservation,
|
||||
sessionKey: params.sessionKey,
|
||||
storePath: params.storePath,
|
||||
}).catch((rollbackError: unknown) => {
|
||||
log.warn(
|
||||
`failed to roll back interrupted main session recovery attempt ${params.sessionKey}: ${String(rollbackError)}`,
|
||||
);
|
||||
scheduleRestartRecoveryReservationRollback({
|
||||
kind:
|
||||
dispatchStarted && !explicitlyRejected ? "abandon_reservation" : "cancel_reservation",
|
||||
reservation: rollbackReservation,
|
||||
sessionKey: params.sessionKey,
|
||||
storePath: params.storePath,
|
||||
});
|
||||
});
|
||||
}
|
||||
log.warn(
|
||||
`failed to resume interrupted main session ${params.sessionKey}: ${error instanceof Error ? (error.stack ?? error.message) : String(error)}`,
|
||||
);
|
||||
return "failed";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,228 @@
|
||||
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
|
||||
import type { InternalSessionEntry as SessionEntry } from "../config/sessions.js";
|
||||
import { loadSessionEntry } from "../config/sessions/session-accessor.js";
|
||||
import { appendAssistantMessageToSessionTranscript } from "../config/sessions/transcript.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import type { GatewayRecoveryRuntime } from "../gateway/server-instance-runtime.types.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import { resolveAgentIdFromSessionKey } from "../routing/session-key.js";
|
||||
import type { DeliveryContext } from "../utils/delivery-context.shared.js";
|
||||
import {
|
||||
isMainSessionRecoveryExhausted,
|
||||
type MainSessionRecoveryObservation,
|
||||
} from "./main-session-recovery-state.js";
|
||||
import { commitMainSessionRecovery } from "./main-session-recovery-store.js";
|
||||
import { buildUnresumableSessionNoticeIdempotencyKey } from "./main-session-restart-claim.js";
|
||||
import { resolveRestartRecoveryDeliveryContext } from "./main-session-restart-dispatch.js";
|
||||
|
||||
const log = createSubsystemLogger("main-session-restart-recovery");
|
||||
const TOMBSTONED_SESSION_NOTICE =
|
||||
"I couldn't recover this session after repeated gateway restarts. " +
|
||||
"Use /new or /reset to start a replacement session.";
|
||||
|
||||
async function claimMainRestartRecoveryTombstone(params: {
|
||||
observation: MainSessionRecoveryObservation;
|
||||
reason: string;
|
||||
storePath: string;
|
||||
sessionKey: string;
|
||||
}): Promise<SessionEntry | null> {
|
||||
const claim = await commitMainSessionRecovery({
|
||||
command: {
|
||||
kind: "tombstone",
|
||||
now: Date.now(),
|
||||
observation: params.observation,
|
||||
reason: params.reason,
|
||||
},
|
||||
requireWriteSuccess: true,
|
||||
target: { sessionKey: params.sessionKey, storePath: params.storePath },
|
||||
});
|
||||
if (claim.transition.kind !== "tombstoned" || !claim.entry) {
|
||||
return null;
|
||||
}
|
||||
log.warn(`tombstoned main-session restart recovery: ${params.sessionKey} (${params.reason})`);
|
||||
return claim.entry;
|
||||
}
|
||||
|
||||
export async function tombstoneMainRestartRecoveryWithNotice(params: {
|
||||
cfg?: OpenClawConfig;
|
||||
entry: SessionEntry;
|
||||
gatewayRuntime: GatewayRecoveryRuntime;
|
||||
observation: MainSessionRecoveryObservation;
|
||||
reason: string;
|
||||
sessionKey: string;
|
||||
storePath: string;
|
||||
}): Promise<"notice_failed" | "skipped" | "tombstoned"> {
|
||||
const deliveryContext = resolveRestartRecoveryDeliveryContext({
|
||||
cfg: params.cfg,
|
||||
entry: params.entry,
|
||||
includeSessionDeliveryFallback: true,
|
||||
sessionKey: params.sessionKey,
|
||||
});
|
||||
if (!deliveryContext) {
|
||||
// The transcript notice and tombstone share one SQLite transaction so a
|
||||
// foreground takeover cannot leave behind a false terminal notice.
|
||||
let entry = params.entry;
|
||||
let observation = params.observation;
|
||||
for (let attempt = 0; attempt < 3; attempt += 1) {
|
||||
const notice = await writeUnresumableSessionNotice({
|
||||
...params,
|
||||
entry,
|
||||
observation,
|
||||
text: TOMBSTONED_SESSION_NOTICE,
|
||||
});
|
||||
if (notice === "written") {
|
||||
return "tombstoned";
|
||||
}
|
||||
if (notice === "failed") {
|
||||
return "notice_failed";
|
||||
}
|
||||
const current = loadSessionEntry({
|
||||
sessionKey: params.sessionKey,
|
||||
storePath: params.storePath,
|
||||
readConsistency: "latest",
|
||||
}) as SessionEntry | undefined;
|
||||
const state = current?.mainRestartRecovery;
|
||||
if (
|
||||
!current ||
|
||||
current.sessionId !== params.entry.sessionId ||
|
||||
state?.cycleId !== params.observation.cycleId ||
|
||||
state.tombstone ||
|
||||
!isMainSessionRecoveryExhausted(current)
|
||||
) {
|
||||
return "skipped";
|
||||
}
|
||||
entry = current;
|
||||
observation = {
|
||||
sessionId: current.sessionId,
|
||||
cycleId: state.cycleId,
|
||||
revision: state.revision,
|
||||
};
|
||||
}
|
||||
return "notice_failed";
|
||||
}
|
||||
const tombstonedEntry = await claimMainRestartRecoveryTombstone(params);
|
||||
if (!tombstonedEntry) {
|
||||
return "skipped";
|
||||
}
|
||||
if (deliveryContext) {
|
||||
await sendUnresumableSessionNotice({
|
||||
deliveryContext,
|
||||
entry: tombstonedEntry,
|
||||
gatewayRuntime: params.gatewayRuntime,
|
||||
reason: params.reason,
|
||||
sessionKey: params.sessionKey,
|
||||
text: TOMBSTONED_SESSION_NOTICE,
|
||||
});
|
||||
}
|
||||
return "tombstoned";
|
||||
}
|
||||
|
||||
async function sendUnresumableSessionNotice(params: {
|
||||
deliveryContext: DeliveryContext;
|
||||
entry: SessionEntry;
|
||||
gatewayRuntime: GatewayRecoveryRuntime;
|
||||
reason: string;
|
||||
sessionKey: string;
|
||||
text: string;
|
||||
}): Promise<void> {
|
||||
const messageParams: Record<string, unknown> = {
|
||||
to: params.deliveryContext.to,
|
||||
message: params.text,
|
||||
bestEffort: true,
|
||||
...(params.deliveryContext.threadId != null
|
||||
? { threadId: params.deliveryContext.threadId }
|
||||
: {}),
|
||||
};
|
||||
const actionParams: Record<string, unknown> = {
|
||||
channel: params.deliveryContext.channel,
|
||||
action: "send",
|
||||
sessionKey: params.sessionKey,
|
||||
sessionId: params.entry.sessionId,
|
||||
idempotencyKey: buildUnresumableSessionNoticeIdempotencyKey(params.entry),
|
||||
params: messageParams,
|
||||
};
|
||||
const accountId = normalizeOptionalString(params.deliveryContext.accountId);
|
||||
if (accountId) {
|
||||
actionParams.accountId = accountId;
|
||||
}
|
||||
try {
|
||||
await params.gatewayRuntime.sendRecoveryNotice(actionParams, 10_000);
|
||||
log.info(
|
||||
`sent interrupted main session recovery notice: ${params.sessionKey} (${params.reason})`,
|
||||
);
|
||||
} catch (error) {
|
||||
log.warn(
|
||||
`failed to send interrupted main session recovery notice ${params.sessionKey}: ${String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function writeUnresumableSessionNotice(params: {
|
||||
entry: SessionEntry;
|
||||
observation: MainSessionRecoveryObservation;
|
||||
reason: string;
|
||||
sessionKey: string;
|
||||
storePath: string;
|
||||
text: string;
|
||||
}): Promise<"failed" | "stale" | "written"> {
|
||||
const recoveryState = params.entry.mainRestartRecovery;
|
||||
if (
|
||||
!recoveryState ||
|
||||
recoveryState.cycleId !== params.observation.cycleId ||
|
||||
recoveryState.revision !== params.observation.revision
|
||||
) {
|
||||
return "stale";
|
||||
}
|
||||
const now = Date.now();
|
||||
const result = await appendAssistantMessageToSessionTranscript({
|
||||
agentId: resolveAgentIdFromSessionKey(params.sessionKey),
|
||||
sessionKey: params.sessionKey,
|
||||
expectedSessionId: params.entry.sessionId,
|
||||
expectedSessionState: {
|
||||
abortedLastRun: params.entry.abortedLastRun,
|
||||
mainRestartRecoveryCycleId: params.observation.cycleId,
|
||||
mainRestartRecoveryRevision: params.observation.revision,
|
||||
restartRecoveryBeforeAgentReplyState: params.entry.restartRecoveryBeforeAgentReplyState,
|
||||
restartRecoveryDeliveryReceiptState: params.entry.restartRecoveryDeliveryReceiptState,
|
||||
restartRecoveryDeliveryToolCallId: params.entry.restartRecoveryDeliveryToolCallId,
|
||||
restartRecoveryDeliveryRequestFingerprint:
|
||||
params.entry.restartRecoveryDeliveryRequestFingerprint,
|
||||
restartRecoveryDeliveryRunId: params.entry.restartRecoveryDeliveryRunId,
|
||||
restartRecoveryDeliverySourceRunId: params.entry.restartRecoveryDeliverySourceRunId,
|
||||
restartRecoveryRequesterAccountId: params.entry.restartRecoveryRequesterAccountId,
|
||||
restartRecoveryRequesterSenderId: params.entry.restartRecoveryRequesterSenderId,
|
||||
restartRecoverySameChannelThreadRequired:
|
||||
params.entry.restartRecoverySameChannelThreadRequired,
|
||||
restartRecoverySourceIngress: params.entry.restartRecoverySourceIngress,
|
||||
restartRecoverySourceReplyDeliveryMode: params.entry.restartRecoverySourceReplyDeliveryMode,
|
||||
restartRecoveryTerminalRunIds: params.entry.restartRecoveryTerminalRunIds,
|
||||
status: params.entry.status,
|
||||
updatedAt: params.entry.updatedAt,
|
||||
},
|
||||
sessionLifecyclePatch: {
|
||||
abortedLastRun: false,
|
||||
endedAt: now,
|
||||
mainRestartRecovery: {
|
||||
...recoveryState,
|
||||
revision: recoveryState.revision + 1,
|
||||
tombstone: { reason: params.reason },
|
||||
},
|
||||
runtimeMs: Math.max(0, now - (params.entry.startedAt ?? now)),
|
||||
status: "failed",
|
||||
updatedAt: now,
|
||||
},
|
||||
storePath: params.storePath,
|
||||
text: params.text,
|
||||
idempotencyKey: buildUnresumableSessionNoticeIdempotencyKey(params.entry),
|
||||
}).catch((error: unknown) => ({ ok: false as const, reason: String(error) }));
|
||||
if (!result.ok) {
|
||||
log.warn(
|
||||
`failed to write interrupted main session notice ${params.sessionKey}: ${result.reason}`,
|
||||
);
|
||||
}
|
||||
return result.ok
|
||||
? "written"
|
||||
: "code" in result && result.code === "session-rebound"
|
||||
? "stale"
|
||||
: "failed";
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -2,14 +2,15 @@
|
||||
* Post-restart recovery for main sessions interrupted while holding a transcript lock.
|
||||
*/
|
||||
|
||||
import { randomUUID } from "node:crypto";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
|
||||
import { isSilentReplyPayloadText, SILENT_REPLY_TOKEN } from "../auto-reply/tokens.js";
|
||||
import { resolveStateDir } from "../config/paths.js";
|
||||
import {
|
||||
type InternalSessionEntry as SessionEntry,
|
||||
type RestartRecoveryRun,
|
||||
type SessionEntry,
|
||||
resolveSessionWorkStartError,
|
||||
resolveAllAgentSessionStoreTargetsSync,
|
||||
resolveSessionFilePath,
|
||||
@@ -18,6 +19,7 @@ import {
|
||||
import { buildRestartRecoveryClaimCleanupPatch } from "../config/sessions/restart-recovery-state.js";
|
||||
import {
|
||||
applySessionEntryReplacements,
|
||||
loadExactSessionEntry,
|
||||
listSessionEntriesByStatus,
|
||||
persistSessionTranscriptTurn,
|
||||
type SessionTranscriptTurnExpectedState,
|
||||
@@ -51,6 +53,12 @@ import {
|
||||
listActiveEmbeddedRunSessionIds,
|
||||
listActiveEmbeddedRunSessionKeys,
|
||||
} from "./embedded-agent-runner/run-state.js";
|
||||
import {
|
||||
isMainRestartRecoveryCandidate,
|
||||
transitionMainSessionRecovery,
|
||||
type MainSessionRecoveryObservation,
|
||||
} from "./main-session-recovery-state.js";
|
||||
import { commitMainSessionRecovery } from "./main-session-recovery-store.js";
|
||||
import {
|
||||
buildUnresumableSessionNoticeIdempotencyKey,
|
||||
loadExpectedRestartRecoveryClaim,
|
||||
@@ -63,6 +71,7 @@ import {
|
||||
resolveRestartRecoveryDeliveryContext,
|
||||
resumeMainSession,
|
||||
} from "./main-session-restart-dispatch.js";
|
||||
import { tombstoneMainRestartRecoveryWithNotice } from "./main-session-restart-recovery-failure.js";
|
||||
import { resolveAgentSessionDirs } from "./session-dirs.js";
|
||||
import type { SessionLockInspection } from "./session-write-lock.js";
|
||||
|
||||
@@ -74,6 +83,34 @@ const UNRESUMABLE_SESSION_NOTICE =
|
||||
"I was interrupted by a gateway restart and couldn't safely resume the previous turn. " +
|
||||
"Please send that last request again and I'll pick it up cleanly.";
|
||||
|
||||
type ExpectedRestartRecoveryTarget = {
|
||||
canonicalSessionKey?: string;
|
||||
sessionId: string;
|
||||
sessionKey: string;
|
||||
};
|
||||
|
||||
type ExhaustedRestartRecoveryTarget = ExpectedRestartRecoveryTarget & {
|
||||
storePath: string;
|
||||
};
|
||||
|
||||
function loadExpectedRestartRecoveryTarget(params: {
|
||||
expected: ExpectedRestartRecoveryTarget;
|
||||
storePath: string;
|
||||
}): SessionEntry | undefined {
|
||||
const exact = loadExactSessionEntry({
|
||||
sessionKey: params.expected.sessionKey,
|
||||
storePath: params.storePath,
|
||||
readConsistency: "latest",
|
||||
});
|
||||
const entry = exact?.sessionKey === params.expected.sessionKey ? exact.entry : undefined;
|
||||
return entry?.sessionId === params.expected.sessionId &&
|
||||
entry.status === "running" &&
|
||||
entry.abortedLastRun === true &&
|
||||
isMainRestartRecoveryCandidate(entry, params.expected.sessionKey)
|
||||
? entry
|
||||
: undefined;
|
||||
}
|
||||
|
||||
function shouldSkipMainRecovery(entry: SessionEntry, sessionKey: string): boolean {
|
||||
if (typeof entry.spawnDepth === "number" && entry.spawnDepth > 0) {
|
||||
return true;
|
||||
@@ -270,13 +307,6 @@ export async function markRestartAbortedMainSessions(params: {
|
||||
continue;
|
||||
}
|
||||
const wasRunning = entry.status === "running";
|
||||
entry.status = "running";
|
||||
entry.abortedLastRun = true;
|
||||
if (!wasRunning) {
|
||||
entry.startedAt = undefined;
|
||||
entry.endedAt = undefined;
|
||||
entry.runtimeMs = undefined;
|
||||
}
|
||||
const recoveryRuns = new Map<string, RestartRecoveryRun>();
|
||||
for (const run of entry.restartRecoveryRuns ?? []) {
|
||||
if (run.lifecycleGeneration === currentLifecycleGeneration) {
|
||||
@@ -305,7 +335,13 @@ export async function markRestartAbortedMainSessions(params: {
|
||||
? a.lifecycleGeneration.localeCompare(b.lifecycleGeneration)
|
||||
: a.runId.localeCompare(b.runId),
|
||||
);
|
||||
entry.updatedAt = Date.now();
|
||||
transitionMainSessionRecovery(entry, {
|
||||
kind: "mark_interrupted",
|
||||
cycleId: randomUUID(),
|
||||
now: Date.now(),
|
||||
resetRuntime: !wasRunning,
|
||||
runs: entry.restartRecoveryRuns,
|
||||
});
|
||||
replacements.push({ sessionKey, entry });
|
||||
counts.marked++;
|
||||
}
|
||||
@@ -379,8 +415,11 @@ export async function markStartupOrphanedMainSessionsForRecovery(params: {
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
entry.abortedLastRun = true;
|
||||
entry.updatedAt = Date.now();
|
||||
transitionMainSessionRecovery(entry, {
|
||||
kind: "mark_interrupted",
|
||||
cycleId: randomUUID(),
|
||||
now: Date.now(),
|
||||
});
|
||||
replacements.push({ sessionKey, entry });
|
||||
counts.marked++;
|
||||
}
|
||||
@@ -1018,60 +1057,25 @@ function resolveMainSessionResumePolicy(
|
||||
}
|
||||
|
||||
async function markSessionFailed(params: {
|
||||
expectedRecoveryRunId?: string;
|
||||
expectedRecoverySourceRunId?: string;
|
||||
expectedSessionId: string;
|
||||
observation: MainSessionRecoveryObservation;
|
||||
storePath: string;
|
||||
sessionKey: string;
|
||||
reason: string;
|
||||
}): Promise<boolean> {
|
||||
const marked = await applySessionEntryReplacements({
|
||||
sessionKeys: [params.sessionKey],
|
||||
storePath: params.storePath,
|
||||
update: (entries) => {
|
||||
const current = entries.find((entry) => entry.sessionKey === params.sessionKey);
|
||||
const entry = current?.entry;
|
||||
if (
|
||||
!entry ||
|
||||
entry.sessionId !== params.expectedSessionId ||
|
||||
entry.status !== "running" ||
|
||||
entry.abortedLastRun !== true ||
|
||||
normalizeOptionalString(entry.restartRecoveryDeliveryRunId) !==
|
||||
params.expectedRecoveryRunId ||
|
||||
normalizeOptionalString(entry.restartRecoveryDeliverySourceRunId) !==
|
||||
params.expectedRecoverySourceRunId
|
||||
) {
|
||||
return { result: false };
|
||||
}
|
||||
entry.status = "failed";
|
||||
entry.abortedLastRun = true;
|
||||
entry.endedAt = Date.now();
|
||||
entry.updatedAt = entry.endedAt;
|
||||
entry.pendingFinalDelivery = undefined;
|
||||
entry.pendingFinalDeliveryText = undefined;
|
||||
entry.pendingFinalDeliveryCreatedAt = undefined;
|
||||
entry.pendingFinalDeliveryLastAttemptAt = undefined;
|
||||
entry.pendingFinalDeliveryAttemptCount = undefined;
|
||||
entry.pendingFinalDeliveryLastError = undefined;
|
||||
entry.pendingFinalDeliveryContext = undefined;
|
||||
entry.pendingFinalDeliveryIntentId = undefined;
|
||||
Object.assign(
|
||||
entry,
|
||||
buildRestartRecoveryClaimCleanupPatch({
|
||||
entry,
|
||||
recordTerminalSource: true,
|
||||
}),
|
||||
);
|
||||
return {
|
||||
result: true,
|
||||
replacements: [{ sessionKey: params.sessionKey, entry }],
|
||||
};
|
||||
const marked = await commitMainSessionRecovery({
|
||||
command: {
|
||||
kind: "fail_recovery",
|
||||
now: Date.now(),
|
||||
observation: params.observation,
|
||||
},
|
||||
requireWriteSuccess: true,
|
||||
target: { sessionKey: params.sessionKey, storePath: params.storePath },
|
||||
});
|
||||
if (marked) {
|
||||
if (marked.transition.kind === "failed") {
|
||||
log.warn(`marked interrupted main session failed: ${params.sessionKey} (${params.reason})`);
|
||||
return true;
|
||||
}
|
||||
return marked;
|
||||
return false;
|
||||
}
|
||||
|
||||
type RecoveryCheckpointCompletion =
|
||||
@@ -1389,6 +1393,7 @@ async function failUnresumableMainSession(params: {
|
||||
entry: SessionEntry;
|
||||
expectedRecoverySourceRunId?: string;
|
||||
gatewayRuntime: GatewayRecoveryRuntime;
|
||||
observation: MainSessionRecoveryObservation;
|
||||
reason: string;
|
||||
sessionKey: string;
|
||||
storePath: string;
|
||||
@@ -1411,9 +1416,7 @@ async function failUnresumableMainSession(params: {
|
||||
return "failed";
|
||||
}
|
||||
const marked = await markSessionFailed({
|
||||
expectedRecoveryRunId: normalizeOptionalString(params.entry.restartRecoveryDeliveryRunId),
|
||||
expectedRecoverySourceRunId: params.expectedRecoverySourceRunId,
|
||||
expectedSessionId: params.entry.sessionId,
|
||||
observation: params.observation,
|
||||
storePath: params.storePath,
|
||||
sessionKey: params.sessionKey,
|
||||
reason: params.reason,
|
||||
@@ -1467,7 +1470,11 @@ export async function markRestartAbortedMainSessionsFromLocks(params: {
|
||||
if (!entryLockPaths.some((lockPath) => interruptedLockPaths.has(lockPath))) {
|
||||
continue;
|
||||
}
|
||||
entry.abortedLastRun = true;
|
||||
transitionMainSessionRecovery(entry, {
|
||||
kind: "mark_interrupted",
|
||||
cycleId: randomUUID(),
|
||||
now: Date.now(),
|
||||
});
|
||||
replacements.push({ sessionKey, entry });
|
||||
counts.marked++;
|
||||
}
|
||||
@@ -1508,9 +1515,12 @@ function resolveRecoveryDispatchSessionKey(params: {
|
||||
|
||||
async function recoverStore(params: {
|
||||
cfg?: OpenClawConfig;
|
||||
observationOnly?: boolean;
|
||||
onExhaustedTarget?: (target: ExhaustedRestartRecoveryTarget) => void;
|
||||
storePath: string;
|
||||
resumedSessionKeys: Set<string>;
|
||||
expectedClaim?: ExpectedRestartRecoveryClaim;
|
||||
expectedTarget?: ExpectedRestartRecoveryTarget;
|
||||
sessionWorkAdmissionHandoffId?: string;
|
||||
activeSessionIds?: Iterable<string>;
|
||||
activeSessionKeys?: Iterable<string>;
|
||||
@@ -1535,6 +1545,12 @@ async function recoverStore(params: {
|
||||
storePath: params.storePath,
|
||||
});
|
||||
entries = entry ? [{ sessionKey: params.expectedClaim.sessionKey, entry }] : [];
|
||||
} else if (params.expectedTarget) {
|
||||
const entry = loadExpectedRestartRecoveryTarget({
|
||||
expected: params.expectedTarget,
|
||||
storePath: params.storePath,
|
||||
});
|
||||
entries = entry ? [{ sessionKey: params.expectedTarget.sessionKey, entry }] : [];
|
||||
} else {
|
||||
entries = listSessionEntriesByStatus({ storePath: params.storePath }, ["running"]);
|
||||
}
|
||||
@@ -1544,9 +1560,10 @@ async function recoverStore(params: {
|
||||
return result;
|
||||
}
|
||||
|
||||
for (const { sessionKey, entry } of entries.toSorted((a, b) =>
|
||||
for (const { sessionKey, entry: loadedEntry } of entries.toSorted((a, b) =>
|
||||
a.sessionKey.localeCompare(b.sessionKey),
|
||||
)) {
|
||||
let entry = loadedEntry;
|
||||
if (!entry || entry.status !== "running" || entry.abortedLastRun !== true) {
|
||||
continue;
|
||||
}
|
||||
@@ -1568,7 +1585,9 @@ async function recoverStore(params: {
|
||||
continue;
|
||||
}
|
||||
const dispatchSessionKey =
|
||||
params.expectedClaim?.canonicalSessionKey ?? resolvedDispatchSessionKey;
|
||||
params.expectedClaim?.canonicalSessionKey ??
|
||||
params.expectedTarget?.canonicalSessionKey ??
|
||||
resolvedDispatchSessionKey;
|
||||
if (
|
||||
hasCurrentProcessOwner({
|
||||
activeSessionIds: resolveActiveSessionIds(),
|
||||
@@ -1586,6 +1605,77 @@ async function recoverStore(params: {
|
||||
continue;
|
||||
}
|
||||
|
||||
const observed = await commitMainSessionRecovery({
|
||||
command: {
|
||||
kind: "observe",
|
||||
cycleId: randomUUID(),
|
||||
lifecycleGeneration: getAgentEventLifecycleGeneration(),
|
||||
sessionKey,
|
||||
},
|
||||
requireWriteSuccess: true,
|
||||
target: { sessionKey, storePath: params.storePath },
|
||||
});
|
||||
if (!observed.entry || observed.transition.kind !== "observed") {
|
||||
result.skipped++;
|
||||
continue;
|
||||
}
|
||||
entry = observed.entry;
|
||||
const recoveryView = observed.transition.view;
|
||||
if (
|
||||
recoveryView.status === "inactive" ||
|
||||
recoveryView.status === "blocked" ||
|
||||
recoveryView.status === "tombstoned"
|
||||
) {
|
||||
result.skipped++;
|
||||
continue;
|
||||
}
|
||||
if (recoveryView.status === "exhausted") {
|
||||
const tombstone = await tombstoneMainRestartRecoveryWithNotice({
|
||||
cfg: params.cfg,
|
||||
entry,
|
||||
gatewayRuntime: params.gatewayRuntime,
|
||||
observation: recoveryView.observation,
|
||||
reason: recoveryView.reason,
|
||||
sessionKey,
|
||||
storePath: params.storePath,
|
||||
});
|
||||
if (tombstone === "notice_failed") {
|
||||
result.failed++;
|
||||
} else {
|
||||
result.skipped++;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (params.observationOnly) {
|
||||
result.skipped++;
|
||||
continue;
|
||||
}
|
||||
const recordResumeResult = (resumeResult: Awaited<ReturnType<typeof resumeMainSession>>) => {
|
||||
if (resumeResult === "resumed") {
|
||||
params.resumedSessionKeys.add(resumeDedupeKey);
|
||||
result.recovered++;
|
||||
} else if (resumeResult === "skipped") {
|
||||
result.skipped++;
|
||||
} else {
|
||||
result.failed++;
|
||||
const current = loadExpectedRestartRecoveryTarget({
|
||||
expected: { sessionId: entry.sessionId, sessionKey },
|
||||
storePath: params.storePath,
|
||||
});
|
||||
if (
|
||||
current?.mainRestartRecovery?.chargedAttempts === MAX_RECOVERY_RETRIES &&
|
||||
!current.mainRestartRecovery.reservation
|
||||
) {
|
||||
params.onExhaustedTarget?.({
|
||||
canonicalSessionKey: dispatchSessionKey,
|
||||
sessionId: entry.sessionId,
|
||||
sessionKey,
|
||||
storePath: params.storePath,
|
||||
});
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if (
|
||||
requiresRestartRecoveryMessageActionAuthority(entry) &&
|
||||
!hasRestartRecoveryMessageActionAuthority(entry)
|
||||
@@ -1597,6 +1687,7 @@ async function recoverStore(params: {
|
||||
entry.restartRecoveryDeliverySourceRunId,
|
||||
),
|
||||
gatewayRuntime: params.gatewayRuntime,
|
||||
observation: recoveryView.observation,
|
||||
reason: "message-tool-only recovery authority is unavailable",
|
||||
sessionKey,
|
||||
storePath: params.storePath,
|
||||
@@ -1627,6 +1718,7 @@ async function recoverStore(params: {
|
||||
entry,
|
||||
expectedRecoverySourceRunId,
|
||||
gatewayRuntime: params.gatewayRuntime,
|
||||
observation: recoveryView.observation,
|
||||
reason: resumeBlockReason,
|
||||
sessionKey,
|
||||
storePath: params.storePath,
|
||||
@@ -1647,6 +1739,8 @@ async function recoverStore(params: {
|
||||
canonicalSessionKey: dispatchSessionKey,
|
||||
cfg: params.cfg,
|
||||
entry,
|
||||
observation: recoveryView.observation,
|
||||
recoveryAttempt: recoveryView.nextAttempt,
|
||||
storePath: params.storePath,
|
||||
sessionKey,
|
||||
pendingFinalDeliveryText: entry.pendingFinalDeliveryText,
|
||||
@@ -1654,12 +1748,7 @@ async function recoverStore(params: {
|
||||
sessionWorkAdmissionHandoffId: params.sessionWorkAdmissionHandoffId,
|
||||
gatewayRuntime: params.gatewayRuntime,
|
||||
});
|
||||
if (resumed) {
|
||||
params.resumedSessionKeys.add(resumeDedupeKey);
|
||||
result.recovered++;
|
||||
} else {
|
||||
result.failed++;
|
||||
}
|
||||
recordResumeResult(resumed);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1691,18 +1780,15 @@ async function recoverStore(params: {
|
||||
canonicalSessionKey: dispatchSessionKey,
|
||||
cfg: params.cfg,
|
||||
entry,
|
||||
observation: recoveryView.observation,
|
||||
recoveryAttempt: recoveryView.nextAttempt,
|
||||
storePath: params.storePath,
|
||||
sessionKey,
|
||||
pendingFinalDeliveryText: entry.pendingFinalDeliveryText,
|
||||
sessionWorkAdmissionHandoffId: params.sessionWorkAdmissionHandoffId,
|
||||
gatewayRuntime: params.gatewayRuntime,
|
||||
});
|
||||
if (resumed) {
|
||||
params.resumedSessionKeys.add(resumeDedupeKey);
|
||||
result.recovered++;
|
||||
} else {
|
||||
result.failed++;
|
||||
}
|
||||
recordResumeResult(resumed);
|
||||
continue;
|
||||
}
|
||||
log.warn(`failed to read transcript for ${sessionKey}: ${String(err)}`);
|
||||
@@ -1718,6 +1804,8 @@ async function recoverStore(params: {
|
||||
canonicalSessionKey: dispatchSessionKey,
|
||||
cfg: params.cfg,
|
||||
entry,
|
||||
observation: recoveryView.observation,
|
||||
recoveryAttempt: recoveryView.nextAttempt,
|
||||
storePath: params.storePath,
|
||||
sessionKey,
|
||||
pendingFinalDeliveryText: entry.pendingFinalDeliveryText,
|
||||
@@ -1725,12 +1813,7 @@ async function recoverStore(params: {
|
||||
sessionWorkAdmissionHandoffId: params.sessionWorkAdmissionHandoffId,
|
||||
gatewayRuntime: params.gatewayRuntime,
|
||||
});
|
||||
if (resumed) {
|
||||
params.resumedSessionKeys.add(resumeDedupeKey);
|
||||
result.recovered++;
|
||||
} else {
|
||||
result.failed++;
|
||||
}
|
||||
recordResumeResult(resumed);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -1767,6 +1850,7 @@ async function recoverStore(params: {
|
||||
entry,
|
||||
expectedRecoverySourceRunId,
|
||||
gatewayRuntime: params.gatewayRuntime,
|
||||
observation: recoveryView.observation,
|
||||
reason: completion.reason,
|
||||
sessionKey,
|
||||
storePath: params.storePath,
|
||||
@@ -1781,6 +1865,7 @@ async function recoverStore(params: {
|
||||
entry,
|
||||
expectedRecoverySourceRunId,
|
||||
gatewayRuntime: params.gatewayRuntime,
|
||||
observation: recoveryView.observation,
|
||||
reason: resumePolicy.reason,
|
||||
sessionKey,
|
||||
storePath: params.storePath,
|
||||
@@ -1796,6 +1881,8 @@ async function recoverStore(params: {
|
||||
canonicalSessionKey: dispatchSessionKey,
|
||||
cfg: params.cfg,
|
||||
entry,
|
||||
observation: recoveryView.observation,
|
||||
recoveryAttempt: recoveryView.nextAttempt,
|
||||
storePath: params.storePath,
|
||||
sessionKey,
|
||||
pendingFinalDeliveryText: entry.pendingFinalDeliveryText,
|
||||
@@ -1804,12 +1891,7 @@ async function recoverStore(params: {
|
||||
sessionWorkAdmissionHandoffId: params.sessionWorkAdmissionHandoffId,
|
||||
gatewayRuntime: params.gatewayRuntime,
|
||||
});
|
||||
if (resumed) {
|
||||
params.resumedSessionKeys.add(resumeDedupeKey);
|
||||
result.recovered++;
|
||||
} else {
|
||||
result.failed++;
|
||||
}
|
||||
recordResumeResult(resumed);
|
||||
}
|
||||
|
||||
return result;
|
||||
@@ -1833,8 +1915,9 @@ async function resolveRestartRecoveryStorePaths(params: {
|
||||
return [...storePaths].toSorted((a, b) => a.localeCompare(b));
|
||||
}
|
||||
|
||||
export async function recoverRestartAbortedMainSessions(params: {
|
||||
async function recoverRestartAbortedMainSessionsWithOptions(params: {
|
||||
cfg?: OpenClawConfig;
|
||||
onExhaustedTarget?: (target: ExhaustedRestartRecoveryTarget) => void;
|
||||
stateDir?: string;
|
||||
resumedSessionKeys?: Set<string>;
|
||||
activeSessionIds?: Iterable<string>;
|
||||
@@ -1847,6 +1930,7 @@ export async function recoverRestartAbortedMainSessions(params: {
|
||||
for (const storePath of await resolveRestartRecoveryStorePaths(params)) {
|
||||
const storeResult = await recoverStore({
|
||||
cfg: params.cfg,
|
||||
onExhaustedTarget: params.onExhaustedTarget,
|
||||
storePath,
|
||||
resumedSessionKeys,
|
||||
activeSessionIds: params.activeSessionIds,
|
||||
@@ -1866,6 +1950,17 @@ export async function recoverRestartAbortedMainSessions(params: {
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function recoverRestartAbortedMainSessions(params: {
|
||||
cfg?: OpenClawConfig;
|
||||
stateDir?: string;
|
||||
resumedSessionKeys?: Set<string>;
|
||||
activeSessionIds?: Iterable<string>;
|
||||
activeSessionKeys?: Iterable<string>;
|
||||
gatewayRuntime: GatewayRecoveryRuntime;
|
||||
}): Promise<{ recovered: number; failed: number; skipped: number }> {
|
||||
return await recoverRestartAbortedMainSessionsWithOptions(params);
|
||||
}
|
||||
|
||||
/** Retries one exact durable Control UI row from its owning per-agent SQLite store. */
|
||||
export async function retryRestartAbortedMainSessionRecovery(params: {
|
||||
canonicalSessionKey?: string;
|
||||
@@ -1918,16 +2013,147 @@ export async function retryRestartAbortedMainSessionRecovery(params: {
|
||||
);
|
||||
} finally {
|
||||
cancelSessionWorkAdmissionHandoff(handoffId);
|
||||
admission.release();
|
||||
}
|
||||
}
|
||||
|
||||
export async function recoverStartupOrphanedMainSessions(params: {
|
||||
/** Reconciles one interrupted row after its final foreground owner releases. */
|
||||
export async function retryRestartAbortedMainSessionRecoveryAfterOwnerRelease(params: {
|
||||
cfg?: OpenClawConfig;
|
||||
expectedSessionId: string;
|
||||
sessionKey: string;
|
||||
storePath: string;
|
||||
gatewayRuntime: GatewayRecoveryRuntime;
|
||||
}): Promise<{ recovered: number; failed: number; skipped: number }> {
|
||||
return await recoverExpectedRestartRecoveryTarget(params);
|
||||
}
|
||||
|
||||
async function recoverExpectedRestartRecoveryTarget(params: {
|
||||
canonicalSessionKey?: string;
|
||||
cfg?: OpenClawConfig;
|
||||
expectedSessionId: string;
|
||||
observationOnly?: boolean;
|
||||
sessionKey: string;
|
||||
storePath: string;
|
||||
gatewayRuntime: GatewayRecoveryRuntime;
|
||||
}): Promise<{ recovered: number; failed: number; skipped: number }> {
|
||||
const expectedTarget: ExpectedRestartRecoveryTarget = {
|
||||
canonicalSessionKey: params.canonicalSessionKey,
|
||||
sessionId: params.expectedSessionId,
|
||||
sessionKey: params.sessionKey,
|
||||
};
|
||||
const assertTargetCurrent = () => {
|
||||
if (
|
||||
!loadExpectedRestartRecoveryTarget({ expected: expectedTarget, storePath: params.storePath })
|
||||
) {
|
||||
throw new Error("restart recovery session ownership changed before owner-release retry");
|
||||
}
|
||||
};
|
||||
if (
|
||||
!loadExpectedRestartRecoveryTarget({ expected: expectedTarget, storePath: params.storePath })
|
||||
) {
|
||||
return { recovered: 0, failed: 0, skipped: 0 };
|
||||
}
|
||||
const admission = await beginSessionWorkAdmission({
|
||||
scope: params.storePath,
|
||||
identities: [params.sessionKey, params.expectedSessionId],
|
||||
assertAllowed: assertTargetCurrent,
|
||||
revalidateAllowed: assertTargetCurrent,
|
||||
});
|
||||
const handoffId = admission.createHandoff();
|
||||
try {
|
||||
return await admission.run(
|
||||
async () =>
|
||||
await recoverStore({
|
||||
cfg: params.cfg,
|
||||
observationOnly: params.observationOnly,
|
||||
storePath: params.storePath,
|
||||
resumedSessionKeys: new Set<string>(),
|
||||
expectedTarget,
|
||||
sessionWorkAdmissionHandoffId: handoffId,
|
||||
gatewayRuntime: params.gatewayRuntime,
|
||||
}),
|
||||
);
|
||||
} finally {
|
||||
cancelSessionWorkAdmissionHandoff(handoffId);
|
||||
admission.release();
|
||||
}
|
||||
}
|
||||
|
||||
export function scheduleRestartAbortedMainSessionRecoveryAfterOwnerRelease(params: {
|
||||
delayMs?: number;
|
||||
expectedSessionId: string;
|
||||
getConfig: () => OpenClawConfig;
|
||||
getGatewayRuntime: () => GatewayRecoveryRuntime | undefined;
|
||||
maxRetries?: number;
|
||||
sessionKey: string;
|
||||
storePath: string;
|
||||
}): void {
|
||||
const retryDelayMs = params.delayMs ?? DEFAULT_RECOVERY_DELAY_MS;
|
||||
const maxRetries = params.maxRetries ?? MAX_RECOVERY_RETRIES;
|
||||
const scheduleAttempt = (attempt: number, delayMs: number) => {
|
||||
const run = () => {
|
||||
void runWithGatewayIndependentRootWorkAdmission(async () => {
|
||||
const gatewayRuntime = params.getGatewayRuntime();
|
||||
if (!gatewayRuntime) {
|
||||
throw new Error("Gateway recovery runtime is unavailable");
|
||||
}
|
||||
return await retryRestartAbortedMainSessionRecoveryAfterOwnerRelease({
|
||||
cfg: params.getConfig(),
|
||||
expectedSessionId: params.expectedSessionId,
|
||||
sessionKey: params.sessionKey,
|
||||
storePath: params.storePath,
|
||||
gatewayRuntime,
|
||||
});
|
||||
})
|
||||
.then((result) => {
|
||||
const stillPending = loadExpectedRestartRecoveryTarget({
|
||||
expected: {
|
||||
sessionId: params.expectedSessionId,
|
||||
sessionKey: params.sessionKey,
|
||||
},
|
||||
storePath: params.storePath,
|
||||
});
|
||||
if (
|
||||
(result.failed > 0 || (result.recovered === 0 && stillPending)) &&
|
||||
attempt < maxRetries
|
||||
) {
|
||||
scheduleAttempt(attempt + 1, retryDelayMs * 2 ** (attempt - 1));
|
||||
} else if (
|
||||
attempt === maxRetries &&
|
||||
stillPending?.mainRestartRecovery?.chargedAttempts === MAX_RECOVERY_RETRIES &&
|
||||
!stillPending.mainRestartRecovery.reservation
|
||||
) {
|
||||
// The last ambiguous dispatch consumed the final durable charge.
|
||||
// One exact observation tombstones exhaustion without dispatching again.
|
||||
scheduleAttempt(attempt + 1, 0);
|
||||
}
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
if (attempt < maxRetries) {
|
||||
scheduleAttempt(attempt + 1, retryDelayMs * 2 ** (attempt - 1));
|
||||
} else {
|
||||
log.warn(`main-session owner-release recovery failed: ${String(error)}`);
|
||||
}
|
||||
});
|
||||
};
|
||||
if (delayMs <= 0) {
|
||||
run();
|
||||
} else {
|
||||
setTimeout(run, delayMs).unref?.();
|
||||
}
|
||||
};
|
||||
scheduleAttempt(1, 0);
|
||||
}
|
||||
|
||||
async function recoverStartupOrphanedMainSessionsWithOptions(params: {
|
||||
cfg?: OpenClawConfig;
|
||||
stateDir?: string;
|
||||
activeSessionIds?: Iterable<string>;
|
||||
activeSessionKeys?: Iterable<string>;
|
||||
updatedBeforeMs?: number;
|
||||
resumedSessionKeys?: Set<string>;
|
||||
onExhaustedTarget?: (target: ExhaustedRestartRecoveryTarget) => void;
|
||||
gatewayRuntime: GatewayRecoveryRuntime;
|
||||
}): Promise<{ marked: number; recovered: number; failed: number; skipped: number }> {
|
||||
const startupRecoveryCutoffMs = params.updatedBeforeMs ?? Date.now();
|
||||
@@ -1938,8 +2164,9 @@ export async function recoverStartupOrphanedMainSessions(params: {
|
||||
activeSessionKeys: params.activeSessionKeys,
|
||||
updatedBeforeMs: startupRecoveryCutoffMs,
|
||||
});
|
||||
const recovered = await recoverRestartAbortedMainSessions({
|
||||
const recovered = await recoverRestartAbortedMainSessionsWithOptions({
|
||||
cfg: params.cfg,
|
||||
onExhaustedTarget: params.onExhaustedTarget,
|
||||
stateDir: params.stateDir,
|
||||
resumedSessionKeys: params.resumedSessionKeys,
|
||||
activeSessionIds: params.activeSessionIds,
|
||||
@@ -1954,6 +2181,18 @@ export async function recoverStartupOrphanedMainSessions(params: {
|
||||
};
|
||||
}
|
||||
|
||||
export async function recoverStartupOrphanedMainSessions(params: {
|
||||
cfg?: OpenClawConfig;
|
||||
stateDir?: string;
|
||||
activeSessionIds?: Iterable<string>;
|
||||
activeSessionKeys?: Iterable<string>;
|
||||
updatedBeforeMs?: number;
|
||||
resumedSessionKeys?: Set<string>;
|
||||
gatewayRuntime: GatewayRecoveryRuntime;
|
||||
}): Promise<{ marked: number; recovered: number; failed: number; skipped: number }> {
|
||||
return await recoverStartupOrphanedMainSessionsWithOptions(params);
|
||||
}
|
||||
|
||||
export function scheduleRestartAbortedMainSessionRecovery(params: {
|
||||
cfg?: OpenClawConfig;
|
||||
delayMs?: number;
|
||||
@@ -1969,29 +2208,60 @@ export function scheduleRestartAbortedMainSessionRecovery(params: {
|
||||
const startupRecoveryCutoffMs = Date.now();
|
||||
|
||||
const runRecoveryAttempt = (attempt: number, delay: number) => {
|
||||
const exhaustedTargets = new Map<string, ExhaustedRestartRecoveryTarget>();
|
||||
const reconcileExhaustedTargets = async () => {
|
||||
const outcomes = await Promise.allSettled(
|
||||
[...exhaustedTargets.values()].map((target) =>
|
||||
runWithGatewayIndependentRootWorkAdmission(
|
||||
async () =>
|
||||
await recoverExpectedRestartRecoveryTarget({
|
||||
canonicalSessionKey: target.canonicalSessionKey,
|
||||
cfg: params.cfg,
|
||||
expectedSessionId: target.sessionId,
|
||||
observationOnly: true,
|
||||
sessionKey: target.sessionKey,
|
||||
storePath: target.storePath,
|
||||
gatewayRuntime: params.gatewayRuntime,
|
||||
}),
|
||||
),
|
||||
),
|
||||
);
|
||||
for (const outcome of outcomes) {
|
||||
if (outcome.status === "rejected") {
|
||||
log.warn(`main-session exhaustion reconciliation failed: ${String(outcome.reason)}`);
|
||||
}
|
||||
}
|
||||
};
|
||||
// Delayed retries outlive startup; each attempt must independently block
|
||||
// host suspension while it reads and rewrites recovery session state.
|
||||
void runWithGatewayIndependentRootWorkAdmission(
|
||||
async () =>
|
||||
await recoverStartupOrphanedMainSessions({
|
||||
await recoverStartupOrphanedMainSessionsWithOptions({
|
||||
cfg: params.cfg,
|
||||
onExhaustedTarget: (target) => {
|
||||
exhaustedTargets.set(`${target.storePath}\u0000${target.sessionKey}`, target);
|
||||
},
|
||||
stateDir: params.stateDir,
|
||||
resumedSessionKeys,
|
||||
updatedBeforeMs: startupRecoveryCutoffMs,
|
||||
gatewayRuntime: params.gatewayRuntime,
|
||||
}),
|
||||
)
|
||||
.then((result) => {
|
||||
.then(async (result) => {
|
||||
if (result.failed > 0 && attempt < maxRetries) {
|
||||
scheduleAttempt(attempt + 1, delay * RETRY_BACKOFF_MULTIPLIER);
|
||||
} else if (result.failed > 0 && attempt === maxRetries && exhaustedTargets.size > 0) {
|
||||
// Reconcile only exact rows whose final dispatch retained its durable charge.
|
||||
await reconcileExhaustedTargets();
|
||||
}
|
||||
})
|
||||
.catch((err: unknown) => {
|
||||
.catch(async (err: unknown) => {
|
||||
if (attempt < maxRetries) {
|
||||
log.warn(`main-session restart recovery failed: ${String(err)}`);
|
||||
scheduleAttempt(attempt + 1, delay * RETRY_BACKOFF_MULTIPLIER);
|
||||
} else {
|
||||
log.warn(`main-session restart recovery gave up: ${String(err)}`);
|
||||
await reconcileExhaustedTargets();
|
||||
}
|
||||
});
|
||||
};
|
||||
|
||||
@@ -678,7 +678,7 @@ describe("subagent-orphan-recovery", () => {
|
||||
const store = mockSingleAbortedSession({
|
||||
subagentRecovery: {
|
||||
automaticAttempts: 2,
|
||||
lastAttemptAt: now - 30_000,
|
||||
lastAttemptAt: now - 2 * 60_000,
|
||||
lastRunId: "previous-run",
|
||||
},
|
||||
});
|
||||
@@ -710,6 +710,31 @@ describe("subagent-orphan-recovery", () => {
|
||||
expect(recovery.wedgedReason).toContain("recovery blocked");
|
||||
});
|
||||
|
||||
it("starts a new attempt burst after the two-minute re-wedge window", async () => {
|
||||
const now = Date.now();
|
||||
const expiredRecovery = {
|
||||
automaticAttempts: 2,
|
||||
lastAttemptAt: now - 2 * 60_000 - 1,
|
||||
lastRunId: "previous-run",
|
||||
};
|
||||
const store = mockSingleAbortedSession({ subagentRecovery: expiredRecovery });
|
||||
|
||||
const result = await recoverOrphanedSubagentSessions({
|
||||
getActiveRuns: () => createActiveRuns(createTestRunRecord()),
|
||||
});
|
||||
|
||||
expect(result.recovered).toBe(1);
|
||||
expect(dispatchAgent).toHaveBeenCalledOnce();
|
||||
expect(sessionAccessor.patchSessionEntry).toHaveBeenCalledOnce();
|
||||
const sessionEntry = requireRecord(
|
||||
store["agent:main:subagent:test-session-1"],
|
||||
"updated session entry",
|
||||
);
|
||||
const recovery = requireRecord(sessionEntry.subagentRecovery, "subagent recovery");
|
||||
expect(recovery.automaticAttempts).toBe(1);
|
||||
expect(recovery.lastRunId).toBe("run-1");
|
||||
});
|
||||
|
||||
it("skips already tombstoned wedged sessions without rewriting them", async () => {
|
||||
mockSingleAbortedSession({
|
||||
subagentRecovery: {
|
||||
|
||||
@@ -8,7 +8,10 @@ import type { SessionEntry } from "../config/sessions.js";
|
||||
const SUBAGENT_RECOVERY_MAX_AUTOMATIC_ATTEMPTS = 2;
|
||||
const SUBAGENT_RECOVERY_REWEDGE_WINDOW_MS = 2 * 60_000;
|
||||
|
||||
/** Decision returned before attempting automatic subagent orphan recovery. */
|
||||
function normalizeRecoveryAttempts(value: unknown): number {
|
||||
return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.floor(value) : 0;
|
||||
}
|
||||
|
||||
type SubagentRecoveryGate =
|
||||
| {
|
||||
allowed: true;
|
||||
@@ -31,7 +34,6 @@ function isRecentRecoveryAttempt(entry: SessionEntry, now: number): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
/** Returns true when recovery has been tombstoned for a session entry. */
|
||||
export function isSubagentRecoveryWedgedEntry(entry: unknown): boolean {
|
||||
if (!entry || typeof entry !== "object") {
|
||||
return false;
|
||||
@@ -44,7 +46,6 @@ export function isSubagentRecoveryWedgedEntry(entry: unknown): boolean {
|
||||
);
|
||||
}
|
||||
|
||||
/** Formats the operator-facing reason for a wedged recovery entry. */
|
||||
export function formatSubagentRecoveryWedgedReason(entry: SessionEntry): string {
|
||||
return (
|
||||
entry.subagentRecovery?.wedgedReason?.trim() ||
|
||||
@@ -52,7 +53,6 @@ export function formatSubagentRecoveryWedgedReason(entry: SessionEntry): string
|
||||
);
|
||||
}
|
||||
|
||||
/** Checks whether automatic orphan recovery may run for this session entry. */
|
||||
export function evaluateSubagentRecoveryGate(
|
||||
entry: SessionEntry,
|
||||
now: number,
|
||||
@@ -66,7 +66,7 @@ export function evaluateSubagentRecoveryGate(
|
||||
}
|
||||
|
||||
const previousAttempts = isRecentRecoveryAttempt(entry, now)
|
||||
? Math.max(0, entry.subagentRecovery?.automaticAttempts ?? 0)
|
||||
? normalizeRecoveryAttempts(entry.subagentRecovery?.automaticAttempts)
|
||||
: 0;
|
||||
if (previousAttempts >= SUBAGENT_RECOVERY_MAX_AUTOMATIC_ATTEMPTS) {
|
||||
return {
|
||||
@@ -84,7 +84,6 @@ export function evaluateSubagentRecoveryGate(
|
||||
};
|
||||
}
|
||||
|
||||
/** Records one accepted automatic orphan-recovery attempt. */
|
||||
export function markSubagentRecoveryAttempt(params: {
|
||||
entry: SessionEntry;
|
||||
now: number;
|
||||
@@ -98,11 +97,10 @@ export function markSubagentRecoveryAttempt(params: {
|
||||
};
|
||||
}
|
||||
|
||||
/** Tombstones automatic recovery until maintenance or doctor clears the state. */
|
||||
export function markSubagentRecoveryWedged(params: {
|
||||
entry: SessionEntry;
|
||||
now: number;
|
||||
runId?: string;
|
||||
runId: string;
|
||||
reason: string;
|
||||
}): void {
|
||||
params.entry.abortedLastRun = false;
|
||||
@@ -113,14 +111,13 @@ export function markSubagentRecoveryWedged(params: {
|
||||
SUBAGENT_RECOVERY_MAX_AUTOMATIC_ATTEMPTS,
|
||||
),
|
||||
lastAttemptAt: params.entry.subagentRecovery?.lastAttemptAt ?? params.now,
|
||||
...(params.runId ? { lastRunId: params.runId } : {}),
|
||||
lastRunId: params.runId,
|
||||
wedgedAt: params.now,
|
||||
wedgedReason: params.reason,
|
||||
};
|
||||
params.entry.updatedAt = params.now;
|
||||
}
|
||||
|
||||
/** Clears stale abort state when a wedged entry should no longer look runnable. */
|
||||
export function clearWedgedSubagentRecoveryAbort(entry: SessionEntry, now: number): boolean {
|
||||
if (!isSubagentRecoveryWedgedEntry(entry) || entry.abortedLastRun !== true) {
|
||||
return false;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// Handles session reset requests produced during agent runner execution.
|
||||
import { transitionMainSessionRecovery } from "../../agents/main-session-recovery-state.js";
|
||||
import type { SessionEntry } from "../../config/sessions.js";
|
||||
import { resolveAgentIdFromSessionKey } from "../../config/sessions.js";
|
||||
import { persistSessionResetLifecycle } from "../../config/sessions/session-accessor.js";
|
||||
@@ -101,6 +102,7 @@ export async function resetReplyRunSession(params: {
|
||||
memoryFlushLastFailedAt: undefined,
|
||||
memoryFlushLastFailureError: undefined,
|
||||
};
|
||||
transitionMainSessionRecovery(nextEntry, { kind: "clear" });
|
||||
const agentId = resolveAgentIdFromSessionKey(params.sessionKey);
|
||||
const nextSessionFile = formatSqliteSessionFileMarker({
|
||||
agentId,
|
||||
|
||||
@@ -340,6 +340,7 @@ describe("getReplyFromConfig fast test bootstrap", () => {
|
||||
pendingFinalDeliveryCreatedAt: 1,
|
||||
pendingFinalDeliveryAttemptCount: 4,
|
||||
pendingFinalDeliveryLastError: null,
|
||||
pendingFinalDeliveryIntentId: "stale-heartbeat-intent",
|
||||
},
|
||||
});
|
||||
const cfg = withFastReplyConfig({
|
||||
@@ -361,6 +362,7 @@ describe("getReplyFromConfig fast test bootstrap", () => {
|
||||
expect(stored.pendingFinalDelivery).toBeUndefined();
|
||||
expect(stored.pendingFinalDeliveryText).toBeUndefined();
|
||||
expect(stored.pendingFinalDeliveryAttemptCount).toBeUndefined();
|
||||
expect(stored.pendingFinalDeliveryIntentId).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps non-ack heartbeat pending delivery without direct replay", async () => {
|
||||
|
||||
@@ -62,7 +62,10 @@ import {
|
||||
} from "./inbound-media.js";
|
||||
import { emitPreAgentMessageHooks } from "./message-preprocess-hooks.js";
|
||||
import { createFastTestModelSelectionState, createModelSelectionState } from "./model-selection.js";
|
||||
import { sanitizePendingFinalDeliveryText } from "./pending-final-delivery.js";
|
||||
import {
|
||||
PENDING_FINAL_DELIVERY_CLEAR_PATCH,
|
||||
sanitizePendingFinalDeliveryText,
|
||||
} from "./pending-final-delivery.js";
|
||||
import { attachProgressNarratorToReplyOptions } from "./progress-narrator.js";
|
||||
import { createReplyTimingTracker } from "./reply-timing-tracker.js";
|
||||
import { initSessionState, resolveReplySessionPreprocessingState } from "./session.js";
|
||||
@@ -553,13 +556,7 @@ export async function getReplyFromConfig(
|
||||
resolveHeartbeatAckMaxChars(cfg, agentId),
|
||||
);
|
||||
if (heartbeatPending.shouldClear) {
|
||||
sessionEntry.pendingFinalDelivery = undefined;
|
||||
sessionEntry.pendingFinalDeliveryText = undefined;
|
||||
sessionEntry.pendingFinalDeliveryCreatedAt = undefined;
|
||||
sessionEntry.pendingFinalDeliveryLastAttemptAt = undefined;
|
||||
sessionEntry.pendingFinalDeliveryAttemptCount = undefined;
|
||||
sessionEntry.pendingFinalDeliveryLastError = undefined;
|
||||
sessionEntry.pendingFinalDeliveryContext = undefined;
|
||||
Object.assign(sessionEntry, PENDING_FINAL_DELIVERY_CLEAR_PATCH);
|
||||
sessionEntryHandle.replaceCurrent(sessionEntry);
|
||||
if (sessionKey && sessionStore) {
|
||||
sessionStore[sessionKey] = sessionEntry;
|
||||
@@ -568,15 +565,7 @@ export async function getReplyFromConfig(
|
||||
const { updateSessionEntry } = await import("../../config/sessions/session-accessor.js");
|
||||
await updateSessionEntry(
|
||||
{ storePath, sessionKey },
|
||||
() => ({
|
||||
pendingFinalDelivery: undefined,
|
||||
pendingFinalDeliveryText: undefined,
|
||||
pendingFinalDeliveryCreatedAt: undefined,
|
||||
pendingFinalDeliveryLastAttemptAt: undefined,
|
||||
pendingFinalDeliveryAttemptCount: undefined,
|
||||
pendingFinalDeliveryLastError: undefined,
|
||||
pendingFinalDeliveryContext: undefined,
|
||||
}),
|
||||
() => ({ ...PENDING_FINAL_DELIVERY_CLEAR_PATCH }),
|
||||
{
|
||||
skipMaintenance: true,
|
||||
takeCacheOwnership: true,
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { SessionEntry } from "../../config/sessions/types.js";
|
||||
import { normalizeReplyPayloadsForDelivery } from "../../infra/outbound/payloads.js";
|
||||
import {
|
||||
isSilentReplyPayloadText,
|
||||
@@ -7,7 +8,6 @@ import {
|
||||
stripLeadingSilentToken,
|
||||
stripSilentToken,
|
||||
} from "../tokens.js";
|
||||
/** Sanitizes pending final delivery text before channel-visible output. */
|
||||
import type { ReplyPayload } from "../types.js";
|
||||
import { stripInternalMetadataForDisplay } from "./display-text-sanitize.js";
|
||||
import { normalizeReplyPayload } from "./normalize-reply.js";
|
||||
@@ -87,6 +87,19 @@ export function buildPendingFinalDeliveryText(payloads: ReplyPayload[]): string
|
||||
return sanitizePendingFinalDeliveryText(text);
|
||||
}
|
||||
|
||||
// A delivered or discarded final must lose the whole record. Keeping this list
|
||||
// centralized prevents new ownership fields from leaving a phantom pending delivery.
|
||||
export const PENDING_FINAL_DELIVERY_CLEAR_PATCH = {
|
||||
pendingFinalDelivery: undefined,
|
||||
pendingFinalDeliveryText: undefined,
|
||||
pendingFinalDeliveryCreatedAt: undefined,
|
||||
pendingFinalDeliveryLastAttemptAt: undefined,
|
||||
pendingFinalDeliveryAttemptCount: undefined,
|
||||
pendingFinalDeliveryLastError: undefined,
|
||||
pendingFinalDeliveryContext: undefined,
|
||||
pendingFinalDeliveryIntentId: undefined,
|
||||
} as const satisfies Partial<SessionEntry>;
|
||||
|
||||
function collectDurableMediaDirectives(payload: ReplyPayload): string[] {
|
||||
if (payload.sensitiveMedia === true) {
|
||||
return [];
|
||||
@@ -144,7 +157,7 @@ function hasUnrecoverableNormalizedDeliveryShape(payload: ReplyPayload): boolean
|
||||
);
|
||||
}
|
||||
|
||||
/** Sanitizes final pending-delivery text and removes silent control tokens. */
|
||||
/** Sanitizes pending final delivery text before channel-visible output. */
|
||||
export function sanitizePendingFinalDeliveryText(text: string): string {
|
||||
let stripped = stripInternalMetadataForDisplay(text).trim();
|
||||
if (isSilentReplyPayloadText(stripped, SILENT_REPLY_TOKEN)) {
|
||||
|
||||
@@ -4,10 +4,11 @@ import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../../test/helpers/temp-dir.js";
|
||||
import {
|
||||
deleteSessionEntryLifecycle,
|
||||
loadSessionEntry,
|
||||
replaceSessionEntry,
|
||||
replaceSessionEntrySync,
|
||||
} from "../../config/sessions/session-accessor.js";
|
||||
import type { SessionEntry } from "../../config/sessions/types.js";
|
||||
import type { InternalSessionEntry as SessionEntry } from "../../config/sessions/types.js";
|
||||
import {
|
||||
resetDiagnosticRunActivityForTest,
|
||||
RUN_STALE_TAKEOVER_MS,
|
||||
@@ -28,6 +29,15 @@ import {
|
||||
import { testing } from "./reply-run-registry.test-support.js";
|
||||
import { admitReplyTurn, runWithReplyOperationLifecycleAdmission } from "./reply-turn-admission.js";
|
||||
|
||||
const recoveryOwnerReleaseMocks = vi.hoisted(() => ({
|
||||
schedulePendingTarget: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../../agents/main-session-recovery-owner-release.js", async (importOriginal) => ({
|
||||
...(await importOriginal<typeof import("../../agents/main-session-recovery-owner-release.js")>()),
|
||||
scheduleMainSessionRecoveryPendingTarget: recoveryOwnerReleaseMocks.schedulePendingTarget,
|
||||
}));
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
function createDeferred() {
|
||||
@@ -49,10 +59,18 @@ function createSessionStore(entries: Record<string, object>): string {
|
||||
return storePath;
|
||||
}
|
||||
|
||||
async function readSessionEntry(
|
||||
storePath: string,
|
||||
sessionKey: string,
|
||||
): Promise<SessionEntry | undefined> {
|
||||
return loadSessionEntry({ sessionKey, storePath });
|
||||
}
|
||||
|
||||
describe("reply turn admission", () => {
|
||||
afterEach(() => {
|
||||
testing.resetReplyRunRegistry();
|
||||
resetDiagnosticRunActivityForTest();
|
||||
recoveryOwnerReleaseMocks.schedulePendingTarget.mockClear();
|
||||
});
|
||||
|
||||
it("rejects a reply when an archive commits before admission", async () => {
|
||||
@@ -324,6 +342,210 @@ describe("reply turn admission", () => {
|
||||
expect(mutationRan).toBe(true);
|
||||
});
|
||||
|
||||
it.each(["visible", "heartbeat", "queued_followup"] as const)(
|
||||
"fences restart recovery from %s reply admission until the operation clears",
|
||||
async (kind) => {
|
||||
const sessionKey = `agent:main:telegram:topic:recovery-race:${kind}`;
|
||||
const sessionId = "interrupted-session";
|
||||
const storePath = createSessionStore({
|
||||
[sessionKey]: {
|
||||
sessionId,
|
||||
updatedAt: 100,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 1,
|
||||
chargedAttempts: 2,
|
||||
},
|
||||
},
|
||||
});
|
||||
const admission = await admitReplyTurn({
|
||||
sessionKey,
|
||||
sessionId,
|
||||
expectedSessionId: sessionId,
|
||||
storePath,
|
||||
kind,
|
||||
resetTriggered: false,
|
||||
});
|
||||
expect(admission.status).toBe("owned");
|
||||
if (admission.status !== "owned") {
|
||||
return;
|
||||
}
|
||||
|
||||
const claimedEntry = await readSessionEntry(storePath, sessionKey);
|
||||
admission.operation.complete();
|
||||
await vi.waitFor(async () => {
|
||||
const entry = await readSessionEntry(storePath, sessionKey);
|
||||
expect(entry?.mainRestartRecovery?.foregroundClaims).toBeUndefined();
|
||||
});
|
||||
|
||||
expect(claimedEntry?.mainRestartRecovery).toMatchObject({
|
||||
foregroundClaims: {
|
||||
tokens: [expect.any(String)],
|
||||
},
|
||||
});
|
||||
await expect(readSessionEntry(storePath, sessionKey)).resolves.toMatchObject({
|
||||
sessionId,
|
||||
status: "running",
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["visible", "heartbeat"] as const)(
|
||||
"rejects %s reply admission for a tombstoned recovery session",
|
||||
async (kind) => {
|
||||
const sessionKey = `agent:main:telegram:topic:recovery-tombstone:${kind}`;
|
||||
const sessionId = "tombstoned-session";
|
||||
const storePath = createSessionStore({
|
||||
[sessionKey]: {
|
||||
sessionId,
|
||||
updatedAt: 100,
|
||||
status: "failed",
|
||||
abortedLastRun: false,
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 4,
|
||||
chargedAttempts: 3,
|
||||
tombstone: { reason: "automatic recovery exhausted" },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await expect(
|
||||
admitReplyTurn({
|
||||
sessionKey,
|
||||
sessionId,
|
||||
expectedSessionId: sessionId,
|
||||
storePath,
|
||||
kind,
|
||||
resetTriggered: false,
|
||||
}),
|
||||
).rejects.toThrow(/changed while starting work/i);
|
||||
},
|
||||
);
|
||||
|
||||
it("drops a queued followup for an admitted recovery fence", async () => {
|
||||
const sessionKey = "agent:main:telegram:topic:admitted-recovery";
|
||||
const sessionId = "admitted-recovery-session";
|
||||
const storePath = createSessionStore({
|
||||
[sessionKey]: {
|
||||
sessionId,
|
||||
updatedAt: 100,
|
||||
status: "running",
|
||||
abortedLastRun: false,
|
||||
restartRecoveryRuns: [{ runId: "recovery-run", lifecycleGeneration: "generation-1" }],
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 3,
|
||||
chargedAttempts: 1,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await expect(
|
||||
admitReplyTurn({
|
||||
sessionKey,
|
||||
sessionId,
|
||||
expectedSessionId: sessionId,
|
||||
storePath,
|
||||
kind: "queued_followup",
|
||||
resetTriggered: false,
|
||||
}),
|
||||
).resolves.toEqual({ status: "skipped", reason: "lifecycle-invalidated" });
|
||||
});
|
||||
|
||||
it("schedules released recovery only after retained admission exits", async () => {
|
||||
const sourceSessionKey = "agent:main:telegram:slash:recovery-adoption";
|
||||
const sessionKey = "agent:main:telegram:topic:recovery-adoption";
|
||||
const sessionId = "interrupted-session";
|
||||
const storePath = createSessionStore({
|
||||
[sessionKey]: {
|
||||
sessionId,
|
||||
updatedAt: 100,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 1,
|
||||
chargedAttempts: 0,
|
||||
},
|
||||
},
|
||||
});
|
||||
const blocker = createReplyOperation({
|
||||
sessionKey,
|
||||
sessionId,
|
||||
resetTriggered: false,
|
||||
});
|
||||
const reservation = createReplyOperation({
|
||||
sessionKey: sourceSessionKey,
|
||||
sessionId: "source-session",
|
||||
resetTriggered: false,
|
||||
});
|
||||
|
||||
const result = await admitReplyTurn({
|
||||
sessionKey,
|
||||
sessionId: reservation.sessionId,
|
||||
expectedSessionId: sessionId,
|
||||
storePath,
|
||||
kind: "visible",
|
||||
resetTriggered: false,
|
||||
waitForActive: false,
|
||||
retainLifecycleAdmissionOnActive: true,
|
||||
adoptOperation: reservation,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ status: "skipped", reason: "active-run" });
|
||||
expect(recoveryOwnerReleaseMocks.schedulePendingTarget).not.toHaveBeenCalled();
|
||||
await expect(readSessionEntry(storePath, sessionKey)).resolves.not.toHaveProperty(
|
||||
"mainRestartRecovery.foregroundClaims",
|
||||
);
|
||||
if (result.status === "skipped") {
|
||||
result.lifecycleAdmission?.release();
|
||||
}
|
||||
await vi.waitFor(() => {
|
||||
expect(recoveryOwnerReleaseMocks.schedulePendingTarget).toHaveBeenCalledWith({
|
||||
sessionId,
|
||||
sessionKey,
|
||||
storePath,
|
||||
});
|
||||
});
|
||||
|
||||
blocker.complete();
|
||||
reservation.complete();
|
||||
});
|
||||
|
||||
it("leaves interrupted subagent sessions to the subagent recovery owner", async () => {
|
||||
const sessionKey = "agent:main:subagent:child-1";
|
||||
const sessionId = "subagent-session";
|
||||
const storePath = createSessionStore({
|
||||
[sessionKey]: {
|
||||
sessionId,
|
||||
updatedAt: 100,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
spawnDepth: 1,
|
||||
},
|
||||
});
|
||||
|
||||
const admission = await admitReplyTurn({
|
||||
sessionKey,
|
||||
sessionId,
|
||||
expectedSessionId: sessionId,
|
||||
storePath,
|
||||
kind: "visible",
|
||||
resetTriggered: false,
|
||||
});
|
||||
|
||||
expect(admission.status).toBe("owned");
|
||||
if (admission.status === "owned") {
|
||||
admission.operation.complete();
|
||||
}
|
||||
await expect(readSessionEntry(storePath, sessionKey)).resolves.not.toHaveProperty(
|
||||
"mainRestartRecovery",
|
||||
);
|
||||
});
|
||||
|
||||
it("holds interrupted queued reply work until its owner exits", async () => {
|
||||
const sessionKey = "agent:main:telegram:topic:queued-delete";
|
||||
const sessionId = "session-before-delete";
|
||||
|
||||
@@ -1,11 +1,22 @@
|
||||
import { scheduleMainSessionRecoveryPendingTarget } from "../../agents/main-session-recovery-owner-release.js";
|
||||
import { isMainRestartRecoveryCandidate } from "../../agents/main-session-recovery-state.js";
|
||||
import {
|
||||
claimMainSessionRecoveryOwner,
|
||||
releaseMainSessionRecoveryOwner,
|
||||
type MainSessionRecoveryPendingTarget,
|
||||
type MainSessionRecoveryOwnerLease,
|
||||
} from "../../agents/main-session-recovery-store.js";
|
||||
// Decides whether an inbound turn may start, queue, or abort a reply run.
|
||||
import { resolveSessionWorkStartError } from "../../config/sessions/lifecycle.js";
|
||||
import { loadSessionEntry } from "../../config/sessions/session-accessor.js";
|
||||
import type { SessionEntry } from "../../config/sessions/types.js";
|
||||
import type { InternalSessionEntry, SessionEntry } from "../../config/sessions/types.js";
|
||||
import { getAgentEventLifecycleGeneration } from "../../infra/agent-events.js";
|
||||
import { formatErrorMessage } from "../../infra/errors.js";
|
||||
import {
|
||||
getDiagnosticSessionActivitySnapshot,
|
||||
resolveRunStaleThresholdMs,
|
||||
} from "../../logging/diagnostic-run-activity.js";
|
||||
import { createSubsystemLogger } from "../../logging/subsystem.js";
|
||||
import {
|
||||
beginSessionWorkAdmission,
|
||||
type SessionWorkAdmissionLease,
|
||||
@@ -40,14 +51,26 @@ type ReplyTurnAdmission =
|
||||
|
||||
class QueuedFollowupLifecycleInvalidatedError extends Error {}
|
||||
|
||||
const log = createSubsystemLogger("auto-reply/reply-turn-admission");
|
||||
const lifecycleAdmissionByOperation = new WeakMap<ReplyOperation, SessionWorkAdmissionLease>();
|
||||
|
||||
async function releaseReplyRecoveryOwner(
|
||||
lease: MainSessionRecoveryOwnerLease | undefined,
|
||||
): Promise<MainSessionRecoveryPendingTarget | undefined> {
|
||||
try {
|
||||
return await releaseMainSessionRecoveryOwner(lease);
|
||||
} catch (error) {
|
||||
log.warn(`failed to release main-session recovery reply owner: ${formatErrorMessage(error)}`);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** Runs owner work with its admission marked as the initiating lifecycle context. */
|
||||
export async function runWithReplyOperationLifecycleAdmission<T>(
|
||||
operation: ReplyOperation | undefined,
|
||||
operation: ReplyOperation,
|
||||
run: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
const admission = operation ? lifecycleAdmissionByOperation.get(operation) : undefined;
|
||||
const admission = lifecycleAdmissionByOperation.get(operation);
|
||||
return admission ? await admission.run(run) : await run();
|
||||
}
|
||||
|
||||
@@ -154,7 +177,8 @@ async function admitReplyTurnWithWaitSignal(
|
||||
try {
|
||||
const storePath = params.storePath;
|
||||
let operation: ReplyOperation | undefined;
|
||||
let admittedSessionEntry: SessionEntry | undefined;
|
||||
let admittedSessionEntry: InternalSessionEntry | undefined;
|
||||
let recoveryOwnerLease: MainSessionRecoveryOwnerLease | undefined;
|
||||
let interruptedBeforeOperation = false;
|
||||
const admission = storePath
|
||||
? await beginSessionWorkAdmission({
|
||||
@@ -172,7 +196,7 @@ async function admitReplyTurnWithWaitSignal(
|
||||
sessionKey: params.sessionKey,
|
||||
readConsistency: "latest",
|
||||
});
|
||||
admittedSessionEntry = currentEntry;
|
||||
admittedSessionEntry = currentEntry as InternalSessionEntry | undefined;
|
||||
if (expectedSessionId && !currentEntry) {
|
||||
rejectLifecycleInvalidatedWork({
|
||||
kind: params.kind,
|
||||
@@ -229,14 +253,37 @@ async function admitReplyTurnWithWaitSignal(
|
||||
},
|
||||
})
|
||||
: undefined;
|
||||
if (interruptedBeforeOperation) {
|
||||
admission?.release();
|
||||
rejectLifecycleInvalidatedWork({
|
||||
kind: params.kind,
|
||||
message: `Session "${params.sessionKey}" changed while starting work. Retry.`,
|
||||
});
|
||||
}
|
||||
try {
|
||||
if (
|
||||
storePath &&
|
||||
!params.resetTriggered &&
|
||||
admittedSessionEntry &&
|
||||
((admittedSessionEntry.status === "running" &&
|
||||
(admittedSessionEntry.abortedLastRun === true ||
|
||||
admittedSessionEntry.restartRecoveryRuns !== undefined ||
|
||||
admittedSessionEntry.mainRestartRecovery !== undefined)) ||
|
||||
admittedSessionEntry.mainRestartRecovery?.tombstone !== undefined) &&
|
||||
isMainRestartRecoveryCandidate(admittedSessionEntry, params.sessionKey)
|
||||
) {
|
||||
const ownerClaim = await claimMainSessionRecoveryOwner({
|
||||
lifecycleGeneration: getAgentEventLifecycleGeneration(),
|
||||
sessionId,
|
||||
target: { sessionKey: params.sessionKey, storePath },
|
||||
});
|
||||
if (ownerClaim.kind === "invalidated") {
|
||||
rejectLifecycleInvalidatedWork({
|
||||
kind: params.kind,
|
||||
message: `Session "${params.sessionKey}" changed while starting work. Retry.`,
|
||||
});
|
||||
}
|
||||
recoveryOwnerLease = ownerClaim.kind === "claimed" ? ownerClaim.lease : undefined;
|
||||
}
|
||||
if (interruptedBeforeOperation || isAbortSignalAborted(params.upstreamAbortSignal)) {
|
||||
rejectLifecycleInvalidatedWork({
|
||||
kind: params.kind,
|
||||
message: `Session "${params.sessionKey}" changed while starting work. Retry.`,
|
||||
});
|
||||
}
|
||||
if (params.adoptOperation) {
|
||||
// The dispatch closures own this object's abort/delivery lifecycle,
|
||||
// so the reservation must move rather than be recreated. Throws
|
||||
@@ -255,11 +302,17 @@ async function admitReplyTurnWithWaitSignal(
|
||||
});
|
||||
}
|
||||
} catch (error) {
|
||||
const pendingRecovery = recoveryOwnerLease
|
||||
? await releaseReplyRecoveryOwner(recoveryOwnerLease)
|
||||
: undefined;
|
||||
if (
|
||||
error instanceof ReplyRunAlreadyActiveError &&
|
||||
admission &&
|
||||
params.retainLifecycleAdmissionOnActive
|
||||
) {
|
||||
void admission.released.then(() => {
|
||||
scheduleMainSessionRecoveryPendingTarget(pendingRecovery);
|
||||
});
|
||||
return {
|
||||
status: "skipped",
|
||||
reason: "active-run",
|
||||
@@ -268,6 +321,7 @@ async function admitReplyTurnWithWaitSignal(
|
||||
};
|
||||
}
|
||||
admission?.release();
|
||||
scheduleMainSessionRecoveryPendingTarget(pendingRecovery);
|
||||
throw error;
|
||||
}
|
||||
if (admission) {
|
||||
@@ -281,7 +335,11 @@ async function admitReplyTurnWithWaitSignal(
|
||||
lifecycleAdmissionByOperation.set(operation, admission);
|
||||
runAfterReplyOperationClear(operation, () => {
|
||||
lifecycleAdmissionByOperation.delete(operation);
|
||||
admission.release();
|
||||
// Keep reset/delete behind durable owner release and its writer lock.
|
||||
void releaseReplyRecoveryOwner(recoveryOwnerLease).then((pendingTarget) => {
|
||||
admission.release();
|
||||
scheduleMainSessionRecoveryPendingTarget(pendingTarget);
|
||||
});
|
||||
});
|
||||
}
|
||||
return {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// Prepares parent-context fork metadata for guarded reply session initialization.
|
||||
import { buildMainSessionRecoveryClearPatch } from "../../agents/main-session-recovery-clear.js";
|
||||
import type { SessionEntry } from "../../config/sessions.js";
|
||||
import { forkSessionFromParent, resolveParentForkDecision } from "./session-fork.js";
|
||||
|
||||
@@ -52,8 +53,11 @@ export async function prepareReplySessionParentFork(params: {
|
||||
`parentTokens=${decision.parentTokens ?? "unknown"}`,
|
||||
);
|
||||
params.warn(`forked session created: file=${fork.sessionFile}`);
|
||||
// The fork replaces this thread's transcript identity; recovery state from
|
||||
// the preseed row must not govern a later interruption of the fork.
|
||||
return {
|
||||
...params.sessionEntry,
|
||||
...buildMainSessionRecoveryClearPatch(params.sessionEntry),
|
||||
sessionId: fork.sessionId,
|
||||
sessionFile: fork.sessionFile,
|
||||
forkedFromParent: true,
|
||||
|
||||
@@ -8,7 +8,7 @@ import { testing as sessionMcpTesting } from "../../agents/agent-bundle-mcp-runt
|
||||
import { getOrCreateSessionMcpRuntime } from "../../agents/agent-bundle-mcp-tools.js";
|
||||
import * as bootstrapCache from "../../agents/bootstrap-cache.js";
|
||||
import type { OpenClawConfig } from "../../config/config.js";
|
||||
import type { SessionEntry } from "../../config/sessions.js";
|
||||
import type { InternalSessionEntry as SessionEntry } from "../../config/sessions.js";
|
||||
import {
|
||||
appendTranscriptEvent,
|
||||
listSessionEntries,
|
||||
@@ -784,6 +784,14 @@ describe("initSessionState thread forking", () => {
|
||||
updatedAt: Date.now(),
|
||||
totalTokens: 0,
|
||||
totalTokensFresh: true,
|
||||
abortedLastRun: true,
|
||||
restartRecoveryRuns: [{ runId: "old-run", lifecycleGeneration: "old-generation" }],
|
||||
mainRestartRecovery: {
|
||||
cycleId: "old-cycle",
|
||||
revision: 4,
|
||||
chargedAttempts: 3,
|
||||
tombstone: { reason: "old transcript exhausted" },
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
@@ -805,6 +813,9 @@ describe("initSessionState thread forking", () => {
|
||||
expect(first.sessionEntry.forkedFromParent).toBe(true);
|
||||
expect(first.sessionEntry.totalTokens).toBeUndefined();
|
||||
expect(first.sessionEntry.totalTokensFresh).toBe(false);
|
||||
expect(first.sessionEntry.abortedLastRun).toBe(false);
|
||||
expect(first.sessionEntry.restartRecoveryRuns).toBeUndefined();
|
||||
expect((first.sessionEntry as SessionEntry).mainRestartRecovery).toBeUndefined();
|
||||
|
||||
const second = await initSessionState({
|
||||
ctx: {
|
||||
@@ -820,6 +831,8 @@ describe("initSessionState thread forking", () => {
|
||||
expect(second.sessionEntry.forkedFromParent).toBe(true);
|
||||
expect(second.sessionEntry.totalTokens).toBeUndefined();
|
||||
expect(second.sessionEntry.totalTokensFresh).toBe(false);
|
||||
expect(second.sessionEntry.restartRecoveryRuns).toBeUndefined();
|
||||
expect((second.sessionEntry as SessionEntry).mainRestartRecovery).toBeUndefined();
|
||||
warn.mockRestore();
|
||||
});
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ import {
|
||||
} from "../config/sessions/session-accessor.js";
|
||||
import { parseSqliteSessionFileMarker } from "../config/sessions/sqlite-marker.js";
|
||||
import { clearSessionStoreCacheForTest } from "../config/sessions/store.js";
|
||||
import type { SessionEntry } from "../config/sessions/types.js";
|
||||
import type { InternalSessionEntry as SessionEntry } from "../config/sessions/types.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { emitAgentEvent, onAgentEvent, resetAgentEventsForTest } from "../infra/agent-events.js";
|
||||
import type { PluginProviderRegistration } from "../plugins/registry.test-fixtures.js";
|
||||
@@ -1100,6 +1100,7 @@ describe("agentCommand", () => {
|
||||
|
||||
expect(prepared.sessionStore).not.toBe(cached);
|
||||
expect(prepared.sessionEntry).not.toBe(cached);
|
||||
expect(prepared).not.toHaveProperty("recoveryCandidateEntry");
|
||||
expect(prepared.sessionStore?.[sessionKey]).toBe(prepared.sessionEntry);
|
||||
expect(prepared.sessionStore?.["agent:main:other"]).toBeUndefined();
|
||||
});
|
||||
|
||||
@@ -0,0 +1,83 @@
|
||||
import { uniqueStrings } from "@openclaw/normalization-core/string-normalization";
|
||||
import { inspectMainSessionRecoveryHealth } from "../agents/main-session-recovery-lifecycle.js";
|
||||
import { transitionMainSessionRecovery } from "../agents/main-session-recovery-state.js";
|
||||
import {
|
||||
applySessionEntryReplacements,
|
||||
listSessionEntries,
|
||||
} from "../config/sessions/session-accessor.js";
|
||||
|
||||
type MainSessionRecoveryDoctorParams = {
|
||||
agentId: string;
|
||||
storePath: string;
|
||||
warnings: string[];
|
||||
changes: string[];
|
||||
confirmRepair: (params: { message: string; initialValue?: boolean }) => Promise<boolean>;
|
||||
countLabel: (count: number, singular: string, plural?: string) => string;
|
||||
};
|
||||
|
||||
export async function noteMainSessionRecoveryIntegrity(
|
||||
params: MainSessionRecoveryDoctorParams,
|
||||
): Promise<number> {
|
||||
const entries = listSessionEntries({ agentId: params.agentId, storePath: params.storePath });
|
||||
const wedged = entries.flatMap(({ entry, sessionKey }) => {
|
||||
const health = inspectMainSessionRecoveryHealth(entry);
|
||||
return health.status === "tombstoned" ? [{ key: sessionKey, health }] : [];
|
||||
});
|
||||
if (wedged.length === 0) {
|
||||
return entries.length;
|
||||
}
|
||||
|
||||
const wedgedCount = params.countLabel(wedged.length, "wedged main session");
|
||||
params.warnings.push(
|
||||
[
|
||||
`- Found ${wedgedCount} with automatic restart recovery tombstoned.`,
|
||||
" OpenClaw will not auto-resume these sessions again; inspect the failed turn, then use /new or reset to replace the session.",
|
||||
` Examples: ${wedged
|
||||
.slice(0, 3)
|
||||
.map(({ key }) => key)
|
||||
.join(", ")}`,
|
||||
].join("\n"),
|
||||
);
|
||||
|
||||
const visibleReasons = uniqueStrings(wedged.map(({ health }) => health.reason)).slice(0, 2);
|
||||
if (visibleReasons.length > 0) {
|
||||
params.warnings.push(visibleReasons.map((reason) => ` Reason: ${reason}`).join("\n"));
|
||||
}
|
||||
|
||||
const staleAborted = wedged.filter(({ health }) => health.repair === "clear_stale_abort");
|
||||
if (staleAborted.length === 0) {
|
||||
return entries.length;
|
||||
}
|
||||
const staleCount = params.countLabel(staleAborted.length, "wedged main session");
|
||||
if (
|
||||
!(await params.confirmRepair({
|
||||
message: `Clear stale aborted recovery flags for ${staleCount}?`,
|
||||
initialValue: true,
|
||||
}))
|
||||
) {
|
||||
return entries.length;
|
||||
}
|
||||
|
||||
const repairedAt = Date.now();
|
||||
// Revalidate under the writer lock because session state can change while Doctor prompts.
|
||||
const repaired = await applySessionEntryReplacements<number>({
|
||||
sessionKeys: staleAborted.map(({ key }) => key),
|
||||
storePath: params.storePath,
|
||||
update: (currentEntries) => {
|
||||
const replacements = currentEntries.flatMap(({ sessionKey, entry }) => {
|
||||
const transition = transitionMainSessionRecovery(entry, {
|
||||
kind: "doctor_repair",
|
||||
now: repairedAt,
|
||||
});
|
||||
return transition.kind === "doctor_repaired" ? [{ sessionKey, entry }] : [];
|
||||
});
|
||||
return { replacements, result: replacements.length };
|
||||
},
|
||||
});
|
||||
if (repaired > 0) {
|
||||
params.changes.push(
|
||||
`- Cleared aborted restart-recovery flags for ${params.countLabel(repaired, "wedged main session")}.`,
|
||||
);
|
||||
}
|
||||
return entries.length;
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
import path from "node:path";
|
||||
|
||||
export function countLabel(count: number, singular: string, plural = `${singular}s`): string {
|
||||
return `${count} ${count === 1 ? singular : plural}`;
|
||||
}
|
||||
|
||||
export function formatFilePreview(paths: string[], limit = 3): string {
|
||||
const names = paths.slice(0, limit).map((filePath) => path.basename(filePath));
|
||||
const remaining = paths.length - names.length;
|
||||
return remaining > 0 ? `${names.join(", ")}, and ${remaining} more` : names.join(", ");
|
||||
}
|
||||
@@ -39,7 +39,9 @@ import { parseAgentSessionKey } from "../sessions/session-key-utils.js";
|
||||
import { shortenHomePath } from "../utils.js";
|
||||
import { repairHeartbeatPoisonedMainSession } from "./doctor-heartbeat-main-session-repair.js";
|
||||
import { describeHeartbeatSessionTargetIssues } from "./doctor-heartbeat-session-target.js";
|
||||
import { noteMainSessionRecoveryIntegrity } from "./doctor-main-session-recovery.js";
|
||||
import { runPluginSessionStateDoctorRepairs } from "./doctor-session-state-providers.js";
|
||||
import { countLabel, formatFilePreview } from "./doctor-state-integrity-format.js";
|
||||
|
||||
const STATE_INTEGRITY_CHECK_ID = "core/doctor/state-integrity";
|
||||
|
||||
@@ -52,19 +54,6 @@ type DoctorPrompterLike = {
|
||||
note?: typeof note;
|
||||
};
|
||||
|
||||
function countLabel(count: number, singular: string, plural = `${singular}s`): string {
|
||||
return `${count} ${count === 1 ? singular : plural}`;
|
||||
}
|
||||
|
||||
function formatFilePreview(paths: string[], limit = 3): string {
|
||||
const names = paths.slice(0, limit).map((filePath) => path.basename(filePath));
|
||||
const remaining = paths.length - names.length;
|
||||
if (remaining > 0) {
|
||||
return `${names.join(", ")}, and ${remaining} more`;
|
||||
}
|
||||
return names.join(", ");
|
||||
}
|
||||
|
||||
function existsDir(dir: string): boolean {
|
||||
try {
|
||||
return fs.existsSync(dir) && fs.statSync(dir).isDirectory();
|
||||
@@ -1276,7 +1265,15 @@ export async function noteStateIntegrity(
|
||||
const store = loadSessionStore(storePath, { skipCache: true, clone: false });
|
||||
const sessionPathOpts = resolveSessionFilePathOptions({ agentId, storePath });
|
||||
const entries = Object.entries(store).filter(([, entry]) => entry && typeof entry === "object");
|
||||
if (entries.length > 0) {
|
||||
const canonicalEntryCount = await noteMainSessionRecoveryIntegrity({
|
||||
agentId,
|
||||
storePath: absoluteStorePath,
|
||||
warnings,
|
||||
changes,
|
||||
confirmRepair: (params) => prompter.confirmRuntimeRepair(params),
|
||||
countLabel,
|
||||
});
|
||||
if (entries.length > 0 || canonicalEntryCount > 0) {
|
||||
const recent = entries
|
||||
.slice()
|
||||
.toSorted((a, b) => {
|
||||
|
||||
@@ -150,17 +150,19 @@ export function createExpiringMapCache<TKey, TValue>(options: {
|
||||
}
|
||||
|
||||
type FileStatSnapshot = {
|
||||
mtimeMs: number;
|
||||
ctimeNs: bigint;
|
||||
mtimeNs: bigint;
|
||||
sizeBytes: number;
|
||||
};
|
||||
|
||||
/** Captures the file attributes used by cache invalidation without exposing fs.Stats. */
|
||||
export function getFileStatSnapshot(filePath: string): FileStatSnapshot | undefined {
|
||||
try {
|
||||
const stats = fs.statSync(filePath);
|
||||
const stats = fs.statSync(filePath, { bigint: true });
|
||||
return {
|
||||
mtimeMs: stats.mtimeMs,
|
||||
sizeBytes: stats.size,
|
||||
ctimeNs: stats.ctimeNs,
|
||||
mtimeNs: stats.mtimeNs,
|
||||
sizeBytes: Number(stats.size),
|
||||
};
|
||||
} catch {
|
||||
return undefined;
|
||||
|
||||
@@ -6,6 +6,7 @@ export * from "./sessions/goals.js";
|
||||
export * from "./sessions/artifacts.js";
|
||||
export * from "./sessions/metadata.js";
|
||||
export * from "./sessions/main-session.js";
|
||||
export type { MainRestartRecoveryState } from "./sessions/main-session-recovery.types.js";
|
||||
export * from "./sessions/main-session.runtime.js";
|
||||
export * from "./sessions/lifecycle.js";
|
||||
export * from "./sessions/paths.js";
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
export type MainRestartRecoveryState = {
|
||||
/** Stable identity for one interrupted episode; prevents clear-and-rewedge ABA matches. */
|
||||
cycleId: string;
|
||||
/** Monotonic identity for observations within the current recovery cycle. */
|
||||
revision: number;
|
||||
/** Attempts charged when their reservation is persisted, before dispatch. */
|
||||
chargedAttempts: number;
|
||||
reservation?: {
|
||||
runId: string;
|
||||
attempt: number;
|
||||
lifecycleGeneration: string;
|
||||
};
|
||||
foregroundClaims?: {
|
||||
lifecycleGeneration: string;
|
||||
tokens: string[];
|
||||
/** Run identity for claims that have crossed the actual agent-run boundary. */
|
||||
runIdsByClaimId?: Record<string, string>;
|
||||
};
|
||||
tombstone?: { reason: string };
|
||||
};
|
||||
@@ -470,15 +470,17 @@ export function buildRestartRecoveryClaimCleanupPatch(params: {
|
||||
entry: SessionEntry;
|
||||
recordTerminalSource: boolean;
|
||||
terminalDeliveryEvidence?: RestartRecoveryTerminalDeliveryEvidenceResult;
|
||||
terminalRunId?: string;
|
||||
terminalSourceRunId?: string;
|
||||
}): Partial<SessionEntry> {
|
||||
const sourceRunId =
|
||||
normalizeRunId(params.terminalSourceRunId) ??
|
||||
normalizeRunId(params.entry.restartRecoveryDeliverySourceRunId);
|
||||
const terminalRunIds =
|
||||
params.recordTerminalSource && sourceRunId
|
||||
params.recordTerminalSource && (sourceRunId || params.terminalRunId)
|
||||
? mergeRestartRecoveryTerminalRunIds(params.entry.restartRecoveryTerminalRunIds, [
|
||||
sourceRunId,
|
||||
...(sourceRunId ? [sourceRunId] : []),
|
||||
...(params.terminalRunId ? [params.terminalRunId] : []),
|
||||
])
|
||||
: undefined;
|
||||
const terminalDeliveryEvidence =
|
||||
|
||||
@@ -4,7 +4,7 @@ import {
|
||||
projectSessionSnapshotChanges,
|
||||
sessionSnapshotTouchedFieldsConflict,
|
||||
} from "./session-snapshot-merge.js";
|
||||
import { mergeSessionEntry, type SessionEntry } from "./types.js";
|
||||
import { mergeSessionEntry, type InternalSessionEntry as SessionEntry } from "./types.js";
|
||||
|
||||
const initial: SessionEntry = {
|
||||
sessionId: "session-1",
|
||||
@@ -13,7 +13,7 @@ const initial: SessionEntry = {
|
||||
model: "claude-opus-4-6",
|
||||
};
|
||||
|
||||
describe("session snapshot model merge", () => {
|
||||
describe("session snapshot merge", () => {
|
||||
it("projects same-provider model changes as an atomic pair", () => {
|
||||
const next = { ...initial, model: "claude-sonnet-4-6", updatedAt: 2 };
|
||||
const patch = projectSessionSnapshotChanges({ initial, next, current: initial });
|
||||
@@ -340,4 +340,274 @@ describe("session snapshot model merge", () => {
|
||||
|
||||
expect(mergeSessionSnapshotChanges({ initial, next, current })).toEqual(current);
|
||||
});
|
||||
|
||||
it("preserves the recovery aggregate when a restart marker wins a stale healthy clear", () => {
|
||||
const initialRecovery: SessionEntry = {
|
||||
...initial,
|
||||
abortedLastRun: true,
|
||||
restartRecoveryRuns: [{ runId: "run-1", lifecycleGeneration: "generation-1" }],
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 2,
|
||||
chargedAttempts: 0,
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration: "generation-1",
|
||||
tokens: ["owner-1"],
|
||||
},
|
||||
},
|
||||
};
|
||||
const next: SessionEntry = {
|
||||
...initialRecovery,
|
||||
updatedAt: 2,
|
||||
abortedLastRun: false,
|
||||
restartRecoveryRuns: undefined,
|
||||
mainRestartRecovery: undefined,
|
||||
};
|
||||
const current: SessionEntry = {
|
||||
...initialRecovery,
|
||||
updatedAt: 3,
|
||||
restartRecoveryRuns: [
|
||||
...(initialRecovery.restartRecoveryRuns ?? []),
|
||||
{ runId: "run-1", lifecycleGeneration: "generation-2" },
|
||||
],
|
||||
};
|
||||
|
||||
const merged = mergeSessionSnapshotChanges({ initial: initialRecovery, next, current });
|
||||
|
||||
expect(merged.abortedLastRun).toBe(true);
|
||||
expect(merged.restartRecoveryRuns).toEqual(current.restartRecoveryRuns);
|
||||
expect(merged.mainRestartRecovery).toEqual(current.mainRestartRecovery);
|
||||
});
|
||||
|
||||
it("marks a claimed recovery healthy without erasing its owner aggregate", () => {
|
||||
const initialRecovery: SessionEntry = {
|
||||
...initial,
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 1,
|
||||
chargedAttempts: 1,
|
||||
},
|
||||
};
|
||||
const next: SessionEntry = {
|
||||
...initialRecovery,
|
||||
updatedAt: 2,
|
||||
abortedLastRun: false,
|
||||
mainRestartRecovery: undefined,
|
||||
};
|
||||
const current: SessionEntry = {
|
||||
...initialRecovery,
|
||||
updatedAt: 3,
|
||||
mainRestartRecovery: {
|
||||
...initialRecovery.mainRestartRecovery!,
|
||||
revision: 2,
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration: "generation-1",
|
||||
tokens: ["owner-1"],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const merged = mergeSessionSnapshotChanges({ initial: initialRecovery, next, current });
|
||||
|
||||
expect(merged.abortedLastRun).toBe(false);
|
||||
expect(merged.restartRecoveryRuns).toBeUndefined();
|
||||
expect(merged.mainRestartRecovery).toEqual(current.mainRestartRecovery);
|
||||
});
|
||||
|
||||
it("clears recovery after lifecycle settlement consumes its run fence", () => {
|
||||
const initialRecovery: SessionEntry = {
|
||||
...initial,
|
||||
abortedLastRun: true,
|
||||
restartRecoveryRuns: [
|
||||
{ runId: "interrupted-run", lifecycleGeneration: "generation-1" },
|
||||
{ runId: "recovery-run", lifecycleGeneration: "generation-1" },
|
||||
],
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 3,
|
||||
chargedAttempts: 1,
|
||||
},
|
||||
};
|
||||
const next: SessionEntry = {
|
||||
...initialRecovery,
|
||||
updatedAt: 2,
|
||||
abortedLastRun: false,
|
||||
restartRecoveryRuns: undefined,
|
||||
mainRestartRecovery: undefined,
|
||||
};
|
||||
const current: SessionEntry = {
|
||||
...structuredClone(initialRecovery),
|
||||
updatedAt: 3,
|
||||
abortedLastRun: false,
|
||||
restartRecoveryRuns: [{ runId: "interrupted-run", lifecycleGeneration: "generation-1" }],
|
||||
};
|
||||
|
||||
const merged = mergeSessionSnapshotChanges({ initial: initialRecovery, next, current });
|
||||
|
||||
expect(merged.abortedLastRun).toBe(false);
|
||||
expect(merged.restartRecoveryRuns).toBeUndefined();
|
||||
expect(merged.mainRestartRecovery).toBeUndefined();
|
||||
});
|
||||
|
||||
it("preserves every concurrent owner while marking a recovered session healthy", () => {
|
||||
const initialRecovery: SessionEntry = {
|
||||
...initial,
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 1,
|
||||
chargedAttempts: 1,
|
||||
},
|
||||
};
|
||||
const next: SessionEntry = {
|
||||
...initialRecovery,
|
||||
updatedAt: 2,
|
||||
abortedLastRun: false,
|
||||
mainRestartRecovery: undefined,
|
||||
};
|
||||
const current: SessionEntry = {
|
||||
...initialRecovery,
|
||||
updatedAt: 3,
|
||||
mainRestartRecovery: {
|
||||
...initialRecovery.mainRestartRecovery!,
|
||||
revision: 3,
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration: "generation-1",
|
||||
tokens: ["owner-1", "owner-2"],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const merged = mergeSessionSnapshotChanges({ initial: initialRecovery, next, current });
|
||||
|
||||
expect(merged.abortedLastRun).toBe(false);
|
||||
expect(merged.mainRestartRecovery?.foregroundClaims?.tokens).toEqual(["owner-1", "owner-2"]);
|
||||
});
|
||||
|
||||
it("reasserts a terminal abort after another owner marks the session healthy", () => {
|
||||
const initialRecovery: SessionEntry = {
|
||||
...initial,
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 3,
|
||||
chargedAttempts: 1,
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration: "generation-1",
|
||||
tokens: ["owner-1", "owner-2"],
|
||||
},
|
||||
},
|
||||
};
|
||||
const next: SessionEntry = {
|
||||
...structuredClone(initialRecovery),
|
||||
updatedAt: 2,
|
||||
};
|
||||
const current: SessionEntry = {
|
||||
...structuredClone(initialRecovery),
|
||||
updatedAt: 3,
|
||||
abortedLastRun: false,
|
||||
mainRestartRecovery: {
|
||||
...initialRecovery.mainRestartRecovery!,
|
||||
revision: 4,
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration: "generation-1",
|
||||
tokens: ["owner-2"],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const merged = mergeSessionSnapshotChanges({
|
||||
initial: initialRecovery,
|
||||
next,
|
||||
current,
|
||||
reassertAbortedLastRun: true,
|
||||
});
|
||||
|
||||
expect(merged.abortedLastRun).toBe(true);
|
||||
expect(merged.mainRestartRecovery).toEqual(current.mainRestartRecovery);
|
||||
});
|
||||
|
||||
it("does not attach a stale terminal abort to a replacement recovery cycle", () => {
|
||||
const initialRecovery: SessionEntry = {
|
||||
...initial,
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-old",
|
||||
revision: 2,
|
||||
chargedAttempts: 1,
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration: "generation-old",
|
||||
tokens: ["old-owner"],
|
||||
},
|
||||
},
|
||||
};
|
||||
const current: SessionEntry = {
|
||||
...structuredClone(initialRecovery),
|
||||
updatedAt: 3,
|
||||
abortedLastRun: false,
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-new",
|
||||
revision: 1,
|
||||
chargedAttempts: 0,
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration: "generation-new",
|
||||
tokens: ["new-owner"],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const merged = mergeSessionSnapshotChanges({
|
||||
initial: initialRecovery,
|
||||
next: { ...structuredClone(initialRecovery), updatedAt: 2 },
|
||||
current,
|
||||
reassertAbortedLastRun: true,
|
||||
});
|
||||
|
||||
expect(merged.abortedLastRun).toBe(false);
|
||||
expect(merged.mainRestartRecovery).toEqual(current.mainRestartRecovery);
|
||||
});
|
||||
|
||||
it("preserves a concurrent foreground claim over a partial recovery update", () => {
|
||||
const initialRecovery: SessionEntry = {
|
||||
...initial,
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 1,
|
||||
chargedAttempts: 0,
|
||||
},
|
||||
};
|
||||
const next: SessionEntry = {
|
||||
...initialRecovery,
|
||||
updatedAt: 2,
|
||||
mainRestartRecovery: {
|
||||
...initialRecovery.mainRestartRecovery!,
|
||||
revision: 2,
|
||||
chargedAttempts: 1,
|
||||
reservation: {
|
||||
runId: "recovery-1",
|
||||
attempt: 1,
|
||||
lifecycleGeneration: "generation-1",
|
||||
},
|
||||
},
|
||||
};
|
||||
const current: SessionEntry = {
|
||||
...initialRecovery,
|
||||
updatedAt: 3,
|
||||
mainRestartRecovery: {
|
||||
...initialRecovery.mainRestartRecovery!,
|
||||
revision: 2,
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration: "generation-1",
|
||||
tokens: ["owner-1"],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const merged = mergeSessionSnapshotChanges({ initial: initialRecovery, next, current });
|
||||
|
||||
expect(merged.mainRestartRecovery).toEqual(current.mainRestartRecovery);
|
||||
expect(merged.mainRestartRecovery?.reservation).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import type { SessionEntry } from "./types.js";
|
||||
import type { InternalSessionEntry as SessionEntry } from "./types.js";
|
||||
|
||||
type SessionEntryRecord = Partial<Record<keyof SessionEntry, unknown>>;
|
||||
|
||||
@@ -42,6 +42,12 @@ const MODEL_OVERRIDE_CONFLICT_DEPENDENT_FIELDS = ["thinkingLevel"] as const sati
|
||||
keyof SessionEntry
|
||||
>;
|
||||
|
||||
const MAIN_SESSION_RECOVERY_TRANSACTION_FIELDS = [
|
||||
"abortedLastRun",
|
||||
"restartRecoveryRuns",
|
||||
"mainRestartRecovery",
|
||||
] as const satisfies ReadonlyArray<keyof SessionEntry>;
|
||||
|
||||
function anySessionFieldChanged(
|
||||
before: SessionEntryRecord,
|
||||
after: SessionEntryRecord,
|
||||
@@ -50,11 +56,69 @@ function anySessionFieldChanged(
|
||||
return fields.some((field) => !isDeepStrictEqual(before[field], after[field]));
|
||||
}
|
||||
|
||||
function mainSessionRecoveryTransactionChanged(before: SessionEntry, after: SessionEntry): boolean {
|
||||
const beforeState = before.mainRestartRecovery;
|
||||
const afterState = after.mainRestartRecovery;
|
||||
return (
|
||||
before.abortedLastRun !== after.abortedLastRun ||
|
||||
!isDeepStrictEqual(before.restartRecoveryRuns, after.restartRecoveryRuns) ||
|
||||
beforeState?.cycleId !== afterState?.cycleId ||
|
||||
beforeState?.chargedAttempts !== afterState?.chargedAttempts ||
|
||||
!isDeepStrictEqual(beforeState?.reservation, afterState?.reservation) ||
|
||||
!isDeepStrictEqual(beforeState?.tombstone, afterState?.tombstone)
|
||||
);
|
||||
}
|
||||
|
||||
function mainSessionRecoveryOwnershipChanged(before: SessionEntry, after: SessionEntry): boolean {
|
||||
return (
|
||||
before.mainRestartRecovery?.revision !== after.mainRestartRecovery?.revision ||
|
||||
!isDeepStrictEqual(
|
||||
before.mainRestartRecovery?.foregroundClaims,
|
||||
after.mainRestartRecovery?.foregroundClaims,
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
function mainSessionRecoveryCycleStateUnchanged(
|
||||
before: SessionEntry,
|
||||
after: SessionEntry,
|
||||
): boolean {
|
||||
const beforeState = before.mainRestartRecovery;
|
||||
const afterState = after.mainRestartRecovery;
|
||||
if (!beforeState || !afterState) {
|
||||
return false;
|
||||
}
|
||||
return (
|
||||
beforeState.cycleId === afterState.cycleId &&
|
||||
beforeState.chargedAttempts === afterState.chargedAttempts &&
|
||||
isDeepStrictEqual(beforeState.reservation, afterState.reservation) &&
|
||||
isDeepStrictEqual(beforeState.tombstone, afterState.tombstone)
|
||||
);
|
||||
}
|
||||
|
||||
function restartRecoveryRunsOnlyConsumed(before: SessionEntry, after: SessionEntry): boolean {
|
||||
const initialRuns = new Set(
|
||||
(before.restartRecoveryRuns ?? []).map((run) => `${run.runId}\u0000${run.lifecycleGeneration}`),
|
||||
);
|
||||
return (after.restartRecoveryRuns ?? []).every((run) =>
|
||||
initialRuns.has(`${run.runId}\u0000${run.lifecycleGeneration}`),
|
||||
);
|
||||
}
|
||||
|
||||
function isCanonicalMainSessionRecoveryClear(entry: SessionEntry): boolean {
|
||||
return (
|
||||
entry.abortedLastRun === false &&
|
||||
entry.restartRecoveryRuns === undefined &&
|
||||
entry.mainRestartRecovery === undefined
|
||||
);
|
||||
}
|
||||
|
||||
/** Projects run-local snapshot changes without restoring concurrently changed fields. */
|
||||
export function projectSessionSnapshotChanges(params: {
|
||||
initial: SessionEntry;
|
||||
next: SessionEntry;
|
||||
current: SessionEntry;
|
||||
reassertAbortedLastRun?: boolean;
|
||||
reassertLiveModelSwitchPending?: boolean;
|
||||
}): Partial<SessionEntry> {
|
||||
if (params.current.sessionId !== params.initial.sessionId) {
|
||||
@@ -131,10 +195,60 @@ export function projectSessionSnapshotChanges(params: {
|
||||
patch.modelProvider = params.next.modelProvider;
|
||||
}
|
||||
|
||||
const mainRecoveryChanged = mainSessionRecoveryTransactionChanged(params.initial, params.next);
|
||||
const mainRecoveryChangedConcurrently = mainSessionRecoveryTransactionChanged(
|
||||
params.initial,
|
||||
params.current,
|
||||
);
|
||||
const mainRecoveryOwnershipChangedConcurrently = mainSessionRecoveryOwnershipChanged(
|
||||
params.initial,
|
||||
params.current,
|
||||
);
|
||||
const initialForegroundClaims = params.initial.mainRestartRecovery?.foregroundClaims;
|
||||
const currentForegroundClaims = params.current.mainRestartRecovery?.foregroundClaims;
|
||||
if (
|
||||
params.reassertAbortedLastRun &&
|
||||
params.next.abortedLastRun === true &&
|
||||
initialForegroundClaims &&
|
||||
currentForegroundClaims &&
|
||||
params.initial.mainRestartRecovery?.cycleId === params.current.mainRestartRecovery?.cycleId &&
|
||||
initialForegroundClaims.lifecycleGeneration === currentForegroundClaims.lifecycleGeneration
|
||||
) {
|
||||
// A terminal abort is authoritative even when true matches this owner's
|
||||
// initial snapshot. Preserve the concurrently narrowed owner aggregate.
|
||||
patch.abortedLastRun = true;
|
||||
}
|
||||
const currentOnlyConsumedLifecycleFences =
|
||||
isCanonicalMainSessionRecoveryClear(params.next) &&
|
||||
!mainRecoveryOwnershipChangedConcurrently &&
|
||||
mainSessionRecoveryCycleStateUnchanged(params.initial, params.current) &&
|
||||
restartRecoveryRunsOnlyConsumed(params.initial, params.current);
|
||||
if (
|
||||
mainRecoveryChanged &&
|
||||
(!mainRecoveryChangedConcurrently || currentOnlyConsumedLifecycleFences)
|
||||
) {
|
||||
// Apply all three fields together: a stale healthy flag can otherwise hide a newer marker.
|
||||
// A healthy run first marks its claim non-interrupted; token-scoped release
|
||||
// removes the aggregate only after the final concurrent owner exits.
|
||||
if (
|
||||
mainRecoveryOwnershipChangedConcurrently &&
|
||||
isCanonicalMainSessionRecoveryClear(params.next)
|
||||
) {
|
||||
patch.abortedLastRun = false;
|
||||
} else if (!mainRecoveryOwnershipChangedConcurrently) {
|
||||
for (const field of MAIN_SESSION_RECOVERY_TRANSACTION_FIELDS) {
|
||||
patchRecord[field] = Object.hasOwn(params.next, field) ? next[field] : undefined;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const field of fields) {
|
||||
if (
|
||||
field === "model" ||
|
||||
field === "modelProvider" ||
|
||||
MAIN_SESSION_RECOVERY_TRANSACTION_FIELDS.includes(
|
||||
field as (typeof MAIN_SESSION_RECOVERY_TRANSACTION_FIELDS)[number],
|
||||
) ||
|
||||
SESSION_MODEL_OVERRIDE_TRANSACTION_FIELDS.includes(
|
||||
field as (typeof SESSION_MODEL_OVERRIDE_TRANSACTION_FIELDS)[number],
|
||||
)
|
||||
@@ -274,6 +388,7 @@ export function mergeSessionSnapshotChanges(params: {
|
||||
initial: SessionEntry;
|
||||
next: SessionEntry;
|
||||
current: SessionEntry;
|
||||
reassertAbortedLastRun?: boolean;
|
||||
reassertLiveModelSwitchPending?: boolean;
|
||||
}): SessionEntry {
|
||||
const merged = { ...params.current };
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
import type { SessionRestartRecoveryState } from "./restart-recovery-types.js";
|
||||
import type { SessionEntry } from "./types.js";
|
||||
import type { InternalSessionEntry as SessionEntry } from "./types.js";
|
||||
|
||||
type SessionRunStatus = "running" | "done" | "failed" | "killed" | "timeout";
|
||||
|
||||
/** Authoritative lifecycle snapshot required for an atomic transcript admission. */
|
||||
export type SessionTranscriptTurnExpectedState = {
|
||||
abortedLastRun: boolean | undefined;
|
||||
/** Fences recovery-only transcript writes against concurrent ownership changes. */
|
||||
mainRestartRecoveryCycleId?: string;
|
||||
mainRestartRecoveryRevision?: number;
|
||||
restartRecoveryBeforeAgentReplyState: SessionRestartRecoveryState["restartRecoveryBeforeAgentReplyState"];
|
||||
restartRecoveryDeliveryReceiptState: SessionRestartRecoveryState["restartRecoveryDeliveryReceiptState"];
|
||||
restartRecoveryDeliveryToolCallId: SessionRestartRecoveryState["restartRecoveryDeliveryToolCallId"];
|
||||
@@ -34,6 +37,7 @@ export type SessionTranscriptTurnLifecyclePatch = {
|
||||
pendingFinalDeliveryLastAttemptAt?: SessionEntry["pendingFinalDeliveryLastAttemptAt"];
|
||||
pendingFinalDeliveryLastError?: SessionEntry["pendingFinalDeliveryLastError"];
|
||||
pendingFinalDeliveryText?: SessionEntry["pendingFinalDeliveryText"];
|
||||
mainRestartRecovery?: SessionEntry["mainRestartRecovery"];
|
||||
restartRecoveryBeforeAgentReplyState?: SessionRestartRecoveryState["restartRecoveryBeforeAgentReplyState"];
|
||||
restartRecoveryDeliveryReceiptState?: SessionRestartRecoveryState["restartRecoveryDeliveryReceiptState"];
|
||||
restartRecoveryDeliveryToolCallId?: SessionRestartRecoveryState["restartRecoveryDeliveryToolCallId"];
|
||||
|
||||
@@ -6,7 +6,7 @@ import type {
|
||||
SessionTranscriptTurnExpectedState,
|
||||
SessionTranscriptTurnLifecyclePatch,
|
||||
} from "./session-transcript-turn-lifecycle.types.js";
|
||||
import type { SessionEntry } from "./types.js";
|
||||
import type { InternalSessionEntry as SessionEntry } from "./types.js";
|
||||
|
||||
export function sessionMatchesExpectedTranscriptTurn<T extends { entry: SessionEntry }>(
|
||||
selected: T | undefined,
|
||||
@@ -24,6 +24,12 @@ export function sessionMatchesExpectedTranscriptTurn<T extends { entry: SessionE
|
||||
selected.entry.lifecycleRevision === expected.expectedLifecycleRevision) &&
|
||||
(expectedState === undefined ||
|
||||
(selected.entry.abortedLastRun === expectedState.abortedLastRun &&
|
||||
(expectedState.mainRestartRecoveryCycleId === undefined ||
|
||||
selected.entry.mainRestartRecovery?.cycleId ===
|
||||
expectedState.mainRestartRecoveryCycleId) &&
|
||||
(expectedState.mainRestartRecoveryRevision === undefined ||
|
||||
selected.entry.mainRestartRecovery?.revision ===
|
||||
expectedState.mainRestartRecoveryRevision) &&
|
||||
selected.entry.restartRecoveryBeforeAgentReplyState ===
|
||||
expectedState.restartRecoveryBeforeAgentReplyState &&
|
||||
selected.entry.restartRecoveryDeliveryReceiptState ===
|
||||
|
||||
@@ -894,7 +894,8 @@ describe("session store writer queue", () => {
|
||||
writeSessionStoreCache({
|
||||
storePath,
|
||||
store,
|
||||
mtimeMs: 1,
|
||||
ctimeNs: 1n,
|
||||
mtimeNs: 1n,
|
||||
sizeBytes: serialized.length,
|
||||
serialized,
|
||||
cloneSerialized: serialized,
|
||||
@@ -904,13 +905,45 @@ describe("session store writer queue", () => {
|
||||
|
||||
const cached = readSessionStoreCache({
|
||||
storePath,
|
||||
mtimeMs: 1,
|
||||
ctimeNs: 1n,
|
||||
mtimeNs: 1n,
|
||||
sizeBytes: serialized.length,
|
||||
});
|
||||
|
||||
expect(cached?.[key]?.sessionId).toBe("s-serialized-cache");
|
||||
});
|
||||
|
||||
it("invalidates session store cache when ctime nanoseconds change inside the same millisecond", () => {
|
||||
const key = "agent:main:ctime-ns-cache";
|
||||
const storePath = "/tmp/openclaw-ctime-ns-cache-test.json";
|
||||
const store = {
|
||||
[key]: {
|
||||
sessionId: "s-ctime-ns-cache",
|
||||
updatedAt: Date.now(),
|
||||
},
|
||||
} satisfies Record<string, SessionEntry>;
|
||||
const serialized = JSON.stringify(store);
|
||||
writeSessionStoreCache({
|
||||
storePath,
|
||||
store,
|
||||
ctimeNs: 1_000_000n,
|
||||
mtimeNs: 1_000_000n,
|
||||
sizeBytes: serialized.length,
|
||||
serialized,
|
||||
cloneSerialized: serialized,
|
||||
takeOwnership: true,
|
||||
});
|
||||
|
||||
const cached = readSessionStoreCache({
|
||||
storePath,
|
||||
ctimeNs: 1_000_001n,
|
||||
mtimeNs: 1_000_000n,
|
||||
sizeBytes: serialized.length,
|
||||
});
|
||||
|
||||
expect(cached).toBeNull();
|
||||
});
|
||||
|
||||
it("returns an owned parsed store for fresh skip-cache loads without cloning again", async () => {
|
||||
const key = "agent:main:owned-skip-cache";
|
||||
const { storePath } = await makeTmpStore({
|
||||
|
||||
@@ -17,7 +17,8 @@ export type SessionStoreSnapshotEntry = DeepReadonly<SessionEntry>;
|
||||
|
||||
type SessionStoreCacheEntry = {
|
||||
store: Record<string, SessionEntry>;
|
||||
mtimeMs?: number;
|
||||
ctimeNs?: bigint;
|
||||
mtimeNs?: bigint;
|
||||
sizeBytes?: number;
|
||||
serialized?: string;
|
||||
};
|
||||
@@ -298,7 +299,8 @@ export function dropSessionStoreObjectCache(storePath: string): void {
|
||||
|
||||
export function readSessionStoreCache(params: {
|
||||
storePath: string;
|
||||
mtimeMs?: number;
|
||||
ctimeNs?: bigint;
|
||||
mtimeNs?: bigint;
|
||||
sizeBytes?: number;
|
||||
clone?: boolean;
|
||||
}): Record<string, SessionEntry> | null {
|
||||
@@ -306,7 +308,11 @@ export function readSessionStoreCache(params: {
|
||||
if (!cached) {
|
||||
return null;
|
||||
}
|
||||
if (params.mtimeMs !== cached.mtimeMs || params.sizeBytes !== cached.sizeBytes) {
|
||||
if (
|
||||
params.ctimeNs !== cached.ctimeNs ||
|
||||
params.mtimeNs !== cached.mtimeNs ||
|
||||
params.sizeBytes !== cached.sizeBytes
|
||||
) {
|
||||
invalidateSessionStoreCache(params.storePath);
|
||||
return null;
|
||||
}
|
||||
@@ -318,14 +324,19 @@ export function readSessionStoreCache(params: {
|
||||
|
||||
export function takeMutableSessionStoreCache(params: {
|
||||
storePath: string;
|
||||
mtimeMs?: number;
|
||||
ctimeNs?: bigint;
|
||||
mtimeNs?: bigint;
|
||||
sizeBytes?: number;
|
||||
}): Record<string, SessionEntry> | null {
|
||||
const cached = SESSION_STORE_CACHE.get(params.storePath);
|
||||
if (!cached) {
|
||||
return null;
|
||||
}
|
||||
if (params.mtimeMs !== cached.mtimeMs || params.sizeBytes !== cached.sizeBytes) {
|
||||
if (
|
||||
params.ctimeNs !== cached.ctimeNs ||
|
||||
params.mtimeNs !== cached.mtimeNs ||
|
||||
params.sizeBytes !== cached.sizeBytes
|
||||
) {
|
||||
invalidateSessionStoreCache(params.storePath);
|
||||
return null;
|
||||
}
|
||||
@@ -336,7 +347,8 @@ export function takeMutableSessionStoreCache(params: {
|
||||
export function writeSessionStoreCache(params: {
|
||||
storePath: string;
|
||||
store: Record<string, SessionEntry>;
|
||||
mtimeMs?: number;
|
||||
ctimeNs?: bigint;
|
||||
mtimeNs?: bigint;
|
||||
sizeBytes?: number;
|
||||
serialized?: string;
|
||||
serializedPromptRefs?: ReadonlyMap<string, SessionSkillPromptRef>;
|
||||
@@ -352,7 +364,8 @@ export function writeSessionStoreCache(params: {
|
||||
}
|
||||
SESSION_STORE_CACHE.set(params.storePath, {
|
||||
store,
|
||||
mtimeMs: params.mtimeMs,
|
||||
ctimeNs: params.ctimeNs,
|
||||
mtimeNs: params.mtimeNs,
|
||||
sizeBytes: params.sizeBytes,
|
||||
serialized: params.cloneSerialized,
|
||||
});
|
||||
|
||||
@@ -396,8 +396,7 @@ export function loadSessionStore(
|
||||
const currentFileStat = getFileStatSnapshot(storePath);
|
||||
const cached = readSessionStoreCache({
|
||||
storePath,
|
||||
mtimeMs: currentFileStat?.mtimeMs,
|
||||
sizeBytes: currentFileStat?.sizeBytes,
|
||||
...currentFileStat,
|
||||
clone: opts.clone,
|
||||
});
|
||||
if (cached) {
|
||||
@@ -409,7 +408,6 @@ export function loadSessionStore(
|
||||
// transiently invalid content while another process is swapping the file.
|
||||
let store: Record<string, SessionEntry> = {};
|
||||
const fileStat = getFileStatSnapshot(storePath);
|
||||
const mtimeMs = fileStat?.mtimeMs;
|
||||
let serializedFromDisk: string | undefined;
|
||||
const maxReadAttempts = 3;
|
||||
const retryBuf = maxReadAttempts > 1 ? new Int32Array(new SharedArrayBuffer(4)) : undefined;
|
||||
@@ -515,8 +513,7 @@ export function loadSessionStore(
|
||||
writeSessionStoreCache({
|
||||
storePath,
|
||||
store,
|
||||
mtimeMs,
|
||||
sizeBytes: fileStat?.sizeBytes,
|
||||
...fileStat,
|
||||
serialized: serializedFromDisk,
|
||||
cloneSerialized: serializedFromDisk,
|
||||
takeOwnership: serializedFromDisk !== undefined,
|
||||
|
||||
@@ -9,7 +9,7 @@ import {
|
||||
recordInboundSessionMeta,
|
||||
updateSessionLastRoute,
|
||||
} from "./session-accessor.js";
|
||||
import type { SessionEntry } from "./types.js";
|
||||
import type { InternalSessionEntry as SessionEntry } from "./types.js";
|
||||
|
||||
// Materializes the SQLite-backed session store as a keyed object so key
|
||||
// normalization/migration assertions keep matching the old JSON-store shape.
|
||||
@@ -60,6 +60,50 @@ function createSignalGroupContext(): MsgContext {
|
||||
};
|
||||
}
|
||||
|
||||
function createRecoveryEntry(sessionId: string): SessionEntry {
|
||||
return {
|
||||
sessionId,
|
||||
updatedAt: 1,
|
||||
abortedLastRun: true,
|
||||
restartRecoveryRuns: [
|
||||
{ runId: "initial-wedged-run", lifecycleGeneration: "gen-1" },
|
||||
{ runId: "recovery-run-1", lifecycleGeneration: "gen-2" },
|
||||
],
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 3,
|
||||
chargedAttempts: 2,
|
||||
},
|
||||
subagentRecovery: {
|
||||
automaticAttempts: 2,
|
||||
lastAttemptAt: 3,
|
||||
wedgedAt: 4,
|
||||
wedgedReason: "automatic_attempt_budget_exceeded",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function expectRecoveryMarkers(entry: SessionEntry | undefined): void {
|
||||
expect(entry).toMatchObject({
|
||||
abortedLastRun: true,
|
||||
restartRecoveryRuns: [
|
||||
{ runId: "initial-wedged-run", lifecycleGeneration: "gen-1" },
|
||||
{ runId: "recovery-run-1", lifecycleGeneration: "gen-2" },
|
||||
],
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 3,
|
||||
chargedAttempts: 2,
|
||||
},
|
||||
subagentRecovery: {
|
||||
automaticAttempts: 2,
|
||||
lastAttemptAt: 3,
|
||||
wedgedAt: 4,
|
||||
wedgedReason: "automatic_attempt_budget_exceeded",
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
describe("session store key normalization", () => {
|
||||
const suiteRootTracker = createSuiteTempRootTracker({
|
||||
prefix: "openclaw-session-key-normalize-",
|
||||
@@ -203,6 +247,39 @@ describe("session store key normalization", () => {
|
||||
expect(store[CANONICAL_KEY]?.origin?.provider).toBe("webchat");
|
||||
});
|
||||
|
||||
it("preserves recovery markers when recording inbound metadata", async () => {
|
||||
await seedStore(storePath, {
|
||||
[CANONICAL_KEY]: createRecoveryEntry("recovered-session"),
|
||||
});
|
||||
|
||||
await recordInboundSessionMeta({
|
||||
storePath,
|
||||
sessionKey: MIXED_CASE_KEY,
|
||||
ctx: createInboundContext(),
|
||||
});
|
||||
|
||||
const store = loadStore(storePath);
|
||||
expectRecoveryMarkers(store[CANONICAL_KEY]);
|
||||
expect(store[CANONICAL_KEY]?.origin?.provider).toBe("webchat");
|
||||
});
|
||||
|
||||
it("preserves recovery markers when updating the last route", async () => {
|
||||
await seedStore(storePath, {
|
||||
[CANONICAL_KEY]: createRecoveryEntry("route-session"),
|
||||
});
|
||||
|
||||
await updateSessionLastRoute({
|
||||
storePath,
|
||||
sessionKey: CANONICAL_KEY,
|
||||
channel: "webchat",
|
||||
to: "webchat:user-1",
|
||||
});
|
||||
|
||||
const store = loadStore(storePath);
|
||||
expectRecoveryMarkers(store[CANONICAL_KEY]);
|
||||
expect(store[CANONICAL_KEY]?.lastTo).toBe("webchat:user-1");
|
||||
});
|
||||
|
||||
it("records Signal group metadata under the mixed-case opaque group id", async () => {
|
||||
await recordInboundSessionMeta({
|
||||
storePath,
|
||||
|
||||
@@ -367,8 +367,7 @@ function updateSessionStoreWriteCaches(params: {
|
||||
writeSessionStoreCache({
|
||||
storePath: params.storePath,
|
||||
store: params.store,
|
||||
mtimeMs: fileStat?.mtimeMs,
|
||||
sizeBytes: fileStat?.sizeBytes,
|
||||
...fileStat,
|
||||
serialized: params.serialized,
|
||||
serializedPromptRefs: params.serializedPromptRefs,
|
||||
cloneSerialized: params.cloneSerialized,
|
||||
@@ -386,7 +385,8 @@ function restoreUnchangedSessionStoreCache(
|
||||
const loadedFileStat = writerStoreFileStats.get(store) ?? null;
|
||||
const currentFileStat = getFileStatSnapshot(storePath) ?? null;
|
||||
if (
|
||||
loadedFileStat?.mtimeMs !== currentFileStat?.mtimeMs ||
|
||||
loadedFileStat?.ctimeNs !== currentFileStat?.ctimeNs ||
|
||||
loadedFileStat?.mtimeNs !== currentFileStat?.mtimeNs ||
|
||||
loadedFileStat?.sizeBytes !== currentFileStat?.sizeBytes
|
||||
) {
|
||||
invalidateSessionStoreCache(storePath);
|
||||
@@ -398,8 +398,7 @@ function restoreUnchangedSessionStoreCache(
|
||||
writeSessionStoreCache({
|
||||
storePath,
|
||||
store,
|
||||
mtimeMs: loadedFileStat?.mtimeMs,
|
||||
sizeBytes: loadedFileStat?.sizeBytes,
|
||||
...loadedFileStat,
|
||||
serialized,
|
||||
serializedPromptRefs,
|
||||
takeOwnership: true,
|
||||
@@ -598,8 +597,7 @@ function loadMutableSessionStoreForWriter(storePath: string): Record<string, Ses
|
||||
if (isSessionStoreCacheEnabled()) {
|
||||
const cached = takeMutableSessionStoreCache({
|
||||
storePath,
|
||||
mtimeMs: currentFileStat?.mtimeMs,
|
||||
sizeBytes: currentFileStat?.sizeBytes,
|
||||
...currentFileStat,
|
||||
});
|
||||
if (cached) {
|
||||
writerStoreFileStats.set(cached, currentFileStat ?? null);
|
||||
|
||||
@@ -31,6 +31,7 @@ import {
|
||||
type SessionTranscriptTurnWriteContext,
|
||||
type SessionTranscriptTurnExpectedState,
|
||||
} from "./session-accessor.js";
|
||||
import type { SessionTranscriptTurnLifecyclePatch } from "./session-transcript-turn-lifecycle.types.js";
|
||||
import { parseSqliteSessionFileMarker, type SqliteSessionFileMarker } from "./sqlite-marker.js";
|
||||
import { resolveSessionStoreEntry } from "./store.js";
|
||||
import {
|
||||
@@ -435,6 +436,7 @@ export async function appendAssistantMessageToSessionTranscript(params: {
|
||||
expectedSessionId?: string;
|
||||
expectedLifecycleRevision?: string;
|
||||
expectedSessionState?: SessionTranscriptTurnExpectedState;
|
||||
sessionLifecyclePatch?: SessionTranscriptTurnLifecyclePatch;
|
||||
text?: string;
|
||||
mediaUrls?: string[];
|
||||
idempotencyKey?: string;
|
||||
@@ -466,6 +468,9 @@ export async function appendAssistantMessageToSessionTranscript(params: {
|
||||
? { expectedLifecycleRevision: params.expectedLifecycleRevision }
|
||||
: {}),
|
||||
...(params.expectedSessionState ? { expectedSessionState: params.expectedSessionState } : {}),
|
||||
...(params.sessionLifecyclePatch
|
||||
? { sessionLifecyclePatch: params.sessionLifecyclePatch }
|
||||
: {}),
|
||||
storePath: params.storePath,
|
||||
idempotencyKey: params.idempotencyKey,
|
||||
updateMode: params.updateMode,
|
||||
@@ -504,6 +509,7 @@ export async function appendExactAssistantMessageToSessionTranscript(params: {
|
||||
expectedSessionId?: string;
|
||||
expectedLifecycleRevision?: string;
|
||||
expectedSessionState?: SessionTranscriptTurnExpectedState;
|
||||
sessionLifecyclePatch?: SessionTranscriptTurnLifecyclePatch;
|
||||
message: SessionTranscriptAssistantMessage;
|
||||
idempotencyKey?: string;
|
||||
storePath?: string;
|
||||
@@ -602,6 +608,9 @@ export async function appendExactAssistantMessageToSessionTranscript(params: {
|
||||
...(params.expectedSessionState
|
||||
? { expectedSessionState: params.expectedSessionState }
|
||||
: {}),
|
||||
...(params.sessionLifecyclePatch
|
||||
? { sessionLifecyclePatch: params.sessionLifecyclePatch }
|
||||
: {}),
|
||||
...(params.config ? { config: params.config } : {}),
|
||||
updateMode: params.updateMode ?? "inline",
|
||||
touchSessionEntry: true,
|
||||
|
||||
@@ -12,6 +12,7 @@ import type { ChannelRouteRef } from "../../plugin-sdk/channel-route.js";
|
||||
import type { Skill } from "../../skills/loading/skill-contract.js";
|
||||
import type { DeliveryContext } from "../../utils/delivery-context.types.js";
|
||||
import type { TtsAutoMode } from "../types.tts.js";
|
||||
import type { MainRestartRecoveryState } from "./main-session-recovery.types.js";
|
||||
import type { SessionRestartRecoveryState } from "./restart-recovery-types.js";
|
||||
import type { SessionEntryProvenance } from "./session-entry-provenance.js";
|
||||
import { rewriteSessionFileForNewSessionId } from "./session-file-rotation.js";
|
||||
@@ -494,6 +495,11 @@ export type SessionEntry = SessionRestartRecoveryState &
|
||||
acp?: SessionAcpMeta;
|
||||
};
|
||||
|
||||
/** Internal durable fields excluded from public/plugin session projections. */
|
||||
export type InternalSessionEntry = SessionEntry & {
|
||||
mainRestartRecovery?: MainRestartRecoveryState;
|
||||
};
|
||||
|
||||
export function isTerminalSessionStatus(
|
||||
status: unknown,
|
||||
): status is Exclude<NonNullable<SessionEntry["status"]>, "running"> {
|
||||
|
||||
@@ -10,6 +10,7 @@ import { parseExecApprovalFollowupApprovalId } from "../../agents/bash-tools.exe
|
||||
import { normalizeSpawnedRunMetadata } from "../../agents/spawned-context.js";
|
||||
import { getAgentEventLifecycleGeneration } from "../../infra/agent-events.js";
|
||||
import {
|
||||
isMainSessionRestartRecoveryInputProvenance,
|
||||
normalizeInputProvenance,
|
||||
shouldPreserveUserFacingSessionStateForInputProvenance,
|
||||
} from "../../sessions/input-provenance.js";
|
||||
@@ -45,6 +46,7 @@ type AgentRequestPreflight = {
|
||||
execApprovalFollowupApprovalId?: string;
|
||||
normalizedSpawned: ReturnType<typeof normalizeSpawnedRunMetadata>;
|
||||
inputProvenance: ReturnType<typeof normalizeInputProvenance>;
|
||||
isRestartRecoveryResumeRun: boolean;
|
||||
preserveUserFacingSessionModelState: boolean;
|
||||
sessionEffects?: "visible" | "internal";
|
||||
suppressVisibleSessionEffects: boolean;
|
||||
@@ -215,6 +217,8 @@ export function prepareAgentRequestPreflight(
|
||||
groupSpace: request.groupSpace,
|
||||
}),
|
||||
inputProvenance,
|
||||
isRestartRecoveryResumeRun:
|
||||
canUseInternalRuntimeHandoff && isMainSessionRestartRecoveryInputProvenance(inputProvenance),
|
||||
preserveUserFacingSessionModelState:
|
||||
canUseInternalRuntimeHandoff &&
|
||||
shouldPreserveUserFacingSessionStateForInputProvenance(inputProvenance),
|
||||
|
||||
@@ -5,6 +5,10 @@ import {
|
||||
isEmbeddedAgentRunAbortableForRunId,
|
||||
retainEmbeddedAgentRunAbortabilityForRunId,
|
||||
} from "../../agents/embedded-agent-runner/runs.js";
|
||||
import {
|
||||
commitMainSessionRecovery,
|
||||
type MainSessionRecoveryPendingTarget,
|
||||
} from "../../agents/main-session-recovery-store.js";
|
||||
import { resolvePersistedOverrideModelRef } from "../../agents/model-selection.js";
|
||||
import { resolveProviderIdForAuth } from "../../agents/provider-auth-aliases.js";
|
||||
import { resolveEffectiveAgentRuntime } from "../../agents/thinking-runtime.js";
|
||||
@@ -45,6 +49,9 @@ export type PreparedAgentRunDispatch = {
|
||||
lifecycleStorePath: string;
|
||||
resolvedThreadId?: string | number;
|
||||
dispatchTaskTrackingMode: Exclude<GatewayAgentTaskTrackingMode, "plugin_subagent">;
|
||||
restoreAdmittedRestartRecoveryInterrupted?: () => Promise<
|
||||
MainSessionRecoveryPendingTarget | undefined
|
||||
>;
|
||||
};
|
||||
|
||||
export async function prepareAgentRunDispatch(params: {
|
||||
@@ -73,6 +80,7 @@ export async function prepareAgentRunDispatch(params: {
|
||||
pendingChatRun?: { sessionKey: string; agentId?: string };
|
||||
inputProvenance?: InputProvenance;
|
||||
isOneShotModelRun: boolean;
|
||||
isRestartRecoveryResumeRun: boolean;
|
||||
runId: string;
|
||||
agentDedupeKeys: readonly string[];
|
||||
context: GatewayRequestHandlerOptions["context"];
|
||||
@@ -300,6 +308,75 @@ export async function prepareAgentRunDispatch(params: {
|
||||
dispatchTaskTrackingMode = "cli";
|
||||
}
|
||||
}
|
||||
let restoreAdmittedRestartRecoveryInterrupted:
|
||||
| (() => Promise<MainSessionRecoveryPendingTarget | undefined>)
|
||||
| undefined;
|
||||
if (params.isRestartRecoveryResumeRun) {
|
||||
const recoverySessionKey = params.resolvedSessionKey;
|
||||
if (!recoverySessionKey) {
|
||||
activeRunAbort.cleanup({ force: true });
|
||||
activeGatewayWorkAdmission.release();
|
||||
params.respond(
|
||||
false,
|
||||
undefined,
|
||||
errorShape(ErrorCodes.UNAVAILABLE, "restart recovery session target is unavailable"),
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
const recoveryAdmission = await commitMainSessionRecovery({
|
||||
command: {
|
||||
kind: "admit_recovery",
|
||||
lifecycleGeneration: params.lifecycleGeneration,
|
||||
now: Date.now(),
|
||||
runId: params.runId,
|
||||
sessionId: params.request.expectedExistingSessionId ?? params.getAdmittedSessionId(),
|
||||
},
|
||||
requireWriteSuccess: true,
|
||||
target: { sessionKey: recoverySessionKey, storePath: lifecycleStorePath },
|
||||
});
|
||||
if (recoveryAdmission.transition.kind !== "admitted_recovery") {
|
||||
throw new Error(
|
||||
`Session "${recoverySessionKey}" restart recovery reservation is stale; recovery was skipped.`,
|
||||
);
|
||||
}
|
||||
const admittedRecoverySessionKey = recoveryAdmission.sessionKey ?? recoverySessionKey;
|
||||
let restored = false;
|
||||
restoreAdmittedRestartRecoveryInterrupted = async () => {
|
||||
if (restored) {
|
||||
return undefined;
|
||||
}
|
||||
const recovery = await commitMainSessionRecovery({
|
||||
command: {
|
||||
kind: "mark_admitted_recovery_interrupted",
|
||||
lifecycleGeneration: params.lifecycleGeneration,
|
||||
now: Date.now(),
|
||||
runId: params.runId,
|
||||
sessionId: params.request.expectedExistingSessionId ?? params.getAdmittedSessionId(),
|
||||
},
|
||||
requireWriteSuccess: true,
|
||||
target: { sessionKey: admittedRecoverySessionKey, storePath: lifecycleStorePath },
|
||||
});
|
||||
restored = true;
|
||||
const expectedSessionId =
|
||||
params.request.expectedExistingSessionId ?? params.getAdmittedSessionId();
|
||||
return recovery.transition.kind === "applied" &&
|
||||
recovery.entry?.sessionId === expectedSessionId &&
|
||||
recovery.sessionKey
|
||||
? {
|
||||
sessionId: recovery.entry.sessionId,
|
||||
sessionKey: recovery.sessionKey,
|
||||
storePath: lifecycleStorePath,
|
||||
}
|
||||
: undefined;
|
||||
};
|
||||
} catch (err) {
|
||||
activeRunAbort.cleanup({ force: true });
|
||||
activeGatewayWorkAdmission.release();
|
||||
params.respond(false, undefined, errorShape(ErrorCodes.UNAVAILABLE, formatForLog(err)));
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
const accepted = {
|
||||
runId: params.runId,
|
||||
sessionKey: params.resolvedSessionKey,
|
||||
@@ -336,5 +413,6 @@ export async function prepareAgentRunDispatch(params: {
|
||||
lifecycleStorePath,
|
||||
resolvedThreadId,
|
||||
dispatchTaskTrackingMode,
|
||||
restoreAdmittedRestartRecoveryInterrupted,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -3,12 +3,13 @@ import {
|
||||
buildAgentRunTerminalOutcome,
|
||||
type AgentRunTerminalOutcome,
|
||||
} from "../../agents/agent-run-terminal-outcome.js";
|
||||
import type { MainSessionRecoveryPendingTarget } from "../../agents/main-session-recovery-store.js";
|
||||
import { isAgentRunRestartAbortReason } from "../../agents/run-termination.js";
|
||||
import {
|
||||
normalizeAgentRunTimeoutPhase,
|
||||
normalizeProviderStarted,
|
||||
} from "../../agents/run-timeout-attribution.js";
|
||||
import { agentCommandFromIngress } from "../../commands/agent.js";
|
||||
import { agentCommandFromGatewayIngress } from "../../commands/agent.js";
|
||||
import { isAbortError } from "../../infra/abort-signal.js";
|
||||
import { clearAgentRunContext } from "../../infra/agent-events.js";
|
||||
import { readErrorName } from "../../infra/errors.js";
|
||||
@@ -77,7 +78,7 @@ export function deleteGatewayDedupeEntries(params: {
|
||||
}
|
||||
|
||||
export function dispatchAgentRunFromGateway(params: {
|
||||
ingressOpts: Parameters<typeof agentCommandFromIngress>[0];
|
||||
ingressOpts: Parameters<typeof agentCommandFromGatewayIngress>[0];
|
||||
runId: string;
|
||||
dedupeKeys: readonly string[];
|
||||
/**
|
||||
@@ -90,6 +91,7 @@ export function dispatchAgentRunFromGateway(params: {
|
||||
respond: GatewayRequestHandlerOptions["respond"];
|
||||
context: GatewayRequestHandlerOptions["context"];
|
||||
taskTrackingMode: Exclude<GatewayAgentTaskTrackingMode, "plugin_subagent">;
|
||||
restoreAdmittedRecovery?: () => Promise<MainSessionRecoveryPendingTarget | undefined>;
|
||||
onSettled?: (outcome: {
|
||||
terminalOutcome: AgentRunTerminalOutcome;
|
||||
onRecovered?: () => void;
|
||||
@@ -139,7 +141,9 @@ export function dispatchAgentRunFromGateway(params: {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
void agentCommandFromIngress(params.ingressOpts, defaultRuntime, params.context.deps)
|
||||
void agentCommandFromGatewayIngress(params.ingressOpts, defaultRuntime, params.context.deps, {
|
||||
restoreAdmittedRecovery: params.restoreAdmittedRecovery,
|
||||
})
|
||||
.then(async (result) => {
|
||||
const aborted = result?.meta?.aborted === true;
|
||||
const timeoutAttribution = readAgentRunTimeoutAttribution(result?.meta);
|
||||
|
||||
@@ -2,6 +2,16 @@ import { ErrorCodes, errorShape } from "../../../packages/gateway-protocol/src/i
|
||||
import type { AgentRunTerminalOutcome } from "../../agents/agent-run-terminal-outcome.js";
|
||||
import { consumeExecApprovalFollowupRuntimeHandoff } from "../../agents/bash-tools.exec-approval-followup-state.js";
|
||||
import { runAgentHarnessBeforeMessageWriteHook } from "../../agents/harness/hook-helpers.js";
|
||||
import { scheduleMainSessionRecoveryPendingTarget } from "../../agents/main-session-recovery-owner-release.js";
|
||||
import {
|
||||
restoreAdmittedRecoveryWithRetries,
|
||||
scheduleAdmittedRecoveryRestore,
|
||||
} from "../../agents/main-session-recovery-restore.js";
|
||||
import {
|
||||
releaseMainSessionRecoveryOwner,
|
||||
type MainSessionRecoveryPendingTarget,
|
||||
type MainSessionRecoveryOwnerLease,
|
||||
} from "../../agents/main-session-recovery-store.js";
|
||||
import { resolveIngressWorkspaceOverrideForSessionRun } from "../../agents/spawned-context.js";
|
||||
import {
|
||||
setChannelSourceTurnId,
|
||||
@@ -45,6 +55,7 @@ import type { GatewayRequestHandlerOptions } from "./types.js";
|
||||
|
||||
export function startAgentRunExecution(params: {
|
||||
prepared: PreparedAgentRunDispatch;
|
||||
mainRestartRecoveryOwnerLease?: MainSessionRecoveryOwnerLease;
|
||||
request: AgentRunRequest;
|
||||
cfg: OpenClawConfig;
|
||||
cfgForAgent?: OpenClawConfig;
|
||||
@@ -59,6 +70,7 @@ export function startAgentRunExecution(params: {
|
||||
isNewSession: boolean;
|
||||
isRawModelRun: boolean;
|
||||
isOneShotModelRun: boolean;
|
||||
isRestartRecoveryResumeRun: boolean;
|
||||
suppressVisibleSessionEffects: boolean;
|
||||
message: string;
|
||||
images: Array<{ type: "image"; data: string; mimeType: string }>;
|
||||
@@ -101,8 +113,10 @@ export function startAgentRunExecution(params: {
|
||||
void prepared.activeGatewayWorkAdmission.run(async () => {
|
||||
await yieldAfterAgentAcceptedAck();
|
||||
let dispatched = false;
|
||||
let pendingRecovery: MainSessionRecoveryPendingTarget | undefined;
|
||||
try {
|
||||
if (prepared.activeRunAbort.controller.signal.aborted) {
|
||||
pendingRecovery = await prepared.restoreAdmittedRestartRecoveryInterrupted?.();
|
||||
const stopReason = resolveAbortedAgentStopReason(prepared.activeRunAbort.entry);
|
||||
setAbortedAgentDedupeEntries({
|
||||
dedupe: params.context.dedupe,
|
||||
@@ -382,6 +396,10 @@ export function startAgentRunExecution(params: {
|
||||
sessionEntry: params.sessionEntry,
|
||||
}),
|
||||
allowGatewaySubagentBinding: true,
|
||||
...(params.mainRestartRecoveryOwnerLease
|
||||
? { mainRestartRecoveryOwnerLease: params.mainRestartRecoveryOwnerLease }
|
||||
: {}),
|
||||
...(params.isRestartRecoveryResumeRun ? { mainRestartRecoveryAdmitted: true } : {}),
|
||||
allowModelOverride: prepared.effectiveAllowModelOverride,
|
||||
},
|
||||
runId: params.runId,
|
||||
@@ -398,6 +416,7 @@ export function startAgentRunExecution(params: {
|
||||
respond: params.respond,
|
||||
context: params.context,
|
||||
taskTrackingMode: prepared.dispatchTaskTrackingMode,
|
||||
restoreAdmittedRecovery: prepared.restoreAdmittedRestartRecoveryInterrupted,
|
||||
});
|
||||
dispatched = true;
|
||||
} catch (err) {
|
||||
@@ -419,9 +438,38 @@ export function startAgentRunExecution(params: {
|
||||
} finally {
|
||||
if (!dispatched) {
|
||||
try {
|
||||
await params.releaseCronContinuationClaimWithRecovery();
|
||||
if (prepared.restoreAdmittedRestartRecoveryInterrupted) {
|
||||
try {
|
||||
pendingRecovery ??= await restoreAdmittedRecoveryWithRetries(
|
||||
prepared.restoreAdmittedRestartRecoveryInterrupted,
|
||||
);
|
||||
} catch (err) {
|
||||
params.context.logGateway.warn(
|
||||
`failed to restore undispatched restart recovery: ${formatForLog(err)}`,
|
||||
);
|
||||
scheduleAdmittedRecoveryRestore(prepared.restoreAdmittedRestartRecoveryInterrupted);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
cleanupAdmittedRun({ force: true });
|
||||
try {
|
||||
await params.releaseCronContinuationClaimWithRecovery();
|
||||
} finally {
|
||||
try {
|
||||
pendingRecovery ??= await releaseMainSessionRecoveryOwner(
|
||||
params.mainRestartRecoveryOwnerLease,
|
||||
);
|
||||
} catch (err) {
|
||||
params.context.logGateway.warn(
|
||||
`failed to release undispatched main restart recovery owner: ${formatForLog(err)}`,
|
||||
);
|
||||
} finally {
|
||||
try {
|
||||
cleanupAdmittedRun({ force: true });
|
||||
} finally {
|
||||
scheduleMainSessionRecoveryPendingTarget(pendingRecovery);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
|
||||
import { ErrorCodes, errorShape } from "../../../packages/gateway-protocol/src/index.js";
|
||||
import { scheduleMainSessionRecoveryPendingTarget } from "../../agents/main-session-recovery-owner-release.js";
|
||||
import {
|
||||
releaseMainSessionRecoveryOwner,
|
||||
type MainSessionRecoveryOwnerLease,
|
||||
} from "../../agents/main-session-recovery-store.js";
|
||||
import { mergeSessionEntry, type SessionEntry } from "../../config/sessions.js";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
import { normalizeDeliveryContext } from "../../utils/delivery-context.shared.js";
|
||||
@@ -46,6 +51,7 @@ export const agentRunHandler: GatewayRequestHandlers["agent"] = async ({
|
||||
execApprovalFollowupApprovalId,
|
||||
normalizedSpawned,
|
||||
inputProvenance,
|
||||
isRestartRecoveryResumeRun,
|
||||
preserveUserFacingSessionModelState,
|
||||
sessionEffects,
|
||||
suppressVisibleSessionEffects,
|
||||
@@ -108,6 +114,7 @@ export const agentRunHandler: GatewayRequestHandlers["agent"] = async ({
|
||||
let agentId = routing.agentId;
|
||||
let requestedSessionKey = routing.requestedSessionKey;
|
||||
let gatewayAdmissionTransferred = false;
|
||||
let mainRestartRecoveryOwnerLease: MainSessionRecoveryOwnerLease | undefined;
|
||||
let releaseGatewayAdmission = () => {};
|
||||
const cronContinuation = createCronContinuationController({
|
||||
runId,
|
||||
@@ -329,6 +336,7 @@ export const agentRunHandler: GatewayRequestHandlers["agent"] = async ({
|
||||
sessionAgentId,
|
||||
mainSessionKey,
|
||||
lifecycleGeneration,
|
||||
isRestartRecoveryResumeRun,
|
||||
runId,
|
||||
agentId,
|
||||
suppressVisibleSessionEffects,
|
||||
@@ -355,6 +363,9 @@ export const agentRunHandler: GatewayRequestHandlers["agent"] = async ({
|
||||
},
|
||||
getAdmittedSessionId: () => admittedSessionId,
|
||||
setCronContinuationClaim: cronContinuation.setClaim,
|
||||
setMainRestartRecoveryOwnerLease: (lease) => {
|
||||
mainRestartRecoveryOwnerLease = lease;
|
||||
},
|
||||
respond,
|
||||
});
|
||||
if (!persistedSession) {
|
||||
@@ -425,6 +436,7 @@ export const agentRunHandler: GatewayRequestHandlers["agent"] = async ({
|
||||
pendingChatRun,
|
||||
inputProvenance,
|
||||
isOneShotModelRun,
|
||||
isRestartRecoveryResumeRun,
|
||||
runId,
|
||||
agentDedupeKeys,
|
||||
context,
|
||||
@@ -448,6 +460,7 @@ export const agentRunHandler: GatewayRequestHandlers["agent"] = async ({
|
||||
gatewayAdmissionTransferred = true;
|
||||
startAgentRunExecution({
|
||||
prepared: preparedDispatch,
|
||||
mainRestartRecoveryOwnerLease,
|
||||
request,
|
||||
cfg,
|
||||
cfgForAgent,
|
||||
@@ -462,6 +475,7 @@ export const agentRunHandler: GatewayRequestHandlers["agent"] = async ({
|
||||
isNewSession,
|
||||
isRawModelRun,
|
||||
isOneShotModelRun,
|
||||
isRestartRecoveryResumeRun,
|
||||
suppressVisibleSessionEffects,
|
||||
message,
|
||||
images,
|
||||
@@ -490,11 +504,28 @@ export const agentRunHandler: GatewayRequestHandlers["agent"] = async ({
|
||||
respond,
|
||||
releaseCronContinuationClaimWithRecovery,
|
||||
});
|
||||
mainRestartRecoveryOwnerLease = undefined;
|
||||
} finally {
|
||||
if (!gatewayAdmissionTransferred) {
|
||||
releaseGatewayAdmission();
|
||||
await releaseCronContinuationClaimWithRecovery();
|
||||
try {
|
||||
if (!gatewayAdmissionTransferred) {
|
||||
let pendingRecovery: Awaited<ReturnType<typeof releaseMainSessionRecoveryOwner>> =
|
||||
undefined;
|
||||
try {
|
||||
pendingRecovery = await releaseMainSessionRecoveryOwner(mainRestartRecoveryOwnerLease);
|
||||
} finally {
|
||||
try {
|
||||
releaseGatewayAdmission();
|
||||
} finally {
|
||||
try {
|
||||
await releaseCronContinuationClaimWithRecovery();
|
||||
} finally {
|
||||
scheduleMainSessionRecoveryPendingTarget(pendingRecovery);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
clearUnacceptedAgentDedupe();
|
||||
}
|
||||
clearUnacceptedAgentDedupe();
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
|
||||
import { resolveTrustedGroupId } from "../../agents/agent-tools.policy.js";
|
||||
import { clearAllCliSessions } from "../../agents/cli-session.js";
|
||||
import { buildMainSessionRecoveryClearPatch } from "../../agents/main-session-recovery-clear.js";
|
||||
import {
|
||||
evaluateSessionFreshness,
|
||||
hasTerminalMainSessionTranscriptNewerThanRegistrySync,
|
||||
@@ -201,11 +202,15 @@ export function buildAgentSessionPatch(params: {
|
||||
freshRecoverableTerminalSession &&
|
||||
!freshSessionRotatedSinceLoad &&
|
||||
patchSessionId === params.freshEntry?.sessionId;
|
||||
const automaticRecoveryClearPatch = shouldClearRotatedState
|
||||
? buildMainSessionRecoveryClearPatch(params.freshEntry)
|
||||
: {};
|
||||
const patch: Partial<SessionEntry> = {
|
||||
sessionId: patchSessionId,
|
||||
updatedAt: params.now,
|
||||
...(freshIsNewSession && !freshSessionRotatedSinceLoad ? { sessionStartedAt: params.now } : {}),
|
||||
...(params.touchInteraction ? { lastInteractionAt: params.now } : {}),
|
||||
...automaticRecoveryClearPatch,
|
||||
...(effectiveDeliveryFields.route ? { route: effectiveDeliveryFields.route } : {}),
|
||||
...(effectiveDeliveryFields.deliveryContext
|
||||
? { deliveryContext: effectiveDeliveryFields.deliveryContext }
|
||||
|
||||
@@ -1,10 +1,17 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { normalizeOptionalString } from "@openclaw/normalization-core/string-coerce";
|
||||
import { ErrorCodes, errorShape } from "../../../packages/gateway-protocol/src/index.js";
|
||||
import {
|
||||
isMainSessionRecoveryExhausted,
|
||||
transitionMainSessionRecovery,
|
||||
} from "../../agents/main-session-recovery-state.js";
|
||||
import type { MainSessionRecoveryOwnerLease } from "../../agents/main-session-recovery-store.js";
|
||||
import {
|
||||
mergeSessionEntry,
|
||||
resolveSessionLifecycleTimestamps,
|
||||
resolveSessionWorkStartError,
|
||||
type SessionEntry,
|
||||
type InternalSessionEntry,
|
||||
type SessionFreshness,
|
||||
} from "../../config/sessions.js";
|
||||
import {
|
||||
@@ -69,6 +76,7 @@ export async function persistAgentSessionPhase(params: {
|
||||
sessionAgentId: string;
|
||||
mainSessionKey: string;
|
||||
lifecycleGeneration: string;
|
||||
isRestartRecoveryResumeRun: boolean;
|
||||
runId: string;
|
||||
agentId?: string;
|
||||
suppressVisibleSessionEffects: boolean;
|
||||
@@ -98,6 +106,7 @@ export async function persistAgentSessionPhase(params: {
|
||||
}) => void;
|
||||
getAdmittedSessionId: () => string;
|
||||
setCronContinuationClaim: (claim: CronContinuationClaim) => void;
|
||||
setMainRestartRecoveryOwnerLease: (lease: MainSessionRecoveryOwnerLease) => void;
|
||||
respond: GatewayRequestHandlerOptions["respond"];
|
||||
}): Promise<AgentSessionPersistResult | undefined> {
|
||||
let patchBuild = params.initialPatchBuild;
|
||||
@@ -106,6 +115,7 @@ export async function persistAgentSessionPhase(params: {
|
||||
let sessionPersistedBeforeGatewayAdmission = params.initialSessionPersistedBeforeGatewayAdmission;
|
||||
let supersededSessionId = params.initialSupersededSessionId;
|
||||
let restoredCronContinuation: RestoredCronContinuation | undefined;
|
||||
let mainRestartRecoveryOwnerLease: MainSessionRecoveryOwnerLease | undefined;
|
||||
let skipAgentInitialSessionTouch = false;
|
||||
const recoveredSessionStartedAt =
|
||||
!patchBuild.isNewSession &&
|
||||
@@ -133,6 +143,7 @@ export async function persistAgentSessionPhase(params: {
|
||||
let archivedDuringStoreUpdateError: string | undefined;
|
||||
let deletedDuringStoreUpdateError: string | undefined;
|
||||
let restoredCronContinuationError: string | undefined;
|
||||
let restartRecoveryReservationConflict: string | undefined;
|
||||
try {
|
||||
persisted =
|
||||
(await patchSessionEntryTarget(
|
||||
@@ -164,6 +175,18 @@ export async function persistAgentSessionPhase(params: {
|
||||
archivedDuringStoreUpdateError = archivedError;
|
||||
throw new Error(archivedError);
|
||||
}
|
||||
const internalFreshEntry = freshEntry as InternalSessionEntry | undefined;
|
||||
if (
|
||||
!params.isRestartRecoveryResumeRun &&
|
||||
internalFreshEntry &&
|
||||
(internalFreshEntry.mainRestartRecovery?.tombstone ||
|
||||
isMainSessionRecoveryExhausted(internalFreshEntry))
|
||||
) {
|
||||
restartRecoveryReservationConflict =
|
||||
`Session "${params.canonicalSessionKey}" is quarantined after restart recovery ` +
|
||||
"exhaustion; use /new or /reset before starting new work.";
|
||||
throw new Error(restartRecoveryReservationConflict);
|
||||
}
|
||||
let entryForPatch = freshEntry;
|
||||
if (params.restoredCronContinuationIdentity) {
|
||||
const marker = freshEntry?.cronRunContinuation;
|
||||
@@ -235,6 +258,38 @@ export async function persistAgentSessionPhase(params: {
|
||||
sessionKey: params.canonicalSessionKey,
|
||||
storePath: params.storePath,
|
||||
});
|
||||
const recoveryTransition = params.isRestartRecoveryResumeRun
|
||||
? transitionMainSessionRecovery(merged as InternalSessionEntry, {
|
||||
kind: "validate_recovery",
|
||||
lifecycleGeneration: params.lifecycleGeneration,
|
||||
runId: params.runId,
|
||||
sessionId: params.request.expectedExistingSessionId ?? merged.sessionId,
|
||||
})
|
||||
: transitionMainSessionRecovery(merged as InternalSessionEntry, {
|
||||
kind: "claim_foreground",
|
||||
cycleId: randomUUID(),
|
||||
lifecycleGeneration: params.lifecycleGeneration,
|
||||
sessionId: merged.sessionId,
|
||||
sessionKey: params.canonicalSessionKey,
|
||||
claimId: mainRestartRecoveryOwnerLease?.claimId ?? randomUUID(),
|
||||
runId: params.runId,
|
||||
});
|
||||
if (
|
||||
params.isRestartRecoveryResumeRun &&
|
||||
recoveryTransition.kind !== "recovery_validated"
|
||||
) {
|
||||
restartRecoveryReservationConflict =
|
||||
`Session "${params.canonicalSessionKey}" restart recovery reservation is stale; ` +
|
||||
"recovery was skipped.";
|
||||
throw new Error(restartRecoveryReservationConflict);
|
||||
}
|
||||
if (recoveryTransition.kind === "foreground_claimed") {
|
||||
mainRestartRecoveryOwnerLease = {
|
||||
...recoveryTransition.claim,
|
||||
storePath: params.storePath,
|
||||
};
|
||||
params.setMainRestartRecoveryOwnerLease(mainRestartRecoveryOwnerLease);
|
||||
}
|
||||
if (
|
||||
params.request.deliver === true &&
|
||||
resolveSendPolicy({
|
||||
@@ -291,6 +346,14 @@ export async function persistAgentSessionPhase(params: {
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
if (restartRecoveryReservationConflict) {
|
||||
params.respond(
|
||||
false,
|
||||
undefined,
|
||||
errorShape(ErrorCodes.UNAVAILABLE, restartRecoveryReservationConflict),
|
||||
);
|
||||
return undefined;
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
if (
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
import { expectDefined } from "@openclaw/normalization-core";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { registerExecApprovalFollowupRuntimeHandoff } from "../../agents/bash-tools.exec-approval-followup-state.js";
|
||||
import type { InternalSessionEntry as SessionEntry } from "../../config/sessions.js";
|
||||
import { runExclusiveSessionLifecycleMutation } from "../../sessions/session-lifecycle-admission.js";
|
||||
import { setGatewayDedupeEntry } from "./agent-job.js";
|
||||
import {
|
||||
getAgentTestMocks,
|
||||
@@ -1998,6 +2000,201 @@ describe("gateway agent handler chat.abort integration", () => {
|
||||
expectRecordFields(errorArgs[3], { runId });
|
||||
});
|
||||
|
||||
it("restores admitted restart recovery if pre-dispatch reactivation fails", async () => {
|
||||
const sessionKey = "agent:main:main";
|
||||
const sessionId = "recovery-session";
|
||||
const runId = "recovery-reactivation-fails";
|
||||
const storePath = "/tmp/sessions.json";
|
||||
const store: Record<string, SessionEntry> = {
|
||||
[sessionKey]: {
|
||||
sessionId,
|
||||
updatedAt: Date.now() - 10_000,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 1,
|
||||
chargedAttempts: 1,
|
||||
reservation: {
|
||||
runId,
|
||||
attempt: 1,
|
||||
lifecycleGeneration: "test-generation",
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
mocks.loadSessionEntry.mockImplementation(() => ({
|
||||
cfg: {},
|
||||
storePath,
|
||||
entry: structuredClone(store[sessionKey]),
|
||||
canonicalKey: sessionKey,
|
||||
}));
|
||||
mocks.updateSessionStore.mockImplementation(async (_path, updater) => await updater(store));
|
||||
mocks.getLatestSubagentRunByChildSessionKey.mockReturnValueOnce({
|
||||
runId: "previous-run",
|
||||
childSessionKey: sessionKey,
|
||||
controllerSessionKey: sessionKey,
|
||||
ownerKey: sessionKey,
|
||||
scopeKind: "session",
|
||||
requesterDisplayKey: "main",
|
||||
task: "old task",
|
||||
cleanup: "keep",
|
||||
createdAt: 1,
|
||||
startedAt: 2,
|
||||
endedAt: 3,
|
||||
outcome: { status: "ok" },
|
||||
});
|
||||
mocks.replaceSubagentRunAfterSteer.mockRejectedValueOnce(new Error("reactivate boom"));
|
||||
|
||||
const respond = vi.fn();
|
||||
await invokeAgent(
|
||||
{
|
||||
message: "resume after restart",
|
||||
agentId: "main",
|
||||
sessionKey,
|
||||
sessionId,
|
||||
expectedExistingSessionId: sessionId,
|
||||
idempotencyKey: runId,
|
||||
inputProvenance: {
|
||||
kind: "internal_system",
|
||||
sourceSessionKey: sessionKey,
|
||||
sourceTool: "main_session_restart_recovery",
|
||||
},
|
||||
},
|
||||
{ client: backendGatewayClient(), reqId: runId, respond },
|
||||
);
|
||||
|
||||
expect(mocks.agentCommand).not.toHaveBeenCalled();
|
||||
expect(store[sessionKey]).toMatchObject({
|
||||
sessionId,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery: {
|
||||
chargedAttempts: 1,
|
||||
},
|
||||
});
|
||||
expect(store[sessionKey]?.mainRestartRecovery?.reservation).toBeUndefined();
|
||||
expect(
|
||||
respond.mock.calls.some(
|
||||
([ok, payload]) =>
|
||||
ok === false && (payload as { runId?: string; status?: string })?.runId === runId,
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("releases a foreground recovery owner if pre-dispatch reactivation fails", async () => {
|
||||
const sessionKey = "agent:main:main";
|
||||
const sessionId = "interrupted-session";
|
||||
const runId = "foreground-reactivation-fails";
|
||||
const storePath = "/tmp/sessions.json";
|
||||
const store: Record<string, SessionEntry> = {
|
||||
[sessionKey]: {
|
||||
sessionId,
|
||||
updatedAt: Date.now() - 10_000,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 1,
|
||||
chargedAttempts: 1,
|
||||
},
|
||||
},
|
||||
};
|
||||
mocks.loadSessionEntry.mockImplementation(() => ({
|
||||
cfg: {},
|
||||
storePath,
|
||||
entry: structuredClone(store[sessionKey]),
|
||||
canonicalKey: sessionKey,
|
||||
}));
|
||||
mocks.updateSessionStore.mockImplementation(async (_path, updater) => await updater(store));
|
||||
mocks.getLatestSubagentRunByChildSessionKey.mockReturnValueOnce({
|
||||
runId: "previous-run",
|
||||
childSessionKey: sessionKey,
|
||||
controllerSessionKey: sessionKey,
|
||||
ownerKey: sessionKey,
|
||||
scopeKind: "session",
|
||||
requesterDisplayKey: "main",
|
||||
task: "old task",
|
||||
cleanup: "keep",
|
||||
createdAt: 1,
|
||||
startedAt: 2,
|
||||
endedAt: 3,
|
||||
outcome: { status: "ok" },
|
||||
});
|
||||
mocks.replaceSubagentRunAfterSteer.mockRejectedValueOnce(new Error("reactivate boom"));
|
||||
|
||||
await invokeAgent(
|
||||
{
|
||||
message: "new foreground turn",
|
||||
agentId: "main",
|
||||
sessionKey,
|
||||
sessionId,
|
||||
idempotencyKey: runId,
|
||||
},
|
||||
{ client: backendGatewayClient(), reqId: runId, respond: vi.fn() },
|
||||
);
|
||||
|
||||
expect(mocks.agentCommand).not.toHaveBeenCalled();
|
||||
expect(store[sessionKey]?.mainRestartRecovery?.foregroundClaims).toBeUndefined();
|
||||
});
|
||||
|
||||
it("releases gateway admission when foreground owner cleanup exhausts retries", async () => {
|
||||
const sessionKey = "agent:main:main";
|
||||
const sessionId = "interrupted-session";
|
||||
const runId = "foreground-release-fails";
|
||||
const storePath = "/tmp/sessions.json";
|
||||
const store: Record<string, SessionEntry> = {
|
||||
[sessionKey]: {
|
||||
sessionId,
|
||||
updatedAt: Date.now() - 10_000,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 1,
|
||||
chargedAttempts: 1,
|
||||
},
|
||||
},
|
||||
};
|
||||
mocks.loadSessionEntry.mockImplementation(() => ({
|
||||
cfg: {},
|
||||
storePath,
|
||||
entry: structuredClone(store[sessionKey]),
|
||||
canonicalKey: sessionKey,
|
||||
}));
|
||||
mocks.updateSessionStore.mockImplementation(async (_path, updater) => await updater(store));
|
||||
mocks.applySessionEntryReplacements.mockRejectedValue(new Error("owner release write failed"));
|
||||
|
||||
await expect(
|
||||
invokeAgent(
|
||||
{
|
||||
message: "new foreground turn",
|
||||
agentId: "main",
|
||||
sessionKey,
|
||||
sessionId,
|
||||
deliver: true,
|
||||
replyChannel: "telegram",
|
||||
bestEffortDeliver: false,
|
||||
idempotencyKey: runId,
|
||||
},
|
||||
{
|
||||
client: backendGatewayClient(),
|
||||
reqId: runId,
|
||||
respond: vi.fn(),
|
||||
flushDispatch: false,
|
||||
},
|
||||
),
|
||||
).rejects.toThrow("owner release write failed");
|
||||
await expect(
|
||||
runExclusiveSessionLifecycleMutation({
|
||||
scope: storePath,
|
||||
identities: [sessionKey, sessionId],
|
||||
signal: AbortSignal.timeout(100),
|
||||
run: async () => "released",
|
||||
}),
|
||||
).resolves.toBe("released");
|
||||
});
|
||||
|
||||
it("does not dispatch a duplicate agent run when dedupe was evicted but the run is active", async () => {
|
||||
prime();
|
||||
mocks.agentCommand.mockResolvedValueOnce({
|
||||
|
||||
@@ -1254,6 +1254,48 @@ describe("gateway agent handler", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("clears automatic recovery quarantine state when a user turn rotates the session id", async () => {
|
||||
vi.useFakeTimers({ toFake: ["Date"] });
|
||||
setDateOnlyFakeClockActive(true);
|
||||
vi.setSystemTime(new Date("2026-05-07T12:00:00.000Z"));
|
||||
const staleEntry = {
|
||||
sessionId: "quarantined-session-id",
|
||||
updatedAt: 0,
|
||||
sessionStartedAt: 0,
|
||||
lastInteractionAt: 0,
|
||||
abortedLastRun: true,
|
||||
restartRecoveryRuns: [
|
||||
{ runId: "initial-wedged-run", lifecycleGeneration: "gen-1" },
|
||||
{ runId: "recovery-run-1", lifecycleGeneration: "gen-2" },
|
||||
],
|
||||
mainRestartRecovery: {
|
||||
automaticAttempts: 2,
|
||||
lastAttemptAt: 3,
|
||||
lastRunId: "recovery-run-1",
|
||||
},
|
||||
subagentRecovery: {
|
||||
automaticAttempts: 2,
|
||||
lastAttemptAt: 3,
|
||||
wedgedAt: 4,
|
||||
wedgedReason: "automatic_attempt_budget_exceeded",
|
||||
},
|
||||
};
|
||||
mockMainSessionEntry(staleEntry);
|
||||
|
||||
const capturedEntry = await runMainAgentAndCaptureEntry("test-idem-rotated-recovery-clear");
|
||||
|
||||
expect(capturedEntry.sessionId).not.toBe("quarantined-session-id");
|
||||
expect(capturedEntry.abortedLastRun).toBeUndefined();
|
||||
expect(capturedEntry.restartRecoveryRuns).toBeUndefined();
|
||||
expect(capturedEntry.mainRestartRecovery).toBeUndefined();
|
||||
expect(capturedEntry.subagentRecovery).toEqual({
|
||||
automaticAttempts: 2,
|
||||
lastAttemptAt: 3,
|
||||
wedgedAt: 4,
|
||||
wedgedReason: "automatic_attempt_budget_exceeded",
|
||||
});
|
||||
});
|
||||
|
||||
it("drops a stale transcript path when a stale session rotates ids", async () => {
|
||||
vi.useFakeTimers({ toFake: ["Date"] });
|
||||
setDateOnlyFakeClockActive(true);
|
||||
|
||||
@@ -37,6 +37,7 @@ vi.mock("../../config/config.js", () => ({
|
||||
}));
|
||||
|
||||
vi.mock("../../commands/agent.js", () => ({
|
||||
agentCommandFromGatewayIngress: agentIngressMocks.agentCommandFromIngress,
|
||||
agentCommandFromIngress: agentIngressMocks.agentCommandFromIngress,
|
||||
}));
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ const performGatewaySessionResetMock = vi.hoisted(() => vi.fn());
|
||||
const parseMessageWithAttachmentsMock = vi.hoisted(() => vi.fn());
|
||||
|
||||
vi.mock("../../commands/agent.js", () => ({
|
||||
agentCommandFromGatewayIngress: agentCommandFromIngressMock,
|
||||
agentCommandFromIngress: agentCommandFromIngressMock,
|
||||
}));
|
||||
|
||||
|
||||
@@ -46,6 +46,73 @@ const mocks = getAgentTestMocks();
|
||||
describe("gateway agent handler", () => {
|
||||
afterEach(describe0AfterEach0);
|
||||
|
||||
it("rejects ordinary work on a restart-recovery tombstone", async () => {
|
||||
const entry = {
|
||||
sessionId: "tombstoned-session",
|
||||
updatedAt: Date.now(),
|
||||
status: "failed",
|
||||
abortedLastRun: false,
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-exhausted",
|
||||
revision: 4,
|
||||
chargedAttempts: 3,
|
||||
tombstone: { reason: "automatic recovery exhausted" },
|
||||
},
|
||||
};
|
||||
mockMainSessionEntry(entry);
|
||||
mocks.updateSessionStore.mockImplementation(
|
||||
async (_path, updater) => await updater({ "agent:main:main": structuredClone(entry) }),
|
||||
);
|
||||
const commandCallCount = mocks.agentCommand.mock.calls.length;
|
||||
const respond = vi.fn();
|
||||
|
||||
await invokeAgent(
|
||||
{
|
||||
message: "continue old work",
|
||||
sessionKey: "agent:main:main",
|
||||
idempotencyKey: "tombstone-reuse",
|
||||
},
|
||||
{ reqId: "tombstone-reuse", respond },
|
||||
);
|
||||
|
||||
expect(mocks.agentCommand).toHaveBeenCalledTimes(commandCallCount);
|
||||
const error = expectRespondError(respond, { code: ErrorCodes.UNAVAILABLE });
|
||||
expectStringFieldContains(error, "message", "quarantined after restart recovery exhaustion");
|
||||
});
|
||||
|
||||
it("rejects ordinary work while restart recovery exhaustion is being tombstoned", async () => {
|
||||
const entry = {
|
||||
sessionId: "exhausted-session",
|
||||
updatedAt: Date.now(),
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-exhausted",
|
||||
revision: 4,
|
||||
chargedAttempts: 3,
|
||||
},
|
||||
};
|
||||
mockMainSessionEntry(entry);
|
||||
mocks.updateSessionStore.mockImplementation(
|
||||
async (_path, updater) => await updater({ "agent:main:main": structuredClone(entry) }),
|
||||
);
|
||||
const commandCallCount = mocks.agentCommand.mock.calls.length;
|
||||
const respond = vi.fn();
|
||||
|
||||
await invokeAgent(
|
||||
{
|
||||
message: "continue old work",
|
||||
sessionKey: "agent:main:main",
|
||||
idempotencyKey: "exhausted-reuse",
|
||||
},
|
||||
{ reqId: "exhausted-reuse", respond },
|
||||
);
|
||||
|
||||
expect(mocks.agentCommand).toHaveBeenCalledTimes(commandCallCount);
|
||||
const error = expectRespondError(respond, { code: ErrorCodes.UNAVAILABLE });
|
||||
expectStringFieldContains(error, "message", "quarantined after restart recovery exhaustion");
|
||||
});
|
||||
|
||||
it("does not restore elevated defaults from idempotency key suffixes", async () => {
|
||||
const bashElevated = {
|
||||
enabled: true,
|
||||
|
||||
@@ -130,6 +130,7 @@ vi.mock("../../config/sessions/session-accessor.js", async () => {
|
||||
|
||||
vi.mock("../../commands/agent.js", () => ({
|
||||
agentCommand: mocks.agentCommand,
|
||||
agentCommandFromGatewayIngress: mocks.agentCommand,
|
||||
agentCommandFromIngress: mocks.agentCommand,
|
||||
}));
|
||||
|
||||
@@ -677,7 +678,7 @@ export async function runMainAgentAndCaptureEntry(idempotencyKey: string) {
|
||||
[canonicalKey]: existingEntry,
|
||||
};
|
||||
const result = await updater(store);
|
||||
capturedEntry = result as Record<string, unknown>;
|
||||
capturedEntry = structuredClone(store[canonicalKey]) as Record<string, unknown>;
|
||||
return result;
|
||||
});
|
||||
mocks.agentCommand.mockResolvedValue({
|
||||
|
||||
@@ -37,12 +37,6 @@ const hoisted = vi.hoisted(() => {
|
||||
marked: 0,
|
||||
skipped: 0,
|
||||
}));
|
||||
const recoverStartupOrphanedMainSessions = vi.fn(async () => ({
|
||||
marked: 0,
|
||||
recovered: 0,
|
||||
failed: 0,
|
||||
skipped: 0,
|
||||
}));
|
||||
const scheduleRestartAbortedMainSessionRecovery = vi.fn();
|
||||
const scheduleRestartSentinelWake =
|
||||
vi.fn<typeof import("./server-restart-sentinel.js").scheduleRestartSentinelWake>();
|
||||
@@ -96,7 +90,6 @@ const hoisted = vi.hoisted(() => {
|
||||
scheduleSubagentOrphanRecovery,
|
||||
markRestartAbortedMainSessionsFromLocks,
|
||||
markStartupOrphanedMainSessionsForRecovery,
|
||||
recoverStartupOrphanedMainSessions,
|
||||
scheduleRestartAbortedMainSessionRecovery,
|
||||
scheduleRestartSentinelWake,
|
||||
refreshLatestUpdateRestartSentinel,
|
||||
@@ -133,7 +126,6 @@ vi.mock("../agents/subagent-registry.js", () => ({
|
||||
vi.mock("../agents/main-session-restart-recovery.js", () => ({
|
||||
markRestartAbortedMainSessionsFromLocks: hoisted.markRestartAbortedMainSessionsFromLocks,
|
||||
markStartupOrphanedMainSessionsForRecovery: hoisted.markStartupOrphanedMainSessionsForRecovery,
|
||||
recoverStartupOrphanedMainSessions: hoisted.recoverStartupOrphanedMainSessions,
|
||||
scheduleRestartAbortedMainSessionRecovery: hoisted.scheduleRestartAbortedMainSessionRecovery,
|
||||
}));
|
||||
|
||||
@@ -343,13 +335,6 @@ describe("startGatewayPostAttachRuntime", () => {
|
||||
marked: 0,
|
||||
skipped: 0,
|
||||
});
|
||||
hoisted.recoverStartupOrphanedMainSessions.mockReset();
|
||||
hoisted.recoverStartupOrphanedMainSessions.mockResolvedValue({
|
||||
marked: 0,
|
||||
recovered: 0,
|
||||
failed: 0,
|
||||
skipped: 0,
|
||||
});
|
||||
hoisted.scheduleRestartAbortedMainSessionRecovery.mockClear();
|
||||
hoisted.scheduleRestartSentinelWake.mockClear();
|
||||
hoisted.refreshLatestUpdateRestartSentinel.mockReset();
|
||||
|
||||
@@ -4,6 +4,7 @@ import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { expect, test, vi } from "vitest";
|
||||
import { listSessionEntries, loadSessionEntry } from "../config/sessions/session-accessor.js";
|
||||
import type { InternalSessionEntry } from "../config/sessions/types.js";
|
||||
import { beginSessionWorkAdmission } from "../sessions/session-lifecycle-admission.js";
|
||||
import { embeddedRunMock, testState, writeSessionStore } from "./test-helpers.js";
|
||||
import {
|
||||
@@ -350,6 +351,44 @@ test("sessions.reset emits internal command hook with reason", async () => {
|
||||
expect(event.context?.previousSessionEntry?.sessionId).toBe("sess-main");
|
||||
});
|
||||
|
||||
test("sessions.reset removes automatic recovery state from the replacement session", async () => {
|
||||
const { dir } = await createSessionStoreDir();
|
||||
await writeSingleLineSession(dir, "sess-recovery", "hello");
|
||||
await writeMainSessionEntry("sess-recovery", {
|
||||
abortedLastRun: true,
|
||||
restartRecoveryRuns: [{ runId: "recovery-run", lifecycleGeneration: "generation-1" }],
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 5,
|
||||
chargedAttempts: 3,
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration: "generation-1",
|
||||
tokens: ["foreground-owner"],
|
||||
},
|
||||
tombstone: {
|
||||
reason: "exhausted",
|
||||
},
|
||||
},
|
||||
subagentRecovery: {
|
||||
automaticAttempts: 2,
|
||||
lastAttemptAt: 10,
|
||||
lastRunId: "child-recovery-run",
|
||||
wedgedAt: 20,
|
||||
wedgedReason: "child exhausted",
|
||||
},
|
||||
});
|
||||
|
||||
await resetMainSession();
|
||||
|
||||
const store = await loadGatewaySessionStoreForKey("main");
|
||||
const replacement = store["agent:main:main"];
|
||||
expect(replacement?.sessionId).not.toBe("sess-recovery");
|
||||
expect(replacement?.abortedLastRun).toBe(false);
|
||||
expect(replacement?.restartRecoveryRuns).toBeUndefined();
|
||||
expect((replacement as InternalSessionEntry | undefined)?.mainRestartRecovery).toBeUndefined();
|
||||
expect(replacement?.subagentRecovery).toBeUndefined();
|
||||
});
|
||||
|
||||
test("sessions.reset does not begin cleanup after losing lifecycle ownership", async () => {
|
||||
const { dir } = await createSessionStoreDir();
|
||||
await writeSingleLineSession(dir, "sess-main", "hello");
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
import { buildMainSessionRecoveryClearPatch } from "../agents/main-session-recovery-clear.js";
|
||||
import type { SessionEntry } from "../config/sessions.js";
|
||||
|
||||
export function buildCheckpointSessionResetPatch(params: {
|
||||
entry: SessionEntry;
|
||||
sessionId: string;
|
||||
sessionFile: string;
|
||||
}): Partial<SessionEntry> {
|
||||
// A checkpoint clone has a new transcript identity, so prior recovery ownership cannot follow it.
|
||||
return {
|
||||
...buildMainSessionRecoveryClearPatch(params.entry),
|
||||
sessionId: params.sessionId,
|
||||
sessionFile: params.sessionFile,
|
||||
updatedAt: Date.now(),
|
||||
systemSent: false,
|
||||
abortedLastRun: false,
|
||||
startedAt: undefined,
|
||||
endedAt: undefined,
|
||||
runtimeMs: undefined,
|
||||
status: undefined,
|
||||
};
|
||||
}
|
||||
@@ -33,6 +33,7 @@ import { scanSessionTranscriptTree } from "../config/sessions/transcript-tree.js
|
||||
import { CURRENT_SESSION_VERSION } from "../config/sessions/version.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import { createSubsystemLogger } from "../logging/subsystem.js";
|
||||
import { buildCheckpointSessionResetPatch } from "./session-compaction-checkpoint-entry.js";
|
||||
import { resolveGatewaySessionStoreTarget } from "./session-utils.js";
|
||||
|
||||
const log = createSubsystemLogger("gateway/session-compaction-checkpoints");
|
||||
@@ -587,15 +588,11 @@ function cloneCheckpointSessionEntry(params: {
|
||||
}): SessionEntry {
|
||||
return {
|
||||
...params.currentEntry,
|
||||
sessionId: params.nextSessionId,
|
||||
sessionFile: params.nextSessionFile,
|
||||
updatedAt: Date.now(),
|
||||
systemSent: false,
|
||||
abortedLastRun: false,
|
||||
startedAt: undefined,
|
||||
endedAt: undefined,
|
||||
runtimeMs: undefined,
|
||||
status: undefined,
|
||||
...buildCheckpointSessionResetPatch({
|
||||
entry: params.currentEntry,
|
||||
sessionId: params.nextSessionId,
|
||||
sessionFile: params.nextSessionFile,
|
||||
}),
|
||||
inputTokens: undefined,
|
||||
outputTokens: undefined,
|
||||
cacheRead: undefined,
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import { buildMainSessionRecoveryClearPatch } from "../agents/main-session-recovery-clear.js";
|
||||
import type { SessionEntry } from "../config/sessions.js";
|
||||
|
||||
export function buildForkedGatewaySessionEntry(
|
||||
entry: SessionEntry,
|
||||
fork: { sessionId: string; sessionFile: string },
|
||||
): SessionEntry {
|
||||
// Replacing the transcript identity also replaces the recovery episode owned by the old row.
|
||||
return {
|
||||
...entry,
|
||||
...buildMainSessionRecoveryClearPatch(entry),
|
||||
sessionId: fork.sessionId,
|
||||
sessionFile: fork.sessionFile,
|
||||
forkedFromParent: true,
|
||||
totalTokens: undefined,
|
||||
totalTokensFresh: false,
|
||||
};
|
||||
}
|
||||
@@ -51,6 +51,7 @@ import {
|
||||
runExclusiveSessionLifecycleMutation,
|
||||
} from "../sessions/session-lifecycle-admission.js";
|
||||
import { createLazyRuntimeModule } from "../shared/lazy-runtime.js";
|
||||
import { buildForkedGatewaySessionEntry } from "./session-create-fork-entry.js";
|
||||
import { resolveSessionStoreAgentId, resolveSessionStoreKey } from "./session-store-key.js";
|
||||
import { loadSessionEntry, resolveGatewaySessionStoreTarget } from "./session-utils.js";
|
||||
import { applySessionsPatchToStore } from "./sessions-patch.js";
|
||||
@@ -700,14 +701,7 @@ export async function createGatewaySession(params: {
|
||||
}
|
||||
return {
|
||||
...initialized,
|
||||
entry: {
|
||||
...entry,
|
||||
sessionId: fork.sessionId,
|
||||
sessionFile: fork.sessionFile,
|
||||
forkedFromParent: true,
|
||||
totalTokens: undefined,
|
||||
totalTokensFresh: false,
|
||||
},
|
||||
entry: buildForkedGatewaySessionEntry(entry, fork),
|
||||
};
|
||||
},
|
||||
params.initialEntry
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
* Session lifecycle state derivation tests.
|
||||
*/
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { SessionEntry } from "../config/sessions.js";
|
||||
import type { InternalSessionEntry as SessionEntry } from "../config/sessions.js";
|
||||
import { getAgentEventLifecycleGeneration } from "../infra/agent-events.js";
|
||||
|
||||
const persistenceMocks = vi.hoisted(() => ({
|
||||
loadSessionEntry: vi.fn(),
|
||||
@@ -258,6 +259,11 @@ describe("session lifecycle state", () => {
|
||||
});
|
||||
|
||||
it("preserves recovery state for a late interrupted-run event", async () => {
|
||||
const mainRestartRecovery = {
|
||||
cycleId: "cycle-1",
|
||||
revision: 2,
|
||||
chargedAttempts: 2,
|
||||
};
|
||||
const persisted = await persistLifecycle(
|
||||
{
|
||||
sessionId: "session-id",
|
||||
@@ -265,6 +271,7 @@ describe("session lifecycle state", () => {
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
restartRecoveryRuns: [{ runId: "restart-run", lifecycleGeneration: "pre-restart" }],
|
||||
mainRestartRecovery,
|
||||
},
|
||||
{
|
||||
ts: 2_000,
|
||||
@@ -279,6 +286,123 @@ describe("session lifecycle state", () => {
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
restartRecoveryRuns: [{ runId: "restart-run", lifecycleGeneration: "pre-restart" }],
|
||||
mainRestartRecovery,
|
||||
});
|
||||
});
|
||||
|
||||
it("ignores an unidentified completion while recovery remains pending", async () => {
|
||||
const persisted = await persistLifecycle(
|
||||
{
|
||||
sessionId: "session-id",
|
||||
updatedAt: 1_000,
|
||||
startedAt: 1_050,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
restartRecoveryRuns: [{ runId: "restart-run", lifecycleGeneration: "pre-restart" }],
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 2,
|
||||
chargedAttempts: 2,
|
||||
},
|
||||
},
|
||||
{
|
||||
ts: 2_000,
|
||||
sessionId: "session-id",
|
||||
data: { phase: "end", endedAt: 1_800 },
|
||||
},
|
||||
);
|
||||
|
||||
expect(persisted).toMatchObject({
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
});
|
||||
expect(persisted.restartRecoveryRuns).toEqual([
|
||||
{ runId: "restart-run", lifecycleGeneration: "pre-restart" },
|
||||
]);
|
||||
expect(persisted.mainRestartRecovery).toMatchObject({ cycleId: "cycle-1" });
|
||||
});
|
||||
|
||||
it("applies the terminal snapshot for the foreground owner run", async () => {
|
||||
const lifecycleGeneration = getAgentEventLifecycleGeneration();
|
||||
const persisted = await persistLifecycle(
|
||||
{
|
||||
sessionId: "session-id",
|
||||
updatedAt: 1_000,
|
||||
startedAt: 1_050,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
restartRecoveryRuns: [
|
||||
{ runId: "interrupted-run", lifecycleGeneration: "pre-restart" },
|
||||
{ runId: "foreground-run", lifecycleGeneration },
|
||||
],
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 2,
|
||||
chargedAttempts: 2,
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration,
|
||||
tokens: ["owner-claim"],
|
||||
runIdsByClaimId: { "owner-claim": "foreground-run" },
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
ts: 2_000,
|
||||
sessionId: "session-id",
|
||||
runId: "foreground-run",
|
||||
lifecycleGeneration,
|
||||
data: { phase: "end", endedAt: 1_800 },
|
||||
},
|
||||
);
|
||||
|
||||
expect(persisted).toMatchObject({
|
||||
status: "done",
|
||||
endedAt: 1_800,
|
||||
abortedLastRun: false,
|
||||
});
|
||||
expect(persisted.restartRecoveryRuns).toBeUndefined();
|
||||
expect(persisted.mainRestartRecovery).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not settle a foreground owner from a stale lifecycle generation", async () => {
|
||||
const persisted = await persistLifecycle(
|
||||
{
|
||||
sessionId: "session-id",
|
||||
updatedAt: 1_000,
|
||||
startedAt: 1_050,
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
restartRecoveryRuns: [
|
||||
{ runId: "interrupted-run", lifecycleGeneration: "pre-restart" },
|
||||
{ runId: "foreground-run", lifecycleGeneration: "pre-restart" },
|
||||
],
|
||||
mainRestartRecovery: {
|
||||
cycleId: "cycle-1",
|
||||
revision: 2,
|
||||
chargedAttempts: 2,
|
||||
foregroundClaims: {
|
||||
lifecycleGeneration: "pre-restart",
|
||||
tokens: ["owner-claim"],
|
||||
runIdsByClaimId: { "owner-claim": "foreground-run" },
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
ts: 2_000,
|
||||
sessionId: "session-id",
|
||||
runId: "foreground-run",
|
||||
lifecycleGeneration: "pre-restart",
|
||||
data: { phase: "end", endedAt: 1_800 },
|
||||
},
|
||||
);
|
||||
|
||||
expect(persisted).toMatchObject({
|
||||
status: "running",
|
||||
abortedLastRun: true,
|
||||
restartRecoveryRuns: [{ runId: "interrupted-run", lifecycleGeneration: "pre-restart" }],
|
||||
mainRestartRecovery: {
|
||||
foregroundClaims: { tokens: ["owner-claim"] },
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -5,9 +5,13 @@ import {
|
||||
buildAgentRunTerminalOutcome,
|
||||
type AgentRunTerminalOutcome,
|
||||
} from "../agents/agent-run-terminal-outcome.js";
|
||||
import type { SessionEntry } from "../config/sessions.js";
|
||||
import {
|
||||
isMainSessionRecoveryLifecycleEvent,
|
||||
projectMainSessionRecoveryLifecycle,
|
||||
} from "../agents/main-session-recovery-lifecycle.js";
|
||||
import type { InternalSessionEntry as SessionEntry } from "../config/sessions.js";
|
||||
import { updateSessionEntry } from "../config/sessions/session-accessor.js";
|
||||
import type { AgentEventPayload } from "../infra/agent-events.js";
|
||||
import { getAgentEventLifecycleGeneration, type AgentEventPayload } from "../infra/agent-events.js";
|
||||
import { parseCronRunScopeSuffix } from "../sessions/session-key-utils.js";
|
||||
import { loadSessionEntry } from "./session-utils.js";
|
||||
import type { GatewaySessionRow, SessionRunStatus } from "./session-utils.types.js";
|
||||
@@ -46,6 +50,7 @@ type PersistedLifecycleSessionShape = Pick<
|
||||
| "runtimeMs"
|
||||
| "abortedLastRun"
|
||||
| "restartRecoveryRuns"
|
||||
| "mainRestartRecovery"
|
||||
>;
|
||||
|
||||
type GatewaySessionLifecycleSnapshot = Partial<LifecycleSessionShape>;
|
||||
@@ -192,37 +197,21 @@ function derivePersistedSessionLifecyclePatch(params: {
|
||||
entry?: Partial<PersistedLifecycleSessionShape> | null;
|
||||
event: LifecycleEventLike;
|
||||
}): Partial<PersistedLifecycleSessionShape> {
|
||||
if (isRestartRecoveryLifecycleEvent(params)) {
|
||||
return {};
|
||||
}
|
||||
const snapshot = deriveGatewaySessionLifecycleSnapshot({
|
||||
session: params.entry ?? undefined,
|
||||
event: params.event,
|
||||
});
|
||||
const patch: Partial<PersistedLifecycleSessionShape> = {
|
||||
const snapshotPatch: Partial<PersistedLifecycleSessionShape> = {
|
||||
...snapshot,
|
||||
updatedAt: typeof snapshot.updatedAt === "number" ? snapshot.updatedAt : undefined,
|
||||
};
|
||||
const runId = params.event.runId?.trim();
|
||||
const lifecycleGeneration = params.event.lifecycleGeneration?.trim();
|
||||
const restartRecoveryRuns = params.entry?.restartRecoveryRuns;
|
||||
if (
|
||||
resolveLifecyclePhase(params.event) !== "start" &&
|
||||
runId &&
|
||||
lifecycleGeneration &&
|
||||
restartRecoveryRuns?.some(
|
||||
(run) => run.runId === runId && run.lifecycleGeneration === lifecycleGeneration,
|
||||
)
|
||||
) {
|
||||
const remainingRuns = restartRecoveryRuns.filter(
|
||||
(run) => run.runId !== runId || run.lifecycleGeneration !== lifecycleGeneration,
|
||||
);
|
||||
if (remainingRuns.length > 0) {
|
||||
return { restartRecoveryRuns: remainingRuns };
|
||||
}
|
||||
patch.restartRecoveryRuns = undefined;
|
||||
}
|
||||
return patch;
|
||||
const projection = projectMainSessionRecoveryLifecycle({
|
||||
currentLifecycleGeneration: getAgentEventLifecycleGeneration(),
|
||||
entry: params.entry,
|
||||
event: params.event,
|
||||
snapshotPatch,
|
||||
});
|
||||
return projection.action === "suppress" ? {} : projection.patch;
|
||||
}
|
||||
|
||||
export function deriveGatewaySessionLifecycleProjectionPatch(params: {
|
||||
@@ -238,21 +227,7 @@ export function isRestartRecoveryLifecycleEvent(params: {
|
||||
entry?: Pick<SessionEntry, "restartRecoveryRuns"> | null;
|
||||
event: Pick<LifecycleEventLike, "runId" | "lifecycleGeneration" | "data">;
|
||||
}): boolean {
|
||||
const runId = params.event.runId?.trim();
|
||||
const lifecycleGeneration = params.event.lifecycleGeneration?.trim();
|
||||
const phase = resolveLifecyclePhase(params.event);
|
||||
const interrupted = params.event.data?.stopReason === "restart";
|
||||
const matchesRecoveryRun = Boolean(
|
||||
runId &&
|
||||
lifecycleGeneration &&
|
||||
params.entry?.restartRecoveryRuns?.some(
|
||||
(run) => run.runId === runId && run.lifecycleGeneration === lifecycleGeneration,
|
||||
),
|
||||
);
|
||||
return (
|
||||
matchesRecoveryRun &&
|
||||
(phase === "start" || ((phase === "end" || phase === "error") && interrupted))
|
||||
);
|
||||
return isMainSessionRecoveryLifecycleEvent(params);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -251,6 +251,7 @@ vi.mock("../commands/status.js", () => ({
|
||||
}));
|
||||
vi.mock("../commands/agent.js", () => ({
|
||||
agentCommand,
|
||||
agentCommandFromGatewayIngress: agentCommand,
|
||||
agentCommandFromIngress: agentCommand,
|
||||
}));
|
||||
vi.mock("../agents/btw.js", () => ({
|
||||
|
||||
@@ -8,7 +8,7 @@ import path from "node:path";
|
||||
import { expectDefined } from "@openclaw/normalization-core";
|
||||
import type { AssistantMessage, UserMessage } from "openclaw/plugin-sdk/llm";
|
||||
import { afterAll, beforeAll, beforeEach, expect, vi } from "vitest";
|
||||
import type { SessionEntry } from "../../config/sessions.js";
|
||||
import type { InternalSessionEntry as SessionEntry } from "../../config/sessions.js";
|
||||
import {
|
||||
loadTranscriptEvents,
|
||||
persistSessionTranscriptTurn,
|
||||
|
||||
@@ -7,4 +7,5 @@ export type * from "../config/types.js";
|
||||
export type { ConfigWriteAfterWrite } from "../config/runtime-snapshot.js";
|
||||
export type { ChannelGroupPolicy } from "../config/group-policy.js";
|
||||
export type { SessionResetMode } from "../config/sessions/reset.js";
|
||||
export type { SessionEntry, SessionScope } from "../config/sessions/types.js";
|
||||
export type { SessionEntry } from "../config/sessions/types.js";
|
||||
export type { SessionScope } from "../config/sessions/types.js";
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
import { MAIN_SESSION_RECOVERY_CLEAR_PATCH } from "../agents/main-session-recovery-clear.js";
|
||||
import type { SessionAccessScope } from "../config/sessions/session-accessor.js";
|
||||
import type { InternalSessionEntry, SessionEntry } from "../config/sessions/types.js";
|
||||
|
||||
export type SessionStoreReadParams = {
|
||||
agentId?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
hydrateSkillPromptRefs?: boolean;
|
||||
readConsistency?: "latest";
|
||||
sessionKey: string;
|
||||
storePath?: string;
|
||||
};
|
||||
|
||||
export function toSessionAccessScope(params: SessionStoreReadParams): SessionAccessScope {
|
||||
// Keep plugin-facing options separate from internal accessor-only controls.
|
||||
return {
|
||||
sessionKey: params.sessionKey,
|
||||
...(params.agentId !== undefined ? { agentId: params.agentId } : {}),
|
||||
...(params.env !== undefined ? { env: params.env } : {}),
|
||||
...(params.hydrateSkillPromptRefs !== undefined
|
||||
? { hydrateSkillPromptRefs: params.hydrateSkillPromptRefs }
|
||||
: {}),
|
||||
...(params.readConsistency !== undefined ? { readConsistency: params.readConsistency } : {}),
|
||||
...(params.storePath !== undefined ? { storePath: params.storePath } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function projectPluginSessionEntry(entry: InternalSessionEntry): SessionEntry {
|
||||
const { mainRestartRecovery: _mainRestartRecovery, ...publicEntry } = entry;
|
||||
return {
|
||||
...publicEntry,
|
||||
...(entry.restartRecoveryRuns
|
||||
? { restartRecoveryRuns: entry.restartRecoveryRuns.map((run) => ({ ...run })) }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function projectPluginSessionEntryPatch(
|
||||
patch: Partial<InternalSessionEntry>,
|
||||
): Partial<SessionEntry> {
|
||||
const { mainRestartRecovery: _mainRestartRecovery, ...publicPatch } = patch;
|
||||
return publicPatch;
|
||||
}
|
||||
|
||||
export function projectPluginSessionStore(
|
||||
store: Record<string, InternalSessionEntry>,
|
||||
): Record<string, SessionEntry> {
|
||||
return Object.fromEntries(
|
||||
Object.entries(store).map(([sessionKey, entry]) => [
|
||||
sessionKey,
|
||||
projectPluginSessionEntry(entry),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
export function activeRecoveryFieldsForSameSession(
|
||||
existingEntry: InternalSessionEntry | undefined,
|
||||
nextSessionId: string | undefined,
|
||||
): Partial<InternalSessionEntry> | undefined {
|
||||
if (
|
||||
!existingEntry ||
|
||||
existingEntry.sessionId !== nextSessionId ||
|
||||
existingEntry.mainRestartRecovery === undefined
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
abortedLastRun: existingEntry.abortedLastRun,
|
||||
restartRecoveryRuns: existingEntry.restartRecoveryRuns,
|
||||
mainRestartRecovery: existingEntry.mainRestartRecovery,
|
||||
};
|
||||
}
|
||||
|
||||
export function clearRecoveryStateForRotatedSessionPatch(
|
||||
existingEntry: InternalSessionEntry,
|
||||
publicPatch: Partial<SessionEntry>,
|
||||
): Partial<InternalSessionEntry> {
|
||||
return Object.hasOwn(publicPatch, "sessionId") &&
|
||||
publicPatch.sessionId !== existingEntry.sessionId
|
||||
? { ...publicPatch, ...MAIN_SESSION_RECOVERY_CLEAR_PATCH }
|
||||
: publicPatch;
|
||||
}
|
||||
|
||||
export function reconcilePluginSessionStore(params: {
|
||||
internalStore: Record<string, InternalSessionEntry>;
|
||||
publicStore: Record<string, SessionEntry>;
|
||||
}): void {
|
||||
for (const sessionKey of Object.keys(params.internalStore)) {
|
||||
if (!Object.hasOwn(params.publicStore, sessionKey)) {
|
||||
delete params.internalStore[sessionKey];
|
||||
}
|
||||
}
|
||||
for (const [sessionKey, publicEntry] of Object.entries(params.publicStore)) {
|
||||
const projectedEntry = projectPluginSessionEntry(publicEntry as InternalSessionEntry);
|
||||
const existingRecovery = activeRecoveryFieldsForSameSession(
|
||||
params.internalStore[sessionKey],
|
||||
projectedEntry.sessionId,
|
||||
);
|
||||
const existingEntry = params.internalStore[sessionKey];
|
||||
params.internalStore[sessionKey] =
|
||||
existingEntry && existingEntry.sessionId !== projectedEntry.sessionId
|
||||
? { ...projectedEntry, ...MAIN_SESSION_RECOVERY_CLEAR_PATCH }
|
||||
: existingRecovery
|
||||
? { ...projectedEntry, ...existingRecovery }
|
||||
: projectedEntry;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
import {
|
||||
patchSessionEntry as patchAccessorSessionEntry,
|
||||
updateSessionEntry,
|
||||
} from "../config/sessions/session-accessor.js";
|
||||
import { normalizeResolvedMaintenanceConfigInput } from "../config/sessions/store-maintenance.js";
|
||||
import type { ResolvedSessionMaintenanceConfigInput } from "../config/sessions/store.js";
|
||||
import type { InternalSessionEntry, SessionEntry } from "../config/sessions/types.js";
|
||||
import {
|
||||
clearRecoveryStateForRotatedSessionPatch,
|
||||
projectPluginSessionEntry,
|
||||
projectPluginSessionEntryPatch,
|
||||
activeRecoveryFieldsForSameSession,
|
||||
type SessionStoreReadParams,
|
||||
toSessionAccessScope,
|
||||
} from "./session-store-runtime-internal.js";
|
||||
|
||||
type SessionStoreEntryUpdate = (
|
||||
entry: SessionEntry,
|
||||
) => Promise<Partial<SessionEntry> | null> | Partial<SessionEntry> | null;
|
||||
|
||||
type SessionStoreEntryPatch = (
|
||||
entry: SessionEntry,
|
||||
context: { existingEntry?: SessionEntry },
|
||||
) => Promise<Partial<SessionEntry> | null> | Partial<SessionEntry> | null;
|
||||
|
||||
type PatchSessionEntryParams = SessionStoreReadParams & {
|
||||
fallbackEntry?: SessionEntry;
|
||||
maintenanceConfig?: ResolvedSessionMaintenanceConfigInput;
|
||||
preserveActivity?: boolean;
|
||||
requireWriteSuccess?: boolean;
|
||||
replaceEntry?: boolean;
|
||||
skipMaintenance?: boolean;
|
||||
update: SessionStoreEntryPatch;
|
||||
};
|
||||
|
||||
type UpdateSessionStoreEntryParams = {
|
||||
storePath: string;
|
||||
sessionKey: string;
|
||||
update: SessionStoreEntryUpdate;
|
||||
skipMaintenance?: boolean;
|
||||
takeCacheOwnership?: boolean;
|
||||
requireWriteSuccess?: boolean;
|
||||
};
|
||||
|
||||
type UpsertSessionEntryParams = SessionStoreReadParams & { entry: SessionEntry };
|
||||
|
||||
/** Patches one public plugin session entry without exposing core recovery coordination. */
|
||||
export async function patchPluginSessionEntry(
|
||||
params: PatchSessionEntryParams,
|
||||
): Promise<SessionEntry | null> {
|
||||
const entry = await patchAccessorSessionEntry(
|
||||
toSessionAccessScope(params),
|
||||
async (internalEntry, context) => {
|
||||
const persistedEntry = internalEntry as InternalSessionEntry;
|
||||
const patch = await params.update(projectPluginSessionEntry(internalEntry), {
|
||||
existingEntry: context.existingEntry
|
||||
? projectPluginSessionEntry(context.existingEntry)
|
||||
: undefined,
|
||||
});
|
||||
if (!patch) {
|
||||
return null;
|
||||
}
|
||||
const publicPatch = projectPluginSessionEntryPatch(patch);
|
||||
const nextSessionId = Object.hasOwn(publicPatch, "sessionId")
|
||||
? publicPatch.sessionId
|
||||
: persistedEntry.sessionId;
|
||||
const existingRecovery = activeRecoveryFieldsForSameSession(persistedEntry, nextSessionId);
|
||||
return params.replaceEntry
|
||||
? existingRecovery
|
||||
? { ...publicPatch, ...existingRecovery }
|
||||
: clearRecoveryStateForRotatedSessionPatch(persistedEntry, publicPatch)
|
||||
: existingRecovery
|
||||
? { ...publicPatch, ...existingRecovery }
|
||||
: clearRecoveryStateForRotatedSessionPatch(persistedEntry, publicPatch);
|
||||
},
|
||||
{
|
||||
fallbackEntry: params.fallbackEntry
|
||||
? projectPluginSessionEntry(params.fallbackEntry)
|
||||
: undefined,
|
||||
maintenanceConfig:
|
||||
params.maintenanceConfig !== undefined
|
||||
? normalizeResolvedMaintenanceConfigInput(params.maintenanceConfig)
|
||||
: undefined,
|
||||
preserveActivity: params.preserveActivity,
|
||||
requireWriteSuccess: params.requireWriteSuccess,
|
||||
replaceEntry: params.replaceEntry,
|
||||
skipMaintenance: params.skipMaintenance,
|
||||
},
|
||||
);
|
||||
return entry ? projectPluginSessionEntry(entry) : null;
|
||||
}
|
||||
|
||||
/** Updates one public plugin session entry by store path and session key. */
|
||||
export async function updatePluginSessionStoreEntry(
|
||||
params: UpdateSessionStoreEntryParams,
|
||||
): Promise<SessionEntry | null> {
|
||||
const entry = await updateSessionEntry(
|
||||
{ sessionKey: params.sessionKey, storePath: params.storePath },
|
||||
async (internalEntry) => {
|
||||
const patch = await params.update(projectPluginSessionEntry(internalEntry));
|
||||
if (!patch) {
|
||||
return null;
|
||||
}
|
||||
const persistedEntry = internalEntry as InternalSessionEntry;
|
||||
const publicPatch = projectPluginSessionEntryPatch(patch);
|
||||
const nextSessionId = Object.hasOwn(publicPatch, "sessionId")
|
||||
? publicPatch.sessionId
|
||||
: persistedEntry.sessionId;
|
||||
const existingRecovery = activeRecoveryFieldsForSameSession(persistedEntry, nextSessionId);
|
||||
return existingRecovery
|
||||
? { ...publicPatch, ...existingRecovery }
|
||||
: clearRecoveryStateForRotatedSessionPatch(persistedEntry, publicPatch);
|
||||
},
|
||||
{
|
||||
skipMaintenance: params.skipMaintenance,
|
||||
takeCacheOwnership: params.takeCacheOwnership,
|
||||
requireWriteSuccess: params.requireWriteSuccess,
|
||||
},
|
||||
);
|
||||
return entry ? projectPluginSessionEntry(entry) : null;
|
||||
}
|
||||
|
||||
/** Replaces or creates one public plugin session entry. */
|
||||
export async function upsertPluginSessionEntry(params: UpsertSessionEntryParams): Promise<void> {
|
||||
const publicEntry = projectPluginSessionEntry(params.entry);
|
||||
await patchAccessorSessionEntry(
|
||||
toSessionAccessScope(params),
|
||||
(internalEntry) => {
|
||||
const persistedEntry = internalEntry as InternalSessionEntry;
|
||||
const existingRecovery = activeRecoveryFieldsForSameSession(
|
||||
persistedEntry,
|
||||
publicEntry.sessionId,
|
||||
);
|
||||
return existingRecovery
|
||||
? { ...publicEntry, ...existingRecovery }
|
||||
: clearRecoveryStateForRotatedSessionPatch(persistedEntry, publicEntry);
|
||||
},
|
||||
{ fallbackEntry: publicEntry, replaceEntry: true },
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,85 @@
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { useAutoCleanupTempDirTracker } from "../../test/helpers/temp-dir.js";
|
||||
import { loadSessionEntry as loadInternalSessionEntry } from "../config/sessions/session-accessor.js";
|
||||
import {
|
||||
patchSessionEntry,
|
||||
upsertSessionEntry,
|
||||
type SessionEntry,
|
||||
} from "./session-store-runtime.js";
|
||||
|
||||
const tempDirs = useAutoCleanupTempDirTracker(afterEach);
|
||||
|
||||
describe("session-store-runtime recovery boundary", () => {
|
||||
let tempDir: string;
|
||||
let storePath: string;
|
||||
|
||||
beforeEach(() => {
|
||||
tempDir = tempDirs.make("openclaw-sdk-session-recovery-");
|
||||
storePath = path.join(tempDir, "sessions.json");
|
||||
});
|
||||
|
||||
it("allows public recovery fields to change without an active core transaction", async () => {
|
||||
const sessionKey = "agent:main:healthy-public-recovery";
|
||||
await upsertSessionEntry({
|
||||
agentId: "main",
|
||||
sessionKey,
|
||||
storePath,
|
||||
entry: {
|
||||
sessionId: "healthy-session",
|
||||
updatedAt: 10,
|
||||
},
|
||||
});
|
||||
|
||||
await patchSessionEntry({
|
||||
sessionKey,
|
||||
storePath,
|
||||
update: () => ({
|
||||
abortedLastRun: true,
|
||||
restartRecoveryRuns: [{ lifecycleGeneration: "generation-1", runId: "run-1" }],
|
||||
}),
|
||||
});
|
||||
|
||||
expect(loadInternalSessionEntry({ sessionKey, storePath })).toMatchObject({
|
||||
abortedLastRun: true,
|
||||
restartRecoveryRuns: [{ lifecycleGeneration: "generation-1", runId: "run-1" }],
|
||||
sessionId: "healthy-session",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects core recovery state from runtime-escaped creation inputs", async () => {
|
||||
const mainRestartRecovery = {
|
||||
chargedAttempts: 1,
|
||||
cycleId: "cycle-injected",
|
||||
revision: 1,
|
||||
};
|
||||
const patchSessionKey = "agent:main:patch-created";
|
||||
await patchSessionEntry({
|
||||
fallbackEntry: {
|
||||
mainRestartRecovery,
|
||||
sessionId: "patch-created",
|
||||
updatedAt: 10,
|
||||
} as unknown as SessionEntry,
|
||||
sessionKey: patchSessionKey,
|
||||
storePath,
|
||||
update: () => ({ updatedAt: 20 }),
|
||||
});
|
||||
expect(loadInternalSessionEntry({ sessionKey: patchSessionKey, storePath })).not.toHaveProperty(
|
||||
"mainRestartRecovery",
|
||||
);
|
||||
|
||||
const upsertSessionKey = "agent:main:upsert-created";
|
||||
await upsertSessionEntry({
|
||||
entry: {
|
||||
mainRestartRecovery,
|
||||
sessionId: "upsert-created",
|
||||
updatedAt: 10,
|
||||
} as unknown as SessionEntry,
|
||||
sessionKey: upsertSessionKey,
|
||||
storePath,
|
||||
});
|
||||
expect(
|
||||
loadInternalSessionEntry({ sessionKey: upsertSessionKey, storePath }),
|
||||
).not.toHaveProperty("mainRestartRecovery");
|
||||
});
|
||||
});
|
||||
@@ -5,7 +5,12 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
appendTranscriptEvent,
|
||||
appendTranscriptMessage,
|
||||
loadSessionEntry as loadInternalSessionEntry,
|
||||
patchSessionEntry as patchInternalSessionEntry,
|
||||
replaceSessionEntry as replaceInternalSessionEntry,
|
||||
} from "../config/sessions/session-accessor.js";
|
||||
import type { InternalSessionEntry } from "../config/sessions/types.js";
|
||||
import type { SessionEntry as ConfigSessionEntry } from "./config-types.js";
|
||||
import {
|
||||
cleanupSessionLifecycleArtifacts,
|
||||
deleteSessionEntry,
|
||||
@@ -26,6 +31,14 @@ import {
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
const LEGACY_TRANSCRIPT_INSPECTION_MAX_BYTES = 16 * 1024 * 1024;
|
||||
const sessionEntryKeepsRecoveryPrivate: "mainRestartRecovery" extends keyof SessionEntry
|
||||
? false
|
||||
: true = true;
|
||||
const configSessionEntryKeepsRecoveryPrivate: "mainRestartRecovery" extends keyof ConfigSessionEntry
|
||||
? false
|
||||
: true = true;
|
||||
void sessionEntryKeepsRecoveryPrivate;
|
||||
void configSessionEntryKeepsRecoveryPrivate;
|
||||
|
||||
describe("session-store-runtime compatibility surface", () => {
|
||||
let tempDir: string;
|
||||
@@ -145,7 +158,7 @@ describe("session-store-runtime compatibility surface", () => {
|
||||
agentId: "main",
|
||||
sessionsDir: path.dirname(storePath),
|
||||
});
|
||||
expect(transcriptPath).toBe(path.join(tempDir, `${sessionId}.jsonl`));
|
||||
expect(transcriptPath).toBe(path.join(fs.realpathSync(tempDir), `${sessionId}.jsonl`));
|
||||
expect(fs.statSync(transcriptPath).mode & 0o777).toBe(0o600);
|
||||
expect(
|
||||
fs
|
||||
@@ -162,7 +175,9 @@ describe("session-store-runtime compatibility surface", () => {
|
||||
]);
|
||||
const codexSidecarPath = `${transcriptPath}.codex-app-server.json`;
|
||||
fs.writeFileSync(codexSidecarPath, "{}\n", { mode: 0o600 });
|
||||
expect(codexSidecarPath).toBe(path.join(tempDir, `${sessionId}.jsonl.codex-app-server.json`));
|
||||
expect(codexSidecarPath).toBe(
|
||||
path.join(fs.realpathSync(tempDir), `${sessionId}.jsonl.codex-app-server.json`),
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves beta.5 doctor archives after deleting the compatibility row", async () => {
|
||||
@@ -320,6 +335,104 @@ describe("session-store-runtime compatibility surface", () => {
|
||||
expect(fs.existsSync(storePath)).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps recovery state private across deprecated whole-store mutations", async () => {
|
||||
const keptKey = "agent:main:telegram:direct:compat-recovery";
|
||||
const removedKey = "agent:main:telegram:direct:compat-removed";
|
||||
const mainRestartRecovery = {
|
||||
chargedAttempts: 2,
|
||||
cycleId: "compat-cycle",
|
||||
revision: 4,
|
||||
};
|
||||
await seedSessionEntry(keptKey, {
|
||||
abortedLastRun: true,
|
||||
model: "gpt-5.5",
|
||||
restartRecoveryRuns: [{ lifecycleGeneration: "compat-generation", runId: "compat-run" }],
|
||||
sessionId: "compat-recovery-session",
|
||||
updatedAt: 10,
|
||||
});
|
||||
await patchInternalSessionEntry(
|
||||
{ agentId: "main", sessionKey: keptKey, storePath },
|
||||
() => ({ mainRestartRecovery }) as Partial<InternalSessionEntry>,
|
||||
);
|
||||
await seedSessionEntry(removedKey, {
|
||||
sessionId: "compat-removed-session",
|
||||
updatedAt: 10,
|
||||
});
|
||||
|
||||
const compatibilityStore = loadSessionStore(storePath);
|
||||
expect(Object.keys(compatibilityStore)).toContain(keptKey);
|
||||
expect(compatibilityStore[keptKey]).not.toHaveProperty("mainRestartRecovery");
|
||||
await updateSessionStore(
|
||||
storePath,
|
||||
(store) => {
|
||||
expect(store[keptKey]).not.toHaveProperty("mainRestartRecovery");
|
||||
const escapedStore = store as unknown as Record<string, InternalSessionEntry>;
|
||||
escapedStore[keptKey] = {
|
||||
mainRestartRecovery: {
|
||||
chargedAttempts: 99,
|
||||
cycleId: "injected-cycle",
|
||||
revision: 99,
|
||||
},
|
||||
model: "gpt-5.6",
|
||||
sessionId: "compat-recovery-session",
|
||||
updatedAt: 20,
|
||||
};
|
||||
delete escapedStore[removedKey];
|
||||
escapedStore["agent:main:telegram:direct:compat-created"] = {
|
||||
mainRestartRecovery: {
|
||||
chargedAttempts: 1,
|
||||
cycleId: "created-injection",
|
||||
revision: 1,
|
||||
},
|
||||
sessionId: "compat-created-session",
|
||||
updatedAt: 20,
|
||||
};
|
||||
},
|
||||
{ skipMaintenance: true },
|
||||
);
|
||||
|
||||
expect(loadInternalSessionEntry({ sessionKey: keptKey, storePath })).toMatchObject({
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery,
|
||||
model: "gpt-5.6",
|
||||
restartRecoveryRuns: [{ lifecycleGeneration: "compat-generation", runId: "compat-run" }],
|
||||
});
|
||||
expect(loadInternalSessionEntry({ sessionKey: removedKey, storePath })).toBeUndefined();
|
||||
expect(
|
||||
loadInternalSessionEntry({
|
||||
sessionKey: "agent:main:telegram:direct:compat-created",
|
||||
storePath,
|
||||
}),
|
||||
).not.toHaveProperty("mainRestartRecovery");
|
||||
|
||||
await updateSessionStore(
|
||||
storePath,
|
||||
(store) => {
|
||||
const escapedStore = store as unknown as Record<string, InternalSessionEntry>;
|
||||
escapedStore[keptKey] = {
|
||||
...escapedStore[keptKey]!,
|
||||
mainRestartRecovery: {
|
||||
chargedAttempts: 99,
|
||||
cycleId: "replacement-injection",
|
||||
revision: 99,
|
||||
},
|
||||
sessionId: "compat-replacement-session",
|
||||
};
|
||||
},
|
||||
{ skipMaintenance: true },
|
||||
);
|
||||
expect(loadInternalSessionEntry({ sessionKey: keptKey, storePath })).toMatchObject({
|
||||
abortedLastRun: false,
|
||||
sessionId: "compat-replacement-session",
|
||||
});
|
||||
const replacedEntry = loadInternalSessionEntry({
|
||||
sessionKey: keptKey,
|
||||
storePath,
|
||||
}) as InternalSessionEntry | undefined;
|
||||
expect(replacedEntry?.mainRestartRecovery).toBeUndefined();
|
||||
expect(replacedEntry?.restartRecoveryRuns).toBeUndefined();
|
||||
});
|
||||
|
||||
it("serializes compatibility callbacks with concurrent row writes", async () => {
|
||||
const sessionKey = "agent:main:serialized";
|
||||
await seedSessionEntry(sessionKey, {
|
||||
@@ -483,6 +596,242 @@ describe("session-store-runtime compatibility surface", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("hides core recovery state and preserves it across public mutations", async () => {
|
||||
const sessionKey = "agent:main:recovery-owned";
|
||||
const mainRestartRecovery = {
|
||||
chargedAttempts: 1,
|
||||
cycleId: "cycle-1",
|
||||
reservation: {
|
||||
attempt: 1,
|
||||
lifecycleGeneration: "generation-1",
|
||||
runId: "run-1",
|
||||
},
|
||||
revision: 1,
|
||||
};
|
||||
await replaceInternalSessionEntry({ sessionKey, storePath }, {
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery,
|
||||
model: "gpt-5.5",
|
||||
restartRecoveryRuns: [{ lifecycleGeneration: "generation-1", runId: "run-1" }],
|
||||
sessionId: "session-recovery",
|
||||
updatedAt: 10,
|
||||
} as InternalSessionEntry);
|
||||
|
||||
expect(getSessionEntry({ sessionKey, storePath })).not.toHaveProperty("mainRestartRecovery");
|
||||
expect(listSessionEntries({ storePath })[0]?.entry).not.toHaveProperty("mainRestartRecovery");
|
||||
|
||||
await patchSessionEntry({
|
||||
sessionKey,
|
||||
storePath,
|
||||
update: (entry) => {
|
||||
entry.restartRecoveryRuns?.splice(0);
|
||||
return {
|
||||
abortedLastRun: false,
|
||||
mainRestartRecovery: undefined,
|
||||
model: "gpt-5.6",
|
||||
restartRecoveryRuns: undefined,
|
||||
} as unknown as Partial<SessionEntry>;
|
||||
},
|
||||
});
|
||||
expect(loadInternalSessionEntry({ sessionKey, storePath })).toMatchObject({
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery,
|
||||
model: "gpt-5.6",
|
||||
restartRecoveryRuns: [{ lifecycleGeneration: "generation-1", runId: "run-1" }],
|
||||
});
|
||||
|
||||
await updateSessionStoreEntry({
|
||||
sessionKey,
|
||||
storePath,
|
||||
update: () => ({ abortedLastRun: false, restartRecoveryRuns: undefined }),
|
||||
});
|
||||
expect(loadInternalSessionEntry({ sessionKey, storePath })).toMatchObject({
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery,
|
||||
restartRecoveryRuns: [{ lifecycleGeneration: "generation-1", runId: "run-1" }],
|
||||
});
|
||||
|
||||
await upsertSessionEntry({
|
||||
sessionKey,
|
||||
storePath,
|
||||
entry: {
|
||||
sessionId: "session-recovery",
|
||||
updatedAt: 20,
|
||||
},
|
||||
});
|
||||
expect(loadInternalSessionEntry({ sessionKey, storePath })).toMatchObject({
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery,
|
||||
restartRecoveryRuns: [{ lifecycleGeneration: "generation-1", runId: "run-1" }],
|
||||
sessionId: "session-recovery",
|
||||
updatedAt: 20,
|
||||
});
|
||||
expect(loadInternalSessionEntry({ sessionKey, storePath })?.model).toBeUndefined();
|
||||
});
|
||||
|
||||
it("clears core recovery state when public replacements change session identity", async () => {
|
||||
const patchKey = "agent:main:telegram:direct:patch-rotation";
|
||||
const upsertKey = "agent:main:telegram:direct:upsert-rotation";
|
||||
const upsertStorePath = path.join(tempDir, "upsert-sessions.json");
|
||||
const mainRestartRecovery = {
|
||||
chargedAttempts: 1,
|
||||
cycleId: "rotation-cycle",
|
||||
revision: 1,
|
||||
};
|
||||
await seedSessionEntry(patchKey, {
|
||||
abortedLastRun: true,
|
||||
restartRecoveryRuns: [{ lifecycleGeneration: "patch-generation", runId: "patch-run" }],
|
||||
sessionId: "patch-before",
|
||||
updatedAt: 10,
|
||||
});
|
||||
await patchInternalSessionEntry(
|
||||
{ agentId: "main", sessionKey: patchKey, storePath },
|
||||
() =>
|
||||
({
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery,
|
||||
restartRecoveryRuns: [{ lifecycleGeneration: "patch-generation", runId: "patch-run" }],
|
||||
}) as Partial<InternalSessionEntry>,
|
||||
);
|
||||
await upsertSessionEntry({
|
||||
agentId: "main",
|
||||
entry: { sessionId: "upsert-before", updatedAt: 10 },
|
||||
sessionKey: upsertKey,
|
||||
storePath: upsertStorePath,
|
||||
});
|
||||
await patchInternalSessionEntry(
|
||||
{ agentId: "main", sessionKey: upsertKey, storePath: upsertStorePath },
|
||||
() =>
|
||||
({
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery,
|
||||
restartRecoveryRuns: [{ lifecycleGeneration: "upsert-generation", runId: "upsert-run" }],
|
||||
}) as Partial<InternalSessionEntry>,
|
||||
);
|
||||
|
||||
await patchSessionEntry({
|
||||
replaceEntry: true,
|
||||
sessionKey: patchKey,
|
||||
storePath,
|
||||
update: () => ({ sessionId: "patch-after", updatedAt: 20 }),
|
||||
});
|
||||
await upsertSessionEntry({
|
||||
entry: {
|
||||
abortedLastRun: true,
|
||||
restartRecoveryRuns: [{ lifecycleGeneration: "upsert-generation", runId: "upsert-run" }],
|
||||
sessionId: "upsert-after",
|
||||
updatedAt: 20,
|
||||
},
|
||||
sessionKey: upsertKey,
|
||||
storePath: upsertStorePath,
|
||||
});
|
||||
|
||||
expect(loadInternalSessionEntry({ sessionKey: patchKey, storePath })).toMatchObject({
|
||||
abortedLastRun: false,
|
||||
sessionId: "patch-after",
|
||||
});
|
||||
expect(
|
||||
loadInternalSessionEntry({ sessionKey: patchKey, storePath })?.restartRecoveryRuns,
|
||||
).toBeUndefined();
|
||||
expect(loadInternalSessionEntry({ sessionKey: patchKey, storePath })).not.toHaveProperty(
|
||||
"mainRestartRecovery",
|
||||
);
|
||||
expect(
|
||||
loadInternalSessionEntry({ sessionKey: upsertKey, storePath: upsertStorePath }),
|
||||
).toMatchObject({
|
||||
sessionId: "upsert-after",
|
||||
});
|
||||
expect(
|
||||
loadInternalSessionEntry({ sessionKey: upsertKey, storePath: upsertStorePath })
|
||||
?.abortedLastRun,
|
||||
).not.toBe(true);
|
||||
expect(
|
||||
loadInternalSessionEntry({ sessionKey: upsertKey, storePath: upsertStorePath })
|
||||
?.restartRecoveryRuns,
|
||||
).toBeUndefined();
|
||||
expect(
|
||||
loadInternalSessionEntry({ sessionKey: upsertKey, storePath: upsertStorePath }),
|
||||
).not.toHaveProperty("mainRestartRecovery");
|
||||
});
|
||||
|
||||
it("clears core recovery state when public patches change session identity", async () => {
|
||||
const patchKey = "agent:main:telegram:direct:patch-rotation";
|
||||
const updateKey = "agent:main:telegram:direct:update-rotation";
|
||||
const updateStorePath = path.join(tempDir, "update-patch-sessions.json");
|
||||
const mainRestartRecovery = {
|
||||
chargedAttempts: 1,
|
||||
cycleId: "rotation-cycle",
|
||||
revision: 1,
|
||||
};
|
||||
await seedSessionEntry(patchKey, {
|
||||
abortedLastRun: true,
|
||||
restartRecoveryRuns: [{ lifecycleGeneration: "patch-generation", runId: "patch-run" }],
|
||||
sessionId: "patch-before",
|
||||
updatedAt: 10,
|
||||
});
|
||||
await upsertSessionEntry({
|
||||
agentId: "main",
|
||||
entry: { sessionId: "update-before", updatedAt: 10 },
|
||||
sessionKey: updateKey,
|
||||
storePath: updateStorePath,
|
||||
});
|
||||
await patchInternalSessionEntry(
|
||||
{ agentId: "main", sessionKey: patchKey, storePath },
|
||||
() =>
|
||||
({
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery,
|
||||
restartRecoveryRuns: [{ lifecycleGeneration: "patch-generation", runId: "patch-run" }],
|
||||
}) as Partial<InternalSessionEntry>,
|
||||
);
|
||||
await patchInternalSessionEntry(
|
||||
{ agentId: "main", sessionKey: updateKey, storePath: updateStorePath },
|
||||
() =>
|
||||
({
|
||||
abortedLastRun: true,
|
||||
mainRestartRecovery,
|
||||
restartRecoveryRuns: [{ lifecycleGeneration: "update-generation", runId: "update-run" }],
|
||||
}) as Partial<InternalSessionEntry>,
|
||||
);
|
||||
|
||||
await patchSessionEntry({
|
||||
sessionKey: patchKey,
|
||||
skipMaintenance: true,
|
||||
storePath,
|
||||
update: () => ({ sessionId: "patch-after", updatedAt: 20 }),
|
||||
});
|
||||
await updateSessionStoreEntry({
|
||||
sessionKey: updateKey,
|
||||
skipMaintenance: true,
|
||||
storePath: updateStorePath,
|
||||
update: () => ({ sessionId: "update-after", updatedAt: 20 }),
|
||||
});
|
||||
|
||||
expect(loadInternalSessionEntry({ sessionKey: patchKey, storePath })).toMatchObject({
|
||||
abortedLastRun: false,
|
||||
sessionId: "patch-after",
|
||||
});
|
||||
expect(
|
||||
loadInternalSessionEntry({ sessionKey: patchKey, storePath })?.restartRecoveryRuns,
|
||||
).toBeUndefined();
|
||||
expect(loadInternalSessionEntry({ sessionKey: patchKey, storePath })).not.toHaveProperty(
|
||||
"mainRestartRecovery",
|
||||
);
|
||||
expect(
|
||||
loadInternalSessionEntry({ sessionKey: updateKey, storePath: updateStorePath }),
|
||||
).toMatchObject({
|
||||
abortedLastRun: false,
|
||||
sessionId: "update-after",
|
||||
});
|
||||
expect(
|
||||
loadInternalSessionEntry({ sessionKey: updateKey, storePath: updateStorePath })
|
||||
?.restartRecoveryRuns,
|
||||
).toBeUndefined();
|
||||
expect(
|
||||
loadInternalSessionEntry({ sessionKey: updateKey, storePath: updateStorePath }),
|
||||
).not.toHaveProperty("mainRestartRecovery");
|
||||
});
|
||||
|
||||
it("preserves resolved maintenance settings through entry patches", async () => {
|
||||
const staleSessionKey = "agent:main:stale";
|
||||
const activeSessionKey = "agent:main:active";
|
||||
|
||||
@@ -17,13 +17,9 @@ import {
|
||||
loadTranscriptEventsSync as loadAccessorTranscriptEventsSync,
|
||||
listSessionEntries as listAccessorSessionEntries,
|
||||
loadSessionEntry,
|
||||
patchSessionEntry as patchAccessorSessionEntry,
|
||||
readSessionUpdatedAt as readAccessorSessionUpdatedAt,
|
||||
readTranscriptStatsSync as readAccessorTranscriptStatsSync,
|
||||
replaceSessionEntry,
|
||||
resolveTranscriptSessionKeyBySessionId as resolveAccessorTranscriptSessionKeyBySessionId,
|
||||
type SessionAccessScope,
|
||||
updateSessionEntry,
|
||||
} from "../config/sessions/session-accessor.js";
|
||||
import { resolveSqliteTargetFromSessionStorePath } from "../config/sessions/session-sqlite-target.js";
|
||||
import {
|
||||
@@ -31,11 +27,26 @@ import {
|
||||
parseSqliteSessionFileMarker,
|
||||
} from "../config/sessions/sqlite-marker.js";
|
||||
import { resolveSessionStoreEntry as resolveSessionStoreEntryFromStore } from "../config/sessions/store-entry.js";
|
||||
import { normalizeResolvedMaintenanceConfigInput } from "../config/sessions/store-maintenance.js";
|
||||
import type { ResolvedSessionMaintenanceConfigInput } from "../config/sessions/store.js";
|
||||
import type { AmbientTranscriptWatermark, SessionEntry } from "../config/sessions/types.js";
|
||||
import type {
|
||||
AmbientTranscriptWatermark,
|
||||
InternalSessionEntry,
|
||||
SessionEntry,
|
||||
} from "../config/sessions/types.js";
|
||||
import { replaceFileAtomicSync } from "../infra/replace-file.js";
|
||||
import { resolveAgentIdFromSessionKey } from "../routing/session-key.js";
|
||||
import {
|
||||
projectPluginSessionEntry,
|
||||
projectPluginSessionStore,
|
||||
reconcilePluginSessionStore,
|
||||
type SessionStoreReadParams,
|
||||
toSessionAccessScope,
|
||||
} from "./session-store-runtime-internal.js";
|
||||
import {
|
||||
patchPluginSessionEntry,
|
||||
updatePluginSessionStoreEntry,
|
||||
upsertPluginSessionEntry,
|
||||
} from "./session-store-runtime-mutations.js";
|
||||
import type { SessionTranscriptEvent } from "./session-transcript-runtime.js";
|
||||
|
||||
const SQLITE_SESSION_STORE_BACKUP_SUFFIXES = ["", "-wal", "-shm", "-journal"] as const;
|
||||
@@ -45,15 +56,6 @@ const LEGACY_TRANSCRIPT_INSPECTION_MAX_BYTES = 16 * 1024 * 1024;
|
||||
// caller. This map is not a general replacement for target-aware SDK methods.
|
||||
const legacyStoreAgentIds = new Map<string, string>();
|
||||
|
||||
type SessionStoreReadParams = {
|
||||
agentId?: string;
|
||||
env?: NodeJS.ProcessEnv;
|
||||
hydrateSkillPromptRefs?: boolean;
|
||||
readConsistency?: "latest";
|
||||
sessionKey: string;
|
||||
storePath?: string;
|
||||
};
|
||||
|
||||
type SessionStoreListParams = Partial<Omit<SessionStoreReadParams, "sessionKey">>;
|
||||
|
||||
type SessionStoreEntrySummary = {
|
||||
@@ -93,10 +95,6 @@ type PatchSessionEntryParams = SessionStoreReadParams & {
|
||||
update: SessionStoreEntryPatch;
|
||||
};
|
||||
|
||||
type ReadAmbientTranscriptWatermarkParams = SessionStoreReadParams & {
|
||||
key: string;
|
||||
};
|
||||
|
||||
type UpdateSessionStoreEntryParams = {
|
||||
storePath: string;
|
||||
sessionKey: string;
|
||||
@@ -106,8 +104,10 @@ type UpdateSessionStoreEntryParams = {
|
||||
requireWriteSuccess?: boolean;
|
||||
};
|
||||
|
||||
type UpsertSessionEntryParams = SessionStoreReadParams & {
|
||||
entry: SessionEntry;
|
||||
type UpsertSessionEntryParams = SessionStoreReadParams & { entry: SessionEntry };
|
||||
|
||||
type ReadAmbientTranscriptWatermarkParams = SessionStoreReadParams & {
|
||||
key: string;
|
||||
};
|
||||
|
||||
type DeleteSessionEntryParams = SessionStoreReadParams & {
|
||||
@@ -131,21 +131,6 @@ type SessionLifecycleArtifactsCleanupResult = {
|
||||
removedEntries: number;
|
||||
};
|
||||
|
||||
function toSessionAccessScope(params: SessionStoreReadParams): SessionAccessScope {
|
||||
// Maintainer note: keep this adapter narrow so plugin callers retain the
|
||||
// object-parameter API while internal accessor-only options stay private.
|
||||
return {
|
||||
sessionKey: params.sessionKey,
|
||||
...(params.agentId !== undefined ? { agentId: params.agentId } : {}),
|
||||
...(params.env !== undefined ? { env: params.env } : {}),
|
||||
...(params.hydrateSkillPromptRefs !== undefined
|
||||
? { hydrateSkillPromptRefs: params.hydrateSkillPromptRefs }
|
||||
: {}),
|
||||
...(params.readConsistency !== undefined ? { readConsistency: params.readConsistency } : {}),
|
||||
...(params.storePath !== undefined ? { storePath: params.storePath } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveLegacySessionStoreTarget(storePath: string): {
|
||||
agentId?: string;
|
||||
storePath: string;
|
||||
@@ -244,13 +229,14 @@ export function loadSessionStore(
|
||||
hydrateSkillPromptRefs: options.hydrateSkillPromptRefs,
|
||||
}).map(({ sessionKey, entry }) => {
|
||||
const sessionId = entry.sessionId?.trim();
|
||||
if (entry.sessionFile || !sessionId) {
|
||||
return [sessionKey, entry];
|
||||
const projectedEntry = projectPluginSessionEntry(entry as InternalSessionEntry);
|
||||
if (projectedEntry.sessionFile || !sessionId) {
|
||||
return [sessionKey, projectedEntry];
|
||||
}
|
||||
return [
|
||||
sessionKey,
|
||||
{
|
||||
...entry,
|
||||
...projectedEntry,
|
||||
// SQLite does not persist sessionFile. Beta.5 needs a locator only in
|
||||
// this detached projection so its file-based doctor reaches the bridge.
|
||||
sessionFile: formatSqliteSessionFileMarker({
|
||||
@@ -285,9 +271,17 @@ export async function updateSessionStore<T>(
|
||||
storePath: target.storePath,
|
||||
skipMaintenance: options.skipMaintenance,
|
||||
update: async (store) => {
|
||||
const result = await mutator(store);
|
||||
const internalStore = store as Record<string, InternalSessionEntry>;
|
||||
const publicStore = projectPluginSessionStore(internalStore);
|
||||
const result = await mutator(publicStore);
|
||||
const persist = !options.skipSaveWhenResult?.(result);
|
||||
if (persist) {
|
||||
// The deprecated callback owns public row changes and deletions, but
|
||||
// core recovery coordination remains invisible and non-overwritable.
|
||||
reconcilePluginSessionStore({ internalStore, publicStore });
|
||||
}
|
||||
return {
|
||||
persist: !options.skipSaveWhenResult?.(result),
|
||||
persist,
|
||||
result,
|
||||
};
|
||||
},
|
||||
@@ -346,7 +340,8 @@ export function resolveSessionStoreEntry(params: {
|
||||
|
||||
/** Loads one session entry by agent/session identity. */
|
||||
export function getSessionEntry(params: SessionStoreReadParams): SessionEntry | undefined {
|
||||
return loadSessionEntry(toSessionAccessScope(params));
|
||||
const entry = loadSessionEntry(toSessionAccessScope(params));
|
||||
return entry ? projectPluginSessionEntry(entry) : undefined;
|
||||
}
|
||||
|
||||
/** Lists session entries for one agent. */
|
||||
@@ -360,7 +355,10 @@ export function listSessionEntries(
|
||||
? { hydrateSkillPromptRefs: params.hydrateSkillPromptRefs }
|
||||
: {}),
|
||||
...(params.storePath !== undefined ? { storePath: params.storePath } : {}),
|
||||
});
|
||||
}).map(({ sessionKey, entry }) => ({
|
||||
sessionKey,
|
||||
entry: projectPluginSessionEntry(entry),
|
||||
}));
|
||||
}
|
||||
|
||||
/** Reads transcript events for a live SQLite-backed session identity. */
|
||||
@@ -399,17 +397,7 @@ export function resolveTranscriptSessionKeyBySessionId(params: {
|
||||
export async function patchSessionEntry(
|
||||
params: PatchSessionEntryParams,
|
||||
): Promise<SessionEntry | null> {
|
||||
return await patchAccessorSessionEntry(toSessionAccessScope(params), params.update, {
|
||||
fallbackEntry: params.fallbackEntry,
|
||||
maintenanceConfig:
|
||||
params.maintenanceConfig !== undefined
|
||||
? normalizeResolvedMaintenanceConfigInput(params.maintenanceConfig)
|
||||
: undefined,
|
||||
preserveActivity: params.preserveActivity,
|
||||
requireWriteSuccess: params.requireWriteSuccess,
|
||||
replaceEntry: params.replaceEntry,
|
||||
skipMaintenance: params.skipMaintenance,
|
||||
});
|
||||
return await patchPluginSessionEntry(params);
|
||||
}
|
||||
|
||||
/** Reads the last activity timestamp for one session entry. */
|
||||
@@ -430,23 +418,12 @@ export function readAmbientTranscriptWatermark(
|
||||
export async function updateSessionStoreEntry(
|
||||
params: UpdateSessionStoreEntryParams,
|
||||
): Promise<SessionEntry | null> {
|
||||
return await updateSessionEntry(
|
||||
{
|
||||
sessionKey: params.sessionKey,
|
||||
storePath: params.storePath,
|
||||
},
|
||||
params.update,
|
||||
{
|
||||
skipMaintenance: params.skipMaintenance,
|
||||
takeCacheOwnership: params.takeCacheOwnership,
|
||||
requireWriteSuccess: params.requireWriteSuccess,
|
||||
},
|
||||
);
|
||||
return await updatePluginSessionStoreEntry(params);
|
||||
}
|
||||
|
||||
/** Replaces or creates one session entry by agent/session identity. */
|
||||
export async function upsertSessionEntry(params: UpsertSessionEntryParams): Promise<void> {
|
||||
await replaceSessionEntry(toSessionAccessScope(params), params.entry);
|
||||
await upsertPluginSessionEntry(params);
|
||||
}
|
||||
|
||||
/** Deletes one session entry by agent/session identity. */
|
||||
@@ -539,4 +516,5 @@ export {
|
||||
resolveThreadFlag,
|
||||
} from "../config/sessions/reset.js";
|
||||
export { resolveSendPolicy } from "../sessions/send-policy.js";
|
||||
export type { SessionEntry, SessionScope } from "../config/sessions/types.js";
|
||||
export type { SessionEntry } from "../config/sessions/types.js";
|
||||
export type { SessionScope } from "../config/sessions/types.js";
|
||||
|
||||
@@ -289,6 +289,11 @@ describe("plugin session extension SessionEntry projection", () => {
|
||||
description: "bad slot",
|
||||
sessionEntrySlotKey: "updatedAt",
|
||||
});
|
||||
api.registerSessionExtension({
|
||||
namespace: "main-recovery",
|
||||
description: "bad main recovery slot",
|
||||
sessionEntrySlotKey: "mainRestartRecovery",
|
||||
});
|
||||
api.registerSessionExtension({
|
||||
namespace: "recovery",
|
||||
description: "bad fresh-main slot",
|
||||
@@ -305,6 +310,10 @@ describe("plugin session extension SessionEntry projection", () => {
|
||||
pluginId: "slot-collision",
|
||||
message: "sessionEntrySlotKey is reserved by SessionEntry: updatedAt",
|
||||
},
|
||||
{
|
||||
pluginId: "slot-collision",
|
||||
message: "sessionEntrySlotKey is reserved by SessionEntry: mainRestartRecovery",
|
||||
},
|
||||
{
|
||||
pluginId: "slot-collision",
|
||||
message: "sessionEntrySlotKey is reserved by SessionEntry: subagentRecovery",
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
/** Reserves session-entry keys so plugin extension slots cannot collide with core session state. */
|
||||
import type { SessionEntry } from "../config/sessions/types.js";
|
||||
import type { InternalSessionEntry as SessionEntry } from "../config/sessions/types.js";
|
||||
|
||||
const SESSION_ENTRY_RESERVED_SLOT_KEY_LIST = [
|
||||
"__proto__",
|
||||
@@ -33,6 +33,7 @@ const SESSION_ENTRY_RESERVED_SLOT_KEY_LIST = [
|
||||
"subagentControlScope",
|
||||
"inheritedToolDeny",
|
||||
"inheritedToolAllow",
|
||||
"mainRestartRecovery",
|
||||
"subagentRecovery",
|
||||
"pluginOwnerId",
|
||||
"systemSent",
|
||||
|
||||
@@ -17,6 +17,8 @@ export type InputProvenance = {
|
||||
sourceTool?: string;
|
||||
};
|
||||
|
||||
export const MAIN_SESSION_RESTART_RECOVERY_SOURCE_TOOL = "main_session_restart_recovery" as const;
|
||||
|
||||
export const INTER_SESSION_PROMPT_PREFIX_BASE = "[Inter-session message]";
|
||||
const AGENT_MEDIATED_COMPLETION_SOURCE_TOOLS = [
|
||||
"agent_harness_task",
|
||||
@@ -74,6 +76,15 @@ export function isInterSessionInputProvenance(value: unknown): boolean {
|
||||
return normalizeInputProvenance(value)?.kind === "inter_session";
|
||||
}
|
||||
|
||||
export function isMainSessionRestartRecoveryInputProvenance(value: unknown): boolean {
|
||||
const provenance = normalizeInputProvenance(value);
|
||||
return (
|
||||
provenance?.kind === "internal_system" &&
|
||||
normalizeOptionalString(provenance.sourceTool)?.toLowerCase() ===
|
||||
MAIN_SESSION_RESTART_RECOVERY_SOURCE_TOOL
|
||||
);
|
||||
}
|
||||
|
||||
const AGENT_MEDIATED_COMPLETION_SOURCE_TOOL_SET: ReadonlySet<string> = new Set(
|
||||
AGENT_MEDIATED_COMPLETION_SOURCE_TOOLS,
|
||||
);
|
||||
|
||||
@@ -428,6 +428,7 @@ export async function beginSessionWorkAdmission(params: {
|
||||
return createSessionWorkAdmissionHandoff(admission, lease);
|
||||
},
|
||||
release,
|
||||
released: admission.released,
|
||||
run: async <T>(run: () => Promise<T>) => {
|
||||
const current = new Set(CURRENT_SESSION_WORK_ADMISSIONS.getStore());
|
||||
current.add(admission);
|
||||
|
||||
@@ -5,6 +5,7 @@ import { normalizeSessionIdentities } from "./session-lifecycle-identity.js";
|
||||
export type SessionWorkAdmissionLease = {
|
||||
createHandoff: () => string;
|
||||
release: () => void;
|
||||
released: Promise<void>;
|
||||
run: <T>(run: () => Promise<T>) => Promise<T>;
|
||||
};
|
||||
|
||||
|
||||
Reference in New Issue
Block a user