mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 10:16:44 +00:00
feat(cloud-workers): session placement, dispatch, and worker turn routing (#106332)
* feat(gateway-protocol): add session placement schema Closed state discriminator for session execution placement (local/requested/provisioning/syncing/starting/active/draining/reconciling/reclaimed/failed), sessions.dispatch params, and worker-admission transcript/live cursor extensions. Swift protocol models mirror the schema. * feat(state): add worker session placement table worker_session_placements rows carry placement state, transition generation, worker ownership metadata, ACK cursors, and the turn claim columns used for atomic admission. * feat(cloud-workers): add durable placement state machine store SQLite-backed placement store split by concern: state table (placement-state), discriminated record types + shape invariants (placement-record), row codec + CAS transition values (placement-row-codec), atomic turn-claim admission/release/waiters (placement-turn-claims), and lifecycle CAS transitions (placement-store). * feat(cloud-workers): sync workspaces and attach sessions to worker environments Environment service session attachment + turn credentials, tunnel workspace commands over a dedicated SSH runner, and git/plain workspace sync into $HOME/.openclaw-worker/workspaces with an immutable manifest. Symlink escapes are rejected locally before transfer (macOS openrsync stat-fails them opaquely) and again by the remote manifest guard. * feat(worker): run one-shot embedded turns from launch descriptors Worker runtime executes a single embedded turn from a stdin launch descriptor and reports completed/failed/fenced on stdout for the gateway launcher. Terminal lifecycle live events are deferred past the final transcript flush; transcript projection helpers are shared via transcript-message instead of duplicated in the runtime. * feat(cloud-workers): dispatch placements and route worker turns Dispatch service drives local->requested->provisioning->syncing->starting->active with failure teardown (placement-dispatch-failure) and restart/runtime recovery incl. lost-worker reclaim (placement-dispatch-recovery). Worker turn launcher claims the placement turn atomically, builds a windowed launch descriptor (worker-turn-payload), runs the remote one-shot worker, and reconciles the committed transcript; agent runners route turns through the session placement admission provider. * feat(gateway): expose session placement RPCs and startup reconciliation sessions.dispatch RPC with lifecycle admission barriers, operator-facing placement projection on session listings, placement-aware session reset guard, and startup/interval reconciliation wiring for worker placements.
This commit is contained in:
@@ -368,6 +368,7 @@ enum class GatewayMethod(
|
||||
ApprovalGet("approval.get"),
|
||||
ApprovalResolve("approval.resolve"),
|
||||
SessionsSearch("sessions.search"),
|
||||
SessionsDispatch("sessions.dispatch"),
|
||||
}
|
||||
|
||||
enum class GatewayEvent(
|
||||
|
||||
@@ -68,6 +68,19 @@ public enum NodePresenceAliveReason: String, Codable, Sendable {
|
||||
case connect = "connect"
|
||||
}
|
||||
|
||||
public enum SessionPlacementState: String, Codable, Sendable {
|
||||
case local = "local"
|
||||
case requested = "requested"
|
||||
case provisioning = "provisioning"
|
||||
case syncing = "syncing"
|
||||
case starting = "starting"
|
||||
case active = "active"
|
||||
case draining = "draining"
|
||||
case reconciling = "reconciling"
|
||||
case reclaimed = "reclaimed"
|
||||
case failed = "failed"
|
||||
}
|
||||
|
||||
public enum SessionFileKind: String, Codable, Sendable {
|
||||
case modified = "modified"
|
||||
case read = "read"
|
||||
@@ -3332,6 +3345,526 @@ public struct SessionOperationEvent: Codable, Sendable {
|
||||
}
|
||||
}
|
||||
|
||||
public struct LocalSessionPlacement: Codable, Sendable {
|
||||
public let state: String
|
||||
public let generation: Int
|
||||
public let createdatms: Int
|
||||
public let updatedatms: Int
|
||||
public let statechangedatms: Int
|
||||
|
||||
public init(
|
||||
state: String,
|
||||
generation: Int,
|
||||
createdatms: Int,
|
||||
updatedatms: Int,
|
||||
statechangedatms: Int)
|
||||
{
|
||||
self.state = state
|
||||
self.generation = generation
|
||||
self.createdatms = createdatms
|
||||
self.updatedatms = updatedatms
|
||||
self.statechangedatms = statechangedatms
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case state
|
||||
case generation
|
||||
case createdatms = "createdAtMs"
|
||||
case updatedatms = "updatedAtMs"
|
||||
case statechangedatms = "stateChangedAtMs"
|
||||
}
|
||||
}
|
||||
|
||||
public struct RequestedSessionPlacement: Codable, Sendable {
|
||||
public let state: String
|
||||
public let generation: Int
|
||||
public let createdatms: Int
|
||||
public let updatedatms: Int
|
||||
public let statechangedatms: Int
|
||||
|
||||
public init(
|
||||
state: String,
|
||||
generation: Int,
|
||||
createdatms: Int,
|
||||
updatedatms: Int,
|
||||
statechangedatms: Int)
|
||||
{
|
||||
self.state = state
|
||||
self.generation = generation
|
||||
self.createdatms = createdatms
|
||||
self.updatedatms = updatedatms
|
||||
self.statechangedatms = statechangedatms
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case state
|
||||
case generation
|
||||
case createdatms = "createdAtMs"
|
||||
case updatedatms = "updatedAtMs"
|
||||
case statechangedatms = "stateChangedAtMs"
|
||||
}
|
||||
}
|
||||
|
||||
public struct ProvisioningSessionPlacement: Codable, Sendable {
|
||||
public let state: String
|
||||
public let generation: Int
|
||||
public let createdatms: Int
|
||||
public let updatedatms: Int
|
||||
public let statechangedatms: Int
|
||||
public let environmentid: String?
|
||||
|
||||
public init(
|
||||
state: String,
|
||||
generation: Int,
|
||||
createdatms: Int,
|
||||
updatedatms: Int,
|
||||
statechangedatms: Int,
|
||||
environmentid: String? = nil)
|
||||
{
|
||||
self.state = state
|
||||
self.generation = generation
|
||||
self.createdatms = createdatms
|
||||
self.updatedatms = updatedatms
|
||||
self.statechangedatms = statechangedatms
|
||||
self.environmentid = environmentid
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case state
|
||||
case generation
|
||||
case createdatms = "createdAtMs"
|
||||
case updatedatms = "updatedAtMs"
|
||||
case statechangedatms = "stateChangedAtMs"
|
||||
case environmentid = "environmentId"
|
||||
}
|
||||
}
|
||||
|
||||
public struct SyncingSessionPlacement: Codable, Sendable {
|
||||
public let state: String
|
||||
public let generation: Int
|
||||
public let createdatms: Int
|
||||
public let updatedatms: Int
|
||||
public let statechangedatms: Int
|
||||
public let environmentid: String
|
||||
public let workerbundlehash: String
|
||||
|
||||
public init(
|
||||
state: String,
|
||||
generation: Int,
|
||||
createdatms: Int,
|
||||
updatedatms: Int,
|
||||
statechangedatms: Int,
|
||||
environmentid: String,
|
||||
workerbundlehash: String)
|
||||
{
|
||||
self.state = state
|
||||
self.generation = generation
|
||||
self.createdatms = createdatms
|
||||
self.updatedatms = updatedatms
|
||||
self.statechangedatms = statechangedatms
|
||||
self.environmentid = environmentid
|
||||
self.workerbundlehash = workerbundlehash
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case state
|
||||
case generation
|
||||
case createdatms = "createdAtMs"
|
||||
case updatedatms = "updatedAtMs"
|
||||
case statechangedatms = "stateChangedAtMs"
|
||||
case environmentid = "environmentId"
|
||||
case workerbundlehash = "workerBundleHash"
|
||||
}
|
||||
}
|
||||
|
||||
public struct StartingSessionPlacement: Codable, Sendable {
|
||||
public let state: String
|
||||
public let generation: Int
|
||||
public let createdatms: Int
|
||||
public let updatedatms: Int
|
||||
public let statechangedatms: Int
|
||||
public let environmentid: String
|
||||
public let workerbundlehash: String
|
||||
public let workspacebasemanifestref: String
|
||||
public let remoteworkspacedir: String
|
||||
|
||||
public init(
|
||||
state: String,
|
||||
generation: Int,
|
||||
createdatms: Int,
|
||||
updatedatms: Int,
|
||||
statechangedatms: Int,
|
||||
environmentid: String,
|
||||
workerbundlehash: String,
|
||||
workspacebasemanifestref: String,
|
||||
remoteworkspacedir: String)
|
||||
{
|
||||
self.state = state
|
||||
self.generation = generation
|
||||
self.createdatms = createdatms
|
||||
self.updatedatms = updatedatms
|
||||
self.statechangedatms = statechangedatms
|
||||
self.environmentid = environmentid
|
||||
self.workerbundlehash = workerbundlehash
|
||||
self.workspacebasemanifestref = workspacebasemanifestref
|
||||
self.remoteworkspacedir = remoteworkspacedir
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case state
|
||||
case generation
|
||||
case createdatms = "createdAtMs"
|
||||
case updatedatms = "updatedAtMs"
|
||||
case statechangedatms = "stateChangedAtMs"
|
||||
case environmentid = "environmentId"
|
||||
case workerbundlehash = "workerBundleHash"
|
||||
case workspacebasemanifestref = "workspaceBaseManifestRef"
|
||||
case remoteworkspacedir = "remoteWorkspaceDir"
|
||||
}
|
||||
}
|
||||
|
||||
public struct ActiveWorkerSessionPlacement: Codable, Sendable {
|
||||
public let state: String
|
||||
public let generation: Int
|
||||
public let createdatms: Int
|
||||
public let updatedatms: Int
|
||||
public let statechangedatms: Int
|
||||
public let environmentid: String
|
||||
public let activeownerepoch: Int
|
||||
public let workerbundlehash: String
|
||||
public let workspacebasemanifestref: String
|
||||
public let remoteworkspacedir: String
|
||||
public let lasttranscriptackcursor: Int?
|
||||
public let lastliveeventackcursor: Int?
|
||||
|
||||
public init(
|
||||
state: String,
|
||||
generation: Int,
|
||||
createdatms: Int,
|
||||
updatedatms: Int,
|
||||
statechangedatms: Int,
|
||||
environmentid: String,
|
||||
activeownerepoch: Int,
|
||||
workerbundlehash: String,
|
||||
workspacebasemanifestref: String,
|
||||
remoteworkspacedir: String,
|
||||
lasttranscriptackcursor: Int? = nil,
|
||||
lastliveeventackcursor: Int? = nil)
|
||||
{
|
||||
self.state = state
|
||||
self.generation = generation
|
||||
self.createdatms = createdatms
|
||||
self.updatedatms = updatedatms
|
||||
self.statechangedatms = statechangedatms
|
||||
self.environmentid = environmentid
|
||||
self.activeownerepoch = activeownerepoch
|
||||
self.workerbundlehash = workerbundlehash
|
||||
self.workspacebasemanifestref = workspacebasemanifestref
|
||||
self.remoteworkspacedir = remoteworkspacedir
|
||||
self.lasttranscriptackcursor = lasttranscriptackcursor
|
||||
self.lastliveeventackcursor = lastliveeventackcursor
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case state
|
||||
case generation
|
||||
case createdatms = "createdAtMs"
|
||||
case updatedatms = "updatedAtMs"
|
||||
case statechangedatms = "stateChangedAtMs"
|
||||
case environmentid = "environmentId"
|
||||
case activeownerepoch = "activeOwnerEpoch"
|
||||
case workerbundlehash = "workerBundleHash"
|
||||
case workspacebasemanifestref = "workspaceBaseManifestRef"
|
||||
case remoteworkspacedir = "remoteWorkspaceDir"
|
||||
case lasttranscriptackcursor = "lastTranscriptAckCursor"
|
||||
case lastliveeventackcursor = "lastLiveEventAckCursor"
|
||||
}
|
||||
}
|
||||
|
||||
public struct DrainingSessionPlacement: Codable, Sendable {
|
||||
public let state: String
|
||||
public let generation: Int
|
||||
public let createdatms: Int
|
||||
public let updatedatms: Int
|
||||
public let statechangedatms: Int
|
||||
public let environmentid: String
|
||||
public let activeownerepoch: Int
|
||||
public let workerbundlehash: String
|
||||
public let workspacebasemanifestref: String
|
||||
public let remoteworkspacedir: String
|
||||
public let lasttranscriptackcursor: Int?
|
||||
public let lastliveeventackcursor: Int?
|
||||
|
||||
public init(
|
||||
state: String,
|
||||
generation: Int,
|
||||
createdatms: Int,
|
||||
updatedatms: Int,
|
||||
statechangedatms: Int,
|
||||
environmentid: String,
|
||||
activeownerepoch: Int,
|
||||
workerbundlehash: String,
|
||||
workspacebasemanifestref: String,
|
||||
remoteworkspacedir: String,
|
||||
lasttranscriptackcursor: Int? = nil,
|
||||
lastliveeventackcursor: Int? = nil)
|
||||
{
|
||||
self.state = state
|
||||
self.generation = generation
|
||||
self.createdatms = createdatms
|
||||
self.updatedatms = updatedatms
|
||||
self.statechangedatms = statechangedatms
|
||||
self.environmentid = environmentid
|
||||
self.activeownerepoch = activeownerepoch
|
||||
self.workerbundlehash = workerbundlehash
|
||||
self.workspacebasemanifestref = workspacebasemanifestref
|
||||
self.remoteworkspacedir = remoteworkspacedir
|
||||
self.lasttranscriptackcursor = lasttranscriptackcursor
|
||||
self.lastliveeventackcursor = lastliveeventackcursor
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case state
|
||||
case generation
|
||||
case createdatms = "createdAtMs"
|
||||
case updatedatms = "updatedAtMs"
|
||||
case statechangedatms = "stateChangedAtMs"
|
||||
case environmentid = "environmentId"
|
||||
case activeownerepoch = "activeOwnerEpoch"
|
||||
case workerbundlehash = "workerBundleHash"
|
||||
case workspacebasemanifestref = "workspaceBaseManifestRef"
|
||||
case remoteworkspacedir = "remoteWorkspaceDir"
|
||||
case lasttranscriptackcursor = "lastTranscriptAckCursor"
|
||||
case lastliveeventackcursor = "lastLiveEventAckCursor"
|
||||
}
|
||||
}
|
||||
|
||||
public struct ReconcilingSessionPlacement: Codable, Sendable {
|
||||
public let state: String
|
||||
public let generation: Int
|
||||
public let createdatms: Int
|
||||
public let updatedatms: Int
|
||||
public let statechangedatms: Int
|
||||
public let environmentid: String
|
||||
public let activeownerepoch: Int
|
||||
public let workerbundlehash: String
|
||||
public let workspacebasemanifestref: String
|
||||
public let remoteworkspacedir: String
|
||||
public let lasttranscriptackcursor: Int?
|
||||
public let lastliveeventackcursor: Int?
|
||||
|
||||
public init(
|
||||
state: String,
|
||||
generation: Int,
|
||||
createdatms: Int,
|
||||
updatedatms: Int,
|
||||
statechangedatms: Int,
|
||||
environmentid: String,
|
||||
activeownerepoch: Int,
|
||||
workerbundlehash: String,
|
||||
workspacebasemanifestref: String,
|
||||
remoteworkspacedir: String,
|
||||
lasttranscriptackcursor: Int? = nil,
|
||||
lastliveeventackcursor: Int? = nil)
|
||||
{
|
||||
self.state = state
|
||||
self.generation = generation
|
||||
self.createdatms = createdatms
|
||||
self.updatedatms = updatedatms
|
||||
self.statechangedatms = statechangedatms
|
||||
self.environmentid = environmentid
|
||||
self.activeownerepoch = activeownerepoch
|
||||
self.workerbundlehash = workerbundlehash
|
||||
self.workspacebasemanifestref = workspacebasemanifestref
|
||||
self.remoteworkspacedir = remoteworkspacedir
|
||||
self.lasttranscriptackcursor = lasttranscriptackcursor
|
||||
self.lastliveeventackcursor = lastliveeventackcursor
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case state
|
||||
case generation
|
||||
case createdatms = "createdAtMs"
|
||||
case updatedatms = "updatedAtMs"
|
||||
case statechangedatms = "stateChangedAtMs"
|
||||
case environmentid = "environmentId"
|
||||
case activeownerepoch = "activeOwnerEpoch"
|
||||
case workerbundlehash = "workerBundleHash"
|
||||
case workspacebasemanifestref = "workspaceBaseManifestRef"
|
||||
case remoteworkspacedir = "remoteWorkspaceDir"
|
||||
case lasttranscriptackcursor = "lastTranscriptAckCursor"
|
||||
case lastliveeventackcursor = "lastLiveEventAckCursor"
|
||||
}
|
||||
}
|
||||
|
||||
public struct ReclaimedSessionPlacement: Codable, Sendable {
|
||||
public let state: String
|
||||
public let generation: Int
|
||||
public let createdatms: Int
|
||||
public let updatedatms: Int
|
||||
public let statechangedatms: Int
|
||||
public let environmentid: String?
|
||||
public let activeownerepoch: Int?
|
||||
public let workspacebasemanifestref: String?
|
||||
public let remoteworkspacedir: String?
|
||||
public let workerbundlehash: String?
|
||||
public let lasttranscriptackcursor: Int?
|
||||
public let lastliveeventackcursor: Int?
|
||||
|
||||
public init(
|
||||
state: String,
|
||||
generation: Int,
|
||||
createdatms: Int,
|
||||
updatedatms: Int,
|
||||
statechangedatms: Int,
|
||||
environmentid: String? = nil,
|
||||
activeownerepoch: Int? = nil,
|
||||
workspacebasemanifestref: String? = nil,
|
||||
remoteworkspacedir: String? = nil,
|
||||
workerbundlehash: String? = nil,
|
||||
lasttranscriptackcursor: Int? = nil,
|
||||
lastliveeventackcursor: Int? = nil)
|
||||
{
|
||||
self.state = state
|
||||
self.generation = generation
|
||||
self.createdatms = createdatms
|
||||
self.updatedatms = updatedatms
|
||||
self.statechangedatms = statechangedatms
|
||||
self.environmentid = environmentid
|
||||
self.activeownerepoch = activeownerepoch
|
||||
self.workspacebasemanifestref = workspacebasemanifestref
|
||||
self.remoteworkspacedir = remoteworkspacedir
|
||||
self.workerbundlehash = workerbundlehash
|
||||
self.lasttranscriptackcursor = lasttranscriptackcursor
|
||||
self.lastliveeventackcursor = lastliveeventackcursor
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case state
|
||||
case generation
|
||||
case createdatms = "createdAtMs"
|
||||
case updatedatms = "updatedAtMs"
|
||||
case statechangedatms = "stateChangedAtMs"
|
||||
case environmentid = "environmentId"
|
||||
case activeownerepoch = "activeOwnerEpoch"
|
||||
case workspacebasemanifestref = "workspaceBaseManifestRef"
|
||||
case remoteworkspacedir = "remoteWorkspaceDir"
|
||||
case workerbundlehash = "workerBundleHash"
|
||||
case lasttranscriptackcursor = "lastTranscriptAckCursor"
|
||||
case lastliveeventackcursor = "lastLiveEventAckCursor"
|
||||
}
|
||||
}
|
||||
|
||||
public struct FailedSessionPlacement: Codable, Sendable {
|
||||
public let state: String
|
||||
public let generation: Int
|
||||
public let createdatms: Int
|
||||
public let updatedatms: Int
|
||||
public let statechangedatms: Int
|
||||
public let environmentid: String?
|
||||
public let activeownerepoch: Int?
|
||||
public let workspacebasemanifestref: String?
|
||||
public let remoteworkspacedir: String?
|
||||
public let workerbundlehash: String?
|
||||
public let lasttranscriptackcursor: Int?
|
||||
public let lastliveeventackcursor: Int?
|
||||
public let recoveryerror: String
|
||||
|
||||
public init(
|
||||
state: String,
|
||||
generation: Int,
|
||||
createdatms: Int,
|
||||
updatedatms: Int,
|
||||
statechangedatms: Int,
|
||||
environmentid: String? = nil,
|
||||
activeownerepoch: Int? = nil,
|
||||
workspacebasemanifestref: String? = nil,
|
||||
remoteworkspacedir: String? = nil,
|
||||
workerbundlehash: String? = nil,
|
||||
lasttranscriptackcursor: Int? = nil,
|
||||
lastliveeventackcursor: Int? = nil,
|
||||
recoveryerror: String)
|
||||
{
|
||||
self.state = state
|
||||
self.generation = generation
|
||||
self.createdatms = createdatms
|
||||
self.updatedatms = updatedatms
|
||||
self.statechangedatms = statechangedatms
|
||||
self.environmentid = environmentid
|
||||
self.activeownerepoch = activeownerepoch
|
||||
self.workspacebasemanifestref = workspacebasemanifestref
|
||||
self.remoteworkspacedir = remoteworkspacedir
|
||||
self.workerbundlehash = workerbundlehash
|
||||
self.lasttranscriptackcursor = lasttranscriptackcursor
|
||||
self.lastliveeventackcursor = lastliveeventackcursor
|
||||
self.recoveryerror = recoveryerror
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case state
|
||||
case generation
|
||||
case createdatms = "createdAtMs"
|
||||
case updatedatms = "updatedAtMs"
|
||||
case statechangedatms = "stateChangedAtMs"
|
||||
case environmentid = "environmentId"
|
||||
case activeownerepoch = "activeOwnerEpoch"
|
||||
case workspacebasemanifestref = "workspaceBaseManifestRef"
|
||||
case remoteworkspacedir = "remoteWorkspaceDir"
|
||||
case workerbundlehash = "workerBundleHash"
|
||||
case lasttranscriptackcursor = "lastTranscriptAckCursor"
|
||||
case lastliveeventackcursor = "lastLiveEventAckCursor"
|
||||
case recoveryerror = "recoveryError"
|
||||
}
|
||||
}
|
||||
|
||||
public struct SessionsDispatchParams: Codable, Sendable {
|
||||
public let key: String
|
||||
public let agentid: String?
|
||||
public let profileid: String
|
||||
|
||||
public init(
|
||||
key: String,
|
||||
agentid: String? = nil,
|
||||
profileid: String)
|
||||
{
|
||||
self.key = key
|
||||
self.agentid = agentid
|
||||
self.profileid = profileid
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case key
|
||||
case agentid = "agentId"
|
||||
case profileid = "profileId"
|
||||
}
|
||||
}
|
||||
|
||||
public struct SessionsDispatchResult: Codable, Sendable {
|
||||
public let ok: Bool
|
||||
public let key: String
|
||||
public let sessionid: String
|
||||
public let placement: ActiveWorkerSessionPlacement
|
||||
|
||||
public init(
|
||||
ok: Bool,
|
||||
key: String,
|
||||
sessionid: String,
|
||||
placement: ActiveWorkerSessionPlacement)
|
||||
{
|
||||
self.ok = ok
|
||||
self.key = key
|
||||
self.sessionid = sessionid
|
||||
self.placement = placement
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case ok
|
||||
case key
|
||||
case sessionid = "sessionId"
|
||||
case placement
|
||||
}
|
||||
}
|
||||
|
||||
public struct SessionsCompactionListParams: Codable, Sendable {
|
||||
public let key: String
|
||||
public let agentid: String?
|
||||
@@ -12359,6 +12892,61 @@ public enum GatewaySuspendStatusResult: Codable, Sendable {
|
||||
}
|
||||
}
|
||||
|
||||
public enum SessionPlacement: Codable, Sendable {
|
||||
case local(LocalSessionPlacement)
|
||||
case requested(RequestedSessionPlacement)
|
||||
case provisioning(ProvisioningSessionPlacement)
|
||||
case syncing(SyncingSessionPlacement)
|
||||
case starting(StartingSessionPlacement)
|
||||
case active(ActiveWorkerSessionPlacement)
|
||||
case draining(DrainingSessionPlacement)
|
||||
case reconciling(ReconcilingSessionPlacement)
|
||||
case reclaimed(ReclaimedSessionPlacement)
|
||||
case failed(FailedSessionPlacement)
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case discriminator = "state"
|
||||
}
|
||||
|
||||
public init(from decoder: Decoder) throws {
|
||||
let container = try decoder.container(keyedBy: CodingKeys.self)
|
||||
let discriminator = try container.decode(String.self, forKey: .discriminator)
|
||||
switch discriminator {
|
||||
case "local": self = try .local(LocalSessionPlacement(from: decoder))
|
||||
case "requested": self = try .requested(RequestedSessionPlacement(from: decoder))
|
||||
case "provisioning": self = try .provisioning(ProvisioningSessionPlacement(from: decoder))
|
||||
case "syncing": self = try .syncing(SyncingSessionPlacement(from: decoder))
|
||||
case "starting": self = try .starting(StartingSessionPlacement(from: decoder))
|
||||
case "active": self = try .active(ActiveWorkerSessionPlacement(from: decoder))
|
||||
case "draining": self = try .draining(DrainingSessionPlacement(from: decoder))
|
||||
case "reconciling": self = try .reconciling(ReconcilingSessionPlacement(from: decoder))
|
||||
case "reclaimed": self = try .reclaimed(ReclaimedSessionPlacement(from: decoder))
|
||||
case "failed": self = try .failed(FailedSessionPlacement(from: decoder))
|
||||
default:
|
||||
throw DecodingError.dataCorruptedError(
|
||||
forKey: .discriminator,
|
||||
in: container,
|
||||
debugDescription: "Unknown SessionPlacement discriminator value"
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
public func encode(to encoder: Encoder) throws {
|
||||
switch self {
|
||||
case .local(let value): try value.encode(to: encoder)
|
||||
case .requested(let value): try value.encode(to: encoder)
|
||||
case .provisioning(let value): try value.encode(to: encoder)
|
||||
case .syncing(let value): try value.encode(to: encoder)
|
||||
case .starting(let value): try value.encode(to: encoder)
|
||||
case .active(let value): try value.encode(to: encoder)
|
||||
case .draining(let value): try value.encode(to: encoder)
|
||||
case .reconciling(let value): try value.encode(to: encoder)
|
||||
case .reclaimed(let value): try value.encode(to: encoder)
|
||||
case .failed(let value): try value.encode(to: encoder)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public enum AuditActivityEventV1: Codable, Sendable {
|
||||
case agentRun(AuditActivityAgentRunV1)
|
||||
case toolAction(AuditActivityToolActionV1)
|
||||
|
||||
+4
-2
@@ -46,8 +46,10 @@ Worker mode does not start channels, Gateway HTTP surfaces, or plugin auto-start
|
||||
beyond the assigned session toolset. It uses a throwaway state directory and has
|
||||
no standing provider or forge credentials.
|
||||
|
||||
Worker-to-worker session dispatch is not exposed in this mode. Agent dispatch
|
||||
and placement remain gateway-owned milestone-3 surfaces.
|
||||
Worker-to-worker session dispatch is not exposed in this mode. Placement and
|
||||
dispatch remain gateway-owned: an operator can dispatch an existing local,
|
||||
managed-worktree session through the Gateway, while a worker process cannot
|
||||
dispatch itself or another worker.
|
||||
|
||||
The prepared assignment carries the transcript context, accepted base leaf,
|
||||
commit sequence, and live-event cursor. On a tunnel reconnect, the process
|
||||
|
||||
@@ -515,13 +515,14 @@ methods. Treat this as feature discovery, not a full enumeration of
|
||||
</Accordion>
|
||||
|
||||
<Accordion title="Session control">
|
||||
- `sessions.list` returns the current session index, including per-row `agentRuntime` metadata when an agent runtime backend is configured.
|
||||
- `sessions.list` returns the current session index, including per-row `agentRuntime` metadata when an agent runtime backend is configured. When cloud-worker placement is enabled or durable recovery state exists, session rows also include a closed `placement` state (`local`, `requested`, `provisioning`, `syncing`, `starting`, `active`, `draining`, `reconciling`, `reclaimed`, or `failed`) plus state-specific environment, owner-epoch, workspace, bundle, ACK-cursor, or recovery fields.
|
||||
- `sessions.subscribe` and `sessions.unsubscribe` toggle session change event subscriptions for the current WS client.
|
||||
- `sessions.messages.subscribe` and `sessions.messages.unsubscribe` toggle transcript/message event subscriptions for one session. Pass `includeApprovals: true` to also receive sanitized `session.approval` lifecycle events for approvals whose persisted audience includes that exact session and whose reviewer binding authorizes the subscribing client. The subscribe response then includes a bounded pending `approvalReplay`; it is authoritative when `truncated` is false. The opt-in is per subscribe call, not sticky: re-subscribing to the same session without `includeApprovals: true` removes an existing approval subscription. In addition to normal session-read authority, this opt-in requires `operator.admin`, or `operator.approvals` on a paired device.
|
||||
- `sessions.preview` returns bounded transcript previews for specific session keys.
|
||||
- `sessions.describe` returns one gateway session row for an exact session key.
|
||||
- `sessions.resolve` resolves or canonicalizes a session target.
|
||||
- `sessions.create` creates a new session entry. `worktree: true` provisions a managed worktree; optional `worktreeBaseRef`/`worktreeName` select the base ref and branch name, and `execNode` (`operator.admin`) binds session exec to a node host. The created worktree is echoed in the result and persisted on the session row (`worktree: { id, branch, repoRoot }`). When the entry is created but its nested initial `chat.send` is rejected, the successful result includes `runStarted: false` and `runError`; clients can preserve the prompt and retry against the returned session key.
|
||||
- `sessions.dispatch` (`operator.admin`) moves an existing local OpenClaw session with a session-owned managed worktree to a configured cloud-worker profile. Pass `{ key, profileId, agentId? }`. The method is absent when no worker profile is configured, closes local turn admission before draining active work, and returns only after placement reaches `active` worker ownership. Dispatch is one-way; worker-to-local pull-back is not part of this RPC.
|
||||
- `sessions.groups.list`, `sessions.groups.put`, `sessions.groups.rename`, and `sessions.groups.delete` manage the gateway-owned custom session group catalog (names + display order). Membership stays on each session's `category` field; rename and delete update member sessions server-side.
|
||||
- `sessions.send` sends a message into an existing session.
|
||||
- `sessions.steer` is the interrupt-and-steer variant for an active session.
|
||||
|
||||
@@ -343,11 +343,15 @@ import {
|
||||
SessionFileEntrySchema,
|
||||
SessionFileKindSchema,
|
||||
SessionFileRelevanceSchema,
|
||||
SessionPlacementSchema,
|
||||
SessionPlacementStateSchema,
|
||||
SessionWorktreeInfoSchema,
|
||||
SessionsCreateParamsSchema,
|
||||
SessionsCreateResultSchema,
|
||||
SessionsDeleteParamsSchema,
|
||||
SessionsDescribeParamsSchema,
|
||||
SessionsDispatchParamsSchema,
|
||||
SessionsDispatchResultSchema,
|
||||
SessionGroupSchema,
|
||||
SessionsGroupsDeleteParamsSchema,
|
||||
SessionsGroupsListParamsSchema,
|
||||
@@ -706,6 +710,8 @@ export const validateSessionsFilesSetParams = lazyCompile(SessionsFilesSetParams
|
||||
export const validateSessionsDiffParams = lazyCompile(SessionsDiffParamsSchema);
|
||||
export const validateSessionsCreateParams = lazyCompile(SessionsCreateParamsSchema);
|
||||
export const validateSessionsSendParams = lazyCompile(SessionsSendParamsSchema);
|
||||
export const validateSessionsDispatchParams = lazyCompile(SessionsDispatchParamsSchema);
|
||||
export const validateSessionsDispatchResult = lazyCompile(SessionsDispatchResultSchema);
|
||||
export const validateSessionsMessagesSubscribeParams = lazyCompile(
|
||||
SessionsMessagesSubscribeParamsSchema,
|
||||
);
|
||||
@@ -1058,9 +1064,13 @@ export {
|
||||
SessionsCompactionGetParamsSchema,
|
||||
SessionsCompactionBranchParamsSchema,
|
||||
SessionsCompactionRestoreParamsSchema,
|
||||
SessionPlacementStateSchema,
|
||||
SessionPlacementSchema,
|
||||
SessionWorktreeInfoSchema,
|
||||
SessionsCreateParamsSchema,
|
||||
SessionsCreateResultSchema,
|
||||
SessionsDispatchParamsSchema,
|
||||
SessionsDispatchResultSchema,
|
||||
SessionsSendParamsSchema,
|
||||
SessionsAbortParamsSchema,
|
||||
SessionsPatchParamsSchema,
|
||||
@@ -1600,7 +1610,11 @@ export type {
|
||||
SessionsDescribeParams,
|
||||
SessionsResolveParams,
|
||||
SessionOperationEvent,
|
||||
SessionPlacementState,
|
||||
SessionPlacement,
|
||||
SessionWorktreeInfo,
|
||||
SessionsDispatchParams,
|
||||
SessionsDispatchResult,
|
||||
SessionsCreateResult,
|
||||
SessionsPatchParams,
|
||||
SessionsResetParams,
|
||||
|
||||
@@ -30,6 +30,7 @@ export * from "./schema/nodes.js";
|
||||
export * from "./schema/protocol-schemas.js";
|
||||
export * from "./schema/push.js";
|
||||
export * from "./schema/secrets.js";
|
||||
export * from "./schema/session-placement.js";
|
||||
export * from "./schema/sessions.js";
|
||||
export * from "./schema/sessions-catalog.js";
|
||||
export * from "./schema/snapshot.js";
|
||||
|
||||
@@ -362,6 +362,7 @@ import {
|
||||
SecretsResolveParamsSchema,
|
||||
SecretsResolveResultSchema,
|
||||
} from "./secrets.js";
|
||||
import { SessionPlacementProtocolSchemas } from "./session-placement.js";
|
||||
import {
|
||||
SessionCatalogCapabilitiesSchema,
|
||||
SessionCatalogDescriptorSchema,
|
||||
@@ -629,6 +630,7 @@ export const ProtocolSchemas = {
|
||||
SessionsSearchResult: SessionsSearchResultSchema,
|
||||
SessionCompactionCheckpoint: SessionCompactionCheckpointSchema,
|
||||
SessionOperationEvent: SessionOperationEventSchema,
|
||||
...SessionPlacementProtocolSchemas,
|
||||
SessionsCompactionListParams: SessionsCompactionListParamsSchema,
|
||||
SessionsCompactionGetParams: SessionsCompactionGetParamsSchema,
|
||||
SessionsCompactionBranchParams: SessionsCompactionBranchParamsSchema,
|
||||
|
||||
@@ -0,0 +1,255 @@
|
||||
import { Value } from "typebox/value";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
SessionPlacementSchema,
|
||||
SessionPlacementStateSchema,
|
||||
validateSessionsDispatchParams,
|
||||
validateSessionsDispatchResult,
|
||||
} from "../index.js";
|
||||
|
||||
const placementStates = [
|
||||
"local",
|
||||
"requested",
|
||||
"provisioning",
|
||||
"syncing",
|
||||
"starting",
|
||||
"active",
|
||||
"draining",
|
||||
"reconciling",
|
||||
"reclaimed",
|
||||
"failed",
|
||||
] as const;
|
||||
|
||||
const basePlacement = {
|
||||
generation: 4,
|
||||
createdAtMs: 100,
|
||||
updatedAtMs: 200,
|
||||
stateChangedAtMs: 150,
|
||||
};
|
||||
const workerBundleHash = "a".repeat(64);
|
||||
const environmentFields = {
|
||||
environmentId: "environment-1",
|
||||
workerBundleHash,
|
||||
};
|
||||
const workspaceFields = {
|
||||
workspaceBaseManifestRef: "manifest-1",
|
||||
remoteWorkspaceDir: "/workspace/session-1",
|
||||
};
|
||||
const workerOwnedFields = {
|
||||
...environmentFields,
|
||||
...workspaceFields,
|
||||
activeOwnerEpoch: 7,
|
||||
};
|
||||
|
||||
describe("session dispatch protocol schemas", () => {
|
||||
it("accepts only the dedicated dispatch selector and configured profile", () => {
|
||||
expect(
|
||||
validateSessionsDispatchParams({
|
||||
key: "agent:main:dispatch",
|
||||
agentId: "main",
|
||||
profileId: "development",
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(validateSessionsDispatchParams({ key: "agent:main:dispatch" })).toBe(false);
|
||||
expect(
|
||||
validateSessionsDispatchParams({
|
||||
key: "agent:main:dispatch",
|
||||
profileId: "development",
|
||||
task: "run remotely",
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps placement states closed", () => {
|
||||
for (const state of placementStates) {
|
||||
expect(Value.Check(SessionPlacementStateSchema, state)).toBe(true);
|
||||
}
|
||||
expect(Value.Check(SessionPlacementStateSchema, "unknown")).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps local and requested placement free of worker metadata", () => {
|
||||
expect(Value.Check(SessionPlacementSchema, { state: "local", ...basePlacement })).toBe(true);
|
||||
expect(Value.Check(SessionPlacementSchema, { state: "requested", ...basePlacement })).toBe(
|
||||
true,
|
||||
);
|
||||
expect(
|
||||
Value.Check(SessionPlacementSchema, {
|
||||
state: "local",
|
||||
...basePlacement,
|
||||
environmentId: "environment-1",
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(
|
||||
Value.Check(SessionPlacementSchema, {
|
||||
state: "requested",
|
||||
...basePlacement,
|
||||
workerBundleHash,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("allows only the optional reserved environment while provisioning", () => {
|
||||
expect(
|
||||
Value.Check(SessionPlacementSchema, {
|
||||
state: "provisioning",
|
||||
...basePlacement,
|
||||
environmentId: "environment-1",
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(Value.Check(SessionPlacementSchema, { state: "provisioning", ...basePlacement })).toBe(
|
||||
true,
|
||||
);
|
||||
expect(
|
||||
Value.Check(SessionPlacementSchema, {
|
||||
state: "provisioning",
|
||||
...basePlacement,
|
||||
...environmentFields,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("requires the provisioned bundle while syncing", () => {
|
||||
expect(
|
||||
Value.Check(SessionPlacementSchema, {
|
||||
state: "syncing",
|
||||
...basePlacement,
|
||||
...environmentFields,
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
Value.Check(SessionPlacementSchema, {
|
||||
state: "syncing",
|
||||
...basePlacement,
|
||||
environmentId: "environment-1",
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(
|
||||
Value.Check(SessionPlacementSchema, {
|
||||
state: "syncing",
|
||||
...basePlacement,
|
||||
...environmentFields,
|
||||
...workspaceFields,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("requires workspace identity while starting", () => {
|
||||
expect(
|
||||
Value.Check(SessionPlacementSchema, {
|
||||
state: "starting",
|
||||
...basePlacement,
|
||||
...environmentFields,
|
||||
...workspaceFields,
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
Value.Check(SessionPlacementSchema, {
|
||||
state: "starting",
|
||||
...basePlacement,
|
||||
...environmentFields,
|
||||
remoteWorkspaceDir: "/workspace/session-1",
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(
|
||||
Value.Check(SessionPlacementSchema, {
|
||||
state: "starting",
|
||||
...basePlacement,
|
||||
...environmentFields,
|
||||
...workspaceFields,
|
||||
lastTranscriptAckCursor: 0,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it.each(["active", "draining", "reconciling"] as const)(
|
||||
"requires complete worker ownership for %s placement",
|
||||
(state) => {
|
||||
expect(
|
||||
Value.Check(SessionPlacementSchema, {
|
||||
state,
|
||||
...basePlacement,
|
||||
...workerOwnedFields,
|
||||
lastTranscriptAckCursor: 2,
|
||||
lastLiveEventAckCursor: 9,
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
Value.Check(SessionPlacementSchema, {
|
||||
state,
|
||||
...basePlacement,
|
||||
environmentId: "environment-1",
|
||||
activeOwnerEpoch: 7,
|
||||
workerBundleHash,
|
||||
}),
|
||||
).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
it("preserves optional provenance only in terminal states", () => {
|
||||
expect(Value.Check(SessionPlacementSchema, { state: "reclaimed", ...basePlacement })).toBe(
|
||||
true,
|
||||
);
|
||||
expect(
|
||||
Value.Check(SessionPlacementSchema, {
|
||||
state: "reclaimed",
|
||||
...basePlacement,
|
||||
...workerOwnedFields,
|
||||
}),
|
||||
).toBe(true);
|
||||
});
|
||||
|
||||
it("requires recovery evidence for failed placement", () => {
|
||||
const failed = {
|
||||
state: "failed" as const,
|
||||
...basePlacement,
|
||||
...workerOwnedFields,
|
||||
recoveryError: "worker admission failed",
|
||||
};
|
||||
expect(Value.Check(SessionPlacementSchema, failed)).toBe(true);
|
||||
expect(
|
||||
Value.Check(SessionPlacementSchema, {
|
||||
state: "failed",
|
||||
...basePlacement,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("accepts only active worker ownership in successful dispatch results", () => {
|
||||
const active = {
|
||||
state: "active" as const,
|
||||
...basePlacement,
|
||||
...workerOwnedFields,
|
||||
};
|
||||
expect(
|
||||
validateSessionsDispatchResult({
|
||||
ok: true,
|
||||
key: "agent:main:dispatch",
|
||||
sessionId: "session-1",
|
||||
placement: active,
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(
|
||||
validateSessionsDispatchResult({
|
||||
ok: true,
|
||||
key: "agent:main:dispatch",
|
||||
sessionId: "session-1",
|
||||
placement: {
|
||||
state: "failed",
|
||||
...basePlacement,
|
||||
recoveryError: "worker admission failed",
|
||||
},
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects unknown placement fields", () => {
|
||||
expect(
|
||||
Value.Check(SessionPlacementSchema, {
|
||||
state: "active",
|
||||
...basePlacement,
|
||||
...workerOwnedFields,
|
||||
unexpected: true,
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,198 @@
|
||||
import type { Static } from "typebox";
|
||||
import { Type } from "typebox";
|
||||
import { NonEmptyString } from "./primitives.js";
|
||||
|
||||
/** Durable gateway ownership states for one session execution placement. */
|
||||
export const SessionPlacementStateSchema = Type.Union([
|
||||
Type.Literal("local"),
|
||||
Type.Literal("requested"),
|
||||
Type.Literal("provisioning"),
|
||||
Type.Literal("syncing"),
|
||||
Type.Literal("starting"),
|
||||
Type.Literal("active"),
|
||||
Type.Literal("draining"),
|
||||
Type.Literal("reconciling"),
|
||||
Type.Literal("reclaimed"),
|
||||
Type.Literal("failed"),
|
||||
]);
|
||||
|
||||
const SessionPlacementTimingProperties = {
|
||||
generation: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
|
||||
createdAtMs: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
|
||||
updatedAtMs: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
|
||||
stateChangedAtMs: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
|
||||
};
|
||||
|
||||
const SessionPlacementOwnerEpochSchema = Type.Integer({
|
||||
minimum: 1,
|
||||
maximum: Number.MAX_SAFE_INTEGER,
|
||||
});
|
||||
|
||||
const WorkerBundleHashSchema = Type.String({
|
||||
minLength: 64,
|
||||
maxLength: 64,
|
||||
pattern: "^[a-f0-9]{64}$",
|
||||
});
|
||||
|
||||
const SessionPlacementWorkspaceProperties = {
|
||||
workspaceBaseManifestRef: NonEmptyString,
|
||||
remoteWorkspaceDir: NonEmptyString,
|
||||
};
|
||||
|
||||
const SessionPlacementAckProperties = {
|
||||
lastTranscriptAckCursor: Type.Optional(
|
||||
Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
|
||||
),
|
||||
lastLiveEventAckCursor: Type.Optional(
|
||||
Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
|
||||
),
|
||||
};
|
||||
|
||||
const TerminalSessionPlacementProperties = {
|
||||
environmentId: Type.Optional(NonEmptyString),
|
||||
activeOwnerEpoch: Type.Optional(SessionPlacementOwnerEpochSchema),
|
||||
workspaceBaseManifestRef: Type.Optional(NonEmptyString),
|
||||
remoteWorkspaceDir: Type.Optional(NonEmptyString),
|
||||
workerBundleHash: Type.Optional(WorkerBundleHashSchema),
|
||||
...SessionPlacementAckProperties,
|
||||
};
|
||||
|
||||
function createUnownedSessionPlacementSchema<const State extends "local" | "requested">(
|
||||
state: State,
|
||||
) {
|
||||
return Type.Object(
|
||||
{ state: Type.Literal(state), ...SessionPlacementTimingProperties },
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
}
|
||||
|
||||
function createWorkerOwnedSessionPlacementSchema<
|
||||
const State extends "active" | "draining" | "reconciling",
|
||||
>(state: State) {
|
||||
return Type.Object(
|
||||
{
|
||||
state: Type.Literal(state),
|
||||
...SessionPlacementTimingProperties,
|
||||
environmentId: NonEmptyString,
|
||||
activeOwnerEpoch: SessionPlacementOwnerEpochSchema,
|
||||
workerBundleHash: WorkerBundleHashSchema,
|
||||
...SessionPlacementWorkspaceProperties,
|
||||
...SessionPlacementAckProperties,
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
}
|
||||
|
||||
export const LocalSessionPlacementSchema = createUnownedSessionPlacementSchema("local");
|
||||
export const RequestedSessionPlacementSchema = createUnownedSessionPlacementSchema("requested");
|
||||
|
||||
export const ProvisioningSessionPlacementSchema = Type.Object(
|
||||
{
|
||||
state: Type.Literal("provisioning"),
|
||||
...SessionPlacementTimingProperties,
|
||||
environmentId: Type.Optional(NonEmptyString),
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
|
||||
export const SyncingSessionPlacementSchema = Type.Object(
|
||||
{
|
||||
state: Type.Literal("syncing"),
|
||||
...SessionPlacementTimingProperties,
|
||||
environmentId: NonEmptyString,
|
||||
workerBundleHash: WorkerBundleHashSchema,
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
|
||||
export const StartingSessionPlacementSchema = Type.Object(
|
||||
{
|
||||
state: Type.Literal("starting"),
|
||||
...SessionPlacementTimingProperties,
|
||||
environmentId: NonEmptyString,
|
||||
workerBundleHash: WorkerBundleHashSchema,
|
||||
...SessionPlacementWorkspaceProperties,
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
|
||||
export const ActiveWorkerSessionPlacementSchema = createWorkerOwnedSessionPlacementSchema("active");
|
||||
export const DrainingSessionPlacementSchema = createWorkerOwnedSessionPlacementSchema("draining");
|
||||
export const ReconcilingSessionPlacementSchema =
|
||||
createWorkerOwnedSessionPlacementSchema("reconciling");
|
||||
|
||||
export const ReclaimedSessionPlacementSchema = Type.Object(
|
||||
{
|
||||
state: Type.Literal("reclaimed"),
|
||||
...SessionPlacementTimingProperties,
|
||||
...TerminalSessionPlacementProperties,
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
|
||||
export const FailedSessionPlacementSchema = Type.Object(
|
||||
{
|
||||
state: Type.Literal("failed"),
|
||||
...SessionPlacementTimingProperties,
|
||||
...TerminalSessionPlacementProperties,
|
||||
recoveryError: NonEmptyString,
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
|
||||
/** Gateway-visible placement projection; `state` remains the closed discriminator. */
|
||||
export const SessionPlacementSchema = Type.Union([
|
||||
LocalSessionPlacementSchema,
|
||||
RequestedSessionPlacementSchema,
|
||||
ProvisioningSessionPlacementSchema,
|
||||
SyncingSessionPlacementSchema,
|
||||
StartingSessionPlacementSchema,
|
||||
ActiveWorkerSessionPlacementSchema,
|
||||
DrainingSessionPlacementSchema,
|
||||
ReconcilingSessionPlacementSchema,
|
||||
ReclaimedSessionPlacementSchema,
|
||||
FailedSessionPlacementSchema,
|
||||
]);
|
||||
|
||||
/** Requests one-way dispatch of an existing local session to a configured worker profile. */
|
||||
export const SessionsDispatchParamsSchema = Type.Object(
|
||||
{
|
||||
key: NonEmptyString,
|
||||
agentId: Type.Optional(NonEmptyString),
|
||||
profileId: NonEmptyString,
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
|
||||
/** Result returned once session dispatch reaches durable worker ownership. */
|
||||
export const SessionsDispatchResultSchema = Type.Object(
|
||||
{
|
||||
ok: Type.Literal(true),
|
||||
key: NonEmptyString,
|
||||
sessionId: NonEmptyString,
|
||||
placement: ActiveWorkerSessionPlacementSchema,
|
||||
},
|
||||
{ additionalProperties: false },
|
||||
);
|
||||
|
||||
export const SessionPlacementProtocolSchemas = {
|
||||
SessionPlacementState: SessionPlacementStateSchema,
|
||||
LocalSessionPlacement: LocalSessionPlacementSchema,
|
||||
RequestedSessionPlacement: RequestedSessionPlacementSchema,
|
||||
ProvisioningSessionPlacement: ProvisioningSessionPlacementSchema,
|
||||
SyncingSessionPlacement: SyncingSessionPlacementSchema,
|
||||
StartingSessionPlacement: StartingSessionPlacementSchema,
|
||||
ActiveWorkerSessionPlacement: ActiveWorkerSessionPlacementSchema,
|
||||
DrainingSessionPlacement: DrainingSessionPlacementSchema,
|
||||
ReconcilingSessionPlacement: ReconcilingSessionPlacementSchema,
|
||||
ReclaimedSessionPlacement: ReclaimedSessionPlacementSchema,
|
||||
FailedSessionPlacement: FailedSessionPlacementSchema,
|
||||
SessionPlacement: SessionPlacementSchema,
|
||||
SessionsDispatchParams: SessionsDispatchParamsSchema,
|
||||
SessionsDispatchResult: SessionsDispatchResultSchema,
|
||||
} as const;
|
||||
|
||||
export type SessionPlacementState = Static<typeof SessionPlacementStateSchema>;
|
||||
export type SessionPlacement = Static<typeof SessionPlacementSchema>;
|
||||
export type SessionsDispatchParams = Static<typeof SessionsDispatchParamsSchema>;
|
||||
export type SessionsDispatchResult = Static<typeof SessionsDispatchResultSchema>;
|
||||
@@ -46,6 +46,7 @@ const connectParams = {
|
||||
environmentId: "worker-1",
|
||||
credential,
|
||||
sessionId: null,
|
||||
runId: null,
|
||||
ownerEpoch: 1,
|
||||
rpcSetVersion: WORKER_RPC_SET_VERSION,
|
||||
handshake,
|
||||
@@ -190,6 +191,29 @@ describe("worker protocol schemas", () => {
|
||||
params: connectParams,
|
||||
}),
|
||||
).toBe(true);
|
||||
const missingRunId = structuredClone(connectParams);
|
||||
Reflect.deleteProperty(missingRunId.admission, "runId");
|
||||
expect(
|
||||
validateWorkerConnectRequestFrame({
|
||||
type: "req",
|
||||
id: "connect-missing-run",
|
||||
method: "connect",
|
||||
params: missingRunId,
|
||||
}),
|
||||
).toBe(false);
|
||||
for (const admission of [
|
||||
{ ...connectParams.admission, sessionId: null, runId: "run-1" },
|
||||
{ ...connectParams.admission, sessionId: "session-1", runId: null },
|
||||
]) {
|
||||
expect(
|
||||
validateWorkerConnectRequestFrame({
|
||||
type: "req",
|
||||
id: "connect-mismatched-session-run",
|
||||
method: "connect",
|
||||
params: { ...connectParams, admission },
|
||||
}),
|
||||
).toBe(false);
|
||||
}
|
||||
expect(
|
||||
Value.Check(WorkerAdmissionResponseFrameSchema, {
|
||||
type: "res",
|
||||
@@ -449,6 +473,7 @@ describe("worker protocol schemas", () => {
|
||||
|
||||
it("keeps worker close reasons closed", () => {
|
||||
expect(Value.Check(WorkerProtocolCloseReasonSchema, "credential-replaced")).toBe(true);
|
||||
expect(Value.Check(WorkerProtocolCloseReasonSchema, "placement-mismatch")).toBe(true);
|
||||
expect(Value.Check(WorkerProtocolCloseReasonSchema, "not-a-worker-reason")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -58,6 +58,27 @@ export const WorkerAdmissionHandshakeSchema = closedObject({
|
||||
}),
|
||||
});
|
||||
|
||||
const WorkerConnectAdmissionCommonProperties = {
|
||||
environmentId: WorkerIdentifierSchema,
|
||||
credential: WorkerCredentialSchema,
|
||||
ownerEpoch: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
|
||||
rpcSetVersion: Type.Integer({ minimum: 1, maximum: Number.MAX_SAFE_INTEGER }),
|
||||
handshake: WorkerAdmissionHandshakeSchema,
|
||||
};
|
||||
|
||||
const WorkerConnectAdmissionSchema = Type.Union([
|
||||
closedObject({
|
||||
...WorkerConnectAdmissionCommonProperties,
|
||||
sessionId: Type.Null(),
|
||||
runId: Type.Null(),
|
||||
}),
|
||||
closedObject({
|
||||
...WorkerConnectAdmissionCommonProperties,
|
||||
sessionId: WorkerIdentifierSchema,
|
||||
runId: WorkerIdentifierSchema,
|
||||
}),
|
||||
]);
|
||||
|
||||
/** Dedicated first-frame payload accepted only on the worker ingress. */
|
||||
export const WorkerConnectParamsSchema = closedObject({
|
||||
minProtocol: Type.Integer({ minimum: 1 }),
|
||||
@@ -69,14 +90,7 @@ export const WorkerConnectParamsSchema = closedObject({
|
||||
mode: Type.Literal(GATEWAY_CLIENT_MODES.WORKER),
|
||||
}),
|
||||
role: Type.Literal("worker"),
|
||||
admission: closedObject({
|
||||
environmentId: WorkerIdentifierSchema,
|
||||
credential: WorkerCredentialSchema,
|
||||
sessionId: Type.Union([WorkerIdentifierSchema, Type.Null()]),
|
||||
ownerEpoch: Type.Integer({ minimum: 0, maximum: Number.MAX_SAFE_INTEGER }),
|
||||
rpcSetVersion: Type.Integer({ minimum: 1, maximum: Number.MAX_SAFE_INTEGER }),
|
||||
handshake: WorkerAdmissionHandshakeSchema,
|
||||
}),
|
||||
admission: WorkerConnectAdmissionSchema,
|
||||
});
|
||||
|
||||
export const WorkerConnectRequestFrameSchema = closedObject({
|
||||
@@ -94,6 +108,7 @@ export const WorkerAdmissionFailureReasonSchema = Type.Union([
|
||||
Type.Literal("bundle-mismatch"),
|
||||
Type.Literal("version-mismatch"),
|
||||
Type.Literal("session-mismatch"),
|
||||
Type.Literal("placement-mismatch"),
|
||||
Type.Literal("owner-epoch-mismatch"),
|
||||
Type.Literal("rpc-set-mismatch"),
|
||||
Type.Literal("protocol-features-mismatch"),
|
||||
|
||||
@@ -66,6 +66,7 @@ import type { AgentRunSessionTarget } from "../run-session-target.js";
|
||||
import { resolveAgentRunAbortLifecycleFields } from "../run-termination.js";
|
||||
import { buildAgentRuntimeAuthPlan } from "../runtime-plan/auth.js";
|
||||
import type { AgentMessage } from "../runtime/index.js";
|
||||
import { withLocalSessionPlacementTurnAdmission } from "../session-placement-admission.js";
|
||||
import { buildUsageWithNoCost } from "../stream-message-shared.js";
|
||||
import {
|
||||
buildClaudeCliFallbackContextPrelude,
|
||||
@@ -715,127 +716,139 @@ export function runAgentAttempt(params: {
|
||||
...mutableCliSessionStore,
|
||||
}
|
||||
: undefined;
|
||||
return runCliAgent({
|
||||
sessionId: params.sessionId,
|
||||
sessionKey: params.sessionKey,
|
||||
sessionEntry: params.sessionEntry,
|
||||
agentId: params.sessionAgentId,
|
||||
trigger: "user",
|
||||
sessionFile: params.sessionFile,
|
||||
storePath: params.storePath,
|
||||
workspaceDir: params.workspaceDir,
|
||||
cwd: params.cwd,
|
||||
config: params.cfg,
|
||||
prompt: cliPrompt,
|
||||
transcriptPrompt: params.transcriptBody,
|
||||
modelProvider: params.providerOverride,
|
||||
provider: cliExecutionProvider,
|
||||
model: params.modelOverride,
|
||||
thinkLevel: params.resolvedThinkLevel,
|
||||
timeoutMs: params.timeoutMs,
|
||||
runTimeoutOverrideMs: params.runTimeoutOverrideMs,
|
||||
runId: params.runId,
|
||||
lifecycleGeneration: params.lifecycleGeneration,
|
||||
lane: params.opts.lane,
|
||||
extraSystemPrompt: params.opts.extraSystemPrompt,
|
||||
inputProvenance: params.opts.inputProvenance,
|
||||
sourceReplyDeliveryMode: params.opts.sourceReplyDeliveryMode,
|
||||
requireExplicitMessageTarget:
|
||||
params.opts.requireExplicitMessageTarget ?? isSubagentSessionKey(params.sessionKey),
|
||||
cliSessionBindingFacts: params.opts.cliSessionBindingFacts,
|
||||
cliSessionId: nextCliSessionId,
|
||||
cliSessionBinding:
|
||||
nextCliSessionId === activeCliSessionBinding?.sessionId
|
||||
? activeCliSessionBinding
|
||||
: undefined,
|
||||
forkCliSessionOnResume,
|
||||
...(forkStoreParams
|
||||
? {
|
||||
claimCliSessionFork: async () => {
|
||||
const claimed = await consumeCliSessionForkInStore(forkStoreParams);
|
||||
if (claimed) {
|
||||
params.sessionEntry = claimed;
|
||||
}
|
||||
return Boolean(claimed);
|
||||
},
|
||||
restoreCliSessionFork: async () => {
|
||||
const restored = await restoreCliSessionForkInStore(forkStoreParams);
|
||||
if (restored) {
|
||||
params.sessionEntry = restored;
|
||||
}
|
||||
},
|
||||
persistCliSessionForkSuccessor: async (successorCliSessionId: string) => {
|
||||
const persisted = await persistCliSessionForkSuccessorInStore({
|
||||
...forkStoreParams,
|
||||
successorCliSessionId,
|
||||
});
|
||||
if (!persisted) {
|
||||
throw new Error("CLI session fork successor could not be persisted");
|
||||
}
|
||||
params.sessionEntry = persisted;
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
authProfileId,
|
||||
bootstrapPromptWarningSignaturesSeen,
|
||||
bootstrapPromptWarningSignature,
|
||||
// Image discovery must use the original turn, before retry/history decoration.
|
||||
imagePrompt: params.body,
|
||||
// Fallback prompts repeat the current task, so prompt-local images must
|
||||
// accompany every CLI process. Native dedupe requires a runtime receipt.
|
||||
images: params.opts.images,
|
||||
imageOrder: params.opts.imageOrder,
|
||||
skillsSnapshot: params.skillsSnapshot,
|
||||
messageChannel: params.messageChannel,
|
||||
streamParams: params.opts.streamParams,
|
||||
messageProvider: params.opts.messageProvider ?? params.messageChannel,
|
||||
currentChannelId: params.runContext.currentChannelId,
|
||||
chatId: params.runContext.chatId,
|
||||
channelContext: params.runContext.channelContext,
|
||||
currentThreadTs: params.runContext.currentThreadTs,
|
||||
currentInboundAudio: params.runContext.currentInboundAudio,
|
||||
approvalReviewerDeviceId: params.opts.approvalReviewerDeviceId,
|
||||
agentAccountId: params.runContext.accountId,
|
||||
senderId: params.runContext.senderId,
|
||||
senderIsOwner: params.opts.senderIsOwner,
|
||||
bashElevated: params.opts.bashElevated,
|
||||
groupId: params.runContext.groupId,
|
||||
groupChannel: params.runContext.groupChannel,
|
||||
groupSpace: params.runContext.groupSpace,
|
||||
spawnedBy: params.spawnedBy,
|
||||
toolsAllow: resolveCliRuntimeToolsAllow(
|
||||
params.opts.toolsAllow,
|
||||
params.opts.toolsAllowIsDefault,
|
||||
),
|
||||
cleanupBundleMcpOnRunEnd: params.opts.cleanupBundleMcpOnRunEnd,
|
||||
cleanupCliLiveSessionOnRunEnd: params.opts.cleanupCliLiveSessionOnRunEnd,
|
||||
oneShotCliRun: params.opts.oneShotCliRun,
|
||||
userTurnTranscriptRecorder: params.userTurnTranscriptRecorder,
|
||||
suppressNextUserMessagePersistence: params.suppressPromptPersistenceOnRetry === true,
|
||||
...(mutableCliSessionStore && !forkCliSessionOnResume
|
||||
? {
|
||||
onBeforeFreshCliSessionRetry: async (retry) => {
|
||||
if (
|
||||
hasNewGeneratedMediaTaskForSessionKey(params.sessionKey, mediaTaskIdsBefore) ||
|
||||
retry.sessionId !== activeCliSessionBinding?.sessionId
|
||||
) {
|
||||
return false;
|
||||
return withLocalSessionPlacementTurnAdmission(
|
||||
{
|
||||
sessionId: params.sessionId,
|
||||
sessionKey: params.sessionKey ?? params.sessionId,
|
||||
agentId: params.sessionAgentId,
|
||||
runId: params.runId,
|
||||
},
|
||||
() =>
|
||||
runCliAgent({
|
||||
sessionId: params.sessionId,
|
||||
sessionKey: params.sessionKey,
|
||||
sessionEntry: params.sessionEntry,
|
||||
agentId: params.sessionAgentId,
|
||||
trigger: "user",
|
||||
sessionFile: params.sessionFile,
|
||||
storePath: params.storePath,
|
||||
workspaceDir: params.workspaceDir,
|
||||
cwd: params.cwd,
|
||||
config: params.cfg,
|
||||
prompt: cliPrompt,
|
||||
transcriptPrompt: params.transcriptBody,
|
||||
modelProvider: params.providerOverride,
|
||||
provider: cliExecutionProvider,
|
||||
model: params.modelOverride,
|
||||
thinkLevel: params.resolvedThinkLevel,
|
||||
timeoutMs: params.timeoutMs,
|
||||
runTimeoutOverrideMs: params.runTimeoutOverrideMs,
|
||||
runId: params.runId,
|
||||
lifecycleGeneration: params.lifecycleGeneration,
|
||||
lane: params.opts.lane,
|
||||
extraSystemPrompt: params.opts.extraSystemPrompt,
|
||||
inputProvenance: params.opts.inputProvenance,
|
||||
sourceReplyDeliveryMode: params.opts.sourceReplyDeliveryMode,
|
||||
requireExplicitMessageTarget:
|
||||
params.opts.requireExplicitMessageTarget ?? isSubagentSessionKey(params.sessionKey),
|
||||
cliSessionBindingFacts: params.opts.cliSessionBindingFacts,
|
||||
cliSessionId: nextCliSessionId,
|
||||
cliSessionBinding:
|
||||
nextCliSessionId === activeCliSessionBinding?.sessionId
|
||||
? activeCliSessionBinding
|
||||
: undefined,
|
||||
forkCliSessionOnResume,
|
||||
...(forkStoreParams
|
||||
? {
|
||||
claimCliSessionFork: async () => {
|
||||
const claimed = await consumeCliSessionForkInStore(forkStoreParams);
|
||||
if (claimed) {
|
||||
params.sessionEntry = claimed;
|
||||
}
|
||||
return Boolean(claimed);
|
||||
},
|
||||
restoreCliSessionFork: async () => {
|
||||
const restored = await restoreCliSessionForkInStore(forkStoreParams);
|
||||
if (restored) {
|
||||
params.sessionEntry = restored;
|
||||
}
|
||||
},
|
||||
persistCliSessionForkSuccessor: async (successorCliSessionId: string) => {
|
||||
const persisted = await persistCliSessionForkSuccessorInStore({
|
||||
...forkStoreParams,
|
||||
successorCliSessionId,
|
||||
});
|
||||
if (!persisted) {
|
||||
throw new Error("CLI session fork successor could not be persisted");
|
||||
}
|
||||
params.sessionEntry = persisted;
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
authProfileId,
|
||||
bootstrapPromptWarningSignaturesSeen,
|
||||
bootstrapPromptWarningSignature,
|
||||
// Image discovery must use the original turn, before retry/history decoration.
|
||||
imagePrompt: params.body,
|
||||
// Fallback prompts repeat the current task, so prompt-local images must
|
||||
// accompany every CLI process. Native dedupe requires a runtime receipt.
|
||||
images: params.opts.images,
|
||||
imageOrder: params.opts.imageOrder,
|
||||
skillsSnapshot: params.skillsSnapshot,
|
||||
messageChannel: params.messageChannel,
|
||||
streamParams: params.opts.streamParams,
|
||||
messageProvider: params.opts.messageProvider ?? params.messageChannel,
|
||||
currentChannelId: params.runContext.currentChannelId,
|
||||
chatId: params.runContext.chatId,
|
||||
channelContext: params.runContext.channelContext,
|
||||
currentThreadTs: params.runContext.currentThreadTs,
|
||||
currentInboundAudio: params.runContext.currentInboundAudio,
|
||||
approvalReviewerDeviceId: params.opts.approvalReviewerDeviceId,
|
||||
agentAccountId: params.runContext.accountId,
|
||||
senderId: params.runContext.senderId,
|
||||
senderIsOwner: params.opts.senderIsOwner,
|
||||
bashElevated: params.opts.bashElevated,
|
||||
groupId: params.runContext.groupId,
|
||||
groupChannel: params.runContext.groupChannel,
|
||||
groupSpace: params.runContext.groupSpace,
|
||||
spawnedBy: params.spawnedBy,
|
||||
toolsAllow: resolveCliRuntimeToolsAllow(
|
||||
params.opts.toolsAllow,
|
||||
params.opts.toolsAllowIsDefault,
|
||||
),
|
||||
cleanupBundleMcpOnRunEnd: params.opts.cleanupBundleMcpOnRunEnd,
|
||||
cleanupCliLiveSessionOnRunEnd: params.opts.cleanupCliLiveSessionOnRunEnd,
|
||||
oneShotCliRun: params.opts.oneShotCliRun,
|
||||
userTurnTranscriptRecorder: params.userTurnTranscriptRecorder,
|
||||
suppressNextUserMessagePersistence: params.suppressPromptPersistenceOnRetry === true,
|
||||
...(mutableCliSessionStore && !forkCliSessionOnResume
|
||||
? {
|
||||
onBeforeFreshCliSessionRetry: async (retry) => {
|
||||
if (
|
||||
hasNewGeneratedMediaTaskForSessionKey(
|
||||
params.sessionKey,
|
||||
mediaTaskIdsBefore,
|
||||
) ||
|
||||
retry.sessionId !== activeCliSessionBinding?.sessionId
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
|
||||
log.warn(
|
||||
`CLI session failed, clearing before fresh retry: provider=${sanitizeForLog(cliExecutionProvider)} sessionKey=${mutableCliSessionStore.sessionKey} reason=${sanitizeForLog(retry.reason)}`,
|
||||
);
|
||||
log.warn(
|
||||
`CLI session failed, clearing before fresh retry: provider=${sanitizeForLog(cliExecutionProvider)} sessionKey=${mutableCliSessionStore.sessionKey} reason=${sanitizeForLog(retry.reason)}`,
|
||||
);
|
||||
|
||||
params.sessionEntry =
|
||||
(await clearCliSessionInStore({
|
||||
provider: cliExecutionProvider,
|
||||
...mutableCliSessionStore,
|
||||
})) ?? params.sessionEntry;
|
||||
return true;
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
});
|
||||
params.sessionEntry =
|
||||
(await clearCliSessionInStore({
|
||||
provider: cliExecutionProvider,
|
||||
...mutableCliSessionStore,
|
||||
})) ?? params.sessionEntry;
|
||||
return true;
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
}),
|
||||
);
|
||||
};
|
||||
return resolveReusableCliSessionBinding().then(async (activeCliSessionBinding) => {
|
||||
try {
|
||||
|
||||
@@ -168,6 +168,7 @@ import {
|
||||
import type { AgentRuntimePlan } from "../runtime-plan/types.js";
|
||||
import type { AgentRuntimeAuthPlan } from "../runtime-plan/types.js";
|
||||
import { ensureRuntimePluginsLoaded } from "../runtime-plugins.js";
|
||||
import { withSessionPlacementTurnAdmission } from "../session-placement-admission.js";
|
||||
import {
|
||||
resolveSessionSuspensionReason,
|
||||
resolveSessionSuspensionTarget,
|
||||
@@ -889,12 +890,15 @@ async function runEmbeddedAgentInternal(
|
||||
});
|
||||
}
|
||||
};
|
||||
const enqueueGlobal = <T>(task: () => Promise<T>, opts?: CommandQueueEnqueueOptions) => {
|
||||
const enqueueGlobal = (
|
||||
task: () => Promise<EmbeddedAgentRunResult>,
|
||||
opts?: CommandQueueEnqueueOptions,
|
||||
) => {
|
||||
const globalOpts: CommandQueueEnqueueOptions = {
|
||||
...opts,
|
||||
priority: sessionQueuePriority,
|
||||
};
|
||||
const taskWithCurrentLifecycle = () => {
|
||||
const taskWithCurrentLifecycle = async () => {
|
||||
params.onLaneWait?.({ waitMs: 0, queuedAhead: 0, waiting: false });
|
||||
throwIfAborted();
|
||||
const currentLifecycleGeneration = getAgentEventLifecycleGeneration();
|
||||
@@ -914,16 +918,29 @@ async function runEmbeddedAgentInternal(
|
||||
lifecycleGeneration = currentLifecycleGeneration;
|
||||
params = { ...params, lifecycleGeneration };
|
||||
}
|
||||
// Queue waits can outlive the durable harness binding that admitted a run.
|
||||
// Recheck only after lifecycle admission, before any run context or hook can execute.
|
||||
// Queue waits can outlive durable harness and placement bindings.
|
||||
// Recheck and claim only after lifecycle admission, before context or hooks execute.
|
||||
assertAgentHarnessRunAdmission(params);
|
||||
claimAgentRunContext(params.runId, {
|
||||
...existingContext,
|
||||
sessionKey: params.sessionKey ?? existingContext?.sessionKey,
|
||||
sessionId: params.sessionId ?? existingContext?.sessionId,
|
||||
lifecycleGeneration,
|
||||
});
|
||||
return withAgentRunLifecycleGeneration(lifecycleGeneration, task);
|
||||
return await withAgentRunLifecycleGeneration(lifecycleGeneration, () =>
|
||||
withSessionPlacementTurnAdmission(
|
||||
{
|
||||
sessionId: params.sessionId,
|
||||
...(params.agentId ? { agentId: params.agentId } : {}),
|
||||
...(params.sessionKey ? { sessionKey: params.sessionKey } : {}),
|
||||
runId: params.runId,
|
||||
},
|
||||
params,
|
||||
() => {
|
||||
claimAgentRunContext(params.runId, {
|
||||
...existingContext,
|
||||
sessionKey: params.sessionKey ?? existingContext?.sessionKey,
|
||||
sessionId: params.sessionId ?? existingContext?.sessionId,
|
||||
lifecycleGeneration,
|
||||
});
|
||||
return task();
|
||||
},
|
||||
),
|
||||
);
|
||||
};
|
||||
if (params.enqueue) {
|
||||
return params.enqueue(taskWithCurrentLifecycle, withLaneTimeout(withRunLaneWait(globalOpts)));
|
||||
|
||||
@@ -0,0 +1,191 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
installSessionPlacementAdmissionProvider,
|
||||
installSessionPlacementResetGuard,
|
||||
resolveSessionPlacementResetBlock,
|
||||
type LocalTurnPlacementClaim,
|
||||
type SessionPlacementAdmissionProvider,
|
||||
withLocalSessionPlacementTurnAdmission,
|
||||
withSessionPlacementTurnAdmission,
|
||||
} from "./session-placement-admission.js";
|
||||
|
||||
let uninstallProvider: (() => void) | undefined;
|
||||
let uninstallResetGuard: (() => void) | undefined;
|
||||
const executeLocalTurn: SessionPlacementAdmissionProvider["executeLocalTurn"] = async (
|
||||
_claim,
|
||||
runLocal,
|
||||
) => await runLocal();
|
||||
|
||||
afterEach(() => {
|
||||
uninstallProvider?.();
|
||||
uninstallProvider = undefined;
|
||||
uninstallResetGuard?.();
|
||||
uninstallResetGuard = undefined;
|
||||
});
|
||||
|
||||
describe("local turn placement admission", () => {
|
||||
const turnParams = {
|
||||
sessionId: "session-1",
|
||||
sessionFile: "/tmp/session-1.jsonl",
|
||||
workspaceDir: "/tmp/workspace",
|
||||
prompt: "test",
|
||||
timeoutMs: 1_000,
|
||||
runId: "run-1",
|
||||
};
|
||||
|
||||
it("delegates the final turn decision to the installed provider", async () => {
|
||||
const events: string[] = [];
|
||||
uninstallProvider = installSessionPlacementAdmissionProvider({
|
||||
executeLocalTurn,
|
||||
executeTurn: async (claim, params, runLocal) => {
|
||||
events.push("claim");
|
||||
expect(claim).toEqual({
|
||||
sessionId: "session-1",
|
||||
sessionKey: "agent:main:main",
|
||||
runId: "run-1",
|
||||
});
|
||||
expect(params).toBe(turnParams);
|
||||
const result = await runLocal();
|
||||
events.push("release");
|
||||
return result;
|
||||
},
|
||||
});
|
||||
|
||||
const result = await withSessionPlacementTurnAdmission(
|
||||
{
|
||||
sessionId: "session-1",
|
||||
sessionKey: "agent:main:main",
|
||||
runId: "run-1",
|
||||
},
|
||||
turnParams,
|
||||
async () => {
|
||||
events.push("turn");
|
||||
return { meta: { durationMs: 1 } };
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.meta.durationMs).toBe(1);
|
||||
expect(events).toEqual(["claim", "turn", "release"]);
|
||||
});
|
||||
|
||||
it("does not start a local turn when the provider routes remotely", async () => {
|
||||
const turn = vi.fn(async () => ({ meta: { durationMs: 1 } }));
|
||||
const executeTurn = vi.fn<SessionPlacementAdmissionProvider["executeTurn"]>(async () => ({
|
||||
payloads: [{ text: "remote" }],
|
||||
meta: { durationMs: 2 },
|
||||
}));
|
||||
uninstallProvider = installSessionPlacementAdmissionProvider({
|
||||
executeLocalTurn,
|
||||
executeTurn,
|
||||
});
|
||||
|
||||
const result = await withSessionPlacementTurnAdmission(
|
||||
{ sessionId: "session-2", runId: "run-2" },
|
||||
{ ...turnParams, sessionId: "session-2", runId: "run-2" },
|
||||
turn,
|
||||
);
|
||||
expect(result.payloads).toEqual([{ text: "remote" }]);
|
||||
expect(executeTurn).toHaveBeenCalledOnce();
|
||||
expect(executeTurn.mock.calls[0]?.[0]).toEqual({ sessionId: "session-2", runId: "run-2" });
|
||||
expect(turn).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not resurrect a replaced provider during uninstall", async () => {
|
||||
const firstClaim = vi.fn(
|
||||
async (_claim, _params, runLocal: () => Promise<{ meta: { durationMs: number } }>) =>
|
||||
await runLocal(),
|
||||
);
|
||||
const uninstallFirst = installSessionPlacementAdmissionProvider({
|
||||
executeLocalTurn,
|
||||
executeTurn: firstClaim,
|
||||
});
|
||||
const secondClaim = vi.fn(
|
||||
async (_claim, _params, runLocal: () => Promise<{ meta: { durationMs: number } }>) =>
|
||||
await runLocal(),
|
||||
);
|
||||
const uninstallSecond = installSessionPlacementAdmissionProvider({
|
||||
executeLocalTurn,
|
||||
executeTurn: secondClaim,
|
||||
});
|
||||
uninstallProvider = uninstallSecond;
|
||||
|
||||
uninstallFirst();
|
||||
await withSessionPlacementTurnAdmission(
|
||||
{ sessionId: "session-4", runId: "run-4" },
|
||||
{ ...turnParams, sessionId: "session-4", runId: "run-4" },
|
||||
async () => ({ meta: { durationMs: 1 } }),
|
||||
);
|
||||
expect(firstClaim).not.toHaveBeenCalled();
|
||||
expect(secondClaim).toHaveBeenCalledOnce();
|
||||
|
||||
uninstallSecond();
|
||||
uninstallProvider = undefined;
|
||||
await withSessionPlacementTurnAdmission(
|
||||
{ sessionId: "session-5", runId: "run-5" },
|
||||
{ ...turnParams, sessionId: "session-5", runId: "run-5" },
|
||||
async () => ({ meta: { durationMs: 1 } }),
|
||||
);
|
||||
expect(firstClaim).not.toHaveBeenCalled();
|
||||
expect(secondClaim).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("delegates generic local execution through the placement gate", async () => {
|
||||
const events: string[] = [];
|
||||
uninstallProvider = installSessionPlacementAdmissionProvider({
|
||||
async executeLocalTurn<T>(
|
||||
claim: LocalTurnPlacementClaim,
|
||||
runLocal: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
events.push("claim");
|
||||
expect(claim).toEqual({
|
||||
sessionId: "session-cli",
|
||||
sessionKey: "agent:main:cli",
|
||||
agentId: "main",
|
||||
runId: "run-cli",
|
||||
});
|
||||
const result = await runLocal();
|
||||
events.push("release");
|
||||
return result;
|
||||
},
|
||||
executeTurn: async (_claim, _params, runLocal) => await runLocal(),
|
||||
});
|
||||
|
||||
const result = await withLocalSessionPlacementTurnAdmission(
|
||||
{
|
||||
sessionId: "session-cli",
|
||||
sessionKey: "agent:main:cli",
|
||||
agentId: "main",
|
||||
runId: "run-cli",
|
||||
},
|
||||
async () => {
|
||||
events.push("turn");
|
||||
return { kind: "cli", code: 0 } as const;
|
||||
},
|
||||
);
|
||||
|
||||
expect(result).toEqual({ kind: "cli", code: 0 });
|
||||
expect(events).toEqual(["claim", "turn", "release"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("session placement reset guard", () => {
|
||||
it("returns the installed reset block", () => {
|
||||
uninstallResetGuard = installSessionPlacementResetGuard((sessionId) =>
|
||||
sessionId === "session-worker" ? "cloud worker placement is active" : undefined,
|
||||
);
|
||||
|
||||
expect(resolveSessionPlacementResetBlock("session-worker")).toBe(
|
||||
"cloud worker placement is active",
|
||||
);
|
||||
expect(resolveSessionPlacementResetBlock("session-local")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not clear a replacement reset guard during stale uninstall", () => {
|
||||
const uninstallFirst = installSessionPlacementResetGuard(() => "first");
|
||||
uninstallResetGuard = installSessionPlacementResetGuard(() => "second");
|
||||
|
||||
uninstallFirst();
|
||||
|
||||
expect(resolveSessionPlacementResetBlock("session-worker")).toBe("second");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,82 @@
|
||||
import { resolveGlobalSingleton } from "../shared/global-singleton.js";
|
||||
import type { RunEmbeddedAgentParams } from "./embedded-agent-runner/run/params.js";
|
||||
import type { EmbeddedAgentRunResult } from "./embedded-agent-runner/types.js";
|
||||
|
||||
export type LocalTurnPlacementClaim = {
|
||||
sessionId: string;
|
||||
agentId?: string;
|
||||
sessionKey?: string;
|
||||
runId: string;
|
||||
};
|
||||
|
||||
export type SessionPlacementTurnParams = RunEmbeddedAgentParams & { sessionFile: string };
|
||||
|
||||
export type SessionPlacementAdmissionProvider = {
|
||||
executeLocalTurn: <T>(claim: LocalTurnPlacementClaim, runLocal: () => Promise<T>) => Promise<T>;
|
||||
executeTurn: (
|
||||
claim: LocalTurnPlacementClaim,
|
||||
params: SessionPlacementTurnParams,
|
||||
runLocal: () => Promise<EmbeddedAgentRunResult>,
|
||||
) => Promise<EmbeddedAgentRunResult>;
|
||||
};
|
||||
|
||||
type SessionPlacementResetGuard = (sessionId: string) => string | undefined;
|
||||
|
||||
type SessionPlacementAdmissionState = {
|
||||
provider?: SessionPlacementAdmissionProvider;
|
||||
resetGuard?: SessionPlacementResetGuard;
|
||||
};
|
||||
|
||||
// Runtime chunks share one provider. The identity guard keeps an older gateway
|
||||
// shutdown from clearing a newer lifecycle's admission gate.
|
||||
const state = resolveGlobalSingleton(
|
||||
Symbol.for("openclaw.sessionPlacementAdmissionState"),
|
||||
(): SessionPlacementAdmissionState => ({}),
|
||||
);
|
||||
|
||||
export function installSessionPlacementAdmissionProvider(
|
||||
provider: SessionPlacementAdmissionProvider,
|
||||
): () => void {
|
||||
state.provider = provider;
|
||||
return () => {
|
||||
if (state.provider === provider) {
|
||||
state.provider = undefined;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export function installSessionPlacementResetGuard(guard: SessionPlacementResetGuard): () => void {
|
||||
state.resetGuard = guard;
|
||||
return () => {
|
||||
if (state.resetGuard === guard) {
|
||||
state.resetGuard = undefined;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveSessionPlacementResetBlock(sessionId: string): string | undefined {
|
||||
return state.resetGuard?.(sessionId);
|
||||
}
|
||||
|
||||
export async function withSessionPlacementTurnAdmission(
|
||||
claim: LocalTurnPlacementClaim,
|
||||
params: SessionPlacementTurnParams,
|
||||
task: () => Promise<EmbeddedAgentRunResult>,
|
||||
): Promise<EmbeddedAgentRunResult> {
|
||||
const provider = state.provider;
|
||||
if (!provider) {
|
||||
return await task();
|
||||
}
|
||||
return await provider.executeTurn(claim, params, task);
|
||||
}
|
||||
|
||||
export async function withLocalSessionPlacementTurnAdmission<T>(
|
||||
claim: LocalTurnPlacementClaim,
|
||||
task: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
const provider = state.provider;
|
||||
if (!provider) {
|
||||
return await task();
|
||||
}
|
||||
return await provider.executeLocalTurn(claim, task);
|
||||
}
|
||||
@@ -70,6 +70,7 @@ import {
|
||||
resolveAgentRunErrorLifecycleFields,
|
||||
} from "../../agents/run-termination.js";
|
||||
import { buildAgentRuntimeOutcomePlan } from "../../agents/runtime-plan/build.js";
|
||||
import { withLocalSessionPlacementTurnAdmission } from "../../agents/session-placement-admission.js";
|
||||
import { resolveSessionRuntimeOverrideForProvider } from "../../agents/session-runtime-compat.js";
|
||||
import { resolveCandidateThinkingLevel } from "../../agents/thinking-runtime.js";
|
||||
import { resolveGroupSessionKey, type SessionEntry } from "../../config/sessions.js";
|
||||
@@ -2080,196 +2081,210 @@ async function runAgentTurnWithFallbackInternal(
|
||||
},
|
||||
});
|
||||
const result = await agentTurnTiming.measure("cli_run", () =>
|
||||
runCliAgentWithLifecycle({
|
||||
runId,
|
||||
lifecycleGeneration,
|
||||
provider: cliExecutionProvider,
|
||||
startedAt: cliLifecycleStartedAt,
|
||||
emitLifecycleTerminal: false,
|
||||
onAgentRunStart: notifyAgentRunStart,
|
||||
suppressAssistantBridge: params.followupRun.run.silentExpected,
|
||||
onActivity: () => params.replyOperation?.recordActivity(),
|
||||
preserveProgressCallbackStartOrder,
|
||||
onAssistantText: async (text) => {
|
||||
if (!preserveProgressCallbackStartOrder) {
|
||||
const textForTyping = await handlePartialForTyping({
|
||||
text,
|
||||
} as ReplyPayload);
|
||||
if (textForTyping === undefined || !params.opts?.onPartialReply) {
|
||||
return;
|
||||
}
|
||||
await params.opts.onPartialReply({ text: textForTyping });
|
||||
return;
|
||||
}
|
||||
const textForTyping = preparePartialForTyping({ text } as ReplyPayload);
|
||||
if (textForTyping === undefined) {
|
||||
return;
|
||||
}
|
||||
// Assistant and tool CLI bridges drain independently; stage presentation
|
||||
// before typing I/O so a later tool cannot overtake this text.
|
||||
await startPresentationWhileTyping(
|
||||
params.typingSignals.signalTextDelta(textForTyping),
|
||||
() => params.opts?.onPartialReply?.({ text: textForTyping }),
|
||||
);
|
||||
},
|
||||
onReasoningText: createCliReasoningStreamBridge(params.opts?.onReasoningStream),
|
||||
onReasoningProgress: async (payload) => {
|
||||
await params.opts?.onReasoningProgress?.(payload);
|
||||
},
|
||||
onToolEvent: async (payload) => {
|
||||
if (!preserveProgressCallbackStartOrder) {
|
||||
await cliToolSummaryTracker.noteToolEvent(payload);
|
||||
if (payload.phase === "result") {
|
||||
return;
|
||||
}
|
||||
const { name, phase, args } = payload;
|
||||
await Promise.all([
|
||||
params.typingSignals.signalToolStart(),
|
||||
params.opts?.onToolStart?.({
|
||||
name,
|
||||
phase,
|
||||
args,
|
||||
detailMode: params.toolProgressDetail,
|
||||
}),
|
||||
]);
|
||||
return;
|
||||
}
|
||||
const summaryPromise = cliToolSummaryTracker.noteToolEvent(payload);
|
||||
if (payload.phase === "result") {
|
||||
await summaryPromise;
|
||||
return;
|
||||
}
|
||||
const { name, phase, args } = payload;
|
||||
// Tool and assistant CLI bridges drain independently. Start channel
|
||||
// presentation before either bridge can yield and invert source order.
|
||||
await Promise.all([
|
||||
summaryPromise,
|
||||
startPresentationWhileTyping(params.typingSignals.signalToolStart(), () =>
|
||||
params.opts?.onToolStart?.({
|
||||
name,
|
||||
phase,
|
||||
args,
|
||||
detailMode: params.toolProgressDetail,
|
||||
}),
|
||||
),
|
||||
]);
|
||||
},
|
||||
onCommentaryText:
|
||||
params.opts?.commentaryProgressEnabled === true && params.opts.onItemEvent
|
||||
? async (payload) => {
|
||||
await params.opts?.onItemEvent?.({
|
||||
itemId: payload.itemId,
|
||||
kind: "preamble",
|
||||
progressText: payload.text,
|
||||
});
|
||||
}
|
||||
: undefined,
|
||||
onFastModeAutoProgress: async (payload) => {
|
||||
await params.opts?.onToolResult?.(payload);
|
||||
},
|
||||
transformResult:
|
||||
params.followupRun.currentInboundEventKind === "room_event"
|
||||
? (resultLocal) =>
|
||||
keepCliSessionBindingOnlyWhenReused({
|
||||
result: resultLocal,
|
||||
existingSessionId: cliSessionBinding?.sessionId,
|
||||
onDroppedReplacement: () => {
|
||||
droppedCliSessionReplacement = true;
|
||||
},
|
||||
})
|
||||
: undefined,
|
||||
runParams: {
|
||||
withLocalSessionPlacementTurnAdmission(
|
||||
{
|
||||
sessionId: params.followupRun.run.sessionId,
|
||||
sessionKey: params.sessionKey,
|
||||
runtimePolicySessionKey:
|
||||
params.followupRun.run.runtimePolicySessionKey ??
|
||||
params.runtimePolicySessionKey,
|
||||
agentId: params.followupRun.run.agentId,
|
||||
trigger: params.isHeartbeat ? "heartbeat" : "user",
|
||||
sessionFile: params.followupRun.run.sessionFile,
|
||||
workspaceDir: params.followupRun.run.workspaceDir,
|
||||
cwd: params.followupRun.run.cwd,
|
||||
config: runtimeConfig,
|
||||
prompt: params.commandBody,
|
||||
transcriptPrompt: params.transcriptCommandBody,
|
||||
suppressNextUserMessagePersistence: suppressQueuedUserPersistenceForCandidate,
|
||||
userTurnTranscriptRecorder,
|
||||
onUserMessagePersisted: notifyUserMessagePersisted,
|
||||
persistAssistantTranscript:
|
||||
params.followupRun.currentInboundEventKind !== "room_event" &&
|
||||
params.followupRun.run.suppressTranscriptOnlyAssistantPersistence !== true,
|
||||
storePath: params.storePath,
|
||||
currentInboundEventKind: params.followupRun.currentInboundEventKind,
|
||||
currentInboundContext: params.followupRun.currentInboundContext,
|
||||
inputProvenance: params.followupRun.run.inputProvenance,
|
||||
modelProvider: provider,
|
||||
provider: cliExecutionProvider,
|
||||
execOverrides: params.followupRun.run.execOverrides,
|
||||
bashElevated: params.followupRun.run.bashElevated,
|
||||
model,
|
||||
thinkLevel: candidateThinkLevel,
|
||||
fastMode: candidateFastMode.fastMode,
|
||||
fastModeStartedAtMs,
|
||||
fastModeAutoOnSeconds: candidateFastMode.fastModeAutoOnSeconds,
|
||||
fastModeAutoProgressState,
|
||||
isFinalFallbackAttempt: runOptions?.isFinalFallbackAttempt,
|
||||
timeoutMs: params.followupRun.run.timeoutMs,
|
||||
runTimeoutOverrideMs: params.followupRun.run.runTimeoutOverrideMs,
|
||||
runId,
|
||||
lane: runLane,
|
||||
extraSystemPrompt: params.followupRun.run.extraSystemPrompt,
|
||||
sourceReplyDeliveryMode: params.followupRun.run.sourceReplyDeliveryMode,
|
||||
taskSuggestionDeliveryMode: params.followupRun.run.taskSuggestionDeliveryMode,
|
||||
silentReplyPromptMode: params.followupRun.run.silentReplyPromptMode,
|
||||
allowEmptyAssistantReplyAsSilent:
|
||||
params.followupRun.run.allowEmptyAssistantReplyAsSilent,
|
||||
extraSystemPromptStatic: params.followupRun.run.extraSystemPromptStatic,
|
||||
cliSessionBindingFacts: params.followupRun.run.cliSessionBindingFacts,
|
||||
ownerNumbers: params.followupRun.run.ownerNumbers,
|
||||
cliSessionId: cliSessionBinding?.sessionId,
|
||||
cliSessionBinding,
|
||||
authProfileId: authProfile.authProfileId,
|
||||
bootstrapContextMode: params.opts?.bootstrapContextMode,
|
||||
bootstrapContextRunKind,
|
||||
bootstrapPromptWarningSignaturesSeen,
|
||||
bootstrapPromptWarningSignature:
|
||||
bootstrapPromptWarningSignaturesSeen[
|
||||
bootstrapPromptWarningSignaturesSeen.length - 1
|
||||
],
|
||||
images: currentTurnImages.images,
|
||||
imageOrder: currentTurnImages.imageOrder,
|
||||
skillsSnapshot: params.followupRun.run.skillsSnapshot,
|
||||
messageChannel: params.followupRun.originatingChannel ?? undefined,
|
||||
messageProvider: hookMessageProvider,
|
||||
clientCaps: params.followupRun.run.clientCaps,
|
||||
currentChannelId:
|
||||
params.followupRun.originatingTo ??
|
||||
params.sessionCtx.OriginatingTo ??
|
||||
params.sessionCtx.To,
|
||||
senderId: params.followupRun.run.senderId,
|
||||
senderName: params.followupRun.run.senderName,
|
||||
senderUsername: params.followupRun.run.senderUsername,
|
||||
senderE164: params.followupRun.run.senderE164,
|
||||
groupId: params.followupRun.run.groupId,
|
||||
groupChannel: params.followupRun.run.groupChannel,
|
||||
groupSpace: params.followupRun.run.groupSpace,
|
||||
spawnedBy: params.followupRun.run.spawnedBy,
|
||||
chatId: params.followupRun.originatingChatId,
|
||||
channelContext: params.followupRun.run.channelContext,
|
||||
currentThreadTs:
|
||||
cliCurrentThreadId != null ? String(cliCurrentThreadId) : undefined,
|
||||
currentMessageId: cliCurrentMessageId,
|
||||
currentInboundAudio: hasInboundAudio(params.sessionCtx),
|
||||
agentAccountId: params.followupRun.run.agentAccountId,
|
||||
senderIsOwner: params.followupRun.run.senderIsOwner,
|
||||
approvalReviewerDeviceId: params.followupRun.run.approvalReviewerDeviceId,
|
||||
toolsAllow: params.opts?.toolsAllow,
|
||||
disableTools: params.opts?.disableTools,
|
||||
abortSignal: runAbortSignal,
|
||||
onExecutionPhase: signalExecutionPhaseForTyping,
|
||||
replyOperation: params.replyOperation,
|
||||
},
|
||||
}),
|
||||
() =>
|
||||
runCliAgentWithLifecycle({
|
||||
runId,
|
||||
lifecycleGeneration,
|
||||
provider: cliExecutionProvider,
|
||||
startedAt: cliLifecycleStartedAt,
|
||||
emitLifecycleTerminal: false,
|
||||
onAgentRunStart: notifyAgentRunStart,
|
||||
suppressAssistantBridge: params.followupRun.run.silentExpected,
|
||||
onActivity: () => params.replyOperation?.recordActivity(),
|
||||
preserveProgressCallbackStartOrder,
|
||||
onAssistantText: async (text) => {
|
||||
if (!preserveProgressCallbackStartOrder) {
|
||||
const textForTyping = await handlePartialForTyping({
|
||||
text,
|
||||
} as ReplyPayload);
|
||||
if (textForTyping === undefined || !params.opts?.onPartialReply) {
|
||||
return;
|
||||
}
|
||||
await params.opts.onPartialReply({ text: textForTyping });
|
||||
return;
|
||||
}
|
||||
const textForTyping = preparePartialForTyping({ text } as ReplyPayload);
|
||||
if (textForTyping === undefined) {
|
||||
return;
|
||||
}
|
||||
// Assistant and tool CLI bridges drain independently; stage presentation
|
||||
// before typing I/O so a later tool cannot overtake this text.
|
||||
await startPresentationWhileTyping(
|
||||
params.typingSignals.signalTextDelta(textForTyping),
|
||||
() => params.opts?.onPartialReply?.({ text: textForTyping }),
|
||||
);
|
||||
},
|
||||
onReasoningText: createCliReasoningStreamBridge(
|
||||
params.opts?.onReasoningStream,
|
||||
),
|
||||
onReasoningProgress: async (payload) => {
|
||||
await params.opts?.onReasoningProgress?.(payload);
|
||||
},
|
||||
onToolEvent: async (payload) => {
|
||||
if (!preserveProgressCallbackStartOrder) {
|
||||
await cliToolSummaryTracker.noteToolEvent(payload);
|
||||
if (payload.phase === "result") {
|
||||
return;
|
||||
}
|
||||
const { name, phase, args } = payload;
|
||||
await Promise.all([
|
||||
params.typingSignals.signalToolStart(),
|
||||
params.opts?.onToolStart?.({
|
||||
name,
|
||||
phase,
|
||||
args,
|
||||
detailMode: params.toolProgressDetail,
|
||||
}),
|
||||
]);
|
||||
return;
|
||||
}
|
||||
const summaryPromise = cliToolSummaryTracker.noteToolEvent(payload);
|
||||
if (payload.phase === "result") {
|
||||
await summaryPromise;
|
||||
return;
|
||||
}
|
||||
const { name, phase, args } = payload;
|
||||
// Tool and assistant CLI bridges drain independently. Start channel
|
||||
// presentation before either bridge can yield and invert source order.
|
||||
await Promise.all([
|
||||
summaryPromise,
|
||||
startPresentationWhileTyping(params.typingSignals.signalToolStart(), () =>
|
||||
params.opts?.onToolStart?.({
|
||||
name,
|
||||
phase,
|
||||
args,
|
||||
detailMode: params.toolProgressDetail,
|
||||
}),
|
||||
),
|
||||
]);
|
||||
},
|
||||
onCommentaryText:
|
||||
params.opts?.commentaryProgressEnabled === true && params.opts.onItemEvent
|
||||
? async (payload) => {
|
||||
await params.opts?.onItemEvent?.({
|
||||
itemId: payload.itemId,
|
||||
kind: "preamble",
|
||||
progressText: payload.text,
|
||||
});
|
||||
}
|
||||
: undefined,
|
||||
onFastModeAutoProgress: async (payload) => {
|
||||
await params.opts?.onToolResult?.(payload);
|
||||
},
|
||||
transformResult:
|
||||
params.followupRun.currentInboundEventKind === "room_event"
|
||||
? (resultLocal) =>
|
||||
keepCliSessionBindingOnlyWhenReused({
|
||||
result: resultLocal,
|
||||
existingSessionId: cliSessionBinding?.sessionId,
|
||||
onDroppedReplacement: () => {
|
||||
droppedCliSessionReplacement = true;
|
||||
},
|
||||
})
|
||||
: undefined,
|
||||
runParams: {
|
||||
sessionId: params.followupRun.run.sessionId,
|
||||
sessionKey: params.sessionKey,
|
||||
runtimePolicySessionKey:
|
||||
params.followupRun.run.runtimePolicySessionKey ??
|
||||
params.runtimePolicySessionKey,
|
||||
agentId: params.followupRun.run.agentId,
|
||||
trigger: params.isHeartbeat ? "heartbeat" : "user",
|
||||
sessionFile: params.followupRun.run.sessionFile,
|
||||
workspaceDir: params.followupRun.run.workspaceDir,
|
||||
cwd: params.followupRun.run.cwd,
|
||||
config: runtimeConfig,
|
||||
prompt: params.commandBody,
|
||||
transcriptPrompt: params.transcriptCommandBody,
|
||||
suppressNextUserMessagePersistence:
|
||||
suppressQueuedUserPersistenceForCandidate,
|
||||
userTurnTranscriptRecorder,
|
||||
onUserMessagePersisted: notifyUserMessagePersisted,
|
||||
persistAssistantTranscript:
|
||||
params.followupRun.currentInboundEventKind !== "room_event" &&
|
||||
params.followupRun.run.suppressTranscriptOnlyAssistantPersistence !==
|
||||
true,
|
||||
storePath: params.storePath,
|
||||
currentInboundEventKind: params.followupRun.currentInboundEventKind,
|
||||
currentInboundContext: params.followupRun.currentInboundContext,
|
||||
inputProvenance: params.followupRun.run.inputProvenance,
|
||||
modelProvider: provider,
|
||||
provider: cliExecutionProvider,
|
||||
execOverrides: params.followupRun.run.execOverrides,
|
||||
bashElevated: params.followupRun.run.bashElevated,
|
||||
model,
|
||||
thinkLevel: candidateThinkLevel,
|
||||
fastMode: candidateFastMode.fastMode,
|
||||
fastModeStartedAtMs,
|
||||
fastModeAutoOnSeconds: candidateFastMode.fastModeAutoOnSeconds,
|
||||
fastModeAutoProgressState,
|
||||
isFinalFallbackAttempt: runOptions?.isFinalFallbackAttempt,
|
||||
timeoutMs: params.followupRun.run.timeoutMs,
|
||||
runTimeoutOverrideMs: params.followupRun.run.runTimeoutOverrideMs,
|
||||
runId,
|
||||
lane: runLane,
|
||||
extraSystemPrompt: params.followupRun.run.extraSystemPrompt,
|
||||
sourceReplyDeliveryMode: params.followupRun.run.sourceReplyDeliveryMode,
|
||||
taskSuggestionDeliveryMode:
|
||||
params.followupRun.run.taskSuggestionDeliveryMode,
|
||||
silentReplyPromptMode: params.followupRun.run.silentReplyPromptMode,
|
||||
allowEmptyAssistantReplyAsSilent:
|
||||
params.followupRun.run.allowEmptyAssistantReplyAsSilent,
|
||||
extraSystemPromptStatic: params.followupRun.run.extraSystemPromptStatic,
|
||||
cliSessionBindingFacts: params.followupRun.run.cliSessionBindingFacts,
|
||||
ownerNumbers: params.followupRun.run.ownerNumbers,
|
||||
cliSessionId: cliSessionBinding?.sessionId,
|
||||
cliSessionBinding,
|
||||
authProfileId: authProfile.authProfileId,
|
||||
bootstrapContextMode: params.opts?.bootstrapContextMode,
|
||||
bootstrapContextRunKind,
|
||||
bootstrapPromptWarningSignaturesSeen,
|
||||
bootstrapPromptWarningSignature:
|
||||
bootstrapPromptWarningSignaturesSeen[
|
||||
bootstrapPromptWarningSignaturesSeen.length - 1
|
||||
],
|
||||
images: currentTurnImages.images,
|
||||
imageOrder: currentTurnImages.imageOrder,
|
||||
skillsSnapshot: params.followupRun.run.skillsSnapshot,
|
||||
messageChannel: params.followupRun.originatingChannel ?? undefined,
|
||||
messageProvider: hookMessageProvider,
|
||||
clientCaps: params.followupRun.run.clientCaps,
|
||||
currentChannelId:
|
||||
params.followupRun.originatingTo ??
|
||||
params.sessionCtx.OriginatingTo ??
|
||||
params.sessionCtx.To,
|
||||
senderId: params.followupRun.run.senderId,
|
||||
senderName: params.followupRun.run.senderName,
|
||||
senderUsername: params.followupRun.run.senderUsername,
|
||||
senderE164: params.followupRun.run.senderE164,
|
||||
groupId: params.followupRun.run.groupId,
|
||||
groupChannel: params.followupRun.run.groupChannel,
|
||||
groupSpace: params.followupRun.run.groupSpace,
|
||||
spawnedBy: params.followupRun.run.spawnedBy,
|
||||
chatId: params.followupRun.originatingChatId,
|
||||
channelContext: params.followupRun.run.channelContext,
|
||||
currentThreadTs:
|
||||
cliCurrentThreadId != null ? String(cliCurrentThreadId) : undefined,
|
||||
currentMessageId: cliCurrentMessageId,
|
||||
currentInboundAudio: hasInboundAudio(params.sessionCtx),
|
||||
agentAccountId: params.followupRun.run.agentAccountId,
|
||||
senderIsOwner: params.followupRun.run.senderIsOwner,
|
||||
approvalReviewerDeviceId: params.followupRun.run.approvalReviewerDeviceId,
|
||||
toolsAllow: params.opts?.toolsAllow,
|
||||
disableTools: params.opts?.disableTools,
|
||||
abortSignal: runAbortSignal,
|
||||
onExecutionPhase: signalExecutionPhaseForTyping,
|
||||
replyOperation: params.replyOperation,
|
||||
},
|
||||
}),
|
||||
),
|
||||
);
|
||||
if (droppedCliSessionReplacement) {
|
||||
await clearDroppedCliSessionBinding({
|
||||
|
||||
@@ -40,6 +40,7 @@ import {
|
||||
buildAgentRuntimeDeliveryPlan,
|
||||
buildAgentRuntimeOutcomePlan,
|
||||
} from "../../agents/runtime-plan/build.js";
|
||||
import { withLocalSessionPlacementTurnAdmission } from "../../agents/session-placement-admission.js";
|
||||
import { resolveSessionRuntimeOverrideForProvider } from "../../agents/session-runtime-compat.js";
|
||||
import { resolveCandidateThinkingLevel } from "../../agents/thinking-runtime.js";
|
||||
import type { SessionEntry } from "../../config/sessions.js";
|
||||
@@ -1168,186 +1169,202 @@ export function createFollowupRunner(params: {
|
||||
shouldEmitToolOutput: shouldEmitToolOutputProgress,
|
||||
deliver: deliverFollowupToolSummary,
|
||||
});
|
||||
const result = await runCliAgentWithLifecycle({
|
||||
runId,
|
||||
lifecycleGeneration,
|
||||
provider: cliExecutionProvider,
|
||||
startedAt: cliLifecycleStartedAt,
|
||||
emitLifecycleTerminal: false,
|
||||
onAgentRunStart: () => opts?.onAgentRunStart?.(runId),
|
||||
suppressAssistantBridge: run.silentExpected,
|
||||
onActivity: () => replyOperation?.recordActivity(),
|
||||
preserveProgressCallbackStartOrder,
|
||||
onReasoningText: createCliReasoningStreamBridge(progressOpts?.onReasoningStream),
|
||||
onReasoningProgress: async (payload) => {
|
||||
await progressOpts?.onReasoningProgress?.(payload);
|
||||
const result = await withLocalSessionPlacementTurnAdmission(
|
||||
{
|
||||
sessionId: run.sessionId,
|
||||
sessionKey: replySessionKey,
|
||||
agentId: run.agentId,
|
||||
runId,
|
||||
},
|
||||
onToolEvent: async (payload) => {
|
||||
if (!preserveProgressCallbackStartOrder) {
|
||||
await cliToolSummaryTracker.noteToolEvent(payload);
|
||||
if (payload.phase === "result") {
|
||||
return;
|
||||
}
|
||||
await forwardFollowupProgressEvent({
|
||||
evt: {
|
||||
stream: "tool",
|
||||
data: { name: payload.name, phase: payload.phase, args: payload.args },
|
||||
},
|
||||
opts: progressOpts,
|
||||
detailMode: toolProgressDetail,
|
||||
emitChannelProgress: shouldEmitToolResultProgress(),
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (payload.phase === "result") {
|
||||
await cliToolSummaryTracker.noteToolEvent(payload);
|
||||
return;
|
||||
}
|
||||
// CLI bridges drain independently. Start channel presentation before
|
||||
// summary bookkeeping can yield and let later progress overtake this tool.
|
||||
const presentationPromise = forwardFollowupProgressEvent({
|
||||
evt: {
|
||||
stream: "tool",
|
||||
data: { name: payload.name, phase: payload.phase, args: payload.args },
|
||||
() =>
|
||||
runCliAgentWithLifecycle({
|
||||
runId,
|
||||
lifecycleGeneration,
|
||||
provider: cliExecutionProvider,
|
||||
startedAt: cliLifecycleStartedAt,
|
||||
emitLifecycleTerminal: false,
|
||||
onAgentRunStart: () => opts?.onAgentRunStart?.(runId),
|
||||
suppressAssistantBridge: run.silentExpected,
|
||||
onActivity: () => replyOperation?.recordActivity(),
|
||||
preserveProgressCallbackStartOrder,
|
||||
onReasoningText: createCliReasoningStreamBridge(
|
||||
progressOpts?.onReasoningStream,
|
||||
),
|
||||
onReasoningProgress: async (payload) => {
|
||||
await progressOpts?.onReasoningProgress?.(payload);
|
||||
},
|
||||
opts: progressOpts,
|
||||
detailMode: toolProgressDetail,
|
||||
emitChannelProgress: shouldEmitToolResultProgress(),
|
||||
});
|
||||
await Promise.all([
|
||||
presentationPromise,
|
||||
cliToolSummaryTracker.noteToolEvent(payload),
|
||||
]);
|
||||
},
|
||||
onCommentaryText:
|
||||
progressOpts?.commentaryProgressEnabled === true && progressOpts.onItemEvent
|
||||
? async ({ text, itemId }) => {
|
||||
onToolEvent: async (payload) => {
|
||||
if (!preserveProgressCallbackStartOrder) {
|
||||
await cliToolSummaryTracker.noteToolEvent(payload);
|
||||
if (payload.phase === "result") {
|
||||
return;
|
||||
}
|
||||
await forwardFollowupProgressEvent({
|
||||
evt: {
|
||||
stream: "item",
|
||||
data: { kind: "preamble", progressText: text, itemId },
|
||||
stream: "tool",
|
||||
data: {
|
||||
name: payload.name,
|
||||
phase: payload.phase,
|
||||
args: payload.args,
|
||||
},
|
||||
},
|
||||
opts: progressOpts,
|
||||
detailMode: toolProgressDetail,
|
||||
emitChannelProgress: shouldEmitToolResultProgress(),
|
||||
});
|
||||
return;
|
||||
}
|
||||
: undefined,
|
||||
onFastModeAutoProgress: async (payload) => {
|
||||
await enqueueProgressDelivery(async () => {
|
||||
// Mirrors direct dispatch progress suppression: ambient
|
||||
// room events never get automatic fast-mode notices.
|
||||
if (isRoomEventFollowup()) {
|
||||
return;
|
||||
}
|
||||
await sendRunPayloads(
|
||||
[payload],
|
||||
effectiveQueued,
|
||||
{
|
||||
provider,
|
||||
modelId: model,
|
||||
},
|
||||
{ kind: "tool", mirror: false, runId },
|
||||
);
|
||||
});
|
||||
},
|
||||
transformResult:
|
||||
queued.currentInboundEventKind === "room_event"
|
||||
? (resultLocal) =>
|
||||
keepCliSessionBindingOnlyWhenReused({
|
||||
result: resultLocal,
|
||||
existingSessionId: cliSessionBinding?.sessionId,
|
||||
onDroppedReplacement: () => {
|
||||
droppedCliSessionReplacement = true;
|
||||
if (payload.phase === "result") {
|
||||
await cliToolSummaryTracker.noteToolEvent(payload);
|
||||
return;
|
||||
}
|
||||
// CLI bridges drain independently. Start channel presentation before
|
||||
// summary bookkeeping can yield and let later progress overtake this tool.
|
||||
const presentationPromise = forwardFollowupProgressEvent({
|
||||
evt: {
|
||||
stream: "tool",
|
||||
data: { name: payload.name, phase: payload.phase, args: payload.args },
|
||||
},
|
||||
opts: progressOpts,
|
||||
detailMode: toolProgressDetail,
|
||||
emitChannelProgress: shouldEmitToolResultProgress(),
|
||||
});
|
||||
await Promise.all([
|
||||
presentationPromise,
|
||||
cliToolSummaryTracker.noteToolEvent(payload),
|
||||
]);
|
||||
},
|
||||
onCommentaryText:
|
||||
progressOpts?.commentaryProgressEnabled === true && progressOpts.onItemEvent
|
||||
? async ({ text, itemId }) => {
|
||||
await forwardFollowupProgressEvent({
|
||||
evt: {
|
||||
stream: "item",
|
||||
data: { kind: "preamble", progressText: text, itemId },
|
||||
},
|
||||
opts: progressOpts,
|
||||
detailMode: toolProgressDetail,
|
||||
});
|
||||
}
|
||||
: undefined,
|
||||
onFastModeAutoProgress: async (payload) => {
|
||||
await enqueueProgressDelivery(async () => {
|
||||
// Mirrors direct dispatch progress suppression: ambient
|
||||
// room events never get automatic fast-mode notices.
|
||||
if (isRoomEventFollowup()) {
|
||||
return;
|
||||
}
|
||||
await sendRunPayloads(
|
||||
[payload],
|
||||
effectiveQueued,
|
||||
{
|
||||
provider,
|
||||
modelId: model,
|
||||
},
|
||||
})
|
||||
: undefined,
|
||||
runParams: {
|
||||
replyOperation,
|
||||
sessionId: run.sessionId,
|
||||
sessionKey: replySessionKey,
|
||||
runtimePolicySessionKey: run.runtimePolicySessionKey,
|
||||
agentId: run.agentId,
|
||||
trigger: opts?.isHeartbeat === true ? "heartbeat" : "user",
|
||||
sessionFile: run.sessionFile,
|
||||
workspaceDir: run.workspaceDir,
|
||||
cwd: run.cwd,
|
||||
config: runtimeConfig,
|
||||
prompt: queued.prompt,
|
||||
transcriptPrompt: queued.transcriptPrompt,
|
||||
suppressNextUserMessagePersistence: suppressQueuedUserPersistenceForCandidate,
|
||||
userTurnTranscriptRecorder,
|
||||
onUserMessagePersisted: notifyUserMessagePersisted,
|
||||
persistAssistantTranscript:
|
||||
queued.currentInboundEventKind !== "room_event" &&
|
||||
run.suppressTranscriptOnlyAssistantPersistence !== true,
|
||||
storePath,
|
||||
currentInboundEventKind: queued.currentInboundEventKind,
|
||||
currentInboundAudio: queued.currentInboundAudio,
|
||||
currentInboundContext,
|
||||
inputProvenance: run.inputProvenance,
|
||||
modelProvider: provider,
|
||||
provider: cliExecutionProvider,
|
||||
execOverrides: run.execOverrides,
|
||||
bashElevated: run.bashElevated,
|
||||
model,
|
||||
...resolveRunAuthProfile(candidateRun, cliExecutionProvider, {
|
||||
config: runtimeConfig,
|
||||
{ kind: "tool", mirror: false, runId },
|
||||
);
|
||||
});
|
||||
},
|
||||
transformResult:
|
||||
queued.currentInboundEventKind === "room_event"
|
||||
? (resultLocal) =>
|
||||
keepCliSessionBindingOnlyWhenReused({
|
||||
result: resultLocal,
|
||||
existingSessionId: cliSessionBinding?.sessionId,
|
||||
onDroppedReplacement: () => {
|
||||
droppedCliSessionReplacement = true;
|
||||
},
|
||||
})
|
||||
: undefined,
|
||||
runParams: {
|
||||
replyOperation,
|
||||
sessionId: run.sessionId,
|
||||
sessionKey: replySessionKey,
|
||||
runtimePolicySessionKey: run.runtimePolicySessionKey,
|
||||
agentId: run.agentId,
|
||||
trigger: opts?.isHeartbeat === true ? "heartbeat" : "user",
|
||||
sessionFile: run.sessionFile,
|
||||
workspaceDir: run.workspaceDir,
|
||||
cwd: run.cwd,
|
||||
config: runtimeConfig,
|
||||
prompt: queued.prompt,
|
||||
transcriptPrompt: queued.transcriptPrompt,
|
||||
suppressNextUserMessagePersistence:
|
||||
suppressQueuedUserPersistenceForCandidate,
|
||||
userTurnTranscriptRecorder,
|
||||
onUserMessagePersisted: notifyUserMessagePersisted,
|
||||
persistAssistantTranscript:
|
||||
queued.currentInboundEventKind !== "room_event" &&
|
||||
run.suppressTranscriptOnlyAssistantPersistence !== true,
|
||||
storePath,
|
||||
currentInboundEventKind: queued.currentInboundEventKind,
|
||||
currentInboundAudio: queued.currentInboundAudio,
|
||||
currentInboundContext,
|
||||
inputProvenance: run.inputProvenance,
|
||||
modelProvider: provider,
|
||||
provider: cliExecutionProvider,
|
||||
execOverrides: run.execOverrides,
|
||||
bashElevated: run.bashElevated,
|
||||
model,
|
||||
...resolveRunAuthProfile(candidateRun, cliExecutionProvider, {
|
||||
config: runtimeConfig,
|
||||
}),
|
||||
thinkLevel: candidateThinkLevel,
|
||||
fastMode: candidateFastMode.fastMode,
|
||||
fastModeStartedAtMs,
|
||||
fastModeAutoOnSeconds: candidateFastMode.fastModeAutoOnSeconds,
|
||||
fastModeAutoProgressState,
|
||||
isFinalFallbackAttempt: runOptions?.isFinalFallbackAttempt,
|
||||
timeoutMs: run.timeoutMs,
|
||||
runTimeoutOverrideMs: run.runTimeoutOverrideMs,
|
||||
runId,
|
||||
extraSystemPrompt: run.extraSystemPrompt,
|
||||
sourceReplyDeliveryMode: run.sourceReplyDeliveryMode,
|
||||
taskSuggestionDeliveryMode: run.taskSuggestionDeliveryMode,
|
||||
silentReplyPromptMode: run.silentReplyPromptMode,
|
||||
allowEmptyAssistantReplyAsSilent: run.allowEmptyAssistantReplyAsSilent,
|
||||
extraSystemPromptStatic: run.extraSystemPromptStatic,
|
||||
cliSessionBindingFacts: run.cliSessionBindingFacts,
|
||||
ownerNumbers: run.ownerNumbers,
|
||||
cliSessionId: cliSessionBinding?.sessionId,
|
||||
cliSessionBinding,
|
||||
bootstrapPromptWarningSignaturesSeen,
|
||||
bootstrapPromptWarningSignature:
|
||||
bootstrapPromptWarningSignaturesSeen[
|
||||
bootstrapPromptWarningSignaturesSeen.length - 1
|
||||
],
|
||||
images: queuedImages,
|
||||
imageOrder: queuedImageOrder,
|
||||
skillsSnapshot: run.skillsSnapshot,
|
||||
messageChannel: queued.originatingChannel ?? undefined,
|
||||
messageProvider: resolveOriginMessageProvider({
|
||||
originatingChannel: queued.originatingChannel,
|
||||
provider: run.messageProvider,
|
||||
}),
|
||||
clientCaps: run.clientCaps,
|
||||
currentChannelId: queued.originatingTo,
|
||||
senderId: run.senderId,
|
||||
senderName: run.senderName,
|
||||
senderUsername: run.senderUsername,
|
||||
senderE164: run.senderE164,
|
||||
groupId: run.groupId,
|
||||
groupChannel: run.groupChannel,
|
||||
groupSpace: run.groupSpace,
|
||||
spawnedBy: run.spawnedBy,
|
||||
chatId: queued.originatingChatId,
|
||||
channelContext: run.channelContext,
|
||||
currentThreadTs:
|
||||
queued.originatingThreadId != null
|
||||
? String(queued.originatingThreadId)
|
||||
: undefined,
|
||||
currentMessageId: followupCurrentMessageId,
|
||||
agentAccountId: run.agentAccountId,
|
||||
senderIsOwner: run.senderIsOwner,
|
||||
disableTools: opts?.disableTools,
|
||||
abortSignal: runAbortSignal,
|
||||
},
|
||||
}),
|
||||
thinkLevel: candidateThinkLevel,
|
||||
fastMode: candidateFastMode.fastMode,
|
||||
fastModeStartedAtMs,
|
||||
fastModeAutoOnSeconds: candidateFastMode.fastModeAutoOnSeconds,
|
||||
fastModeAutoProgressState,
|
||||
isFinalFallbackAttempt: runOptions?.isFinalFallbackAttempt,
|
||||
timeoutMs: run.timeoutMs,
|
||||
runTimeoutOverrideMs: run.runTimeoutOverrideMs,
|
||||
runId,
|
||||
extraSystemPrompt: run.extraSystemPrompt,
|
||||
sourceReplyDeliveryMode: run.sourceReplyDeliveryMode,
|
||||
taskSuggestionDeliveryMode: run.taskSuggestionDeliveryMode,
|
||||
silentReplyPromptMode: run.silentReplyPromptMode,
|
||||
allowEmptyAssistantReplyAsSilent: run.allowEmptyAssistantReplyAsSilent,
|
||||
extraSystemPromptStatic: run.extraSystemPromptStatic,
|
||||
cliSessionBindingFacts: run.cliSessionBindingFacts,
|
||||
ownerNumbers: run.ownerNumbers,
|
||||
cliSessionId: cliSessionBinding?.sessionId,
|
||||
cliSessionBinding,
|
||||
bootstrapPromptWarningSignaturesSeen,
|
||||
bootstrapPromptWarningSignature:
|
||||
bootstrapPromptWarningSignaturesSeen[
|
||||
bootstrapPromptWarningSignaturesSeen.length - 1
|
||||
],
|
||||
images: queuedImages,
|
||||
imageOrder: queuedImageOrder,
|
||||
skillsSnapshot: run.skillsSnapshot,
|
||||
messageChannel: queued.originatingChannel ?? undefined,
|
||||
messageProvider: resolveOriginMessageProvider({
|
||||
originatingChannel: queued.originatingChannel,
|
||||
provider: run.messageProvider,
|
||||
}),
|
||||
clientCaps: run.clientCaps,
|
||||
currentChannelId: queued.originatingTo,
|
||||
senderId: run.senderId,
|
||||
senderName: run.senderName,
|
||||
senderUsername: run.senderUsername,
|
||||
senderE164: run.senderE164,
|
||||
groupId: run.groupId,
|
||||
groupChannel: run.groupChannel,
|
||||
groupSpace: run.groupSpace,
|
||||
spawnedBy: run.spawnedBy,
|
||||
chatId: queued.originatingChatId,
|
||||
channelContext: run.channelContext,
|
||||
currentThreadTs:
|
||||
queued.originatingThreadId != null
|
||||
? String(queued.originatingThreadId)
|
||||
: undefined,
|
||||
currentMessageId: followupCurrentMessageId,
|
||||
agentAccountId: run.agentAccountId,
|
||||
senderIsOwner: run.senderIsOwner,
|
||||
disableTools: opts?.disableTools,
|
||||
abortSignal: runAbortSignal,
|
||||
},
|
||||
});
|
||||
);
|
||||
if (droppedCliSessionReplacement) {
|
||||
await clearDroppedCliSessionBinding({
|
||||
provider: cliExecutionProvider,
|
||||
|
||||
@@ -338,6 +338,7 @@ export const cliCommandCatalog: readonly CliCommandCatalogEntry[] = [
|
||||
bypassConfigGuard: true,
|
||||
hideBanner: true,
|
||||
loadPlugins: "never",
|
||||
ownsProtocolStdout: true,
|
||||
networkProxy: "bypass",
|
||||
},
|
||||
},
|
||||
|
||||
@@ -191,6 +191,7 @@ describe("command-path-policy", () => {
|
||||
bypassConfigGuard: true,
|
||||
loadPlugins: "never",
|
||||
hideBanner: true,
|
||||
ownsProtocolStdout: true,
|
||||
networkProxy: "bypass",
|
||||
});
|
||||
expectResolvedPolicy(["configure"], {
|
||||
|
||||
@@ -254,6 +254,7 @@ describe("command-startup-policy", () => {
|
||||
expect(shouldBypassConfigGuardForCommandPath(["worker"])).toBe(true);
|
||||
expect(policy.hideBanner).toBe(true);
|
||||
expect(policy.loadPlugins).toBe(false);
|
||||
expect(policy.suppressDoctorStdout).toBe(true);
|
||||
});
|
||||
|
||||
it("suppresses startup stdout for the bare acp protocol", () => {
|
||||
|
||||
@@ -8,6 +8,7 @@ import { runAgentHarnessBeforeMessageWriteHook } from "../../agents/harness/hook
|
||||
import type { ModelCatalogEntry } from "../../agents/model-catalog.types.js";
|
||||
import { resolveCliRuntimeExecutionProvider } from "../../agents/model-runtime-aliases.js";
|
||||
import { wrapUntrustedPromptDataBlock } from "../../agents/sanitize-for-prompt.js";
|
||||
import { withLocalSessionPlacementTurnAdmission } from "../../agents/session-placement-admission.js";
|
||||
import { resolveSessionRuntimeOverrideForProvider } from "../../agents/session-runtime-compat.js";
|
||||
import type { ThinkLevel, VerboseLevel } from "../../auto-reply/thinking.js";
|
||||
import type { CliSessionBinding } from "../../config/sessions.js";
|
||||
@@ -415,55 +416,69 @@ export function createCronPromptExecutor(params: {
|
||||
cliSessionBinding && hasCliSessionReuseMetadata(cliSessionBinding)
|
||||
? cliSessionBinding
|
||||
: undefined;
|
||||
const result = await runCliAgent({
|
||||
sessionId: params.cronSession.sessionEntry.sessionId,
|
||||
sessionKey: params.runSessionKey,
|
||||
sessionEntry: params.cronSession.sessionEntry,
|
||||
agentId: params.agentId,
|
||||
trigger: "cron",
|
||||
jobId: params.job.id,
|
||||
cleanupCliLiveSessionOnRunEnd: params.usesDetachedRunSession === true,
|
||||
sessionFile,
|
||||
workspaceDir: params.workspaceDir,
|
||||
config: params.cfgWithAgentDefaults,
|
||||
prompt: modelPrompt,
|
||||
transcriptPrompt: deliveryTargetRuntimeContext ? promptText : undefined,
|
||||
modelProvider: providerOverride,
|
||||
provider: executionProvider,
|
||||
model: modelOverride,
|
||||
thinkLevel: candidateThinkLevel,
|
||||
timeoutMs: params.timeoutMs,
|
||||
runId: params.cronSession.sessionEntry.sessionId,
|
||||
lane: resolveCronAgentLane(params.lane),
|
||||
allowEmptyAssistantReplyAsSilent,
|
||||
cliSessionId: cliSessionBinding?.sessionId,
|
||||
cliSessionBinding: guardedCliSessionBinding,
|
||||
skillsSnapshot: params.skillsSnapshot,
|
||||
messageChannel,
|
||||
sourceReplyDeliveryMode,
|
||||
requireExplicitMessageTarget: sourceDelivery.messageTool.requireExplicitTarget,
|
||||
cliSessionBindingFacts: {
|
||||
sourceReplyDeliveryMode,
|
||||
requireExplicitMessageTarget: sourceDelivery.messageTool.requireExplicitTarget,
|
||||
// Cron intentionally reuses its durable session id as the run id; turn
|
||||
// claims stay unique via per-claim ids and the worker gate handles this
|
||||
// via credential rotation (see worker-environments/service.ts fences).
|
||||
const runId = params.cronSession.sessionEntry.sessionId;
|
||||
const result = await withLocalSessionPlacementTurnAdmission(
|
||||
{
|
||||
sessionId: params.cronSession.sessionEntry.sessionId,
|
||||
sessionKey: params.runSessionKey,
|
||||
agentId: params.agentId,
|
||||
runId,
|
||||
},
|
||||
toolsAllow: resolveCliRuntimeToolsAllow(
|
||||
params.agentPayload?.toolsAllow,
|
||||
params.agentPayload?.toolsAllowIsDefault,
|
||||
),
|
||||
abortSignal: params.abortSignal,
|
||||
onExecutionStarted: params.onExecutionStarted,
|
||||
onExecutionPhase: params.onExecutionPhase,
|
||||
bootstrapContextMode,
|
||||
bootstrapContextRunKind: "cron",
|
||||
bootstrapPromptWarningSignaturesSeen,
|
||||
bootstrapPromptWarningSignature,
|
||||
fastModeStartedAtMs,
|
||||
fastModeAutoProgressState,
|
||||
isFinalFallbackAttempt: runOptions?.isFinalFallbackAttempt,
|
||||
userTurnTranscriptRecorder,
|
||||
suppressNextUserMessagePersistence:
|
||||
userTurnTranscriptRecorder.hasPersisted() || userTurnTranscriptRecorder.isBlocked(),
|
||||
});
|
||||
() =>
|
||||
runCliAgent({
|
||||
sessionId: params.cronSession.sessionEntry.sessionId,
|
||||
sessionKey: params.runSessionKey,
|
||||
sessionEntry: params.cronSession.sessionEntry,
|
||||
agentId: params.agentId,
|
||||
trigger: "cron",
|
||||
jobId: params.job.id,
|
||||
cleanupCliLiveSessionOnRunEnd: params.usesDetachedRunSession === true,
|
||||
sessionFile,
|
||||
workspaceDir: params.workspaceDir,
|
||||
config: params.cfgWithAgentDefaults,
|
||||
prompt: modelPrompt,
|
||||
transcriptPrompt: deliveryTargetRuntimeContext ? promptText : undefined,
|
||||
modelProvider: providerOverride,
|
||||
provider: executionProvider,
|
||||
model: modelOverride,
|
||||
thinkLevel: candidateThinkLevel,
|
||||
timeoutMs: params.timeoutMs,
|
||||
runId,
|
||||
lane: resolveCronAgentLane(params.lane),
|
||||
allowEmptyAssistantReplyAsSilent,
|
||||
cliSessionId: cliSessionBinding?.sessionId,
|
||||
cliSessionBinding: guardedCliSessionBinding,
|
||||
skillsSnapshot: params.skillsSnapshot,
|
||||
messageChannel,
|
||||
sourceReplyDeliveryMode,
|
||||
requireExplicitMessageTarget: sourceDelivery.messageTool.requireExplicitTarget,
|
||||
cliSessionBindingFacts: {
|
||||
sourceReplyDeliveryMode,
|
||||
requireExplicitMessageTarget: sourceDelivery.messageTool.requireExplicitTarget,
|
||||
},
|
||||
toolsAllow: resolveCliRuntimeToolsAllow(
|
||||
params.agentPayload?.toolsAllow,
|
||||
params.agentPayload?.toolsAllowIsDefault,
|
||||
),
|
||||
abortSignal: params.abortSignal,
|
||||
onExecutionStarted: params.onExecutionStarted,
|
||||
onExecutionPhase: params.onExecutionPhase,
|
||||
bootstrapContextMode,
|
||||
bootstrapContextRunKind: "cron",
|
||||
bootstrapPromptWarningSignaturesSeen,
|
||||
bootstrapPromptWarningSignature,
|
||||
fastModeStartedAtMs,
|
||||
fastModeAutoProgressState,
|
||||
isFinalFallbackAttempt: runOptions?.isFinalFallbackAttempt,
|
||||
userTurnTranscriptRecorder,
|
||||
suppressNextUserMessagePersistence:
|
||||
userTurnTranscriptRecorder.hasPersisted() ||
|
||||
userTurnTranscriptRecorder.isBlocked(),
|
||||
}),
|
||||
);
|
||||
bootstrapPromptWarningSignaturesSeen = resolveBootstrapWarningSignaturesSeen(
|
||||
result.meta?.systemPromptReport,
|
||||
);
|
||||
|
||||
@@ -51,6 +51,7 @@ describe("method scope resolution", () => {
|
||||
["taskSuggestions.dismiss", ["operator.write"]],
|
||||
["config.schema.lookup", ["operator.read"]],
|
||||
["sessions.create", ["operator.write"]],
|
||||
["sessions.dispatch", ["operator.admin"]],
|
||||
["sessions.send", ["operator.write"]],
|
||||
["sessions.abort", ["operator.write"]],
|
||||
["tasks.cancel", ["operator.write"]],
|
||||
|
||||
@@ -345,6 +345,12 @@ const CORE_GATEWAY_METHOD_SPECS: readonly CoreGatewayMethodSpec[] = [
|
||||
{ name: "approval.get", scope: "operator.approvals" },
|
||||
{ name: "approval.resolve", scope: "operator.approvals" },
|
||||
{ name: "sessions.search", scope: "operator.read" },
|
||||
{
|
||||
name: "sessions.dispatch",
|
||||
scope: "operator.admin",
|
||||
startup: true,
|
||||
controlPlaneWrite: true,
|
||||
},
|
||||
] as const;
|
||||
|
||||
const CORE_GATEWAY_METHOD_SPEC_BY_NAME: ReadonlyMap<string, CoreGatewayMethodSpec> = new Map(
|
||||
|
||||
@@ -86,11 +86,11 @@ describe("listGatewayMethods", () => {
|
||||
]);
|
||||
expect(methods).toContain("tts.speak");
|
||||
expect(coreMethods.slice(-5)).toEqual([
|
||||
"sessions.catalog.continue",
|
||||
"sessions.catalog.archive",
|
||||
"approval.get",
|
||||
"approval.resolve",
|
||||
"sessions.search",
|
||||
"sessions.dispatch",
|
||||
]);
|
||||
expect(methods.indexOf("approval.get")).toBeGreaterThan(methods.indexOf("tts.speak"));
|
||||
expect(methods.indexOf("approval.resolve")).toBe(methods.indexOf("approval.get") + 1);
|
||||
|
||||
@@ -0,0 +1,403 @@
|
||||
import { expectDefined } from "@openclaw/normalization-core";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { ErrorCodes } from "../../../packages/gateway-protocol/src/index.js";
|
||||
import type { WorkerSessionPlacementRecord } from "../worker-environments/placement-store.js";
|
||||
import type { GatewayRequestContext, RespondFn } from "./types.js";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
findLiveByOwner: vi.fn(),
|
||||
resolveTarget: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../../agents/worktrees/service.js", () => ({
|
||||
managedWorktrees: {
|
||||
findLiveByOwner: mocks.findLiveByOwner,
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("../session-utils.js", async () => {
|
||||
const actual = await vi.importActual<typeof import("../session-utils.js")>("../session-utils.js");
|
||||
return {
|
||||
...actual,
|
||||
resolveGatewaySessionStoreTargetWithStore: mocks.resolveTarget,
|
||||
};
|
||||
});
|
||||
|
||||
import { sessionsHandlers } from "./sessions.js";
|
||||
|
||||
const sessionKey = "agent:main:cloud-test";
|
||||
const sessionId = "session-cloud-test";
|
||||
|
||||
function reclaimedPlacementRecord(): WorkerSessionPlacementRecord {
|
||||
return {
|
||||
sessionId,
|
||||
agentId: "main",
|
||||
sessionKey,
|
||||
state: "reclaimed",
|
||||
environmentId: "environment-previous",
|
||||
generation: 4,
|
||||
activeOwnerEpoch: 1,
|
||||
workspaceBaseManifestRef: "manifest-previous",
|
||||
remoteWorkspaceDir: "/worker/session-cloud-test",
|
||||
workerBundleHash: "c".repeat(64),
|
||||
lastTranscriptAckCursor: 3,
|
||||
lastLiveEventAckCursor: 2,
|
||||
recoveryError: null,
|
||||
turnClaim: null,
|
||||
createdAtMs: 1,
|
||||
updatedAtMs: 2,
|
||||
stateChangedAtMs: 2,
|
||||
};
|
||||
}
|
||||
|
||||
function targetWithEntry(entry?: {
|
||||
sessionId: string;
|
||||
worktree?: { id: string; branch: string; repoRoot: string };
|
||||
agentHarnessId?: string;
|
||||
agentRuntimeOverride?: string;
|
||||
archivedAt?: number;
|
||||
modelSelectionLocked?: boolean;
|
||||
providerOverride?: string;
|
||||
modelOverride?: string;
|
||||
}) {
|
||||
// Pin an anthropic model by default: the effective-runtime fallback consults
|
||||
// the process-global harness registry, so the default openai model resolves
|
||||
// to "codex" whenever a sibling test in the shard registered that harness.
|
||||
const pinnedEntry = entry
|
||||
? { providerOverride: "anthropic", modelOverride: "claude-test", ...entry }
|
||||
: undefined;
|
||||
return {
|
||||
agentId: "main",
|
||||
storePath: "/tmp/openclaw-agent.sqlite",
|
||||
canonicalKey: sessionKey,
|
||||
storeKeys: [sessionKey],
|
||||
store: pinnedEntry ? { [sessionKey]: pinnedEntry } : {},
|
||||
};
|
||||
}
|
||||
|
||||
function makeContext(overrides: Partial<GatewayRequestContext> = {}): GatewayRequestContext {
|
||||
return {
|
||||
getRuntimeConfig: () => ({
|
||||
cloudWorkers: {
|
||||
profiles: {
|
||||
test: { provider: "fake", region: "test", size: "small" },
|
||||
},
|
||||
},
|
||||
}),
|
||||
...overrides,
|
||||
} as unknown as GatewayRequestContext;
|
||||
}
|
||||
|
||||
async function invoke(context: GatewayRequestContext) {
|
||||
const respond = vi.fn() as unknown as RespondFn;
|
||||
await expectDefined(
|
||||
sessionsHandlers["sessions.dispatch"],
|
||||
'sessionsHandlers["sessions.dispatch"] test invariant',
|
||||
)({
|
||||
req: { id: "dispatch-request" } as never,
|
||||
params: { key: sessionKey, profileId: "test" },
|
||||
respond,
|
||||
context,
|
||||
client: null,
|
||||
isWebchatConnect: () => false,
|
||||
});
|
||||
return respond;
|
||||
}
|
||||
|
||||
describe("sessions.dispatch", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mocks.resolveTarget.mockReturnValue(targetWithEntry());
|
||||
});
|
||||
|
||||
it("stays unavailable without a configured placement dispatcher", async () => {
|
||||
const respond = await invoke(makeContext());
|
||||
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
false,
|
||||
undefined,
|
||||
expect.objectContaining({ code: ErrorCodes.INVALID_REQUEST }),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects a missing session before dispatch", async () => {
|
||||
const dispatch = vi.fn();
|
||||
const respond = await invoke(
|
||||
makeContext({
|
||||
workerPlacementDispatchService: { dispatch },
|
||||
workerSessionPlacementService: { getMany: () => new Map() },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(dispatch).not.toHaveBeenCalled();
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
false,
|
||||
undefined,
|
||||
expect.objectContaining({ code: ErrorCodes.INVALID_REQUEST }),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects sessions without their bound managed worktree", async () => {
|
||||
mocks.resolveTarget.mockReturnValue(targetWithEntry({ sessionId }));
|
||||
const dispatch = vi.fn();
|
||||
const respond = await invoke(
|
||||
makeContext({
|
||||
workerPlacementDispatchService: { dispatch },
|
||||
workerSessionPlacementService: { getMany: () => new Map() },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(dispatch).not.toHaveBeenCalled();
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
false,
|
||||
undefined,
|
||||
expect.objectContaining({
|
||||
code: ErrorCodes.INVALID_REQUEST,
|
||||
message: expect.stringContaining("session-owned managed worktree"),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects dispatch from a nonlocal placement", async () => {
|
||||
mocks.resolveTarget.mockReturnValue(targetWithEntry({ sessionId }));
|
||||
const dispatch = vi.fn();
|
||||
const respond = await invoke(
|
||||
makeContext({
|
||||
workerPlacementDispatchService: { dispatch },
|
||||
workerSessionPlacementService: {
|
||||
getMany: () => new Map([[sessionId, { state: "requested" } as never]]),
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(dispatch).not.toHaveBeenCalled();
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
false,
|
||||
undefined,
|
||||
expect.objectContaining({
|
||||
code: ErrorCodes.INVALID_REQUEST,
|
||||
message: expect.stringContaining("placement requested"),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects sessions owned by an unsupported runtime", async () => {
|
||||
mocks.resolveTarget.mockReturnValue(
|
||||
targetWithEntry({
|
||||
sessionId,
|
||||
agentRuntimeOverride: "codex",
|
||||
worktree: { id: "worktree-1", branch: "openclaw/cloud-test", repoRoot: "/repo" },
|
||||
}),
|
||||
);
|
||||
const dispatch = vi.fn();
|
||||
const respond = await invoke(
|
||||
makeContext({
|
||||
workerPlacementDispatchService: { dispatch },
|
||||
workerSessionPlacementService: { getMany: () => new Map() },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(dispatch).not.toHaveBeenCalled();
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
false,
|
||||
undefined,
|
||||
expect.objectContaining({
|
||||
code: ErrorCodes.INVALID_REQUEST,
|
||||
message: expect.stringContaining("OpenClaw runtime"),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects an archived session before dispatch", async () => {
|
||||
mocks.resolveTarget.mockReturnValue(
|
||||
targetWithEntry({
|
||||
sessionId,
|
||||
archivedAt: 2,
|
||||
worktree: { id: "worktree-1", branch: "openclaw/cloud-test", repoRoot: "/repo" },
|
||||
}),
|
||||
);
|
||||
const dispatch = vi.fn();
|
||||
const respond = await invoke(
|
||||
makeContext({
|
||||
workerPlacementDispatchService: { dispatch },
|
||||
workerSessionPlacementService: { getMany: () => new Map() },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(dispatch).not.toHaveBeenCalled();
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
false,
|
||||
undefined,
|
||||
expect.objectContaining({
|
||||
code: ErrorCodes.INVALID_REQUEST,
|
||||
message: expect.stringContaining("archived"),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("allows an explicitly reclaimed session to dispatch again", async () => {
|
||||
mocks.resolveTarget.mockReturnValue(
|
||||
targetWithEntry({
|
||||
sessionId,
|
||||
worktree: { id: "worktree-1", branch: "openclaw/cloud-test", repoRoot: "/repo" },
|
||||
}),
|
||||
);
|
||||
mocks.findLiveByOwner.mockReturnValue({
|
||||
id: "worktree-1",
|
||||
ownerKind: "session",
|
||||
ownerId: sessionKey,
|
||||
});
|
||||
const dispatchedPlacement: WorkerSessionPlacementRecord = {
|
||||
sessionId,
|
||||
agentId: "main",
|
||||
sessionKey,
|
||||
state: "active",
|
||||
environmentId: "environment-2",
|
||||
generation: 5,
|
||||
activeOwnerEpoch: 2,
|
||||
workspaceBaseManifestRef: "manifest-2",
|
||||
remoteWorkspaceDir: "/worker/session-cloud-test",
|
||||
workerBundleHash: "d".repeat(64),
|
||||
lastTranscriptAckCursor: null,
|
||||
lastLiveEventAckCursor: null,
|
||||
recoveryError: null,
|
||||
turnClaim: null,
|
||||
createdAtMs: 1,
|
||||
updatedAtMs: 3,
|
||||
stateChangedAtMs: 3,
|
||||
};
|
||||
const dispatch = vi.fn().mockResolvedValue(dispatchedPlacement);
|
||||
const respond = await invoke(
|
||||
makeContext({
|
||||
workerPlacementDispatchService: { dispatch },
|
||||
workerSessionPlacementService: {
|
||||
getMany: () => new Map([[sessionId, reclaimedPlacementRecord()]]),
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
expect(dispatch).toHaveBeenCalledWith({
|
||||
sessionId,
|
||||
sessionKey,
|
||||
agentId: "main",
|
||||
profileId: "test",
|
||||
});
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
true,
|
||||
expect.objectContaining({
|
||||
placement: expect.objectContaining({
|
||||
state: "active",
|
||||
environmentId: "environment-2",
|
||||
generation: 5,
|
||||
}),
|
||||
}),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["CLI", "claude-cli"],
|
||||
["plugin", "test-harness"],
|
||||
])("rejects sessions assigned to a configured %s runtime", async (_kind, runtimeId) => {
|
||||
const modelRef = "anthropic/claude-test";
|
||||
mocks.resolveTarget.mockReturnValue(
|
||||
targetWithEntry({
|
||||
sessionId,
|
||||
providerOverride: "anthropic",
|
||||
modelOverride: "claude-test",
|
||||
worktree: { id: "worktree-1", branch: "openclaw/cloud-test", repoRoot: "/repo" },
|
||||
}),
|
||||
);
|
||||
const dispatch = vi.fn();
|
||||
const respond = await invoke(
|
||||
makeContext({
|
||||
getRuntimeConfig: () => ({
|
||||
cloudWorkers: {
|
||||
profiles: {
|
||||
test: { provider: "fake", region: "test", size: "small" },
|
||||
},
|
||||
},
|
||||
agents: {
|
||||
defaults: {
|
||||
models: {
|
||||
[modelRef]: { agentRuntime: { id: runtimeId } },
|
||||
},
|
||||
},
|
||||
},
|
||||
}),
|
||||
workerPlacementDispatchService: { dispatch },
|
||||
workerSessionPlacementService: { getMany: () => new Map() },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(dispatch).not.toHaveBeenCalled();
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
false,
|
||||
undefined,
|
||||
expect.objectContaining({
|
||||
code: ErrorCodes.INVALID_REQUEST,
|
||||
message: expect.stringContaining(runtimeId),
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("dispatches an existing managed-worktree session and projects placement", async () => {
|
||||
mocks.resolveTarget.mockReturnValue(
|
||||
targetWithEntry({
|
||||
sessionId,
|
||||
worktree: { id: "worktree-1", branch: "openclaw/cloud-test", repoRoot: "/repo" },
|
||||
}),
|
||||
);
|
||||
mocks.findLiveByOwner.mockReturnValue({
|
||||
id: "worktree-1",
|
||||
ownerKind: "session",
|
||||
ownerId: sessionKey,
|
||||
});
|
||||
const dispatch = vi.fn().mockResolvedValue({
|
||||
sessionId,
|
||||
agentId: "main",
|
||||
sessionKey,
|
||||
state: "active",
|
||||
environmentId: "environment-1",
|
||||
generation: 5,
|
||||
activeOwnerEpoch: 2,
|
||||
workspaceBaseManifestRef: "manifest-1",
|
||||
remoteWorkspaceDir: "/worker/session-cloud-test",
|
||||
workerBundleHash: "b".repeat(64),
|
||||
lastTranscriptAckCursor: null,
|
||||
lastLiveEventAckCursor: null,
|
||||
recoveryError: null,
|
||||
turnClaim: null,
|
||||
createdAtMs: 1,
|
||||
updatedAtMs: 2,
|
||||
stateChangedAtMs: 2,
|
||||
});
|
||||
const respond = await invoke(
|
||||
makeContext({
|
||||
workerPlacementDispatchService: { dispatch },
|
||||
workerSessionPlacementService: { getMany: () => new Map() },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(dispatch).toHaveBeenCalledWith({
|
||||
sessionId,
|
||||
sessionKey,
|
||||
agentId: "main",
|
||||
profileId: "test",
|
||||
});
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
true,
|
||||
expect.objectContaining({
|
||||
ok: true,
|
||||
key: sessionKey,
|
||||
sessionId,
|
||||
placement: expect.objectContaining({
|
||||
state: "active",
|
||||
environmentId: "environment-1",
|
||||
activeOwnerEpoch: 2,
|
||||
}),
|
||||
}),
|
||||
undefined,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -12,7 +12,9 @@ import { GATEWAY_CLIENT_IDS } from "../../../packages/gateway-protocol/src/clien
|
||||
import {
|
||||
ErrorCodes,
|
||||
errorShape,
|
||||
type SessionPlacement,
|
||||
type SessionOperationEvent,
|
||||
type SessionsPatchParams,
|
||||
validateSessionsAbortParams,
|
||||
validateSessionsCleanupParams,
|
||||
validateSessionsCompactParams,
|
||||
@@ -23,6 +25,7 @@ import {
|
||||
validateSessionsCreateParams,
|
||||
validateSessionsDeleteParams,
|
||||
validateSessionsDescribeParams,
|
||||
validateSessionsDispatchParams,
|
||||
validateSessionsGroupsDeleteParams,
|
||||
validateSessionsGroupsListParams,
|
||||
validateSessionsGroupsPutParams,
|
||||
@@ -149,6 +152,11 @@ import {
|
||||
import { projectSessionsPatchEntry } from "../sessions-patch.js";
|
||||
import { resolveSessionKeyFromResolveParams } from "../sessions-resolve.js";
|
||||
import { asWorkerInferenceControl } from "../worker-environments/inference-control.js";
|
||||
import { projectWorkerSessionPlacement } from "../worker-environments/placement-projector.js";
|
||||
import {
|
||||
isWorkerPlacementSessionRuntimeSupported,
|
||||
resolveWorkerPlacementSessionRuntime,
|
||||
} from "../worker-environments/placement-session-runtime.js";
|
||||
import { resolveWorkerSessionTarget } from "../worker-environments/session-target.js";
|
||||
import { setGatewayDedupeEntry } from "./agent-job.js";
|
||||
import { chatHandlers } from "./chat.js";
|
||||
@@ -174,6 +182,81 @@ const compactionCheckpointStore = createFileBackedCompactionCheckpointStore();
|
||||
const MODEL_SELECTION_LOCKED_CHECKPOINT_MESSAGE =
|
||||
"Checkpoint branch and restore are unavailable while model selection is locked.";
|
||||
|
||||
class SessionWorkerPlacementMutationError extends Error {
|
||||
constructor(
|
||||
readonly placementState: SessionPlacement["state"],
|
||||
action: "delete" | "reset" | "restore",
|
||||
key: string,
|
||||
) {
|
||||
super(`Session ${key} cannot ${action} while cloud worker placement is ${placementState}.`);
|
||||
}
|
||||
}
|
||||
|
||||
function resolveSessionWorkerPlacementMutationError(params: {
|
||||
action: "delete" | "reset" | "restore";
|
||||
context: GatewayRequestContext;
|
||||
key: string;
|
||||
sessionId: string | undefined;
|
||||
}): SessionWorkerPlacementMutationError | undefined {
|
||||
if (!params.sessionId) {
|
||||
return undefined;
|
||||
}
|
||||
const placement = params.context.workerSessionPlacementService
|
||||
?.getMany([params.sessionId])
|
||||
.get(params.sessionId);
|
||||
if (
|
||||
!placement ||
|
||||
placement.state === "local" ||
|
||||
(params.action === "delete" && placement.state === "reclaimed")
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
return new SessionWorkerPlacementMutationError(placement.state, params.action, params.key);
|
||||
}
|
||||
|
||||
function respondSessionWorkerPlacementMutationError(
|
||||
error: SessionWorkerPlacementMutationError,
|
||||
respond: RespondFn,
|
||||
): void {
|
||||
respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, error.message));
|
||||
}
|
||||
|
||||
function resolveSessionWorkerPlacementPatchError(params: {
|
||||
agentId: string;
|
||||
cfg: OpenClawConfig;
|
||||
context: GatewayRequestContext;
|
||||
entry: SessionEntry | undefined;
|
||||
key: string;
|
||||
patch: SessionsPatchParams;
|
||||
sessionKey: string;
|
||||
validateModelRuntime: boolean;
|
||||
}): string | undefined {
|
||||
const placement = params.entry?.sessionId
|
||||
? params.context.workerSessionPlacementService
|
||||
?.getMany([params.entry.sessionId])
|
||||
.get(params.entry.sessionId)
|
||||
: undefined;
|
||||
if (!placement || placement.state === "local") {
|
||||
return undefined;
|
||||
}
|
||||
if (params.patch.archived !== undefined) {
|
||||
return `Session ${params.key} cannot change archive state while cloud worker placement is ${placement.state}.`;
|
||||
}
|
||||
if (!params.validateModelRuntime || params.patch.model === undefined || !params.entry) {
|
||||
return undefined;
|
||||
}
|
||||
const runtime = resolveWorkerPlacementSessionRuntime({
|
||||
cfg: params.cfg,
|
||||
entry: params.entry,
|
||||
agentId: params.agentId,
|
||||
sessionKey: params.sessionKey,
|
||||
});
|
||||
if (isWorkerPlacementSessionRuntimeSupported(runtime)) {
|
||||
return undefined;
|
||||
}
|
||||
return `Session ${params.key} cannot select the ${runtime} runtime while cloud worker placement is ${placement.state}.`;
|
||||
}
|
||||
|
||||
function filterSessionStoreToConfiguredAgents(
|
||||
cfg: OpenClawConfig,
|
||||
store: Record<string, SessionEntry>,
|
||||
@@ -363,7 +446,7 @@ function emitSessionOperation(
|
||||
}
|
||||
|
||||
function rejectWebchatSessionMutation(params: {
|
||||
action: "patch" | "delete" | "compact" | "restore";
|
||||
action: "patch" | "delete" | "compact" | "restore" | "dispatch";
|
||||
client: GatewayClient | null;
|
||||
isWebchatConnect: (params: GatewayClient["connect"] | null | undefined) => boolean;
|
||||
respond: RespondFn;
|
||||
@@ -385,6 +468,14 @@ function rejectWebchatSessionMutation(params: {
|
||||
return true;
|
||||
}
|
||||
|
||||
function isWorkerDispatchInputError(error: unknown): boolean {
|
||||
if (typeof error !== "object" || error === null || !("code" in error)) {
|
||||
return false;
|
||||
}
|
||||
const code = error.code;
|
||||
return code === "invalid_profile" || code === "profile_not_found" || code === "invalid_state";
|
||||
}
|
||||
|
||||
function isAgentMainSessionKey(cfg: OpenClawConfig, sessionKey: string): boolean {
|
||||
const parsed = parseAgentSessionKey(sessionKey);
|
||||
if (!parsed) {
|
||||
@@ -967,10 +1058,16 @@ export const sessionsHandlers: GatewayRequestHandlers = {
|
||||
},
|
||||
},
|
||||
);
|
||||
const placementsBySessionId = context.workerSessionPlacementService?.getMany(
|
||||
result.sessions.flatMap((session) => (session.sessionId ? [session.sessionId] : [])),
|
||||
);
|
||||
const sessions = measureDiagnosticsTimelineSpanSync(
|
||||
"gateway.sessions.list.active_run_flags",
|
||||
() => {
|
||||
return result.sessions.map((session) => {
|
||||
const placementRecord = session.sessionId
|
||||
? placementsBySessionId?.get(session.sessionId)
|
||||
: undefined;
|
||||
const activeRunState = resolveVisibleActiveSessionRunState({
|
||||
context,
|
||||
requestedKey: session.key,
|
||||
@@ -981,6 +1078,9 @@ export const sessionsHandlers: GatewayRequestHandlers = {
|
||||
});
|
||||
return Object.assign({}, session, {
|
||||
hasActiveRun: activeRunState.active,
|
||||
...(placementRecord
|
||||
? { placement: projectWorkerSessionPlacement(placementRecord) }
|
||||
: {}),
|
||||
...(activeRunState.runIds.length > 0
|
||||
? { activeRunIds: activeRunState.runIds }
|
||||
: {}),
|
||||
@@ -1282,7 +1382,16 @@ export const sessionsHandlers: GatewayRequestHandlers = {
|
||||
includeLastMessage: p.includeLastMessage,
|
||||
transcriptUsageMaxBytes: 64 * 1024,
|
||||
});
|
||||
respond(true, { session: row }, undefined);
|
||||
const placement = row.sessionId
|
||||
? context.workerSessionPlacementService?.getMany([row.sessionId]).get(row.sessionId)
|
||||
: undefined;
|
||||
respond(
|
||||
true,
|
||||
{
|
||||
session: placement ? { ...row, placement: projectWorkerSessionPlacement(placement) } : row,
|
||||
},
|
||||
undefined,
|
||||
);
|
||||
},
|
||||
"sessions.resolve": async ({ params, respond, context }) => {
|
||||
if (!assertValidParams(params, validateSessionsResolveParams, "sessions.resolve", respond)) {
|
||||
@@ -1924,6 +2033,16 @@ export const sessionsHandlers: GatewayRequestHandlers = {
|
||||
);
|
||||
return;
|
||||
}
|
||||
const initialPlacementError = resolveSessionWorkerPlacementMutationError({
|
||||
action: "restore",
|
||||
context,
|
||||
key,
|
||||
sessionId: entry.sessionId,
|
||||
});
|
||||
if (initialPlacementError) {
|
||||
respondSessionWorkerPlacementMutationError(initialPlacementError, respond);
|
||||
return;
|
||||
}
|
||||
const lifecycleIdentities = [
|
||||
key,
|
||||
canonicalKey,
|
||||
@@ -1935,6 +2054,7 @@ export const sessionsHandlers: GatewayRequestHandlers = {
|
||||
let admittedWorkReleased = true;
|
||||
let restoreTargetStillCurrent = true;
|
||||
let restoreBlockedByModelLock = false;
|
||||
let restorePlacementError: SessionWorkerPlacementMutationError | undefined;
|
||||
// Restore replaces the active transcript identity. Hold the same lifecycle fence as
|
||||
// compaction so neither operation can publish state from the other's obsolete session.
|
||||
await runExclusiveSessionLifecycleMutation({
|
||||
@@ -1960,6 +2080,15 @@ export const sessionsHandlers: GatewayRequestHandlers = {
|
||||
if (restoreBlockedByModelLock) {
|
||||
return;
|
||||
}
|
||||
restorePlacementError = resolveSessionWorkerPlacementMutationError({
|
||||
action: "restore",
|
||||
context,
|
||||
key,
|
||||
sessionId: current.entry?.sessionId,
|
||||
});
|
||||
if (restorePlacementError) {
|
||||
return;
|
||||
}
|
||||
clearSessionQueues([
|
||||
key,
|
||||
current.canonicalKey,
|
||||
@@ -1993,6 +2122,10 @@ export const sessionsHandlers: GatewayRequestHandlers = {
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (restorePlacementError) {
|
||||
respondSessionWorkerPlacementMutationError(restorePlacementError, respond);
|
||||
return;
|
||||
}
|
||||
if (!admittedWorkReleased) {
|
||||
respond(
|
||||
false,
|
||||
@@ -2113,6 +2246,145 @@ export const sessionsHandlers: GatewayRequestHandlers = {
|
||||
},
|
||||
});
|
||||
},
|
||||
"sessions.dispatch": async ({ params, respond, context, client, isWebchatConnect }) => {
|
||||
if (!assertValidParams(params, validateSessionsDispatchParams, "sessions.dispatch", respond)) {
|
||||
return;
|
||||
}
|
||||
const key = requireSessionKey(params.key, respond);
|
||||
if (!key) {
|
||||
return;
|
||||
}
|
||||
if (rejectWebchatSessionMutation({ action: "dispatch", client, isWebchatConnect, respond })) {
|
||||
return;
|
||||
}
|
||||
const dispatchService = context.workerPlacementDispatchService;
|
||||
const placementReader = context.workerSessionPlacementService;
|
||||
if (!dispatchService || !placementReader) {
|
||||
respond(
|
||||
false,
|
||||
undefined,
|
||||
errorShape(ErrorCodes.INVALID_REQUEST, "cloud worker dispatch is not configured"),
|
||||
);
|
||||
return;
|
||||
}
|
||||
const cfg = context.getRuntimeConfig();
|
||||
const requestedAgent = resolveRequestedGlobalAgentId(cfg, key, params.agentId);
|
||||
if (!requestedAgent.ok) {
|
||||
respond(false, undefined, requestedAgent.error);
|
||||
return;
|
||||
}
|
||||
if (!Object.hasOwn(cfg.cloudWorkers?.profiles ?? {}, params.profileId)) {
|
||||
respond(
|
||||
false,
|
||||
undefined,
|
||||
errorShape(
|
||||
ErrorCodes.INVALID_REQUEST,
|
||||
`cloud worker profile is not configured: ${params.profileId}`,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
const target = loadAccessorSessionEntryForGatewayTarget({
|
||||
key,
|
||||
cfg,
|
||||
agentId: requestedAgent.agentId,
|
||||
});
|
||||
const entry = target.entry;
|
||||
const sessionId = normalizeOptionalString(entry?.sessionId);
|
||||
if (!entry || !sessionId) {
|
||||
respond(
|
||||
false,
|
||||
undefined,
|
||||
errorShape(ErrorCodes.INVALID_REQUEST, `session not found: ${key}`),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (entry.archivedAt !== undefined) {
|
||||
respond(
|
||||
false,
|
||||
undefined,
|
||||
errorShape(ErrorCodes.INVALID_REQUEST, "cannot dispatch an archived session"),
|
||||
);
|
||||
return;
|
||||
}
|
||||
const sessionRuntime = resolveWorkerPlacementSessionRuntime({
|
||||
cfg,
|
||||
entry,
|
||||
agentId: target.target.agentId,
|
||||
sessionKey: target.canonicalKey,
|
||||
});
|
||||
if (!isWorkerPlacementSessionRuntimeSupported(sessionRuntime)) {
|
||||
respond(
|
||||
false,
|
||||
undefined,
|
||||
errorShape(
|
||||
ErrorCodes.INVALID_REQUEST,
|
||||
`cloud worker dispatch requires the OpenClaw runtime, not ${sessionRuntime}`,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
const existingPlacement = placementReader.getMany([sessionId]).get(sessionId);
|
||||
if (
|
||||
existingPlacement &&
|
||||
existingPlacement.state !== "local" &&
|
||||
existingPlacement.state !== "reclaimed"
|
||||
) {
|
||||
respond(
|
||||
false,
|
||||
undefined,
|
||||
errorShape(
|
||||
ErrorCodes.INVALID_REQUEST,
|
||||
`session cannot dispatch from placement ${existingPlacement.state}`,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
const worktree = managedWorktrees.findLiveByOwner("session", target.canonicalKey);
|
||||
if (
|
||||
!target.entry?.worktree?.id ||
|
||||
!worktree ||
|
||||
worktree.id !== target.entry.worktree.id ||
|
||||
worktree.ownerId !== target.canonicalKey
|
||||
) {
|
||||
respond(
|
||||
false,
|
||||
undefined,
|
||||
errorShape(
|
||||
ErrorCodes.INVALID_REQUEST,
|
||||
"sessions.dispatch requires a session-owned managed worktree",
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const placement = await dispatchService.dispatch({
|
||||
sessionId,
|
||||
sessionKey: target.canonicalKey,
|
||||
agentId: target.target.agentId,
|
||||
profileId: params.profileId,
|
||||
});
|
||||
respond(
|
||||
true,
|
||||
{
|
||||
ok: true,
|
||||
key: target.canonicalKey,
|
||||
sessionId,
|
||||
placement: projectWorkerSessionPlacement(placement),
|
||||
},
|
||||
undefined,
|
||||
);
|
||||
} catch (error) {
|
||||
respond(
|
||||
false,
|
||||
undefined,
|
||||
errorShape(
|
||||
isWorkerDispatchInputError(error) ? ErrorCodes.INVALID_REQUEST : ErrorCodes.UNAVAILABLE,
|
||||
formatErrorMessage(error),
|
||||
),
|
||||
);
|
||||
}
|
||||
},
|
||||
"sessions.send": async ({ req, params, respond, context, client, isWebchatConnect }) => {
|
||||
await handleSessionSend({
|
||||
method: "sessions.send",
|
||||
@@ -2354,6 +2626,20 @@ export const sessionsHandlers: GatewayRequestHandlers = {
|
||||
respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, missingHarnessSessionError));
|
||||
return;
|
||||
}
|
||||
const initialPlacementPatchError = resolveSessionWorkerPlacementPatchError({
|
||||
agentId: target.agentId,
|
||||
cfg,
|
||||
context,
|
||||
entry: lifecycleEntry,
|
||||
key,
|
||||
patch: p,
|
||||
sessionKey: canonicalKey,
|
||||
validateModelRuntime: false,
|
||||
});
|
||||
if (initialPlacementPatchError) {
|
||||
respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, initialPlacementPatchError));
|
||||
return;
|
||||
}
|
||||
const lifecycleIdentities = [canonicalKey, key, lifecycleEntry?.sessionId];
|
||||
if (p.archived === true && isSessionLifecycleMutationActive(storePath, lifecycleIdentities)) {
|
||||
respond(
|
||||
@@ -2436,8 +2722,8 @@ export const sessionsHandlers: GatewayRequestHandlers = {
|
||||
});
|
||||
return { primaryKey, candidateKeys: migratedTarget.storeKeys };
|
||||
},
|
||||
project: async ({ primaryKey, existingEntry, entries }) =>
|
||||
await projectSessionsPatchEntry({
|
||||
project: async ({ primaryKey, existingEntry, entries }) => {
|
||||
const projected = await projectSessionsPatchEntry({
|
||||
cfg,
|
||||
entries,
|
||||
existingEntry,
|
||||
@@ -2445,7 +2731,27 @@ export const sessionsHandlers: GatewayRequestHandlers = {
|
||||
agentId: requestedAgentId,
|
||||
patch: p,
|
||||
loadGatewayModelCatalog: loadPatchModelCatalog,
|
||||
}),
|
||||
});
|
||||
if (!projected.ok) {
|
||||
return projected;
|
||||
}
|
||||
const placementPatchError = resolveSessionWorkerPlacementPatchError({
|
||||
agentId: target.agentId,
|
||||
cfg,
|
||||
context,
|
||||
entry: projected.entry,
|
||||
key,
|
||||
patch: p,
|
||||
sessionKey: canonicalKey,
|
||||
validateModelRuntime: true,
|
||||
});
|
||||
return placementPatchError
|
||||
? {
|
||||
ok: false,
|
||||
error: errorShape(ErrorCodes.INVALID_REQUEST, placementPatchError),
|
||||
}
|
||||
: projected;
|
||||
},
|
||||
});
|
||||
};
|
||||
const applied = await runExclusiveSessionLifecycleMutation({
|
||||
@@ -2774,6 +3080,16 @@ export const sessionsHandlers: GatewayRequestHandlers = {
|
||||
if (rejectExpectedSessionMismatch(initialDeleteEntry)) {
|
||||
return;
|
||||
}
|
||||
const initialPlacementError = resolveSessionWorkerPlacementMutationError({
|
||||
action: "delete",
|
||||
context,
|
||||
key,
|
||||
sessionId: normalizeOptionalString(initialDeleteEntry?.sessionId),
|
||||
});
|
||||
if (initialPlacementError) {
|
||||
respondSessionWorkerPlacementMutationError(initialPlacementError, respond);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
rejectPluginRuntimeDeleteMismatch({
|
||||
client,
|
||||
@@ -2821,6 +3137,7 @@ export const sessionsHandlers: GatewayRequestHandlers = {
|
||||
let admittedWorkReleased = true;
|
||||
let expectedSessionStillCurrent = true;
|
||||
let deleteBlockedByModelLock = false;
|
||||
let deleteBlockedByWorkerPlacement = false;
|
||||
const deletion = await runExclusiveSessionLifecycleMutation({
|
||||
scope: storePath,
|
||||
identities: deleteLifecycleIdentities,
|
||||
@@ -2834,6 +3151,17 @@ export const sessionsHandlers: GatewayRequestHandlers = {
|
||||
if (!expectedSessionStillCurrent) {
|
||||
return;
|
||||
}
|
||||
const placementError = resolveSessionWorkerPlacementMutationError({
|
||||
action: "delete",
|
||||
context,
|
||||
key,
|
||||
sessionId: normalizeOptionalString(preparedEntry?.sessionId),
|
||||
});
|
||||
if (placementError) {
|
||||
deleteBlockedByWorkerPlacement = true;
|
||||
respondSessionWorkerPlacementMutationError(placementError, respond);
|
||||
return;
|
||||
}
|
||||
admittedWorkReleased = await interruptSessionWorkAdmissions({
|
||||
scope: storePath,
|
||||
identities: deleteLifecycleIdentities,
|
||||
@@ -2841,7 +3169,11 @@ export const sessionsHandlers: GatewayRequestHandlers = {
|
||||
});
|
||||
},
|
||||
run: async () => {
|
||||
if (deleteBlockedByModelLock || !expectedSessionStillCurrent) {
|
||||
if (
|
||||
deleteBlockedByModelLock ||
|
||||
deleteBlockedByWorkerPlacement ||
|
||||
!expectedSessionStillCurrent
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
if (!admittedWorkReleased) {
|
||||
|
||||
@@ -40,7 +40,11 @@ import type { DedupeEntry } from "../server-shared.js";
|
||||
import type { GatewayEventLoopHealth } from "../server/event-loop-health.js";
|
||||
import type { TerminalLaunchResolution } from "../terminal/launch.js";
|
||||
import type { TerminalSessionManager } from "../terminal/session-manager.js";
|
||||
import type { WorkerEnvironmentServiceContract } from "../worker-environments/service-contract.js";
|
||||
import type { WorkerSessionPlacementReader } from "../worker-environments/placement-projector.js";
|
||||
import type {
|
||||
WorkerEnvironmentServiceContract,
|
||||
WorkerPlacementDispatchContract,
|
||||
} from "../worker-environments/service-contract.js";
|
||||
|
||||
/**
|
||||
* Shared gateway request types used by every server-method module.
|
||||
@@ -142,6 +146,10 @@ export type GatewayRequestContext = {
|
||||
nodeRegistry: NodeRegistry;
|
||||
/** Durable cloud-worker lifecycle; absent from lightweight in-process contexts. */
|
||||
workerEnvironmentService?: WorkerEnvironmentServiceContract;
|
||||
/** Durable per-session worker placement; absent when cloud workers are disabled. */
|
||||
workerSessionPlacementService?: WorkerSessionPlacementReader;
|
||||
/** One-way local-to-worker dispatch; absent when cloud workers are disabled. */
|
||||
workerPlacementDispatchService?: WorkerPlacementDispatchContract;
|
||||
// Operator terminal session store. Absent in local/in-process contexts where
|
||||
// no PTY surface is served.
|
||||
terminalSessions?: TerminalSessionManager;
|
||||
|
||||
@@ -48,6 +48,8 @@ type GatewayRequestContextParams = {
|
||||
enforceSharedGatewayAuthGenerationForConfigWrite: (nextConfig: OpenClawConfig) => void;
|
||||
nodeRegistry: GatewayRequestContext["nodeRegistry"];
|
||||
workerEnvironmentService?: GatewayRequestContext["workerEnvironmentService"];
|
||||
workerSessionPlacementService?: GatewayRequestContext["workerSessionPlacementService"];
|
||||
workerPlacementDispatchService?: GatewayRequestContext["workerPlacementDispatchService"];
|
||||
terminalSessions?: GatewayRequestContext["terminalSessions"];
|
||||
agentRunSeq: GatewayRequestContext["agentRunSeq"];
|
||||
chatAbortControllers: GatewayRequestContext["chatAbortControllers"];
|
||||
@@ -211,6 +213,12 @@ export function createGatewayRequestContext(
|
||||
...(params.workerEnvironmentService
|
||||
? { workerEnvironmentService: params.workerEnvironmentService }
|
||||
: {}),
|
||||
...(params.workerSessionPlacementService
|
||||
? { workerSessionPlacementService: params.workerSessionPlacementService }
|
||||
: {}),
|
||||
...(params.workerPlacementDispatchService
|
||||
? { workerPlacementDispatchService: params.workerPlacementDispatchService }
|
||||
: {}),
|
||||
terminalSessions: params.terminalSessions,
|
||||
agentRunSeq: params.agentRunSeq,
|
||||
chatAbortControllers: params.chatAbortControllers,
|
||||
|
||||
@@ -2173,16 +2173,26 @@ describe("startGatewayPostAttachRuntime", () => {
|
||||
expect(startGatewaySidecarsValue).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("starts the worker environment sidecar before releasing startup-gated methods", async () => {
|
||||
it("reconciles worker placement before starting channels and sidecars", async () => {
|
||||
let finishReconcile: (() => void) | undefined;
|
||||
const reconcileReady = new Promise<void>((resolve) => {
|
||||
finishReconcile = resolve;
|
||||
});
|
||||
const startupOrder: string[] = [];
|
||||
const workerSidecar = { stop: vi.fn() };
|
||||
const startWorkerEnvironmentRuntime = vi.fn(async () => {
|
||||
startupOrder.push("worker-reconcile");
|
||||
await reconcileReady;
|
||||
startupOrder.push("worker-ready");
|
||||
return workerSidecar;
|
||||
});
|
||||
const startGatewaySidecarsValue = vi.fn(async () => {
|
||||
startupOrder.push("gateway-sidecars");
|
||||
return {
|
||||
pluginServices: null,
|
||||
postReadySidecars: [],
|
||||
};
|
||||
});
|
||||
const onGatewayLifetimeSidecars = vi.fn();
|
||||
const unavailableGatewayMethods = new Set<string>(STARTUP_UNAVAILABLE_GATEWAY_METHODS);
|
||||
|
||||
@@ -2195,25 +2205,47 @@ describe("startGatewayPostAttachRuntime", () => {
|
||||
onGatewayLifetimeSidecars,
|
||||
},
|
||||
createPostAttachRuntimeDeps({
|
||||
startGatewaySidecars: vi.fn(async () => ({
|
||||
pluginServices: null,
|
||||
postReadySidecars: [],
|
||||
})),
|
||||
startGatewaySidecars: startGatewaySidecarsValue,
|
||||
}),
|
||||
);
|
||||
|
||||
await vi.waitFor(() => {
|
||||
expect(startWorkerEnvironmentRuntime).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
expect(startGatewaySidecarsValue).not.toHaveBeenCalled();
|
||||
expect(startupOrder).toEqual(["worker-reconcile"]);
|
||||
expect([...unavailableGatewayMethods]).toEqual([...STARTUP_UNAVAILABLE_GATEWAY_METHODS]);
|
||||
|
||||
finishReconcile?.();
|
||||
await vi.waitFor(() => {
|
||||
expect([...unavailableGatewayMethods]).toEqual([]);
|
||||
expect(startGatewaySidecarsValue).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
expect(startupOrder).toEqual(["worker-reconcile", "worker-ready", "gateway-sidecars"]);
|
||||
expect([...unavailableGatewayMethods]).toEqual([]);
|
||||
expect(onGatewayLifetimeSidecars).toHaveBeenCalledWith(expect.arrayContaining([workerSidecar]));
|
||||
});
|
||||
|
||||
it("stops worker placement runtime when channel and sidecar startup fails", async () => {
|
||||
const workerSidecar = { stop: vi.fn(async () => {}) };
|
||||
const startupError = new Error("sidecar startup failed");
|
||||
|
||||
await expect(
|
||||
startGatewayPostAttachRuntime(
|
||||
{
|
||||
...createPostAttachParams(),
|
||||
startWorkerEnvironmentRuntime: vi.fn(() => workerSidecar),
|
||||
},
|
||||
createPostAttachRuntimeDeps({
|
||||
startGatewaySidecars: vi.fn(async () => {
|
||||
throw startupError;
|
||||
}),
|
||||
}),
|
||||
),
|
||||
).rejects.toBe(startupError);
|
||||
|
||||
expect(workerSidecar.stop).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not start the worker environment sidecar after close begins", async () => {
|
||||
const startWorkerEnvironmentRuntime = vi.fn(() => ({ stop: vi.fn() }));
|
||||
const startGatewaySidecarsValue = vi.fn(async () => ({
|
||||
|
||||
@@ -1273,28 +1273,35 @@ export async function startGatewayPostAttachRuntime(
|
||||
? Promise.resolve({ pluginServices: null, pluginRegistry, postReadySidecars: [] })
|
||||
: waitForSidecarStartTurn().then(async () => {
|
||||
await loadStartupPluginsIfNeeded();
|
||||
params.log.info("starting channels and sidecars...");
|
||||
const loaderStatsBefore = getPluginModuleLoaderStats();
|
||||
const result = await measureStartup(params.startupTrace, "sidecars.total", () =>
|
||||
runtimeDeps.startGatewaySidecars({
|
||||
cfg: params.gatewayPluginConfigAtStart,
|
||||
pluginRegistry,
|
||||
defaultWorkspaceDir: params.defaultWorkspaceDir,
|
||||
deps: params.deps,
|
||||
startChannels: params.startChannels,
|
||||
log: params.log,
|
||||
logHooks: params.logHooks,
|
||||
logChannels: params.logChannels,
|
||||
startupTrace: params.startupTrace,
|
||||
onChannelsStarted: params.onChannelsStarted,
|
||||
onPluginServices: reportPluginServices,
|
||||
shouldStartPluginServices: () => params.isClosing?.() !== true,
|
||||
startupOutcomes,
|
||||
}),
|
||||
);
|
||||
const workerEnvironmentSidecar = params.isClosing?.()
|
||||
? null
|
||||
: ((await params.startWorkerEnvironmentRuntime?.()) ?? null);
|
||||
params.log.info("starting channels and sidecars...");
|
||||
const loaderStatsBefore = getPluginModuleLoaderStats();
|
||||
const result = await (async () => {
|
||||
try {
|
||||
return await measureStartup(params.startupTrace, "sidecars.total", () =>
|
||||
runtimeDeps.startGatewaySidecars({
|
||||
cfg: params.gatewayPluginConfigAtStart,
|
||||
pluginRegistry,
|
||||
defaultWorkspaceDir: params.defaultWorkspaceDir,
|
||||
deps: params.deps,
|
||||
startChannels: params.startChannels,
|
||||
log: params.log,
|
||||
logHooks: params.logHooks,
|
||||
logChannels: params.logChannels,
|
||||
startupTrace: params.startupTrace,
|
||||
onChannelsStarted: params.onChannelsStarted,
|
||||
onPluginServices: reportPluginServices,
|
||||
shouldStartPluginServices: () => params.isClosing?.() !== true,
|
||||
startupOutcomes,
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
await workerEnvironmentSidecar?.stop();
|
||||
throw error;
|
||||
}
|
||||
})();
|
||||
const loaderStatsAfter = getPluginModuleLoaderStats();
|
||||
params.startupTrace?.detail("sidecars.plugin-loader", [
|
||||
["callsCount", loaderStatsAfter.calls - loaderStatsBefore.calls],
|
||||
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
import { createLazyRuntimeModule } from "../shared/lazy-runtime.js";
|
||||
import type { WorkerBundleProducer, WorkerNpmArtifact } from "./worker-environments/bundle.js";
|
||||
import type { WorkerLiveEventReceiver } from "./worker-environments/live-events.js";
|
||||
import type { WorkerSessionPlacementStore } from "./worker-environments/placement-store.js";
|
||||
import type { WorkerEnvironmentService } from "./worker-environments/service.js";
|
||||
|
||||
type WorkerEnvironmentStore = ReturnType<
|
||||
@@ -24,6 +25,8 @@ export type GatewayWorkerEnvironmentStartupState = {
|
||||
listDurableProviderIds: () => string[];
|
||||
records: WorkerEnvironmentRecord[];
|
||||
store: WorkerEnvironmentStore;
|
||||
placementStore: WorkerSessionPlacementStore;
|
||||
hasNonlocalPlacementRecords: boolean;
|
||||
};
|
||||
|
||||
export type GatewayWorkerEnvironmentRuntime = {
|
||||
@@ -39,8 +42,13 @@ const loadWorkerInferenceRuntimeModule = createLazyRuntimeModule(
|
||||
);
|
||||
|
||||
export async function loadGatewayWorkerEnvironmentStartupState(): Promise<GatewayWorkerEnvironmentStartupState> {
|
||||
const { createWorkerEnvironmentStore } = await import("./worker-environments/store.js");
|
||||
const [{ createWorkerEnvironmentStore }, { createWorkerSessionPlacementStore }] =
|
||||
await Promise.all([
|
||||
import("./worker-environments/store.js"),
|
||||
import("./worker-environments/placement-store.js"),
|
||||
]);
|
||||
const store = createWorkerEnvironmentStore();
|
||||
const placementStore = createWorkerSessionPlacementStore();
|
||||
const records = store.list();
|
||||
const durableProviderIds = uniqueStrings(
|
||||
records.flatMap((record) =>
|
||||
@@ -51,7 +59,15 @@ export async function loadGatewayWorkerEnvironmentStartupState(): Promise<Gatewa
|
||||
);
|
||||
const listDurableProviderIds = () =>
|
||||
uniqueStrings(store.listForReconcile().map((record) => record.providerId));
|
||||
return { durableProviderIds, listDurableProviderIds, records, store };
|
||||
return {
|
||||
durableProviderIds,
|
||||
listDurableProviderIds,
|
||||
records,
|
||||
store,
|
||||
placementStore,
|
||||
// Non-local placements must revive the worker service even without configured profiles.
|
||||
hasNonlocalPlacementRecords: placementStore.listForReconcile().length > 0,
|
||||
};
|
||||
}
|
||||
|
||||
export async function createGatewayWorkerEnvironmentRuntime(params: {
|
||||
@@ -63,16 +79,21 @@ export async function createGatewayWorkerEnvironmentRuntime(params: {
|
||||
const [
|
||||
{ createWorkerEnvironmentService },
|
||||
{ createWorkerLiveEventReceiver },
|
||||
{ createWorkerSessionPlacementGate },
|
||||
{ createWorkerTranscriptCommitter },
|
||||
{ createWorkerTunnelManager },
|
||||
{ resolveWorkerProvider },
|
||||
] = await Promise.all([
|
||||
import("./worker-environments/service.js"),
|
||||
import("./worker-environments/live-events.js"),
|
||||
import("./worker-environments/placement-worker-gate.js"),
|
||||
import("./worker-environments/transcript-commit.js"),
|
||||
import("./worker-environments/tunnel.js"),
|
||||
import("../plugins/worker-provider-registry.js"),
|
||||
]);
|
||||
// A crashed gateway can leak local turn claims; drop them before workers re-admit turns.
|
||||
params.startup.placementStore.clearLocalTurnClaimsAfterRestart();
|
||||
const placementGate = createWorkerSessionPlacementGate(params.startup.placementStore);
|
||||
let workerBundleProducer: WorkerBundleProducer | undefined;
|
||||
let workerNpmArtifact: Promise<WorkerNpmArtifact> | undefined;
|
||||
const prepareInstallation = async (install: "bundle" | "npm") => {
|
||||
@@ -128,6 +149,7 @@ export async function createGatewayWorkerEnvironmentRuntime(params: {
|
||||
const workerInferenceRuntime = await loadWorkerInferenceRuntimeModule();
|
||||
return await workerInferenceRuntime.executeWorkerInference(inferenceParams);
|
||||
},
|
||||
placementStore: placementGate,
|
||||
liveEvents: workerLiveEvents,
|
||||
resolveSshIdentity: async ({ provider, leaseId, profile, keyRef }) => {
|
||||
const workerRuntime = await loadWorkerEnvironmentRuntimeModule();
|
||||
|
||||
@@ -0,0 +1,429 @@
|
||||
import {
|
||||
installSessionPlacementAdmissionProvider,
|
||||
installSessionPlacementResetGuard,
|
||||
} from "../agents/session-placement-admission.js";
|
||||
import { clearSessionQueues } from "../auto-reply/reply/queue/cleanup.js";
|
||||
import { getRuntimeConfig } from "../config/config.js";
|
||||
import { runExclusiveSessionStoreWrite } from "../config/sessions/store-writer.js";
|
||||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
import {
|
||||
interruptSessionWorkAdmissions,
|
||||
runExclusiveSessionLifecycleMutation,
|
||||
SESSION_WORK_ADMISSION_DRAIN_TIMEOUT_MS,
|
||||
} from "../sessions/session-lifecycle-admission.js";
|
||||
import { createLazyRuntimeModule } from "../shared/lazy-runtime.js";
|
||||
import {
|
||||
createWorkerPlacementDispatchService,
|
||||
type WorkerPlacementDispatchService,
|
||||
} from "./worker-environments/placement-dispatch.js";
|
||||
import type { WorkerSessionPlacementStore } from "./worker-environments/placement-store.js";
|
||||
import type { WorkerEnvironmentService } from "./worker-environments/service.js";
|
||||
import { createWorkerSessionTurnPlacementProvider } from "./worker-environments/worker-turn-launcher.js";
|
||||
|
||||
const WORKER_PLACEMENT_RECONCILE_INTERVAL_MS = 60_000;
|
||||
|
||||
const loadWorkerPlacementSessionRuntimeModule = createLazyRuntimeModule(async () => {
|
||||
const [placementSessionRuntime, { managedWorktrees }, sessionUtils] = await Promise.all([
|
||||
import("./worker-environments/placement-session-runtime.js"),
|
||||
import("../agents/worktrees/service.js"),
|
||||
import("./session-utils.js"),
|
||||
]);
|
||||
return {
|
||||
isWorkerPlacementSessionRuntimeSupported:
|
||||
placementSessionRuntime.isWorkerPlacementSessionRuntimeSupported,
|
||||
managedWorktrees,
|
||||
resolveWorkerPlacementSessionRuntime:
|
||||
placementSessionRuntime.resolveWorkerPlacementSessionRuntime,
|
||||
resolveFreshestSessionEntryFromStoreKeys: sessionUtils.resolveFreshestSessionEntryFromStoreKeys,
|
||||
resolveGatewaySessionStoreTargetWithStore:
|
||||
sessionUtils.resolveGatewaySessionStoreTargetWithStore,
|
||||
};
|
||||
});
|
||||
|
||||
class WorkerDispatchTargetChangedError extends Error {
|
||||
readonly code = "invalid_state";
|
||||
}
|
||||
|
||||
/** Serializes reconciliation sweeps against in-flight dispatches so a sweep never
|
||||
* observes a placement mid-transition. Dispatches wait out any pending sweep. */
|
||||
function coordinateWorkerPlacementDispatch(
|
||||
service: WorkerPlacementDispatchService,
|
||||
): WorkerPlacementDispatchService {
|
||||
let activeDispatchCount = 0;
|
||||
let reconciliation: Promise<void> | undefined;
|
||||
const dispatchIdleWaiters = new Set<() => void>();
|
||||
const waitForDispatchIdle = (): Promise<void> => {
|
||||
if (activeDispatchCount === 0) {
|
||||
return Promise.resolve();
|
||||
}
|
||||
return new Promise<void>((resolve) => {
|
||||
dispatchIdleWaiters.add(resolve);
|
||||
});
|
||||
};
|
||||
const runReconciliation = (operation: () => Promise<void>): Promise<void> => {
|
||||
if (reconciliation) {
|
||||
return reconciliation;
|
||||
}
|
||||
const current = (async () => {
|
||||
await waitForDispatchIdle();
|
||||
await operation();
|
||||
})();
|
||||
reconciliation = current;
|
||||
const clearCurrent = () => {
|
||||
if (reconciliation === current) {
|
||||
reconciliation = undefined;
|
||||
}
|
||||
};
|
||||
void current.then(clearCurrent, clearCurrent);
|
||||
return current;
|
||||
};
|
||||
return {
|
||||
dispatch: async (request) => {
|
||||
for (;;) {
|
||||
const pendingReconciliation = reconciliation;
|
||||
if (!pendingReconciliation) {
|
||||
break;
|
||||
}
|
||||
await pendingReconciliation.catch(() => undefined);
|
||||
}
|
||||
activeDispatchCount += 1;
|
||||
try {
|
||||
return await service.dispatch(request);
|
||||
} finally {
|
||||
activeDispatchCount -= 1;
|
||||
if (activeDispatchCount === 0) {
|
||||
const waiters = [...dispatchIdleWaiters];
|
||||
dispatchIdleWaiters.clear();
|
||||
for (const resolve of waiters) {
|
||||
resolve();
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
reconcile: () => runReconciliation(service.reconcile),
|
||||
reconcileActive: () => runReconciliation(service.reconcileActive),
|
||||
};
|
||||
}
|
||||
|
||||
type WorkerPlacementSidecar = { stop: () => Promise<void> };
|
||||
|
||||
export type GatewayWorkerPlacementRuntimeParams = {
|
||||
placements: WorkerSessionPlacementStore;
|
||||
environments: WorkerEnvironmentService;
|
||||
admitNewPlacements: boolean;
|
||||
revokeSessionAuthority: (request: { sessionId: string; sessionKeys: readonly string[] }) => void;
|
||||
warn: (message: string) => void;
|
||||
};
|
||||
|
||||
export type GatewayWorkerPlacementRuntime = ReturnType<typeof createGatewayWorkerPlacementRuntime>;
|
||||
|
||||
export function createGatewayWorkerPlacementRuntime(params: GatewayWorkerPlacementRuntimeParams) {
|
||||
const dispatchService = coordinateWorkerPlacementDispatch(
|
||||
createWorkerPlacementDispatchService({
|
||||
placements: params.placements,
|
||||
environments: params.environments,
|
||||
runLocalBarrier: async ({ sessionId, sessionKey, agentId, startDispatch }) => {
|
||||
const {
|
||||
isWorkerPlacementSessionRuntimeSupported,
|
||||
managedWorktrees,
|
||||
resolveFreshestSessionEntryFromStoreKeys,
|
||||
resolveGatewaySessionStoreTargetWithStore,
|
||||
resolveWorkerPlacementSessionRuntime,
|
||||
} = await loadWorkerPlacementSessionRuntimeModule();
|
||||
const target = resolveGatewaySessionStoreTargetWithStore({
|
||||
cfg: getRuntimeConfig(),
|
||||
key: sessionKey,
|
||||
agentId,
|
||||
clone: false,
|
||||
});
|
||||
const lifecycleIdentities = [
|
||||
sessionKey,
|
||||
target.canonicalKey,
|
||||
...target.storeKeys,
|
||||
sessionId,
|
||||
];
|
||||
let placement: ReturnType<typeof startDispatch> | undefined;
|
||||
await runExclusiveSessionLifecycleMutation({
|
||||
scope: target.storePath,
|
||||
identities: lifecycleIdentities,
|
||||
prepare: async () => {
|
||||
const currentConfig = getRuntimeConfig();
|
||||
const currentTarget = resolveGatewaySessionStoreTargetWithStore({
|
||||
cfg: currentConfig,
|
||||
key: sessionKey,
|
||||
agentId,
|
||||
clone: false,
|
||||
});
|
||||
const currentEntry = resolveFreshestSessionEntryFromStoreKeys(
|
||||
currentTarget.store,
|
||||
currentTarget.storeKeys,
|
||||
);
|
||||
const worktree = managedWorktrees.findLiveByOwner(
|
||||
"session",
|
||||
currentTarget.canonicalKey,
|
||||
);
|
||||
if (
|
||||
currentTarget.storePath !== target.storePath ||
|
||||
currentTarget.canonicalKey !== target.canonicalKey ||
|
||||
currentTarget.agentId !== target.agentId ||
|
||||
currentEntry?.sessionId !== sessionId ||
|
||||
!currentEntry.worktree?.id ||
|
||||
!worktree ||
|
||||
worktree.id !== currentEntry.worktree.id ||
|
||||
worktree.ownerId !== currentTarget.canonicalKey
|
||||
) {
|
||||
throw new WorkerDispatchTargetChangedError(
|
||||
`Session ${sessionKey} changed before cloud worker dispatch. Retry.`,
|
||||
);
|
||||
}
|
||||
if (currentEntry.archivedAt !== undefined) {
|
||||
throw new WorkerDispatchTargetChangedError(
|
||||
`Session ${sessionKey} was archived before cloud worker dispatch. Retry.`,
|
||||
);
|
||||
}
|
||||
const currentRuntime = resolveWorkerPlacementSessionRuntime({
|
||||
cfg: currentConfig,
|
||||
entry: currentEntry,
|
||||
agentId: currentTarget.agentId,
|
||||
sessionKey: currentTarget.canonicalKey,
|
||||
});
|
||||
if (!isWorkerPlacementSessionRuntimeSupported(currentRuntime)) {
|
||||
throw new WorkerDispatchTargetChangedError(
|
||||
`Session ${sessionKey} runtime changed to ${currentRuntime} before cloud worker dispatch. Retry.`,
|
||||
);
|
||||
}
|
||||
placement = startDispatch();
|
||||
clearSessionQueues(lifecycleIdentities);
|
||||
params.revokeSessionAuthority({
|
||||
sessionId,
|
||||
sessionKeys: lifecycleIdentities,
|
||||
});
|
||||
const released = await interruptSessionWorkAdmissions({
|
||||
scope: target.storePath,
|
||||
identities: lifecycleIdentities,
|
||||
timeoutMs: SESSION_WORK_ADMISSION_DRAIN_TIMEOUT_MS,
|
||||
});
|
||||
if (!released) {
|
||||
throw new Error(`Session ${sessionKey} is still active; dispatch stopped`);
|
||||
}
|
||||
await params.placements.waitForTurnClaimRelease(sessionId, {
|
||||
timeoutMs: SESSION_WORK_ADMISSION_DRAIN_TIMEOUT_MS,
|
||||
});
|
||||
await runExclusiveSessionStoreWrite(target.storePath, async () => {}, {
|
||||
reentrant: true,
|
||||
});
|
||||
},
|
||||
run: async () => {
|
||||
if (!placement) {
|
||||
throw new Error(`Session ${sessionKey} dispatch barrier did not start`);
|
||||
}
|
||||
},
|
||||
});
|
||||
if (!placement) {
|
||||
throw new Error(`Session ${sessionKey} dispatch barrier did not complete`);
|
||||
}
|
||||
return placement;
|
||||
},
|
||||
runActivationBarrier: async ({ sessionId, sessionKey, agentId, activate }) => {
|
||||
const {
|
||||
isWorkerPlacementSessionRuntimeSupported,
|
||||
managedWorktrees,
|
||||
resolveFreshestSessionEntryFromStoreKeys,
|
||||
resolveGatewaySessionStoreTargetWithStore,
|
||||
resolveWorkerPlacementSessionRuntime,
|
||||
} = await loadWorkerPlacementSessionRuntimeModule();
|
||||
const target = resolveGatewaySessionStoreTargetWithStore({
|
||||
cfg: getRuntimeConfig(),
|
||||
key: sessionKey,
|
||||
agentId,
|
||||
clone: false,
|
||||
});
|
||||
const lifecycleIdentities = [
|
||||
sessionKey,
|
||||
target.canonicalKey,
|
||||
...target.storeKeys,
|
||||
sessionId,
|
||||
];
|
||||
let activePlacement: ReturnType<typeof activate> | undefined;
|
||||
await runExclusiveSessionLifecycleMutation({
|
||||
scope: target.storePath,
|
||||
identities: lifecycleIdentities,
|
||||
run: async () => {
|
||||
const currentConfig = getRuntimeConfig();
|
||||
const currentTarget = resolveGatewaySessionStoreTargetWithStore({
|
||||
cfg: currentConfig,
|
||||
key: sessionKey,
|
||||
agentId,
|
||||
clone: false,
|
||||
});
|
||||
const currentEntry = resolveFreshestSessionEntryFromStoreKeys(
|
||||
currentTarget.store,
|
||||
currentTarget.storeKeys,
|
||||
);
|
||||
const worktree = managedWorktrees.findLiveByOwner(
|
||||
"session",
|
||||
currentTarget.canonicalKey,
|
||||
);
|
||||
if (
|
||||
currentTarget.storePath !== target.storePath ||
|
||||
currentTarget.canonicalKey !== target.canonicalKey ||
|
||||
currentTarget.agentId !== target.agentId ||
|
||||
currentEntry?.sessionId !== sessionId ||
|
||||
!currentEntry.worktree?.id ||
|
||||
!worktree ||
|
||||
worktree.id !== currentEntry.worktree.id ||
|
||||
worktree.ownerId !== currentTarget.canonicalKey
|
||||
) {
|
||||
throw new WorkerDispatchTargetChangedError(
|
||||
`Session ${sessionKey} changed before cloud worker activation. Retry.`,
|
||||
);
|
||||
}
|
||||
if (currentEntry.archivedAt !== undefined) {
|
||||
throw new WorkerDispatchTargetChangedError(
|
||||
`Session ${sessionKey} was archived before cloud worker activation. Retry.`,
|
||||
);
|
||||
}
|
||||
const currentRuntime = resolveWorkerPlacementSessionRuntime({
|
||||
cfg: currentConfig,
|
||||
entry: currentEntry,
|
||||
agentId: currentTarget.agentId,
|
||||
sessionKey: currentTarget.canonicalKey,
|
||||
});
|
||||
if (!isWorkerPlacementSessionRuntimeSupported(currentRuntime)) {
|
||||
throw new WorkerDispatchTargetChangedError(
|
||||
`Session ${sessionKey} runtime changed to ${currentRuntime} before cloud worker activation. Retry.`,
|
||||
);
|
||||
}
|
||||
activePlacement = activate();
|
||||
},
|
||||
});
|
||||
if (!activePlacement) {
|
||||
throw new Error(`Session ${sessionKey} activation barrier did not complete`);
|
||||
}
|
||||
return activePlacement;
|
||||
},
|
||||
resolveWorkspacePath: async ({ sessionId, sessionKey, agentId }) => {
|
||||
const {
|
||||
managedWorktrees,
|
||||
resolveFreshestSessionEntryFromStoreKeys,
|
||||
resolveGatewaySessionStoreTargetWithStore,
|
||||
} = await loadWorkerPlacementSessionRuntimeModule();
|
||||
const target = resolveGatewaySessionStoreTargetWithStore({
|
||||
cfg: getRuntimeConfig(),
|
||||
key: sessionKey,
|
||||
agentId,
|
||||
clone: false,
|
||||
});
|
||||
const sessionEntry = resolveFreshestSessionEntryFromStoreKeys(
|
||||
target.store,
|
||||
target.storeKeys,
|
||||
);
|
||||
const worktree = managedWorktrees.findLiveByOwner("session", target.canonicalKey);
|
||||
if (
|
||||
sessionEntry?.sessionId !== sessionId ||
|
||||
!sessionEntry.worktree?.id ||
|
||||
!worktree ||
|
||||
worktree.id !== sessionEntry.worktree.id ||
|
||||
worktree.ownerId !== target.canonicalKey
|
||||
) {
|
||||
throw new Error(
|
||||
`Session ${sessionKey} dispatch requires a session-owned managed worktree`,
|
||||
);
|
||||
}
|
||||
return worktree.path;
|
||||
},
|
||||
}),
|
||||
);
|
||||
const admissionProvider = createWorkerSessionTurnPlacementProvider({
|
||||
environments: params.environments,
|
||||
placements: params.placements,
|
||||
admitNewPlacements: params.admitNewPlacements,
|
||||
});
|
||||
const startRuntime = async (hooks: {
|
||||
isClosePreludeStarted: () => boolean;
|
||||
registerSidecar: (sidecar: WorkerPlacementSidecar) => void;
|
||||
}): Promise<WorkerPlacementSidecar | null> => {
|
||||
const uninstallPlacementAdmission = installSessionPlacementAdmissionProvider(admissionProvider);
|
||||
const uninstallPlacementResetGuard = installSessionPlacementResetGuard((sessionId) => {
|
||||
const placement = params.placements.get(sessionId);
|
||||
if (!placement || placement.state === "local") {
|
||||
return undefined;
|
||||
}
|
||||
return `cloud worker placement is ${placement.state}`;
|
||||
});
|
||||
let placementReconcileInterval: ReturnType<typeof setInterval> | undefined;
|
||||
let placementReconcileInFlight: Promise<void> | undefined;
|
||||
let stopped = false;
|
||||
const reconcileActivePlacements = (): Promise<void> => {
|
||||
if (stopped) {
|
||||
return Promise.resolve();
|
||||
}
|
||||
if (placementReconcileInFlight) {
|
||||
return placementReconcileInFlight;
|
||||
}
|
||||
const current = dispatchService.reconcileActive();
|
||||
placementReconcileInFlight = current;
|
||||
const clearCurrent = () => {
|
||||
if (placementReconcileInFlight === current) {
|
||||
placementReconcileInFlight = undefined;
|
||||
}
|
||||
};
|
||||
void current.then(clearCurrent, (error: unknown) => {
|
||||
params.warn(`Worker placement reconcile sweep failed: ${formatErrorMessage(error)}`);
|
||||
clearCurrent();
|
||||
});
|
||||
return current;
|
||||
};
|
||||
const sidecar: WorkerPlacementSidecar = {
|
||||
stop: async () => {
|
||||
if (stopped) {
|
||||
return;
|
||||
}
|
||||
stopped = true;
|
||||
clearInterval(placementReconcileInterval);
|
||||
placementReconcileInterval = undefined;
|
||||
uninstallPlacementAdmission();
|
||||
uninstallPlacementResetGuard();
|
||||
const environmentStop = params.environments.stop();
|
||||
const stopResults = await Promise.allSettled([
|
||||
...(placementReconcileInFlight ? [placementReconcileInFlight] : []),
|
||||
environmentStop,
|
||||
]);
|
||||
const environmentStopResult = stopResults.at(-1);
|
||||
if (environmentStopResult?.status === "rejected") {
|
||||
throw environmentStopResult.reason;
|
||||
}
|
||||
},
|
||||
};
|
||||
// Close must see the drain handle before reconciliation can yield.
|
||||
hooks.registerSidecar(sidecar);
|
||||
// Track startup reconciliation in the shared in-flight slot so a concurrent
|
||||
// close prelude drains it before uninstalling guards and stopping environments.
|
||||
const startupReconcile = dispatchService.reconcile();
|
||||
placementReconcileInFlight = startupReconcile;
|
||||
try {
|
||||
try {
|
||||
await startupReconcile;
|
||||
} finally {
|
||||
if (placementReconcileInFlight === startupReconcile) {
|
||||
placementReconcileInFlight = undefined;
|
||||
}
|
||||
}
|
||||
if (hooks.isClosePreludeStarted()) {
|
||||
await sidecar.stop();
|
||||
return null;
|
||||
}
|
||||
params.environments.start();
|
||||
placementReconcileInterval = setInterval(
|
||||
() => void reconcileActivePlacements(),
|
||||
WORKER_PLACEMENT_RECONCILE_INTERVAL_MS,
|
||||
);
|
||||
placementReconcileInterval.unref?.();
|
||||
return sidecar;
|
||||
} catch (error) {
|
||||
await sidecar.stop();
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
return { dispatchService, admissionProvider, placements: params.placements, startRuntime };
|
||||
}
|
||||
+86
-12
@@ -85,6 +85,7 @@ import { createAuthRateLimiter, type AuthRateLimiter } from "./auth-rate-limit.j
|
||||
import { resolveGatewayAuth } from "./auth.js";
|
||||
import type { RestartRecoveryCandidate } from "./chat-abort.js";
|
||||
import type { ExecApprovalManager } from "./exec-approval-manager.js";
|
||||
import { revokeAttachGrantsForSession } from "./mcp-grant-store.js";
|
||||
import { ADMIN_SCOPE } from "./method-scopes.js";
|
||||
import {
|
||||
STARTUP_UNAVAILABLE_GATEWAY_METHODS,
|
||||
@@ -164,6 +165,9 @@ const loadGatewayModelCatalogModule = createLazyRuntimeModule(
|
||||
const loadWorkerEnvironmentStartupModule = createLazyRuntimeModule(
|
||||
() => import("./server-worker-environment-startup.js"),
|
||||
);
|
||||
const loadWorkerPlacementStartupModule = createLazyRuntimeModule(
|
||||
() => import("./server-worker-placement-startup.js"),
|
||||
);
|
||||
|
||||
export async function resetModelCatalogCacheForTest(): Promise<void> {
|
||||
const { resetModelCatalogCacheForTest: resetModelCatalogCacheForTestLocal } =
|
||||
@@ -177,6 +181,21 @@ const MAX_MEDIA_TTL_HOURS = 24 * 7;
|
||||
const POST_READY_MAINTENANCE_DELAY_MS = 250;
|
||||
const RETAINED_PLUGIN_CLEANUP_DELAY_MS = 30_000;
|
||||
|
||||
function approvalRequestTargetsSession(
|
||||
request: unknown,
|
||||
sessionKeys: ReadonlySet<string>,
|
||||
sessionId: string,
|
||||
): boolean {
|
||||
if (typeof request !== "object" || request === null) {
|
||||
return false;
|
||||
}
|
||||
const record = request as { sessionKey?: unknown; sessionId?: unknown };
|
||||
return (
|
||||
(typeof record.sessionId === "string" && record.sessionId === sessionId) ||
|
||||
(typeof record.sessionKey === "string" && sessionKeys.has(record.sessionKey))
|
||||
);
|
||||
}
|
||||
|
||||
type GatewayStartupChannelPlugin = {
|
||||
id: ChannelId;
|
||||
gatewayMethods?: readonly string[];
|
||||
@@ -901,9 +920,12 @@ export async function startGatewayServer(
|
||||
}
|
||||
let { pluginRegistry, baseGatewayMethods } = pluginBootstrap;
|
||||
// Unconfigured clean installs get no service; durable rows still need list/status projection.
|
||||
const hasConfiguredWorkerProfiles =
|
||||
Object.keys(gatewayPluginConfigAtStart.cloudWorkers?.profiles ?? {}).length > 0;
|
||||
const shouldStartWorkerEnvironmentService =
|
||||
Object.keys(gatewayPluginConfigAtStart.cloudWorkers?.profiles ?? {}).length > 0 ||
|
||||
Boolean(workerEnvironmentStartup?.records.length);
|
||||
hasConfiguredWorkerProfiles ||
|
||||
Boolean(workerEnvironmentStartup?.records.length) ||
|
||||
Boolean(workerEnvironmentStartup?.hasNonlocalPlacementRecords);
|
||||
let resolveWorkerGatewayEndpoint: () =>
|
||||
| { host: "127.0.0.1" | "::1"; port: number }
|
||||
| undefined = () => undefined;
|
||||
@@ -920,6 +942,30 @@ export async function startGatewayServer(
|
||||
})
|
||||
: {};
|
||||
const { workerEnvironmentService, workerLiveEvents } = workerEnvironmentRuntime;
|
||||
// Assigned once approval managers exist; placement dispatch must not run before then.
|
||||
let revokeWorkerDispatchSessionAuthority = (_params: {
|
||||
sessionId: string;
|
||||
sessionKeys: readonly string[];
|
||||
}): void => {
|
||||
throw new Error("Worker dispatch authority revocation is not ready");
|
||||
};
|
||||
const workerPlacementRuntime =
|
||||
workerEnvironmentService && workerEnvironmentStartup
|
||||
? await startupTrace.measure("worker-environments.placement-runtime", async () => {
|
||||
const placementModule = await loadWorkerPlacementStartupModule();
|
||||
return placementModule.createGatewayWorkerPlacementRuntime({
|
||||
placements: workerEnvironmentStartup.placementStore,
|
||||
environments: workerEnvironmentService,
|
||||
admitNewPlacements: hasConfiguredWorkerProfiles,
|
||||
revokeSessionAuthority: (request) => revokeWorkerDispatchSessionAuthority(request),
|
||||
warn: (message) => log.warn(message),
|
||||
});
|
||||
})
|
||||
: undefined;
|
||||
// Without configured profiles, existing placements still reconcile but new dispatches stay off.
|
||||
const workerPlacementDispatchAvailable = hasConfiguredWorkerProfiles
|
||||
? workerPlacementRuntime?.dispatchService
|
||||
: undefined;
|
||||
const channelLogs = Object.fromEntries(
|
||||
listGatewayStartupChannelPlugins().map((plugin) => [plugin.id, logChannels.child(plugin.id)]),
|
||||
) as Record<ChannelId, ReturnType<typeof createSubsystemLogger>>;
|
||||
@@ -937,7 +983,9 @@ export async function startGatewayServer(
|
||||
return methods;
|
||||
};
|
||||
const listActiveGatewayMethods = (nextBaseGatewayMethods: string[]) =>
|
||||
uniqueStrings([...nextBaseGatewayMethods, ...listStartupChannelGatewayMethods()]);
|
||||
uniqueStrings([...nextBaseGatewayMethods, ...listStartupChannelGatewayMethods()]).filter(
|
||||
(method) => workerPlacementDispatchAvailable || method !== "sessions.dispatch",
|
||||
);
|
||||
const runtimeConfig = await startupTrace.measure("runtime.config", async () => {
|
||||
const { resolveGatewayRuntimeConfig } = await import("./server-runtime-config.js");
|
||||
return resolveGatewayRuntimeConfig({
|
||||
@@ -1549,6 +1597,22 @@ export async function startGatewayServer(
|
||||
});
|
||||
approvalManagersForReplay.exec = execApprovalManager;
|
||||
approvalManagersForReplay.plugin = pluginApprovalManager;
|
||||
revokeWorkerDispatchSessionAuthority = ({ sessionId, sessionKeys }) => {
|
||||
const keys = new Set(sessionKeys);
|
||||
for (const sessionKey of keys) {
|
||||
revokeAttachGrantsForSession(sessionKey);
|
||||
}
|
||||
for (const record of execApprovalManager.listPendingRecords()) {
|
||||
if (approvalRequestTargetsSession(record.request, keys, sessionId)) {
|
||||
execApprovalManager.expire(record.id, "worker-dispatch");
|
||||
}
|
||||
}
|
||||
for (const record of pluginApprovalManager.listPendingRecords()) {
|
||||
if (approvalRequestTargetsSession(record.request, keys, sessionId)) {
|
||||
pluginApprovalManager.expire(record.id, "worker-dispatch");
|
||||
}
|
||||
}
|
||||
};
|
||||
const attachedGatewayExtraHandlers: GatewayRequestHandlers = {
|
||||
...pluginRegistry.gatewayHandlers,
|
||||
...extraHandlers,
|
||||
@@ -1568,8 +1632,10 @@ export async function startGatewayServer(
|
||||
}
|
||||
const coreDescriptors = createCoreGatewayMethodDescriptors(coreDescriptorHandlers).filter(
|
||||
(descriptor) =>
|
||||
workerEnvironmentService ||
|
||||
(descriptor.name !== "environments.create" && descriptor.name !== "environments.destroy"),
|
||||
(workerEnvironmentService ||
|
||||
(descriptor.name !== "environments.create" &&
|
||||
descriptor.name !== "environments.destroy")) &&
|
||||
(workerPlacementDispatchAvailable || descriptor.name !== "sessions.dispatch"),
|
||||
);
|
||||
return createGatewayMethodRegistry([
|
||||
...coreDescriptors,
|
||||
@@ -1816,6 +1882,12 @@ export async function startGatewayServer(
|
||||
},
|
||||
nodeRegistry,
|
||||
...(workerEnvironmentService ? { workerEnvironmentService } : {}),
|
||||
...(workerPlacementRuntime
|
||||
? { workerSessionPlacementService: workerPlacementRuntime.placements }
|
||||
: {}),
|
||||
...(workerPlacementDispatchAvailable
|
||||
? { workerPlacementDispatchService: workerPlacementDispatchAvailable }
|
||||
: {}),
|
||||
terminalSessions,
|
||||
agentRunSeq,
|
||||
chatAbortControllers,
|
||||
@@ -2054,17 +2126,19 @@ export async function startGatewayServer(
|
||||
runtimeState.gatewayLifetimeSidecars = [];
|
||||
}
|
||||
},
|
||||
...(workerEnvironmentService
|
||||
...(workerPlacementRuntime
|
||||
? {
|
||||
startWorkerEnvironmentRuntime: () => {
|
||||
startWorkerEnvironmentRuntime: async () => {
|
||||
if (closePreludeStarted) {
|
||||
return null;
|
||||
}
|
||||
const sidecar = { stop: () => workerEnvironmentService.stop() };
|
||||
// Close must see the drain handle before reconciliation can yield.
|
||||
runtimeState.gatewayLifetimeSidecars.push(sidecar);
|
||||
workerEnvironmentService.start();
|
||||
return sidecar;
|
||||
return await workerPlacementRuntime.startRuntime({
|
||||
isClosePreludeStarted: () => closePreludeStarted,
|
||||
// Close must see the drain handle before reconciliation can yield.
|
||||
registerSidecar: (sidecar) => {
|
||||
runtimeState.gatewayLifetimeSidecars.push(sidecar);
|
||||
},
|
||||
});
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
import { expect, test, vi } from "vitest";
|
||||
import type { GatewaySessionRow } from "./session-utils.types.js";
|
||||
import { writeSessionStore } from "./test-helpers.js";
|
||||
import {
|
||||
directSessionReq,
|
||||
setupGatewaySessionsTestHarness,
|
||||
} from "./test/server-sessions.test-helpers.js";
|
||||
import type { WorkerSessionPlacementReader } from "./worker-environments/placement-projector.js";
|
||||
import type { WorkerSessionPlacementRecord } from "./worker-environments/placement-store.js";
|
||||
|
||||
const { createSessionStoreDir } = setupGatewaySessionsTestHarness();
|
||||
|
||||
function activePlacementRecord(): WorkerSessionPlacementRecord {
|
||||
return {
|
||||
sessionId: "sess-main",
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
state: "active",
|
||||
environmentId: "env-placement",
|
||||
generation: 7,
|
||||
activeOwnerEpoch: 12,
|
||||
workspaceBaseManifestRef: "manifest-base",
|
||||
remoteWorkspaceDir: "/workspace/main",
|
||||
workerBundleHash: ["a", "b"].join("").repeat(32),
|
||||
lastTranscriptAckCursor: 23,
|
||||
lastLiveEventAckCursor: 9,
|
||||
recoveryError: null,
|
||||
turnClaim: null,
|
||||
createdAtMs: 100,
|
||||
updatedAtMs: 300,
|
||||
stateChangedAtMs: 200,
|
||||
};
|
||||
}
|
||||
|
||||
async function seedSessionRows(): Promise<void> {
|
||||
await createSessionStoreDir();
|
||||
await writeSessionStore({
|
||||
entries: {
|
||||
main: { sessionId: "sess-main", updatedAt: 200 },
|
||||
"agent:main:other": { sessionId: "sess-other", updatedAt: 100 },
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
test("sessions.list omits placement when the worker placement service is disabled", async () => {
|
||||
await seedSessionRows();
|
||||
|
||||
const result = await directSessionReq<{ sessions: GatewaySessionRow[] }>("sessions.list", {});
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
expect(result.payload?.sessions).toHaveLength(2);
|
||||
expect(result.payload?.sessions.every((session) => session.placement === undefined)).toBe(true);
|
||||
});
|
||||
|
||||
test("sessions.list batch-projects durable worker placement", async () => {
|
||||
await seedSessionRows();
|
||||
const placement = activePlacementRecord();
|
||||
const getMany = vi.fn<WorkerSessionPlacementReader["getMany"]>((sessionIds) => {
|
||||
expect(sessionIds).toEqual(expect.arrayContaining(["sess-main", "sess-other"]));
|
||||
return new Map([[placement.sessionId, placement]]);
|
||||
});
|
||||
|
||||
const result = await directSessionReq<{ sessions: GatewaySessionRow[] }>(
|
||||
"sessions.list",
|
||||
{},
|
||||
{
|
||||
context: { workerSessionPlacementService: { getMany } },
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
expect(getMany).toHaveBeenCalledTimes(1);
|
||||
const main = result.payload?.sessions.find((session) => session.sessionId === "sess-main");
|
||||
const other = result.payload?.sessions.find((session) => session.sessionId === "sess-other");
|
||||
expect(main?.placement).toEqual({
|
||||
state: "active",
|
||||
environmentId: "env-placement",
|
||||
generation: 7,
|
||||
activeOwnerEpoch: 12,
|
||||
workspaceBaseManifestRef: "manifest-base",
|
||||
remoteWorkspaceDir: "/workspace/main",
|
||||
workerBundleHash: ["a", "b"].join("").repeat(32),
|
||||
lastTranscriptAckCursor: 23,
|
||||
lastLiveEventAckCursor: 9,
|
||||
createdAtMs: 100,
|
||||
updatedAtMs: 300,
|
||||
stateChangedAtMs: 200,
|
||||
});
|
||||
expect(other?.placement).toBeUndefined();
|
||||
});
|
||||
|
||||
test("sessions.describe projects durable worker placement", async () => {
|
||||
await seedSessionRows();
|
||||
const placement = activePlacementRecord();
|
||||
const getMany = vi.fn<WorkerSessionPlacementReader["getMany"]>((sessionIds) => {
|
||||
expect(sessionIds).toEqual(["sess-main"]);
|
||||
return new Map([[placement.sessionId, placement]]);
|
||||
});
|
||||
|
||||
const result = await directSessionReq<{ session: GatewaySessionRow | null }>(
|
||||
"sessions.describe",
|
||||
{ key: "main" },
|
||||
{
|
||||
context: { workerSessionPlacementService: { getMany } },
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.ok).toBe(true);
|
||||
expect(getMany).toHaveBeenCalledTimes(1);
|
||||
expect(result.payload?.session?.placement).toEqual({
|
||||
state: "active",
|
||||
environmentId: "env-placement",
|
||||
generation: 7,
|
||||
activeOwnerEpoch: 12,
|
||||
workspaceBaseManifestRef: "manifest-base",
|
||||
remoteWorkspaceDir: "/workspace/main",
|
||||
workerBundleHash: ["a", "b"].join("").repeat(32),
|
||||
lastTranscriptAckCursor: 23,
|
||||
lastLiveEventAckCursor: 9,
|
||||
createdAtMs: 100,
|
||||
updatedAtMs: 300,
|
||||
stateChangedAtMs: 200,
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,246 @@
|
||||
import { afterEach, expect, test } from "vitest";
|
||||
import { installSessionPlacementResetGuard } from "../agents/session-placement-admission.js";
|
||||
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
|
||||
import { loadSessionEntry } from "./session-utils.js";
|
||||
import { embeddedRunMock, writeSessionStore } from "./test-helpers.js";
|
||||
import {
|
||||
directSessionReq,
|
||||
sessionStoreEntry,
|
||||
setupGatewaySessionsTestHarness,
|
||||
} from "./test/server-sessions.test-helpers.js";
|
||||
import type { WorkerSessionPlacementReader } from "./worker-environments/placement-projector.js";
|
||||
import type { WorkerSessionPlacementRecord } from "./worker-environments/placement-store.js";
|
||||
|
||||
const { createSessionStoreDir, seedActiveMainSession } = setupGatewaySessionsTestHarness();
|
||||
let uninstallResetGuard: (() => void) | undefined;
|
||||
|
||||
afterEach(() => {
|
||||
uninstallResetGuard?.();
|
||||
uninstallResetGuard = undefined;
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
});
|
||||
|
||||
function placementRecord(
|
||||
sessionId: string,
|
||||
state: "active" | "local",
|
||||
): WorkerSessionPlacementRecord {
|
||||
const identity = {
|
||||
sessionId,
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:worker-session",
|
||||
turnClaim: null,
|
||||
createdAtMs: 1,
|
||||
updatedAtMs: 2,
|
||||
stateChangedAtMs: 2,
|
||||
};
|
||||
if (state === "active") {
|
||||
return {
|
||||
...identity,
|
||||
state,
|
||||
generation: 2,
|
||||
environmentId: "worker-environment",
|
||||
activeOwnerEpoch: 1,
|
||||
workspaceBaseManifestRef: "manifest-ref",
|
||||
remoteWorkspaceDir: "/workspace",
|
||||
workerBundleHash: "bundle-hash",
|
||||
lastTranscriptAckCursor: null,
|
||||
lastLiveEventAckCursor: null,
|
||||
recoveryError: null,
|
||||
};
|
||||
}
|
||||
return {
|
||||
...identity,
|
||||
state,
|
||||
generation: 0,
|
||||
environmentId: null,
|
||||
activeOwnerEpoch: null,
|
||||
workspaceBaseManifestRef: null,
|
||||
remoteWorkspaceDir: null,
|
||||
workerBundleHash: null,
|
||||
lastTranscriptAckCursor: null,
|
||||
lastLiveEventAckCursor: null,
|
||||
recoveryError: null,
|
||||
};
|
||||
}
|
||||
|
||||
function terminalPlacementRecord(
|
||||
sessionId: string,
|
||||
state: "failed" | "reclaimed",
|
||||
): WorkerSessionPlacementRecord {
|
||||
const terminalMetadata = {
|
||||
environmentId: "worker-environment",
|
||||
activeOwnerEpoch: 1,
|
||||
workspaceBaseManifestRef: "manifest-ref",
|
||||
remoteWorkspaceDir: "/workspace",
|
||||
workerBundleHash: "bundle-hash",
|
||||
lastTranscriptAckCursor: null,
|
||||
lastLiveEventAckCursor: null,
|
||||
};
|
||||
const identity = {
|
||||
sessionId,
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:worker-session",
|
||||
generation: 2,
|
||||
turnClaim: null,
|
||||
createdAtMs: 1,
|
||||
updatedAtMs: 2,
|
||||
stateChangedAtMs: 2,
|
||||
};
|
||||
if (state === "failed") {
|
||||
return {
|
||||
...identity,
|
||||
...terminalMetadata,
|
||||
state,
|
||||
recoveryError: "worker recovery stopped",
|
||||
};
|
||||
}
|
||||
return {
|
||||
...identity,
|
||||
...terminalMetadata,
|
||||
state,
|
||||
recoveryError: null,
|
||||
};
|
||||
}
|
||||
|
||||
function sequencedPlacementReader(
|
||||
records: readonly WorkerSessionPlacementRecord[],
|
||||
): WorkerSessionPlacementReader {
|
||||
let readIndex = 0;
|
||||
return {
|
||||
getMany(sessionIds) {
|
||||
const record = records[Math.min(readIndex, records.length - 1)];
|
||||
readIndex += 1;
|
||||
const result = new Map<string, WorkerSessionPlacementRecord>();
|
||||
if (record && sessionIds.includes(record.sessionId)) {
|
||||
result.set(record.sessionId, record);
|
||||
}
|
||||
return result;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test("sessions.reset rechecks worker placement inside the lifecycle fence", async () => {
|
||||
await seedActiveMainSession();
|
||||
let resetGuardReadCount = 0;
|
||||
uninstallResetGuard = installSessionPlacementResetGuard((sessionId) => {
|
||||
expect(sessionId).toBe("sess-main");
|
||||
resetGuardReadCount += 1;
|
||||
return resetGuardReadCount === 1 ? undefined : "cloud worker placement is active";
|
||||
});
|
||||
|
||||
const reset = await directSessionReq("sessions.reset", { key: "main" });
|
||||
|
||||
expect(reset.ok).toBe(false);
|
||||
expect(reset.error?.message).toContain("cloud worker placement is active");
|
||||
expect(resetGuardReadCount).toBe(2);
|
||||
expect(loadSessionEntry("main").entry?.sessionId).toBe("sess-main");
|
||||
expect(embeddedRunMock.abortCalls).toEqual([]);
|
||||
});
|
||||
|
||||
test("sessions.delete rechecks worker placement before destructive cleanup", async () => {
|
||||
await createSessionStoreDir();
|
||||
const sessionKey = "discord:group:worker-session";
|
||||
const sessionId = "sess-worker-delete";
|
||||
await writeSessionStore({ entries: { [sessionKey]: sessionStoreEntry(sessionId) } });
|
||||
const placementReader = sequencedPlacementReader([
|
||||
placementRecord(sessionId, "local"),
|
||||
placementRecord(sessionId, "active"),
|
||||
]);
|
||||
|
||||
const deleted = await directSessionReq(
|
||||
"sessions.delete",
|
||||
{ key: sessionKey },
|
||||
{
|
||||
context: { workerSessionPlacementService: placementReader },
|
||||
},
|
||||
);
|
||||
|
||||
expect(deleted.ok).toBe(false);
|
||||
expect(deleted.error?.message).toContain("cloud worker placement is active");
|
||||
expect(loadSessionEntry(sessionKey).entry?.sessionId).toBe(sessionId);
|
||||
expect(embeddedRunMock.abortCalls).toEqual([]);
|
||||
});
|
||||
|
||||
test("sessions.delete rejects failed placement with unresolved worker ownership", async () => {
|
||||
await createSessionStoreDir();
|
||||
const sessionKey = "discord:group:failed-worker-session";
|
||||
const sessionId = "sess-failed-worker-delete";
|
||||
await writeSessionStore({ entries: { [sessionKey]: sessionStoreEntry(sessionId) } });
|
||||
const placementReader = sequencedPlacementReader([terminalPlacementRecord(sessionId, "failed")]);
|
||||
|
||||
const deleted = await directSessionReq(
|
||||
"sessions.delete",
|
||||
{ key: sessionKey },
|
||||
{
|
||||
context: { workerSessionPlacementService: placementReader },
|
||||
},
|
||||
);
|
||||
|
||||
expect(deleted.ok).toBe(false);
|
||||
expect(deleted.error?.message).toContain("cloud worker placement is failed");
|
||||
expect(loadSessionEntry(sessionKey).entry?.sessionId).toBe(sessionId);
|
||||
expect(embeddedRunMock.abortCalls).toEqual([]);
|
||||
});
|
||||
|
||||
test("sessions.delete allows reclaimed placement with no live worker owner", async () => {
|
||||
await createSessionStoreDir();
|
||||
const sessionKey = "discord:group:reclaimed-worker-session";
|
||||
const sessionId = "sess-reclaimed-worker-delete";
|
||||
await writeSessionStore({ entries: { [sessionKey]: sessionStoreEntry(sessionId) } });
|
||||
const placementReader = sequencedPlacementReader([
|
||||
terminalPlacementRecord(sessionId, "reclaimed"),
|
||||
]);
|
||||
|
||||
const deleted = await directSessionReq(
|
||||
"sessions.delete",
|
||||
{ key: sessionKey },
|
||||
{
|
||||
context: { workerSessionPlacementService: placementReader },
|
||||
},
|
||||
);
|
||||
|
||||
expect(deleted.ok).toBe(true);
|
||||
expect(deleted.payload).toMatchObject({ ok: true, deleted: true });
|
||||
expect(loadSessionEntry(sessionKey).entry).toBeUndefined();
|
||||
});
|
||||
|
||||
test("sessions.compaction.restore rechecks worker placement inside the lifecycle fence", async () => {
|
||||
await createSessionStoreDir();
|
||||
const sessionKey = "discord:group:worker-restore";
|
||||
const sessionId = "sess-worker-restore";
|
||||
const checkpointId = "checkpoint-worker-restore";
|
||||
await writeSessionStore({
|
||||
entries: {
|
||||
[sessionKey]: sessionStoreEntry(sessionId, {
|
||||
compactionCheckpoints: [
|
||||
{
|
||||
checkpointId,
|
||||
sessionKey,
|
||||
sessionId,
|
||||
createdAt: 1,
|
||||
reason: "manual",
|
||||
preCompaction: { sessionId },
|
||||
postCompaction: { sessionId },
|
||||
},
|
||||
],
|
||||
}),
|
||||
},
|
||||
});
|
||||
const placementReader = sequencedPlacementReader([
|
||||
placementRecord(sessionId, "local"),
|
||||
placementRecord(sessionId, "active"),
|
||||
]);
|
||||
|
||||
const restored = await directSessionReq(
|
||||
"sessions.compaction.restore",
|
||||
{ key: sessionKey, checkpointId },
|
||||
{
|
||||
context: { workerSessionPlacementService: placementReader },
|
||||
},
|
||||
);
|
||||
|
||||
expect(restored.ok).toBe(false);
|
||||
expect(restored.error?.message).toContain("cloud worker placement is active");
|
||||
expect(loadSessionEntry(sessionKey).entry?.sessionId).toBe(sessionId);
|
||||
expect(embeddedRunMock.abortCalls).toEqual([]);
|
||||
});
|
||||
@@ -53,6 +53,7 @@ const WORKER_CONNECT: WorkerConnectParams = {
|
||||
environmentId: "worker-1",
|
||||
credential: CREDENTIAL,
|
||||
sessionId: null,
|
||||
runId: null,
|
||||
ownerEpoch: 1,
|
||||
rpcSetVersion: 1,
|
||||
handshake: HANDSHAKE,
|
||||
@@ -63,6 +64,7 @@ const IDENTITY: WorkerConnectionIdentity = {
|
||||
credentialHash: "h".repeat(43),
|
||||
bundleHash: HANDSHAKE.bundleHash,
|
||||
sessionId: null,
|
||||
runId: null,
|
||||
ownerEpoch: 1,
|
||||
rpcSetVersion: 1,
|
||||
protocolFeatures: [...HANDSHAKE.protocolFeatures],
|
||||
@@ -90,6 +92,7 @@ const LIVE_EVENT = {
|
||||
const ATTACHED_IDENTITY: WorkerConnectionIdentity = {
|
||||
...IDENTITY,
|
||||
sessionId: "session-1",
|
||||
runId: "run-1",
|
||||
};
|
||||
const INFERENCE_IDS = {
|
||||
runEpoch: 1,
|
||||
|
||||
@@ -1246,6 +1246,7 @@ describe("session.message websocket events", () => {
|
||||
credentialHash: ["fanout", "credential", "hash"].join("-"),
|
||||
bundleHash: "f".repeat(64),
|
||||
sessionId,
|
||||
runId: "run-fanout",
|
||||
ownerEpoch: 4,
|
||||
rpcSetVersion: 1,
|
||||
protocolFeatures: ["worker-live-event-v1", "worker-transcript-commit-v1"],
|
||||
|
||||
@@ -20,6 +20,7 @@ import { clearAllCliSessions } from "../agents/cli-session.js";
|
||||
import { abortEmbeddedAgentRun, waitForEmbeddedAgentRunEnd } from "../agents/embedded-agent.js";
|
||||
import { resetRegisteredAgentHarnessSessions } from "../agents/harness/registry.js";
|
||||
import { resolveSessionModelRef } from "../agents/session-model-ref.js";
|
||||
import { resolveSessionPlacementResetBlock } from "../agents/session-placement-admission.js";
|
||||
import { stopSubagentsForRequester } from "../auto-reply/reply/abort.js";
|
||||
import {
|
||||
buildSessionEndHookPayload,
|
||||
@@ -927,6 +928,18 @@ export async function performGatewaySessionReset(params: {
|
||||
error: errorShape(ErrorCodes.INVALID_REQUEST, MODEL_SELECTION_LOCKED_RESET_MESSAGE),
|
||||
};
|
||||
}
|
||||
const initialPlacementBlock = initialResetEntry?.sessionId
|
||||
? resolveSessionPlacementResetBlock(initialResetEntry.sessionId)
|
||||
: undefined;
|
||||
if (initialPlacementBlock) {
|
||||
return {
|
||||
ok: false,
|
||||
error: errorShape(
|
||||
ErrorCodes.INVALID_REQUEST,
|
||||
`Session ${params.key} cannot reset while ${initialPlacementBlock}.`,
|
||||
),
|
||||
};
|
||||
}
|
||||
const resetLifecycleIdentities = [
|
||||
resetTarget.target.canonicalKey,
|
||||
params.key,
|
||||
@@ -979,6 +992,18 @@ export async function performGatewaySessionReset(params: {
|
||||
params.key,
|
||||
requestedAgentId ? { agentId: requestedAgentId } : undefined,
|
||||
);
|
||||
const placementBlock = entry?.sessionId
|
||||
? resolveSessionPlacementResetBlock(entry.sessionId)
|
||||
: undefined;
|
||||
if (placementBlock) {
|
||||
return {
|
||||
ok: false,
|
||||
error: errorShape(
|
||||
ErrorCodes.INVALID_REQUEST,
|
||||
`Session ${params.key} cannot reset while ${placementBlock}.`,
|
||||
),
|
||||
};
|
||||
}
|
||||
const archivedSessionError = resolveSessionWorkStartError(canonicalKey, entry);
|
||||
if (archivedSessionError) {
|
||||
return {
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
// Shared Gateway session projection types.
|
||||
// Keeps server methods and Control UI payloads aligned.
|
||||
import type { FastMode } from "@openclaw/normalization-core/string-coerce";
|
||||
import type { SessionPlacement } from "../../packages/gateway-protocol/src/index.js";
|
||||
import type { ChatType } from "../channels/chat-type.js";
|
||||
import type {
|
||||
SessionCompactionCheckpoint,
|
||||
@@ -77,6 +78,7 @@ export type GatewaySessionRow = {
|
||||
lastReadAt?: number;
|
||||
lastActivityAt?: number;
|
||||
sessionId?: string;
|
||||
placement?: SessionPlacement;
|
||||
systemSent?: boolean;
|
||||
abortedLastRun?: boolean;
|
||||
thinkingLevel?: string;
|
||||
|
||||
@@ -52,15 +52,23 @@ describe("worker admission", () => {
|
||||
} as WorkerEnvironmentStore;
|
||||
});
|
||||
|
||||
const admission = (overrides: Partial<WorkerConnectParams["admission"]> = {}) => ({
|
||||
environmentId: "worker-1",
|
||||
credential: CREDENTIAL,
|
||||
sessionId: null,
|
||||
ownerEpoch: 1,
|
||||
rpcSetVersion: 1,
|
||||
handshake: RECEIPT,
|
||||
...overrides,
|
||||
});
|
||||
type AdmissionCommon = Omit<WorkerConnectParams["admission"], "sessionId" | "runId">;
|
||||
type AdmissionOverrides =
|
||||
| (Partial<AdmissionCommon> & { sessionId?: null; runId?: null })
|
||||
| (Partial<AdmissionCommon> & { sessionId: string; runId: string });
|
||||
const admission = (overrides: AdmissionOverrides = {}): WorkerConnectParams["admission"] => {
|
||||
const common = {
|
||||
environmentId: overrides.environmentId ?? "worker-1",
|
||||
credential: overrides.credential ?? CREDENTIAL,
|
||||
ownerEpoch: overrides.ownerEpoch ?? 1,
|
||||
rpcSetVersion: overrides.rpcSetVersion ?? 1,
|
||||
handshake: overrides.handshake ?? RECEIPT,
|
||||
};
|
||||
if (typeof overrides.sessionId === "string" && typeof overrides.runId === "string") {
|
||||
return { ...common, sessionId: overrides.sessionId, runId: overrides.runId };
|
||||
}
|
||||
return { ...common, sessionId: null, runId: null };
|
||||
};
|
||||
const admit = (workerAdmission = admission(), expectedBuild: typeof RECEIPT = RECEIPT) =>
|
||||
admitWorkerConnection({ store, admission: workerAdmission, expectedBuild, nowMs });
|
||||
|
||||
@@ -72,6 +80,7 @@ describe("worker admission", () => {
|
||||
identity: {
|
||||
environmentId: "worker-1",
|
||||
sessionId: null,
|
||||
runId: null,
|
||||
ownerEpoch: 1,
|
||||
rpcSetVersion: 1,
|
||||
protocolFeatures: ["worker-heartbeat-v1"],
|
||||
@@ -86,7 +95,7 @@ describe("worker admission", () => {
|
||||
["environment-mismatch", () => admission({ environmentId: " worker-1 " })],
|
||||
["bundle-mismatch", () => admission({ handshake: { ...RECEIPT, bundleHash: "b".repeat(64) } })],
|
||||
["version-mismatch", () => admission({ handshake: { ...RECEIPT, openclawVersion: "other" } })],
|
||||
["session-mismatch", () => admission({ sessionId: "session-other" })],
|
||||
["session-mismatch", () => admission({ sessionId: "session-other", runId: "run-other" })],
|
||||
["owner-epoch-mismatch", () => admission({ ownerEpoch: 2 })],
|
||||
["rpc-set-mismatch", () => admission({ rpcSetVersion: 2 })],
|
||||
[
|
||||
|
||||
@@ -93,6 +93,9 @@ export function admitWorkerConnection(params: {
|
||||
if (admission.sessionId !== credential.sessionId) {
|
||||
return { ok: false, reason: "session-mismatch" };
|
||||
}
|
||||
if ((admission.sessionId === null) !== (admission.runId === null)) {
|
||||
return { ok: false, reason: "session-mismatch" };
|
||||
}
|
||||
if (
|
||||
admission.ownerEpoch !== credential.ownerEpoch ||
|
||||
admission.ownerEpoch !== environment.ownerEpoch
|
||||
@@ -121,6 +124,7 @@ export function admitWorkerConnection(params: {
|
||||
credentialHash: credential.credentialHash,
|
||||
bundleHash: credential.bundleHash,
|
||||
sessionId: credential.sessionId,
|
||||
runId: admission.runId,
|
||||
ownerEpoch: credential.ownerEpoch,
|
||||
rpcSetVersion: credential.rpcSetVersion,
|
||||
protocolFeatures: [...environment.bootstrapReceipt.protocolFeatures],
|
||||
|
||||
@@ -4,6 +4,7 @@ export type WorkerConnectionIdentity = {
|
||||
credentialHash: string;
|
||||
bundleHash: string;
|
||||
sessionId: string | null;
|
||||
runId: string | null;
|
||||
ownerEpoch: number;
|
||||
rpcSetVersion: number;
|
||||
protocolFeatures: string[];
|
||||
|
||||
@@ -73,6 +73,7 @@ const identity: WorkerConnectionIdentity = {
|
||||
credentialHash: ["credential", "hash", "runtime", "test"].join("-"),
|
||||
bundleHash: "bundle-hash-runtime-test",
|
||||
sessionId: SESSION_ID,
|
||||
runId: "run-runtime-test",
|
||||
ownerEpoch: 3,
|
||||
rpcSetVersion: 1,
|
||||
protocolFeatures: ["worker-inference-v1"],
|
||||
|
||||
@@ -41,6 +41,7 @@ const IDENTITY: WorkerConnectionIdentity = {
|
||||
credentialHash: ["fixture", "digest"].join("-"),
|
||||
bundleHash: ["fixture", "bundle", "digest"].join("-"),
|
||||
sessionId: REQUEST.sessionId,
|
||||
runId: REQUEST.runId,
|
||||
ownerEpoch: REQUEST.runEpoch,
|
||||
rpcSetVersion: 1,
|
||||
protocolFeatures: ["worker-inference-v1"],
|
||||
|
||||
@@ -27,6 +27,7 @@ const IDENTITY: WorkerConnectionIdentity = {
|
||||
credentialHash: "d",
|
||||
bundleHash: "b",
|
||||
sessionId: REQUEST.sessionId,
|
||||
runId: REQUEST.runId,
|
||||
ownerEpoch: REQUEST.runEpoch,
|
||||
rpcSetVersion: 1,
|
||||
protocolFeatures: ["worker-inference-v1"],
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
claimAgentRunContext,
|
||||
clearAgentRunContext,
|
||||
emitAgentEvent,
|
||||
getAgentEventLifecycleGeneration,
|
||||
getAgentRunContext,
|
||||
onAgentRuntimeEvent,
|
||||
sweepStaleRunContexts,
|
||||
@@ -33,6 +34,7 @@ const ID: Identity = {
|
||||
credentialHash: ["credential", "hash", "live"].join("-"),
|
||||
bundleHash: "b".repeat(64),
|
||||
sessionId: SID,
|
||||
runId: RUN,
|
||||
ownerEpoch: EPOCH,
|
||||
rpcSetVersion: 1,
|
||||
protocolFeatures: ["worker-live-event-v1"],
|
||||
@@ -455,6 +457,51 @@ describe("worker live events", () => {
|
||||
fail(msg(1, "pending", 0, "run-pending"), "invalid-event");
|
||||
});
|
||||
|
||||
it("adopts a compatible pre-registered gateway run context", () => {
|
||||
const lifecycleGeneration = getAgentEventLifecycleGeneration();
|
||||
claimAgentRunContext(RUN, {
|
||||
...LOCAL,
|
||||
isControlUiVisible: false,
|
||||
lifecycleGeneration,
|
||||
});
|
||||
|
||||
ack(msg(1, "worker"));
|
||||
|
||||
expect(getAgentRunContext(RUN)).toMatchObject({
|
||||
...LOCAL,
|
||||
isControlUiVisible: false,
|
||||
lifecycleGeneration,
|
||||
projectSessionActive: true,
|
||||
});
|
||||
expect(deltas()).toEqual(["worker"]);
|
||||
});
|
||||
|
||||
it("rejects pre-registered gateway run contexts with mismatched identity", () => {
|
||||
const lifecycleGeneration = getAgentEventLifecycleGeneration();
|
||||
const mismatches: Array<{
|
||||
context: Parameters<typeof claimAgentRunContext>[1];
|
||||
name: string;
|
||||
}> = [
|
||||
{ name: "session-id", context: { ...LOCAL, sessionId: `${SID}-other` } },
|
||||
{ name: "session-key", context: { ...LOCAL, sessionKey: `${KEY}-other` } },
|
||||
{ name: "agent-id", context: { ...LOCAL, agentId: "other" } },
|
||||
{ name: "lifecycle", context: { ...LOCAL, lifecycleGeneration: "other-lifecycle" } },
|
||||
{ name: "visibility", context: { ...LOCAL, isControlUiVisible: true } },
|
||||
];
|
||||
|
||||
for (const mismatch of mismatches) {
|
||||
const runId = `run-mismatch-${mismatch.name}`;
|
||||
claimAgentRunContext(runId, {
|
||||
isControlUiVisible: false,
|
||||
lifecycleGeneration,
|
||||
...mismatch.context,
|
||||
});
|
||||
fail(msg(1, "blocked", 0, runId), "invalid-event");
|
||||
clearAgentRunContext(runId);
|
||||
}
|
||||
expect(events).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps run ids exclusive", () => {
|
||||
const local = "run-local-first";
|
||||
claimAgentRunContext(local, LOCAL);
|
||||
|
||||
@@ -569,11 +569,19 @@ export function createWorkerLiveEventReceiver(options: WorkerLiveEventReceiverOp
|
||||
}
|
||||
const lifecycleGeneration = getAgentEventLifecycleGeneration();
|
||||
const existingContext = getAgentRunContext(runId);
|
||||
if (existingContext?.lifecycleGeneration === lifecycleGeneration) {
|
||||
return invalidEvent();
|
||||
}
|
||||
// Turn placement owns wider visibility; otherwise scope to this session.
|
||||
const controlUiVisible = false;
|
||||
const adoptExistingUnowned = existingContext !== undefined;
|
||||
if (
|
||||
existingContext &&
|
||||
(existingContext.sessionId !== window.sessionId ||
|
||||
existingContext.sessionKey !== window.target.sessionKey ||
|
||||
existingContext.agentId !== window.target.agentId ||
|
||||
existingContext.lifecycleGeneration !== lifecycleGeneration ||
|
||||
existingContext.isControlUiVisible !== controlUiVisible)
|
||||
) {
|
||||
return invalidEvent();
|
||||
}
|
||||
const claimId = claimAgentRunContext(
|
||||
runId,
|
||||
{
|
||||
@@ -585,6 +593,7 @@ export function createWorkerLiveEventReceiver(options: WorkerLiveEventReceiverOp
|
||||
sessionKey: window.target.sessionKey,
|
||||
},
|
||||
{
|
||||
adoptExistingUnowned,
|
||||
exclusive: true,
|
||||
onClearRequested: (clearedClaimId) => {
|
||||
if (window.activeRuns.get(runId)?.claimId === clearedClaimId) {
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
|
||||
import { formatErrorMessage } from "../../infra/errors.js";
|
||||
import { redactSensitiveText } from "../../logging/redact.js";
|
||||
import type {
|
||||
createWorkerSessionPlacementStore,
|
||||
WorkerSessionPlacementRecord,
|
||||
} from "./placement-store.js";
|
||||
import type { WorkerEnvironmentService } from "./service.js";
|
||||
|
||||
export type WorkerDispatchPlacement = WorkerSessionPlacementRecord;
|
||||
export type WorkerActiveDispatchPlacement = Extract<
|
||||
WorkerSessionPlacementRecord,
|
||||
{ state: "active" }
|
||||
>;
|
||||
export type WorkerFailedDispatchPlacement = Extract<WorkerDispatchPlacement, { state: "failed" }>;
|
||||
export type WorkerStartingDispatchPlacement = Extract<
|
||||
WorkerDispatchPlacement,
|
||||
{ state: "starting" }
|
||||
>;
|
||||
type WorkerDrainingDispatchPlacement = Extract<WorkerDispatchPlacement, { state: "draining" }>;
|
||||
type WorkerReconcilingDispatchPlacement = Extract<
|
||||
WorkerDispatchPlacement,
|
||||
{ state: "reconciling" }
|
||||
>;
|
||||
|
||||
export type WorkerDispatchPlacementStore = Pick<
|
||||
ReturnType<typeof createWorkerSessionPlacementStore>,
|
||||
| "adoptActive"
|
||||
| "fail"
|
||||
| "get"
|
||||
| "listForReconcile"
|
||||
| "startDispatch"
|
||||
| "startDrain"
|
||||
| "startReconcile"
|
||||
| "transition"
|
||||
>;
|
||||
|
||||
export type WorkerDispatchEnvironmentService = Pick<
|
||||
WorkerEnvironmentService,
|
||||
"attachSession" | "create" | "destroy" | "get" | "reconcileOnce" | "startTunnel" | "stopTunnel"
|
||||
>;
|
||||
|
||||
export type WorkerActivationBarrier = (params: {
|
||||
sessionId: string;
|
||||
sessionKey: string;
|
||||
agentId: string;
|
||||
activate: () => WorkerActiveDispatchPlacement;
|
||||
}) => Promise<WorkerActiveDispatchPlacement>;
|
||||
|
||||
const RECOVERY_ERROR_LIMIT = 1_024;
|
||||
|
||||
function boundedError(error: unknown): string {
|
||||
const redacted = redactSensitiveText(formatErrorMessage(error), { mode: "tools" })
|
||||
.replace(/\s+/gu, " ")
|
||||
.trim();
|
||||
return truncateUtf16Safe(redacted || "unknown dispatch failure", RECOVERY_ERROR_LIMIT);
|
||||
}
|
||||
|
||||
export function isUnavailableEnvironment(
|
||||
environment: NonNullable<ReturnType<WorkerEnvironmentService["get"]>>,
|
||||
): boolean {
|
||||
return (
|
||||
environment.state === "draining" ||
|
||||
environment.state === "destroying" ||
|
||||
environment.state === "destroyed" ||
|
||||
environment.state === "failed" ||
|
||||
environment.state === "orphaned"
|
||||
);
|
||||
}
|
||||
|
||||
export function createPlacementFailureActions(deps: {
|
||||
placements: WorkerDispatchPlacementStore;
|
||||
environments: WorkerDispatchEnvironmentService;
|
||||
}) {
|
||||
const { environments, placements } = deps;
|
||||
|
||||
const updateFailure = (
|
||||
placement: WorkerDispatchPlacement,
|
||||
error: unknown,
|
||||
): WorkerDispatchPlacement =>
|
||||
placements.fail({
|
||||
sessionId: placement.sessionId,
|
||||
expectedGeneration: placement.generation,
|
||||
recoveryError: boundedError(error),
|
||||
});
|
||||
|
||||
const cleanupEnvironment = async (params: {
|
||||
environmentId: string;
|
||||
ownerEpoch: number | null;
|
||||
}): Promise<string[]> => {
|
||||
const teardownErrors: string[] = [];
|
||||
try {
|
||||
await environments.stopTunnel(params.environmentId, params.ownerEpoch ?? undefined);
|
||||
} catch (error) {
|
||||
teardownErrors.push(`tunnel stop: ${boundedError(error)}`);
|
||||
}
|
||||
try {
|
||||
await environments.destroy(params.environmentId);
|
||||
} catch (error) {
|
||||
teardownErrors.push(`environment destroy: ${boundedError(error)}`);
|
||||
}
|
||||
return teardownErrors;
|
||||
};
|
||||
|
||||
const teardownEnvironment = async (params: {
|
||||
placement: WorkerDispatchPlacement;
|
||||
environmentId: string | null;
|
||||
ownerEpoch: number | null;
|
||||
primaryError: unknown;
|
||||
}): Promise<void> => {
|
||||
const environmentId = params.environmentId;
|
||||
const teardownErrors = environmentId
|
||||
? await cleanupEnvironment({
|
||||
environmentId,
|
||||
ownerEpoch: params.ownerEpoch,
|
||||
})
|
||||
: [];
|
||||
const recoveryError = [boundedError(params.primaryError), ...teardownErrors].join("; ");
|
||||
updateFailure(
|
||||
params.placement,
|
||||
new Error(truncateUtf16Safe(recoveryError, RECOVERY_ERROR_LIMIT)),
|
||||
);
|
||||
};
|
||||
|
||||
const retryFailedTeardown = async (placement: WorkerFailedDispatchPlacement): Promise<void> => {
|
||||
if (!placement.environmentId) {
|
||||
return;
|
||||
}
|
||||
const environment = environments.get(placement.environmentId);
|
||||
if (
|
||||
!environment ||
|
||||
environment.state === "destroyed" ||
|
||||
environment.state === "failed" ||
|
||||
environment.state === "orphaned"
|
||||
) {
|
||||
return;
|
||||
}
|
||||
const teardownErrors = await cleanupEnvironment({
|
||||
environmentId: placement.environmentId,
|
||||
ownerEpoch: placement.activeOwnerEpoch,
|
||||
});
|
||||
if (teardownErrors.length > 0) {
|
||||
const recoveryError = [placement.recoveryError, ...teardownErrors].filter(Boolean).join("; ");
|
||||
placements.fail({
|
||||
sessionId: placement.sessionId,
|
||||
expectedGeneration: placement.generation,
|
||||
recoveryError: truncateUtf16Safe(recoveryError, RECOVERY_ERROR_LIMIT),
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const startDrain = (
|
||||
placement: WorkerActiveDispatchPlacement,
|
||||
): WorkerDrainingDispatchPlacement => {
|
||||
const draining = placements.startDrain({
|
||||
sessionId: placement.sessionId,
|
||||
environmentId: placement.environmentId,
|
||||
ownerEpoch: placement.activeOwnerEpoch,
|
||||
expectedGeneration: placement.generation,
|
||||
});
|
||||
if (draining.state !== "draining") {
|
||||
throw new Error("Worker placement drain did not produce a draining placement");
|
||||
}
|
||||
return draining;
|
||||
};
|
||||
|
||||
const startReconcile = (
|
||||
placement: WorkerDrainingDispatchPlacement,
|
||||
): WorkerReconcilingDispatchPlacement => {
|
||||
const reconciling = placements.startReconcile({
|
||||
sessionId: placement.sessionId,
|
||||
environmentId: placement.environmentId,
|
||||
ownerEpoch: placement.activeOwnerEpoch,
|
||||
expectedGeneration: placement.generation,
|
||||
});
|
||||
if (reconciling.state !== "reconciling") {
|
||||
throw new Error("Worker placement reconcile did not produce a reconciling placement");
|
||||
}
|
||||
return reconciling;
|
||||
};
|
||||
|
||||
const advanceReclaimed = (placement: WorkerDrainingDispatchPlacement): void => {
|
||||
// Lost-worker recovery has no live workspace to pull back. Deliberate inbound
|
||||
// reconciliation remains a separate migration workflow.
|
||||
const reconciling = startReconcile(placement);
|
||||
const reclaimed = placements.transition({
|
||||
sessionId: reconciling.sessionId,
|
||||
from: "reconciling",
|
||||
to: "reclaimed",
|
||||
expectedGeneration: reconciling.generation,
|
||||
});
|
||||
if (reclaimed.state !== "reclaimed") {
|
||||
throw new Error("Worker placement reclaim did not produce a reclaimed placement");
|
||||
}
|
||||
};
|
||||
|
||||
const finishDrainingFailure = (
|
||||
placement: WorkerDrainingDispatchPlacement,
|
||||
error: unknown,
|
||||
teardownErrors: readonly string[],
|
||||
): void => {
|
||||
const reconciling = startReconcile(placement);
|
||||
const recoveryError = [boundedError(error), ...teardownErrors].join("; ");
|
||||
updateFailure(reconciling, new Error(truncateUtf16Safe(recoveryError, RECOVERY_ERROR_LIMIT)));
|
||||
};
|
||||
|
||||
const failDraining = async (
|
||||
placement: WorkerDrainingDispatchPlacement,
|
||||
error: unknown,
|
||||
): Promise<void> => {
|
||||
const teardownErrors = await cleanupEnvironment({
|
||||
environmentId: placement.environmentId,
|
||||
ownerEpoch: placement.activeOwnerEpoch,
|
||||
});
|
||||
finishDrainingFailure(placement, error, teardownErrors);
|
||||
};
|
||||
|
||||
const reclaimActive = async (
|
||||
placement: WorkerActiveDispatchPlacement,
|
||||
environment: ReturnType<WorkerEnvironmentService["get"]>,
|
||||
claimedTurnError: Error,
|
||||
): Promise<void> => {
|
||||
const draining = startDrain(placement);
|
||||
if (draining.turnClaim) {
|
||||
await failDraining(draining, claimedTurnError);
|
||||
return;
|
||||
}
|
||||
if (environment && !isUnavailableEnvironment(environment)) {
|
||||
const teardownErrors = await cleanupEnvironment({
|
||||
environmentId: placement.environmentId,
|
||||
ownerEpoch: placement.activeOwnerEpoch,
|
||||
});
|
||||
if (teardownErrors.length > 0) {
|
||||
finishDrainingFailure(
|
||||
draining,
|
||||
new Error(`Worker reclaim teardown failed: ${teardownErrors.join("; ")}`),
|
||||
[],
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
advanceReclaimed(draining);
|
||||
};
|
||||
|
||||
const failActive = async (
|
||||
placement: WorkerActiveDispatchPlacement,
|
||||
error: unknown,
|
||||
): Promise<void> => {
|
||||
const draining = startDrain(placement);
|
||||
await failDraining(draining, error);
|
||||
};
|
||||
|
||||
return {
|
||||
failActive,
|
||||
failDraining,
|
||||
reclaimActive,
|
||||
retryFailedTeardown,
|
||||
teardownEnvironment,
|
||||
};
|
||||
}
|
||||
|
||||
export type PlacementFailureActions = ReturnType<typeof createPlacementFailureActions>;
|
||||
@@ -0,0 +1,231 @@
|
||||
import {
|
||||
isUnavailableEnvironment,
|
||||
type PlacementFailureActions,
|
||||
type WorkerActivationBarrier,
|
||||
type WorkerActiveDispatchPlacement,
|
||||
type WorkerDispatchEnvironmentService,
|
||||
type WorkerDispatchPlacement,
|
||||
type WorkerDispatchPlacementStore,
|
||||
type WorkerFailedDispatchPlacement,
|
||||
type WorkerStartingDispatchPlacement,
|
||||
} from "./placement-dispatch-failure.js";
|
||||
import type { WorkerEnvironmentService } from "./service.js";
|
||||
|
||||
function sameActiveEnvironment(
|
||||
placement: WorkerActiveDispatchPlacement,
|
||||
environment: ReturnType<WorkerEnvironmentService["get"]>,
|
||||
): environment is NonNullable<typeof environment> {
|
||||
return Boolean(
|
||||
environment &&
|
||||
environment.state === "attached" &&
|
||||
placement.environmentId &&
|
||||
environment.environmentId === placement.environmentId &&
|
||||
placement.activeOwnerEpoch !== null &&
|
||||
environment.ownerEpoch === placement.activeOwnerEpoch &&
|
||||
placement.workerBundleHash &&
|
||||
environment.bootstrapReceipt?.bundleHash === placement.workerBundleHash &&
|
||||
environment.attachedSessionIds.length === 1 &&
|
||||
environment.attachedSessionIds[0] === placement.sessionId,
|
||||
);
|
||||
}
|
||||
|
||||
function isStartingPlacement(
|
||||
placement: WorkerDispatchPlacement,
|
||||
): placement is WorkerStartingDispatchPlacement {
|
||||
return placement.state === "starting";
|
||||
}
|
||||
|
||||
function isFailedPlacement(
|
||||
placement: WorkerDispatchPlacement,
|
||||
): placement is WorkerFailedDispatchPlacement {
|
||||
return placement.state === "failed";
|
||||
}
|
||||
|
||||
export function createPlacementRecoveryActions(deps: {
|
||||
placements: WorkerDispatchPlacementStore;
|
||||
environments: WorkerDispatchEnvironmentService;
|
||||
runActivationBarrier: WorkerActivationBarrier;
|
||||
failure: PlacementFailureActions;
|
||||
}) {
|
||||
const { environments, failure, placements } = deps;
|
||||
|
||||
const adoptActive = async (placement: WorkerActiveDispatchPlacement): Promise<void> => {
|
||||
// Worker turns are one-shot SSH children owned by the previous gateway process. A durable
|
||||
// claim cannot prove that child remains live after restart, so fence the whole placement.
|
||||
if (placement.turnClaim) {
|
||||
const error = new Error(
|
||||
"Active worker turn claim cannot be proven live after gateway restart",
|
||||
);
|
||||
await failure.failActive(placement, error);
|
||||
return;
|
||||
}
|
||||
const environment = placement.environmentId
|
||||
? environments.get(placement.environmentId)
|
||||
: undefined;
|
||||
if (!environment || isUnavailableEnvironment(environment)) {
|
||||
await failure.reclaimActive(
|
||||
placement,
|
||||
environment,
|
||||
new Error("Active worker disappeared during restart reconciliation"),
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (!sameActiveEnvironment(placement, environment)) {
|
||||
await failure.reclaimActive(
|
||||
placement,
|
||||
environment,
|
||||
new Error("Active worker placement does not match its environment owner"),
|
||||
);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await environments.startTunnel({
|
||||
environmentId: environment.environmentId,
|
||||
ownerEpoch: environment.ownerEpoch,
|
||||
});
|
||||
placements.adoptActive({
|
||||
sessionId: placement.sessionId,
|
||||
expectedGeneration: placement.generation,
|
||||
environmentId: environment.environmentId,
|
||||
ownerEpoch: environment.ownerEpoch,
|
||||
});
|
||||
} catch (error) {
|
||||
await failure.failActive(placement, error);
|
||||
}
|
||||
};
|
||||
|
||||
const resumeStarting = async (placement: WorkerStartingDispatchPlacement): Promise<void> => {
|
||||
const environment = placement.environmentId
|
||||
? environments.get(placement.environmentId)
|
||||
: undefined;
|
||||
const expectedBundle = placement.workerBundleHash;
|
||||
const hasSyncedWorkspace = Boolean(
|
||||
placement.workspaceBaseManifestRef && placement.remoteWorkspaceDir,
|
||||
);
|
||||
const canResume =
|
||||
environment &&
|
||||
expectedBundle &&
|
||||
environment.bootstrapReceipt?.bundleHash === expectedBundle &&
|
||||
hasSyncedWorkspace;
|
||||
if (!canResume) {
|
||||
const error = new Error("Interrupted worker dispatch cannot safely resume");
|
||||
await failure.teardownEnvironment({
|
||||
placement,
|
||||
environmentId: placement.environmentId,
|
||||
ownerEpoch: environment?.ownerEpoch ?? null,
|
||||
primaryError: error,
|
||||
});
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const ownerEpoch =
|
||||
environment.state === "attached" &&
|
||||
environment.attachedSessionIds.length === 1 &&
|
||||
environment.attachedSessionIds[0] === placement.sessionId
|
||||
? environment.ownerEpoch
|
||||
: environment.state === "ready" || environment.state === "idle"
|
||||
? (
|
||||
await environments.attachSession({
|
||||
environmentId: environment.environmentId,
|
||||
ownerEpoch: environment.ownerEpoch,
|
||||
sessionId: placement.sessionId,
|
||||
})
|
||||
).ownerEpoch
|
||||
: undefined;
|
||||
if (ownerEpoch === undefined) {
|
||||
throw new Error(`Worker environment cannot resume dispatch from ${environment.state}`);
|
||||
}
|
||||
await environments.startTunnel({ environmentId: environment.environmentId, ownerEpoch });
|
||||
await deps.runActivationBarrier({
|
||||
sessionId: placement.sessionId,
|
||||
sessionKey: placement.sessionKey,
|
||||
agentId: placement.agentId,
|
||||
activate: () => {
|
||||
const activated = placements.transition({
|
||||
sessionId: placement.sessionId,
|
||||
from: "starting",
|
||||
to: "active",
|
||||
expectedGeneration: placement.generation,
|
||||
patch: { activeOwnerEpoch: ownerEpoch },
|
||||
});
|
||||
if (activated.state !== "active") {
|
||||
throw new Error("Worker dispatch activation did not produce an active placement");
|
||||
}
|
||||
return activated;
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
await failure.teardownEnvironment({
|
||||
placement,
|
||||
environmentId: environment.environmentId,
|
||||
ownerEpoch: environment.ownerEpoch,
|
||||
primaryError: error,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const reconcile = async (): Promise<void> => {
|
||||
await environments.reconcileOnce();
|
||||
for (const placement of placements.listForReconcile()) {
|
||||
if (placement.state === "local" || placement.state === "reclaimed") {
|
||||
continue;
|
||||
}
|
||||
if (placement.state === "active") {
|
||||
await adoptActive(placement);
|
||||
continue;
|
||||
}
|
||||
if (isFailedPlacement(placement)) {
|
||||
await failure.retryFailedTeardown(placement);
|
||||
continue;
|
||||
}
|
||||
if (isStartingPlacement(placement)) {
|
||||
await resumeStarting(placement);
|
||||
continue;
|
||||
}
|
||||
const error = new Error(`Worker dispatch interrupted in ${placement.state}`);
|
||||
if (placement.state === "draining") {
|
||||
await failure.failDraining(placement, error);
|
||||
continue;
|
||||
}
|
||||
await failure.teardownEnvironment({
|
||||
placement,
|
||||
environmentId: placement.environmentId,
|
||||
ownerEpoch: placement.activeOwnerEpoch,
|
||||
primaryError: error,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
// Runtime sweeps must not classify a live dispatch preparation as a crash. They only repair
|
||||
// durable active ownership and retry teardown already fenced by a previous failure.
|
||||
const reconcileActive = async (): Promise<void> => {
|
||||
await environments.reconcileOnce();
|
||||
for (const placement of placements.listForReconcile()) {
|
||||
if (isFailedPlacement(placement)) {
|
||||
await failure.retryFailedTeardown(placement);
|
||||
continue;
|
||||
}
|
||||
if (placement.state !== "active") {
|
||||
continue;
|
||||
}
|
||||
const environment = environments.get(placement.environmentId);
|
||||
if (!environment || isUnavailableEnvironment(environment)) {
|
||||
await failure.reclaimActive(
|
||||
placement,
|
||||
environment,
|
||||
new Error("Active worker disappeared during an admitted turn"),
|
||||
);
|
||||
continue;
|
||||
}
|
||||
if (!sameActiveEnvironment(placement, environment)) {
|
||||
await failure.reclaimActive(
|
||||
placement,
|
||||
environment,
|
||||
new Error("Active worker placement does not match its environment owner"),
|
||||
);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return { reconcile, reconcileActive };
|
||||
}
|
||||
@@ -0,0 +1,694 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
openOpenClawStateDatabase,
|
||||
type OpenClawStateDatabase,
|
||||
} from "../../state/openclaw-state-db.js";
|
||||
import type { MintedWorkerCredential } from "./credential.js";
|
||||
import type {
|
||||
WorkerDispatchEnvironmentService,
|
||||
WorkerDispatchPlacementStore,
|
||||
} from "./placement-dispatch-failure.js";
|
||||
import { createWorkerPlacementDispatchService } from "./placement-dispatch.js";
|
||||
import {
|
||||
createWorkerSessionPlacementStore,
|
||||
type WorkerSessionPlacementRecord,
|
||||
} from "./placement-store.js";
|
||||
import { workerEnvironmentIdForIdempotencyKey } from "./service.js";
|
||||
import type {
|
||||
WorkerEnvironmentBootstrapReceipt,
|
||||
WorkerEnvironmentProfileSnapshot,
|
||||
WorkerEnvironmentSshEndpoint,
|
||||
} from "./store.js";
|
||||
import type { WorkerTunnelHandle } from "./tunnel.js";
|
||||
|
||||
type WorkerDispatchRequest = Parameters<
|
||||
ReturnType<typeof createWorkerPlacementDispatchService>["dispatch"]
|
||||
>[0];
|
||||
|
||||
const BUNDLE_HASH = "a".repeat(64);
|
||||
const MANIFEST_REF = `sha256:${"b".repeat(64)}`;
|
||||
const HOST_KEY = [["ssh", "ed25519"].join("-"), "AAAA"].join(" ");
|
||||
const REQUEST: WorkerDispatchRequest = {
|
||||
sessionId: "session-1",
|
||||
sessionKey: "agent:main:session-1",
|
||||
agentId: "main",
|
||||
profileId: "development",
|
||||
};
|
||||
|
||||
type PlacementStore = ReturnType<typeof createWorkerSessionPlacementStore>;
|
||||
type DispatchEnvironmentRecord = Awaited<ReturnType<WorkerDispatchEnvironmentService["create"]>>;
|
||||
type DispatchStage =
|
||||
| "barrier"
|
||||
| "workspace"
|
||||
| "create"
|
||||
| "tunnel:ready"
|
||||
| "sync"
|
||||
| "attach"
|
||||
| "tunnel:attached"
|
||||
| "activation";
|
||||
|
||||
function seedStartingPlacement(
|
||||
store: PlacementStore,
|
||||
environmentId: string,
|
||||
): WorkerSessionPlacementRecord {
|
||||
let current = store.startDispatch(REQUEST);
|
||||
current = store.transition({
|
||||
sessionId: REQUEST.sessionId,
|
||||
from: "requested",
|
||||
to: "provisioning",
|
||||
expectedGeneration: current.generation,
|
||||
patch: { environmentId },
|
||||
});
|
||||
current = store.transition({
|
||||
sessionId: REQUEST.sessionId,
|
||||
from: "provisioning",
|
||||
to: "syncing",
|
||||
expectedGeneration: current.generation,
|
||||
patch: { workerBundleHash: BUNDLE_HASH },
|
||||
});
|
||||
current = store.transition({
|
||||
sessionId: REQUEST.sessionId,
|
||||
from: "syncing",
|
||||
to: "starting",
|
||||
expectedGeneration: current.generation,
|
||||
patch: {
|
||||
workspaceBaseManifestRef: MANIFEST_REF,
|
||||
remoteWorkspaceDir: "/worker/workspace",
|
||||
},
|
||||
});
|
||||
return current;
|
||||
}
|
||||
|
||||
function seedActivePlacement(
|
||||
store: PlacementStore,
|
||||
params: { environmentId: string; ownerEpoch: number },
|
||||
): WorkerSessionPlacementRecord {
|
||||
const current = seedStartingPlacement(store, params.environmentId);
|
||||
return store.transition({
|
||||
sessionId: REQUEST.sessionId,
|
||||
from: "starting",
|
||||
to: "active",
|
||||
expectedGeneration: current.generation,
|
||||
patch: { activeOwnerEpoch: params.ownerEpoch },
|
||||
});
|
||||
}
|
||||
|
||||
function createHarness(
|
||||
placementStore: PlacementStore,
|
||||
options: { failAt?: DispatchStage; destroyFails?: boolean; claimOnDrain?: boolean } = {},
|
||||
) {
|
||||
const log: string[] = [];
|
||||
const fail = (stage: DispatchStage) => {
|
||||
log.push(stage);
|
||||
if (options.failAt === stage) {
|
||||
throw new Error(`${stage} failed`);
|
||||
}
|
||||
};
|
||||
const placements: WorkerDispatchPlacementStore = {
|
||||
get: (sessionId) => placementStore.get(sessionId),
|
||||
startDispatch: (params) => {
|
||||
log.push("placement:requested");
|
||||
return placementStore.startDispatch(params);
|
||||
},
|
||||
transition: (params) => {
|
||||
log.push(`placement:${params.to}`);
|
||||
return placementStore.transition(params);
|
||||
},
|
||||
fail: (params) => {
|
||||
log.push("placement:failed");
|
||||
return placementStore.fail(params);
|
||||
},
|
||||
listForReconcile: () => placementStore.listForReconcile(),
|
||||
startDrain: (params) => {
|
||||
log.push("placement:draining");
|
||||
if (options.claimOnDrain) {
|
||||
placementStore.claimTurn({
|
||||
sessionId: params.sessionId,
|
||||
sessionKey: REQUEST.sessionKey,
|
||||
agentId: REQUEST.agentId,
|
||||
claimId: "claim-on-drain",
|
||||
runId: "run-on-drain",
|
||||
owner: {
|
||||
kind: "worker",
|
||||
environmentId: params.environmentId,
|
||||
ownerEpoch: params.ownerEpoch,
|
||||
},
|
||||
});
|
||||
}
|
||||
return placementStore.startDrain(params);
|
||||
},
|
||||
startReconcile: (params) => {
|
||||
log.push("placement:reconciling");
|
||||
return placementStore.startReconcile(params);
|
||||
},
|
||||
adoptActive: (params) => {
|
||||
log.push("placement:adopted");
|
||||
return placementStore.adoptActive(params);
|
||||
},
|
||||
};
|
||||
const environmentId = workerEnvironmentIdForIdempotencyKey(
|
||||
`session-dispatch:${REQUEST.sessionId}:1`,
|
||||
);
|
||||
const profileSnapshot: WorkerEnvironmentProfileSnapshot = {
|
||||
settings: { region: "test" },
|
||||
};
|
||||
const bootstrapReceipt: WorkerEnvironmentBootstrapReceipt = {
|
||||
bundleHash: BUNDLE_HASH,
|
||||
openclawVersion: "2026.7.2",
|
||||
protocolFeatures: [],
|
||||
};
|
||||
const sshEndpoint: WorkerEnvironmentSshEndpoint = {
|
||||
host: "worker.example.test",
|
||||
port: 22,
|
||||
user: "worker",
|
||||
hostKey: HOST_KEY,
|
||||
keyRef: { source: "file", provider: "worker-keys", id: "/key" },
|
||||
};
|
||||
const environmentBase = {
|
||||
environmentId,
|
||||
providerId: "fake",
|
||||
profileId: "development",
|
||||
profileSnapshot,
|
||||
provisionOperationId: "provision-1",
|
||||
bootstrapReceipt,
|
||||
teardownTerminalState: null,
|
||||
lastError: null,
|
||||
createdAtMs: 1,
|
||||
updatedAtMs: 1,
|
||||
stateChangedAtMs: 1,
|
||||
idleSinceAtMs: null,
|
||||
destroyRequestedAtMs: null,
|
||||
leaseId: "lease-1",
|
||||
sshEndpoint,
|
||||
};
|
||||
const ready = {
|
||||
...environmentBase,
|
||||
state: "ready",
|
||||
ownerEpoch: 1,
|
||||
attachedSessionIds: [],
|
||||
tunnelStatus: "connected",
|
||||
} satisfies DispatchEnvironmentRecord;
|
||||
const attached = {
|
||||
...environmentBase,
|
||||
state: "attached",
|
||||
ownerEpoch: 2,
|
||||
attachedSessionIds: [REQUEST.sessionId],
|
||||
tunnelStatus: "connected",
|
||||
} satisfies DispatchEnvironmentRecord;
|
||||
let currentEnvironment: ReturnType<WorkerDispatchEnvironmentService["get"]> = ready;
|
||||
const destroyedEnvironment = (ownerEpoch: number): DispatchEnvironmentRecord => ({
|
||||
...environmentBase,
|
||||
state: "destroyed",
|
||||
ownerEpoch,
|
||||
attachedSessionIds: [],
|
||||
tunnelStatus: "stopped",
|
||||
});
|
||||
const tunnelHandle = (ownerEpoch: number): WorkerTunnelHandle => ({
|
||||
environmentId: ready.environmentId,
|
||||
ownerEpoch,
|
||||
remoteSocketPath: "/worker/gateway.sock",
|
||||
runWorkspaceCommand: vi.fn(async () => ({
|
||||
stdout: "",
|
||||
stderr: "",
|
||||
code: 0,
|
||||
signal: null,
|
||||
killed: false,
|
||||
termination: "exit" as const,
|
||||
})),
|
||||
syncWorkspace: vi.fn(async () => {
|
||||
fail("sync");
|
||||
return {
|
||||
mode: "git" as const,
|
||||
remoteWorkspaceDir: "/worker/workspace",
|
||||
manifestRef: MANIFEST_REF,
|
||||
};
|
||||
}),
|
||||
stop: vi.fn(async () => {}),
|
||||
});
|
||||
const credential: MintedWorkerCredential = {
|
||||
credential: ["worker", "credential", "fixture"].join("-"),
|
||||
deliveryId: "c".repeat(43),
|
||||
environmentId: ready.environmentId,
|
||||
bundleHash: BUNDLE_HASH,
|
||||
sessionId: REQUEST.sessionId,
|
||||
rpcSetVersion: 1,
|
||||
ownerEpoch: 2,
|
||||
expiresAtMs: 10_000,
|
||||
};
|
||||
const environments: WorkerDispatchEnvironmentService = {
|
||||
create: vi.fn(async () => {
|
||||
fail("create");
|
||||
return ready;
|
||||
}),
|
||||
get: vi.fn(() => currentEnvironment),
|
||||
attachSession: vi.fn(async () => {
|
||||
fail("attach");
|
||||
currentEnvironment = attached;
|
||||
return credential;
|
||||
}),
|
||||
startTunnel: vi.fn(async ({ ownerEpoch }) => {
|
||||
fail(ownerEpoch === 1 ? "tunnel:ready" : "tunnel:attached");
|
||||
return tunnelHandle(ownerEpoch);
|
||||
}),
|
||||
stopTunnel: vi.fn(async () => {
|
||||
log.push("teardown:stop");
|
||||
}),
|
||||
destroy: vi.fn(async () => {
|
||||
log.push("teardown:destroy");
|
||||
if (options.destroyFails) {
|
||||
throw new Error("destroy pending");
|
||||
}
|
||||
const destroyed = destroyedEnvironment((currentEnvironment?.ownerEpoch ?? 1) + 1);
|
||||
currentEnvironment = destroyed;
|
||||
return destroyed;
|
||||
}),
|
||||
reconcileOnce: vi.fn(async () => {
|
||||
log.push("environment:reconcile");
|
||||
}),
|
||||
};
|
||||
const service = createWorkerPlacementDispatchService({
|
||||
placements,
|
||||
environments,
|
||||
runLocalBarrier: async ({ startDispatch }) => {
|
||||
log.push("barrier");
|
||||
const placement = startDispatch();
|
||||
if (options.failAt === "barrier") {
|
||||
throw new Error("barrier failed");
|
||||
}
|
||||
return placement;
|
||||
},
|
||||
runActivationBarrier: async ({ activate }) => {
|
||||
fail("activation");
|
||||
return activate();
|
||||
},
|
||||
resolveWorkspacePath: async () => {
|
||||
fail("workspace");
|
||||
return "/gateway/workspace";
|
||||
},
|
||||
});
|
||||
return {
|
||||
log,
|
||||
placements: {
|
||||
current: () => placementStore.get(REQUEST.sessionId),
|
||||
seedStarting: () => seedStartingPlacement(placementStore, environmentId),
|
||||
seedActive: (ownerEpoch: number) =>
|
||||
seedActivePlacement(placementStore, { environmentId, ownerEpoch }),
|
||||
seedDraining: (ownerEpoch: number) => {
|
||||
const active = seedActivePlacement(placementStore, { environmentId, ownerEpoch });
|
||||
if (active.state !== "active") {
|
||||
throw new Error("active placement fixture was not active");
|
||||
}
|
||||
return placementStore.startDrain({
|
||||
sessionId: active.sessionId,
|
||||
environmentId: active.environmentId,
|
||||
ownerEpoch: active.activeOwnerEpoch,
|
||||
expectedGeneration: active.generation,
|
||||
});
|
||||
},
|
||||
},
|
||||
environments,
|
||||
markEnvironmentDestroyed: () => {
|
||||
currentEnvironment = destroyedEnvironment((currentEnvironment?.ownerEpoch ?? 1) + 1);
|
||||
},
|
||||
markEnvironmentOwnerEpoch: (ownerEpoch: number) => {
|
||||
currentEnvironment = { ...attached, ownerEpoch };
|
||||
},
|
||||
service,
|
||||
ready,
|
||||
attached,
|
||||
};
|
||||
}
|
||||
|
||||
describe("worker placement dispatch", () => {
|
||||
let root: string;
|
||||
let database: OpenClawStateDatabase;
|
||||
let placementStore: PlacementStore;
|
||||
|
||||
beforeEach(async () => {
|
||||
root = await fs.mkdtemp(path.join(await fs.realpath(os.tmpdir()), "openclaw-dispatch-"));
|
||||
database = openOpenClawStateDatabase({ env: { OPENCLAW_STATE_DIR: root } });
|
||||
placementStore = createWorkerSessionPlacementStore({ database, now: () => 1_000 });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("orders the migration barrier, provisioning, sync, attachment, and activation", async () => {
|
||||
const harness = createHarness(placementStore);
|
||||
|
||||
await expect(harness.service.dispatch(REQUEST)).resolves.toMatchObject({
|
||||
state: "active",
|
||||
environmentId: harness.ready.environmentId,
|
||||
activeOwnerEpoch: 2,
|
||||
workspaceBaseManifestRef: MANIFEST_REF,
|
||||
remoteWorkspaceDir: "/worker/workspace",
|
||||
workerBundleHash: BUNDLE_HASH,
|
||||
});
|
||||
|
||||
expect(harness.log).toEqual([
|
||||
"barrier",
|
||||
"placement:requested",
|
||||
"workspace",
|
||||
"placement:provisioning",
|
||||
"create",
|
||||
"placement:syncing",
|
||||
"tunnel:ready",
|
||||
"sync",
|
||||
"placement:starting",
|
||||
"attach",
|
||||
"tunnel:attached",
|
||||
"activation",
|
||||
"placement:active",
|
||||
]);
|
||||
});
|
||||
|
||||
it.each<DispatchStage>([
|
||||
"barrier",
|
||||
"workspace",
|
||||
"create",
|
||||
"tunnel:ready",
|
||||
"sync",
|
||||
"attach",
|
||||
"tunnel:attached",
|
||||
"activation",
|
||||
])("fails closed and tears down acquired resources when %s fails", async (failAt) => {
|
||||
const harness = createHarness(placementStore, { failAt });
|
||||
|
||||
await expect(harness.service.dispatch(REQUEST)).rejects.toThrow(`${failAt} failed`);
|
||||
|
||||
expect(harness.placements.current()).toMatchObject({
|
||||
state: "failed",
|
||||
recoveryError: `${failAt} failed`,
|
||||
});
|
||||
const failedAt = harness.log.indexOf("placement:failed");
|
||||
expect(failedAt).toBeGreaterThan(-1);
|
||||
const environmentAcquired = !["barrier", "workspace"].includes(failAt);
|
||||
expect(harness.log.includes("teardown:stop")).toBe(environmentAcquired);
|
||||
expect(harness.log.includes("teardown:destroy")).toBe(environmentAcquired);
|
||||
if (environmentAcquired) {
|
||||
expect(failedAt).toBeGreaterThan(harness.log.indexOf("teardown:destroy"));
|
||||
}
|
||||
});
|
||||
|
||||
it("does not fail or tear down a dispatch owned by another invocation", async () => {
|
||||
placementStore.startDispatch(REQUEST);
|
||||
const harness = createHarness(placementStore);
|
||||
|
||||
await expect(harness.service.dispatch(REQUEST)).rejects.toThrow(
|
||||
"Cannot dispatch session session-1 from placement requested",
|
||||
);
|
||||
|
||||
expect(harness.placements.current()).toMatchObject({ state: "requested" });
|
||||
expect(harness.log).not.toContain("placement:failed");
|
||||
expect(harness.log).not.toContain("teardown:destroy");
|
||||
});
|
||||
|
||||
it("persists pending teardown evidence after placement is fenced", async () => {
|
||||
const harness = createHarness(placementStore, { failAt: "sync", destroyFails: true });
|
||||
|
||||
await expect(harness.service.dispatch(REQUEST)).rejects.toThrow("sync failed");
|
||||
|
||||
expect(harness.placements.current()).toMatchObject({
|
||||
state: "failed",
|
||||
recoveryError: expect.stringContaining("environment destroy: destroy pending"),
|
||||
});
|
||||
expect(harness.log.filter((entry) => entry === "placement:failed")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("adopts an exact active environment after restart without reprovisioning", async () => {
|
||||
const harness = createHarness(placementStore);
|
||||
await harness.environments.attachSession({
|
||||
environmentId: harness.ready.environmentId,
|
||||
ownerEpoch: harness.ready.ownerEpoch,
|
||||
sessionId: REQUEST.sessionId,
|
||||
});
|
||||
harness.placements.seedActive(harness.attached.ownerEpoch);
|
||||
harness.log.length = 0;
|
||||
|
||||
await harness.service.reconcile();
|
||||
|
||||
expect(harness.log).toEqual(["environment:reconcile", "tunnel:attached", "placement:adopted"]);
|
||||
expect(harness.environments.create).not.toHaveBeenCalled();
|
||||
expect(harness.environments.destroy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reclaims an active placement whose environment is already terminal after restart", async () => {
|
||||
const harness = createHarness(placementStore);
|
||||
harness.placements.seedActive(harness.attached.ownerEpoch);
|
||||
harness.markEnvironmentDestroyed();
|
||||
harness.log.length = 0;
|
||||
|
||||
await harness.service.reconcile();
|
||||
|
||||
expect(harness.placements.current()).toMatchObject({
|
||||
state: "reclaimed",
|
||||
environmentId: harness.ready.environmentId,
|
||||
activeOwnerEpoch: harness.attached.ownerEpoch,
|
||||
});
|
||||
expect(harness.log).toEqual([
|
||||
"environment:reconcile",
|
||||
"placement:draining",
|
||||
"placement:reconciling",
|
||||
"placement:reclaimed",
|
||||
]);
|
||||
expect(harness.environments.startTunnel).not.toHaveBeenCalled();
|
||||
expect(harness.environments.destroy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("fails closed when an active worker turn claim cannot be proven live after restart", async () => {
|
||||
const harness = createHarness(placementStore);
|
||||
await harness.environments.attachSession({
|
||||
environmentId: harness.ready.environmentId,
|
||||
ownerEpoch: harness.ready.ownerEpoch,
|
||||
sessionId: REQUEST.sessionId,
|
||||
});
|
||||
harness.placements.seedActive(harness.attached.ownerEpoch);
|
||||
placementStore.claimTurn({
|
||||
...REQUEST,
|
||||
claimId: "claim-1",
|
||||
runId: "run-1",
|
||||
owner: {
|
||||
kind: "worker",
|
||||
environmentId: harness.attached.environmentId,
|
||||
ownerEpoch: harness.attached.ownerEpoch,
|
||||
},
|
||||
});
|
||||
harness.log.length = 0;
|
||||
|
||||
await harness.service.reconcile();
|
||||
|
||||
expect(harness.placements.current()).toMatchObject({
|
||||
state: "failed",
|
||||
turnClaim: null,
|
||||
recoveryError: "Active worker turn claim cannot be proven live after gateway restart",
|
||||
});
|
||||
expect(harness.log).toEqual([
|
||||
"environment:reconcile",
|
||||
"placement:draining",
|
||||
"teardown:stop",
|
||||
"teardown:destroy",
|
||||
"placement:reconciling",
|
||||
"placement:failed",
|
||||
]);
|
||||
expect(harness.environments.startTunnel).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("resumes a synced starting placement after restart", async () => {
|
||||
const harness = createHarness(placementStore);
|
||||
harness.placements.seedStarting();
|
||||
harness.log.length = 0;
|
||||
|
||||
await harness.service.reconcile();
|
||||
|
||||
expect(harness.placements.current()).toMatchObject({
|
||||
state: "active",
|
||||
environmentId: harness.ready.environmentId,
|
||||
activeOwnerEpoch: harness.attached.ownerEpoch,
|
||||
});
|
||||
expect(harness.log).toEqual([
|
||||
"environment:reconcile",
|
||||
"attach",
|
||||
"tunnel:attached",
|
||||
"activation",
|
||||
"placement:active",
|
||||
]);
|
||||
expect(harness.environments.create).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("finishes an interrupted drain through reconciliation before failure", async () => {
|
||||
const harness = createHarness(placementStore);
|
||||
harness.placements.seedDraining(harness.attached.ownerEpoch);
|
||||
harness.log.length = 0;
|
||||
|
||||
await harness.service.reconcile();
|
||||
|
||||
expect(harness.placements.current()).toMatchObject({
|
||||
state: "failed",
|
||||
turnClaim: null,
|
||||
recoveryError: "Worker dispatch interrupted in draining",
|
||||
});
|
||||
expect(harness.log).toEqual([
|
||||
"environment:reconcile",
|
||||
"teardown:stop",
|
||||
"teardown:destroy",
|
||||
"placement:reconciling",
|
||||
"placement:failed",
|
||||
]);
|
||||
});
|
||||
|
||||
it("drains, tears down, and reclaims an idle active placement with a mismatched owner", async () => {
|
||||
const harness = createHarness(placementStore);
|
||||
harness.placements.seedActive(99);
|
||||
|
||||
await harness.service.reconcile();
|
||||
|
||||
expect(harness.placements.current()).toMatchObject({
|
||||
state: "reclaimed",
|
||||
});
|
||||
expect(harness.log).toEqual([
|
||||
"environment:reconcile",
|
||||
"placement:draining",
|
||||
"teardown:stop",
|
||||
"teardown:destroy",
|
||||
"placement:reconciling",
|
||||
"placement:reclaimed",
|
||||
]);
|
||||
|
||||
const destroyCalls = vi.mocked(harness.environments.destroy).mock.calls.length;
|
||||
await harness.service.reconcile();
|
||||
expect(harness.environments.destroy).toHaveBeenCalledTimes(destroyCalls);
|
||||
});
|
||||
|
||||
it("preserves a live active turn claim during runtime reconciliation", async () => {
|
||||
const harness = createHarness(placementStore);
|
||||
await harness.environments.attachSession({
|
||||
environmentId: harness.ready.environmentId,
|
||||
ownerEpoch: harness.ready.ownerEpoch,
|
||||
sessionId: REQUEST.sessionId,
|
||||
});
|
||||
harness.placements.seedActive(harness.attached.ownerEpoch);
|
||||
placementStore.claimTurn({
|
||||
...REQUEST,
|
||||
claimId: "claim-1",
|
||||
runId: "run-1",
|
||||
owner: {
|
||||
kind: "worker",
|
||||
environmentId: harness.attached.environmentId,
|
||||
ownerEpoch: harness.attached.ownerEpoch,
|
||||
},
|
||||
});
|
||||
harness.log.length = 0;
|
||||
|
||||
await harness.service.reconcileActive();
|
||||
|
||||
expect(harness.placements.current()).toMatchObject({
|
||||
state: "active",
|
||||
turnClaim: {
|
||||
claimId: "claim-1",
|
||||
runId: "run-1",
|
||||
owner: "worker",
|
||||
},
|
||||
});
|
||||
expect(harness.log).toEqual(["environment:reconcile"]);
|
||||
expect(harness.environments.startTunnel).not.toHaveBeenCalled();
|
||||
expect(harness.environments.destroy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("fences a live turn before tearing down a mismatched runtime owner", async () => {
|
||||
const harness = createHarness(placementStore);
|
||||
await harness.environments.attachSession({
|
||||
environmentId: harness.ready.environmentId,
|
||||
ownerEpoch: harness.ready.ownerEpoch,
|
||||
sessionId: REQUEST.sessionId,
|
||||
});
|
||||
harness.placements.seedActive(harness.attached.ownerEpoch);
|
||||
placementStore.claimTurn({
|
||||
...REQUEST,
|
||||
claimId: "claim-1",
|
||||
runId: "run-1",
|
||||
owner: {
|
||||
kind: "worker",
|
||||
environmentId: harness.attached.environmentId,
|
||||
ownerEpoch: harness.attached.ownerEpoch,
|
||||
},
|
||||
});
|
||||
harness.markEnvironmentOwnerEpoch(harness.attached.ownerEpoch + 1);
|
||||
harness.log.length = 0;
|
||||
|
||||
await harness.service.reconcileActive();
|
||||
|
||||
expect(harness.placements.current()).toMatchObject({
|
||||
state: "failed",
|
||||
turnClaim: null,
|
||||
recoveryError: "Active worker placement does not match its environment owner",
|
||||
});
|
||||
expect(harness.log).toEqual([
|
||||
"environment:reconcile",
|
||||
"placement:draining",
|
||||
"teardown:stop",
|
||||
"teardown:destroy",
|
||||
"placement:reconciling",
|
||||
"placement:failed",
|
||||
]);
|
||||
});
|
||||
|
||||
it("reclaims a terminal active environment during runtime reconciliation", async () => {
|
||||
const harness = createHarness(placementStore);
|
||||
harness.placements.seedActive(harness.attached.ownerEpoch);
|
||||
harness.markEnvironmentDestroyed();
|
||||
harness.log.length = 0;
|
||||
|
||||
await harness.service.reconcileActive();
|
||||
|
||||
expect(harness.placements.current()).toMatchObject({ state: "reclaimed" });
|
||||
expect(harness.log).toEqual([
|
||||
"environment:reconcile",
|
||||
"placement:draining",
|
||||
"placement:reconciling",
|
||||
"placement:reclaimed",
|
||||
]);
|
||||
expect(harness.environments.destroy).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("fences a turn admitted immediately before runtime drain", async () => {
|
||||
const harness = createHarness(placementStore, { claimOnDrain: true });
|
||||
harness.placements.seedActive(harness.attached.ownerEpoch);
|
||||
harness.markEnvironmentDestroyed();
|
||||
harness.log.length = 0;
|
||||
|
||||
await harness.service.reconcileActive();
|
||||
|
||||
expect(harness.placements.current()).toMatchObject({
|
||||
state: "failed",
|
||||
turnClaim: null,
|
||||
recoveryError: "Active worker disappeared during an admitted turn",
|
||||
});
|
||||
expect(harness.log).toEqual([
|
||||
"environment:reconcile",
|
||||
"placement:draining",
|
||||
"teardown:stop",
|
||||
"teardown:destroy",
|
||||
"placement:reconciling",
|
||||
"placement:failed",
|
||||
]);
|
||||
});
|
||||
|
||||
it("leaves in-flight dispatch preparation untouched during runtime reconciliation", async () => {
|
||||
const harness = createHarness(placementStore);
|
||||
harness.placements.seedStarting();
|
||||
harness.log.length = 0;
|
||||
|
||||
await harness.service.reconcileActive();
|
||||
|
||||
expect(harness.placements.current()).toMatchObject({ state: "starting" });
|
||||
expect(harness.log).toEqual(["environment:reconcile"]);
|
||||
expect(harness.environments.attachSession).not.toHaveBeenCalled();
|
||||
expect(harness.environments.destroy).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,178 @@
|
||||
import {
|
||||
createPlacementFailureActions,
|
||||
type WorkerActivationBarrier,
|
||||
type WorkerActiveDispatchPlacement,
|
||||
type WorkerDispatchEnvironmentService,
|
||||
type WorkerDispatchPlacement,
|
||||
type WorkerDispatchPlacementStore,
|
||||
} from "./placement-dispatch-failure.js";
|
||||
import { createPlacementRecoveryActions } from "./placement-dispatch-recovery.js";
|
||||
import type { WorkerPlacementDispatchRequest } from "./service-contract.js";
|
||||
import { type WorkerEnvironmentService, workerEnvironmentIdForIdempotencyKey } from "./service.js";
|
||||
|
||||
type WorkerLocalDispatchBarrier = (params: {
|
||||
sessionId: string;
|
||||
sessionKey: string;
|
||||
agentId: string;
|
||||
startDispatch: () => WorkerDispatchPlacement;
|
||||
}) => Promise<WorkerDispatchPlacement>;
|
||||
|
||||
type WorkerPlacementDispatchOptions = {
|
||||
placements: WorkerDispatchPlacementStore;
|
||||
environments: WorkerDispatchEnvironmentService;
|
||||
runLocalBarrier: WorkerLocalDispatchBarrier;
|
||||
runActivationBarrier: WorkerActivationBarrier;
|
||||
resolveWorkspacePath: (params: {
|
||||
sessionId: string;
|
||||
sessionKey: string;
|
||||
agentId: string;
|
||||
}) => Promise<string>;
|
||||
};
|
||||
|
||||
function requireProvisionedEnvironment(
|
||||
environment: Awaited<ReturnType<WorkerEnvironmentService["create"]>>,
|
||||
expectedEnvironmentId: string,
|
||||
): { environmentId: string; ownerEpoch: number; bundleHash: string } {
|
||||
if (
|
||||
(environment.state !== "ready" && environment.state !== "idle") ||
|
||||
!environment.bootstrapReceipt ||
|
||||
environment.environmentId !== expectedEnvironmentId
|
||||
) {
|
||||
throw new Error(`Worker environment is not dispatchable: ${environment.state}`);
|
||||
}
|
||||
return {
|
||||
environmentId: environment.environmentId,
|
||||
ownerEpoch: environment.ownerEpoch,
|
||||
bundleHash: environment.bootstrapReceipt.bundleHash,
|
||||
};
|
||||
}
|
||||
|
||||
export function createWorkerPlacementDispatchService(options: WorkerPlacementDispatchOptions) {
|
||||
const { environments, placements } = options;
|
||||
const failure = createPlacementFailureActions({ environments, placements });
|
||||
const recovery = createPlacementRecoveryActions({
|
||||
environments,
|
||||
failure,
|
||||
placements,
|
||||
runActivationBarrier: options.runActivationBarrier,
|
||||
});
|
||||
|
||||
const dispatch = async (
|
||||
request: WorkerPlacementDispatchRequest,
|
||||
): Promise<WorkerActiveDispatchPlacement> => {
|
||||
let placement: WorkerDispatchPlacement | undefined;
|
||||
let environmentId: string | null = null;
|
||||
let ownerEpoch: number | null = null;
|
||||
try {
|
||||
placement = await options.runLocalBarrier({
|
||||
sessionId: request.sessionId,
|
||||
sessionKey: request.sessionKey,
|
||||
agentId: request.agentId,
|
||||
startDispatch: () => {
|
||||
placement = placements.startDispatch({
|
||||
sessionId: request.sessionId,
|
||||
sessionKey: request.sessionKey,
|
||||
agentId: request.agentId,
|
||||
});
|
||||
return placement;
|
||||
},
|
||||
});
|
||||
const localPath = await options.resolveWorkspacePath(request);
|
||||
const idempotencyKey = `session-dispatch:${request.sessionId}:${placement.generation}`;
|
||||
const expectedEnvironmentId = workerEnvironmentIdForIdempotencyKey(idempotencyKey);
|
||||
placement = placements.transition({
|
||||
sessionId: request.sessionId,
|
||||
from: "requested",
|
||||
to: "provisioning",
|
||||
expectedGeneration: placement.generation,
|
||||
patch: { environmentId: expectedEnvironmentId },
|
||||
});
|
||||
const environment = await environments.create(request.profileId, idempotencyKey);
|
||||
const provisioned = requireProvisionedEnvironment(environment, expectedEnvironmentId);
|
||||
environmentId = provisioned.environmentId;
|
||||
ownerEpoch = provisioned.ownerEpoch;
|
||||
placement = placements.transition({
|
||||
sessionId: request.sessionId,
|
||||
from: "provisioning",
|
||||
to: "syncing",
|
||||
expectedGeneration: placement.generation,
|
||||
patch: {
|
||||
environmentId,
|
||||
workerBundleHash: provisioned.bundleHash,
|
||||
},
|
||||
});
|
||||
const readyTunnel = await environments.startTunnel({ environmentId, ownerEpoch });
|
||||
const synced = await readyTunnel.syncWorkspace({
|
||||
localPath,
|
||||
sessionId: request.sessionId,
|
||||
generation: placement.generation,
|
||||
});
|
||||
placement = placements.transition({
|
||||
sessionId: request.sessionId,
|
||||
from: "syncing",
|
||||
to: "starting",
|
||||
expectedGeneration: placement.generation,
|
||||
patch: {
|
||||
workspaceBaseManifestRef: synced.manifestRef,
|
||||
remoteWorkspaceDir: synced.remoteWorkspaceDir,
|
||||
},
|
||||
});
|
||||
const credential = await environments.attachSession({
|
||||
environmentId,
|
||||
ownerEpoch,
|
||||
sessionId: request.sessionId,
|
||||
});
|
||||
ownerEpoch = credential.ownerEpoch;
|
||||
await environments.startTunnel({ environmentId, ownerEpoch });
|
||||
const startingPlacement = placement;
|
||||
const activePlacement = await options.runActivationBarrier({
|
||||
sessionId: request.sessionId,
|
||||
sessionKey: request.sessionKey,
|
||||
agentId: request.agentId,
|
||||
activate: () => {
|
||||
const activated = placements.transition({
|
||||
sessionId: request.sessionId,
|
||||
from: "starting",
|
||||
to: "active",
|
||||
expectedGeneration: startingPlacement.generation,
|
||||
patch: { activeOwnerEpoch: ownerEpoch },
|
||||
});
|
||||
if (activated.state !== "active") {
|
||||
throw new Error("Worker dispatch activation did not produce an active placement");
|
||||
}
|
||||
return activated;
|
||||
},
|
||||
});
|
||||
return activePlacement;
|
||||
} catch (error) {
|
||||
const current = placement ? placements.get(request.sessionId) : undefined;
|
||||
if (current && current.state !== "local" && current.state !== "reclaimed") {
|
||||
if (current.state === "active") {
|
||||
await failure.failActive(current, error);
|
||||
} else {
|
||||
const currentEnvironmentId = environmentId ?? current.environmentId;
|
||||
const currentEnvironment = currentEnvironmentId
|
||||
? environments.get(currentEnvironmentId)
|
||||
: undefined;
|
||||
await failure.teardownEnvironment({
|
||||
placement: current,
|
||||
environmentId: currentEnvironment?.environmentId ?? null,
|
||||
ownerEpoch: ownerEpoch ?? currentEnvironment?.ownerEpoch ?? null,
|
||||
primaryError: error,
|
||||
});
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
dispatch,
|
||||
reconcile: recovery.reconcile,
|
||||
reconcileActive: recovery.reconcileActive,
|
||||
};
|
||||
}
|
||||
|
||||
export type WorkerPlacementDispatchService = ReturnType<
|
||||
typeof createWorkerPlacementDispatchService
|
||||
>;
|
||||
@@ -0,0 +1,107 @@
|
||||
import { Value } from "typebox/value";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { SessionPlacementSchema } from "../../../packages/gateway-protocol/src/index.js";
|
||||
import { projectWorkerSessionPlacement } from "./placement-projector.js";
|
||||
import type { WorkerSessionPlacementRecord } from "./placement-store.js";
|
||||
|
||||
const BUNDLE_HASH = "a".repeat(64);
|
||||
|
||||
const RECORD_BASE = {
|
||||
sessionId: "session-1",
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:session-1",
|
||||
generation: 4,
|
||||
workspaceBaseManifestRef: null,
|
||||
remoteWorkspaceDir: null,
|
||||
workerBundleHash: null,
|
||||
lastTranscriptAckCursor: null,
|
||||
lastLiveEventAckCursor: null,
|
||||
recoveryError: null,
|
||||
turnClaim: null,
|
||||
createdAtMs: 100,
|
||||
updatedAtMs: 200,
|
||||
stateChangedAtMs: 150,
|
||||
};
|
||||
|
||||
describe("worker placement projection", () => {
|
||||
it("emits only fields valid for each placement discriminator", () => {
|
||||
const records = [
|
||||
{
|
||||
...RECORD_BASE,
|
||||
state: "local",
|
||||
environmentId: null,
|
||||
activeOwnerEpoch: null,
|
||||
},
|
||||
{
|
||||
...RECORD_BASE,
|
||||
state: "provisioning",
|
||||
environmentId: "environment-1",
|
||||
activeOwnerEpoch: null,
|
||||
},
|
||||
{
|
||||
...RECORD_BASE,
|
||||
state: "reclaimed",
|
||||
environmentId: "environment-1",
|
||||
activeOwnerEpoch: 7,
|
||||
workspaceBaseManifestRef: "manifest-1",
|
||||
remoteWorkspaceDir: "/workspace",
|
||||
workerBundleHash: BUNDLE_HASH,
|
||||
},
|
||||
{
|
||||
...RECORD_BASE,
|
||||
state: "failed",
|
||||
environmentId: "environment-1",
|
||||
activeOwnerEpoch: 7,
|
||||
recoveryError: "worker unavailable",
|
||||
},
|
||||
] satisfies WorkerSessionPlacementRecord[];
|
||||
|
||||
const projected = records.map(projectWorkerSessionPlacement);
|
||||
|
||||
expect(projected).toEqual([
|
||||
{
|
||||
state: "local",
|
||||
generation: 4,
|
||||
createdAtMs: 100,
|
||||
updatedAtMs: 200,
|
||||
stateChangedAtMs: 150,
|
||||
},
|
||||
{
|
||||
state: "provisioning",
|
||||
generation: 4,
|
||||
createdAtMs: 100,
|
||||
updatedAtMs: 200,
|
||||
stateChangedAtMs: 150,
|
||||
environmentId: "environment-1",
|
||||
},
|
||||
{
|
||||
state: "reclaimed",
|
||||
generation: 4,
|
||||
createdAtMs: 100,
|
||||
updatedAtMs: 200,
|
||||
stateChangedAtMs: 150,
|
||||
environmentId: "environment-1",
|
||||
activeOwnerEpoch: 7,
|
||||
workspaceBaseManifestRef: "manifest-1",
|
||||
remoteWorkspaceDir: "/workspace",
|
||||
workerBundleHash: BUNDLE_HASH,
|
||||
},
|
||||
{
|
||||
state: "failed",
|
||||
generation: 4,
|
||||
createdAtMs: 100,
|
||||
updatedAtMs: 200,
|
||||
stateChangedAtMs: 150,
|
||||
environmentId: "environment-1",
|
||||
activeOwnerEpoch: 7,
|
||||
recoveryError: "worker unavailable",
|
||||
},
|
||||
]);
|
||||
for (const placement of projected) {
|
||||
expect(Value.Check(SessionPlacementSchema, placement)).toBe(true);
|
||||
expect(placement).not.toHaveProperty("sessionId");
|
||||
expect(placement).not.toHaveProperty("sessionKey");
|
||||
expect(placement).not.toHaveProperty("turnClaim");
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,133 @@
|
||||
import type { SessionPlacement } from "../../../packages/gateway-protocol/src/index.js";
|
||||
import type { WorkerSessionPlacementRecord } from "./placement-store.js";
|
||||
|
||||
export type WorkerSessionPlacementReader = {
|
||||
getMany(sessionIds: readonly string[]): ReadonlyMap<string, WorkerSessionPlacementRecord>;
|
||||
};
|
||||
|
||||
/** Removes gateway-only identity and turn-claim fields from the operator projection. */
|
||||
export function projectWorkerSessionPlacement(
|
||||
record: WorkerSessionPlacementRecord,
|
||||
): SessionPlacement {
|
||||
const timing = {
|
||||
generation: record.generation,
|
||||
createdAtMs: record.createdAtMs,
|
||||
updatedAtMs: record.updatedAtMs,
|
||||
stateChangedAtMs: record.stateChangedAtMs,
|
||||
};
|
||||
switch (record.state) {
|
||||
case "local":
|
||||
return { state: "local", ...timing };
|
||||
case "requested":
|
||||
return { state: "requested", ...timing };
|
||||
case "provisioning":
|
||||
return {
|
||||
state: "provisioning",
|
||||
...timing,
|
||||
...(record.environmentId ? { environmentId: record.environmentId } : {}),
|
||||
};
|
||||
case "syncing":
|
||||
return {
|
||||
state: "syncing",
|
||||
...timing,
|
||||
environmentId: record.environmentId,
|
||||
workerBundleHash: record.workerBundleHash,
|
||||
};
|
||||
case "starting":
|
||||
return {
|
||||
state: "starting",
|
||||
...timing,
|
||||
environmentId: record.environmentId,
|
||||
workerBundleHash: record.workerBundleHash,
|
||||
workspaceBaseManifestRef: record.workspaceBaseManifestRef,
|
||||
remoteWorkspaceDir: record.remoteWorkspaceDir,
|
||||
};
|
||||
case "active":
|
||||
return {
|
||||
state: "active",
|
||||
...timing,
|
||||
environmentId: record.environmentId,
|
||||
activeOwnerEpoch: record.activeOwnerEpoch,
|
||||
workerBundleHash: record.workerBundleHash,
|
||||
workspaceBaseManifestRef: record.workspaceBaseManifestRef,
|
||||
remoteWorkspaceDir: record.remoteWorkspaceDir,
|
||||
...(record.lastTranscriptAckCursor !== null
|
||||
? { lastTranscriptAckCursor: record.lastTranscriptAckCursor }
|
||||
: {}),
|
||||
...(record.lastLiveEventAckCursor !== null
|
||||
? { lastLiveEventAckCursor: record.lastLiveEventAckCursor }
|
||||
: {}),
|
||||
};
|
||||
case "draining":
|
||||
return {
|
||||
state: "draining",
|
||||
...timing,
|
||||
environmentId: record.environmentId,
|
||||
activeOwnerEpoch: record.activeOwnerEpoch,
|
||||
workerBundleHash: record.workerBundleHash,
|
||||
workspaceBaseManifestRef: record.workspaceBaseManifestRef,
|
||||
remoteWorkspaceDir: record.remoteWorkspaceDir,
|
||||
...(record.lastTranscriptAckCursor !== null
|
||||
? { lastTranscriptAckCursor: record.lastTranscriptAckCursor }
|
||||
: {}),
|
||||
...(record.lastLiveEventAckCursor !== null
|
||||
? { lastLiveEventAckCursor: record.lastLiveEventAckCursor }
|
||||
: {}),
|
||||
};
|
||||
case "reconciling":
|
||||
return {
|
||||
state: "reconciling",
|
||||
...timing,
|
||||
environmentId: record.environmentId,
|
||||
activeOwnerEpoch: record.activeOwnerEpoch,
|
||||
workerBundleHash: record.workerBundleHash,
|
||||
workspaceBaseManifestRef: record.workspaceBaseManifestRef,
|
||||
remoteWorkspaceDir: record.remoteWorkspaceDir,
|
||||
...(record.lastTranscriptAckCursor !== null
|
||||
? { lastTranscriptAckCursor: record.lastTranscriptAckCursor }
|
||||
: {}),
|
||||
...(record.lastLiveEventAckCursor !== null
|
||||
? { lastLiveEventAckCursor: record.lastLiveEventAckCursor }
|
||||
: {}),
|
||||
};
|
||||
case "reclaimed":
|
||||
return {
|
||||
state: "reclaimed",
|
||||
...timing,
|
||||
...(record.environmentId ? { environmentId: record.environmentId } : {}),
|
||||
...(record.activeOwnerEpoch !== null ? { activeOwnerEpoch: record.activeOwnerEpoch } : {}),
|
||||
...(record.workspaceBaseManifestRef
|
||||
? { workspaceBaseManifestRef: record.workspaceBaseManifestRef }
|
||||
: {}),
|
||||
...(record.remoteWorkspaceDir ? { remoteWorkspaceDir: record.remoteWorkspaceDir } : {}),
|
||||
...(record.workerBundleHash ? { workerBundleHash: record.workerBundleHash } : {}),
|
||||
...(record.lastTranscriptAckCursor !== null
|
||||
? { lastTranscriptAckCursor: record.lastTranscriptAckCursor }
|
||||
: {}),
|
||||
...(record.lastLiveEventAckCursor !== null
|
||||
? { lastLiveEventAckCursor: record.lastLiveEventAckCursor }
|
||||
: {}),
|
||||
};
|
||||
case "failed":
|
||||
return {
|
||||
state: "failed",
|
||||
...timing,
|
||||
...(record.environmentId ? { environmentId: record.environmentId } : {}),
|
||||
...(record.activeOwnerEpoch !== null ? { activeOwnerEpoch: record.activeOwnerEpoch } : {}),
|
||||
...(record.workspaceBaseManifestRef
|
||||
? { workspaceBaseManifestRef: record.workspaceBaseManifestRef }
|
||||
: {}),
|
||||
...(record.remoteWorkspaceDir ? { remoteWorkspaceDir: record.remoteWorkspaceDir } : {}),
|
||||
...(record.workerBundleHash ? { workerBundleHash: record.workerBundleHash } : {}),
|
||||
...(record.lastTranscriptAckCursor !== null
|
||||
? { lastTranscriptAckCursor: record.lastTranscriptAckCursor }
|
||||
: {}),
|
||||
...(record.lastLiveEventAckCursor !== null
|
||||
? { lastLiveEventAckCursor: record.lastLiveEventAckCursor }
|
||||
: {}),
|
||||
recoveryError: record.recoveryError,
|
||||
};
|
||||
}
|
||||
// Exhaustive over placement states; the return satisfies consistent-return.
|
||||
return record satisfies never;
|
||||
}
|
||||
@@ -0,0 +1,371 @@
|
||||
import type { WorkerSessionPlacementState } from "./placement-state.js";
|
||||
|
||||
export type WorkerSessionPlacementIdentity = {
|
||||
sessionId: string;
|
||||
agentId: string;
|
||||
sessionKey: string;
|
||||
};
|
||||
|
||||
export type WorkerSessionTurnOwner =
|
||||
| { kind: "local" }
|
||||
| { kind: "worker"; environmentId: string; ownerEpoch: number };
|
||||
|
||||
export type WorkerSessionTurnClaim = {
|
||||
sessionId: string;
|
||||
claimId: string;
|
||||
runId: string;
|
||||
placementGeneration: number;
|
||||
owner: WorkerSessionTurnOwner;
|
||||
};
|
||||
|
||||
export type PersistedTurnClaim =
|
||||
| {
|
||||
owner: "local";
|
||||
claimId: string;
|
||||
runId: string;
|
||||
generation: number;
|
||||
ownerEpoch: null;
|
||||
}
|
||||
| {
|
||||
owner: "worker";
|
||||
claimId: string;
|
||||
runId: string;
|
||||
generation: number;
|
||||
ownerEpoch: number;
|
||||
};
|
||||
|
||||
type PersistedLocalTurnClaim = Extract<PersistedTurnClaim, { owner: "local" }>;
|
||||
type PersistedWorkerTurnClaim = Extract<PersistedTurnClaim, { owner: "worker" }>;
|
||||
|
||||
type PlacementRecordBase<TurnClaim extends PersistedTurnClaim | null> =
|
||||
WorkerSessionPlacementIdentity & {
|
||||
generation: number;
|
||||
turnClaim: TurnClaim;
|
||||
createdAtMs: number;
|
||||
updatedAtMs: number;
|
||||
stateChangedAtMs: number;
|
||||
};
|
||||
|
||||
type UnclaimedPlacementRecordBase = PlacementRecordBase<null>;
|
||||
type LocalClaimablePlacementRecordBase = PlacementRecordBase<PersistedLocalTurnClaim | null>;
|
||||
type WorkerClaimablePlacementRecordBase = PlacementRecordBase<PersistedWorkerTurnClaim | null>;
|
||||
|
||||
export type EmptyWorkerPlacementMetadata = {
|
||||
environmentId: null;
|
||||
activeOwnerEpoch: null;
|
||||
workspaceBaseManifestRef: null;
|
||||
remoteWorkspaceDir: null;
|
||||
workerBundleHash: null;
|
||||
lastTranscriptAckCursor: null;
|
||||
lastLiveEventAckCursor: null;
|
||||
recoveryError: null;
|
||||
};
|
||||
|
||||
type ProvisioningPlacementMetadata = {
|
||||
environmentId: string | null;
|
||||
activeOwnerEpoch: null;
|
||||
workspaceBaseManifestRef: null;
|
||||
remoteWorkspaceDir: null;
|
||||
workerBundleHash: null;
|
||||
lastTranscriptAckCursor: null;
|
||||
lastLiveEventAckCursor: null;
|
||||
recoveryError: null;
|
||||
};
|
||||
|
||||
type SyncingPlacementMetadata = {
|
||||
environmentId: string;
|
||||
activeOwnerEpoch: null;
|
||||
workspaceBaseManifestRef: null;
|
||||
remoteWorkspaceDir: null;
|
||||
workerBundleHash: string;
|
||||
lastTranscriptAckCursor: null;
|
||||
lastLiveEventAckCursor: null;
|
||||
recoveryError: null;
|
||||
};
|
||||
|
||||
type StartingPlacementMetadata = {
|
||||
environmentId: string;
|
||||
activeOwnerEpoch: null;
|
||||
workspaceBaseManifestRef: string;
|
||||
remoteWorkspaceDir: string;
|
||||
workerBundleHash: string;
|
||||
lastTranscriptAckCursor: null;
|
||||
lastLiveEventAckCursor: null;
|
||||
recoveryError: null;
|
||||
};
|
||||
|
||||
export type OwnedWorkerPlacementMetadata = {
|
||||
environmentId: string;
|
||||
activeOwnerEpoch: number;
|
||||
workspaceBaseManifestRef: string;
|
||||
remoteWorkspaceDir: string;
|
||||
workerBundleHash: string;
|
||||
lastTranscriptAckCursor: number | null;
|
||||
lastLiveEventAckCursor: number | null;
|
||||
recoveryError: null;
|
||||
};
|
||||
|
||||
type TerminalPlacementMetadata = {
|
||||
environmentId: string | null;
|
||||
activeOwnerEpoch: number | null;
|
||||
workspaceBaseManifestRef: string | null;
|
||||
remoteWorkspaceDir: string | null;
|
||||
workerBundleHash: string | null;
|
||||
lastTranscriptAckCursor: number | null;
|
||||
lastLiveEventAckCursor: number | null;
|
||||
};
|
||||
|
||||
type LocalPlacementRecord = LocalClaimablePlacementRecordBase &
|
||||
EmptyWorkerPlacementMetadata & {
|
||||
state: "local";
|
||||
};
|
||||
type RequestedPlacementRecord = LocalClaimablePlacementRecordBase &
|
||||
EmptyWorkerPlacementMetadata & {
|
||||
state: "requested";
|
||||
};
|
||||
type ProvisioningPlacementRecord = UnclaimedPlacementRecordBase &
|
||||
ProvisioningPlacementMetadata & {
|
||||
state: "provisioning";
|
||||
};
|
||||
type SyncingPlacementRecord = UnclaimedPlacementRecordBase &
|
||||
SyncingPlacementMetadata & {
|
||||
state: "syncing";
|
||||
};
|
||||
type StartingPlacementRecord = UnclaimedPlacementRecordBase &
|
||||
StartingPlacementMetadata & {
|
||||
state: "starting";
|
||||
};
|
||||
type ActivePlacementRecord = WorkerClaimablePlacementRecordBase &
|
||||
OwnedWorkerPlacementMetadata & {
|
||||
state: "active";
|
||||
};
|
||||
type DrainingPlacementRecord = WorkerClaimablePlacementRecordBase &
|
||||
OwnedWorkerPlacementMetadata & {
|
||||
state: "draining";
|
||||
};
|
||||
type ReconcilingPlacementRecord = UnclaimedPlacementRecordBase &
|
||||
OwnedWorkerPlacementMetadata & {
|
||||
state: "reconciling";
|
||||
};
|
||||
type ReclaimedPlacementRecord = UnclaimedPlacementRecordBase &
|
||||
OwnedWorkerPlacementMetadata & {
|
||||
state: "reclaimed";
|
||||
};
|
||||
type FailedPlacementRecord = LocalClaimablePlacementRecordBase &
|
||||
TerminalPlacementMetadata & {
|
||||
state: "failed";
|
||||
recoveryError: string;
|
||||
};
|
||||
|
||||
export type WorkerSessionPlacementRecord =
|
||||
| LocalPlacementRecord
|
||||
| RequestedPlacementRecord
|
||||
| ProvisioningPlacementRecord
|
||||
| SyncingPlacementRecord
|
||||
| StartingPlacementRecord
|
||||
| ActivePlacementRecord
|
||||
| DrainingPlacementRecord
|
||||
| ReconcilingPlacementRecord
|
||||
| ReclaimedPlacementRecord
|
||||
| FailedPlacementRecord;
|
||||
|
||||
export type WorkerSessionPlacementTransitionPatch = {
|
||||
environmentId?: string | null;
|
||||
activeOwnerEpoch?: number | null;
|
||||
workspaceBaseManifestRef?: string | null;
|
||||
remoteWorkspaceDir?: string | null;
|
||||
workerBundleHash?: string | null;
|
||||
lastTranscriptAckCursor?: number | null;
|
||||
lastLiveEventAckCursor?: number | null;
|
||||
recoveryError?: string | null;
|
||||
};
|
||||
|
||||
export function required(value: string, field: string): string {
|
||||
const normalized = value.trim();
|
||||
if (!normalized) {
|
||||
throw new Error(`Worker session placement ${field} must be a non-empty string`);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
export function nullableRequired(value: string | null, field: string): string | null {
|
||||
return value === null ? null : required(value, field);
|
||||
}
|
||||
|
||||
export function normalizeEpoch(value: number, field: string): number {
|
||||
if (!Number.isSafeInteger(value) || value < 1) {
|
||||
throw new Error(`Worker session placement ${field} must be a positive safe integer`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function normalizeCursor(value: number | null, field: string): number | null {
|
||||
if (value !== null && (!Number.isSafeInteger(value) || value < 0)) {
|
||||
throw new Error(`Worker session placement ${field} must be a non-negative safe integer`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export function advanceCursor(
|
||||
current: number | null,
|
||||
value: number | undefined,
|
||||
field: string,
|
||||
): number | null {
|
||||
if (value === undefined) {
|
||||
return current;
|
||||
}
|
||||
const next = normalizeCursor(value, field);
|
||||
if (next === null || current === null) {
|
||||
return next ?? current;
|
||||
}
|
||||
return Math.max(current, next);
|
||||
}
|
||||
|
||||
export function normalizeIdentity(
|
||||
input: WorkerSessionPlacementIdentity,
|
||||
): WorkerSessionPlacementIdentity {
|
||||
return {
|
||||
sessionId: required(input.sessionId, "session id"),
|
||||
agentId: required(input.agentId, "agent id"),
|
||||
sessionKey: required(input.sessionKey, "session key"),
|
||||
};
|
||||
}
|
||||
|
||||
export function nextGeneration(generation: number): number {
|
||||
const next = generation + 1;
|
||||
if (!Number.isSafeInteger(next)) {
|
||||
throw new Error("Worker session placement generation is exhausted");
|
||||
}
|
||||
return next;
|
||||
}
|
||||
|
||||
export function localTurnClaimForState(
|
||||
turnClaim: PersistedTurnClaim | null,
|
||||
state: "local" | "requested" | "failed",
|
||||
): PersistedLocalTurnClaim | null {
|
||||
if (turnClaim?.owner === "worker") {
|
||||
throw new Error(`Worker turn claim cannot survive placement ${state}`);
|
||||
}
|
||||
return turnClaim;
|
||||
}
|
||||
|
||||
export function workerTurnClaimForState(
|
||||
turnClaim: PersistedTurnClaim | null,
|
||||
state: "active" | "draining",
|
||||
): PersistedWorkerTurnClaim | null {
|
||||
if (turnClaim?.owner === "local") {
|
||||
throw new Error(`Local turn claim cannot survive placement ${state}`);
|
||||
}
|
||||
return turnClaim;
|
||||
}
|
||||
|
||||
export function unclaimedTurnForState(
|
||||
turnClaim: PersistedTurnClaim | null,
|
||||
state: "provisioning" | "syncing" | "starting" | "reconciling" | "reclaimed",
|
||||
): null {
|
||||
if (turnClaim !== null) {
|
||||
throw new Error(`Turn claim cannot survive placement ${state}`);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function assertRecordShape(record: {
|
||||
state: WorkerSessionPlacementState;
|
||||
environmentId: string | null;
|
||||
activeOwnerEpoch: number | null;
|
||||
workspaceBaseManifestRef: string | null;
|
||||
remoteWorkspaceDir: string | null;
|
||||
workerBundleHash: string | null;
|
||||
lastTranscriptAckCursor: number | null;
|
||||
lastLiveEventAckCursor: number | null;
|
||||
recoveryError: string | null;
|
||||
turnClaim: PersistedTurnClaim | null;
|
||||
}): void {
|
||||
if (record.state === "local" || record.state === "requested") {
|
||||
if (
|
||||
record.environmentId !== null ||
|
||||
record.activeOwnerEpoch !== null ||
|
||||
record.workspaceBaseManifestRef !== null ||
|
||||
record.remoteWorkspaceDir !== null ||
|
||||
record.workerBundleHash !== null ||
|
||||
record.lastTranscriptAckCursor !== null ||
|
||||
record.lastLiveEventAckCursor !== null ||
|
||||
record.recoveryError !== null
|
||||
) {
|
||||
throw new Error(`Worker session placement ${record.state} cannot retain worker metadata`);
|
||||
}
|
||||
} else if (record.state === "provisioning") {
|
||||
if (
|
||||
record.activeOwnerEpoch !== null ||
|
||||
record.workspaceBaseManifestRef !== null ||
|
||||
record.remoteWorkspaceDir !== null ||
|
||||
record.workerBundleHash !== null ||
|
||||
record.lastTranscriptAckCursor !== null ||
|
||||
record.lastLiveEventAckCursor !== null ||
|
||||
record.recoveryError !== null
|
||||
) {
|
||||
throw new Error("Provisioning worker session placement can only retain an environment id");
|
||||
}
|
||||
} else if (record.state === "syncing") {
|
||||
if (
|
||||
!record.environmentId ||
|
||||
record.activeOwnerEpoch !== null ||
|
||||
record.workspaceBaseManifestRef !== null ||
|
||||
record.remoteWorkspaceDir !== null ||
|
||||
!record.workerBundleHash ||
|
||||
record.lastTranscriptAckCursor !== null ||
|
||||
record.lastLiveEventAckCursor !== null ||
|
||||
record.recoveryError !== null
|
||||
) {
|
||||
throw new Error("Syncing worker session placement requires an environment and bundle");
|
||||
}
|
||||
} else if (record.state === "starting") {
|
||||
if (
|
||||
!record.environmentId ||
|
||||
record.activeOwnerEpoch !== null ||
|
||||
!record.workspaceBaseManifestRef ||
|
||||
!record.remoteWorkspaceDir ||
|
||||
!record.workerBundleHash ||
|
||||
record.lastTranscriptAckCursor !== null ||
|
||||
record.lastLiveEventAckCursor !== null ||
|
||||
record.recoveryError !== null
|
||||
) {
|
||||
throw new Error("Starting worker session placement requires complete workspace metadata");
|
||||
}
|
||||
} else if (
|
||||
record.state === "active" ||
|
||||
record.state === "draining" ||
|
||||
record.state === "reconciling" ||
|
||||
record.state === "reclaimed"
|
||||
) {
|
||||
if (
|
||||
!record.environmentId ||
|
||||
record.activeOwnerEpoch === null ||
|
||||
!record.workspaceBaseManifestRef ||
|
||||
!record.remoteWorkspaceDir ||
|
||||
!record.workerBundleHash ||
|
||||
record.recoveryError !== null
|
||||
) {
|
||||
throw new Error(
|
||||
`Worker session placement ${record.state} requires complete worker ownership`,
|
||||
);
|
||||
}
|
||||
normalizeEpoch(record.activeOwnerEpoch, "active owner epoch");
|
||||
} else if (!record.recoveryError) {
|
||||
throw new Error("Failed worker session placement requires a recovery error");
|
||||
}
|
||||
if (
|
||||
record.turnClaim?.owner === "local" &&
|
||||
record.state !== "local" &&
|
||||
record.state !== "requested" &&
|
||||
record.state !== "failed"
|
||||
) {
|
||||
throw new Error("Local turn claim requires local, dispatch-barrier, or failed placement");
|
||||
}
|
||||
if (record.turnClaim?.owner === "worker") {
|
||||
const workerMayFinish = record.state === "active" || record.state === "draining";
|
||||
if (!workerMayFinish || record.activeOwnerEpoch !== record.turnClaim.ownerEpoch) {
|
||||
throw new Error("Worker turn claim requires the active or draining worker owner epoch");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,431 @@
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import type { Selectable } from "kysely";
|
||||
import {
|
||||
executeSqliteQuerySync,
|
||||
executeSqliteQueryTakeFirstSync,
|
||||
getNodeSqliteKysely,
|
||||
} from "../../infra/kysely-sync.js";
|
||||
import type {
|
||||
DB as StateDatabase,
|
||||
WorkerSessionPlacements,
|
||||
} from "../../state/openclaw-state-db.generated.js";
|
||||
import {
|
||||
assertRecordShape,
|
||||
localTurnClaimForState,
|
||||
nextGeneration,
|
||||
normalizeCursor,
|
||||
normalizeEpoch,
|
||||
nullableRequired,
|
||||
required,
|
||||
unclaimedTurnForState,
|
||||
workerTurnClaimForState,
|
||||
type EmptyWorkerPlacementMetadata,
|
||||
type OwnedWorkerPlacementMetadata,
|
||||
type PersistedTurnClaim,
|
||||
type WorkerSessionPlacementIdentity,
|
||||
type WorkerSessionPlacementRecord,
|
||||
type WorkerSessionPlacementTransitionPatch,
|
||||
} from "./placement-record.js";
|
||||
import { parseWorkerSessionPlacementState } from "./placement-state.js";
|
||||
|
||||
type PlacementRow = Selectable<WorkerSessionPlacements>;
|
||||
type PlacementDatabase = Pick<StateDatabase, "worker_session_placements">;
|
||||
|
||||
export const query = (db: DatabaseSync) => getNodeSqliteKysely<PlacementDatabase>(db);
|
||||
|
||||
const EMPTY_WORKER_METADATA: EmptyWorkerPlacementMetadata = {
|
||||
environmentId: null,
|
||||
activeOwnerEpoch: null,
|
||||
workspaceBaseManifestRef: null,
|
||||
remoteWorkspaceDir: null,
|
||||
workerBundleHash: null,
|
||||
lastTranscriptAckCursor: null,
|
||||
lastLiveEventAckCursor: null,
|
||||
recoveryError: null,
|
||||
};
|
||||
|
||||
function parseTurnClaim(row: PlacementRow): PersistedTurnClaim | null {
|
||||
if (row.turn_claim_owner === null) {
|
||||
return null;
|
||||
}
|
||||
const claimId = required(row.turn_claim_id ?? "", "turn claim id");
|
||||
const runId = required(row.turn_claim_run_id ?? "", "turn claim run id");
|
||||
const generation = row.turn_claim_generation;
|
||||
if (generation === null || !Number.isSafeInteger(generation) || generation < 0) {
|
||||
throw new Error("Worker session placement turn claim generation is invalid");
|
||||
}
|
||||
if (row.turn_claim_owner === "local") {
|
||||
if (row.turn_claim_owner_epoch !== null) {
|
||||
throw new Error("Local turn claim cannot retain a worker owner epoch");
|
||||
}
|
||||
return { owner: "local", claimId, runId, generation, ownerEpoch: null };
|
||||
}
|
||||
if (row.turn_claim_owner === "worker") {
|
||||
return {
|
||||
owner: "worker",
|
||||
claimId,
|
||||
runId,
|
||||
generation,
|
||||
ownerEpoch: normalizeEpoch(row.turn_claim_owner_epoch ?? 0, "turn claim owner epoch"),
|
||||
};
|
||||
}
|
||||
throw new Error(`Invalid worker session turn claim owner: ${row.turn_claim_owner}`);
|
||||
}
|
||||
|
||||
type ParsedWorkerMetadata = {
|
||||
environmentId: string | null;
|
||||
activeOwnerEpoch: number | null;
|
||||
workspaceBaseManifestRef: string | null;
|
||||
remoteWorkspaceDir: string | null;
|
||||
workerBundleHash: string | null;
|
||||
lastTranscriptAckCursor: number | null;
|
||||
lastLiveEventAckCursor: number | null;
|
||||
};
|
||||
|
||||
function ownedWorkerMetadata(
|
||||
parsed: ParsedWorkerMetadata,
|
||||
state: "active" | "draining" | "reconciling" | "reclaimed",
|
||||
): OwnedWorkerPlacementMetadata {
|
||||
if (
|
||||
parsed.environmentId === null ||
|
||||
parsed.activeOwnerEpoch === null ||
|
||||
parsed.workspaceBaseManifestRef === null ||
|
||||
parsed.remoteWorkspaceDir === null ||
|
||||
parsed.workerBundleHash === null
|
||||
) {
|
||||
throw new Error(`Worker session placement ${state} requires complete worker ownership`);
|
||||
}
|
||||
return {
|
||||
environmentId: parsed.environmentId,
|
||||
activeOwnerEpoch: parsed.activeOwnerEpoch,
|
||||
workspaceBaseManifestRef: parsed.workspaceBaseManifestRef,
|
||||
remoteWorkspaceDir: parsed.remoteWorkspaceDir,
|
||||
workerBundleHash: parsed.workerBundleHash,
|
||||
lastTranscriptAckCursor: parsed.lastTranscriptAckCursor,
|
||||
lastLiveEventAckCursor: parsed.lastLiveEventAckCursor,
|
||||
recoveryError: null,
|
||||
};
|
||||
}
|
||||
|
||||
export function fromRow(row: PlacementRow): WorkerSessionPlacementRecord {
|
||||
const state = parseWorkerSessionPlacementState(row.state);
|
||||
const parsed: ParsedWorkerMetadata = {
|
||||
environmentId:
|
||||
row.environment_id === null ? null : required(row.environment_id, "environment id"),
|
||||
activeOwnerEpoch:
|
||||
row.active_owner_epoch === null
|
||||
? null
|
||||
: normalizeEpoch(row.active_owner_epoch, "active owner epoch"),
|
||||
workspaceBaseManifestRef: nullableRequired(
|
||||
row.workspace_base_manifest_ref,
|
||||
"workspace base manifest ref",
|
||||
),
|
||||
remoteWorkspaceDir: nullableRequired(row.remote_workspace_dir, "remote workspace directory"),
|
||||
workerBundleHash: nullableRequired(row.worker_bundle_hash, "worker bundle hash"),
|
||||
lastTranscriptAckCursor: normalizeCursor(
|
||||
row.last_transcript_ack_cursor,
|
||||
"transcript ACK cursor",
|
||||
),
|
||||
lastLiveEventAckCursor: normalizeCursor(row.last_live_event_ack_cursor, "live ACK cursor"),
|
||||
};
|
||||
const recoveryError = nullableRequired(row.recovery_error, "recovery error");
|
||||
const turnClaim = parseTurnClaim(row);
|
||||
const base = {
|
||||
sessionId: row.session_id,
|
||||
agentId: row.agent_id,
|
||||
sessionKey: row.session_key,
|
||||
generation: row.transition_generation,
|
||||
createdAtMs: row.created_at_ms,
|
||||
updatedAtMs: row.updated_at_ms,
|
||||
stateChangedAtMs: row.state_changed_at_ms,
|
||||
};
|
||||
assertRecordShape({ state, ...parsed, recoveryError, turnClaim });
|
||||
switch (state) {
|
||||
case "local": {
|
||||
return {
|
||||
...base,
|
||||
state,
|
||||
turnClaim: localTurnClaimForState(turnClaim, state),
|
||||
...EMPTY_WORKER_METADATA,
|
||||
};
|
||||
}
|
||||
case "requested": {
|
||||
return {
|
||||
...base,
|
||||
state,
|
||||
turnClaim: localTurnClaimForState(turnClaim, state),
|
||||
...EMPTY_WORKER_METADATA,
|
||||
};
|
||||
}
|
||||
case "provisioning": {
|
||||
return {
|
||||
...base,
|
||||
state,
|
||||
turnClaim: unclaimedTurnForState(turnClaim, state),
|
||||
...EMPTY_WORKER_METADATA,
|
||||
environmentId: parsed.environmentId,
|
||||
};
|
||||
}
|
||||
case "syncing": {
|
||||
if (parsed.environmentId === null || parsed.workerBundleHash === null) {
|
||||
throw new Error("Syncing worker session placement requires an environment and bundle");
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
state,
|
||||
turnClaim: unclaimedTurnForState(turnClaim, state),
|
||||
...EMPTY_WORKER_METADATA,
|
||||
environmentId: parsed.environmentId,
|
||||
workerBundleHash: parsed.workerBundleHash,
|
||||
};
|
||||
}
|
||||
case "starting": {
|
||||
if (
|
||||
parsed.environmentId === null ||
|
||||
parsed.workspaceBaseManifestRef === null ||
|
||||
parsed.remoteWorkspaceDir === null ||
|
||||
parsed.workerBundleHash === null
|
||||
) {
|
||||
throw new Error("Starting worker session placement requires complete workspace metadata");
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
state,
|
||||
turnClaim: unclaimedTurnForState(turnClaim, state),
|
||||
...EMPTY_WORKER_METADATA,
|
||||
environmentId: parsed.environmentId,
|
||||
workspaceBaseManifestRef: parsed.workspaceBaseManifestRef,
|
||||
remoteWorkspaceDir: parsed.remoteWorkspaceDir,
|
||||
workerBundleHash: parsed.workerBundleHash,
|
||||
};
|
||||
}
|
||||
case "active": {
|
||||
return {
|
||||
...base,
|
||||
state,
|
||||
turnClaim: workerTurnClaimForState(turnClaim, state),
|
||||
...ownedWorkerMetadata(parsed, state),
|
||||
};
|
||||
}
|
||||
case "draining": {
|
||||
return {
|
||||
...base,
|
||||
state,
|
||||
turnClaim: workerTurnClaimForState(turnClaim, state),
|
||||
...ownedWorkerMetadata(parsed, state),
|
||||
};
|
||||
}
|
||||
case "reconciling": {
|
||||
return {
|
||||
...base,
|
||||
state,
|
||||
turnClaim: unclaimedTurnForState(turnClaim, state),
|
||||
...ownedWorkerMetadata(parsed, state),
|
||||
};
|
||||
}
|
||||
case "reclaimed": {
|
||||
return {
|
||||
...base,
|
||||
state,
|
||||
turnClaim: unclaimedTurnForState(turnClaim, state),
|
||||
...ownedWorkerMetadata(parsed, state),
|
||||
};
|
||||
}
|
||||
case "failed": {
|
||||
if (recoveryError === null) {
|
||||
throw new Error("Failed worker session placement requires a recovery error");
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
state,
|
||||
turnClaim: localTurnClaimForState(turnClaim, state),
|
||||
environmentId: parsed.environmentId,
|
||||
activeOwnerEpoch: parsed.activeOwnerEpoch,
|
||||
workspaceBaseManifestRef: parsed.workspaceBaseManifestRef,
|
||||
remoteWorkspaceDir: parsed.remoteWorkspaceDir,
|
||||
workerBundleHash: parsed.workerBundleHash,
|
||||
lastTranscriptAckCursor: parsed.lastTranscriptAckCursor,
|
||||
lastLiveEventAckCursor: parsed.lastLiveEventAckCursor,
|
||||
recoveryError,
|
||||
};
|
||||
}
|
||||
}
|
||||
// Exhaustive over placement states; the return satisfies consistent-return.
|
||||
return state satisfies never;
|
||||
}
|
||||
|
||||
export function find(
|
||||
db: DatabaseSync,
|
||||
sessionId: string,
|
||||
): WorkerSessionPlacementRecord | undefined {
|
||||
const row = executeSqliteQueryTakeFirstSync(
|
||||
db,
|
||||
query(db)
|
||||
.selectFrom("worker_session_placements")
|
||||
.selectAll()
|
||||
.where("session_id", "=", sessionId),
|
||||
);
|
||||
return row ? fromRow(row) : undefined;
|
||||
}
|
||||
|
||||
export function getRequired(db: DatabaseSync, sessionId: string): WorkerSessionPlacementRecord {
|
||||
const record = find(db, sessionId);
|
||||
if (!record) {
|
||||
throw new Error(`Unknown worker session placement: ${sessionId}`);
|
||||
}
|
||||
return record;
|
||||
}
|
||||
|
||||
function assertIdentity(
|
||||
record: WorkerSessionPlacementRecord,
|
||||
identity: WorkerSessionPlacementIdentity,
|
||||
): void {
|
||||
if (record.agentId !== identity.agentId || record.sessionKey !== identity.sessionKey) {
|
||||
throw new Error(`Worker session placement identity changed for ${identity.sessionId}`);
|
||||
}
|
||||
}
|
||||
|
||||
function insertLocal(
|
||||
db: DatabaseSync,
|
||||
identity: WorkerSessionPlacementIdentity,
|
||||
nowMs: number,
|
||||
): WorkerSessionPlacementRecord {
|
||||
executeSqliteQuerySync(
|
||||
db,
|
||||
query(db).insertInto("worker_session_placements").values({
|
||||
session_id: identity.sessionId,
|
||||
agent_id: identity.agentId,
|
||||
session_key: identity.sessionKey,
|
||||
state: "local",
|
||||
environment_id: null,
|
||||
transition_generation: 0,
|
||||
active_owner_epoch: null,
|
||||
workspace_base_manifest_ref: null,
|
||||
remote_workspace_dir: null,
|
||||
worker_bundle_hash: null,
|
||||
last_transcript_ack_cursor: null,
|
||||
last_live_event_ack_cursor: null,
|
||||
recovery_error: null,
|
||||
turn_claim_owner: null,
|
||||
turn_claim_id: null,
|
||||
turn_claim_run_id: null,
|
||||
turn_claim_generation: null,
|
||||
turn_claim_owner_epoch: null,
|
||||
created_at_ms: nowMs,
|
||||
updated_at_ms: nowMs,
|
||||
state_changed_at_ms: nowMs,
|
||||
}),
|
||||
);
|
||||
return getRequired(db, identity.sessionId);
|
||||
}
|
||||
|
||||
export function ensureLocal(
|
||||
db: DatabaseSync,
|
||||
identity: WorkerSessionPlacementIdentity,
|
||||
nowMs: number,
|
||||
): WorkerSessionPlacementRecord {
|
||||
const current = find(db, identity.sessionId);
|
||||
if (current) {
|
||||
assertIdentity(current, identity);
|
||||
return current;
|
||||
}
|
||||
return insertLocal(db, identity, nowMs);
|
||||
}
|
||||
|
||||
export function transitionValues(
|
||||
current: WorkerSessionPlacementRecord,
|
||||
to: WorkerSessionPlacementRecord["state"],
|
||||
patch: WorkerSessionPlacementTransitionPatch,
|
||||
nowMs: number,
|
||||
): PlacementRow {
|
||||
const environmentId =
|
||||
to === "local" || to === "requested"
|
||||
? null
|
||||
: patch.environmentId === undefined
|
||||
? current.environmentId
|
||||
: patch.environmentId === null
|
||||
? null
|
||||
: required(patch.environmentId, "environment id");
|
||||
const activeOwnerEpoch =
|
||||
to === "local" ||
|
||||
to === "requested" ||
|
||||
to === "provisioning" ||
|
||||
to === "syncing" ||
|
||||
to === "starting"
|
||||
? null
|
||||
: patch.activeOwnerEpoch === undefined
|
||||
? current.activeOwnerEpoch
|
||||
: patch.activeOwnerEpoch === null
|
||||
? null
|
||||
: normalizeEpoch(patch.activeOwnerEpoch, "active owner epoch");
|
||||
const generation = nextGeneration(current.generation);
|
||||
const clearsWorkerMetadata = to === "local" || to === "requested";
|
||||
const values: PlacementRow = {
|
||||
session_id: current.sessionId,
|
||||
agent_id: current.agentId,
|
||||
session_key: current.sessionKey,
|
||||
state: to,
|
||||
environment_id: environmentId,
|
||||
transition_generation: generation,
|
||||
active_owner_epoch: activeOwnerEpoch,
|
||||
workspace_base_manifest_ref: clearsWorkerMetadata
|
||||
? null
|
||||
: patch.workspaceBaseManifestRef === undefined
|
||||
? current.workspaceBaseManifestRef
|
||||
: patch.workspaceBaseManifestRef === null
|
||||
? null
|
||||
: required(patch.workspaceBaseManifestRef, "workspace base manifest ref"),
|
||||
remote_workspace_dir: clearsWorkerMetadata
|
||||
? null
|
||||
: patch.remoteWorkspaceDir === undefined
|
||||
? current.remoteWorkspaceDir
|
||||
: patch.remoteWorkspaceDir === null
|
||||
? null
|
||||
: required(patch.remoteWorkspaceDir, "remote workspace directory"),
|
||||
worker_bundle_hash: clearsWorkerMetadata
|
||||
? null
|
||||
: patch.workerBundleHash === undefined
|
||||
? current.workerBundleHash
|
||||
: patch.workerBundleHash === null
|
||||
? null
|
||||
: required(patch.workerBundleHash, "worker bundle hash"),
|
||||
last_transcript_ack_cursor: clearsWorkerMetadata
|
||||
? null
|
||||
: patch.lastTranscriptAckCursor === undefined
|
||||
? current.lastTranscriptAckCursor
|
||||
: normalizeCursor(patch.lastTranscriptAckCursor, "transcript ACK cursor"),
|
||||
last_live_event_ack_cursor: clearsWorkerMetadata
|
||||
? null
|
||||
: patch.lastLiveEventAckCursor === undefined
|
||||
? current.lastLiveEventAckCursor
|
||||
: normalizeCursor(patch.lastLiveEventAckCursor, "live ACK cursor"),
|
||||
recovery_error: clearsWorkerMetadata
|
||||
? null
|
||||
: patch.recoveryError === undefined
|
||||
? current.recoveryError
|
||||
: patch.recoveryError === null
|
||||
? null
|
||||
: required(patch.recoveryError, "recovery error"),
|
||||
turn_claim_owner: null,
|
||||
turn_claim_id: null,
|
||||
turn_claim_run_id: null,
|
||||
turn_claim_generation: null,
|
||||
turn_claim_owner_epoch: null,
|
||||
created_at_ms: current.createdAtMs,
|
||||
updated_at_ms: nowMs,
|
||||
state_changed_at_ms: nowMs,
|
||||
};
|
||||
assertRecordShape({
|
||||
state: to,
|
||||
environmentId,
|
||||
activeOwnerEpoch,
|
||||
workspaceBaseManifestRef: values.workspace_base_manifest_ref,
|
||||
remoteWorkspaceDir: values.remote_workspace_dir,
|
||||
workerBundleHash: values.worker_bundle_hash,
|
||||
lastTranscriptAckCursor: values.last_transcript_ack_cursor,
|
||||
lastLiveEventAckCursor: values.last_live_event_ack_cursor,
|
||||
recoveryError: values.recovery_error,
|
||||
turnClaim: null,
|
||||
});
|
||||
return values;
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
import {
|
||||
isDefaultAgentRuntimeId,
|
||||
OPENCLAW_AGENT_RUNTIME_ID,
|
||||
} from "../../agents/agent-runtime-id.js";
|
||||
import { resolveSessionModelRef } from "../../agents/session-model-ref.js";
|
||||
import { resolvePersistedSessionRuntimeId } from "../../agents/session-runtime-compat.js";
|
||||
import { resolveEffectiveAgentRuntime } from "../../agents/thinking-runtime.js";
|
||||
import type { SessionEntry } from "../../config/sessions.js";
|
||||
import type { OpenClawConfig } from "../../config/types.openclaw.js";
|
||||
|
||||
export function resolveWorkerPlacementSessionRuntime(params: {
|
||||
cfg: OpenClawConfig;
|
||||
entry: SessionEntry;
|
||||
agentId: string;
|
||||
sessionKey: string;
|
||||
}): string {
|
||||
const persistedRuntime = resolvePersistedSessionRuntimeId(params.entry);
|
||||
if (persistedRuntime && !isDefaultAgentRuntimeId(persistedRuntime)) {
|
||||
return persistedRuntime;
|
||||
}
|
||||
const selectedModel = resolveSessionModelRef(params.cfg, params.entry, params.agentId);
|
||||
return resolveEffectiveAgentRuntime({
|
||||
cfg: params.cfg,
|
||||
provider: selectedModel.provider,
|
||||
modelId: selectedModel.model,
|
||||
agentId: params.agentId,
|
||||
sessionKey: params.sessionKey,
|
||||
});
|
||||
}
|
||||
|
||||
export function isWorkerPlacementSessionRuntimeSupported(runtime: string): boolean {
|
||||
return runtime === OPENCLAW_AGENT_RUNTIME_ID;
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
const WORKER_SESSION_PLACEMENT_STATES = [
|
||||
"local",
|
||||
"requested",
|
||||
"provisioning",
|
||||
"syncing",
|
||||
"starting",
|
||||
"active",
|
||||
"draining",
|
||||
"reconciling",
|
||||
"reclaimed",
|
||||
"failed",
|
||||
] as const;
|
||||
|
||||
export type WorkerSessionPlacementState = (typeof WORKER_SESSION_PLACEMENT_STATES)[number];
|
||||
|
||||
type WorkerSessionPlacementTransition = {
|
||||
[From in WorkerSessionPlacementState]: readonly WorkerSessionPlacementState[];
|
||||
};
|
||||
|
||||
const WORKER_SESSION_PLACEMENT_TRANSITIONS = {
|
||||
local: ["requested"],
|
||||
requested: ["provisioning", "failed"],
|
||||
provisioning: ["syncing", "failed"],
|
||||
syncing: ["starting", "failed"],
|
||||
starting: ["active", "failed"],
|
||||
active: ["draining"],
|
||||
draining: ["reconciling"],
|
||||
reconciling: ["local", "reclaimed", "failed"],
|
||||
reclaimed: ["requested"],
|
||||
failed: [],
|
||||
} as const satisfies WorkerSessionPlacementTransition;
|
||||
|
||||
export function parseWorkerSessionPlacementState(value: string): WorkerSessionPlacementState {
|
||||
if ((WORKER_SESSION_PLACEMENT_STATES as readonly string[]).includes(value)) {
|
||||
return value as WorkerSessionPlacementState;
|
||||
}
|
||||
throw new Error(`Invalid worker session placement state: ${value}`);
|
||||
}
|
||||
|
||||
export function canTransitionWorkerSessionPlacement(
|
||||
from: WorkerSessionPlacementState,
|
||||
to: WorkerSessionPlacementState,
|
||||
): boolean {
|
||||
return (
|
||||
WORKER_SESSION_PLACEMENT_TRANSITIONS[from] as readonly WorkerSessionPlacementState[]
|
||||
).includes(to);
|
||||
}
|
||||
@@ -0,0 +1,695 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
openOpenClawStateDatabase,
|
||||
type OpenClawStateDatabase,
|
||||
} from "../../state/openclaw-state-db.js";
|
||||
import type { WorkerSessionPlacementIdentity } from "./placement-record.js";
|
||||
import {
|
||||
createWorkerSessionPlacementStore,
|
||||
type WorkerSessionPlacementStore,
|
||||
} from "./placement-store.js";
|
||||
|
||||
const SESSION: WorkerSessionPlacementIdentity = {
|
||||
sessionId: "session-placement",
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:placement",
|
||||
};
|
||||
|
||||
describe("worker session placement store", () => {
|
||||
let root: string;
|
||||
let database: OpenClawStateDatabase;
|
||||
let store: WorkerSessionPlacementStore;
|
||||
let nowMs: number;
|
||||
|
||||
beforeEach(async () => {
|
||||
root = await fs.mkdtemp(path.join(await fs.realpath(os.tmpdir()), "openclaw-placement-"));
|
||||
database = openOpenClawStateDatabase({ env: { OPENCLAW_STATE_DIR: root } });
|
||||
nowMs = 1_000;
|
||||
store = createWorkerSessionPlacementStore({ database, now: () => nowMs });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function advanceToActive(identity: WorkerSessionPlacementIdentity = SESSION) {
|
||||
let placement = store.startDispatch(identity);
|
||||
placement = store.transition({
|
||||
sessionId: identity.sessionId,
|
||||
from: "requested",
|
||||
to: "provisioning",
|
||||
expectedGeneration: placement.generation,
|
||||
patch: { environmentId: `environment-${identity.sessionId}` },
|
||||
});
|
||||
placement = store.transition({
|
||||
sessionId: identity.sessionId,
|
||||
from: "provisioning",
|
||||
to: "syncing",
|
||||
expectedGeneration: placement.generation,
|
||||
patch: { workerBundleHash: "a".repeat(64) },
|
||||
});
|
||||
placement = store.transition({
|
||||
sessionId: identity.sessionId,
|
||||
from: "syncing",
|
||||
to: "starting",
|
||||
expectedGeneration: placement.generation,
|
||||
patch: {
|
||||
workspaceBaseManifestRef: `manifest-${identity.sessionId}`,
|
||||
remoteWorkspaceDir: `/workspace/${identity.sessionId}`,
|
||||
},
|
||||
});
|
||||
const active = store.transition({
|
||||
sessionId: identity.sessionId,
|
||||
from: "starting",
|
||||
to: "active",
|
||||
expectedGeneration: placement.generation,
|
||||
patch: { activeOwnerEpoch: 7 },
|
||||
});
|
||||
if (active.state !== "active") {
|
||||
throw new Error("expected active worker placement");
|
||||
}
|
||||
return active;
|
||||
}
|
||||
|
||||
it("persists the placement lifecycle and rejects stale transition generations", () => {
|
||||
const requested = store.startDispatch(SESSION);
|
||||
expect(requested).toMatchObject({
|
||||
state: "requested",
|
||||
generation: 1,
|
||||
environmentId: null,
|
||||
activeOwnerEpoch: null,
|
||||
});
|
||||
|
||||
const provisioning = store.transition({
|
||||
sessionId: SESSION.sessionId,
|
||||
from: "requested",
|
||||
to: "provisioning",
|
||||
expectedGeneration: requested.generation,
|
||||
patch: { environmentId: "environment-placement" },
|
||||
});
|
||||
expect(provisioning).toMatchObject({
|
||||
state: "provisioning",
|
||||
generation: 2,
|
||||
environmentId: "environment-placement",
|
||||
});
|
||||
expect(() =>
|
||||
store.transition({
|
||||
sessionId: SESSION.sessionId,
|
||||
from: "provisioning",
|
||||
to: "syncing",
|
||||
expectedGeneration: 1,
|
||||
}),
|
||||
).toThrow("expected provisioning@1, found provisioning@2");
|
||||
expect(() =>
|
||||
store.transition({
|
||||
sessionId: SESSION.sessionId,
|
||||
from: "provisioning",
|
||||
to: "active",
|
||||
expectedGeneration: provisioning.generation,
|
||||
}),
|
||||
).toThrow("Illegal worker session placement transition");
|
||||
|
||||
const failed = store.fail({
|
||||
sessionId: SESSION.sessionId,
|
||||
expectedGeneration: provisioning.generation,
|
||||
recoveryError: "workspace synchronization failed",
|
||||
});
|
||||
expect(failed).toMatchObject({
|
||||
state: "failed",
|
||||
generation: 3,
|
||||
recoveryError: "workspace synchronization failed",
|
||||
});
|
||||
expect(() =>
|
||||
store.fail({
|
||||
sessionId: SESSION.sessionId,
|
||||
expectedGeneration: failed.generation - 1,
|
||||
recoveryError: "stale teardown failure",
|
||||
}),
|
||||
).toThrow("changed before failure");
|
||||
expect(store.get(SESSION.sessionId)?.recoveryError).toBe("workspace synchronization failed");
|
||||
expect(
|
||||
store.fail({ sessionId: SESSION.sessionId, recoveryError: "teardown retry failed" }),
|
||||
).toMatchObject({
|
||||
state: "failed",
|
||||
generation: failed.generation,
|
||||
recoveryError: "teardown retry failed",
|
||||
});
|
||||
});
|
||||
|
||||
it("requires each placement phase to persist its complete metadata", () => {
|
||||
const requested = store.startDispatch(SESSION);
|
||||
const provisioning = store.transition({
|
||||
sessionId: SESSION.sessionId,
|
||||
from: "requested",
|
||||
to: "provisioning",
|
||||
expectedGeneration: requested.generation,
|
||||
patch: { environmentId: "environment-placement" },
|
||||
});
|
||||
expect(provisioning).toMatchObject({
|
||||
workspaceBaseManifestRef: null,
|
||||
remoteWorkspaceDir: null,
|
||||
workerBundleHash: null,
|
||||
lastTranscriptAckCursor: null,
|
||||
lastLiveEventAckCursor: null,
|
||||
});
|
||||
|
||||
expect(() =>
|
||||
store.transition({
|
||||
sessionId: SESSION.sessionId,
|
||||
from: "provisioning",
|
||||
to: "syncing",
|
||||
expectedGeneration: provisioning.generation,
|
||||
}),
|
||||
).toThrow("requires an environment and bundle");
|
||||
const syncing = store.transition({
|
||||
sessionId: SESSION.sessionId,
|
||||
from: "provisioning",
|
||||
to: "syncing",
|
||||
expectedGeneration: provisioning.generation,
|
||||
patch: { workerBundleHash: "a".repeat(64) },
|
||||
});
|
||||
|
||||
expect(() =>
|
||||
store.transition({
|
||||
sessionId: SESSION.sessionId,
|
||||
from: "syncing",
|
||||
to: "starting",
|
||||
expectedGeneration: syncing.generation,
|
||||
patch: { workspaceBaseManifestRef: "manifest-placement" },
|
||||
}),
|
||||
).toThrow("requires complete workspace metadata");
|
||||
expect(
|
||||
store.transition({
|
||||
sessionId: SESSION.sessionId,
|
||||
from: "syncing",
|
||||
to: "starting",
|
||||
expectedGeneration: syncing.generation,
|
||||
patch: {
|
||||
workspaceBaseManifestRef: "manifest-placement",
|
||||
remoteWorkspaceDir: "/workspace/placement",
|
||||
},
|
||||
}),
|
||||
).toMatchObject({
|
||||
state: "starting",
|
||||
environmentId: "environment-placement",
|
||||
workerBundleHash: "a".repeat(64),
|
||||
workspaceBaseManifestRef: "manifest-placement",
|
||||
remoteWorkspaceDir: "/workspace/placement",
|
||||
});
|
||||
});
|
||||
|
||||
it("drains and reconciles worker ownership before returning local", () => {
|
||||
const active = advanceToActive();
|
||||
const draining = store.transition({
|
||||
sessionId: SESSION.sessionId,
|
||||
from: "active",
|
||||
to: "draining",
|
||||
expectedGeneration: active.generation,
|
||||
});
|
||||
const reconciling = store.startReconcile({
|
||||
sessionId: SESSION.sessionId,
|
||||
environmentId: active.environmentId,
|
||||
ownerEpoch: active.activeOwnerEpoch,
|
||||
expectedGeneration: draining.generation,
|
||||
});
|
||||
const local = store.transition({
|
||||
sessionId: SESSION.sessionId,
|
||||
from: "reconciling",
|
||||
to: "local",
|
||||
expectedGeneration: reconciling.generation,
|
||||
});
|
||||
expect(local).toMatchObject({
|
||||
state: "local",
|
||||
environmentId: null,
|
||||
activeOwnerEpoch: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects reclaim before worker ownership reaches reconciliation", () => {
|
||||
const requested = store.startDispatch(SESSION);
|
||||
expect(() =>
|
||||
store.transition({
|
||||
sessionId: SESSION.sessionId,
|
||||
from: "requested",
|
||||
to: "reclaimed",
|
||||
expectedGeneration: requested.generation,
|
||||
}),
|
||||
).toThrow("Illegal worker session placement transition");
|
||||
expect(
|
||||
store.fail({
|
||||
sessionId: SESSION.sessionId,
|
||||
expectedGeneration: requested.generation,
|
||||
recoveryError: "dispatch stopped before provisioning",
|
||||
}),
|
||||
).toMatchObject({ state: "failed" });
|
||||
});
|
||||
|
||||
it("closes local admission before draining the existing local turn", async () => {
|
||||
const localClaim = store.claimTurn({
|
||||
...SESSION,
|
||||
owner: { kind: "local" },
|
||||
claimId: "local-claim",
|
||||
runId: "run-local",
|
||||
});
|
||||
const requested = store.startDispatch(SESSION);
|
||||
expect(requested).toMatchObject({ state: "requested", generation: 1 });
|
||||
expect(requested.turnClaim).toMatchObject({ owner: "local", generation: 0 });
|
||||
|
||||
expect(() =>
|
||||
store.claimTurn({
|
||||
...SESSION,
|
||||
owner: { kind: "local" },
|
||||
claimId: "new-local-claim",
|
||||
runId: "new-local-run",
|
||||
}),
|
||||
).toThrow("already has an active turn claim");
|
||||
expect(() =>
|
||||
store.claimTurn({
|
||||
...SESSION,
|
||||
owner: localClaim.owner,
|
||||
claimId: localClaim.claimId,
|
||||
runId: localClaim.runId,
|
||||
}),
|
||||
).toThrow("already has an active turn claim");
|
||||
expect(() =>
|
||||
store.transition({
|
||||
sessionId: SESSION.sessionId,
|
||||
from: "requested",
|
||||
to: "provisioning",
|
||||
expectedGeneration: requested.generation,
|
||||
patch: { environmentId: "environment-placement" },
|
||||
}),
|
||||
).toThrow("during an active turn");
|
||||
|
||||
const released = store.waitForTurnClaimRelease(SESSION.sessionId, { timeoutMs: 1_000 });
|
||||
store.releaseTurn(localClaim);
|
||||
await released;
|
||||
expect(
|
||||
store.transition({
|
||||
sessionId: SESSION.sessionId,
|
||||
from: "requested",
|
||||
to: "provisioning",
|
||||
expectedGeneration: requested.generation,
|
||||
patch: { environmentId: "environment-placement" },
|
||||
}),
|
||||
).toMatchObject({ state: "provisioning", turnClaim: null });
|
||||
});
|
||||
|
||||
it("keeps the draining local claim releasable when the dispatch barrier fails", () => {
|
||||
const localClaim = store.claimTurn({
|
||||
...SESSION,
|
||||
owner: { kind: "local" },
|
||||
claimId: "local-barrier-claim",
|
||||
runId: "local-barrier-run",
|
||||
});
|
||||
const requested = store.startDispatch(SESSION);
|
||||
const failed = store.fail({
|
||||
sessionId: SESSION.sessionId,
|
||||
expectedGeneration: requested.generation,
|
||||
recoveryError: "local drain timed out",
|
||||
});
|
||||
|
||||
expect(failed).toMatchObject({
|
||||
state: "failed",
|
||||
recoveryError: "local drain timed out",
|
||||
turnClaim: { owner: "local", claimId: localClaim.claimId },
|
||||
});
|
||||
expect(() =>
|
||||
store.claimTurn({
|
||||
...SESSION,
|
||||
owner: { kind: "local" },
|
||||
claimId: "new-local-claim",
|
||||
runId: "new-local-run",
|
||||
}),
|
||||
).toThrow("already has an active turn claim");
|
||||
expect(store.releaseTurn(localClaim)).toMatchObject({ state: "failed", turnClaim: null });
|
||||
});
|
||||
|
||||
it("does not let a stale claim release a later turn that reuses the run id", () => {
|
||||
const firstClaim = store.claimTurn({
|
||||
...SESSION,
|
||||
owner: { kind: "local" },
|
||||
claimId: "first-claim-token",
|
||||
runId: "reused-run",
|
||||
});
|
||||
store.releaseTurn(firstClaim);
|
||||
const secondClaim = store.claimTurn({
|
||||
...SESSION,
|
||||
owner: { kind: "local" },
|
||||
claimId: "second-claim-token",
|
||||
runId: firstClaim.runId,
|
||||
});
|
||||
|
||||
expect(() => store.releaseTurn(firstClaim)).toThrow("turn claim changed before release");
|
||||
expect(store.validateTurnClaim(secondClaim)).toBe(true);
|
||||
expect(store.get(SESSION.sessionId)?.turnClaim).toMatchObject({
|
||||
claimId: secondClaim.claimId,
|
||||
runId: secondClaim.runId,
|
||||
});
|
||||
});
|
||||
|
||||
it("allows a reset session id to reuse its canonical session key", () => {
|
||||
const firstClaim = store.claimTurn({
|
||||
...SESSION,
|
||||
owner: { kind: "local" },
|
||||
claimId: "first-session-claim",
|
||||
runId: "first-session-run",
|
||||
});
|
||||
store.releaseTurn(firstClaim);
|
||||
|
||||
const rotated = store.claimTurn({
|
||||
...SESSION,
|
||||
sessionId: "session-placement-rotated",
|
||||
owner: { kind: "local" },
|
||||
claimId: "rotated-session-claim",
|
||||
runId: "rotated-session-run",
|
||||
});
|
||||
expect(rotated.sessionId).toBe("session-placement-rotated");
|
||||
expect(store.list().map((record) => record.sessionId)).toEqual([
|
||||
SESSION.sessionId,
|
||||
"session-placement-rotated",
|
||||
]);
|
||||
});
|
||||
|
||||
it("admits exactly the active placement owner and fences stale worker epochs", () => {
|
||||
const active = advanceToActive();
|
||||
expect(() =>
|
||||
store.claimTurn({
|
||||
...SESSION,
|
||||
owner: { kind: "local" },
|
||||
claimId: "local-after-dispatch",
|
||||
runId: "local-after-dispatch-run",
|
||||
}),
|
||||
).toThrow("Local turn rejected");
|
||||
expect(() =>
|
||||
store.claimTurn({
|
||||
...SESSION,
|
||||
owner: {
|
||||
kind: "worker",
|
||||
environmentId: active.environmentId,
|
||||
ownerEpoch: active.activeOwnerEpoch + 1,
|
||||
},
|
||||
claimId: "stale-worker",
|
||||
runId: "stale-worker-run",
|
||||
}),
|
||||
).toThrow("stale owner");
|
||||
|
||||
const workerClaim = store.claimTurn({
|
||||
...SESSION,
|
||||
owner: {
|
||||
kind: "worker",
|
||||
environmentId: active.environmentId,
|
||||
ownerEpoch: active.activeOwnerEpoch,
|
||||
},
|
||||
claimId: "worker-claim",
|
||||
runId: "worker-run",
|
||||
});
|
||||
expect(store.validateTurnClaim(workerClaim)).toBe(true);
|
||||
expect(
|
||||
store.validateTurnClaim({
|
||||
...workerClaim,
|
||||
owner: {
|
||||
kind: "worker",
|
||||
environmentId: "environment-stale",
|
||||
ownerEpoch: active.activeOwnerEpoch,
|
||||
},
|
||||
}),
|
||||
).toBe(false);
|
||||
expect(
|
||||
store.validateWorkerOwner({
|
||||
sessionId: SESSION.sessionId,
|
||||
environmentId: active.environmentId,
|
||||
ownerEpoch: active.activeOwnerEpoch,
|
||||
}),
|
||||
).toBe(true);
|
||||
expect(() =>
|
||||
store.claimTurn({
|
||||
...SESSION,
|
||||
owner: {
|
||||
kind: "worker",
|
||||
environmentId: active.environmentId,
|
||||
ownerEpoch: active.activeOwnerEpoch,
|
||||
},
|
||||
claimId: "competing-worker",
|
||||
runId: "competing-worker-run",
|
||||
}),
|
||||
).toThrow("already has an active turn claim");
|
||||
expect(() =>
|
||||
store.claimTurn({
|
||||
...SESSION,
|
||||
owner: workerClaim.owner,
|
||||
claimId: workerClaim.claimId,
|
||||
runId: workerClaim.runId,
|
||||
}),
|
||||
).toThrow("already has an active turn claim");
|
||||
expect(() =>
|
||||
store.fail({
|
||||
sessionId: SESSION.sessionId,
|
||||
expectedGeneration: active.generation,
|
||||
recoveryError: "active worker disappeared",
|
||||
}),
|
||||
).toThrow("Cannot fail worker session placement from active");
|
||||
const draining = store.startDrain({
|
||||
sessionId: SESSION.sessionId,
|
||||
environmentId: active.environmentId,
|
||||
ownerEpoch: active.activeOwnerEpoch,
|
||||
expectedGeneration: active.generation,
|
||||
});
|
||||
const reconciling = store.startReconcile({
|
||||
sessionId: SESSION.sessionId,
|
||||
environmentId: active.environmentId,
|
||||
ownerEpoch: active.activeOwnerEpoch,
|
||||
expectedGeneration: draining.generation,
|
||||
});
|
||||
expect(
|
||||
store.transition({
|
||||
sessionId: SESSION.sessionId,
|
||||
from: "reconciling",
|
||||
to: "failed",
|
||||
expectedGeneration: reconciling.generation,
|
||||
patch: { recoveryError: "active worker disappeared" },
|
||||
}),
|
||||
).toMatchObject({ state: "failed", turnClaim: null });
|
||||
expect(store.validateTurnClaim(workerClaim)).toBe(false);
|
||||
});
|
||||
|
||||
it("clears dead local claims on restart while adopting active worker ownership", () => {
|
||||
const localIdentity = {
|
||||
...SESSION,
|
||||
sessionId: "session-local-restart",
|
||||
sessionKey: "agent:main:local-restart",
|
||||
};
|
||||
store.claimTurn({
|
||||
...localIdentity,
|
||||
owner: { kind: "local" },
|
||||
claimId: "local-before-restart",
|
||||
runId: "local-restart-run",
|
||||
});
|
||||
const active = advanceToActive();
|
||||
const workerClaim = store.claimTurn({
|
||||
...SESSION,
|
||||
owner: {
|
||||
kind: "worker",
|
||||
environmentId: active.environmentId,
|
||||
ownerEpoch: active.activeOwnerEpoch,
|
||||
},
|
||||
claimId: "worker-before-restart",
|
||||
runId: "worker-restart-run",
|
||||
});
|
||||
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
database = openOpenClawStateDatabase({ env: { OPENCLAW_STATE_DIR: root } });
|
||||
store = createWorkerSessionPlacementStore({ database, now: () => nowMs });
|
||||
|
||||
expect(store.clearLocalTurnClaimsAfterRestart()).toBe(1);
|
||||
expect(store.get(localIdentity.sessionId)?.turnClaim).toBeNull();
|
||||
expect(store.validateTurnClaim(workerClaim)).toBe(true);
|
||||
expect(
|
||||
store.adoptActive({
|
||||
sessionId: SESSION.sessionId,
|
||||
environmentId: active.environmentId,
|
||||
ownerEpoch: active.activeOwnerEpoch,
|
||||
expectedGeneration: active.generation,
|
||||
}),
|
||||
).toMatchObject({ state: "active", turnClaim: { owner: "worker" } });
|
||||
expect(store.listForReconcile().map((record) => record.sessionId)).toEqual([SESSION.sessionId]);
|
||||
expect(store.list().map((record) => record.sessionId)).toEqual([
|
||||
localIdentity.sessionId,
|
||||
SESSION.sessionId,
|
||||
]);
|
||||
});
|
||||
|
||||
it("closes worker admission before draining the active turn", async () => {
|
||||
const active = advanceToActive();
|
||||
const workerClaim = store.claimTurn({
|
||||
...SESSION,
|
||||
owner: {
|
||||
kind: "worker",
|
||||
environmentId: active.environmentId,
|
||||
ownerEpoch: active.activeOwnerEpoch,
|
||||
},
|
||||
claimId: "worker-drain-claim",
|
||||
runId: "worker-drain-run",
|
||||
});
|
||||
|
||||
const draining = store.startDrain({
|
||||
sessionId: SESSION.sessionId,
|
||||
environmentId: active.environmentId,
|
||||
ownerEpoch: active.activeOwnerEpoch,
|
||||
expectedGeneration: active.generation,
|
||||
});
|
||||
expect(draining).toMatchObject({
|
||||
state: "draining",
|
||||
generation: active.generation + 1,
|
||||
turnClaim: { owner: "worker", claimId: workerClaim.claimId },
|
||||
});
|
||||
expect(store.validateTurnClaim(workerClaim)).toBe(true);
|
||||
|
||||
const released = store.waitForTurnClaimRelease(SESSION.sessionId, { timeoutMs: 1_000 });
|
||||
expect(store.releaseTurn(workerClaim)).toMatchObject({ state: "draining", turnClaim: null });
|
||||
await released;
|
||||
expect(() =>
|
||||
store.claimTurn({
|
||||
...SESSION,
|
||||
owner: workerClaim.owner,
|
||||
claimId: "worker-after-drain",
|
||||
runId: "worker-after-drain-run",
|
||||
}),
|
||||
).toThrow("stale owner");
|
||||
expect(
|
||||
store.startReconcile({
|
||||
sessionId: SESSION.sessionId,
|
||||
environmentId: active.environmentId,
|
||||
ownerEpoch: active.activeOwnerEpoch,
|
||||
expectedGeneration: draining.generation,
|
||||
}),
|
||||
).toMatchObject({ state: "reconciling", turnClaim: null });
|
||||
});
|
||||
|
||||
it("atomically fences a drained claim before its worker is reclaimed", async () => {
|
||||
const active = advanceToActive();
|
||||
const workerClaim = store.claimTurn({
|
||||
...SESSION,
|
||||
owner: {
|
||||
kind: "worker",
|
||||
environmentId: active.environmentId,
|
||||
ownerEpoch: active.activeOwnerEpoch,
|
||||
},
|
||||
claimId: "worker-reclaim-claim",
|
||||
runId: "worker-reclaim-run",
|
||||
});
|
||||
const released = store.waitForTurnClaimRelease(SESSION.sessionId, { timeoutMs: 1_000 });
|
||||
|
||||
const draining = store.startDrain({
|
||||
sessionId: SESSION.sessionId,
|
||||
environmentId: active.environmentId,
|
||||
ownerEpoch: active.activeOwnerEpoch,
|
||||
expectedGeneration: active.generation,
|
||||
});
|
||||
expect(() =>
|
||||
store.fail({
|
||||
sessionId: SESSION.sessionId,
|
||||
expectedGeneration: draining.generation,
|
||||
recoveryError: "worker teardown not yet fenced",
|
||||
}),
|
||||
).toThrow("Cannot fail worker session placement from draining");
|
||||
expect(() =>
|
||||
store.transition({
|
||||
sessionId: SESSION.sessionId,
|
||||
from: "draining",
|
||||
to: "reclaimed",
|
||||
expectedGeneration: draining.generation,
|
||||
}),
|
||||
).toThrow("Illegal worker session placement transition");
|
||||
expect(() =>
|
||||
store.transition({
|
||||
sessionId: SESSION.sessionId,
|
||||
from: "draining",
|
||||
to: "reconciling",
|
||||
expectedGeneration: draining.generation,
|
||||
}),
|
||||
).toThrow("Use startReconcile after fencing the drained worker environment");
|
||||
expect(() =>
|
||||
store.startReconcile({
|
||||
sessionId: SESSION.sessionId,
|
||||
environmentId: active.environmentId,
|
||||
ownerEpoch: active.activeOwnerEpoch,
|
||||
expectedGeneration: draining.generation - 1,
|
||||
}),
|
||||
).toThrow("Cannot reconcile stale worker placement");
|
||||
const reconciling = store.startReconcile({
|
||||
sessionId: SESSION.sessionId,
|
||||
environmentId: active.environmentId,
|
||||
ownerEpoch: active.activeOwnerEpoch,
|
||||
expectedGeneration: draining.generation,
|
||||
});
|
||||
const reclaimed = store.transition({
|
||||
sessionId: SESSION.sessionId,
|
||||
from: "reconciling",
|
||||
to: "reclaimed",
|
||||
expectedGeneration: reconciling.generation,
|
||||
});
|
||||
expect(reclaimed).toMatchObject({ state: "reclaimed", turnClaim: null });
|
||||
await released;
|
||||
expect(store.validateTurnClaim(workerClaim)).toBe(false);
|
||||
expect(store.startDispatch(SESSION)).toMatchObject({
|
||||
state: "requested",
|
||||
generation: reclaimed.generation + 1,
|
||||
environmentId: null,
|
||||
activeOwnerEpoch: null,
|
||||
workspaceBaseManifestRef: null,
|
||||
remoteWorkspaceDir: null,
|
||||
workerBundleHash: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("binds acknowledged cursors to the exact normalized worker claim", () => {
|
||||
const active = advanceToActive();
|
||||
const firstClaim = store.claimTurn({
|
||||
...SESSION,
|
||||
owner: {
|
||||
kind: "worker",
|
||||
environmentId: ` ${active.environmentId} `,
|
||||
ownerEpoch: active.activeOwnerEpoch,
|
||||
},
|
||||
claimId: "worker-ack-first",
|
||||
runId: "worker-ack-first-run",
|
||||
});
|
||||
expect(firstClaim.owner).toEqual({
|
||||
kind: "worker",
|
||||
environmentId: active.environmentId,
|
||||
ownerEpoch: active.activeOwnerEpoch,
|
||||
});
|
||||
store.releaseTurn(firstClaim);
|
||||
const currentClaim = store.claimTurn({
|
||||
...SESSION,
|
||||
owner: firstClaim.owner,
|
||||
claimId: "worker-ack-current",
|
||||
runId: "worker-ack-current-run",
|
||||
});
|
||||
|
||||
expect(() => store.updateAckCursors({ claim: firstClaim, transcript: 4 })).toThrow(
|
||||
"Cannot ACK stale worker turn",
|
||||
);
|
||||
expect(store.get(SESSION.sessionId)?.lastTranscriptAckCursor).toBeNull();
|
||||
expect(
|
||||
store.updateAckCursors({
|
||||
claim: currentClaim,
|
||||
transcript: 4,
|
||||
liveEvent: 9,
|
||||
}),
|
||||
).toMatchObject({ lastTranscriptAckCursor: 4, lastLiveEventAckCursor: 9 });
|
||||
expect(
|
||||
store.updateAckCursors({
|
||||
claim: currentClaim,
|
||||
transcript: 3,
|
||||
liveEvent: 8,
|
||||
}),
|
||||
).toMatchObject({ lastTranscriptAckCursor: 4, lastLiveEventAckCursor: 9 });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,414 @@
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { executeSqliteQuerySync } from "../../infra/kysely-sync.js";
|
||||
import {
|
||||
openOpenClawStateDatabase,
|
||||
runOpenClawStateWriteTransaction,
|
||||
type OpenClawStateDatabase,
|
||||
} from "../../state/openclaw-state-db.js";
|
||||
import {
|
||||
assertRecordShape,
|
||||
nextGeneration,
|
||||
normalizeEpoch,
|
||||
normalizeIdentity,
|
||||
required,
|
||||
type WorkerSessionPlacementIdentity,
|
||||
type WorkerSessionPlacementRecord,
|
||||
type WorkerSessionPlacementTransitionPatch,
|
||||
} from "./placement-record.js";
|
||||
import {
|
||||
ensureLocal,
|
||||
find,
|
||||
fromRow,
|
||||
getRequired,
|
||||
query,
|
||||
transitionValues,
|
||||
} from "./placement-row-codec.js";
|
||||
import {
|
||||
canTransitionWorkerSessionPlacement,
|
||||
type WorkerSessionPlacementState,
|
||||
} from "./placement-state.js";
|
||||
import {
|
||||
createPlacementTurnClaimOps,
|
||||
signalTurnClaimRelease,
|
||||
type PlacementStoreRuntime,
|
||||
} from "./placement-turn-claims.js";
|
||||
|
||||
export type { WorkerSessionPlacementRecord, WorkerSessionTurnClaim } from "./placement-record.js";
|
||||
|
||||
function updateTransition(
|
||||
db: DatabaseSync,
|
||||
current: WorkerSessionPlacementRecord,
|
||||
to: WorkerSessionPlacementState,
|
||||
patch: WorkerSessionPlacementTransitionPatch,
|
||||
nowMs: number,
|
||||
): WorkerSessionPlacementRecord {
|
||||
const values = transitionValues(current, to, patch, nowMs);
|
||||
const result = executeSqliteQuerySync(
|
||||
db,
|
||||
query(db)
|
||||
.updateTable("worker_session_placements")
|
||||
.set(values)
|
||||
.where("session_id", "=", current.sessionId)
|
||||
.where("state", "=", current.state)
|
||||
.where("transition_generation", "=", current.generation)
|
||||
.where("turn_claim_owner", "is", null),
|
||||
);
|
||||
if (result.numAffectedRows !== 1n) {
|
||||
throw new Error(`Worker session placement ${current.sessionId} changed during transition`);
|
||||
}
|
||||
return getRequired(db, current.sessionId);
|
||||
}
|
||||
|
||||
export function createWorkerSessionPlacementStore(
|
||||
options: { database?: OpenClawStateDatabase; now?: () => number } = {},
|
||||
) {
|
||||
const path = (options.database ?? openOpenClawStateDatabase()).path;
|
||||
const now = options.now ?? Date.now;
|
||||
const runtime: PlacementStoreRuntime = {
|
||||
path,
|
||||
now,
|
||||
read: () => openOpenClawStateDatabase({ path }).db,
|
||||
write: (operation) => runOpenClawStateWriteTransaction(({ db }) => operation(db), { path }),
|
||||
};
|
||||
const { read, write } = runtime;
|
||||
|
||||
return {
|
||||
...createPlacementTurnClaimOps(runtime),
|
||||
|
||||
get(sessionId: string): WorkerSessionPlacementRecord | undefined {
|
||||
return find(read(), required(sessionId, "session id"));
|
||||
},
|
||||
|
||||
getMany(sessionIds: readonly string[]): ReadonlyMap<string, WorkerSessionPlacementRecord> {
|
||||
const normalizedIds = [
|
||||
...new Set(sessionIds.map((sessionId) => required(sessionId, "session id"))),
|
||||
];
|
||||
const records = new Map<string, WorkerSessionPlacementRecord>();
|
||||
const db = read();
|
||||
for (let offset = 0; offset < normalizedIds.length; offset += 250) {
|
||||
const chunk = normalizedIds.slice(offset, offset + 250);
|
||||
for (const row of executeSqliteQuerySync(
|
||||
db,
|
||||
query(db)
|
||||
.selectFrom("worker_session_placements")
|
||||
.selectAll()
|
||||
.where("session_id", "in", chunk),
|
||||
).rows) {
|
||||
const record = fromRow(row);
|
||||
records.set(record.sessionId, record);
|
||||
}
|
||||
}
|
||||
return records;
|
||||
},
|
||||
|
||||
startDispatch(input: WorkerSessionPlacementIdentity): WorkerSessionPlacementRecord {
|
||||
const identity = normalizeIdentity(input);
|
||||
return write((db) => {
|
||||
const current = ensureLocal(db, identity, now());
|
||||
if (current.state !== "local" && current.state !== "reclaimed") {
|
||||
throw new Error(
|
||||
`Cannot dispatch session ${identity.sessionId} from placement ${current.state}`,
|
||||
);
|
||||
}
|
||||
const updatedAtMs = now();
|
||||
// Preserve an in-flight local claim while closing admission. Reclaimed
|
||||
// placement has no live owner and starts a fresh worker generation.
|
||||
const result = executeSqliteQuerySync(
|
||||
db,
|
||||
query(db)
|
||||
.updateTable("worker_session_placements")
|
||||
.set({
|
||||
state: "requested",
|
||||
environment_id: null,
|
||||
transition_generation: nextGeneration(current.generation),
|
||||
active_owner_epoch: null,
|
||||
workspace_base_manifest_ref: null,
|
||||
remote_workspace_dir: null,
|
||||
worker_bundle_hash: null,
|
||||
last_transcript_ack_cursor: null,
|
||||
last_live_event_ack_cursor: null,
|
||||
recovery_error: null,
|
||||
updated_at_ms: updatedAtMs,
|
||||
state_changed_at_ms: updatedAtMs,
|
||||
})
|
||||
.where("session_id", "=", current.sessionId)
|
||||
.where("state", "=", current.state)
|
||||
.where("transition_generation", "=", current.generation),
|
||||
);
|
||||
if (result.numAffectedRows !== 1n) {
|
||||
throw new Error(
|
||||
`Session ${identity.sessionId} placement changed during dispatch barrier`,
|
||||
);
|
||||
}
|
||||
return getRequired(db, identity.sessionId);
|
||||
});
|
||||
},
|
||||
|
||||
transition(input: {
|
||||
sessionId: string;
|
||||
from: WorkerSessionPlacementState;
|
||||
to: WorkerSessionPlacementState;
|
||||
expectedGeneration: number;
|
||||
patch?: WorkerSessionPlacementTransitionPatch;
|
||||
}): WorkerSessionPlacementRecord {
|
||||
if (!canTransitionWorkerSessionPlacement(input.from, input.to)) {
|
||||
throw new Error(
|
||||
`Illegal worker session placement transition: ${input.from} -> ${input.to}`,
|
||||
);
|
||||
}
|
||||
if (input.from === "draining" && input.to === "reconciling") {
|
||||
throw new Error("Use startReconcile after fencing the drained worker environment");
|
||||
}
|
||||
const sessionId = required(input.sessionId, "session id");
|
||||
return write((db) => {
|
||||
const current = getRequired(db, sessionId);
|
||||
if (current.state !== input.from || current.generation !== input.expectedGeneration) {
|
||||
throw new Error(
|
||||
`Worker session placement ${sessionId} changed: expected ${input.from}@${input.expectedGeneration}, found ${current.state}@${current.generation}`,
|
||||
);
|
||||
}
|
||||
if (current.turnClaim) {
|
||||
throw new Error(`Cannot transition session ${sessionId} during an active turn`);
|
||||
}
|
||||
return updateTransition(db, current, input.to, input.patch ?? {}, now());
|
||||
});
|
||||
},
|
||||
|
||||
startDrain(input: {
|
||||
sessionId: string;
|
||||
environmentId: string;
|
||||
ownerEpoch: number;
|
||||
expectedGeneration: number;
|
||||
}): WorkerSessionPlacementRecord {
|
||||
const sessionId = required(input.sessionId, "session id");
|
||||
const environmentId = required(input.environmentId, "environment id");
|
||||
const ownerEpoch = normalizeEpoch(input.ownerEpoch, "active owner epoch");
|
||||
return write((db) => {
|
||||
const current = getRequired(db, sessionId);
|
||||
if (
|
||||
current.state !== "active" ||
|
||||
current.generation !== input.expectedGeneration ||
|
||||
current.environmentId !== environmentId ||
|
||||
current.activeOwnerEpoch !== ownerEpoch
|
||||
) {
|
||||
throw new Error(`Cannot drain stale worker placement for session ${sessionId}`);
|
||||
}
|
||||
// Draining closes new admission first. The already-admitted worker may
|
||||
// finish under its old claim before reconciliation advances ownership.
|
||||
const values = transitionValues(current, "draining", {}, now());
|
||||
const turnClaim = current.turnClaim;
|
||||
if (turnClaim) {
|
||||
values.turn_claim_owner = turnClaim.owner;
|
||||
values.turn_claim_id = turnClaim.claimId;
|
||||
values.turn_claim_run_id = turnClaim.runId;
|
||||
values.turn_claim_generation = turnClaim.generation;
|
||||
values.turn_claim_owner_epoch = turnClaim.ownerEpoch;
|
||||
}
|
||||
assertRecordShape({
|
||||
state: "draining",
|
||||
environmentId,
|
||||
activeOwnerEpoch: ownerEpoch,
|
||||
workspaceBaseManifestRef: values.workspace_base_manifest_ref,
|
||||
remoteWorkspaceDir: values.remote_workspace_dir,
|
||||
workerBundleHash: values.worker_bundle_hash,
|
||||
lastTranscriptAckCursor: values.last_transcript_ack_cursor,
|
||||
lastLiveEventAckCursor: values.last_live_event_ack_cursor,
|
||||
recoveryError: values.recovery_error,
|
||||
turnClaim,
|
||||
});
|
||||
const result = executeSqliteQuerySync(
|
||||
db,
|
||||
query(db)
|
||||
.updateTable("worker_session_placements")
|
||||
.set(values)
|
||||
.where("session_id", "=", sessionId)
|
||||
.where("state", "=", "active")
|
||||
.where("transition_generation", "=", current.generation)
|
||||
.where("environment_id", "=", environmentId)
|
||||
.where("active_owner_epoch", "=", ownerEpoch),
|
||||
);
|
||||
if (result.numAffectedRows !== 1n) {
|
||||
throw new Error(`Worker session placement ${sessionId} changed during drain`);
|
||||
}
|
||||
return getRequired(db, sessionId);
|
||||
});
|
||||
},
|
||||
|
||||
startReconcile(input: {
|
||||
sessionId: string;
|
||||
environmentId: string;
|
||||
ownerEpoch: number;
|
||||
expectedGeneration: number;
|
||||
}): WorkerSessionPlacementRecord {
|
||||
const sessionId = required(input.sessionId, "session id");
|
||||
const environmentId = required(input.environmentId, "environment id");
|
||||
const ownerEpoch = normalizeEpoch(input.ownerEpoch, "active owner epoch");
|
||||
const outcome = write((db) => {
|
||||
const current = getRequired(db, sessionId);
|
||||
if (
|
||||
current.state !== "draining" ||
|
||||
current.generation !== input.expectedGeneration ||
|
||||
current.environmentId !== environmentId ||
|
||||
current.activeOwnerEpoch !== ownerEpoch
|
||||
) {
|
||||
throw new Error(`Cannot reconcile stale worker placement for session ${sessionId}`);
|
||||
}
|
||||
// The caller has already fenced the environment. Clear its last claim
|
||||
// in the same CAS that opens the post-worker reconciliation phase.
|
||||
const releasedClaim = current.turnClaim !== null;
|
||||
const values = transitionValues(current, "reconciling", {}, now());
|
||||
const update = query(db)
|
||||
.updateTable("worker_session_placements")
|
||||
.set(values)
|
||||
.where("session_id", "=", sessionId)
|
||||
.where("state", "=", "draining")
|
||||
.where("transition_generation", "=", current.generation)
|
||||
.where("environment_id", "=", environmentId)
|
||||
.where("active_owner_epoch", "=", ownerEpoch);
|
||||
const guardedUpdate = current.turnClaim
|
||||
? update
|
||||
.where("turn_claim_owner", "=", "worker")
|
||||
.where("turn_claim_id", "=", current.turnClaim.claimId)
|
||||
.where("turn_claim_run_id", "=", current.turnClaim.runId)
|
||||
.where("turn_claim_generation", "=", current.turnClaim.generation)
|
||||
.where("turn_claim_owner_epoch", "=", current.turnClaim.ownerEpoch)
|
||||
: update.where("turn_claim_owner", "is", null);
|
||||
const result = executeSqliteQuerySync(db, guardedUpdate);
|
||||
if (result.numAffectedRows !== 1n) {
|
||||
throw new Error(`Worker session placement ${sessionId} changed during reconcile`);
|
||||
}
|
||||
return { record: getRequired(db, sessionId), releasedClaim };
|
||||
});
|
||||
if (outcome.releasedClaim) {
|
||||
signalTurnClaimRelease(path, sessionId);
|
||||
}
|
||||
return outcome.record;
|
||||
},
|
||||
|
||||
validateWorkerOwner(input: {
|
||||
sessionId: string;
|
||||
environmentId: string;
|
||||
ownerEpoch: number;
|
||||
}): boolean {
|
||||
const current = find(read(), required(input.sessionId, "session id"));
|
||||
return (
|
||||
current?.state === "active" &&
|
||||
current.environmentId === required(input.environmentId, "environment id") &&
|
||||
current.activeOwnerEpoch === normalizeEpoch(input.ownerEpoch, "active owner epoch")
|
||||
);
|
||||
},
|
||||
|
||||
fail(input: {
|
||||
sessionId: string;
|
||||
recoveryError: string;
|
||||
expectedGeneration?: number;
|
||||
}): WorkerSessionPlacementRecord {
|
||||
const sessionId = required(input.sessionId, "session id");
|
||||
const recoveryError = required(input.recoveryError, "recovery error");
|
||||
const outcome = write((db) => {
|
||||
const current = getRequired(db, sessionId);
|
||||
if (
|
||||
input.expectedGeneration !== undefined &&
|
||||
current.generation !== input.expectedGeneration
|
||||
) {
|
||||
throw new Error(`Worker session placement ${sessionId} changed before failure`);
|
||||
}
|
||||
if (current.state === "failed") {
|
||||
const result = executeSqliteQuerySync(
|
||||
db,
|
||||
query(db)
|
||||
.updateTable("worker_session_placements")
|
||||
.set({ recovery_error: recoveryError, updated_at_ms: now() })
|
||||
.where("session_id", "=", sessionId)
|
||||
.where("state", "=", "failed")
|
||||
.where("transition_generation", "=", current.generation),
|
||||
);
|
||||
if (result.numAffectedRows !== 1n) {
|
||||
throw new Error(`Worker session placement ${sessionId} changed during failure update`);
|
||||
}
|
||||
return { record: getRequired(db, sessionId), releasedClaim: false };
|
||||
}
|
||||
if (!canTransitionWorkerSessionPlacement(current.state, "failed")) {
|
||||
throw new Error(`Cannot fail worker session placement from ${current.state}`);
|
||||
}
|
||||
const localClaim = current.turnClaim?.owner === "local" ? current.turnClaim : null;
|
||||
const updatedAtMs = now();
|
||||
const result = executeSqliteQuerySync(
|
||||
db,
|
||||
query(db)
|
||||
.updateTable("worker_session_placements")
|
||||
.set({
|
||||
state: "failed",
|
||||
transition_generation: nextGeneration(current.generation),
|
||||
recovery_error: recoveryError,
|
||||
turn_claim_owner: localClaim ? "local" : null,
|
||||
turn_claim_id: localClaim?.claimId ?? null,
|
||||
turn_claim_run_id: localClaim?.runId ?? null,
|
||||
turn_claim_generation: localClaim?.generation ?? null,
|
||||
turn_claim_owner_epoch: null,
|
||||
updated_at_ms: updatedAtMs,
|
||||
state_changed_at_ms: updatedAtMs,
|
||||
})
|
||||
.where("session_id", "=", sessionId)
|
||||
.where("state", "=", current.state)
|
||||
.where("transition_generation", "=", current.generation),
|
||||
);
|
||||
if (result.numAffectedRows !== 1n) {
|
||||
throw new Error(`Worker session placement ${sessionId} changed during failure`);
|
||||
}
|
||||
return {
|
||||
record: getRequired(db, sessionId),
|
||||
releasedClaim: current.turnClaim?.owner === "worker",
|
||||
};
|
||||
});
|
||||
if (outcome.releasedClaim) {
|
||||
signalTurnClaimRelease(path, sessionId);
|
||||
}
|
||||
return outcome.record;
|
||||
},
|
||||
|
||||
adoptActive(input: {
|
||||
sessionId: string;
|
||||
environmentId: string;
|
||||
ownerEpoch: number;
|
||||
expectedGeneration?: number;
|
||||
}): WorkerSessionPlacementRecord {
|
||||
const sessionId = required(input.sessionId, "session id");
|
||||
const environmentId = required(input.environmentId, "environment id");
|
||||
const ownerEpoch = normalizeEpoch(input.ownerEpoch, "active owner epoch");
|
||||
const current = getRequired(read(), sessionId);
|
||||
if (
|
||||
current.state !== "active" ||
|
||||
current.environmentId !== environmentId ||
|
||||
current.activeOwnerEpoch !== ownerEpoch ||
|
||||
(input.expectedGeneration !== undefined && current.generation !== input.expectedGeneration)
|
||||
) {
|
||||
throw new Error(`Cannot adopt stale worker placement for session ${sessionId}`);
|
||||
}
|
||||
return current;
|
||||
},
|
||||
|
||||
listForReconcile(): WorkerSessionPlacementRecord[] {
|
||||
const db = read();
|
||||
return executeSqliteQuerySync(
|
||||
db,
|
||||
query(db)
|
||||
.selectFrom("worker_session_placements")
|
||||
.selectAll()
|
||||
.where("state", "not in", ["local", "reclaimed"])
|
||||
.orderBy("updated_at_ms")
|
||||
.orderBy("session_id"),
|
||||
).rows.map(fromRow);
|
||||
},
|
||||
|
||||
list(): WorkerSessionPlacementRecord[] {
|
||||
const db = read();
|
||||
return executeSqliteQuerySync(
|
||||
db,
|
||||
query(db).selectFrom("worker_session_placements").selectAll().orderBy("session_id"),
|
||||
).rows.map(fromRow);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export type WorkerSessionPlacementStore = ReturnType<typeof createWorkerSessionPlacementStore>;
|
||||
@@ -0,0 +1,352 @@
|
||||
import type { DatabaseSync } from "node:sqlite";
|
||||
import { executeSqliteQuerySync } from "../../infra/kysely-sync.js";
|
||||
import {
|
||||
advanceCursor,
|
||||
normalizeEpoch,
|
||||
normalizeIdentity,
|
||||
required,
|
||||
type WorkerSessionPlacementIdentity,
|
||||
type WorkerSessionPlacementRecord,
|
||||
type WorkerSessionTurnClaim,
|
||||
type WorkerSessionTurnOwner,
|
||||
} from "./placement-record.js";
|
||||
import { ensureLocal, find, getRequired, query } from "./placement-row-codec.js";
|
||||
|
||||
/** One database path backs many store instances; waits and releases meet here. */
|
||||
export type PlacementStoreRuntime = {
|
||||
path: string;
|
||||
now: () => number;
|
||||
read: () => DatabaseSync;
|
||||
write: <T>(operation: (db: DatabaseSync) => T) => T;
|
||||
};
|
||||
|
||||
type TurnClaimReleaseWaiter = () => void;
|
||||
const turnClaimReleaseWaiters = new Map<string, Map<string, Set<TurnClaimReleaseWaiter>>>();
|
||||
|
||||
function waitersFor(path: string, sessionId: string): Set<TurnClaimReleaseWaiter> {
|
||||
let bySession = turnClaimReleaseWaiters.get(path);
|
||||
if (!bySession) {
|
||||
bySession = new Map();
|
||||
turnClaimReleaseWaiters.set(path, bySession);
|
||||
}
|
||||
let waiters = bySession.get(sessionId);
|
||||
if (!waiters) {
|
||||
waiters = new Set();
|
||||
bySession.set(sessionId, waiters);
|
||||
}
|
||||
return waiters;
|
||||
}
|
||||
|
||||
export function signalTurnClaimRelease(path: string, sessionId: string): void {
|
||||
const bySession = turnClaimReleaseWaiters.get(path);
|
||||
const waiters = bySession?.get(sessionId);
|
||||
if (!waiters) {
|
||||
return;
|
||||
}
|
||||
bySession?.delete(sessionId);
|
||||
if (bySession?.size === 0) {
|
||||
turnClaimReleaseWaiters.delete(path);
|
||||
}
|
||||
for (const resolve of waiters) {
|
||||
resolve();
|
||||
}
|
||||
}
|
||||
|
||||
export function createPlacementTurnClaimOps(runtime: PlacementStoreRuntime) {
|
||||
const { path, now, read, write } = runtime;
|
||||
|
||||
return {
|
||||
claimTurn(
|
||||
input: WorkerSessionPlacementIdentity & {
|
||||
owner: WorkerSessionTurnOwner;
|
||||
claimId: string;
|
||||
runId: string;
|
||||
},
|
||||
): WorkerSessionTurnClaim {
|
||||
const identity = normalizeIdentity(input);
|
||||
const claimId = required(input.claimId, "turn claim id");
|
||||
const runId = required(input.runId, "turn claim run id");
|
||||
const owner: WorkerSessionTurnOwner =
|
||||
input.owner.kind === "local"
|
||||
? { kind: "local" }
|
||||
: {
|
||||
kind: "worker",
|
||||
environmentId: required(input.owner.environmentId, "turn owner environment id"),
|
||||
ownerEpoch: normalizeEpoch(input.owner.ownerEpoch, "turn owner epoch"),
|
||||
};
|
||||
return write((db) => {
|
||||
const current = ensureLocal(db, identity, now());
|
||||
if (current.turnClaim) {
|
||||
throw new Error(`Session ${identity.sessionId} already has an active turn claim`);
|
||||
}
|
||||
if (owner.kind === "local") {
|
||||
if (current.state !== "local") {
|
||||
throw new Error(
|
||||
`Local turn rejected for session ${identity.sessionId} in placement ${current.state}`,
|
||||
);
|
||||
}
|
||||
} else if (
|
||||
current.state !== "active" ||
|
||||
current.environmentId !== owner.environmentId ||
|
||||
current.activeOwnerEpoch !== owner.ownerEpoch
|
||||
) {
|
||||
throw new Error(`Worker turn rejected for session ${identity.sessionId}: stale owner`);
|
||||
}
|
||||
const result = executeSqliteQuerySync(
|
||||
db,
|
||||
query(db)
|
||||
.updateTable("worker_session_placements")
|
||||
.set({
|
||||
turn_claim_owner: owner.kind,
|
||||
turn_claim_id: claimId,
|
||||
turn_claim_run_id: runId,
|
||||
turn_claim_generation: current.generation,
|
||||
turn_claim_owner_epoch: owner.kind === "worker" ? owner.ownerEpoch : null,
|
||||
updated_at_ms: now(),
|
||||
})
|
||||
.where("session_id", "=", current.sessionId)
|
||||
.where("state", "=", current.state)
|
||||
.where("transition_generation", "=", current.generation)
|
||||
.where("turn_claim_owner", "is", null),
|
||||
);
|
||||
if (result.numAffectedRows !== 1n) {
|
||||
throw new Error(`Session ${identity.sessionId} placement changed during turn admission`);
|
||||
}
|
||||
return {
|
||||
sessionId: current.sessionId,
|
||||
claimId,
|
||||
runId,
|
||||
placementGeneration: current.generation,
|
||||
owner,
|
||||
};
|
||||
});
|
||||
},
|
||||
|
||||
releaseTurn(claim: WorkerSessionTurnClaim): WorkerSessionPlacementRecord {
|
||||
const sessionId = required(claim.sessionId, "session id");
|
||||
const claimId = required(claim.claimId, "turn claim id");
|
||||
const runId = required(claim.runId, "turn claim run id");
|
||||
const released = write((db) => {
|
||||
const current = getRequired(db, sessionId);
|
||||
const persisted = current.turnClaim;
|
||||
const workerMayFinish = current.state === "active" || current.state === "draining";
|
||||
if (
|
||||
!persisted ||
|
||||
persisted.claimId !== claimId ||
|
||||
persisted.runId !== runId ||
|
||||
persisted.generation !== claim.placementGeneration ||
|
||||
persisted.owner !== claim.owner.kind ||
|
||||
(claim.owner.kind === "worker" &&
|
||||
(persisted.ownerEpoch !== claim.owner.ownerEpoch ||
|
||||
!workerMayFinish ||
|
||||
current.environmentId !== claim.owner.environmentId ||
|
||||
current.activeOwnerEpoch !== claim.owner.ownerEpoch))
|
||||
) {
|
||||
throw new Error(`Session ${sessionId} turn claim changed before release`);
|
||||
}
|
||||
const result = executeSqliteQuerySync(
|
||||
db,
|
||||
query(db)
|
||||
.updateTable("worker_session_placements")
|
||||
.set({
|
||||
turn_claim_owner: null,
|
||||
turn_claim_id: null,
|
||||
turn_claim_run_id: null,
|
||||
turn_claim_generation: null,
|
||||
turn_claim_owner_epoch: null,
|
||||
updated_at_ms: now(),
|
||||
})
|
||||
.where("session_id", "=", sessionId)
|
||||
.where("turn_claim_id", "=", claimId)
|
||||
.where("turn_claim_run_id", "=", runId)
|
||||
.where("turn_claim_generation", "=", claim.placementGeneration),
|
||||
);
|
||||
if (result.numAffectedRows !== 1n) {
|
||||
throw new Error(`Session ${sessionId} turn claim changed during release`);
|
||||
}
|
||||
return getRequired(db, sessionId);
|
||||
});
|
||||
signalTurnClaimRelease(path, sessionId);
|
||||
return released;
|
||||
},
|
||||
|
||||
clearLocalTurnClaimsAfterRestart(): number {
|
||||
const clearedSessionIds = write((db) => {
|
||||
const sessionIds = executeSqliteQuerySync(
|
||||
db,
|
||||
query(db)
|
||||
.selectFrom("worker_session_placements")
|
||||
.select("session_id")
|
||||
.where("turn_claim_owner", "=", "local"),
|
||||
).rows.map((row) => row.session_id);
|
||||
const result = executeSqliteQuerySync(
|
||||
db,
|
||||
query(db)
|
||||
.updateTable("worker_session_placements")
|
||||
.set({
|
||||
turn_claim_owner: null,
|
||||
turn_claim_id: null,
|
||||
turn_claim_run_id: null,
|
||||
turn_claim_generation: null,
|
||||
turn_claim_owner_epoch: null,
|
||||
updated_at_ms: now(),
|
||||
})
|
||||
.where("turn_claim_owner", "=", "local"),
|
||||
);
|
||||
if (result.numAffectedRows !== BigInt(sessionIds.length)) {
|
||||
throw new Error("Local turn claims changed during restart recovery");
|
||||
}
|
||||
return sessionIds;
|
||||
});
|
||||
for (const sessionId of clearedSessionIds) {
|
||||
signalTurnClaimRelease(path, sessionId);
|
||||
}
|
||||
return clearedSessionIds.length;
|
||||
},
|
||||
|
||||
async waitForTurnClaimRelease(
|
||||
sessionIdInput: string,
|
||||
waitOptions: { timeoutMs: number; signal?: AbortSignal },
|
||||
): Promise<void> {
|
||||
const sessionId = required(sessionIdInput, "session id");
|
||||
if (!Number.isSafeInteger(waitOptions.timeoutMs) || waitOptions.timeoutMs < 0) {
|
||||
throw new Error("Worker session turn claim wait timeout must be a non-negative integer");
|
||||
}
|
||||
if (!find(read(), sessionId)?.turnClaim) {
|
||||
return;
|
||||
}
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
let settled = false;
|
||||
const waiters = waitersFor(path, sessionId);
|
||||
const finish = (error?: Error) => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
waitOptions.signal?.removeEventListener("abort", onAbort);
|
||||
waiters.delete(onRelease);
|
||||
if (waiters.size === 0) {
|
||||
const bySession = turnClaimReleaseWaiters.get(path);
|
||||
bySession?.delete(sessionId);
|
||||
if (bySession?.size === 0) {
|
||||
turnClaimReleaseWaiters.delete(path);
|
||||
}
|
||||
}
|
||||
if (error) {
|
||||
reject(error);
|
||||
} else {
|
||||
resolve();
|
||||
}
|
||||
};
|
||||
const onRelease = () => finish();
|
||||
const onAbort = () => finish(new Error(`Turn claim wait aborted for session ${sessionId}`));
|
||||
const timer = setTimeout(
|
||||
() => finish(new Error(`Timed out waiting for session ${sessionId} turn claim release`)),
|
||||
waitOptions.timeoutMs,
|
||||
);
|
||||
waiters.add(onRelease);
|
||||
waitOptions.signal?.addEventListener("abort", onAbort, { once: true });
|
||||
// Register first, then reread. This closes the release-between-check-and-wait race.
|
||||
if (!find(read(), sessionId)?.turnClaim) {
|
||||
finish();
|
||||
} else if (waitOptions.signal?.aborted) {
|
||||
onAbort();
|
||||
}
|
||||
});
|
||||
},
|
||||
|
||||
validateTurnClaim(claim: WorkerSessionTurnClaim): boolean {
|
||||
const current = find(read(), required(claim.sessionId, "session id"));
|
||||
const persisted = current?.turnClaim;
|
||||
return (
|
||||
persisted !== undefined &&
|
||||
persisted !== null &&
|
||||
persisted.claimId === claim.claimId &&
|
||||
persisted.runId === claim.runId &&
|
||||
persisted.generation === claim.placementGeneration &&
|
||||
persisted.owner === claim.owner.kind &&
|
||||
(claim.owner.kind === "local" ||
|
||||
(persisted.ownerEpoch === claim.owner.ownerEpoch &&
|
||||
(current?.state === "active" || current?.state === "draining") &&
|
||||
current.environmentId === claim.owner.environmentId &&
|
||||
current.activeOwnerEpoch === claim.owner.ownerEpoch))
|
||||
);
|
||||
},
|
||||
|
||||
updateAckCursors(input: {
|
||||
claim: WorkerSessionTurnClaim;
|
||||
transcript?: number;
|
||||
liveEvent?: number;
|
||||
}): WorkerSessionPlacementRecord {
|
||||
const sessionId = required(input.claim.sessionId, "session id");
|
||||
const claimId = required(input.claim.claimId, "turn claim id");
|
||||
const runId = required(input.claim.runId, "turn claim run id");
|
||||
if (
|
||||
!Number.isSafeInteger(input.claim.placementGeneration) ||
|
||||
input.claim.placementGeneration < 0
|
||||
) {
|
||||
throw new Error("Worker session placement turn claim generation is invalid");
|
||||
}
|
||||
if (input.claim.owner.kind !== "worker") {
|
||||
throw new Error("Only a worker turn claim can acknowledge worker cursors");
|
||||
}
|
||||
const placementGeneration = input.claim.placementGeneration;
|
||||
const environmentId = required(input.claim.owner.environmentId, "environment id");
|
||||
const ownerEpoch = normalizeEpoch(input.claim.owner.ownerEpoch, "active owner epoch");
|
||||
return write((db) => {
|
||||
const current = getRequired(db, sessionId);
|
||||
const persisted = current.turnClaim;
|
||||
const workerMayFinish = current.state === "active" || current.state === "draining";
|
||||
if (
|
||||
!workerMayFinish ||
|
||||
current.environmentId !== environmentId ||
|
||||
current.activeOwnerEpoch !== ownerEpoch ||
|
||||
persisted?.owner !== "worker" ||
|
||||
persisted.claimId !== claimId ||
|
||||
persisted.runId !== runId ||
|
||||
persisted.generation !== placementGeneration ||
|
||||
persisted.ownerEpoch !== ownerEpoch
|
||||
) {
|
||||
throw new Error(`Cannot ACK stale worker turn for session ${sessionId}`);
|
||||
}
|
||||
// Successful RPC replays can carry an older sequence. Preserve the
|
||||
// durable high-water mark while acknowledging the idempotent replay.
|
||||
const transcript = advanceCursor(
|
||||
current.lastTranscriptAckCursor,
|
||||
input.transcript,
|
||||
"transcript ACK cursor",
|
||||
);
|
||||
const liveEvent = advanceCursor(
|
||||
current.lastLiveEventAckCursor,
|
||||
input.liveEvent,
|
||||
"live ACK cursor",
|
||||
);
|
||||
const result = executeSqliteQuerySync(
|
||||
db,
|
||||
query(db)
|
||||
.updateTable("worker_session_placements")
|
||||
.set({
|
||||
last_transcript_ack_cursor: transcript,
|
||||
last_live_event_ack_cursor: liveEvent,
|
||||
updated_at_ms: now(),
|
||||
})
|
||||
.where("session_id", "=", sessionId)
|
||||
.where("state", "=", current.state)
|
||||
.where("transition_generation", "=", current.generation)
|
||||
.where("environment_id", "=", environmentId)
|
||||
.where("active_owner_epoch", "=", ownerEpoch)
|
||||
.where("turn_claim_owner", "=", "worker")
|
||||
.where("turn_claim_id", "=", claimId)
|
||||
.where("turn_claim_run_id", "=", runId)
|
||||
.where("turn_claim_generation", "=", placementGeneration)
|
||||
.where("turn_claim_owner_epoch", "=", ownerEpoch),
|
||||
);
|
||||
if (result.numAffectedRows !== 1n) {
|
||||
throw new Error(`Worker session placement ${sessionId} changed during ACK`);
|
||||
}
|
||||
return getRequired(db, sessionId);
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,153 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
closeOpenClawStateDatabaseForTest,
|
||||
openOpenClawStateDatabase,
|
||||
type OpenClawStateDatabase,
|
||||
} from "../../state/openclaw-state-db.js";
|
||||
import type { WorkerSessionPlacementIdentity } from "./placement-record.js";
|
||||
import {
|
||||
createWorkerSessionPlacementStore,
|
||||
type WorkerSessionPlacementStore,
|
||||
} from "./placement-store.js";
|
||||
import { createWorkerSessionPlacementGate } from "./placement-worker-gate.js";
|
||||
|
||||
const SESSION: WorkerSessionPlacementIdentity = {
|
||||
sessionId: "session-worker-gate",
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:worker-gate",
|
||||
};
|
||||
const ENVIRONMENT_ID = "environment-worker-gate";
|
||||
const OWNER_EPOCH = 7;
|
||||
|
||||
describe("worker session placement gate", () => {
|
||||
let root: string;
|
||||
let database: OpenClawStateDatabase;
|
||||
let store: WorkerSessionPlacementStore;
|
||||
|
||||
beforeEach(async () => {
|
||||
root = await fs.mkdtemp(path.join(await fs.realpath(os.tmpdir()), "openclaw-worker-gate-"));
|
||||
database = openOpenClawStateDatabase({ env: { OPENCLAW_STATE_DIR: root } });
|
||||
store = createWorkerSessionPlacementStore({ database });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
function activate() {
|
||||
let placement = store.startDispatch(SESSION);
|
||||
placement = store.transition({
|
||||
sessionId: SESSION.sessionId,
|
||||
from: "requested",
|
||||
to: "provisioning",
|
||||
expectedGeneration: placement.generation,
|
||||
patch: { environmentId: ENVIRONMENT_ID },
|
||||
});
|
||||
placement = store.transition({
|
||||
sessionId: SESSION.sessionId,
|
||||
from: "provisioning",
|
||||
to: "syncing",
|
||||
expectedGeneration: placement.generation,
|
||||
patch: { workerBundleHash: "a".repeat(64) },
|
||||
});
|
||||
placement = store.transition({
|
||||
sessionId: SESSION.sessionId,
|
||||
from: "syncing",
|
||||
to: "starting",
|
||||
expectedGeneration: placement.generation,
|
||||
patch: {
|
||||
workspaceBaseManifestRef: "manifest-worker-gate",
|
||||
remoteWorkspaceDir: "/workspace/worker-gate",
|
||||
},
|
||||
});
|
||||
return store.transition({
|
||||
sessionId: SESSION.sessionId,
|
||||
from: "starting",
|
||||
to: "active",
|
||||
expectedGeneration: placement.generation,
|
||||
patch: { activeOwnerEpoch: OWNER_EPOCH },
|
||||
});
|
||||
}
|
||||
|
||||
function preclaim(runId: string) {
|
||||
const placement = activate();
|
||||
return store.claimTurn({
|
||||
sessionId: placement.sessionId,
|
||||
agentId: placement.agentId,
|
||||
sessionKey: placement.sessionKey,
|
||||
claimId: `claim:${runId}`,
|
||||
runId,
|
||||
owner: { kind: "worker", environmentId: ENVIRONMENT_ID, ownerEpoch: OWNER_EPOCH },
|
||||
});
|
||||
}
|
||||
|
||||
it("accepts only the exact gateway-preclaimed worker run", () => {
|
||||
const runId = "run-worker-gate";
|
||||
preclaim(runId);
|
||||
const gate = createWorkerSessionPlacementGate(store);
|
||||
const binding = {
|
||||
sessionId: SESSION.sessionId,
|
||||
environmentId: ENVIRONMENT_ID,
|
||||
ownerEpoch: OWNER_EPOCH,
|
||||
runId,
|
||||
};
|
||||
|
||||
expect(gate.validateWorkerTurn(binding)).toBe(true);
|
||||
expect(gate.validateWorkerTurn({ ...binding, runId: "run-competing" })).toBe(false);
|
||||
expect(gate.validateWorkerTurn({ ...binding, ownerEpoch: OWNER_EPOCH + 1 })).toBe(false);
|
||||
});
|
||||
|
||||
it("updates exact-owner cursors and rejects stale descriptor replay", () => {
|
||||
const runId = "run-worker-ack";
|
||||
const claim = preclaim(runId);
|
||||
const gate = createWorkerSessionPlacementGate(store);
|
||||
const binding = {
|
||||
sessionId: SESSION.sessionId,
|
||||
environmentId: ENVIRONMENT_ID,
|
||||
ownerEpoch: OWNER_EPOCH,
|
||||
runId,
|
||||
};
|
||||
|
||||
gate.updateAckCursors({ ...binding, transcriptSeq: 4, liveSeq: 9 });
|
||||
expect(store.get(SESSION.sessionId)).toMatchObject({
|
||||
generation: claim.placementGeneration,
|
||||
lastTranscriptAckCursor: 4,
|
||||
lastLiveEventAckCursor: 9,
|
||||
});
|
||||
store.releaseTurn(claim);
|
||||
expect(store.get(SESSION.sessionId)?.turnClaim).toBeNull();
|
||||
expect(gate.validateWorkerTurn(binding)).toBe(false);
|
||||
});
|
||||
|
||||
it("lets the admitted worker finish acknowledgements after draining closes admission", () => {
|
||||
const runId = "run-worker-draining-ack";
|
||||
const claim = preclaim(runId);
|
||||
const active = store.get(SESSION.sessionId);
|
||||
if (active?.state !== "active") {
|
||||
throw new Error("expected active placement");
|
||||
}
|
||||
store.startDrain({
|
||||
sessionId: SESSION.sessionId,
|
||||
environmentId: active.environmentId,
|
||||
ownerEpoch: active.activeOwnerEpoch,
|
||||
expectedGeneration: active.generation,
|
||||
});
|
||||
const gate = createWorkerSessionPlacementGate(store);
|
||||
const binding = {
|
||||
sessionId: SESSION.sessionId,
|
||||
environmentId: ENVIRONMENT_ID,
|
||||
ownerEpoch: OWNER_EPOCH,
|
||||
runId,
|
||||
};
|
||||
|
||||
expect(gate.validateWorkerTurn(binding)).toBe(true);
|
||||
gate.updateAckCursors({ ...binding, transcriptSeq: 5 });
|
||||
expect(store.get(SESSION.sessionId)?.lastTranscriptAckCursor).toBe(5);
|
||||
store.releaseTurn(claim);
|
||||
expect(gate.validateWorkerTurn(binding)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,60 @@
|
||||
import type {
|
||||
WorkerSessionPlacementRecord,
|
||||
WorkerSessionPlacementStore,
|
||||
WorkerSessionTurnClaim,
|
||||
} from "./placement-store.js";
|
||||
import type { WorkerPlacementTurnBinding, WorkerSessionPlacementGate } from "./service.js";
|
||||
|
||||
function claimForBinding(
|
||||
record: WorkerSessionPlacementRecord | undefined,
|
||||
binding: WorkerPlacementTurnBinding,
|
||||
): WorkerSessionTurnClaim | undefined {
|
||||
const persisted = record?.turnClaim;
|
||||
if (
|
||||
!record ||
|
||||
(record.state !== "active" && record.state !== "draining") ||
|
||||
record.environmentId !== binding.environmentId ||
|
||||
record.activeOwnerEpoch !== binding.ownerEpoch ||
|
||||
persisted?.owner !== "worker" ||
|
||||
persisted.runId !== binding.runId ||
|
||||
persisted.ownerEpoch !== binding.ownerEpoch
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
sessionId: binding.sessionId,
|
||||
claimId: persisted.claimId,
|
||||
runId: persisted.runId,
|
||||
placementGeneration: persisted.generation,
|
||||
owner: {
|
||||
kind: "worker",
|
||||
environmentId: binding.environmentId,
|
||||
ownerEpoch: binding.ownerEpoch,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createWorkerSessionPlacementGate(
|
||||
store: WorkerSessionPlacementStore,
|
||||
): WorkerSessionPlacementGate {
|
||||
const validateWorkerTurn = (binding: WorkerPlacementTurnBinding): boolean => {
|
||||
const claim = claimForBinding(store.get(binding.sessionId), binding);
|
||||
return claim ? store.validateTurnClaim(claim) : false;
|
||||
};
|
||||
|
||||
return {
|
||||
validateWorkerTurn,
|
||||
|
||||
updateAckCursors(binding): void {
|
||||
const claim = claimForBinding(store.get(binding.sessionId), binding);
|
||||
if (!claim) {
|
||||
throw new Error(`Cannot ACK stale worker turn for session ${binding.sessionId}`);
|
||||
}
|
||||
store.updateAckCursors({
|
||||
claim,
|
||||
...(binding.transcriptSeq === undefined ? {} : { transcript: binding.transcriptSeq }),
|
||||
...(binding.liveSeq === undefined ? {} : { liveEvent: binding.liveSeq }),
|
||||
});
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
import type { WorkerSessionPlacementRecord } from "./placement-record.js";
|
||||
import type { WorkerEnvironmentState } from "./state.js";
|
||||
import type {
|
||||
WorkerTunnelHandle,
|
||||
@@ -27,3 +28,18 @@ export type WorkerEnvironmentServiceContract = {
|
||||
startTunnel(request: WorkerTunnelRequest): Promise<WorkerTunnelHandle>;
|
||||
stopTunnel(environmentId: string, ownerEpoch?: number): Promise<void>;
|
||||
};
|
||||
|
||||
export type WorkerPlacementDispatchRequest = {
|
||||
sessionId: string;
|
||||
sessionKey: string;
|
||||
agentId: string;
|
||||
profileId: string;
|
||||
};
|
||||
|
||||
// Leaf dispatch contract: GatewayRequestContext must not import the dispatch
|
||||
// runtime (it reaches agents/plugins and closes an import cycle through core).
|
||||
export type WorkerPlacementDispatchContract = {
|
||||
dispatch(
|
||||
request: WorkerPlacementDispatchRequest,
|
||||
): Promise<Extract<WorkerSessionPlacementRecord, { state: "active" }>>;
|
||||
};
|
||||
|
||||
@@ -129,6 +129,7 @@ describe("worker environment service", () => {
|
||||
| "tunnelManager"
|
||||
| "generateWorkerCredential"
|
||||
| "liveEvents"
|
||||
| "placementStore"
|
||||
| "workerCredentialTtlMs"
|
||||
>
|
||||
> = {},
|
||||
@@ -243,6 +244,7 @@ describe("worker environment service", () => {
|
||||
environmentId,
|
||||
credential: [CREDENTIAL, environmentId].join("-"),
|
||||
sessionId: null,
|
||||
runId: null,
|
||||
ownerEpoch: 1,
|
||||
rpcSetVersion: 1,
|
||||
handshake: BOOTSTRAP_RECEIPT,
|
||||
@@ -269,6 +271,7 @@ describe("worker environment service", () => {
|
||||
credentialHash: credential.credentialHash,
|
||||
bundleHash: credential.bundleHash,
|
||||
sessionId,
|
||||
runId: "run-1",
|
||||
ownerEpoch: attached.ownerEpoch,
|
||||
rpcSetVersion: credential.rpcSetVersion,
|
||||
protocolFeatures: [...attached.bootstrapReceipt.protocolFeatures],
|
||||
@@ -282,7 +285,7 @@ describe("worker environment service", () => {
|
||||
return {
|
||||
runEpoch: identity.ownerEpoch,
|
||||
sessionId: identity.sessionId ?? "session-missing",
|
||||
runId: "run-inference",
|
||||
runId: identity.runId ?? "run-missing",
|
||||
turnId: "turn-inference",
|
||||
modelRef: { provider: "fake", model: "model-test" },
|
||||
context: { messages: [] },
|
||||
@@ -435,6 +438,551 @@ describe("worker environment service", () => {
|
||||
expect(applyTranscriptCommit).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("admits only a gateway-preclaimed worker placement and fences later requests", async () => {
|
||||
const environmentId = "worker-placement-fence";
|
||||
const sessionId = "session-placement-fence";
|
||||
const identity = seedAttachedIdentity(environmentId, sessionId);
|
||||
const placementStore = {
|
||||
validateWorkerTurn: vi.fn(() => true),
|
||||
updateAckCursors: vi.fn(),
|
||||
};
|
||||
const workerService = createService(createProvider(), { placementStore });
|
||||
const admission = {
|
||||
environmentId,
|
||||
credential: [CREDENTIAL, environmentId, sessionId].join("-"),
|
||||
sessionId,
|
||||
runId: "run-1",
|
||||
ownerEpoch: identity.ownerEpoch,
|
||||
rpcSetVersion: 1,
|
||||
handshake: BOOTSTRAP_RECEIPT,
|
||||
};
|
||||
|
||||
await expect(workerService.admitWorker(admission)).resolves.toMatchObject({ ok: true });
|
||||
await expect(workerService.admitWorker(admission)).resolves.toMatchObject({ ok: true });
|
||||
expect(placementStore.validateWorkerTurn).toHaveBeenLastCalledWith({
|
||||
sessionId,
|
||||
environmentId,
|
||||
ownerEpoch: identity.ownerEpoch,
|
||||
runId: "run-1",
|
||||
});
|
||||
expect(placementStore.validateWorkerTurn).toHaveBeenCalledTimes(2);
|
||||
expect(workerService.validateWorkerConnection(identity)).toBeNull();
|
||||
|
||||
const warmEnvironmentId = "worker-placement-warm";
|
||||
seedReady(warmEnvironmentId);
|
||||
const warmAdmission = await workerService.admitWorker(admissionFor(warmEnvironmentId));
|
||||
expect(warmAdmission).toMatchObject({ ok: true });
|
||||
if (!warmAdmission.ok) {
|
||||
throw new Error("warm worker admission failed");
|
||||
}
|
||||
expect(workerService.validateWorkerConnection(warmAdmission.identity)).toBeNull();
|
||||
expect(placementStore.validateWorkerTurn).toHaveBeenCalledTimes(3);
|
||||
|
||||
placementStore.validateWorkerTurn.mockReturnValue(false);
|
||||
await expect(
|
||||
workerService.admitWorker({ ...admission, runId: "run-conflict" }),
|
||||
).resolves.toEqual({ ok: false, reason: "placement-mismatch" });
|
||||
|
||||
placementStore.validateWorkerTurn.mockReturnValue(true);
|
||||
nowMs += 10_000;
|
||||
expect(workerService.validateWorkerConnection(identity)).toBeNull();
|
||||
expect(workerService.validateWorkerConnection(warmAdmission.identity)).toBe(
|
||||
"credential-expired",
|
||||
);
|
||||
await expect(workerService.admitWorker(admission)).resolves.toEqual({
|
||||
ok: false,
|
||||
reason: "credential-expired",
|
||||
});
|
||||
|
||||
placementStore.validateWorkerTurn.mockReturnValue(false);
|
||||
expect(workerService.validateWorkerConnection(identity)).toBe("placement-mismatch");
|
||||
await expect(
|
||||
workerService.commitTranscript(identity, {
|
||||
runEpoch: identity.ownerEpoch,
|
||||
seq: 1,
|
||||
baseLeafId: null,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "fenced" }],
|
||||
timestamp: 1,
|
||||
},
|
||||
],
|
||||
}),
|
||||
).resolves.toEqual({ ok: false, closeReason: "placement-mismatch" });
|
||||
});
|
||||
|
||||
it("persists worker transcript and terminal live ACK cursors", async () => {
|
||||
const identity = seedAttachedIdentity("worker-placement-ack", "session-placement-ack");
|
||||
const placementStore = {
|
||||
validateWorkerTurn: vi.fn(() => true),
|
||||
updateAckCursors: vi.fn(),
|
||||
};
|
||||
const applyTranscriptCommit = vi.fn(async () => ({
|
||||
ok: true as const,
|
||||
result: { entryIds: ["entry-placement"], newLeafId: "entry-placement" },
|
||||
}));
|
||||
const liveEvents = createLiveEvents({
|
||||
apply: vi.fn(
|
||||
({
|
||||
request,
|
||||
}: Parameters<NonNullable<WorkerEnvironmentServiceOptions["liveEvents"]>["apply"]>[0]) => ({
|
||||
ok: true as const,
|
||||
result: { ackedSeq: request.seq },
|
||||
}),
|
||||
),
|
||||
});
|
||||
const workerService = createService(createProvider(), {
|
||||
applyTranscriptCommit,
|
||||
liveEvents,
|
||||
placementStore,
|
||||
});
|
||||
const binding = {
|
||||
sessionId: identity.sessionId ?? "session-missing",
|
||||
environmentId: identity.environmentId,
|
||||
ownerEpoch: identity.ownerEpoch,
|
||||
runId: identity.runId ?? "run-missing",
|
||||
};
|
||||
|
||||
await expect(
|
||||
workerService.commitTranscript(identity, {
|
||||
runEpoch: identity.ownerEpoch,
|
||||
seq: 7,
|
||||
baseLeafId: null,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "commit" }],
|
||||
timestamp: 1,
|
||||
},
|
||||
],
|
||||
}),
|
||||
).resolves.toMatchObject({ ok: true });
|
||||
expect(placementStore.updateAckCursors).toHaveBeenCalledWith({
|
||||
...binding,
|
||||
transcriptSeq: 7,
|
||||
});
|
||||
|
||||
await expect(
|
||||
workerService.pushLiveEvent(identity, {
|
||||
runEpoch: identity.ownerEpoch,
|
||||
lastAckedSeq: 0,
|
||||
seq: 1,
|
||||
runId: binding.runId,
|
||||
event: { kind: "lifecycle", payload: { phase: "end", endedAt: 2 } },
|
||||
}),
|
||||
).resolves.toEqual({ ok: true, result: { ackedSeq: 1 } });
|
||||
expect(placementStore.updateAckCursors).toHaveBeenLastCalledWith({
|
||||
...binding,
|
||||
liveSeq: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not ACK a transcript commit after its worker claim is fenced", async () => {
|
||||
const identity = seedAttachedIdentity("worker-placement-race", "session-placement-race");
|
||||
const placementStore = {
|
||||
validateWorkerTurn: vi.fn(() => true),
|
||||
updateAckCursors: vi.fn(),
|
||||
};
|
||||
let finishCommit: (() => void) | undefined;
|
||||
const commitBlocked = new Promise<void>((resolve) => {
|
||||
finishCommit = resolve;
|
||||
});
|
||||
const applyTranscriptCommit = vi.fn(async () => {
|
||||
await commitBlocked;
|
||||
return {
|
||||
ok: true as const,
|
||||
result: { entryIds: ["entry-placement-race"], newLeafId: "entry-placement-race" },
|
||||
};
|
||||
});
|
||||
const workerService = createService(createProvider(), {
|
||||
applyTranscriptCommit,
|
||||
placementStore,
|
||||
});
|
||||
|
||||
const commit = workerService.commitTranscript(identity, {
|
||||
runEpoch: identity.ownerEpoch,
|
||||
seq: 1,
|
||||
baseLeafId: null,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "commit before claim fence" }],
|
||||
timestamp: 1,
|
||||
},
|
||||
],
|
||||
});
|
||||
await vi.waitFor(() => expect(applyTranscriptCommit).toHaveBeenCalledOnce());
|
||||
placementStore.validateWorkerTurn.mockReturnValue(false);
|
||||
finishCommit?.();
|
||||
|
||||
await expect(commit).resolves.toEqual({ ok: false, closeReason: "placement-mismatch" });
|
||||
expect(placementStore.validateWorkerTurn).toHaveBeenCalledTimes(2);
|
||||
expect(placementStore.updateAckCursors).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("advances the transcript cursor when a stale-base commit consumes its sequence", async () => {
|
||||
const identity = seedAttachedIdentity("worker-placement-stale", "session-placement-stale");
|
||||
const placementStore = {
|
||||
validateWorkerTurn: vi.fn(() => true),
|
||||
updateAckCursors: vi.fn(),
|
||||
};
|
||||
const applyTranscriptCommit = vi
|
||||
.fn<NonNullable<WorkerEnvironmentServiceOptions["applyTranscriptCommit"]>>()
|
||||
.mockResolvedValueOnce({ ok: false, reason: "stale-base-leaf" })
|
||||
.mockResolvedValueOnce({ ok: false, reason: "invalid-batch" });
|
||||
const workerService = createService(createProvider(), {
|
||||
applyTranscriptCommit,
|
||||
placementStore,
|
||||
});
|
||||
const request = {
|
||||
runEpoch: identity.ownerEpoch,
|
||||
seq: 11,
|
||||
baseLeafId: "stale-leaf",
|
||||
messages: [
|
||||
{
|
||||
role: "user" as const,
|
||||
content: [{ type: "text" as const, text: "stale commit" }],
|
||||
timestamp: 1,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
await expect(workerService.commitTranscript(identity, request)).resolves.toEqual({
|
||||
ok: false,
|
||||
reason: "stale-base-leaf",
|
||||
});
|
||||
expect(placementStore.updateAckCursors).toHaveBeenCalledWith({
|
||||
sessionId: identity.sessionId,
|
||||
environmentId: identity.environmentId,
|
||||
ownerEpoch: identity.ownerEpoch,
|
||||
runId: identity.runId,
|
||||
transcriptSeq: 11,
|
||||
});
|
||||
|
||||
await expect(
|
||||
workerService.commitTranscript(identity, { ...request, seq: 12 }),
|
||||
).resolves.toEqual({ ok: false, reason: "invalid-batch" });
|
||||
expect(placementStore.updateAckCursors).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("fences after a buffered terminal event becomes acknowledged by a gap fill", async () => {
|
||||
const identity = seedAttachedIdentity("worker-placement-gap", "session-placement-gap");
|
||||
const placementStore = {
|
||||
validateWorkerTurn: vi.fn(() => true),
|
||||
updateAckCursors: vi.fn(),
|
||||
};
|
||||
const applyTranscriptCommit = vi.fn(async () => ({
|
||||
ok: true as const,
|
||||
result: { entryIds: ["entry-after-terminal-gap"], newLeafId: "entry-after-terminal-gap" },
|
||||
}));
|
||||
const liveApply = vi.fn(
|
||||
({
|
||||
request,
|
||||
}: Parameters<NonNullable<WorkerEnvironmentServiceOptions["liveEvents"]>["apply"]>[0]) => ({
|
||||
ok: true as const,
|
||||
result: { ackedSeq: request.seq === 1 ? 2 : 0 },
|
||||
}),
|
||||
);
|
||||
const workerService = createService(createProvider(), {
|
||||
applyTranscriptCommit,
|
||||
liveEvents: createLiveEvents({ apply: liveApply }),
|
||||
placementStore,
|
||||
});
|
||||
|
||||
await expect(
|
||||
workerService.pushLiveEvent(identity, {
|
||||
runEpoch: identity.ownerEpoch,
|
||||
lastAckedSeq: 0,
|
||||
seq: 2,
|
||||
runId: identity.runId ?? "run-missing",
|
||||
event: { kind: "lifecycle", payload: { phase: "end", endedAt: 2 } },
|
||||
}),
|
||||
).resolves.toEqual({ ok: true, result: { ackedSeq: 0 } });
|
||||
expect(placementStore.updateAckCursors).toHaveBeenCalledWith({
|
||||
sessionId: identity.sessionId,
|
||||
environmentId: identity.environmentId,
|
||||
ownerEpoch: identity.ownerEpoch,
|
||||
runId: identity.runId,
|
||||
liveSeq: 0,
|
||||
});
|
||||
|
||||
await expect(
|
||||
workerService.pushLiveEvent(identity, {
|
||||
runEpoch: identity.ownerEpoch,
|
||||
lastAckedSeq: 0,
|
||||
seq: 1,
|
||||
runId: identity.runId ?? "run-missing",
|
||||
event: { kind: "assistant", payload: { text: "fills gap", delta: "fills gap" } },
|
||||
}),
|
||||
).resolves.toEqual({ ok: true, result: { ackedSeq: 2 } });
|
||||
await expect(
|
||||
workerService.commitTranscript(identity, {
|
||||
runEpoch: identity.ownerEpoch,
|
||||
seq: 1,
|
||||
baseLeafId: null,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "late transcript" }],
|
||||
timestamp: 1,
|
||||
},
|
||||
],
|
||||
}),
|
||||
).resolves.toEqual({ ok: false, closeReason: "placement-mismatch" });
|
||||
await expect(
|
||||
workerService.pushLiveEvent(identity, {
|
||||
runEpoch: identity.ownerEpoch,
|
||||
lastAckedSeq: 2,
|
||||
seq: 3,
|
||||
runId: identity.runId ?? "run-missing",
|
||||
event: { kind: "assistant", payload: { text: "late", delta: "late" } },
|
||||
}),
|
||||
).resolves.toEqual({ ok: false, closeReason: "placement-mismatch" });
|
||||
expect(applyTranscriptCommit).not.toHaveBeenCalled();
|
||||
expect(liveApply).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("applies a terminal ACK only after its transcript commit finishes", async () => {
|
||||
const identity = seedAttachedIdentity("worker-placement-order", "session-placement-order");
|
||||
const placementStore = {
|
||||
validateWorkerTurn: vi.fn(() => true),
|
||||
updateAckCursors: vi.fn(),
|
||||
};
|
||||
let finishCommit: (() => void) | undefined;
|
||||
const commitBlocked = new Promise<void>((resolve) => {
|
||||
finishCommit = resolve;
|
||||
});
|
||||
const applyTranscriptCommit = vi.fn(async () => {
|
||||
await commitBlocked;
|
||||
return {
|
||||
ok: true as const,
|
||||
result: { entryIds: ["entry-order"], newLeafId: "entry-order" },
|
||||
};
|
||||
});
|
||||
const workerService = createService(createProvider(), {
|
||||
applyTranscriptCommit,
|
||||
liveEvents: createLiveEvents({
|
||||
apply: vi.fn(
|
||||
({
|
||||
request,
|
||||
}: Parameters<
|
||||
NonNullable<WorkerEnvironmentServiceOptions["liveEvents"]>["apply"]
|
||||
>[0]) => ({ ok: true as const, result: { ackedSeq: request.seq } }),
|
||||
),
|
||||
}),
|
||||
placementStore,
|
||||
});
|
||||
|
||||
const commit = workerService.commitTranscript(identity, {
|
||||
runEpoch: identity.ownerEpoch,
|
||||
seq: 1,
|
||||
baseLeafId: null,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "commit before terminal" }],
|
||||
timestamp: 1,
|
||||
},
|
||||
],
|
||||
});
|
||||
await vi.waitFor(() => expect(applyTranscriptCommit).toHaveBeenCalledOnce());
|
||||
const terminal = workerService.pushLiveEvent(identity, {
|
||||
runEpoch: identity.ownerEpoch,
|
||||
lastAckedSeq: 0,
|
||||
seq: 1,
|
||||
runId: identity.runId ?? "run-missing",
|
||||
event: { kind: "lifecycle", payload: { phase: "end", endedAt: 2 } },
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(placementStore.updateAckCursors).not.toHaveBeenCalled();
|
||||
|
||||
finishCommit?.();
|
||||
await expect(commit).resolves.toMatchObject({ ok: true });
|
||||
await expect(terminal).resolves.toEqual({ ok: true, result: { ackedSeq: 1 } });
|
||||
expect(placementStore.updateAckCursors.mock.calls).toEqual([
|
||||
[
|
||||
{
|
||||
sessionId: identity.sessionId,
|
||||
environmentId: identity.environmentId,
|
||||
ownerEpoch: identity.ownerEpoch,
|
||||
runId: identity.runId,
|
||||
transcriptSeq: 1,
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
sessionId: identity.sessionId,
|
||||
environmentId: identity.environmentId,
|
||||
ownerEpoch: identity.ownerEpoch,
|
||||
runId: identity.runId,
|
||||
liveSeq: 1,
|
||||
},
|
||||
],
|
||||
]);
|
||||
});
|
||||
|
||||
it("fences post-terminal mutations while preserving sequenced replays", async () => {
|
||||
const identity = seedAttachedIdentity("worker-terminal-fence", "session-terminal-fence");
|
||||
const placementStore = {
|
||||
validateWorkerTurn: vi.fn(() => true),
|
||||
updateAckCursors: vi.fn(),
|
||||
};
|
||||
const applyTranscriptCommit = vi.fn(async () => ({
|
||||
ok: true as const,
|
||||
result: { entryIds: ["entry-terminal"], newLeafId: "entry-terminal" },
|
||||
}));
|
||||
const liveApply = vi.fn(
|
||||
({
|
||||
request,
|
||||
}: Parameters<NonNullable<WorkerEnvironmentServiceOptions["liveEvents"]>["apply"]>[0]) => ({
|
||||
ok: true as const,
|
||||
result: { ackedSeq: request.seq },
|
||||
}),
|
||||
);
|
||||
const executeInference = vi.fn<WorkerEnvironmentServiceOptions["executeInference"]>(
|
||||
async () => ({
|
||||
type: "error",
|
||||
reason: "provider-error",
|
||||
message: "Provider request failed",
|
||||
}),
|
||||
);
|
||||
const workerService = createService(createProvider(), {
|
||||
applyTranscriptCommit,
|
||||
executeInference,
|
||||
liveEvents: createLiveEvents({ apply: liveApply }),
|
||||
placementStore,
|
||||
});
|
||||
const transcript = {
|
||||
runEpoch: identity.ownerEpoch,
|
||||
seq: 1,
|
||||
baseLeafId: null,
|
||||
messages: [
|
||||
{
|
||||
role: "user" as const,
|
||||
content: [{ type: "text" as const, text: "terminal fence" }],
|
||||
timestamp: 1,
|
||||
},
|
||||
],
|
||||
};
|
||||
const terminal = {
|
||||
runEpoch: identity.ownerEpoch,
|
||||
lastAckedSeq: 0,
|
||||
seq: 1,
|
||||
runId: identity.runId ?? "run-missing",
|
||||
event: { kind: "lifecycle" as const, payload: { phase: "end" as const, endedAt: 2 } },
|
||||
};
|
||||
|
||||
await expect(workerService.commitTranscript(identity, transcript)).resolves.toMatchObject({
|
||||
ok: true,
|
||||
});
|
||||
await expect(workerService.pushLiveEvent(identity, terminal)).resolves.toEqual({
|
||||
ok: true,
|
||||
result: { ackedSeq: 1 },
|
||||
});
|
||||
|
||||
await expect(workerService.commitTranscript(identity, transcript)).resolves.toMatchObject({
|
||||
ok: true,
|
||||
});
|
||||
await expect(
|
||||
workerService.commitTranscript(identity, { ...transcript, seq: 2 }),
|
||||
).resolves.toEqual({ ok: false, closeReason: "placement-mismatch" });
|
||||
expect(applyTranscriptCommit).toHaveBeenCalledTimes(2);
|
||||
|
||||
await expect(workerService.pushLiveEvent(identity, terminal)).resolves.toEqual({
|
||||
ok: true,
|
||||
result: { ackedSeq: 1 },
|
||||
});
|
||||
await expect(
|
||||
workerService.pushLiveEvent(identity, {
|
||||
...terminal,
|
||||
seq: 2,
|
||||
event: { kind: "assistant", payload: { text: "late", delta: "late" } },
|
||||
}),
|
||||
).resolves.toEqual({ ok: false, closeReason: "placement-mismatch" });
|
||||
expect(liveApply).toHaveBeenCalledTimes(2);
|
||||
|
||||
expect(
|
||||
workerService.startInference(identity, inferenceRequest(identity), {
|
||||
connectionId: "connection-terminal-fence",
|
||||
send: vi.fn(),
|
||||
}),
|
||||
).toEqual({ ok: false, closeReason: "placement-mismatch" });
|
||||
expect(workerService.cancelInference(identity, inferenceRequest(identity))).toEqual({
|
||||
ok: false,
|
||||
closeReason: "placement-mismatch",
|
||||
});
|
||||
expect(executeInference).not.toHaveBeenCalled();
|
||||
|
||||
const rotatedCredentialHash = hashWorkerCredential(
|
||||
["rotated", identity.environmentId, identity.sessionId].join("-"),
|
||||
);
|
||||
database.db
|
||||
.prepare(
|
||||
"UPDATE worker_environment_credentials SET credential_hash = ? WHERE environment_id = ?",
|
||||
)
|
||||
.run(rotatedCredentialHash, identity.environmentId);
|
||||
const rotatedIdentity = { ...identity, credentialHash: rotatedCredentialHash };
|
||||
await expect(
|
||||
workerService.commitTranscript(rotatedIdentity, { ...transcript, seq: 2 }),
|
||||
).resolves.toMatchObject({ ok: true });
|
||||
expect(applyTranscriptCommit).toHaveBeenCalledTimes(3);
|
||||
});
|
||||
|
||||
it("does not treat a terminal event on an already ACKed sequence as authoritative", async () => {
|
||||
const identity = seedAttachedIdentity("worker-terminal-reuse", "session-terminal-reuse");
|
||||
const applyTranscriptCommit = vi.fn(async () => ({
|
||||
ok: true as const,
|
||||
result: { entryIds: ["entry-after-reuse"], newLeafId: "entry-after-reuse" },
|
||||
}));
|
||||
const workerService = createService(createProvider(), {
|
||||
applyTranscriptCommit,
|
||||
liveEvents: createLiveEvents({
|
||||
apply: vi.fn(
|
||||
({
|
||||
request,
|
||||
}: Parameters<
|
||||
NonNullable<WorkerEnvironmentServiceOptions["liveEvents"]>["apply"]
|
||||
>[0]) => ({ ok: true as const, result: { ackedSeq: request.seq } }),
|
||||
),
|
||||
}),
|
||||
placementStore: {
|
||||
validateWorkerTurn: vi.fn(() => true),
|
||||
updateAckCursors: vi.fn(),
|
||||
},
|
||||
});
|
||||
const event = {
|
||||
runEpoch: identity.ownerEpoch,
|
||||
lastAckedSeq: 0,
|
||||
seq: 1,
|
||||
runId: identity.runId ?? "run-missing",
|
||||
event: { kind: "assistant" as const, payload: { text: "first", delta: "first" } },
|
||||
};
|
||||
|
||||
await expect(workerService.pushLiveEvent(identity, event)).resolves.toMatchObject({ ok: true });
|
||||
await expect(
|
||||
workerService.pushLiveEvent(identity, {
|
||||
...event,
|
||||
event: { kind: "lifecycle", payload: { phase: "end", endedAt: 2 } },
|
||||
}),
|
||||
).resolves.toMatchObject({ ok: true });
|
||||
await expect(
|
||||
workerService.commitTranscript(identity, {
|
||||
runEpoch: identity.ownerEpoch,
|
||||
seq: 1,
|
||||
baseLeafId: null,
|
||||
messages: [
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "still mutable" }],
|
||||
timestamp: 1,
|
||||
},
|
||||
],
|
||||
}),
|
||||
).resolves.toMatchObject({ ok: true });
|
||||
expect(applyTranscriptCommit).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("fences inference by epoch and the durable session credential", async () => {
|
||||
const identity = seedAttachedIdentity("worker-inference-fence", "session-inference-fence");
|
||||
const executeInference = vi.fn<WorkerEnvironmentServiceOptions["executeInference"]>(
|
||||
@@ -1526,6 +2074,7 @@ describe("worker environment service", () => {
|
||||
ownerEpoch: request.ownerEpoch,
|
||||
remoteSocketPath: "/tmp/worker/gateway.sock",
|
||||
runWorkspaceCommand: vi.fn(),
|
||||
syncWorkspace: vi.fn(),
|
||||
stop: async () => {},
|
||||
};
|
||||
}),
|
||||
|
||||
@@ -93,6 +93,15 @@ const serviceError = (code: WorkerEnvironmentServiceErrorCode, message: string)
|
||||
new WorkerEnvironmentServiceError(code, message);
|
||||
const ORPHANED_LEASE_ERROR = "Worker provider no longer recognizes the lease";
|
||||
|
||||
function workerEnvironmentIdempotencyDigest(idempotencyKey: string): string {
|
||||
return createHash("sha256").update(idempotencyKey).digest("hex");
|
||||
}
|
||||
|
||||
export function workerEnvironmentIdForIdempotencyKey(idempotencyKey: string): string {
|
||||
const digest = workerEnvironmentIdempotencyDigest(idempotencyKey);
|
||||
return `worker:${digest.slice(0, 32)}`;
|
||||
}
|
||||
|
||||
type WorkerEnvironmentServiceOptions = {
|
||||
store: WorkerEnvironmentStore;
|
||||
getConfig: () => OpenClawConfig;
|
||||
@@ -131,6 +140,42 @@ type WorkerEnvironmentServiceOptions = {
|
||||
>;
|
||||
executeInference: WorkerInferenceExecutor;
|
||||
inferenceStore?: WorkerInferenceStore;
|
||||
placementStore?: WorkerSessionPlacementGate;
|
||||
};
|
||||
|
||||
export type WorkerPlacementTurnBinding = Readonly<{
|
||||
sessionId: string;
|
||||
environmentId: string;
|
||||
ownerEpoch: number;
|
||||
runId: string;
|
||||
}>;
|
||||
|
||||
type WorkerProcessTurnBinding = WorkerPlacementTurnBinding & {
|
||||
credentialHash: string;
|
||||
};
|
||||
|
||||
type WorkerTerminalTurnFence = WorkerProcessTurnBinding & {
|
||||
transcriptSeq: number;
|
||||
liveSeq: number;
|
||||
};
|
||||
|
||||
type WorkerPendingTerminalTurnFence = WorkerProcessTurnBinding & {
|
||||
terminalLiveSeq: number;
|
||||
};
|
||||
|
||||
type WorkerTurnRequest =
|
||||
| { kind: "inference" }
|
||||
| { kind: "live"; seq: number }
|
||||
| { kind: "transcript"; seq: number };
|
||||
|
||||
export type WorkerSessionPlacementGate = {
|
||||
validateWorkerTurn(binding: WorkerPlacementTurnBinding): boolean;
|
||||
updateAckCursors(
|
||||
binding: WorkerPlacementTurnBinding & {
|
||||
transcriptSeq?: number;
|
||||
liveSeq?: number;
|
||||
},
|
||||
): void;
|
||||
};
|
||||
|
||||
type WorkerTranscriptCommitApplicationResult =
|
||||
@@ -211,6 +256,9 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService
|
||||
const operations = new KeyedAsyncQueue();
|
||||
const activeOperations = new Set<Promise<unknown>>();
|
||||
const pendingCredentials = new Map<string, MintedWorkerCredential>();
|
||||
const observedAckCursors = new Map<string, WorkerTerminalTurnFence>();
|
||||
const pendingTerminalTurnFences = new Map<string, WorkerPendingTerminalTurnFence>();
|
||||
const terminalTurnFences = new Map<string, WorkerTerminalTurnFence>();
|
||||
const now = options.now ?? Date.now;
|
||||
const inference = createWorkerInferenceManager({
|
||||
execute: options.executeInference,
|
||||
@@ -223,6 +271,86 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService
|
||||
let unsubscribeSessionIdentityMutation: (() => void) | undefined;
|
||||
let stopping = false;
|
||||
|
||||
const placementBinding = (
|
||||
identity: WorkerConnectionIdentity,
|
||||
): WorkerPlacementTurnBinding | undefined => {
|
||||
if (!identity.sessionId || !identity.runId) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
sessionId: identity.sessionId,
|
||||
environmentId: identity.environmentId,
|
||||
ownerEpoch: identity.ownerEpoch,
|
||||
runId: identity.runId,
|
||||
};
|
||||
};
|
||||
|
||||
const processTurnBinding = (
|
||||
identity: WorkerConnectionIdentity,
|
||||
): WorkerProcessTurnBinding | undefined => {
|
||||
const placement = placementBinding(identity);
|
||||
return placement ? { ...placement, credentialHash: identity.credentialHash } : undefined;
|
||||
};
|
||||
|
||||
const matchesTurnBinding = (
|
||||
left: WorkerProcessTurnBinding,
|
||||
right: WorkerProcessTurnBinding,
|
||||
): boolean =>
|
||||
left.sessionId === right.sessionId &&
|
||||
left.environmentId === right.environmentId &&
|
||||
left.ownerEpoch === right.ownerEpoch &&
|
||||
left.runId === right.runId &&
|
||||
safeEqualSecret(left.credentialHash, right.credentialHash);
|
||||
|
||||
const recordAckCursor = (
|
||||
binding: WorkerProcessTurnBinding,
|
||||
cursor: { transcriptSeq: number } | { liveSeq: number },
|
||||
): WorkerTerminalTurnFence => {
|
||||
const current = observedAckCursors.get(binding.sessionId);
|
||||
const currentTurn = current && matchesTurnBinding(current, binding) ? current : undefined;
|
||||
const next: WorkerTerminalTurnFence = {
|
||||
...binding,
|
||||
transcriptSeq:
|
||||
"transcriptSeq" in cursor
|
||||
? Math.max(currentTurn?.transcriptSeq ?? 0, cursor.transcriptSeq)
|
||||
: (currentTurn?.transcriptSeq ?? 0),
|
||||
liveSeq:
|
||||
"liveSeq" in cursor
|
||||
? Math.max(currentTurn?.liveSeq ?? 0, cursor.liveSeq)
|
||||
: (currentTurn?.liveSeq ?? 0),
|
||||
};
|
||||
observedAckCursors.set(binding.sessionId, next);
|
||||
return next;
|
||||
};
|
||||
|
||||
const observedAckCursorFor = (
|
||||
binding: WorkerProcessTurnBinding,
|
||||
): WorkerTerminalTurnFence | undefined => {
|
||||
const observed = observedAckCursors.get(binding.sessionId);
|
||||
return observed && matchesTurnBinding(observed, binding) ? observed : undefined;
|
||||
};
|
||||
|
||||
const validateWorkerPlacement = (
|
||||
identity: WorkerConnectionIdentity,
|
||||
): { durableClaim: boolean; valid: boolean } => {
|
||||
if (!options.placementStore) {
|
||||
return { durableClaim: false, valid: true };
|
||||
}
|
||||
if (identity.sessionId === null && identity.runId === null) {
|
||||
return { durableClaim: false, valid: true };
|
||||
}
|
||||
const binding = placementBinding(identity);
|
||||
const valid = binding ? options.placementStore.validateWorkerTurn(binding) : false;
|
||||
return { durableClaim: valid, valid };
|
||||
};
|
||||
|
||||
const isTerminalLiveEvent = (request: WorkerLiveEventParams): boolean =>
|
||||
request.event.kind === "lifecycle" &&
|
||||
(request.event.payload.phase === "end" ||
|
||||
(request.event.payload.phase === "error" &&
|
||||
(request.event.payload.aborted === true ||
|
||||
request.event.payload.fallbackExhaustedFailure === true)));
|
||||
|
||||
const project = (record: WorkerEnvironmentRecord) => ({
|
||||
...record,
|
||||
tunnelStatus: tunnels?.status(record.environmentId) ?? ("stopped" as const),
|
||||
@@ -753,7 +881,7 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService
|
||||
if (!normalizedProfileId || normalizedProfileId !== profileId) {
|
||||
throw serviceError("invalid_profile", "Worker profile id must be non-empty and trimmed");
|
||||
}
|
||||
const digest = createHash("sha256").update(idempotencyKey).digest("hex");
|
||||
const digest = workerEnvironmentIdempotencyDigest(idempotencyKey);
|
||||
const environmentId = `worker:${digest.slice(0, 32)}`;
|
||||
return withLock(environmentId, async () => {
|
||||
if (stopping) {
|
||||
@@ -1038,6 +1166,9 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService
|
||||
await Promise.allSettled(activeOperations);
|
||||
}
|
||||
pendingCredentials.clear();
|
||||
observedAckCursors.clear();
|
||||
pendingTerminalTurnFences.clear();
|
||||
terminalTurnFences.clear();
|
||||
options.liveEvents?.clear();
|
||||
};
|
||||
|
||||
@@ -1077,6 +1208,7 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService
|
||||
const validateAttachedWorkerRequest = (
|
||||
identity: WorkerConnectionIdentity,
|
||||
runEpoch: number,
|
||||
request: WorkerTurnRequest,
|
||||
):
|
||||
| { ok: true }
|
||||
| { ok: false; closeReason: WorkerProtocolCloseReason }
|
||||
@@ -1084,11 +1216,29 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService
|
||||
if (stopping) {
|
||||
return { ok: false, closeReason: "environment-unavailable" };
|
||||
}
|
||||
const placement = validateWorkerPlacement(identity);
|
||||
if (!placement.valid) {
|
||||
return { ok: false, closeReason: "placement-mismatch" };
|
||||
}
|
||||
const turnBinding = processTurnBinding(identity);
|
||||
const terminalFence = identity.sessionId
|
||||
? terminalTurnFences.get(identity.sessionId)
|
||||
: undefined;
|
||||
if (turnBinding && terminalFence && matchesTurnBinding(terminalFence, turnBinding)) {
|
||||
const isReplay =
|
||||
(request.kind === "transcript" && request.seq <= terminalFence.transcriptSeq) ||
|
||||
(request.kind === "live" && request.seq <= terminalFence.liveSeq);
|
||||
if (!isReplay) {
|
||||
return { ok: false, closeReason: "placement-mismatch" };
|
||||
}
|
||||
}
|
||||
const credential = store.getCredential(identity.environmentId);
|
||||
if (!credential || !safeEqualSecret(credential.credentialHash, identity.credentialHash)) {
|
||||
return { ok: false, closeReason: "credential-replaced" };
|
||||
}
|
||||
if (now() >= credential.expiresAtMs) {
|
||||
// TTL limits admission and reconnect. An already-admitted exact durable
|
||||
// turn stays usable until its terminal ACK or placement fence.
|
||||
if (now() >= credential.expiresAtMs && !placement.durableClaim) {
|
||||
return { ok: false, closeReason: "credential-expired" };
|
||||
}
|
||||
const environment = store.get(identity.environmentId);
|
||||
@@ -1111,6 +1261,11 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService
|
||||
) {
|
||||
return { ok: false, reason: "session-not-attached" };
|
||||
}
|
||||
if (turnBinding && terminalFence && !matchesTurnBinding(terminalFence, turnBinding)) {
|
||||
// Credential rotation identifies a new process turn even when a caller
|
||||
// intentionally reuses its durable run id (for example, cron sessions).
|
||||
terminalTurnFences.delete(turnBinding.sessionId);
|
||||
}
|
||||
return { ok: true };
|
||||
};
|
||||
|
||||
@@ -1119,32 +1274,118 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService
|
||||
request: WorkerTranscriptCommitParams,
|
||||
): Promise<WorkerTranscriptCommitServiceResult> =>
|
||||
withLock(identity.environmentId, async () => {
|
||||
const binding = validateAttachedWorkerRequest(identity, request.runEpoch);
|
||||
const binding = validateAttachedWorkerRequest(identity, request.runEpoch, {
|
||||
kind: "transcript",
|
||||
seq: request.seq,
|
||||
});
|
||||
if (!binding.ok) {
|
||||
return binding;
|
||||
}
|
||||
if (!options.applyTranscriptCommit) {
|
||||
return { ok: false, closeReason: "gateway-unavailable" };
|
||||
}
|
||||
return await options.applyTranscriptCommit({ identity, request });
|
||||
const result = await options.applyTranscriptCommit({ identity, request });
|
||||
// Transcript persistence awaits outside the placement transaction. Revalidate the durable
|
||||
// claim before exposing an ACK so reclamation cannot admit both owners for one session.
|
||||
const currentBinding = validateAttachedWorkerRequest(identity, request.runEpoch, {
|
||||
kind: "transcript",
|
||||
seq: request.seq,
|
||||
});
|
||||
if (!currentBinding.ok) {
|
||||
return currentBinding;
|
||||
}
|
||||
// Stale base is a terminal sequenced outcome. Advance its durable cursor
|
||||
// so the next worker commit cannot reuse the consumed sequence number.
|
||||
if (result.ok || result.reason === "stale-base-leaf") {
|
||||
const placement = placementBinding(identity);
|
||||
const processTurn = processTurnBinding(identity);
|
||||
if (!placement || !processTurn) {
|
||||
return { ok: false, closeReason: "placement-mismatch" };
|
||||
}
|
||||
options.placementStore?.updateAckCursors({ ...placement, transcriptSeq: request.seq });
|
||||
recordAckCursor(processTurn, { transcriptSeq: request.seq });
|
||||
}
|
||||
return result;
|
||||
});
|
||||
|
||||
const pushLiveEvent = (
|
||||
const applyLiveEvent = (
|
||||
identity: WorkerConnectionIdentity,
|
||||
request: WorkerLiveEventParams,
|
||||
): WorkerLiveEventServiceResult => {
|
||||
const binding = validateAttachedWorkerRequest(identity, request.runEpoch, {
|
||||
kind: "live",
|
||||
seq: request.seq,
|
||||
});
|
||||
if (!binding.ok) {
|
||||
if ("closeReason" in binding) {
|
||||
return binding;
|
||||
}
|
||||
return { ok: false, details: { reason: binding.reason } };
|
||||
}
|
||||
if (request.runId !== identity.runId) {
|
||||
return { ok: false, closeReason: "placement-mismatch" };
|
||||
}
|
||||
if (!options.liveEvents) {
|
||||
return { ok: false, closeReason: "gateway-unavailable" };
|
||||
}
|
||||
// The caller holds the environment lock, preserving order with transcript
|
||||
// commits and the terminal mutation fence while this synchronous receiver runs.
|
||||
const result = options.liveEvents.apply({ identity, request });
|
||||
if (result.ok) {
|
||||
const placement = placementBinding(identity);
|
||||
const processTurn = processTurnBinding(identity);
|
||||
if (!placement || !processTurn) {
|
||||
return { ok: false, closeReason: "placement-mismatch" };
|
||||
}
|
||||
options.placementStore?.updateAckCursors({
|
||||
...placement,
|
||||
liveSeq: result.result.ackedSeq,
|
||||
});
|
||||
recordAckCursor(processTurn, { liveSeq: result.result.ackedSeq });
|
||||
}
|
||||
return result;
|
||||
};
|
||||
|
||||
const pushLiveEvent = async (
|
||||
identity: WorkerConnectionIdentity,
|
||||
request: WorkerLiveEventParams,
|
||||
): Promise<WorkerLiveEventServiceResult> => {
|
||||
const binding = validateAttachedWorkerRequest(identity, request.runEpoch);
|
||||
if (!binding.ok) {
|
||||
if ("closeReason" in binding) {
|
||||
return Promise.resolve(binding);
|
||||
return await withLock(identity.environmentId, async () => {
|
||||
const placement = placementBinding(identity);
|
||||
const processTurn = processTurnBinding(identity);
|
||||
const observed = processTurn ? observedAckCursorFor(processTurn) : undefined;
|
||||
const wasNewSequence = request.seq > (observed?.liveSeq ?? 0);
|
||||
const result = applyLiveEvent(identity, request);
|
||||
if (!result.ok || !placement || !processTurn) {
|
||||
return result;
|
||||
}
|
||||
return Promise.resolve({ ok: false, details: { reason: binding.reason } });
|
||||
}
|
||||
if (!options.liveEvents) {
|
||||
return Promise.resolve({ ok: false, closeReason: "gateway-unavailable" });
|
||||
}
|
||||
// Publish after authoritative validation without blocking on lifecycle work.
|
||||
return Promise.resolve(options.liveEvents.apply({ identity, request }));
|
||||
const pending = pendingTerminalTurnFences.get(placement.sessionId);
|
||||
if (pending && !matchesTurnBinding(pending, processTurn)) {
|
||||
pendingTerminalTurnFences.delete(placement.sessionId);
|
||||
}
|
||||
if (isTerminalLiveEvent(request) && wasNewSequence) {
|
||||
pendingTerminalTurnFences.set(placement.sessionId, {
|
||||
...processTurn,
|
||||
terminalLiveSeq: request.seq,
|
||||
});
|
||||
}
|
||||
const terminal = pendingTerminalTurnFences.get(placement.sessionId);
|
||||
if (
|
||||
terminal &&
|
||||
matchesTurnBinding(terminal, processTurn) &&
|
||||
result.result.ackedSeq >= terminal.terminalLiveSeq
|
||||
) {
|
||||
// A gap fill can ACK a previously buffered terminal event. Fence from
|
||||
// the observed high-water marks, not only from the request carrying it.
|
||||
terminalTurnFences.set(
|
||||
placement.sessionId,
|
||||
observedAckCursorFor(processTurn) ??
|
||||
recordAckCursor(processTurn, { liveSeq: result.result.ackedSeq }),
|
||||
);
|
||||
pendingTerminalTurnFences.delete(placement.sessionId);
|
||||
}
|
||||
return result;
|
||||
});
|
||||
};
|
||||
|
||||
const revalidateInference = (
|
||||
@@ -1154,7 +1395,9 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService
|
||||
if (request.sessionId !== identity.sessionId) {
|
||||
return "session-not-attached";
|
||||
}
|
||||
const binding = validateAttachedWorkerRequest(identity, request.runEpoch);
|
||||
const binding = validateAttachedWorkerRequest(identity, request.runEpoch, {
|
||||
kind: "inference",
|
||||
});
|
||||
return binding.ok ? null : "reason" in binding ? binding.reason : "session-not-attached";
|
||||
};
|
||||
|
||||
@@ -1163,10 +1406,12 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService
|
||||
request: WorkerInferenceStartParams,
|
||||
sink: WorkerInferenceSink,
|
||||
): WorkerInferenceStartServiceResult => {
|
||||
if (request.sessionId !== identity.sessionId) {
|
||||
if (request.sessionId !== identity.sessionId || request.runId !== identity.runId) {
|
||||
return { ok: false, reason: "session-not-attached" };
|
||||
}
|
||||
const binding = validateAttachedWorkerRequest(identity, request.runEpoch);
|
||||
const binding = validateAttachedWorkerRequest(identity, request.runEpoch, {
|
||||
kind: "inference",
|
||||
});
|
||||
if (!binding.ok) {
|
||||
return binding;
|
||||
}
|
||||
@@ -1182,10 +1427,12 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService
|
||||
identity: WorkerConnectionIdentity,
|
||||
request: WorkerInferenceCancelParams,
|
||||
): WorkerInferenceCancelServiceResult => {
|
||||
if (request.sessionId !== identity.sessionId) {
|
||||
if (request.sessionId !== identity.sessionId || request.runId !== identity.runId) {
|
||||
return { ok: false, reason: "session-not-attached" };
|
||||
}
|
||||
const binding = validateAttachedWorkerRequest(identity, request.runEpoch);
|
||||
const binding = validateAttachedWorkerRequest(identity, request.runEpoch, {
|
||||
kind: "inference",
|
||||
});
|
||||
if (!binding.ok) {
|
||||
return binding;
|
||||
}
|
||||
@@ -1227,12 +1474,41 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService
|
||||
if (stopping) {
|
||||
return { ok: false, reason: "environment-unavailable" } as const;
|
||||
}
|
||||
return admitWorkerConnection({ store, admission, expectedBuild, nowMs: now() });
|
||||
const admitted = admitWorkerConnection({ store, admission, expectedBuild, nowMs: now() });
|
||||
if (
|
||||
!admitted.ok ||
|
||||
!options.placementStore ||
|
||||
(admitted.identity.sessionId === null && admitted.identity.runId === null)
|
||||
) {
|
||||
return admitted;
|
||||
}
|
||||
const placement = placementBinding(admitted.identity);
|
||||
if (!placement || !options.placementStore.validateWorkerTurn(placement)) {
|
||||
return { ok: false, reason: "placement-mismatch" } as const;
|
||||
}
|
||||
return admitted;
|
||||
},
|
||||
validateWorkerConnection: (identity: WorkerConnectionIdentity) => {
|
||||
if (stopping) {
|
||||
return "environment-unavailable" as const;
|
||||
}
|
||||
const placement = validateWorkerPlacement(identity);
|
||||
if (!placement.valid) {
|
||||
return "placement-mismatch" as const;
|
||||
}
|
||||
const environmentFailure = validateWorkerConnectionIdentity({
|
||||
store,
|
||||
identity,
|
||||
nowMs: now(),
|
||||
});
|
||||
if (
|
||||
environmentFailure &&
|
||||
!(environmentFailure === "credential-expired" && placement.durableClaim)
|
||||
) {
|
||||
return environmentFailure;
|
||||
}
|
||||
return null;
|
||||
},
|
||||
validateWorkerConnection: (identity: WorkerConnectionIdentity) =>
|
||||
stopping
|
||||
? ("environment-unavailable" as const)
|
||||
: validateWorkerConnectionIdentity({ store, identity, nowMs: now() }),
|
||||
commitTranscript,
|
||||
pushLiveEvent,
|
||||
startInference,
|
||||
@@ -1246,6 +1522,36 @@ export function createWorkerEnvironmentService(options: WorkerEnvironmentService
|
||||
attachSession,
|
||||
takeMintedCredential: (binding: WorkerCredentialBinding) =>
|
||||
readPendingCredential(binding)?.grant,
|
||||
acquireTurnCredential: (binding: WorkerCredentialBinding & { sessionId: string }) =>
|
||||
withLock(binding.environmentId, async () => {
|
||||
const pending = readPendingCredential(binding)?.grant;
|
||||
if (pending) {
|
||||
return pending;
|
||||
}
|
||||
const environment = store.get(binding.environmentId);
|
||||
if (
|
||||
!environment ||
|
||||
environment.state !== "attached" ||
|
||||
environment.ownerEpoch !== binding.ownerEpoch ||
|
||||
environment.attachedSessionIds.length !== 1 ||
|
||||
environment.attachedSessionIds[0] !== binding.sessionId
|
||||
) {
|
||||
throw serviceError("invalid_state", "Worker session credential owner is not attached");
|
||||
}
|
||||
const previous = store.getCredential(binding.environmentId);
|
||||
const minted = mintCredentialLocked(binding);
|
||||
const grant = stageCredential(minted.grant);
|
||||
if (previous?.sessionId === binding.sessionId) {
|
||||
options.liveEvents?.rotateCredential({
|
||||
credentialHash: minted.credentialHash,
|
||||
environmentId: binding.environmentId,
|
||||
previousCredentialHash: previous.credentialHash,
|
||||
runEpoch: binding.ownerEpoch,
|
||||
sessionId: binding.sessionId,
|
||||
});
|
||||
}
|
||||
return grant;
|
||||
}),
|
||||
acknowledgeCredentialDelivery: (claim: WorkerCredentialDeliveryClaim): boolean => {
|
||||
const pending = readPendingCredential(claim);
|
||||
if (!pending || pending.grant.deliveryId !== claim.deliveryId) {
|
||||
|
||||
@@ -38,6 +38,7 @@ const IDENTITY: WorkerConnectionIdentity = {
|
||||
credentialHash: ["credential", "hash", "a"].join("-"),
|
||||
bundleHash: "b".repeat(64),
|
||||
sessionId: SESSION_ID,
|
||||
runId: "run-worker-transcript",
|
||||
ownerEpoch: RUN_EPOCH,
|
||||
rpcSetVersion: 1,
|
||||
protocolFeatures: ["worker-transcript-commit-v1"],
|
||||
|
||||
@@ -7,10 +7,23 @@ export type WorkerTunnelRequest = {
|
||||
ownerEpoch: number;
|
||||
};
|
||||
|
||||
type WorkerWorkspaceCommand = {
|
||||
export type WorkerWorkspaceCommand = {
|
||||
argv: readonly string[];
|
||||
input?: string;
|
||||
timeoutMs?: number;
|
||||
signal?: AbortSignal;
|
||||
};
|
||||
|
||||
export type WorkerWorkspaceSyncRequest = {
|
||||
localPath: string;
|
||||
sessionId: string;
|
||||
generation: number;
|
||||
};
|
||||
|
||||
export type WorkerWorkspaceSyncResult = {
|
||||
mode: "git" | "plain";
|
||||
remoteWorkspaceDir: string;
|
||||
manifestRef: string;
|
||||
};
|
||||
|
||||
export type WorkerTunnelHandle = {
|
||||
@@ -18,5 +31,6 @@ export type WorkerTunnelHandle = {
|
||||
ownerEpoch: number;
|
||||
remoteSocketPath: string;
|
||||
runWorkspaceCommand(command: WorkerWorkspaceCommand): Promise<SpawnResult>;
|
||||
syncWorkspace(request: WorkerWorkspaceSyncRequest): Promise<WorkerWorkspaceSyncResult>;
|
||||
stop(): Promise<void>;
|
||||
};
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { sliceUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
|
||||
import { redactSensitiveText } from "../../logging/redact.js";
|
||||
import {
|
||||
runCommandWithTimeout,
|
||||
type CommandOptions,
|
||||
type SpawnResult,
|
||||
} from "../../process/exec.js";
|
||||
|
||||
export const WORKER_TUNNEL_READY_MARKER = "OPENCLAW_WORKER_TUNNEL_READY";
|
||||
|
||||
const STOP_GRACE_MS = 1_500;
|
||||
const STDERR_LIMIT = 4_096;
|
||||
|
||||
type WorkerSshProcessExit = {
|
||||
code: number | null;
|
||||
signal: NodeJS.Signals | null;
|
||||
};
|
||||
|
||||
export type WorkerSshProcess = {
|
||||
ready: Promise<void>;
|
||||
exited: Promise<WorkerSshProcessExit>;
|
||||
stop(): Promise<void>;
|
||||
};
|
||||
|
||||
export type WorkerSshRunner = {
|
||||
start(argv: string[], options: CommandOptions): WorkerSshProcess;
|
||||
run(argv: string[], options: CommandOptions): Promise<SpawnResult>;
|
||||
};
|
||||
|
||||
export function workerSshProcessError(stderr: string): Error {
|
||||
const detail = redactSensitiveText(stderr, { mode: "tools" }).replace(/\s+/gu, " ").trim();
|
||||
return new Error(detail ? `Worker SSH tunnel failed: ${detail}` : "Worker SSH tunnel failed");
|
||||
}
|
||||
|
||||
/** Production runner that treats the remote post-forward marker as connection readiness. */
|
||||
export function createWorkerSshRunner(): WorkerSshRunner {
|
||||
return {
|
||||
run: runCommandWithTimeout,
|
||||
start(argv, options) {
|
||||
const [command, ...args] = argv;
|
||||
if (!command) {
|
||||
throw new Error("Worker SSH runner requires a command");
|
||||
}
|
||||
const child = spawn(command, args, {
|
||||
env: options.baseEnv,
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
windowsHide: true,
|
||||
});
|
||||
let closed = false;
|
||||
let readySettled = false;
|
||||
let resolveReady!: () => void;
|
||||
let rejectReady!: (error: Error) => void;
|
||||
let resolveExited!: (exit: WorkerSshProcessExit) => void;
|
||||
const ready = new Promise<void>((resolve, reject) => {
|
||||
resolveReady = resolve;
|
||||
rejectReady = reject;
|
||||
});
|
||||
const exited = new Promise<WorkerSshProcessExit>((resolve) => {
|
||||
resolveExited = resolve;
|
||||
});
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
const settleReadyError = () => {
|
||||
if (readySettled) {
|
||||
return;
|
||||
}
|
||||
readySettled = true;
|
||||
rejectReady(workerSshProcessError(stderr));
|
||||
};
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stdout.on("error", () => {});
|
||||
child.stdout.on("data", (chunk: string) => {
|
||||
if (readySettled) {
|
||||
return;
|
||||
}
|
||||
stdout = sliceUtf16Safe(`${stdout}${chunk}`, -STDERR_LIMIT);
|
||||
if (stdout.split(/\r?\n/u).includes(WORKER_TUNNEL_READY_MARKER)) {
|
||||
readySettled = true;
|
||||
resolveReady();
|
||||
}
|
||||
});
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stderr.on("error", () => {});
|
||||
child.stderr.on("data", (chunk: string) => {
|
||||
stderr = sliceUtf16Safe(`${stderr}${chunk}`, -STDERR_LIMIT);
|
||||
});
|
||||
child.once("error", settleReadyError);
|
||||
child.once("close", (code, signal) => {
|
||||
closed = true;
|
||||
settleReadyError();
|
||||
resolveExited({ code, signal });
|
||||
});
|
||||
child.stdin.on("error", () => {});
|
||||
if (options.input !== undefined) {
|
||||
child.stdin.end(options.input);
|
||||
} else {
|
||||
child.stdin.end();
|
||||
}
|
||||
|
||||
let stopPromise: Promise<void> | undefined;
|
||||
return {
|
||||
ready,
|
||||
exited,
|
||||
stop() {
|
||||
return (stopPromise ??= (async () => {
|
||||
if (closed) {
|
||||
return;
|
||||
}
|
||||
child.kill("SIGTERM");
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
await Promise.race([
|
||||
exited,
|
||||
new Promise<void>((resolve) => {
|
||||
timer = setTimeout(resolve, STOP_GRACE_MS);
|
||||
timer.unref?.();
|
||||
}),
|
||||
]);
|
||||
clearTimeout(timer);
|
||||
if (!closed) {
|
||||
child.kill("SIGKILL");
|
||||
await exited;
|
||||
}
|
||||
})());
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -1,11 +1,20 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { WorkerSshEndpoint } from "../../plugins/types.js";
|
||||
import type { CommandOptions, SpawnResult } from "../../process/exec.js";
|
||||
import {
|
||||
runCommandWithTimeout,
|
||||
type CommandOptions,
|
||||
type SpawnResult,
|
||||
} from "../../process/exec.js";
|
||||
import {
|
||||
createWorkerSshRunner,
|
||||
type WorkerSshProcess,
|
||||
type WorkerSshRunner,
|
||||
} from "./tunnel-ssh-runner.js";
|
||||
import { createWorkerTunnelManager } from "./tunnel.js";
|
||||
|
||||
type WorkerTunnelOptions = NonNullable<Parameters<typeof createWorkerTunnelManager>[0]>;
|
||||
type WorkerSshRunner = NonNullable<WorkerTunnelOptions["runner"]>;
|
||||
type WorkerSshProcess = ReturnType<WorkerSshRunner["start"]>;
|
||||
type WorkerSshProcessExit = Awaited<WorkerSshProcess["exited"]>;
|
||||
|
||||
const HOST_KEY = [["ssh", "ed25519"].join("-"), "AAAA"].join(" ");
|
||||
@@ -17,10 +26,10 @@ const SSH: WorkerSshEndpoint = {
|
||||
keyRef: { source: "file", provider: "workers", id: "/identity" },
|
||||
};
|
||||
|
||||
function success(): SpawnResult {
|
||||
function success(stdout = "", stderr = ""): SpawnResult {
|
||||
return {
|
||||
stdout: "",
|
||||
stderr: "",
|
||||
stdout,
|
||||
stderr,
|
||||
code: 0,
|
||||
signal: null,
|
||||
killed: false,
|
||||
@@ -71,7 +80,7 @@ class FakeProcess implements WorkerSshProcess {
|
||||
}
|
||||
}
|
||||
|
||||
function fakeRunner() {
|
||||
function fakeRunner(onRun?: (argv: string[], options: CommandOptions) => SpawnResult | undefined) {
|
||||
const starts: Array<{ argv: string[]; options: CommandOptions; process: FakeProcess }> = [];
|
||||
const runs: Array<{ argv: string[]; options: CommandOptions }> = [];
|
||||
const runner: WorkerSshRunner = {
|
||||
@@ -82,12 +91,77 @@ function fakeRunner() {
|
||||
},
|
||||
async run(argv, options) {
|
||||
runs.push({ argv, options });
|
||||
return success();
|
||||
return onRun?.(argv, options) ?? success();
|
||||
},
|
||||
};
|
||||
return { runner, runs, starts };
|
||||
}
|
||||
|
||||
function localWorkspaceRunner(remoteHome: string) {
|
||||
const starts: Array<{ argv: string[]; options: CommandOptions; process: FakeProcess }> = [];
|
||||
const runner: WorkerSshRunner = {
|
||||
start(argv, options) {
|
||||
const process = new FakeProcess();
|
||||
starts.push({ argv, options, process });
|
||||
return process;
|
||||
},
|
||||
async run(argv, options) {
|
||||
if (argv[0] === "git") {
|
||||
return await runCommandWithTimeout(argv, options);
|
||||
}
|
||||
if (argv[0] === "rsync") {
|
||||
const localArgv = [...argv];
|
||||
const remoteShellIndex = localArgv.indexOf("-e");
|
||||
if (remoteShellIndex >= 0) {
|
||||
localArgv.splice(remoteShellIndex, 2);
|
||||
}
|
||||
const destination = localArgv.at(-1);
|
||||
const separator = destination?.indexOf(":") ?? -1;
|
||||
if (!destination || separator < 0) {
|
||||
throw new Error("missing test rsync destination");
|
||||
}
|
||||
const remotePath = destination.slice(separator + 1);
|
||||
// Prod rsync targets the absolute directory returned by the setup script,
|
||||
// which already lives under the fake remote HOME.
|
||||
const localDestination = path.isAbsolute(remotePath)
|
||||
? remotePath
|
||||
: path.join(remoteHome, remotePath);
|
||||
localArgv[localArgv.length - 1] = localDestination;
|
||||
await fs.mkdir(
|
||||
destination.endsWith("/") ? localDestination : path.dirname(localDestination),
|
||||
{ recursive: true },
|
||||
);
|
||||
return await runCommandWithTimeout(localArgv, options);
|
||||
}
|
||||
if (argv[0] === "ssh") {
|
||||
if (options.input?.includes("unsafe worker tunnel directory")) {
|
||||
return success();
|
||||
}
|
||||
const remoteCommand = argv.at(-1);
|
||||
if (!remoteCommand) {
|
||||
throw new Error("missing test SSH remote command");
|
||||
}
|
||||
return await runCommandWithTimeout(["sh", "-c", remoteCommand], {
|
||||
...options,
|
||||
baseEnv: { ...options.baseEnv, HOME: remoteHome },
|
||||
});
|
||||
}
|
||||
throw new Error(`unexpected test command: ${argv[0] ?? "missing"}`);
|
||||
},
|
||||
};
|
||||
return { runner, starts };
|
||||
}
|
||||
|
||||
async function git(root: string, ...args: string[]): Promise<string> {
|
||||
const result = await runCommandWithTimeout(["git", "-C", root, ...args], {
|
||||
timeoutMs: 30_000,
|
||||
});
|
||||
if (result.code !== 0) {
|
||||
throw new Error(result.stderr || result.stdout || `git ${args[0] ?? "command"} failed`);
|
||||
}
|
||||
return result.stdout.trim();
|
||||
}
|
||||
|
||||
const resolveIdentity = async () => ({ kind: "path", path: "/keys/worker" }) as const;
|
||||
|
||||
async function waitForStarts(starts: unknown[], count: number) {
|
||||
@@ -115,7 +189,7 @@ describe("worker tunnel manager", () => {
|
||||
expect(tunnel?.argv).toContain("StreamLocalBindUnlink=yes");
|
||||
expect(tunnel?.options.input).not.toContain("rm -f");
|
||||
expect(tunnel?.argv[tunnel.argv.indexOf("-R") + 1]).toMatch(
|
||||
/^\/tmp\/ocw-[a-f0-9]+\/gateway\.sock:127\.0\.0\.1:18789$/u,
|
||||
/^\/tmp\/ocw-[a-f0-9]{16}-3\/gateway\.sock:127\.0\.0\.1:18789$/u,
|
||||
);
|
||||
tunnel?.process.becomeReady();
|
||||
const handle = await starting;
|
||||
@@ -134,6 +208,293 @@ describe("worker tunnel manager", () => {
|
||||
expect(manager.status("worker:one")).toBe("stopped");
|
||||
});
|
||||
|
||||
it("syncs a dirty workspace over pinned rsync and records an immutable manifest", async () => {
|
||||
const manifestRef = `sha256:${"b".repeat(64)}`;
|
||||
const remoteWorkspaceDir = "/home/worker/.openclaw-worker/workspaces/env/session/7";
|
||||
const localPath = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-worker-sync-test-"));
|
||||
await fs.writeFile(path.join(localPath, ".worktreeinclude"), "cache/*.bin\n");
|
||||
await git(localPath, "init");
|
||||
await git(localPath, "config", "user.name", "Worker Sync Test");
|
||||
await git(localPath, "config", "user.email", "worker-sync@example.invalid");
|
||||
await fs.mkdir(path.join(localPath, "src"), { recursive: true });
|
||||
await fs.writeFile(path.join(localPath, "src/tracked.ts"), "tracked\n");
|
||||
await git(localPath, "add", ".worktreeinclude", "src/tracked.ts");
|
||||
await git(localPath, "commit", "-m", "base");
|
||||
const commit = await git(localPath, "rev-parse", "HEAD");
|
||||
const fake = fakeRunner((argv, options) => {
|
||||
if (argv.includes("--show-toplevel")) {
|
||||
return success(`${localPath}\n`);
|
||||
}
|
||||
if (argv.includes("--verify")) {
|
||||
return success(`${commit}\n`);
|
||||
}
|
||||
if (options.input?.includes("unsafe worker workspace directory")) {
|
||||
return success(`${remoteWorkspaceDir}\n`);
|
||||
}
|
||||
if (argv.at(-1)?.includes("worker workspace symlink escapes")) {
|
||||
return success(`${manifestRef}\n`);
|
||||
}
|
||||
return undefined;
|
||||
});
|
||||
const manager = createWorkerTunnelManager({ runner: fake.runner });
|
||||
const starting = manager.start({
|
||||
environmentId: "worker:sync",
|
||||
ownerEpoch: 5,
|
||||
ssh: SSH,
|
||||
gateway: { host: "127.0.0.1", port: 18789 },
|
||||
resolveIdentity,
|
||||
});
|
||||
await waitForStarts(fake.starts, 1);
|
||||
fake.starts[0]?.process.becomeReady();
|
||||
const handle = await starting;
|
||||
|
||||
try {
|
||||
await expect(
|
||||
handle.syncWorkspace({
|
||||
localPath,
|
||||
sessionId: "session:one",
|
||||
generation: 7,
|
||||
}),
|
||||
).resolves.toEqual({ mode: "git", remoteWorkspaceDir, manifestRef });
|
||||
|
||||
const transfer = fake.runs.findLast((entry) => entry.argv[0] === "rsync");
|
||||
expect(transfer?.argv).toContain("--checksum");
|
||||
expect(transfer?.argv).toContain(`${localPath}/`);
|
||||
expect(transfer?.argv.at(-1)).toBe(`worker@worker.example.test:${remoteWorkspaceDir}/`);
|
||||
expect(transfer?.argv).not.toContain("--protect-args");
|
||||
expect(transfer?.argv.some((arg) => arg.startsWith("--files-from="))).toBe(true);
|
||||
const remoteShell = transfer?.argv[transfer.argv.indexOf("-e") + 1];
|
||||
expect(remoteShell).toContain("ClearAllForwardings=yes");
|
||||
expect(remoteShell).toContain("ControlMaster=no");
|
||||
expect(remoteShell).toContain("ControlPath=none");
|
||||
const manifest = fake.runs.find((entry) =>
|
||||
entry.argv.at(-1)?.includes("worker workspace symlink escapes"),
|
||||
);
|
||||
expect(manifest?.argv.at(-1)).toContain(commit);
|
||||
} finally {
|
||||
await handle.stop();
|
||||
await fs.rm(localPath, { recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("fails workspace sync before manifest creation when rsync fails", async () => {
|
||||
const remoteWorkspaceDir = "/home/worker/.openclaw-worker/workspaces/env/session/2";
|
||||
const fake = fakeRunner((argv, options) => {
|
||||
if (argv[0] === "git") {
|
||||
return { ...success(), code: 128 };
|
||||
}
|
||||
if (argv[0] === "rsync") {
|
||||
return { ...success("", "transfer denied"), code: 23 };
|
||||
}
|
||||
if (options.input?.includes("unsafe worker workspace directory")) {
|
||||
return success(`${remoteWorkspaceDir}\n`);
|
||||
}
|
||||
return undefined;
|
||||
});
|
||||
const manager = createWorkerTunnelManager({ runner: fake.runner });
|
||||
const starting = manager.start({
|
||||
environmentId: "worker:sync-failure",
|
||||
ownerEpoch: 2,
|
||||
ssh: SSH,
|
||||
gateway: { host: "127.0.0.1", port: 18789 },
|
||||
resolveIdentity,
|
||||
});
|
||||
await waitForStarts(fake.starts, 1);
|
||||
fake.starts[0]?.process.becomeReady();
|
||||
const handle = await starting;
|
||||
|
||||
await expect(
|
||||
handle.syncWorkspace({
|
||||
localPath: "/gateway/worktrees/session-two",
|
||||
sessionId: "session:two",
|
||||
generation: 2,
|
||||
}),
|
||||
).rejects.toThrow("Worker workspace sync failed: transfer denied");
|
||||
expect(
|
||||
fake.runs.some((entry) => entry.argv.at(-1)?.includes("worker workspace symlink escapes")),
|
||||
).toBe(false);
|
||||
|
||||
await handle.stop();
|
||||
});
|
||||
|
||||
it("materializes a large dirty git workspace as a credential-free commit-capable clone", async () => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-worker-git-sync-"));
|
||||
const localPath = path.join(root, "local");
|
||||
const remoteHome = path.join(root, "remote-home");
|
||||
await Promise.all([
|
||||
fs.mkdir(path.join(localPath, "generated"), { recursive: true }),
|
||||
fs.mkdir(remoteHome, { recursive: true }),
|
||||
]);
|
||||
await git(localPath, "init");
|
||||
await git(localPath, "config", "user.name", "Worker Sync Test");
|
||||
await git(localPath, "config", "user.email", "worker-sync@example.invalid");
|
||||
await Promise.all([
|
||||
fs.writeFile(path.join(localPath, ".gitignore"), "cache/**\nprivate/**\n"),
|
||||
fs.writeFile(path.join(localPath, ".worktreeinclude"), "cache/allowed.txt\n"),
|
||||
fs.writeFile(path.join(localPath, "gone.txt"), "delete me\n"),
|
||||
fs.writeFile(path.join(localPath, "rename-old.txt"), "rename me\n"),
|
||||
fs.writeFile(path.join(localPath, "modified.txt"), "before\n"),
|
||||
]);
|
||||
const largeFiles = Array.from(
|
||||
{ length: 1_800 },
|
||||
(_, index) => `generated/long-worker-file-name-${String(index).padStart(4, "0")}.txt`,
|
||||
);
|
||||
await Promise.all(
|
||||
largeFiles.map((file, index) => fs.writeFile(path.join(localPath, file), `${index}\n`)),
|
||||
);
|
||||
await git(localPath, "add", ".");
|
||||
await git(localPath, "commit", "-m", "base");
|
||||
const firstBase = await git(localPath, "rev-parse", "HEAD");
|
||||
await fs.mkdir(path.join(localPath, "vendor/sub/.git"), { recursive: true });
|
||||
await fs.writeFile(path.join(localPath, "vendor/sub/.git/secret"), "must not transfer\n");
|
||||
await git(localPath, "update-index", "--add", "--cacheinfo", `160000,${firstBase},vendor/sub`);
|
||||
await git(localPath, "commit", "-m", "record submodule");
|
||||
const baseCommit = await git(localPath, "rev-parse", "HEAD");
|
||||
|
||||
await Promise.all([
|
||||
fs.rm(path.join(localPath, "gone.txt")),
|
||||
fs.rename(path.join(localPath, "rename-old.txt"), path.join(localPath, "rename-new.txt")),
|
||||
fs.writeFile(path.join(localPath, "modified.txt"), "after\n"),
|
||||
fs.mkdir(path.join(localPath, "cache"), { recursive: true }),
|
||||
fs.mkdir(path.join(localPath, "private"), { recursive: true }),
|
||||
]);
|
||||
await Promise.all([
|
||||
fs.writeFile(path.join(localPath, "cache/allowed.txt"), "allowed\n"),
|
||||
fs.writeFile(path.join(localPath, "private/ignored.txt"), "private\n"),
|
||||
]);
|
||||
|
||||
const fake = localWorkspaceRunner(remoteHome);
|
||||
const manager = createWorkerTunnelManager({ runner: fake.runner });
|
||||
const starting = manager.start({
|
||||
environmentId: "worker:real-git-sync",
|
||||
ownerEpoch: 11,
|
||||
ssh: SSH,
|
||||
gateway: { host: "127.0.0.1", port: 18789 },
|
||||
resolveIdentity,
|
||||
});
|
||||
await waitForStarts(fake.starts, 1);
|
||||
fake.starts[0]?.process.becomeReady();
|
||||
const handle = await starting;
|
||||
|
||||
try {
|
||||
const result = await handle.syncWorkspace({
|
||||
localPath,
|
||||
sessionId: "session:real-git-sync",
|
||||
generation: 1,
|
||||
});
|
||||
expect(result.mode).toBe("git");
|
||||
expect(result.manifestRef).toMatch(/^sha256:[a-f0-9]{64}$/u);
|
||||
await expect(
|
||||
fs.readFile(path.join(result.remoteWorkspaceDir, largeFiles[0] ?? ""), "utf8"),
|
||||
).resolves.toBe("0\n");
|
||||
await expect(
|
||||
fs.readFile(path.join(result.remoteWorkspaceDir, largeFiles.at(-1) ?? ""), "utf8"),
|
||||
).resolves.toBe("1799\n");
|
||||
await expect(fs.access(path.join(result.remoteWorkspaceDir, "gone.txt"))).rejects.toThrow();
|
||||
await expect(
|
||||
fs.readFile(path.join(result.remoteWorkspaceDir, "rename-new.txt"), "utf8"),
|
||||
).resolves.toBe("rename me\n");
|
||||
await expect(
|
||||
fs.readFile(path.join(result.remoteWorkspaceDir, "cache/allowed.txt"), "utf8"),
|
||||
).resolves.toBe("allowed\n");
|
||||
await expect(
|
||||
fs.access(path.join(result.remoteWorkspaceDir, "private/ignored.txt")),
|
||||
).rejects.toThrow();
|
||||
await expect(
|
||||
fs.access(path.join(result.remoteWorkspaceDir, "vendor/sub/.git/secret")),
|
||||
).rejects.toThrow();
|
||||
expect(await git(result.remoteWorkspaceDir, "rev-parse", "HEAD")).toBe(baseCommit);
|
||||
expect(await git(result.remoteWorkspaceDir, "rev-list", "--count", "HEAD")).toBe("1");
|
||||
expect(await git(result.remoteWorkspaceDir, "remote")).toBe("");
|
||||
const status = await runCommandWithTimeout(
|
||||
["git", "-C", result.remoteWorkspaceDir, "status", "--porcelain"],
|
||||
{ timeoutMs: 30_000 },
|
||||
);
|
||||
const statusLines = status.stdout.split("\n").filter(Boolean);
|
||||
expect(statusLines).toContain(" D gone.txt");
|
||||
expect(statusLines).toContain("?? rename-new.txt");
|
||||
await git(result.remoteWorkspaceDir, "add", "-A");
|
||||
await git(result.remoteWorkspaceDir, "commit", "-m", "worker commit");
|
||||
await git(result.remoteWorkspaceDir, "merge-base", "--is-ancestor", baseCommit, "HEAD");
|
||||
|
||||
const manifestPath = path.join(
|
||||
remoteHome,
|
||||
".openclaw-worker/manifests",
|
||||
`${result.manifestRef.slice("sha256:".length)}.json`,
|
||||
);
|
||||
const manifest = JSON.parse(await fs.readFile(manifestPath, "utf8")) as {
|
||||
entries: Array<{ path: string }>;
|
||||
};
|
||||
expect(manifest.entries.some((entry) => entry.path === ".git")).toBe(false);
|
||||
expect(manifest.entries.some((entry) => entry.path.startsWith(".git/"))).toBe(false);
|
||||
} finally {
|
||||
await handle.stop();
|
||||
await fs.rm(root, { recursive: true });
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
it("mirrors plain workspaces and rejects escaping symlinks in a git overlay", async () => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-worker-sync-modes-"));
|
||||
const plainPath = path.join(root, "plain");
|
||||
const gitPath = path.join(root, "git");
|
||||
const remoteHome = path.join(root, "remote-home");
|
||||
await Promise.all([
|
||||
fs.mkdir(path.join(plainPath, "nested/.git"), { recursive: true }),
|
||||
fs.mkdir(gitPath, { recursive: true }),
|
||||
fs.mkdir(remoteHome, { recursive: true }),
|
||||
]);
|
||||
await Promise.all([
|
||||
fs.writeFile(path.join(plainPath, "hello.txt"), "plain\n"),
|
||||
fs.writeFile(path.join(plainPath, "nested/.git/config"), "private metadata\n"),
|
||||
]);
|
||||
await git(gitPath, "init");
|
||||
await git(gitPath, "config", "user.name", "Worker Sync Test");
|
||||
await git(gitPath, "config", "user.email", "worker-sync@example.invalid");
|
||||
await fs.writeFile(path.join(gitPath, "tracked.txt"), "tracked\n");
|
||||
await git(gitPath, "add", "tracked.txt");
|
||||
await git(gitPath, "commit", "-m", "base");
|
||||
await fs.symlink(path.join(root, "outside"), path.join(gitPath, "escape"));
|
||||
|
||||
const fake = localWorkspaceRunner(remoteHome);
|
||||
const manager = createWorkerTunnelManager({ runner: fake.runner });
|
||||
const starting = manager.start({
|
||||
environmentId: "worker:real-sync-modes",
|
||||
ownerEpoch: 12,
|
||||
ssh: SSH,
|
||||
gateway: { host: "127.0.0.1", port: 18789 },
|
||||
resolveIdentity,
|
||||
});
|
||||
await waitForStarts(fake.starts, 1);
|
||||
fake.starts[0]?.process.becomeReady();
|
||||
const handle = await starting;
|
||||
|
||||
try {
|
||||
const plain = await handle.syncWorkspace({
|
||||
localPath: plainPath,
|
||||
sessionId: "session:plain-sync",
|
||||
generation: 1,
|
||||
});
|
||||
expect(plain.mode).toBe("plain");
|
||||
await expect(
|
||||
fs.readFile(path.join(plain.remoteWorkspaceDir, "hello.txt"), "utf8"),
|
||||
).resolves.toBe("plain\n");
|
||||
await expect(
|
||||
fs.access(path.join(plain.remoteWorkspaceDir, "nested/.git/config")),
|
||||
).rejects.toThrow();
|
||||
|
||||
await expect(
|
||||
handle.syncWorkspace({
|
||||
localPath: gitPath,
|
||||
sessionId: "session:symlink-sync",
|
||||
generation: 2,
|
||||
}),
|
||||
).rejects.toThrow("worker workspace symlink escapes the sync root");
|
||||
} finally {
|
||||
await handle.stop();
|
||||
await fs.rm(root, { recursive: true });
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
it("reconnects with capped backoff after unexpected exits and failed attempts", async () => {
|
||||
const fake = fakeRunner();
|
||||
const delays: number[] = [];
|
||||
@@ -309,3 +670,16 @@ describe("worker tunnel manager", () => {
|
||||
expect(fake.starts).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("createWorkerSshRunner diagnostic tails", () => {
|
||||
it("keeps SSH tunnel failure stderr on a valid UTF-16 boundary", async () => {
|
||||
const retained = "b".repeat(4095);
|
||||
const child = createWorkerSshRunner().start(
|
||||
[process.execPath, "-e", `process.stderr.write(${JSON.stringify(`a😀${retained}`)})`],
|
||||
{ timeoutMs: 10_000, baseEnv: process.env },
|
||||
);
|
||||
|
||||
await expect(child.ready).rejects.toThrow(`Worker SSH tunnel failed: ${retained}`);
|
||||
await child.exited;
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,15 +1,7 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { randomBytes } from "node:crypto";
|
||||
import { sliceUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
|
||||
import { RetrySupervisor } from "../../../packages/retry/src/index.js";
|
||||
import { sleepWithAbort, type BackoffPolicy } from "../../infra/backoff.js";
|
||||
import { redactSensitiveText } from "../../logging/redact.js";
|
||||
import type { WorkerSshEndpoint } from "../../plugins/types.js";
|
||||
import {
|
||||
runCommandWithTimeout,
|
||||
type CommandOptions,
|
||||
type SpawnResult,
|
||||
} from "../../process/exec.js";
|
||||
import type { SpawnResult } from "../../process/exec.js";
|
||||
import {
|
||||
prepareWorkerSsh,
|
||||
type PreparedWorkerSsh,
|
||||
@@ -23,14 +15,18 @@ import type {
|
||||
WorkerTunnelRequest,
|
||||
WorkerTunnelStatus,
|
||||
} from "./tunnel-contract.js";
|
||||
import {
|
||||
createWorkerSshRunner,
|
||||
type WorkerSshProcess,
|
||||
type WorkerSshRunner,
|
||||
workerSshProcessError,
|
||||
WORKER_TUNNEL_READY_MARKER,
|
||||
} from "./tunnel-ssh-runner.js";
|
||||
import { createWorkerWorkspaceActions, stableWorkerPathComponent } from "./workspace-sync.js";
|
||||
|
||||
export type { WorkerTunnelHandle } from "./tunnel-contract.js";
|
||||
const READY_MARKER = "OPENCLAW_WORKER_TUNNEL_READY";
|
||||
const REMOTE_SOCKET_NAME = "gateway.sock";
|
||||
const REMOTE_SETUP_TIMEOUT_MS = 20_000;
|
||||
const WORKSPACE_TIMEOUT_MS = 10 * 60_000;
|
||||
const STOP_GRACE_MS = 1_500;
|
||||
const STDERR_LIMIT = 4_096;
|
||||
const DEFAULT_STABLE_CONNECTION_MS = 30_000;
|
||||
const DEFAULT_BACKOFF: BackoffPolicy = {
|
||||
initialMs: 250,
|
||||
@@ -43,7 +39,14 @@ const REMOTE_SOCKET_SETUP_SCRIPT = String.raw`set -eu
|
||||
directory=$1
|
||||
socket=$2
|
||||
umask 077
|
||||
mkdir -p -- "$directory"
|
||||
if [ -e "$directory" ] || [ -L "$directory" ]; then
|
||||
if [ ! -d "$directory" ] || [ -L "$directory" ]; then
|
||||
printf '%s\n' 'unsafe worker tunnel directory' >&2
|
||||
exit 2
|
||||
fi
|
||||
else
|
||||
mkdir -- "$directory"
|
||||
fi
|
||||
chmod 700 -- "$directory"
|
||||
rm -f -- "$socket"
|
||||
`;
|
||||
@@ -51,7 +54,7 @@ rm -f -- "$socket"
|
||||
const REMOTE_TUNNEL_READY_SCRIPT = String.raw`set -eu
|
||||
socket=$1
|
||||
test -S "$socket"
|
||||
printf '%s\n' '${READY_MARKER}'
|
||||
printf '%s\n' '${WORKER_TUNNEL_READY_MARKER}'
|
||||
trap 'exit 0' HUP INT TERM
|
||||
while :; do sleep 3600; done
|
||||
`;
|
||||
@@ -63,22 +66,6 @@ rm -f -- "$socket"
|
||||
rmdir -- "$directory" 2>/dev/null || true
|
||||
`;
|
||||
|
||||
type WorkerSshProcessExit = {
|
||||
code: number | null;
|
||||
signal: NodeJS.Signals | null;
|
||||
};
|
||||
|
||||
type WorkerSshProcess = {
|
||||
ready: Promise<void>;
|
||||
exited: Promise<WorkerSshProcessExit>;
|
||||
stop(): Promise<void>;
|
||||
};
|
||||
|
||||
type WorkerSshRunner = {
|
||||
start(argv: string[], options: CommandOptions): WorkerSshProcess;
|
||||
run(argv: string[], options: CommandOptions): Promise<SpawnResult>;
|
||||
};
|
||||
|
||||
type WorkerTunnelStartRequest = WorkerTunnelRequest & {
|
||||
gateway: { host: "127.0.0.1" | "::1"; port: number };
|
||||
ssh: WorkerSshEndpoint;
|
||||
@@ -113,106 +100,6 @@ type WorkerTunnelManagerOptions = {
|
||||
stableConnectionMs?: number;
|
||||
};
|
||||
|
||||
function processError(stderr: string): Error {
|
||||
const detail = redactSensitiveText(stderr, { mode: "tools" }).replace(/\s+/gu, " ").trim();
|
||||
return new Error(detail ? `Worker SSH tunnel failed: ${detail}` : "Worker SSH tunnel failed");
|
||||
}
|
||||
|
||||
/** Production runner that treats the remote post-forward marker as connection readiness. */
|
||||
function createWorkerSshRunner(): WorkerSshRunner {
|
||||
return {
|
||||
run: runCommandWithTimeout,
|
||||
start(argv, options) {
|
||||
const [command, ...args] = argv;
|
||||
if (!command) {
|
||||
throw new Error("Worker SSH runner requires a command");
|
||||
}
|
||||
const child = spawn(command, args, {
|
||||
env: options.baseEnv,
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
windowsHide: true,
|
||||
});
|
||||
let closed = false;
|
||||
let readySettled = false;
|
||||
let resolveReady!: () => void;
|
||||
let rejectReady!: (error: Error) => void;
|
||||
let resolveExited!: (exit: WorkerSshProcessExit) => void;
|
||||
const ready = new Promise<void>((resolve, reject) => {
|
||||
resolveReady = resolve;
|
||||
rejectReady = reject;
|
||||
});
|
||||
const exited = new Promise<WorkerSshProcessExit>((resolve) => {
|
||||
resolveExited = resolve;
|
||||
});
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
const settleReadyError = () => {
|
||||
if (readySettled) {
|
||||
return;
|
||||
}
|
||||
readySettled = true;
|
||||
rejectReady(processError(stderr));
|
||||
};
|
||||
child.stdout.setEncoding("utf8");
|
||||
child.stdout.on("error", () => {});
|
||||
child.stdout.on("data", (chunk: string) => {
|
||||
if (readySettled) {
|
||||
return;
|
||||
}
|
||||
stdout = sliceUtf16Safe(`${stdout}${chunk}`, -STDERR_LIMIT);
|
||||
if (stdout.split(/\r?\n/u).includes(READY_MARKER)) {
|
||||
readySettled = true;
|
||||
resolveReady();
|
||||
}
|
||||
});
|
||||
child.stderr.setEncoding("utf8");
|
||||
child.stderr.on("error", () => {});
|
||||
child.stderr.on("data", (chunk: string) => {
|
||||
stderr = sliceUtf16Safe(`${stderr}${chunk}`, -STDERR_LIMIT);
|
||||
});
|
||||
child.once("error", settleReadyError);
|
||||
child.once("close", (code, signal) => {
|
||||
closed = true;
|
||||
settleReadyError();
|
||||
resolveExited({ code, signal });
|
||||
});
|
||||
child.stdin.on("error", () => {});
|
||||
if (options.input !== undefined) {
|
||||
child.stdin.end(options.input);
|
||||
} else {
|
||||
child.stdin.end();
|
||||
}
|
||||
|
||||
let stopPromise: Promise<void> | undefined;
|
||||
return {
|
||||
ready,
|
||||
exited,
|
||||
stop() {
|
||||
return (stopPromise ??= (async () => {
|
||||
if (closed) {
|
||||
return;
|
||||
}
|
||||
child.kill("SIGTERM");
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
await Promise.race([
|
||||
exited,
|
||||
new Promise<void>((resolve) => {
|
||||
timer = setTimeout(resolve, STOP_GRACE_MS);
|
||||
timer.unref?.();
|
||||
}),
|
||||
]);
|
||||
clearTimeout(timer);
|
||||
if (!closed) {
|
||||
child.kill("SIGKILL");
|
||||
await exited;
|
||||
}
|
||||
})());
|
||||
},
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function success(result: SpawnResult): boolean {
|
||||
return result.termination === "exit" && result.code === 0;
|
||||
}
|
||||
@@ -285,7 +172,7 @@ export function createWorkerTunnelManager(options: WorkerTunnelManagerOptions =
|
||||
});
|
||||
const result = await runner.run(command.argv, command.options);
|
||||
if (!success(result)) {
|
||||
throw processError(result.stderr || result.stdout);
|
||||
throw workerSshProcessError(result.stderr || result.stdout);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -304,36 +191,14 @@ export function createWorkerTunnelManager(options: WorkerTunnelManagerOptions =
|
||||
environmentId: entry.environmentId,
|
||||
ownerEpoch: entry.ownerEpoch,
|
||||
remoteSocketPath: entry.remoteSocketPath,
|
||||
async runWorkspaceCommand(command) {
|
||||
if (!isCurrent(entry) || !entry.prepared || entry.status !== "connected") {
|
||||
throw new Error("Worker tunnel owner is no longer connected");
|
||||
}
|
||||
const task = runner.run(
|
||||
[
|
||||
"ssh",
|
||||
...workerSshOptions(entry.prepared, { forwarding: "disabled" }),
|
||||
"-a",
|
||||
"-x",
|
||||
"-T",
|
||||
"-p",
|
||||
String(entry.prepared.port),
|
||||
"--",
|
||||
entry.prepared.sshTarget,
|
||||
workerSshRemoteCommand(command.argv),
|
||||
],
|
||||
workerSshCommandOptions({
|
||||
input: command.input,
|
||||
timeoutMs: command.timeoutMs ?? WORKSPACE_TIMEOUT_MS,
|
||||
signal: entry.abortController.signal,
|
||||
}),
|
||||
);
|
||||
entry.workspaceTasks.add(task);
|
||||
void task.then(
|
||||
() => entry.workspaceTasks.delete(task),
|
||||
() => entry.workspaceTasks.delete(task),
|
||||
);
|
||||
return await task;
|
||||
},
|
||||
...createWorkerWorkspaceActions({
|
||||
environmentId: entry.environmentId,
|
||||
ownerSignal: entry.abortController.signal,
|
||||
isConnected: () => isCurrent(entry) && entry.status === "connected",
|
||||
getPrepared: () => entry.prepared,
|
||||
runner,
|
||||
tasks: entry.workspaceTasks,
|
||||
}),
|
||||
stop: () => stop(entry.environmentId, entry.ownerEpoch),
|
||||
});
|
||||
|
||||
@@ -469,7 +334,8 @@ export function createWorkerTunnelManager(options: WorkerTunnelManagerOptions =
|
||||
rejectReady = reject;
|
||||
});
|
||||
void ready.catch(() => undefined);
|
||||
const remoteDirectory = `/tmp/ocw-${randomBytes(8).toString("hex")}`;
|
||||
const environmentKey = stableWorkerPathComponent(request.environmentId, 16);
|
||||
const remoteDirectory = `/tmp/ocw-${environmentKey}-${request.ownerEpoch}`;
|
||||
const entry: TunnelEntry = {
|
||||
environmentId: request.environmentId,
|
||||
ownerEpoch: request.ownerEpoch,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,464 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { truncateUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
|
||||
import { mapThinkingLevelForProvider } from "../../agents/embedded-agent-runner/utils.js";
|
||||
import type {
|
||||
LocalTurnPlacementClaim,
|
||||
SessionPlacementAdmissionProvider,
|
||||
SessionPlacementTurnParams,
|
||||
} from "../../agents/session-placement-admission.js";
|
||||
import { convertToLlm } from "../../agents/sessions/messages.js";
|
||||
import { SessionManager } from "../../agents/sessions/session-manager.js";
|
||||
import { formatErrorMessage } from "../../infra/errors.js";
|
||||
import { redactSensitiveText } from "../../logging/redact.js";
|
||||
import { parseWorkerLaunchDescriptor } from "../../worker/launch-descriptor.js";
|
||||
import type {
|
||||
WorkerSessionPlacementRecord,
|
||||
WorkerSessionPlacementStore,
|
||||
WorkerSessionTurnClaim,
|
||||
} from "./placement-store.js";
|
||||
import type { WorkerEnvironmentService } from "./service.js";
|
||||
import {
|
||||
assertSupportedTurn,
|
||||
assistantText,
|
||||
buildWorkerAgentMeta,
|
||||
fitLaunchDescriptor,
|
||||
parseRuntimeResult,
|
||||
windowInitialMessages,
|
||||
} from "./worker-turn-payload.js";
|
||||
|
||||
const WORKER_LAUNCH_SCRIPT = 'exec node "$HOME/.openclaw-worker/$1/openclaw.mjs" worker';
|
||||
|
||||
type WorkerTurnEnvironmentService = Pick<
|
||||
WorkerEnvironmentService,
|
||||
| "acknowledgeCredentialDelivery"
|
||||
| "acquireTurnCredential"
|
||||
| "destroy"
|
||||
| "get"
|
||||
| "startTunnel"
|
||||
| "stopTunnel"
|
||||
>;
|
||||
|
||||
type ActiveWorkerPlacement = Extract<WorkerSessionPlacementRecord, { state: "active" }>;
|
||||
type ReclaimedWorkerPlacement = Extract<WorkerSessionPlacementRecord, { state: "reclaimed" }>;
|
||||
|
||||
type WorkerTurnLauncherOptions = {
|
||||
admitNewPlacements?: boolean;
|
||||
environments: WorkerTurnEnvironmentService;
|
||||
placements: WorkerSessionPlacementStore;
|
||||
redispatchReclaimed?: (placement: ReclaimedWorkerPlacement) => Promise<ActiveWorkerPlacement>;
|
||||
};
|
||||
|
||||
class WorkerTurnExecutionError extends Error {}
|
||||
|
||||
function required(value: string | undefined, field: string): string {
|
||||
const normalized = value?.trim();
|
||||
if (!normalized) {
|
||||
throw new Error(`Worker turn ${field} is required`);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
|
||||
async function waitForTurnOperation<T>(params: {
|
||||
operation: Promise<T>;
|
||||
signal?: AbortSignal;
|
||||
timeoutMs: number;
|
||||
}): Promise<T> {
|
||||
const timeout = AbortSignal.timeout(params.timeoutMs);
|
||||
const signal = params.signal ? AbortSignal.any([params.signal, timeout]) : timeout;
|
||||
const abortError = () =>
|
||||
signal.reason instanceof Error
|
||||
? signal.reason
|
||||
: new Error("Cloud worker operation aborted", { cause: signal.reason });
|
||||
if (signal.aborted) {
|
||||
throw abortError();
|
||||
}
|
||||
return await new Promise<T>((resolve, reject) => {
|
||||
const onAbort = () => reject(abortError());
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
params.operation.then(resolve, reject).finally(() => {
|
||||
signal.removeEventListener("abort", onAbort);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function resolvePlacementIdentity(
|
||||
claim: LocalTurnPlacementClaim,
|
||||
placement: WorkerSessionPlacementRecord | undefined,
|
||||
) {
|
||||
return {
|
||||
sessionId: claim.sessionId,
|
||||
agentId: placement?.agentId ?? required(claim.agentId, "agent id"),
|
||||
sessionKey: placement?.sessionKey ?? required(claim.sessionKey, "session key"),
|
||||
};
|
||||
}
|
||||
|
||||
function requireActivePlacement(placement: WorkerSessionPlacementRecord): ActiveWorkerPlacement {
|
||||
if (
|
||||
placement.state !== "active" ||
|
||||
!placement.remoteWorkspaceDir ||
|
||||
!placement.workerBundleHash
|
||||
) {
|
||||
throw new Error(`Worker turn rejected in placement ${placement.state}`);
|
||||
}
|
||||
return placement;
|
||||
}
|
||||
|
||||
function releaseClaimIfOwned(
|
||||
placements: WorkerSessionPlacementStore,
|
||||
turnClaim: WorkerSessionTurnClaim,
|
||||
): void {
|
||||
if (placements.validateTurnClaim(turnClaim)) {
|
||||
placements.releaseTurn(turnClaim);
|
||||
}
|
||||
}
|
||||
|
||||
async function executeLocalTurn<T>(params: {
|
||||
claim: LocalTurnPlacementClaim;
|
||||
placements: WorkerSessionPlacementStore;
|
||||
runLocal: () => Promise<T>;
|
||||
}): Promise<T> {
|
||||
const current = params.placements.get(params.claim.sessionId);
|
||||
const turnClaim = params.placements.claimTurn({
|
||||
...resolvePlacementIdentity(params.claim, current),
|
||||
claimId: randomUUID(),
|
||||
runId: params.claim.runId,
|
||||
owner: { kind: "local" },
|
||||
});
|
||||
try {
|
||||
return await params.runLocal();
|
||||
} finally {
|
||||
releaseClaimIfOwned(params.placements, turnClaim);
|
||||
}
|
||||
}
|
||||
|
||||
function recoveryError(error: unknown): string {
|
||||
const message = redactSensitiveText(formatErrorMessage(error), { mode: "tools" })
|
||||
.replace(/\s+/gu, " ")
|
||||
.trim();
|
||||
return truncateUtf16Safe(message || "cloud worker turn failed", 1_024);
|
||||
}
|
||||
|
||||
async function failHandedOffTurn(params: {
|
||||
environments: WorkerTurnEnvironmentService;
|
||||
placements: WorkerSessionPlacementStore;
|
||||
placement: ActiveWorkerPlacement;
|
||||
error: unknown;
|
||||
}): Promise<void> {
|
||||
const primaryFailure = recoveryError(params.error);
|
||||
const failures = [primaryFailure];
|
||||
let draining: WorkerSessionPlacementRecord;
|
||||
try {
|
||||
draining = params.placements.startDrain({
|
||||
sessionId: params.placement.sessionId,
|
||||
environmentId: params.placement.environmentId,
|
||||
ownerEpoch: params.placement.activeOwnerEpoch,
|
||||
expectedGeneration: params.placement.generation,
|
||||
});
|
||||
} catch {
|
||||
// Exact drain ownership failed. Do not tear down an environment that may
|
||||
// now belong to a newer placement generation.
|
||||
return;
|
||||
}
|
||||
if (draining.state !== "draining") {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await params.environments.stopTunnel(
|
||||
params.placement.environmentId,
|
||||
params.placement.activeOwnerEpoch,
|
||||
);
|
||||
} catch (error) {
|
||||
failures.push(`tunnel stop: ${recoveryError(error)}`);
|
||||
}
|
||||
try {
|
||||
await params.environments.destroy(params.placement.environmentId);
|
||||
} catch (error) {
|
||||
failures.push(`environment destroy: ${recoveryError(error)}`);
|
||||
}
|
||||
try {
|
||||
// Both teardown calls returned through the environment queue. Fence stale
|
||||
// worker RPC durably now; failed teardown remains eligible for retry.
|
||||
const reconciling = params.placements.startReconcile({
|
||||
sessionId: draining.sessionId,
|
||||
environmentId: draining.environmentId,
|
||||
ownerEpoch: draining.activeOwnerEpoch,
|
||||
expectedGeneration: draining.generation,
|
||||
});
|
||||
if (reconciling.state !== "reconciling") {
|
||||
return;
|
||||
}
|
||||
params.placements.fail({
|
||||
sessionId: reconciling.sessionId,
|
||||
expectedGeneration: reconciling.generation,
|
||||
recoveryError: truncateUtf16Safe(failures.join("; "), 1_024),
|
||||
});
|
||||
} catch {
|
||||
// Leave the durable draining or reconciling row for startup reconciliation.
|
||||
}
|
||||
}
|
||||
|
||||
async function executeWorkerTurn(params: {
|
||||
environments: WorkerTurnEnvironmentService;
|
||||
onHandoff: () => void;
|
||||
placement: ActiveWorkerPlacement;
|
||||
placements: WorkerSessionPlacementStore;
|
||||
turn: SessionPlacementTurnParams;
|
||||
}) {
|
||||
const { placement, turn } = params;
|
||||
const modelRef = assertSupportedTurn(turn);
|
||||
const environment = params.environments.get(placement.environmentId);
|
||||
if (
|
||||
!environment ||
|
||||
environment.state !== "attached" ||
|
||||
environment.ownerEpoch !== placement.activeOwnerEpoch ||
|
||||
environment.bootstrapReceipt?.bundleHash !== placement.workerBundleHash ||
|
||||
environment.attachedSessionIds.length !== 1 ||
|
||||
environment.attachedSessionIds[0] !== placement.sessionId
|
||||
) {
|
||||
throw new Error("Active worker placement does not match its attached environment");
|
||||
}
|
||||
|
||||
const startedAt = Date.now();
|
||||
turn.onExecutionStarted?.({ lifecycleGeneration: turn.lifecycleGeneration });
|
||||
turn.onExecutionPhase?.({ phase: "runner_entered", backend: "cloud-worker" });
|
||||
const manager = SessionManager.open(turn.sessionFile);
|
||||
const userMessageAlreadyPersisted =
|
||||
turn.suppressNextUserMessagePersistence === true ||
|
||||
turn.userTurnTranscriptRecorder?.hasPersisted() === true;
|
||||
const contextMessages = convertToLlm(manager.buildSessionContext().messages);
|
||||
const leaf = manager.getLeafEntry();
|
||||
const initialMessages = windowInitialMessages(
|
||||
userMessageAlreadyPersisted && leaf?.type === "message" && leaf.message.role === "user"
|
||||
? contextMessages.slice(0, -1)
|
||||
: contextMessages,
|
||||
);
|
||||
let baseLeafId = manager.getLeafId();
|
||||
if (!userMessageAlreadyPersisted) {
|
||||
const persisted = turn.userTurnTranscriptRecorder
|
||||
? await turn.userTurnTranscriptRecorder.persistApproved({ cwd: turn.workspaceDir })
|
||||
: undefined;
|
||||
if (persisted) {
|
||||
baseLeafId = persisted.messageId;
|
||||
turn.userTurnTranscriptRecorder?.markRuntimePersisted(persisted.message);
|
||||
turn.onUserMessagePersisted?.(persisted.message);
|
||||
} else if (turn.userTurnTranscriptRecorder?.hasPersisted()) {
|
||||
baseLeafId = SessionManager.open(turn.sessionFile).getLeafId();
|
||||
} else if (!turn.userTurnTranscriptRecorder) {
|
||||
const message = {
|
||||
role: "user" as const,
|
||||
content: [{ type: "text" as const, text: turn.transcriptPrompt ?? turn.prompt }],
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
baseLeafId = manager.appendMessage(message);
|
||||
turn.onUserMessagePersisted?.(message);
|
||||
} else {
|
||||
throw new Error("Cloud worker turn could not persist its canonical user message");
|
||||
}
|
||||
}
|
||||
turn.onExecutionPhase?.({
|
||||
phase: "model_resolution",
|
||||
backend: "cloud-worker",
|
||||
provider: modelRef.provider,
|
||||
model: modelRef.model,
|
||||
});
|
||||
|
||||
const credential = await params.environments.acquireTurnCredential({
|
||||
environmentId: placement.environmentId,
|
||||
ownerEpoch: placement.activeOwnerEpoch,
|
||||
sessionId: placement.sessionId,
|
||||
});
|
||||
const tunnel = await waitForTurnOperation({
|
||||
operation: params.environments.startTunnel({
|
||||
environmentId: placement.environmentId,
|
||||
ownerEpoch: placement.activeOwnerEpoch,
|
||||
}),
|
||||
...(turn.abortSignal ? { signal: turn.abortSignal } : {}),
|
||||
timeoutMs: turn.timeoutMs,
|
||||
});
|
||||
const reasoning = mapThinkingLevelForProvider(turn.thinkLevel);
|
||||
const descriptor = fitLaunchDescriptor(
|
||||
(windowedMessages) =>
|
||||
parseWorkerLaunchDescriptor({
|
||||
version: 1,
|
||||
socketPath: tunnel.remoteSocketPath,
|
||||
admission: {
|
||||
environmentId: placement.environmentId,
|
||||
credential: credential.credential,
|
||||
sessionId: placement.sessionId,
|
||||
ownerEpoch: placement.activeOwnerEpoch,
|
||||
rpcSetVersion: credential.rpcSetVersion,
|
||||
handshake: environment.bootstrapReceipt,
|
||||
},
|
||||
assignment: {
|
||||
runId: turn.runId,
|
||||
turnId: randomUUID(),
|
||||
prompt: turn.prompt,
|
||||
suppressPromptTranscript: true,
|
||||
workspaceDir: placement.remoteWorkspaceDir,
|
||||
modelRef,
|
||||
inferenceOptions: reasoning ? { reasoning } : {},
|
||||
...(turn.extraSystemPrompt === undefined ? {} : { systemPrompt: turn.extraSystemPrompt }),
|
||||
initialMessages: windowedMessages,
|
||||
transcript: {
|
||||
baseLeafId,
|
||||
nextSeq: (placement.lastTranscriptAckCursor ?? 0) + 1,
|
||||
},
|
||||
liveEvents: {
|
||||
ackedSeq: placement.lastLiveEventAckCursor ?? 0,
|
||||
nextSeq: (placement.lastLiveEventAckCursor ?? 0) + 1,
|
||||
},
|
||||
},
|
||||
}),
|
||||
initialMessages,
|
||||
);
|
||||
turn.userTurnTranscriptRecorder?.markSentToProvider?.();
|
||||
turn.onExecutionPhase?.({ phase: "attempt_dispatch", backend: "cloud-worker" });
|
||||
const handoffAbort = new AbortController();
|
||||
params.onHandoff();
|
||||
const processPromise = tunnel.runWorkspaceCommand({
|
||||
argv: ["sh", "-c", WORKER_LAUNCH_SCRIPT, "openclaw-worker", placement.workerBundleHash],
|
||||
input: JSON.stringify(descriptor),
|
||||
timeoutMs: turn.timeoutMs,
|
||||
signal: turn.abortSignal
|
||||
? AbortSignal.any([turn.abortSignal, handoffAbort.signal])
|
||||
: handoffAbort.signal,
|
||||
});
|
||||
turn.onExecutionPhase?.({ phase: "process_spawned", backend: "cloud-worker" });
|
||||
let credentialDelivered: boolean;
|
||||
try {
|
||||
credentialDelivered = params.environments.acknowledgeCredentialDelivery(credential);
|
||||
} catch (error) {
|
||||
handoffAbort.abort();
|
||||
await processPromise.catch(() => undefined);
|
||||
throw new Error("Cloud worker credential handoff failed", { cause: error });
|
||||
}
|
||||
if (!credentialDelivered) {
|
||||
handoffAbort.abort();
|
||||
await processPromise.catch(() => undefined);
|
||||
throw new Error("Cloud worker credential owner changed during process handoff");
|
||||
}
|
||||
const processResult = await processPromise;
|
||||
if (processResult.code !== 0 || processResult.signal !== null || processResult.killed) {
|
||||
throw new Error("Cloud worker process failed before completing the turn");
|
||||
}
|
||||
const runtimeResult = parseRuntimeResult(processResult.stdout);
|
||||
if (runtimeResult.status === "fenced") {
|
||||
throw new Error(`Cloud worker turn was fenced: ${runtimeResult.reason}`);
|
||||
}
|
||||
if (runtimeResult.status === "failed") {
|
||||
throw new WorkerTurnExecutionError("Cloud worker turn failed");
|
||||
}
|
||||
|
||||
const completed = SessionManager.open(turn.sessionFile);
|
||||
const currentPlacement = params.placements.get(placement.sessionId);
|
||||
if (
|
||||
runtimeResult.transcriptLeafId !== completed.getLeafId() ||
|
||||
runtimeResult.transcriptNextSeq !== (currentPlacement?.lastTranscriptAckCursor ?? 0) + 1
|
||||
) {
|
||||
throw new Error("Cloud worker result does not match its committed transcript acknowledgement");
|
||||
}
|
||||
const terminal = runtimeResult.transcriptLeafId
|
||||
? completed.getEntry(runtimeResult.transcriptLeafId)
|
||||
: undefined;
|
||||
if (!terminal || terminal.type !== "message" || terminal.message.role !== "assistant") {
|
||||
throw new Error("Cloud worker completed without a terminal assistant transcript message");
|
||||
}
|
||||
const text = assistantText(terminal.message);
|
||||
const baseIndex = completed.getBranch().findIndex((entry) => entry.id === baseLeafId);
|
||||
const workerMessages = completed
|
||||
.getBranch()
|
||||
.slice(baseIndex + 1)
|
||||
.flatMap((entry) => (entry.type === "message" ? [entry.message] : []));
|
||||
return {
|
||||
...(text ? { payloads: [{ text }] } : {}),
|
||||
meta: {
|
||||
durationMs: Date.now() - startedAt,
|
||||
agentMeta: {
|
||||
sessionId: placement.sessionId,
|
||||
sessionFile: turn.sessionFile,
|
||||
...buildWorkerAgentMeta({ messages: workerMessages, modelRef }),
|
||||
},
|
||||
stopReason: terminal.message.stopReason,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function createWorkerSessionTurnPlacementProvider(
|
||||
options: WorkerTurnLauncherOptions,
|
||||
): SessionPlacementAdmissionProvider {
|
||||
return {
|
||||
async executeLocalTurn<T>(claim: LocalTurnPlacementClaim, runLocal: () => Promise<T>) {
|
||||
if (!options.placements.get(claim.sessionId) && options.admitNewPlacements === false) {
|
||||
return await runLocal();
|
||||
}
|
||||
return await executeLocalTurn({ claim, placements: options.placements, runLocal });
|
||||
},
|
||||
async executeTurn(claim, turn, runLocal) {
|
||||
const current = options.placements.get(claim.sessionId);
|
||||
if (
|
||||
!current &&
|
||||
(options.admitNewPlacements === false ||
|
||||
(turn.modelRun === true && !claim.sessionKey?.trim()))
|
||||
) {
|
||||
return await runLocal();
|
||||
}
|
||||
if (!current || current.state === "local") {
|
||||
return await executeLocalTurn({ claim, placements: options.placements, runLocal });
|
||||
}
|
||||
let routablePlacement = current;
|
||||
if (routablePlacement.state === "reclaimed") {
|
||||
if (!options.redispatchReclaimed) {
|
||||
throw new Error("Reclaimed worker placement requires redispatch");
|
||||
}
|
||||
routablePlacement = await options.redispatchReclaimed(routablePlacement);
|
||||
}
|
||||
const identity = resolvePlacementIdentity(claim, routablePlacement);
|
||||
const placement = requireActivePlacement(routablePlacement);
|
||||
const turnClaim = options.placements.claimTurn({
|
||||
...identity,
|
||||
claimId: randomUUID(),
|
||||
runId: claim.runId,
|
||||
owner: {
|
||||
kind: "worker",
|
||||
environmentId: placement.environmentId,
|
||||
ownerEpoch: placement.activeOwnerEpoch,
|
||||
},
|
||||
});
|
||||
let handedOff = false;
|
||||
try {
|
||||
const result = await executeWorkerTurn({
|
||||
environments: options.environments,
|
||||
onHandoff: () => {
|
||||
handedOff = true;
|
||||
},
|
||||
placement,
|
||||
placements: options.placements,
|
||||
turn,
|
||||
});
|
||||
if (!options.placements.validateTurnClaim(turnClaim)) {
|
||||
throw new Error("Cloud worker turn ownership changed before result reconciliation");
|
||||
}
|
||||
options.placements.releaseTurn(turnClaim);
|
||||
return result;
|
||||
} catch (error) {
|
||||
if (error instanceof WorkerTurnExecutionError) {
|
||||
if (options.placements.validateTurnClaim(turnClaim)) {
|
||||
options.placements.releaseTurn(turnClaim);
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
if (handedOff) {
|
||||
await failHandedOffTurn({
|
||||
environments: options.environments,
|
||||
placements: options.placements,
|
||||
placement,
|
||||
error,
|
||||
});
|
||||
} else {
|
||||
releaseClaimIfOwned(options.placements, turnClaim);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,197 @@
|
||||
import type { WorkerTranscriptMessage } from "../../../packages/gateway-protocol/src/schema/worker-admission.js";
|
||||
import {
|
||||
WORKER_INFERENCE_MAX_CONTEXT_MESSAGES,
|
||||
WORKER_PROTOCOL_MAX_INFERENCE_PAYLOAD_BYTES,
|
||||
} from "../../../packages/gateway-protocol/src/schema/worker-inference.js";
|
||||
import {
|
||||
isDefaultAgentRuntimeId,
|
||||
normalizeOptionalAgentRuntimeId,
|
||||
OPENCLAW_AGENT_RUNTIME_ID,
|
||||
} from "../../agents/agent-runtime-id.js";
|
||||
import {
|
||||
buildUsageAgentMetaFields,
|
||||
resolveReportedModelRef,
|
||||
} from "../../agents/embedded-agent-runner/run/helpers.js";
|
||||
import {
|
||||
createUsageAccumulator,
|
||||
mergeUsageIntoAccumulator,
|
||||
} from "../../agents/embedded-agent-runner/usage-accumulator.js";
|
||||
import { resolveDefaultModelForAgent } from "../../agents/model-selection-config.js";
|
||||
import type { AgentMessage } from "../../agents/runtime/index.js";
|
||||
import type { SessionPlacementTurnParams } from "../../agents/session-placement-admission.js";
|
||||
import { resolveEffectiveAgentRuntime } from "../../agents/thinking-runtime.js";
|
||||
import { hasNonzeroUsage, normalizeUsage } from "../../agents/usage.js";
|
||||
import type { WorkerLaunchDescriptor } from "../../worker/launch-descriptor.js";
|
||||
import { toWorkerTranscriptMessage } from "../../worker/transcript-message.js";
|
||||
import type { WorkerRuntimeResult } from "../../worker/worker.runtime.js";
|
||||
|
||||
export function windowInitialMessages(messages: AgentMessage[]): WorkerTranscriptMessage[] {
|
||||
const projected = messages.flatMap((message) => {
|
||||
const value = toWorkerTranscriptMessage(message);
|
||||
return value ? [value] : [];
|
||||
});
|
||||
if (projected.length <= WORKER_INFERENCE_MAX_CONTEXT_MESSAGES) {
|
||||
return projected;
|
||||
}
|
||||
const minimumStart = projected.length - WORKER_INFERENCE_MAX_CONTEXT_MESSAGES;
|
||||
const completeTurnStart = projected.findIndex(
|
||||
(message, index) => index >= minimumStart && message.role === "user",
|
||||
);
|
||||
if (completeTurnStart < 0) {
|
||||
throw new Error("Worker turn transcript has no complete context window");
|
||||
}
|
||||
return projected.slice(completeTurnStart);
|
||||
}
|
||||
|
||||
export function fitLaunchDescriptor(
|
||||
build: (initialMessages: WorkerTranscriptMessage[]) => WorkerLaunchDescriptor,
|
||||
messages: WorkerTranscriptMessage[],
|
||||
): WorkerLaunchDescriptor {
|
||||
let initialMessages = messages;
|
||||
while (true) {
|
||||
const descriptor = build(initialMessages);
|
||||
if (
|
||||
Buffer.byteLength(JSON.stringify(descriptor), "utf8") <=
|
||||
WORKER_PROTOCOL_MAX_INFERENCE_PAYLOAD_BYTES
|
||||
) {
|
||||
return descriptor;
|
||||
}
|
||||
const nextTurn = initialMessages.findIndex(
|
||||
(message, index) => index > 0 && message.role === "user",
|
||||
);
|
||||
if (nextTurn < 0) {
|
||||
throw new Error("Worker turn context exceeds the launch descriptor payload limit");
|
||||
}
|
||||
initialMessages = initialMessages.slice(nextTurn);
|
||||
}
|
||||
}
|
||||
|
||||
export function parseRuntimeResult(stdout: string): WorkerRuntimeResult {
|
||||
let value: unknown;
|
||||
try {
|
||||
value = JSON.parse(stdout.trim()) as unknown;
|
||||
} catch (error) {
|
||||
throw new Error("Worker process returned invalid output", { cause: error });
|
||||
}
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
throw new Error("Worker process returned invalid output");
|
||||
}
|
||||
const result = value as Record<string, unknown>;
|
||||
if (
|
||||
result.status === "failed" &&
|
||||
result.reason === "turn-failed" &&
|
||||
Object.keys(result).every((key) => ["status", "reason"].includes(key))
|
||||
) {
|
||||
return result as WorkerRuntimeResult;
|
||||
}
|
||||
if (
|
||||
result.status === "completed" &&
|
||||
(result.transcriptLeafId === null || typeof result.transcriptLeafId === "string") &&
|
||||
typeof result.transcriptNextSeq === "number" &&
|
||||
Number.isSafeInteger(result.transcriptNextSeq) &&
|
||||
result.transcriptNextSeq >= 1 &&
|
||||
Object.keys(result).every((key) =>
|
||||
["status", "transcriptLeafId", "transcriptNextSeq"].includes(key),
|
||||
)
|
||||
) {
|
||||
return result as WorkerRuntimeResult;
|
||||
}
|
||||
if (
|
||||
result.status === "fenced" &&
|
||||
(result.reason === "credential-replaced" || result.reason === "owner-epoch-mismatch") &&
|
||||
Object.keys(result).every((key) => ["status", "reason"].includes(key))
|
||||
) {
|
||||
return result as WorkerRuntimeResult;
|
||||
}
|
||||
throw new Error("Worker process returned invalid output");
|
||||
}
|
||||
|
||||
export function assistantText(message: AgentMessage): string {
|
||||
if (message.role !== "assistant") {
|
||||
return "";
|
||||
}
|
||||
return message.content.flatMap((part) => (part.type === "text" ? [part.text] : [])).join("");
|
||||
}
|
||||
|
||||
export function buildWorkerAgentMeta(params: {
|
||||
messages: AgentMessage[];
|
||||
modelRef: { provider: string; model: string };
|
||||
}) {
|
||||
const usageAccumulator = createUsageAccumulator();
|
||||
const assistants = params.messages.filter(
|
||||
(message): message is Extract<AgentMessage, { role: "assistant" }> =>
|
||||
message.role === "assistant",
|
||||
);
|
||||
let lastRunPromptUsage: ReturnType<typeof normalizeUsage>;
|
||||
for (const assistant of assistants) {
|
||||
const usage = normalizeUsage(assistant.usage);
|
||||
mergeUsageIntoAccumulator(usageAccumulator, usage);
|
||||
if (hasNonzeroUsage(usage)) {
|
||||
lastRunPromptUsage = usage;
|
||||
}
|
||||
}
|
||||
const lastAssistant = assistants.at(-1);
|
||||
const usageMeta = buildUsageAgentMetaFields({
|
||||
usageAccumulator,
|
||||
lastAssistantUsage: lastAssistant?.usage,
|
||||
lastRunPromptUsage,
|
||||
lastTurnTotal: lastRunPromptUsage?.total,
|
||||
});
|
||||
const reportedModelRef = resolveReportedModelRef({
|
||||
...params.modelRef,
|
||||
assistant: lastAssistant,
|
||||
});
|
||||
return {
|
||||
provider: reportedModelRef.provider,
|
||||
model: reportedModelRef.model,
|
||||
usage: usageMeta.usage,
|
||||
lastCallUsage: usageMeta.lastCallUsage,
|
||||
promptTokens: usageMeta.promptTokens,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveTurnModelRef(params: SessionPlacementTurnParams): {
|
||||
provider: string;
|
||||
model: string;
|
||||
} {
|
||||
const explicitProvider = params.provider?.trim();
|
||||
const explicitModel = params.model?.trim();
|
||||
const defaults =
|
||||
explicitProvider && explicitModel
|
||||
? undefined
|
||||
: resolveDefaultModelForAgent({ cfg: params.config ?? {}, agentId: params.agentId });
|
||||
return {
|
||||
provider: explicitProvider ?? defaults?.provider ?? "",
|
||||
model: explicitModel ?? defaults?.model ?? "",
|
||||
};
|
||||
}
|
||||
|
||||
export function assertSupportedTurn(params: SessionPlacementTurnParams): {
|
||||
provider: string;
|
||||
model: string;
|
||||
} {
|
||||
if (params.images?.length || params.imageOrder?.length) {
|
||||
throw new Error("Cloud worker turns do not yet support current-turn image input");
|
||||
}
|
||||
if (params.clientTools?.length) {
|
||||
throw new Error("Cloud worker turns do not support client-provided tools");
|
||||
}
|
||||
const modelRef = resolveTurnModelRef(params);
|
||||
const explicitRuntime =
|
||||
normalizeOptionalAgentRuntimeId(params.agentHarnessId) ??
|
||||
normalizeOptionalAgentRuntimeId(params.agentHarnessRuntimeOverride);
|
||||
const runtime =
|
||||
explicitRuntime && !isDefaultAgentRuntimeId(explicitRuntime)
|
||||
? explicitRuntime
|
||||
: resolveEffectiveAgentRuntime({
|
||||
cfg: params.config ?? {},
|
||||
provider: modelRef.provider,
|
||||
modelId: modelRef.model,
|
||||
agentId: params.agentId,
|
||||
sessionKey: params.sessionKey,
|
||||
});
|
||||
if (runtime !== OPENCLAW_AGENT_RUNTIME_ID) {
|
||||
throw new Error(`Cloud worker turns require the OpenClaw runtime, not ${runtime}`);
|
||||
}
|
||||
return modelRef;
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { runLocalCommandToFile } from "./workspace-sync-local.js";
|
||||
|
||||
async function waitForFile(filePath: string): Promise<void> {
|
||||
const deadline = Date.now() + 5_000;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
await fs.access(filePath);
|
||||
return;
|
||||
} catch {
|
||||
await new Promise<void>((resolve) => {
|
||||
setTimeout(resolve, 10);
|
||||
});
|
||||
}
|
||||
}
|
||||
throw new Error(`Timed out waiting for ${filePath}`);
|
||||
}
|
||||
|
||||
describe("runLocalCommandToFile", () => {
|
||||
it("force-kills a command that ignores abort termination", async () => {
|
||||
const root = await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-workspace-sync-"));
|
||||
const outputPath = path.join(root, "output");
|
||||
const readyPath = path.join(root, "ready");
|
||||
const controller = new AbortController();
|
||||
const operation = runLocalCommandToFile({
|
||||
argv: [
|
||||
process.execPath,
|
||||
"-e",
|
||||
[
|
||||
'const fs = require("node:fs");',
|
||||
'process.on("SIGTERM", () => {});',
|
||||
'fs.writeFileSync(process.argv[1], "ready");',
|
||||
"setInterval(() => {}, 1000);",
|
||||
].join(""),
|
||||
readyPath,
|
||||
],
|
||||
outputPath,
|
||||
signal: controller.signal,
|
||||
timeoutMs: 10_000,
|
||||
});
|
||||
|
||||
try {
|
||||
await waitForFile(readyPath);
|
||||
const abortedAt = Date.now();
|
||||
controller.abort();
|
||||
await expect(operation).rejects.toThrow("Worker workspace file enumeration was aborted");
|
||||
expect(Date.now() - abortedAt).toBeLessThan(3_000);
|
||||
} finally {
|
||||
controller.abort();
|
||||
await operation.catch(() => undefined);
|
||||
await fs.rm(root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,235 @@
|
||||
import { spawn } from "node:child_process";
|
||||
import { createReadStream } from "node:fs";
|
||||
import fs from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { sliceUtf16Safe } from "@openclaw/normalization-core/utf16-slice";
|
||||
import { killProcessTree } from "../../process/kill-tree.js";
|
||||
import { workerSshCommandOptions } from "./ssh.js";
|
||||
|
||||
const STDERR_LIMIT = 4_096;
|
||||
const COMMAND_KILL_GRACE_MS = 300;
|
||||
const COMMAND_CLOSE_GRACE_MS = 1_000;
|
||||
|
||||
function validateGitRelativePath(file: string): string {
|
||||
if (
|
||||
!file ||
|
||||
path.posix.isAbsolute(file) ||
|
||||
path.posix.normalize(file) !== file ||
|
||||
file === ".." ||
|
||||
file.startsWith("../")
|
||||
) {
|
||||
throw new Error("Worker workspace git file list contains an unsafe path");
|
||||
}
|
||||
return file;
|
||||
}
|
||||
|
||||
async function* readNulFile(filePath: string): AsyncGenerator<string> {
|
||||
let pending = Buffer.alloc(0);
|
||||
for await (const value of createReadStream(filePath)) {
|
||||
const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value);
|
||||
const buffer = pending.length === 0 ? chunk : Buffer.concat([pending, chunk]);
|
||||
let offset = 0;
|
||||
for (;;) {
|
||||
const separator = buffer.indexOf(0, offset);
|
||||
if (separator < 0) {
|
||||
break;
|
||||
}
|
||||
yield validateGitRelativePath(buffer.subarray(offset, separator).toString("utf8"));
|
||||
offset = separator + 1;
|
||||
}
|
||||
pending = Buffer.from(buffer.subarray(offset));
|
||||
}
|
||||
if (pending.length > 0) {
|
||||
throw new Error("Worker workspace git file list is not NUL terminated");
|
||||
}
|
||||
}
|
||||
|
||||
export async function runLocalCommandToFile(params: {
|
||||
argv: string[];
|
||||
inputPath?: string;
|
||||
outputPath: string;
|
||||
signal: AbortSignal;
|
||||
timeoutMs: number;
|
||||
}): Promise<void> {
|
||||
const [command, ...args] = params.argv;
|
||||
if (!command) {
|
||||
throw new Error("Worker workspace command requires an executable");
|
||||
}
|
||||
const output = await fs.open(params.outputPath, "wx", 0o600);
|
||||
const input = params.inputPath ? await fs.open(params.inputPath, "r") : undefined;
|
||||
let stderr = "";
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
let terminationTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
let abort: (() => void) | undefined;
|
||||
try {
|
||||
if (params.signal.aborted) {
|
||||
throw new Error("Worker workspace file enumeration was aborted");
|
||||
}
|
||||
const child = spawn(command, args, {
|
||||
env: workerSshCommandOptions({ timeoutMs: params.timeoutMs }).baseEnv,
|
||||
stdio: [input?.fd ?? "ignore", output.fd, "pipe"],
|
||||
...(process.platform !== "win32" ? { detached: true } : {}),
|
||||
windowsHide: true,
|
||||
});
|
||||
const childStderr = child.stderr;
|
||||
if (!childStderr) {
|
||||
throw new Error("Worker workspace command has no stderr pipe");
|
||||
}
|
||||
childStderr.setEncoding("utf8");
|
||||
childStderr.on("data", (chunk: string) => {
|
||||
stderr = sliceUtf16Safe(`${stderr}${chunk}`, -STDERR_LIMIT);
|
||||
});
|
||||
const result = await new Promise<{ code: number | null; error?: Error }>((resolve) => {
|
||||
let settled = false;
|
||||
const finish = (value: { code: number | null; error?: Error }) => {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
resolve(value);
|
||||
};
|
||||
let terminationStarted = false;
|
||||
const terminate = () => {
|
||||
if (settled || terminationStarted) {
|
||||
return;
|
||||
}
|
||||
terminationStarted = true;
|
||||
const pid = child.pid;
|
||||
if (typeof pid === "number" && pid > 0) {
|
||||
killProcessTree(pid, { graceMs: COMMAND_KILL_GRACE_MS });
|
||||
} else {
|
||||
child.kill("SIGTERM");
|
||||
}
|
||||
// A descendant can retain stderr even after the direct child exits. Bound
|
||||
// shutdown so placement replacement cannot wait forever on that pipe.
|
||||
terminationTimer = setTimeout(() => {
|
||||
if (typeof pid === "number" && pid > 0) {
|
||||
killProcessTree(pid, { force: true });
|
||||
} else {
|
||||
child.kill("SIGKILL");
|
||||
}
|
||||
childStderr.destroy();
|
||||
finish({ code: null });
|
||||
}, COMMAND_KILL_GRACE_MS + COMMAND_CLOSE_GRACE_MS);
|
||||
terminationTimer.unref?.();
|
||||
};
|
||||
child.once("error", (error) => finish({ code: null, error }));
|
||||
child.once("close", (code) => finish({ code }));
|
||||
abort = terminate;
|
||||
params.signal.addEventListener("abort", abort, { once: true });
|
||||
timer = setTimeout(terminate, params.timeoutMs);
|
||||
timer.unref?.();
|
||||
if (params.signal.aborted) {
|
||||
terminate();
|
||||
}
|
||||
});
|
||||
if (result.error) {
|
||||
throw result.error;
|
||||
}
|
||||
if (params.signal.aborted) {
|
||||
throw new Error("Worker workspace file enumeration was aborted");
|
||||
}
|
||||
if (result.code !== 0) {
|
||||
throw new Error(
|
||||
stderr.trim()
|
||||
? `Worker workspace file enumeration failed: ${stderr.trim()}`
|
||||
: "Worker workspace file enumeration failed",
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
clearTimeout(terminationTimer);
|
||||
if (abort) {
|
||||
params.signal.removeEventListener("abort", abort);
|
||||
}
|
||||
await output.close();
|
||||
await input?.close();
|
||||
}
|
||||
}
|
||||
|
||||
function hasErrorCode(error: unknown, code: string): boolean {
|
||||
return (
|
||||
typeof error === "object" &&
|
||||
error !== null &&
|
||||
"code" in error &&
|
||||
typeof error.code === "string" &&
|
||||
error.code === code
|
||||
);
|
||||
}
|
||||
|
||||
export async function writeEligibleGitFiles(params: {
|
||||
gitRoot: string;
|
||||
eligiblePath: string;
|
||||
ignoredPath: string;
|
||||
selectedPath: string;
|
||||
outputPath: string;
|
||||
}): Promise<void> {
|
||||
const output = await fs.open(params.outputPath, "wx", 0o600);
|
||||
const canonicalRoot = await fs.realpath(params.gitRoot);
|
||||
let buffered: string[] = [];
|
||||
let bufferedBytes = 0;
|
||||
const flush = async () => {
|
||||
if (buffered.length === 0) {
|
||||
return;
|
||||
}
|
||||
await output.write(buffered.join(""));
|
||||
buffered = [];
|
||||
bufferedBytes = 0;
|
||||
};
|
||||
const appendIfTransferable = async (file: string) => {
|
||||
const absolute = path.join(canonicalRoot, file);
|
||||
const stats = await fs.lstat(absolute).catch((error: unknown) => {
|
||||
if (hasErrorCode(error, "ENOENT")) {
|
||||
return undefined;
|
||||
}
|
||||
throw error;
|
||||
});
|
||||
// Gitlinks are directories. Keep their commit in the base repository without
|
||||
// recursively copying nested repositories or their credential-bearing metadata.
|
||||
if (!stats || (!stats.isFile() && !stats.isSymbolicLink())) {
|
||||
return;
|
||||
}
|
||||
if (stats.isSymbolicLink()) {
|
||||
// Mirrors the remote manifest guard, but before transfer: macOS openrsync
|
||||
// stat-fails escaping links with an opaque error instead of copying them.
|
||||
const target = await fs.readlink(absolute);
|
||||
const resolvedTarget = path.resolve(path.dirname(absolute), target);
|
||||
if (
|
||||
resolvedTarget !== canonicalRoot &&
|
||||
!resolvedTarget.startsWith(canonicalRoot + path.sep)
|
||||
) {
|
||||
throw new Error(`worker workspace symlink escapes the sync root: ${file}`);
|
||||
}
|
||||
}
|
||||
const record = `${file}\0`;
|
||||
buffered.push(record);
|
||||
bufferedBytes += Buffer.byteLength(record);
|
||||
if (bufferedBytes >= 64 * 1024) {
|
||||
await flush();
|
||||
}
|
||||
};
|
||||
try {
|
||||
for await (const file of readNulFile(params.eligiblePath)) {
|
||||
await appendIfTransferable(file);
|
||||
}
|
||||
const ignored = readNulFile(params.ignoredPath)[Symbol.asyncIterator]();
|
||||
const selected = readNulFile(params.selectedPath)[Symbol.asyncIterator]();
|
||||
let ignoredItem = await ignored.next();
|
||||
let selectedItem = await selected.next();
|
||||
while (!ignoredItem.done && !selectedItem.done) {
|
||||
const order = Buffer.compare(Buffer.from(ignoredItem.value), Buffer.from(selectedItem.value));
|
||||
if (order === 0) {
|
||||
await appendIfTransferable(ignoredItem.value);
|
||||
ignoredItem = await ignored.next();
|
||||
selectedItem = await selected.next();
|
||||
} else if (order < 0) {
|
||||
ignoredItem = await ignored.next();
|
||||
} else {
|
||||
selectedItem = await selected.next();
|
||||
}
|
||||
}
|
||||
await flush();
|
||||
} finally {
|
||||
await output.close();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,185 @@
|
||||
export const REMOTE_WORKSPACE_SETUP_SCRIPT = String.raw`set -eu
|
||||
relative=$1
|
||||
root=$HOME/.openclaw-worker
|
||||
|
||||
ensure_private_directory() {
|
||||
directory=$1
|
||||
if [ -e "$directory" ] || [ -L "$directory" ]; then
|
||||
if [ ! -d "$directory" ] || [ -L "$directory" ]; then
|
||||
printf '%s\n' 'unsafe worker workspace directory' >&2
|
||||
exit 2
|
||||
fi
|
||||
else
|
||||
mkdir "$directory"
|
||||
fi
|
||||
chmod 700 "$directory"
|
||||
}
|
||||
|
||||
ensure_private_directory "$root"
|
||||
current=$root
|
||||
old_ifs=$IFS
|
||||
IFS=/
|
||||
set -- $relative
|
||||
IFS=$old_ifs
|
||||
for segment in "$@"; do
|
||||
current=$current/$segment
|
||||
ensure_private_directory "$current"
|
||||
done
|
||||
cd "$current"
|
||||
find . -mindepth 1 -maxdepth 1 -exec rm -rf -- {} +
|
||||
pwd -P
|
||||
`;
|
||||
|
||||
export const REMOTE_GIT_WORKSPACE_SETUP_SCRIPT = String.raw`set -eu
|
||||
workspace=$1
|
||||
pack=$2
|
||||
base=$3
|
||||
author_name=$4
|
||||
author_email=$5
|
||||
cd "$workspace"
|
||||
if ! command -v git >/dev/null 2>&1; then
|
||||
printf '%s\n' 'git is required for a git worker workspace' >&2
|
||||
exit 2
|
||||
fi
|
||||
case ${"${"}#base} in
|
||||
40) git init -q . ;;
|
||||
64) git init -q --object-format=sha256 . ;;
|
||||
*) printf '%s\n' 'invalid worker git base object id' >&2; exit 2 ;;
|
||||
esac
|
||||
git index-pack --stdin < "$pack" >/dev/null
|
||||
printf '%s\n' "$base" > .git/shallow
|
||||
actual=$(git rev-parse --verify "$base^{commit}")
|
||||
if [ "$actual" != "$base" ]; then
|
||||
printf '%s\n' 'worker git base does not match the synced pack' >&2
|
||||
exit 2
|
||||
fi
|
||||
git update-ref refs/heads/openclaw-worker "$base"
|
||||
git symbolic-ref HEAD refs/heads/openclaw-worker
|
||||
git read-tree "$base"
|
||||
git ls-files --stage -z | node -e '
|
||||
const childProcess = require("node:child_process");
|
||||
const chunks = [];
|
||||
process.stdin.on("data", (chunk) => chunks.push(chunk));
|
||||
process.stdin.on("end", () => {
|
||||
const paths = Buffer.concat(chunks)
|
||||
.toString("utf8")
|
||||
.split("\0")
|
||||
.filter(Boolean)
|
||||
.flatMap((record) => {
|
||||
const separator = record.indexOf("\t");
|
||||
return separator >= 0 && record.startsWith("160000 ") ? [record.slice(separator + 1)] : [];
|
||||
});
|
||||
if (paths.length > 0) {
|
||||
childProcess.execFileSync("git", ["update-index", "--skip-worktree", "--", ...paths]);
|
||||
}
|
||||
});'
|
||||
rm -f -- "$pack"
|
||||
if [ -n "$author_name" ]; then git config user.name "$author_name"; fi
|
||||
if [ -n "$author_email" ]; then git config user.email "$author_email"; fi
|
||||
`;
|
||||
|
||||
export const REMOTE_WORKSPACE_MANIFEST_JS = String.raw`const crypto = require("node:crypto");
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
const root = fs.realpathSync(process.argv[1]);
|
||||
const baseCommit = process.argv[2] || null;
|
||||
const entries = [];
|
||||
function fail(message) {
|
||||
throw new Error(message);
|
||||
}
|
||||
function walk(relativeDirectory) {
|
||||
const absoluteDirectory = relativeDirectory ? path.join(root, relativeDirectory) : root;
|
||||
for (const name of fs.readdirSync(absoluteDirectory).sort()) {
|
||||
if (!relativeDirectory && name === ".git") {
|
||||
continue;
|
||||
}
|
||||
const relative = relativeDirectory ? relativeDirectory + "/" + name : name;
|
||||
const absolute = path.join(root, relative);
|
||||
const stats = fs.lstatSync(absolute);
|
||||
const mode = stats.mode & 0o777;
|
||||
if (stats.isDirectory()) {
|
||||
entries.push({ path: relative, type: "directory", mode });
|
||||
walk(relative);
|
||||
} else if (stats.isFile()) {
|
||||
entries.push({
|
||||
path: relative,
|
||||
type: "file",
|
||||
mode,
|
||||
size: stats.size,
|
||||
sha256: null,
|
||||
});
|
||||
} else if (stats.isSymbolicLink()) {
|
||||
const target = fs.readlinkSync(absolute);
|
||||
const resolvedTarget = path.resolve(path.dirname(absolute), target);
|
||||
if (resolvedTarget !== root && !resolvedTarget.startsWith(root + path.sep)) {
|
||||
fail("worker workspace symlink escapes the sync root: " + relative);
|
||||
}
|
||||
entries.push({ path: relative, type: "symlink", mode, target });
|
||||
} else {
|
||||
fail("unsupported worker workspace entry: " + relative);
|
||||
}
|
||||
}
|
||||
}
|
||||
async function hashFiles() {
|
||||
for (const entry of entries) {
|
||||
if (entry.type !== "file") {
|
||||
continue;
|
||||
}
|
||||
const hash = crypto.createHash("sha256");
|
||||
const stream = fs.createReadStream(path.join(root, entry.path));
|
||||
for await (const chunk of stream) {
|
||||
hash.update(chunk);
|
||||
}
|
||||
entry.sha256 = hash.digest("hex");
|
||||
}
|
||||
}
|
||||
function ensurePrivateDirectory(directory) {
|
||||
try {
|
||||
const stats = fs.lstatSync(directory);
|
||||
if (stats.isSymbolicLink() || !stats.isDirectory()) {
|
||||
fail("unsafe worker manifest directory");
|
||||
}
|
||||
} catch (error) {
|
||||
if (error && error.code === "ENOENT") {
|
||||
fs.mkdirSync(directory, { mode: 0o700 });
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
fs.chmodSync(directory, 0o700);
|
||||
}
|
||||
async function main() {
|
||||
walk("");
|
||||
await hashFiles();
|
||||
const manifest = JSON.stringify({ version: 1, baseCommit, entries });
|
||||
const digest = crypto.createHash("sha256").update(manifest).digest("hex");
|
||||
const workerRoot = path.join(process.env.HOME, ".openclaw-worker");
|
||||
const manifestRoot = path.join(workerRoot, "manifests");
|
||||
ensurePrivateDirectory(workerRoot);
|
||||
ensurePrivateDirectory(manifestRoot);
|
||||
const manifestPath = path.join(manifestRoot, digest + ".json");
|
||||
const temporaryPath = manifestPath + "." + process.pid + "." + crypto.randomBytes(4).toString("hex");
|
||||
fs.writeFileSync(temporaryPath, manifest, { encoding: "utf8", flag: "wx", mode: 0o600 });
|
||||
try {
|
||||
try {
|
||||
fs.linkSync(temporaryPath, manifestPath);
|
||||
} catch (error) {
|
||||
const existing = error && error.code === "EEXIST" ? fs.lstatSync(manifestPath) : null;
|
||||
if (
|
||||
!existing ||
|
||||
existing.isSymbolicLink() ||
|
||||
!existing.isFile() ||
|
||||
fs.readFileSync(manifestPath, "utf8") !== manifest
|
||||
) {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
fs.rmSync(temporaryPath, { force: true });
|
||||
}
|
||||
process.stdout.write("sha256:" + digest + "\n");
|
||||
}
|
||||
main().catch((error) => {
|
||||
process.stderr.write(String(error && error.stack ? error.stack : error) + "\n");
|
||||
process.exitCode = 1;
|
||||
});`;
|
||||
@@ -0,0 +1,407 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { redactSensitiveText } from "../../logging/redact.js";
|
||||
import type { CommandOptions, SpawnResult } from "../../process/exec.js";
|
||||
import {
|
||||
type PreparedWorkerSsh,
|
||||
workerSshCommandOptions,
|
||||
workerSshOptions,
|
||||
workerSshRemoteCommand,
|
||||
} from "./ssh.js";
|
||||
import type {
|
||||
WorkerTunnelHandle,
|
||||
WorkerWorkspaceCommand,
|
||||
WorkerWorkspaceSyncRequest,
|
||||
WorkerWorkspaceSyncResult,
|
||||
} from "./tunnel-contract.js";
|
||||
import { runLocalCommandToFile, writeEligibleGitFiles } from "./workspace-sync-local.js";
|
||||
import {
|
||||
REMOTE_GIT_WORKSPACE_SETUP_SCRIPT,
|
||||
REMOTE_WORKSPACE_MANIFEST_JS,
|
||||
REMOTE_WORKSPACE_SETUP_SCRIPT,
|
||||
} from "./workspace-sync-scripts.js";
|
||||
|
||||
const REMOTE_SETUP_TIMEOUT_MS = 20_000;
|
||||
const WORKSPACE_TIMEOUT_MS = 10 * 60_000;
|
||||
// Relative to the $HOME/.openclaw-worker root owned by REMOTE_WORKSPACE_SETUP_SCRIPT;
|
||||
// rsync targets must use the returned absolute directory, never this relative path.
|
||||
const REMOTE_WORKSPACE_ROOT = "workspaces";
|
||||
const REMOTE_GIT_PACK_NAME = ".openclaw-base.pack";
|
||||
const GIT_COMMIT_PATTERN = /^[a-f0-9]{40}(?:[a-f0-9]{24})?$/u;
|
||||
const MANIFEST_REF_PATTERN = /^sha256:[a-f0-9]{64}$/u;
|
||||
|
||||
type WorkerWorkspaceRunner = {
|
||||
run(argv: string[], options: CommandOptions): Promise<SpawnResult>;
|
||||
};
|
||||
|
||||
type WorkerWorkspaceActionsOptions = {
|
||||
environmentId: string;
|
||||
ownerSignal: AbortSignal;
|
||||
isConnected: () => boolean;
|
||||
getPrepared: () => PreparedWorkerSsh | undefined;
|
||||
runner: WorkerWorkspaceRunner;
|
||||
tasks: Set<Promise<unknown>>;
|
||||
};
|
||||
|
||||
function success(result: SpawnResult): boolean {
|
||||
return result.termination === "exit" && result.code === 0;
|
||||
}
|
||||
|
||||
function workspaceSyncError(result: SpawnResult): Error {
|
||||
const detail = redactSensitiveText(result.stderr || result.stdout, { mode: "tools" })
|
||||
.replace(/\s+/gu, " ")
|
||||
.trim();
|
||||
return new Error(
|
||||
detail ? `Worker workspace sync failed: ${detail}` : "Worker workspace sync failed",
|
||||
);
|
||||
}
|
||||
|
||||
export function stableWorkerPathComponent(value: string, length: number): string {
|
||||
return createHash("sha256").update(value).digest("hex").slice(0, length);
|
||||
}
|
||||
|
||||
function validateWorkspaceSyncRequest(request: WorkerWorkspaceSyncRequest): void {
|
||||
if (!request.sessionId.trim()) {
|
||||
throw new Error("Worker workspace session id must be non-empty");
|
||||
}
|
||||
if (!path.isAbsolute(request.localPath)) {
|
||||
throw new Error("Worker workspace local path must be absolute");
|
||||
}
|
||||
if (!Number.isSafeInteger(request.generation) || request.generation < 0) {
|
||||
throw new Error("Worker workspace generation must be a non-negative safe integer");
|
||||
}
|
||||
}
|
||||
|
||||
function parseRemoteWorkspaceDirectory(stdout: string): string {
|
||||
const lines = stdout.split(/\r?\n/u).filter(Boolean);
|
||||
const directory = lines.length === 1 ? lines[0] : undefined;
|
||||
if (
|
||||
!directory ||
|
||||
!path.posix.isAbsolute(directory) ||
|
||||
path.posix.normalize(directory) !== directory ||
|
||||
directory === "/"
|
||||
) {
|
||||
throw new Error("Worker workspace setup returned an invalid remote directory");
|
||||
}
|
||||
return directory;
|
||||
}
|
||||
|
||||
function parseManifestRef(stdout: string): string {
|
||||
const lines = stdout.split(/\r?\n/u).filter(Boolean);
|
||||
const manifestRef = lines.length === 1 ? lines[0] : undefined;
|
||||
if (!manifestRef || !MANIFEST_REF_PATTERN.test(manifestRef)) {
|
||||
throw new Error("Worker workspace sync returned an invalid manifest reference");
|
||||
}
|
||||
return manifestRef;
|
||||
}
|
||||
|
||||
/** Binds workspace commands and synchronization to one connected tunnel owner. */
|
||||
export function createWorkerWorkspaceActions(
|
||||
options: WorkerWorkspaceActionsOptions,
|
||||
): Pick<WorkerTunnelHandle, "runWorkspaceCommand" | "syncWorkspace"> {
|
||||
const track = <T>(task: Promise<T>): Promise<T> => {
|
||||
options.tasks.add(task);
|
||||
void task.then(
|
||||
() => options.tasks.delete(task),
|
||||
() => options.tasks.delete(task),
|
||||
);
|
||||
return task;
|
||||
};
|
||||
|
||||
const requirePrepared = (): PreparedWorkerSsh => {
|
||||
const prepared = options.getPrepared();
|
||||
if (!options.isConnected() || !prepared) {
|
||||
throw new Error("Worker tunnel owner is no longer connected");
|
||||
}
|
||||
return prepared;
|
||||
};
|
||||
|
||||
const runTask = (argv: string[], commandOptions: CommandOptions): Promise<SpawnResult> =>
|
||||
track(options.runner.run(argv, commandOptions));
|
||||
|
||||
const runWorkspaceCommand = async (command: WorkerWorkspaceCommand): Promise<SpawnResult> => {
|
||||
const prepared = requirePrepared();
|
||||
return await runTask(
|
||||
[
|
||||
"ssh",
|
||||
...workerSshOptions(prepared, { forwarding: "disabled" }),
|
||||
"-a",
|
||||
"-x",
|
||||
"-T",
|
||||
"-p",
|
||||
String(prepared.port),
|
||||
"--",
|
||||
prepared.sshTarget,
|
||||
workerSshRemoteCommand(command.argv),
|
||||
],
|
||||
workerSshCommandOptions({
|
||||
input: command.input,
|
||||
timeoutMs: command.timeoutMs ?? WORKSPACE_TIMEOUT_MS,
|
||||
signal: command.signal
|
||||
? AbortSignal.any([options.ownerSignal, command.signal])
|
||||
: options.ownerSignal,
|
||||
}),
|
||||
);
|
||||
};
|
||||
|
||||
const syncWorkspaceImpl = async (
|
||||
request: WorkerWorkspaceSyncRequest,
|
||||
): Promise<WorkerWorkspaceSyncResult> => {
|
||||
validateWorkspaceSyncRequest(request);
|
||||
const prepared = requirePrepared();
|
||||
const environmentKey = stableWorkerPathComponent(options.environmentId, 16);
|
||||
const sessionKey = stableWorkerPathComponent(request.sessionId, 32);
|
||||
const remoteRelative = [
|
||||
REMOTE_WORKSPACE_ROOT,
|
||||
environmentKey,
|
||||
sessionKey,
|
||||
String(request.generation),
|
||||
].join("/");
|
||||
const setup = await runWorkspaceCommand({
|
||||
argv: ["sh", "-s", "--", remoteRelative],
|
||||
input: REMOTE_WORKSPACE_SETUP_SCRIPT,
|
||||
});
|
||||
if (!success(setup)) {
|
||||
throw workspaceSyncError(setup);
|
||||
}
|
||||
const remoteWorkspaceDir = parseRemoteWorkspaceDirectory(setup.stdout.trim());
|
||||
|
||||
const gitRootResult = await runTask(
|
||||
["git", "-C", request.localPath, "rev-parse", "--show-toplevel"],
|
||||
workerSshCommandOptions({
|
||||
timeoutMs: REMOTE_SETUP_TIMEOUT_MS,
|
||||
signal: options.ownerSignal,
|
||||
}),
|
||||
);
|
||||
const mode = success(gitRootResult) ? "git" : "plain";
|
||||
let baseCommit = "";
|
||||
let gitRoot = request.localPath;
|
||||
const temporaryDirectory = await fs.mkdtemp(
|
||||
path.join(os.tmpdir(), "openclaw-worker-workspace-sync-"),
|
||||
);
|
||||
const rsyncSsh = workerSshRemoteCommand([
|
||||
"ssh",
|
||||
...workerSshOptions(prepared, { forwarding: "disabled" }),
|
||||
"-a",
|
||||
"-x",
|
||||
"-T",
|
||||
"-p",
|
||||
String(prepared.port),
|
||||
]);
|
||||
try {
|
||||
let fileListPath: string | undefined;
|
||||
if (mode === "git") {
|
||||
gitRoot = gitRootResult.stdout.trim();
|
||||
const [canonicalRequestPath, canonicalGitRoot] = await Promise.all([
|
||||
fs.realpath(request.localPath),
|
||||
fs.realpath(gitRoot),
|
||||
]);
|
||||
if (canonicalRequestPath !== canonicalGitRoot) {
|
||||
throw new Error("Worker git workspace sync requires the managed worktree root");
|
||||
}
|
||||
const gitBase = await runTask(
|
||||
["git", "-C", gitRoot, "rev-parse", "--verify", "HEAD"],
|
||||
workerSshCommandOptions({
|
||||
timeoutMs: REMOTE_SETUP_TIMEOUT_MS,
|
||||
signal: options.ownerSignal,
|
||||
}),
|
||||
);
|
||||
if (!success(gitBase)) {
|
||||
throw new Error("Worker git workspace has no base commit");
|
||||
}
|
||||
baseCommit = gitBase.stdout.trim();
|
||||
if (!GIT_COMMIT_PATTERN.test(baseCommit)) {
|
||||
throw new Error("Worker workspace git base is not a commit id");
|
||||
}
|
||||
|
||||
const eligiblePath = path.join(temporaryDirectory, "eligible");
|
||||
const ignoredPath = path.join(temporaryDirectory, "ignored");
|
||||
const selectedPath = path.join(temporaryDirectory, "selected");
|
||||
fileListPath = path.join(temporaryDirectory, "transfer-list");
|
||||
await runLocalCommandToFile({
|
||||
argv: [
|
||||
"git",
|
||||
"-C",
|
||||
gitRoot,
|
||||
"ls-files",
|
||||
"--full-name",
|
||||
"--cached",
|
||||
"--others",
|
||||
"--exclude-standard",
|
||||
"-z",
|
||||
],
|
||||
outputPath: eligiblePath,
|
||||
signal: options.ownerSignal,
|
||||
timeoutMs: WORKSPACE_TIMEOUT_MS,
|
||||
});
|
||||
const worktreeIncludePath = path.join(gitRoot, ".worktreeinclude");
|
||||
const worktreeInclude = await fs.lstat(worktreeIncludePath).catch(() => undefined);
|
||||
if (worktreeInclude?.isFile()) {
|
||||
await runLocalCommandToFile({
|
||||
argv: [
|
||||
"git",
|
||||
"-C",
|
||||
gitRoot,
|
||||
"ls-files",
|
||||
"--full-name",
|
||||
"--others",
|
||||
"--ignored",
|
||||
"--exclude-standard",
|
||||
"-z",
|
||||
],
|
||||
outputPath: ignoredPath,
|
||||
signal: options.ownerSignal,
|
||||
timeoutMs: WORKSPACE_TIMEOUT_MS,
|
||||
});
|
||||
await runLocalCommandToFile({
|
||||
argv: [
|
||||
"git",
|
||||
"-C",
|
||||
gitRoot,
|
||||
"ls-files",
|
||||
"--full-name",
|
||||
"--others",
|
||||
"--ignored",
|
||||
`--exclude-from=${worktreeIncludePath}`,
|
||||
"-z",
|
||||
],
|
||||
outputPath: selectedPath,
|
||||
signal: options.ownerSignal,
|
||||
timeoutMs: WORKSPACE_TIMEOUT_MS,
|
||||
});
|
||||
} else {
|
||||
await Promise.all([
|
||||
fs.writeFile(ignoredPath, "", { mode: 0o600 }),
|
||||
fs.writeFile(selectedPath, "", { mode: 0o600 }),
|
||||
]);
|
||||
}
|
||||
await writeEligibleGitFiles({
|
||||
gitRoot,
|
||||
eligiblePath,
|
||||
ignoredPath,
|
||||
selectedPath,
|
||||
outputPath: fileListPath,
|
||||
});
|
||||
|
||||
const objectListPath = path.join(temporaryDirectory, "base-objects");
|
||||
const packPath = path.join(temporaryDirectory, "base.pack");
|
||||
await runLocalCommandToFile({
|
||||
argv: [
|
||||
"git",
|
||||
"-C",
|
||||
gitRoot,
|
||||
"rev-list",
|
||||
"--objects",
|
||||
"--no-object-names",
|
||||
`${baseCommit}^{tree}`,
|
||||
],
|
||||
outputPath: objectListPath,
|
||||
signal: options.ownerSignal,
|
||||
timeoutMs: WORKSPACE_TIMEOUT_MS,
|
||||
});
|
||||
await fs.appendFile(objectListPath, `${baseCommit}\n`);
|
||||
await runLocalCommandToFile({
|
||||
argv: ["git", "-C", gitRoot, "pack-objects", "--stdout"],
|
||||
inputPath: objectListPath,
|
||||
outputPath: packPath,
|
||||
signal: options.ownerSignal,
|
||||
timeoutMs: WORKSPACE_TIMEOUT_MS,
|
||||
});
|
||||
const packTransfer = await runTask(
|
||||
[
|
||||
"rsync",
|
||||
"--archive",
|
||||
"--checksum",
|
||||
"-e",
|
||||
rsyncSsh,
|
||||
"--",
|
||||
packPath,
|
||||
`${prepared.scpTarget}:${remoteWorkspaceDir}/${REMOTE_GIT_PACK_NAME}`,
|
||||
],
|
||||
workerSshCommandOptions({
|
||||
timeoutMs: WORKSPACE_TIMEOUT_MS,
|
||||
signal: options.ownerSignal,
|
||||
}),
|
||||
);
|
||||
if (!success(packTransfer)) {
|
||||
throw workspaceSyncError(packTransfer);
|
||||
}
|
||||
const [authorName, authorEmail] = await Promise.all(
|
||||
["user.name", "user.email"].map(async (key) => {
|
||||
const result = await runTask(
|
||||
["git", "-C", gitRoot, "config", "--get", key],
|
||||
workerSshCommandOptions({
|
||||
timeoutMs: REMOTE_SETUP_TIMEOUT_MS,
|
||||
signal: options.ownerSignal,
|
||||
}),
|
||||
);
|
||||
return success(result) ? result.stdout.trim() : "";
|
||||
}),
|
||||
);
|
||||
const seeded = await runWorkspaceCommand({
|
||||
argv: [
|
||||
"sh",
|
||||
"-s",
|
||||
"--",
|
||||
remoteWorkspaceDir,
|
||||
path.posix.join(remoteWorkspaceDir, REMOTE_GIT_PACK_NAME),
|
||||
baseCommit,
|
||||
authorName ?? "",
|
||||
authorEmail ?? "",
|
||||
],
|
||||
input: REMOTE_GIT_WORKSPACE_SETUP_SCRIPT,
|
||||
});
|
||||
if (!success(seeded)) {
|
||||
throw workspaceSyncError(seeded);
|
||||
}
|
||||
}
|
||||
|
||||
const localSource = gitRoot.endsWith(path.sep) ? gitRoot : `${gitRoot}${path.sep}`;
|
||||
const transfer = await runTask(
|
||||
[
|
||||
"rsync",
|
||||
"--archive",
|
||||
"--checksum",
|
||||
"--exclude=.git",
|
||||
...(fileListPath ? ["--recursive", "--from0", `--files-from=${fileListPath}`] : []),
|
||||
"-e",
|
||||
rsyncSsh,
|
||||
"--",
|
||||
localSource,
|
||||
`${prepared.scpTarget}:${remoteWorkspaceDir}/`,
|
||||
],
|
||||
workerSshCommandOptions({
|
||||
timeoutMs: WORKSPACE_TIMEOUT_MS,
|
||||
signal: options.ownerSignal,
|
||||
}),
|
||||
);
|
||||
if (!success(transfer)) {
|
||||
throw workspaceSyncError(transfer);
|
||||
}
|
||||
|
||||
const manifest = await runWorkspaceCommand({
|
||||
argv: ["node", "-e", REMOTE_WORKSPACE_MANIFEST_JS, remoteWorkspaceDir, baseCommit],
|
||||
});
|
||||
if (!success(manifest)) {
|
||||
throw workspaceSyncError(manifest);
|
||||
}
|
||||
return {
|
||||
mode,
|
||||
remoteWorkspaceDir,
|
||||
manifestRef: parseManifestRef(manifest.stdout.trim()),
|
||||
};
|
||||
} finally {
|
||||
await fs.rm(temporaryDirectory, { recursive: true, force: true });
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
runWorkspaceCommand,
|
||||
syncWorkspace(request) {
|
||||
// Keep the outer task registered across local-file phases so tunnel stop drains all owner work.
|
||||
return track(syncWorkspaceImpl(request));
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -223,6 +223,45 @@ describe("agent-events sequencing", () => {
|
||||
expect(seen).toEqual(["worker"]);
|
||||
});
|
||||
|
||||
test("explicitly adopts only an unowned same-generation context", () => {
|
||||
const lifecycleGeneration = getAgentEventLifecycleGeneration();
|
||||
registerAgentRunContext("adopted-run", {
|
||||
agentId: "main",
|
||||
isControlUiVisible: false,
|
||||
lifecycleGeneration,
|
||||
sessionId: "session-adopted",
|
||||
sessionKey: "agent:main:adopted",
|
||||
});
|
||||
|
||||
const claimId = claimAgentRunContext(
|
||||
"adopted-run",
|
||||
{
|
||||
agentId: "main",
|
||||
isControlUiVisible: false,
|
||||
lifecycleGeneration,
|
||||
sessionId: "session-adopted",
|
||||
sessionKey: "agent:main:adopted",
|
||||
},
|
||||
{
|
||||
adoptExistingUnowned: true,
|
||||
exclusive: true,
|
||||
ownsContext: true,
|
||||
trackOwner: true,
|
||||
},
|
||||
);
|
||||
expect(claimId).toBeDefined();
|
||||
expect(
|
||||
claimAgentRunContext(
|
||||
"adopted-run",
|
||||
{ lifecycleGeneration, sessionKey: "agent:main:adopted" },
|
||||
{ adoptExistingUnowned: true, exclusive: true, trackOwner: true },
|
||||
),
|
||||
).toBeUndefined();
|
||||
|
||||
releaseAgentRunContext("adopted-run", claimId);
|
||||
expect(getAgentRunContext("adopted-run")).toBeUndefined();
|
||||
});
|
||||
|
||||
test("full event reset clears tracked ownership", () => {
|
||||
const lifecycleGeneration = getAgentEventLifecycleGeneration();
|
||||
claimAgentRunContext(
|
||||
|
||||
@@ -300,6 +300,8 @@ export function claimAgentRunContext(
|
||||
runId: string,
|
||||
context: AgentRunContext,
|
||||
options: {
|
||||
/** Adopt a same-generation context only when no tracked execution owns it. */
|
||||
adoptExistingUnowned?: boolean;
|
||||
trackOwner?: boolean;
|
||||
ownsContext?: boolean;
|
||||
exclusive?: boolean;
|
||||
@@ -316,11 +318,16 @@ export function claimAgentRunContext(
|
||||
const existingOwners = ownersById.get(runId);
|
||||
const currentOwners =
|
||||
existingOwners?.lifecycleGeneration === lifecycleGeneration ? existingOwners : undefined;
|
||||
const adoptsExistingUnowned =
|
||||
options.exclusive === true &&
|
||||
options.adoptExistingUnowned === true &&
|
||||
existing?.lifecycleGeneration === lifecycleGeneration &&
|
||||
currentOwners === undefined;
|
||||
if (
|
||||
currentOwners?.exclusiveClaimId ||
|
||||
(options.exclusive &&
|
||||
(existing?.lifecycleGeneration === lifecycleGeneration ||
|
||||
(currentOwners?.claimIds.size ?? 0) > 0))
|
||||
((existing?.lifecycleGeneration === lifecycleGeneration && !adoptsExistingUnowned) ||
|
||||
currentOwners !== undefined))
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
+25
@@ -1149,6 +1149,30 @@ export interface WorkerInferenceTurns {
|
||||
updated_at_ms: number;
|
||||
}
|
||||
|
||||
export interface WorkerSessionPlacements {
|
||||
active_owner_epoch: number | null;
|
||||
agent_id: string;
|
||||
created_at_ms: number;
|
||||
environment_id: string | null;
|
||||
last_live_event_ack_cursor: number | null;
|
||||
last_transcript_ack_cursor: number | null;
|
||||
recovery_error: string | null;
|
||||
remote_workspace_dir: string | null;
|
||||
session_id: string;
|
||||
session_key: string;
|
||||
state: string;
|
||||
state_changed_at_ms: number;
|
||||
transition_generation: Generated<number>;
|
||||
turn_claim_generation: number | null;
|
||||
turn_claim_id: string | null;
|
||||
turn_claim_owner: string | null;
|
||||
turn_claim_owner_epoch: number | null;
|
||||
turn_claim_run_id: string | null;
|
||||
updated_at_ms: number;
|
||||
worker_bundle_hash: string | null;
|
||||
workspace_base_manifest_ref: string | null;
|
||||
}
|
||||
|
||||
export interface WorkerTranscriptCommitHeads {
|
||||
environment_id: string;
|
||||
next_seq: number;
|
||||
@@ -1272,6 +1296,7 @@ export interface DB {
|
||||
worker_environment_credentials: WorkerEnvironmentCredentials;
|
||||
worker_environments: WorkerEnvironments;
|
||||
worker_inference_turns: WorkerInferenceTurns;
|
||||
worker_session_placements: WorkerSessionPlacements;
|
||||
worker_transcript_commit_heads: WorkerTranscriptCommitHeads;
|
||||
worker_transcript_commits: WorkerTranscriptCommits;
|
||||
workspace_setup_state: WorkspaceSetupState;
|
||||
|
||||
@@ -46,6 +46,71 @@ function createTempStateDir(): string {
|
||||
return makeTempDir(stateDbTempDirs, "openclaw-state-db-");
|
||||
}
|
||||
|
||||
type PlacementConstraintProbe = {
|
||||
sessionId: string;
|
||||
state: string;
|
||||
environmentId: string | null;
|
||||
activeOwnerEpoch: number | null;
|
||||
workerBundleHash: string | null;
|
||||
recoveryError: string | null;
|
||||
workspaceBaseManifestRef?: string;
|
||||
remoteWorkspaceDir?: string;
|
||||
lastTranscriptAckCursor?: number;
|
||||
lastLiveEventAckCursor?: number;
|
||||
turnClaimOwner?: "local" | "worker";
|
||||
turnClaimOwnerEpoch?: number;
|
||||
};
|
||||
|
||||
function insertPlacementConstraintProbe(
|
||||
database: DatabaseSync,
|
||||
input: PlacementConstraintProbe,
|
||||
): void {
|
||||
const hasClaim = input.turnClaimOwner !== undefined;
|
||||
database
|
||||
.prepare(
|
||||
`INSERT INTO worker_session_placements (
|
||||
session_id,
|
||||
agent_id,
|
||||
session_key,
|
||||
state,
|
||||
environment_id,
|
||||
active_owner_epoch,
|
||||
workspace_base_manifest_ref,
|
||||
remote_workspace_dir,
|
||||
worker_bundle_hash,
|
||||
last_transcript_ack_cursor,
|
||||
last_live_event_ack_cursor,
|
||||
recovery_error,
|
||||
turn_claim_owner,
|
||||
turn_claim_id,
|
||||
turn_claim_run_id,
|
||||
turn_claim_generation,
|
||||
turn_claim_owner_epoch,
|
||||
created_at_ms,
|
||||
updated_at_ms,
|
||||
state_changed_at_ms
|
||||
) VALUES (?, 'main', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, 1, 1)`,
|
||||
)
|
||||
.run(
|
||||
input.sessionId,
|
||||
`agent:main:${input.sessionId}`,
|
||||
input.state,
|
||||
input.environmentId,
|
||||
input.activeOwnerEpoch,
|
||||
input.workspaceBaseManifestRef ?? null,
|
||||
input.remoteWorkspaceDir ?? null,
|
||||
input.workerBundleHash,
|
||||
input.lastTranscriptAckCursor ?? null,
|
||||
input.lastLiveEventAckCursor ?? null,
|
||||
input.recoveryError,
|
||||
input.turnClaimOwner ?? null,
|
||||
hasClaim ? `${input.sessionId}-claim` : null,
|
||||
hasClaim ? `${input.sessionId}-run` : null,
|
||||
hasClaim ? 0 : null,
|
||||
input.turnClaimOwnerEpoch ?? null,
|
||||
);
|
||||
}
|
||||
|
||||
function statfsFixture(type: number): ReturnType<typeof fs.statfsSync> {
|
||||
return {
|
||||
type,
|
||||
@@ -656,6 +721,375 @@ describe("openclaw state database", () => {
|
||||
expect(database.path).toBe(path.join(stateDir, "state", "openclaw.sqlite"));
|
||||
});
|
||||
|
||||
it("rejects a placement turn claim tuple without an owner", () => {
|
||||
const database = openOpenClawStateDatabase({
|
||||
env: { OPENCLAW_STATE_DIR: createTempStateDir() },
|
||||
});
|
||||
|
||||
expect(() =>
|
||||
database.db
|
||||
.prepare(
|
||||
`INSERT INTO worker_session_placements (
|
||||
session_id,
|
||||
agent_id,
|
||||
session_key,
|
||||
state,
|
||||
turn_claim_id,
|
||||
turn_claim_run_id,
|
||||
turn_claim_generation,
|
||||
created_at_ms,
|
||||
updated_at_ms,
|
||||
state_changed_at_ms
|
||||
) VALUES (?, 'main', 'agent:main:placement-claim', 'local', ?, ?, 0, 1, 1, 1)`,
|
||||
)
|
||||
.run("session-placement-claim", "claim-without-owner", "run-without-owner"),
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
const validPlacementShapes = [
|
||||
{
|
||||
name: "local placement",
|
||||
sessionId: "session-local-valid",
|
||||
state: "local",
|
||||
environmentId: null,
|
||||
activeOwnerEpoch: null,
|
||||
workerBundleHash: null,
|
||||
recoveryError: null,
|
||||
},
|
||||
{
|
||||
name: "requested placement",
|
||||
sessionId: "session-requested-valid",
|
||||
state: "requested",
|
||||
environmentId: null,
|
||||
activeOwnerEpoch: null,
|
||||
workerBundleHash: null,
|
||||
recoveryError: null,
|
||||
},
|
||||
{
|
||||
name: "provisioning placement before environment allocation",
|
||||
sessionId: "session-provisioning-pending-valid",
|
||||
state: "provisioning",
|
||||
environmentId: null,
|
||||
activeOwnerEpoch: null,
|
||||
workerBundleHash: null,
|
||||
recoveryError: null,
|
||||
},
|
||||
{
|
||||
name: "provisioning placement after environment allocation",
|
||||
sessionId: "session-provisioning-allocated-valid",
|
||||
state: "provisioning",
|
||||
environmentId: "environment-provisioning",
|
||||
activeOwnerEpoch: null,
|
||||
workerBundleHash: null,
|
||||
recoveryError: null,
|
||||
},
|
||||
{
|
||||
name: "syncing placement",
|
||||
sessionId: "session-syncing-valid",
|
||||
state: "syncing",
|
||||
environmentId: "environment-syncing",
|
||||
activeOwnerEpoch: null,
|
||||
workerBundleHash: "bundle-syncing",
|
||||
recoveryError: null,
|
||||
},
|
||||
{
|
||||
name: "starting placement",
|
||||
sessionId: "session-starting-valid",
|
||||
state: "starting",
|
||||
environmentId: "environment-starting",
|
||||
activeOwnerEpoch: null,
|
||||
workspaceBaseManifestRef: "manifest-starting",
|
||||
remoteWorkspaceDir: "/workspace/starting",
|
||||
workerBundleHash: "bundle-starting",
|
||||
recoveryError: null,
|
||||
},
|
||||
{
|
||||
name: "active placement",
|
||||
sessionId: "session-active-valid",
|
||||
state: "active",
|
||||
environmentId: "environment-active",
|
||||
activeOwnerEpoch: 7,
|
||||
workspaceBaseManifestRef: "manifest-active",
|
||||
remoteWorkspaceDir: "/workspace/active",
|
||||
workerBundleHash: "bundle-active",
|
||||
lastTranscriptAckCursor: 3,
|
||||
lastLiveEventAckCursor: 4,
|
||||
recoveryError: null,
|
||||
},
|
||||
{
|
||||
name: "draining placement",
|
||||
sessionId: "session-draining-valid",
|
||||
state: "draining",
|
||||
environmentId: "environment-draining",
|
||||
activeOwnerEpoch: 7,
|
||||
workspaceBaseManifestRef: "manifest-draining",
|
||||
remoteWorkspaceDir: "/workspace/draining",
|
||||
workerBundleHash: "bundle-draining",
|
||||
recoveryError: null,
|
||||
},
|
||||
{
|
||||
name: "reconciling placement",
|
||||
sessionId: "session-reconciling-valid",
|
||||
state: "reconciling",
|
||||
environmentId: "environment-reconciling",
|
||||
activeOwnerEpoch: 7,
|
||||
workspaceBaseManifestRef: "manifest-reconciling",
|
||||
remoteWorkspaceDir: "/workspace/reconciling",
|
||||
workerBundleHash: "bundle-reconciling",
|
||||
recoveryError: null,
|
||||
},
|
||||
{
|
||||
name: "reclaimed placement with full provenance",
|
||||
sessionId: "session-reclaimed-valid",
|
||||
state: "reclaimed",
|
||||
environmentId: "environment-reclaimed",
|
||||
activeOwnerEpoch: 7,
|
||||
workspaceBaseManifestRef: "manifest-reclaimed",
|
||||
remoteWorkspaceDir: "/workspace/reclaimed",
|
||||
workerBundleHash: "bundle-reclaimed",
|
||||
recoveryError: null,
|
||||
},
|
||||
{
|
||||
name: "failed placement with recovery detail",
|
||||
sessionId: "session-failed-valid",
|
||||
state: "failed",
|
||||
environmentId: "environment-failed",
|
||||
activeOwnerEpoch: null,
|
||||
workerBundleHash: null,
|
||||
recoveryError: "worker placement failed",
|
||||
},
|
||||
] satisfies Array<PlacementConstraintProbe & { name: string }>;
|
||||
|
||||
it.each(validPlacementShapes)("allows a valid $name", (input) => {
|
||||
const database = openOpenClawStateDatabase({
|
||||
env: { OPENCLAW_STATE_DIR: createTempStateDir() },
|
||||
});
|
||||
|
||||
expect(() => insertPlacementConstraintProbe(database.db, input)).not.toThrow();
|
||||
});
|
||||
|
||||
const invalidPlacementShapes = [
|
||||
{
|
||||
name: "local environment",
|
||||
sessionId: "session-local-environment",
|
||||
state: "local",
|
||||
environmentId: "environment-local",
|
||||
activeOwnerEpoch: null,
|
||||
workerBundleHash: null,
|
||||
recoveryError: null,
|
||||
},
|
||||
{
|
||||
name: "syncing without environment",
|
||||
sessionId: "session-syncing-environment",
|
||||
state: "syncing",
|
||||
environmentId: null,
|
||||
activeOwnerEpoch: null,
|
||||
workerBundleHash: "bundle-hash",
|
||||
recoveryError: null,
|
||||
},
|
||||
{
|
||||
name: "syncing workspace metadata",
|
||||
sessionId: "session-syncing-workspace",
|
||||
state: "syncing",
|
||||
environmentId: "environment-syncing",
|
||||
activeOwnerEpoch: null,
|
||||
workspaceBaseManifestRef: "manifest-syncing",
|
||||
remoteWorkspaceDir: "/workspace/syncing",
|
||||
workerBundleHash: "bundle-hash",
|
||||
recoveryError: null,
|
||||
},
|
||||
{
|
||||
name: "active without owner epoch",
|
||||
sessionId: "session-active-epoch",
|
||||
state: "active",
|
||||
environmentId: "environment-active",
|
||||
activeOwnerEpoch: null,
|
||||
workerBundleHash: "bundle-hash",
|
||||
recoveryError: null,
|
||||
workspaceBaseManifestRef: "manifest-active",
|
||||
remoteWorkspaceDir: "/workspace/active",
|
||||
},
|
||||
{
|
||||
name: "active without worker bundle",
|
||||
sessionId: "session-active-bundle",
|
||||
state: "active",
|
||||
environmentId: "environment-active",
|
||||
activeOwnerEpoch: 7,
|
||||
workerBundleHash: null,
|
||||
recoveryError: null,
|
||||
workspaceBaseManifestRef: "manifest-active",
|
||||
remoteWorkspaceDir: "/workspace/active",
|
||||
},
|
||||
{
|
||||
name: "starting without manifest",
|
||||
sessionId: "session-starting-manifest",
|
||||
state: "starting",
|
||||
environmentId: "environment-starting",
|
||||
activeOwnerEpoch: null,
|
||||
workerBundleHash: "bundle-hash",
|
||||
recoveryError: null,
|
||||
remoteWorkspaceDir: "/workspace/starting",
|
||||
},
|
||||
{
|
||||
name: "starting owner epoch",
|
||||
sessionId: "session-starting-epoch",
|
||||
state: "starting",
|
||||
environmentId: "environment-starting",
|
||||
activeOwnerEpoch: 7,
|
||||
workspaceBaseManifestRef: "manifest-starting",
|
||||
remoteWorkspaceDir: "/workspace/starting",
|
||||
workerBundleHash: "bundle-hash",
|
||||
recoveryError: null,
|
||||
},
|
||||
{
|
||||
name: "requested worker metadata",
|
||||
sessionId: "session-requested-metadata",
|
||||
state: "requested",
|
||||
environmentId: null,
|
||||
activeOwnerEpoch: null,
|
||||
workerBundleHash: "bundle-hash",
|
||||
recoveryError: null,
|
||||
},
|
||||
{
|
||||
name: "provisioning worker bundle",
|
||||
sessionId: "session-provisioning-bundle",
|
||||
state: "provisioning",
|
||||
environmentId: "environment-provisioning",
|
||||
activeOwnerEpoch: null,
|
||||
workerBundleHash: "bundle-hash",
|
||||
recoveryError: null,
|
||||
},
|
||||
{
|
||||
name: "active recovery error",
|
||||
sessionId: "session-active-recovery",
|
||||
state: "active",
|
||||
environmentId: "environment-active",
|
||||
activeOwnerEpoch: 7,
|
||||
workspaceBaseManifestRef: "manifest-active",
|
||||
remoteWorkspaceDir: "/workspace/active",
|
||||
workerBundleHash: "bundle-hash",
|
||||
recoveryError: "unexpected active recovery detail",
|
||||
},
|
||||
{
|
||||
name: "reclaimed placement without full provenance",
|
||||
sessionId: "session-reclaimed-provenance",
|
||||
state: "reclaimed",
|
||||
environmentId: "environment-reclaimed",
|
||||
activeOwnerEpoch: null,
|
||||
workerBundleHash: "bundle-hash",
|
||||
recoveryError: null,
|
||||
},
|
||||
{
|
||||
name: "reclaimed recovery error",
|
||||
sessionId: "session-reclaimed-recovery",
|
||||
state: "reclaimed",
|
||||
environmentId: "environment-reclaimed",
|
||||
activeOwnerEpoch: 7,
|
||||
workspaceBaseManifestRef: "manifest-reclaimed",
|
||||
remoteWorkspaceDir: "/workspace/reclaimed",
|
||||
workerBundleHash: "bundle-hash",
|
||||
recoveryError: "unexpected reclaimed recovery detail",
|
||||
},
|
||||
{
|
||||
name: "failed without recovery error",
|
||||
sessionId: "session-failed-recovery",
|
||||
state: "failed",
|
||||
environmentId: null,
|
||||
activeOwnerEpoch: null,
|
||||
workerBundleHash: null,
|
||||
recoveryError: null,
|
||||
},
|
||||
] satisfies Array<PlacementConstraintProbe & { name: string }>;
|
||||
|
||||
it.each(invalidPlacementShapes)("rejects a placement with $name", (input) => {
|
||||
const database = openOpenClawStateDatabase({
|
||||
env: { OPENCLAW_STATE_DIR: createTempStateDir() },
|
||||
});
|
||||
|
||||
expect(() => insertPlacementConstraintProbe(database.db, input)).toThrow();
|
||||
});
|
||||
|
||||
const invalidPlacementClaimOwners = [
|
||||
{
|
||||
name: "local claim on active placement",
|
||||
state: "active",
|
||||
activeOwnerEpoch: 7,
|
||||
turnClaimOwner: "local",
|
||||
turnClaimOwnerEpoch: undefined,
|
||||
},
|
||||
{
|
||||
name: "worker claim on reconciling placement",
|
||||
state: "reconciling",
|
||||
activeOwnerEpoch: 7,
|
||||
turnClaimOwner: "worker",
|
||||
turnClaimOwnerEpoch: 7,
|
||||
},
|
||||
{
|
||||
name: "stale worker owner epoch",
|
||||
state: "active",
|
||||
activeOwnerEpoch: 7,
|
||||
turnClaimOwner: "worker",
|
||||
turnClaimOwnerEpoch: 8,
|
||||
},
|
||||
{
|
||||
name: "worker claim on reclaimed placement",
|
||||
state: "reclaimed",
|
||||
activeOwnerEpoch: 7,
|
||||
turnClaimOwner: "worker",
|
||||
turnClaimOwnerEpoch: 7,
|
||||
},
|
||||
] satisfies Array<{
|
||||
name: string;
|
||||
state: string;
|
||||
activeOwnerEpoch: number;
|
||||
turnClaimOwner: "local" | "worker";
|
||||
turnClaimOwnerEpoch: number | undefined;
|
||||
}>;
|
||||
|
||||
it.each(invalidPlacementClaimOwners)("rejects a placement with $name", (input) => {
|
||||
const database = openOpenClawStateDatabase({
|
||||
env: { OPENCLAW_STATE_DIR: createTempStateDir() },
|
||||
});
|
||||
|
||||
expect(() =>
|
||||
insertPlacementConstraintProbe(database.db, {
|
||||
sessionId: `session-${input.state}-${input.turnClaimOwner}`,
|
||||
state: input.state,
|
||||
environmentId: `environment-${input.state}`,
|
||||
activeOwnerEpoch: input.activeOwnerEpoch,
|
||||
workspaceBaseManifestRef: `manifest-${input.state}`,
|
||||
remoteWorkspaceDir: `/workspace/${input.state}`,
|
||||
workerBundleHash: "bundle-hash",
|
||||
recoveryError: null,
|
||||
turnClaimOwner: input.turnClaimOwner,
|
||||
...(input.turnClaimOwnerEpoch === undefined
|
||||
? {}
|
||||
: { turnClaimOwnerEpoch: input.turnClaimOwnerEpoch }),
|
||||
}),
|
||||
).toThrow();
|
||||
});
|
||||
|
||||
it("allows an exact worker claim while placement drains", () => {
|
||||
const database = openOpenClawStateDatabase({
|
||||
env: { OPENCLAW_STATE_DIR: createTempStateDir() },
|
||||
});
|
||||
|
||||
expect(() =>
|
||||
insertPlacementConstraintProbe(database.db, {
|
||||
sessionId: "session-draining-worker",
|
||||
state: "draining",
|
||||
environmentId: "environment-draining",
|
||||
activeOwnerEpoch: 7,
|
||||
workspaceBaseManifestRef: "manifest-draining",
|
||||
remoteWorkspaceDir: "/workspace/draining",
|
||||
workerBundleHash: "bundle-hash",
|
||||
recoveryError: null,
|
||||
turnClaimOwner: "worker",
|
||||
turnClaimOwnerEpoch: 7,
|
||||
}),
|
||||
).not.toThrow();
|
||||
});
|
||||
|
||||
it("repairs a same-name shared-state uniqueness index", () => {
|
||||
const stateDir = createTempStateDir();
|
||||
const env = { OPENCLAW_STATE_DIR: stateDir };
|
||||
|
||||
@@ -1586,6 +1586,122 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_worker_environments_provider_lease
|
||||
ON worker_environments(provider_id, lease_id)
|
||||
WHERE lease_id IS NOT NULL;
|
||||
|
||||
-- Session placement lives in the shared state database so local admission,
|
||||
-- worker admission, and environment attachment use one durable authority.
|
||||
CREATE TABLE IF NOT EXISTS worker_session_placements (
|
||||
session_id TEXT NOT NULL PRIMARY KEY,
|
||||
agent_id TEXT NOT NULL,
|
||||
session_key TEXT NOT NULL,
|
||||
state TEXT NOT NULL CHECK (
|
||||
state IN (
|
||||
'local',
|
||||
'requested',
|
||||
'provisioning',
|
||||
'syncing',
|
||||
'starting',
|
||||
'active',
|
||||
'draining',
|
||||
'reconciling',
|
||||
'reclaimed',
|
||||
'failed'
|
||||
)
|
||||
),
|
||||
environment_id TEXT,
|
||||
transition_generation INTEGER NOT NULL DEFAULT 0 CHECK (transition_generation >= 0),
|
||||
active_owner_epoch INTEGER CHECK (active_owner_epoch IS NULL OR active_owner_epoch >= 1),
|
||||
workspace_base_manifest_ref TEXT,
|
||||
remote_workspace_dir TEXT,
|
||||
worker_bundle_hash TEXT,
|
||||
last_transcript_ack_cursor INTEGER CHECK (
|
||||
last_transcript_ack_cursor IS NULL OR last_transcript_ack_cursor >= 0
|
||||
),
|
||||
last_live_event_ack_cursor INTEGER CHECK (
|
||||
last_live_event_ack_cursor IS NULL OR last_live_event_ack_cursor >= 0
|
||||
),
|
||||
recovery_error TEXT,
|
||||
turn_claim_owner TEXT CHECK (turn_claim_owner IN ('local', 'worker')),
|
||||
turn_claim_id TEXT,
|
||||
turn_claim_run_id TEXT,
|
||||
turn_claim_generation INTEGER CHECK (
|
||||
turn_claim_generation IS NULL OR turn_claim_generation >= 0
|
||||
),
|
||||
turn_claim_owner_epoch INTEGER CHECK (
|
||||
turn_claim_owner_epoch IS NULL OR turn_claim_owner_epoch >= 1
|
||||
),
|
||||
created_at_ms INTEGER NOT NULL,
|
||||
updated_at_ms INTEGER NOT NULL,
|
||||
state_changed_at_ms INTEGER NOT NULL,
|
||||
CHECK (
|
||||
(state IN ('local', 'requested')
|
||||
AND environment_id IS NULL AND active_owner_epoch IS NULL
|
||||
AND workspace_base_manifest_ref IS NULL AND remote_workspace_dir IS NULL
|
||||
AND worker_bundle_hash IS NULL
|
||||
AND last_transcript_ack_cursor IS NULL AND last_live_event_ack_cursor IS NULL
|
||||
AND recovery_error IS NULL)
|
||||
OR
|
||||
(state IS 'provisioning'
|
||||
AND active_owner_epoch IS NULL
|
||||
AND workspace_base_manifest_ref IS NULL AND remote_workspace_dir IS NULL
|
||||
AND worker_bundle_hash IS NULL
|
||||
AND last_transcript_ack_cursor IS NULL AND last_live_event_ack_cursor IS NULL
|
||||
AND recovery_error IS NULL)
|
||||
OR
|
||||
(state IS 'syncing'
|
||||
AND environment_id IS NOT NULL AND active_owner_epoch IS NULL
|
||||
AND workspace_base_manifest_ref IS NULL AND remote_workspace_dir IS NULL
|
||||
AND worker_bundle_hash IS NOT NULL
|
||||
AND last_transcript_ack_cursor IS NULL AND last_live_event_ack_cursor IS NULL
|
||||
AND recovery_error IS NULL)
|
||||
OR
|
||||
(state IS 'starting'
|
||||
AND environment_id IS NOT NULL AND active_owner_epoch IS NULL
|
||||
AND workspace_base_manifest_ref IS NOT NULL AND remote_workspace_dir IS NOT NULL
|
||||
AND worker_bundle_hash IS NOT NULL
|
||||
AND last_transcript_ack_cursor IS NULL AND last_live_event_ack_cursor IS NULL
|
||||
AND recovery_error IS NULL)
|
||||
OR
|
||||
(state IN ('active', 'draining', 'reconciling')
|
||||
AND environment_id IS NOT NULL AND active_owner_epoch IS NOT NULL
|
||||
AND workspace_base_manifest_ref IS NOT NULL AND remote_workspace_dir IS NOT NULL
|
||||
AND worker_bundle_hash IS NOT NULL AND recovery_error IS NULL)
|
||||
OR
|
||||
(state IS 'reclaimed'
|
||||
AND environment_id IS NOT NULL AND active_owner_epoch IS NOT NULL
|
||||
AND workspace_base_manifest_ref IS NOT NULL AND remote_workspace_dir IS NOT NULL
|
||||
AND worker_bundle_hash IS NOT NULL AND recovery_error IS NULL
|
||||
AND turn_claim_owner IS NULL AND turn_claim_id IS NULL AND turn_claim_run_id IS NULL
|
||||
AND turn_claim_generation IS NULL AND turn_claim_owner_epoch IS NULL)
|
||||
OR
|
||||
(state IS 'failed' AND recovery_error IS NOT NULL)
|
||||
),
|
||||
CHECK (
|
||||
(turn_claim_owner IS NULL AND turn_claim_id IS NULL AND turn_claim_run_id IS NULL
|
||||
AND turn_claim_generation IS NULL AND turn_claim_owner_epoch IS NULL)
|
||||
OR
|
||||
(turn_claim_owner IS 'local' AND turn_claim_id IS NOT NULL
|
||||
AND turn_claim_run_id IS NOT NULL AND turn_claim_generation IS NOT NULL
|
||||
AND turn_claim_owner_epoch IS NULL)
|
||||
OR
|
||||
(turn_claim_owner IS 'worker' AND turn_claim_id IS NOT NULL
|
||||
AND turn_claim_run_id IS NOT NULL AND turn_claim_generation IS NOT NULL
|
||||
AND turn_claim_owner_epoch IS NOT NULL)
|
||||
),
|
||||
CHECK (
|
||||
turn_claim_owner IS NULL
|
||||
OR
|
||||
(turn_claim_owner IS 'local' AND state IN ('local', 'requested', 'failed'))
|
||||
OR
|
||||
(turn_claim_owner IS 'worker' AND state IN ('active', 'draining')
|
||||
AND turn_claim_owner_epoch IS active_owner_epoch)
|
||||
)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_worker_session_placements_session_key
|
||||
ON worker_session_placements(agent_id, session_key);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_worker_session_placements_reconcile
|
||||
ON worker_session_placements(updated_at_ms, session_id);
|
||||
|
||||
-- One active, opaque admission credential per worker environment. Plaintext
|
||||
-- may be retried until delivery acknowledgement but never enters durable state.
|
||||
CREATE TABLE IF NOT EXISTS worker_environment_credentials (
|
||||
|
||||
@@ -1581,6 +1581,122 @@ CREATE UNIQUE INDEX IF NOT EXISTS idx_worker_environments_provider_lease
|
||||
ON worker_environments(provider_id, lease_id)
|
||||
WHERE lease_id IS NOT NULL;
|
||||
|
||||
-- Session placement lives in the shared state database so local admission,
|
||||
-- worker admission, and environment attachment use one durable authority.
|
||||
CREATE TABLE IF NOT EXISTS worker_session_placements (
|
||||
session_id TEXT NOT NULL PRIMARY KEY,
|
||||
agent_id TEXT NOT NULL,
|
||||
session_key TEXT NOT NULL,
|
||||
state TEXT NOT NULL CHECK (
|
||||
state IN (
|
||||
'local',
|
||||
'requested',
|
||||
'provisioning',
|
||||
'syncing',
|
||||
'starting',
|
||||
'active',
|
||||
'draining',
|
||||
'reconciling',
|
||||
'reclaimed',
|
||||
'failed'
|
||||
)
|
||||
),
|
||||
environment_id TEXT,
|
||||
transition_generation INTEGER NOT NULL DEFAULT 0 CHECK (transition_generation >= 0),
|
||||
active_owner_epoch INTEGER CHECK (active_owner_epoch IS NULL OR active_owner_epoch >= 1),
|
||||
workspace_base_manifest_ref TEXT,
|
||||
remote_workspace_dir TEXT,
|
||||
worker_bundle_hash TEXT,
|
||||
last_transcript_ack_cursor INTEGER CHECK (
|
||||
last_transcript_ack_cursor IS NULL OR last_transcript_ack_cursor >= 0
|
||||
),
|
||||
last_live_event_ack_cursor INTEGER CHECK (
|
||||
last_live_event_ack_cursor IS NULL OR last_live_event_ack_cursor >= 0
|
||||
),
|
||||
recovery_error TEXT,
|
||||
turn_claim_owner TEXT CHECK (turn_claim_owner IN ('local', 'worker')),
|
||||
turn_claim_id TEXT,
|
||||
turn_claim_run_id TEXT,
|
||||
turn_claim_generation INTEGER CHECK (
|
||||
turn_claim_generation IS NULL OR turn_claim_generation >= 0
|
||||
),
|
||||
turn_claim_owner_epoch INTEGER CHECK (
|
||||
turn_claim_owner_epoch IS NULL OR turn_claim_owner_epoch >= 1
|
||||
),
|
||||
created_at_ms INTEGER NOT NULL,
|
||||
updated_at_ms INTEGER NOT NULL,
|
||||
state_changed_at_ms INTEGER NOT NULL,
|
||||
CHECK (
|
||||
(state IN ('local', 'requested')
|
||||
AND environment_id IS NULL AND active_owner_epoch IS NULL
|
||||
AND workspace_base_manifest_ref IS NULL AND remote_workspace_dir IS NULL
|
||||
AND worker_bundle_hash IS NULL
|
||||
AND last_transcript_ack_cursor IS NULL AND last_live_event_ack_cursor IS NULL
|
||||
AND recovery_error IS NULL)
|
||||
OR
|
||||
(state IS 'provisioning'
|
||||
AND active_owner_epoch IS NULL
|
||||
AND workspace_base_manifest_ref IS NULL AND remote_workspace_dir IS NULL
|
||||
AND worker_bundle_hash IS NULL
|
||||
AND last_transcript_ack_cursor IS NULL AND last_live_event_ack_cursor IS NULL
|
||||
AND recovery_error IS NULL)
|
||||
OR
|
||||
(state IS 'syncing'
|
||||
AND environment_id IS NOT NULL AND active_owner_epoch IS NULL
|
||||
AND workspace_base_manifest_ref IS NULL AND remote_workspace_dir IS NULL
|
||||
AND worker_bundle_hash IS NOT NULL
|
||||
AND last_transcript_ack_cursor IS NULL AND last_live_event_ack_cursor IS NULL
|
||||
AND recovery_error IS NULL)
|
||||
OR
|
||||
(state IS 'starting'
|
||||
AND environment_id IS NOT NULL AND active_owner_epoch IS NULL
|
||||
AND workspace_base_manifest_ref IS NOT NULL AND remote_workspace_dir IS NOT NULL
|
||||
AND worker_bundle_hash IS NOT NULL
|
||||
AND last_transcript_ack_cursor IS NULL AND last_live_event_ack_cursor IS NULL
|
||||
AND recovery_error IS NULL)
|
||||
OR
|
||||
(state IN ('active', 'draining', 'reconciling')
|
||||
AND environment_id IS NOT NULL AND active_owner_epoch IS NOT NULL
|
||||
AND workspace_base_manifest_ref IS NOT NULL AND remote_workspace_dir IS NOT NULL
|
||||
AND worker_bundle_hash IS NOT NULL AND recovery_error IS NULL)
|
||||
OR
|
||||
(state IS 'reclaimed'
|
||||
AND environment_id IS NOT NULL AND active_owner_epoch IS NOT NULL
|
||||
AND workspace_base_manifest_ref IS NOT NULL AND remote_workspace_dir IS NOT NULL
|
||||
AND worker_bundle_hash IS NOT NULL AND recovery_error IS NULL
|
||||
AND turn_claim_owner IS NULL AND turn_claim_id IS NULL AND turn_claim_run_id IS NULL
|
||||
AND turn_claim_generation IS NULL AND turn_claim_owner_epoch IS NULL)
|
||||
OR
|
||||
(state IS 'failed' AND recovery_error IS NOT NULL)
|
||||
),
|
||||
CHECK (
|
||||
(turn_claim_owner IS NULL AND turn_claim_id IS NULL AND turn_claim_run_id IS NULL
|
||||
AND turn_claim_generation IS NULL AND turn_claim_owner_epoch IS NULL)
|
||||
OR
|
||||
(turn_claim_owner IS 'local' AND turn_claim_id IS NOT NULL
|
||||
AND turn_claim_run_id IS NOT NULL AND turn_claim_generation IS NOT NULL
|
||||
AND turn_claim_owner_epoch IS NULL)
|
||||
OR
|
||||
(turn_claim_owner IS 'worker' AND turn_claim_id IS NOT NULL
|
||||
AND turn_claim_run_id IS NOT NULL AND turn_claim_generation IS NOT NULL
|
||||
AND turn_claim_owner_epoch IS NOT NULL)
|
||||
),
|
||||
CHECK (
|
||||
turn_claim_owner IS NULL
|
||||
OR
|
||||
(turn_claim_owner IS 'local' AND state IN ('local', 'requested', 'failed'))
|
||||
OR
|
||||
(turn_claim_owner IS 'worker' AND state IN ('active', 'draining')
|
||||
AND turn_claim_owner_epoch IS active_owner_epoch)
|
||||
)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_worker_session_placements_session_key
|
||||
ON worker_session_placements(agent_id, session_key);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_worker_session_placements_reconcile
|
||||
ON worker_session_placements(updated_at_ms, session_id);
|
||||
|
||||
-- One active, opaque admission credential per worker environment. Plaintext
|
||||
-- may be retried until delivery acknowledgement but never enters durable state.
|
||||
CREATE TABLE IF NOT EXISTS worker_environment_credentials (
|
||||
|
||||
@@ -166,6 +166,7 @@ type WorkerLiveRuntime = {
|
||||
handleSessionEvent: (event: AgentSessionEvent) => void;
|
||||
enqueueRunFailure: (failure: { aborted: boolean; error: Error }) => void;
|
||||
flush: () => Promise<void>;
|
||||
emitTerminal: () => Promise<void>;
|
||||
};
|
||||
|
||||
export function createWorkerLiveRuntime(client: WorkerLiveClient): WorkerLiveRuntime {
|
||||
@@ -219,6 +220,9 @@ export function createWorkerLiveRuntime(client: WorkerLiveClient): WorkerLiveRun
|
||||
};
|
||||
const startedAt = Date.now();
|
||||
let lifecycleFinished = false;
|
||||
// Terminal lifecycle events are deferred past the final transcript flush so the
|
||||
// gateway never sees an end/error before the authoritative transcript commit.
|
||||
let terminalLiveEvent: WorkerLiveEvent | undefined;
|
||||
let streamedText = "";
|
||||
let streamedThinking = "";
|
||||
const handleSessionEvent = (event: AgentSessionEvent) => {
|
||||
@@ -310,17 +314,18 @@ export function createWorkerLiveRuntime(client: WorkerLiveClient): WorkerLiveRun
|
||||
.toReversed()
|
||||
.find((message): message is AssistantMessage => message.role === "assistant");
|
||||
if (lastAssistant?.stopReason === "error") {
|
||||
enqueueLive({
|
||||
terminalLiveEvent = {
|
||||
kind: "lifecycle",
|
||||
payload: {
|
||||
phase: "error",
|
||||
startedAt,
|
||||
endedAt: Date.now(),
|
||||
error: lastAssistant.errorMessage ?? "Worker inference failed.",
|
||||
fallbackExhaustedFailure: true,
|
||||
},
|
||||
});
|
||||
};
|
||||
} else if (lastAssistant?.stopReason === "aborted") {
|
||||
enqueueLive({
|
||||
terminalLiveEvent = {
|
||||
kind: "lifecycle",
|
||||
payload: {
|
||||
phase: "end",
|
||||
@@ -329,12 +334,12 @@ export function createWorkerLiveRuntime(client: WorkerLiveClient): WorkerLiveRun
|
||||
stopReason: "aborted",
|
||||
aborted: true,
|
||||
},
|
||||
});
|
||||
};
|
||||
} else {
|
||||
enqueueLive({
|
||||
terminalLiveEvent = {
|
||||
kind: "lifecycle",
|
||||
payload: { phase: "end", startedAt, endedAt: Date.now() },
|
||||
});
|
||||
};
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -343,7 +348,7 @@ export function createWorkerLiveRuntime(client: WorkerLiveClient): WorkerLiveRun
|
||||
return;
|
||||
}
|
||||
if (failure.aborted) {
|
||||
enqueueLive({
|
||||
terminalLiveEvent = {
|
||||
kind: "lifecycle",
|
||||
payload: {
|
||||
phase: "end",
|
||||
@@ -352,18 +357,27 @@ export function createWorkerLiveRuntime(client: WorkerLiveClient): WorkerLiveRun
|
||||
stopReason: "aborted",
|
||||
aborted: true,
|
||||
},
|
||||
});
|
||||
};
|
||||
} else {
|
||||
enqueueLive({
|
||||
terminalLiveEvent = {
|
||||
kind: "lifecycle",
|
||||
payload: {
|
||||
phase: "error",
|
||||
startedAt,
|
||||
endedAt: Date.now(),
|
||||
error: failure.error.message,
|
||||
fallbackExhaustedFailure: true,
|
||||
},
|
||||
});
|
||||
};
|
||||
}
|
||||
};
|
||||
return { handleSessionEvent, enqueueRunFailure, flush };
|
||||
// Emits directly (not via the degradable preview queue): the terminal event drives
|
||||
// gateway turn settlement and must survive a degraded live stream.
|
||||
const emitTerminal = async () => {
|
||||
if (!terminalLiveEvent) {
|
||||
return;
|
||||
}
|
||||
await client.emit(boundLiveEvent(terminalLiveEvent));
|
||||
};
|
||||
return { handleSessionEvent, enqueueRunFailure, flush, emitTerminal };
|
||||
}
|
||||
|
||||
@@ -4,126 +4,14 @@ import type { WorkerInferenceContext } from "../../packages/gateway-protocol/src
|
||||
import { WORKER_INFERENCE_MAX_CONTEXT_MESSAGES } from "../../packages/gateway-protocol/src/schema/worker-inference.js";
|
||||
import type { AgentMessage } from "../agents/runtime/index.js";
|
||||
import type { AgentSessionWriteLockRunner } from "../agents/sessions/agent-session.js";
|
||||
import type { AssistantMessage, Context, Message } from "../llm/types.js";
|
||||
import { isWorkerTranscriptMessageFrameSafe } from "./transcript-message.js";
|
||||
|
||||
function cloneTextContent(part: { type: "text"; text: string; textSignature?: string }) {
|
||||
return {
|
||||
type: "text" as const,
|
||||
text: part.text,
|
||||
...(part.textSignature ? { textSignature: part.textSignature } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function cloneImageContent(part: { type: "image"; data: string; mimeType: string }) {
|
||||
return { type: "image" as const, data: part.data, mimeType: part.mimeType };
|
||||
}
|
||||
|
||||
function cloneUsage(message: AssistantMessage): WorkerTranscriptMessage & { role: "assistant" } {
|
||||
return {
|
||||
role: "assistant",
|
||||
content: message.content.map((part) => {
|
||||
if (part.type === "text") {
|
||||
return cloneTextContent(part);
|
||||
}
|
||||
if (part.type === "thinking") {
|
||||
return {
|
||||
type: "thinking" as const,
|
||||
thinking: part.thinking,
|
||||
...(part.thinkingSignature ? { thinkingSignature: part.thinkingSignature } : {}),
|
||||
...(part.redacted === undefined ? {} : { redacted: part.redacted }),
|
||||
};
|
||||
}
|
||||
return {
|
||||
type: "toolCall" as const,
|
||||
id: part.id,
|
||||
name: part.name,
|
||||
arguments: structuredClone(part.arguments),
|
||||
...(part.thoughtSignature ? { thoughtSignature: part.thoughtSignature } : {}),
|
||||
...(part.executionMode ? { executionMode: part.executionMode } : {}),
|
||||
};
|
||||
}),
|
||||
api: message.api,
|
||||
provider: message.provider,
|
||||
model: message.model,
|
||||
...(message.responseModel ? { responseModel: message.responseModel } : {}),
|
||||
...(message.responseId ? { responseId: message.responseId } : {}),
|
||||
...(message.diagnostics
|
||||
? {
|
||||
diagnostics: message.diagnostics.map((diagnostic) => ({
|
||||
type: diagnostic.type,
|
||||
timestamp: diagnostic.timestamp,
|
||||
...(diagnostic.error
|
||||
? {
|
||||
error: {
|
||||
...(diagnostic.error.name ? { name: diagnostic.error.name } : {}),
|
||||
message: diagnostic.error.message,
|
||||
...(diagnostic.error.stack ? { stack: diagnostic.error.stack } : {}),
|
||||
...(diagnostic.error.code === undefined ? {} : { code: diagnostic.error.code }),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
...(diagnostic.details ? { details: structuredClone(diagnostic.details) } : {}),
|
||||
})),
|
||||
}
|
||||
: {}),
|
||||
usage: {
|
||||
input: message.usage.input,
|
||||
output: message.usage.output,
|
||||
cacheRead: message.usage.cacheRead,
|
||||
cacheWrite: message.usage.cacheWrite,
|
||||
...(message.usage.contextUsage
|
||||
? { contextUsage: structuredClone(message.usage.contextUsage) }
|
||||
: {}),
|
||||
totalTokens: message.usage.totalTokens,
|
||||
cost: {
|
||||
input: message.usage.cost.input,
|
||||
output: message.usage.cost.output,
|
||||
cacheRead: message.usage.cost.cacheRead,
|
||||
cacheWrite: message.usage.cost.cacheWrite,
|
||||
total: message.usage.cost.total,
|
||||
...(message.usage.cost.totalOrigin ? { totalOrigin: message.usage.cost.totalOrigin } : {}),
|
||||
},
|
||||
},
|
||||
stopReason: message.stopReason,
|
||||
...(message.errorMessage ? { errorMessage: message.errorMessage } : {}),
|
||||
...(message.errorCode ? { errorCode: message.errorCode } : {}),
|
||||
...(message.errorType ? { errorType: message.errorType } : {}),
|
||||
...(message.errorBody ? { errorBody: message.errorBody } : {}),
|
||||
timestamp: message.timestamp,
|
||||
};
|
||||
}
|
||||
|
||||
export function toWorkerTranscriptMessage(
|
||||
message: AgentMessage,
|
||||
): WorkerTranscriptMessage | undefined {
|
||||
if (message.role === "user") {
|
||||
const content =
|
||||
typeof message.content === "string"
|
||||
? [{ type: "text" as const, text: message.content }]
|
||||
: message.content.map((part) =>
|
||||
part.type === "text" ? cloneTextContent(part) : cloneImageContent(part),
|
||||
);
|
||||
return { role: "user", content, timestamp: message.timestamp };
|
||||
}
|
||||
if (message.role === "assistant") {
|
||||
return cloneUsage(message);
|
||||
}
|
||||
if (message.role === "toolResult") {
|
||||
return {
|
||||
role: "toolResult",
|
||||
toolCallId: message.toolCallId,
|
||||
toolName: message.toolName,
|
||||
content: message.content.map((part) =>
|
||||
part.type === "text" ? cloneTextContent(part) : cloneImageContent(part),
|
||||
),
|
||||
...(message.details === undefined ? {} : { details: structuredClone(message.details) }),
|
||||
isError: message.isError,
|
||||
timestamp: message.timestamp,
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
import type { Context, Message } from "../llm/types.js";
|
||||
import {
|
||||
cloneImageContent,
|
||||
cloneTextContent,
|
||||
cloneUsage,
|
||||
isWorkerTranscriptMessageFrameSafe,
|
||||
toWorkerTranscriptMessage,
|
||||
} from "./transcript-message.js";
|
||||
|
||||
export function toAgentMessage(message: WorkerTranscriptMessage): Message {
|
||||
if (message.role === "user") {
|
||||
|
||||
@@ -26,8 +26,8 @@ import {
|
||||
createWorkerTranscriptRuntime,
|
||||
toAgentMessage,
|
||||
toWorkerInferenceContext,
|
||||
toWorkerTranscriptMessage,
|
||||
} from "./embedded-agent-transcript.runtime.js";
|
||||
import { toWorkerTranscriptMessage } from "./transcript-message.js";
|
||||
|
||||
const LOCAL_WORKER_TOOL_NAMES = [
|
||||
"read",
|
||||
@@ -75,6 +75,7 @@ type RunWorkerEmbeddedTurnParams = {
|
||||
transcript: WorkerEmbeddedTranscriptClient;
|
||||
live: WorkerEmbeddedLiveClient;
|
||||
initialMessages?: WorkerTranscriptMessage[];
|
||||
suppressPromptTranscript?: boolean;
|
||||
systemPrompt?: string;
|
||||
inferenceOptions?: WorkerInferenceOptions;
|
||||
signal?: AbortSignal;
|
||||
@@ -110,7 +111,7 @@ export async function runWorkerEmbeddedTurn(
|
||||
noPromptTemplates: true,
|
||||
noThemes: true,
|
||||
noContextFiles: true,
|
||||
...(params.systemPrompt === undefined ? {} : { systemPrompt: params.systemPrompt }),
|
||||
...(params.systemPrompt === undefined ? {} : { appendSystemPrompt: [params.systemPrompt] }),
|
||||
agentsFilesOverride: () => ({ agentsFiles: contextFiles }),
|
||||
});
|
||||
await resourceLoader.reload();
|
||||
@@ -122,6 +123,7 @@ export async function runWorkerEmbeddedTurn(
|
||||
|
||||
const transcriptRuntime = createWorkerTranscriptRuntime(params.transcript);
|
||||
const sessionManager = guardSessionManager(baseSessionManager, {
|
||||
suppressNextUserMessagePersistence: params.suppressPromptTranscript,
|
||||
onMessagePersisted: transcriptRuntime.onMessagePersisted,
|
||||
});
|
||||
|
||||
@@ -221,13 +223,14 @@ export async function runWorkerEmbeddedTurn(
|
||||
|
||||
let finalTranscriptFailure: Error | undefined;
|
||||
try {
|
||||
if (!params.signal?.aborted) {
|
||||
try {
|
||||
await transcriptRuntime.withSessionWriteLock(() => undefined);
|
||||
} catch (error) {
|
||||
finalTranscriptFailure = toError(error, "Worker transcript flush failed.");
|
||||
}
|
||||
await liveRuntime.flush();
|
||||
try {
|
||||
await transcriptRuntime.withSessionWriteLock(() => undefined);
|
||||
} catch (error) {
|
||||
finalTranscriptFailure = toError(error, "Worker transcript flush failed.");
|
||||
}
|
||||
await liveRuntime.flush();
|
||||
if (finalTranscriptFailure === undefined) {
|
||||
await liveRuntime.emitTerminal();
|
||||
}
|
||||
} finally {
|
||||
params.signal?.removeEventListener("abort", abortTurn);
|
||||
|
||||
@@ -28,6 +28,7 @@ function launchDescriptor(): WorkerLaunchDescriptor {
|
||||
runId: "run-1",
|
||||
turnId: "turn-1",
|
||||
prompt: "Inspect the workspace.",
|
||||
suppressPromptTranscript: false,
|
||||
workspaceDir: "/tmp/openclaw-worker/workspace",
|
||||
modelRef: { provider: "provider-1", model: "model-1" },
|
||||
inferenceOptions: { reasoning: "medium", maxTokens: 512 },
|
||||
@@ -52,7 +53,7 @@ describe("worker launch descriptor", () => {
|
||||
expect(buildWorkerConnectParams(descriptor)).toMatchObject({
|
||||
role: "worker",
|
||||
client: { id: "openclaw-worker", mode: "worker", version: "2026.7.12" },
|
||||
admission: descriptor.admission,
|
||||
admission: { ...descriptor.admission, runId: descriptor.assignment.runId },
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -31,6 +31,7 @@ type WorkerLaunchAssignment = {
|
||||
runId: string;
|
||||
turnId: string;
|
||||
prompt: string;
|
||||
suppressPromptTranscript: boolean;
|
||||
workspaceDir: string;
|
||||
modelRef: WorkerInferenceModelRef;
|
||||
inferenceOptions: WorkerInferenceOptions;
|
||||
@@ -46,7 +47,7 @@ type WorkerLaunchAssignment = {
|
||||
};
|
||||
};
|
||||
|
||||
type WorkerLaunchAdmission = WorkerConnectParams["admission"] & {
|
||||
type WorkerLaunchAdmission = Omit<WorkerConnectParams["admission"], "runId"> & {
|
||||
sessionId: string;
|
||||
};
|
||||
|
||||
@@ -94,6 +95,7 @@ function parseAssignment(value: unknown): WorkerLaunchAssignment | undefined {
|
||||
"runId",
|
||||
"turnId",
|
||||
"prompt",
|
||||
"suppressPromptTranscript",
|
||||
"workspaceDir",
|
||||
"modelRef",
|
||||
"inferenceOptions",
|
||||
@@ -110,6 +112,7 @@ function parseAssignment(value: unknown): WorkerLaunchAssignment | undefined {
|
||||
!isIdentifier(value.runId) ||
|
||||
!isIdentifier(value.turnId) ||
|
||||
typeof value.prompt !== "string" ||
|
||||
typeof value.suppressPromptTranscript !== "boolean" ||
|
||||
!isIdentifier(value.workspaceDir) ||
|
||||
!path.isAbsolute(value.workspaceDir) ||
|
||||
(value.systemPrompt !== undefined && typeof value.systemPrompt !== "string") ||
|
||||
@@ -146,7 +149,7 @@ function parseAssignment(value: unknown): WorkerLaunchAssignment | undefined {
|
||||
}
|
||||
|
||||
export function buildWorkerConnectParams(
|
||||
descriptor: Pick<WorkerLaunchDescriptor, "admission">,
|
||||
descriptor: Pick<WorkerLaunchDescriptor, "admission" | "assignment">,
|
||||
): WorkerConnectParams {
|
||||
return {
|
||||
minProtocol: PROTOCOL_VERSION,
|
||||
@@ -158,7 +161,10 @@ export function buildWorkerConnectParams(
|
||||
mode: GATEWAY_CLIENT_MODES.WORKER,
|
||||
},
|
||||
role: "worker",
|
||||
admission: descriptor.admission,
|
||||
admission: {
|
||||
...descriptor.admission,
|
||||
runId: descriptor.assignment.runId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -6,9 +6,131 @@ import {
|
||||
WORKER_PROTOCOL_MAX_IDENTIFIER_LENGTH,
|
||||
WORKER_PROTOCOL_MAX_PAYLOAD_BYTES,
|
||||
} from "../../packages/gateway-protocol/src/schema/worker-admission.js";
|
||||
import type { AgentMessage } from "../agents/runtime/index.js";
|
||||
import type { AssistantMessage } from "../llm/types.js";
|
||||
|
||||
const SIZE_FRAME_ID = "00000000-0000-4000-8000-000000000000";
|
||||
|
||||
export function cloneTextContent(part: { type: "text"; text: string; textSignature?: string }) {
|
||||
return {
|
||||
type: "text" as const,
|
||||
text: part.text,
|
||||
...(part.textSignature ? { textSignature: part.textSignature } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function cloneImageContent(part: { type: "image"; data: string; mimeType: string }) {
|
||||
return { type: "image" as const, data: part.data, mimeType: part.mimeType };
|
||||
}
|
||||
|
||||
export function cloneUsage(
|
||||
message: AssistantMessage,
|
||||
): WorkerTranscriptMessage & { role: "assistant" } {
|
||||
return {
|
||||
role: "assistant",
|
||||
content: message.content.map((part) => {
|
||||
if (part.type === "text") {
|
||||
return cloneTextContent(part);
|
||||
}
|
||||
if (part.type === "thinking") {
|
||||
return {
|
||||
type: "thinking" as const,
|
||||
thinking: part.thinking,
|
||||
...(part.thinkingSignature ? { thinkingSignature: part.thinkingSignature } : {}),
|
||||
...(part.redacted === undefined ? {} : { redacted: part.redacted }),
|
||||
};
|
||||
}
|
||||
return {
|
||||
type: "toolCall" as const,
|
||||
id: part.id,
|
||||
name: part.name,
|
||||
arguments: structuredClone(part.arguments),
|
||||
...(part.thoughtSignature ? { thoughtSignature: part.thoughtSignature } : {}),
|
||||
...(part.executionMode ? { executionMode: part.executionMode } : {}),
|
||||
};
|
||||
}),
|
||||
api: message.api,
|
||||
provider: message.provider,
|
||||
model: message.model,
|
||||
...(message.responseModel ? { responseModel: message.responseModel } : {}),
|
||||
...(message.responseId ? { responseId: message.responseId } : {}),
|
||||
...(message.diagnostics
|
||||
? {
|
||||
diagnostics: message.diagnostics.map((diagnostic) => ({
|
||||
type: diagnostic.type,
|
||||
timestamp: diagnostic.timestamp,
|
||||
...(diagnostic.error
|
||||
? {
|
||||
error: {
|
||||
...(diagnostic.error.name ? { name: diagnostic.error.name } : {}),
|
||||
message: diagnostic.error.message,
|
||||
...(diagnostic.error.stack ? { stack: diagnostic.error.stack } : {}),
|
||||
...(diagnostic.error.code === undefined ? {} : { code: diagnostic.error.code }),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
...(diagnostic.details ? { details: structuredClone(diagnostic.details) } : {}),
|
||||
})),
|
||||
}
|
||||
: {}),
|
||||
usage: {
|
||||
input: message.usage.input,
|
||||
output: message.usage.output,
|
||||
cacheRead: message.usage.cacheRead,
|
||||
cacheWrite: message.usage.cacheWrite,
|
||||
...(message.usage.contextUsage
|
||||
? { contextUsage: structuredClone(message.usage.contextUsage) }
|
||||
: {}),
|
||||
totalTokens: message.usage.totalTokens,
|
||||
cost: {
|
||||
input: message.usage.cost.input,
|
||||
output: message.usage.cost.output,
|
||||
cacheRead: message.usage.cost.cacheRead,
|
||||
cacheWrite: message.usage.cost.cacheWrite,
|
||||
total: message.usage.cost.total,
|
||||
...(message.usage.cost.totalOrigin ? { totalOrigin: message.usage.cost.totalOrigin } : {}),
|
||||
},
|
||||
},
|
||||
stopReason: message.stopReason,
|
||||
...(message.errorMessage ? { errorMessage: message.errorMessage } : {}),
|
||||
...(message.errorCode ? { errorCode: message.errorCode } : {}),
|
||||
...(message.errorType ? { errorType: message.errorType } : {}),
|
||||
...(message.errorBody ? { errorBody: message.errorBody } : {}),
|
||||
timestamp: message.timestamp,
|
||||
};
|
||||
}
|
||||
|
||||
export function toWorkerTranscriptMessage(
|
||||
message: AgentMessage,
|
||||
): WorkerTranscriptMessage | undefined {
|
||||
if (message.role === "user") {
|
||||
const content =
|
||||
typeof message.content === "string"
|
||||
? [{ type: "text" as const, text: message.content }]
|
||||
: message.content.map((part) =>
|
||||
part.type === "text" ? cloneTextContent(part) : cloneImageContent(part),
|
||||
);
|
||||
return { role: "user", content, timestamp: message.timestamp };
|
||||
}
|
||||
if (message.role === "assistant") {
|
||||
return cloneUsage(message);
|
||||
}
|
||||
if (message.role === "toolResult") {
|
||||
return {
|
||||
role: "toolResult",
|
||||
toolCallId: message.toolCallId,
|
||||
toolName: message.toolName,
|
||||
content: message.content.map((part) =>
|
||||
part.type === "text" ? cloneTextContent(part) : cloneImageContent(part),
|
||||
),
|
||||
...(message.details === undefined ? {} : { details: structuredClone(message.details) }),
|
||||
isError: message.isError,
|
||||
timestamp: message.timestamp,
|
||||
};
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function isWorkerTranscriptMessageFrameSafe(message: WorkerTranscriptMessage): boolean {
|
||||
const frame: WorkerTranscriptCommitRequestFrame = {
|
||||
type: "req",
|
||||
|
||||
@@ -320,6 +320,7 @@ class ComposedGatewayHarness {
|
||||
baseLeafId?: string | null;
|
||||
initialSeq?: number;
|
||||
initialAckedSeq?: number;
|
||||
runId?: string;
|
||||
} = {},
|
||||
): WorkerClients {
|
||||
const epoch = params.epoch ?? this.epoch;
|
||||
@@ -336,12 +337,13 @@ class ComposedGatewayHarness {
|
||||
handshake: HANDSHAKE,
|
||||
},
|
||||
assignment: {
|
||||
runId: RUN_ID,
|
||||
runId: params.runId ?? RUN_ID,
|
||||
turnId: "fault-turn",
|
||||
prompt: "fault injection",
|
||||
workspaceDir: this.root,
|
||||
modelRef: MODEL_REF,
|
||||
inferenceOptions: {},
|
||||
suppressPromptTranscript: false,
|
||||
initialMessages: [],
|
||||
transcript: { baseLeafId: params.baseLeafId ?? null, nextSeq: params.initialSeq ?? 1 },
|
||||
liveEvents: {
|
||||
@@ -857,10 +859,13 @@ describe("cloud worker milestone 2 fault injection", () => {
|
||||
await oldInferenceRejected;
|
||||
|
||||
harness.providerPlan = { kind: "immediate", text: "new owner reply" };
|
||||
// Milestone-3 admission binds the worker to a single run; the fresh owner
|
||||
// must be admitted for the run it executes.
|
||||
const fresh = harness.createClients({
|
||||
admissionProof: REPLACEMENT_CREDENTIAL,
|
||||
epoch: newEpoch,
|
||||
baseLeafId: oldCommit.newLeafId,
|
||||
runId: "fresh-run",
|
||||
});
|
||||
clients.push(fresh);
|
||||
await fresh.connection.start();
|
||||
|
||||
@@ -128,6 +128,7 @@ class FakeWorkerGateway {
|
||||
readonly acceptedTranscriptRequests: WorkerTranscriptCommitParams[] = [];
|
||||
readonly liveEventRequests: WorkerLiveEventParams[] = [];
|
||||
readonly inferenceRequests: WorkerInferenceStartParams[] = [];
|
||||
readonly applicationOrder: string[] = [];
|
||||
|
||||
constructor(private readonly options: FakeGatewayOptions = {}) {
|
||||
this.httpServer = createServer();
|
||||
@@ -315,6 +316,7 @@ class FakeWorkerGateway {
|
||||
return;
|
||||
}
|
||||
this.acceptedTranscriptRequests.push(structuredClone(frame.params));
|
||||
this.applicationOrder.push(`transcript:${frame.params.seq}`);
|
||||
this.send(socket, {
|
||||
type: "res",
|
||||
id: frame.id,
|
||||
@@ -331,6 +333,11 @@ class FakeWorkerGateway {
|
||||
private handleLiveEvent(socket: WebSocket, frame: WorkerLiveEventRequestFrame): void {
|
||||
this.methods.push(frame.method);
|
||||
this.liveEventRequests.push(structuredClone(frame.params));
|
||||
this.applicationOrder.push(
|
||||
frame.params.event.kind === "lifecycle"
|
||||
? `live:lifecycle:${frame.params.event.payload.phase}`
|
||||
: `live:${frame.params.event.kind}`,
|
||||
);
|
||||
if (this.options.silenceFirstLiveEvent && !this.droppedLiveEvent) {
|
||||
this.droppedLiveEvent = true;
|
||||
return;
|
||||
@@ -706,6 +713,7 @@ function descriptor(socketPath: string, workspaceDir: string): WorkerLaunchDescr
|
||||
runId: RUN_ID,
|
||||
turnId: "worker-turn",
|
||||
prompt: "Complete the worker turn.",
|
||||
suppressPromptTranscript: false,
|
||||
workspaceDir,
|
||||
modelRef: MODEL_REF,
|
||||
inferenceOptions: { reasoning: "off" },
|
||||
@@ -754,6 +762,14 @@ describe("worker runtime", () => {
|
||||
expect(gateway.inferenceRequests[0]?.context.systemPrompt).toContain("worker-bootstrap-marker");
|
||||
const toolNames = gateway.inferenceRequests[0]?.context.tools?.map((tool) => tool.name) ?? [];
|
||||
expect(toolNames).toHaveLength(6);
|
||||
const terminalIndex = gateway.applicationOrder.findIndex(
|
||||
(entry) => entry === "live:lifecycle:end",
|
||||
);
|
||||
const finalTranscriptIndex = gateway.applicationOrder.findLastIndex((entry) =>
|
||||
entry.startsWith("transcript:"),
|
||||
);
|
||||
expect(finalTranscriptIndex).toBeGreaterThanOrEqual(0);
|
||||
expect(terminalIndex).toBeGreaterThan(finalTranscriptIndex);
|
||||
expect(toolNames).toEqual(
|
||||
expect.arrayContaining(["read", "write", "edit", "apply_patch", "exec", "process"]),
|
||||
);
|
||||
@@ -806,7 +822,7 @@ describe("worker runtime", () => {
|
||||
gateway.liveEventRequests.some(
|
||||
(request) => request.event.kind === "lifecycle" && request.event.payload.phase === "error",
|
||||
),
|
||||
).toBe(true);
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("renumbers live events after a gateway cursor reset without aborting the run", async () => {
|
||||
@@ -824,10 +840,10 @@ describe("worker runtime", () => {
|
||||
expect(gateway.liveEventRequests[1]?.event).toEqual(gateway.liveEventRequests[0]?.event);
|
||||
});
|
||||
|
||||
it("degrades hard live-event failures without affecting inference or transcript commits", async () => {
|
||||
it("requires authoritative terminal delivery after degrading preview live events", async () => {
|
||||
const { gateway, launch } = await setup({ liveFailure: "capacity-exceeded" });
|
||||
|
||||
await expect(runWorkerDescriptor(launch)).resolves.toMatchObject({ status: "completed" });
|
||||
await expect(runWorkerDescriptor(launch)).rejects.toThrow("worker live event rejected");
|
||||
|
||||
expect(gateway.inferenceRequests).toHaveLength(1);
|
||||
expect(
|
||||
@@ -835,7 +851,11 @@ describe("worker runtime", () => {
|
||||
.flatMap((request) => request.messages)
|
||||
.map((message) => message.role),
|
||||
).toEqual(["user", "assistant"]);
|
||||
expect(gateway.liveEventRequests).toHaveLength(1);
|
||||
expect(gateway.liveEventRequests.length).toBeGreaterThanOrEqual(2);
|
||||
expect(gateway.liveEventRequests.at(-1)?.event).toMatchObject({
|
||||
kind: "lifecycle",
|
||||
payload: { phase: "end" },
|
||||
});
|
||||
});
|
||||
|
||||
it("degrades a repeated no-progress live resync without hanging the run", async () => {
|
||||
@@ -849,7 +869,11 @@ describe("worker runtime", () => {
|
||||
|
||||
expect(gateway.inferenceRequests).toHaveLength(1);
|
||||
expect(gateway.acceptedTranscriptRequests).toHaveLength(2);
|
||||
expect(gateway.liveEventRequests).toHaveLength(2);
|
||||
expect(gateway.liveEventRequests).toHaveLength(3);
|
||||
expect(gateway.liveEventRequests.at(-1)?.event).toMatchObject({
|
||||
kind: "lifecycle",
|
||||
payload: { phase: "end" },
|
||||
});
|
||||
});
|
||||
|
||||
it("fails closed when worker admission is rejected", async () => {
|
||||
@@ -887,6 +911,10 @@ describe("worker runtime", () => {
|
||||
|
||||
await expect(result).rejects.toThrow("operator stopped worker");
|
||||
expect(gateway.methods).toContain("worker.inference.cancel");
|
||||
expect(gateway.liveEventRequests.at(-1)?.event).toMatchObject({
|
||||
kind: "lifecycle",
|
||||
payload: { phase: "end", aborted: true },
|
||||
});
|
||||
});
|
||||
|
||||
it("bounds shutdown when remote inference cancellation cannot settle", async () => {
|
||||
@@ -905,14 +933,17 @@ describe("worker runtime", () => {
|
||||
});
|
||||
|
||||
it.each([
|
||||
["error", "fixture provider failed", "error", "error"],
|
||||
["cancelled", "fixture inference cancelled", "aborted", "end"],
|
||||
["error", "error", "error"],
|
||||
["cancelled", "aborted", "end"],
|
||||
] as const)(
|
||||
"reports remote inference %s terminals as failed turns",
|
||||
async (plan, message, stopReason, lifecyclePhase) => {
|
||||
async (plan, stopReason, lifecyclePhase) => {
|
||||
const { gateway, launch } = await setup({ inferencePlans: [plan] });
|
||||
|
||||
await expect(runWorkerDescriptor(launch)).rejects.toThrow(message);
|
||||
await expect(runWorkerDescriptor(launch)).resolves.toEqual({
|
||||
status: "failed",
|
||||
reason: "turn-failed",
|
||||
});
|
||||
const assistant = gateway.transcriptRequests
|
||||
.flatMap((request) => request.messages)
|
||||
.toReversed()
|
||||
@@ -926,6 +957,19 @@ describe("worker runtime", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps an unacknowledged failed-turn terminal as an infrastructure failure", async () => {
|
||||
const { gateway, launch } = await setup({
|
||||
inferencePlans: ["error"],
|
||||
liveFailure: "capacity-exceeded",
|
||||
});
|
||||
|
||||
await expect(runWorkerDescriptor(launch)).rejects.toThrow("worker live event rejected");
|
||||
expect(gateway.liveEventRequests.at(-1)?.event).toMatchObject({
|
||||
kind: "lifecycle",
|
||||
payload: { phase: "error" },
|
||||
});
|
||||
});
|
||||
|
||||
it("fails closed when a heartbeat is rejected without fencing", async () => {
|
||||
const { launch } = await setup({
|
||||
inferencePlans: ["hold"],
|
||||
@@ -958,9 +1002,10 @@ describe("worker runtime", () => {
|
||||
async (plan) => {
|
||||
const { gateway, launch } = await setup({ inferencePlans: [plan] });
|
||||
|
||||
await expect(runWorkerDescriptor(launch)).rejects.toThrow(
|
||||
"Worker inference result exceeds the transcript message limit.",
|
||||
);
|
||||
await expect(runWorkerDescriptor(launch)).resolves.toEqual({
|
||||
status: "failed",
|
||||
reason: "turn-failed",
|
||||
});
|
||||
const assistant = gateway.transcriptRequests
|
||||
.flatMap((request) => request.messages)
|
||||
.toReversed()
|
||||
|
||||
@@ -9,8 +9,11 @@ import {
|
||||
WorkerTranscriptCommitClient,
|
||||
} from "./worker-rpc-clients.js";
|
||||
|
||||
type WorkerRuntimeResult =
|
||||
// Cross-process contract: serialized to stdout by runWorkerCommand and parsed by the
|
||||
// gateway worker turn launcher.
|
||||
export type WorkerRuntimeResult =
|
||||
| { status: "completed"; transcriptLeafId: string | null; transcriptNextSeq: number }
|
||||
| { status: "failed"; reason: "turn-failed" }
|
||||
| { status: "fenced"; reason: "credential-replaced" | "owner-epoch-mismatch" };
|
||||
|
||||
const WORKER_REMOTE_CANCEL_GRACE_MS = 1_000;
|
||||
@@ -52,6 +55,7 @@ export async function runWorkerDescriptor(
|
||||
|
||||
const abortController = new AbortController();
|
||||
let turnStarted = false;
|
||||
let terminalLiveAcked = false;
|
||||
let forcedStopTimer: NodeJS.Timeout | undefined;
|
||||
const connection = createWorkerConnection({
|
||||
socketPath: descriptor.socketPath,
|
||||
@@ -121,6 +125,7 @@ export async function runWorkerDescriptor(
|
||||
sessionKey: `worker:${descriptor.admission.sessionId}`,
|
||||
runId: descriptor.assignment.runId,
|
||||
prompt: descriptor.assignment.prompt,
|
||||
suppressPromptTranscript: descriptor.assignment.suppressPromptTranscript,
|
||||
modelRef: descriptor.assignment.modelRef,
|
||||
initialMessages: descriptor.assignment.initialMessages,
|
||||
...(descriptor.assignment.systemPrompt === undefined
|
||||
@@ -136,6 +141,12 @@ export async function runWorkerDescriptor(
|
||||
live: {
|
||||
emit: async (event) => {
|
||||
await live.emit(descriptor.assignment.runId, event);
|
||||
if (
|
||||
event.kind === "lifecycle" &&
|
||||
(event.payload.phase === "end" || event.payload.phase === "error")
|
||||
) {
|
||||
terminalLiveAcked = true;
|
||||
}
|
||||
},
|
||||
},
|
||||
signal: abortController.signal,
|
||||
@@ -148,6 +159,12 @@ export async function runWorkerDescriptor(
|
||||
if (fenced) {
|
||||
return fenced;
|
||||
}
|
||||
if (options.signal?.aborted) {
|
||||
throw toError(options.signal.reason, "worker interrupted");
|
||||
}
|
||||
if (terminalLiveAcked && connection.state.kind === "ready") {
|
||||
return { status: "failed", reason: "turn-failed" };
|
||||
}
|
||||
throw toError(error, "worker session failed");
|
||||
}
|
||||
const fenced = fencedResult(connection.state);
|
||||
|
||||
Reference in New Issue
Block a user