From 0f95e66b7f841ca3e0d78635d643724159d43e31 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Sun, 19 Jul 2026 01:06:49 -0700 Subject: [PATCH] feat(talk): add durable client voice sessions (#111216) Live-append voice transcripts into the agent session and persist a per-agent SQLite call record across relay and client transcript paths. Add run-scoped spoken confirmation for high-impact actions, mutation digests, bootstrap-context injection, talk.client.transcript and talk.client.close protocol methods, and Control UI adoption. This adds zero new configuration. Co-authored-by: Clifton King --- .../openclaw/app/gateway/GatewayProtocol.kt | 2 + .../OpenClawProtocol/GatewayModels.swift | 78 +- docs/gateway/protocol.md | 6 +- docs/nodes/talk.md | 4 + packages/gateway-protocol/src/index.ts | 12 + .../gateway-protocol/src/schema/channels.ts | 44 +- .../src/schema/protocol-schemas.ts | 6 + src/agents/agent-tools.before-tool-call.ts | 25 + src/gateway/method-scopes.test.ts | 4 + src/gateway/methods/core-descriptors.ts | 2 + src/gateway/server-methods-list.test.ts | 2 + src/gateway/server-methods.ts | 2 + .../server-methods/talk-client.test.ts | 282 +++++++ src/gateway/server-methods/talk-client.ts | 254 +++++- src/gateway/server-methods/talk-session.ts | 60 +- src/gateway/server-methods/talk-shared.ts | 66 +- src/gateway/server-methods/talk.test.ts | 147 +++- src/gateway/talk-agent-consult.ts | 50 +- src/gateway/talk-realtime-relay.test.ts | 202 +++++ src/gateway/talk-realtime-relay.ts | 162 ++++ src/infra/diagnostic-events.ts | 2 + src/talk/agent-consult-tool.test.ts | 14 + src/talk/agent-consult-tool.ts | 14 +- .../client-voice-confirmation.test-support.ts | 15 + src/talk/client-voice-confirmation.test.ts | 501 ++++++++++++ src/talk/client-voice-confirmation.ts | 341 ++++++++ src/talk/client-voice-session-store.ts | 161 ++++ src/talk/client-voice-session.test-support.ts | 22 + src/talk/client-voice-session.test.ts | 579 ++++++++++++++ src/talk/client-voice-session.ts | 732 ++++++++++++++++++ test/vitest/vitest.gateway-client.config.ts | 2 +- .../pages/chat/realtime-talk-consult.test.ts | 19 +- ui/src/pages/chat/realtime-talk-shared.ts | 8 + ui/src/pages/chat/realtime-talk.test.ts | 373 ++++++++- ui/src/pages/chat/realtime-talk.ts | 227 +++++- 35 files changed, 4328 insertions(+), 92 deletions(-) create mode 100644 src/gateway/server-methods/talk-client.test.ts create mode 100644 src/talk/client-voice-confirmation.test-support.ts create mode 100644 src/talk/client-voice-confirmation.test.ts create mode 100644 src/talk/client-voice-confirmation.ts create mode 100644 src/talk/client-voice-session-store.ts create mode 100644 src/talk/client-voice-session.test-support.ts create mode 100644 src/talk/client-voice-session.test.ts create mode 100644 src/talk/client-voice-session.ts diff --git a/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewayProtocol.kt b/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewayProtocol.kt index d512336893b..044481f4fb3 100644 --- a/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewayProtocol.kt +++ b/apps/android/app/src/main/java/ai/openclaw/app/gateway/GatewayProtocol.kt @@ -203,6 +203,8 @@ enum class GatewayMethod( TalkCatalog("talk.catalog"), TalkConfig("talk.config"), TalkClientCreate("talk.client.create"), + TalkClientTranscript("talk.client.transcript"), + TalkClientClose("talk.client.close"), TalkClientToolCall("talk.client.toolCall"), TalkClientSteer("talk.client.steer"), TalkSessionCreate("talk.session.create"), diff --git a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift index 6efa9f512ac..d48ac2e81a3 100644 --- a/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift +++ b/apps/shared/OpenClawKit/Sources/OpenClawProtocol/GatewayModels.swift @@ -8116,6 +8116,7 @@ public struct TalkCatalogResult: Codable, Sendable { public struct TalkClientCreateParams: Codable, Sendable { public let sessionkey: String? + public let voicesessionid: String? public let provider: String? public let model: String? public let voice: String? @@ -8126,10 +8127,11 @@ public struct TalkClientCreateParams: Codable, Sendable { public let mode: AnyCodable? public let transport: AnyCodable? public let brain: AnyCodable? - public let capabilities: [String]? + public let capabilities: [AnyCodable]? public init( sessionkey: String? = nil, + voicesessionid: String? = nil, provider: String? = nil, model: String? = nil, voice: String? = nil, @@ -8140,9 +8142,10 @@ public struct TalkClientCreateParams: Codable, Sendable { mode: AnyCodable? = nil, transport: AnyCodable? = nil, brain: AnyCodable? = nil, - capabilities: [String]? = nil) + capabilities: [AnyCodable]? = nil) { self.sessionkey = sessionkey + self.voicesessionid = voicesessionid self.provider = provider self.model = model self.voice = voice @@ -8158,6 +8161,7 @@ public struct TalkClientCreateParams: Codable, Sendable { private enum CodingKeys: String, CodingKey { case sessionkey = "sessionKey" + case voicesessionid = "voiceSessionId" case provider case model case voice @@ -8172,6 +8176,38 @@ public struct TalkClientCreateParams: Codable, Sendable { } } +public struct TalkClientCloseParams: Codable, Sendable { + public let sessionkey: String + public let voicesessionid: String + + public init( + sessionkey: String, + voicesessionid: String) + { + self.sessionkey = sessionkey + self.voicesessionid = voicesessionid + } + + private enum CodingKeys: String, CodingKey { + case sessionkey = "sessionKey" + case voicesessionid = "voiceSessionId" + } +} + +public struct TalkClientMutationResult: Codable, Sendable { + public let ok: Bool + + public init( + ok: Bool) + { + self.ok = ok + } + + private enum CodingKeys: String, CodingKey { + case ok + } +} + public struct TalkClientSteerParams: Codable, Sendable { public let sessionkey: String public let text: String @@ -8270,6 +8306,7 @@ public struct TalkAgentControlResult: Codable, Sendable { public struct TalkClientToolCallParams: Codable, Sendable { public let sessionkey: String + public let voicesessionid: String? public let callid: String public let name: String public let args: AnyCodable? @@ -8277,12 +8314,14 @@ public struct TalkClientToolCallParams: Codable, Sendable { public init( sessionkey: String, + voicesessionid: String? = nil, callid: String, name: String, args: AnyCodable? = nil, relaysessionid: String? = nil) { self.sessionkey = sessionkey + self.voicesessionid = voicesessionid self.callid = callid self.name = name self.args = args @@ -8291,6 +8330,7 @@ public struct TalkClientToolCallParams: Codable, Sendable { private enum CodingKeys: String, CodingKey { case sessionkey = "sessionKey" + case voicesessionid = "voiceSessionId" case callid = "callId" case name case args @@ -8316,6 +8356,40 @@ public struct TalkClientToolCallResult: Codable, Sendable { } } +public struct TalkClientTranscriptParams: Codable, Sendable { + public let sessionkey: String + public let voicesessionid: String + public let entryid: String + public let role: AnyCodable + public let text: String + public let timestamp: Double? + + public init( + sessionkey: String, + voicesessionid: String, + entryid: String, + role: AnyCodable, + text: String, + timestamp: Double? = nil) + { + self.sessionkey = sessionkey + self.voicesessionid = voicesessionid + self.entryid = entryid + self.role = role + self.text = text + self.timestamp = timestamp + } + + private enum CodingKeys: String, CodingKey { + case sessionkey = "sessionKey" + case voicesessionid = "voiceSessionId" + case entryid = "entryId" + case role + case text + case timestamp + } +} + public struct TalkConfigParams: Codable, Sendable { public let includesecrets: Bool? diff --git a/docs/gateway/protocol.md b/docs/gateway/protocol.md index eb0ab9d0fe9..461afb7c2c6 100644 --- a/docs/gateway/protocol.md +++ b/docs/gateway/protocol.md @@ -496,8 +496,10 @@ methods. Treat this as feature discovery, not a full enumeration of - `talk.session.steer` sends active-run voice control into a gateway-owned agent-backed Talk session: `{ sessionId, text, mode? }`, where `mode` is `status`, `steer`, `cancel`, or `followup`; omitted mode is classified from the spoken text. - `talk.session.close` closes a gateway-owned relay, transcription, or managed-room session and emits terminal Talk events. - `talk.mode` sets/broadcasts the current Talk mode state for WebChat/Control UI clients. - - `talk.client.create` creates a client-owned realtime provider session using `webrtc` or `provider-websocket` while the gateway owns config, credentials, instructions, and tool policy. - - `talk.client.toolCall` lets client-owned realtime transports forward provider tool calls to gateway policy. The first supported tool is `openclaw_agent_consult`; clients get a run id and wait for normal chat lifecycle events before submitting the provider-specific tool result. + - `talk.client.create` creates or resumes a client-owned realtime provider session using `webrtc` or `provider-websocket` while the gateway owns credentials, instructions, tool policy, and the returned `voiceSessionId`. Clients pass `sessionKey` and reuse `voiceSessionId` when replacing the provider transport during one call. + - `talk.client.transcript` appends one finalized `{ role, text }` item to the normal agent session. The required `entryId` is idempotent within `voiceSessionId`; retries do not duplicate transcript messages. + - `talk.client.close` closes the logical voice session after pending transcript writes. Closing is idempotent and may deliver a mutation-only call digest to the session's last non-WebChat channel. + - `talk.client.toolCall` lets client-owned realtime transports forward provider tool calls to gateway policy. The first supported tool is `openclaw_agent_consult`; clients get a run id and wait for normal chat lifecycle events before submitting the provider-specific tool result. Voice-bound high-impact actions return `VOICE_CONFIRMATION_REQUIRED:` until a later finalized user utterance explicitly confirms that exact action and the next consult supplies the `confirmationId`. - `talk.client.steer` sends active-run voice control for client-owned realtime transports. The gateway resolves the active embedded run from `sessionKey` and returns a structured accepted/rejected result instead of silently dropping steering. - `talk.event` is the single Talk event channel for realtime, transcription, STT/TTS, managed-room, telephony, and meeting adapters. - `talk.speak` synthesizes speech through the active Talk speech provider. diff --git a/docs/nodes/talk.md b/docs/nodes/talk.md index 5e2da216460..984a59c188d 100644 --- a/docs/nodes/talk.md +++ b/docs/nodes/talk.md @@ -18,6 +18,10 @@ Native Talk is a continuous loop: listen for speech, send the transcript to the Client-owned realtime Talk forwards provider tool calls through `talk.client.toolCall` instead of calling `chat.send` directly. While a realtime consult is active, clients can call `talk.client.steer` or `talk.session.steer` to classify spoken input as `status`, `steer`, `cancel`, or `followup`. Accepted steering queues into the active embedded run; rejected steering returns a reason such as `no_active_run`, `not_streaming`, or `compacting`. +Finalized realtime user and assistant utterances are always appended live to the active agent session, so later chat and voice turns share one history. Client-owned transports report their finalized transcripts with stable entry ids; Gateway relay sessions append the same events server-side. Provider sessions also receive the bounded realtime profile context used by Discord voice. + +Voice-originated consult runs require a new, exact spoken confirmation before high-impact actions such as sending messages, controlling nodes, browser/computer actions, service changes, destructive shell commands, or publication. The confirmation applies only to the exact blocked tool arguments and is consumed once; unrelated concurrent runs remain unaffected. When a call closes, OpenClaw can send a compact **Voice call changes** digest for mutating tools to the session's last non-WebChat delivery target. + Transcription-only Talk emits the same Talk event envelope as realtime and STT/TTS sessions, but uses `mode: "transcription"` and `brain: "none"`. All Talk sessions broadcast events on the `talk.event` channel; clients subscribe to it for partial/final transcript updates (`transcript.delta`/`transcript.done`) and other session telemetry. Browser Video Talk is available for OpenAI Realtime WebRTC and Google Live diff --git a/packages/gateway-protocol/src/index.ts b/packages/gateway-protocol/src/index.ts index d1e7827b730..0468660bca8 100644 --- a/packages/gateway-protocol/src/index.ts +++ b/packages/gateway-protocol/src/index.ts @@ -96,10 +96,13 @@ import { TalkCatalogResultSchema, TalkClientCreateParamsSchema, TalkClientCreateResultSchema, + TalkClientCloseParamsSchema, + TalkClientMutationResultSchema, TalkAgentControlResultSchema, TalkClientSteerParamsSchema, TalkClientToolCallParamsSchema, TalkClientToolCallResultSchema, + TalkClientTranscriptParamsSchema, TalkConfigParamsSchema, TalkConfigResultSchema, TalkSessionAppendAudioParamsSchema, @@ -808,8 +811,11 @@ export const validateTalkConfigParams = lazyCompile(TalkConfigParamsSchema); export const validateTalkConfigResult = lazyCompile(TalkConfigResultSchema); export const validateTalkClientCreateParams = lazyCompile(TalkClientCreateParamsSchema); export const validateTalkClientCreateResult = lazyCompile(TalkClientCreateResultSchema); +export const validateTalkClientCloseParams = lazyCompile(TalkClientCloseParamsSchema); +export const validateTalkClientMutationResult = lazyCompile(TalkClientMutationResultSchema); export const validateTalkClientToolCallParams = lazyCompile(TalkClientToolCallParamsSchema); export const validateTalkClientToolCallResult = lazyCompile(TalkClientToolCallResultSchema); +export const validateTalkClientTranscriptParams = lazyCompile(TalkClientTranscriptParamsSchema); export const validateTalkClientSteerParams = lazyCompile(TalkClientSteerParamsSchema); export const validateTalkAgentControlResult = lazyCompile(TalkAgentControlResultSchema); export const validateTalkSessionCreateParams = lazyCompile(TalkSessionCreateParamsSchema); @@ -1234,10 +1240,13 @@ export { TalkCatalogResultSchema, TalkClientCreateParamsSchema, TalkClientCreateResultSchema, + TalkClientCloseParamsSchema, + TalkClientMutationResultSchema, TalkAgentControlResultSchema, TalkClientSteerParamsSchema, TalkClientToolCallParamsSchema, TalkClientToolCallResultSchema, + TalkClientTranscriptParamsSchema, TalkConfigParamsSchema, TalkConfigResultSchema, TalkSessionAppendAudioParamsSchema, @@ -1560,10 +1569,13 @@ export type { TalkCatalogResult, TalkClientCreateParams, TalkClientCreateResult, + TalkClientCloseParams, + TalkClientMutationResult, TalkClientSteerParams, TalkAgentControlResult, TalkClientToolCallParams, TalkClientToolCallResult, + TalkClientTranscriptParams, TalkConfigParams, TalkConfigResult, TalkSessionAppendAudioParams, diff --git a/packages/gateway-protocol/src/schema/channels.ts b/packages/gateway-protocol/src/schema/channels.ts index 75928485955..229e7dad6a1 100644 --- a/packages/gateway-protocol/src/schema/channels.ts +++ b/packages/gateway-protocol/src/schema/channels.ts @@ -186,9 +186,14 @@ export const TalkEventSchema = Type.Object( }, ); +// Voice ids compose into transcript idempotency keys (`voice::`), so the +// charset excludes the `:` delimiter to keep distinct id pairs collision-free. +const VoiceIdString = Type.String({ pattern: "^[A-Za-z0-9_-]{1,128}$" }); + /** Creates a browser-facing Talk client session. */ export const TalkClientCreateParamsSchema = closedObject({ - sessionKey: Type.Optional(Type.String()), + sessionKey: Type.Optional(NonEmptyString), + voiceSessionId: Type.Optional(VoiceIdString), provider: Type.Optional(Type.String()), model: Type.Optional(Type.String()), voice: Type.Optional(Type.String()), @@ -199,18 +204,44 @@ export const TalkClientCreateParamsSchema = closedObject({ mode: Type.Optional(TalkModeSchema), transport: Type.Optional(TalkTransportSchema), brain: Type.Optional(TalkBrainSchema), - capabilities: Type.Optional(Type.Array(Type.Literal("camera-frame"), { uniqueItems: true })), + capabilities: Type.Optional( + Type.Array(Type.Union([Type.Literal("camera-frame"), Type.Literal("voice-transcript")]), { + uniqueItems: true, + }), + ), }); /** Tool-call request from a browser/client session back into the agent runtime. */ export const TalkClientToolCallParamsSchema = closedObject({ sessionKey: NonEmptyString, + voiceSessionId: Type.Optional(VoiceIdString), callId: NonEmptyString, name: NonEmptyString, args: Type.Optional(Type.Unknown()), relaySessionId: Type.Optional(NonEmptyString), }); +/** One finalized transcript item from a client-owned Talk session. */ +export const TalkClientTranscriptParamsSchema = closedObject({ + sessionKey: NonEmptyString, + voiceSessionId: VoiceIdString, + entryId: VoiceIdString, + role: Type.Union([Type.Literal("user"), Type.Literal("assistant")]), + text: NonEmptyString, + timestamp: Type.Optional(Type.Number()), +}); + +/** Logical close for a client-owned Talk session. */ +export const TalkClientCloseParamsSchema = closedObject({ + sessionKey: NonEmptyString, + voiceSessionId: VoiceIdString, +}); + +/** Result for client-owned transcript and close mutations. */ +export const TalkClientMutationResultSchema = closedObject({ + ok: Type.Literal(true), +}); + /** Agent run identity returned after accepting a Talk client tool call. */ export const TalkClientToolCallResultSchema = closedObject({ runId: NonEmptyString, @@ -456,6 +487,7 @@ export const TalkSessionOkResultSchema = closedObject({ const BrowserRealtimeWebRtcSdpSessionSchema = closedObject({ provider: NonEmptyString, transport: Type.Literal("webrtc"), + voiceSessionId: NonEmptyString, clientSecret: NonEmptyString, offerUrl: Type.Optional(Type.String()), offerHeaders: Type.Optional(Type.Record(Type.String(), Type.String())), @@ -468,6 +500,7 @@ const BrowserRealtimeWebRtcSdpSessionSchema = closedObject({ const BrowserRealtimeJsonPcmWebSocketSessionSchema = closedObject({ provider: NonEmptyString, transport: Type.Literal("provider-websocket"), + voiceSessionId: NonEmptyString, protocol: NonEmptyString, clientSecret: NonEmptyString, websocketUrl: NonEmptyString, @@ -482,6 +515,8 @@ const BrowserRealtimeJsonPcmWebSocketSessionSchema = closedObject({ const BrowserRealtimeGatewayRelaySessionSchema = closedObject({ provider: NonEmptyString, transport: Type.Literal("gateway-relay"), + // Server-owned: older gateways omit it and clients derive it from relaySessionId. + voiceSessionId: Type.Optional(NonEmptyString), relaySessionId: NonEmptyString, audio: BrowserRealtimeAudioContractSchema, model: Type.Optional(Type.String()), @@ -493,6 +528,8 @@ const BrowserRealtimeGatewayRelaySessionSchema = closedObject({ const BrowserRealtimeManagedRoomSessionSchema = closedObject({ provider: NonEmptyString, transport: Type.Literal("managed-room"), + // Server-owned rooms carry no client voice bookkeeping yet. + voiceSessionId: Type.Optional(NonEmptyString), roomUrl: NonEmptyString, token: Type.Optional(Type.String()), model: Type.Optional(Type.String()), @@ -740,6 +777,9 @@ export type TalkClientSteerParams = Static; export type TalkAgentControlResult = Static; export type TalkClientToolCallParams = Static; export type TalkClientToolCallResult = Static; +export type TalkClientTranscriptParams = Static; +export type TalkClientCloseParams = Static; +export type TalkClientMutationResult = Static; export type TalkSessionCreateParams = Static; export type TalkSessionCreateResult = Static; export type TalkSessionJoinParams = Static; diff --git a/packages/gateway-protocol/src/schema/protocol-schemas.ts b/packages/gateway-protocol/src/schema/protocol-schemas.ts index 54d62d1cf88..5801fafa4c1 100644 --- a/packages/gateway-protocol/src/schema/protocol-schemas.ts +++ b/packages/gateway-protocol/src/schema/protocol-schemas.ts @@ -187,10 +187,13 @@ import { TalkCatalogResultSchema, TalkClientCreateParamsSchema, TalkClientCreateResultSchema, + TalkClientCloseParamsSchema, + TalkClientMutationResultSchema, TalkAgentControlResultSchema, TalkClientSteerParamsSchema, TalkClientToolCallParamsSchema, TalkClientToolCallResultSchema, + TalkClientTranscriptParamsSchema, TalkConfigParamsSchema, TalkConfigResultSchema, TalkSessionAppendAudioParamsSchema, @@ -850,10 +853,13 @@ export const ProtocolSchemas = { TalkCatalogResult: TalkCatalogResultSchema, TalkClientCreateParams: TalkClientCreateParamsSchema, TalkClientCreateResult: TalkClientCreateResultSchema, + TalkClientCloseParams: TalkClientCloseParamsSchema, + TalkClientMutationResult: TalkClientMutationResultSchema, TalkClientSteerParams: TalkClientSteerParamsSchema, TalkAgentControlResult: TalkAgentControlResultSchema, TalkClientToolCallParams: TalkClientToolCallParamsSchema, TalkClientToolCallResult: TalkClientToolCallResultSchema, + TalkClientTranscriptParams: TalkClientTranscriptParamsSchema, TalkConfigParams: TalkConfigParamsSchema, TalkConfigResult: TalkConfigResultSchema, TalkSessionAppendAudioParams: TalkSessionAppendAudioParamsSchema, diff --git a/src/agents/agent-tools.before-tool-call.ts b/src/agents/agent-tools.before-tool-call.ts index f61086635f8..e8f048664d0 100644 --- a/src/agents/agent-tools.before-tool-call.ts +++ b/src/agents/agent-tools.before-tool-call.ts @@ -71,6 +71,11 @@ import { } from "../skills/loading/source.js"; import type { SkillSnapshot, SkillTelemetrySource, SkillUsagePath } from "../skills/types.js"; import { resolveSkillWorkshopToolApproval } from "../skills/workshop/policy.js"; +import { resolveClientVoiceToolConfirmationPolicy } from "../talk/client-voice-confirmation.js"; +import { + isClientVoiceSessionConfirmable, + resolveClientVoiceRunBinding, +} from "../talk/client-voice-session.js"; import { isPlainObject, truncateUtf16Safe } from "../utils.js"; import { adjustedParamsByToolCallId, @@ -84,6 +89,7 @@ import { } from "./agent-tools.before-tool-call.state.js"; import { normalizeFileToolPathParam } from "./agent-tools.params.js"; import { resolveAgentRunAbortLifecycleFields } from "./run-termination.js"; +import { buildToolMutationState } from "./tool-mutation.js"; export { consumeAdjustedParamsForToolCall, consumePreExecutionBlockedToolCall, @@ -1533,6 +1539,24 @@ export async function runBeforeToolCallHook(args: { ...(args.ctx?.config ? { config: args.ctx.config } : {}), ...(args.ctx?.workspaceDir ? { workspaceDir: args.ctx.workspaceDir } : {}), }); + const voiceRun = resolveClientVoiceRunBinding(args.ctx?.runId); + const voiceConfirmation = resolveClientVoiceToolConfirmationPolicy({ + agentId: voiceRun?.agentId, + voiceSessionId: voiceRun?.voiceSessionId, + runId: args.ctx?.runId, + toolName, + toolParams: normalizedParams, + ...(voiceRun ? { isConfirmable: () => isClientVoiceSessionConfirmable(voiceRun) } : {}), + }); + if (!voiceConfirmation.allowed) { + return { + blocked: true, + kind: "veto", + deniedReason: "plugin-before-tool-call", + reason: voiceConfirmation.reason, + params, + }; + } if (!initialCorePolicyResult && !shouldRunTrustedPolicies && !hasBeforeToolCallHooks) { return { blocked: false, params }; } @@ -1793,6 +1817,7 @@ export function wrapToolWithBeforeToolCallHook( ...diagnosticIdentity, ...(toolCallId && { toolCallId }), paramsSummary: summarizeToolParams(toolParams), + mutatingAction: buildToolMutationState(normalizedToolName, toolParams).mutatingAction, }); const recordPreExecutionError = ( error: unknown, diff --git a/src/gateway/method-scopes.test.ts b/src/gateway/method-scopes.test.ts index 75945378fef..9ba552443be 100644 --- a/src/gateway/method-scopes.test.ts +++ b/src/gateway/method-scopes.test.ts @@ -87,6 +87,8 @@ describe("method scope resolution", () => { ["node.pair.approve", ["operator.pairing"]], ["poll", ["operator.write"]], ["talk.client.create", ["operator.write"]], + ["talk.client.transcript", ["operator.write"]], + ["talk.client.close", ["operator.write"]], ["talk.client.toolCall", ["operator.write"]], ["talk.client.steer", ["operator.write"]], ["talk.session.create", ["operator.write"]], @@ -518,6 +520,8 @@ describe("operator scope authorization", () => { it("allows operator.write clients to use unified Talk sessions", () => { for (const method of [ "talk.client.create", + "talk.client.transcript", + "talk.client.close", "talk.client.toolCall", "talk.client.steer", "talk.session.create", diff --git a/src/gateway/methods/core-descriptors.ts b/src/gateway/methods/core-descriptors.ts index 022a113200b..4ef1845715a 100644 --- a/src/gateway/methods/core-descriptors.ts +++ b/src/gateway/methods/core-descriptors.ts @@ -87,6 +87,8 @@ const CORE_GATEWAY_METHOD_SPECS: readonly CoreGatewayMethodSpec[] = [ // Params-aware: reading redacted config needs read; includeSecrets also needs talk secrets. { name: "talk.config", scope: "dynamic" }, { name: "talk.client.create", scope: "operator.write" }, + { name: "talk.client.transcript", scope: "operator.write" }, + { name: "talk.client.close", scope: "operator.write" }, { name: "talk.client.toolCall", scope: "operator.write" }, { name: "talk.client.steer", scope: "operator.write" }, { name: "talk.session.create", scope: "operator.write" }, diff --git a/src/gateway/server-methods-list.test.ts b/src/gateway/server-methods-list.test.ts index 150c5b92340..a41a9761f03 100644 --- a/src/gateway/server-methods-list.test.ts +++ b/src/gateway/server-methods-list.test.ts @@ -149,6 +149,8 @@ describe("listGatewayMethods", () => { it("advertises the versioned Talk session RPCs", () => { const methods = listGatewayMethods(); expect(methods).toContain("talk.client.create"); + expect(methods).toContain("talk.client.transcript"); + expect(methods).toContain("talk.client.close"); expect(methods).toContain("talk.client.toolCall"); expect(methods).toContain("talk.client.steer"); expect(methods).toContain("talk.session.create"); diff --git a/src/gateway/server-methods.ts b/src/gateway/server-methods.ts index 33d2d42df5f..606cb137a10 100644 --- a/src/gateway/server-methods.ts +++ b/src/gateway/server-methods.ts @@ -578,6 +578,8 @@ export const coreGatewayHandlers: GatewayRequestHandlers = { "talk.session.steer", "talk.session.close", "talk.client.create", + "talk.client.transcript", + "talk.client.close", "talk.client.toolCall", "talk.client.steer", "talk.catalog", diff --git a/src/gateway/server-methods/talk-client.test.ts b/src/gateway/server-methods/talk-client.test.ts new file mode 100644 index 00000000000..0c84c7f15a4 --- /dev/null +++ b/src/gateway/server-methods/talk-client.test.ts @@ -0,0 +1,282 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + loadSessionEntry, + readSessionTranscriptMessageEvents, + replaceSessionEntry, +} from "../../config/sessions/session-accessor.js"; +import { closeOpenClawAgentDatabasesForTest } from "../../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js"; +import { + authorizeClientVoiceConfirmation, + resolveClientVoiceToolConfirmationPolicy, +} from "../../talk/client-voice-confirmation.js"; +import { resetClientVoiceConfirmationStateForTest } from "../../talk/client-voice-confirmation.test-support.js"; +import { + closeClientVoiceSession, + createOrResumeClientVoiceSession, + ensureClientVoiceAgentSessionEntry, +} from "../../talk/client-voice-session.js"; +import { clientVoiceSessionTesting } from "../../talk/client-voice-session.test-support.js"; +import { captureEnv, setTestEnvValue } from "../../test-utils/env.js"; +import { talkClientHandlers } from "./talk-client.js"; + +const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]); +const sessionKey = "agent:main:main"; +const sessionId = "voice-transcript-session"; +let tempDir: string; + +async function invokeTranscript(params: Record) { + const respond = vi.fn(); + await talkClientHandlers["talk.client.transcript"]?.({ + params, + respond, + context: { getRuntimeConfig: () => ({}) }, + } as never); + return respond; +} + +async function invokeClose(params: Record) { + const respond = vi.fn(); + await talkClientHandlers["talk.client.close"]?.({ + params, + respond, + context: { getRuntimeConfig: () => ({}) }, + client: { connId: "conn-close" }, + } as never); + return respond; +} + +describe("talk.client.transcript", () => { + beforeEach(async () => { + tempDir = await fs.realpath( + await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-talk-transcript-")), + ); + setTestEnvValue("OPENCLAW_STATE_DIR", tempDir); + await replaceSessionEntry( + { agentId: "main", sessionKey }, + { sessionId, updatedAt: Date.now() }, + ); + }); + + afterEach(async () => { + clientVoiceSessionTesting.reset(); + resetClientVoiceConfirmationStateForTest(); + vi.useRealTimers(); + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + envSnapshot.restore(); + await fs.rm(tempDir, { recursive: true, force: true }); + }); + + it("appends finalized messages once by event id", async () => { + const voiceSessionId = createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey, + origin: "client", + }); + const params = { + sessionKey, + voiceSessionId, + entryId: "1", + role: "user", + text: "hello from voice", + timestamp: 123, + }; + + expect(await invokeTranscript(params)).toHaveBeenCalledWith(true, { ok: true }, undefined); + expect(await invokeTranscript(params)).toHaveBeenCalledWith(true, { ok: true }, undefined); + const events = readSessionTranscriptMessageEvents({ agentId: "main", sessionId }); + expect(events).toHaveLength(1); + expect(events[0]?.event).toMatchObject({ + id: `voice:${voiceSessionId}:1`, + message: { + role: "user", + content: [{ type: "text", text: "hello from voice" }], + timestamp: 123, + provenance: { kind: "realtime_voice", sourceChannel: "talk" }, + }, + }); + }); + + it("appends before the session has ever received a chat turn", async () => { + const talkFirstSessionKey = "agent:main:talk-first"; + await ensureClientVoiceAgentSessionEntry({ + agentId: "main", + sessionKey: talkFirstSessionKey, + }); + const voiceSessionId = createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey: talkFirstSessionKey, + provider: "google", + origin: "client", + }); + + expect( + await invokeTranscript({ + sessionKey: talkFirstSessionKey, + voiceSessionId, + entryId: "1", + role: "user", + text: "heard before the first consult", + }), + ).toHaveBeenCalledWith(true, { ok: true }, undefined); + + const talkFirstEntry = loadSessionEntry({ + agentId: "main", + sessionKey: talkFirstSessionKey, + }); + expect(talkFirstEntry?.sessionId).toBeTruthy(); + expect( + readSessionTranscriptMessageEvents({ + agentId: "main", + sessionId: talkFirstEntry?.sessionId ?? "missing", + }), + ).toHaveLength(1); + }); + + it("uses server observation time for spoken-confirmation freshness", async () => { + vi.useFakeTimers(); + vi.setSystemTime(100); + const voiceSessionId = createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey, + origin: "client", + }); + await invokeTranscript({ + sessionKey, + voiceSessionId, + entryId: "early-yes", + role: "user", + text: "yes", + timestamp: 10_000, + }); + const policy = resolveClientVoiceToolConfirmationPolicy({ + agentId: "main", + voiceSessionId, + runId: "run-later", + toolName: "message", + toolParams: { action: "send", message: "later" }, + now: 200, + }); + expect(policy.allowed).toBe(false); + if (policy.allowed) { + throw new Error("expected confirmation request"); + } + const confirmationId = policy.reason.match(/VOICE_CONFIRMATION_REQUIRED:([^\s]+)/)?.[1]; + expect(confirmationId).toBeTruthy(); + + expect(() => + authorizeClientVoiceConfirmation({ + agentId: "main", + voiceSessionId, + confirmationId: confirmationId ?? "missing", + now: 201, + }), + ).toThrow("explicit spoken confirmation"); + }); + + it("accepts an idempotent close retry after the first response is lost", async () => { + const voiceSessionId = createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey, + origin: "client", + }); + const params = { sessionKey, voiceSessionId }; + + expect(await invokeClose(params)).toHaveBeenCalledWith(true, { ok: true }, undefined); + expect(await invokeClose(params)).toHaveBeenCalledWith(true, { ok: true }, undefined); + }); + + it("truncates UTF-16 safely and writes assistant metadata", async () => { + const voiceSessionId = createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey, + provider: "google", + origin: "client", + }); + const respond = await invokeTranscript({ + sessionKey, + voiceSessionId, + entryId: "assistant-1", + role: "assistant", + text: `${"x".repeat(7_999)}😀tail`, + }); + + expect(respond).toHaveBeenCalledWith(true, { ok: true }, undefined); + const event = readSessionTranscriptMessageEvents({ agentId: "main", sessionId })[0]?.event as + | { message?: { content?: Array<{ text?: string }> } } + | undefined; + const text = event?.message?.content?.[0]?.text; + expect(text).toBe("x".repeat(7_999)); + expect(event).toMatchObject({ + message: { + api: "realtime", + provider: "google", + model: "realtime-voice", + stopReason: "stop", + }, + }); + }); + + it("uses the neutral provider label for records created before provider tracking", async () => { + const voiceSessionId = createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey, + origin: "client", + }); + + expect( + await invokeTranscript({ + sessionKey, + voiceSessionId, + entryId: "legacy-assistant", + role: "assistant", + text: "legacy provider reply", + }), + ).toHaveBeenCalledWith(true, { ok: true }, undefined); + expect( + readSessionTranscriptMessageEvents({ agentId: "main", sessionId })[0]?.event, + ).toMatchObject({ + message: { api: "realtime", provider: "realtime", model: "realtime-voice" }, + }); + }); + + it.each([ + ["missing", "voice-missing", "voice session not found"], + ["closed", "voice-closed", "voice session is closed"], + ["relay", "voice-relay", "does not allow this transcript source"], + ])("rejects %s voice records", async (kind, voiceSessionId, expected) => { + if (kind !== "missing") { + createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey, + origin: kind === "relay" ? "relay" : "client", + voiceSessionId, + }); + } + if (kind === "closed") { + await closeClientVoiceSession({ + agentId: "main", + sessionKey, + voiceSessionId, + config: {}, + }); + } + + const respond = await invokeTranscript({ + sessionKey, + voiceSessionId, + entryId: "1", + role: "user", + text: "hello", + }); + expect(respond).toHaveBeenCalledWith( + false, + undefined, + expect.objectContaining({ message: expect.stringContaining(expected) }), + ); + }); +}); diff --git a/src/gateway/server-methods/talk-client.ts b/src/gateway/server-methods/talk-client.ts index 9d7a28ef542..ffefd84a3dc 100644 --- a/src/gateway/server-methods/talk-client.ts +++ b/src/gateway/server-methods/talk-client.ts @@ -8,28 +8,71 @@ import { ErrorCodes, errorShape, formatValidationErrors, + validateTalkClientCloseParams, validateTalkClientCreateParams, validateTalkClientSteerParams, validateTalkClientToolCallParams, + validateTalkClientTranscriptParams, } from "../../../packages/gateway-protocol/src/index.js"; +import { + buildAgentMainSessionKey, + resolveAgentIdFromSessionKey, +} from "../../routing/session-key.js"; import { REALTIME_VOICE_AGENT_CONSULT_TOOL, REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, + parseRealtimeVoiceAgentConsultArgs, } from "../../talk/agent-consult-tool.js"; import { REALTIME_VOICE_AGENT_CONTROL_TOOL } from "../../talk/agent-run-control-shared.js"; import { controlRealtimeVoiceAgentRun } from "../../talk/agent-run-control.js"; +import { + authorizeClientVoiceConfirmation, + bindAuthorizedClientVoiceConfirmation, + type ClientVoiceConfirmationGrant, +} from "../../talk/client-voice-confirmation.js"; +import { + appendClientVoiceTranscript, + assertClientVoiceSessionOpen, + closeClientVoiceSession, + closeStaleClientVoiceSessions, + createOrResumeClientVoiceSession, + ensureClientVoiceAgentSessionEntry, + registerClientVoiceConsultRun, + resolveClientVoiceSessionOrigin, + resolveOpenClientVoiceSessionId, +} from "../../talk/client-voice-session.js"; import { REALTIME_VOICE_DESCRIBE_VIEW_TOOL } from "../../talk/describe-view-tool.js"; import { resolveConfiguredRealtimeVoiceProvider } from "../../talk/provider-resolver.js"; import { startTalkRealtimeAgentConsult } from "../talk-agent-consult.js"; +import { + ensureTalkRealtimeRelayVoiceSession, + flushTalkRealtimeRelayVoiceWrites, +} from "../talk-realtime-relay.js"; import { formatForLog } from "../ws-log.js"; import { buildRealtimeInstructions, buildRealtimeVoiceLaunchOptions, buildTalkRealtimeConfig, isUnsupportedBrowserWebRtcSession, + resolveTalkRealtimeProviderInstructions, } from "./talk-shared.js"; import type { GatewayRequestHandlers } from "./types.js"; +const LEGACY_VOICE_BINDING_TTL_MS = 6 * 60 * 60_000; +const legacyVoiceSessionByClient = new Map(); + +function legacyVoiceBindingKey(connId: string, sessionKey: string): string { + return `${connId}\0${sessionKey}`; +} + +function pruneLegacyVoiceBindings(now = Date.now()): void { + for (const [key, binding] of legacyVoiceSessionByClient) { + if (binding.expiresAt <= now) { + legacyVoiceSessionByClient.delete(key); + } + } +} + /** * Gateway methods for browser-owned realtime Talk sessions. * @@ -37,7 +80,7 @@ import type { GatewayRequestHandlers } from "./types.js"; * calls back into OpenClaw agent consult runs. */ export const talkClientHandlers: GatewayRequestHandlers = { - "talk.client.create": async ({ params, respond, context }) => { + "talk.client.create": async ({ params, respond, context, client }) => { if (!validateTalkClientCreateParams(params)) { respond( false, @@ -50,6 +93,8 @@ export const talkClientHandlers: GatewayRequestHandlers = { return; } const typedParams = params as { + sessionKey?: string; + voiceSessionId?: string; provider?: string; model?: string; voice?: string; @@ -143,6 +188,17 @@ export const talkClientHandlers: GatewayRequestHandlers = { requested: typedParams, defaults: realtimeConfig, }); + const realtimeContext = await resolveTalkRealtimeProviderInstructions({ + config: runtimeConfig, + configuredInstructions: realtimeConfig.instructions, + sessionKey: typedParams.sessionKey, + // Legacy creates can drift to another agent's session at toolCall time, so + // the default agent's profile must not leak into the provider session. + requireSessionKeyForProfile: true, + warn: (message) => context.logGateway.warn(`talk realtime context: ${message}`), + }); + const { agentId, requestedSessionKey } = realtimeContext; + const sessionKey = requestedSessionKey ?? buildAgentMainSessionKey({ agentId }); if (resolution.provider.createBrowserSession && transport !== "gateway-relay") { const tools = [REALTIME_VOICE_AGENT_CONSULT_TOOL, REALTIME_VOICE_AGENT_CONTROL_TOOL]; if (wantsCameraFrames) { @@ -151,16 +207,50 @@ export const talkClientHandlers: GatewayRequestHandlers = { const session = await resolution.provider.createBrowserSession({ cfg: runtimeConfig, providerConfig: resolution.providerConfig, - instructions: buildRealtimeInstructions(realtimeConfig.instructions), + instructions: buildRealtimeInstructions(realtimeContext.instructions), tools, ...launchOptions, }); + // Client-owned voice records are minted only for client-owned transports; + // relay sessions are created via talk.session.create and keyed by relaySessionId. + // Widening this guard would hand relay calls a mismatched voiceSessionId. if ( (session.transport === "webrtc" || session.transport === "provider-websocket") && !isUnsupportedBrowserWebRtcSession(session) && (!transport || session.transport === transport) ) { - respond(true, session, undefined); + // Recovering 6h-abandoned calls (and retrying their digests) is not on the + // start path; running it inline would delay use of time-sensitive provider + // credentials behind slow channel sends. Fire it off the response path. + void closeStaleClientVoiceSessions({ + agentId, + config: runtimeConfig, + excludeVoiceSessionId: normalizeOptionalString(typedParams.voiceSessionId), + warn: (message) => context.logGateway.warn(`talk voice session recovery: ${message}`), + }).catch((error: unknown) => + context.logGateway.warn(`talk voice session recovery failed: ${formatForLog(error)}`), + ); + await ensureClientVoiceAgentSessionEntry({ agentId, sessionKey }); + const voiceSessionId = createOrResumeClientVoiceSession({ + agentId, + sessionKey, + provider: resolution.provider.id, + origin: "client", + // Deployed clients sent sessionKey before transcripts existed, so capability + // must be negotiated explicitly; declaring it turns the confirmation gate on. + transcriptCapable: typedParams.capabilities?.includes("voice-transcript") === true, + voiceSessionId: normalizeOptionalString(typedParams.voiceSessionId), + }); + const connId = normalizeOptionalString(client?.connId); + if (connId) { + const now = Date.now(); + pruneLegacyVoiceBindings(now); + legacyVoiceSessionByClient.set( + legacyVoiceBindingKey(connId, typedParams.sessionKey?.trim() || sessionKey), + { voiceSessionId, expiresAt: now + LEGACY_VOICE_BINDING_TTL_MS }, + ); + } + respond(true, { ...session, voiceSessionId }, undefined); return; } if (transport) { @@ -209,6 +299,82 @@ export const talkClientHandlers: GatewayRequestHandlers = { return; } + const agentId = resolveAgentIdFromSessionKey(params.sessionKey); + const relaySessionId = normalizeOptionalString(params.relaySessionId); + const connId = normalizeOptionalString(request.client?.connId); + pruneLegacyVoiceBindings(); + const explicitVoiceSessionId = normalizeOptionalString(params.voiceSessionId); + if (relaySessionId && explicitVoiceSessionId && explicitVoiceSessionId !== relaySessionId) { + respond( + false, + undefined, + errorShape(ErrorCodes.INVALID_REQUEST, "relaySessionId and voiceSessionId must match"), + ); + return; + } + let confirmationGrant: ClientVoiceConfirmationGrant | undefined; + let voiceSessionId: string; + try { + // Shipped clients may consult without ever creating a voice session (old app, + // restarted gateway, ambiguous open records). Implicitly create one instead of + // erroring so confirmation and mutation evidence stay always-on. + voiceSessionId = + explicitVoiceSessionId ?? + relaySessionId ?? + (connId + ? legacyVoiceSessionByClient.get(legacyVoiceBindingKey(connId, params.sessionKey)) + ?.voiceSessionId + : undefined) ?? + resolveOpenClientVoiceSessionId({ agentId, sessionKey: params.sessionKey }) ?? + createOrResumeClientVoiceSession({ + agentId, + sessionKey: params.sessionKey, + origin: "client", + }); + // Pin the resolved id to this connection so a legacy client's later consults + // reuse one record instead of forking a new never-closed session each time. + if (connId && !relaySessionId) { + const now = Date.now(); + pruneLegacyVoiceBindings(now); + legacyVoiceSessionByClient.set(legacyVoiceBindingKey(connId, params.sessionKey), { + voiceSessionId, + expiresAt: now + LEGACY_VOICE_BINDING_TTL_MS, + }); + } + if (relaySessionId && connId) { + // Initialize the canonical session row BEFORE binding: the bind drains the + // relay's buffered finals into transcript appends, which fail without it. + await ensureClientVoiceAgentSessionEntry({ agentId, sessionKey: params.sessionKey }); + ensureTalkRealtimeRelayVoiceSession({ + relaySessionId, + connId, + sessionKey: params.sessionKey, + }); + await flushTalkRealtimeRelayVoiceWrites({ relaySessionId, connId }); + } + const parsedArgs = parseRealtimeVoiceAgentConsultArgs(params.args ?? {}); + const origin = assertClientVoiceSessionOpen({ + agentId, + sessionKey: params.sessionKey, + voiceSessionId, + }); + if (origin === "relay" && (!relaySessionId || !connId)) { + throw new Error( + "relay-owned voice sessions require relaySessionId and connection ownership", + ); + } + if (parsedArgs.confirmationId) { + confirmationGrant = authorizeClientVoiceConfirmation({ + agentId, + voiceSessionId, + confirmationId: parsedArgs.confirmationId, + }); + } + } catch (err) { + respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, formatForLog(err))); + return; + } + const result = await startTalkRealtimeAgentConsult({ context: request.context, client: request.client, @@ -218,7 +384,19 @@ export const talkClientHandlers: GatewayRequestHandlers = { callId: params.callId, args: params.args ?? {}, relaySessionId: normalizeOptionalString(params.relaySessionId), - connId: normalizeOptionalString(request.client?.connId), + connId, + onRunStarted: (runId) => { + registerClientVoiceConsultRun({ + agentId, + sessionKey: params.sessionKey, + voiceSessionId, + runId, + config: request.context.getRuntimeConfig(), + }); + if (confirmationGrant) { + bindAuthorizedClientVoiceConfirmation({ grant: confirmationGrant, runId }); + } + }, }); if (!result.ok) { respond(false, undefined, result.error); @@ -233,6 +411,74 @@ export const talkClientHandlers: GatewayRequestHandlers = { undefined, ); }, + "talk.client.transcript": async ({ params, respond, context }) => { + if (!validateTalkClientTranscriptParams(params)) { + respond( + false, + undefined, + errorShape( + ErrorCodes.INVALID_REQUEST, + `invalid talk.client.transcript params: ${formatValidationErrors(validateTalkClientTranscriptParams.errors)}`, + ), + ); + return; + } + try { + await appendClientVoiceTranscript({ + agentId: resolveAgentIdFromSessionKey(params.sessionKey), + sessionKey: params.sessionKey, + voiceSessionId: params.voiceSessionId, + entryId: params.entryId, + role: params.role, + text: params.text, + ...(params.timestamp !== undefined ? { timestamp: params.timestamp } : {}), + config: context.getRuntimeConfig(), + }); + respond(true, { ok: true }, undefined); + } catch (err) { + respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, formatForLog(err))); + } + }, + "talk.client.close": async ({ params, respond, context, client }) => { + if (!validateTalkClientCloseParams(params)) { + respond( + false, + undefined, + errorShape( + ErrorCodes.INVALID_REQUEST, + `invalid talk.client.close params: ${formatValidationErrors(validateTalkClientCloseParams.errors)}`, + ), + ); + return; + } + try { + const agentId = resolveAgentIdFromSessionKey(params.sessionKey); + const origin = resolveClientVoiceSessionOrigin({ + agentId, + sessionKey: params.sessionKey, + voiceSessionId: params.voiceSessionId, + }); + if (origin === "relay") { + throw new Error("relay-owned voice sessions close through talk.session.stop"); + } + await closeClientVoiceSession({ + agentId, + sessionKey: params.sessionKey, + voiceSessionId: params.voiceSessionId, + config: context.getRuntimeConfig(), + }); + const connId = normalizeOptionalString(client?.connId); + if (connId) { + const key = legacyVoiceBindingKey(connId, params.sessionKey); + if (legacyVoiceSessionByClient.get(key)?.voiceSessionId === params.voiceSessionId) { + legacyVoiceSessionByClient.delete(key); + } + } + respond(true, { ok: true }, undefined); + } catch (err) { + respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, formatForLog(err))); + } + }, "talk.client.steer": async ({ params, respond, client, context }) => { if (!validateTalkClientSteerParams(params)) { respond( diff --git a/src/gateway/server-methods/talk-session.ts b/src/gateway/server-methods/talk-session.ts index acdd38b0229..5ed952a5cc8 100644 --- a/src/gateway/server-methods/talk-session.ts +++ b/src/gateway/server-methods/talk-session.ts @@ -19,8 +19,8 @@ import { import { REALTIME_VOICE_AGENT_CONSULT_TOOL } from "../../talk/agent-consult-tool.js"; import { REALTIME_VOICE_AGENT_CONTROL_TOOL } from "../../talk/agent-run-control-shared.js"; import { controlRealtimeVoiceAgentRun } from "../../talk/agent-run-control.js"; +import { ensureClientVoiceAgentSessionEntry } from "../../talk/client-voice-session.js"; import { resolveConfiguredRealtimeVoiceProvider } from "../../talk/provider-resolver.js"; -import type { TalkBrain, TalkMode, TalkTransport } from "../../talk/talk-events.js"; import { ADMIN_SCOPE } from "../operator-scopes.js"; import { resolveSessionKeyFromResolveParams } from "../sessions-resolve.js"; import { @@ -63,49 +63,20 @@ import { buildTalkRealtimeConfig, buildTalkTranscriptionConfig, canUseTalkDirectTools, + normalizeTalkSessionBrain, + normalizeTalkSessionMode, + normalizeTalkSessionTransport, resolveConfiguredRealtimeTranscriptionProvider, + resolveTalkRealtimeProviderInstructions, talkHandoffErrorCode, withRealtimeBrowserOverrides, } from "./talk-shared.js"; import type { GatewayRequestContext, GatewayRequestHandlers, RespondFn } from "./types.js"; import { assertValidParams } from "./validation.js"; -/** - * Gateway-managed Talk session methods for managed rooms and audio relays. - * The public `sessionId` is resolved through the unified registry so each RPC - * can enforce the correct connection ownership for its concrete backend. - */ +/** Gateway-managed Talk sessions resolve public ids through connection-owned unified records. */ type ManagedRoomTalkSession = Extract; -function normalizeTalkSessionMode(params: { mode?: string; transport?: string }): TalkMode { - const mode = normalizeOptionalLowercaseString(params.mode) as TalkMode | undefined; - if (mode) { - return mode; - } - return normalizeOptionalLowercaseString(params.transport) === "managed-room" - ? "stt-tts" - : "realtime"; -} - -function normalizeTalkSessionTransport(params: { - mode: TalkMode; - transport?: string; -}): TalkTransport { - const transport = normalizeOptionalLowercaseString(params.transport) as TalkTransport | undefined; - if (transport) { - return transport; - } - return params.mode === "stt-tts" ? "managed-room" : "gateway-relay"; -} - -function normalizeTalkSessionBrain(params: { mode: TalkMode; brain?: string }): TalkBrain { - const brain = normalizeOptionalLowercaseString(params.brain) as TalkBrain | undefined; - if (brain) { - return brain; - } - return params.mode === "transcription" ? "none" : "agent-consult"; -} - function isActiveManagedRoomClient( session: { handoffId: string }, connId: string | undefined, @@ -321,16 +292,29 @@ export const talkSessionHandlers: GatewayRequestHandlers = { requested: params, defaults: realtimeConfig, }); + const realtimeContext = await resolveTalkRealtimeProviderInstructions({ + config: runtimeConfig, + configuredInstructions: realtimeConfig.instructions, + sessionKey: params.sessionKey, + requireSessionKeyForProfile: true, + warn: (message) => context.logGateway.warn(`talk realtime context: ${message}`), + }); + if (realtimeContext.requestedSessionKey) { + await ensureClientVoiceAgentSessionEntry({ + agentId: realtimeContext.agentId, + sessionKey: realtimeContext.requestedSessionKey, + }); + } const session = createTalkRealtimeRelaySession({ context, connId, cfg: runtimeConfig, provider: resolution.provider, providerConfig: withRealtimeBrowserOverrides(resolution.providerConfig, launchOptions), - instructions: buildRealtimeInstructions(realtimeConfig.instructions), + instructions: buildRealtimeInstructions(realtimeContext.instructions), tools: [REALTIME_VOICE_AGENT_CONSULT_TOOL, REALTIME_VOICE_AGENT_CONTROL_TOOL], model: launchOptions.model, - sessionKey: normalizeOptionalString(params.sessionKey), + sessionKey: realtimeContext.requestedSessionKey, voice: launchOptions.voice, language: normalizeOptionalLowercaseString(params.language), forceAgentConsultOnFinalTranscript: @@ -344,6 +328,8 @@ export const talkSessionHandlers: GatewayRequestHandlers = { respondOk(respond, { ...session, sessionId: session.relaySessionId, + // The relay session is the logical voice call; clients need not synthesize it. + voiceSessionId: session.relaySessionId, mode, brain, }); diff --git a/src/gateway/server-methods/talk-shared.ts b/src/gateway/server-methods/talk-shared.ts index e4c54ebd0c2..8dce1c76f8c 100644 --- a/src/gateway/server-methods/talk-shared.ts +++ b/src/gateway/server-methods/talk-shared.ts @@ -13,6 +13,8 @@ import { resolveSupportedVoiceModelRefs, type VoiceModelProvider, } from "../../../packages/speech-core/voice-models.js"; +import { resolveDefaultAgentId } from "../../agents/agent-scope.js"; +import { resolveRealtimeBootstrapContextInstructions } from "../../agents/realtime-bootstrap-context.js"; import type { TalkRealtimeConfig } from "../../config/types.gateway.js"; import type { OpenClawConfig } from "../../config/types.js"; import { @@ -20,6 +22,7 @@ import { listRealtimeTranscriptionProviders, } from "../../realtime-transcription/provider-registry.js"; import type { RealtimeTranscriptionProviderConfig } from "../../realtime-transcription/provider-types.js"; +import { resolveAgentIdFromSessionKey } from "../../routing/session-key.js"; import { REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME } from "../../talk/agent-consult-tool.js"; import { REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME } from "../../talk/agent-run-control-shared.js"; import { listRealtimeVoiceProviders } from "../../talk/provider-registry.js"; @@ -27,10 +30,71 @@ import type { RealtimeVoiceBrowserSession, RealtimeVoiceProviderConfig, } from "../../talk/provider-types.js"; -import type { TalkEvent } from "../../talk/talk-events.js"; +import type { TalkBrain, TalkEvent, TalkMode, TalkTransport } from "../../talk/talk-events.js"; import { ADMIN_SCOPE } from "../operator-scopes.js"; import type { TalkHandoffTurnResult } from "../talk-handoff.js"; +/** Resolve the Talk session mode, defaulting managed-room transports to stt-tts. */ +export function normalizeTalkSessionMode(params: { mode?: string; transport?: string }): TalkMode { + return ( + (normalizeOptionalLowercaseString(params.mode) as TalkMode | undefined) ?? + (normalizeOptionalLowercaseString(params.transport) === "managed-room" ? "stt-tts" : "realtime") + ); +} + +/** Resolve the Talk session transport from mode when the client omits it. */ +export function normalizeTalkSessionTransport(params: { + mode: TalkMode; + transport?: string; +}): TalkTransport { + const transport = normalizeOptionalLowercaseString(params.transport) as TalkTransport | undefined; + if (transport) { + return transport; + } + return params.mode === "stt-tts" ? "managed-room" : "gateway-relay"; +} + +/** Resolve the Talk session brain, defaulting transcription sessions to none. */ +export function normalizeTalkSessionBrain(params: { mode: TalkMode; brain?: string }): TalkBrain { + const brain = normalizeOptionalLowercaseString(params.brain) as TalkBrain | undefined; + if (brain) { + return brain; + } + return params.mode === "transcription" ? "none" : "agent-consult"; +} + +export async function resolveTalkRealtimeProviderInstructions(params: { + config: OpenClawConfig; + configuredInstructions?: string; + sessionKey?: unknown; + /** Relay sessions bind their agent lazily; injecting a guessed profile would mix agents. */ + requireSessionKeyForProfile?: boolean; + warn: (message: string) => void; +}): Promise<{ agentId: string; instructions: string; requestedSessionKey?: string }> { + const requestedSessionKey = normalizeOptionalString(params.sessionKey); + // Older clients can prefetch without a key. Client-owned creates bind to the + // default agent immediately, so its workspace profile stays consistent there. + const agentId = requestedSessionKey + ? resolveAgentIdFromSessionKey(requestedSessionKey) + : resolveDefaultAgentId(params.config); + const bootstrapContext = + params.requireSessionKeyForProfile && !requestedSessionKey + ? undefined + : await resolveRealtimeBootstrapContextInstructions({ + agentId, + config: params.config, + sessionKey: requestedSessionKey, + warn: params.warn, + }); + return { + agentId, + instructions: [params.configuredInstructions, bootstrapContext] + .filter((entry): entry is string => Boolean(entry?.trim())) + .join("\n\n"), + ...(requestedSessionKey ? { requestedSessionKey } : {}), + }; +} + export function canUseTalkDirectTools(client: { connect?: { scopes?: string[] } } | null): boolean { const scopes = Array.isArray(client?.connect?.scopes) ? client.connect.scopes : []; return scopes.includes(ADMIN_SCOPE); diff --git a/src/gateway/server-methods/talk.test.ts b/src/gateway/server-methods/talk.test.ts index 74985d3a080..a727423a638 100644 --- a/src/gateway/server-methods/talk.test.ts +++ b/src/gateway/server-methods/talk.test.ts @@ -36,6 +36,8 @@ const mocks = vi.hoisted(() => ({ cancelTalkRealtimeRelayTurn: vi.fn(), stopTalkRealtimeRelaySession: vi.fn(), registerTalkRealtimeRelayAgentRun: vi.fn(), + flushTalkRealtimeRelayVoiceWrites: vi.fn(async () => undefined), + ensureTalkRealtimeRelayVoiceSession: vi.fn(), submitTalkRealtimeRelayToolResult: vi.fn(), createTalkTranscriptionRelaySession: vi.fn(), sendTalkTranscriptionRelayAudio: vi.fn(), @@ -45,6 +47,15 @@ const mocks = vi.hoisted(() => ({ controlRealtimeVoiceAgentRun: vi.fn(), steerTalkRealtimeRelayAgentRun: vi.fn(), resolveSessionKeyFromResolveParams: vi.fn(), + resolveRealtimeBootstrapContextInstructions: vi.fn( + async (): Promise => undefined, + ), + closeStaleClientVoiceSessions: vi.fn(async () => 0), + createOrResumeClientVoiceSession: vi.fn(() => "voice-test"), + ensureClientVoiceAgentSessionEntry: vi.fn(async () => undefined), + assertClientVoiceSessionOpen: vi.fn(), + registerClientVoiceConsultRun: vi.fn(), + resolveOpenClientVoiceSessionId: vi.fn(), })); vi.mock("../../config/config.js", () => ({ @@ -82,6 +93,23 @@ vi.mock("../../talk/agent-run-control.js", () => ({ controlRealtimeVoiceAgentRun: mocks.controlRealtimeVoiceAgentRun, })); +vi.mock("../../agents/realtime-bootstrap-context.js", () => ({ + resolveRealtimeBootstrapContextInstructions: mocks.resolveRealtimeBootstrapContextInstructions, +})); + +vi.mock("../../talk/client-voice-session.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + assertClientVoiceSessionOpen: mocks.assertClientVoiceSessionOpen, + closeStaleClientVoiceSessions: mocks.closeStaleClientVoiceSessions, + createOrResumeClientVoiceSession: mocks.createOrResumeClientVoiceSession, + ensureClientVoiceAgentSessionEntry: mocks.ensureClientVoiceAgentSessionEntry, + registerClientVoiceConsultRun: mocks.registerClientVoiceConsultRun, + resolveOpenClientVoiceSessionId: mocks.resolveOpenClientVoiceSessionId, + }; +}); + vi.mock("./chat.js", () => ({ chatHandlers: { "chat.send": mocks.chatSend, @@ -99,6 +127,8 @@ vi.mock("../talk-realtime-relay.js", async (importOriginal) => { acknowledgeTalkRealtimeRelayMark: mocks.acknowledgeTalkRealtimeRelayMark, cancelTalkRealtimeRelayTurn: mocks.cancelTalkRealtimeRelayTurn, createTalkRealtimeRelaySession: mocks.createTalkRealtimeRelaySession, + ensureTalkRealtimeRelayVoiceSession: mocks.ensureTalkRealtimeRelayVoiceSession, + flushTalkRealtimeRelayVoiceWrites: mocks.flushTalkRealtimeRelayVoiceWrites, registerTalkRealtimeRelayAgentRun: mocks.registerTalkRealtimeRelayAgentRun, sendTalkRealtimeRelayAudio: mocks.sendTalkRealtimeRelayAudio, steerTalkRealtimeRelayAgentRun: mocks.steerTalkRealtimeRelayAgentRun, @@ -1484,6 +1514,7 @@ describe("talk.session unified handlers", () => { )({ req: { type: "req", id: "1", method: "talk.session.create" }, params: { + sessionKey: "agent:main:main", mode: "realtime", transport: "gateway-relay", brain: "agent-consult", @@ -1520,6 +1551,10 @@ describe("talk.session unified handlers", () => { providerConfigs: { openai: { apiKey: "openai-key" } }, defaultModel: "gpt-realtime-default", }); + expect(mocks.ensureClientVoiceAgentSessionEntry).toHaveBeenCalledWith({ + agentId: "main", + sessionKey: "agent:main:main", + }); const relayCreateInput = mockCallArg(mocks.createTalkRealtimeRelaySession) as Record< string, unknown @@ -2418,6 +2453,96 @@ describe("talk.client.toolCall handler", () => { ); }); + it("implicitly creates a voice session for consults without a binding", async () => { + const respond = vi.fn(); + + await expectDefined( + talkHandlers["talk.client.toolCall"], + 'talkHandlers["talk.client.toolCall"] test invariant', + )({ + req: { type: "req", id: "1", method: "talk.client.toolCall" }, + params: { + sessionKey: "main", + callId: "call-unbound", + name: "openclaw_agent_consult", + args: { question: "Do something" }, + }, + client: { connId: "conn-1" } as never, + isWebchatConnect: () => false, + respond: respond as never, + context: { getRuntimeConfig: () => ({}) as OpenClawConfig } as never, + }); + + expect(mocks.createOrResumeClientVoiceSession).toHaveBeenCalledWith({ + agentId: "main", + sessionKey: "main", + origin: "client", + }); + expect(mocks.registerClientVoiceConsultRun).toHaveBeenCalledWith( + expect.objectContaining({ voiceSessionId: "voice-test", runId: "run-voice-1" }), + ); + expect(mocks.chatSend).toHaveBeenCalledTimes(1); + expect(respond).toHaveBeenCalledWith(true, expect.anything(), undefined); + }); + + it("resolves a legacy consult to the open client voice record", async () => { + mocks.resolveOpenClientVoiceSessionId.mockReturnValueOnce("voice-test"); + const respond = vi.fn(); + + await expectDefined( + talkHandlers["talk.client.toolCall"], + 'talkHandlers["talk.client.toolCall"] test invariant', + )({ + req: { type: "req", id: "legacy", method: "talk.client.toolCall" }, + params: { + sessionKey: "main", + callId: "call-legacy", + name: "openclaw_agent_consult", + args: { question: "Continue the call" }, + }, + client: { connId: "conn-legacy" } as never, + isWebchatConnect: () => false, + respond: respond as never, + context: { getRuntimeConfig: () => ({}) as OpenClawConfig } as never, + }); + + expect(mocks.assertClientVoiceSessionOpen).toHaveBeenCalledWith({ + agentId: "main", + sessionKey: "main", + voiceSessionId: "voice-test", + }); + expectRespondOk(respond, { runId: "run-voice-1" }); + }); + + it("requires relay connection ownership for relay-origin voice records", async () => { + mocks.assertClientVoiceSessionOpen.mockReturnValueOnce("relay"); + const respond = vi.fn(); + + await expectDefined( + talkHandlers["talk.client.toolCall"], + 'talkHandlers["talk.client.toolCall"] test invariant', + )({ + req: { type: "req", id: "relay-owner", method: "talk.client.toolCall" }, + params: { + sessionKey: "main", + voiceSessionId: "relay-secret", + callId: "call-relay-owner", + name: "openclaw_agent_consult", + args: { question: "Continue" }, + }, + client: { connId: "other-conn" } as never, + isWebchatConnect: () => false, + respond: respond as never, + context: { getRuntimeConfig: () => ({}) as OpenClawConfig } as never, + }); + + expect(mocks.chatSend).not.toHaveBeenCalled(); + expectRespondError(respond, { + code: ErrorCodes.INVALID_REQUEST, + message: "Error: relay-owned voice sessions require relaySessionId and connection ownership", + }); + }); + it("starts agent consult through gateway policy instead of exposing chat.send to browser clients", async () => { const respond = vi.fn(); @@ -2428,6 +2553,7 @@ describe("talk.client.toolCall handler", () => { req: { type: "req", id: "1", method: "talk.client.toolCall" }, params: { sessionKey: "main", + voiceSessionId: "voice-test", callId: "call-1", name: "openclaw_agent_consult", args: { question: "What is in this repo?", responseStyle: "one sentence" }, @@ -2470,6 +2596,7 @@ describe("talk.client.toolCall handler", () => { req: { type: "req", id: "1", method: "talk.client.toolCall" }, params: { sessionKey: "main", + voiceSessionId: "voice-test", callId: "call-active", name: "openclaw_agent_consult", args: { question: "What is running?" }, @@ -2497,6 +2624,7 @@ describe("talk.client.toolCall handler", () => { req: { type: "req", id: "1", method: "talk.client.toolCall" }, params: { sessionKey: "main", + voiceSessionId: "voice-test", callId: "call-1", name: "openclaw_agent_consult", args: { question: "Are the basement lights off?" }, @@ -2533,6 +2661,7 @@ describe("talk.client.toolCall handler", () => { req: { type: "req", id: "1", method: "talk.client.toolCall" }, params: { sessionKey: "main", + voiceSessionId: "relay-1", relaySessionId: "relay-1", callId: "call-1", name: "openclaw_agent_consult", @@ -2581,6 +2710,7 @@ describe("talk.client.toolCall handler", () => { req: { type: "req", id: "1", method: "talk.client.toolCall" }, params: { sessionKey: "main", + voiceSessionId: "relay-1", relaySessionId: "relay-1", callId: "call-1", name: "openclaw_agent_consult", @@ -2749,6 +2879,8 @@ describe("talk.client.steer handler", () => { describe("talk.client.create handler", () => { beforeEach(() => { vi.clearAllMocks(); + mocks.resolveRealtimeBootstrapContextInstructions.mockResolvedValue(undefined); + mocks.createOrResumeClientVoiceSession.mockReturnValue("voice-test"); }); it("builds realtime launch defaults from talk.realtime", () => { @@ -2772,6 +2904,7 @@ describe("talk.client.create handler", () => { }); it("uses talk.realtime provider, model, voice, and instructions without reading speech provider config", async () => { + mocks.resolveRealtimeBootstrapContextInstructions.mockResolvedValue("Bounded profile context."); const createBrowserSession = vi.fn(async (_input: unknown) => ({ provider: "openai", transport: "webrtc" as const, @@ -2838,6 +2971,7 @@ describe("talk.client.create handler", () => { reasoningEffort: "low", }); expect(createInput.instructions).toContain("Additional realtime instructions:\nSpeak warmly."); + expect(createInput.instructions).toContain("Bounded profile context."); expect(createInput.instructions).toContain("tool-backed actions"); expect(createInput.instructions).toContain("Let me check that for you"); expect(createInput.tools).not.toContainEqual( @@ -2846,7 +2980,18 @@ describe("talk.client.create handler", () => { expect(createInput).not.toHaveProperty("provider"); expect(createInput).not.toHaveProperty("providers"); expect(createInput).not.toHaveProperty("transport"); - expectRespondOk(respond, { provider: "openai", transport: "webrtc" }); + expect(mocks.ensureClientVoiceAgentSessionEntry).toHaveBeenCalledWith({ + agentId: "main", + sessionKey: "main", + }); + expect(mocks.createOrResumeClientVoiceSession).toHaveBeenCalledWith( + expect.objectContaining({ provider: "openai" }), + ); + expectRespondOk(respond, { + provider: "openai", + transport: "webrtc", + voiceSessionId: "voice-test", + }); }); it("adds describe_view to camera clients whose provider supports video frames", async () => { diff --git a/src/gateway/talk-agent-consult.ts b/src/gateway/talk-agent-consult.ts index 7f2a0bd8007..eb7c6831de3 100644 --- a/src/gateway/talk-agent-consult.ts +++ b/src/gateway/talk-agent-consult.ts @@ -10,6 +10,7 @@ import { } from "../../packages/gateway-protocol/src/index.js"; import { normalizeTalkSection } from "../config/talk.js"; import { buildRealtimeVoiceAgentConsultChatMessage } from "../talk/agent-consult-tool.js"; +import { abortChatRunById } from "./chat-abort.js"; import { chatHandlers } from "./server-methods/chat.js"; import type { GatewayClient, @@ -66,6 +67,7 @@ export async function startTalkRealtimeAgentConsult(params: { args: unknown; relaySessionId?: string; connId?: string; + onRunStarted?: (runId: string) => void; }): Promise< { ok: true; runId: string; idempotencyKey: string } | { ok: false; error: ErrorShape } > { @@ -77,6 +79,7 @@ export async function startTalkRealtimeAgentConsult(params: { } const idempotencyKey = `talk-${params.callId}-${randomUUID()}`; const normalizedTalk = normalizeTalkSection(params.context.getRuntimeConfig().talk); + let acknowledgedRunId: string | undefined; const chatResponse = await new Promise< { ok: true; result: unknown } | { ok: false; error: ErrorShape } | undefined >((resolve) => { @@ -106,6 +109,37 @@ export async function startTalkRealtimeAgentConsult(params: { }, respond: (ok: boolean, result?: unknown, error?: ErrorShape) => { acknowledged = true; + if (ok && !terminalTalkChatSendAckError(normalizeTalkChatSendAckStatus(result))) { + const candidateRunId = + result && typeof result === "object" && !Array.isArray(result) + ? (result as Record).runId + : undefined; + const runId = typeof candidateRunId === "string" ? candidateRunId : idempotencyKey; + try { + if (params.relaySessionId && params.connId) { + registerTalkRealtimeRelayAgentRun({ + relaySessionId: params.relaySessionId, + connId: params.connId, + sessionKey: params.sessionKey, + runId, + callId: params.callId, + }); + } + params.onRunStarted?.(runId); + acknowledgedRunId = runId; + } catch (registrationError) { + abortChatRunById(params.context, { + runId, + sessionKey: params.sessionKey, + stopReason: "voice session binding failed", + }); + resolve({ + ok: false, + error: errorShape(ErrorCodes.UNAVAILABLE, formatForLog(registrationError)), + }); + return; + } + } resolve( ok ? { ok: true, result } @@ -152,20 +186,6 @@ export async function startTalkRealtimeAgentConsult(params: { if (terminalAckError) { return { ok: false, error: terminalAckError }; } - const runId = - result && typeof result === "object" && !Array.isArray(result) - ? typeof (result as Record).runId === "string" - ? (result as Record).runId - : idempotencyKey - : idempotencyKey; - if (params.relaySessionId && params.connId) { - registerTalkRealtimeRelayAgentRun({ - relaySessionId: params.relaySessionId, - connId: params.connId, - sessionKey: params.sessionKey, - runId: expectDefined(runId, "talk agent run id"), - callId: params.callId, - }); - } + const runId = expectDefined(acknowledgedRunId, "talk agent run id"); return { ok: true, runId: expectDefined(runId, "talk agent run id"), idempotencyKey }; } diff --git a/src/gateway/talk-realtime-relay.test.ts b/src/gateway/talk-realtime-relay.test.ts index ab2d4bb24be..53e19e3b224 100644 --- a/src/gateway/talk-realtime-relay.test.ts +++ b/src/gateway/talk-realtime-relay.test.ts @@ -1,18 +1,30 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; /** * Tests talk realtime relay event forwarding and connection cleanup. */ import { afterEach, describe, expect, it, vi } from "vitest"; import { setActiveEmbeddedRun } from "../agents/embedded-agent-runner/runs.js"; import { testing as embeddedRunTesting } from "../agents/embedded-agent-runner/runs.test-support.js"; +import { + readSessionTranscriptMessageEvents, + replaceSessionEntry, +} from "../config/sessions/session-accessor.js"; import type { RealtimeVoiceProviderPlugin } from "../plugins/types.js"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { clientVoiceSessionTesting } from "../talk/client-voice-session.test-support.js"; import type { RealtimeVoiceBridge, RealtimeVoiceBridgeCreateRequest, } from "../talk/provider-types.js"; +import { captureEnv, setTestEnvValue } from "../test-utils/env.js"; import { acknowledgeTalkRealtimeRelayMark, cancelTalkRealtimeRelayTurn, createTalkRealtimeRelaySession as createTalkRealtimeRelaySessionRaw, + flushTalkRealtimeRelayVoiceWrites, registerTalkRealtimeRelayAgentRun, sendTalkRealtimeRelayAudio, steerTalkRealtimeRelayAgentRun, @@ -74,6 +86,196 @@ describe("talk realtime gateway relay", () => { }; } + it("appends finalized relay transcripts to the canonical agent session", async () => { + const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]); + const tempDir = await fs.realpath( + await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-relay-voice-")), + ); + setTestEnvValue("OPENCLAW_STATE_DIR", tempDir); + let bridgeRequest: RealtimeVoiceBridgeCreateRequest | undefined; + try { + await replaceSessionEntry( + { agentId: "main", sessionKey: "agent:main:main" }, + { sessionId: "relay-voice-session", updatedAt: Date.now() }, + ); + const provider = createIdleRelayProvider(); + provider.createBridge = (request) => { + bridgeRequest = request; + return createIdleRelayProvider().createBridge?.(request) as RealtimeVoiceBridge; + }; + const session = createTalkRealtimeRelaySession({ + context: { + broadcastToConnIds: vi.fn(), + chatAbortControllers: new Map(), + getRuntimeConfig: () => ({}), + logGateway: { warn: vi.fn() }, + } as never, + connId: "conn-voice", + cfg: {}, + provider, + providerConfig: {}, + instructions: "brief", + tools: [], + sessionKey: "agent:main:main", + }); + bridgeRequest?.onTranscript?.("user", "relay hello", true); + bridgeRequest?.onTranscript?.("assistant", "relay response", true); + await flushTalkRealtimeRelayVoiceWrites({ + relaySessionId: session.relaySessionId, + connId: "conn-voice", + }); + + const events = readSessionTranscriptMessageEvents({ + agentId: "main", + sessionId: "relay-voice-session", + }); + expect(events).toHaveLength(2); + expect(events[0]?.event).toMatchObject({ + id: `voice:${session.relaySessionId}:1`, + message: { role: "user", content: [{ type: "text", text: "relay hello" }] }, + }); + expect(events[1]?.event).toMatchObject({ + id: `voice:${session.relaySessionId}:2`, + message: { + role: "assistant", + content: [{ type: "text", text: "relay response" }], + api: "realtime", + provider: "relay-test", + model: "realtime-voice", + }, + }); + stopTalkRealtimeRelaySession({ + relaySessionId: session.relaySessionId, + connId: "conn-voice", + }); + await vi.waitFor(() => + expect(clientVoiceSessionTesting.readRecord("main", session.relaySessionId)?.status).toBe( + "closed", + ), + ); + } finally { + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + envSnapshot.restore(); + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it("creates the relay voice record before binding a transcript-free consult", async () => { + const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]); + const tempDir = await fs.realpath( + await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-relay-voice-consult-")), + ); + setTestEnvValue("OPENCLAW_STATE_DIR", tempDir); + try { + const session = createTalkRealtimeRelaySession({ + context: { + broadcastToConnIds: vi.fn(), + chatAbortControllers: new Map(), + getRuntimeConfig: () => ({}), + logGateway: { warn: vi.fn() }, + } as never, + connId: "conn-consult", + cfg: {}, + provider: createIdleRelayProvider(), + providerConfig: {}, + instructions: "brief", + tools: [], + sessionKey: "agent:main:main", + }); + expect(clientVoiceSessionTesting.readRecord("main", session.relaySessionId)).toBeUndefined(); + + registerTalkRealtimeRelayAgentRun({ + relaySessionId: session.relaySessionId, + connId: "conn-consult", + sessionKey: "agent:main:main", + runId: "run-before-transcript", + }); + + expect(clientVoiceSessionTesting.readRecord("main", session.relaySessionId)).toMatchObject({ + status: "open", + consultRunIds: ["run-before-transcript"], + }); + stopTalkRealtimeRelaySession({ + relaySessionId: session.relaySessionId, + connId: "conn-consult", + }); + await vi.waitFor(() => + expect(clientVoiceSessionTesting.readRecord("main", session.relaySessionId)?.status).toBe( + "closed", + ), + ); + } finally { + clientVoiceSessionTesting.reset(); + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + envSnapshot.restore(); + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + + it("logs relay transcript append failures", async () => { + const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]); + const tempDir = await fs.realpath( + await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-relay-voice-failure-")), + ); + setTestEnvValue("OPENCLAW_STATE_DIR", tempDir); + let bridgeRequest: RealtimeVoiceBridgeCreateRequest | undefined; + const warn = vi.fn(); + try { + vi.useFakeTimers(); + const provider = createIdleRelayProvider(); + provider.createBridge = (request) => { + bridgeRequest = request; + return createIdleRelayProvider().createBridge?.(request) as RealtimeVoiceBridge; + }; + const session = createTalkRealtimeRelaySession({ + context: { + broadcastToConnIds: vi.fn(), + getRuntimeConfig: () => ({}), + logGateway: { warn }, + } as never, + connId: "conn-voice-failure", + cfg: {}, + provider, + providerConfig: {}, + instructions: "brief", + tools: [], + sessionKey: "agent:main:missing", + }); + bridgeRequest?.onTranscript?.("user", "cannot persist", true); + await vi.advanceTimersByTimeAsync(0); + expect(warn).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(500); + expect(warn).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1_999); + expect(warn).not.toHaveBeenCalled(); + await vi.advanceTimersByTimeAsync(1); + await flushTalkRealtimeRelayVoiceWrites({ + relaySessionId: session.relaySessionId, + connId: "conn-voice-failure", + }); + + expect(warn).toHaveBeenCalledExactlyOnceWith( + expect.stringContaining("realtime relay transcript append failed"), + ); + stopTalkRealtimeRelaySession({ + relaySessionId: session.relaySessionId, + connId: "conn-voice-failure", + }); + await vi.waitFor(() => + expect(clientVoiceSessionTesting.readRecord("main", session.relaySessionId)?.status).toBe( + "closed", + ), + ); + } finally { + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + envSnapshot.restore(); + await fs.rm(tempDir, { recursive: true, force: true }); + } + }); + function createDeferredVoid(): { promise: Promise; resolve: () => void } { let resolve!: () => void; const promise = new Promise((accept) => { diff --git a/src/gateway/talk-realtime-relay.ts b/src/gateway/talk-realtime-relay.ts index fd7b65912f0..6d7ed60c3e6 100644 --- a/src/gateway/talk-realtime-relay.ts +++ b/src/gateway/talk-realtime-relay.ts @@ -5,6 +5,7 @@ import { resolveExpiresAtMsFromDurationMs } from "@openclaw/normalization-core/n import type { OpenClawConfig } from "../config/types.js"; import { formatErrorMessage } from "../infra/errors.js"; import type { RealtimeVoiceProviderPlugin } from "../plugins/types.js"; +import { resolveAgentIdFromSessionKey } from "../routing/session-key.js"; import { REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, buildRealtimeVoiceAgentConsultWorkingResponse, @@ -16,6 +17,12 @@ import { shouldAutoControlRealtimeVoiceAgentText, type RealtimeVoiceAgentControlResult, } from "../talk/agent-run-control.js"; +import { + appendRelayVoiceTranscript, + closeClientVoiceSession, + createOrResumeClientVoiceSession, + registerClientVoiceConsultRun, +} from "../talk/client-voice-session.js"; import { readSpeakableRealtimeVoiceToolResult } from "../talk/consult-question.js"; import { createRealtimeVoiceForcedConsultCoordinator, @@ -134,6 +141,7 @@ type RelaySession = { expiresAtMs: number; cleanupTimer: ReturnType; activeAgentRuns: Map; + provider: string; activeAgentToolCalls: Map; completedAgentToolCalls: Set; // Cancelled calls retain their original turn long enough to terminally satisfy @@ -154,6 +162,11 @@ type RelaySession = { toolResultEpoch: number; forcedConsults: RealtimeVoiceForcedConsultCoordinator; transcript: RealtimeVoiceTranscriptEntry[]; + voiceConfig?: OpenClawConfig; + voiceSessionCreated: boolean; + voiceTranscriptSeq: number; + voiceTranscriptWrites: Promise; + pendingVoiceTranscripts: Array<{ role: "user" | "assistant"; text: string }>; }; type CreateTalkRealtimeRelaySessionParams = { @@ -182,6 +195,128 @@ type TalkRealtimeRelaySessionResult = { }; const relaySessions = new Map(); +const RELAY_TRANSCRIPT_RETRY_DELAYS_MS = [0, 500, 2_000] as const; + +function logRelayVoiceFailure(session: RelaySession, message: string, error: unknown): void { + session.context.logGateway?.warn(`${message}: ${formatErrorMessage(error)}`); +} + +function ensureRelayVoiceSession(session: RelaySession): boolean { + if (session.voiceSessionCreated) { + return true; + } + if (!session.sessionKey) { + return false; + } + try { + createOrResumeClientVoiceSession({ + agentId: resolveAgentIdFromSessionKey(session.sessionKey), + sessionKey: session.sessionKey, + provider: session.provider, + origin: "relay", + voiceSessionId: session.id, + }); + session.voiceSessionCreated = true; + return true; + } catch (error) { + logRelayVoiceFailure(session, "realtime relay voice session create failed", error); + return false; + } +} + +const MAX_PENDING_VOICE_TRANSCRIPTS = 40; + +function enqueueRelayVoiceTranscript( + session: RelaySession, + role: "user" | "assistant", + text: string, +): void { + if (!session.sessionKey) { + // Lazy-bound relays hear audio before talk.client.toolCall supplies the session + // key; buffer bounded finals so the call's opening turns survive the binding. + // Never-binding callers accept best-effort loss: they had no persistence before. + session.pendingVoiceTranscripts.push({ role, text }); + if (session.pendingVoiceTranscripts.length > MAX_PENDING_VOICE_TRANSCRIPTS) { + session.pendingVoiceTranscripts.shift(); + } + return; + } + if (!ensureRelayVoiceSession(session)) { + return; + } + session.voiceTranscriptSeq += 1; + const entryId = String(session.voiceTranscriptSeq); + const sessionKey = session.sessionKey; + session.voiceTranscriptWrites = session.voiceTranscriptWrites + .then(async () => { + let lastError: unknown; + for (const delayMs of RELAY_TRANSCRIPT_RETRY_DELAYS_MS) { + if (delayMs > 0) { + await new Promise((resolve) => { + setTimeout(resolve, delayMs); + }); + } + try { + await appendRelayVoiceTranscript({ + agentId: resolveAgentIdFromSessionKey(sessionKey), + sessionKey, + voiceSessionId: session.id, + entryId, + role, + text, + ...(session.voiceConfig ? { config: session.voiceConfig } : {}), + }); + return; + } catch (error) { + lastError = error; + } + } + throw lastError; + }) + .catch((error: unknown) => { + logRelayVoiceFailure(session, "realtime relay transcript append failed", error); + }); +} + +function closeRelayVoiceSession(session: RelaySession): void { + if (!session.sessionKey || !ensureRelayVoiceSession(session)) { + return; + } + const sessionKey = session.sessionKey; + void session.voiceTranscriptWrites + .then(async () => { + const config = session.voiceConfig ?? session.context.getRuntimeConfig(); + await closeClientVoiceSession({ + agentId: resolveAgentIdFromSessionKey(sessionKey), + sessionKey, + voiceSessionId: session.id, + config, + }); + }) + .catch((error: unknown) => { + logRelayVoiceFailure(session, "realtime relay voice session close failed", error); + }); +} + +/** Ensure a gateway-relay call has its durable record before transcript-free RPCs. */ +export function ensureTalkRealtimeRelayVoiceSession(params: { + relaySessionId: string; + connId: string; + sessionKey: string; +}): void { + const session = getRelaySession(params.relaySessionId, params.connId); + if (session.sessionKey && session.sessionKey !== params.sessionKey) { + throw new Error("Realtime relay session belongs to another agent session"); + } + session.sessionKey = params.sessionKey; + if (!ensureRelayVoiceSession(session)) { + throw new Error("Realtime relay voice session could not be created"); + } + const buffered = session.pendingVoiceTranscripts.splice(0); + for (const entry of buffered) { + enqueueRelayVoiceTranscript(session, entry.role, entry.text); + } +} function isWorkingToolResult(result: unknown): boolean { return ( @@ -538,6 +673,7 @@ function closeRelaySession(session: RelaySession, reason: "completed" | "error") clearTimeout(session.cleanupTimer); abortRelayAgentRuns(session, reason === "error" ? "relay-error" : "relay-closed"); session.bridge.close(); + closeRelayVoiceSession(session); broadcastToOwner(session.context, session.connId, { relaySessionId: session.id, type: "close", @@ -708,6 +844,7 @@ export function createTalkRealtimeRelaySession( const turnId = relay ? ensureRelayTurn(relay) : undefined; if (final && relay) { recordRealtimeVoiceTranscript(relay.transcript, role, text); + enqueueRelayVoiceTranscript(relay, role, text); } const eventType = role === "assistant" @@ -845,6 +982,7 @@ export function createTalkRealtimeRelaySession( forgetUnifiedTalkSession(relaySessionId); clearTimeout(active.cleanupTimer); abortRelayAgentRuns(active, "relay-closed"); + closeRelayVoiceSession(active); if (!ready && !failureEmitted) { const issue = realtimeRelayIssue({ message: "Realtime provider closed before the session became ready.", @@ -879,6 +1017,7 @@ export function createTalkRealtimeRelaySession( } }, RELAY_SESSION_TTL_MS), activeAgentRuns: new Map(), + provider: params.provider.id, activeAgentToolCalls: new Map(), completedAgentToolCalls: new Set(), cancelledAgentToolCalls: new Map(), @@ -890,6 +1029,11 @@ export function createTalkRealtimeRelaySession( toolResultEpoch: 0, forcedConsults: createRealtimeVoiceForcedConsultCoordinator(), transcript: [], + ...(params.cfg ? { voiceConfig: params.cfg } : {}), + voiceSessionCreated: false, + voiceTranscriptSeq: 0, + voiceTranscriptWrites: Promise.resolve(), + pendingVoiceTranscripts: [], }; relayRef.current = relay; relay.cleanupTimer.unref?.(); @@ -1352,6 +1496,24 @@ export function registerTalkRealtimeRelayAgentRun(params: { if (!session.sessionKey) { session.sessionKey = params.sessionKey; } + if (!ensureRelayVoiceSession(session)) { + throw new Error("Realtime relay voice session could not be created for agent consult"); + } + registerClientVoiceConsultRun({ + agentId: resolveAgentIdFromSessionKey(params.sessionKey), + sessionKey: params.sessionKey, + voiceSessionId: session.id, + runId: params.runId, + }); +} + +/** Wait for server-owned final transcript appends before a relay consult is authorized. */ +export async function flushTalkRealtimeRelayVoiceWrites(params: { + relaySessionId: string; + connId: string; +}): Promise { + const session = getRelaySession(params.relaySessionId, params.connId); + await session.voiceTranscriptWrites; } /** Applies realtime voice-control text to the active agent-consult chat run. */ diff --git a/src/infra/diagnostic-events.ts b/src/infra/diagnostic-events.ts index 4bc6f3f63d1..38745aee7e8 100644 --- a/src/infra/diagnostic-events.ts +++ b/src/infra/diagnostic-events.ts @@ -482,6 +482,8 @@ type DiagnosticToolExecutionBaseEvent = DiagnosticBaseEvent & { toolOwner?: string; toolCallId?: string; paramsSummary?: DiagnosticToolParamsSummary; + /** Deterministic mutation classification computed before tool execution. */ + mutatingAction?: boolean; }; export type DiagnosticToolExecutionStartedEvent = DiagnosticToolExecutionBaseEvent & { diff --git a/src/talk/agent-consult-tool.test.ts b/src/talk/agent-consult-tool.test.ts index 1144059f6e0..568a78c332c 100644 --- a/src/talk/agent-consult-tool.test.ts +++ b/src/talk/agent-consult-tool.test.ts @@ -30,6 +30,20 @@ describe("realtime voice agent consult tool", () => { ); }); + it("normalizes a server-issued spoken confirmation id", () => { + expect( + parseRealtimeVoiceAgentConsultArgs({ + question: "Send it now", + confirmationId: " confirm-123 ", + }), + ).toStrictEqual({ + question: "Send it now", + context: undefined, + responseStyle: undefined, + confirmationId: "confirm-123", + }); + }); + it("accepts provider question aliases from realtime tool calls", () => { expect(parseRealtimeVoiceAgentConsultArgs({ prompt: " Check the repo. " })).toStrictEqual({ context: undefined, diff --git a/src/talk/agent-consult-tool.ts b/src/talk/agent-consult-tool.ts index 14ed3fa4179..09c513fbe80 100644 --- a/src/talk/agent-consult-tool.ts +++ b/src/talk/agent-consult-tool.ts @@ -26,6 +26,7 @@ export type RealtimeVoiceAgentConsultArgs = { question: string; context?: string; responseStyle?: string; + confirmationId?: string; }; /** Compact transcript entry included in delegated agent prompts. */ export type RealtimeVoiceAgentConsultTranscriptEntry = { @@ -54,6 +55,11 @@ export const REALTIME_VOICE_AGENT_CONSULT_TOOL: RealtimeVoiceTool = { type: "string", description: "Optional style hint for the spoken answer.", }, + confirmationId: { + type: "string", + description: + "Server-issued confirmation id from a prior VOICE_CONFIRMATION_REQUIRED result, supplied only after the user explicitly confirms aloud.", + }, }, required: ["question"], }, @@ -177,10 +183,14 @@ export function parseRealtimeVoiceAgentConsultArgs(args: unknown): RealtimeVoice if (!question) { throw new Error("question required"); } + const context = readConsultStringArg(args, "context"); + const responseStyle = readConsultStringArg(args, "responseStyle"); + const confirmationId = readConsultStringArg(args, "confirmationId"); return { question, - context: readConsultStringArg(args, "context"), - responseStyle: readConsultStringArg(args, "responseStyle"), + context, + responseStyle, + ...(confirmationId ? { confirmationId } : {}), }; } diff --git a/src/talk/client-voice-confirmation.test-support.ts b/src/talk/client-voice-confirmation.test-support.ts new file mode 100644 index 00000000000..2507e7d946d --- /dev/null +++ b/src/talk/client-voice-confirmation.test-support.ts @@ -0,0 +1,15 @@ +import "./client-voice-confirmation.js"; + +type ClientVoiceConfirmationTestApi = { + resetClientVoiceConfirmationStateForTest(): void; +}; + +function getTestApi(): ClientVoiceConfirmationTestApi { + return (globalThis as Record)[ + Symbol.for("openclaw.clientVoiceConfirmationTestApi") + ] as ClientVoiceConfirmationTestApi; +} + +export function resetClientVoiceConfirmationStateForTest(): void { + getTestApi().resetClientVoiceConfirmationStateForTest(); +} diff --git a/src/talk/client-voice-confirmation.test.ts b/src/talk/client-voice-confirmation.test.ts new file mode 100644 index 00000000000..19be95c35d4 --- /dev/null +++ b/src/talk/client-voice-confirmation.test.ts @@ -0,0 +1,501 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { + authorizeClientVoiceConfirmation as authorizeClientVoiceConfirmationForTest, + bindAuthorizedClientVoiceConfirmation, + deactivateClientVoiceConfirmationSession, + noteClientVoiceConfirmationUtterance as noteClientVoiceConfirmationUtteranceForTest, + releaseClientVoiceConfirmationRun, + resolveClientVoiceToolConfirmationPolicy as resolveClientVoiceToolConfirmationPolicyForTest, +} from "./client-voice-confirmation.js"; +import { resetClientVoiceConfirmationStateForTest } from "./client-voice-confirmation.test-support.js"; + +function authorizeClientVoiceConfirmation( + params: Omit[0], "agentId">, +) { + return authorizeClientVoiceConfirmationForTest({ agentId: "main", ...params }); +} + +function noteClientVoiceConfirmationUtterance( + params: Omit[0], "agentId">, +): void { + noteClientVoiceConfirmationUtteranceForTest({ agentId: "main", ...params }); +} + +function resolveClientVoiceToolConfirmationPolicy( + params: Omit[0], "agentId">, +) { + return resolveClientVoiceToolConfirmationPolicyForTest({ agentId: "main", ...params }); +} + +function confirmationIdFrom(reason: string): string { + const match = reason.match(/VOICE_CONFIRMATION_REQUIRED:([^\s]+)/); + if (!match?.[1]) { + throw new Error(`missing confirmation id: ${reason}`); + } + return match[1]; +} + +function block(params: { + voiceSessionId: string; + runId?: string; + toolName?: string; + toolParams?: unknown; + now?: number; +}) { + const result = resolveClientVoiceToolConfirmationPolicy({ + voiceSessionId: params.voiceSessionId, + runId: params.runId, + toolName: params.toolName ?? "message", + toolParams: params.toolParams ?? { action: "send", message: "hello" }, + now: params.now, + }); + expect(result.allowed).toBe(false); + if (result.allowed) { + throw new Error("expected blocked voice action"); + } + return confirmationIdFrom(result.reason); +} + +describe("client voice confirmation", () => { + afterEach(() => resetClientVoiceConfirmationStateForTest()); + + it("does not pause a concurrent non-voice run sharing the session key", () => { + block({ voiceSessionId: "voice-1", runId: "voice-run" }); + + expect( + resolveClientVoiceToolConfirmationPolicy({ + toolName: "message", + toolParams: { action: "send", message: "other run" }, + }), + ).toEqual({ allowed: true }); + }); + + it.each([ + ["exec", "git clean -fdx"], + ["bash", "mv a b"], + ])( + "requires confirmation for an unlisted destructive shell command: %s %s", + (toolName, command) => { + expect( + resolveClientVoiceToolConfirmationPolicy({ + voiceSessionId: "voice-1", + runId: "voice-run", + toolName, + toolParams: { command }, + }).allowed, + ).toBe(false); + }, + ); + + it.each(["ls -la", "grep -n TODO README.md"])( + "does not require confirmation for a classified read-only shell command: %s", + (command) => { + expect( + resolveClientVoiceToolConfirmationPolicy({ + voiceSessionId: "voice-1", + runId: "voice-run", + toolName: "exec", + toolParams: { command }, + }), + ).toEqual({ allowed: true }); + }, + ); + + it("requires confirmation before delegating work outside the voice-bound run", () => { + expect( + resolveClientVoiceToolConfirmationPolicy({ + voiceSessionId: "voice-1", + runId: "voice-run", + toolName: "sessions_send", + toolParams: { sessionKey: "agent:main:child", message: "send this" }, + }).allowed, + ).toBe(false); + }); + + it("keeps pre-gate behavior for sessions that cannot report spoken approvals", () => { + expect( + resolveClientVoiceToolConfirmationPolicy({ + voiceSessionId: "voice-legacy", + runId: "voice-run", + toolName: "message", + toolParams: { action: "send", to: "user", message: "hi" }, + isConfirmable: () => false, + }), + ).toEqual({ allowed: true }); + expect( + resolveClientVoiceToolConfirmationPolicy({ + voiceSessionId: "voice-modern", + runId: "voice-run", + toolName: "message", + toolParams: { action: "send", to: "user", message: "hi" }, + isConfirmable: () => true, + }).allowed, + ).toBe(false); + }); + + it("keeps workspace-local writes confirmation-free", () => { + expect( + resolveClientVoiceToolConfirmationPolicy({ + voiceSessionId: "voice-1", + runId: "voice-run", + toolName: "write", + toolParams: { path: "notes.txt", content: "local change" }, + }), + ).toEqual({ allowed: true }); + }); + + it("keeps a challenge authorizable until the run is bound, then consumes it", () => { + const confirmationId = block({ + voiceSessionId: "voice-1", + toolParams: { action: "send", message: "A" }, + now: 100, + }); + noteClientVoiceConfirmationUtterance({ + voiceSessionId: "voice-1", + text: "yes", + timestamp: 101, + }); + // A failed/retried consult can re-authorize the same challenge before it binds. + const first = authorizeClientVoiceConfirmation({ + voiceSessionId: "voice-1", + confirmationId, + now: 102, + }); + expect( + authorizeClientVoiceConfirmation({ voiceSessionId: "voice-1", confirmationId, now: 103 }) + .fingerprint, + ).toBe(first.fingerprint); + bindAuthorizedClientVoiceConfirmation({ grant: first, runId: "run-approved" }); + // After binding the run, the challenge is consumed and cannot re-authorize. + expect(() => + authorizeClientVoiceConfirmation({ voiceSessionId: "voice-1", confirmationId, now: 104 }), + ).toThrow("missing, expired, or belongs to another action"); + }); + + it("binds approval to the exact tool fingerprint", () => { + const confirmationId = block({ + voiceSessionId: "voice-1", + toolParams: { action: "send", message: "A" }, + now: 100, + }); + noteClientVoiceConfirmationUtterance({ + voiceSessionId: "voice-1", + text: "Yes, do it.", + timestamp: 101, + }); + const grant = authorizeClientVoiceConfirmation({ + voiceSessionId: "voice-1", + confirmationId, + now: 102, + }); + bindAuthorizedClientVoiceConfirmation({ grant, runId: "run-approved" }); + + expect( + resolveClientVoiceToolConfirmationPolicy({ + voiceSessionId: "voice-1", + runId: "run-approved", + toolName: "message", + toolParams: { action: "send", message: "B" }, + now: 103, + }).allowed, + ).toBe(false); + expect( + resolveClientVoiceToolConfirmationPolicy({ + voiceSessionId: "voice-1", + runId: "run-approved", + toolName: "message", + toolParams: { action: "send", message: "A" }, + now: 104, + }), + ).toEqual({ allowed: true }); + }); + + it("rejects expired confirmations", () => { + const confirmationId = block({ voiceSessionId: "voice-1", now: 1_000 }); + noteClientVoiceConfirmationUtterance({ + voiceSessionId: "voice-1", + text: "confirm", + timestamp: 1_001, + }); + + expect(() => + authorizeClientVoiceConfirmation({ + voiceSessionId: "voice-1", + confirmationId, + now: 121_001, + }), + ).toThrow("missing, expired"); + }); + + it.each(["no", "don't do it", "don’t do it", "do not proceed", "cancel"])( + "a refusal invalidates the pending confirmation for %j", + (text) => { + const confirmationId = block({ voiceSessionId: "voice-1", now: 100 }); + noteClientVoiceConfirmationUtterance({ + voiceSessionId: "voice-1", + text, + timestamp: 101, + }); + + expect(() => + authorizeClientVoiceConfirmation({ + voiceSessionId: "voice-1", + confirmationId, + now: 102, + }), + ).toThrow("missing, expired, or belongs to another action"); + }, + ); + + it("consumes an approved fingerprint once", () => { + const toolParams = { action: "send", message: "A" }; + const confirmationId = block({ voiceSessionId: "voice-1", toolParams, now: 100 }); + noteClientVoiceConfirmationUtterance({ + voiceSessionId: "voice-1", + text: "go ahead", + timestamp: 101, + }); + const grant = authorizeClientVoiceConfirmation({ + voiceSessionId: "voice-1", + confirmationId, + now: 102, + }); + bindAuthorizedClientVoiceConfirmation({ grant, runId: "run-approved" }); + + expect( + resolveClientVoiceToolConfirmationPolicy({ + voiceSessionId: "voice-1", + runId: "run-approved", + toolName: "message", + toolParams, + now: 103, + }), + ).toEqual({ allowed: true }); + expect( + resolveClientVoiceToolConfirmationPolicy({ + voiceSessionId: "voice-1", + runId: "run-approved", + toolName: "message", + toolParams, + now: 104, + }).allowed, + ).toBe(false); + }); + + it("consumes one spoken affirmation for only one pending action", () => { + const first = block({ + voiceSessionId: "voice-1", + runId: "run-1", + toolParams: { action: "send", message: "A" }, + now: 100, + }); + const second = block({ + voiceSessionId: "voice-1", + runId: "run-2", + toolParams: { action: "send", message: "B" }, + now: 100, + }); + noteClientVoiceConfirmationUtterance({ + voiceSessionId: "voice-1", + text: "yes", + timestamp: 101, + }); + + expect(() => + authorizeClientVoiceConfirmation({ + voiceSessionId: "voice-1", + confirmationId: first, + now: 102, + }), + ).toThrow("newer confirmation request supersedes"); + // Binding the newer grant consumes the shared affirmation and its challenge. + const grant = authorizeClientVoiceConfirmation({ + voiceSessionId: "voice-1", + confirmationId: second, + now: 102, + }); + bindAuthorizedClientVoiceConfirmation({ grant, runId: "run-2" }); + expect(() => + authorizeClientVoiceConfirmation({ + voiceSessionId: "voice-1", + confirmationId: first, + now: 103, + }), + ).toThrow("explicit spoken confirmation"); + }); + + it("binds an approved fingerprint to its follow-up run", () => { + const toolParams = { action: "send", message: "same" }; + const confirmationId = block({ + voiceSessionId: "voice-1", + runId: "run-original", + toolParams, + now: 100, + }); + noteClientVoiceConfirmationUtterance({ + voiceSessionId: "voice-1", + text: "proceed", + timestamp: 101, + }); + const grant = authorizeClientVoiceConfirmation({ + voiceSessionId: "voice-1", + confirmationId, + now: 102, + }); + bindAuthorizedClientVoiceConfirmation({ grant, runId: "run-approved" }); + + expect( + resolveClientVoiceToolConfirmationPolicy({ + voiceSessionId: "voice-1", + runId: "run-other", + toolName: "message", + toolParams, + now: 103, + }).allowed, + ).toBe(false); + expect( + resolveClientVoiceToolConfirmationPolicy({ + voiceSessionId: "voice-1", + runId: "run-approved", + toolName: "message", + toolParams, + now: 104, + }), + ).toEqual({ allowed: true }); + }); + + it("only the newest pending challenge can be authorized", () => { + const olderId = block({ + voiceSessionId: "voice-1", + runId: "run-1", + toolParams: { action: "send", message: "older" }, + now: 100, + }); + const newerId = block({ + voiceSessionId: "voice-1", + runId: "run-1", + toolParams: { action: "send", message: "newer" }, + now: 110, + }); + noteClientVoiceConfirmationUtterance({ + voiceSessionId: "voice-1", + text: "yes", + timestamp: 111, + }); + expect(() => + authorizeClientVoiceConfirmation({ + voiceSessionId: "voice-1", + confirmationId: olderId, + now: 112, + }), + ).toThrow("newer confirmation request supersedes"); + expect( + authorizeClientVoiceConfirmation({ + voiceSessionId: "voice-1", + confirmationId: newerId, + now: 113, + }).fingerprint, + ).toBeTruthy(); + }); + + it("invalidates a pending confirmation when the user refuses", () => { + const toolParams = { action: "send", message: "declined" }; + const confirmationId = block({ + voiceSessionId: "voice-1", + runId: "run-1", + toolParams, + now: 100, + }); + noteClientVoiceConfirmationUtterance({ + voiceSessionId: "voice-1", + text: "No, cancel that", + timestamp: 101, + }); + noteClientVoiceConfirmationUtterance({ + voiceSessionId: "voice-1", + text: "yes", + timestamp: 102, + }); + expect(() => + authorizeClientVoiceConfirmation({ + voiceSessionId: "voice-1", + confirmationId, + now: 103, + }), + ).toThrow("missing, expired, or belongs to another action"); + }); + + it("keeps a live run's grant across call close and releases it on run completion", () => { + const toolParams = { action: "send", message: "confirmed then hangup" }; + const confirmationId = block({ + voiceSessionId: "voice-1", + runId: "run-original", + toolParams, + now: 100, + }); + noteClientVoiceConfirmationUtterance({ + voiceSessionId: "voice-1", + text: "yes do it", + timestamp: 101, + }); + const grant = authorizeClientVoiceConfirmation({ + voiceSessionId: "voice-1", + confirmationId, + now: 102, + }); + bindAuthorizedClientVoiceConfirmation({ grant, runId: "run-live" }); + + deactivateClientVoiceConfirmationSession("main", "voice-1", ["run-live"]); + expect( + resolveClientVoiceToolConfirmationPolicy({ + voiceSessionId: "voice-1", + runId: "run-live", + toolName: "message", + toolParams, + now: 103, + }), + ).toEqual({ allowed: true }); + + bindAuthorizedClientVoiceConfirmation({ grant, runId: "run-done" }); + releaseClientVoiceConfirmationRun("main", "voice-1", "run-done"); + expect( + resolveClientVoiceToolConfirmationPolicy({ + voiceSessionId: "voice-1", + runId: "run-done", + toolName: "message", + toolParams, + now: 104, + }).allowed, + ).toBe(false); + }); + + it("isolates same-named voice sessions across agents", () => { + const blocked = resolveClientVoiceToolConfirmationPolicyForTest({ + agentId: "agent-b", + voiceSessionId: "shared-id", + runId: "run-b", + toolName: "message", + toolParams: { action: "send", message: "B" }, + now: 100, + }); + expect(blocked.allowed).toBe(false); + if (blocked.allowed) { + throw new Error("expected confirmation request"); + } + noteClientVoiceConfirmationUtteranceForTest({ + agentId: "agent-a", + voiceSessionId: "shared-id", + text: "yes", + timestamp: 101, + }); + + expect(() => + authorizeClientVoiceConfirmationForTest({ + agentId: "agent-b", + voiceSessionId: "shared-id", + confirmationId: confirmationIdFrom(blocked.reason), + now: 102, + }), + ).toThrow("explicit spoken confirmation"); + }); +}); diff --git a/src/talk/client-voice-confirmation.ts b/src/talk/client-voice-confirmation.ts new file mode 100644 index 00000000000..d4f93f08320 --- /dev/null +++ b/src/talk/client-voice-confirmation.ts @@ -0,0 +1,341 @@ +/** In-memory spoken confirmation binding for high-impact Talk actions. */ +import { createHash, randomUUID } from "node:crypto"; +import { buildToolMutationState } from "../agents/tool-mutation.js"; + +const CONFIRMATION_TTL_MS = 2 * 60_000; + +type PendingVoiceConfirmation = { + confirmationId: string; + agentId: string; + voiceSessionId: string; + runId?: string; + fingerprint: string; + toolName: string; + createdAt: number; + /** Monotonic tiebreaker: same-millisecond challenges must still have one newest. */ + seq: number; + expiresAt: number; +}; + +type RecentVoiceUserUtterance = { + text: string; + timestamp: number; +}; + +export type ClientVoiceConfirmationGrant = { + agentId: string; + voiceSessionId: string; + confirmationId: string; + fingerprint: string; + expiresAt: number; +}; + +const pendingConfirmations = new Map(); +let confirmationSeq = 0; +const approvedFingerprints = new Map>>(); +const recentUserUtterances = new Map(); + +function confirmationScopeKey(agentId: string, voiceSessionId: string): string { + return `${agentId}\0${voiceSessionId}`; +} + +function stableToolFingerprint(toolName: string, params: unknown): string { + const normalize = (value: unknown): unknown => { + if (Array.isArray(value)) { + return value.map(normalize); + } + if (!value || typeof value !== "object") { + return value; + } + return Object.fromEntries( + Object.entries(value as Record) + .toSorted(([left], [right]) => left.localeCompare(right)) + .map(([key, entry]) => [key, normalize(entry)]), + ); + }; + return createHash("sha256") + .update(`${toolName}\0${JSON.stringify(normalize(params))}`) + .digest("hex"); +} + +function requiresHighImpactVoiceConfirmation(toolName: string, params: unknown): boolean { + const normalizedTool = toolName.trim().toLowerCase(); + if (!buildToolMutationState(normalizedTool, params).mutatingAction) { + return false; + } + if ( + ["message", "gateway", "nodes", "browser", "computer", "canvas", "cron", "process"].includes( + normalizedTool, + ) + ) { + return true; + } + // Workspace-local edits stay bound to this run. Session delegation is gated because + // delegated runs leave the voice binding and otherwise bypass spoken confirmation. + if ( + ["write", "edit", "apply_patch", "create_goal", "update_goal", "get_goal"].includes( + normalizedTool, + ) + ) { + return false; + } + return true; +} + +function consumeApprovedFingerprint( + voiceSessionId: string, + runId: string | undefined, + fingerprint: string, + now: number, +): boolean { + if (!runId) { + return false; + } + const approvedByRun = approvedFingerprints.get(voiceSessionId); + const approved = approvedByRun?.get(runId); + const expiresAt = approved?.get(fingerprint); + if (!expiresAt || expiresAt < now) { + approved?.delete(fingerprint); + return false; + } + approved?.delete(fingerprint); + return true; +} + +/** Record a finalized user utterance after the durable transcript append succeeds. */ +export function noteClientVoiceConfirmationUtterance(params: { + agentId: string; + voiceSessionId: string; + text: string; + timestamp: number; +}): void { + recentUserUtterances.set(confirmationScopeKey(params.agentId, params.voiceSessionId), { + text: params.text, + timestamp: params.timestamp, + }); + // A spoken refusal kills the outstanding challenge: a later unrelated "yes" + // must not resurrect an action the user already declined. + if (REFUSAL_PATTERN.test(normalizeUtterance(params.text))) { + for (const [confirmationId, confirmation] of pendingConfirmations) { + if ( + confirmation.agentId === params.agentId && + confirmation.voiceSessionId === params.voiceSessionId && + confirmation.createdAt < params.timestamp + ) { + pendingConfirmations.delete(confirmationId); + } + } + } +} + +/** Pause a high-impact action for one voice-bound run until its exact fingerprint is approved. */ +export function resolveClientVoiceToolConfirmationPolicy(params: { + agentId?: string; + voiceSessionId?: string; + runId?: string; + toolName: string; + toolParams: unknown; + isConfirmable?: () => boolean; + now?: number; +}): { allowed: true } | { allowed: false; reason: string } { + if (!params.agentId || !params.voiceSessionId) { + return { allowed: true }; + } + if (!requiresHighImpactVoiceConfirmation(params.toolName, params.toolParams)) { + return { allowed: true }; + } + // Sessions that cannot report spoken approvals (legacy clients without transcript + // RPCs) keep pre-gate behavior; a pause they can never confirm is a dead end. + // This is not a client trust boundary: authenticated clients can already run any + // tool via chat.send. The gate guards against voice-channel misfires only. + if (params.isConfirmable && !params.isConfirmable()) { + return { allowed: true }; + } + const now = params.now ?? Date.now(); + const fingerprint = stableToolFingerprint(params.toolName, params.toolParams); + const scopeKey = confirmationScopeKey(params.agentId, params.voiceSessionId); + if (consumeApprovedFingerprint(scopeKey, params.runId, fingerprint, now)) { + return { allowed: true }; + } + const existing = [...pendingConfirmations.values()].find( + (entry) => + entry.voiceSessionId === params.voiceSessionId && + entry.agentId === params.agentId && + entry.runId === params.runId && + entry.fingerprint === fingerprint && + entry.expiresAt >= now, + ); + const confirmation = + existing ?? + ({ + confirmationId: randomUUID(), + agentId: params.agentId, + voiceSessionId: params.voiceSessionId, + ...(params.runId ? { runId: params.runId } : {}), + fingerprint, + toolName: params.toolName, + createdAt: now, + seq: ++confirmationSeq, + expiresAt: now + CONFIRMATION_TTL_MS, + } satisfies PendingVoiceConfirmation); + pendingConfirmations.set(confirmation.confirmationId, confirmation); + return { + allowed: false, + reason: + `VOICE_CONFIRMATION_REQUIRED:${confirmation.confirmationId} ` + + `The high-impact voice action "${params.toolName}" was not executed. ` + + "Ask the user for explicit spoken confirmation, then call openclaw_agent_consult again with this confirmationId.", + }; +} + +const REFUSAL_PATTERN = /\b(no|don't|do not|cancel|stop|never mind)\b/; + +function normalizeUtterance(text: string): string { + return ( + text + .trim() + .toLowerCase() + // STT commonly emits typographic apostrophes; fold them so "don't" (U+2019) + // matches the refusal pattern and cannot slip past as a non-refusal. + .replace(/[‘’ʼ]/g, "'") + .replace(/[,;:.!?]+/g, "") + .replace(/\s+/g, " ") + ); +} + +function isExplicitAffirmation(text: string): boolean { + const normalized = normalizeUtterance(text); + if (REFUSAL_PATTERN.test(normalized)) { + return false; + } + // English-only phrases are an accepted first version; localized matching is follow-up work. + return /^(yes|yes do it|do it|confirm|confirmed|go ahead|proceed|send it|make the change|restart it)$/.test( + normalized, + ); +} + +/** Bind a later affirmative utterance to one exact paused action. */ +export function authorizeClientVoiceConfirmation(params: { + agentId: string; + voiceSessionId: string; + confirmationId: string; + now?: number; +}): ClientVoiceConfirmationGrant { + const confirmation = pendingConfirmations.get(params.confirmationId); + const now = params.now ?? Date.now(); + if ( + !confirmation || + confirmation.agentId !== params.agentId || + confirmation.voiceSessionId !== params.voiceSessionId || + confirmation.expiresAt < now + ) { + throw new Error("voice confirmation is missing, expired, or belongs to another action"); + } + // A bare "yes" can only answer the question the model asked last; authorizing an + // older challenge would let the model swap in a different pending action. + for (const entry of pendingConfirmations.values()) { + if ( + entry.agentId === params.agentId && + entry.voiceSessionId === params.voiceSessionId && + entry.seq > confirmation.seq + ) { + throw new Error("a newer confirmation request supersedes this one; ask again"); + } + } + const scopeKey = confirmationScopeKey(params.agentId, params.voiceSessionId); + const affirmation = recentUserUtterances.get(scopeKey); + if ( + !affirmation || + affirmation.timestamp <= confirmation.createdAt || + !isExplicitAffirmation(affirmation.text) + ) { + throw new Error("explicit spoken confirmation was not found after the action request"); + } + // Validate only; the challenge and affirmation are consumed at bind time, once the + // consult run is established. This keeps a failed/lost-response consult retryable + // with the same confirmationId instead of leaving the action unconfirmable. + return { + agentId: params.agentId, + voiceSessionId: params.voiceSessionId, + confirmationId: params.confirmationId, + fingerprint: confirmation.fingerprint, + expiresAt: confirmation.expiresAt, + }; +} + +/** Bind a validated spoken grant to the one follow-up run and consume the challenge. */ +export function bindAuthorizedClientVoiceConfirmation(params: { + grant: ClientVoiceConfirmationGrant; + runId: string; +}): void { + const scopeKey = confirmationScopeKey(params.grant.agentId, params.grant.voiceSessionId); + const approvedByRun = approvedFingerprints.get(scopeKey) ?? new Map(); + const approved = approvedByRun.get(params.runId) ?? new Map(); + approved.set(params.grant.fingerprint, params.grant.expiresAt); + approvedByRun.set(params.runId, approved); + approvedFingerprints.set(scopeKey, approvedByRun); + // Consume now that the run exists: one spoken affirmation authorizes one action. + pendingConfirmations.delete(params.grant.confirmationId); + recentUserUtterances.delete(scopeKey); +} + +/** + * Remove ephemeral confirmation state when the logical call closes. Approved + * grants for still-live consult runs survive: a spoken "yes" followed by hangup + * must not re-block the confirmed action its run is about to execute. + */ +export function deactivateClientVoiceConfirmationSession( + agentId: string, + voiceSessionId: string, + liveRunIds: readonly string[] = [], +): void { + const scopeKey = confirmationScopeKey(agentId, voiceSessionId); + recentUserUtterances.delete(scopeKey); + const approvedByRun = approvedFingerprints.get(scopeKey); + if (approvedByRun) { + const live = new Set(liveRunIds); + for (const runId of approvedByRun.keys()) { + if (!live.has(runId)) { + approvedByRun.delete(runId); + } + } + if (approvedByRun.size === 0) { + approvedFingerprints.delete(scopeKey); + } + } + for (const [confirmationId, confirmation] of pendingConfirmations) { + if (confirmation.agentId === agentId && confirmation.voiceSessionId === voiceSessionId) { + pendingConfirmations.delete(confirmationId); + } + } +} + +/** Drop a completed run's surviving grants once its lifecycle ends. */ +export function releaseClientVoiceConfirmationRun( + agentId: string, + voiceSessionId: string, + runId: string, +): void { + const scopeKey = confirmationScopeKey(agentId, voiceSessionId); + const approvedByRun = approvedFingerprints.get(scopeKey); + if (!approvedByRun) { + return; + } + approvedByRun.delete(runId); + if (approvedByRun.size === 0) { + approvedFingerprints.delete(scopeKey); + } +} + +/** Test-only reset for process-global state. */ +function resetClientVoiceConfirmationStateForTest(): void { + pendingConfirmations.clear(); + approvedFingerprints.clear(); + recentUserUtterances.clear(); +} + +if (process.env.VITEST || process.env.NODE_ENV === "test") { + (globalThis as Record)[ + Symbol.for("openclaw.clientVoiceConfirmationTestApi") + ] = { resetClientVoiceConfirmationStateForTest }; +} diff --git a/src/talk/client-voice-session-store.ts b/src/talk/client-voice-session-store.ts new file mode 100644 index 00000000000..2897da7ce98 --- /dev/null +++ b/src/talk/client-voice-session-store.ts @@ -0,0 +1,161 @@ +/** SQLite-backed persistence for durable per-agent Talk voice-call records. */ +import { + openOpenClawAgentDatabase, + type OpenClawAgentDatabase, +} from "../state/openclaw-agent-db.js"; + +export const VOICE_SESSION_CACHE_SCOPE = "talk-client-voice-sessions"; +export const VOICE_SESSION_RECORD_VERSION = 1; +export const VOICE_SESSION_MAX_TRANSCRIPT_CHARS = 8_000; +export const VOICE_SESSION_STALE_AFTER_MS = 6 * 60 * 60_000; + +export type ClientVoiceToolEffect = { + runId: string; + toolCallId?: string; + toolName: string; + startedAt: number; + finishedAt?: number; + status: "started" | "succeeded" | "failed" | "cancelled" | "blocked"; +}; + +export type ClientVoiceSessionRecord = { + version: typeof VOICE_SESSION_RECORD_VERSION; + voiceSessionId: string; + agentId: string; + sessionKey: string; + provider?: string; + origin: "client" | "relay"; + status: "open" | "closed"; + createdAt: number; + updatedAt: number; + closedAt?: number; + consultRunIds: string[]; + effects: ClientVoiceToolEffect[]; + digestDeliveredAt?: number; + /** Declared at create when the client speaks the transcript protocol (sent sessionKey). */ + transcriptCapable?: boolean; + /** Set once a finalized user utterance persisted; gates spoken confirmation capability. */ + hasUserTranscript?: boolean; +}; + +export type ClientVoiceRunBinding = { + agentId: string; + voiceSessionId: string; + sessionKey: string; +}; + +function parseVoiceSessionRecord(value: unknown): ClientVoiceSessionRecord | undefined { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return undefined; + } + const record = value as Partial; + if ( + record.version !== VOICE_SESSION_RECORD_VERSION || + typeof record.voiceSessionId !== "string" || + typeof record.agentId !== "string" || + typeof record.sessionKey !== "string" || + (record.provider !== undefined && + (typeof record.provider !== "string" || !record.provider.trim())) || + (record.origin !== "client" && record.origin !== "relay") || + (record.status !== "open" && record.status !== "closed") || + typeof record.createdAt !== "number" || + typeof record.updatedAt !== "number" + ) { + return undefined; + } + const consultRunIds = Array.isArray(record.consultRunIds) + ? record.consultRunIds.filter((entry): entry is string => typeof entry === "string") + : []; + const effects = Array.isArray(record.effects) + ? record.effects.filter((entry): entry is ClientVoiceToolEffect => { + if (!entry || typeof entry !== "object") { + return false; + } + const effect = entry as Partial; + return ( + typeof effect.runId === "string" && + typeof effect.toolName === "string" && + typeof effect.startedAt === "number" && + (effect.status === "started" || + effect.status === "succeeded" || + effect.status === "failed" || + effect.status === "cancelled" || + effect.status === "blocked") + ); + }) + : []; + const provider = record.provider?.trim(); + return { + ...record, + ...(provider ? { provider } : {}), + consultRunIds, + effects, + } as ClientVoiceSessionRecord; +} + +export function parseStoredVoiceSessionRecord( + valueJson: unknown, +): ClientVoiceSessionRecord | undefined { + if (typeof valueJson !== "string") { + return undefined; + } + try { + return parseVoiceSessionRecord(JSON.parse(valueJson)); + } catch { + return undefined; + } +} + +export function readVoiceSessionRecord( + agentId: string, + voiceSessionId: string, +): ClientVoiceSessionRecord | undefined { + const database = openOpenClawAgentDatabase({ agentId }); + const row = database.db + .prepare("SELECT value_json FROM cache_entries WHERE scope = ? AND key = ?") + .get(VOICE_SESSION_CACHE_SCOPE, voiceSessionId) as { value_json?: unknown } | undefined; + return parseStoredVoiceSessionRecord(row?.value_json); +} + +export function readVoiceSessionRecordInTransaction( + database: OpenClawAgentDatabase, + voiceSessionId: string, +): ClientVoiceSessionRecord | undefined { + const row = database.db + .prepare("SELECT value_json FROM cache_entries WHERE scope = ? AND key = ?") + .get(VOICE_SESSION_CACHE_SCOPE, voiceSessionId) as { value_json?: unknown } | undefined; + return parseStoredVoiceSessionRecord(row?.value_json); +} + +export function writeVoiceSessionRecordInTransaction( + database: OpenClawAgentDatabase, + record: ClientVoiceSessionRecord, +): void { + database.db + .prepare( + `INSERT INTO cache_entries (scope, key, value_json, blob, expires_at, updated_at) + VALUES (?, ?, ?, NULL, NULL, ?) + ON CONFLICT(scope, key) DO UPDATE SET + value_json = excluded.value_json, + updated_at = excluded.updated_at`, + ) + .run( + VOICE_SESSION_CACHE_SCOPE, + record.voiceSessionId, + JSON.stringify(record), + record.updatedAt, + ); +} + +export function assertVoiceSessionOwnership( + record: ClientVoiceSessionRecord, + params: { agentId: string; sessionKey: string }, +): void { + if (record.agentId !== params.agentId || record.sessionKey !== params.sessionKey) { + throw new Error("voice session does not belong to this agent session"); + } +} + +export function operationKey(agentId: string, voiceSessionId: string): string { + return `${agentId}\0${voiceSessionId}`; +} diff --git a/src/talk/client-voice-session.test-support.ts b/src/talk/client-voice-session.test-support.ts new file mode 100644 index 00000000000..775923ef51f --- /dev/null +++ b/src/talk/client-voice-session.test-support.ts @@ -0,0 +1,22 @@ +import type { ClientVoiceSessionRecord } from "./client-voice-session-store.js"; +import "./client-voice-session.js"; + +type ClientVoiceSessionTestApi = { + readRecord(agentId: string, voiceSessionId: string): ClientVoiceSessionRecord | undefined; + reset(): void; +}; + +function getTestApi(): ClientVoiceSessionTestApi { + return (globalThis as Record)[ + Symbol.for("openclaw.clientVoiceSessionTestApi") + ] as ClientVoiceSessionTestApi; +} + +export const clientVoiceSessionTesting = { + readRecord(agentId: string, voiceSessionId: string): ClientVoiceSessionRecord | undefined { + return getTestApi().readRecord(agentId, voiceSessionId); + }, + reset(): void { + getTestApi().reset(); + }, +}; diff --git a/src/talk/client-voice-session.test.ts b/src/talk/client-voice-session.test.ts new file mode 100644 index 00000000000..7222779a5ae --- /dev/null +++ b/src/talk/client-voice-session.test.ts @@ -0,0 +1,579 @@ +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { replaceSessionEntry } from "../config/sessions/session-accessor.js"; +import { + emitTrustedDiagnosticEvent, + waitForDiagnosticEventsDrained, +} from "../infra/diagnostic-events.js"; +import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js"; +import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js"; +import { captureEnv, setTestEnvValue } from "../test-utils/env.js"; +import { + closeClientVoiceSession, + closeStaleClientVoiceSessions, + createOrResumeClientVoiceSession, + isClientVoiceSessionConfirmable, + registerClientVoiceConsultRun, + resolveClientVoiceRunBinding, + resolveOpenClientVoiceSessionId, +} from "./client-voice-session.js"; +import { clientVoiceSessionTesting } from "./client-voice-session.test-support.js"; + +const { sendDurableMessageBatch } = vi.hoisted(() => ({ + sendDurableMessageBatch: vi.fn(async () => ({ status: "sent" })), +})); + +vi.mock("../channels/message/runtime.js", () => ({ sendDurableMessageBatch })); + +const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]); +let tempDir: string; + +async function seedSession( + sessionKey: string, + route: { lastChannel?: string; lastTo?: string } = {}, +): Promise { + await replaceSessionEntry( + { agentId: "main", sessionKey }, + { + sessionId: `session-${sessionKey.replaceAll(":", "-")}`, + updatedAt: Date.now(), + ...route, + }, + ); +} + +function recordMutation(voiceSessionId: string, runId = `run-${voiceSessionId}`): void { + registerClientVoiceConsultRun({ + agentId: "main", + sessionKey: "agent:main:main", + voiceSessionId, + runId, + }); + emitTrustedDiagnosticEvent({ + type: "tool.execution.started", + runId, + toolCallId: `call-${runId}`, + toolName: "message", + mutatingAction: true, + }); + emitTrustedDiagnosticEvent({ + type: "tool.execution.completed", + runId, + toolCallId: `call-${runId}`, + toolName: "message", + durationMs: 5, + }); +} + +async function completeRun(runId: string): Promise { + emitTrustedDiagnosticEvent({ + type: "run.completed", + runId, + durationMs: 5, + outcome: "completed", + }); + await waitForDiagnosticEventsDrained(); +} + +describe("client voice session", () => { + beforeEach(async () => { + tempDir = await fs.realpath( + await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-voice-session-")), + ); + setTestEnvValue("OPENCLAW_STATE_DIR", tempDir); + sendDurableMessageBatch.mockClear(); + }); + + afterEach(async () => { + clientVoiceSessionTesting.reset(); + closeOpenClawAgentDatabasesForTest(); + closeOpenClawStateDatabaseForTest(); + envSnapshot.restore(); + await fs.rm(tempDir, { recursive: true, force: true }); + }); + + it("creates, resumes, and enforces ownership and open state", async () => { + const voiceSessionId = createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + provider: "google", + origin: "client", + voiceSessionId: "voice-1", + now: 10, + }); + expect( + createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + origin: "client", + voiceSessionId, + now: 20, + }), + ).toBe(voiceSessionId); + expect(clientVoiceSessionTesting.readRecord("main", voiceSessionId)).toMatchObject({ + provider: "google", + }); + expect(() => + createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + provider: "openai", + origin: "client", + voiceSessionId, + }), + ).toThrow("provider does not match"); + expect(() => + createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:other", + origin: "client", + voiceSessionId, + }), + ).toThrow("does not belong"); + + await closeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + voiceSessionId, + config: {}, + now: 30, + }); + expect(() => + createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + origin: "client", + voiceSessionId, + }), + ).toThrow("already closed"); + }); + + it("marks confirmability by declared capability, relay origin, or observed transcript", () => { + const capable = createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + origin: "client", + transcriptCapable: true, + voiceSessionId: "voice-capable", + }); + const legacy = createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + origin: "client", + voiceSessionId: "voice-legacy", + }); + const relay = createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + origin: "relay", + voiceSessionId: "voice-relay", + }); + const binding = (voiceSessionId: string) => ({ + agentId: "main", + sessionKey: "agent:main:main", + voiceSessionId, + }); + expect(isClientVoiceSessionConfirmable(binding(capable))).toBe(true); + expect(isClientVoiceSessionConfirmable(binding(legacy))).toBe(false); + expect(isClientVoiceSessionConfirmable(binding(relay))).toBe(true); + }); + + it("closes idempotently without changing the first close time", async () => { + const voiceSessionId = createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + origin: "client", + now: 10, + }); + await closeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + voiceSessionId, + config: {}, + now: 20, + }); + await closeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + voiceSessionId, + config: {}, + now: 30, + }); + + expect(clientVoiceSessionTesting.readRecord("main", voiceSessionId)).toMatchObject({ + status: "closed", + closedAt: 20, + }); + }); + + it("keeps active consult runs voice-bound after the call closes", async () => { + const voiceSessionId = createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + origin: "client", + }); + registerClientVoiceConsultRun({ + agentId: "main", + sessionKey: "agent:main:main", + voiceSessionId, + runId: "run-active", + }); + + await closeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + voiceSessionId, + config: {}, + }); + + expect(resolveClientVoiceRunBinding("run-active")).toMatchObject({ voiceSessionId }); + }); + + it("resolves the open client record for legacy tool calls", () => { + const voiceSessionId = createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + origin: "client", + }); + + expect( + resolveOpenClientVoiceSessionId({ agentId: "main", sessionKey: "agent:main:main" }), + ).toBe(voiceSessionId); + expect( + resolveOpenClientVoiceSessionId({ agentId: "main", sessionKey: "agent:main:other" }), + ).toBeUndefined(); + createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + origin: "client", + }); + expect( + resolveOpenClientVoiceSessionId({ agentId: "main", sessionKey: "agent:main:main" }), + ).toBeUndefined(); + }); + + it("keeps repeated tool-call ids separate across consult runs", () => { + const voiceSessionId = createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + origin: "client", + }); + for (const runId of ["run-1", "run-2"]) { + registerClientVoiceConsultRun({ + agentId: "main", + sessionKey: "agent:main:main", + voiceSessionId, + runId, + }); + emitTrustedDiagnosticEvent({ + type: "tool.execution.started", + runId, + toolCallId: "call-1", + toolName: "message", + mutatingAction: true, + }); + } + + expect(clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.effects).toHaveLength(2); + }); + + it("closes stale records and leaves recent records open", async () => { + const stale = createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:stale", + origin: "client", + now: 1, + }); + const recent = createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:recent", + origin: "client", + now: 6 * 60 * 60_000, + }); + + expect( + await closeStaleClientVoiceSessions({ + agentId: "main", + config: {}, + now: 6 * 60 * 60_000 + 2, + }), + ).toBe(1); + expect(clientVoiceSessionTesting.readRecord("main", stale)?.status).toBe("closed"); + expect(clientVoiceSessionTesting.readRecord("main", recent)?.status).toBe("open"); + }); + + it("records only mutating started effects and updates their terminal status", () => { + const voiceSessionId = createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + origin: "client", + }); + registerClientVoiceConsultRun({ + agentId: "main", + sessionKey: "agent:main:main", + voiceSessionId, + runId: "run-1", + }); + emitTrustedDiagnosticEvent({ + type: "tool.execution.started", + runId: "run-1", + toolCallId: "read-1", + toolName: "read", + mutatingAction: false, + }); + emitTrustedDiagnosticEvent({ + type: "tool.execution.started", + runId: "run-1", + toolCallId: "message-1", + toolName: "message", + mutatingAction: true, + }); + emitTrustedDiagnosticEvent({ + type: "tool.execution.error", + runId: "run-1", + toolCallId: "message-1", + toolName: "message", + durationMs: 4, + errorCategory: "aborted", + terminalReason: "cancelled", + }); + + expect(clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.effects).toEqual([ + expect.objectContaining({ + toolCallId: "message-1", + toolName: "message", + status: "cancelled", + finishedAt: expect.any(Number), + }), + ]); + }); + + it("records post-close effects and defers the digest until the last consult completes", async () => { + await seedSession("agent:main:main", { + lastChannel: "discord", + lastTo: "channel:voice-updates", + }); + const voiceSessionId = createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + origin: "client", + }); + for (const runId of ["run-1", "run-2"]) { + registerClientVoiceConsultRun({ + agentId: "main", + sessionKey: "agent:main:main", + voiceSessionId, + runId, + }); + } + + await closeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + voiceSessionId, + config: {}, + }); + expect(sendDurableMessageBatch).not.toHaveBeenCalled(); + + for (const runId of ["run-1", "run-2"]) { + emitTrustedDiagnosticEvent({ + type: "tool.execution.started", + runId, + toolCallId: `call-${runId}`, + toolName: "message", + mutatingAction: true, + }); + emitTrustedDiagnosticEvent({ + type: "tool.execution.completed", + runId, + toolCallId: `call-${runId}`, + toolName: "message", + durationMs: 5, + }); + } + expect(clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.effects).toEqual([ + expect.objectContaining({ runId: "run-1", status: "succeeded" }), + expect.objectContaining({ runId: "run-2", status: "succeeded" }), + ]); + + await completeRun("run-1"); + expect(sendDurableMessageBatch).not.toHaveBeenCalled(); + await completeRun("run-2"); + await vi.waitFor(() => expect(sendDurableMessageBatch).toHaveBeenCalledTimes(1)); + expect(sendDurableMessageBatch).toHaveBeenCalledWith( + expect.objectContaining({ + payloads: [{ text: "Voice call changes\n- message: succeeded\n- message: succeeded" }], + }), + ); + expect(clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.digestDeliveredAt).toEqual( + expect.any(Number), + ); + + await closeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + voiceSessionId, + config: {}, + }); + expect(sendDurableMessageBatch).toHaveBeenCalledTimes(1); + }); + + it("retries a deferred digest on the next voice session after a failed delivery", async () => { + await seedSession("agent:main:main", { + lastChannel: "discord", + lastTo: "channel:voice-updates", + }); + const voiceSessionId = createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + origin: "client", + }); + registerClientVoiceConsultRun({ + agentId: "main", + sessionKey: "agent:main:main", + voiceSessionId, + runId: "run-live", + }); + emitTrustedDiagnosticEvent({ + type: "tool.execution.started", + runId: "run-live", + toolCallId: "call-run-live", + toolName: "message", + mutatingAction: true, + }); + emitTrustedDiagnosticEvent({ + type: "tool.execution.completed", + runId: "run-live", + toolCallId: "call-run-live", + toolName: "message", + durationMs: 5, + }); + // Call ends while the consult still runs, so the digest is deferred. + await closeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + voiceSessionId, + config: {}, + }); + sendDurableMessageBatch.mockRejectedValueOnce(new Error("channel offline")); + await completeRun("run-live"); + expect( + clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.digestDeliveredAt, + ).toBeUndefined(); + + // Starting the next voice session retries the deferred digest. + await closeStaleClientVoiceSessions({ agentId: "main", config: {} }); + await vi.waitFor(() => + expect( + clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.digestDeliveredAt, + ).toEqual(expect.any(Number)), + ); + }); + + it("retries the mutation digest after a transient send failure", async () => { + await seedSession("agent:main:main", { + lastChannel: "discord", + lastTo: "channel:voice-updates", + }); + const voiceSessionId = createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + origin: "client", + }); + recordMutation(voiceSessionId); + await completeRun(`run-${voiceSessionId}`); + sendDurableMessageBatch.mockRejectedValueOnce(new Error("channel offline")); + + await closeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + voiceSessionId, + config: {}, + }).catch(() => undefined); + expect( + clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.digestDeliveredAt, + ).toBeUndefined(); + + await closeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + voiceSessionId, + config: {}, + }); + expect(sendDurableMessageBatch).toHaveBeenCalledTimes(2); + expect(clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.digestDeliveredAt).toEqual( + expect.any(Number), + ); + }); + + it("delivers one mutation digest and skips webchat or missing targets", async () => { + await seedSession("agent:main:main", { + lastChannel: "discord", + lastTo: "channel:voice-updates", + }); + const delivered = createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + origin: "client", + }); + recordMutation(delivered); + await completeRun(`run-${delivered}`); + await closeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + voiceSessionId: delivered, + config: {}, + }); + await closeClientVoiceSession({ + agentId: "main", + sessionKey: "agent:main:main", + voiceSessionId: delivered, + config: {}, + }); + expect(sendDurableMessageBatch).toHaveBeenCalledTimes(1); + expect(sendDurableMessageBatch).toHaveBeenCalledWith( + expect.objectContaining({ + durability: "required", + requireUnknownSendReconciliation: true, + payloads: [{ text: "Voice call changes\n- message: succeeded" }], + }), + ); + + for (const [voiceSessionId, route] of [ + ["voice-webchat", { lastChannel: "webchat", lastTo: "browser" }], + ["voice-no-target", {}], + ] as const) { + const sessionKey = `agent:main:${voiceSessionId}`; + await seedSession(sessionKey, route); + createOrResumeClientVoiceSession({ + agentId: "main", + sessionKey, + origin: "client", + voiceSessionId, + }); + registerClientVoiceConsultRun({ + agentId: "main", + sessionKey, + voiceSessionId, + runId: `run-${voiceSessionId}`, + }); + emitTrustedDiagnosticEvent({ + type: "tool.execution.started", + runId: `run-${voiceSessionId}`, + toolCallId: `call-${voiceSessionId}`, + toolName: "message", + mutatingAction: true, + }); + await completeRun(`run-${voiceSessionId}`); + await closeClientVoiceSession({ + agentId: "main", + sessionKey, + voiceSessionId, + config: {}, + }); + } + expect(sendDurableMessageBatch).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/talk/client-voice-session.ts b/src/talk/client-voice-session.ts new file mode 100644 index 00000000000..7719c3dd58f --- /dev/null +++ b/src/talk/client-voice-session.ts @@ -0,0 +1,732 @@ +/** Durable per-agent voice-call records for Talk continuity and mutation evidence. */ +import { randomUUID } from "node:crypto"; +import { + appendTranscriptMessage, + loadSessionEntry, + upsertSessionEntry, +} from "../config/sessions/session-accessor.js"; +import type { OpenClawConfig } from "../config/types.openclaw.js"; +import { + onTrustedInternalDiagnosticEvent, + onTrustedToolExecutionEvent, + type TrustedToolExecutionEvent, +} from "../infra/diagnostic-events.js"; +import { buildOutboundSessionContext } from "../infra/outbound/session-context.js"; +import { resolveSessionDeliveryTarget } from "../infra/outbound/targets-session.js"; +import { + openOpenClawAgentDatabase, + runOpenClawAgentWriteTransaction, +} from "../state/openclaw-agent-db.js"; +import { truncateUtf16Safe } from "../utils.js"; +import { + deactivateClientVoiceConfirmationSession, + noteClientVoiceConfirmationUtterance, + releaseClientVoiceConfirmationRun, +} from "./client-voice-confirmation.js"; +import { + assertVoiceSessionOwnership as assertOwnership, + type ClientVoiceRunBinding, + type ClientVoiceSessionRecord, + type ClientVoiceToolEffect, + operationKey, + parseStoredVoiceSessionRecord as parseStoredRecord, + readVoiceSessionRecord as readRecord, + readVoiceSessionRecordInTransaction as readRecordInTransaction, + VOICE_SESSION_CACHE_SCOPE as CACHE_SCOPE, + VOICE_SESSION_MAX_TRANSCRIPT_CHARS as MAX_TRANSCRIPT_CHARS, + VOICE_SESSION_RECORD_VERSION as RECORD_VERSION, + VOICE_SESSION_STALE_AFTER_MS as STALE_AFTER_MS, + writeVoiceSessionRecordInTransaction as writeRecordInTransaction, +} from "./client-voice-session-store.js"; + +const voiceSessionByRunId = new Map(); +const voiceSessionOperations = new Map>(); +const deferredDigestConfigs = new Map(); +let unsubscribeToolEffects: (() => void) | undefined; +let unsubscribeRunCompletion: (() => void) | undefined; + +function hasLiveConsultRun(record: ClientVoiceSessionRecord): boolean { + return record.consultRunIds.some((runId) => { + const binding = voiceSessionByRunId.get(runId); + return ( + binding?.agentId === record.agentId && + binding.voiceSessionId === record.voiceSessionId && + binding.sessionKey === record.sessionKey + ); + }); +} + +async function runVoiceSessionOperation( + agentId: string, + voiceSessionId: string, + operation: () => Promise, +): Promise { + const key = operationKey(agentId, voiceSessionId); + const previous = voiceSessionOperations.get(key) ?? Promise.resolve(); + const current = previous.then(operation, operation); + const settled = current.then( + () => undefined, + () => undefined, + ); + voiceSessionOperations.set(key, settled); + void settled.then(() => { + if (voiceSessionOperations.get(key) === settled) { + voiceSessionOperations.delete(key); + } + }); + return await current; +} + +function effectStatus(event: TrustedToolExecutionEvent): ClientVoiceToolEffect["status"] { + if (event.type === "tool.execution.started") { + return "started"; + } + if (event.type === "tool.execution.completed") { + return "succeeded"; + } + if (event.type === "tool.execution.blocked") { + return "blocked"; + } + return event.terminalReason === "cancelled" ? "cancelled" : "failed"; +} + +function recordClientVoiceToolEffect(event: TrustedToolExecutionEvent): void { + const runId = event.runId; + if (!runId) { + return; + } + const binding = voiceSessionByRunId.get(runId); + if (!binding) { + return; + } + runOpenClawAgentWriteTransaction( + (database) => { + const record = readRecordInTransaction(database, binding.voiceSessionId); + if (!record) { + return; + } + const existing = event.toolCallId + ? record.effects.find( + (effect) => effect.runId === runId && effect.toolCallId === event.toolCallId, + ) + : record.effects.findLast( + (effect) => + effect.runId === runId && + effect.toolName === event.toolName && + effect.status === "started", + ); + if (event.type !== "tool.execution.started" && !existing) { + return; + } + if (event.type !== "tool.execution.started" && existing) { + existing.status = effectStatus(event); + existing.finishedAt = event.ts; + } else if (event.mutatingAction === true && (!event.toolCallId || !existing)) { + record.effects.push({ + runId, + ...(event.toolCallId ? { toolCallId: event.toolCallId } : {}), + toolName: event.toolName, + startedAt: event.ts, + status: "started", + }); + } + record.updatedAt = Date.now(); + writeRecordInTransaction(database, record); + }, + { agentId: binding.agentId }, + ); +} + +function ensureToolEffectSubscription(): void { + unsubscribeToolEffects ??= onTrustedToolExecutionEvent(recordClientVoiceToolEffect); + unsubscribeRunCompletion ??= onTrustedInternalDiagnosticEvent((event) => { + if (event.type !== "run.completed") { + return; + } + const binding = voiceSessionByRunId.get(event.runId); + if (!binding) { + return; + } + voiceSessionByRunId.delete(event.runId); + releaseClientVoiceConfirmationRun(binding.agentId, binding.voiceSessionId, event.runId); + void finishDeferredMutationDigest(binding).catch((error: unknown) => { + console.warn( + `[talk] deferred voice mutation digest failed: ${error instanceof Error ? error.message : String(error)}`, + ); + }); + }); +} + +/** Create a call record or resume the same open call across transport restarts. */ +export function createOrResumeClientVoiceSession(params: { + agentId: string; + sessionKey: string; + provider?: string; + origin: "client" | "relay"; + transcriptCapable?: boolean; + voiceSessionId?: string; + now?: number; +}): string { + const voiceSessionId = params.voiceSessionId?.trim() || randomUUID(); + const provider = params.provider?.trim() || undefined; + const now = params.now ?? Date.now(); + runOpenClawAgentWriteTransaction( + (database) => { + const existing = readRecordInTransaction(database, voiceSessionId); + if (existing) { + assertOwnership(existing, params); + if (existing.origin !== params.origin) { + throw new Error("voice session origin does not match"); + } + if (existing.status !== "open") { + throw new Error("voice session is already closed"); + } + if (existing.provider && provider && existing.provider !== provider) { + throw new Error("voice session provider does not match"); + } + if (!existing.provider && provider) { + existing.provider = provider; + } + if (params.transcriptCapable === true) { + existing.transcriptCapable = true; + } + existing.updatedAt = now; + writeRecordInTransaction(database, existing); + return; + } + writeRecordInTransaction(database, { + version: RECORD_VERSION, + voiceSessionId, + agentId: params.agentId, + sessionKey: params.sessionKey, + ...(provider ? { provider } : {}), + origin: params.origin, + ...(params.transcriptCapable === true ? { transcriptCapable: true } : {}), + status: "open", + createdAt: now, + updatedAt: now, + consultRunIds: [], + effects: [], + }); + }, + { agentId: params.agentId }, + ); + return voiceSessionId; +} + +/** Ensure Talk has the same canonical agent-session row that chat turns append to. */ +export async function ensureClientVoiceAgentSessionEntry(params: { + agentId: string; + sessionKey: string; +}): Promise { + const existing = loadSessionEntry(params); + if (existing?.sessionId) { + return; + } + const created = await upsertSessionEntry(params, {}); + if (!created?.sessionId) { + throw new Error(`agent session could not be initialized (${params.sessionKey})`); + } +} + +/** Correlate a consult run with its open call for confirmation and mutation evidence. */ +export function registerClientVoiceConsultRun(params: { + agentId: string; + sessionKey: string; + voiceSessionId: string; + runId: string; + config?: OpenClawConfig; +}): void { + let recordClosed = false; + runOpenClawAgentWriteTransaction( + (database) => { + const record = readRecordInTransaction(database, params.voiceSessionId); + if (!record) { + throw new Error("voice session not found"); + } + assertOwnership(record, params); + recordClosed = record.status === "closed"; + // A close can race in while chat.send is still acking this run. The run has + // already started, so bind it anyway (even on a closed record) to keep effect + // capture; aborting here would drop a just-confirmed high-impact action. + if (!record.consultRunIds.includes(params.runId)) { + record.consultRunIds.push(params.runId); + record.updatedAt = Date.now(); + writeRecordInTransaction(database, record); + } + }, + { agentId: params.agentId }, + ); + voiceSessionByRunId.set(params.runId, { + agentId: params.agentId, + voiceSessionId: params.voiceSessionId, + sessionKey: params.sessionKey, + }); + // Bound to a call that already closed: its digest may have been delivered and its + // retry config cleared, so re-arm it to cover this late run's mutations. + if (recordClosed && params.config) { + deferredDigestConfigs.set(operationKey(params.agentId, params.voiceSessionId), params.config); + } + ensureToolEffectSubscription(); +} + +/** Return the open voice-call binding for one executing run. */ +export function resolveClientVoiceRunBinding(runId?: string): ClientVoiceRunBinding | undefined { + return runId ? voiceSessionByRunId.get(runId) : undefined; +} + +/** + * Confirmation applies only when the session can observe spoken approvals: + * relay sessions (server hears utterances) or clients that report transcripts. + * Legacy clients without transcript reporting keep pre-gate behavior. + */ +export function isClientVoiceSessionConfirmable(binding: ClientVoiceRunBinding): boolean { + const record = readRecord(binding.agentId, binding.voiceSessionId); + return ( + record?.origin === "relay" || + record?.transcriptCapable === true || + record?.hasUserTranscript === true + ); +} + +/** Validate ownership and open state before starting a voice-bound consult. */ +export function assertClientVoiceSessionOpen(params: { + agentId: string; + sessionKey: string; + voiceSessionId: string; +}): "client" | "relay" { + const record = readRecord(params.agentId, params.voiceSessionId); + if (!record) { + throw new Error("voice session not found"); + } + assertOwnership(record, params); + if (record.status !== "open") { + throw new Error("voice session is closed"); + } + return record.origin; +} + +/** Validate durable ownership without rejecting an idempotent close retry. */ +export function resolveClientVoiceSessionOrigin(params: { + agentId: string; + sessionKey: string; + voiceSessionId: string; +}): "client" | "relay" { + const record = readRecord(params.agentId, params.voiceSessionId); + if (!record) { + throw new Error("voice session not found"); + } + assertOwnership(record, params); + return record.origin; +} + +/** Resolve the newest open client-owned call for legacy tool-call clients. */ +export function resolveOpenClientVoiceSessionId(params: { + agentId: string; + sessionKey: string; +}): string | undefined { + const database = openOpenClawAgentDatabase({ agentId: params.agentId }); + const rows = database.db + .prepare("SELECT value_json FROM cache_entries WHERE scope = ? ORDER BY updated_at DESC") + .all(CACHE_SCOPE) as Array<{ value_json?: unknown }>; + let match: string | undefined; + for (const row of rows) { + const record = parseStoredRecord(row.value_json); + if ( + record?.origin === "client" && + record.status === "open" && + record.agentId === params.agentId && + record.sessionKey === params.sessionKey + ) { + if (match) { + return undefined; + } + match = record.voiceSessionId; + } + } + return match; +} + +function buildPersistedVoiceMessage(params: { + role: "user" | "assistant"; + text: string; + timestamp: number; + provider: string; +}): Record { + const provenance = { kind: "realtime_voice", sourceChannel: "talk" }; + if (params.role === "user") { + return { + role: "user", + content: [{ type: "text", text: params.text }], + timestamp: params.timestamp, + provenance, + }; + } + return { + role: "assistant", + content: [{ type: "text", text: params.text }], + api: "realtime", + provider: params.provider, + model: "realtime-voice", + stopReason: "stop", + timestamp: params.timestamp, + provenance, + }; +} + +async function appendVoiceTranscript(params: { + agentId: string; + sessionKey: string; + voiceSessionId: string; + origin: "client" | "relay"; + entryId: string; + role: "user" | "assistant"; + text: string; + timestamp?: number; + config?: OpenClawConfig; +}): Promise { + await runVoiceSessionOperation(params.agentId, params.voiceSessionId, async () => { + const text = truncateUtf16Safe(params.text.trim(), MAX_TRANSCRIPT_CHARS); + if (!text) { + return; + } + const record = readRecord(params.agentId, params.voiceSessionId); + if (!record) { + throw new Error("voice session not found"); + } + assertOwnership(record, params); + if (record.status !== "open") { + throw new Error("voice session is closed"); + } + if (record.origin !== params.origin) { + throw new Error("voice session origin does not allow this transcript source"); + } + const sessionEntry = loadSessionEntry({ + agentId: params.agentId, + sessionKey: params.sessionKey, + }); + if (!sessionEntry?.sessionId) { + throw new Error(`agent session not found (${params.sessionKey})`); + } + const observedAt = Date.now(); + const timestamp = params.timestamp ?? observedAt; + await appendTranscriptMessage( + { + agentId: params.agentId, + sessionId: sessionEntry.sessionId, + sessionKey: params.sessionKey, + }, + { + ...(params.config ? { config: params.config } : {}), + eventId: `voice:${params.voiceSessionId}:${params.entryId}`, + message: buildPersistedVoiceMessage({ + role: params.role, + text, + timestamp, + provider: record.provider ?? "realtime", + }), + now: timestamp, + }, + ); + runOpenClawAgentWriteTransaction( + (database) => { + const current = readRecordInTransaction(database, params.voiceSessionId); + if (!current) { + throw new Error("voice session disappeared during transcript append"); + } + assertOwnership(current, params); + // Reaching here means this exact eventId is durably persisted (fresh append or + // idempotent dedup of our own prior write). Arm confirmation bookkeeping in both + // cases so a retry after a partial failure still records the user utterance. + if (params.role === "user") { + current.hasUserTranscript = true; + } + current.updatedAt = Date.now(); + writeRecordInTransaction(database, current); + }, + { agentId: params.agentId }, + ); + if (params.role === "user") { + noteClientVoiceConfirmationUtterance({ + agentId: params.agentId, + voiceSessionId: params.voiceSessionId, + text, + timestamp: observedAt, + }); + } + }); +} + +/** Append one finalized client-owned transcript item idempotently. */ +export async function appendClientVoiceTranscript( + params: Omit[0], "origin">, +): Promise { + await appendVoiceTranscript({ ...params, origin: "client" }); +} + +/** Append one finalized relay-owned transcript item idempotently. */ +export async function appendRelayVoiceTranscript( + params: Omit[0], "origin">, +): Promise { + await appendVoiceTranscript({ ...params, origin: "relay" }); +} + +// The mutation digest is a best-effort informational summary, not a durable record: +// the authoritative effects live on the voice record and the canonical transcript is +// persisted independently. Accepted tradeoffs (retry state is process-local like the +// rest of this subsystem): a digest already delivered at close does not reissue for a +// run that binds afterward, and an undelivered digest whose retry map is cleared by a +// gateway restart is not rediscovered. Both lose a summary message, never real data. +function formatMutationDigest(effects: ClientVoiceToolEffect[]): string | undefined { + if (effects.length === 0) { + return undefined; + } + return [ + "Voice call changes", + ...effects + .slice(0, 12) + .map( + (effect) => + `- ${effect.toolName}: ${effect.status === "started" ? "outcome not confirmed" : effect.status}`, + ), + ].join("\n"); +} + +async function deliverMutationDigest( + record: ClientVoiceSessionRecord, + config: OpenClawConfig, + target: { channel: string; to: string; accountId?: string; threadId?: string | number | null }, + text: string, +): Promise { + const { sendDurableMessageBatch } = await import("../channels/message/runtime.js"); + const send = await sendDurableMessageBatch({ + cfg: config, + channel: target.channel, + to: target.to, + ...(target.accountId ? { accountId: target.accountId } : {}), + ...(target.threadId != null ? { threadId: target.threadId } : {}), + payloads: [{ text }], + durability: "required", + requireUnknownSendReconciliation: true, + session: buildOutboundSessionContext({ + cfg: config, + agentId: record.agentId, + sessionKey: record.sessionKey, + policySessionKey: record.sessionKey, + }), + }); + if (send.status === "failed" || send.status === "partial_failed") { + throw send.error; + } +} + +async function deliverMutationDigestOnce( + record: ClientVoiceSessionRecord, + config: OpenClawConfig, +): Promise { + if (record.digestDeliveredAt) { + return; + } + const text = formatMutationDigest(record.effects); + if (!text) { + return; + } + const entry = loadSessionEntry({ agentId: record.agentId, sessionKey: record.sessionKey }); + const target = resolveSessionDeliveryTarget({ entry, requestedChannel: "last" }); + if (!target.channel || target.channel === "webchat" || !target.to) { + return; + } + // Send first, mark after: a transient send failure (the common case) then leaves + // the marker unset so close/stale-sweep retries. The rare crash between a durable + // send and this marker can re-send one informational digest, which is acceptable. + await deliverMutationDigest( + record, + config, + { + channel: target.channel, + to: target.to, + accountId: target.accountId, + threadId: target.threadId, + }, + text, + ); + const deliveredAt = Date.now(); + runOpenClawAgentWriteTransaction( + (database) => { + const current = readRecordInTransaction(database, record.voiceSessionId); + if (!current || current.digestDeliveredAt) { + return; + } + current.digestDeliveredAt = deliveredAt; + current.updatedAt = deliveredAt; + writeRecordInTransaction(database, current); + }, + { agentId: record.agentId }, + ); +} + +async function finishDeferredMutationDigest(binding: { + agentId: string; + voiceSessionId: string; +}): Promise { + const key = operationKey(binding.agentId, binding.voiceSessionId); + const config = deferredDigestConfigs.get(key); + if (!config) { + return; + } + await runVoiceSessionOperation(binding.agentId, binding.voiceSessionId, async () => { + const record = readRecord(binding.agentId, binding.voiceSessionId); + if (!record || record.status !== "closed" || hasLiveConsultRun(record)) { + return; + } + // Deliver first; only drop the retry config once the send succeeds, so a + // transient failure here is retried by the next closeStaleClientVoiceSessions. + await deliverMutationDigestOnce(record, config); + deferredDigestConfigs.delete(key); + }); +} + +/** Retry deferred digests whose delivery previously failed after the call closed. */ +async function retryDeferredMutationDigests(agentId: string): Promise { + for (const key of Array.from(deferredDigestConfigs.keys())) { + const [entryAgentId, voiceSessionId] = key.split("\0"); + if (entryAgentId !== agentId || !voiceSessionId) { + continue; + } + try { + await finishDeferredMutationDigest({ agentId, voiceSessionId }); + } catch { + // Still undeliverable; a later voice session retries again. + } + } +} + +async function closeClientVoiceSessionInternal(params: { + agentId: string; + sessionKey: string; + voiceSessionId: string; + config: OpenClawConfig; + now?: number; +}): Promise { + const existing = readRecord(params.agentId, params.voiceSessionId); + if (!existing) { + throw new Error("voice session not found"); + } + assertOwnership(existing, params); + const now = params.now ?? Date.now(); + runOpenClawAgentWriteTransaction( + (database) => { + const current = readRecordInTransaction(database, params.voiceSessionId); + if (!current) { + throw new Error("voice session disappeared during close"); + } + assertOwnership(current, params); + if (current.status === "open") { + current.status = "closed"; + current.closedAt = now; + current.updatedAt = now; + writeRecordInTransaction(database, current); + } + }, + { agentId: params.agentId }, + ); + const closed = readRecord(params.agentId, params.voiceSessionId); + if (!closed) { + throw new Error("voice session disappeared after close"); + } + // Transport close does not end consult runs: live bindings keep effect capture active, + // approved grants stay valid for those runs, and the digest waits for the last run.completed. + const liveRunIds = closed.consultRunIds.filter((runId) => { + const binding = voiceSessionByRunId.get(runId); + return binding?.voiceSessionId === params.voiceSessionId && binding.agentId === params.agentId; + }); + deactivateClientVoiceConfirmationSession(params.agentId, params.voiceSessionId, liveRunIds); + const key = operationKey(params.agentId, params.voiceSessionId); + // Both the deferred and immediate paths retain the retry config until a send + // actually succeeds, so a channel outage does not permanently lose the digest. + deferredDigestConfigs.set(key, params.config); + if (hasLiveConsultRun(closed)) { + return; + } + await deliverMutationDigestOnce(closed, params.config); + // Intentionally unconditional: a consult that late-binds during the await above has + // its mutations omitted from this already-sent point-in-time summary. Re-sending to + // include them would double-notify the user for one call, which is a worse outcome + // for an informational digest than omitting a straggler action (still on the record). + deferredDigestConfigs.delete(key); +} + +/** Close a logical voice call and deliver its mutation digest at most once. */ +export async function closeClientVoiceSession(params: { + agentId: string; + sessionKey: string; + voiceSessionId: string; + config: OpenClawConfig; + now?: number; +}): Promise { + await runVoiceSessionOperation(params.agentId, params.voiceSessionId, async () => { + await closeClientVoiceSessionInternal(params); + }); +} + +/** Close abandoned open calls idle for the fixed six-hour recovery window. */ +export async function closeStaleClientVoiceSessions(params: { + agentId: string; + config: OpenClawConfig; + excludeVoiceSessionId?: string; + now?: number; + warn?: (message: string) => void; +}): Promise { + const now = params.now ?? Date.now(); + // A new voice session is the natural retry point for a digest whose delivery + // failed after its own call had already closed and its runs completed. + await retryDeferredMutationDigests(params.agentId); + const database = openOpenClawAgentDatabase({ agentId: params.agentId }); + const rows = database.db + .prepare("SELECT value_json FROM cache_entries WHERE scope = ? AND updated_at <= ?") + .all(CACHE_SCOPE, now - STALE_AFTER_MS) as Array<{ value_json?: unknown }>; + const stale = rows.flatMap((row) => { + const record = parseStoredRecord(row.value_json); + return record && + record.status === "open" && + record.voiceSessionId !== params.excludeVoiceSessionId + ? [record] + : []; + }); + let closed = 0; + for (const record of stale) { + try { + await closeClientVoiceSession({ + agentId: params.agentId, + sessionKey: record.sessionKey, + voiceSessionId: record.voiceSessionId, + config: params.config, + now, + }); + closed += 1; + } catch (error) { + params.warn?.( + `failed to close stale voice session ${record.voiceSessionId}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } + return closed; +} + +const clientVoiceSessionTesting = { + readRecord, + reset(): void { + voiceSessionByRunId.clear(); + voiceSessionOperations.clear(); + deferredDigestConfigs.clear(); + unsubscribeToolEffects?.(); + unsubscribeToolEffects = undefined; + unsubscribeRunCompletion?.(); + unsubscribeRunCompletion = undefined; + }, +}; + +if (process.env.VITEST || process.env.NODE_ENV === "test") { + (globalThis as Record)[Symbol.for("openclaw.clientVoiceSessionTestApi")] = + clientVoiceSessionTesting; +} diff --git a/test/vitest/vitest.gateway-client.config.ts b/test/vitest/vitest.gateway-client.config.ts index d031313a262..e8d871b4478 100644 --- a/test/vitest/vitest.gateway-client.config.ts +++ b/test/vitest/vitest.gateway-client.config.ts @@ -13,7 +13,7 @@ export function createGatewayClientVitestConfig(env?: Record { it("submits realtime consults through the Gateway tool-call endpoint", async () => { + const order: string[] = []; let listener: ((event: { event: string; payload?: unknown }) => void) | undefined; const request = vi.fn(async (method: string, _params: unknown) => { if (method === "talk.client.toolCall") { + order.push("tool-call"); setImmediate(() => { listener?.({ event: "chat", @@ -40,11 +42,16 @@ describe("RealtimeTalkSession consult handoff", () => { }; }); const submit = vi.fn(); + const flushTranscriptWrites = vi.fn(async () => { + order.push("flush"); + }); await submitRealtimeTalkConsult({ ctx: { client: { request, addEventListener }, sessionKey: "agent:main:main", + voiceSessionId: "voice-1", + flushTranscriptWrites, callbacks: {}, } as never, callId: "call-1", @@ -53,13 +60,23 @@ describe("RealtimeTalkSession consult handoff", () => { }); const toolCall = requireFirstMockCall(request.mock.calls, "Gateway request") as - | [string, { sessionKey?: string; name?: string; args?: { question?: string } }] + | [ + string, + { + sessionKey?: string; + voiceSessionId?: string; + name?: string; + args?: { question?: string }; + }, + ] | undefined; expect(toolCall?.[0]).toBe("talk.client.toolCall"); expect(toolCall?.[1]?.sessionKey).toBe("agent:main:main"); + expect(toolCall?.[1]).toMatchObject({ voiceSessionId: "voice-1" }); expect(toolCall?.[1]?.name).toBe("openclaw_agent_consult"); expect(toolCall?.[1]?.args).toEqual({ question: "Are the basement lights off?" }); expect(submit).toHaveBeenCalledWith("call-1", { result: "Basement lights are off." }); + expect(order).toEqual(["flush", "tool-call"]); }); it("prefers source-reply final text over an earlier empty Talk consult final", async () => { diff --git a/ui/src/pages/chat/realtime-talk-shared.ts b/ui/src/pages/chat/realtime-talk-shared.ts index 05a91419699..8c8a7d70755 100644 --- a/ui/src/pages/chat/realtime-talk-shared.ts +++ b/ui/src/pages/chat/realtime-talk-shared.ts @@ -45,6 +45,7 @@ type RealtimeTalkAudioContract = { export type RealtimeTalkWebRtcSdpSessionResult = { provider: string; transport: "webrtc"; + voiceSessionId?: string; clientSecret: string; offerUrl?: string; offerHeaders?: Record; @@ -58,6 +59,7 @@ export type RealtimeTalkWebRtcSdpSessionResult = { export type RealtimeTalkJsonPcmWebSocketSessionResult = { provider: string; transport: "provider-websocket"; + voiceSessionId?: string; protocol: string; clientSecret: string; websocketUrl: string; @@ -73,6 +75,7 @@ export type RealtimeTalkJsonPcmWebSocketSessionResult = { export type RealtimeTalkGatewayRelaySessionResult = { provider: string; transport: "gateway-relay"; + voiceSessionId?: string; relaySessionId: string; audio: RealtimeTalkAudioContract; model?: string; @@ -85,6 +88,7 @@ export type RealtimeTalkGatewayRelaySessionResult = { type RealtimeTalkManagedRoomSessionResult = { provider: string; transport: "managed-room"; + voiceSessionId?: string; roomUrl: string; token?: string; model?: string; @@ -110,6 +114,8 @@ export type RealtimeTalkTransport = { export type RealtimeTalkTransportContext = { client: GatewayBrowserClient; sessionKey: string; + voiceSessionId?: string; + flushTranscriptWrites?: () => Promise; callbacks: RealtimeTalkCallbacks; inputDeviceId?: string; videoDeviceId?: string; @@ -572,10 +578,12 @@ export async function submitRealtimeTalkConsult(params: { try { const args = typeof params.args === "string" ? JSON.parse(params.args || "{}") : (params.args ?? {}); + await ctx.flushTranscriptWrites?.(); const response = await ctx.client.request<{ runId?: string; idempotencyKey?: string }>( "talk.client.toolCall", { sessionKey: ctx.sessionKey, + ...(ctx.voiceSessionId ? { voiceSessionId: ctx.voiceSessionId } : {}), callId, name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME, args, diff --git a/ui/src/pages/chat/realtime-talk.test.ts b/ui/src/pages/chat/realtime-talk.test.ts index 597c62aa5a5..58cc79a5cb5 100644 --- a/ui/src/pages/chat/realtime-talk.test.ts +++ b/ui/src/pages/chat/realtime-talk.test.ts @@ -26,7 +26,7 @@ const { webRtcSetVideoEnabled: vi.fn(async () => undefined), googleSwitchCamera: vi.fn(async () => undefined), webRtcSwitchCamera: vi.fn(async () => undefined), - googleCtor: vi.fn(function () { + googleCtor: vi.fn(function (_session: unknown, _ctx: unknown) { return { start: googleStart, stop: googleStop, @@ -34,10 +34,10 @@ const { switchCamera: googleSwitchCamera, }; }), - relayCtor: vi.fn(function () { + relayCtor: vi.fn(function (_session: unknown, _ctx: unknown) { return { start: relayStart, stop: relayStop }; }), - webRtcCtor: vi.fn(function () { + webRtcCtor: vi.fn(function (_session: unknown, _ctx: unknown) { return { start: webRtcStart, stop: webRtcStop, @@ -81,6 +81,7 @@ describe("RealtimeTalkSession", () => { it("starts the Google Live WebSocket transport from a generic session result", async () => { const request = vi.fn(async () => ({ provider: "google", + voiceSessionId: "voice-1", transport: "provider-websocket", protocol: "google-live-bidi", clientSecret: "auth_tokens/session", @@ -97,7 +98,10 @@ describe("RealtimeTalkSession", () => { await session.start(); - expect(request).toHaveBeenCalledWith("talk.client.create", { sessionKey: "main" }); + expect(request).toHaveBeenCalledWith("talk.client.create", { + sessionKey: "main", + capabilities: ["voice-transcript"], + }); expect(googleCtor).toHaveBeenCalledTimes(1); expect(googleStart).toHaveBeenCalledTimes(1); expect(webRtcCtor).not.toHaveBeenCalled(); @@ -108,6 +112,7 @@ describe("RealtimeTalkSession", () => { it("defaults legacy session results without an explicit transport to WebRTC", async () => { const request = vi.fn(async () => ({ provider: "openai", + voiceSessionId: "voice-1", clientSecret: "auth_tokens/session", })); const session = new RealtimeTalkSession({ request } as never, "main"); @@ -122,6 +127,7 @@ describe("RealtimeTalkSession", () => { it("accepts legacy WebRTC transport names", async () => { const request = vi.fn(async () => ({ provider: "openai", + voiceSessionId: "voice-1", transport: "webrtc-sdp", clientSecret: "secret", })); @@ -136,6 +142,7 @@ describe("RealtimeTalkSession", () => { it("accepts legacy provider WebSocket transport names", async () => { const request = vi.fn(async () => ({ provider: "example", + voiceSessionId: "voice-1", transport: "json-pcm-websocket", clientSecret: "secret", protocol: "google-live-bidi", @@ -209,6 +216,7 @@ describe("RealtimeTalkSession", () => { sessionKey: "main", provider: "xai", transport: "gateway-relay", + capabilities: ["voice-transcript"], }); expect(request).toHaveBeenNthCalledWith(2, "talk.session.create", { sessionKey: "main", @@ -263,7 +271,7 @@ describe("RealtimeTalkSession", () => { sessionKey: "main", provider: "openai", transport: "gateway-relay", - capabilities: ["camera-frame"], + capabilities: ["voice-transcript", "camera-frame"], }); expect(request).toHaveBeenNthCalledWith(3, "talk.session.create", { sessionKey: "main", @@ -280,6 +288,7 @@ describe("RealtimeTalkSession", () => { it("starts the WebRTC transport for canonical WebRTC sessions", async () => { const request = vi.fn(async () => ({ provider: "openai", + voiceSessionId: "voice-1", transport: "webrtc", clientSecret: "secret", })); @@ -298,6 +307,7 @@ describe("RealtimeTalkSession", () => { it("passes launch options to client-owned realtime session creation", async () => { const request = vi.fn(async () => ({ provider: "openai", + voiceSessionId: "voice-1", transport: "webrtc", clientSecret: "secret", })); @@ -330,6 +340,7 @@ describe("RealtimeTalkSession", () => { silenceDurationMs: 650, prefixPaddingMs: 250, reasoningEffort: "low", + capabilities: ["voice-transcript"], }); expect(webRtcCtor).toHaveBeenCalledWith( expect.any(Object), @@ -349,6 +360,7 @@ describe("RealtimeTalkSession", () => { } return { provider: "openai", + voiceSessionId: "voice-1", transport: "webrtc", clientSecret: "secret", }; @@ -363,7 +375,7 @@ describe("RealtimeTalkSession", () => { expect(request).toHaveBeenNthCalledWith(1, "talk.catalog", {}); expect(request).toHaveBeenNthCalledWith(2, "talk.client.create", { sessionKey: "main", - capabilities: ["camera-frame"], + capabilities: ["voice-transcript", "camera-frame"], }); expect(onVideoCapability).toHaveBeenCalledWith(true); expect(webRtcCtor).toHaveBeenCalledWith( @@ -392,6 +404,7 @@ describe("RealtimeTalkSession", () => { return { provider: "openai", transport: "webrtc", + voiceSessionId: "voice-settings-camera", clientSecret: "secret", }; }); @@ -424,7 +437,12 @@ describe("RealtimeTalkSession", () => { }, }; } - return { provider: "openai", transport: "webrtc", clientSecret: "secret" }; + return { + provider: "openai", + transport: "webrtc", + voiceSessionId: "voice-pending-camera", + clientSecret: "secret", + }; }); const session = new RealtimeTalkSession({ request } as never, "main", { onVideoCapability: vi.fn(), @@ -454,6 +472,7 @@ describe("RealtimeTalkSession", () => { } return { provider: "openai", + voiceSessionId: "voice-1", transport: "webrtc", clientSecret: "secret", }; @@ -465,7 +484,10 @@ describe("RealtimeTalkSession", () => { await session.start(); - expect(request).toHaveBeenNthCalledWith(2, "talk.client.create", { sessionKey: "main" }); + expect(request).toHaveBeenNthCalledWith(2, "talk.client.create", { + sessionKey: "main", + capabilities: ["voice-transcript"], + }); expect(onVideoCapability).toHaveBeenCalledWith(false); }); @@ -491,7 +513,7 @@ describe("RealtimeTalkSession", () => { await expect(session.start()).rejects.toBe(clientError); expect(request.mock.calls).toEqual([ - ["talk.client.create", { sessionKey: "main" }], + ["talk.client.create", { sessionKey: "main", capabilities: ["voice-transcript"] }], ["talk.config", {}], ]); expect(relayCtor).not.toHaveBeenCalled(); @@ -592,7 +614,7 @@ describe("RealtimeTalkSession", () => { await expect(session.start()).rejects.toBe(clientError); expect(request.mock.calls).toEqual([ - ["talk.client.create", { sessionKey: "main" }], + ["talk.client.create", { sessionKey: "main", capabilities: ["voice-transcript"] }], ["talk.config", {}], ]); expect(relayCtor).not.toHaveBeenCalled(); @@ -614,9 +636,338 @@ describe("RealtimeTalkSession", () => { await expect(session.start()).rejects.toBe(clientError); expect(request.mock.calls).toEqual([ - ["talk.client.create", { sessionKey: "main" }], + ["talk.client.create", { sessionKey: "main", capabilities: ["voice-transcript"] }], ["talk.config", {}], ]); expect(relayCtor).not.toHaveBeenCalled(); }); + + it("retries finalized transcript writes in order", async () => { + vi.useFakeTimers(); + try { + const transcriptEntryIds: string[] = []; + let firstAttempt = true; + const request = vi.fn(async (method: string, params?: { entryId?: string }) => { + if (method === "talk.client.create") { + return { + provider: "openai", + transport: "webrtc", + voiceSessionId: "voice-queue", + clientSecret: "secret", + }; + } + if (method === "talk.client.transcript") { + transcriptEntryIds.push(String(params?.entryId)); + if (params?.entryId === "1" && firstAttempt) { + firstAttempt = false; + throw new Error("temporary failure"); + } + return { ok: true }; + } + return { ok: true }; + }); + const session = new RealtimeTalkSession({ request } as never, "agent:main:main"); + await session.start(); + const context = webRtcCtor.mock.calls[0]?.[1] as { + callbacks: { + onTranscript?: (entry: { + role: "user" | "assistant"; + text: string; + final: boolean; + }) => void; + }; + }; + context.callbacks.onTranscript?.({ role: "user", text: "first", final: true }); + context.callbacks.onTranscript?.({ role: "assistant", text: "second", final: true }); + + await vi.advanceTimersByTimeAsync(500); + await vi.waitFor(() => expect(transcriptEntryIds).toEqual(["1", "1", "2"])); + session.stop(); + await vi.runAllTimersAsync(); + } finally { + vi.useRealTimers(); + } + }); + + it("continues entry ids when the same voice session replaces its transport", async () => { + const transcriptEntryIds: string[] = []; + const request = vi.fn(async (method: string, params?: { entryId?: string }) => { + if (method === "talk.client.create") { + return { + provider: "openai", + transport: "webrtc", + voiceSessionId: "voice-resume", + clientSecret: "secret", + }; + } + if (method === "talk.client.transcript") { + transcriptEntryIds.push(String(params?.entryId)); + } + return { ok: true }; + }); + const session = new RealtimeTalkSession({ request } as never, "agent:main:main", {}); + + await session.start(); + const firstContext = webRtcCtor.mock.calls[0]?.[1] as { + callbacks: { + onTranscript?: (entry: { + role: "user" | "assistant"; + text: string; + final: boolean; + }) => void; + }; + flushTranscriptWrites?: () => Promise; + }; + firstContext.callbacks.onTranscript?.({ role: "user", text: "first", final: true }); + await firstContext.flushTranscriptWrites?.(); + + await session.start(); + const secondContext = webRtcCtor.mock.calls[1]?.[1] as typeof firstContext; + secondContext.callbacks.onTranscript?.({ role: "assistant", text: "second", final: true }); + await secondContext.flushTranscriptWrites?.(); + + expect(transcriptEntryIds).toEqual(["1", "2"]); + expect(request.mock.calls.filter(([method]) => method === "talk.client.create")).toEqual([ + ["talk.client.create", { sessionKey: "agent:main:main", capabilities: ["voice-transcript"] }], + [ + "talk.client.create", + { + sessionKey: "agent:main:main", + voiceSessionId: "voice-resume", + capabilities: ["voice-transcript"], + }, + ], + ]); + session.stop(); + await Promise.resolve(); + }); + + it("surfaces transcript failure after three attempts", async () => { + vi.useFakeTimers(); + const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined); + try { + const request = vi.fn(async (method: string) => { + if (method === "talk.client.create") { + return { + provider: "openai", + transport: "webrtc", + voiceSessionId: "voice-failure", + clientSecret: "secret", + }; + } + if (method === "talk.client.transcript") { + throw new Error("still unavailable"); + } + return { ok: true }; + }); + const onStatus = vi.fn(); + const session = new RealtimeTalkSession({ request } as never, "agent:main:main", { + onStatus, + }); + await session.start(); + const context = webRtcCtor.mock.calls[0]?.[1] as { + callbacks: { + onTranscript?: (entry: { + role: "user" | "assistant"; + text: string; + final: boolean; + }) => void; + }; + }; + context.callbacks.onTranscript?.({ role: "user", text: "save me", final: true }); + + await vi.advanceTimersByTimeAsync(2_500); + await vi.waitFor(() => + expect(onStatus).toHaveBeenCalledWith( + "error", + expect.stringContaining("Voice transcript could not be saved"), + ), + ); + expect( + request.mock.calls.filter(([method]) => method === "talk.client.transcript"), + ).toHaveLength(3); + expect(warn).toHaveBeenCalled(); + session.stop(); + await vi.runAllTimersAsync(); + } finally { + warn.mockRestore(); + vi.useRealTimers(); + } + }); + + it("retries logical voice-session close after transient failures", async () => { + vi.useFakeTimers(); + try { + let closeAttempts = 0; + const request = vi.fn(async (method: string) => { + if (method === "talk.client.create") { + return { + provider: "openai", + transport: "webrtc", + voiceSessionId: "voice-close-retry", + clientSecret: "secret", + }; + } + if (method === "talk.client.close" && ++closeAttempts < 3) { + throw new Error("temporary close failure"); + } + return { ok: true }; + }); + const session = new RealtimeTalkSession({ request } as never, "agent:main:main"); + await session.start(); + + session.stop(); + await vi.runAllTimersAsync(); + + expect(closeAttempts).toBe(3); + } finally { + vi.useRealTimers(); + } + }); + + it("starts a new call without resuming the voice session being closed", async () => { + let createCount = 0; + let finishClose: (() => void) | undefined; + const closing = new Promise((resolve) => { + finishClose = resolve; + }); + const request = vi.fn(async (method: string) => { + if (method === "talk.client.create") { + createCount += 1; + return { + provider: "openai", + transport: "webrtc", + voiceSessionId: `voice-${createCount}`, + clientSecret: "secret", + }; + } + if (method === "talk.client.close") { + await closing; + } + return { ok: true }; + }); + const session = new RealtimeTalkSession({ request } as never, "agent:main:main"); + await session.start(); + + session.stop(); + await session.start(); + + const creates = request.mock.calls.filter(([method]) => method === "talk.client.create"); + expect(creates).toEqual([ + ["talk.client.create", { sessionKey: "agent:main:main", capabilities: ["voice-transcript"] }], + ["talk.client.create", { sessionKey: "agent:main:main", capabilities: ["voice-transcript"] }], + ]); + finishClose?.(); + await Promise.resolve(); + }); + + it("ignores final transcript callbacks emitted after shutdown begins", async () => { + const request = vi.fn(async (method: string) => { + if (method === "talk.client.create") { + return { + provider: "openai", + transport: "webrtc", + voiceSessionId: "voice-shutdown", + clientSecret: "secret", + }; + } + return { ok: true }; + }); + const session = new RealtimeTalkSession({ request } as never, "agent:main:main"); + await session.start(); + const context = webRtcCtor.mock.calls[0]?.[1] as { + callbacks: { + onTranscript?: (entry: { + role: "user" | "assistant"; + text: string; + final: boolean; + }) => void; + }; + }; + + session.stop(); + context.callbacks.onTranscript?.({ role: "user", text: "too late", final: true }); + await Promise.resolve(); + + expect(request.mock.calls.some(([method]) => method === "talk.client.transcript")).toBe(false); + }); + + it("drops a previous transport's delayed transcript after stop and restart", async () => { + let createCount = 0; + const request = vi.fn(async (method: string) => { + if (method === "talk.client.create") { + createCount += 1; + return { + provider: "openai", + transport: "webrtc", + voiceSessionId: `voice-${createCount}`, + clientSecret: "secret", + }; + } + return { ok: true }; + }); + const onTranscript = vi.fn(); + const session = new RealtimeTalkSession({ request } as never, "agent:main:main", { + onTranscript, + }); + await session.start(); + const previousContext = webRtcCtor.mock.calls[0]?.[1] as { + callbacks: { + onTranscript?: (entry: { + role: "user" | "assistant"; + text: string; + final: boolean; + }) => void; + }; + }; + + session.stop(); + await session.start(); + previousContext.callbacks.onTranscript?.({ + role: "user", + text: "stale transcript", + final: true, + }); + await Promise.resolve(); + + expect(onTranscript).not.toHaveBeenCalled(); + expect(request.mock.calls.some(([method]) => method === "talk.client.transcript")).toBe(false); + }); + + it("does not report Gateway relay transcripts through the client RPC", async () => { + const request = vi.fn(async (method: string) => { + if (method === "talk.client.create") { + return { + provider: "openai", + transport: "gateway-relay", + relaySessionId: "relay-voice", + audio: { + inputEncoding: "pcm16", + inputSampleRateHz: 24_000, + outputEncoding: "pcm16", + outputSampleRateHz: 24_000, + }, + }; + } + return { ok: true }; + }); + const session = new RealtimeTalkSession({ request } as never, "agent:main:main"); + await session.start(); + const context = relayCtor.mock.calls[0]?.[1] as { + callbacks: { + onTranscript?: (entry: { + role: "user" | "assistant"; + text: string; + final: boolean; + }) => void; + }; + }; + context.callbacks.onTranscript?.({ role: "user", text: "server owns this", final: true }); + await Promise.resolve(); + + expect(request.mock.calls.some(([method]) => method === "talk.client.transcript")).toBe(false); + session.stop(); + await Promise.resolve(); + expect(request.mock.calls.some(([method]) => method === "talk.client.close")).toBe(false); + }); }); diff --git a/ui/src/pages/chat/realtime-talk.ts b/ui/src/pages/chat/realtime-talk.ts index da44bff2c22..d4de594fa82 100644 --- a/ui/src/pages/chat/realtime-talk.ts +++ b/ui/src/pages/chat/realtime-talk.ts @@ -58,6 +58,13 @@ export async function switchActiveRealtimeTalkCameras( type RealtimeTalkLaunchTransport = NonNullable; +type DetachedVoiceSession = { + voiceSessionId: string; + serverOwned: boolean; + generation: number; + transcriptWrites: Promise; +}; + type RealtimeTalkConfigResult = { config?: { talk?: { @@ -104,9 +111,6 @@ function createTransport( ctx, ); } - if (transport === "managed-room") { - throw new Error("Managed-room realtime Talk sessions are not available in this UI yet"); - } const unknownTransport = (session as { transport?: string }).transport ?? "unknown"; throw new Error(`Unsupported realtime Talk transport: ${unknownTransport}`); } @@ -115,8 +119,17 @@ function resolveTransport(session: RealtimeTalkSessionResult): string { return normalizeTalkTransport((session as { transport?: string }).transport) ?? "webrtc"; } +function transcriptWriteError(error: unknown, fallback: string): Error { + return error instanceof Error ? error : new Error(fallback, { cause: error }); +} + function compactLaunchParams( - params: RealtimeTalkLaunchOptions & { sessionKey: string; mode?: string; brain?: string }, + params: RealtimeTalkLaunchOptions & { + sessionKey: string; + voiceSessionId?: string; + mode?: string; + brain?: string; + }, ): Record { return Object.fromEntries(Object.entries(params).filter(([, value]) => value !== undefined)); } @@ -126,6 +139,12 @@ export class RealtimeTalkSession { private closed = false; private videoEnabled = false; private videoOperation = 0; + private voiceSessionId: string | undefined; + private transportGeneration = 0; + private readonly transcriptSeqByVoiceSessionId = new Map(); + private acceptingTranscripts = false; + private serverOwnedVoiceSession = false; + private transcriptWrites: Promise = Promise.resolve(); constructor( private readonly client: GatewayBrowserClient, @@ -142,16 +161,48 @@ export class RealtimeTalkSession { if (this.closed) { return; } - const session = await this.createSession( - providerVideoCapable ? { ...this.options, capabilities: ["camera-frame"] } : this.options, - ); + // Declaring voice-transcript arms the server-side spoken-confirmation gate; + // this client reports every finalized utterance, so the gate is completable. + const capabilities: Array<"camera-frame" | "voice-transcript"> = ["voice-transcript"]; + if (providerVideoCapable) { + capabilities.push("camera-frame"); + } + const session = await this.createSession({ ...this.options, capabilities }); + const transport = resolveTransport(session); + // Managed-room stays unsupported here and carries no voice bookkeeping; + // reject it before the voice-session requirement produces a misleading error. + if (transport === "managed-room") { + throw new Error("Managed-room realtime Talk sessions are not available in this UI yet"); + } + const voiceSessionId = + session.voiceSessionId ?? + (transport === "gateway-relay" + ? (session as RealtimeTalkGatewayRelaySessionResult).relaySessionId + : undefined); + if (!voiceSessionId) { + throw new Error("Realtime Talk session did not return a voice session id"); + } + this.voiceSessionId = voiceSessionId; + this.acceptingTranscripts = true; + this.serverOwnedVoiceSession = transport === "gateway-relay"; if (this.closed) { + const detached = this.detachVoiceSession(); + if (detached) { + this.closeLogicalVoiceSession(detached); + } return; } + this.transportGeneration += 1; + const callbacks = + transport === "gateway-relay" + ? this.callbacks + : this.clientOwnedTranscriptCallbacks(voiceSessionId, this.transportGeneration); this.transport = createTransport(session, { client: this.client, sessionKey: this.sessionKey, - callbacks: this.callbacks, + voiceSessionId, + flushTranscriptWrites: async () => await this.transcriptWrites, + callbacks, inputDeviceId: this.localOptions.inputDeviceId, videoDeviceId: this.localOptions.videoDeviceId, consultThinkingLevel: session.consultThinkingLevel, @@ -185,18 +236,22 @@ export class RealtimeTalkSession { } private async createSession( - options: RealtimeTalkLaunchOptions & { capabilities?: Array<"camera-frame"> }, + options: RealtimeTalkLaunchOptions & { + capabilities?: Array<"camera-frame" | "voice-transcript">; + }, ): Promise { + const launchOptions = { ...options }; try { return await this.client.request( "talk.client.create", compactLaunchParams({ sessionKey: this.sessionKey, - ...options, + voiceSessionId: this.voiceSessionId, + ...launchOptions, }), ); } catch (error) { - let transport = options.transport; + let transport = launchOptions.transport; if (!transport) { let result: RealtimeTalkConfigResult; try { @@ -218,10 +273,10 @@ export class RealtimeTalkSession { if (transport && transport !== "gateway-relay") { throw error; } - const gatewayOptions = { ...options }; + const gatewayOptions = { ...launchOptions }; delete gatewayOptions.capabilities; try { - return await this.client.request( + const relaySession = await this.client.request( "talk.session.create", compactLaunchParams({ sessionKey: this.sessionKey, @@ -231,6 +286,13 @@ export class RealtimeTalkSession { brain: "agent-consult", }), ); + return resolveTransport(relaySession) === "gateway-relay" + ? { + ...relaySession, + voiceSessionId: (relaySession as RealtimeTalkGatewayRelaySessionResult) + .relaySessionId, + } + : relaySession; } catch { throw error; } @@ -243,8 +305,147 @@ export class RealtimeTalkSession { this.videoEnabled = false; activeRealtimeTalkSessions.delete(this); this.callbacks.onStatus?.("idle"); + const detached = this.detachVoiceSession(); this.transport?.stop(); this.transport = null; + if (detached) { + this.closeLogicalVoiceSession(detached); + } + } + + private clientOwnedTranscriptCallbacks( + owningVoiceSessionId: string, + owningGeneration: number, + ): RealtimeTalkCallbacks { + return { + ...this.callbacks, + onTranscript: (entry) => { + // Transport replacement can reuse the voice session id, so a retired + // transport's late finals are fenced by generation, not id alone. + if ( + this.transportGeneration !== owningGeneration || + this.voiceSessionId !== owningVoiceSessionId || + !this.acceptingTranscripts + ) { + return; + } + // Persist before notifying: a consumer callback that stops or throws must + // not be able to drop an already-finalized utterance from the write tail. + if (entry.final) { + const transcriptSeq = + (this.transcriptSeqByVoiceSessionId.get(owningVoiceSessionId) ?? 0) + 1; + this.transcriptSeqByVoiceSessionId.set(owningVoiceSessionId, transcriptSeq); + const entryId = String(transcriptSeq); + // One promise tail preserves transcript order and makes consult flushes + // observe the same dedupe sequence used by close. + const write = this.transcriptWrites.then(async () => { + await this.writeTranscriptWithRetry({ + voiceSessionId: owningVoiceSessionId, + entryId, + role: entry.role, + text: entry.text, + }); + }); + this.transcriptWrites = write.catch((error: unknown) => { + // The utterance exists only in client memory; after retries and surfacing the error, + // keeping the record open cannot recover it, while server entryId dedupe preserves order. + // Deferring close would only shift the identical loss to the 6h stale sweep. + const detail = `Voice transcript could not be saved: ${error instanceof Error ? error.message : String(error)}`; + console.warn(detail, error); + // Only surface to the user if this transport is still the active one; a + // retired call's late failure must not error a healthy replacement call. + if (this.transportGeneration === owningGeneration) { + this.callbacks.onStatus?.("error", detail); + } + }); + } + this.callbacks.onTranscript?.(entry); + }, + }; + } + + private async writeTranscriptWithRetry(params: { + voiceSessionId: string; + entryId: string; + role: "user" | "assistant"; + text: string; + }): Promise { + const retryDelaysMs = [0, 500, 2_000]; + let lastError: unknown; + for (const delayMs of retryDelaysMs) { + if (delayMs > 0) { + await new Promise((resolve) => { + setTimeout(resolve, delayMs); + }); + } + try { + await this.client.request("talk.client.transcript", { + sessionKey: this.sessionKey, + voiceSessionId: params.voiceSessionId, + entryId: params.entryId, + role: params.role, + text: params.text, + timestamp: Date.now(), + }); + return; + } catch (error) { + lastError = error; + } + } + throw transcriptWriteError(lastError, "voice transcript save failed"); + } + + private detachVoiceSession(): DetachedVoiceSession | undefined { + const voiceSessionId = this.voiceSessionId; + if (!voiceSessionId) { + return undefined; + } + const detached = { + voiceSessionId, + serverOwned: this.serverOwnedVoiceSession, + generation: this.transportGeneration, + transcriptWrites: this.transcriptWrites, + } satisfies DetachedVoiceSession; + this.voiceSessionId = undefined; + this.acceptingTranscripts = false; + this.serverOwnedVoiceSession = false; + this.transcriptWrites = Promise.resolve(); + return detached; + } + + private closeLogicalVoiceSession(detached: DetachedVoiceSession): void { + if (detached.serverOwned) { + return; + } + void detached.transcriptWrites + .then(async () => { + let lastError: unknown; + for (const delayMs of [0, 500, 2_000]) { + if (delayMs > 0) { + await new Promise((resolve) => { + setTimeout(resolve, delayMs); + }); + } + try { + await this.client.request("talk.client.close", { + sessionKey: this.sessionKey, + voiceSessionId: detached.voiceSessionId, + }); + return; + } catch (error) { + lastError = error; + } + } + throw transcriptWriteError(lastError, "Realtime Talk voice session close failed"); + }) + .catch((error: unknown) => { + console.warn("Realtime Talk voice session close failed", error); + // Suppress if a newer transport has started: closing the old call is its own + // teardown and must not push the active replacement call into an error state. + if (this.transportGeneration === detached.generation) { + this.callbacks.onStatus?.("error", "Realtime Talk voice session close failed"); + } + }); } async setVideoEnabled(enabled: boolean): Promise {