mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 02:06:43 +00:00
feat(talk): add durable client voice sessions (#111216)
Live-append voice transcripts into the agent session and persist a per-agent SQLite call record across relay and client transcript paths. Add run-scoped spoken confirmation for high-impact actions, mutation digests, bootstrap-context injection, talk.client.transcript and talk.client.close protocol methods, and Control UI adoption. This adds zero new configuration. Co-authored-by: Clifton King <clifton@users.noreply.github.com>
This commit is contained in:
co-authored by
Clifton King
parent
83fc53a22f
commit
0f95e66b7f
@@ -203,6 +203,8 @@ enum class GatewayMethod(
|
||||
TalkCatalog("talk.catalog"),
|
||||
TalkConfig("talk.config"),
|
||||
TalkClientCreate("talk.client.create"),
|
||||
TalkClientTranscript("talk.client.transcript"),
|
||||
TalkClientClose("talk.client.close"),
|
||||
TalkClientToolCall("talk.client.toolCall"),
|
||||
TalkClientSteer("talk.client.steer"),
|
||||
TalkSessionCreate("talk.session.create"),
|
||||
|
||||
@@ -8116,6 +8116,7 @@ public struct TalkCatalogResult: Codable, Sendable {
|
||||
|
||||
public struct TalkClientCreateParams: Codable, Sendable {
|
||||
public let sessionkey: String?
|
||||
public let voicesessionid: String?
|
||||
public let provider: String?
|
||||
public let model: String?
|
||||
public let voice: String?
|
||||
@@ -8126,10 +8127,11 @@ public struct TalkClientCreateParams: Codable, Sendable {
|
||||
public let mode: AnyCodable?
|
||||
public let transport: AnyCodable?
|
||||
public let brain: AnyCodable?
|
||||
public let capabilities: [String]?
|
||||
public let capabilities: [AnyCodable]?
|
||||
|
||||
public init(
|
||||
sessionkey: String? = nil,
|
||||
voicesessionid: String? = nil,
|
||||
provider: String? = nil,
|
||||
model: String? = nil,
|
||||
voice: String? = nil,
|
||||
@@ -8140,9 +8142,10 @@ public struct TalkClientCreateParams: Codable, Sendable {
|
||||
mode: AnyCodable? = nil,
|
||||
transport: AnyCodable? = nil,
|
||||
brain: AnyCodable? = nil,
|
||||
capabilities: [String]? = nil)
|
||||
capabilities: [AnyCodable]? = nil)
|
||||
{
|
||||
self.sessionkey = sessionkey
|
||||
self.voicesessionid = voicesessionid
|
||||
self.provider = provider
|
||||
self.model = model
|
||||
self.voice = voice
|
||||
@@ -8158,6 +8161,7 @@ public struct TalkClientCreateParams: Codable, Sendable {
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case sessionkey = "sessionKey"
|
||||
case voicesessionid = "voiceSessionId"
|
||||
case provider
|
||||
case model
|
||||
case voice
|
||||
@@ -8172,6 +8176,38 @@ public struct TalkClientCreateParams: Codable, Sendable {
|
||||
}
|
||||
}
|
||||
|
||||
public struct TalkClientCloseParams: Codable, Sendable {
|
||||
public let sessionkey: String
|
||||
public let voicesessionid: String
|
||||
|
||||
public init(
|
||||
sessionkey: String,
|
||||
voicesessionid: String)
|
||||
{
|
||||
self.sessionkey = sessionkey
|
||||
self.voicesessionid = voicesessionid
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case sessionkey = "sessionKey"
|
||||
case voicesessionid = "voiceSessionId"
|
||||
}
|
||||
}
|
||||
|
||||
public struct TalkClientMutationResult: Codable, Sendable {
|
||||
public let ok: Bool
|
||||
|
||||
public init(
|
||||
ok: Bool)
|
||||
{
|
||||
self.ok = ok
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case ok
|
||||
}
|
||||
}
|
||||
|
||||
public struct TalkClientSteerParams: Codable, Sendable {
|
||||
public let sessionkey: String
|
||||
public let text: String
|
||||
@@ -8270,6 +8306,7 @@ public struct TalkAgentControlResult: Codable, Sendable {
|
||||
|
||||
public struct TalkClientToolCallParams: Codable, Sendable {
|
||||
public let sessionkey: String
|
||||
public let voicesessionid: String?
|
||||
public let callid: String
|
||||
public let name: String
|
||||
public let args: AnyCodable?
|
||||
@@ -8277,12 +8314,14 @@ public struct TalkClientToolCallParams: Codable, Sendable {
|
||||
|
||||
public init(
|
||||
sessionkey: String,
|
||||
voicesessionid: String? = nil,
|
||||
callid: String,
|
||||
name: String,
|
||||
args: AnyCodable? = nil,
|
||||
relaysessionid: String? = nil)
|
||||
{
|
||||
self.sessionkey = sessionkey
|
||||
self.voicesessionid = voicesessionid
|
||||
self.callid = callid
|
||||
self.name = name
|
||||
self.args = args
|
||||
@@ -8291,6 +8330,7 @@ public struct TalkClientToolCallParams: Codable, Sendable {
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case sessionkey = "sessionKey"
|
||||
case voicesessionid = "voiceSessionId"
|
||||
case callid = "callId"
|
||||
case name
|
||||
case args
|
||||
@@ -8316,6 +8356,40 @@ public struct TalkClientToolCallResult: Codable, Sendable {
|
||||
}
|
||||
}
|
||||
|
||||
public struct TalkClientTranscriptParams: Codable, Sendable {
|
||||
public let sessionkey: String
|
||||
public let voicesessionid: String
|
||||
public let entryid: String
|
||||
public let role: AnyCodable
|
||||
public let text: String
|
||||
public let timestamp: Double?
|
||||
|
||||
public init(
|
||||
sessionkey: String,
|
||||
voicesessionid: String,
|
||||
entryid: String,
|
||||
role: AnyCodable,
|
||||
text: String,
|
||||
timestamp: Double? = nil)
|
||||
{
|
||||
self.sessionkey = sessionkey
|
||||
self.voicesessionid = voicesessionid
|
||||
self.entryid = entryid
|
||||
self.role = role
|
||||
self.text = text
|
||||
self.timestamp = timestamp
|
||||
}
|
||||
|
||||
private enum CodingKeys: String, CodingKey {
|
||||
case sessionkey = "sessionKey"
|
||||
case voicesessionid = "voiceSessionId"
|
||||
case entryid = "entryId"
|
||||
case role
|
||||
case text
|
||||
case timestamp
|
||||
}
|
||||
}
|
||||
|
||||
public struct TalkConfigParams: Codable, Sendable {
|
||||
public let includesecrets: Bool?
|
||||
|
||||
|
||||
@@ -496,8 +496,10 @@ methods. Treat this as feature discovery, not a full enumeration of
|
||||
- `talk.session.steer` sends active-run voice control into a gateway-owned agent-backed Talk session: `{ sessionId, text, mode? }`, where `mode` is `status`, `steer`, `cancel`, or `followup`; omitted mode is classified from the spoken text.
|
||||
- `talk.session.close` closes a gateway-owned relay, transcription, or managed-room session and emits terminal Talk events.
|
||||
- `talk.mode` sets/broadcasts the current Talk mode state for WebChat/Control UI clients.
|
||||
- `talk.client.create` creates a client-owned realtime provider session using `webrtc` or `provider-websocket` while the gateway owns config, credentials, instructions, and tool policy.
|
||||
- `talk.client.toolCall` lets client-owned realtime transports forward provider tool calls to gateway policy. The first supported tool is `openclaw_agent_consult`; clients get a run id and wait for normal chat lifecycle events before submitting the provider-specific tool result.
|
||||
- `talk.client.create` creates or resumes a client-owned realtime provider session using `webrtc` or `provider-websocket` while the gateway owns credentials, instructions, tool policy, and the returned `voiceSessionId`. Clients pass `sessionKey` and reuse `voiceSessionId` when replacing the provider transport during one call.
|
||||
- `talk.client.transcript` appends one finalized `{ role, text }` item to the normal agent session. The required `entryId` is idempotent within `voiceSessionId`; retries do not duplicate transcript messages.
|
||||
- `talk.client.close` closes the logical voice session after pending transcript writes. Closing is idempotent and may deliver a mutation-only call digest to the session's last non-WebChat channel.
|
||||
- `talk.client.toolCall` lets client-owned realtime transports forward provider tool calls to gateway policy. The first supported tool is `openclaw_agent_consult`; clients get a run id and wait for normal chat lifecycle events before submitting the provider-specific tool result. Voice-bound high-impact actions return `VOICE_CONFIRMATION_REQUIRED:<id>` until a later finalized user utterance explicitly confirms that exact action and the next consult supplies the `confirmationId`.
|
||||
- `talk.client.steer` sends active-run voice control for client-owned realtime transports. The gateway resolves the active embedded run from `sessionKey` and returns a structured accepted/rejected result instead of silently dropping steering.
|
||||
- `talk.event` is the single Talk event channel for realtime, transcription, STT/TTS, managed-room, telephony, and meeting adapters.
|
||||
- `talk.speak` synthesizes speech through the active Talk speech provider.
|
||||
|
||||
@@ -18,6 +18,10 @@ Native Talk is a continuous loop: listen for speech, send the transcript to the
|
||||
|
||||
Client-owned realtime Talk forwards provider tool calls through `talk.client.toolCall` instead of calling `chat.send` directly. While a realtime consult is active, clients can call `talk.client.steer` or `talk.session.steer` to classify spoken input as `status`, `steer`, `cancel`, or `followup`. Accepted steering queues into the active embedded run; rejected steering returns a reason such as `no_active_run`, `not_streaming`, or `compacting`.
|
||||
|
||||
Finalized realtime user and assistant utterances are always appended live to the active agent session, so later chat and voice turns share one history. Client-owned transports report their finalized transcripts with stable entry ids; Gateway relay sessions append the same events server-side. Provider sessions also receive the bounded realtime profile context used by Discord voice.
|
||||
|
||||
Voice-originated consult runs require a new, exact spoken confirmation before high-impact actions such as sending messages, controlling nodes, browser/computer actions, service changes, destructive shell commands, or publication. The confirmation applies only to the exact blocked tool arguments and is consumed once; unrelated concurrent runs remain unaffected. When a call closes, OpenClaw can send a compact **Voice call changes** digest for mutating tools to the session's last non-WebChat delivery target.
|
||||
|
||||
Transcription-only Talk emits the same Talk event envelope as realtime and STT/TTS sessions, but uses `mode: "transcription"` and `brain: "none"`. All Talk sessions broadcast events on the `talk.event` channel; clients subscribe to it for partial/final transcript updates (`transcript.delta`/`transcript.done`) and other session telemetry.
|
||||
|
||||
Browser Video Talk is available for OpenAI Realtime WebRTC and Google Live
|
||||
|
||||
@@ -96,10 +96,13 @@ import {
|
||||
TalkCatalogResultSchema,
|
||||
TalkClientCreateParamsSchema,
|
||||
TalkClientCreateResultSchema,
|
||||
TalkClientCloseParamsSchema,
|
||||
TalkClientMutationResultSchema,
|
||||
TalkAgentControlResultSchema,
|
||||
TalkClientSteerParamsSchema,
|
||||
TalkClientToolCallParamsSchema,
|
||||
TalkClientToolCallResultSchema,
|
||||
TalkClientTranscriptParamsSchema,
|
||||
TalkConfigParamsSchema,
|
||||
TalkConfigResultSchema,
|
||||
TalkSessionAppendAudioParamsSchema,
|
||||
@@ -808,8 +811,11 @@ export const validateTalkConfigParams = lazyCompile(TalkConfigParamsSchema);
|
||||
export const validateTalkConfigResult = lazyCompile(TalkConfigResultSchema);
|
||||
export const validateTalkClientCreateParams = lazyCompile(TalkClientCreateParamsSchema);
|
||||
export const validateTalkClientCreateResult = lazyCompile(TalkClientCreateResultSchema);
|
||||
export const validateTalkClientCloseParams = lazyCompile(TalkClientCloseParamsSchema);
|
||||
export const validateTalkClientMutationResult = lazyCompile(TalkClientMutationResultSchema);
|
||||
export const validateTalkClientToolCallParams = lazyCompile(TalkClientToolCallParamsSchema);
|
||||
export const validateTalkClientToolCallResult = lazyCompile(TalkClientToolCallResultSchema);
|
||||
export const validateTalkClientTranscriptParams = lazyCompile(TalkClientTranscriptParamsSchema);
|
||||
export const validateTalkClientSteerParams = lazyCompile(TalkClientSteerParamsSchema);
|
||||
export const validateTalkAgentControlResult = lazyCompile(TalkAgentControlResultSchema);
|
||||
export const validateTalkSessionCreateParams = lazyCompile(TalkSessionCreateParamsSchema);
|
||||
@@ -1234,10 +1240,13 @@ export {
|
||||
TalkCatalogResultSchema,
|
||||
TalkClientCreateParamsSchema,
|
||||
TalkClientCreateResultSchema,
|
||||
TalkClientCloseParamsSchema,
|
||||
TalkClientMutationResultSchema,
|
||||
TalkAgentControlResultSchema,
|
||||
TalkClientSteerParamsSchema,
|
||||
TalkClientToolCallParamsSchema,
|
||||
TalkClientToolCallResultSchema,
|
||||
TalkClientTranscriptParamsSchema,
|
||||
TalkConfigParamsSchema,
|
||||
TalkConfigResultSchema,
|
||||
TalkSessionAppendAudioParamsSchema,
|
||||
@@ -1560,10 +1569,13 @@ export type {
|
||||
TalkCatalogResult,
|
||||
TalkClientCreateParams,
|
||||
TalkClientCreateResult,
|
||||
TalkClientCloseParams,
|
||||
TalkClientMutationResult,
|
||||
TalkClientSteerParams,
|
||||
TalkAgentControlResult,
|
||||
TalkClientToolCallParams,
|
||||
TalkClientToolCallResult,
|
||||
TalkClientTranscriptParams,
|
||||
TalkConfigParams,
|
||||
TalkConfigResult,
|
||||
TalkSessionAppendAudioParams,
|
||||
|
||||
@@ -186,9 +186,14 @@ export const TalkEventSchema = Type.Object(
|
||||
},
|
||||
);
|
||||
|
||||
// Voice ids compose into transcript idempotency keys (`voice:<id>:<entry>`), so the
|
||||
// charset excludes the `:` delimiter to keep distinct id pairs collision-free.
|
||||
const VoiceIdString = Type.String({ pattern: "^[A-Za-z0-9_-]{1,128}$" });
|
||||
|
||||
/** Creates a browser-facing Talk client session. */
|
||||
export const TalkClientCreateParamsSchema = closedObject({
|
||||
sessionKey: Type.Optional(Type.String()),
|
||||
sessionKey: Type.Optional(NonEmptyString),
|
||||
voiceSessionId: Type.Optional(VoiceIdString),
|
||||
provider: Type.Optional(Type.String()),
|
||||
model: Type.Optional(Type.String()),
|
||||
voice: Type.Optional(Type.String()),
|
||||
@@ -199,18 +204,44 @@ export const TalkClientCreateParamsSchema = closedObject({
|
||||
mode: Type.Optional(TalkModeSchema),
|
||||
transport: Type.Optional(TalkTransportSchema),
|
||||
brain: Type.Optional(TalkBrainSchema),
|
||||
capabilities: Type.Optional(Type.Array(Type.Literal("camera-frame"), { uniqueItems: true })),
|
||||
capabilities: Type.Optional(
|
||||
Type.Array(Type.Union([Type.Literal("camera-frame"), Type.Literal("voice-transcript")]), {
|
||||
uniqueItems: true,
|
||||
}),
|
||||
),
|
||||
});
|
||||
|
||||
/** Tool-call request from a browser/client session back into the agent runtime. */
|
||||
export const TalkClientToolCallParamsSchema = closedObject({
|
||||
sessionKey: NonEmptyString,
|
||||
voiceSessionId: Type.Optional(VoiceIdString),
|
||||
callId: NonEmptyString,
|
||||
name: NonEmptyString,
|
||||
args: Type.Optional(Type.Unknown()),
|
||||
relaySessionId: Type.Optional(NonEmptyString),
|
||||
});
|
||||
|
||||
/** One finalized transcript item from a client-owned Talk session. */
|
||||
export const TalkClientTranscriptParamsSchema = closedObject({
|
||||
sessionKey: NonEmptyString,
|
||||
voiceSessionId: VoiceIdString,
|
||||
entryId: VoiceIdString,
|
||||
role: Type.Union([Type.Literal("user"), Type.Literal("assistant")]),
|
||||
text: NonEmptyString,
|
||||
timestamp: Type.Optional(Type.Number()),
|
||||
});
|
||||
|
||||
/** Logical close for a client-owned Talk session. */
|
||||
export const TalkClientCloseParamsSchema = closedObject({
|
||||
sessionKey: NonEmptyString,
|
||||
voiceSessionId: VoiceIdString,
|
||||
});
|
||||
|
||||
/** Result for client-owned transcript and close mutations. */
|
||||
export const TalkClientMutationResultSchema = closedObject({
|
||||
ok: Type.Literal(true),
|
||||
});
|
||||
|
||||
/** Agent run identity returned after accepting a Talk client tool call. */
|
||||
export const TalkClientToolCallResultSchema = closedObject({
|
||||
runId: NonEmptyString,
|
||||
@@ -456,6 +487,7 @@ export const TalkSessionOkResultSchema = closedObject({
|
||||
const BrowserRealtimeWebRtcSdpSessionSchema = closedObject({
|
||||
provider: NonEmptyString,
|
||||
transport: Type.Literal("webrtc"),
|
||||
voiceSessionId: NonEmptyString,
|
||||
clientSecret: NonEmptyString,
|
||||
offerUrl: Type.Optional(Type.String()),
|
||||
offerHeaders: Type.Optional(Type.Record(Type.String(), Type.String())),
|
||||
@@ -468,6 +500,7 @@ const BrowserRealtimeWebRtcSdpSessionSchema = closedObject({
|
||||
const BrowserRealtimeJsonPcmWebSocketSessionSchema = closedObject({
|
||||
provider: NonEmptyString,
|
||||
transport: Type.Literal("provider-websocket"),
|
||||
voiceSessionId: NonEmptyString,
|
||||
protocol: NonEmptyString,
|
||||
clientSecret: NonEmptyString,
|
||||
websocketUrl: NonEmptyString,
|
||||
@@ -482,6 +515,8 @@ const BrowserRealtimeJsonPcmWebSocketSessionSchema = closedObject({
|
||||
const BrowserRealtimeGatewayRelaySessionSchema = closedObject({
|
||||
provider: NonEmptyString,
|
||||
transport: Type.Literal("gateway-relay"),
|
||||
// Server-owned: older gateways omit it and clients derive it from relaySessionId.
|
||||
voiceSessionId: Type.Optional(NonEmptyString),
|
||||
relaySessionId: NonEmptyString,
|
||||
audio: BrowserRealtimeAudioContractSchema,
|
||||
model: Type.Optional(Type.String()),
|
||||
@@ -493,6 +528,8 @@ const BrowserRealtimeGatewayRelaySessionSchema = closedObject({
|
||||
const BrowserRealtimeManagedRoomSessionSchema = closedObject({
|
||||
provider: NonEmptyString,
|
||||
transport: Type.Literal("managed-room"),
|
||||
// Server-owned rooms carry no client voice bookkeeping yet.
|
||||
voiceSessionId: Type.Optional(NonEmptyString),
|
||||
roomUrl: NonEmptyString,
|
||||
token: Type.Optional(Type.String()),
|
||||
model: Type.Optional(Type.String()),
|
||||
@@ -740,6 +777,9 @@ export type TalkClientSteerParams = Static<typeof TalkClientSteerParamsSchema>;
|
||||
export type TalkAgentControlResult = Static<typeof TalkAgentControlResultSchema>;
|
||||
export type TalkClientToolCallParams = Static<typeof TalkClientToolCallParamsSchema>;
|
||||
export type TalkClientToolCallResult = Static<typeof TalkClientToolCallResultSchema>;
|
||||
export type TalkClientTranscriptParams = Static<typeof TalkClientTranscriptParamsSchema>;
|
||||
export type TalkClientCloseParams = Static<typeof TalkClientCloseParamsSchema>;
|
||||
export type TalkClientMutationResult = Static<typeof TalkClientMutationResultSchema>;
|
||||
export type TalkSessionCreateParams = Static<typeof TalkSessionCreateParamsSchema>;
|
||||
export type TalkSessionCreateResult = Static<typeof TalkSessionCreateResultSchema>;
|
||||
export type TalkSessionJoinParams = Static<typeof TalkSessionJoinParamsSchema>;
|
||||
|
||||
@@ -187,10 +187,13 @@ import {
|
||||
TalkCatalogResultSchema,
|
||||
TalkClientCreateParamsSchema,
|
||||
TalkClientCreateResultSchema,
|
||||
TalkClientCloseParamsSchema,
|
||||
TalkClientMutationResultSchema,
|
||||
TalkAgentControlResultSchema,
|
||||
TalkClientSteerParamsSchema,
|
||||
TalkClientToolCallParamsSchema,
|
||||
TalkClientToolCallResultSchema,
|
||||
TalkClientTranscriptParamsSchema,
|
||||
TalkConfigParamsSchema,
|
||||
TalkConfigResultSchema,
|
||||
TalkSessionAppendAudioParamsSchema,
|
||||
@@ -850,10 +853,13 @@ export const ProtocolSchemas = {
|
||||
TalkCatalogResult: TalkCatalogResultSchema,
|
||||
TalkClientCreateParams: TalkClientCreateParamsSchema,
|
||||
TalkClientCreateResult: TalkClientCreateResultSchema,
|
||||
TalkClientCloseParams: TalkClientCloseParamsSchema,
|
||||
TalkClientMutationResult: TalkClientMutationResultSchema,
|
||||
TalkClientSteerParams: TalkClientSteerParamsSchema,
|
||||
TalkAgentControlResult: TalkAgentControlResultSchema,
|
||||
TalkClientToolCallParams: TalkClientToolCallParamsSchema,
|
||||
TalkClientToolCallResult: TalkClientToolCallResultSchema,
|
||||
TalkClientTranscriptParams: TalkClientTranscriptParamsSchema,
|
||||
TalkConfigParams: TalkConfigParamsSchema,
|
||||
TalkConfigResult: TalkConfigResultSchema,
|
||||
TalkSessionAppendAudioParams: TalkSessionAppendAudioParamsSchema,
|
||||
|
||||
@@ -71,6 +71,11 @@ import {
|
||||
} from "../skills/loading/source.js";
|
||||
import type { SkillSnapshot, SkillTelemetrySource, SkillUsagePath } from "../skills/types.js";
|
||||
import { resolveSkillWorkshopToolApproval } from "../skills/workshop/policy.js";
|
||||
import { resolveClientVoiceToolConfirmationPolicy } from "../talk/client-voice-confirmation.js";
|
||||
import {
|
||||
isClientVoiceSessionConfirmable,
|
||||
resolveClientVoiceRunBinding,
|
||||
} from "../talk/client-voice-session.js";
|
||||
import { isPlainObject, truncateUtf16Safe } from "../utils.js";
|
||||
import {
|
||||
adjustedParamsByToolCallId,
|
||||
@@ -84,6 +89,7 @@ import {
|
||||
} from "./agent-tools.before-tool-call.state.js";
|
||||
import { normalizeFileToolPathParam } from "./agent-tools.params.js";
|
||||
import { resolveAgentRunAbortLifecycleFields } from "./run-termination.js";
|
||||
import { buildToolMutationState } from "./tool-mutation.js";
|
||||
export {
|
||||
consumeAdjustedParamsForToolCall,
|
||||
consumePreExecutionBlockedToolCall,
|
||||
@@ -1533,6 +1539,24 @@ export async function runBeforeToolCallHook(args: {
|
||||
...(args.ctx?.config ? { config: args.ctx.config } : {}),
|
||||
...(args.ctx?.workspaceDir ? { workspaceDir: args.ctx.workspaceDir } : {}),
|
||||
});
|
||||
const voiceRun = resolveClientVoiceRunBinding(args.ctx?.runId);
|
||||
const voiceConfirmation = resolveClientVoiceToolConfirmationPolicy({
|
||||
agentId: voiceRun?.agentId,
|
||||
voiceSessionId: voiceRun?.voiceSessionId,
|
||||
runId: args.ctx?.runId,
|
||||
toolName,
|
||||
toolParams: normalizedParams,
|
||||
...(voiceRun ? { isConfirmable: () => isClientVoiceSessionConfirmable(voiceRun) } : {}),
|
||||
});
|
||||
if (!voiceConfirmation.allowed) {
|
||||
return {
|
||||
blocked: true,
|
||||
kind: "veto",
|
||||
deniedReason: "plugin-before-tool-call",
|
||||
reason: voiceConfirmation.reason,
|
||||
params,
|
||||
};
|
||||
}
|
||||
if (!initialCorePolicyResult && !shouldRunTrustedPolicies && !hasBeforeToolCallHooks) {
|
||||
return { blocked: false, params };
|
||||
}
|
||||
@@ -1793,6 +1817,7 @@ export function wrapToolWithBeforeToolCallHook(
|
||||
...diagnosticIdentity,
|
||||
...(toolCallId && { toolCallId }),
|
||||
paramsSummary: summarizeToolParams(toolParams),
|
||||
mutatingAction: buildToolMutationState(normalizedToolName, toolParams).mutatingAction,
|
||||
});
|
||||
const recordPreExecutionError = (
|
||||
error: unknown,
|
||||
|
||||
@@ -87,6 +87,8 @@ describe("method scope resolution", () => {
|
||||
["node.pair.approve", ["operator.pairing"]],
|
||||
["poll", ["operator.write"]],
|
||||
["talk.client.create", ["operator.write"]],
|
||||
["talk.client.transcript", ["operator.write"]],
|
||||
["talk.client.close", ["operator.write"]],
|
||||
["talk.client.toolCall", ["operator.write"]],
|
||||
["talk.client.steer", ["operator.write"]],
|
||||
["talk.session.create", ["operator.write"]],
|
||||
@@ -518,6 +520,8 @@ describe("operator scope authorization", () => {
|
||||
it("allows operator.write clients to use unified Talk sessions", () => {
|
||||
for (const method of [
|
||||
"talk.client.create",
|
||||
"talk.client.transcript",
|
||||
"talk.client.close",
|
||||
"talk.client.toolCall",
|
||||
"talk.client.steer",
|
||||
"talk.session.create",
|
||||
|
||||
@@ -87,6 +87,8 @@ const CORE_GATEWAY_METHOD_SPECS: readonly CoreGatewayMethodSpec[] = [
|
||||
// Params-aware: reading redacted config needs read; includeSecrets also needs talk secrets.
|
||||
{ name: "talk.config", scope: "dynamic" },
|
||||
{ name: "talk.client.create", scope: "operator.write" },
|
||||
{ name: "talk.client.transcript", scope: "operator.write" },
|
||||
{ name: "talk.client.close", scope: "operator.write" },
|
||||
{ name: "talk.client.toolCall", scope: "operator.write" },
|
||||
{ name: "talk.client.steer", scope: "operator.write" },
|
||||
{ name: "talk.session.create", scope: "operator.write" },
|
||||
|
||||
@@ -149,6 +149,8 @@ describe("listGatewayMethods", () => {
|
||||
it("advertises the versioned Talk session RPCs", () => {
|
||||
const methods = listGatewayMethods();
|
||||
expect(methods).toContain("talk.client.create");
|
||||
expect(methods).toContain("talk.client.transcript");
|
||||
expect(methods).toContain("talk.client.close");
|
||||
expect(methods).toContain("talk.client.toolCall");
|
||||
expect(methods).toContain("talk.client.steer");
|
||||
expect(methods).toContain("talk.session.create");
|
||||
|
||||
@@ -578,6 +578,8 @@ export const coreGatewayHandlers: GatewayRequestHandlers = {
|
||||
"talk.session.steer",
|
||||
"talk.session.close",
|
||||
"talk.client.create",
|
||||
"talk.client.transcript",
|
||||
"talk.client.close",
|
||||
"talk.client.toolCall",
|
||||
"talk.client.steer",
|
||||
"talk.catalog",
|
||||
|
||||
@@ -0,0 +1,282 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
loadSessionEntry,
|
||||
readSessionTranscriptMessageEvents,
|
||||
replaceSessionEntry,
|
||||
} from "../../config/sessions/session-accessor.js";
|
||||
import { closeOpenClawAgentDatabasesForTest } from "../../state/openclaw-agent-db.js";
|
||||
import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js";
|
||||
import {
|
||||
authorizeClientVoiceConfirmation,
|
||||
resolveClientVoiceToolConfirmationPolicy,
|
||||
} from "../../talk/client-voice-confirmation.js";
|
||||
import { resetClientVoiceConfirmationStateForTest } from "../../talk/client-voice-confirmation.test-support.js";
|
||||
import {
|
||||
closeClientVoiceSession,
|
||||
createOrResumeClientVoiceSession,
|
||||
ensureClientVoiceAgentSessionEntry,
|
||||
} from "../../talk/client-voice-session.js";
|
||||
import { clientVoiceSessionTesting } from "../../talk/client-voice-session.test-support.js";
|
||||
import { captureEnv, setTestEnvValue } from "../../test-utils/env.js";
|
||||
import { talkClientHandlers } from "./talk-client.js";
|
||||
|
||||
const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
|
||||
const sessionKey = "agent:main:main";
|
||||
const sessionId = "voice-transcript-session";
|
||||
let tempDir: string;
|
||||
|
||||
async function invokeTranscript(params: Record<string, unknown>) {
|
||||
const respond = vi.fn();
|
||||
await talkClientHandlers["talk.client.transcript"]?.({
|
||||
params,
|
||||
respond,
|
||||
context: { getRuntimeConfig: () => ({}) },
|
||||
} as never);
|
||||
return respond;
|
||||
}
|
||||
|
||||
async function invokeClose(params: Record<string, unknown>) {
|
||||
const respond = vi.fn();
|
||||
await talkClientHandlers["talk.client.close"]?.({
|
||||
params,
|
||||
respond,
|
||||
context: { getRuntimeConfig: () => ({}) },
|
||||
client: { connId: "conn-close" },
|
||||
} as never);
|
||||
return respond;
|
||||
}
|
||||
|
||||
describe("talk.client.transcript", () => {
|
||||
beforeEach(async () => {
|
||||
tempDir = await fs.realpath(
|
||||
await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-talk-transcript-")),
|
||||
);
|
||||
setTestEnvValue("OPENCLAW_STATE_DIR", tempDir);
|
||||
await replaceSessionEntry(
|
||||
{ agentId: "main", sessionKey },
|
||||
{ sessionId, updatedAt: Date.now() },
|
||||
);
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
clientVoiceSessionTesting.reset();
|
||||
resetClientVoiceConfirmationStateForTest();
|
||||
vi.useRealTimers();
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
envSnapshot.restore();
|
||||
await fs.rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("appends finalized messages once by event id", async () => {
|
||||
const voiceSessionId = createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey,
|
||||
origin: "client",
|
||||
});
|
||||
const params = {
|
||||
sessionKey,
|
||||
voiceSessionId,
|
||||
entryId: "1",
|
||||
role: "user",
|
||||
text: "hello from voice",
|
||||
timestamp: 123,
|
||||
};
|
||||
|
||||
expect(await invokeTranscript(params)).toHaveBeenCalledWith(true, { ok: true }, undefined);
|
||||
expect(await invokeTranscript(params)).toHaveBeenCalledWith(true, { ok: true }, undefined);
|
||||
const events = readSessionTranscriptMessageEvents({ agentId: "main", sessionId });
|
||||
expect(events).toHaveLength(1);
|
||||
expect(events[0]?.event).toMatchObject({
|
||||
id: `voice:${voiceSessionId}:1`,
|
||||
message: {
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "hello from voice" }],
|
||||
timestamp: 123,
|
||||
provenance: { kind: "realtime_voice", sourceChannel: "talk" },
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("appends before the session has ever received a chat turn", async () => {
|
||||
const talkFirstSessionKey = "agent:main:talk-first";
|
||||
await ensureClientVoiceAgentSessionEntry({
|
||||
agentId: "main",
|
||||
sessionKey: talkFirstSessionKey,
|
||||
});
|
||||
const voiceSessionId = createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: talkFirstSessionKey,
|
||||
provider: "google",
|
||||
origin: "client",
|
||||
});
|
||||
|
||||
expect(
|
||||
await invokeTranscript({
|
||||
sessionKey: talkFirstSessionKey,
|
||||
voiceSessionId,
|
||||
entryId: "1",
|
||||
role: "user",
|
||||
text: "heard before the first consult",
|
||||
}),
|
||||
).toHaveBeenCalledWith(true, { ok: true }, undefined);
|
||||
|
||||
const talkFirstEntry = loadSessionEntry({
|
||||
agentId: "main",
|
||||
sessionKey: talkFirstSessionKey,
|
||||
});
|
||||
expect(talkFirstEntry?.sessionId).toBeTruthy();
|
||||
expect(
|
||||
readSessionTranscriptMessageEvents({
|
||||
agentId: "main",
|
||||
sessionId: talkFirstEntry?.sessionId ?? "missing",
|
||||
}),
|
||||
).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("uses server observation time for spoken-confirmation freshness", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(100);
|
||||
const voiceSessionId = createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey,
|
||||
origin: "client",
|
||||
});
|
||||
await invokeTranscript({
|
||||
sessionKey,
|
||||
voiceSessionId,
|
||||
entryId: "early-yes",
|
||||
role: "user",
|
||||
text: "yes",
|
||||
timestamp: 10_000,
|
||||
});
|
||||
const policy = resolveClientVoiceToolConfirmationPolicy({
|
||||
agentId: "main",
|
||||
voiceSessionId,
|
||||
runId: "run-later",
|
||||
toolName: "message",
|
||||
toolParams: { action: "send", message: "later" },
|
||||
now: 200,
|
||||
});
|
||||
expect(policy.allowed).toBe(false);
|
||||
if (policy.allowed) {
|
||||
throw new Error("expected confirmation request");
|
||||
}
|
||||
const confirmationId = policy.reason.match(/VOICE_CONFIRMATION_REQUIRED:([^\s]+)/)?.[1];
|
||||
expect(confirmationId).toBeTruthy();
|
||||
|
||||
expect(() =>
|
||||
authorizeClientVoiceConfirmation({
|
||||
agentId: "main",
|
||||
voiceSessionId,
|
||||
confirmationId: confirmationId ?? "missing",
|
||||
now: 201,
|
||||
}),
|
||||
).toThrow("explicit spoken confirmation");
|
||||
});
|
||||
|
||||
it("accepts an idempotent close retry after the first response is lost", async () => {
|
||||
const voiceSessionId = createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey,
|
||||
origin: "client",
|
||||
});
|
||||
const params = { sessionKey, voiceSessionId };
|
||||
|
||||
expect(await invokeClose(params)).toHaveBeenCalledWith(true, { ok: true }, undefined);
|
||||
expect(await invokeClose(params)).toHaveBeenCalledWith(true, { ok: true }, undefined);
|
||||
});
|
||||
|
||||
it("truncates UTF-16 safely and writes assistant metadata", async () => {
|
||||
const voiceSessionId = createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey,
|
||||
provider: "google",
|
||||
origin: "client",
|
||||
});
|
||||
const respond = await invokeTranscript({
|
||||
sessionKey,
|
||||
voiceSessionId,
|
||||
entryId: "assistant-1",
|
||||
role: "assistant",
|
||||
text: `${"x".repeat(7_999)}😀tail`,
|
||||
});
|
||||
|
||||
expect(respond).toHaveBeenCalledWith(true, { ok: true }, undefined);
|
||||
const event = readSessionTranscriptMessageEvents({ agentId: "main", sessionId })[0]?.event as
|
||||
| { message?: { content?: Array<{ text?: string }> } }
|
||||
| undefined;
|
||||
const text = event?.message?.content?.[0]?.text;
|
||||
expect(text).toBe("x".repeat(7_999));
|
||||
expect(event).toMatchObject({
|
||||
message: {
|
||||
api: "realtime",
|
||||
provider: "google",
|
||||
model: "realtime-voice",
|
||||
stopReason: "stop",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("uses the neutral provider label for records created before provider tracking", async () => {
|
||||
const voiceSessionId = createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey,
|
||||
origin: "client",
|
||||
});
|
||||
|
||||
expect(
|
||||
await invokeTranscript({
|
||||
sessionKey,
|
||||
voiceSessionId,
|
||||
entryId: "legacy-assistant",
|
||||
role: "assistant",
|
||||
text: "legacy provider reply",
|
||||
}),
|
||||
).toHaveBeenCalledWith(true, { ok: true }, undefined);
|
||||
expect(
|
||||
readSessionTranscriptMessageEvents({ agentId: "main", sessionId })[0]?.event,
|
||||
).toMatchObject({
|
||||
message: { api: "realtime", provider: "realtime", model: "realtime-voice" },
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
["missing", "voice-missing", "voice session not found"],
|
||||
["closed", "voice-closed", "voice session is closed"],
|
||||
["relay", "voice-relay", "does not allow this transcript source"],
|
||||
])("rejects %s voice records", async (kind, voiceSessionId, expected) => {
|
||||
if (kind !== "missing") {
|
||||
createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey,
|
||||
origin: kind === "relay" ? "relay" : "client",
|
||||
voiceSessionId,
|
||||
});
|
||||
}
|
||||
if (kind === "closed") {
|
||||
await closeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey,
|
||||
voiceSessionId,
|
||||
config: {},
|
||||
});
|
||||
}
|
||||
|
||||
const respond = await invokeTranscript({
|
||||
sessionKey,
|
||||
voiceSessionId,
|
||||
entryId: "1",
|
||||
role: "user",
|
||||
text: "hello",
|
||||
});
|
||||
expect(respond).toHaveBeenCalledWith(
|
||||
false,
|
||||
undefined,
|
||||
expect.objectContaining({ message: expect.stringContaining(expected) }),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -8,28 +8,71 @@ import {
|
||||
ErrorCodes,
|
||||
errorShape,
|
||||
formatValidationErrors,
|
||||
validateTalkClientCloseParams,
|
||||
validateTalkClientCreateParams,
|
||||
validateTalkClientSteerParams,
|
||||
validateTalkClientToolCallParams,
|
||||
validateTalkClientTranscriptParams,
|
||||
} from "../../../packages/gateway-protocol/src/index.js";
|
||||
import {
|
||||
buildAgentMainSessionKey,
|
||||
resolveAgentIdFromSessionKey,
|
||||
} from "../../routing/session-key.js";
|
||||
import {
|
||||
REALTIME_VOICE_AGENT_CONSULT_TOOL,
|
||||
REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
|
||||
parseRealtimeVoiceAgentConsultArgs,
|
||||
} from "../../talk/agent-consult-tool.js";
|
||||
import { REALTIME_VOICE_AGENT_CONTROL_TOOL } from "../../talk/agent-run-control-shared.js";
|
||||
import { controlRealtimeVoiceAgentRun } from "../../talk/agent-run-control.js";
|
||||
import {
|
||||
authorizeClientVoiceConfirmation,
|
||||
bindAuthorizedClientVoiceConfirmation,
|
||||
type ClientVoiceConfirmationGrant,
|
||||
} from "../../talk/client-voice-confirmation.js";
|
||||
import {
|
||||
appendClientVoiceTranscript,
|
||||
assertClientVoiceSessionOpen,
|
||||
closeClientVoiceSession,
|
||||
closeStaleClientVoiceSessions,
|
||||
createOrResumeClientVoiceSession,
|
||||
ensureClientVoiceAgentSessionEntry,
|
||||
registerClientVoiceConsultRun,
|
||||
resolveClientVoiceSessionOrigin,
|
||||
resolveOpenClientVoiceSessionId,
|
||||
} from "../../talk/client-voice-session.js";
|
||||
import { REALTIME_VOICE_DESCRIBE_VIEW_TOOL } from "../../talk/describe-view-tool.js";
|
||||
import { resolveConfiguredRealtimeVoiceProvider } from "../../talk/provider-resolver.js";
|
||||
import { startTalkRealtimeAgentConsult } from "../talk-agent-consult.js";
|
||||
import {
|
||||
ensureTalkRealtimeRelayVoiceSession,
|
||||
flushTalkRealtimeRelayVoiceWrites,
|
||||
} from "../talk-realtime-relay.js";
|
||||
import { formatForLog } from "../ws-log.js";
|
||||
import {
|
||||
buildRealtimeInstructions,
|
||||
buildRealtimeVoiceLaunchOptions,
|
||||
buildTalkRealtimeConfig,
|
||||
isUnsupportedBrowserWebRtcSession,
|
||||
resolveTalkRealtimeProviderInstructions,
|
||||
} from "./talk-shared.js";
|
||||
import type { GatewayRequestHandlers } from "./types.js";
|
||||
|
||||
const LEGACY_VOICE_BINDING_TTL_MS = 6 * 60 * 60_000;
|
||||
const legacyVoiceSessionByClient = new Map<string, { voiceSessionId: string; expiresAt: number }>();
|
||||
|
||||
function legacyVoiceBindingKey(connId: string, sessionKey: string): string {
|
||||
return `${connId}\0${sessionKey}`;
|
||||
}
|
||||
|
||||
function pruneLegacyVoiceBindings(now = Date.now()): void {
|
||||
for (const [key, binding] of legacyVoiceSessionByClient) {
|
||||
if (binding.expiresAt <= now) {
|
||||
legacyVoiceSessionByClient.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Gateway methods for browser-owned realtime Talk sessions.
|
||||
*
|
||||
@@ -37,7 +80,7 @@ import type { GatewayRequestHandlers } from "./types.js";
|
||||
* calls back into OpenClaw agent consult runs.
|
||||
*/
|
||||
export const talkClientHandlers: GatewayRequestHandlers = {
|
||||
"talk.client.create": async ({ params, respond, context }) => {
|
||||
"talk.client.create": async ({ params, respond, context, client }) => {
|
||||
if (!validateTalkClientCreateParams(params)) {
|
||||
respond(
|
||||
false,
|
||||
@@ -50,6 +93,8 @@ export const talkClientHandlers: GatewayRequestHandlers = {
|
||||
return;
|
||||
}
|
||||
const typedParams = params as {
|
||||
sessionKey?: string;
|
||||
voiceSessionId?: string;
|
||||
provider?: string;
|
||||
model?: string;
|
||||
voice?: string;
|
||||
@@ -143,6 +188,17 @@ export const talkClientHandlers: GatewayRequestHandlers = {
|
||||
requested: typedParams,
|
||||
defaults: realtimeConfig,
|
||||
});
|
||||
const realtimeContext = await resolveTalkRealtimeProviderInstructions({
|
||||
config: runtimeConfig,
|
||||
configuredInstructions: realtimeConfig.instructions,
|
||||
sessionKey: typedParams.sessionKey,
|
||||
// Legacy creates can drift to another agent's session at toolCall time, so
|
||||
// the default agent's profile must not leak into the provider session.
|
||||
requireSessionKeyForProfile: true,
|
||||
warn: (message) => context.logGateway.warn(`talk realtime context: ${message}`),
|
||||
});
|
||||
const { agentId, requestedSessionKey } = realtimeContext;
|
||||
const sessionKey = requestedSessionKey ?? buildAgentMainSessionKey({ agentId });
|
||||
if (resolution.provider.createBrowserSession && transport !== "gateway-relay") {
|
||||
const tools = [REALTIME_VOICE_AGENT_CONSULT_TOOL, REALTIME_VOICE_AGENT_CONTROL_TOOL];
|
||||
if (wantsCameraFrames) {
|
||||
@@ -151,16 +207,50 @@ export const talkClientHandlers: GatewayRequestHandlers = {
|
||||
const session = await resolution.provider.createBrowserSession({
|
||||
cfg: runtimeConfig,
|
||||
providerConfig: resolution.providerConfig,
|
||||
instructions: buildRealtimeInstructions(realtimeConfig.instructions),
|
||||
instructions: buildRealtimeInstructions(realtimeContext.instructions),
|
||||
tools,
|
||||
...launchOptions,
|
||||
});
|
||||
// Client-owned voice records are minted only for client-owned transports;
|
||||
// relay sessions are created via talk.session.create and keyed by relaySessionId.
|
||||
// Widening this guard would hand relay calls a mismatched voiceSessionId.
|
||||
if (
|
||||
(session.transport === "webrtc" || session.transport === "provider-websocket") &&
|
||||
!isUnsupportedBrowserWebRtcSession(session) &&
|
||||
(!transport || session.transport === transport)
|
||||
) {
|
||||
respond(true, session, undefined);
|
||||
// Recovering 6h-abandoned calls (and retrying their digests) is not on the
|
||||
// start path; running it inline would delay use of time-sensitive provider
|
||||
// credentials behind slow channel sends. Fire it off the response path.
|
||||
void closeStaleClientVoiceSessions({
|
||||
agentId,
|
||||
config: runtimeConfig,
|
||||
excludeVoiceSessionId: normalizeOptionalString(typedParams.voiceSessionId),
|
||||
warn: (message) => context.logGateway.warn(`talk voice session recovery: ${message}`),
|
||||
}).catch((error: unknown) =>
|
||||
context.logGateway.warn(`talk voice session recovery failed: ${formatForLog(error)}`),
|
||||
);
|
||||
await ensureClientVoiceAgentSessionEntry({ agentId, sessionKey });
|
||||
const voiceSessionId = createOrResumeClientVoiceSession({
|
||||
agentId,
|
||||
sessionKey,
|
||||
provider: resolution.provider.id,
|
||||
origin: "client",
|
||||
// Deployed clients sent sessionKey before transcripts existed, so capability
|
||||
// must be negotiated explicitly; declaring it turns the confirmation gate on.
|
||||
transcriptCapable: typedParams.capabilities?.includes("voice-transcript") === true,
|
||||
voiceSessionId: normalizeOptionalString(typedParams.voiceSessionId),
|
||||
});
|
||||
const connId = normalizeOptionalString(client?.connId);
|
||||
if (connId) {
|
||||
const now = Date.now();
|
||||
pruneLegacyVoiceBindings(now);
|
||||
legacyVoiceSessionByClient.set(
|
||||
legacyVoiceBindingKey(connId, typedParams.sessionKey?.trim() || sessionKey),
|
||||
{ voiceSessionId, expiresAt: now + LEGACY_VOICE_BINDING_TTL_MS },
|
||||
);
|
||||
}
|
||||
respond(true, { ...session, voiceSessionId }, undefined);
|
||||
return;
|
||||
}
|
||||
if (transport) {
|
||||
@@ -209,6 +299,82 @@ export const talkClientHandlers: GatewayRequestHandlers = {
|
||||
return;
|
||||
}
|
||||
|
||||
const agentId = resolveAgentIdFromSessionKey(params.sessionKey);
|
||||
const relaySessionId = normalizeOptionalString(params.relaySessionId);
|
||||
const connId = normalizeOptionalString(request.client?.connId);
|
||||
pruneLegacyVoiceBindings();
|
||||
const explicitVoiceSessionId = normalizeOptionalString(params.voiceSessionId);
|
||||
if (relaySessionId && explicitVoiceSessionId && explicitVoiceSessionId !== relaySessionId) {
|
||||
respond(
|
||||
false,
|
||||
undefined,
|
||||
errorShape(ErrorCodes.INVALID_REQUEST, "relaySessionId and voiceSessionId must match"),
|
||||
);
|
||||
return;
|
||||
}
|
||||
let confirmationGrant: ClientVoiceConfirmationGrant | undefined;
|
||||
let voiceSessionId: string;
|
||||
try {
|
||||
// Shipped clients may consult without ever creating a voice session (old app,
|
||||
// restarted gateway, ambiguous open records). Implicitly create one instead of
|
||||
// erroring so confirmation and mutation evidence stay always-on.
|
||||
voiceSessionId =
|
||||
explicitVoiceSessionId ??
|
||||
relaySessionId ??
|
||||
(connId
|
||||
? legacyVoiceSessionByClient.get(legacyVoiceBindingKey(connId, params.sessionKey))
|
||||
?.voiceSessionId
|
||||
: undefined) ??
|
||||
resolveOpenClientVoiceSessionId({ agentId, sessionKey: params.sessionKey }) ??
|
||||
createOrResumeClientVoiceSession({
|
||||
agentId,
|
||||
sessionKey: params.sessionKey,
|
||||
origin: "client",
|
||||
});
|
||||
// Pin the resolved id to this connection so a legacy client's later consults
|
||||
// reuse one record instead of forking a new never-closed session each time.
|
||||
if (connId && !relaySessionId) {
|
||||
const now = Date.now();
|
||||
pruneLegacyVoiceBindings(now);
|
||||
legacyVoiceSessionByClient.set(legacyVoiceBindingKey(connId, params.sessionKey), {
|
||||
voiceSessionId,
|
||||
expiresAt: now + LEGACY_VOICE_BINDING_TTL_MS,
|
||||
});
|
||||
}
|
||||
if (relaySessionId && connId) {
|
||||
// Initialize the canonical session row BEFORE binding: the bind drains the
|
||||
// relay's buffered finals into transcript appends, which fail without it.
|
||||
await ensureClientVoiceAgentSessionEntry({ agentId, sessionKey: params.sessionKey });
|
||||
ensureTalkRealtimeRelayVoiceSession({
|
||||
relaySessionId,
|
||||
connId,
|
||||
sessionKey: params.sessionKey,
|
||||
});
|
||||
await flushTalkRealtimeRelayVoiceWrites({ relaySessionId, connId });
|
||||
}
|
||||
const parsedArgs = parseRealtimeVoiceAgentConsultArgs(params.args ?? {});
|
||||
const origin = assertClientVoiceSessionOpen({
|
||||
agentId,
|
||||
sessionKey: params.sessionKey,
|
||||
voiceSessionId,
|
||||
});
|
||||
if (origin === "relay" && (!relaySessionId || !connId)) {
|
||||
throw new Error(
|
||||
"relay-owned voice sessions require relaySessionId and connection ownership",
|
||||
);
|
||||
}
|
||||
if (parsedArgs.confirmationId) {
|
||||
confirmationGrant = authorizeClientVoiceConfirmation({
|
||||
agentId,
|
||||
voiceSessionId,
|
||||
confirmationId: parsedArgs.confirmationId,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, formatForLog(err)));
|
||||
return;
|
||||
}
|
||||
|
||||
const result = await startTalkRealtimeAgentConsult({
|
||||
context: request.context,
|
||||
client: request.client,
|
||||
@@ -218,7 +384,19 @@ export const talkClientHandlers: GatewayRequestHandlers = {
|
||||
callId: params.callId,
|
||||
args: params.args ?? {},
|
||||
relaySessionId: normalizeOptionalString(params.relaySessionId),
|
||||
connId: normalizeOptionalString(request.client?.connId),
|
||||
connId,
|
||||
onRunStarted: (runId) => {
|
||||
registerClientVoiceConsultRun({
|
||||
agentId,
|
||||
sessionKey: params.sessionKey,
|
||||
voiceSessionId,
|
||||
runId,
|
||||
config: request.context.getRuntimeConfig(),
|
||||
});
|
||||
if (confirmationGrant) {
|
||||
bindAuthorizedClientVoiceConfirmation({ grant: confirmationGrant, runId });
|
||||
}
|
||||
},
|
||||
});
|
||||
if (!result.ok) {
|
||||
respond(false, undefined, result.error);
|
||||
@@ -233,6 +411,74 @@ export const talkClientHandlers: GatewayRequestHandlers = {
|
||||
undefined,
|
||||
);
|
||||
},
|
||||
"talk.client.transcript": async ({ params, respond, context }) => {
|
||||
if (!validateTalkClientTranscriptParams(params)) {
|
||||
respond(
|
||||
false,
|
||||
undefined,
|
||||
errorShape(
|
||||
ErrorCodes.INVALID_REQUEST,
|
||||
`invalid talk.client.transcript params: ${formatValidationErrors(validateTalkClientTranscriptParams.errors)}`,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await appendClientVoiceTranscript({
|
||||
agentId: resolveAgentIdFromSessionKey(params.sessionKey),
|
||||
sessionKey: params.sessionKey,
|
||||
voiceSessionId: params.voiceSessionId,
|
||||
entryId: params.entryId,
|
||||
role: params.role,
|
||||
text: params.text,
|
||||
...(params.timestamp !== undefined ? { timestamp: params.timestamp } : {}),
|
||||
config: context.getRuntimeConfig(),
|
||||
});
|
||||
respond(true, { ok: true }, undefined);
|
||||
} catch (err) {
|
||||
respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, formatForLog(err)));
|
||||
}
|
||||
},
|
||||
"talk.client.close": async ({ params, respond, context, client }) => {
|
||||
if (!validateTalkClientCloseParams(params)) {
|
||||
respond(
|
||||
false,
|
||||
undefined,
|
||||
errorShape(
|
||||
ErrorCodes.INVALID_REQUEST,
|
||||
`invalid talk.client.close params: ${formatValidationErrors(validateTalkClientCloseParams.errors)}`,
|
||||
),
|
||||
);
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const agentId = resolveAgentIdFromSessionKey(params.sessionKey);
|
||||
const origin = resolveClientVoiceSessionOrigin({
|
||||
agentId,
|
||||
sessionKey: params.sessionKey,
|
||||
voiceSessionId: params.voiceSessionId,
|
||||
});
|
||||
if (origin === "relay") {
|
||||
throw new Error("relay-owned voice sessions close through talk.session.stop");
|
||||
}
|
||||
await closeClientVoiceSession({
|
||||
agentId,
|
||||
sessionKey: params.sessionKey,
|
||||
voiceSessionId: params.voiceSessionId,
|
||||
config: context.getRuntimeConfig(),
|
||||
});
|
||||
const connId = normalizeOptionalString(client?.connId);
|
||||
if (connId) {
|
||||
const key = legacyVoiceBindingKey(connId, params.sessionKey);
|
||||
if (legacyVoiceSessionByClient.get(key)?.voiceSessionId === params.voiceSessionId) {
|
||||
legacyVoiceSessionByClient.delete(key);
|
||||
}
|
||||
}
|
||||
respond(true, { ok: true }, undefined);
|
||||
} catch (err) {
|
||||
respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, formatForLog(err)));
|
||||
}
|
||||
},
|
||||
"talk.client.steer": async ({ params, respond, client, context }) => {
|
||||
if (!validateTalkClientSteerParams(params)) {
|
||||
respond(
|
||||
|
||||
@@ -19,8 +19,8 @@ import {
|
||||
import { REALTIME_VOICE_AGENT_CONSULT_TOOL } from "../../talk/agent-consult-tool.js";
|
||||
import { REALTIME_VOICE_AGENT_CONTROL_TOOL } from "../../talk/agent-run-control-shared.js";
|
||||
import { controlRealtimeVoiceAgentRun } from "../../talk/agent-run-control.js";
|
||||
import { ensureClientVoiceAgentSessionEntry } from "../../talk/client-voice-session.js";
|
||||
import { resolveConfiguredRealtimeVoiceProvider } from "../../talk/provider-resolver.js";
|
||||
import type { TalkBrain, TalkMode, TalkTransport } from "../../talk/talk-events.js";
|
||||
import { ADMIN_SCOPE } from "../operator-scopes.js";
|
||||
import { resolveSessionKeyFromResolveParams } from "../sessions-resolve.js";
|
||||
import {
|
||||
@@ -63,49 +63,20 @@ import {
|
||||
buildTalkRealtimeConfig,
|
||||
buildTalkTranscriptionConfig,
|
||||
canUseTalkDirectTools,
|
||||
normalizeTalkSessionBrain,
|
||||
normalizeTalkSessionMode,
|
||||
normalizeTalkSessionTransport,
|
||||
resolveConfiguredRealtimeTranscriptionProvider,
|
||||
resolveTalkRealtimeProviderInstructions,
|
||||
talkHandoffErrorCode,
|
||||
withRealtimeBrowserOverrides,
|
||||
} from "./talk-shared.js";
|
||||
import type { GatewayRequestContext, GatewayRequestHandlers, RespondFn } from "./types.js";
|
||||
import { assertValidParams } from "./validation.js";
|
||||
|
||||
/**
|
||||
* Gateway-managed Talk session methods for managed rooms and audio relays.
|
||||
* The public `sessionId` is resolved through the unified registry so each RPC
|
||||
* can enforce the correct connection ownership for its concrete backend.
|
||||
*/
|
||||
/** Gateway-managed Talk sessions resolve public ids through connection-owned unified records. */
|
||||
type ManagedRoomTalkSession = Extract<UnifiedTalkSessionRecord, { kind: "managed-room" }>;
|
||||
|
||||
function normalizeTalkSessionMode(params: { mode?: string; transport?: string }): TalkMode {
|
||||
const mode = normalizeOptionalLowercaseString(params.mode) as TalkMode | undefined;
|
||||
if (mode) {
|
||||
return mode;
|
||||
}
|
||||
return normalizeOptionalLowercaseString(params.transport) === "managed-room"
|
||||
? "stt-tts"
|
||||
: "realtime";
|
||||
}
|
||||
|
||||
function normalizeTalkSessionTransport(params: {
|
||||
mode: TalkMode;
|
||||
transport?: string;
|
||||
}): TalkTransport {
|
||||
const transport = normalizeOptionalLowercaseString(params.transport) as TalkTransport | undefined;
|
||||
if (transport) {
|
||||
return transport;
|
||||
}
|
||||
return params.mode === "stt-tts" ? "managed-room" : "gateway-relay";
|
||||
}
|
||||
|
||||
function normalizeTalkSessionBrain(params: { mode: TalkMode; brain?: string }): TalkBrain {
|
||||
const brain = normalizeOptionalLowercaseString(params.brain) as TalkBrain | undefined;
|
||||
if (brain) {
|
||||
return brain;
|
||||
}
|
||||
return params.mode === "transcription" ? "none" : "agent-consult";
|
||||
}
|
||||
|
||||
function isActiveManagedRoomClient(
|
||||
session: { handoffId: string },
|
||||
connId: string | undefined,
|
||||
@@ -321,16 +292,29 @@ export const talkSessionHandlers: GatewayRequestHandlers = {
|
||||
requested: params,
|
||||
defaults: realtimeConfig,
|
||||
});
|
||||
const realtimeContext = await resolveTalkRealtimeProviderInstructions({
|
||||
config: runtimeConfig,
|
||||
configuredInstructions: realtimeConfig.instructions,
|
||||
sessionKey: params.sessionKey,
|
||||
requireSessionKeyForProfile: true,
|
||||
warn: (message) => context.logGateway.warn(`talk realtime context: ${message}`),
|
||||
});
|
||||
if (realtimeContext.requestedSessionKey) {
|
||||
await ensureClientVoiceAgentSessionEntry({
|
||||
agentId: realtimeContext.agentId,
|
||||
sessionKey: realtimeContext.requestedSessionKey,
|
||||
});
|
||||
}
|
||||
const session = createTalkRealtimeRelaySession({
|
||||
context,
|
||||
connId,
|
||||
cfg: runtimeConfig,
|
||||
provider: resolution.provider,
|
||||
providerConfig: withRealtimeBrowserOverrides(resolution.providerConfig, launchOptions),
|
||||
instructions: buildRealtimeInstructions(realtimeConfig.instructions),
|
||||
instructions: buildRealtimeInstructions(realtimeContext.instructions),
|
||||
tools: [REALTIME_VOICE_AGENT_CONSULT_TOOL, REALTIME_VOICE_AGENT_CONTROL_TOOL],
|
||||
model: launchOptions.model,
|
||||
sessionKey: normalizeOptionalString(params.sessionKey),
|
||||
sessionKey: realtimeContext.requestedSessionKey,
|
||||
voice: launchOptions.voice,
|
||||
language: normalizeOptionalLowercaseString(params.language),
|
||||
forceAgentConsultOnFinalTranscript:
|
||||
@@ -344,6 +328,8 @@ export const talkSessionHandlers: GatewayRequestHandlers = {
|
||||
respondOk(respond, {
|
||||
...session,
|
||||
sessionId: session.relaySessionId,
|
||||
// The relay session is the logical voice call; clients need not synthesize it.
|
||||
voiceSessionId: session.relaySessionId,
|
||||
mode,
|
||||
brain,
|
||||
});
|
||||
|
||||
@@ -13,6 +13,8 @@ import {
|
||||
resolveSupportedVoiceModelRefs,
|
||||
type VoiceModelProvider,
|
||||
} from "../../../packages/speech-core/voice-models.js";
|
||||
import { resolveDefaultAgentId } from "../../agents/agent-scope.js";
|
||||
import { resolveRealtimeBootstrapContextInstructions } from "../../agents/realtime-bootstrap-context.js";
|
||||
import type { TalkRealtimeConfig } from "../../config/types.gateway.js";
|
||||
import type { OpenClawConfig } from "../../config/types.js";
|
||||
import {
|
||||
@@ -20,6 +22,7 @@ import {
|
||||
listRealtimeTranscriptionProviders,
|
||||
} from "../../realtime-transcription/provider-registry.js";
|
||||
import type { RealtimeTranscriptionProviderConfig } from "../../realtime-transcription/provider-types.js";
|
||||
import { resolveAgentIdFromSessionKey } from "../../routing/session-key.js";
|
||||
import { REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME } from "../../talk/agent-consult-tool.js";
|
||||
import { REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME } from "../../talk/agent-run-control-shared.js";
|
||||
import { listRealtimeVoiceProviders } from "../../talk/provider-registry.js";
|
||||
@@ -27,10 +30,71 @@ import type {
|
||||
RealtimeVoiceBrowserSession,
|
||||
RealtimeVoiceProviderConfig,
|
||||
} from "../../talk/provider-types.js";
|
||||
import type { TalkEvent } from "../../talk/talk-events.js";
|
||||
import type { TalkBrain, TalkEvent, TalkMode, TalkTransport } from "../../talk/talk-events.js";
|
||||
import { ADMIN_SCOPE } from "../operator-scopes.js";
|
||||
import type { TalkHandoffTurnResult } from "../talk-handoff.js";
|
||||
|
||||
/** Resolve the Talk session mode, defaulting managed-room transports to stt-tts. */
|
||||
export function normalizeTalkSessionMode(params: { mode?: string; transport?: string }): TalkMode {
|
||||
return (
|
||||
(normalizeOptionalLowercaseString(params.mode) as TalkMode | undefined) ??
|
||||
(normalizeOptionalLowercaseString(params.transport) === "managed-room" ? "stt-tts" : "realtime")
|
||||
);
|
||||
}
|
||||
|
||||
/** Resolve the Talk session transport from mode when the client omits it. */
|
||||
export function normalizeTalkSessionTransport(params: {
|
||||
mode: TalkMode;
|
||||
transport?: string;
|
||||
}): TalkTransport {
|
||||
const transport = normalizeOptionalLowercaseString(params.transport) as TalkTransport | undefined;
|
||||
if (transport) {
|
||||
return transport;
|
||||
}
|
||||
return params.mode === "stt-tts" ? "managed-room" : "gateway-relay";
|
||||
}
|
||||
|
||||
/** Resolve the Talk session brain, defaulting transcription sessions to none. */
|
||||
export function normalizeTalkSessionBrain(params: { mode: TalkMode; brain?: string }): TalkBrain {
|
||||
const brain = normalizeOptionalLowercaseString(params.brain) as TalkBrain | undefined;
|
||||
if (brain) {
|
||||
return brain;
|
||||
}
|
||||
return params.mode === "transcription" ? "none" : "agent-consult";
|
||||
}
|
||||
|
||||
export async function resolveTalkRealtimeProviderInstructions(params: {
|
||||
config: OpenClawConfig;
|
||||
configuredInstructions?: string;
|
||||
sessionKey?: unknown;
|
||||
/** Relay sessions bind their agent lazily; injecting a guessed profile would mix agents. */
|
||||
requireSessionKeyForProfile?: boolean;
|
||||
warn: (message: string) => void;
|
||||
}): Promise<{ agentId: string; instructions: string; requestedSessionKey?: string }> {
|
||||
const requestedSessionKey = normalizeOptionalString(params.sessionKey);
|
||||
// Older clients can prefetch without a key. Client-owned creates bind to the
|
||||
// default agent immediately, so its workspace profile stays consistent there.
|
||||
const agentId = requestedSessionKey
|
||||
? resolveAgentIdFromSessionKey(requestedSessionKey)
|
||||
: resolveDefaultAgentId(params.config);
|
||||
const bootstrapContext =
|
||||
params.requireSessionKeyForProfile && !requestedSessionKey
|
||||
? undefined
|
||||
: await resolveRealtimeBootstrapContextInstructions({
|
||||
agentId,
|
||||
config: params.config,
|
||||
sessionKey: requestedSessionKey,
|
||||
warn: params.warn,
|
||||
});
|
||||
return {
|
||||
agentId,
|
||||
instructions: [params.configuredInstructions, bootstrapContext]
|
||||
.filter((entry): entry is string => Boolean(entry?.trim()))
|
||||
.join("\n\n"),
|
||||
...(requestedSessionKey ? { requestedSessionKey } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function canUseTalkDirectTools(client: { connect?: { scopes?: string[] } } | null): boolean {
|
||||
const scopes = Array.isArray(client?.connect?.scopes) ? client.connect.scopes : [];
|
||||
return scopes.includes(ADMIN_SCOPE);
|
||||
|
||||
@@ -36,6 +36,8 @@ const mocks = vi.hoisted(() => ({
|
||||
cancelTalkRealtimeRelayTurn: vi.fn(),
|
||||
stopTalkRealtimeRelaySession: vi.fn(),
|
||||
registerTalkRealtimeRelayAgentRun: vi.fn(),
|
||||
flushTalkRealtimeRelayVoiceWrites: vi.fn(async () => undefined),
|
||||
ensureTalkRealtimeRelayVoiceSession: vi.fn(),
|
||||
submitTalkRealtimeRelayToolResult: vi.fn(),
|
||||
createTalkTranscriptionRelaySession: vi.fn(),
|
||||
sendTalkTranscriptionRelayAudio: vi.fn(),
|
||||
@@ -45,6 +47,15 @@ const mocks = vi.hoisted(() => ({
|
||||
controlRealtimeVoiceAgentRun: vi.fn(),
|
||||
steerTalkRealtimeRelayAgentRun: vi.fn(),
|
||||
resolveSessionKeyFromResolveParams: vi.fn(),
|
||||
resolveRealtimeBootstrapContextInstructions: vi.fn(
|
||||
async (): Promise<string | undefined> => undefined,
|
||||
),
|
||||
closeStaleClientVoiceSessions: vi.fn(async () => 0),
|
||||
createOrResumeClientVoiceSession: vi.fn(() => "voice-test"),
|
||||
ensureClientVoiceAgentSessionEntry: vi.fn(async () => undefined),
|
||||
assertClientVoiceSessionOpen: vi.fn(),
|
||||
registerClientVoiceConsultRun: vi.fn(),
|
||||
resolveOpenClientVoiceSessionId: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock("../../config/config.js", () => ({
|
||||
@@ -82,6 +93,23 @@ vi.mock("../../talk/agent-run-control.js", () => ({
|
||||
controlRealtimeVoiceAgentRun: mocks.controlRealtimeVoiceAgentRun,
|
||||
}));
|
||||
|
||||
vi.mock("../../agents/realtime-bootstrap-context.js", () => ({
|
||||
resolveRealtimeBootstrapContextInstructions: mocks.resolveRealtimeBootstrapContextInstructions,
|
||||
}));
|
||||
|
||||
vi.mock("../../talk/client-voice-session.js", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("../../talk/client-voice-session.js")>();
|
||||
return {
|
||||
...actual,
|
||||
assertClientVoiceSessionOpen: mocks.assertClientVoiceSessionOpen,
|
||||
closeStaleClientVoiceSessions: mocks.closeStaleClientVoiceSessions,
|
||||
createOrResumeClientVoiceSession: mocks.createOrResumeClientVoiceSession,
|
||||
ensureClientVoiceAgentSessionEntry: mocks.ensureClientVoiceAgentSessionEntry,
|
||||
registerClientVoiceConsultRun: mocks.registerClientVoiceConsultRun,
|
||||
resolveOpenClientVoiceSessionId: mocks.resolveOpenClientVoiceSessionId,
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("./chat.js", () => ({
|
||||
chatHandlers: {
|
||||
"chat.send": mocks.chatSend,
|
||||
@@ -99,6 +127,8 @@ vi.mock("../talk-realtime-relay.js", async (importOriginal) => {
|
||||
acknowledgeTalkRealtimeRelayMark: mocks.acknowledgeTalkRealtimeRelayMark,
|
||||
cancelTalkRealtimeRelayTurn: mocks.cancelTalkRealtimeRelayTurn,
|
||||
createTalkRealtimeRelaySession: mocks.createTalkRealtimeRelaySession,
|
||||
ensureTalkRealtimeRelayVoiceSession: mocks.ensureTalkRealtimeRelayVoiceSession,
|
||||
flushTalkRealtimeRelayVoiceWrites: mocks.flushTalkRealtimeRelayVoiceWrites,
|
||||
registerTalkRealtimeRelayAgentRun: mocks.registerTalkRealtimeRelayAgentRun,
|
||||
sendTalkRealtimeRelayAudio: mocks.sendTalkRealtimeRelayAudio,
|
||||
steerTalkRealtimeRelayAgentRun: mocks.steerTalkRealtimeRelayAgentRun,
|
||||
@@ -1484,6 +1514,7 @@ describe("talk.session unified handlers", () => {
|
||||
)({
|
||||
req: { type: "req", id: "1", method: "talk.session.create" },
|
||||
params: {
|
||||
sessionKey: "agent:main:main",
|
||||
mode: "realtime",
|
||||
transport: "gateway-relay",
|
||||
brain: "agent-consult",
|
||||
@@ -1520,6 +1551,10 @@ describe("talk.session unified handlers", () => {
|
||||
providerConfigs: { openai: { apiKey: "openai-key" } },
|
||||
defaultModel: "gpt-realtime-default",
|
||||
});
|
||||
expect(mocks.ensureClientVoiceAgentSessionEntry).toHaveBeenCalledWith({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
});
|
||||
const relayCreateInput = mockCallArg(mocks.createTalkRealtimeRelaySession) as Record<
|
||||
string,
|
||||
unknown
|
||||
@@ -2418,6 +2453,96 @@ describe("talk.client.toolCall handler", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("implicitly creates a voice session for consults without a binding", async () => {
|
||||
const respond = vi.fn();
|
||||
|
||||
await expectDefined(
|
||||
talkHandlers["talk.client.toolCall"],
|
||||
'talkHandlers["talk.client.toolCall"] test invariant',
|
||||
)({
|
||||
req: { type: "req", id: "1", method: "talk.client.toolCall" },
|
||||
params: {
|
||||
sessionKey: "main",
|
||||
callId: "call-unbound",
|
||||
name: "openclaw_agent_consult",
|
||||
args: { question: "Do something" },
|
||||
},
|
||||
client: { connId: "conn-1" } as never,
|
||||
isWebchatConnect: () => false,
|
||||
respond: respond as never,
|
||||
context: { getRuntimeConfig: () => ({}) as OpenClawConfig } as never,
|
||||
});
|
||||
|
||||
expect(mocks.createOrResumeClientVoiceSession).toHaveBeenCalledWith({
|
||||
agentId: "main",
|
||||
sessionKey: "main",
|
||||
origin: "client",
|
||||
});
|
||||
expect(mocks.registerClientVoiceConsultRun).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ voiceSessionId: "voice-test", runId: "run-voice-1" }),
|
||||
);
|
||||
expect(mocks.chatSend).toHaveBeenCalledTimes(1);
|
||||
expect(respond).toHaveBeenCalledWith(true, expect.anything(), undefined);
|
||||
});
|
||||
|
||||
it("resolves a legacy consult to the open client voice record", async () => {
|
||||
mocks.resolveOpenClientVoiceSessionId.mockReturnValueOnce("voice-test");
|
||||
const respond = vi.fn();
|
||||
|
||||
await expectDefined(
|
||||
talkHandlers["talk.client.toolCall"],
|
||||
'talkHandlers["talk.client.toolCall"] test invariant',
|
||||
)({
|
||||
req: { type: "req", id: "legacy", method: "talk.client.toolCall" },
|
||||
params: {
|
||||
sessionKey: "main",
|
||||
callId: "call-legacy",
|
||||
name: "openclaw_agent_consult",
|
||||
args: { question: "Continue the call" },
|
||||
},
|
||||
client: { connId: "conn-legacy" } as never,
|
||||
isWebchatConnect: () => false,
|
||||
respond: respond as never,
|
||||
context: { getRuntimeConfig: () => ({}) as OpenClawConfig } as never,
|
||||
});
|
||||
|
||||
expect(mocks.assertClientVoiceSessionOpen).toHaveBeenCalledWith({
|
||||
agentId: "main",
|
||||
sessionKey: "main",
|
||||
voiceSessionId: "voice-test",
|
||||
});
|
||||
expectRespondOk(respond, { runId: "run-voice-1" });
|
||||
});
|
||||
|
||||
it("requires relay connection ownership for relay-origin voice records", async () => {
|
||||
mocks.assertClientVoiceSessionOpen.mockReturnValueOnce("relay");
|
||||
const respond = vi.fn();
|
||||
|
||||
await expectDefined(
|
||||
talkHandlers["talk.client.toolCall"],
|
||||
'talkHandlers["talk.client.toolCall"] test invariant',
|
||||
)({
|
||||
req: { type: "req", id: "relay-owner", method: "talk.client.toolCall" },
|
||||
params: {
|
||||
sessionKey: "main",
|
||||
voiceSessionId: "relay-secret",
|
||||
callId: "call-relay-owner",
|
||||
name: "openclaw_agent_consult",
|
||||
args: { question: "Continue" },
|
||||
},
|
||||
client: { connId: "other-conn" } as never,
|
||||
isWebchatConnect: () => false,
|
||||
respond: respond as never,
|
||||
context: { getRuntimeConfig: () => ({}) as OpenClawConfig } as never,
|
||||
});
|
||||
|
||||
expect(mocks.chatSend).not.toHaveBeenCalled();
|
||||
expectRespondError(respond, {
|
||||
code: ErrorCodes.INVALID_REQUEST,
|
||||
message: "Error: relay-owned voice sessions require relaySessionId and connection ownership",
|
||||
});
|
||||
});
|
||||
|
||||
it("starts agent consult through gateway policy instead of exposing chat.send to browser clients", async () => {
|
||||
const respond = vi.fn();
|
||||
|
||||
@@ -2428,6 +2553,7 @@ describe("talk.client.toolCall handler", () => {
|
||||
req: { type: "req", id: "1", method: "talk.client.toolCall" },
|
||||
params: {
|
||||
sessionKey: "main",
|
||||
voiceSessionId: "voice-test",
|
||||
callId: "call-1",
|
||||
name: "openclaw_agent_consult",
|
||||
args: { question: "What is in this repo?", responseStyle: "one sentence" },
|
||||
@@ -2470,6 +2596,7 @@ describe("talk.client.toolCall handler", () => {
|
||||
req: { type: "req", id: "1", method: "talk.client.toolCall" },
|
||||
params: {
|
||||
sessionKey: "main",
|
||||
voiceSessionId: "voice-test",
|
||||
callId: "call-active",
|
||||
name: "openclaw_agent_consult",
|
||||
args: { question: "What is running?" },
|
||||
@@ -2497,6 +2624,7 @@ describe("talk.client.toolCall handler", () => {
|
||||
req: { type: "req", id: "1", method: "talk.client.toolCall" },
|
||||
params: {
|
||||
sessionKey: "main",
|
||||
voiceSessionId: "voice-test",
|
||||
callId: "call-1",
|
||||
name: "openclaw_agent_consult",
|
||||
args: { question: "Are the basement lights off?" },
|
||||
@@ -2533,6 +2661,7 @@ describe("talk.client.toolCall handler", () => {
|
||||
req: { type: "req", id: "1", method: "talk.client.toolCall" },
|
||||
params: {
|
||||
sessionKey: "main",
|
||||
voiceSessionId: "relay-1",
|
||||
relaySessionId: "relay-1",
|
||||
callId: "call-1",
|
||||
name: "openclaw_agent_consult",
|
||||
@@ -2581,6 +2710,7 @@ describe("talk.client.toolCall handler", () => {
|
||||
req: { type: "req", id: "1", method: "talk.client.toolCall" },
|
||||
params: {
|
||||
sessionKey: "main",
|
||||
voiceSessionId: "relay-1",
|
||||
relaySessionId: "relay-1",
|
||||
callId: "call-1",
|
||||
name: "openclaw_agent_consult",
|
||||
@@ -2749,6 +2879,8 @@ describe("talk.client.steer handler", () => {
|
||||
describe("talk.client.create handler", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mocks.resolveRealtimeBootstrapContextInstructions.mockResolvedValue(undefined);
|
||||
mocks.createOrResumeClientVoiceSession.mockReturnValue("voice-test");
|
||||
});
|
||||
|
||||
it("builds realtime launch defaults from talk.realtime", () => {
|
||||
@@ -2772,6 +2904,7 @@ describe("talk.client.create handler", () => {
|
||||
});
|
||||
|
||||
it("uses talk.realtime provider, model, voice, and instructions without reading speech provider config", async () => {
|
||||
mocks.resolveRealtimeBootstrapContextInstructions.mockResolvedValue("Bounded profile context.");
|
||||
const createBrowserSession = vi.fn(async (_input: unknown) => ({
|
||||
provider: "openai",
|
||||
transport: "webrtc" as const,
|
||||
@@ -2838,6 +2971,7 @@ describe("talk.client.create handler", () => {
|
||||
reasoningEffort: "low",
|
||||
});
|
||||
expect(createInput.instructions).toContain("Additional realtime instructions:\nSpeak warmly.");
|
||||
expect(createInput.instructions).toContain("Bounded profile context.");
|
||||
expect(createInput.instructions).toContain("tool-backed actions");
|
||||
expect(createInput.instructions).toContain("Let me check that for you");
|
||||
expect(createInput.tools).not.toContainEqual(
|
||||
@@ -2846,7 +2980,18 @@ describe("talk.client.create handler", () => {
|
||||
expect(createInput).not.toHaveProperty("provider");
|
||||
expect(createInput).not.toHaveProperty("providers");
|
||||
expect(createInput).not.toHaveProperty("transport");
|
||||
expectRespondOk(respond, { provider: "openai", transport: "webrtc" });
|
||||
expect(mocks.ensureClientVoiceAgentSessionEntry).toHaveBeenCalledWith({
|
||||
agentId: "main",
|
||||
sessionKey: "main",
|
||||
});
|
||||
expect(mocks.createOrResumeClientVoiceSession).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ provider: "openai" }),
|
||||
);
|
||||
expectRespondOk(respond, {
|
||||
provider: "openai",
|
||||
transport: "webrtc",
|
||||
voiceSessionId: "voice-test",
|
||||
});
|
||||
});
|
||||
|
||||
it("adds describe_view to camera clients whose provider supports video frames", async () => {
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
} from "../../packages/gateway-protocol/src/index.js";
|
||||
import { normalizeTalkSection } from "../config/talk.js";
|
||||
import { buildRealtimeVoiceAgentConsultChatMessage } from "../talk/agent-consult-tool.js";
|
||||
import { abortChatRunById } from "./chat-abort.js";
|
||||
import { chatHandlers } from "./server-methods/chat.js";
|
||||
import type {
|
||||
GatewayClient,
|
||||
@@ -66,6 +67,7 @@ export async function startTalkRealtimeAgentConsult(params: {
|
||||
args: unknown;
|
||||
relaySessionId?: string;
|
||||
connId?: string;
|
||||
onRunStarted?: (runId: string) => void;
|
||||
}): Promise<
|
||||
{ ok: true; runId: string; idempotencyKey: string } | { ok: false; error: ErrorShape }
|
||||
> {
|
||||
@@ -77,6 +79,7 @@ export async function startTalkRealtimeAgentConsult(params: {
|
||||
}
|
||||
const idempotencyKey = `talk-${params.callId}-${randomUUID()}`;
|
||||
const normalizedTalk = normalizeTalkSection(params.context.getRuntimeConfig().talk);
|
||||
let acknowledgedRunId: string | undefined;
|
||||
const chatResponse = await new Promise<
|
||||
{ ok: true; result: unknown } | { ok: false; error: ErrorShape } | undefined
|
||||
>((resolve) => {
|
||||
@@ -106,6 +109,37 @@ export async function startTalkRealtimeAgentConsult(params: {
|
||||
},
|
||||
respond: (ok: boolean, result?: unknown, error?: ErrorShape) => {
|
||||
acknowledged = true;
|
||||
if (ok && !terminalTalkChatSendAckError(normalizeTalkChatSendAckStatus(result))) {
|
||||
const candidateRunId =
|
||||
result && typeof result === "object" && !Array.isArray(result)
|
||||
? (result as Record<string, unknown>).runId
|
||||
: undefined;
|
||||
const runId = typeof candidateRunId === "string" ? candidateRunId : idempotencyKey;
|
||||
try {
|
||||
if (params.relaySessionId && params.connId) {
|
||||
registerTalkRealtimeRelayAgentRun({
|
||||
relaySessionId: params.relaySessionId,
|
||||
connId: params.connId,
|
||||
sessionKey: params.sessionKey,
|
||||
runId,
|
||||
callId: params.callId,
|
||||
});
|
||||
}
|
||||
params.onRunStarted?.(runId);
|
||||
acknowledgedRunId = runId;
|
||||
} catch (registrationError) {
|
||||
abortChatRunById(params.context, {
|
||||
runId,
|
||||
sessionKey: params.sessionKey,
|
||||
stopReason: "voice session binding failed",
|
||||
});
|
||||
resolve({
|
||||
ok: false,
|
||||
error: errorShape(ErrorCodes.UNAVAILABLE, formatForLog(registrationError)),
|
||||
});
|
||||
return;
|
||||
}
|
||||
}
|
||||
resolve(
|
||||
ok
|
||||
? { ok: true, result }
|
||||
@@ -152,20 +186,6 @@ export async function startTalkRealtimeAgentConsult(params: {
|
||||
if (terminalAckError) {
|
||||
return { ok: false, error: terminalAckError };
|
||||
}
|
||||
const runId =
|
||||
result && typeof result === "object" && !Array.isArray(result)
|
||||
? typeof (result as Record<string, unknown>).runId === "string"
|
||||
? (result as Record<string, string>).runId
|
||||
: idempotencyKey
|
||||
: idempotencyKey;
|
||||
if (params.relaySessionId && params.connId) {
|
||||
registerTalkRealtimeRelayAgentRun({
|
||||
relaySessionId: params.relaySessionId,
|
||||
connId: params.connId,
|
||||
sessionKey: params.sessionKey,
|
||||
runId: expectDefined(runId, "talk agent run id"),
|
||||
callId: params.callId,
|
||||
});
|
||||
}
|
||||
const runId = expectDefined(acknowledgedRunId, "talk agent run id");
|
||||
return { ok: true, runId: expectDefined(runId, "talk agent run id"), idempotencyKey };
|
||||
}
|
||||
|
||||
@@ -1,18 +1,30 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
/**
|
||||
* Tests talk realtime relay event forwarding and connection cleanup.
|
||||
*/
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { setActiveEmbeddedRun } from "../agents/embedded-agent-runner/runs.js";
|
||||
import { testing as embeddedRunTesting } from "../agents/embedded-agent-runner/runs.test-support.js";
|
||||
import {
|
||||
readSessionTranscriptMessageEvents,
|
||||
replaceSessionEntry,
|
||||
} from "../config/sessions/session-accessor.js";
|
||||
import type { RealtimeVoiceProviderPlugin } from "../plugins/types.js";
|
||||
import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js";
|
||||
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
|
||||
import { clientVoiceSessionTesting } from "../talk/client-voice-session.test-support.js";
|
||||
import type {
|
||||
RealtimeVoiceBridge,
|
||||
RealtimeVoiceBridgeCreateRequest,
|
||||
} from "../talk/provider-types.js";
|
||||
import { captureEnv, setTestEnvValue } from "../test-utils/env.js";
|
||||
import {
|
||||
acknowledgeTalkRealtimeRelayMark,
|
||||
cancelTalkRealtimeRelayTurn,
|
||||
createTalkRealtimeRelaySession as createTalkRealtimeRelaySessionRaw,
|
||||
flushTalkRealtimeRelayVoiceWrites,
|
||||
registerTalkRealtimeRelayAgentRun,
|
||||
sendTalkRealtimeRelayAudio,
|
||||
steerTalkRealtimeRelayAgentRun,
|
||||
@@ -74,6 +86,196 @@ describe("talk realtime gateway relay", () => {
|
||||
};
|
||||
}
|
||||
|
||||
it("appends finalized relay transcripts to the canonical agent session", async () => {
|
||||
const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
|
||||
const tempDir = await fs.realpath(
|
||||
await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-relay-voice-")),
|
||||
);
|
||||
setTestEnvValue("OPENCLAW_STATE_DIR", tempDir);
|
||||
let bridgeRequest: RealtimeVoiceBridgeCreateRequest | undefined;
|
||||
try {
|
||||
await replaceSessionEntry(
|
||||
{ agentId: "main", sessionKey: "agent:main:main" },
|
||||
{ sessionId: "relay-voice-session", updatedAt: Date.now() },
|
||||
);
|
||||
const provider = createIdleRelayProvider();
|
||||
provider.createBridge = (request) => {
|
||||
bridgeRequest = request;
|
||||
return createIdleRelayProvider().createBridge?.(request) as RealtimeVoiceBridge;
|
||||
};
|
||||
const session = createTalkRealtimeRelaySession({
|
||||
context: {
|
||||
broadcastToConnIds: vi.fn(),
|
||||
chatAbortControllers: new Map(),
|
||||
getRuntimeConfig: () => ({}),
|
||||
logGateway: { warn: vi.fn() },
|
||||
} as never,
|
||||
connId: "conn-voice",
|
||||
cfg: {},
|
||||
provider,
|
||||
providerConfig: {},
|
||||
instructions: "brief",
|
||||
tools: [],
|
||||
sessionKey: "agent:main:main",
|
||||
});
|
||||
bridgeRequest?.onTranscript?.("user", "relay hello", true);
|
||||
bridgeRequest?.onTranscript?.("assistant", "relay response", true);
|
||||
await flushTalkRealtimeRelayVoiceWrites({
|
||||
relaySessionId: session.relaySessionId,
|
||||
connId: "conn-voice",
|
||||
});
|
||||
|
||||
const events = readSessionTranscriptMessageEvents({
|
||||
agentId: "main",
|
||||
sessionId: "relay-voice-session",
|
||||
});
|
||||
expect(events).toHaveLength(2);
|
||||
expect(events[0]?.event).toMatchObject({
|
||||
id: `voice:${session.relaySessionId}:1`,
|
||||
message: { role: "user", content: [{ type: "text", text: "relay hello" }] },
|
||||
});
|
||||
expect(events[1]?.event).toMatchObject({
|
||||
id: `voice:${session.relaySessionId}:2`,
|
||||
message: {
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: "relay response" }],
|
||||
api: "realtime",
|
||||
provider: "relay-test",
|
||||
model: "realtime-voice",
|
||||
},
|
||||
});
|
||||
stopTalkRealtimeRelaySession({
|
||||
relaySessionId: session.relaySessionId,
|
||||
connId: "conn-voice",
|
||||
});
|
||||
await vi.waitFor(() =>
|
||||
expect(clientVoiceSessionTesting.readRecord("main", session.relaySessionId)?.status).toBe(
|
||||
"closed",
|
||||
),
|
||||
);
|
||||
} finally {
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
envSnapshot.restore();
|
||||
await fs.rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("creates the relay voice record before binding a transcript-free consult", async () => {
|
||||
const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
|
||||
const tempDir = await fs.realpath(
|
||||
await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-relay-voice-consult-")),
|
||||
);
|
||||
setTestEnvValue("OPENCLAW_STATE_DIR", tempDir);
|
||||
try {
|
||||
const session = createTalkRealtimeRelaySession({
|
||||
context: {
|
||||
broadcastToConnIds: vi.fn(),
|
||||
chatAbortControllers: new Map(),
|
||||
getRuntimeConfig: () => ({}),
|
||||
logGateway: { warn: vi.fn() },
|
||||
} as never,
|
||||
connId: "conn-consult",
|
||||
cfg: {},
|
||||
provider: createIdleRelayProvider(),
|
||||
providerConfig: {},
|
||||
instructions: "brief",
|
||||
tools: [],
|
||||
sessionKey: "agent:main:main",
|
||||
});
|
||||
expect(clientVoiceSessionTesting.readRecord("main", session.relaySessionId)).toBeUndefined();
|
||||
|
||||
registerTalkRealtimeRelayAgentRun({
|
||||
relaySessionId: session.relaySessionId,
|
||||
connId: "conn-consult",
|
||||
sessionKey: "agent:main:main",
|
||||
runId: "run-before-transcript",
|
||||
});
|
||||
|
||||
expect(clientVoiceSessionTesting.readRecord("main", session.relaySessionId)).toMatchObject({
|
||||
status: "open",
|
||||
consultRunIds: ["run-before-transcript"],
|
||||
});
|
||||
stopTalkRealtimeRelaySession({
|
||||
relaySessionId: session.relaySessionId,
|
||||
connId: "conn-consult",
|
||||
});
|
||||
await vi.waitFor(() =>
|
||||
expect(clientVoiceSessionTesting.readRecord("main", session.relaySessionId)?.status).toBe(
|
||||
"closed",
|
||||
),
|
||||
);
|
||||
} finally {
|
||||
clientVoiceSessionTesting.reset();
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
envSnapshot.restore();
|
||||
await fs.rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
it("logs relay transcript append failures", async () => {
|
||||
const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
|
||||
const tempDir = await fs.realpath(
|
||||
await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-relay-voice-failure-")),
|
||||
);
|
||||
setTestEnvValue("OPENCLAW_STATE_DIR", tempDir);
|
||||
let bridgeRequest: RealtimeVoiceBridgeCreateRequest | undefined;
|
||||
const warn = vi.fn();
|
||||
try {
|
||||
vi.useFakeTimers();
|
||||
const provider = createIdleRelayProvider();
|
||||
provider.createBridge = (request) => {
|
||||
bridgeRequest = request;
|
||||
return createIdleRelayProvider().createBridge?.(request) as RealtimeVoiceBridge;
|
||||
};
|
||||
const session = createTalkRealtimeRelaySession({
|
||||
context: {
|
||||
broadcastToConnIds: vi.fn(),
|
||||
getRuntimeConfig: () => ({}),
|
||||
logGateway: { warn },
|
||||
} as never,
|
||||
connId: "conn-voice-failure",
|
||||
cfg: {},
|
||||
provider,
|
||||
providerConfig: {},
|
||||
instructions: "brief",
|
||||
tools: [],
|
||||
sessionKey: "agent:main:missing",
|
||||
});
|
||||
bridgeRequest?.onTranscript?.("user", "cannot persist", true);
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
expect(warn).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(500);
|
||||
expect(warn).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(1_999);
|
||||
expect(warn).not.toHaveBeenCalled();
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
await flushTalkRealtimeRelayVoiceWrites({
|
||||
relaySessionId: session.relaySessionId,
|
||||
connId: "conn-voice-failure",
|
||||
});
|
||||
|
||||
expect(warn).toHaveBeenCalledExactlyOnceWith(
|
||||
expect.stringContaining("realtime relay transcript append failed"),
|
||||
);
|
||||
stopTalkRealtimeRelaySession({
|
||||
relaySessionId: session.relaySessionId,
|
||||
connId: "conn-voice-failure",
|
||||
});
|
||||
await vi.waitFor(() =>
|
||||
expect(clientVoiceSessionTesting.readRecord("main", session.relaySessionId)?.status).toBe(
|
||||
"closed",
|
||||
),
|
||||
);
|
||||
} finally {
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
envSnapshot.restore();
|
||||
await fs.rm(tempDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
function createDeferredVoid(): { promise: Promise<void>; resolve: () => void } {
|
||||
let resolve!: () => void;
|
||||
const promise = new Promise<void>((accept) => {
|
||||
|
||||
@@ -5,6 +5,7 @@ import { resolveExpiresAtMsFromDurationMs } from "@openclaw/normalization-core/n
|
||||
import type { OpenClawConfig } from "../config/types.js";
|
||||
import { formatErrorMessage } from "../infra/errors.js";
|
||||
import type { RealtimeVoiceProviderPlugin } from "../plugins/types.js";
|
||||
import { resolveAgentIdFromSessionKey } from "../routing/session-key.js";
|
||||
import {
|
||||
REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
|
||||
buildRealtimeVoiceAgentConsultWorkingResponse,
|
||||
@@ -16,6 +17,12 @@ import {
|
||||
shouldAutoControlRealtimeVoiceAgentText,
|
||||
type RealtimeVoiceAgentControlResult,
|
||||
} from "../talk/agent-run-control.js";
|
||||
import {
|
||||
appendRelayVoiceTranscript,
|
||||
closeClientVoiceSession,
|
||||
createOrResumeClientVoiceSession,
|
||||
registerClientVoiceConsultRun,
|
||||
} from "../talk/client-voice-session.js";
|
||||
import { readSpeakableRealtimeVoiceToolResult } from "../talk/consult-question.js";
|
||||
import {
|
||||
createRealtimeVoiceForcedConsultCoordinator,
|
||||
@@ -134,6 +141,7 @@ type RelaySession = {
|
||||
expiresAtMs: number;
|
||||
cleanupTimer: ReturnType<typeof setTimeout>;
|
||||
activeAgentRuns: Map<string, string>;
|
||||
provider: string;
|
||||
activeAgentToolCalls: Map<string, string>;
|
||||
completedAgentToolCalls: Set<string>;
|
||||
// Cancelled calls retain their original turn long enough to terminally satisfy
|
||||
@@ -154,6 +162,11 @@ type RelaySession = {
|
||||
toolResultEpoch: number;
|
||||
forcedConsults: RealtimeVoiceForcedConsultCoordinator;
|
||||
transcript: RealtimeVoiceTranscriptEntry[];
|
||||
voiceConfig?: OpenClawConfig;
|
||||
voiceSessionCreated: boolean;
|
||||
voiceTranscriptSeq: number;
|
||||
voiceTranscriptWrites: Promise<void>;
|
||||
pendingVoiceTranscripts: Array<{ role: "user" | "assistant"; text: string }>;
|
||||
};
|
||||
|
||||
type CreateTalkRealtimeRelaySessionParams = {
|
||||
@@ -182,6 +195,128 @@ type TalkRealtimeRelaySessionResult = {
|
||||
};
|
||||
|
||||
const relaySessions = new Map<string, RelaySession>();
|
||||
const RELAY_TRANSCRIPT_RETRY_DELAYS_MS = [0, 500, 2_000] as const;
|
||||
|
||||
function logRelayVoiceFailure(session: RelaySession, message: string, error: unknown): void {
|
||||
session.context.logGateway?.warn(`${message}: ${formatErrorMessage(error)}`);
|
||||
}
|
||||
|
||||
function ensureRelayVoiceSession(session: RelaySession): boolean {
|
||||
if (session.voiceSessionCreated) {
|
||||
return true;
|
||||
}
|
||||
if (!session.sessionKey) {
|
||||
return false;
|
||||
}
|
||||
try {
|
||||
createOrResumeClientVoiceSession({
|
||||
agentId: resolveAgentIdFromSessionKey(session.sessionKey),
|
||||
sessionKey: session.sessionKey,
|
||||
provider: session.provider,
|
||||
origin: "relay",
|
||||
voiceSessionId: session.id,
|
||||
});
|
||||
session.voiceSessionCreated = true;
|
||||
return true;
|
||||
} catch (error) {
|
||||
logRelayVoiceFailure(session, "realtime relay voice session create failed", error);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
const MAX_PENDING_VOICE_TRANSCRIPTS = 40;
|
||||
|
||||
function enqueueRelayVoiceTranscript(
|
||||
session: RelaySession,
|
||||
role: "user" | "assistant",
|
||||
text: string,
|
||||
): void {
|
||||
if (!session.sessionKey) {
|
||||
// Lazy-bound relays hear audio before talk.client.toolCall supplies the session
|
||||
// key; buffer bounded finals so the call's opening turns survive the binding.
|
||||
// Never-binding callers accept best-effort loss: they had no persistence before.
|
||||
session.pendingVoiceTranscripts.push({ role, text });
|
||||
if (session.pendingVoiceTranscripts.length > MAX_PENDING_VOICE_TRANSCRIPTS) {
|
||||
session.pendingVoiceTranscripts.shift();
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (!ensureRelayVoiceSession(session)) {
|
||||
return;
|
||||
}
|
||||
session.voiceTranscriptSeq += 1;
|
||||
const entryId = String(session.voiceTranscriptSeq);
|
||||
const sessionKey = session.sessionKey;
|
||||
session.voiceTranscriptWrites = session.voiceTranscriptWrites
|
||||
.then(async () => {
|
||||
let lastError: unknown;
|
||||
for (const delayMs of RELAY_TRANSCRIPT_RETRY_DELAYS_MS) {
|
||||
if (delayMs > 0) {
|
||||
await new Promise<void>((resolve) => {
|
||||
setTimeout(resolve, delayMs);
|
||||
});
|
||||
}
|
||||
try {
|
||||
await appendRelayVoiceTranscript({
|
||||
agentId: resolveAgentIdFromSessionKey(sessionKey),
|
||||
sessionKey,
|
||||
voiceSessionId: session.id,
|
||||
entryId,
|
||||
role,
|
||||
text,
|
||||
...(session.voiceConfig ? { config: session.voiceConfig } : {}),
|
||||
});
|
||||
return;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
}
|
||||
}
|
||||
throw lastError;
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
logRelayVoiceFailure(session, "realtime relay transcript append failed", error);
|
||||
});
|
||||
}
|
||||
|
||||
function closeRelayVoiceSession(session: RelaySession): void {
|
||||
if (!session.sessionKey || !ensureRelayVoiceSession(session)) {
|
||||
return;
|
||||
}
|
||||
const sessionKey = session.sessionKey;
|
||||
void session.voiceTranscriptWrites
|
||||
.then(async () => {
|
||||
const config = session.voiceConfig ?? session.context.getRuntimeConfig();
|
||||
await closeClientVoiceSession({
|
||||
agentId: resolveAgentIdFromSessionKey(sessionKey),
|
||||
sessionKey,
|
||||
voiceSessionId: session.id,
|
||||
config,
|
||||
});
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
logRelayVoiceFailure(session, "realtime relay voice session close failed", error);
|
||||
});
|
||||
}
|
||||
|
||||
/** Ensure a gateway-relay call has its durable record before transcript-free RPCs. */
|
||||
export function ensureTalkRealtimeRelayVoiceSession(params: {
|
||||
relaySessionId: string;
|
||||
connId: string;
|
||||
sessionKey: string;
|
||||
}): void {
|
||||
const session = getRelaySession(params.relaySessionId, params.connId);
|
||||
if (session.sessionKey && session.sessionKey !== params.sessionKey) {
|
||||
throw new Error("Realtime relay session belongs to another agent session");
|
||||
}
|
||||
session.sessionKey = params.sessionKey;
|
||||
if (!ensureRelayVoiceSession(session)) {
|
||||
throw new Error("Realtime relay voice session could not be created");
|
||||
}
|
||||
const buffered = session.pendingVoiceTranscripts.splice(0);
|
||||
for (const entry of buffered) {
|
||||
enqueueRelayVoiceTranscript(session, entry.role, entry.text);
|
||||
}
|
||||
}
|
||||
|
||||
function isWorkingToolResult(result: unknown): boolean {
|
||||
return (
|
||||
@@ -538,6 +673,7 @@ function closeRelaySession(session: RelaySession, reason: "completed" | "error")
|
||||
clearTimeout(session.cleanupTimer);
|
||||
abortRelayAgentRuns(session, reason === "error" ? "relay-error" : "relay-closed");
|
||||
session.bridge.close();
|
||||
closeRelayVoiceSession(session);
|
||||
broadcastToOwner(session.context, session.connId, {
|
||||
relaySessionId: session.id,
|
||||
type: "close",
|
||||
@@ -708,6 +844,7 @@ export function createTalkRealtimeRelaySession(
|
||||
const turnId = relay ? ensureRelayTurn(relay) : undefined;
|
||||
if (final && relay) {
|
||||
recordRealtimeVoiceTranscript(relay.transcript, role, text);
|
||||
enqueueRelayVoiceTranscript(relay, role, text);
|
||||
}
|
||||
const eventType =
|
||||
role === "assistant"
|
||||
@@ -845,6 +982,7 @@ export function createTalkRealtimeRelaySession(
|
||||
forgetUnifiedTalkSession(relaySessionId);
|
||||
clearTimeout(active.cleanupTimer);
|
||||
abortRelayAgentRuns(active, "relay-closed");
|
||||
closeRelayVoiceSession(active);
|
||||
if (!ready && !failureEmitted) {
|
||||
const issue = realtimeRelayIssue({
|
||||
message: "Realtime provider closed before the session became ready.",
|
||||
@@ -879,6 +1017,7 @@ export function createTalkRealtimeRelaySession(
|
||||
}
|
||||
}, RELAY_SESSION_TTL_MS),
|
||||
activeAgentRuns: new Map(),
|
||||
provider: params.provider.id,
|
||||
activeAgentToolCalls: new Map(),
|
||||
completedAgentToolCalls: new Set(),
|
||||
cancelledAgentToolCalls: new Map(),
|
||||
@@ -890,6 +1029,11 @@ export function createTalkRealtimeRelaySession(
|
||||
toolResultEpoch: 0,
|
||||
forcedConsults: createRealtimeVoiceForcedConsultCoordinator(),
|
||||
transcript: [],
|
||||
...(params.cfg ? { voiceConfig: params.cfg } : {}),
|
||||
voiceSessionCreated: false,
|
||||
voiceTranscriptSeq: 0,
|
||||
voiceTranscriptWrites: Promise.resolve(),
|
||||
pendingVoiceTranscripts: [],
|
||||
};
|
||||
relayRef.current = relay;
|
||||
relay.cleanupTimer.unref?.();
|
||||
@@ -1352,6 +1496,24 @@ export function registerTalkRealtimeRelayAgentRun(params: {
|
||||
if (!session.sessionKey) {
|
||||
session.sessionKey = params.sessionKey;
|
||||
}
|
||||
if (!ensureRelayVoiceSession(session)) {
|
||||
throw new Error("Realtime relay voice session could not be created for agent consult");
|
||||
}
|
||||
registerClientVoiceConsultRun({
|
||||
agentId: resolveAgentIdFromSessionKey(params.sessionKey),
|
||||
sessionKey: params.sessionKey,
|
||||
voiceSessionId: session.id,
|
||||
runId: params.runId,
|
||||
});
|
||||
}
|
||||
|
||||
/** Wait for server-owned final transcript appends before a relay consult is authorized. */
|
||||
export async function flushTalkRealtimeRelayVoiceWrites(params: {
|
||||
relaySessionId: string;
|
||||
connId: string;
|
||||
}): Promise<void> {
|
||||
const session = getRelaySession(params.relaySessionId, params.connId);
|
||||
await session.voiceTranscriptWrites;
|
||||
}
|
||||
|
||||
/** Applies realtime voice-control text to the active agent-consult chat run. */
|
||||
|
||||
@@ -482,6 +482,8 @@ type DiagnosticToolExecutionBaseEvent = DiagnosticBaseEvent & {
|
||||
toolOwner?: string;
|
||||
toolCallId?: string;
|
||||
paramsSummary?: DiagnosticToolParamsSummary;
|
||||
/** Deterministic mutation classification computed before tool execution. */
|
||||
mutatingAction?: boolean;
|
||||
};
|
||||
|
||||
export type DiagnosticToolExecutionStartedEvent = DiagnosticToolExecutionBaseEvent & {
|
||||
|
||||
@@ -30,6 +30,20 @@ describe("realtime voice agent consult tool", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("normalizes a server-issued spoken confirmation id", () => {
|
||||
expect(
|
||||
parseRealtimeVoiceAgentConsultArgs({
|
||||
question: "Send it now",
|
||||
confirmationId: " confirm-123 ",
|
||||
}),
|
||||
).toStrictEqual({
|
||||
question: "Send it now",
|
||||
context: undefined,
|
||||
responseStyle: undefined,
|
||||
confirmationId: "confirm-123",
|
||||
});
|
||||
});
|
||||
|
||||
it("accepts provider question aliases from realtime tool calls", () => {
|
||||
expect(parseRealtimeVoiceAgentConsultArgs({ prompt: " Check the repo. " })).toStrictEqual({
|
||||
context: undefined,
|
||||
|
||||
@@ -26,6 +26,7 @@ export type RealtimeVoiceAgentConsultArgs = {
|
||||
question: string;
|
||||
context?: string;
|
||||
responseStyle?: string;
|
||||
confirmationId?: string;
|
||||
};
|
||||
/** Compact transcript entry included in delegated agent prompts. */
|
||||
export type RealtimeVoiceAgentConsultTranscriptEntry = {
|
||||
@@ -54,6 +55,11 @@ export const REALTIME_VOICE_AGENT_CONSULT_TOOL: RealtimeVoiceTool = {
|
||||
type: "string",
|
||||
description: "Optional style hint for the spoken answer.",
|
||||
},
|
||||
confirmationId: {
|
||||
type: "string",
|
||||
description:
|
||||
"Server-issued confirmation id from a prior VOICE_CONFIRMATION_REQUIRED result, supplied only after the user explicitly confirms aloud.",
|
||||
},
|
||||
},
|
||||
required: ["question"],
|
||||
},
|
||||
@@ -177,10 +183,14 @@ export function parseRealtimeVoiceAgentConsultArgs(args: unknown): RealtimeVoice
|
||||
if (!question) {
|
||||
throw new Error("question required");
|
||||
}
|
||||
const context = readConsultStringArg(args, "context");
|
||||
const responseStyle = readConsultStringArg(args, "responseStyle");
|
||||
const confirmationId = readConsultStringArg(args, "confirmationId");
|
||||
return {
|
||||
question,
|
||||
context: readConsultStringArg(args, "context"),
|
||||
responseStyle: readConsultStringArg(args, "responseStyle"),
|
||||
context,
|
||||
responseStyle,
|
||||
...(confirmationId ? { confirmationId } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
import "./client-voice-confirmation.js";
|
||||
|
||||
type ClientVoiceConfirmationTestApi = {
|
||||
resetClientVoiceConfirmationStateForTest(): void;
|
||||
};
|
||||
|
||||
function getTestApi(): ClientVoiceConfirmationTestApi {
|
||||
return (globalThis as Record<PropertyKey, unknown>)[
|
||||
Symbol.for("openclaw.clientVoiceConfirmationTestApi")
|
||||
] as ClientVoiceConfirmationTestApi;
|
||||
}
|
||||
|
||||
export function resetClientVoiceConfirmationStateForTest(): void {
|
||||
getTestApi().resetClientVoiceConfirmationStateForTest();
|
||||
}
|
||||
@@ -0,0 +1,501 @@
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
authorizeClientVoiceConfirmation as authorizeClientVoiceConfirmationForTest,
|
||||
bindAuthorizedClientVoiceConfirmation,
|
||||
deactivateClientVoiceConfirmationSession,
|
||||
noteClientVoiceConfirmationUtterance as noteClientVoiceConfirmationUtteranceForTest,
|
||||
releaseClientVoiceConfirmationRun,
|
||||
resolveClientVoiceToolConfirmationPolicy as resolveClientVoiceToolConfirmationPolicyForTest,
|
||||
} from "./client-voice-confirmation.js";
|
||||
import { resetClientVoiceConfirmationStateForTest } from "./client-voice-confirmation.test-support.js";
|
||||
|
||||
function authorizeClientVoiceConfirmation(
|
||||
params: Omit<Parameters<typeof authorizeClientVoiceConfirmationForTest>[0], "agentId">,
|
||||
) {
|
||||
return authorizeClientVoiceConfirmationForTest({ agentId: "main", ...params });
|
||||
}
|
||||
|
||||
function noteClientVoiceConfirmationUtterance(
|
||||
params: Omit<Parameters<typeof noteClientVoiceConfirmationUtteranceForTest>[0], "agentId">,
|
||||
): void {
|
||||
noteClientVoiceConfirmationUtteranceForTest({ agentId: "main", ...params });
|
||||
}
|
||||
|
||||
function resolveClientVoiceToolConfirmationPolicy(
|
||||
params: Omit<Parameters<typeof resolveClientVoiceToolConfirmationPolicyForTest>[0], "agentId">,
|
||||
) {
|
||||
return resolveClientVoiceToolConfirmationPolicyForTest({ agentId: "main", ...params });
|
||||
}
|
||||
|
||||
function confirmationIdFrom(reason: string): string {
|
||||
const match = reason.match(/VOICE_CONFIRMATION_REQUIRED:([^\s]+)/);
|
||||
if (!match?.[1]) {
|
||||
throw new Error(`missing confirmation id: ${reason}`);
|
||||
}
|
||||
return match[1];
|
||||
}
|
||||
|
||||
function block(params: {
|
||||
voiceSessionId: string;
|
||||
runId?: string;
|
||||
toolName?: string;
|
||||
toolParams?: unknown;
|
||||
now?: number;
|
||||
}) {
|
||||
const result = resolveClientVoiceToolConfirmationPolicy({
|
||||
voiceSessionId: params.voiceSessionId,
|
||||
runId: params.runId,
|
||||
toolName: params.toolName ?? "message",
|
||||
toolParams: params.toolParams ?? { action: "send", message: "hello" },
|
||||
now: params.now,
|
||||
});
|
||||
expect(result.allowed).toBe(false);
|
||||
if (result.allowed) {
|
||||
throw new Error("expected blocked voice action");
|
||||
}
|
||||
return confirmationIdFrom(result.reason);
|
||||
}
|
||||
|
||||
describe("client voice confirmation", () => {
|
||||
afterEach(() => resetClientVoiceConfirmationStateForTest());
|
||||
|
||||
it("does not pause a concurrent non-voice run sharing the session key", () => {
|
||||
block({ voiceSessionId: "voice-1", runId: "voice-run" });
|
||||
|
||||
expect(
|
||||
resolveClientVoiceToolConfirmationPolicy({
|
||||
toolName: "message",
|
||||
toolParams: { action: "send", message: "other run" },
|
||||
}),
|
||||
).toEqual({ allowed: true });
|
||||
});
|
||||
|
||||
it.each([
|
||||
["exec", "git clean -fdx"],
|
||||
["bash", "mv a b"],
|
||||
])(
|
||||
"requires confirmation for an unlisted destructive shell command: %s %s",
|
||||
(toolName, command) => {
|
||||
expect(
|
||||
resolveClientVoiceToolConfirmationPolicy({
|
||||
voiceSessionId: "voice-1",
|
||||
runId: "voice-run",
|
||||
toolName,
|
||||
toolParams: { command },
|
||||
}).allowed,
|
||||
).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["ls -la", "grep -n TODO README.md"])(
|
||||
"does not require confirmation for a classified read-only shell command: %s",
|
||||
(command) => {
|
||||
expect(
|
||||
resolveClientVoiceToolConfirmationPolicy({
|
||||
voiceSessionId: "voice-1",
|
||||
runId: "voice-run",
|
||||
toolName: "exec",
|
||||
toolParams: { command },
|
||||
}),
|
||||
).toEqual({ allowed: true });
|
||||
},
|
||||
);
|
||||
|
||||
it("requires confirmation before delegating work outside the voice-bound run", () => {
|
||||
expect(
|
||||
resolveClientVoiceToolConfirmationPolicy({
|
||||
voiceSessionId: "voice-1",
|
||||
runId: "voice-run",
|
||||
toolName: "sessions_send",
|
||||
toolParams: { sessionKey: "agent:main:child", message: "send this" },
|
||||
}).allowed,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps pre-gate behavior for sessions that cannot report spoken approvals", () => {
|
||||
expect(
|
||||
resolveClientVoiceToolConfirmationPolicy({
|
||||
voiceSessionId: "voice-legacy",
|
||||
runId: "voice-run",
|
||||
toolName: "message",
|
||||
toolParams: { action: "send", to: "user", message: "hi" },
|
||||
isConfirmable: () => false,
|
||||
}),
|
||||
).toEqual({ allowed: true });
|
||||
expect(
|
||||
resolveClientVoiceToolConfirmationPolicy({
|
||||
voiceSessionId: "voice-modern",
|
||||
runId: "voice-run",
|
||||
toolName: "message",
|
||||
toolParams: { action: "send", to: "user", message: "hi" },
|
||||
isConfirmable: () => true,
|
||||
}).allowed,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps workspace-local writes confirmation-free", () => {
|
||||
expect(
|
||||
resolveClientVoiceToolConfirmationPolicy({
|
||||
voiceSessionId: "voice-1",
|
||||
runId: "voice-run",
|
||||
toolName: "write",
|
||||
toolParams: { path: "notes.txt", content: "local change" },
|
||||
}),
|
||||
).toEqual({ allowed: true });
|
||||
});
|
||||
|
||||
it("keeps a challenge authorizable until the run is bound, then consumes it", () => {
|
||||
const confirmationId = block({
|
||||
voiceSessionId: "voice-1",
|
||||
toolParams: { action: "send", message: "A" },
|
||||
now: 100,
|
||||
});
|
||||
noteClientVoiceConfirmationUtterance({
|
||||
voiceSessionId: "voice-1",
|
||||
text: "yes",
|
||||
timestamp: 101,
|
||||
});
|
||||
// A failed/retried consult can re-authorize the same challenge before it binds.
|
||||
const first = authorizeClientVoiceConfirmation({
|
||||
voiceSessionId: "voice-1",
|
||||
confirmationId,
|
||||
now: 102,
|
||||
});
|
||||
expect(
|
||||
authorizeClientVoiceConfirmation({ voiceSessionId: "voice-1", confirmationId, now: 103 })
|
||||
.fingerprint,
|
||||
).toBe(first.fingerprint);
|
||||
bindAuthorizedClientVoiceConfirmation({ grant: first, runId: "run-approved" });
|
||||
// After binding the run, the challenge is consumed and cannot re-authorize.
|
||||
expect(() =>
|
||||
authorizeClientVoiceConfirmation({ voiceSessionId: "voice-1", confirmationId, now: 104 }),
|
||||
).toThrow("missing, expired, or belongs to another action");
|
||||
});
|
||||
|
||||
it("binds approval to the exact tool fingerprint", () => {
|
||||
const confirmationId = block({
|
||||
voiceSessionId: "voice-1",
|
||||
toolParams: { action: "send", message: "A" },
|
||||
now: 100,
|
||||
});
|
||||
noteClientVoiceConfirmationUtterance({
|
||||
voiceSessionId: "voice-1",
|
||||
text: "Yes, do it.",
|
||||
timestamp: 101,
|
||||
});
|
||||
const grant = authorizeClientVoiceConfirmation({
|
||||
voiceSessionId: "voice-1",
|
||||
confirmationId,
|
||||
now: 102,
|
||||
});
|
||||
bindAuthorizedClientVoiceConfirmation({ grant, runId: "run-approved" });
|
||||
|
||||
expect(
|
||||
resolveClientVoiceToolConfirmationPolicy({
|
||||
voiceSessionId: "voice-1",
|
||||
runId: "run-approved",
|
||||
toolName: "message",
|
||||
toolParams: { action: "send", message: "B" },
|
||||
now: 103,
|
||||
}).allowed,
|
||||
).toBe(false);
|
||||
expect(
|
||||
resolveClientVoiceToolConfirmationPolicy({
|
||||
voiceSessionId: "voice-1",
|
||||
runId: "run-approved",
|
||||
toolName: "message",
|
||||
toolParams: { action: "send", message: "A" },
|
||||
now: 104,
|
||||
}),
|
||||
).toEqual({ allowed: true });
|
||||
});
|
||||
|
||||
it("rejects expired confirmations", () => {
|
||||
const confirmationId = block({ voiceSessionId: "voice-1", now: 1_000 });
|
||||
noteClientVoiceConfirmationUtterance({
|
||||
voiceSessionId: "voice-1",
|
||||
text: "confirm",
|
||||
timestamp: 1_001,
|
||||
});
|
||||
|
||||
expect(() =>
|
||||
authorizeClientVoiceConfirmation({
|
||||
voiceSessionId: "voice-1",
|
||||
confirmationId,
|
||||
now: 121_001,
|
||||
}),
|
||||
).toThrow("missing, expired");
|
||||
});
|
||||
|
||||
it.each(["no", "don't do it", "don’t do it", "do not proceed", "cancel"])(
|
||||
"a refusal invalidates the pending confirmation for %j",
|
||||
(text) => {
|
||||
const confirmationId = block({ voiceSessionId: "voice-1", now: 100 });
|
||||
noteClientVoiceConfirmationUtterance({
|
||||
voiceSessionId: "voice-1",
|
||||
text,
|
||||
timestamp: 101,
|
||||
});
|
||||
|
||||
expect(() =>
|
||||
authorizeClientVoiceConfirmation({
|
||||
voiceSessionId: "voice-1",
|
||||
confirmationId,
|
||||
now: 102,
|
||||
}),
|
||||
).toThrow("missing, expired, or belongs to another action");
|
||||
},
|
||||
);
|
||||
|
||||
it("consumes an approved fingerprint once", () => {
|
||||
const toolParams = { action: "send", message: "A" };
|
||||
const confirmationId = block({ voiceSessionId: "voice-1", toolParams, now: 100 });
|
||||
noteClientVoiceConfirmationUtterance({
|
||||
voiceSessionId: "voice-1",
|
||||
text: "go ahead",
|
||||
timestamp: 101,
|
||||
});
|
||||
const grant = authorizeClientVoiceConfirmation({
|
||||
voiceSessionId: "voice-1",
|
||||
confirmationId,
|
||||
now: 102,
|
||||
});
|
||||
bindAuthorizedClientVoiceConfirmation({ grant, runId: "run-approved" });
|
||||
|
||||
expect(
|
||||
resolveClientVoiceToolConfirmationPolicy({
|
||||
voiceSessionId: "voice-1",
|
||||
runId: "run-approved",
|
||||
toolName: "message",
|
||||
toolParams,
|
||||
now: 103,
|
||||
}),
|
||||
).toEqual({ allowed: true });
|
||||
expect(
|
||||
resolveClientVoiceToolConfirmationPolicy({
|
||||
voiceSessionId: "voice-1",
|
||||
runId: "run-approved",
|
||||
toolName: "message",
|
||||
toolParams,
|
||||
now: 104,
|
||||
}).allowed,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("consumes one spoken affirmation for only one pending action", () => {
|
||||
const first = block({
|
||||
voiceSessionId: "voice-1",
|
||||
runId: "run-1",
|
||||
toolParams: { action: "send", message: "A" },
|
||||
now: 100,
|
||||
});
|
||||
const second = block({
|
||||
voiceSessionId: "voice-1",
|
||||
runId: "run-2",
|
||||
toolParams: { action: "send", message: "B" },
|
||||
now: 100,
|
||||
});
|
||||
noteClientVoiceConfirmationUtterance({
|
||||
voiceSessionId: "voice-1",
|
||||
text: "yes",
|
||||
timestamp: 101,
|
||||
});
|
||||
|
||||
expect(() =>
|
||||
authorizeClientVoiceConfirmation({
|
||||
voiceSessionId: "voice-1",
|
||||
confirmationId: first,
|
||||
now: 102,
|
||||
}),
|
||||
).toThrow("newer confirmation request supersedes");
|
||||
// Binding the newer grant consumes the shared affirmation and its challenge.
|
||||
const grant = authorizeClientVoiceConfirmation({
|
||||
voiceSessionId: "voice-1",
|
||||
confirmationId: second,
|
||||
now: 102,
|
||||
});
|
||||
bindAuthorizedClientVoiceConfirmation({ grant, runId: "run-2" });
|
||||
expect(() =>
|
||||
authorizeClientVoiceConfirmation({
|
||||
voiceSessionId: "voice-1",
|
||||
confirmationId: first,
|
||||
now: 103,
|
||||
}),
|
||||
).toThrow("explicit spoken confirmation");
|
||||
});
|
||||
|
||||
it("binds an approved fingerprint to its follow-up run", () => {
|
||||
const toolParams = { action: "send", message: "same" };
|
||||
const confirmationId = block({
|
||||
voiceSessionId: "voice-1",
|
||||
runId: "run-original",
|
||||
toolParams,
|
||||
now: 100,
|
||||
});
|
||||
noteClientVoiceConfirmationUtterance({
|
||||
voiceSessionId: "voice-1",
|
||||
text: "proceed",
|
||||
timestamp: 101,
|
||||
});
|
||||
const grant = authorizeClientVoiceConfirmation({
|
||||
voiceSessionId: "voice-1",
|
||||
confirmationId,
|
||||
now: 102,
|
||||
});
|
||||
bindAuthorizedClientVoiceConfirmation({ grant, runId: "run-approved" });
|
||||
|
||||
expect(
|
||||
resolveClientVoiceToolConfirmationPolicy({
|
||||
voiceSessionId: "voice-1",
|
||||
runId: "run-other",
|
||||
toolName: "message",
|
||||
toolParams,
|
||||
now: 103,
|
||||
}).allowed,
|
||||
).toBe(false);
|
||||
expect(
|
||||
resolveClientVoiceToolConfirmationPolicy({
|
||||
voiceSessionId: "voice-1",
|
||||
runId: "run-approved",
|
||||
toolName: "message",
|
||||
toolParams,
|
||||
now: 104,
|
||||
}),
|
||||
).toEqual({ allowed: true });
|
||||
});
|
||||
|
||||
it("only the newest pending challenge can be authorized", () => {
|
||||
const olderId = block({
|
||||
voiceSessionId: "voice-1",
|
||||
runId: "run-1",
|
||||
toolParams: { action: "send", message: "older" },
|
||||
now: 100,
|
||||
});
|
||||
const newerId = block({
|
||||
voiceSessionId: "voice-1",
|
||||
runId: "run-1",
|
||||
toolParams: { action: "send", message: "newer" },
|
||||
now: 110,
|
||||
});
|
||||
noteClientVoiceConfirmationUtterance({
|
||||
voiceSessionId: "voice-1",
|
||||
text: "yes",
|
||||
timestamp: 111,
|
||||
});
|
||||
expect(() =>
|
||||
authorizeClientVoiceConfirmation({
|
||||
voiceSessionId: "voice-1",
|
||||
confirmationId: olderId,
|
||||
now: 112,
|
||||
}),
|
||||
).toThrow("newer confirmation request supersedes");
|
||||
expect(
|
||||
authorizeClientVoiceConfirmation({
|
||||
voiceSessionId: "voice-1",
|
||||
confirmationId: newerId,
|
||||
now: 113,
|
||||
}).fingerprint,
|
||||
).toBeTruthy();
|
||||
});
|
||||
|
||||
it("invalidates a pending confirmation when the user refuses", () => {
|
||||
const toolParams = { action: "send", message: "declined" };
|
||||
const confirmationId = block({
|
||||
voiceSessionId: "voice-1",
|
||||
runId: "run-1",
|
||||
toolParams,
|
||||
now: 100,
|
||||
});
|
||||
noteClientVoiceConfirmationUtterance({
|
||||
voiceSessionId: "voice-1",
|
||||
text: "No, cancel that",
|
||||
timestamp: 101,
|
||||
});
|
||||
noteClientVoiceConfirmationUtterance({
|
||||
voiceSessionId: "voice-1",
|
||||
text: "yes",
|
||||
timestamp: 102,
|
||||
});
|
||||
expect(() =>
|
||||
authorizeClientVoiceConfirmation({
|
||||
voiceSessionId: "voice-1",
|
||||
confirmationId,
|
||||
now: 103,
|
||||
}),
|
||||
).toThrow("missing, expired, or belongs to another action");
|
||||
});
|
||||
|
||||
it("keeps a live run's grant across call close and releases it on run completion", () => {
|
||||
const toolParams = { action: "send", message: "confirmed then hangup" };
|
||||
const confirmationId = block({
|
||||
voiceSessionId: "voice-1",
|
||||
runId: "run-original",
|
||||
toolParams,
|
||||
now: 100,
|
||||
});
|
||||
noteClientVoiceConfirmationUtterance({
|
||||
voiceSessionId: "voice-1",
|
||||
text: "yes do it",
|
||||
timestamp: 101,
|
||||
});
|
||||
const grant = authorizeClientVoiceConfirmation({
|
||||
voiceSessionId: "voice-1",
|
||||
confirmationId,
|
||||
now: 102,
|
||||
});
|
||||
bindAuthorizedClientVoiceConfirmation({ grant, runId: "run-live" });
|
||||
|
||||
deactivateClientVoiceConfirmationSession("main", "voice-1", ["run-live"]);
|
||||
expect(
|
||||
resolveClientVoiceToolConfirmationPolicy({
|
||||
voiceSessionId: "voice-1",
|
||||
runId: "run-live",
|
||||
toolName: "message",
|
||||
toolParams,
|
||||
now: 103,
|
||||
}),
|
||||
).toEqual({ allowed: true });
|
||||
|
||||
bindAuthorizedClientVoiceConfirmation({ grant, runId: "run-done" });
|
||||
releaseClientVoiceConfirmationRun("main", "voice-1", "run-done");
|
||||
expect(
|
||||
resolveClientVoiceToolConfirmationPolicy({
|
||||
voiceSessionId: "voice-1",
|
||||
runId: "run-done",
|
||||
toolName: "message",
|
||||
toolParams,
|
||||
now: 104,
|
||||
}).allowed,
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("isolates same-named voice sessions across agents", () => {
|
||||
const blocked = resolveClientVoiceToolConfirmationPolicyForTest({
|
||||
agentId: "agent-b",
|
||||
voiceSessionId: "shared-id",
|
||||
runId: "run-b",
|
||||
toolName: "message",
|
||||
toolParams: { action: "send", message: "B" },
|
||||
now: 100,
|
||||
});
|
||||
expect(blocked.allowed).toBe(false);
|
||||
if (blocked.allowed) {
|
||||
throw new Error("expected confirmation request");
|
||||
}
|
||||
noteClientVoiceConfirmationUtteranceForTest({
|
||||
agentId: "agent-a",
|
||||
voiceSessionId: "shared-id",
|
||||
text: "yes",
|
||||
timestamp: 101,
|
||||
});
|
||||
|
||||
expect(() =>
|
||||
authorizeClientVoiceConfirmationForTest({
|
||||
agentId: "agent-b",
|
||||
voiceSessionId: "shared-id",
|
||||
confirmationId: confirmationIdFrom(blocked.reason),
|
||||
now: 102,
|
||||
}),
|
||||
).toThrow("explicit spoken confirmation");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,341 @@
|
||||
/** In-memory spoken confirmation binding for high-impact Talk actions. */
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { buildToolMutationState } from "../agents/tool-mutation.js";
|
||||
|
||||
const CONFIRMATION_TTL_MS = 2 * 60_000;
|
||||
|
||||
type PendingVoiceConfirmation = {
|
||||
confirmationId: string;
|
||||
agentId: string;
|
||||
voiceSessionId: string;
|
||||
runId?: string;
|
||||
fingerprint: string;
|
||||
toolName: string;
|
||||
createdAt: number;
|
||||
/** Monotonic tiebreaker: same-millisecond challenges must still have one newest. */
|
||||
seq: number;
|
||||
expiresAt: number;
|
||||
};
|
||||
|
||||
type RecentVoiceUserUtterance = {
|
||||
text: string;
|
||||
timestamp: number;
|
||||
};
|
||||
|
||||
export type ClientVoiceConfirmationGrant = {
|
||||
agentId: string;
|
||||
voiceSessionId: string;
|
||||
confirmationId: string;
|
||||
fingerprint: string;
|
||||
expiresAt: number;
|
||||
};
|
||||
|
||||
const pendingConfirmations = new Map<string, PendingVoiceConfirmation>();
|
||||
let confirmationSeq = 0;
|
||||
const approvedFingerprints = new Map<string, Map<string, Map<string, number>>>();
|
||||
const recentUserUtterances = new Map<string, RecentVoiceUserUtterance>();
|
||||
|
||||
function confirmationScopeKey(agentId: string, voiceSessionId: string): string {
|
||||
return `${agentId}\0${voiceSessionId}`;
|
||||
}
|
||||
|
||||
function stableToolFingerprint(toolName: string, params: unknown): string {
|
||||
const normalize = (value: unknown): unknown => {
|
||||
if (Array.isArray(value)) {
|
||||
return value.map(normalize);
|
||||
}
|
||||
if (!value || typeof value !== "object") {
|
||||
return value;
|
||||
}
|
||||
return Object.fromEntries(
|
||||
Object.entries(value as Record<string, unknown>)
|
||||
.toSorted(([left], [right]) => left.localeCompare(right))
|
||||
.map(([key, entry]) => [key, normalize(entry)]),
|
||||
);
|
||||
};
|
||||
return createHash("sha256")
|
||||
.update(`${toolName}\0${JSON.stringify(normalize(params))}`)
|
||||
.digest("hex");
|
||||
}
|
||||
|
||||
function requiresHighImpactVoiceConfirmation(toolName: string, params: unknown): boolean {
|
||||
const normalizedTool = toolName.trim().toLowerCase();
|
||||
if (!buildToolMutationState(normalizedTool, params).mutatingAction) {
|
||||
return false;
|
||||
}
|
||||
if (
|
||||
["message", "gateway", "nodes", "browser", "computer", "canvas", "cron", "process"].includes(
|
||||
normalizedTool,
|
||||
)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
// Workspace-local edits stay bound to this run. Session delegation is gated because
|
||||
// delegated runs leave the voice binding and otherwise bypass spoken confirmation.
|
||||
if (
|
||||
["write", "edit", "apply_patch", "create_goal", "update_goal", "get_goal"].includes(
|
||||
normalizedTool,
|
||||
)
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
function consumeApprovedFingerprint(
|
||||
voiceSessionId: string,
|
||||
runId: string | undefined,
|
||||
fingerprint: string,
|
||||
now: number,
|
||||
): boolean {
|
||||
if (!runId) {
|
||||
return false;
|
||||
}
|
||||
const approvedByRun = approvedFingerprints.get(voiceSessionId);
|
||||
const approved = approvedByRun?.get(runId);
|
||||
const expiresAt = approved?.get(fingerprint);
|
||||
if (!expiresAt || expiresAt < now) {
|
||||
approved?.delete(fingerprint);
|
||||
return false;
|
||||
}
|
||||
approved?.delete(fingerprint);
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Record a finalized user utterance after the durable transcript append succeeds. */
|
||||
export function noteClientVoiceConfirmationUtterance(params: {
|
||||
agentId: string;
|
||||
voiceSessionId: string;
|
||||
text: string;
|
||||
timestamp: number;
|
||||
}): void {
|
||||
recentUserUtterances.set(confirmationScopeKey(params.agentId, params.voiceSessionId), {
|
||||
text: params.text,
|
||||
timestamp: params.timestamp,
|
||||
});
|
||||
// A spoken refusal kills the outstanding challenge: a later unrelated "yes"
|
||||
// must not resurrect an action the user already declined.
|
||||
if (REFUSAL_PATTERN.test(normalizeUtterance(params.text))) {
|
||||
for (const [confirmationId, confirmation] of pendingConfirmations) {
|
||||
if (
|
||||
confirmation.agentId === params.agentId &&
|
||||
confirmation.voiceSessionId === params.voiceSessionId &&
|
||||
confirmation.createdAt < params.timestamp
|
||||
) {
|
||||
pendingConfirmations.delete(confirmationId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Pause a high-impact action for one voice-bound run until its exact fingerprint is approved. */
|
||||
export function resolveClientVoiceToolConfirmationPolicy(params: {
|
||||
agentId?: string;
|
||||
voiceSessionId?: string;
|
||||
runId?: string;
|
||||
toolName: string;
|
||||
toolParams: unknown;
|
||||
isConfirmable?: () => boolean;
|
||||
now?: number;
|
||||
}): { allowed: true } | { allowed: false; reason: string } {
|
||||
if (!params.agentId || !params.voiceSessionId) {
|
||||
return { allowed: true };
|
||||
}
|
||||
if (!requiresHighImpactVoiceConfirmation(params.toolName, params.toolParams)) {
|
||||
return { allowed: true };
|
||||
}
|
||||
// Sessions that cannot report spoken approvals (legacy clients without transcript
|
||||
// RPCs) keep pre-gate behavior; a pause they can never confirm is a dead end.
|
||||
// This is not a client trust boundary: authenticated clients can already run any
|
||||
// tool via chat.send. The gate guards against voice-channel misfires only.
|
||||
if (params.isConfirmable && !params.isConfirmable()) {
|
||||
return { allowed: true };
|
||||
}
|
||||
const now = params.now ?? Date.now();
|
||||
const fingerprint = stableToolFingerprint(params.toolName, params.toolParams);
|
||||
const scopeKey = confirmationScopeKey(params.agentId, params.voiceSessionId);
|
||||
if (consumeApprovedFingerprint(scopeKey, params.runId, fingerprint, now)) {
|
||||
return { allowed: true };
|
||||
}
|
||||
const existing = [...pendingConfirmations.values()].find(
|
||||
(entry) =>
|
||||
entry.voiceSessionId === params.voiceSessionId &&
|
||||
entry.agentId === params.agentId &&
|
||||
entry.runId === params.runId &&
|
||||
entry.fingerprint === fingerprint &&
|
||||
entry.expiresAt >= now,
|
||||
);
|
||||
const confirmation =
|
||||
existing ??
|
||||
({
|
||||
confirmationId: randomUUID(),
|
||||
agentId: params.agentId,
|
||||
voiceSessionId: params.voiceSessionId,
|
||||
...(params.runId ? { runId: params.runId } : {}),
|
||||
fingerprint,
|
||||
toolName: params.toolName,
|
||||
createdAt: now,
|
||||
seq: ++confirmationSeq,
|
||||
expiresAt: now + CONFIRMATION_TTL_MS,
|
||||
} satisfies PendingVoiceConfirmation);
|
||||
pendingConfirmations.set(confirmation.confirmationId, confirmation);
|
||||
return {
|
||||
allowed: false,
|
||||
reason:
|
||||
`VOICE_CONFIRMATION_REQUIRED:${confirmation.confirmationId} ` +
|
||||
`The high-impact voice action "${params.toolName}" was not executed. ` +
|
||||
"Ask the user for explicit spoken confirmation, then call openclaw_agent_consult again with this confirmationId.",
|
||||
};
|
||||
}
|
||||
|
||||
const REFUSAL_PATTERN = /\b(no|don't|do not|cancel|stop|never mind)\b/;
|
||||
|
||||
function normalizeUtterance(text: string): string {
|
||||
return (
|
||||
text
|
||||
.trim()
|
||||
.toLowerCase()
|
||||
// STT commonly emits typographic apostrophes; fold them so "don't" (U+2019)
|
||||
// matches the refusal pattern and cannot slip past as a non-refusal.
|
||||
.replace(/[‘’ʼ]/g, "'")
|
||||
.replace(/[,;:.!?]+/g, "")
|
||||
.replace(/\s+/g, " ")
|
||||
);
|
||||
}
|
||||
|
||||
function isExplicitAffirmation(text: string): boolean {
|
||||
const normalized = normalizeUtterance(text);
|
||||
if (REFUSAL_PATTERN.test(normalized)) {
|
||||
return false;
|
||||
}
|
||||
// English-only phrases are an accepted first version; localized matching is follow-up work.
|
||||
return /^(yes|yes do it|do it|confirm|confirmed|go ahead|proceed|send it|make the change|restart it)$/.test(
|
||||
normalized,
|
||||
);
|
||||
}
|
||||
|
||||
/** Bind a later affirmative utterance to one exact paused action. */
|
||||
export function authorizeClientVoiceConfirmation(params: {
|
||||
agentId: string;
|
||||
voiceSessionId: string;
|
||||
confirmationId: string;
|
||||
now?: number;
|
||||
}): ClientVoiceConfirmationGrant {
|
||||
const confirmation = pendingConfirmations.get(params.confirmationId);
|
||||
const now = params.now ?? Date.now();
|
||||
if (
|
||||
!confirmation ||
|
||||
confirmation.agentId !== params.agentId ||
|
||||
confirmation.voiceSessionId !== params.voiceSessionId ||
|
||||
confirmation.expiresAt < now
|
||||
) {
|
||||
throw new Error("voice confirmation is missing, expired, or belongs to another action");
|
||||
}
|
||||
// A bare "yes" can only answer the question the model asked last; authorizing an
|
||||
// older challenge would let the model swap in a different pending action.
|
||||
for (const entry of pendingConfirmations.values()) {
|
||||
if (
|
||||
entry.agentId === params.agentId &&
|
||||
entry.voiceSessionId === params.voiceSessionId &&
|
||||
entry.seq > confirmation.seq
|
||||
) {
|
||||
throw new Error("a newer confirmation request supersedes this one; ask again");
|
||||
}
|
||||
}
|
||||
const scopeKey = confirmationScopeKey(params.agentId, params.voiceSessionId);
|
||||
const affirmation = recentUserUtterances.get(scopeKey);
|
||||
if (
|
||||
!affirmation ||
|
||||
affirmation.timestamp <= confirmation.createdAt ||
|
||||
!isExplicitAffirmation(affirmation.text)
|
||||
) {
|
||||
throw new Error("explicit spoken confirmation was not found after the action request");
|
||||
}
|
||||
// Validate only; the challenge and affirmation are consumed at bind time, once the
|
||||
// consult run is established. This keeps a failed/lost-response consult retryable
|
||||
// with the same confirmationId instead of leaving the action unconfirmable.
|
||||
return {
|
||||
agentId: params.agentId,
|
||||
voiceSessionId: params.voiceSessionId,
|
||||
confirmationId: params.confirmationId,
|
||||
fingerprint: confirmation.fingerprint,
|
||||
expiresAt: confirmation.expiresAt,
|
||||
};
|
||||
}
|
||||
|
||||
/** Bind a validated spoken grant to the one follow-up run and consume the challenge. */
|
||||
export function bindAuthorizedClientVoiceConfirmation(params: {
|
||||
grant: ClientVoiceConfirmationGrant;
|
||||
runId: string;
|
||||
}): void {
|
||||
const scopeKey = confirmationScopeKey(params.grant.agentId, params.grant.voiceSessionId);
|
||||
const approvedByRun = approvedFingerprints.get(scopeKey) ?? new Map();
|
||||
const approved = approvedByRun.get(params.runId) ?? new Map<string, number>();
|
||||
approved.set(params.grant.fingerprint, params.grant.expiresAt);
|
||||
approvedByRun.set(params.runId, approved);
|
||||
approvedFingerprints.set(scopeKey, approvedByRun);
|
||||
// Consume now that the run exists: one spoken affirmation authorizes one action.
|
||||
pendingConfirmations.delete(params.grant.confirmationId);
|
||||
recentUserUtterances.delete(scopeKey);
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove ephemeral confirmation state when the logical call closes. Approved
|
||||
* grants for still-live consult runs survive: a spoken "yes" followed by hangup
|
||||
* must not re-block the confirmed action its run is about to execute.
|
||||
*/
|
||||
export function deactivateClientVoiceConfirmationSession(
|
||||
agentId: string,
|
||||
voiceSessionId: string,
|
||||
liveRunIds: readonly string[] = [],
|
||||
): void {
|
||||
const scopeKey = confirmationScopeKey(agentId, voiceSessionId);
|
||||
recentUserUtterances.delete(scopeKey);
|
||||
const approvedByRun = approvedFingerprints.get(scopeKey);
|
||||
if (approvedByRun) {
|
||||
const live = new Set(liveRunIds);
|
||||
for (const runId of approvedByRun.keys()) {
|
||||
if (!live.has(runId)) {
|
||||
approvedByRun.delete(runId);
|
||||
}
|
||||
}
|
||||
if (approvedByRun.size === 0) {
|
||||
approvedFingerprints.delete(scopeKey);
|
||||
}
|
||||
}
|
||||
for (const [confirmationId, confirmation] of pendingConfirmations) {
|
||||
if (confirmation.agentId === agentId && confirmation.voiceSessionId === voiceSessionId) {
|
||||
pendingConfirmations.delete(confirmationId);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Drop a completed run's surviving grants once its lifecycle ends. */
|
||||
export function releaseClientVoiceConfirmationRun(
|
||||
agentId: string,
|
||||
voiceSessionId: string,
|
||||
runId: string,
|
||||
): void {
|
||||
const scopeKey = confirmationScopeKey(agentId, voiceSessionId);
|
||||
const approvedByRun = approvedFingerprints.get(scopeKey);
|
||||
if (!approvedByRun) {
|
||||
return;
|
||||
}
|
||||
approvedByRun.delete(runId);
|
||||
if (approvedByRun.size === 0) {
|
||||
approvedFingerprints.delete(scopeKey);
|
||||
}
|
||||
}
|
||||
|
||||
/** Test-only reset for process-global state. */
|
||||
function resetClientVoiceConfirmationStateForTest(): void {
|
||||
pendingConfirmations.clear();
|
||||
approvedFingerprints.clear();
|
||||
recentUserUtterances.clear();
|
||||
}
|
||||
|
||||
if (process.env.VITEST || process.env.NODE_ENV === "test") {
|
||||
(globalThis as Record<PropertyKey, unknown>)[
|
||||
Symbol.for("openclaw.clientVoiceConfirmationTestApi")
|
||||
] = { resetClientVoiceConfirmationStateForTest };
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
/** SQLite-backed persistence for durable per-agent Talk voice-call records. */
|
||||
import {
|
||||
openOpenClawAgentDatabase,
|
||||
type OpenClawAgentDatabase,
|
||||
} from "../state/openclaw-agent-db.js";
|
||||
|
||||
export const VOICE_SESSION_CACHE_SCOPE = "talk-client-voice-sessions";
|
||||
export const VOICE_SESSION_RECORD_VERSION = 1;
|
||||
export const VOICE_SESSION_MAX_TRANSCRIPT_CHARS = 8_000;
|
||||
export const VOICE_SESSION_STALE_AFTER_MS = 6 * 60 * 60_000;
|
||||
|
||||
export type ClientVoiceToolEffect = {
|
||||
runId: string;
|
||||
toolCallId?: string;
|
||||
toolName: string;
|
||||
startedAt: number;
|
||||
finishedAt?: number;
|
||||
status: "started" | "succeeded" | "failed" | "cancelled" | "blocked";
|
||||
};
|
||||
|
||||
export type ClientVoiceSessionRecord = {
|
||||
version: typeof VOICE_SESSION_RECORD_VERSION;
|
||||
voiceSessionId: string;
|
||||
agentId: string;
|
||||
sessionKey: string;
|
||||
provider?: string;
|
||||
origin: "client" | "relay";
|
||||
status: "open" | "closed";
|
||||
createdAt: number;
|
||||
updatedAt: number;
|
||||
closedAt?: number;
|
||||
consultRunIds: string[];
|
||||
effects: ClientVoiceToolEffect[];
|
||||
digestDeliveredAt?: number;
|
||||
/** Declared at create when the client speaks the transcript protocol (sent sessionKey). */
|
||||
transcriptCapable?: boolean;
|
||||
/** Set once a finalized user utterance persisted; gates spoken confirmation capability. */
|
||||
hasUserTranscript?: boolean;
|
||||
};
|
||||
|
||||
export type ClientVoiceRunBinding = {
|
||||
agentId: string;
|
||||
voiceSessionId: string;
|
||||
sessionKey: string;
|
||||
};
|
||||
|
||||
function parseVoiceSessionRecord(value: unknown): ClientVoiceSessionRecord | undefined {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return undefined;
|
||||
}
|
||||
const record = value as Partial<ClientVoiceSessionRecord>;
|
||||
if (
|
||||
record.version !== VOICE_SESSION_RECORD_VERSION ||
|
||||
typeof record.voiceSessionId !== "string" ||
|
||||
typeof record.agentId !== "string" ||
|
||||
typeof record.sessionKey !== "string" ||
|
||||
(record.provider !== undefined &&
|
||||
(typeof record.provider !== "string" || !record.provider.trim())) ||
|
||||
(record.origin !== "client" && record.origin !== "relay") ||
|
||||
(record.status !== "open" && record.status !== "closed") ||
|
||||
typeof record.createdAt !== "number" ||
|
||||
typeof record.updatedAt !== "number"
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
const consultRunIds = Array.isArray(record.consultRunIds)
|
||||
? record.consultRunIds.filter((entry): entry is string => typeof entry === "string")
|
||||
: [];
|
||||
const effects = Array.isArray(record.effects)
|
||||
? record.effects.filter((entry): entry is ClientVoiceToolEffect => {
|
||||
if (!entry || typeof entry !== "object") {
|
||||
return false;
|
||||
}
|
||||
const effect = entry as Partial<ClientVoiceToolEffect>;
|
||||
return (
|
||||
typeof effect.runId === "string" &&
|
||||
typeof effect.toolName === "string" &&
|
||||
typeof effect.startedAt === "number" &&
|
||||
(effect.status === "started" ||
|
||||
effect.status === "succeeded" ||
|
||||
effect.status === "failed" ||
|
||||
effect.status === "cancelled" ||
|
||||
effect.status === "blocked")
|
||||
);
|
||||
})
|
||||
: [];
|
||||
const provider = record.provider?.trim();
|
||||
return {
|
||||
...record,
|
||||
...(provider ? { provider } : {}),
|
||||
consultRunIds,
|
||||
effects,
|
||||
} as ClientVoiceSessionRecord;
|
||||
}
|
||||
|
||||
export function parseStoredVoiceSessionRecord(
|
||||
valueJson: unknown,
|
||||
): ClientVoiceSessionRecord | undefined {
|
||||
if (typeof valueJson !== "string") {
|
||||
return undefined;
|
||||
}
|
||||
try {
|
||||
return parseVoiceSessionRecord(JSON.parse(valueJson));
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
export function readVoiceSessionRecord(
|
||||
agentId: string,
|
||||
voiceSessionId: string,
|
||||
): ClientVoiceSessionRecord | undefined {
|
||||
const database = openOpenClawAgentDatabase({ agentId });
|
||||
const row = database.db
|
||||
.prepare("SELECT value_json FROM cache_entries WHERE scope = ? AND key = ?")
|
||||
.get(VOICE_SESSION_CACHE_SCOPE, voiceSessionId) as { value_json?: unknown } | undefined;
|
||||
return parseStoredVoiceSessionRecord(row?.value_json);
|
||||
}
|
||||
|
||||
export function readVoiceSessionRecordInTransaction(
|
||||
database: OpenClawAgentDatabase,
|
||||
voiceSessionId: string,
|
||||
): ClientVoiceSessionRecord | undefined {
|
||||
const row = database.db
|
||||
.prepare("SELECT value_json FROM cache_entries WHERE scope = ? AND key = ?")
|
||||
.get(VOICE_SESSION_CACHE_SCOPE, voiceSessionId) as { value_json?: unknown } | undefined;
|
||||
return parseStoredVoiceSessionRecord(row?.value_json);
|
||||
}
|
||||
|
||||
export function writeVoiceSessionRecordInTransaction(
|
||||
database: OpenClawAgentDatabase,
|
||||
record: ClientVoiceSessionRecord,
|
||||
): void {
|
||||
database.db
|
||||
.prepare(
|
||||
`INSERT INTO cache_entries (scope, key, value_json, blob, expires_at, updated_at)
|
||||
VALUES (?, ?, ?, NULL, NULL, ?)
|
||||
ON CONFLICT(scope, key) DO UPDATE SET
|
||||
value_json = excluded.value_json,
|
||||
updated_at = excluded.updated_at`,
|
||||
)
|
||||
.run(
|
||||
VOICE_SESSION_CACHE_SCOPE,
|
||||
record.voiceSessionId,
|
||||
JSON.stringify(record),
|
||||
record.updatedAt,
|
||||
);
|
||||
}
|
||||
|
||||
export function assertVoiceSessionOwnership(
|
||||
record: ClientVoiceSessionRecord,
|
||||
params: { agentId: string; sessionKey: string },
|
||||
): void {
|
||||
if (record.agentId !== params.agentId || record.sessionKey !== params.sessionKey) {
|
||||
throw new Error("voice session does not belong to this agent session");
|
||||
}
|
||||
}
|
||||
|
||||
export function operationKey(agentId: string, voiceSessionId: string): string {
|
||||
return `${agentId}\0${voiceSessionId}`;
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
import type { ClientVoiceSessionRecord } from "./client-voice-session-store.js";
|
||||
import "./client-voice-session.js";
|
||||
|
||||
type ClientVoiceSessionTestApi = {
|
||||
readRecord(agentId: string, voiceSessionId: string): ClientVoiceSessionRecord | undefined;
|
||||
reset(): void;
|
||||
};
|
||||
|
||||
function getTestApi(): ClientVoiceSessionTestApi {
|
||||
return (globalThis as Record<PropertyKey, unknown>)[
|
||||
Symbol.for("openclaw.clientVoiceSessionTestApi")
|
||||
] as ClientVoiceSessionTestApi;
|
||||
}
|
||||
|
||||
export const clientVoiceSessionTesting = {
|
||||
readRecord(agentId: string, voiceSessionId: string): ClientVoiceSessionRecord | undefined {
|
||||
return getTestApi().readRecord(agentId, voiceSessionId);
|
||||
},
|
||||
reset(): void {
|
||||
getTestApi().reset();
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,579 @@
|
||||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { replaceSessionEntry } from "../config/sessions/session-accessor.js";
|
||||
import {
|
||||
emitTrustedDiagnosticEvent,
|
||||
waitForDiagnosticEventsDrained,
|
||||
} from "../infra/diagnostic-events.js";
|
||||
import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js";
|
||||
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
|
||||
import { captureEnv, setTestEnvValue } from "../test-utils/env.js";
|
||||
import {
|
||||
closeClientVoiceSession,
|
||||
closeStaleClientVoiceSessions,
|
||||
createOrResumeClientVoiceSession,
|
||||
isClientVoiceSessionConfirmable,
|
||||
registerClientVoiceConsultRun,
|
||||
resolveClientVoiceRunBinding,
|
||||
resolveOpenClientVoiceSessionId,
|
||||
} from "./client-voice-session.js";
|
||||
import { clientVoiceSessionTesting } from "./client-voice-session.test-support.js";
|
||||
|
||||
const { sendDurableMessageBatch } = vi.hoisted(() => ({
|
||||
sendDurableMessageBatch: vi.fn(async () => ({ status: "sent" })),
|
||||
}));
|
||||
|
||||
vi.mock("../channels/message/runtime.js", () => ({ sendDurableMessageBatch }));
|
||||
|
||||
const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
|
||||
let tempDir: string;
|
||||
|
||||
async function seedSession(
|
||||
sessionKey: string,
|
||||
route: { lastChannel?: string; lastTo?: string } = {},
|
||||
): Promise<void> {
|
||||
await replaceSessionEntry(
|
||||
{ agentId: "main", sessionKey },
|
||||
{
|
||||
sessionId: `session-${sessionKey.replaceAll(":", "-")}`,
|
||||
updatedAt: Date.now(),
|
||||
...route,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function recordMutation(voiceSessionId: string, runId = `run-${voiceSessionId}`): void {
|
||||
registerClientVoiceConsultRun({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
voiceSessionId,
|
||||
runId,
|
||||
});
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "tool.execution.started",
|
||||
runId,
|
||||
toolCallId: `call-${runId}`,
|
||||
toolName: "message",
|
||||
mutatingAction: true,
|
||||
});
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "tool.execution.completed",
|
||||
runId,
|
||||
toolCallId: `call-${runId}`,
|
||||
toolName: "message",
|
||||
durationMs: 5,
|
||||
});
|
||||
}
|
||||
|
||||
async function completeRun(runId: string): Promise<void> {
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "run.completed",
|
||||
runId,
|
||||
durationMs: 5,
|
||||
outcome: "completed",
|
||||
});
|
||||
await waitForDiagnosticEventsDrained();
|
||||
}
|
||||
|
||||
describe("client voice session", () => {
|
||||
beforeEach(async () => {
|
||||
tempDir = await fs.realpath(
|
||||
await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-voice-session-")),
|
||||
);
|
||||
setTestEnvValue("OPENCLAW_STATE_DIR", tempDir);
|
||||
sendDurableMessageBatch.mockClear();
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
clientVoiceSessionTesting.reset();
|
||||
closeOpenClawAgentDatabasesForTest();
|
||||
closeOpenClawStateDatabaseForTest();
|
||||
envSnapshot.restore();
|
||||
await fs.rm(tempDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
it("creates, resumes, and enforces ownership and open state", async () => {
|
||||
const voiceSessionId = createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
provider: "google",
|
||||
origin: "client",
|
||||
voiceSessionId: "voice-1",
|
||||
now: 10,
|
||||
});
|
||||
expect(
|
||||
createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
origin: "client",
|
||||
voiceSessionId,
|
||||
now: 20,
|
||||
}),
|
||||
).toBe(voiceSessionId);
|
||||
expect(clientVoiceSessionTesting.readRecord("main", voiceSessionId)).toMatchObject({
|
||||
provider: "google",
|
||||
});
|
||||
expect(() =>
|
||||
createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
provider: "openai",
|
||||
origin: "client",
|
||||
voiceSessionId,
|
||||
}),
|
||||
).toThrow("provider does not match");
|
||||
expect(() =>
|
||||
createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:other",
|
||||
origin: "client",
|
||||
voiceSessionId,
|
||||
}),
|
||||
).toThrow("does not belong");
|
||||
|
||||
await closeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
voiceSessionId,
|
||||
config: {},
|
||||
now: 30,
|
||||
});
|
||||
expect(() =>
|
||||
createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
origin: "client",
|
||||
voiceSessionId,
|
||||
}),
|
||||
).toThrow("already closed");
|
||||
});
|
||||
|
||||
it("marks confirmability by declared capability, relay origin, or observed transcript", () => {
|
||||
const capable = createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
origin: "client",
|
||||
transcriptCapable: true,
|
||||
voiceSessionId: "voice-capable",
|
||||
});
|
||||
const legacy = createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
origin: "client",
|
||||
voiceSessionId: "voice-legacy",
|
||||
});
|
||||
const relay = createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
origin: "relay",
|
||||
voiceSessionId: "voice-relay",
|
||||
});
|
||||
const binding = (voiceSessionId: string) => ({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
voiceSessionId,
|
||||
});
|
||||
expect(isClientVoiceSessionConfirmable(binding(capable))).toBe(true);
|
||||
expect(isClientVoiceSessionConfirmable(binding(legacy))).toBe(false);
|
||||
expect(isClientVoiceSessionConfirmable(binding(relay))).toBe(true);
|
||||
});
|
||||
|
||||
it("closes idempotently without changing the first close time", async () => {
|
||||
const voiceSessionId = createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
origin: "client",
|
||||
now: 10,
|
||||
});
|
||||
await closeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
voiceSessionId,
|
||||
config: {},
|
||||
now: 20,
|
||||
});
|
||||
await closeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
voiceSessionId,
|
||||
config: {},
|
||||
now: 30,
|
||||
});
|
||||
|
||||
expect(clientVoiceSessionTesting.readRecord("main", voiceSessionId)).toMatchObject({
|
||||
status: "closed",
|
||||
closedAt: 20,
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps active consult runs voice-bound after the call closes", async () => {
|
||||
const voiceSessionId = createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
origin: "client",
|
||||
});
|
||||
registerClientVoiceConsultRun({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
voiceSessionId,
|
||||
runId: "run-active",
|
||||
});
|
||||
|
||||
await closeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
voiceSessionId,
|
||||
config: {},
|
||||
});
|
||||
|
||||
expect(resolveClientVoiceRunBinding("run-active")).toMatchObject({ voiceSessionId });
|
||||
});
|
||||
|
||||
it("resolves the open client record for legacy tool calls", () => {
|
||||
const voiceSessionId = createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
origin: "client",
|
||||
});
|
||||
|
||||
expect(
|
||||
resolveOpenClientVoiceSessionId({ agentId: "main", sessionKey: "agent:main:main" }),
|
||||
).toBe(voiceSessionId);
|
||||
expect(
|
||||
resolveOpenClientVoiceSessionId({ agentId: "main", sessionKey: "agent:main:other" }),
|
||||
).toBeUndefined();
|
||||
createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
origin: "client",
|
||||
});
|
||||
expect(
|
||||
resolveOpenClientVoiceSessionId({ agentId: "main", sessionKey: "agent:main:main" }),
|
||||
).toBeUndefined();
|
||||
});
|
||||
|
||||
it("keeps repeated tool-call ids separate across consult runs", () => {
|
||||
const voiceSessionId = createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
origin: "client",
|
||||
});
|
||||
for (const runId of ["run-1", "run-2"]) {
|
||||
registerClientVoiceConsultRun({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
voiceSessionId,
|
||||
runId,
|
||||
});
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "tool.execution.started",
|
||||
runId,
|
||||
toolCallId: "call-1",
|
||||
toolName: "message",
|
||||
mutatingAction: true,
|
||||
});
|
||||
}
|
||||
|
||||
expect(clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.effects).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("closes stale records and leaves recent records open", async () => {
|
||||
const stale = createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:stale",
|
||||
origin: "client",
|
||||
now: 1,
|
||||
});
|
||||
const recent = createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:recent",
|
||||
origin: "client",
|
||||
now: 6 * 60 * 60_000,
|
||||
});
|
||||
|
||||
expect(
|
||||
await closeStaleClientVoiceSessions({
|
||||
agentId: "main",
|
||||
config: {},
|
||||
now: 6 * 60 * 60_000 + 2,
|
||||
}),
|
||||
).toBe(1);
|
||||
expect(clientVoiceSessionTesting.readRecord("main", stale)?.status).toBe("closed");
|
||||
expect(clientVoiceSessionTesting.readRecord("main", recent)?.status).toBe("open");
|
||||
});
|
||||
|
||||
it("records only mutating started effects and updates their terminal status", () => {
|
||||
const voiceSessionId = createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
origin: "client",
|
||||
});
|
||||
registerClientVoiceConsultRun({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
voiceSessionId,
|
||||
runId: "run-1",
|
||||
});
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "tool.execution.started",
|
||||
runId: "run-1",
|
||||
toolCallId: "read-1",
|
||||
toolName: "read",
|
||||
mutatingAction: false,
|
||||
});
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "tool.execution.started",
|
||||
runId: "run-1",
|
||||
toolCallId: "message-1",
|
||||
toolName: "message",
|
||||
mutatingAction: true,
|
||||
});
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "tool.execution.error",
|
||||
runId: "run-1",
|
||||
toolCallId: "message-1",
|
||||
toolName: "message",
|
||||
durationMs: 4,
|
||||
errorCategory: "aborted",
|
||||
terminalReason: "cancelled",
|
||||
});
|
||||
|
||||
expect(clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.effects).toEqual([
|
||||
expect.objectContaining({
|
||||
toolCallId: "message-1",
|
||||
toolName: "message",
|
||||
status: "cancelled",
|
||||
finishedAt: expect.any(Number),
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("records post-close effects and defers the digest until the last consult completes", async () => {
|
||||
await seedSession("agent:main:main", {
|
||||
lastChannel: "discord",
|
||||
lastTo: "channel:voice-updates",
|
||||
});
|
||||
const voiceSessionId = createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
origin: "client",
|
||||
});
|
||||
for (const runId of ["run-1", "run-2"]) {
|
||||
registerClientVoiceConsultRun({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
voiceSessionId,
|
||||
runId,
|
||||
});
|
||||
}
|
||||
|
||||
await closeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
voiceSessionId,
|
||||
config: {},
|
||||
});
|
||||
expect(sendDurableMessageBatch).not.toHaveBeenCalled();
|
||||
|
||||
for (const runId of ["run-1", "run-2"]) {
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "tool.execution.started",
|
||||
runId,
|
||||
toolCallId: `call-${runId}`,
|
||||
toolName: "message",
|
||||
mutatingAction: true,
|
||||
});
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "tool.execution.completed",
|
||||
runId,
|
||||
toolCallId: `call-${runId}`,
|
||||
toolName: "message",
|
||||
durationMs: 5,
|
||||
});
|
||||
}
|
||||
expect(clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.effects).toEqual([
|
||||
expect.objectContaining({ runId: "run-1", status: "succeeded" }),
|
||||
expect.objectContaining({ runId: "run-2", status: "succeeded" }),
|
||||
]);
|
||||
|
||||
await completeRun("run-1");
|
||||
expect(sendDurableMessageBatch).not.toHaveBeenCalled();
|
||||
await completeRun("run-2");
|
||||
await vi.waitFor(() => expect(sendDurableMessageBatch).toHaveBeenCalledTimes(1));
|
||||
expect(sendDurableMessageBatch).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
payloads: [{ text: "Voice call changes\n- message: succeeded\n- message: succeeded" }],
|
||||
}),
|
||||
);
|
||||
expect(clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.digestDeliveredAt).toEqual(
|
||||
expect.any(Number),
|
||||
);
|
||||
|
||||
await closeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
voiceSessionId,
|
||||
config: {},
|
||||
});
|
||||
expect(sendDurableMessageBatch).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("retries a deferred digest on the next voice session after a failed delivery", async () => {
|
||||
await seedSession("agent:main:main", {
|
||||
lastChannel: "discord",
|
||||
lastTo: "channel:voice-updates",
|
||||
});
|
||||
const voiceSessionId = createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
origin: "client",
|
||||
});
|
||||
registerClientVoiceConsultRun({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
voiceSessionId,
|
||||
runId: "run-live",
|
||||
});
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "tool.execution.started",
|
||||
runId: "run-live",
|
||||
toolCallId: "call-run-live",
|
||||
toolName: "message",
|
||||
mutatingAction: true,
|
||||
});
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "tool.execution.completed",
|
||||
runId: "run-live",
|
||||
toolCallId: "call-run-live",
|
||||
toolName: "message",
|
||||
durationMs: 5,
|
||||
});
|
||||
// Call ends while the consult still runs, so the digest is deferred.
|
||||
await closeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
voiceSessionId,
|
||||
config: {},
|
||||
});
|
||||
sendDurableMessageBatch.mockRejectedValueOnce(new Error("channel offline"));
|
||||
await completeRun("run-live");
|
||||
expect(
|
||||
clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.digestDeliveredAt,
|
||||
).toBeUndefined();
|
||||
|
||||
// Starting the next voice session retries the deferred digest.
|
||||
await closeStaleClientVoiceSessions({ agentId: "main", config: {} });
|
||||
await vi.waitFor(() =>
|
||||
expect(
|
||||
clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.digestDeliveredAt,
|
||||
).toEqual(expect.any(Number)),
|
||||
);
|
||||
});
|
||||
|
||||
it("retries the mutation digest after a transient send failure", async () => {
|
||||
await seedSession("agent:main:main", {
|
||||
lastChannel: "discord",
|
||||
lastTo: "channel:voice-updates",
|
||||
});
|
||||
const voiceSessionId = createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
origin: "client",
|
||||
});
|
||||
recordMutation(voiceSessionId);
|
||||
await completeRun(`run-${voiceSessionId}`);
|
||||
sendDurableMessageBatch.mockRejectedValueOnce(new Error("channel offline"));
|
||||
|
||||
await closeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
voiceSessionId,
|
||||
config: {},
|
||||
}).catch(() => undefined);
|
||||
expect(
|
||||
clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.digestDeliveredAt,
|
||||
).toBeUndefined();
|
||||
|
||||
await closeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
voiceSessionId,
|
||||
config: {},
|
||||
});
|
||||
expect(sendDurableMessageBatch).toHaveBeenCalledTimes(2);
|
||||
expect(clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.digestDeliveredAt).toEqual(
|
||||
expect.any(Number),
|
||||
);
|
||||
});
|
||||
|
||||
it("delivers one mutation digest and skips webchat or missing targets", async () => {
|
||||
await seedSession("agent:main:main", {
|
||||
lastChannel: "discord",
|
||||
lastTo: "channel:voice-updates",
|
||||
});
|
||||
const delivered = createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
origin: "client",
|
||||
});
|
||||
recordMutation(delivered);
|
||||
await completeRun(`run-${delivered}`);
|
||||
await closeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
voiceSessionId: delivered,
|
||||
config: {},
|
||||
});
|
||||
await closeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey: "agent:main:main",
|
||||
voiceSessionId: delivered,
|
||||
config: {},
|
||||
});
|
||||
expect(sendDurableMessageBatch).toHaveBeenCalledTimes(1);
|
||||
expect(sendDurableMessageBatch).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
durability: "required",
|
||||
requireUnknownSendReconciliation: true,
|
||||
payloads: [{ text: "Voice call changes\n- message: succeeded" }],
|
||||
}),
|
||||
);
|
||||
|
||||
for (const [voiceSessionId, route] of [
|
||||
["voice-webchat", { lastChannel: "webchat", lastTo: "browser" }],
|
||||
["voice-no-target", {}],
|
||||
] as const) {
|
||||
const sessionKey = `agent:main:${voiceSessionId}`;
|
||||
await seedSession(sessionKey, route);
|
||||
createOrResumeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey,
|
||||
origin: "client",
|
||||
voiceSessionId,
|
||||
});
|
||||
registerClientVoiceConsultRun({
|
||||
agentId: "main",
|
||||
sessionKey,
|
||||
voiceSessionId,
|
||||
runId: `run-${voiceSessionId}`,
|
||||
});
|
||||
emitTrustedDiagnosticEvent({
|
||||
type: "tool.execution.started",
|
||||
runId: `run-${voiceSessionId}`,
|
||||
toolCallId: `call-${voiceSessionId}`,
|
||||
toolName: "message",
|
||||
mutatingAction: true,
|
||||
});
|
||||
await completeRun(`run-${voiceSessionId}`);
|
||||
await closeClientVoiceSession({
|
||||
agentId: "main",
|
||||
sessionKey,
|
||||
voiceSessionId,
|
||||
config: {},
|
||||
});
|
||||
}
|
||||
expect(sendDurableMessageBatch).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,732 @@
|
||||
/** Durable per-agent voice-call records for Talk continuity and mutation evidence. */
|
||||
import { randomUUID } from "node:crypto";
|
||||
import {
|
||||
appendTranscriptMessage,
|
||||
loadSessionEntry,
|
||||
upsertSessionEntry,
|
||||
} from "../config/sessions/session-accessor.js";
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import {
|
||||
onTrustedInternalDiagnosticEvent,
|
||||
onTrustedToolExecutionEvent,
|
||||
type TrustedToolExecutionEvent,
|
||||
} from "../infra/diagnostic-events.js";
|
||||
import { buildOutboundSessionContext } from "../infra/outbound/session-context.js";
|
||||
import { resolveSessionDeliveryTarget } from "../infra/outbound/targets-session.js";
|
||||
import {
|
||||
openOpenClawAgentDatabase,
|
||||
runOpenClawAgentWriteTransaction,
|
||||
} from "../state/openclaw-agent-db.js";
|
||||
import { truncateUtf16Safe } from "../utils.js";
|
||||
import {
|
||||
deactivateClientVoiceConfirmationSession,
|
||||
noteClientVoiceConfirmationUtterance,
|
||||
releaseClientVoiceConfirmationRun,
|
||||
} from "./client-voice-confirmation.js";
|
||||
import {
|
||||
assertVoiceSessionOwnership as assertOwnership,
|
||||
type ClientVoiceRunBinding,
|
||||
type ClientVoiceSessionRecord,
|
||||
type ClientVoiceToolEffect,
|
||||
operationKey,
|
||||
parseStoredVoiceSessionRecord as parseStoredRecord,
|
||||
readVoiceSessionRecord as readRecord,
|
||||
readVoiceSessionRecordInTransaction as readRecordInTransaction,
|
||||
VOICE_SESSION_CACHE_SCOPE as CACHE_SCOPE,
|
||||
VOICE_SESSION_MAX_TRANSCRIPT_CHARS as MAX_TRANSCRIPT_CHARS,
|
||||
VOICE_SESSION_RECORD_VERSION as RECORD_VERSION,
|
||||
VOICE_SESSION_STALE_AFTER_MS as STALE_AFTER_MS,
|
||||
writeVoiceSessionRecordInTransaction as writeRecordInTransaction,
|
||||
} from "./client-voice-session-store.js";
|
||||
|
||||
const voiceSessionByRunId = new Map<string, ClientVoiceRunBinding>();
|
||||
const voiceSessionOperations = new Map<string, Promise<void>>();
|
||||
const deferredDigestConfigs = new Map<string, OpenClawConfig>();
|
||||
let unsubscribeToolEffects: (() => void) | undefined;
|
||||
let unsubscribeRunCompletion: (() => void) | undefined;
|
||||
|
||||
function hasLiveConsultRun(record: ClientVoiceSessionRecord): boolean {
|
||||
return record.consultRunIds.some((runId) => {
|
||||
const binding = voiceSessionByRunId.get(runId);
|
||||
return (
|
||||
binding?.agentId === record.agentId &&
|
||||
binding.voiceSessionId === record.voiceSessionId &&
|
||||
binding.sessionKey === record.sessionKey
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async function runVoiceSessionOperation<T>(
|
||||
agentId: string,
|
||||
voiceSessionId: string,
|
||||
operation: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
const key = operationKey(agentId, voiceSessionId);
|
||||
const previous = voiceSessionOperations.get(key) ?? Promise.resolve();
|
||||
const current = previous.then(operation, operation);
|
||||
const settled = current.then(
|
||||
() => undefined,
|
||||
() => undefined,
|
||||
);
|
||||
voiceSessionOperations.set(key, settled);
|
||||
void settled.then(() => {
|
||||
if (voiceSessionOperations.get(key) === settled) {
|
||||
voiceSessionOperations.delete(key);
|
||||
}
|
||||
});
|
||||
return await current;
|
||||
}
|
||||
|
||||
function effectStatus(event: TrustedToolExecutionEvent): ClientVoiceToolEffect["status"] {
|
||||
if (event.type === "tool.execution.started") {
|
||||
return "started";
|
||||
}
|
||||
if (event.type === "tool.execution.completed") {
|
||||
return "succeeded";
|
||||
}
|
||||
if (event.type === "tool.execution.blocked") {
|
||||
return "blocked";
|
||||
}
|
||||
return event.terminalReason === "cancelled" ? "cancelled" : "failed";
|
||||
}
|
||||
|
||||
function recordClientVoiceToolEffect(event: TrustedToolExecutionEvent): void {
|
||||
const runId = event.runId;
|
||||
if (!runId) {
|
||||
return;
|
||||
}
|
||||
const binding = voiceSessionByRunId.get(runId);
|
||||
if (!binding) {
|
||||
return;
|
||||
}
|
||||
runOpenClawAgentWriteTransaction(
|
||||
(database) => {
|
||||
const record = readRecordInTransaction(database, binding.voiceSessionId);
|
||||
if (!record) {
|
||||
return;
|
||||
}
|
||||
const existing = event.toolCallId
|
||||
? record.effects.find(
|
||||
(effect) => effect.runId === runId && effect.toolCallId === event.toolCallId,
|
||||
)
|
||||
: record.effects.findLast(
|
||||
(effect) =>
|
||||
effect.runId === runId &&
|
||||
effect.toolName === event.toolName &&
|
||||
effect.status === "started",
|
||||
);
|
||||
if (event.type !== "tool.execution.started" && !existing) {
|
||||
return;
|
||||
}
|
||||
if (event.type !== "tool.execution.started" && existing) {
|
||||
existing.status = effectStatus(event);
|
||||
existing.finishedAt = event.ts;
|
||||
} else if (event.mutatingAction === true && (!event.toolCallId || !existing)) {
|
||||
record.effects.push({
|
||||
runId,
|
||||
...(event.toolCallId ? { toolCallId: event.toolCallId } : {}),
|
||||
toolName: event.toolName,
|
||||
startedAt: event.ts,
|
||||
status: "started",
|
||||
});
|
||||
}
|
||||
record.updatedAt = Date.now();
|
||||
writeRecordInTransaction(database, record);
|
||||
},
|
||||
{ agentId: binding.agentId },
|
||||
);
|
||||
}
|
||||
|
||||
function ensureToolEffectSubscription(): void {
|
||||
unsubscribeToolEffects ??= onTrustedToolExecutionEvent(recordClientVoiceToolEffect);
|
||||
unsubscribeRunCompletion ??= onTrustedInternalDiagnosticEvent((event) => {
|
||||
if (event.type !== "run.completed") {
|
||||
return;
|
||||
}
|
||||
const binding = voiceSessionByRunId.get(event.runId);
|
||||
if (!binding) {
|
||||
return;
|
||||
}
|
||||
voiceSessionByRunId.delete(event.runId);
|
||||
releaseClientVoiceConfirmationRun(binding.agentId, binding.voiceSessionId, event.runId);
|
||||
void finishDeferredMutationDigest(binding).catch((error: unknown) => {
|
||||
console.warn(
|
||||
`[talk] deferred voice mutation digest failed: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/** Create a call record or resume the same open call across transport restarts. */
|
||||
export function createOrResumeClientVoiceSession(params: {
|
||||
agentId: string;
|
||||
sessionKey: string;
|
||||
provider?: string;
|
||||
origin: "client" | "relay";
|
||||
transcriptCapable?: boolean;
|
||||
voiceSessionId?: string;
|
||||
now?: number;
|
||||
}): string {
|
||||
const voiceSessionId = params.voiceSessionId?.trim() || randomUUID();
|
||||
const provider = params.provider?.trim() || undefined;
|
||||
const now = params.now ?? Date.now();
|
||||
runOpenClawAgentWriteTransaction(
|
||||
(database) => {
|
||||
const existing = readRecordInTransaction(database, voiceSessionId);
|
||||
if (existing) {
|
||||
assertOwnership(existing, params);
|
||||
if (existing.origin !== params.origin) {
|
||||
throw new Error("voice session origin does not match");
|
||||
}
|
||||
if (existing.status !== "open") {
|
||||
throw new Error("voice session is already closed");
|
||||
}
|
||||
if (existing.provider && provider && existing.provider !== provider) {
|
||||
throw new Error("voice session provider does not match");
|
||||
}
|
||||
if (!existing.provider && provider) {
|
||||
existing.provider = provider;
|
||||
}
|
||||
if (params.transcriptCapable === true) {
|
||||
existing.transcriptCapable = true;
|
||||
}
|
||||
existing.updatedAt = now;
|
||||
writeRecordInTransaction(database, existing);
|
||||
return;
|
||||
}
|
||||
writeRecordInTransaction(database, {
|
||||
version: RECORD_VERSION,
|
||||
voiceSessionId,
|
||||
agentId: params.agentId,
|
||||
sessionKey: params.sessionKey,
|
||||
...(provider ? { provider } : {}),
|
||||
origin: params.origin,
|
||||
...(params.transcriptCapable === true ? { transcriptCapable: true } : {}),
|
||||
status: "open",
|
||||
createdAt: now,
|
||||
updatedAt: now,
|
||||
consultRunIds: [],
|
||||
effects: [],
|
||||
});
|
||||
},
|
||||
{ agentId: params.agentId },
|
||||
);
|
||||
return voiceSessionId;
|
||||
}
|
||||
|
||||
/** Ensure Talk has the same canonical agent-session row that chat turns append to. */
|
||||
export async function ensureClientVoiceAgentSessionEntry(params: {
|
||||
agentId: string;
|
||||
sessionKey: string;
|
||||
}): Promise<void> {
|
||||
const existing = loadSessionEntry(params);
|
||||
if (existing?.sessionId) {
|
||||
return;
|
||||
}
|
||||
const created = await upsertSessionEntry(params, {});
|
||||
if (!created?.sessionId) {
|
||||
throw new Error(`agent session could not be initialized (${params.sessionKey})`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Correlate a consult run with its open call for confirmation and mutation evidence. */
|
||||
export function registerClientVoiceConsultRun(params: {
|
||||
agentId: string;
|
||||
sessionKey: string;
|
||||
voiceSessionId: string;
|
||||
runId: string;
|
||||
config?: OpenClawConfig;
|
||||
}): void {
|
||||
let recordClosed = false;
|
||||
runOpenClawAgentWriteTransaction(
|
||||
(database) => {
|
||||
const record = readRecordInTransaction(database, params.voiceSessionId);
|
||||
if (!record) {
|
||||
throw new Error("voice session not found");
|
||||
}
|
||||
assertOwnership(record, params);
|
||||
recordClosed = record.status === "closed";
|
||||
// A close can race in while chat.send is still acking this run. The run has
|
||||
// already started, so bind it anyway (even on a closed record) to keep effect
|
||||
// capture; aborting here would drop a just-confirmed high-impact action.
|
||||
if (!record.consultRunIds.includes(params.runId)) {
|
||||
record.consultRunIds.push(params.runId);
|
||||
record.updatedAt = Date.now();
|
||||
writeRecordInTransaction(database, record);
|
||||
}
|
||||
},
|
||||
{ agentId: params.agentId },
|
||||
);
|
||||
voiceSessionByRunId.set(params.runId, {
|
||||
agentId: params.agentId,
|
||||
voiceSessionId: params.voiceSessionId,
|
||||
sessionKey: params.sessionKey,
|
||||
});
|
||||
// Bound to a call that already closed: its digest may have been delivered and its
|
||||
// retry config cleared, so re-arm it to cover this late run's mutations.
|
||||
if (recordClosed && params.config) {
|
||||
deferredDigestConfigs.set(operationKey(params.agentId, params.voiceSessionId), params.config);
|
||||
}
|
||||
ensureToolEffectSubscription();
|
||||
}
|
||||
|
||||
/** Return the open voice-call binding for one executing run. */
|
||||
export function resolveClientVoiceRunBinding(runId?: string): ClientVoiceRunBinding | undefined {
|
||||
return runId ? voiceSessionByRunId.get(runId) : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Confirmation applies only when the session can observe spoken approvals:
|
||||
* relay sessions (server hears utterances) or clients that report transcripts.
|
||||
* Legacy clients without transcript reporting keep pre-gate behavior.
|
||||
*/
|
||||
export function isClientVoiceSessionConfirmable(binding: ClientVoiceRunBinding): boolean {
|
||||
const record = readRecord(binding.agentId, binding.voiceSessionId);
|
||||
return (
|
||||
record?.origin === "relay" ||
|
||||
record?.transcriptCapable === true ||
|
||||
record?.hasUserTranscript === true
|
||||
);
|
||||
}
|
||||
|
||||
/** Validate ownership and open state before starting a voice-bound consult. */
|
||||
export function assertClientVoiceSessionOpen(params: {
|
||||
agentId: string;
|
||||
sessionKey: string;
|
||||
voiceSessionId: string;
|
||||
}): "client" | "relay" {
|
||||
const record = readRecord(params.agentId, params.voiceSessionId);
|
||||
if (!record) {
|
||||
throw new Error("voice session not found");
|
||||
}
|
||||
assertOwnership(record, params);
|
||||
if (record.status !== "open") {
|
||||
throw new Error("voice session is closed");
|
||||
}
|
||||
return record.origin;
|
||||
}
|
||||
|
||||
/** Validate durable ownership without rejecting an idempotent close retry. */
|
||||
export function resolveClientVoiceSessionOrigin(params: {
|
||||
agentId: string;
|
||||
sessionKey: string;
|
||||
voiceSessionId: string;
|
||||
}): "client" | "relay" {
|
||||
const record = readRecord(params.agentId, params.voiceSessionId);
|
||||
if (!record) {
|
||||
throw new Error("voice session not found");
|
||||
}
|
||||
assertOwnership(record, params);
|
||||
return record.origin;
|
||||
}
|
||||
|
||||
/** Resolve the newest open client-owned call for legacy tool-call clients. */
|
||||
export function resolveOpenClientVoiceSessionId(params: {
|
||||
agentId: string;
|
||||
sessionKey: string;
|
||||
}): string | undefined {
|
||||
const database = openOpenClawAgentDatabase({ agentId: params.agentId });
|
||||
const rows = database.db
|
||||
.prepare("SELECT value_json FROM cache_entries WHERE scope = ? ORDER BY updated_at DESC")
|
||||
.all(CACHE_SCOPE) as Array<{ value_json?: unknown }>;
|
||||
let match: string | undefined;
|
||||
for (const row of rows) {
|
||||
const record = parseStoredRecord(row.value_json);
|
||||
if (
|
||||
record?.origin === "client" &&
|
||||
record.status === "open" &&
|
||||
record.agentId === params.agentId &&
|
||||
record.sessionKey === params.sessionKey
|
||||
) {
|
||||
if (match) {
|
||||
return undefined;
|
||||
}
|
||||
match = record.voiceSessionId;
|
||||
}
|
||||
}
|
||||
return match;
|
||||
}
|
||||
|
||||
function buildPersistedVoiceMessage(params: {
|
||||
role: "user" | "assistant";
|
||||
text: string;
|
||||
timestamp: number;
|
||||
provider: string;
|
||||
}): Record<string, unknown> {
|
||||
const provenance = { kind: "realtime_voice", sourceChannel: "talk" };
|
||||
if (params.role === "user") {
|
||||
return {
|
||||
role: "user",
|
||||
content: [{ type: "text", text: params.text }],
|
||||
timestamp: params.timestamp,
|
||||
provenance,
|
||||
};
|
||||
}
|
||||
return {
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: params.text }],
|
||||
api: "realtime",
|
||||
provider: params.provider,
|
||||
model: "realtime-voice",
|
||||
stopReason: "stop",
|
||||
timestamp: params.timestamp,
|
||||
provenance,
|
||||
};
|
||||
}
|
||||
|
||||
async function appendVoiceTranscript(params: {
|
||||
agentId: string;
|
||||
sessionKey: string;
|
||||
voiceSessionId: string;
|
||||
origin: "client" | "relay";
|
||||
entryId: string;
|
||||
role: "user" | "assistant";
|
||||
text: string;
|
||||
timestamp?: number;
|
||||
config?: OpenClawConfig;
|
||||
}): Promise<void> {
|
||||
await runVoiceSessionOperation(params.agentId, params.voiceSessionId, async () => {
|
||||
const text = truncateUtf16Safe(params.text.trim(), MAX_TRANSCRIPT_CHARS);
|
||||
if (!text) {
|
||||
return;
|
||||
}
|
||||
const record = readRecord(params.agentId, params.voiceSessionId);
|
||||
if (!record) {
|
||||
throw new Error("voice session not found");
|
||||
}
|
||||
assertOwnership(record, params);
|
||||
if (record.status !== "open") {
|
||||
throw new Error("voice session is closed");
|
||||
}
|
||||
if (record.origin !== params.origin) {
|
||||
throw new Error("voice session origin does not allow this transcript source");
|
||||
}
|
||||
const sessionEntry = loadSessionEntry({
|
||||
agentId: params.agentId,
|
||||
sessionKey: params.sessionKey,
|
||||
});
|
||||
if (!sessionEntry?.sessionId) {
|
||||
throw new Error(`agent session not found (${params.sessionKey})`);
|
||||
}
|
||||
const observedAt = Date.now();
|
||||
const timestamp = params.timestamp ?? observedAt;
|
||||
await appendTranscriptMessage(
|
||||
{
|
||||
agentId: params.agentId,
|
||||
sessionId: sessionEntry.sessionId,
|
||||
sessionKey: params.sessionKey,
|
||||
},
|
||||
{
|
||||
...(params.config ? { config: params.config } : {}),
|
||||
eventId: `voice:${params.voiceSessionId}:${params.entryId}`,
|
||||
message: buildPersistedVoiceMessage({
|
||||
role: params.role,
|
||||
text,
|
||||
timestamp,
|
||||
provider: record.provider ?? "realtime",
|
||||
}),
|
||||
now: timestamp,
|
||||
},
|
||||
);
|
||||
runOpenClawAgentWriteTransaction(
|
||||
(database) => {
|
||||
const current = readRecordInTransaction(database, params.voiceSessionId);
|
||||
if (!current) {
|
||||
throw new Error("voice session disappeared during transcript append");
|
||||
}
|
||||
assertOwnership(current, params);
|
||||
// Reaching here means this exact eventId is durably persisted (fresh append or
|
||||
// idempotent dedup of our own prior write). Arm confirmation bookkeeping in both
|
||||
// cases so a retry after a partial failure still records the user utterance.
|
||||
if (params.role === "user") {
|
||||
current.hasUserTranscript = true;
|
||||
}
|
||||
current.updatedAt = Date.now();
|
||||
writeRecordInTransaction(database, current);
|
||||
},
|
||||
{ agentId: params.agentId },
|
||||
);
|
||||
if (params.role === "user") {
|
||||
noteClientVoiceConfirmationUtterance({
|
||||
agentId: params.agentId,
|
||||
voiceSessionId: params.voiceSessionId,
|
||||
text,
|
||||
timestamp: observedAt,
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/** Append one finalized client-owned transcript item idempotently. */
|
||||
export async function appendClientVoiceTranscript(
|
||||
params: Omit<Parameters<typeof appendVoiceTranscript>[0], "origin">,
|
||||
): Promise<void> {
|
||||
await appendVoiceTranscript({ ...params, origin: "client" });
|
||||
}
|
||||
|
||||
/** Append one finalized relay-owned transcript item idempotently. */
|
||||
export async function appendRelayVoiceTranscript(
|
||||
params: Omit<Parameters<typeof appendVoiceTranscript>[0], "origin">,
|
||||
): Promise<void> {
|
||||
await appendVoiceTranscript({ ...params, origin: "relay" });
|
||||
}
|
||||
|
||||
// The mutation digest is a best-effort informational summary, not a durable record:
|
||||
// the authoritative effects live on the voice record and the canonical transcript is
|
||||
// persisted independently. Accepted tradeoffs (retry state is process-local like the
|
||||
// rest of this subsystem): a digest already delivered at close does not reissue for a
|
||||
// run that binds afterward, and an undelivered digest whose retry map is cleared by a
|
||||
// gateway restart is not rediscovered. Both lose a summary message, never real data.
|
||||
function formatMutationDigest(effects: ClientVoiceToolEffect[]): string | undefined {
|
||||
if (effects.length === 0) {
|
||||
return undefined;
|
||||
}
|
||||
return [
|
||||
"Voice call changes",
|
||||
...effects
|
||||
.slice(0, 12)
|
||||
.map(
|
||||
(effect) =>
|
||||
`- ${effect.toolName}: ${effect.status === "started" ? "outcome not confirmed" : effect.status}`,
|
||||
),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
async function deliverMutationDigest(
|
||||
record: ClientVoiceSessionRecord,
|
||||
config: OpenClawConfig,
|
||||
target: { channel: string; to: string; accountId?: string; threadId?: string | number | null },
|
||||
text: string,
|
||||
): Promise<void> {
|
||||
const { sendDurableMessageBatch } = await import("../channels/message/runtime.js");
|
||||
const send = await sendDurableMessageBatch({
|
||||
cfg: config,
|
||||
channel: target.channel,
|
||||
to: target.to,
|
||||
...(target.accountId ? { accountId: target.accountId } : {}),
|
||||
...(target.threadId != null ? { threadId: target.threadId } : {}),
|
||||
payloads: [{ text }],
|
||||
durability: "required",
|
||||
requireUnknownSendReconciliation: true,
|
||||
session: buildOutboundSessionContext({
|
||||
cfg: config,
|
||||
agentId: record.agentId,
|
||||
sessionKey: record.sessionKey,
|
||||
policySessionKey: record.sessionKey,
|
||||
}),
|
||||
});
|
||||
if (send.status === "failed" || send.status === "partial_failed") {
|
||||
throw send.error;
|
||||
}
|
||||
}
|
||||
|
||||
async function deliverMutationDigestOnce(
|
||||
record: ClientVoiceSessionRecord,
|
||||
config: OpenClawConfig,
|
||||
): Promise<void> {
|
||||
if (record.digestDeliveredAt) {
|
||||
return;
|
||||
}
|
||||
const text = formatMutationDigest(record.effects);
|
||||
if (!text) {
|
||||
return;
|
||||
}
|
||||
const entry = loadSessionEntry({ agentId: record.agentId, sessionKey: record.sessionKey });
|
||||
const target = resolveSessionDeliveryTarget({ entry, requestedChannel: "last" });
|
||||
if (!target.channel || target.channel === "webchat" || !target.to) {
|
||||
return;
|
||||
}
|
||||
// Send first, mark after: a transient send failure (the common case) then leaves
|
||||
// the marker unset so close/stale-sweep retries. The rare crash between a durable
|
||||
// send and this marker can re-send one informational digest, which is acceptable.
|
||||
await deliverMutationDigest(
|
||||
record,
|
||||
config,
|
||||
{
|
||||
channel: target.channel,
|
||||
to: target.to,
|
||||
accountId: target.accountId,
|
||||
threadId: target.threadId,
|
||||
},
|
||||
text,
|
||||
);
|
||||
const deliveredAt = Date.now();
|
||||
runOpenClawAgentWriteTransaction(
|
||||
(database) => {
|
||||
const current = readRecordInTransaction(database, record.voiceSessionId);
|
||||
if (!current || current.digestDeliveredAt) {
|
||||
return;
|
||||
}
|
||||
current.digestDeliveredAt = deliveredAt;
|
||||
current.updatedAt = deliveredAt;
|
||||
writeRecordInTransaction(database, current);
|
||||
},
|
||||
{ agentId: record.agentId },
|
||||
);
|
||||
}
|
||||
|
||||
async function finishDeferredMutationDigest(binding: {
|
||||
agentId: string;
|
||||
voiceSessionId: string;
|
||||
}): Promise<void> {
|
||||
const key = operationKey(binding.agentId, binding.voiceSessionId);
|
||||
const config = deferredDigestConfigs.get(key);
|
||||
if (!config) {
|
||||
return;
|
||||
}
|
||||
await runVoiceSessionOperation(binding.agentId, binding.voiceSessionId, async () => {
|
||||
const record = readRecord(binding.agentId, binding.voiceSessionId);
|
||||
if (!record || record.status !== "closed" || hasLiveConsultRun(record)) {
|
||||
return;
|
||||
}
|
||||
// Deliver first; only drop the retry config once the send succeeds, so a
|
||||
// transient failure here is retried by the next closeStaleClientVoiceSessions.
|
||||
await deliverMutationDigestOnce(record, config);
|
||||
deferredDigestConfigs.delete(key);
|
||||
});
|
||||
}
|
||||
|
||||
/** Retry deferred digests whose delivery previously failed after the call closed. */
|
||||
async function retryDeferredMutationDigests(agentId: string): Promise<void> {
|
||||
for (const key of Array.from(deferredDigestConfigs.keys())) {
|
||||
const [entryAgentId, voiceSessionId] = key.split("\0");
|
||||
if (entryAgentId !== agentId || !voiceSessionId) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await finishDeferredMutationDigest({ agentId, voiceSessionId });
|
||||
} catch {
|
||||
// Still undeliverable; a later voice session retries again.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function closeClientVoiceSessionInternal(params: {
|
||||
agentId: string;
|
||||
sessionKey: string;
|
||||
voiceSessionId: string;
|
||||
config: OpenClawConfig;
|
||||
now?: number;
|
||||
}): Promise<void> {
|
||||
const existing = readRecord(params.agentId, params.voiceSessionId);
|
||||
if (!existing) {
|
||||
throw new Error("voice session not found");
|
||||
}
|
||||
assertOwnership(existing, params);
|
||||
const now = params.now ?? Date.now();
|
||||
runOpenClawAgentWriteTransaction(
|
||||
(database) => {
|
||||
const current = readRecordInTransaction(database, params.voiceSessionId);
|
||||
if (!current) {
|
||||
throw new Error("voice session disappeared during close");
|
||||
}
|
||||
assertOwnership(current, params);
|
||||
if (current.status === "open") {
|
||||
current.status = "closed";
|
||||
current.closedAt = now;
|
||||
current.updatedAt = now;
|
||||
writeRecordInTransaction(database, current);
|
||||
}
|
||||
},
|
||||
{ agentId: params.agentId },
|
||||
);
|
||||
const closed = readRecord(params.agentId, params.voiceSessionId);
|
||||
if (!closed) {
|
||||
throw new Error("voice session disappeared after close");
|
||||
}
|
||||
// Transport close does not end consult runs: live bindings keep effect capture active,
|
||||
// approved grants stay valid for those runs, and the digest waits for the last run.completed.
|
||||
const liveRunIds = closed.consultRunIds.filter((runId) => {
|
||||
const binding = voiceSessionByRunId.get(runId);
|
||||
return binding?.voiceSessionId === params.voiceSessionId && binding.agentId === params.agentId;
|
||||
});
|
||||
deactivateClientVoiceConfirmationSession(params.agentId, params.voiceSessionId, liveRunIds);
|
||||
const key = operationKey(params.agentId, params.voiceSessionId);
|
||||
// Both the deferred and immediate paths retain the retry config until a send
|
||||
// actually succeeds, so a channel outage does not permanently lose the digest.
|
||||
deferredDigestConfigs.set(key, params.config);
|
||||
if (hasLiveConsultRun(closed)) {
|
||||
return;
|
||||
}
|
||||
await deliverMutationDigestOnce(closed, params.config);
|
||||
// Intentionally unconditional: a consult that late-binds during the await above has
|
||||
// its mutations omitted from this already-sent point-in-time summary. Re-sending to
|
||||
// include them would double-notify the user for one call, which is a worse outcome
|
||||
// for an informational digest than omitting a straggler action (still on the record).
|
||||
deferredDigestConfigs.delete(key);
|
||||
}
|
||||
|
||||
/** Close a logical voice call and deliver its mutation digest at most once. */
|
||||
export async function closeClientVoiceSession(params: {
|
||||
agentId: string;
|
||||
sessionKey: string;
|
||||
voiceSessionId: string;
|
||||
config: OpenClawConfig;
|
||||
now?: number;
|
||||
}): Promise<void> {
|
||||
await runVoiceSessionOperation(params.agentId, params.voiceSessionId, async () => {
|
||||
await closeClientVoiceSessionInternal(params);
|
||||
});
|
||||
}
|
||||
|
||||
/** Close abandoned open calls idle for the fixed six-hour recovery window. */
|
||||
export async function closeStaleClientVoiceSessions(params: {
|
||||
agentId: string;
|
||||
config: OpenClawConfig;
|
||||
excludeVoiceSessionId?: string;
|
||||
now?: number;
|
||||
warn?: (message: string) => void;
|
||||
}): Promise<number> {
|
||||
const now = params.now ?? Date.now();
|
||||
// A new voice session is the natural retry point for a digest whose delivery
|
||||
// failed after its own call had already closed and its runs completed.
|
||||
await retryDeferredMutationDigests(params.agentId);
|
||||
const database = openOpenClawAgentDatabase({ agentId: params.agentId });
|
||||
const rows = database.db
|
||||
.prepare("SELECT value_json FROM cache_entries WHERE scope = ? AND updated_at <= ?")
|
||||
.all(CACHE_SCOPE, now - STALE_AFTER_MS) as Array<{ value_json?: unknown }>;
|
||||
const stale = rows.flatMap((row) => {
|
||||
const record = parseStoredRecord(row.value_json);
|
||||
return record &&
|
||||
record.status === "open" &&
|
||||
record.voiceSessionId !== params.excludeVoiceSessionId
|
||||
? [record]
|
||||
: [];
|
||||
});
|
||||
let closed = 0;
|
||||
for (const record of stale) {
|
||||
try {
|
||||
await closeClientVoiceSession({
|
||||
agentId: params.agentId,
|
||||
sessionKey: record.sessionKey,
|
||||
voiceSessionId: record.voiceSessionId,
|
||||
config: params.config,
|
||||
now,
|
||||
});
|
||||
closed += 1;
|
||||
} catch (error) {
|
||||
params.warn?.(
|
||||
`failed to close stale voice session ${record.voiceSessionId}: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
return closed;
|
||||
}
|
||||
|
||||
const clientVoiceSessionTesting = {
|
||||
readRecord,
|
||||
reset(): void {
|
||||
voiceSessionByRunId.clear();
|
||||
voiceSessionOperations.clear();
|
||||
deferredDigestConfigs.clear();
|
||||
unsubscribeToolEffects?.();
|
||||
unsubscribeToolEffects = undefined;
|
||||
unsubscribeRunCompletion?.();
|
||||
unsubscribeRunCompletion = undefined;
|
||||
},
|
||||
};
|
||||
|
||||
if (process.env.VITEST || process.env.NODE_ENV === "test") {
|
||||
(globalThis as Record<PropertyKey, unknown>)[Symbol.for("openclaw.clientVoiceSessionTestApi")] =
|
||||
clientVoiceSessionTesting;
|
||||
}
|
||||
@@ -13,7 +13,7 @@ export function createGatewayClientVitestConfig(env?: Record<string, string | un
|
||||
],
|
||||
{
|
||||
env,
|
||||
exclude: ["src/gateway/**/*server*.test.ts"],
|
||||
exclude: ["src/gateway/**/*server*.test.ts", "src/gateway/server-methods/**/*.test.ts"],
|
||||
// Gateway child projects share one include file; preserve this project's ownership.
|
||||
intersectIncludeFile: true,
|
||||
isolate: true,
|
||||
|
||||
@@ -16,9 +16,11 @@ function requireFirstMockCall(calls: readonly unknown[][], label: string): unkno
|
||||
|
||||
describe("RealtimeTalkSession consult handoff", () => {
|
||||
it("submits realtime consults through the Gateway tool-call endpoint", async () => {
|
||||
const order: string[] = [];
|
||||
let listener: ((event: { event: string; payload?: unknown }) => void) | undefined;
|
||||
const request = vi.fn(async (method: string, _params: unknown) => {
|
||||
if (method === "talk.client.toolCall") {
|
||||
order.push("tool-call");
|
||||
setImmediate(() => {
|
||||
listener?.({
|
||||
event: "chat",
|
||||
@@ -40,11 +42,16 @@ describe("RealtimeTalkSession consult handoff", () => {
|
||||
};
|
||||
});
|
||||
const submit = vi.fn();
|
||||
const flushTranscriptWrites = vi.fn(async () => {
|
||||
order.push("flush");
|
||||
});
|
||||
|
||||
await submitRealtimeTalkConsult({
|
||||
ctx: {
|
||||
client: { request, addEventListener },
|
||||
sessionKey: "agent:main:main",
|
||||
voiceSessionId: "voice-1",
|
||||
flushTranscriptWrites,
|
||||
callbacks: {},
|
||||
} as never,
|
||||
callId: "call-1",
|
||||
@@ -53,13 +60,23 @@ describe("RealtimeTalkSession consult handoff", () => {
|
||||
});
|
||||
|
||||
const toolCall = requireFirstMockCall(request.mock.calls, "Gateway request") as
|
||||
| [string, { sessionKey?: string; name?: string; args?: { question?: string } }]
|
||||
| [
|
||||
string,
|
||||
{
|
||||
sessionKey?: string;
|
||||
voiceSessionId?: string;
|
||||
name?: string;
|
||||
args?: { question?: string };
|
||||
},
|
||||
]
|
||||
| undefined;
|
||||
expect(toolCall?.[0]).toBe("talk.client.toolCall");
|
||||
expect(toolCall?.[1]?.sessionKey).toBe("agent:main:main");
|
||||
expect(toolCall?.[1]).toMatchObject({ voiceSessionId: "voice-1" });
|
||||
expect(toolCall?.[1]?.name).toBe("openclaw_agent_consult");
|
||||
expect(toolCall?.[1]?.args).toEqual({ question: "Are the basement lights off?" });
|
||||
expect(submit).toHaveBeenCalledWith("call-1", { result: "Basement lights are off." });
|
||||
expect(order).toEqual(["flush", "tool-call"]);
|
||||
});
|
||||
|
||||
it("prefers source-reply final text over an earlier empty Talk consult final", async () => {
|
||||
|
||||
@@ -45,6 +45,7 @@ type RealtimeTalkAudioContract = {
|
||||
export type RealtimeTalkWebRtcSdpSessionResult = {
|
||||
provider: string;
|
||||
transport: "webrtc";
|
||||
voiceSessionId?: string;
|
||||
clientSecret: string;
|
||||
offerUrl?: string;
|
||||
offerHeaders?: Record<string, string>;
|
||||
@@ -58,6 +59,7 @@ export type RealtimeTalkWebRtcSdpSessionResult = {
|
||||
export type RealtimeTalkJsonPcmWebSocketSessionResult = {
|
||||
provider: string;
|
||||
transport: "provider-websocket";
|
||||
voiceSessionId?: string;
|
||||
protocol: string;
|
||||
clientSecret: string;
|
||||
websocketUrl: string;
|
||||
@@ -73,6 +75,7 @@ export type RealtimeTalkJsonPcmWebSocketSessionResult = {
|
||||
export type RealtimeTalkGatewayRelaySessionResult = {
|
||||
provider: string;
|
||||
transport: "gateway-relay";
|
||||
voiceSessionId?: string;
|
||||
relaySessionId: string;
|
||||
audio: RealtimeTalkAudioContract;
|
||||
model?: string;
|
||||
@@ -85,6 +88,7 @@ export type RealtimeTalkGatewayRelaySessionResult = {
|
||||
type RealtimeTalkManagedRoomSessionResult = {
|
||||
provider: string;
|
||||
transport: "managed-room";
|
||||
voiceSessionId?: string;
|
||||
roomUrl: string;
|
||||
token?: string;
|
||||
model?: string;
|
||||
@@ -110,6 +114,8 @@ export type RealtimeTalkTransport = {
|
||||
export type RealtimeTalkTransportContext = {
|
||||
client: GatewayBrowserClient;
|
||||
sessionKey: string;
|
||||
voiceSessionId?: string;
|
||||
flushTranscriptWrites?: () => Promise<void>;
|
||||
callbacks: RealtimeTalkCallbacks;
|
||||
inputDeviceId?: string;
|
||||
videoDeviceId?: string;
|
||||
@@ -572,10 +578,12 @@ export async function submitRealtimeTalkConsult(params: {
|
||||
try {
|
||||
const args =
|
||||
typeof params.args === "string" ? JSON.parse(params.args || "{}") : (params.args ?? {});
|
||||
await ctx.flushTranscriptWrites?.();
|
||||
const response = await ctx.client.request<{ runId?: string; idempotencyKey?: string }>(
|
||||
"talk.client.toolCall",
|
||||
{
|
||||
sessionKey: ctx.sessionKey,
|
||||
...(ctx.voiceSessionId ? { voiceSessionId: ctx.voiceSessionId } : {}),
|
||||
callId,
|
||||
name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
|
||||
args,
|
||||
|
||||
@@ -26,7 +26,7 @@ const {
|
||||
webRtcSetVideoEnabled: vi.fn(async () => undefined),
|
||||
googleSwitchCamera: vi.fn(async () => undefined),
|
||||
webRtcSwitchCamera: vi.fn(async () => undefined),
|
||||
googleCtor: vi.fn(function () {
|
||||
googleCtor: vi.fn(function (_session: unknown, _ctx: unknown) {
|
||||
return {
|
||||
start: googleStart,
|
||||
stop: googleStop,
|
||||
@@ -34,10 +34,10 @@ const {
|
||||
switchCamera: googleSwitchCamera,
|
||||
};
|
||||
}),
|
||||
relayCtor: vi.fn(function () {
|
||||
relayCtor: vi.fn(function (_session: unknown, _ctx: unknown) {
|
||||
return { start: relayStart, stop: relayStop };
|
||||
}),
|
||||
webRtcCtor: vi.fn(function () {
|
||||
webRtcCtor: vi.fn(function (_session: unknown, _ctx: unknown) {
|
||||
return {
|
||||
start: webRtcStart,
|
||||
stop: webRtcStop,
|
||||
@@ -81,6 +81,7 @@ describe("RealtimeTalkSession", () => {
|
||||
it("starts the Google Live WebSocket transport from a generic session result", async () => {
|
||||
const request = vi.fn(async () => ({
|
||||
provider: "google",
|
||||
voiceSessionId: "voice-1",
|
||||
transport: "provider-websocket",
|
||||
protocol: "google-live-bidi",
|
||||
clientSecret: "auth_tokens/session",
|
||||
@@ -97,7 +98,10 @@ describe("RealtimeTalkSession", () => {
|
||||
|
||||
await session.start();
|
||||
|
||||
expect(request).toHaveBeenCalledWith("talk.client.create", { sessionKey: "main" });
|
||||
expect(request).toHaveBeenCalledWith("talk.client.create", {
|
||||
sessionKey: "main",
|
||||
capabilities: ["voice-transcript"],
|
||||
});
|
||||
expect(googleCtor).toHaveBeenCalledTimes(1);
|
||||
expect(googleStart).toHaveBeenCalledTimes(1);
|
||||
expect(webRtcCtor).not.toHaveBeenCalled();
|
||||
@@ -108,6 +112,7 @@ describe("RealtimeTalkSession", () => {
|
||||
it("defaults legacy session results without an explicit transport to WebRTC", async () => {
|
||||
const request = vi.fn(async () => ({
|
||||
provider: "openai",
|
||||
voiceSessionId: "voice-1",
|
||||
clientSecret: "auth_tokens/session",
|
||||
}));
|
||||
const session = new RealtimeTalkSession({ request } as never, "main");
|
||||
@@ -122,6 +127,7 @@ describe("RealtimeTalkSession", () => {
|
||||
it("accepts legacy WebRTC transport names", async () => {
|
||||
const request = vi.fn(async () => ({
|
||||
provider: "openai",
|
||||
voiceSessionId: "voice-1",
|
||||
transport: "webrtc-sdp",
|
||||
clientSecret: "secret",
|
||||
}));
|
||||
@@ -136,6 +142,7 @@ describe("RealtimeTalkSession", () => {
|
||||
it("accepts legacy provider WebSocket transport names", async () => {
|
||||
const request = vi.fn(async () => ({
|
||||
provider: "example",
|
||||
voiceSessionId: "voice-1",
|
||||
transport: "json-pcm-websocket",
|
||||
clientSecret: "secret",
|
||||
protocol: "google-live-bidi",
|
||||
@@ -209,6 +216,7 @@ describe("RealtimeTalkSession", () => {
|
||||
sessionKey: "main",
|
||||
provider: "xai",
|
||||
transport: "gateway-relay",
|
||||
capabilities: ["voice-transcript"],
|
||||
});
|
||||
expect(request).toHaveBeenNthCalledWith(2, "talk.session.create", {
|
||||
sessionKey: "main",
|
||||
@@ -263,7 +271,7 @@ describe("RealtimeTalkSession", () => {
|
||||
sessionKey: "main",
|
||||
provider: "openai",
|
||||
transport: "gateway-relay",
|
||||
capabilities: ["camera-frame"],
|
||||
capabilities: ["voice-transcript", "camera-frame"],
|
||||
});
|
||||
expect(request).toHaveBeenNthCalledWith(3, "talk.session.create", {
|
||||
sessionKey: "main",
|
||||
@@ -280,6 +288,7 @@ describe("RealtimeTalkSession", () => {
|
||||
it("starts the WebRTC transport for canonical WebRTC sessions", async () => {
|
||||
const request = vi.fn(async () => ({
|
||||
provider: "openai",
|
||||
voiceSessionId: "voice-1",
|
||||
transport: "webrtc",
|
||||
clientSecret: "secret",
|
||||
}));
|
||||
@@ -298,6 +307,7 @@ describe("RealtimeTalkSession", () => {
|
||||
it("passes launch options to client-owned realtime session creation", async () => {
|
||||
const request = vi.fn(async () => ({
|
||||
provider: "openai",
|
||||
voiceSessionId: "voice-1",
|
||||
transport: "webrtc",
|
||||
clientSecret: "secret",
|
||||
}));
|
||||
@@ -330,6 +340,7 @@ describe("RealtimeTalkSession", () => {
|
||||
silenceDurationMs: 650,
|
||||
prefixPaddingMs: 250,
|
||||
reasoningEffort: "low",
|
||||
capabilities: ["voice-transcript"],
|
||||
});
|
||||
expect(webRtcCtor).toHaveBeenCalledWith(
|
||||
expect.any(Object),
|
||||
@@ -349,6 +360,7 @@ describe("RealtimeTalkSession", () => {
|
||||
}
|
||||
return {
|
||||
provider: "openai",
|
||||
voiceSessionId: "voice-1",
|
||||
transport: "webrtc",
|
||||
clientSecret: "secret",
|
||||
};
|
||||
@@ -363,7 +375,7 @@ describe("RealtimeTalkSession", () => {
|
||||
expect(request).toHaveBeenNthCalledWith(1, "talk.catalog", {});
|
||||
expect(request).toHaveBeenNthCalledWith(2, "talk.client.create", {
|
||||
sessionKey: "main",
|
||||
capabilities: ["camera-frame"],
|
||||
capabilities: ["voice-transcript", "camera-frame"],
|
||||
});
|
||||
expect(onVideoCapability).toHaveBeenCalledWith(true);
|
||||
expect(webRtcCtor).toHaveBeenCalledWith(
|
||||
@@ -392,6 +404,7 @@ describe("RealtimeTalkSession", () => {
|
||||
return {
|
||||
provider: "openai",
|
||||
transport: "webrtc",
|
||||
voiceSessionId: "voice-settings-camera",
|
||||
clientSecret: "secret",
|
||||
};
|
||||
});
|
||||
@@ -424,7 +437,12 @@ describe("RealtimeTalkSession", () => {
|
||||
},
|
||||
};
|
||||
}
|
||||
return { provider: "openai", transport: "webrtc", clientSecret: "secret" };
|
||||
return {
|
||||
provider: "openai",
|
||||
transport: "webrtc",
|
||||
voiceSessionId: "voice-pending-camera",
|
||||
clientSecret: "secret",
|
||||
};
|
||||
});
|
||||
const session = new RealtimeTalkSession({ request } as never, "main", {
|
||||
onVideoCapability: vi.fn(),
|
||||
@@ -454,6 +472,7 @@ describe("RealtimeTalkSession", () => {
|
||||
}
|
||||
return {
|
||||
provider: "openai",
|
||||
voiceSessionId: "voice-1",
|
||||
transport: "webrtc",
|
||||
clientSecret: "secret",
|
||||
};
|
||||
@@ -465,7 +484,10 @@ describe("RealtimeTalkSession", () => {
|
||||
|
||||
await session.start();
|
||||
|
||||
expect(request).toHaveBeenNthCalledWith(2, "talk.client.create", { sessionKey: "main" });
|
||||
expect(request).toHaveBeenNthCalledWith(2, "talk.client.create", {
|
||||
sessionKey: "main",
|
||||
capabilities: ["voice-transcript"],
|
||||
});
|
||||
expect(onVideoCapability).toHaveBeenCalledWith(false);
|
||||
});
|
||||
|
||||
@@ -491,7 +513,7 @@ describe("RealtimeTalkSession", () => {
|
||||
await expect(session.start()).rejects.toBe(clientError);
|
||||
|
||||
expect(request.mock.calls).toEqual([
|
||||
["talk.client.create", { sessionKey: "main" }],
|
||||
["talk.client.create", { sessionKey: "main", capabilities: ["voice-transcript"] }],
|
||||
["talk.config", {}],
|
||||
]);
|
||||
expect(relayCtor).not.toHaveBeenCalled();
|
||||
@@ -592,7 +614,7 @@ describe("RealtimeTalkSession", () => {
|
||||
await expect(session.start()).rejects.toBe(clientError);
|
||||
|
||||
expect(request.mock.calls).toEqual([
|
||||
["talk.client.create", { sessionKey: "main" }],
|
||||
["talk.client.create", { sessionKey: "main", capabilities: ["voice-transcript"] }],
|
||||
["talk.config", {}],
|
||||
]);
|
||||
expect(relayCtor).not.toHaveBeenCalled();
|
||||
@@ -614,9 +636,338 @@ describe("RealtimeTalkSession", () => {
|
||||
await expect(session.start()).rejects.toBe(clientError);
|
||||
|
||||
expect(request.mock.calls).toEqual([
|
||||
["talk.client.create", { sessionKey: "main" }],
|
||||
["talk.client.create", { sessionKey: "main", capabilities: ["voice-transcript"] }],
|
||||
["talk.config", {}],
|
||||
]);
|
||||
expect(relayCtor).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("retries finalized transcript writes in order", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
const transcriptEntryIds: string[] = [];
|
||||
let firstAttempt = true;
|
||||
const request = vi.fn(async (method: string, params?: { entryId?: string }) => {
|
||||
if (method === "talk.client.create") {
|
||||
return {
|
||||
provider: "openai",
|
||||
transport: "webrtc",
|
||||
voiceSessionId: "voice-queue",
|
||||
clientSecret: "secret",
|
||||
};
|
||||
}
|
||||
if (method === "talk.client.transcript") {
|
||||
transcriptEntryIds.push(String(params?.entryId));
|
||||
if (params?.entryId === "1" && firstAttempt) {
|
||||
firstAttempt = false;
|
||||
throw new Error("temporary failure");
|
||||
}
|
||||
return { ok: true };
|
||||
}
|
||||
return { ok: true };
|
||||
});
|
||||
const session = new RealtimeTalkSession({ request } as never, "agent:main:main");
|
||||
await session.start();
|
||||
const context = webRtcCtor.mock.calls[0]?.[1] as {
|
||||
callbacks: {
|
||||
onTranscript?: (entry: {
|
||||
role: "user" | "assistant";
|
||||
text: string;
|
||||
final: boolean;
|
||||
}) => void;
|
||||
};
|
||||
};
|
||||
context.callbacks.onTranscript?.({ role: "user", text: "first", final: true });
|
||||
context.callbacks.onTranscript?.({ role: "assistant", text: "second", final: true });
|
||||
|
||||
await vi.advanceTimersByTimeAsync(500);
|
||||
await vi.waitFor(() => expect(transcriptEntryIds).toEqual(["1", "1", "2"]));
|
||||
session.stop();
|
||||
await vi.runAllTimersAsync();
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("continues entry ids when the same voice session replaces its transport", async () => {
|
||||
const transcriptEntryIds: string[] = [];
|
||||
const request = vi.fn(async (method: string, params?: { entryId?: string }) => {
|
||||
if (method === "talk.client.create") {
|
||||
return {
|
||||
provider: "openai",
|
||||
transport: "webrtc",
|
||||
voiceSessionId: "voice-resume",
|
||||
clientSecret: "secret",
|
||||
};
|
||||
}
|
||||
if (method === "talk.client.transcript") {
|
||||
transcriptEntryIds.push(String(params?.entryId));
|
||||
}
|
||||
return { ok: true };
|
||||
});
|
||||
const session = new RealtimeTalkSession({ request } as never, "agent:main:main", {});
|
||||
|
||||
await session.start();
|
||||
const firstContext = webRtcCtor.mock.calls[0]?.[1] as {
|
||||
callbacks: {
|
||||
onTranscript?: (entry: {
|
||||
role: "user" | "assistant";
|
||||
text: string;
|
||||
final: boolean;
|
||||
}) => void;
|
||||
};
|
||||
flushTranscriptWrites?: () => Promise<void>;
|
||||
};
|
||||
firstContext.callbacks.onTranscript?.({ role: "user", text: "first", final: true });
|
||||
await firstContext.flushTranscriptWrites?.();
|
||||
|
||||
await session.start();
|
||||
const secondContext = webRtcCtor.mock.calls[1]?.[1] as typeof firstContext;
|
||||
secondContext.callbacks.onTranscript?.({ role: "assistant", text: "second", final: true });
|
||||
await secondContext.flushTranscriptWrites?.();
|
||||
|
||||
expect(transcriptEntryIds).toEqual(["1", "2"]);
|
||||
expect(request.mock.calls.filter(([method]) => method === "talk.client.create")).toEqual([
|
||||
["talk.client.create", { sessionKey: "agent:main:main", capabilities: ["voice-transcript"] }],
|
||||
[
|
||||
"talk.client.create",
|
||||
{
|
||||
sessionKey: "agent:main:main",
|
||||
voiceSessionId: "voice-resume",
|
||||
capabilities: ["voice-transcript"],
|
||||
},
|
||||
],
|
||||
]);
|
||||
session.stop();
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
it("surfaces transcript failure after three attempts", async () => {
|
||||
vi.useFakeTimers();
|
||||
const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);
|
||||
try {
|
||||
const request = vi.fn(async (method: string) => {
|
||||
if (method === "talk.client.create") {
|
||||
return {
|
||||
provider: "openai",
|
||||
transport: "webrtc",
|
||||
voiceSessionId: "voice-failure",
|
||||
clientSecret: "secret",
|
||||
};
|
||||
}
|
||||
if (method === "talk.client.transcript") {
|
||||
throw new Error("still unavailable");
|
||||
}
|
||||
return { ok: true };
|
||||
});
|
||||
const onStatus = vi.fn();
|
||||
const session = new RealtimeTalkSession({ request } as never, "agent:main:main", {
|
||||
onStatus,
|
||||
});
|
||||
await session.start();
|
||||
const context = webRtcCtor.mock.calls[0]?.[1] as {
|
||||
callbacks: {
|
||||
onTranscript?: (entry: {
|
||||
role: "user" | "assistant";
|
||||
text: string;
|
||||
final: boolean;
|
||||
}) => void;
|
||||
};
|
||||
};
|
||||
context.callbacks.onTranscript?.({ role: "user", text: "save me", final: true });
|
||||
|
||||
await vi.advanceTimersByTimeAsync(2_500);
|
||||
await vi.waitFor(() =>
|
||||
expect(onStatus).toHaveBeenCalledWith(
|
||||
"error",
|
||||
expect.stringContaining("Voice transcript could not be saved"),
|
||||
),
|
||||
);
|
||||
expect(
|
||||
request.mock.calls.filter(([method]) => method === "talk.client.transcript"),
|
||||
).toHaveLength(3);
|
||||
expect(warn).toHaveBeenCalled();
|
||||
session.stop();
|
||||
await vi.runAllTimersAsync();
|
||||
} finally {
|
||||
warn.mockRestore();
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("retries logical voice-session close after transient failures", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
let closeAttempts = 0;
|
||||
const request = vi.fn(async (method: string) => {
|
||||
if (method === "talk.client.create") {
|
||||
return {
|
||||
provider: "openai",
|
||||
transport: "webrtc",
|
||||
voiceSessionId: "voice-close-retry",
|
||||
clientSecret: "secret",
|
||||
};
|
||||
}
|
||||
if (method === "talk.client.close" && ++closeAttempts < 3) {
|
||||
throw new Error("temporary close failure");
|
||||
}
|
||||
return { ok: true };
|
||||
});
|
||||
const session = new RealtimeTalkSession({ request } as never, "agent:main:main");
|
||||
await session.start();
|
||||
|
||||
session.stop();
|
||||
await vi.runAllTimersAsync();
|
||||
|
||||
expect(closeAttempts).toBe(3);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("starts a new call without resuming the voice session being closed", async () => {
|
||||
let createCount = 0;
|
||||
let finishClose: (() => void) | undefined;
|
||||
const closing = new Promise<void>((resolve) => {
|
||||
finishClose = resolve;
|
||||
});
|
||||
const request = vi.fn(async (method: string) => {
|
||||
if (method === "talk.client.create") {
|
||||
createCount += 1;
|
||||
return {
|
||||
provider: "openai",
|
||||
transport: "webrtc",
|
||||
voiceSessionId: `voice-${createCount}`,
|
||||
clientSecret: "secret",
|
||||
};
|
||||
}
|
||||
if (method === "talk.client.close") {
|
||||
await closing;
|
||||
}
|
||||
return { ok: true };
|
||||
});
|
||||
const session = new RealtimeTalkSession({ request } as never, "agent:main:main");
|
||||
await session.start();
|
||||
|
||||
session.stop();
|
||||
await session.start();
|
||||
|
||||
const creates = request.mock.calls.filter(([method]) => method === "talk.client.create");
|
||||
expect(creates).toEqual([
|
||||
["talk.client.create", { sessionKey: "agent:main:main", capabilities: ["voice-transcript"] }],
|
||||
["talk.client.create", { sessionKey: "agent:main:main", capabilities: ["voice-transcript"] }],
|
||||
]);
|
||||
finishClose?.();
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
it("ignores final transcript callbacks emitted after shutdown begins", async () => {
|
||||
const request = vi.fn(async (method: string) => {
|
||||
if (method === "talk.client.create") {
|
||||
return {
|
||||
provider: "openai",
|
||||
transport: "webrtc",
|
||||
voiceSessionId: "voice-shutdown",
|
||||
clientSecret: "secret",
|
||||
};
|
||||
}
|
||||
return { ok: true };
|
||||
});
|
||||
const session = new RealtimeTalkSession({ request } as never, "agent:main:main");
|
||||
await session.start();
|
||||
const context = webRtcCtor.mock.calls[0]?.[1] as {
|
||||
callbacks: {
|
||||
onTranscript?: (entry: {
|
||||
role: "user" | "assistant";
|
||||
text: string;
|
||||
final: boolean;
|
||||
}) => void;
|
||||
};
|
||||
};
|
||||
|
||||
session.stop();
|
||||
context.callbacks.onTranscript?.({ role: "user", text: "too late", final: true });
|
||||
await Promise.resolve();
|
||||
|
||||
expect(request.mock.calls.some(([method]) => method === "talk.client.transcript")).toBe(false);
|
||||
});
|
||||
|
||||
it("drops a previous transport's delayed transcript after stop and restart", async () => {
|
||||
let createCount = 0;
|
||||
const request = vi.fn(async (method: string) => {
|
||||
if (method === "talk.client.create") {
|
||||
createCount += 1;
|
||||
return {
|
||||
provider: "openai",
|
||||
transport: "webrtc",
|
||||
voiceSessionId: `voice-${createCount}`,
|
||||
clientSecret: "secret",
|
||||
};
|
||||
}
|
||||
return { ok: true };
|
||||
});
|
||||
const onTranscript = vi.fn();
|
||||
const session = new RealtimeTalkSession({ request } as never, "agent:main:main", {
|
||||
onTranscript,
|
||||
});
|
||||
await session.start();
|
||||
const previousContext = webRtcCtor.mock.calls[0]?.[1] as {
|
||||
callbacks: {
|
||||
onTranscript?: (entry: {
|
||||
role: "user" | "assistant";
|
||||
text: string;
|
||||
final: boolean;
|
||||
}) => void;
|
||||
};
|
||||
};
|
||||
|
||||
session.stop();
|
||||
await session.start();
|
||||
previousContext.callbacks.onTranscript?.({
|
||||
role: "user",
|
||||
text: "stale transcript",
|
||||
final: true,
|
||||
});
|
||||
await Promise.resolve();
|
||||
|
||||
expect(onTranscript).not.toHaveBeenCalled();
|
||||
expect(request.mock.calls.some(([method]) => method === "talk.client.transcript")).toBe(false);
|
||||
});
|
||||
|
||||
it("does not report Gateway relay transcripts through the client RPC", async () => {
|
||||
const request = vi.fn(async (method: string) => {
|
||||
if (method === "talk.client.create") {
|
||||
return {
|
||||
provider: "openai",
|
||||
transport: "gateway-relay",
|
||||
relaySessionId: "relay-voice",
|
||||
audio: {
|
||||
inputEncoding: "pcm16",
|
||||
inputSampleRateHz: 24_000,
|
||||
outputEncoding: "pcm16",
|
||||
outputSampleRateHz: 24_000,
|
||||
},
|
||||
};
|
||||
}
|
||||
return { ok: true };
|
||||
});
|
||||
const session = new RealtimeTalkSession({ request } as never, "agent:main:main");
|
||||
await session.start();
|
||||
const context = relayCtor.mock.calls[0]?.[1] as {
|
||||
callbacks: {
|
||||
onTranscript?: (entry: {
|
||||
role: "user" | "assistant";
|
||||
text: string;
|
||||
final: boolean;
|
||||
}) => void;
|
||||
};
|
||||
};
|
||||
context.callbacks.onTranscript?.({ role: "user", text: "server owns this", final: true });
|
||||
await Promise.resolve();
|
||||
|
||||
expect(request.mock.calls.some(([method]) => method === "talk.client.transcript")).toBe(false);
|
||||
session.stop();
|
||||
await Promise.resolve();
|
||||
expect(request.mock.calls.some(([method]) => method === "talk.client.close")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -58,6 +58,13 @@ export async function switchActiveRealtimeTalkCameras(
|
||||
|
||||
type RealtimeTalkLaunchTransport = NonNullable<RealtimeTalkLaunchOptions["transport"]>;
|
||||
|
||||
type DetachedVoiceSession = {
|
||||
voiceSessionId: string;
|
||||
serverOwned: boolean;
|
||||
generation: number;
|
||||
transcriptWrites: Promise<void>;
|
||||
};
|
||||
|
||||
type RealtimeTalkConfigResult = {
|
||||
config?: {
|
||||
talk?: {
|
||||
@@ -104,9 +111,6 @@ function createTransport(
|
||||
ctx,
|
||||
);
|
||||
}
|
||||
if (transport === "managed-room") {
|
||||
throw new Error("Managed-room realtime Talk sessions are not available in this UI yet");
|
||||
}
|
||||
const unknownTransport = (session as { transport?: string }).transport ?? "unknown";
|
||||
throw new Error(`Unsupported realtime Talk transport: ${unknownTransport}`);
|
||||
}
|
||||
@@ -115,8 +119,17 @@ function resolveTransport(session: RealtimeTalkSessionResult): string {
|
||||
return normalizeTalkTransport((session as { transport?: string }).transport) ?? "webrtc";
|
||||
}
|
||||
|
||||
function transcriptWriteError(error: unknown, fallback: string): Error {
|
||||
return error instanceof Error ? error : new Error(fallback, { cause: error });
|
||||
}
|
||||
|
||||
function compactLaunchParams(
|
||||
params: RealtimeTalkLaunchOptions & { sessionKey: string; mode?: string; brain?: string },
|
||||
params: RealtimeTalkLaunchOptions & {
|
||||
sessionKey: string;
|
||||
voiceSessionId?: string;
|
||||
mode?: string;
|
||||
brain?: string;
|
||||
},
|
||||
): Record<string, unknown> {
|
||||
return Object.fromEntries(Object.entries(params).filter(([, value]) => value !== undefined));
|
||||
}
|
||||
@@ -126,6 +139,12 @@ export class RealtimeTalkSession {
|
||||
private closed = false;
|
||||
private videoEnabled = false;
|
||||
private videoOperation = 0;
|
||||
private voiceSessionId: string | undefined;
|
||||
private transportGeneration = 0;
|
||||
private readonly transcriptSeqByVoiceSessionId = new Map<string, number>();
|
||||
private acceptingTranscripts = false;
|
||||
private serverOwnedVoiceSession = false;
|
||||
private transcriptWrites: Promise<void> = Promise.resolve();
|
||||
|
||||
constructor(
|
||||
private readonly client: GatewayBrowserClient,
|
||||
@@ -142,16 +161,48 @@ export class RealtimeTalkSession {
|
||||
if (this.closed) {
|
||||
return;
|
||||
}
|
||||
const session = await this.createSession(
|
||||
providerVideoCapable ? { ...this.options, capabilities: ["camera-frame"] } : this.options,
|
||||
);
|
||||
// Declaring voice-transcript arms the server-side spoken-confirmation gate;
|
||||
// this client reports every finalized utterance, so the gate is completable.
|
||||
const capabilities: Array<"camera-frame" | "voice-transcript"> = ["voice-transcript"];
|
||||
if (providerVideoCapable) {
|
||||
capabilities.push("camera-frame");
|
||||
}
|
||||
const session = await this.createSession({ ...this.options, capabilities });
|
||||
const transport = resolveTransport(session);
|
||||
// Managed-room stays unsupported here and carries no voice bookkeeping;
|
||||
// reject it before the voice-session requirement produces a misleading error.
|
||||
if (transport === "managed-room") {
|
||||
throw new Error("Managed-room realtime Talk sessions are not available in this UI yet");
|
||||
}
|
||||
const voiceSessionId =
|
||||
session.voiceSessionId ??
|
||||
(transport === "gateway-relay"
|
||||
? (session as RealtimeTalkGatewayRelaySessionResult).relaySessionId
|
||||
: undefined);
|
||||
if (!voiceSessionId) {
|
||||
throw new Error("Realtime Talk session did not return a voice session id");
|
||||
}
|
||||
this.voiceSessionId = voiceSessionId;
|
||||
this.acceptingTranscripts = true;
|
||||
this.serverOwnedVoiceSession = transport === "gateway-relay";
|
||||
if (this.closed) {
|
||||
const detached = this.detachVoiceSession();
|
||||
if (detached) {
|
||||
this.closeLogicalVoiceSession(detached);
|
||||
}
|
||||
return;
|
||||
}
|
||||
this.transportGeneration += 1;
|
||||
const callbacks =
|
||||
transport === "gateway-relay"
|
||||
? this.callbacks
|
||||
: this.clientOwnedTranscriptCallbacks(voiceSessionId, this.transportGeneration);
|
||||
this.transport = createTransport(session, {
|
||||
client: this.client,
|
||||
sessionKey: this.sessionKey,
|
||||
callbacks: this.callbacks,
|
||||
voiceSessionId,
|
||||
flushTranscriptWrites: async () => await this.transcriptWrites,
|
||||
callbacks,
|
||||
inputDeviceId: this.localOptions.inputDeviceId,
|
||||
videoDeviceId: this.localOptions.videoDeviceId,
|
||||
consultThinkingLevel: session.consultThinkingLevel,
|
||||
@@ -185,18 +236,22 @@ export class RealtimeTalkSession {
|
||||
}
|
||||
|
||||
private async createSession(
|
||||
options: RealtimeTalkLaunchOptions & { capabilities?: Array<"camera-frame"> },
|
||||
options: RealtimeTalkLaunchOptions & {
|
||||
capabilities?: Array<"camera-frame" | "voice-transcript">;
|
||||
},
|
||||
): Promise<RealtimeTalkSessionResult> {
|
||||
const launchOptions = { ...options };
|
||||
try {
|
||||
return await this.client.request<RealtimeTalkSessionResult>(
|
||||
"talk.client.create",
|
||||
compactLaunchParams({
|
||||
sessionKey: this.sessionKey,
|
||||
...options,
|
||||
voiceSessionId: this.voiceSessionId,
|
||||
...launchOptions,
|
||||
}),
|
||||
);
|
||||
} catch (error) {
|
||||
let transport = options.transport;
|
||||
let transport = launchOptions.transport;
|
||||
if (!transport) {
|
||||
let result: RealtimeTalkConfigResult;
|
||||
try {
|
||||
@@ -218,10 +273,10 @@ export class RealtimeTalkSession {
|
||||
if (transport && transport !== "gateway-relay") {
|
||||
throw error;
|
||||
}
|
||||
const gatewayOptions = { ...options };
|
||||
const gatewayOptions = { ...launchOptions };
|
||||
delete gatewayOptions.capabilities;
|
||||
try {
|
||||
return await this.client.request<RealtimeTalkSessionResult>(
|
||||
const relaySession = await this.client.request<RealtimeTalkSessionResult>(
|
||||
"talk.session.create",
|
||||
compactLaunchParams({
|
||||
sessionKey: this.sessionKey,
|
||||
@@ -231,6 +286,13 @@ export class RealtimeTalkSession {
|
||||
brain: "agent-consult",
|
||||
}),
|
||||
);
|
||||
return resolveTransport(relaySession) === "gateway-relay"
|
||||
? {
|
||||
...relaySession,
|
||||
voiceSessionId: (relaySession as RealtimeTalkGatewayRelaySessionResult)
|
||||
.relaySessionId,
|
||||
}
|
||||
: relaySession;
|
||||
} catch {
|
||||
throw error;
|
||||
}
|
||||
@@ -243,8 +305,147 @@ export class RealtimeTalkSession {
|
||||
this.videoEnabled = false;
|
||||
activeRealtimeTalkSessions.delete(this);
|
||||
this.callbacks.onStatus?.("idle");
|
||||
const detached = this.detachVoiceSession();
|
||||
this.transport?.stop();
|
||||
this.transport = null;
|
||||
if (detached) {
|
||||
this.closeLogicalVoiceSession(detached);
|
||||
}
|
||||
}
|
||||
|
||||
private clientOwnedTranscriptCallbacks(
|
||||
owningVoiceSessionId: string,
|
||||
owningGeneration: number,
|
||||
): RealtimeTalkCallbacks {
|
||||
return {
|
||||
...this.callbacks,
|
||||
onTranscript: (entry) => {
|
||||
// Transport replacement can reuse the voice session id, so a retired
|
||||
// transport's late finals are fenced by generation, not id alone.
|
||||
if (
|
||||
this.transportGeneration !== owningGeneration ||
|
||||
this.voiceSessionId !== owningVoiceSessionId ||
|
||||
!this.acceptingTranscripts
|
||||
) {
|
||||
return;
|
||||
}
|
||||
// Persist before notifying: a consumer callback that stops or throws must
|
||||
// not be able to drop an already-finalized utterance from the write tail.
|
||||
if (entry.final) {
|
||||
const transcriptSeq =
|
||||
(this.transcriptSeqByVoiceSessionId.get(owningVoiceSessionId) ?? 0) + 1;
|
||||
this.transcriptSeqByVoiceSessionId.set(owningVoiceSessionId, transcriptSeq);
|
||||
const entryId = String(transcriptSeq);
|
||||
// One promise tail preserves transcript order and makes consult flushes
|
||||
// observe the same dedupe sequence used by close.
|
||||
const write = this.transcriptWrites.then(async () => {
|
||||
await this.writeTranscriptWithRetry({
|
||||
voiceSessionId: owningVoiceSessionId,
|
||||
entryId,
|
||||
role: entry.role,
|
||||
text: entry.text,
|
||||
});
|
||||
});
|
||||
this.transcriptWrites = write.catch((error: unknown) => {
|
||||
// The utterance exists only in client memory; after retries and surfacing the error,
|
||||
// keeping the record open cannot recover it, while server entryId dedupe preserves order.
|
||||
// Deferring close would only shift the identical loss to the 6h stale sweep.
|
||||
const detail = `Voice transcript could not be saved: ${error instanceof Error ? error.message : String(error)}`;
|
||||
console.warn(detail, error);
|
||||
// Only surface to the user if this transport is still the active one; a
|
||||
// retired call's late failure must not error a healthy replacement call.
|
||||
if (this.transportGeneration === owningGeneration) {
|
||||
this.callbacks.onStatus?.("error", detail);
|
||||
}
|
||||
});
|
||||
}
|
||||
this.callbacks.onTranscript?.(entry);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private async writeTranscriptWithRetry(params: {
|
||||
voiceSessionId: string;
|
||||
entryId: string;
|
||||
role: "user" | "assistant";
|
||||
text: string;
|
||||
}): Promise<void> {
|
||||
const retryDelaysMs = [0, 500, 2_000];
|
||||
let lastError: unknown;
|
||||
for (const delayMs of retryDelaysMs) {
|
||||
if (delayMs > 0) {
|
||||
await new Promise<void>((resolve) => {
|
||||
setTimeout(resolve, delayMs);
|
||||
});
|
||||
}
|
||||
try {
|
||||
await this.client.request("talk.client.transcript", {
|
||||
sessionKey: this.sessionKey,
|
||||
voiceSessionId: params.voiceSessionId,
|
||||
entryId: params.entryId,
|
||||
role: params.role,
|
||||
text: params.text,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
return;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
}
|
||||
}
|
||||
throw transcriptWriteError(lastError, "voice transcript save failed");
|
||||
}
|
||||
|
||||
private detachVoiceSession(): DetachedVoiceSession | undefined {
|
||||
const voiceSessionId = this.voiceSessionId;
|
||||
if (!voiceSessionId) {
|
||||
return undefined;
|
||||
}
|
||||
const detached = {
|
||||
voiceSessionId,
|
||||
serverOwned: this.serverOwnedVoiceSession,
|
||||
generation: this.transportGeneration,
|
||||
transcriptWrites: this.transcriptWrites,
|
||||
} satisfies DetachedVoiceSession;
|
||||
this.voiceSessionId = undefined;
|
||||
this.acceptingTranscripts = false;
|
||||
this.serverOwnedVoiceSession = false;
|
||||
this.transcriptWrites = Promise.resolve();
|
||||
return detached;
|
||||
}
|
||||
|
||||
private closeLogicalVoiceSession(detached: DetachedVoiceSession): void {
|
||||
if (detached.serverOwned) {
|
||||
return;
|
||||
}
|
||||
void detached.transcriptWrites
|
||||
.then(async () => {
|
||||
let lastError: unknown;
|
||||
for (const delayMs of [0, 500, 2_000]) {
|
||||
if (delayMs > 0) {
|
||||
await new Promise<void>((resolve) => {
|
||||
setTimeout(resolve, delayMs);
|
||||
});
|
||||
}
|
||||
try {
|
||||
await this.client.request("talk.client.close", {
|
||||
sessionKey: this.sessionKey,
|
||||
voiceSessionId: detached.voiceSessionId,
|
||||
});
|
||||
return;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
}
|
||||
}
|
||||
throw transcriptWriteError(lastError, "Realtime Talk voice session close failed");
|
||||
})
|
||||
.catch((error: unknown) => {
|
||||
console.warn("Realtime Talk voice session close failed", error);
|
||||
// Suppress if a newer transport has started: closing the old call is its own
|
||||
// teardown and must not push the active replacement call into an error state.
|
||||
if (this.transportGeneration === detached.generation) {
|
||||
this.callbacks.onStatus?.("error", "Realtime Talk voice session close failed");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async setVideoEnabled(enabled: boolean): Promise<void> {
|
||||
|
||||
Reference in New Issue
Block a user