feat(talk): add durable client voice sessions (#111216)

Live-append voice transcripts into the agent session and persist a per-agent SQLite call record across relay and client transcript paths.

Add run-scoped spoken confirmation for high-impact actions, mutation digests, bootstrap-context injection, talk.client.transcript and talk.client.close protocol methods, and Control UI adoption. This adds zero new configuration.

Co-authored-by: Clifton King <clifton@users.noreply.github.com>
This commit is contained in:
Peter Steinberger
2026-07-19 01:06:49 -07:00
committed by GitHub
co-authored by Clifton King
parent 83fc53a22f
commit 0f95e66b7f
35 changed files with 4328 additions and 92 deletions
@@ -203,6 +203,8 @@ enum class GatewayMethod(
TalkCatalog("talk.catalog"),
TalkConfig("talk.config"),
TalkClientCreate("talk.client.create"),
TalkClientTranscript("talk.client.transcript"),
TalkClientClose("talk.client.close"),
TalkClientToolCall("talk.client.toolCall"),
TalkClientSteer("talk.client.steer"),
TalkSessionCreate("talk.session.create"),
@@ -8116,6 +8116,7 @@ public struct TalkCatalogResult: Codable, Sendable {
public struct TalkClientCreateParams: Codable, Sendable {
public let sessionkey: String?
public let voicesessionid: String?
public let provider: String?
public let model: String?
public let voice: String?
@@ -8126,10 +8127,11 @@ public struct TalkClientCreateParams: Codable, Sendable {
public let mode: AnyCodable?
public let transport: AnyCodable?
public let brain: AnyCodable?
public let capabilities: [String]?
public let capabilities: [AnyCodable]?
public init(
sessionkey: String? = nil,
voicesessionid: String? = nil,
provider: String? = nil,
model: String? = nil,
voice: String? = nil,
@@ -8140,9 +8142,10 @@ public struct TalkClientCreateParams: Codable, Sendable {
mode: AnyCodable? = nil,
transport: AnyCodable? = nil,
brain: AnyCodable? = nil,
capabilities: [String]? = nil)
capabilities: [AnyCodable]? = nil)
{
self.sessionkey = sessionkey
self.voicesessionid = voicesessionid
self.provider = provider
self.model = model
self.voice = voice
@@ -8158,6 +8161,7 @@ public struct TalkClientCreateParams: Codable, Sendable {
private enum CodingKeys: String, CodingKey {
case sessionkey = "sessionKey"
case voicesessionid = "voiceSessionId"
case provider
case model
case voice
@@ -8172,6 +8176,38 @@ public struct TalkClientCreateParams: Codable, Sendable {
}
}
public struct TalkClientCloseParams: Codable, Sendable {
public let sessionkey: String
public let voicesessionid: String
public init(
sessionkey: String,
voicesessionid: String)
{
self.sessionkey = sessionkey
self.voicesessionid = voicesessionid
}
private enum CodingKeys: String, CodingKey {
case sessionkey = "sessionKey"
case voicesessionid = "voiceSessionId"
}
}
public struct TalkClientMutationResult: Codable, Sendable {
public let ok: Bool
public init(
ok: Bool)
{
self.ok = ok
}
private enum CodingKeys: String, CodingKey {
case ok
}
}
public struct TalkClientSteerParams: Codable, Sendable {
public let sessionkey: String
public let text: String
@@ -8270,6 +8306,7 @@ public struct TalkAgentControlResult: Codable, Sendable {
public struct TalkClientToolCallParams: Codable, Sendable {
public let sessionkey: String
public let voicesessionid: String?
public let callid: String
public let name: String
public let args: AnyCodable?
@@ -8277,12 +8314,14 @@ public struct TalkClientToolCallParams: Codable, Sendable {
public init(
sessionkey: String,
voicesessionid: String? = nil,
callid: String,
name: String,
args: AnyCodable? = nil,
relaysessionid: String? = nil)
{
self.sessionkey = sessionkey
self.voicesessionid = voicesessionid
self.callid = callid
self.name = name
self.args = args
@@ -8291,6 +8330,7 @@ public struct TalkClientToolCallParams: Codable, Sendable {
private enum CodingKeys: String, CodingKey {
case sessionkey = "sessionKey"
case voicesessionid = "voiceSessionId"
case callid = "callId"
case name
case args
@@ -8316,6 +8356,40 @@ public struct TalkClientToolCallResult: Codable, Sendable {
}
}
public struct TalkClientTranscriptParams: Codable, Sendable {
public let sessionkey: String
public let voicesessionid: String
public let entryid: String
public let role: AnyCodable
public let text: String
public let timestamp: Double?
public init(
sessionkey: String,
voicesessionid: String,
entryid: String,
role: AnyCodable,
text: String,
timestamp: Double? = nil)
{
self.sessionkey = sessionkey
self.voicesessionid = voicesessionid
self.entryid = entryid
self.role = role
self.text = text
self.timestamp = timestamp
}
private enum CodingKeys: String, CodingKey {
case sessionkey = "sessionKey"
case voicesessionid = "voiceSessionId"
case entryid = "entryId"
case role
case text
case timestamp
}
}
public struct TalkConfigParams: Codable, Sendable {
public let includesecrets: Bool?
+4 -2
View File
@@ -496,8 +496,10 @@ methods. Treat this as feature discovery, not a full enumeration of
- `talk.session.steer` sends active-run voice control into a gateway-owned agent-backed Talk session: `{ sessionId, text, mode? }`, where `mode` is `status`, `steer`, `cancel`, or `followup`; omitted mode is classified from the spoken text.
- `talk.session.close` closes a gateway-owned relay, transcription, or managed-room session and emits terminal Talk events.
- `talk.mode` sets/broadcasts the current Talk mode state for WebChat/Control UI clients.
- `talk.client.create` creates a client-owned realtime provider session using `webrtc` or `provider-websocket` while the gateway owns config, credentials, instructions, and tool policy.
- `talk.client.toolCall` lets client-owned realtime transports forward provider tool calls to gateway policy. The first supported tool is `openclaw_agent_consult`; clients get a run id and wait for normal chat lifecycle events before submitting the provider-specific tool result.
- `talk.client.create` creates or resumes a client-owned realtime provider session using `webrtc` or `provider-websocket` while the gateway owns credentials, instructions, tool policy, and the returned `voiceSessionId`. Clients pass `sessionKey` and reuse `voiceSessionId` when replacing the provider transport during one call.
- `talk.client.transcript` appends one finalized `{ role, text }` item to the normal agent session. The required `entryId` is idempotent within `voiceSessionId`; retries do not duplicate transcript messages.
- `talk.client.close` closes the logical voice session after pending transcript writes. Closing is idempotent and may deliver a mutation-only call digest to the session's last non-WebChat channel.
- `talk.client.toolCall` lets client-owned realtime transports forward provider tool calls to gateway policy. The first supported tool is `openclaw_agent_consult`; clients get a run id and wait for normal chat lifecycle events before submitting the provider-specific tool result. Voice-bound high-impact actions return `VOICE_CONFIRMATION_REQUIRED:<id>` until a later finalized user utterance explicitly confirms that exact action and the next consult supplies the `confirmationId`.
- `talk.client.steer` sends active-run voice control for client-owned realtime transports. The gateway resolves the active embedded run from `sessionKey` and returns a structured accepted/rejected result instead of silently dropping steering.
- `talk.event` is the single Talk event channel for realtime, transcription, STT/TTS, managed-room, telephony, and meeting adapters.
- `talk.speak` synthesizes speech through the active Talk speech provider.
+4
View File
@@ -18,6 +18,10 @@ Native Talk is a continuous loop: listen for speech, send the transcript to the
Client-owned realtime Talk forwards provider tool calls through `talk.client.toolCall` instead of calling `chat.send` directly. While a realtime consult is active, clients can call `talk.client.steer` or `talk.session.steer` to classify spoken input as `status`, `steer`, `cancel`, or `followup`. Accepted steering queues into the active embedded run; rejected steering returns a reason such as `no_active_run`, `not_streaming`, or `compacting`.
Finalized realtime user and assistant utterances are always appended live to the active agent session, so later chat and voice turns share one history. Client-owned transports report their finalized transcripts with stable entry ids; Gateway relay sessions append the same events server-side. Provider sessions also receive the bounded realtime profile context used by Discord voice.
Voice-originated consult runs require a new, exact spoken confirmation before high-impact actions such as sending messages, controlling nodes, browser/computer actions, service changes, destructive shell commands, or publication. The confirmation applies only to the exact blocked tool arguments and is consumed once; unrelated concurrent runs remain unaffected. When a call closes, OpenClaw can send a compact **Voice call changes** digest for mutating tools to the session's last non-WebChat delivery target.
Transcription-only Talk emits the same Talk event envelope as realtime and STT/TTS sessions, but uses `mode: "transcription"` and `brain: "none"`. All Talk sessions broadcast events on the `talk.event` channel; clients subscribe to it for partial/final transcript updates (`transcript.delta`/`transcript.done`) and other session telemetry.
Browser Video Talk is available for OpenAI Realtime WebRTC and Google Live
+12
View File
@@ -96,10 +96,13 @@ import {
TalkCatalogResultSchema,
TalkClientCreateParamsSchema,
TalkClientCreateResultSchema,
TalkClientCloseParamsSchema,
TalkClientMutationResultSchema,
TalkAgentControlResultSchema,
TalkClientSteerParamsSchema,
TalkClientToolCallParamsSchema,
TalkClientToolCallResultSchema,
TalkClientTranscriptParamsSchema,
TalkConfigParamsSchema,
TalkConfigResultSchema,
TalkSessionAppendAudioParamsSchema,
@@ -808,8 +811,11 @@ export const validateTalkConfigParams = lazyCompile(TalkConfigParamsSchema);
export const validateTalkConfigResult = lazyCompile(TalkConfigResultSchema);
export const validateTalkClientCreateParams = lazyCompile(TalkClientCreateParamsSchema);
export const validateTalkClientCreateResult = lazyCompile(TalkClientCreateResultSchema);
export const validateTalkClientCloseParams = lazyCompile(TalkClientCloseParamsSchema);
export const validateTalkClientMutationResult = lazyCompile(TalkClientMutationResultSchema);
export const validateTalkClientToolCallParams = lazyCompile(TalkClientToolCallParamsSchema);
export const validateTalkClientToolCallResult = lazyCompile(TalkClientToolCallResultSchema);
export const validateTalkClientTranscriptParams = lazyCompile(TalkClientTranscriptParamsSchema);
export const validateTalkClientSteerParams = lazyCompile(TalkClientSteerParamsSchema);
export const validateTalkAgentControlResult = lazyCompile(TalkAgentControlResultSchema);
export const validateTalkSessionCreateParams = lazyCompile(TalkSessionCreateParamsSchema);
@@ -1234,10 +1240,13 @@ export {
TalkCatalogResultSchema,
TalkClientCreateParamsSchema,
TalkClientCreateResultSchema,
TalkClientCloseParamsSchema,
TalkClientMutationResultSchema,
TalkAgentControlResultSchema,
TalkClientSteerParamsSchema,
TalkClientToolCallParamsSchema,
TalkClientToolCallResultSchema,
TalkClientTranscriptParamsSchema,
TalkConfigParamsSchema,
TalkConfigResultSchema,
TalkSessionAppendAudioParamsSchema,
@@ -1560,10 +1569,13 @@ export type {
TalkCatalogResult,
TalkClientCreateParams,
TalkClientCreateResult,
TalkClientCloseParams,
TalkClientMutationResult,
TalkClientSteerParams,
TalkAgentControlResult,
TalkClientToolCallParams,
TalkClientToolCallResult,
TalkClientTranscriptParams,
TalkConfigParams,
TalkConfigResult,
TalkSessionAppendAudioParams,
@@ -186,9 +186,14 @@ export const TalkEventSchema = Type.Object(
},
);
// Voice ids compose into transcript idempotency keys (`voice:<id>:<entry>`), so the
// charset excludes the `:` delimiter to keep distinct id pairs collision-free.
const VoiceIdString = Type.String({ pattern: "^[A-Za-z0-9_-]{1,128}$" });
/** Creates a browser-facing Talk client session. */
export const TalkClientCreateParamsSchema = closedObject({
sessionKey: Type.Optional(Type.String()),
sessionKey: Type.Optional(NonEmptyString),
voiceSessionId: Type.Optional(VoiceIdString),
provider: Type.Optional(Type.String()),
model: Type.Optional(Type.String()),
voice: Type.Optional(Type.String()),
@@ -199,18 +204,44 @@ export const TalkClientCreateParamsSchema = closedObject({
mode: Type.Optional(TalkModeSchema),
transport: Type.Optional(TalkTransportSchema),
brain: Type.Optional(TalkBrainSchema),
capabilities: Type.Optional(Type.Array(Type.Literal("camera-frame"), { uniqueItems: true })),
capabilities: Type.Optional(
Type.Array(Type.Union([Type.Literal("camera-frame"), Type.Literal("voice-transcript")]), {
uniqueItems: true,
}),
),
});
/** Tool-call request from a browser/client session back into the agent runtime. */
export const TalkClientToolCallParamsSchema = closedObject({
sessionKey: NonEmptyString,
voiceSessionId: Type.Optional(VoiceIdString),
callId: NonEmptyString,
name: NonEmptyString,
args: Type.Optional(Type.Unknown()),
relaySessionId: Type.Optional(NonEmptyString),
});
/** One finalized transcript item from a client-owned Talk session. */
export const TalkClientTranscriptParamsSchema = closedObject({
sessionKey: NonEmptyString,
voiceSessionId: VoiceIdString,
entryId: VoiceIdString,
role: Type.Union([Type.Literal("user"), Type.Literal("assistant")]),
text: NonEmptyString,
timestamp: Type.Optional(Type.Number()),
});
/** Logical close for a client-owned Talk session. */
export const TalkClientCloseParamsSchema = closedObject({
sessionKey: NonEmptyString,
voiceSessionId: VoiceIdString,
});
/** Result for client-owned transcript and close mutations. */
export const TalkClientMutationResultSchema = closedObject({
ok: Type.Literal(true),
});
/** Agent run identity returned after accepting a Talk client tool call. */
export const TalkClientToolCallResultSchema = closedObject({
runId: NonEmptyString,
@@ -456,6 +487,7 @@ export const TalkSessionOkResultSchema = closedObject({
const BrowserRealtimeWebRtcSdpSessionSchema = closedObject({
provider: NonEmptyString,
transport: Type.Literal("webrtc"),
voiceSessionId: NonEmptyString,
clientSecret: NonEmptyString,
offerUrl: Type.Optional(Type.String()),
offerHeaders: Type.Optional(Type.Record(Type.String(), Type.String())),
@@ -468,6 +500,7 @@ const BrowserRealtimeWebRtcSdpSessionSchema = closedObject({
const BrowserRealtimeJsonPcmWebSocketSessionSchema = closedObject({
provider: NonEmptyString,
transport: Type.Literal("provider-websocket"),
voiceSessionId: NonEmptyString,
protocol: NonEmptyString,
clientSecret: NonEmptyString,
websocketUrl: NonEmptyString,
@@ -482,6 +515,8 @@ const BrowserRealtimeJsonPcmWebSocketSessionSchema = closedObject({
const BrowserRealtimeGatewayRelaySessionSchema = closedObject({
provider: NonEmptyString,
transport: Type.Literal("gateway-relay"),
// Server-owned: older gateways omit it and clients derive it from relaySessionId.
voiceSessionId: Type.Optional(NonEmptyString),
relaySessionId: NonEmptyString,
audio: BrowserRealtimeAudioContractSchema,
model: Type.Optional(Type.String()),
@@ -493,6 +528,8 @@ const BrowserRealtimeGatewayRelaySessionSchema = closedObject({
const BrowserRealtimeManagedRoomSessionSchema = closedObject({
provider: NonEmptyString,
transport: Type.Literal("managed-room"),
// Server-owned rooms carry no client voice bookkeeping yet.
voiceSessionId: Type.Optional(NonEmptyString),
roomUrl: NonEmptyString,
token: Type.Optional(Type.String()),
model: Type.Optional(Type.String()),
@@ -740,6 +777,9 @@ export type TalkClientSteerParams = Static<typeof TalkClientSteerParamsSchema>;
export type TalkAgentControlResult = Static<typeof TalkAgentControlResultSchema>;
export type TalkClientToolCallParams = Static<typeof TalkClientToolCallParamsSchema>;
export type TalkClientToolCallResult = Static<typeof TalkClientToolCallResultSchema>;
export type TalkClientTranscriptParams = Static<typeof TalkClientTranscriptParamsSchema>;
export type TalkClientCloseParams = Static<typeof TalkClientCloseParamsSchema>;
export type TalkClientMutationResult = Static<typeof TalkClientMutationResultSchema>;
export type TalkSessionCreateParams = Static<typeof TalkSessionCreateParamsSchema>;
export type TalkSessionCreateResult = Static<typeof TalkSessionCreateResultSchema>;
export type TalkSessionJoinParams = Static<typeof TalkSessionJoinParamsSchema>;
@@ -187,10 +187,13 @@ import {
TalkCatalogResultSchema,
TalkClientCreateParamsSchema,
TalkClientCreateResultSchema,
TalkClientCloseParamsSchema,
TalkClientMutationResultSchema,
TalkAgentControlResultSchema,
TalkClientSteerParamsSchema,
TalkClientToolCallParamsSchema,
TalkClientToolCallResultSchema,
TalkClientTranscriptParamsSchema,
TalkConfigParamsSchema,
TalkConfigResultSchema,
TalkSessionAppendAudioParamsSchema,
@@ -850,10 +853,13 @@ export const ProtocolSchemas = {
TalkCatalogResult: TalkCatalogResultSchema,
TalkClientCreateParams: TalkClientCreateParamsSchema,
TalkClientCreateResult: TalkClientCreateResultSchema,
TalkClientCloseParams: TalkClientCloseParamsSchema,
TalkClientMutationResult: TalkClientMutationResultSchema,
TalkClientSteerParams: TalkClientSteerParamsSchema,
TalkAgentControlResult: TalkAgentControlResultSchema,
TalkClientToolCallParams: TalkClientToolCallParamsSchema,
TalkClientToolCallResult: TalkClientToolCallResultSchema,
TalkClientTranscriptParams: TalkClientTranscriptParamsSchema,
TalkConfigParams: TalkConfigParamsSchema,
TalkConfigResult: TalkConfigResultSchema,
TalkSessionAppendAudioParams: TalkSessionAppendAudioParamsSchema,
@@ -71,6 +71,11 @@ import {
} from "../skills/loading/source.js";
import type { SkillSnapshot, SkillTelemetrySource, SkillUsagePath } from "../skills/types.js";
import { resolveSkillWorkshopToolApproval } from "../skills/workshop/policy.js";
import { resolveClientVoiceToolConfirmationPolicy } from "../talk/client-voice-confirmation.js";
import {
isClientVoiceSessionConfirmable,
resolveClientVoiceRunBinding,
} from "../talk/client-voice-session.js";
import { isPlainObject, truncateUtf16Safe } from "../utils.js";
import {
adjustedParamsByToolCallId,
@@ -84,6 +89,7 @@ import {
} from "./agent-tools.before-tool-call.state.js";
import { normalizeFileToolPathParam } from "./agent-tools.params.js";
import { resolveAgentRunAbortLifecycleFields } from "./run-termination.js";
import { buildToolMutationState } from "./tool-mutation.js";
export {
consumeAdjustedParamsForToolCall,
consumePreExecutionBlockedToolCall,
@@ -1533,6 +1539,24 @@ export async function runBeforeToolCallHook(args: {
...(args.ctx?.config ? { config: args.ctx.config } : {}),
...(args.ctx?.workspaceDir ? { workspaceDir: args.ctx.workspaceDir } : {}),
});
const voiceRun = resolveClientVoiceRunBinding(args.ctx?.runId);
const voiceConfirmation = resolveClientVoiceToolConfirmationPolicy({
agentId: voiceRun?.agentId,
voiceSessionId: voiceRun?.voiceSessionId,
runId: args.ctx?.runId,
toolName,
toolParams: normalizedParams,
...(voiceRun ? { isConfirmable: () => isClientVoiceSessionConfirmable(voiceRun) } : {}),
});
if (!voiceConfirmation.allowed) {
return {
blocked: true,
kind: "veto",
deniedReason: "plugin-before-tool-call",
reason: voiceConfirmation.reason,
params,
};
}
if (!initialCorePolicyResult && !shouldRunTrustedPolicies && !hasBeforeToolCallHooks) {
return { blocked: false, params };
}
@@ -1793,6 +1817,7 @@ export function wrapToolWithBeforeToolCallHook(
...diagnosticIdentity,
...(toolCallId && { toolCallId }),
paramsSummary: summarizeToolParams(toolParams),
mutatingAction: buildToolMutationState(normalizedToolName, toolParams).mutatingAction,
});
const recordPreExecutionError = (
error: unknown,
+4
View File
@@ -87,6 +87,8 @@ describe("method scope resolution", () => {
["node.pair.approve", ["operator.pairing"]],
["poll", ["operator.write"]],
["talk.client.create", ["operator.write"]],
["talk.client.transcript", ["operator.write"]],
["talk.client.close", ["operator.write"]],
["talk.client.toolCall", ["operator.write"]],
["talk.client.steer", ["operator.write"]],
["talk.session.create", ["operator.write"]],
@@ -518,6 +520,8 @@ describe("operator scope authorization", () => {
it("allows operator.write clients to use unified Talk sessions", () => {
for (const method of [
"talk.client.create",
"talk.client.transcript",
"talk.client.close",
"talk.client.toolCall",
"talk.client.steer",
"talk.session.create",
+2
View File
@@ -87,6 +87,8 @@ const CORE_GATEWAY_METHOD_SPECS: readonly CoreGatewayMethodSpec[] = [
// Params-aware: reading redacted config needs read; includeSecrets also needs talk secrets.
{ name: "talk.config", scope: "dynamic" },
{ name: "talk.client.create", scope: "operator.write" },
{ name: "talk.client.transcript", scope: "operator.write" },
{ name: "talk.client.close", scope: "operator.write" },
{ name: "talk.client.toolCall", scope: "operator.write" },
{ name: "talk.client.steer", scope: "operator.write" },
{ name: "talk.session.create", scope: "operator.write" },
+2
View File
@@ -149,6 +149,8 @@ describe("listGatewayMethods", () => {
it("advertises the versioned Talk session RPCs", () => {
const methods = listGatewayMethods();
expect(methods).toContain("talk.client.create");
expect(methods).toContain("talk.client.transcript");
expect(methods).toContain("talk.client.close");
expect(methods).toContain("talk.client.toolCall");
expect(methods).toContain("talk.client.steer");
expect(methods).toContain("talk.session.create");
+2
View File
@@ -578,6 +578,8 @@ export const coreGatewayHandlers: GatewayRequestHandlers = {
"talk.session.steer",
"talk.session.close",
"talk.client.create",
"talk.client.transcript",
"talk.client.close",
"talk.client.toolCall",
"talk.client.steer",
"talk.catalog",
@@ -0,0 +1,282 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
loadSessionEntry,
readSessionTranscriptMessageEvents,
replaceSessionEntry,
} from "../../config/sessions/session-accessor.js";
import { closeOpenClawAgentDatabasesForTest } from "../../state/openclaw-agent-db.js";
import { closeOpenClawStateDatabaseForTest } from "../../state/openclaw-state-db.js";
import {
authorizeClientVoiceConfirmation,
resolveClientVoiceToolConfirmationPolicy,
} from "../../talk/client-voice-confirmation.js";
import { resetClientVoiceConfirmationStateForTest } from "../../talk/client-voice-confirmation.test-support.js";
import {
closeClientVoiceSession,
createOrResumeClientVoiceSession,
ensureClientVoiceAgentSessionEntry,
} from "../../talk/client-voice-session.js";
import { clientVoiceSessionTesting } from "../../talk/client-voice-session.test-support.js";
import { captureEnv, setTestEnvValue } from "../../test-utils/env.js";
import { talkClientHandlers } from "./talk-client.js";
const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
const sessionKey = "agent:main:main";
const sessionId = "voice-transcript-session";
let tempDir: string;
async function invokeTranscript(params: Record<string, unknown>) {
const respond = vi.fn();
await talkClientHandlers["talk.client.transcript"]?.({
params,
respond,
context: { getRuntimeConfig: () => ({}) },
} as never);
return respond;
}
async function invokeClose(params: Record<string, unknown>) {
const respond = vi.fn();
await talkClientHandlers["talk.client.close"]?.({
params,
respond,
context: { getRuntimeConfig: () => ({}) },
client: { connId: "conn-close" },
} as never);
return respond;
}
describe("talk.client.transcript", () => {
beforeEach(async () => {
tempDir = await fs.realpath(
await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-talk-transcript-")),
);
setTestEnvValue("OPENCLAW_STATE_DIR", tempDir);
await replaceSessionEntry(
{ agentId: "main", sessionKey },
{ sessionId, updatedAt: Date.now() },
);
});
afterEach(async () => {
clientVoiceSessionTesting.reset();
resetClientVoiceConfirmationStateForTest();
vi.useRealTimers();
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
envSnapshot.restore();
await fs.rm(tempDir, { recursive: true, force: true });
});
it("appends finalized messages once by event id", async () => {
const voiceSessionId = createOrResumeClientVoiceSession({
agentId: "main",
sessionKey,
origin: "client",
});
const params = {
sessionKey,
voiceSessionId,
entryId: "1",
role: "user",
text: "hello from voice",
timestamp: 123,
};
expect(await invokeTranscript(params)).toHaveBeenCalledWith(true, { ok: true }, undefined);
expect(await invokeTranscript(params)).toHaveBeenCalledWith(true, { ok: true }, undefined);
const events = readSessionTranscriptMessageEvents({ agentId: "main", sessionId });
expect(events).toHaveLength(1);
expect(events[0]?.event).toMatchObject({
id: `voice:${voiceSessionId}:1`,
message: {
role: "user",
content: [{ type: "text", text: "hello from voice" }],
timestamp: 123,
provenance: { kind: "realtime_voice", sourceChannel: "talk" },
},
});
});
it("appends before the session has ever received a chat turn", async () => {
const talkFirstSessionKey = "agent:main:talk-first";
await ensureClientVoiceAgentSessionEntry({
agentId: "main",
sessionKey: talkFirstSessionKey,
});
const voiceSessionId = createOrResumeClientVoiceSession({
agentId: "main",
sessionKey: talkFirstSessionKey,
provider: "google",
origin: "client",
});
expect(
await invokeTranscript({
sessionKey: talkFirstSessionKey,
voiceSessionId,
entryId: "1",
role: "user",
text: "heard before the first consult",
}),
).toHaveBeenCalledWith(true, { ok: true }, undefined);
const talkFirstEntry = loadSessionEntry({
agentId: "main",
sessionKey: talkFirstSessionKey,
});
expect(talkFirstEntry?.sessionId).toBeTruthy();
expect(
readSessionTranscriptMessageEvents({
agentId: "main",
sessionId: talkFirstEntry?.sessionId ?? "missing",
}),
).toHaveLength(1);
});
it("uses server observation time for spoken-confirmation freshness", async () => {
vi.useFakeTimers();
vi.setSystemTime(100);
const voiceSessionId = createOrResumeClientVoiceSession({
agentId: "main",
sessionKey,
origin: "client",
});
await invokeTranscript({
sessionKey,
voiceSessionId,
entryId: "early-yes",
role: "user",
text: "yes",
timestamp: 10_000,
});
const policy = resolveClientVoiceToolConfirmationPolicy({
agentId: "main",
voiceSessionId,
runId: "run-later",
toolName: "message",
toolParams: { action: "send", message: "later" },
now: 200,
});
expect(policy.allowed).toBe(false);
if (policy.allowed) {
throw new Error("expected confirmation request");
}
const confirmationId = policy.reason.match(/VOICE_CONFIRMATION_REQUIRED:([^\s]+)/)?.[1];
expect(confirmationId).toBeTruthy();
expect(() =>
authorizeClientVoiceConfirmation({
agentId: "main",
voiceSessionId,
confirmationId: confirmationId ?? "missing",
now: 201,
}),
).toThrow("explicit spoken confirmation");
});
it("accepts an idempotent close retry after the first response is lost", async () => {
const voiceSessionId = createOrResumeClientVoiceSession({
agentId: "main",
sessionKey,
origin: "client",
});
const params = { sessionKey, voiceSessionId };
expect(await invokeClose(params)).toHaveBeenCalledWith(true, { ok: true }, undefined);
expect(await invokeClose(params)).toHaveBeenCalledWith(true, { ok: true }, undefined);
});
it("truncates UTF-16 safely and writes assistant metadata", async () => {
const voiceSessionId = createOrResumeClientVoiceSession({
agentId: "main",
sessionKey,
provider: "google",
origin: "client",
});
const respond = await invokeTranscript({
sessionKey,
voiceSessionId,
entryId: "assistant-1",
role: "assistant",
text: `${"x".repeat(7_999)}😀tail`,
});
expect(respond).toHaveBeenCalledWith(true, { ok: true }, undefined);
const event = readSessionTranscriptMessageEvents({ agentId: "main", sessionId })[0]?.event as
| { message?: { content?: Array<{ text?: string }> } }
| undefined;
const text = event?.message?.content?.[0]?.text;
expect(text).toBe("x".repeat(7_999));
expect(event).toMatchObject({
message: {
api: "realtime",
provider: "google",
model: "realtime-voice",
stopReason: "stop",
},
});
});
it("uses the neutral provider label for records created before provider tracking", async () => {
const voiceSessionId = createOrResumeClientVoiceSession({
agentId: "main",
sessionKey,
origin: "client",
});
expect(
await invokeTranscript({
sessionKey,
voiceSessionId,
entryId: "legacy-assistant",
role: "assistant",
text: "legacy provider reply",
}),
).toHaveBeenCalledWith(true, { ok: true }, undefined);
expect(
readSessionTranscriptMessageEvents({ agentId: "main", sessionId })[0]?.event,
).toMatchObject({
message: { api: "realtime", provider: "realtime", model: "realtime-voice" },
});
});
it.each([
["missing", "voice-missing", "voice session not found"],
["closed", "voice-closed", "voice session is closed"],
["relay", "voice-relay", "does not allow this transcript source"],
])("rejects %s voice records", async (kind, voiceSessionId, expected) => {
if (kind !== "missing") {
createOrResumeClientVoiceSession({
agentId: "main",
sessionKey,
origin: kind === "relay" ? "relay" : "client",
voiceSessionId,
});
}
if (kind === "closed") {
await closeClientVoiceSession({
agentId: "main",
sessionKey,
voiceSessionId,
config: {},
});
}
const respond = await invokeTranscript({
sessionKey,
voiceSessionId,
entryId: "1",
role: "user",
text: "hello",
});
expect(respond).toHaveBeenCalledWith(
false,
undefined,
expect.objectContaining({ message: expect.stringContaining(expected) }),
);
});
});
+250 -4
View File
@@ -8,28 +8,71 @@ import {
ErrorCodes,
errorShape,
formatValidationErrors,
validateTalkClientCloseParams,
validateTalkClientCreateParams,
validateTalkClientSteerParams,
validateTalkClientToolCallParams,
validateTalkClientTranscriptParams,
} from "../../../packages/gateway-protocol/src/index.js";
import {
buildAgentMainSessionKey,
resolveAgentIdFromSessionKey,
} from "../../routing/session-key.js";
import {
REALTIME_VOICE_AGENT_CONSULT_TOOL,
REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
parseRealtimeVoiceAgentConsultArgs,
} from "../../talk/agent-consult-tool.js";
import { REALTIME_VOICE_AGENT_CONTROL_TOOL } from "../../talk/agent-run-control-shared.js";
import { controlRealtimeVoiceAgentRun } from "../../talk/agent-run-control.js";
import {
authorizeClientVoiceConfirmation,
bindAuthorizedClientVoiceConfirmation,
type ClientVoiceConfirmationGrant,
} from "../../talk/client-voice-confirmation.js";
import {
appendClientVoiceTranscript,
assertClientVoiceSessionOpen,
closeClientVoiceSession,
closeStaleClientVoiceSessions,
createOrResumeClientVoiceSession,
ensureClientVoiceAgentSessionEntry,
registerClientVoiceConsultRun,
resolveClientVoiceSessionOrigin,
resolveOpenClientVoiceSessionId,
} from "../../talk/client-voice-session.js";
import { REALTIME_VOICE_DESCRIBE_VIEW_TOOL } from "../../talk/describe-view-tool.js";
import { resolveConfiguredRealtimeVoiceProvider } from "../../talk/provider-resolver.js";
import { startTalkRealtimeAgentConsult } from "../talk-agent-consult.js";
import {
ensureTalkRealtimeRelayVoiceSession,
flushTalkRealtimeRelayVoiceWrites,
} from "../talk-realtime-relay.js";
import { formatForLog } from "../ws-log.js";
import {
buildRealtimeInstructions,
buildRealtimeVoiceLaunchOptions,
buildTalkRealtimeConfig,
isUnsupportedBrowserWebRtcSession,
resolveTalkRealtimeProviderInstructions,
} from "./talk-shared.js";
import type { GatewayRequestHandlers } from "./types.js";
const LEGACY_VOICE_BINDING_TTL_MS = 6 * 60 * 60_000;
const legacyVoiceSessionByClient = new Map<string, { voiceSessionId: string; expiresAt: number }>();
function legacyVoiceBindingKey(connId: string, sessionKey: string): string {
return `${connId}\0${sessionKey}`;
}
function pruneLegacyVoiceBindings(now = Date.now()): void {
for (const [key, binding] of legacyVoiceSessionByClient) {
if (binding.expiresAt <= now) {
legacyVoiceSessionByClient.delete(key);
}
}
}
/**
* Gateway methods for browser-owned realtime Talk sessions.
*
@@ -37,7 +80,7 @@ import type { GatewayRequestHandlers } from "./types.js";
* calls back into OpenClaw agent consult runs.
*/
export const talkClientHandlers: GatewayRequestHandlers = {
"talk.client.create": async ({ params, respond, context }) => {
"talk.client.create": async ({ params, respond, context, client }) => {
if (!validateTalkClientCreateParams(params)) {
respond(
false,
@@ -50,6 +93,8 @@ export const talkClientHandlers: GatewayRequestHandlers = {
return;
}
const typedParams = params as {
sessionKey?: string;
voiceSessionId?: string;
provider?: string;
model?: string;
voice?: string;
@@ -143,6 +188,17 @@ export const talkClientHandlers: GatewayRequestHandlers = {
requested: typedParams,
defaults: realtimeConfig,
});
const realtimeContext = await resolveTalkRealtimeProviderInstructions({
config: runtimeConfig,
configuredInstructions: realtimeConfig.instructions,
sessionKey: typedParams.sessionKey,
// Legacy creates can drift to another agent's session at toolCall time, so
// the default agent's profile must not leak into the provider session.
requireSessionKeyForProfile: true,
warn: (message) => context.logGateway.warn(`talk realtime context: ${message}`),
});
const { agentId, requestedSessionKey } = realtimeContext;
const sessionKey = requestedSessionKey ?? buildAgentMainSessionKey({ agentId });
if (resolution.provider.createBrowserSession && transport !== "gateway-relay") {
const tools = [REALTIME_VOICE_AGENT_CONSULT_TOOL, REALTIME_VOICE_AGENT_CONTROL_TOOL];
if (wantsCameraFrames) {
@@ -151,16 +207,50 @@ export const talkClientHandlers: GatewayRequestHandlers = {
const session = await resolution.provider.createBrowserSession({
cfg: runtimeConfig,
providerConfig: resolution.providerConfig,
instructions: buildRealtimeInstructions(realtimeConfig.instructions),
instructions: buildRealtimeInstructions(realtimeContext.instructions),
tools,
...launchOptions,
});
// Client-owned voice records are minted only for client-owned transports;
// relay sessions are created via talk.session.create and keyed by relaySessionId.
// Widening this guard would hand relay calls a mismatched voiceSessionId.
if (
(session.transport === "webrtc" || session.transport === "provider-websocket") &&
!isUnsupportedBrowserWebRtcSession(session) &&
(!transport || session.transport === transport)
) {
respond(true, session, undefined);
// Recovering 6h-abandoned calls (and retrying their digests) is not on the
// start path; running it inline would delay use of time-sensitive provider
// credentials behind slow channel sends. Fire it off the response path.
void closeStaleClientVoiceSessions({
agentId,
config: runtimeConfig,
excludeVoiceSessionId: normalizeOptionalString(typedParams.voiceSessionId),
warn: (message) => context.logGateway.warn(`talk voice session recovery: ${message}`),
}).catch((error: unknown) =>
context.logGateway.warn(`talk voice session recovery failed: ${formatForLog(error)}`),
);
await ensureClientVoiceAgentSessionEntry({ agentId, sessionKey });
const voiceSessionId = createOrResumeClientVoiceSession({
agentId,
sessionKey,
provider: resolution.provider.id,
origin: "client",
// Deployed clients sent sessionKey before transcripts existed, so capability
// must be negotiated explicitly; declaring it turns the confirmation gate on.
transcriptCapable: typedParams.capabilities?.includes("voice-transcript") === true,
voiceSessionId: normalizeOptionalString(typedParams.voiceSessionId),
});
const connId = normalizeOptionalString(client?.connId);
if (connId) {
const now = Date.now();
pruneLegacyVoiceBindings(now);
legacyVoiceSessionByClient.set(
legacyVoiceBindingKey(connId, typedParams.sessionKey?.trim() || sessionKey),
{ voiceSessionId, expiresAt: now + LEGACY_VOICE_BINDING_TTL_MS },
);
}
respond(true, { ...session, voiceSessionId }, undefined);
return;
}
if (transport) {
@@ -209,6 +299,82 @@ export const talkClientHandlers: GatewayRequestHandlers = {
return;
}
const agentId = resolveAgentIdFromSessionKey(params.sessionKey);
const relaySessionId = normalizeOptionalString(params.relaySessionId);
const connId = normalizeOptionalString(request.client?.connId);
pruneLegacyVoiceBindings();
const explicitVoiceSessionId = normalizeOptionalString(params.voiceSessionId);
if (relaySessionId && explicitVoiceSessionId && explicitVoiceSessionId !== relaySessionId) {
respond(
false,
undefined,
errorShape(ErrorCodes.INVALID_REQUEST, "relaySessionId and voiceSessionId must match"),
);
return;
}
let confirmationGrant: ClientVoiceConfirmationGrant | undefined;
let voiceSessionId: string;
try {
// Shipped clients may consult without ever creating a voice session (old app,
// restarted gateway, ambiguous open records). Implicitly create one instead of
// erroring so confirmation and mutation evidence stay always-on.
voiceSessionId =
explicitVoiceSessionId ??
relaySessionId ??
(connId
? legacyVoiceSessionByClient.get(legacyVoiceBindingKey(connId, params.sessionKey))
?.voiceSessionId
: undefined) ??
resolveOpenClientVoiceSessionId({ agentId, sessionKey: params.sessionKey }) ??
createOrResumeClientVoiceSession({
agentId,
sessionKey: params.sessionKey,
origin: "client",
});
// Pin the resolved id to this connection so a legacy client's later consults
// reuse one record instead of forking a new never-closed session each time.
if (connId && !relaySessionId) {
const now = Date.now();
pruneLegacyVoiceBindings(now);
legacyVoiceSessionByClient.set(legacyVoiceBindingKey(connId, params.sessionKey), {
voiceSessionId,
expiresAt: now + LEGACY_VOICE_BINDING_TTL_MS,
});
}
if (relaySessionId && connId) {
// Initialize the canonical session row BEFORE binding: the bind drains the
// relay's buffered finals into transcript appends, which fail without it.
await ensureClientVoiceAgentSessionEntry({ agentId, sessionKey: params.sessionKey });
ensureTalkRealtimeRelayVoiceSession({
relaySessionId,
connId,
sessionKey: params.sessionKey,
});
await flushTalkRealtimeRelayVoiceWrites({ relaySessionId, connId });
}
const parsedArgs = parseRealtimeVoiceAgentConsultArgs(params.args ?? {});
const origin = assertClientVoiceSessionOpen({
agentId,
sessionKey: params.sessionKey,
voiceSessionId,
});
if (origin === "relay" && (!relaySessionId || !connId)) {
throw new Error(
"relay-owned voice sessions require relaySessionId and connection ownership",
);
}
if (parsedArgs.confirmationId) {
confirmationGrant = authorizeClientVoiceConfirmation({
agentId,
voiceSessionId,
confirmationId: parsedArgs.confirmationId,
});
}
} catch (err) {
respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, formatForLog(err)));
return;
}
const result = await startTalkRealtimeAgentConsult({
context: request.context,
client: request.client,
@@ -218,7 +384,19 @@ export const talkClientHandlers: GatewayRequestHandlers = {
callId: params.callId,
args: params.args ?? {},
relaySessionId: normalizeOptionalString(params.relaySessionId),
connId: normalizeOptionalString(request.client?.connId),
connId,
onRunStarted: (runId) => {
registerClientVoiceConsultRun({
agentId,
sessionKey: params.sessionKey,
voiceSessionId,
runId,
config: request.context.getRuntimeConfig(),
});
if (confirmationGrant) {
bindAuthorizedClientVoiceConfirmation({ grant: confirmationGrant, runId });
}
},
});
if (!result.ok) {
respond(false, undefined, result.error);
@@ -233,6 +411,74 @@ export const talkClientHandlers: GatewayRequestHandlers = {
undefined,
);
},
"talk.client.transcript": async ({ params, respond, context }) => {
if (!validateTalkClientTranscriptParams(params)) {
respond(
false,
undefined,
errorShape(
ErrorCodes.INVALID_REQUEST,
`invalid talk.client.transcript params: ${formatValidationErrors(validateTalkClientTranscriptParams.errors)}`,
),
);
return;
}
try {
await appendClientVoiceTranscript({
agentId: resolveAgentIdFromSessionKey(params.sessionKey),
sessionKey: params.sessionKey,
voiceSessionId: params.voiceSessionId,
entryId: params.entryId,
role: params.role,
text: params.text,
...(params.timestamp !== undefined ? { timestamp: params.timestamp } : {}),
config: context.getRuntimeConfig(),
});
respond(true, { ok: true }, undefined);
} catch (err) {
respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, formatForLog(err)));
}
},
"talk.client.close": async ({ params, respond, context, client }) => {
if (!validateTalkClientCloseParams(params)) {
respond(
false,
undefined,
errorShape(
ErrorCodes.INVALID_REQUEST,
`invalid talk.client.close params: ${formatValidationErrors(validateTalkClientCloseParams.errors)}`,
),
);
return;
}
try {
const agentId = resolveAgentIdFromSessionKey(params.sessionKey);
const origin = resolveClientVoiceSessionOrigin({
agentId,
sessionKey: params.sessionKey,
voiceSessionId: params.voiceSessionId,
});
if (origin === "relay") {
throw new Error("relay-owned voice sessions close through talk.session.stop");
}
await closeClientVoiceSession({
agentId,
sessionKey: params.sessionKey,
voiceSessionId: params.voiceSessionId,
config: context.getRuntimeConfig(),
});
const connId = normalizeOptionalString(client?.connId);
if (connId) {
const key = legacyVoiceBindingKey(connId, params.sessionKey);
if (legacyVoiceSessionByClient.get(key)?.voiceSessionId === params.voiceSessionId) {
legacyVoiceSessionByClient.delete(key);
}
}
respond(true, { ok: true }, undefined);
} catch (err) {
respond(false, undefined, errorShape(ErrorCodes.INVALID_REQUEST, formatForLog(err)));
}
},
"talk.client.steer": async ({ params, respond, client, context }) => {
if (!validateTalkClientSteerParams(params)) {
respond(
+23 -37
View File
@@ -19,8 +19,8 @@ import {
import { REALTIME_VOICE_AGENT_CONSULT_TOOL } from "../../talk/agent-consult-tool.js";
import { REALTIME_VOICE_AGENT_CONTROL_TOOL } from "../../talk/agent-run-control-shared.js";
import { controlRealtimeVoiceAgentRun } from "../../talk/agent-run-control.js";
import { ensureClientVoiceAgentSessionEntry } from "../../talk/client-voice-session.js";
import { resolveConfiguredRealtimeVoiceProvider } from "../../talk/provider-resolver.js";
import type { TalkBrain, TalkMode, TalkTransport } from "../../talk/talk-events.js";
import { ADMIN_SCOPE } from "../operator-scopes.js";
import { resolveSessionKeyFromResolveParams } from "../sessions-resolve.js";
import {
@@ -63,49 +63,20 @@ import {
buildTalkRealtimeConfig,
buildTalkTranscriptionConfig,
canUseTalkDirectTools,
normalizeTalkSessionBrain,
normalizeTalkSessionMode,
normalizeTalkSessionTransport,
resolveConfiguredRealtimeTranscriptionProvider,
resolveTalkRealtimeProviderInstructions,
talkHandoffErrorCode,
withRealtimeBrowserOverrides,
} from "./talk-shared.js";
import type { GatewayRequestContext, GatewayRequestHandlers, RespondFn } from "./types.js";
import { assertValidParams } from "./validation.js";
/**
* Gateway-managed Talk session methods for managed rooms and audio relays.
* The public `sessionId` is resolved through the unified registry so each RPC
* can enforce the correct connection ownership for its concrete backend.
*/
/** Gateway-managed Talk sessions resolve public ids through connection-owned unified records. */
type ManagedRoomTalkSession = Extract<UnifiedTalkSessionRecord, { kind: "managed-room" }>;
function normalizeTalkSessionMode(params: { mode?: string; transport?: string }): TalkMode {
const mode = normalizeOptionalLowercaseString(params.mode) as TalkMode | undefined;
if (mode) {
return mode;
}
return normalizeOptionalLowercaseString(params.transport) === "managed-room"
? "stt-tts"
: "realtime";
}
function normalizeTalkSessionTransport(params: {
mode: TalkMode;
transport?: string;
}): TalkTransport {
const transport = normalizeOptionalLowercaseString(params.transport) as TalkTransport | undefined;
if (transport) {
return transport;
}
return params.mode === "stt-tts" ? "managed-room" : "gateway-relay";
}
function normalizeTalkSessionBrain(params: { mode: TalkMode; brain?: string }): TalkBrain {
const brain = normalizeOptionalLowercaseString(params.brain) as TalkBrain | undefined;
if (brain) {
return brain;
}
return params.mode === "transcription" ? "none" : "agent-consult";
}
function isActiveManagedRoomClient(
session: { handoffId: string },
connId: string | undefined,
@@ -321,16 +292,29 @@ export const talkSessionHandlers: GatewayRequestHandlers = {
requested: params,
defaults: realtimeConfig,
});
const realtimeContext = await resolveTalkRealtimeProviderInstructions({
config: runtimeConfig,
configuredInstructions: realtimeConfig.instructions,
sessionKey: params.sessionKey,
requireSessionKeyForProfile: true,
warn: (message) => context.logGateway.warn(`talk realtime context: ${message}`),
});
if (realtimeContext.requestedSessionKey) {
await ensureClientVoiceAgentSessionEntry({
agentId: realtimeContext.agentId,
sessionKey: realtimeContext.requestedSessionKey,
});
}
const session = createTalkRealtimeRelaySession({
context,
connId,
cfg: runtimeConfig,
provider: resolution.provider,
providerConfig: withRealtimeBrowserOverrides(resolution.providerConfig, launchOptions),
instructions: buildRealtimeInstructions(realtimeConfig.instructions),
instructions: buildRealtimeInstructions(realtimeContext.instructions),
tools: [REALTIME_VOICE_AGENT_CONSULT_TOOL, REALTIME_VOICE_AGENT_CONTROL_TOOL],
model: launchOptions.model,
sessionKey: normalizeOptionalString(params.sessionKey),
sessionKey: realtimeContext.requestedSessionKey,
voice: launchOptions.voice,
language: normalizeOptionalLowercaseString(params.language),
forceAgentConsultOnFinalTranscript:
@@ -344,6 +328,8 @@ export const talkSessionHandlers: GatewayRequestHandlers = {
respondOk(respond, {
...session,
sessionId: session.relaySessionId,
// The relay session is the logical voice call; clients need not synthesize it.
voiceSessionId: session.relaySessionId,
mode,
brain,
});
+65 -1
View File
@@ -13,6 +13,8 @@ import {
resolveSupportedVoiceModelRefs,
type VoiceModelProvider,
} from "../../../packages/speech-core/voice-models.js";
import { resolveDefaultAgentId } from "../../agents/agent-scope.js";
import { resolveRealtimeBootstrapContextInstructions } from "../../agents/realtime-bootstrap-context.js";
import type { TalkRealtimeConfig } from "../../config/types.gateway.js";
import type { OpenClawConfig } from "../../config/types.js";
import {
@@ -20,6 +22,7 @@ import {
listRealtimeTranscriptionProviders,
} from "../../realtime-transcription/provider-registry.js";
import type { RealtimeTranscriptionProviderConfig } from "../../realtime-transcription/provider-types.js";
import { resolveAgentIdFromSessionKey } from "../../routing/session-key.js";
import { REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME } from "../../talk/agent-consult-tool.js";
import { REALTIME_VOICE_AGENT_CONTROL_TOOL_NAME } from "../../talk/agent-run-control-shared.js";
import { listRealtimeVoiceProviders } from "../../talk/provider-registry.js";
@@ -27,10 +30,71 @@ import type {
RealtimeVoiceBrowserSession,
RealtimeVoiceProviderConfig,
} from "../../talk/provider-types.js";
import type { TalkEvent } from "../../talk/talk-events.js";
import type { TalkBrain, TalkEvent, TalkMode, TalkTransport } from "../../talk/talk-events.js";
import { ADMIN_SCOPE } from "../operator-scopes.js";
import type { TalkHandoffTurnResult } from "../talk-handoff.js";
/** Resolve the Talk session mode, defaulting managed-room transports to stt-tts. */
export function normalizeTalkSessionMode(params: { mode?: string; transport?: string }): TalkMode {
return (
(normalizeOptionalLowercaseString(params.mode) as TalkMode | undefined) ??
(normalizeOptionalLowercaseString(params.transport) === "managed-room" ? "stt-tts" : "realtime")
);
}
/** Resolve the Talk session transport from mode when the client omits it. */
export function normalizeTalkSessionTransport(params: {
mode: TalkMode;
transport?: string;
}): TalkTransport {
const transport = normalizeOptionalLowercaseString(params.transport) as TalkTransport | undefined;
if (transport) {
return transport;
}
return params.mode === "stt-tts" ? "managed-room" : "gateway-relay";
}
/** Resolve the Talk session brain, defaulting transcription sessions to none. */
export function normalizeTalkSessionBrain(params: { mode: TalkMode; brain?: string }): TalkBrain {
const brain = normalizeOptionalLowercaseString(params.brain) as TalkBrain | undefined;
if (brain) {
return brain;
}
return params.mode === "transcription" ? "none" : "agent-consult";
}
export async function resolveTalkRealtimeProviderInstructions(params: {
config: OpenClawConfig;
configuredInstructions?: string;
sessionKey?: unknown;
/** Relay sessions bind their agent lazily; injecting a guessed profile would mix agents. */
requireSessionKeyForProfile?: boolean;
warn: (message: string) => void;
}): Promise<{ agentId: string; instructions: string; requestedSessionKey?: string }> {
const requestedSessionKey = normalizeOptionalString(params.sessionKey);
// Older clients can prefetch without a key. Client-owned creates bind to the
// default agent immediately, so its workspace profile stays consistent there.
const agentId = requestedSessionKey
? resolveAgentIdFromSessionKey(requestedSessionKey)
: resolveDefaultAgentId(params.config);
const bootstrapContext =
params.requireSessionKeyForProfile && !requestedSessionKey
? undefined
: await resolveRealtimeBootstrapContextInstructions({
agentId,
config: params.config,
sessionKey: requestedSessionKey,
warn: params.warn,
});
return {
agentId,
instructions: [params.configuredInstructions, bootstrapContext]
.filter((entry): entry is string => Boolean(entry?.trim()))
.join("\n\n"),
...(requestedSessionKey ? { requestedSessionKey } : {}),
};
}
export function canUseTalkDirectTools(client: { connect?: { scopes?: string[] } } | null): boolean {
const scopes = Array.isArray(client?.connect?.scopes) ? client.connect.scopes : [];
return scopes.includes(ADMIN_SCOPE);
+146 -1
View File
@@ -36,6 +36,8 @@ const mocks = vi.hoisted(() => ({
cancelTalkRealtimeRelayTurn: vi.fn(),
stopTalkRealtimeRelaySession: vi.fn(),
registerTalkRealtimeRelayAgentRun: vi.fn(),
flushTalkRealtimeRelayVoiceWrites: vi.fn(async () => undefined),
ensureTalkRealtimeRelayVoiceSession: vi.fn(),
submitTalkRealtimeRelayToolResult: vi.fn(),
createTalkTranscriptionRelaySession: vi.fn(),
sendTalkTranscriptionRelayAudio: vi.fn(),
@@ -45,6 +47,15 @@ const mocks = vi.hoisted(() => ({
controlRealtimeVoiceAgentRun: vi.fn(),
steerTalkRealtimeRelayAgentRun: vi.fn(),
resolveSessionKeyFromResolveParams: vi.fn(),
resolveRealtimeBootstrapContextInstructions: vi.fn(
async (): Promise<string | undefined> => undefined,
),
closeStaleClientVoiceSessions: vi.fn(async () => 0),
createOrResumeClientVoiceSession: vi.fn(() => "voice-test"),
ensureClientVoiceAgentSessionEntry: vi.fn(async () => undefined),
assertClientVoiceSessionOpen: vi.fn(),
registerClientVoiceConsultRun: vi.fn(),
resolveOpenClientVoiceSessionId: vi.fn(),
}));
vi.mock("../../config/config.js", () => ({
@@ -82,6 +93,23 @@ vi.mock("../../talk/agent-run-control.js", () => ({
controlRealtimeVoiceAgentRun: mocks.controlRealtimeVoiceAgentRun,
}));
vi.mock("../../agents/realtime-bootstrap-context.js", () => ({
resolveRealtimeBootstrapContextInstructions: mocks.resolveRealtimeBootstrapContextInstructions,
}));
vi.mock("../../talk/client-voice-session.js", async (importOriginal) => {
const actual = await importOriginal<typeof import("../../talk/client-voice-session.js")>();
return {
...actual,
assertClientVoiceSessionOpen: mocks.assertClientVoiceSessionOpen,
closeStaleClientVoiceSessions: mocks.closeStaleClientVoiceSessions,
createOrResumeClientVoiceSession: mocks.createOrResumeClientVoiceSession,
ensureClientVoiceAgentSessionEntry: mocks.ensureClientVoiceAgentSessionEntry,
registerClientVoiceConsultRun: mocks.registerClientVoiceConsultRun,
resolveOpenClientVoiceSessionId: mocks.resolveOpenClientVoiceSessionId,
};
});
vi.mock("./chat.js", () => ({
chatHandlers: {
"chat.send": mocks.chatSend,
@@ -99,6 +127,8 @@ vi.mock("../talk-realtime-relay.js", async (importOriginal) => {
acknowledgeTalkRealtimeRelayMark: mocks.acknowledgeTalkRealtimeRelayMark,
cancelTalkRealtimeRelayTurn: mocks.cancelTalkRealtimeRelayTurn,
createTalkRealtimeRelaySession: mocks.createTalkRealtimeRelaySession,
ensureTalkRealtimeRelayVoiceSession: mocks.ensureTalkRealtimeRelayVoiceSession,
flushTalkRealtimeRelayVoiceWrites: mocks.flushTalkRealtimeRelayVoiceWrites,
registerTalkRealtimeRelayAgentRun: mocks.registerTalkRealtimeRelayAgentRun,
sendTalkRealtimeRelayAudio: mocks.sendTalkRealtimeRelayAudio,
steerTalkRealtimeRelayAgentRun: mocks.steerTalkRealtimeRelayAgentRun,
@@ -1484,6 +1514,7 @@ describe("talk.session unified handlers", () => {
)({
req: { type: "req", id: "1", method: "talk.session.create" },
params: {
sessionKey: "agent:main:main",
mode: "realtime",
transport: "gateway-relay",
brain: "agent-consult",
@@ -1520,6 +1551,10 @@ describe("talk.session unified handlers", () => {
providerConfigs: { openai: { apiKey: "openai-key" } },
defaultModel: "gpt-realtime-default",
});
expect(mocks.ensureClientVoiceAgentSessionEntry).toHaveBeenCalledWith({
agentId: "main",
sessionKey: "agent:main:main",
});
const relayCreateInput = mockCallArg(mocks.createTalkRealtimeRelaySession) as Record<
string,
unknown
@@ -2418,6 +2453,96 @@ describe("talk.client.toolCall handler", () => {
);
});
it("implicitly creates a voice session for consults without a binding", async () => {
const respond = vi.fn();
await expectDefined(
talkHandlers["talk.client.toolCall"],
'talkHandlers["talk.client.toolCall"] test invariant',
)({
req: { type: "req", id: "1", method: "talk.client.toolCall" },
params: {
sessionKey: "main",
callId: "call-unbound",
name: "openclaw_agent_consult",
args: { question: "Do something" },
},
client: { connId: "conn-1" } as never,
isWebchatConnect: () => false,
respond: respond as never,
context: { getRuntimeConfig: () => ({}) as OpenClawConfig } as never,
});
expect(mocks.createOrResumeClientVoiceSession).toHaveBeenCalledWith({
agentId: "main",
sessionKey: "main",
origin: "client",
});
expect(mocks.registerClientVoiceConsultRun).toHaveBeenCalledWith(
expect.objectContaining({ voiceSessionId: "voice-test", runId: "run-voice-1" }),
);
expect(mocks.chatSend).toHaveBeenCalledTimes(1);
expect(respond).toHaveBeenCalledWith(true, expect.anything(), undefined);
});
it("resolves a legacy consult to the open client voice record", async () => {
mocks.resolveOpenClientVoiceSessionId.mockReturnValueOnce("voice-test");
const respond = vi.fn();
await expectDefined(
talkHandlers["talk.client.toolCall"],
'talkHandlers["talk.client.toolCall"] test invariant',
)({
req: { type: "req", id: "legacy", method: "talk.client.toolCall" },
params: {
sessionKey: "main",
callId: "call-legacy",
name: "openclaw_agent_consult",
args: { question: "Continue the call" },
},
client: { connId: "conn-legacy" } as never,
isWebchatConnect: () => false,
respond: respond as never,
context: { getRuntimeConfig: () => ({}) as OpenClawConfig } as never,
});
expect(mocks.assertClientVoiceSessionOpen).toHaveBeenCalledWith({
agentId: "main",
sessionKey: "main",
voiceSessionId: "voice-test",
});
expectRespondOk(respond, { runId: "run-voice-1" });
});
it("requires relay connection ownership for relay-origin voice records", async () => {
mocks.assertClientVoiceSessionOpen.mockReturnValueOnce("relay");
const respond = vi.fn();
await expectDefined(
talkHandlers["talk.client.toolCall"],
'talkHandlers["talk.client.toolCall"] test invariant',
)({
req: { type: "req", id: "relay-owner", method: "talk.client.toolCall" },
params: {
sessionKey: "main",
voiceSessionId: "relay-secret",
callId: "call-relay-owner",
name: "openclaw_agent_consult",
args: { question: "Continue" },
},
client: { connId: "other-conn" } as never,
isWebchatConnect: () => false,
respond: respond as never,
context: { getRuntimeConfig: () => ({}) as OpenClawConfig } as never,
});
expect(mocks.chatSend).not.toHaveBeenCalled();
expectRespondError(respond, {
code: ErrorCodes.INVALID_REQUEST,
message: "Error: relay-owned voice sessions require relaySessionId and connection ownership",
});
});
it("starts agent consult through gateway policy instead of exposing chat.send to browser clients", async () => {
const respond = vi.fn();
@@ -2428,6 +2553,7 @@ describe("talk.client.toolCall handler", () => {
req: { type: "req", id: "1", method: "talk.client.toolCall" },
params: {
sessionKey: "main",
voiceSessionId: "voice-test",
callId: "call-1",
name: "openclaw_agent_consult",
args: { question: "What is in this repo?", responseStyle: "one sentence" },
@@ -2470,6 +2596,7 @@ describe("talk.client.toolCall handler", () => {
req: { type: "req", id: "1", method: "talk.client.toolCall" },
params: {
sessionKey: "main",
voiceSessionId: "voice-test",
callId: "call-active",
name: "openclaw_agent_consult",
args: { question: "What is running?" },
@@ -2497,6 +2624,7 @@ describe("talk.client.toolCall handler", () => {
req: { type: "req", id: "1", method: "talk.client.toolCall" },
params: {
sessionKey: "main",
voiceSessionId: "voice-test",
callId: "call-1",
name: "openclaw_agent_consult",
args: { question: "Are the basement lights off?" },
@@ -2533,6 +2661,7 @@ describe("talk.client.toolCall handler", () => {
req: { type: "req", id: "1", method: "talk.client.toolCall" },
params: {
sessionKey: "main",
voiceSessionId: "relay-1",
relaySessionId: "relay-1",
callId: "call-1",
name: "openclaw_agent_consult",
@@ -2581,6 +2710,7 @@ describe("talk.client.toolCall handler", () => {
req: { type: "req", id: "1", method: "talk.client.toolCall" },
params: {
sessionKey: "main",
voiceSessionId: "relay-1",
relaySessionId: "relay-1",
callId: "call-1",
name: "openclaw_agent_consult",
@@ -2749,6 +2879,8 @@ describe("talk.client.steer handler", () => {
describe("talk.client.create handler", () => {
beforeEach(() => {
vi.clearAllMocks();
mocks.resolveRealtimeBootstrapContextInstructions.mockResolvedValue(undefined);
mocks.createOrResumeClientVoiceSession.mockReturnValue("voice-test");
});
it("builds realtime launch defaults from talk.realtime", () => {
@@ -2772,6 +2904,7 @@ describe("talk.client.create handler", () => {
});
it("uses talk.realtime provider, model, voice, and instructions without reading speech provider config", async () => {
mocks.resolveRealtimeBootstrapContextInstructions.mockResolvedValue("Bounded profile context.");
const createBrowserSession = vi.fn(async (_input: unknown) => ({
provider: "openai",
transport: "webrtc" as const,
@@ -2838,6 +2971,7 @@ describe("talk.client.create handler", () => {
reasoningEffort: "low",
});
expect(createInput.instructions).toContain("Additional realtime instructions:\nSpeak warmly.");
expect(createInput.instructions).toContain("Bounded profile context.");
expect(createInput.instructions).toContain("tool-backed actions");
expect(createInput.instructions).toContain("Let me check that for you");
expect(createInput.tools).not.toContainEqual(
@@ -2846,7 +2980,18 @@ describe("talk.client.create handler", () => {
expect(createInput).not.toHaveProperty("provider");
expect(createInput).not.toHaveProperty("providers");
expect(createInput).not.toHaveProperty("transport");
expectRespondOk(respond, { provider: "openai", transport: "webrtc" });
expect(mocks.ensureClientVoiceAgentSessionEntry).toHaveBeenCalledWith({
agentId: "main",
sessionKey: "main",
});
expect(mocks.createOrResumeClientVoiceSession).toHaveBeenCalledWith(
expect.objectContaining({ provider: "openai" }),
);
expectRespondOk(respond, {
provider: "openai",
transport: "webrtc",
voiceSessionId: "voice-test",
});
});
it("adds describe_view to camera clients whose provider supports video frames", async () => {
+35 -15
View File
@@ -10,6 +10,7 @@ import {
} from "../../packages/gateway-protocol/src/index.js";
import { normalizeTalkSection } from "../config/talk.js";
import { buildRealtimeVoiceAgentConsultChatMessage } from "../talk/agent-consult-tool.js";
import { abortChatRunById } from "./chat-abort.js";
import { chatHandlers } from "./server-methods/chat.js";
import type {
GatewayClient,
@@ -66,6 +67,7 @@ export async function startTalkRealtimeAgentConsult(params: {
args: unknown;
relaySessionId?: string;
connId?: string;
onRunStarted?: (runId: string) => void;
}): Promise<
{ ok: true; runId: string; idempotencyKey: string } | { ok: false; error: ErrorShape }
> {
@@ -77,6 +79,7 @@ export async function startTalkRealtimeAgentConsult(params: {
}
const idempotencyKey = `talk-${params.callId}-${randomUUID()}`;
const normalizedTalk = normalizeTalkSection(params.context.getRuntimeConfig().talk);
let acknowledgedRunId: string | undefined;
const chatResponse = await new Promise<
{ ok: true; result: unknown } | { ok: false; error: ErrorShape } | undefined
>((resolve) => {
@@ -106,6 +109,37 @@ export async function startTalkRealtimeAgentConsult(params: {
},
respond: (ok: boolean, result?: unknown, error?: ErrorShape) => {
acknowledged = true;
if (ok && !terminalTalkChatSendAckError(normalizeTalkChatSendAckStatus(result))) {
const candidateRunId =
result && typeof result === "object" && !Array.isArray(result)
? (result as Record<string, unknown>).runId
: undefined;
const runId = typeof candidateRunId === "string" ? candidateRunId : idempotencyKey;
try {
if (params.relaySessionId && params.connId) {
registerTalkRealtimeRelayAgentRun({
relaySessionId: params.relaySessionId,
connId: params.connId,
sessionKey: params.sessionKey,
runId,
callId: params.callId,
});
}
params.onRunStarted?.(runId);
acknowledgedRunId = runId;
} catch (registrationError) {
abortChatRunById(params.context, {
runId,
sessionKey: params.sessionKey,
stopReason: "voice session binding failed",
});
resolve({
ok: false,
error: errorShape(ErrorCodes.UNAVAILABLE, formatForLog(registrationError)),
});
return;
}
}
resolve(
ok
? { ok: true, result }
@@ -152,20 +186,6 @@ export async function startTalkRealtimeAgentConsult(params: {
if (terminalAckError) {
return { ok: false, error: terminalAckError };
}
const runId =
result && typeof result === "object" && !Array.isArray(result)
? typeof (result as Record<string, unknown>).runId === "string"
? (result as Record<string, string>).runId
: idempotencyKey
: idempotencyKey;
if (params.relaySessionId && params.connId) {
registerTalkRealtimeRelayAgentRun({
relaySessionId: params.relaySessionId,
connId: params.connId,
sessionKey: params.sessionKey,
runId: expectDefined(runId, "talk agent run id"),
callId: params.callId,
});
}
const runId = expectDefined(acknowledgedRunId, "talk agent run id");
return { ok: true, runId: expectDefined(runId, "talk agent run id"), idempotencyKey };
}
+202
View File
@@ -1,18 +1,30 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
/**
* Tests talk realtime relay event forwarding and connection cleanup.
*/
import { afterEach, describe, expect, it, vi } from "vitest";
import { setActiveEmbeddedRun } from "../agents/embedded-agent-runner/runs.js";
import { testing as embeddedRunTesting } from "../agents/embedded-agent-runner/runs.test-support.js";
import {
readSessionTranscriptMessageEvents,
replaceSessionEntry,
} from "../config/sessions/session-accessor.js";
import type { RealtimeVoiceProviderPlugin } from "../plugins/types.js";
import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js";
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
import { clientVoiceSessionTesting } from "../talk/client-voice-session.test-support.js";
import type {
RealtimeVoiceBridge,
RealtimeVoiceBridgeCreateRequest,
} from "../talk/provider-types.js";
import { captureEnv, setTestEnvValue } from "../test-utils/env.js";
import {
acknowledgeTalkRealtimeRelayMark,
cancelTalkRealtimeRelayTurn,
createTalkRealtimeRelaySession as createTalkRealtimeRelaySessionRaw,
flushTalkRealtimeRelayVoiceWrites,
registerTalkRealtimeRelayAgentRun,
sendTalkRealtimeRelayAudio,
steerTalkRealtimeRelayAgentRun,
@@ -74,6 +86,196 @@ describe("talk realtime gateway relay", () => {
};
}
it("appends finalized relay transcripts to the canonical agent session", async () => {
const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
const tempDir = await fs.realpath(
await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-relay-voice-")),
);
setTestEnvValue("OPENCLAW_STATE_DIR", tempDir);
let bridgeRequest: RealtimeVoiceBridgeCreateRequest | undefined;
try {
await replaceSessionEntry(
{ agentId: "main", sessionKey: "agent:main:main" },
{ sessionId: "relay-voice-session", updatedAt: Date.now() },
);
const provider = createIdleRelayProvider();
provider.createBridge = (request) => {
bridgeRequest = request;
return createIdleRelayProvider().createBridge?.(request) as RealtimeVoiceBridge;
};
const session = createTalkRealtimeRelaySession({
context: {
broadcastToConnIds: vi.fn(),
chatAbortControllers: new Map(),
getRuntimeConfig: () => ({}),
logGateway: { warn: vi.fn() },
} as never,
connId: "conn-voice",
cfg: {},
provider,
providerConfig: {},
instructions: "brief",
tools: [],
sessionKey: "agent:main:main",
});
bridgeRequest?.onTranscript?.("user", "relay hello", true);
bridgeRequest?.onTranscript?.("assistant", "relay response", true);
await flushTalkRealtimeRelayVoiceWrites({
relaySessionId: session.relaySessionId,
connId: "conn-voice",
});
const events = readSessionTranscriptMessageEvents({
agentId: "main",
sessionId: "relay-voice-session",
});
expect(events).toHaveLength(2);
expect(events[0]?.event).toMatchObject({
id: `voice:${session.relaySessionId}:1`,
message: { role: "user", content: [{ type: "text", text: "relay hello" }] },
});
expect(events[1]?.event).toMatchObject({
id: `voice:${session.relaySessionId}:2`,
message: {
role: "assistant",
content: [{ type: "text", text: "relay response" }],
api: "realtime",
provider: "relay-test",
model: "realtime-voice",
},
});
stopTalkRealtimeRelaySession({
relaySessionId: session.relaySessionId,
connId: "conn-voice",
});
await vi.waitFor(() =>
expect(clientVoiceSessionTesting.readRecord("main", session.relaySessionId)?.status).toBe(
"closed",
),
);
} finally {
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
envSnapshot.restore();
await fs.rm(tempDir, { recursive: true, force: true });
}
});
it("creates the relay voice record before binding a transcript-free consult", async () => {
const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
const tempDir = await fs.realpath(
await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-relay-voice-consult-")),
);
setTestEnvValue("OPENCLAW_STATE_DIR", tempDir);
try {
const session = createTalkRealtimeRelaySession({
context: {
broadcastToConnIds: vi.fn(),
chatAbortControllers: new Map(),
getRuntimeConfig: () => ({}),
logGateway: { warn: vi.fn() },
} as never,
connId: "conn-consult",
cfg: {},
provider: createIdleRelayProvider(),
providerConfig: {},
instructions: "brief",
tools: [],
sessionKey: "agent:main:main",
});
expect(clientVoiceSessionTesting.readRecord("main", session.relaySessionId)).toBeUndefined();
registerTalkRealtimeRelayAgentRun({
relaySessionId: session.relaySessionId,
connId: "conn-consult",
sessionKey: "agent:main:main",
runId: "run-before-transcript",
});
expect(clientVoiceSessionTesting.readRecord("main", session.relaySessionId)).toMatchObject({
status: "open",
consultRunIds: ["run-before-transcript"],
});
stopTalkRealtimeRelaySession({
relaySessionId: session.relaySessionId,
connId: "conn-consult",
});
await vi.waitFor(() =>
expect(clientVoiceSessionTesting.readRecord("main", session.relaySessionId)?.status).toBe(
"closed",
),
);
} finally {
clientVoiceSessionTesting.reset();
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
envSnapshot.restore();
await fs.rm(tempDir, { recursive: true, force: true });
}
});
it("logs relay transcript append failures", async () => {
const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
const tempDir = await fs.realpath(
await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-relay-voice-failure-")),
);
setTestEnvValue("OPENCLAW_STATE_DIR", tempDir);
let bridgeRequest: RealtimeVoiceBridgeCreateRequest | undefined;
const warn = vi.fn();
try {
vi.useFakeTimers();
const provider = createIdleRelayProvider();
provider.createBridge = (request) => {
bridgeRequest = request;
return createIdleRelayProvider().createBridge?.(request) as RealtimeVoiceBridge;
};
const session = createTalkRealtimeRelaySession({
context: {
broadcastToConnIds: vi.fn(),
getRuntimeConfig: () => ({}),
logGateway: { warn },
} as never,
connId: "conn-voice-failure",
cfg: {},
provider,
providerConfig: {},
instructions: "brief",
tools: [],
sessionKey: "agent:main:missing",
});
bridgeRequest?.onTranscript?.("user", "cannot persist", true);
await vi.advanceTimersByTimeAsync(0);
expect(warn).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(500);
expect(warn).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(1_999);
expect(warn).not.toHaveBeenCalled();
await vi.advanceTimersByTimeAsync(1);
await flushTalkRealtimeRelayVoiceWrites({
relaySessionId: session.relaySessionId,
connId: "conn-voice-failure",
});
expect(warn).toHaveBeenCalledExactlyOnceWith(
expect.stringContaining("realtime relay transcript append failed"),
);
stopTalkRealtimeRelaySession({
relaySessionId: session.relaySessionId,
connId: "conn-voice-failure",
});
await vi.waitFor(() =>
expect(clientVoiceSessionTesting.readRecord("main", session.relaySessionId)?.status).toBe(
"closed",
),
);
} finally {
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
envSnapshot.restore();
await fs.rm(tempDir, { recursive: true, force: true });
}
});
function createDeferredVoid(): { promise: Promise<void>; resolve: () => void } {
let resolve!: () => void;
const promise = new Promise<void>((accept) => {
+162
View File
@@ -5,6 +5,7 @@ import { resolveExpiresAtMsFromDurationMs } from "@openclaw/normalization-core/n
import type { OpenClawConfig } from "../config/types.js";
import { formatErrorMessage } from "../infra/errors.js";
import type { RealtimeVoiceProviderPlugin } from "../plugins/types.js";
import { resolveAgentIdFromSessionKey } from "../routing/session-key.js";
import {
REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
buildRealtimeVoiceAgentConsultWorkingResponse,
@@ -16,6 +17,12 @@ import {
shouldAutoControlRealtimeVoiceAgentText,
type RealtimeVoiceAgentControlResult,
} from "../talk/agent-run-control.js";
import {
appendRelayVoiceTranscript,
closeClientVoiceSession,
createOrResumeClientVoiceSession,
registerClientVoiceConsultRun,
} from "../talk/client-voice-session.js";
import { readSpeakableRealtimeVoiceToolResult } from "../talk/consult-question.js";
import {
createRealtimeVoiceForcedConsultCoordinator,
@@ -134,6 +141,7 @@ type RelaySession = {
expiresAtMs: number;
cleanupTimer: ReturnType<typeof setTimeout>;
activeAgentRuns: Map<string, string>;
provider: string;
activeAgentToolCalls: Map<string, string>;
completedAgentToolCalls: Set<string>;
// Cancelled calls retain their original turn long enough to terminally satisfy
@@ -154,6 +162,11 @@ type RelaySession = {
toolResultEpoch: number;
forcedConsults: RealtimeVoiceForcedConsultCoordinator;
transcript: RealtimeVoiceTranscriptEntry[];
voiceConfig?: OpenClawConfig;
voiceSessionCreated: boolean;
voiceTranscriptSeq: number;
voiceTranscriptWrites: Promise<void>;
pendingVoiceTranscripts: Array<{ role: "user" | "assistant"; text: string }>;
};
type CreateTalkRealtimeRelaySessionParams = {
@@ -182,6 +195,128 @@ type TalkRealtimeRelaySessionResult = {
};
const relaySessions = new Map<string, RelaySession>();
const RELAY_TRANSCRIPT_RETRY_DELAYS_MS = [0, 500, 2_000] as const;
function logRelayVoiceFailure(session: RelaySession, message: string, error: unknown): void {
session.context.logGateway?.warn(`${message}: ${formatErrorMessage(error)}`);
}
function ensureRelayVoiceSession(session: RelaySession): boolean {
if (session.voiceSessionCreated) {
return true;
}
if (!session.sessionKey) {
return false;
}
try {
createOrResumeClientVoiceSession({
agentId: resolveAgentIdFromSessionKey(session.sessionKey),
sessionKey: session.sessionKey,
provider: session.provider,
origin: "relay",
voiceSessionId: session.id,
});
session.voiceSessionCreated = true;
return true;
} catch (error) {
logRelayVoiceFailure(session, "realtime relay voice session create failed", error);
return false;
}
}
const MAX_PENDING_VOICE_TRANSCRIPTS = 40;
function enqueueRelayVoiceTranscript(
session: RelaySession,
role: "user" | "assistant",
text: string,
): void {
if (!session.sessionKey) {
// Lazy-bound relays hear audio before talk.client.toolCall supplies the session
// key; buffer bounded finals so the call's opening turns survive the binding.
// Never-binding callers accept best-effort loss: they had no persistence before.
session.pendingVoiceTranscripts.push({ role, text });
if (session.pendingVoiceTranscripts.length > MAX_PENDING_VOICE_TRANSCRIPTS) {
session.pendingVoiceTranscripts.shift();
}
return;
}
if (!ensureRelayVoiceSession(session)) {
return;
}
session.voiceTranscriptSeq += 1;
const entryId = String(session.voiceTranscriptSeq);
const sessionKey = session.sessionKey;
session.voiceTranscriptWrites = session.voiceTranscriptWrites
.then(async () => {
let lastError: unknown;
for (const delayMs of RELAY_TRANSCRIPT_RETRY_DELAYS_MS) {
if (delayMs > 0) {
await new Promise<void>((resolve) => {
setTimeout(resolve, delayMs);
});
}
try {
await appendRelayVoiceTranscript({
agentId: resolveAgentIdFromSessionKey(sessionKey),
sessionKey,
voiceSessionId: session.id,
entryId,
role,
text,
...(session.voiceConfig ? { config: session.voiceConfig } : {}),
});
return;
} catch (error) {
lastError = error;
}
}
throw lastError;
})
.catch((error: unknown) => {
logRelayVoiceFailure(session, "realtime relay transcript append failed", error);
});
}
function closeRelayVoiceSession(session: RelaySession): void {
if (!session.sessionKey || !ensureRelayVoiceSession(session)) {
return;
}
const sessionKey = session.sessionKey;
void session.voiceTranscriptWrites
.then(async () => {
const config = session.voiceConfig ?? session.context.getRuntimeConfig();
await closeClientVoiceSession({
agentId: resolveAgentIdFromSessionKey(sessionKey),
sessionKey,
voiceSessionId: session.id,
config,
});
})
.catch((error: unknown) => {
logRelayVoiceFailure(session, "realtime relay voice session close failed", error);
});
}
/** Ensure a gateway-relay call has its durable record before transcript-free RPCs. */
export function ensureTalkRealtimeRelayVoiceSession(params: {
relaySessionId: string;
connId: string;
sessionKey: string;
}): void {
const session = getRelaySession(params.relaySessionId, params.connId);
if (session.sessionKey && session.sessionKey !== params.sessionKey) {
throw new Error("Realtime relay session belongs to another agent session");
}
session.sessionKey = params.sessionKey;
if (!ensureRelayVoiceSession(session)) {
throw new Error("Realtime relay voice session could not be created");
}
const buffered = session.pendingVoiceTranscripts.splice(0);
for (const entry of buffered) {
enqueueRelayVoiceTranscript(session, entry.role, entry.text);
}
}
function isWorkingToolResult(result: unknown): boolean {
return (
@@ -538,6 +673,7 @@ function closeRelaySession(session: RelaySession, reason: "completed" | "error")
clearTimeout(session.cleanupTimer);
abortRelayAgentRuns(session, reason === "error" ? "relay-error" : "relay-closed");
session.bridge.close();
closeRelayVoiceSession(session);
broadcastToOwner(session.context, session.connId, {
relaySessionId: session.id,
type: "close",
@@ -708,6 +844,7 @@ export function createTalkRealtimeRelaySession(
const turnId = relay ? ensureRelayTurn(relay) : undefined;
if (final && relay) {
recordRealtimeVoiceTranscript(relay.transcript, role, text);
enqueueRelayVoiceTranscript(relay, role, text);
}
const eventType =
role === "assistant"
@@ -845,6 +982,7 @@ export function createTalkRealtimeRelaySession(
forgetUnifiedTalkSession(relaySessionId);
clearTimeout(active.cleanupTimer);
abortRelayAgentRuns(active, "relay-closed");
closeRelayVoiceSession(active);
if (!ready && !failureEmitted) {
const issue = realtimeRelayIssue({
message: "Realtime provider closed before the session became ready.",
@@ -879,6 +1017,7 @@ export function createTalkRealtimeRelaySession(
}
}, RELAY_SESSION_TTL_MS),
activeAgentRuns: new Map(),
provider: params.provider.id,
activeAgentToolCalls: new Map(),
completedAgentToolCalls: new Set(),
cancelledAgentToolCalls: new Map(),
@@ -890,6 +1029,11 @@ export function createTalkRealtimeRelaySession(
toolResultEpoch: 0,
forcedConsults: createRealtimeVoiceForcedConsultCoordinator(),
transcript: [],
...(params.cfg ? { voiceConfig: params.cfg } : {}),
voiceSessionCreated: false,
voiceTranscriptSeq: 0,
voiceTranscriptWrites: Promise.resolve(),
pendingVoiceTranscripts: [],
};
relayRef.current = relay;
relay.cleanupTimer.unref?.();
@@ -1352,6 +1496,24 @@ export function registerTalkRealtimeRelayAgentRun(params: {
if (!session.sessionKey) {
session.sessionKey = params.sessionKey;
}
if (!ensureRelayVoiceSession(session)) {
throw new Error("Realtime relay voice session could not be created for agent consult");
}
registerClientVoiceConsultRun({
agentId: resolveAgentIdFromSessionKey(params.sessionKey),
sessionKey: params.sessionKey,
voiceSessionId: session.id,
runId: params.runId,
});
}
/** Wait for server-owned final transcript appends before a relay consult is authorized. */
export async function flushTalkRealtimeRelayVoiceWrites(params: {
relaySessionId: string;
connId: string;
}): Promise<void> {
const session = getRelaySession(params.relaySessionId, params.connId);
await session.voiceTranscriptWrites;
}
/** Applies realtime voice-control text to the active agent-consult chat run. */
+2
View File
@@ -482,6 +482,8 @@ type DiagnosticToolExecutionBaseEvent = DiagnosticBaseEvent & {
toolOwner?: string;
toolCallId?: string;
paramsSummary?: DiagnosticToolParamsSummary;
/** Deterministic mutation classification computed before tool execution. */
mutatingAction?: boolean;
};
export type DiagnosticToolExecutionStartedEvent = DiagnosticToolExecutionBaseEvent & {
+14
View File
@@ -30,6 +30,20 @@ describe("realtime voice agent consult tool", () => {
);
});
it("normalizes a server-issued spoken confirmation id", () => {
expect(
parseRealtimeVoiceAgentConsultArgs({
question: "Send it now",
confirmationId: " confirm-123 ",
}),
).toStrictEqual({
question: "Send it now",
context: undefined,
responseStyle: undefined,
confirmationId: "confirm-123",
});
});
it("accepts provider question aliases from realtime tool calls", () => {
expect(parseRealtimeVoiceAgentConsultArgs({ prompt: " Check the repo. " })).toStrictEqual({
context: undefined,
+12 -2
View File
@@ -26,6 +26,7 @@ export type RealtimeVoiceAgentConsultArgs = {
question: string;
context?: string;
responseStyle?: string;
confirmationId?: string;
};
/** Compact transcript entry included in delegated agent prompts. */
export type RealtimeVoiceAgentConsultTranscriptEntry = {
@@ -54,6 +55,11 @@ export const REALTIME_VOICE_AGENT_CONSULT_TOOL: RealtimeVoiceTool = {
type: "string",
description: "Optional style hint for the spoken answer.",
},
confirmationId: {
type: "string",
description:
"Server-issued confirmation id from a prior VOICE_CONFIRMATION_REQUIRED result, supplied only after the user explicitly confirms aloud.",
},
},
required: ["question"],
},
@@ -177,10 +183,14 @@ export function parseRealtimeVoiceAgentConsultArgs(args: unknown): RealtimeVoice
if (!question) {
throw new Error("question required");
}
const context = readConsultStringArg(args, "context");
const responseStyle = readConsultStringArg(args, "responseStyle");
const confirmationId = readConsultStringArg(args, "confirmationId");
return {
question,
context: readConsultStringArg(args, "context"),
responseStyle: readConsultStringArg(args, "responseStyle"),
context,
responseStyle,
...(confirmationId ? { confirmationId } : {}),
};
}
@@ -0,0 +1,15 @@
import "./client-voice-confirmation.js";
type ClientVoiceConfirmationTestApi = {
resetClientVoiceConfirmationStateForTest(): void;
};
function getTestApi(): ClientVoiceConfirmationTestApi {
return (globalThis as Record<PropertyKey, unknown>)[
Symbol.for("openclaw.clientVoiceConfirmationTestApi")
] as ClientVoiceConfirmationTestApi;
}
export function resetClientVoiceConfirmationStateForTest(): void {
getTestApi().resetClientVoiceConfirmationStateForTest();
}
+501
View File
@@ -0,0 +1,501 @@
import { afterEach, describe, expect, it } from "vitest";
import {
authorizeClientVoiceConfirmation as authorizeClientVoiceConfirmationForTest,
bindAuthorizedClientVoiceConfirmation,
deactivateClientVoiceConfirmationSession,
noteClientVoiceConfirmationUtterance as noteClientVoiceConfirmationUtteranceForTest,
releaseClientVoiceConfirmationRun,
resolveClientVoiceToolConfirmationPolicy as resolveClientVoiceToolConfirmationPolicyForTest,
} from "./client-voice-confirmation.js";
import { resetClientVoiceConfirmationStateForTest } from "./client-voice-confirmation.test-support.js";
function authorizeClientVoiceConfirmation(
params: Omit<Parameters<typeof authorizeClientVoiceConfirmationForTest>[0], "agentId">,
) {
return authorizeClientVoiceConfirmationForTest({ agentId: "main", ...params });
}
function noteClientVoiceConfirmationUtterance(
params: Omit<Parameters<typeof noteClientVoiceConfirmationUtteranceForTest>[0], "agentId">,
): void {
noteClientVoiceConfirmationUtteranceForTest({ agentId: "main", ...params });
}
function resolveClientVoiceToolConfirmationPolicy(
params: Omit<Parameters<typeof resolveClientVoiceToolConfirmationPolicyForTest>[0], "agentId">,
) {
return resolveClientVoiceToolConfirmationPolicyForTest({ agentId: "main", ...params });
}
function confirmationIdFrom(reason: string): string {
const match = reason.match(/VOICE_CONFIRMATION_REQUIRED:([^\s]+)/);
if (!match?.[1]) {
throw new Error(`missing confirmation id: ${reason}`);
}
return match[1];
}
function block(params: {
voiceSessionId: string;
runId?: string;
toolName?: string;
toolParams?: unknown;
now?: number;
}) {
const result = resolveClientVoiceToolConfirmationPolicy({
voiceSessionId: params.voiceSessionId,
runId: params.runId,
toolName: params.toolName ?? "message",
toolParams: params.toolParams ?? { action: "send", message: "hello" },
now: params.now,
});
expect(result.allowed).toBe(false);
if (result.allowed) {
throw new Error("expected blocked voice action");
}
return confirmationIdFrom(result.reason);
}
describe("client voice confirmation", () => {
afterEach(() => resetClientVoiceConfirmationStateForTest());
it("does not pause a concurrent non-voice run sharing the session key", () => {
block({ voiceSessionId: "voice-1", runId: "voice-run" });
expect(
resolveClientVoiceToolConfirmationPolicy({
toolName: "message",
toolParams: { action: "send", message: "other run" },
}),
).toEqual({ allowed: true });
});
it.each([
["exec", "git clean -fdx"],
["bash", "mv a b"],
])(
"requires confirmation for an unlisted destructive shell command: %s %s",
(toolName, command) => {
expect(
resolveClientVoiceToolConfirmationPolicy({
voiceSessionId: "voice-1",
runId: "voice-run",
toolName,
toolParams: { command },
}).allowed,
).toBe(false);
},
);
it.each(["ls -la", "grep -n TODO README.md"])(
"does not require confirmation for a classified read-only shell command: %s",
(command) => {
expect(
resolveClientVoiceToolConfirmationPolicy({
voiceSessionId: "voice-1",
runId: "voice-run",
toolName: "exec",
toolParams: { command },
}),
).toEqual({ allowed: true });
},
);
it("requires confirmation before delegating work outside the voice-bound run", () => {
expect(
resolveClientVoiceToolConfirmationPolicy({
voiceSessionId: "voice-1",
runId: "voice-run",
toolName: "sessions_send",
toolParams: { sessionKey: "agent:main:child", message: "send this" },
}).allowed,
).toBe(false);
});
it("keeps pre-gate behavior for sessions that cannot report spoken approvals", () => {
expect(
resolveClientVoiceToolConfirmationPolicy({
voiceSessionId: "voice-legacy",
runId: "voice-run",
toolName: "message",
toolParams: { action: "send", to: "user", message: "hi" },
isConfirmable: () => false,
}),
).toEqual({ allowed: true });
expect(
resolveClientVoiceToolConfirmationPolicy({
voiceSessionId: "voice-modern",
runId: "voice-run",
toolName: "message",
toolParams: { action: "send", to: "user", message: "hi" },
isConfirmable: () => true,
}).allowed,
).toBe(false);
});
it("keeps workspace-local writes confirmation-free", () => {
expect(
resolveClientVoiceToolConfirmationPolicy({
voiceSessionId: "voice-1",
runId: "voice-run",
toolName: "write",
toolParams: { path: "notes.txt", content: "local change" },
}),
).toEqual({ allowed: true });
});
it("keeps a challenge authorizable until the run is bound, then consumes it", () => {
const confirmationId = block({
voiceSessionId: "voice-1",
toolParams: { action: "send", message: "A" },
now: 100,
});
noteClientVoiceConfirmationUtterance({
voiceSessionId: "voice-1",
text: "yes",
timestamp: 101,
});
// A failed/retried consult can re-authorize the same challenge before it binds.
const first = authorizeClientVoiceConfirmation({
voiceSessionId: "voice-1",
confirmationId,
now: 102,
});
expect(
authorizeClientVoiceConfirmation({ voiceSessionId: "voice-1", confirmationId, now: 103 })
.fingerprint,
).toBe(first.fingerprint);
bindAuthorizedClientVoiceConfirmation({ grant: first, runId: "run-approved" });
// After binding the run, the challenge is consumed and cannot re-authorize.
expect(() =>
authorizeClientVoiceConfirmation({ voiceSessionId: "voice-1", confirmationId, now: 104 }),
).toThrow("missing, expired, or belongs to another action");
});
it("binds approval to the exact tool fingerprint", () => {
const confirmationId = block({
voiceSessionId: "voice-1",
toolParams: { action: "send", message: "A" },
now: 100,
});
noteClientVoiceConfirmationUtterance({
voiceSessionId: "voice-1",
text: "Yes, do it.",
timestamp: 101,
});
const grant = authorizeClientVoiceConfirmation({
voiceSessionId: "voice-1",
confirmationId,
now: 102,
});
bindAuthorizedClientVoiceConfirmation({ grant, runId: "run-approved" });
expect(
resolveClientVoiceToolConfirmationPolicy({
voiceSessionId: "voice-1",
runId: "run-approved",
toolName: "message",
toolParams: { action: "send", message: "B" },
now: 103,
}).allowed,
).toBe(false);
expect(
resolveClientVoiceToolConfirmationPolicy({
voiceSessionId: "voice-1",
runId: "run-approved",
toolName: "message",
toolParams: { action: "send", message: "A" },
now: 104,
}),
).toEqual({ allowed: true });
});
it("rejects expired confirmations", () => {
const confirmationId = block({ voiceSessionId: "voice-1", now: 1_000 });
noteClientVoiceConfirmationUtterance({
voiceSessionId: "voice-1",
text: "confirm",
timestamp: 1_001,
});
expect(() =>
authorizeClientVoiceConfirmation({
voiceSessionId: "voice-1",
confirmationId,
now: 121_001,
}),
).toThrow("missing, expired");
});
it.each(["no", "don't do it", "dont do it", "do not proceed", "cancel"])(
"a refusal invalidates the pending confirmation for %j",
(text) => {
const confirmationId = block({ voiceSessionId: "voice-1", now: 100 });
noteClientVoiceConfirmationUtterance({
voiceSessionId: "voice-1",
text,
timestamp: 101,
});
expect(() =>
authorizeClientVoiceConfirmation({
voiceSessionId: "voice-1",
confirmationId,
now: 102,
}),
).toThrow("missing, expired, or belongs to another action");
},
);
it("consumes an approved fingerprint once", () => {
const toolParams = { action: "send", message: "A" };
const confirmationId = block({ voiceSessionId: "voice-1", toolParams, now: 100 });
noteClientVoiceConfirmationUtterance({
voiceSessionId: "voice-1",
text: "go ahead",
timestamp: 101,
});
const grant = authorizeClientVoiceConfirmation({
voiceSessionId: "voice-1",
confirmationId,
now: 102,
});
bindAuthorizedClientVoiceConfirmation({ grant, runId: "run-approved" });
expect(
resolveClientVoiceToolConfirmationPolicy({
voiceSessionId: "voice-1",
runId: "run-approved",
toolName: "message",
toolParams,
now: 103,
}),
).toEqual({ allowed: true });
expect(
resolveClientVoiceToolConfirmationPolicy({
voiceSessionId: "voice-1",
runId: "run-approved",
toolName: "message",
toolParams,
now: 104,
}).allowed,
).toBe(false);
});
it("consumes one spoken affirmation for only one pending action", () => {
const first = block({
voiceSessionId: "voice-1",
runId: "run-1",
toolParams: { action: "send", message: "A" },
now: 100,
});
const second = block({
voiceSessionId: "voice-1",
runId: "run-2",
toolParams: { action: "send", message: "B" },
now: 100,
});
noteClientVoiceConfirmationUtterance({
voiceSessionId: "voice-1",
text: "yes",
timestamp: 101,
});
expect(() =>
authorizeClientVoiceConfirmation({
voiceSessionId: "voice-1",
confirmationId: first,
now: 102,
}),
).toThrow("newer confirmation request supersedes");
// Binding the newer grant consumes the shared affirmation and its challenge.
const grant = authorizeClientVoiceConfirmation({
voiceSessionId: "voice-1",
confirmationId: second,
now: 102,
});
bindAuthorizedClientVoiceConfirmation({ grant, runId: "run-2" });
expect(() =>
authorizeClientVoiceConfirmation({
voiceSessionId: "voice-1",
confirmationId: first,
now: 103,
}),
).toThrow("explicit spoken confirmation");
});
it("binds an approved fingerprint to its follow-up run", () => {
const toolParams = { action: "send", message: "same" };
const confirmationId = block({
voiceSessionId: "voice-1",
runId: "run-original",
toolParams,
now: 100,
});
noteClientVoiceConfirmationUtterance({
voiceSessionId: "voice-1",
text: "proceed",
timestamp: 101,
});
const grant = authorizeClientVoiceConfirmation({
voiceSessionId: "voice-1",
confirmationId,
now: 102,
});
bindAuthorizedClientVoiceConfirmation({ grant, runId: "run-approved" });
expect(
resolveClientVoiceToolConfirmationPolicy({
voiceSessionId: "voice-1",
runId: "run-other",
toolName: "message",
toolParams,
now: 103,
}).allowed,
).toBe(false);
expect(
resolveClientVoiceToolConfirmationPolicy({
voiceSessionId: "voice-1",
runId: "run-approved",
toolName: "message",
toolParams,
now: 104,
}),
).toEqual({ allowed: true });
});
it("only the newest pending challenge can be authorized", () => {
const olderId = block({
voiceSessionId: "voice-1",
runId: "run-1",
toolParams: { action: "send", message: "older" },
now: 100,
});
const newerId = block({
voiceSessionId: "voice-1",
runId: "run-1",
toolParams: { action: "send", message: "newer" },
now: 110,
});
noteClientVoiceConfirmationUtterance({
voiceSessionId: "voice-1",
text: "yes",
timestamp: 111,
});
expect(() =>
authorizeClientVoiceConfirmation({
voiceSessionId: "voice-1",
confirmationId: olderId,
now: 112,
}),
).toThrow("newer confirmation request supersedes");
expect(
authorizeClientVoiceConfirmation({
voiceSessionId: "voice-1",
confirmationId: newerId,
now: 113,
}).fingerprint,
).toBeTruthy();
});
it("invalidates a pending confirmation when the user refuses", () => {
const toolParams = { action: "send", message: "declined" };
const confirmationId = block({
voiceSessionId: "voice-1",
runId: "run-1",
toolParams,
now: 100,
});
noteClientVoiceConfirmationUtterance({
voiceSessionId: "voice-1",
text: "No, cancel that",
timestamp: 101,
});
noteClientVoiceConfirmationUtterance({
voiceSessionId: "voice-1",
text: "yes",
timestamp: 102,
});
expect(() =>
authorizeClientVoiceConfirmation({
voiceSessionId: "voice-1",
confirmationId,
now: 103,
}),
).toThrow("missing, expired, or belongs to another action");
});
it("keeps a live run's grant across call close and releases it on run completion", () => {
const toolParams = { action: "send", message: "confirmed then hangup" };
const confirmationId = block({
voiceSessionId: "voice-1",
runId: "run-original",
toolParams,
now: 100,
});
noteClientVoiceConfirmationUtterance({
voiceSessionId: "voice-1",
text: "yes do it",
timestamp: 101,
});
const grant = authorizeClientVoiceConfirmation({
voiceSessionId: "voice-1",
confirmationId,
now: 102,
});
bindAuthorizedClientVoiceConfirmation({ grant, runId: "run-live" });
deactivateClientVoiceConfirmationSession("main", "voice-1", ["run-live"]);
expect(
resolveClientVoiceToolConfirmationPolicy({
voiceSessionId: "voice-1",
runId: "run-live",
toolName: "message",
toolParams,
now: 103,
}),
).toEqual({ allowed: true });
bindAuthorizedClientVoiceConfirmation({ grant, runId: "run-done" });
releaseClientVoiceConfirmationRun("main", "voice-1", "run-done");
expect(
resolveClientVoiceToolConfirmationPolicy({
voiceSessionId: "voice-1",
runId: "run-done",
toolName: "message",
toolParams,
now: 104,
}).allowed,
).toBe(false);
});
it("isolates same-named voice sessions across agents", () => {
const blocked = resolveClientVoiceToolConfirmationPolicyForTest({
agentId: "agent-b",
voiceSessionId: "shared-id",
runId: "run-b",
toolName: "message",
toolParams: { action: "send", message: "B" },
now: 100,
});
expect(blocked.allowed).toBe(false);
if (blocked.allowed) {
throw new Error("expected confirmation request");
}
noteClientVoiceConfirmationUtteranceForTest({
agentId: "agent-a",
voiceSessionId: "shared-id",
text: "yes",
timestamp: 101,
});
expect(() =>
authorizeClientVoiceConfirmationForTest({
agentId: "agent-b",
voiceSessionId: "shared-id",
confirmationId: confirmationIdFrom(blocked.reason),
now: 102,
}),
).toThrow("explicit spoken confirmation");
});
});
+341
View File
@@ -0,0 +1,341 @@
/** In-memory spoken confirmation binding for high-impact Talk actions. */
import { createHash, randomUUID } from "node:crypto";
import { buildToolMutationState } from "../agents/tool-mutation.js";
const CONFIRMATION_TTL_MS = 2 * 60_000;
type PendingVoiceConfirmation = {
confirmationId: string;
agentId: string;
voiceSessionId: string;
runId?: string;
fingerprint: string;
toolName: string;
createdAt: number;
/** Monotonic tiebreaker: same-millisecond challenges must still have one newest. */
seq: number;
expiresAt: number;
};
type RecentVoiceUserUtterance = {
text: string;
timestamp: number;
};
export type ClientVoiceConfirmationGrant = {
agentId: string;
voiceSessionId: string;
confirmationId: string;
fingerprint: string;
expiresAt: number;
};
const pendingConfirmations = new Map<string, PendingVoiceConfirmation>();
let confirmationSeq = 0;
const approvedFingerprints = new Map<string, Map<string, Map<string, number>>>();
const recentUserUtterances = new Map<string, RecentVoiceUserUtterance>();
function confirmationScopeKey(agentId: string, voiceSessionId: string): string {
return `${agentId}\0${voiceSessionId}`;
}
function stableToolFingerprint(toolName: string, params: unknown): string {
const normalize = (value: unknown): unknown => {
if (Array.isArray(value)) {
return value.map(normalize);
}
if (!value || typeof value !== "object") {
return value;
}
return Object.fromEntries(
Object.entries(value as Record<string, unknown>)
.toSorted(([left], [right]) => left.localeCompare(right))
.map(([key, entry]) => [key, normalize(entry)]),
);
};
return createHash("sha256")
.update(`${toolName}\0${JSON.stringify(normalize(params))}`)
.digest("hex");
}
function requiresHighImpactVoiceConfirmation(toolName: string, params: unknown): boolean {
const normalizedTool = toolName.trim().toLowerCase();
if (!buildToolMutationState(normalizedTool, params).mutatingAction) {
return false;
}
if (
["message", "gateway", "nodes", "browser", "computer", "canvas", "cron", "process"].includes(
normalizedTool,
)
) {
return true;
}
// Workspace-local edits stay bound to this run. Session delegation is gated because
// delegated runs leave the voice binding and otherwise bypass spoken confirmation.
if (
["write", "edit", "apply_patch", "create_goal", "update_goal", "get_goal"].includes(
normalizedTool,
)
) {
return false;
}
return true;
}
function consumeApprovedFingerprint(
voiceSessionId: string,
runId: string | undefined,
fingerprint: string,
now: number,
): boolean {
if (!runId) {
return false;
}
const approvedByRun = approvedFingerprints.get(voiceSessionId);
const approved = approvedByRun?.get(runId);
const expiresAt = approved?.get(fingerprint);
if (!expiresAt || expiresAt < now) {
approved?.delete(fingerprint);
return false;
}
approved?.delete(fingerprint);
return true;
}
/** Record a finalized user utterance after the durable transcript append succeeds. */
export function noteClientVoiceConfirmationUtterance(params: {
agentId: string;
voiceSessionId: string;
text: string;
timestamp: number;
}): void {
recentUserUtterances.set(confirmationScopeKey(params.agentId, params.voiceSessionId), {
text: params.text,
timestamp: params.timestamp,
});
// A spoken refusal kills the outstanding challenge: a later unrelated "yes"
// must not resurrect an action the user already declined.
if (REFUSAL_PATTERN.test(normalizeUtterance(params.text))) {
for (const [confirmationId, confirmation] of pendingConfirmations) {
if (
confirmation.agentId === params.agentId &&
confirmation.voiceSessionId === params.voiceSessionId &&
confirmation.createdAt < params.timestamp
) {
pendingConfirmations.delete(confirmationId);
}
}
}
}
/** Pause a high-impact action for one voice-bound run until its exact fingerprint is approved. */
export function resolveClientVoiceToolConfirmationPolicy(params: {
agentId?: string;
voiceSessionId?: string;
runId?: string;
toolName: string;
toolParams: unknown;
isConfirmable?: () => boolean;
now?: number;
}): { allowed: true } | { allowed: false; reason: string } {
if (!params.agentId || !params.voiceSessionId) {
return { allowed: true };
}
if (!requiresHighImpactVoiceConfirmation(params.toolName, params.toolParams)) {
return { allowed: true };
}
// Sessions that cannot report spoken approvals (legacy clients without transcript
// RPCs) keep pre-gate behavior; a pause they can never confirm is a dead end.
// This is not a client trust boundary: authenticated clients can already run any
// tool via chat.send. The gate guards against voice-channel misfires only.
if (params.isConfirmable && !params.isConfirmable()) {
return { allowed: true };
}
const now = params.now ?? Date.now();
const fingerprint = stableToolFingerprint(params.toolName, params.toolParams);
const scopeKey = confirmationScopeKey(params.agentId, params.voiceSessionId);
if (consumeApprovedFingerprint(scopeKey, params.runId, fingerprint, now)) {
return { allowed: true };
}
const existing = [...pendingConfirmations.values()].find(
(entry) =>
entry.voiceSessionId === params.voiceSessionId &&
entry.agentId === params.agentId &&
entry.runId === params.runId &&
entry.fingerprint === fingerprint &&
entry.expiresAt >= now,
);
const confirmation =
existing ??
({
confirmationId: randomUUID(),
agentId: params.agentId,
voiceSessionId: params.voiceSessionId,
...(params.runId ? { runId: params.runId } : {}),
fingerprint,
toolName: params.toolName,
createdAt: now,
seq: ++confirmationSeq,
expiresAt: now + CONFIRMATION_TTL_MS,
} satisfies PendingVoiceConfirmation);
pendingConfirmations.set(confirmation.confirmationId, confirmation);
return {
allowed: false,
reason:
`VOICE_CONFIRMATION_REQUIRED:${confirmation.confirmationId} ` +
`The high-impact voice action "${params.toolName}" was not executed. ` +
"Ask the user for explicit spoken confirmation, then call openclaw_agent_consult again with this confirmationId.",
};
}
const REFUSAL_PATTERN = /\b(no|don't|do not|cancel|stop|never mind)\b/;
function normalizeUtterance(text: string): string {
return (
text
.trim()
.toLowerCase()
// STT commonly emits typographic apostrophes; fold them so "don't" (U+2019)
// matches the refusal pattern and cannot slip past as a non-refusal.
.replace(/[‘’ʼ]/g, "'")
.replace(/[,;:.!?]+/g, "")
.replace(/\s+/g, " ")
);
}
function isExplicitAffirmation(text: string): boolean {
const normalized = normalizeUtterance(text);
if (REFUSAL_PATTERN.test(normalized)) {
return false;
}
// English-only phrases are an accepted first version; localized matching is follow-up work.
return /^(yes|yes do it|do it|confirm|confirmed|go ahead|proceed|send it|make the change|restart it)$/.test(
normalized,
);
}
/** Bind a later affirmative utterance to one exact paused action. */
export function authorizeClientVoiceConfirmation(params: {
agentId: string;
voiceSessionId: string;
confirmationId: string;
now?: number;
}): ClientVoiceConfirmationGrant {
const confirmation = pendingConfirmations.get(params.confirmationId);
const now = params.now ?? Date.now();
if (
!confirmation ||
confirmation.agentId !== params.agentId ||
confirmation.voiceSessionId !== params.voiceSessionId ||
confirmation.expiresAt < now
) {
throw new Error("voice confirmation is missing, expired, or belongs to another action");
}
// A bare "yes" can only answer the question the model asked last; authorizing an
// older challenge would let the model swap in a different pending action.
for (const entry of pendingConfirmations.values()) {
if (
entry.agentId === params.agentId &&
entry.voiceSessionId === params.voiceSessionId &&
entry.seq > confirmation.seq
) {
throw new Error("a newer confirmation request supersedes this one; ask again");
}
}
const scopeKey = confirmationScopeKey(params.agentId, params.voiceSessionId);
const affirmation = recentUserUtterances.get(scopeKey);
if (
!affirmation ||
affirmation.timestamp <= confirmation.createdAt ||
!isExplicitAffirmation(affirmation.text)
) {
throw new Error("explicit spoken confirmation was not found after the action request");
}
// Validate only; the challenge and affirmation are consumed at bind time, once the
// consult run is established. This keeps a failed/lost-response consult retryable
// with the same confirmationId instead of leaving the action unconfirmable.
return {
agentId: params.agentId,
voiceSessionId: params.voiceSessionId,
confirmationId: params.confirmationId,
fingerprint: confirmation.fingerprint,
expiresAt: confirmation.expiresAt,
};
}
/** Bind a validated spoken grant to the one follow-up run and consume the challenge. */
export function bindAuthorizedClientVoiceConfirmation(params: {
grant: ClientVoiceConfirmationGrant;
runId: string;
}): void {
const scopeKey = confirmationScopeKey(params.grant.agentId, params.grant.voiceSessionId);
const approvedByRun = approvedFingerprints.get(scopeKey) ?? new Map();
const approved = approvedByRun.get(params.runId) ?? new Map<string, number>();
approved.set(params.grant.fingerprint, params.grant.expiresAt);
approvedByRun.set(params.runId, approved);
approvedFingerprints.set(scopeKey, approvedByRun);
// Consume now that the run exists: one spoken affirmation authorizes one action.
pendingConfirmations.delete(params.grant.confirmationId);
recentUserUtterances.delete(scopeKey);
}
/**
* Remove ephemeral confirmation state when the logical call closes. Approved
* grants for still-live consult runs survive: a spoken "yes" followed by hangup
* must not re-block the confirmed action its run is about to execute.
*/
export function deactivateClientVoiceConfirmationSession(
agentId: string,
voiceSessionId: string,
liveRunIds: readonly string[] = [],
): void {
const scopeKey = confirmationScopeKey(agentId, voiceSessionId);
recentUserUtterances.delete(scopeKey);
const approvedByRun = approvedFingerprints.get(scopeKey);
if (approvedByRun) {
const live = new Set(liveRunIds);
for (const runId of approvedByRun.keys()) {
if (!live.has(runId)) {
approvedByRun.delete(runId);
}
}
if (approvedByRun.size === 0) {
approvedFingerprints.delete(scopeKey);
}
}
for (const [confirmationId, confirmation] of pendingConfirmations) {
if (confirmation.agentId === agentId && confirmation.voiceSessionId === voiceSessionId) {
pendingConfirmations.delete(confirmationId);
}
}
}
/** Drop a completed run's surviving grants once its lifecycle ends. */
export function releaseClientVoiceConfirmationRun(
agentId: string,
voiceSessionId: string,
runId: string,
): void {
const scopeKey = confirmationScopeKey(agentId, voiceSessionId);
const approvedByRun = approvedFingerprints.get(scopeKey);
if (!approvedByRun) {
return;
}
approvedByRun.delete(runId);
if (approvedByRun.size === 0) {
approvedFingerprints.delete(scopeKey);
}
}
/** Test-only reset for process-global state. */
function resetClientVoiceConfirmationStateForTest(): void {
pendingConfirmations.clear();
approvedFingerprints.clear();
recentUserUtterances.clear();
}
if (process.env.VITEST || process.env.NODE_ENV === "test") {
(globalThis as Record<PropertyKey, unknown>)[
Symbol.for("openclaw.clientVoiceConfirmationTestApi")
] = { resetClientVoiceConfirmationStateForTest };
}
+161
View File
@@ -0,0 +1,161 @@
/** SQLite-backed persistence for durable per-agent Talk voice-call records. */
import {
openOpenClawAgentDatabase,
type OpenClawAgentDatabase,
} from "../state/openclaw-agent-db.js";
export const VOICE_SESSION_CACHE_SCOPE = "talk-client-voice-sessions";
export const VOICE_SESSION_RECORD_VERSION = 1;
export const VOICE_SESSION_MAX_TRANSCRIPT_CHARS = 8_000;
export const VOICE_SESSION_STALE_AFTER_MS = 6 * 60 * 60_000;
export type ClientVoiceToolEffect = {
runId: string;
toolCallId?: string;
toolName: string;
startedAt: number;
finishedAt?: number;
status: "started" | "succeeded" | "failed" | "cancelled" | "blocked";
};
export type ClientVoiceSessionRecord = {
version: typeof VOICE_SESSION_RECORD_VERSION;
voiceSessionId: string;
agentId: string;
sessionKey: string;
provider?: string;
origin: "client" | "relay";
status: "open" | "closed";
createdAt: number;
updatedAt: number;
closedAt?: number;
consultRunIds: string[];
effects: ClientVoiceToolEffect[];
digestDeliveredAt?: number;
/** Declared at create when the client speaks the transcript protocol (sent sessionKey). */
transcriptCapable?: boolean;
/** Set once a finalized user utterance persisted; gates spoken confirmation capability. */
hasUserTranscript?: boolean;
};
export type ClientVoiceRunBinding = {
agentId: string;
voiceSessionId: string;
sessionKey: string;
};
function parseVoiceSessionRecord(value: unknown): ClientVoiceSessionRecord | undefined {
if (!value || typeof value !== "object" || Array.isArray(value)) {
return undefined;
}
const record = value as Partial<ClientVoiceSessionRecord>;
if (
record.version !== VOICE_SESSION_RECORD_VERSION ||
typeof record.voiceSessionId !== "string" ||
typeof record.agentId !== "string" ||
typeof record.sessionKey !== "string" ||
(record.provider !== undefined &&
(typeof record.provider !== "string" || !record.provider.trim())) ||
(record.origin !== "client" && record.origin !== "relay") ||
(record.status !== "open" && record.status !== "closed") ||
typeof record.createdAt !== "number" ||
typeof record.updatedAt !== "number"
) {
return undefined;
}
const consultRunIds = Array.isArray(record.consultRunIds)
? record.consultRunIds.filter((entry): entry is string => typeof entry === "string")
: [];
const effects = Array.isArray(record.effects)
? record.effects.filter((entry): entry is ClientVoiceToolEffect => {
if (!entry || typeof entry !== "object") {
return false;
}
const effect = entry as Partial<ClientVoiceToolEffect>;
return (
typeof effect.runId === "string" &&
typeof effect.toolName === "string" &&
typeof effect.startedAt === "number" &&
(effect.status === "started" ||
effect.status === "succeeded" ||
effect.status === "failed" ||
effect.status === "cancelled" ||
effect.status === "blocked")
);
})
: [];
const provider = record.provider?.trim();
return {
...record,
...(provider ? { provider } : {}),
consultRunIds,
effects,
} as ClientVoiceSessionRecord;
}
export function parseStoredVoiceSessionRecord(
valueJson: unknown,
): ClientVoiceSessionRecord | undefined {
if (typeof valueJson !== "string") {
return undefined;
}
try {
return parseVoiceSessionRecord(JSON.parse(valueJson));
} catch {
return undefined;
}
}
export function readVoiceSessionRecord(
agentId: string,
voiceSessionId: string,
): ClientVoiceSessionRecord | undefined {
const database = openOpenClawAgentDatabase({ agentId });
const row = database.db
.prepare("SELECT value_json FROM cache_entries WHERE scope = ? AND key = ?")
.get(VOICE_SESSION_CACHE_SCOPE, voiceSessionId) as { value_json?: unknown } | undefined;
return parseStoredVoiceSessionRecord(row?.value_json);
}
export function readVoiceSessionRecordInTransaction(
database: OpenClawAgentDatabase,
voiceSessionId: string,
): ClientVoiceSessionRecord | undefined {
const row = database.db
.prepare("SELECT value_json FROM cache_entries WHERE scope = ? AND key = ?")
.get(VOICE_SESSION_CACHE_SCOPE, voiceSessionId) as { value_json?: unknown } | undefined;
return parseStoredVoiceSessionRecord(row?.value_json);
}
export function writeVoiceSessionRecordInTransaction(
database: OpenClawAgentDatabase,
record: ClientVoiceSessionRecord,
): void {
database.db
.prepare(
`INSERT INTO cache_entries (scope, key, value_json, blob, expires_at, updated_at)
VALUES (?, ?, ?, NULL, NULL, ?)
ON CONFLICT(scope, key) DO UPDATE SET
value_json = excluded.value_json,
updated_at = excluded.updated_at`,
)
.run(
VOICE_SESSION_CACHE_SCOPE,
record.voiceSessionId,
JSON.stringify(record),
record.updatedAt,
);
}
export function assertVoiceSessionOwnership(
record: ClientVoiceSessionRecord,
params: { agentId: string; sessionKey: string },
): void {
if (record.agentId !== params.agentId || record.sessionKey !== params.sessionKey) {
throw new Error("voice session does not belong to this agent session");
}
}
export function operationKey(agentId: string, voiceSessionId: string): string {
return `${agentId}\0${voiceSessionId}`;
}
@@ -0,0 +1,22 @@
import type { ClientVoiceSessionRecord } from "./client-voice-session-store.js";
import "./client-voice-session.js";
type ClientVoiceSessionTestApi = {
readRecord(agentId: string, voiceSessionId: string): ClientVoiceSessionRecord | undefined;
reset(): void;
};
function getTestApi(): ClientVoiceSessionTestApi {
return (globalThis as Record<PropertyKey, unknown>)[
Symbol.for("openclaw.clientVoiceSessionTestApi")
] as ClientVoiceSessionTestApi;
}
export const clientVoiceSessionTesting = {
readRecord(agentId: string, voiceSessionId: string): ClientVoiceSessionRecord | undefined {
return getTestApi().readRecord(agentId, voiceSessionId);
},
reset(): void {
getTestApi().reset();
},
};
+579
View File
@@ -0,0 +1,579 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { replaceSessionEntry } from "../config/sessions/session-accessor.js";
import {
emitTrustedDiagnosticEvent,
waitForDiagnosticEventsDrained,
} from "../infra/diagnostic-events.js";
import { closeOpenClawAgentDatabasesForTest } from "../state/openclaw-agent-db.js";
import { closeOpenClawStateDatabaseForTest } from "../state/openclaw-state-db.js";
import { captureEnv, setTestEnvValue } from "../test-utils/env.js";
import {
closeClientVoiceSession,
closeStaleClientVoiceSessions,
createOrResumeClientVoiceSession,
isClientVoiceSessionConfirmable,
registerClientVoiceConsultRun,
resolveClientVoiceRunBinding,
resolveOpenClientVoiceSessionId,
} from "./client-voice-session.js";
import { clientVoiceSessionTesting } from "./client-voice-session.test-support.js";
const { sendDurableMessageBatch } = vi.hoisted(() => ({
sendDurableMessageBatch: vi.fn(async () => ({ status: "sent" })),
}));
vi.mock("../channels/message/runtime.js", () => ({ sendDurableMessageBatch }));
const envSnapshot = captureEnv(["OPENCLAW_STATE_DIR"]);
let tempDir: string;
async function seedSession(
sessionKey: string,
route: { lastChannel?: string; lastTo?: string } = {},
): Promise<void> {
await replaceSessionEntry(
{ agentId: "main", sessionKey },
{
sessionId: `session-${sessionKey.replaceAll(":", "-")}`,
updatedAt: Date.now(),
...route,
},
);
}
function recordMutation(voiceSessionId: string, runId = `run-${voiceSessionId}`): void {
registerClientVoiceConsultRun({
agentId: "main",
sessionKey: "agent:main:main",
voiceSessionId,
runId,
});
emitTrustedDiagnosticEvent({
type: "tool.execution.started",
runId,
toolCallId: `call-${runId}`,
toolName: "message",
mutatingAction: true,
});
emitTrustedDiagnosticEvent({
type: "tool.execution.completed",
runId,
toolCallId: `call-${runId}`,
toolName: "message",
durationMs: 5,
});
}
async function completeRun(runId: string): Promise<void> {
emitTrustedDiagnosticEvent({
type: "run.completed",
runId,
durationMs: 5,
outcome: "completed",
});
await waitForDiagnosticEventsDrained();
}
describe("client voice session", () => {
beforeEach(async () => {
tempDir = await fs.realpath(
await fs.mkdtemp(path.join(os.tmpdir(), "openclaw-voice-session-")),
);
setTestEnvValue("OPENCLAW_STATE_DIR", tempDir);
sendDurableMessageBatch.mockClear();
});
afterEach(async () => {
clientVoiceSessionTesting.reset();
closeOpenClawAgentDatabasesForTest();
closeOpenClawStateDatabaseForTest();
envSnapshot.restore();
await fs.rm(tempDir, { recursive: true, force: true });
});
it("creates, resumes, and enforces ownership and open state", async () => {
const voiceSessionId = createOrResumeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
provider: "google",
origin: "client",
voiceSessionId: "voice-1",
now: 10,
});
expect(
createOrResumeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
origin: "client",
voiceSessionId,
now: 20,
}),
).toBe(voiceSessionId);
expect(clientVoiceSessionTesting.readRecord("main", voiceSessionId)).toMatchObject({
provider: "google",
});
expect(() =>
createOrResumeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
provider: "openai",
origin: "client",
voiceSessionId,
}),
).toThrow("provider does not match");
expect(() =>
createOrResumeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:other",
origin: "client",
voiceSessionId,
}),
).toThrow("does not belong");
await closeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
voiceSessionId,
config: {},
now: 30,
});
expect(() =>
createOrResumeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
origin: "client",
voiceSessionId,
}),
).toThrow("already closed");
});
it("marks confirmability by declared capability, relay origin, or observed transcript", () => {
const capable = createOrResumeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
origin: "client",
transcriptCapable: true,
voiceSessionId: "voice-capable",
});
const legacy = createOrResumeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
origin: "client",
voiceSessionId: "voice-legacy",
});
const relay = createOrResumeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
origin: "relay",
voiceSessionId: "voice-relay",
});
const binding = (voiceSessionId: string) => ({
agentId: "main",
sessionKey: "agent:main:main",
voiceSessionId,
});
expect(isClientVoiceSessionConfirmable(binding(capable))).toBe(true);
expect(isClientVoiceSessionConfirmable(binding(legacy))).toBe(false);
expect(isClientVoiceSessionConfirmable(binding(relay))).toBe(true);
});
it("closes idempotently without changing the first close time", async () => {
const voiceSessionId = createOrResumeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
origin: "client",
now: 10,
});
await closeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
voiceSessionId,
config: {},
now: 20,
});
await closeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
voiceSessionId,
config: {},
now: 30,
});
expect(clientVoiceSessionTesting.readRecord("main", voiceSessionId)).toMatchObject({
status: "closed",
closedAt: 20,
});
});
it("keeps active consult runs voice-bound after the call closes", async () => {
const voiceSessionId = createOrResumeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
origin: "client",
});
registerClientVoiceConsultRun({
agentId: "main",
sessionKey: "agent:main:main",
voiceSessionId,
runId: "run-active",
});
await closeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
voiceSessionId,
config: {},
});
expect(resolveClientVoiceRunBinding("run-active")).toMatchObject({ voiceSessionId });
});
it("resolves the open client record for legacy tool calls", () => {
const voiceSessionId = createOrResumeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
origin: "client",
});
expect(
resolveOpenClientVoiceSessionId({ agentId: "main", sessionKey: "agent:main:main" }),
).toBe(voiceSessionId);
expect(
resolveOpenClientVoiceSessionId({ agentId: "main", sessionKey: "agent:main:other" }),
).toBeUndefined();
createOrResumeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
origin: "client",
});
expect(
resolveOpenClientVoiceSessionId({ agentId: "main", sessionKey: "agent:main:main" }),
).toBeUndefined();
});
it("keeps repeated tool-call ids separate across consult runs", () => {
const voiceSessionId = createOrResumeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
origin: "client",
});
for (const runId of ["run-1", "run-2"]) {
registerClientVoiceConsultRun({
agentId: "main",
sessionKey: "agent:main:main",
voiceSessionId,
runId,
});
emitTrustedDiagnosticEvent({
type: "tool.execution.started",
runId,
toolCallId: "call-1",
toolName: "message",
mutatingAction: true,
});
}
expect(clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.effects).toHaveLength(2);
});
it("closes stale records and leaves recent records open", async () => {
const stale = createOrResumeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:stale",
origin: "client",
now: 1,
});
const recent = createOrResumeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:recent",
origin: "client",
now: 6 * 60 * 60_000,
});
expect(
await closeStaleClientVoiceSessions({
agentId: "main",
config: {},
now: 6 * 60 * 60_000 + 2,
}),
).toBe(1);
expect(clientVoiceSessionTesting.readRecord("main", stale)?.status).toBe("closed");
expect(clientVoiceSessionTesting.readRecord("main", recent)?.status).toBe("open");
});
it("records only mutating started effects and updates their terminal status", () => {
const voiceSessionId = createOrResumeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
origin: "client",
});
registerClientVoiceConsultRun({
agentId: "main",
sessionKey: "agent:main:main",
voiceSessionId,
runId: "run-1",
});
emitTrustedDiagnosticEvent({
type: "tool.execution.started",
runId: "run-1",
toolCallId: "read-1",
toolName: "read",
mutatingAction: false,
});
emitTrustedDiagnosticEvent({
type: "tool.execution.started",
runId: "run-1",
toolCallId: "message-1",
toolName: "message",
mutatingAction: true,
});
emitTrustedDiagnosticEvent({
type: "tool.execution.error",
runId: "run-1",
toolCallId: "message-1",
toolName: "message",
durationMs: 4,
errorCategory: "aborted",
terminalReason: "cancelled",
});
expect(clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.effects).toEqual([
expect.objectContaining({
toolCallId: "message-1",
toolName: "message",
status: "cancelled",
finishedAt: expect.any(Number),
}),
]);
});
it("records post-close effects and defers the digest until the last consult completes", async () => {
await seedSession("agent:main:main", {
lastChannel: "discord",
lastTo: "channel:voice-updates",
});
const voiceSessionId = createOrResumeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
origin: "client",
});
for (const runId of ["run-1", "run-2"]) {
registerClientVoiceConsultRun({
agentId: "main",
sessionKey: "agent:main:main",
voiceSessionId,
runId,
});
}
await closeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
voiceSessionId,
config: {},
});
expect(sendDurableMessageBatch).not.toHaveBeenCalled();
for (const runId of ["run-1", "run-2"]) {
emitTrustedDiagnosticEvent({
type: "tool.execution.started",
runId,
toolCallId: `call-${runId}`,
toolName: "message",
mutatingAction: true,
});
emitTrustedDiagnosticEvent({
type: "tool.execution.completed",
runId,
toolCallId: `call-${runId}`,
toolName: "message",
durationMs: 5,
});
}
expect(clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.effects).toEqual([
expect.objectContaining({ runId: "run-1", status: "succeeded" }),
expect.objectContaining({ runId: "run-2", status: "succeeded" }),
]);
await completeRun("run-1");
expect(sendDurableMessageBatch).not.toHaveBeenCalled();
await completeRun("run-2");
await vi.waitFor(() => expect(sendDurableMessageBatch).toHaveBeenCalledTimes(1));
expect(sendDurableMessageBatch).toHaveBeenCalledWith(
expect.objectContaining({
payloads: [{ text: "Voice call changes\n- message: succeeded\n- message: succeeded" }],
}),
);
expect(clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.digestDeliveredAt).toEqual(
expect.any(Number),
);
await closeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
voiceSessionId,
config: {},
});
expect(sendDurableMessageBatch).toHaveBeenCalledTimes(1);
});
it("retries a deferred digest on the next voice session after a failed delivery", async () => {
await seedSession("agent:main:main", {
lastChannel: "discord",
lastTo: "channel:voice-updates",
});
const voiceSessionId = createOrResumeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
origin: "client",
});
registerClientVoiceConsultRun({
agentId: "main",
sessionKey: "agent:main:main",
voiceSessionId,
runId: "run-live",
});
emitTrustedDiagnosticEvent({
type: "tool.execution.started",
runId: "run-live",
toolCallId: "call-run-live",
toolName: "message",
mutatingAction: true,
});
emitTrustedDiagnosticEvent({
type: "tool.execution.completed",
runId: "run-live",
toolCallId: "call-run-live",
toolName: "message",
durationMs: 5,
});
// Call ends while the consult still runs, so the digest is deferred.
await closeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
voiceSessionId,
config: {},
});
sendDurableMessageBatch.mockRejectedValueOnce(new Error("channel offline"));
await completeRun("run-live");
expect(
clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.digestDeliveredAt,
).toBeUndefined();
// Starting the next voice session retries the deferred digest.
await closeStaleClientVoiceSessions({ agentId: "main", config: {} });
await vi.waitFor(() =>
expect(
clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.digestDeliveredAt,
).toEqual(expect.any(Number)),
);
});
it("retries the mutation digest after a transient send failure", async () => {
await seedSession("agent:main:main", {
lastChannel: "discord",
lastTo: "channel:voice-updates",
});
const voiceSessionId = createOrResumeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
origin: "client",
});
recordMutation(voiceSessionId);
await completeRun(`run-${voiceSessionId}`);
sendDurableMessageBatch.mockRejectedValueOnce(new Error("channel offline"));
await closeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
voiceSessionId,
config: {},
}).catch(() => undefined);
expect(
clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.digestDeliveredAt,
).toBeUndefined();
await closeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
voiceSessionId,
config: {},
});
expect(sendDurableMessageBatch).toHaveBeenCalledTimes(2);
expect(clientVoiceSessionTesting.readRecord("main", voiceSessionId)?.digestDeliveredAt).toEqual(
expect.any(Number),
);
});
it("delivers one mutation digest and skips webchat or missing targets", async () => {
await seedSession("agent:main:main", {
lastChannel: "discord",
lastTo: "channel:voice-updates",
});
const delivered = createOrResumeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
origin: "client",
});
recordMutation(delivered);
await completeRun(`run-${delivered}`);
await closeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
voiceSessionId: delivered,
config: {},
});
await closeClientVoiceSession({
agentId: "main",
sessionKey: "agent:main:main",
voiceSessionId: delivered,
config: {},
});
expect(sendDurableMessageBatch).toHaveBeenCalledTimes(1);
expect(sendDurableMessageBatch).toHaveBeenCalledWith(
expect.objectContaining({
durability: "required",
requireUnknownSendReconciliation: true,
payloads: [{ text: "Voice call changes\n- message: succeeded" }],
}),
);
for (const [voiceSessionId, route] of [
["voice-webchat", { lastChannel: "webchat", lastTo: "browser" }],
["voice-no-target", {}],
] as const) {
const sessionKey = `agent:main:${voiceSessionId}`;
await seedSession(sessionKey, route);
createOrResumeClientVoiceSession({
agentId: "main",
sessionKey,
origin: "client",
voiceSessionId,
});
registerClientVoiceConsultRun({
agentId: "main",
sessionKey,
voiceSessionId,
runId: `run-${voiceSessionId}`,
});
emitTrustedDiagnosticEvent({
type: "tool.execution.started",
runId: `run-${voiceSessionId}`,
toolCallId: `call-${voiceSessionId}`,
toolName: "message",
mutatingAction: true,
});
await completeRun(`run-${voiceSessionId}`);
await closeClientVoiceSession({
agentId: "main",
sessionKey,
voiceSessionId,
config: {},
});
}
expect(sendDurableMessageBatch).toHaveBeenCalledTimes(1);
});
});
+732
View File
@@ -0,0 +1,732 @@
/** Durable per-agent voice-call records for Talk continuity and mutation evidence. */
import { randomUUID } from "node:crypto";
import {
appendTranscriptMessage,
loadSessionEntry,
upsertSessionEntry,
} from "../config/sessions/session-accessor.js";
import type { OpenClawConfig } from "../config/types.openclaw.js";
import {
onTrustedInternalDiagnosticEvent,
onTrustedToolExecutionEvent,
type TrustedToolExecutionEvent,
} from "../infra/diagnostic-events.js";
import { buildOutboundSessionContext } from "../infra/outbound/session-context.js";
import { resolveSessionDeliveryTarget } from "../infra/outbound/targets-session.js";
import {
openOpenClawAgentDatabase,
runOpenClawAgentWriteTransaction,
} from "../state/openclaw-agent-db.js";
import { truncateUtf16Safe } from "../utils.js";
import {
deactivateClientVoiceConfirmationSession,
noteClientVoiceConfirmationUtterance,
releaseClientVoiceConfirmationRun,
} from "./client-voice-confirmation.js";
import {
assertVoiceSessionOwnership as assertOwnership,
type ClientVoiceRunBinding,
type ClientVoiceSessionRecord,
type ClientVoiceToolEffect,
operationKey,
parseStoredVoiceSessionRecord as parseStoredRecord,
readVoiceSessionRecord as readRecord,
readVoiceSessionRecordInTransaction as readRecordInTransaction,
VOICE_SESSION_CACHE_SCOPE as CACHE_SCOPE,
VOICE_SESSION_MAX_TRANSCRIPT_CHARS as MAX_TRANSCRIPT_CHARS,
VOICE_SESSION_RECORD_VERSION as RECORD_VERSION,
VOICE_SESSION_STALE_AFTER_MS as STALE_AFTER_MS,
writeVoiceSessionRecordInTransaction as writeRecordInTransaction,
} from "./client-voice-session-store.js";
const voiceSessionByRunId = new Map<string, ClientVoiceRunBinding>();
const voiceSessionOperations = new Map<string, Promise<void>>();
const deferredDigestConfigs = new Map<string, OpenClawConfig>();
let unsubscribeToolEffects: (() => void) | undefined;
let unsubscribeRunCompletion: (() => void) | undefined;
function hasLiveConsultRun(record: ClientVoiceSessionRecord): boolean {
return record.consultRunIds.some((runId) => {
const binding = voiceSessionByRunId.get(runId);
return (
binding?.agentId === record.agentId &&
binding.voiceSessionId === record.voiceSessionId &&
binding.sessionKey === record.sessionKey
);
});
}
async function runVoiceSessionOperation<T>(
agentId: string,
voiceSessionId: string,
operation: () => Promise<T>,
): Promise<T> {
const key = operationKey(agentId, voiceSessionId);
const previous = voiceSessionOperations.get(key) ?? Promise.resolve();
const current = previous.then(operation, operation);
const settled = current.then(
() => undefined,
() => undefined,
);
voiceSessionOperations.set(key, settled);
void settled.then(() => {
if (voiceSessionOperations.get(key) === settled) {
voiceSessionOperations.delete(key);
}
});
return await current;
}
function effectStatus(event: TrustedToolExecutionEvent): ClientVoiceToolEffect["status"] {
if (event.type === "tool.execution.started") {
return "started";
}
if (event.type === "tool.execution.completed") {
return "succeeded";
}
if (event.type === "tool.execution.blocked") {
return "blocked";
}
return event.terminalReason === "cancelled" ? "cancelled" : "failed";
}
function recordClientVoiceToolEffect(event: TrustedToolExecutionEvent): void {
const runId = event.runId;
if (!runId) {
return;
}
const binding = voiceSessionByRunId.get(runId);
if (!binding) {
return;
}
runOpenClawAgentWriteTransaction(
(database) => {
const record = readRecordInTransaction(database, binding.voiceSessionId);
if (!record) {
return;
}
const existing = event.toolCallId
? record.effects.find(
(effect) => effect.runId === runId && effect.toolCallId === event.toolCallId,
)
: record.effects.findLast(
(effect) =>
effect.runId === runId &&
effect.toolName === event.toolName &&
effect.status === "started",
);
if (event.type !== "tool.execution.started" && !existing) {
return;
}
if (event.type !== "tool.execution.started" && existing) {
existing.status = effectStatus(event);
existing.finishedAt = event.ts;
} else if (event.mutatingAction === true && (!event.toolCallId || !existing)) {
record.effects.push({
runId,
...(event.toolCallId ? { toolCallId: event.toolCallId } : {}),
toolName: event.toolName,
startedAt: event.ts,
status: "started",
});
}
record.updatedAt = Date.now();
writeRecordInTransaction(database, record);
},
{ agentId: binding.agentId },
);
}
function ensureToolEffectSubscription(): void {
unsubscribeToolEffects ??= onTrustedToolExecutionEvent(recordClientVoiceToolEffect);
unsubscribeRunCompletion ??= onTrustedInternalDiagnosticEvent((event) => {
if (event.type !== "run.completed") {
return;
}
const binding = voiceSessionByRunId.get(event.runId);
if (!binding) {
return;
}
voiceSessionByRunId.delete(event.runId);
releaseClientVoiceConfirmationRun(binding.agentId, binding.voiceSessionId, event.runId);
void finishDeferredMutationDigest(binding).catch((error: unknown) => {
console.warn(
`[talk] deferred voice mutation digest failed: ${error instanceof Error ? error.message : String(error)}`,
);
});
});
}
/** Create a call record or resume the same open call across transport restarts. */
export function createOrResumeClientVoiceSession(params: {
agentId: string;
sessionKey: string;
provider?: string;
origin: "client" | "relay";
transcriptCapable?: boolean;
voiceSessionId?: string;
now?: number;
}): string {
const voiceSessionId = params.voiceSessionId?.trim() || randomUUID();
const provider = params.provider?.trim() || undefined;
const now = params.now ?? Date.now();
runOpenClawAgentWriteTransaction(
(database) => {
const existing = readRecordInTransaction(database, voiceSessionId);
if (existing) {
assertOwnership(existing, params);
if (existing.origin !== params.origin) {
throw new Error("voice session origin does not match");
}
if (existing.status !== "open") {
throw new Error("voice session is already closed");
}
if (existing.provider && provider && existing.provider !== provider) {
throw new Error("voice session provider does not match");
}
if (!existing.provider && provider) {
existing.provider = provider;
}
if (params.transcriptCapable === true) {
existing.transcriptCapable = true;
}
existing.updatedAt = now;
writeRecordInTransaction(database, existing);
return;
}
writeRecordInTransaction(database, {
version: RECORD_VERSION,
voiceSessionId,
agentId: params.agentId,
sessionKey: params.sessionKey,
...(provider ? { provider } : {}),
origin: params.origin,
...(params.transcriptCapable === true ? { transcriptCapable: true } : {}),
status: "open",
createdAt: now,
updatedAt: now,
consultRunIds: [],
effects: [],
});
},
{ agentId: params.agentId },
);
return voiceSessionId;
}
/** Ensure Talk has the same canonical agent-session row that chat turns append to. */
export async function ensureClientVoiceAgentSessionEntry(params: {
agentId: string;
sessionKey: string;
}): Promise<void> {
const existing = loadSessionEntry(params);
if (existing?.sessionId) {
return;
}
const created = await upsertSessionEntry(params, {});
if (!created?.sessionId) {
throw new Error(`agent session could not be initialized (${params.sessionKey})`);
}
}
/** Correlate a consult run with its open call for confirmation and mutation evidence. */
export function registerClientVoiceConsultRun(params: {
agentId: string;
sessionKey: string;
voiceSessionId: string;
runId: string;
config?: OpenClawConfig;
}): void {
let recordClosed = false;
runOpenClawAgentWriteTransaction(
(database) => {
const record = readRecordInTransaction(database, params.voiceSessionId);
if (!record) {
throw new Error("voice session not found");
}
assertOwnership(record, params);
recordClosed = record.status === "closed";
// A close can race in while chat.send is still acking this run. The run has
// already started, so bind it anyway (even on a closed record) to keep effect
// capture; aborting here would drop a just-confirmed high-impact action.
if (!record.consultRunIds.includes(params.runId)) {
record.consultRunIds.push(params.runId);
record.updatedAt = Date.now();
writeRecordInTransaction(database, record);
}
},
{ agentId: params.agentId },
);
voiceSessionByRunId.set(params.runId, {
agentId: params.agentId,
voiceSessionId: params.voiceSessionId,
sessionKey: params.sessionKey,
});
// Bound to a call that already closed: its digest may have been delivered and its
// retry config cleared, so re-arm it to cover this late run's mutations.
if (recordClosed && params.config) {
deferredDigestConfigs.set(operationKey(params.agentId, params.voiceSessionId), params.config);
}
ensureToolEffectSubscription();
}
/** Return the open voice-call binding for one executing run. */
export function resolveClientVoiceRunBinding(runId?: string): ClientVoiceRunBinding | undefined {
return runId ? voiceSessionByRunId.get(runId) : undefined;
}
/**
* Confirmation applies only when the session can observe spoken approvals:
* relay sessions (server hears utterances) or clients that report transcripts.
* Legacy clients without transcript reporting keep pre-gate behavior.
*/
export function isClientVoiceSessionConfirmable(binding: ClientVoiceRunBinding): boolean {
const record = readRecord(binding.agentId, binding.voiceSessionId);
return (
record?.origin === "relay" ||
record?.transcriptCapable === true ||
record?.hasUserTranscript === true
);
}
/** Validate ownership and open state before starting a voice-bound consult. */
export function assertClientVoiceSessionOpen(params: {
agentId: string;
sessionKey: string;
voiceSessionId: string;
}): "client" | "relay" {
const record = readRecord(params.agentId, params.voiceSessionId);
if (!record) {
throw new Error("voice session not found");
}
assertOwnership(record, params);
if (record.status !== "open") {
throw new Error("voice session is closed");
}
return record.origin;
}
/** Validate durable ownership without rejecting an idempotent close retry. */
export function resolveClientVoiceSessionOrigin(params: {
agentId: string;
sessionKey: string;
voiceSessionId: string;
}): "client" | "relay" {
const record = readRecord(params.agentId, params.voiceSessionId);
if (!record) {
throw new Error("voice session not found");
}
assertOwnership(record, params);
return record.origin;
}
/** Resolve the newest open client-owned call for legacy tool-call clients. */
export function resolveOpenClientVoiceSessionId(params: {
agentId: string;
sessionKey: string;
}): string | undefined {
const database = openOpenClawAgentDatabase({ agentId: params.agentId });
const rows = database.db
.prepare("SELECT value_json FROM cache_entries WHERE scope = ? ORDER BY updated_at DESC")
.all(CACHE_SCOPE) as Array<{ value_json?: unknown }>;
let match: string | undefined;
for (const row of rows) {
const record = parseStoredRecord(row.value_json);
if (
record?.origin === "client" &&
record.status === "open" &&
record.agentId === params.agentId &&
record.sessionKey === params.sessionKey
) {
if (match) {
return undefined;
}
match = record.voiceSessionId;
}
}
return match;
}
function buildPersistedVoiceMessage(params: {
role: "user" | "assistant";
text: string;
timestamp: number;
provider: string;
}): Record<string, unknown> {
const provenance = { kind: "realtime_voice", sourceChannel: "talk" };
if (params.role === "user") {
return {
role: "user",
content: [{ type: "text", text: params.text }],
timestamp: params.timestamp,
provenance,
};
}
return {
role: "assistant",
content: [{ type: "text", text: params.text }],
api: "realtime",
provider: params.provider,
model: "realtime-voice",
stopReason: "stop",
timestamp: params.timestamp,
provenance,
};
}
async function appendVoiceTranscript(params: {
agentId: string;
sessionKey: string;
voiceSessionId: string;
origin: "client" | "relay";
entryId: string;
role: "user" | "assistant";
text: string;
timestamp?: number;
config?: OpenClawConfig;
}): Promise<void> {
await runVoiceSessionOperation(params.agentId, params.voiceSessionId, async () => {
const text = truncateUtf16Safe(params.text.trim(), MAX_TRANSCRIPT_CHARS);
if (!text) {
return;
}
const record = readRecord(params.agentId, params.voiceSessionId);
if (!record) {
throw new Error("voice session not found");
}
assertOwnership(record, params);
if (record.status !== "open") {
throw new Error("voice session is closed");
}
if (record.origin !== params.origin) {
throw new Error("voice session origin does not allow this transcript source");
}
const sessionEntry = loadSessionEntry({
agentId: params.agentId,
sessionKey: params.sessionKey,
});
if (!sessionEntry?.sessionId) {
throw new Error(`agent session not found (${params.sessionKey})`);
}
const observedAt = Date.now();
const timestamp = params.timestamp ?? observedAt;
await appendTranscriptMessage(
{
agentId: params.agentId,
sessionId: sessionEntry.sessionId,
sessionKey: params.sessionKey,
},
{
...(params.config ? { config: params.config } : {}),
eventId: `voice:${params.voiceSessionId}:${params.entryId}`,
message: buildPersistedVoiceMessage({
role: params.role,
text,
timestamp,
provider: record.provider ?? "realtime",
}),
now: timestamp,
},
);
runOpenClawAgentWriteTransaction(
(database) => {
const current = readRecordInTransaction(database, params.voiceSessionId);
if (!current) {
throw new Error("voice session disappeared during transcript append");
}
assertOwnership(current, params);
// Reaching here means this exact eventId is durably persisted (fresh append or
// idempotent dedup of our own prior write). Arm confirmation bookkeeping in both
// cases so a retry after a partial failure still records the user utterance.
if (params.role === "user") {
current.hasUserTranscript = true;
}
current.updatedAt = Date.now();
writeRecordInTransaction(database, current);
},
{ agentId: params.agentId },
);
if (params.role === "user") {
noteClientVoiceConfirmationUtterance({
agentId: params.agentId,
voiceSessionId: params.voiceSessionId,
text,
timestamp: observedAt,
});
}
});
}
/** Append one finalized client-owned transcript item idempotently. */
export async function appendClientVoiceTranscript(
params: Omit<Parameters<typeof appendVoiceTranscript>[0], "origin">,
): Promise<void> {
await appendVoiceTranscript({ ...params, origin: "client" });
}
/** Append one finalized relay-owned transcript item idempotently. */
export async function appendRelayVoiceTranscript(
params: Omit<Parameters<typeof appendVoiceTranscript>[0], "origin">,
): Promise<void> {
await appendVoiceTranscript({ ...params, origin: "relay" });
}
// The mutation digest is a best-effort informational summary, not a durable record:
// the authoritative effects live on the voice record and the canonical transcript is
// persisted independently. Accepted tradeoffs (retry state is process-local like the
// rest of this subsystem): a digest already delivered at close does not reissue for a
// run that binds afterward, and an undelivered digest whose retry map is cleared by a
// gateway restart is not rediscovered. Both lose a summary message, never real data.
function formatMutationDigest(effects: ClientVoiceToolEffect[]): string | undefined {
if (effects.length === 0) {
return undefined;
}
return [
"Voice call changes",
...effects
.slice(0, 12)
.map(
(effect) =>
`- ${effect.toolName}: ${effect.status === "started" ? "outcome not confirmed" : effect.status}`,
),
].join("\n");
}
async function deliverMutationDigest(
record: ClientVoiceSessionRecord,
config: OpenClawConfig,
target: { channel: string; to: string; accountId?: string; threadId?: string | number | null },
text: string,
): Promise<void> {
const { sendDurableMessageBatch } = await import("../channels/message/runtime.js");
const send = await sendDurableMessageBatch({
cfg: config,
channel: target.channel,
to: target.to,
...(target.accountId ? { accountId: target.accountId } : {}),
...(target.threadId != null ? { threadId: target.threadId } : {}),
payloads: [{ text }],
durability: "required",
requireUnknownSendReconciliation: true,
session: buildOutboundSessionContext({
cfg: config,
agentId: record.agentId,
sessionKey: record.sessionKey,
policySessionKey: record.sessionKey,
}),
});
if (send.status === "failed" || send.status === "partial_failed") {
throw send.error;
}
}
async function deliverMutationDigestOnce(
record: ClientVoiceSessionRecord,
config: OpenClawConfig,
): Promise<void> {
if (record.digestDeliveredAt) {
return;
}
const text = formatMutationDigest(record.effects);
if (!text) {
return;
}
const entry = loadSessionEntry({ agentId: record.agentId, sessionKey: record.sessionKey });
const target = resolveSessionDeliveryTarget({ entry, requestedChannel: "last" });
if (!target.channel || target.channel === "webchat" || !target.to) {
return;
}
// Send first, mark after: a transient send failure (the common case) then leaves
// the marker unset so close/stale-sweep retries. The rare crash between a durable
// send and this marker can re-send one informational digest, which is acceptable.
await deliverMutationDigest(
record,
config,
{
channel: target.channel,
to: target.to,
accountId: target.accountId,
threadId: target.threadId,
},
text,
);
const deliveredAt = Date.now();
runOpenClawAgentWriteTransaction(
(database) => {
const current = readRecordInTransaction(database, record.voiceSessionId);
if (!current || current.digestDeliveredAt) {
return;
}
current.digestDeliveredAt = deliveredAt;
current.updatedAt = deliveredAt;
writeRecordInTransaction(database, current);
},
{ agentId: record.agentId },
);
}
async function finishDeferredMutationDigest(binding: {
agentId: string;
voiceSessionId: string;
}): Promise<void> {
const key = operationKey(binding.agentId, binding.voiceSessionId);
const config = deferredDigestConfigs.get(key);
if (!config) {
return;
}
await runVoiceSessionOperation(binding.agentId, binding.voiceSessionId, async () => {
const record = readRecord(binding.agentId, binding.voiceSessionId);
if (!record || record.status !== "closed" || hasLiveConsultRun(record)) {
return;
}
// Deliver first; only drop the retry config once the send succeeds, so a
// transient failure here is retried by the next closeStaleClientVoiceSessions.
await deliverMutationDigestOnce(record, config);
deferredDigestConfigs.delete(key);
});
}
/** Retry deferred digests whose delivery previously failed after the call closed. */
async function retryDeferredMutationDigests(agentId: string): Promise<void> {
for (const key of Array.from(deferredDigestConfigs.keys())) {
const [entryAgentId, voiceSessionId] = key.split("\0");
if (entryAgentId !== agentId || !voiceSessionId) {
continue;
}
try {
await finishDeferredMutationDigest({ agentId, voiceSessionId });
} catch {
// Still undeliverable; a later voice session retries again.
}
}
}
async function closeClientVoiceSessionInternal(params: {
agentId: string;
sessionKey: string;
voiceSessionId: string;
config: OpenClawConfig;
now?: number;
}): Promise<void> {
const existing = readRecord(params.agentId, params.voiceSessionId);
if (!existing) {
throw new Error("voice session not found");
}
assertOwnership(existing, params);
const now = params.now ?? Date.now();
runOpenClawAgentWriteTransaction(
(database) => {
const current = readRecordInTransaction(database, params.voiceSessionId);
if (!current) {
throw new Error("voice session disappeared during close");
}
assertOwnership(current, params);
if (current.status === "open") {
current.status = "closed";
current.closedAt = now;
current.updatedAt = now;
writeRecordInTransaction(database, current);
}
},
{ agentId: params.agentId },
);
const closed = readRecord(params.agentId, params.voiceSessionId);
if (!closed) {
throw new Error("voice session disappeared after close");
}
// Transport close does not end consult runs: live bindings keep effect capture active,
// approved grants stay valid for those runs, and the digest waits for the last run.completed.
const liveRunIds = closed.consultRunIds.filter((runId) => {
const binding = voiceSessionByRunId.get(runId);
return binding?.voiceSessionId === params.voiceSessionId && binding.agentId === params.agentId;
});
deactivateClientVoiceConfirmationSession(params.agentId, params.voiceSessionId, liveRunIds);
const key = operationKey(params.agentId, params.voiceSessionId);
// Both the deferred and immediate paths retain the retry config until a send
// actually succeeds, so a channel outage does not permanently lose the digest.
deferredDigestConfigs.set(key, params.config);
if (hasLiveConsultRun(closed)) {
return;
}
await deliverMutationDigestOnce(closed, params.config);
// Intentionally unconditional: a consult that late-binds during the await above has
// its mutations omitted from this already-sent point-in-time summary. Re-sending to
// include them would double-notify the user for one call, which is a worse outcome
// for an informational digest than omitting a straggler action (still on the record).
deferredDigestConfigs.delete(key);
}
/** Close a logical voice call and deliver its mutation digest at most once. */
export async function closeClientVoiceSession(params: {
agentId: string;
sessionKey: string;
voiceSessionId: string;
config: OpenClawConfig;
now?: number;
}): Promise<void> {
await runVoiceSessionOperation(params.agentId, params.voiceSessionId, async () => {
await closeClientVoiceSessionInternal(params);
});
}
/** Close abandoned open calls idle for the fixed six-hour recovery window. */
export async function closeStaleClientVoiceSessions(params: {
agentId: string;
config: OpenClawConfig;
excludeVoiceSessionId?: string;
now?: number;
warn?: (message: string) => void;
}): Promise<number> {
const now = params.now ?? Date.now();
// A new voice session is the natural retry point for a digest whose delivery
// failed after its own call had already closed and its runs completed.
await retryDeferredMutationDigests(params.agentId);
const database = openOpenClawAgentDatabase({ agentId: params.agentId });
const rows = database.db
.prepare("SELECT value_json FROM cache_entries WHERE scope = ? AND updated_at <= ?")
.all(CACHE_SCOPE, now - STALE_AFTER_MS) as Array<{ value_json?: unknown }>;
const stale = rows.flatMap((row) => {
const record = parseStoredRecord(row.value_json);
return record &&
record.status === "open" &&
record.voiceSessionId !== params.excludeVoiceSessionId
? [record]
: [];
});
let closed = 0;
for (const record of stale) {
try {
await closeClientVoiceSession({
agentId: params.agentId,
sessionKey: record.sessionKey,
voiceSessionId: record.voiceSessionId,
config: params.config,
now,
});
closed += 1;
} catch (error) {
params.warn?.(
`failed to close stale voice session ${record.voiceSessionId}: ${error instanceof Error ? error.message : String(error)}`,
);
}
}
return closed;
}
const clientVoiceSessionTesting = {
readRecord,
reset(): void {
voiceSessionByRunId.clear();
voiceSessionOperations.clear();
deferredDigestConfigs.clear();
unsubscribeToolEffects?.();
unsubscribeToolEffects = undefined;
unsubscribeRunCompletion?.();
unsubscribeRunCompletion = undefined;
},
};
if (process.env.VITEST || process.env.NODE_ENV === "test") {
(globalThis as Record<PropertyKey, unknown>)[Symbol.for("openclaw.clientVoiceSessionTestApi")] =
clientVoiceSessionTesting;
}
+1 -1
View File
@@ -13,7 +13,7 @@ export function createGatewayClientVitestConfig(env?: Record<string, string | un
],
{
env,
exclude: ["src/gateway/**/*server*.test.ts"],
exclude: ["src/gateway/**/*server*.test.ts", "src/gateway/server-methods/**/*.test.ts"],
// Gateway child projects share one include file; preserve this project's ownership.
intersectIncludeFile: true,
isolate: true,
@@ -16,9 +16,11 @@ function requireFirstMockCall(calls: readonly unknown[][], label: string): unkno
describe("RealtimeTalkSession consult handoff", () => {
it("submits realtime consults through the Gateway tool-call endpoint", async () => {
const order: string[] = [];
let listener: ((event: { event: string; payload?: unknown }) => void) | undefined;
const request = vi.fn(async (method: string, _params: unknown) => {
if (method === "talk.client.toolCall") {
order.push("tool-call");
setImmediate(() => {
listener?.({
event: "chat",
@@ -40,11 +42,16 @@ describe("RealtimeTalkSession consult handoff", () => {
};
});
const submit = vi.fn();
const flushTranscriptWrites = vi.fn(async () => {
order.push("flush");
});
await submitRealtimeTalkConsult({
ctx: {
client: { request, addEventListener },
sessionKey: "agent:main:main",
voiceSessionId: "voice-1",
flushTranscriptWrites,
callbacks: {},
} as never,
callId: "call-1",
@@ -53,13 +60,23 @@ describe("RealtimeTalkSession consult handoff", () => {
});
const toolCall = requireFirstMockCall(request.mock.calls, "Gateway request") as
| [string, { sessionKey?: string; name?: string; args?: { question?: string } }]
| [
string,
{
sessionKey?: string;
voiceSessionId?: string;
name?: string;
args?: { question?: string };
},
]
| undefined;
expect(toolCall?.[0]).toBe("talk.client.toolCall");
expect(toolCall?.[1]?.sessionKey).toBe("agent:main:main");
expect(toolCall?.[1]).toMatchObject({ voiceSessionId: "voice-1" });
expect(toolCall?.[1]?.name).toBe("openclaw_agent_consult");
expect(toolCall?.[1]?.args).toEqual({ question: "Are the basement lights off?" });
expect(submit).toHaveBeenCalledWith("call-1", { result: "Basement lights are off." });
expect(order).toEqual(["flush", "tool-call"]);
});
it("prefers source-reply final text over an earlier empty Talk consult final", async () => {
@@ -45,6 +45,7 @@ type RealtimeTalkAudioContract = {
export type RealtimeTalkWebRtcSdpSessionResult = {
provider: string;
transport: "webrtc";
voiceSessionId?: string;
clientSecret: string;
offerUrl?: string;
offerHeaders?: Record<string, string>;
@@ -58,6 +59,7 @@ export type RealtimeTalkWebRtcSdpSessionResult = {
export type RealtimeTalkJsonPcmWebSocketSessionResult = {
provider: string;
transport: "provider-websocket";
voiceSessionId?: string;
protocol: string;
clientSecret: string;
websocketUrl: string;
@@ -73,6 +75,7 @@ export type RealtimeTalkJsonPcmWebSocketSessionResult = {
export type RealtimeTalkGatewayRelaySessionResult = {
provider: string;
transport: "gateway-relay";
voiceSessionId?: string;
relaySessionId: string;
audio: RealtimeTalkAudioContract;
model?: string;
@@ -85,6 +88,7 @@ export type RealtimeTalkGatewayRelaySessionResult = {
type RealtimeTalkManagedRoomSessionResult = {
provider: string;
transport: "managed-room";
voiceSessionId?: string;
roomUrl: string;
token?: string;
model?: string;
@@ -110,6 +114,8 @@ export type RealtimeTalkTransport = {
export type RealtimeTalkTransportContext = {
client: GatewayBrowserClient;
sessionKey: string;
voiceSessionId?: string;
flushTranscriptWrites?: () => Promise<void>;
callbacks: RealtimeTalkCallbacks;
inputDeviceId?: string;
videoDeviceId?: string;
@@ -572,10 +578,12 @@ export async function submitRealtimeTalkConsult(params: {
try {
const args =
typeof params.args === "string" ? JSON.parse(params.args || "{}") : (params.args ?? {});
await ctx.flushTranscriptWrites?.();
const response = await ctx.client.request<{ runId?: string; idempotencyKey?: string }>(
"talk.client.toolCall",
{
sessionKey: ctx.sessionKey,
...(ctx.voiceSessionId ? { voiceSessionId: ctx.voiceSessionId } : {}),
callId,
name: REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
args,
+362 -11
View File
@@ -26,7 +26,7 @@ const {
webRtcSetVideoEnabled: vi.fn(async () => undefined),
googleSwitchCamera: vi.fn(async () => undefined),
webRtcSwitchCamera: vi.fn(async () => undefined),
googleCtor: vi.fn(function () {
googleCtor: vi.fn(function (_session: unknown, _ctx: unknown) {
return {
start: googleStart,
stop: googleStop,
@@ -34,10 +34,10 @@ const {
switchCamera: googleSwitchCamera,
};
}),
relayCtor: vi.fn(function () {
relayCtor: vi.fn(function (_session: unknown, _ctx: unknown) {
return { start: relayStart, stop: relayStop };
}),
webRtcCtor: vi.fn(function () {
webRtcCtor: vi.fn(function (_session: unknown, _ctx: unknown) {
return {
start: webRtcStart,
stop: webRtcStop,
@@ -81,6 +81,7 @@ describe("RealtimeTalkSession", () => {
it("starts the Google Live WebSocket transport from a generic session result", async () => {
const request = vi.fn(async () => ({
provider: "google",
voiceSessionId: "voice-1",
transport: "provider-websocket",
protocol: "google-live-bidi",
clientSecret: "auth_tokens/session",
@@ -97,7 +98,10 @@ describe("RealtimeTalkSession", () => {
await session.start();
expect(request).toHaveBeenCalledWith("talk.client.create", { sessionKey: "main" });
expect(request).toHaveBeenCalledWith("talk.client.create", {
sessionKey: "main",
capabilities: ["voice-transcript"],
});
expect(googleCtor).toHaveBeenCalledTimes(1);
expect(googleStart).toHaveBeenCalledTimes(1);
expect(webRtcCtor).not.toHaveBeenCalled();
@@ -108,6 +112,7 @@ describe("RealtimeTalkSession", () => {
it("defaults legacy session results without an explicit transport to WebRTC", async () => {
const request = vi.fn(async () => ({
provider: "openai",
voiceSessionId: "voice-1",
clientSecret: "auth_tokens/session",
}));
const session = new RealtimeTalkSession({ request } as never, "main");
@@ -122,6 +127,7 @@ describe("RealtimeTalkSession", () => {
it("accepts legacy WebRTC transport names", async () => {
const request = vi.fn(async () => ({
provider: "openai",
voiceSessionId: "voice-1",
transport: "webrtc-sdp",
clientSecret: "secret",
}));
@@ -136,6 +142,7 @@ describe("RealtimeTalkSession", () => {
it("accepts legacy provider WebSocket transport names", async () => {
const request = vi.fn(async () => ({
provider: "example",
voiceSessionId: "voice-1",
transport: "json-pcm-websocket",
clientSecret: "secret",
protocol: "google-live-bidi",
@@ -209,6 +216,7 @@ describe("RealtimeTalkSession", () => {
sessionKey: "main",
provider: "xai",
transport: "gateway-relay",
capabilities: ["voice-transcript"],
});
expect(request).toHaveBeenNthCalledWith(2, "talk.session.create", {
sessionKey: "main",
@@ -263,7 +271,7 @@ describe("RealtimeTalkSession", () => {
sessionKey: "main",
provider: "openai",
transport: "gateway-relay",
capabilities: ["camera-frame"],
capabilities: ["voice-transcript", "camera-frame"],
});
expect(request).toHaveBeenNthCalledWith(3, "talk.session.create", {
sessionKey: "main",
@@ -280,6 +288,7 @@ describe("RealtimeTalkSession", () => {
it("starts the WebRTC transport for canonical WebRTC sessions", async () => {
const request = vi.fn(async () => ({
provider: "openai",
voiceSessionId: "voice-1",
transport: "webrtc",
clientSecret: "secret",
}));
@@ -298,6 +307,7 @@ describe("RealtimeTalkSession", () => {
it("passes launch options to client-owned realtime session creation", async () => {
const request = vi.fn(async () => ({
provider: "openai",
voiceSessionId: "voice-1",
transport: "webrtc",
clientSecret: "secret",
}));
@@ -330,6 +340,7 @@ describe("RealtimeTalkSession", () => {
silenceDurationMs: 650,
prefixPaddingMs: 250,
reasoningEffort: "low",
capabilities: ["voice-transcript"],
});
expect(webRtcCtor).toHaveBeenCalledWith(
expect.any(Object),
@@ -349,6 +360,7 @@ describe("RealtimeTalkSession", () => {
}
return {
provider: "openai",
voiceSessionId: "voice-1",
transport: "webrtc",
clientSecret: "secret",
};
@@ -363,7 +375,7 @@ describe("RealtimeTalkSession", () => {
expect(request).toHaveBeenNthCalledWith(1, "talk.catalog", {});
expect(request).toHaveBeenNthCalledWith(2, "talk.client.create", {
sessionKey: "main",
capabilities: ["camera-frame"],
capabilities: ["voice-transcript", "camera-frame"],
});
expect(onVideoCapability).toHaveBeenCalledWith(true);
expect(webRtcCtor).toHaveBeenCalledWith(
@@ -392,6 +404,7 @@ describe("RealtimeTalkSession", () => {
return {
provider: "openai",
transport: "webrtc",
voiceSessionId: "voice-settings-camera",
clientSecret: "secret",
};
});
@@ -424,7 +437,12 @@ describe("RealtimeTalkSession", () => {
},
};
}
return { provider: "openai", transport: "webrtc", clientSecret: "secret" };
return {
provider: "openai",
transport: "webrtc",
voiceSessionId: "voice-pending-camera",
clientSecret: "secret",
};
});
const session = new RealtimeTalkSession({ request } as never, "main", {
onVideoCapability: vi.fn(),
@@ -454,6 +472,7 @@ describe("RealtimeTalkSession", () => {
}
return {
provider: "openai",
voiceSessionId: "voice-1",
transport: "webrtc",
clientSecret: "secret",
};
@@ -465,7 +484,10 @@ describe("RealtimeTalkSession", () => {
await session.start();
expect(request).toHaveBeenNthCalledWith(2, "talk.client.create", { sessionKey: "main" });
expect(request).toHaveBeenNthCalledWith(2, "talk.client.create", {
sessionKey: "main",
capabilities: ["voice-transcript"],
});
expect(onVideoCapability).toHaveBeenCalledWith(false);
});
@@ -491,7 +513,7 @@ describe("RealtimeTalkSession", () => {
await expect(session.start()).rejects.toBe(clientError);
expect(request.mock.calls).toEqual([
["talk.client.create", { sessionKey: "main" }],
["talk.client.create", { sessionKey: "main", capabilities: ["voice-transcript"] }],
["talk.config", {}],
]);
expect(relayCtor).not.toHaveBeenCalled();
@@ -592,7 +614,7 @@ describe("RealtimeTalkSession", () => {
await expect(session.start()).rejects.toBe(clientError);
expect(request.mock.calls).toEqual([
["talk.client.create", { sessionKey: "main" }],
["talk.client.create", { sessionKey: "main", capabilities: ["voice-transcript"] }],
["talk.config", {}],
]);
expect(relayCtor).not.toHaveBeenCalled();
@@ -614,9 +636,338 @@ describe("RealtimeTalkSession", () => {
await expect(session.start()).rejects.toBe(clientError);
expect(request.mock.calls).toEqual([
["talk.client.create", { sessionKey: "main" }],
["talk.client.create", { sessionKey: "main", capabilities: ["voice-transcript"] }],
["talk.config", {}],
]);
expect(relayCtor).not.toHaveBeenCalled();
});
it("retries finalized transcript writes in order", async () => {
vi.useFakeTimers();
try {
const transcriptEntryIds: string[] = [];
let firstAttempt = true;
const request = vi.fn(async (method: string, params?: { entryId?: string }) => {
if (method === "talk.client.create") {
return {
provider: "openai",
transport: "webrtc",
voiceSessionId: "voice-queue",
clientSecret: "secret",
};
}
if (method === "talk.client.transcript") {
transcriptEntryIds.push(String(params?.entryId));
if (params?.entryId === "1" && firstAttempt) {
firstAttempt = false;
throw new Error("temporary failure");
}
return { ok: true };
}
return { ok: true };
});
const session = new RealtimeTalkSession({ request } as never, "agent:main:main");
await session.start();
const context = webRtcCtor.mock.calls[0]?.[1] as {
callbacks: {
onTranscript?: (entry: {
role: "user" | "assistant";
text: string;
final: boolean;
}) => void;
};
};
context.callbacks.onTranscript?.({ role: "user", text: "first", final: true });
context.callbacks.onTranscript?.({ role: "assistant", text: "second", final: true });
await vi.advanceTimersByTimeAsync(500);
await vi.waitFor(() => expect(transcriptEntryIds).toEqual(["1", "1", "2"]));
session.stop();
await vi.runAllTimersAsync();
} finally {
vi.useRealTimers();
}
});
it("continues entry ids when the same voice session replaces its transport", async () => {
const transcriptEntryIds: string[] = [];
const request = vi.fn(async (method: string, params?: { entryId?: string }) => {
if (method === "talk.client.create") {
return {
provider: "openai",
transport: "webrtc",
voiceSessionId: "voice-resume",
clientSecret: "secret",
};
}
if (method === "talk.client.transcript") {
transcriptEntryIds.push(String(params?.entryId));
}
return { ok: true };
});
const session = new RealtimeTalkSession({ request } as never, "agent:main:main", {});
await session.start();
const firstContext = webRtcCtor.mock.calls[0]?.[1] as {
callbacks: {
onTranscript?: (entry: {
role: "user" | "assistant";
text: string;
final: boolean;
}) => void;
};
flushTranscriptWrites?: () => Promise<void>;
};
firstContext.callbacks.onTranscript?.({ role: "user", text: "first", final: true });
await firstContext.flushTranscriptWrites?.();
await session.start();
const secondContext = webRtcCtor.mock.calls[1]?.[1] as typeof firstContext;
secondContext.callbacks.onTranscript?.({ role: "assistant", text: "second", final: true });
await secondContext.flushTranscriptWrites?.();
expect(transcriptEntryIds).toEqual(["1", "2"]);
expect(request.mock.calls.filter(([method]) => method === "talk.client.create")).toEqual([
["talk.client.create", { sessionKey: "agent:main:main", capabilities: ["voice-transcript"] }],
[
"talk.client.create",
{
sessionKey: "agent:main:main",
voiceSessionId: "voice-resume",
capabilities: ["voice-transcript"],
},
],
]);
session.stop();
await Promise.resolve();
});
it("surfaces transcript failure after three attempts", async () => {
vi.useFakeTimers();
const warn = vi.spyOn(console, "warn").mockImplementation(() => undefined);
try {
const request = vi.fn(async (method: string) => {
if (method === "talk.client.create") {
return {
provider: "openai",
transport: "webrtc",
voiceSessionId: "voice-failure",
clientSecret: "secret",
};
}
if (method === "talk.client.transcript") {
throw new Error("still unavailable");
}
return { ok: true };
});
const onStatus = vi.fn();
const session = new RealtimeTalkSession({ request } as never, "agent:main:main", {
onStatus,
});
await session.start();
const context = webRtcCtor.mock.calls[0]?.[1] as {
callbacks: {
onTranscript?: (entry: {
role: "user" | "assistant";
text: string;
final: boolean;
}) => void;
};
};
context.callbacks.onTranscript?.({ role: "user", text: "save me", final: true });
await vi.advanceTimersByTimeAsync(2_500);
await vi.waitFor(() =>
expect(onStatus).toHaveBeenCalledWith(
"error",
expect.stringContaining("Voice transcript could not be saved"),
),
);
expect(
request.mock.calls.filter(([method]) => method === "talk.client.transcript"),
).toHaveLength(3);
expect(warn).toHaveBeenCalled();
session.stop();
await vi.runAllTimersAsync();
} finally {
warn.mockRestore();
vi.useRealTimers();
}
});
it("retries logical voice-session close after transient failures", async () => {
vi.useFakeTimers();
try {
let closeAttempts = 0;
const request = vi.fn(async (method: string) => {
if (method === "talk.client.create") {
return {
provider: "openai",
transport: "webrtc",
voiceSessionId: "voice-close-retry",
clientSecret: "secret",
};
}
if (method === "talk.client.close" && ++closeAttempts < 3) {
throw new Error("temporary close failure");
}
return { ok: true };
});
const session = new RealtimeTalkSession({ request } as never, "agent:main:main");
await session.start();
session.stop();
await vi.runAllTimersAsync();
expect(closeAttempts).toBe(3);
} finally {
vi.useRealTimers();
}
});
it("starts a new call without resuming the voice session being closed", async () => {
let createCount = 0;
let finishClose: (() => void) | undefined;
const closing = new Promise<void>((resolve) => {
finishClose = resolve;
});
const request = vi.fn(async (method: string) => {
if (method === "talk.client.create") {
createCount += 1;
return {
provider: "openai",
transport: "webrtc",
voiceSessionId: `voice-${createCount}`,
clientSecret: "secret",
};
}
if (method === "talk.client.close") {
await closing;
}
return { ok: true };
});
const session = new RealtimeTalkSession({ request } as never, "agent:main:main");
await session.start();
session.stop();
await session.start();
const creates = request.mock.calls.filter(([method]) => method === "talk.client.create");
expect(creates).toEqual([
["talk.client.create", { sessionKey: "agent:main:main", capabilities: ["voice-transcript"] }],
["talk.client.create", { sessionKey: "agent:main:main", capabilities: ["voice-transcript"] }],
]);
finishClose?.();
await Promise.resolve();
});
it("ignores final transcript callbacks emitted after shutdown begins", async () => {
const request = vi.fn(async (method: string) => {
if (method === "talk.client.create") {
return {
provider: "openai",
transport: "webrtc",
voiceSessionId: "voice-shutdown",
clientSecret: "secret",
};
}
return { ok: true };
});
const session = new RealtimeTalkSession({ request } as never, "agent:main:main");
await session.start();
const context = webRtcCtor.mock.calls[0]?.[1] as {
callbacks: {
onTranscript?: (entry: {
role: "user" | "assistant";
text: string;
final: boolean;
}) => void;
};
};
session.stop();
context.callbacks.onTranscript?.({ role: "user", text: "too late", final: true });
await Promise.resolve();
expect(request.mock.calls.some(([method]) => method === "talk.client.transcript")).toBe(false);
});
it("drops a previous transport's delayed transcript after stop and restart", async () => {
let createCount = 0;
const request = vi.fn(async (method: string) => {
if (method === "talk.client.create") {
createCount += 1;
return {
provider: "openai",
transport: "webrtc",
voiceSessionId: `voice-${createCount}`,
clientSecret: "secret",
};
}
return { ok: true };
});
const onTranscript = vi.fn();
const session = new RealtimeTalkSession({ request } as never, "agent:main:main", {
onTranscript,
});
await session.start();
const previousContext = webRtcCtor.mock.calls[0]?.[1] as {
callbacks: {
onTranscript?: (entry: {
role: "user" | "assistant";
text: string;
final: boolean;
}) => void;
};
};
session.stop();
await session.start();
previousContext.callbacks.onTranscript?.({
role: "user",
text: "stale transcript",
final: true,
});
await Promise.resolve();
expect(onTranscript).not.toHaveBeenCalled();
expect(request.mock.calls.some(([method]) => method === "talk.client.transcript")).toBe(false);
});
it("does not report Gateway relay transcripts through the client RPC", async () => {
const request = vi.fn(async (method: string) => {
if (method === "talk.client.create") {
return {
provider: "openai",
transport: "gateway-relay",
relaySessionId: "relay-voice",
audio: {
inputEncoding: "pcm16",
inputSampleRateHz: 24_000,
outputEncoding: "pcm16",
outputSampleRateHz: 24_000,
},
};
}
return { ok: true };
});
const session = new RealtimeTalkSession({ request } as never, "agent:main:main");
await session.start();
const context = relayCtor.mock.calls[0]?.[1] as {
callbacks: {
onTranscript?: (entry: {
role: "user" | "assistant";
text: string;
final: boolean;
}) => void;
};
};
context.callbacks.onTranscript?.({ role: "user", text: "server owns this", final: true });
await Promise.resolve();
expect(request.mock.calls.some(([method]) => method === "talk.client.transcript")).toBe(false);
session.stop();
await Promise.resolve();
expect(request.mock.calls.some(([method]) => method === "talk.client.close")).toBe(false);
});
});
+214 -13
View File
@@ -58,6 +58,13 @@ export async function switchActiveRealtimeTalkCameras(
type RealtimeTalkLaunchTransport = NonNullable<RealtimeTalkLaunchOptions["transport"]>;
type DetachedVoiceSession = {
voiceSessionId: string;
serverOwned: boolean;
generation: number;
transcriptWrites: Promise<void>;
};
type RealtimeTalkConfigResult = {
config?: {
talk?: {
@@ -104,9 +111,6 @@ function createTransport(
ctx,
);
}
if (transport === "managed-room") {
throw new Error("Managed-room realtime Talk sessions are not available in this UI yet");
}
const unknownTransport = (session as { transport?: string }).transport ?? "unknown";
throw new Error(`Unsupported realtime Talk transport: ${unknownTransport}`);
}
@@ -115,8 +119,17 @@ function resolveTransport(session: RealtimeTalkSessionResult): string {
return normalizeTalkTransport((session as { transport?: string }).transport) ?? "webrtc";
}
function transcriptWriteError(error: unknown, fallback: string): Error {
return error instanceof Error ? error : new Error(fallback, { cause: error });
}
function compactLaunchParams(
params: RealtimeTalkLaunchOptions & { sessionKey: string; mode?: string; brain?: string },
params: RealtimeTalkLaunchOptions & {
sessionKey: string;
voiceSessionId?: string;
mode?: string;
brain?: string;
},
): Record<string, unknown> {
return Object.fromEntries(Object.entries(params).filter(([, value]) => value !== undefined));
}
@@ -126,6 +139,12 @@ export class RealtimeTalkSession {
private closed = false;
private videoEnabled = false;
private videoOperation = 0;
private voiceSessionId: string | undefined;
private transportGeneration = 0;
private readonly transcriptSeqByVoiceSessionId = new Map<string, number>();
private acceptingTranscripts = false;
private serverOwnedVoiceSession = false;
private transcriptWrites: Promise<void> = Promise.resolve();
constructor(
private readonly client: GatewayBrowserClient,
@@ -142,16 +161,48 @@ export class RealtimeTalkSession {
if (this.closed) {
return;
}
const session = await this.createSession(
providerVideoCapable ? { ...this.options, capabilities: ["camera-frame"] } : this.options,
);
// Declaring voice-transcript arms the server-side spoken-confirmation gate;
// this client reports every finalized utterance, so the gate is completable.
const capabilities: Array<"camera-frame" | "voice-transcript"> = ["voice-transcript"];
if (providerVideoCapable) {
capabilities.push("camera-frame");
}
const session = await this.createSession({ ...this.options, capabilities });
const transport = resolveTransport(session);
// Managed-room stays unsupported here and carries no voice bookkeeping;
// reject it before the voice-session requirement produces a misleading error.
if (transport === "managed-room") {
throw new Error("Managed-room realtime Talk sessions are not available in this UI yet");
}
const voiceSessionId =
session.voiceSessionId ??
(transport === "gateway-relay"
? (session as RealtimeTalkGatewayRelaySessionResult).relaySessionId
: undefined);
if (!voiceSessionId) {
throw new Error("Realtime Talk session did not return a voice session id");
}
this.voiceSessionId = voiceSessionId;
this.acceptingTranscripts = true;
this.serverOwnedVoiceSession = transport === "gateway-relay";
if (this.closed) {
const detached = this.detachVoiceSession();
if (detached) {
this.closeLogicalVoiceSession(detached);
}
return;
}
this.transportGeneration += 1;
const callbacks =
transport === "gateway-relay"
? this.callbacks
: this.clientOwnedTranscriptCallbacks(voiceSessionId, this.transportGeneration);
this.transport = createTransport(session, {
client: this.client,
sessionKey: this.sessionKey,
callbacks: this.callbacks,
voiceSessionId,
flushTranscriptWrites: async () => await this.transcriptWrites,
callbacks,
inputDeviceId: this.localOptions.inputDeviceId,
videoDeviceId: this.localOptions.videoDeviceId,
consultThinkingLevel: session.consultThinkingLevel,
@@ -185,18 +236,22 @@ export class RealtimeTalkSession {
}
private async createSession(
options: RealtimeTalkLaunchOptions & { capabilities?: Array<"camera-frame"> },
options: RealtimeTalkLaunchOptions & {
capabilities?: Array<"camera-frame" | "voice-transcript">;
},
): Promise<RealtimeTalkSessionResult> {
const launchOptions = { ...options };
try {
return await this.client.request<RealtimeTalkSessionResult>(
"talk.client.create",
compactLaunchParams({
sessionKey: this.sessionKey,
...options,
voiceSessionId: this.voiceSessionId,
...launchOptions,
}),
);
} catch (error) {
let transport = options.transport;
let transport = launchOptions.transport;
if (!transport) {
let result: RealtimeTalkConfigResult;
try {
@@ -218,10 +273,10 @@ export class RealtimeTalkSession {
if (transport && transport !== "gateway-relay") {
throw error;
}
const gatewayOptions = { ...options };
const gatewayOptions = { ...launchOptions };
delete gatewayOptions.capabilities;
try {
return await this.client.request<RealtimeTalkSessionResult>(
const relaySession = await this.client.request<RealtimeTalkSessionResult>(
"talk.session.create",
compactLaunchParams({
sessionKey: this.sessionKey,
@@ -231,6 +286,13 @@ export class RealtimeTalkSession {
brain: "agent-consult",
}),
);
return resolveTransport(relaySession) === "gateway-relay"
? {
...relaySession,
voiceSessionId: (relaySession as RealtimeTalkGatewayRelaySessionResult)
.relaySessionId,
}
: relaySession;
} catch {
throw error;
}
@@ -243,8 +305,147 @@ export class RealtimeTalkSession {
this.videoEnabled = false;
activeRealtimeTalkSessions.delete(this);
this.callbacks.onStatus?.("idle");
const detached = this.detachVoiceSession();
this.transport?.stop();
this.transport = null;
if (detached) {
this.closeLogicalVoiceSession(detached);
}
}
private clientOwnedTranscriptCallbacks(
owningVoiceSessionId: string,
owningGeneration: number,
): RealtimeTalkCallbacks {
return {
...this.callbacks,
onTranscript: (entry) => {
// Transport replacement can reuse the voice session id, so a retired
// transport's late finals are fenced by generation, not id alone.
if (
this.transportGeneration !== owningGeneration ||
this.voiceSessionId !== owningVoiceSessionId ||
!this.acceptingTranscripts
) {
return;
}
// Persist before notifying: a consumer callback that stops or throws must
// not be able to drop an already-finalized utterance from the write tail.
if (entry.final) {
const transcriptSeq =
(this.transcriptSeqByVoiceSessionId.get(owningVoiceSessionId) ?? 0) + 1;
this.transcriptSeqByVoiceSessionId.set(owningVoiceSessionId, transcriptSeq);
const entryId = String(transcriptSeq);
// One promise tail preserves transcript order and makes consult flushes
// observe the same dedupe sequence used by close.
const write = this.transcriptWrites.then(async () => {
await this.writeTranscriptWithRetry({
voiceSessionId: owningVoiceSessionId,
entryId,
role: entry.role,
text: entry.text,
});
});
this.transcriptWrites = write.catch((error: unknown) => {
// The utterance exists only in client memory; after retries and surfacing the error,
// keeping the record open cannot recover it, while server entryId dedupe preserves order.
// Deferring close would only shift the identical loss to the 6h stale sweep.
const detail = `Voice transcript could not be saved: ${error instanceof Error ? error.message : String(error)}`;
console.warn(detail, error);
// Only surface to the user if this transport is still the active one; a
// retired call's late failure must not error a healthy replacement call.
if (this.transportGeneration === owningGeneration) {
this.callbacks.onStatus?.("error", detail);
}
});
}
this.callbacks.onTranscript?.(entry);
},
};
}
private async writeTranscriptWithRetry(params: {
voiceSessionId: string;
entryId: string;
role: "user" | "assistant";
text: string;
}): Promise<void> {
const retryDelaysMs = [0, 500, 2_000];
let lastError: unknown;
for (const delayMs of retryDelaysMs) {
if (delayMs > 0) {
await new Promise<void>((resolve) => {
setTimeout(resolve, delayMs);
});
}
try {
await this.client.request("talk.client.transcript", {
sessionKey: this.sessionKey,
voiceSessionId: params.voiceSessionId,
entryId: params.entryId,
role: params.role,
text: params.text,
timestamp: Date.now(),
});
return;
} catch (error) {
lastError = error;
}
}
throw transcriptWriteError(lastError, "voice transcript save failed");
}
private detachVoiceSession(): DetachedVoiceSession | undefined {
const voiceSessionId = this.voiceSessionId;
if (!voiceSessionId) {
return undefined;
}
const detached = {
voiceSessionId,
serverOwned: this.serverOwnedVoiceSession,
generation: this.transportGeneration,
transcriptWrites: this.transcriptWrites,
} satisfies DetachedVoiceSession;
this.voiceSessionId = undefined;
this.acceptingTranscripts = false;
this.serverOwnedVoiceSession = false;
this.transcriptWrites = Promise.resolve();
return detached;
}
private closeLogicalVoiceSession(detached: DetachedVoiceSession): void {
if (detached.serverOwned) {
return;
}
void detached.transcriptWrites
.then(async () => {
let lastError: unknown;
for (const delayMs of [0, 500, 2_000]) {
if (delayMs > 0) {
await new Promise<void>((resolve) => {
setTimeout(resolve, delayMs);
});
}
try {
await this.client.request("talk.client.close", {
sessionKey: this.sessionKey,
voiceSessionId: detached.voiceSessionId,
});
return;
} catch (error) {
lastError = error;
}
}
throw transcriptWriteError(lastError, "Realtime Talk voice session close failed");
})
.catch((error: unknown) => {
console.warn("Realtime Talk voice session close failed", error);
// Suppress if a newer transport has started: closing the old call is its own
// teardown and must not push the active replacement call into an error state.
if (this.transportGeneration === detached.generation) {
this.callbacks.onStatus?.("error", "Realtime Talk voice session close failed");
}
});
}
async setVideoEnabled(enabled: boolean): Promise<void> {