furyhawk/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-07-21 10:16:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(android): derive Voice readiness from Gateway catalog (#98269)

Browse Source 
* fix(android): derive voice readiness from Gateway catalog

Co-authored-by: Colin <colin@solvely.net>

* chore(android): sync native i18n inventory

* test(gateway): use registered realtime provider ids

* fix(gateway): satisfy Talk catalog lint

* chore(android): refresh voice i18n inventory

* fix(talk): preserve runtime readiness semantics

* fix(talk): make catalog readiness authoritative

* fix(talk): validate selected provider readiness

* fix(android): honor authoritative talk readiness

* fix(android): inventory voice readiness copy

* fix(talk): report runtime-selected catalog provider

---------

Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
Colin Johnson
2026-07-02 23:32:03 -07:00
committed by GitHub
co-authored by Peter Steinberger
parent 4e6933c84f
commit 7cfc66ad07
16 changed files with 1353 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/.i18n/native-source.json
+295 -183
View File
File diff suppressed because it is too large Load Diff
apps/android/app/src/main/java/ai/openclaw/app/GatewayTalkSetupReadiness.kt
+216
View File
@@ -0,0 +1,216 @@
package ai.openclaw.app
import ai.openclaw.app.node.asObjectOrNull
import ai.openclaw.app.node.asStringOrNull
import kotlinx.serialization.json.JsonArray
import kotlinx.serialization.json.JsonElement
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.JsonPrimitive
import kotlinx.serialization.json.booleanOrNull
data class GatewayTalkSetupReadiness(
val realtimeTalk: GatewayTalkSetupState,
val dictation: GatewayTalkSetupState,
) {
companion object {
fun unverified(
issue: GatewayTalkSetupIssue = GatewayTalkSetupIssue.CatalogNotLoaded,
): GatewayTalkSetupReadiness =
GatewayTalkSetupReadiness(
realtimeTalk = GatewayTalkSetupState.Unverified(issue),
dictation = GatewayTalkSetupState.Unverified(issue),
)
}
}
sealed interface GatewayTalkSetupState {
data class Ready(
val provider: GatewayTalkProvider,
) : GatewayTalkSetupState
data class NeedsSetup(
val issue: GatewayTalkSetupIssue,
val provider: GatewayTalkProvider? = null,
) : GatewayTalkSetupState
/** Catalog failures must not disable a startup path that the Gateway still validates. */
data class Unverified(
val issue: GatewayTalkSetupIssue,
) : GatewayTalkSetupState
}
enum class GatewayTalkSetupTarget(
val title: String,
) {
REALTIME_TALK("Realtime Talk"),
DICTATION("Dictation"),
}
sealed interface GatewayTalkSetupIssue {
data object CatalogNotLoaded : GatewayTalkSetupIssue
data object CatalogLoadFailed : GatewayTalkSetupIssue
data class GroupMissing(
val target: GatewayTalkSetupTarget,
) : GatewayTalkSetupIssue
data class NoProvider(
val target: GatewayTalkSetupTarget,
) : GatewayTalkSetupIssue
data class UnknownProvider(
val target: GatewayTalkSetupTarget,
val providerId: String,
) : GatewayTalkSetupIssue
data class MissingReadiness(
val target: GatewayTalkSetupTarget,
) : GatewayTalkSetupIssue
data class ConfigureProvider(
val target: GatewayTalkSetupTarget,
) : GatewayTalkSetupIssue
data class MissingActiveProvider(
val target: GatewayTalkSetupTarget,
) : GatewayTalkSetupIssue
data class UnsupportedProvider(
val target: GatewayTalkSetupTarget,
) : GatewayTalkSetupIssue
data class ConfigureSelectedProvider(
val providerLabel: String,
) : GatewayTalkSetupIssue
}
data class GatewayTalkProvider(
val id: String,
val label: String,
)
val GatewayTalkSetupState.isReady: Boolean
get() = this is GatewayTalkSetupState.Ready
val GatewayTalkSetupState.requiresSetup: Boolean
get() = this is GatewayTalkSetupState.NeedsSetup
fun gatewayTalkSetupStatusText(state: GatewayTalkSetupState): String =
when (state) {
is GatewayTalkSetupState.Ready -> "Ready"
is GatewayTalkSetupState.NeedsSetup -> "Needs setup"
is GatewayTalkSetupState.Unverified -> "Unverified"
}
fun gatewayTalkSetupDescription(state: GatewayTalkSetupState): String =
when (state) {
is GatewayTalkSetupState.Ready -> "${state.provider.label} via Gateway relay"
is GatewayTalkSetupState.NeedsSetup -> gatewayTalkSetupIssueDescription(state.issue)
is GatewayTalkSetupState.Unverified -> gatewayTalkSetupIssueDescription(state.issue)
}
private fun gatewayTalkSetupIssueDescription(issue: GatewayTalkSetupIssue): String =
when (issue) {
GatewayTalkSetupIssue.CatalogNotLoaded -> "Gateway talk catalog not loaded"
GatewayTalkSetupIssue.CatalogLoadFailed -> "Could not load Gateway talk catalog"
is GatewayTalkSetupIssue.GroupMissing -> "Gateway did not return ${issue.target.title} setup"
is GatewayTalkSetupIssue.NoProvider -> "No ${issue.target.title} provider is configured on the Gateway"
is GatewayTalkSetupIssue.UnknownProvider -> "Gateway selected unknown provider ${issue.providerId}"
is GatewayTalkSetupIssue.MissingReadiness -> "Gateway did not return ${issue.target.title} readiness"
is GatewayTalkSetupIssue.ConfigureProvider -> "Configure a ${issue.target.title} provider on the Gateway"
is GatewayTalkSetupIssue.MissingActiveProvider ->
"Gateway did not identify the active ${issue.target.title} provider"
is GatewayTalkSetupIssue.UnsupportedProvider ->
"Choose a supported ${issue.target.title} provider on the Gateway"
is GatewayTalkSetupIssue.ConfigureSelectedProvider -> "Configure ${issue.providerLabel} on the Gateway"
}
internal fun parseGatewayTalkSetupReadiness(catalog: JsonObject?): GatewayTalkSetupReadiness {
if (catalog == null) return GatewayTalkSetupReadiness.unverified()
return GatewayTalkSetupReadiness(
realtimeTalk =
parseTalkCatalogGroup(catalog = catalog, key = "realtime", target = GatewayTalkSetupTarget.REALTIME_TALK),
dictation =
parseTalkCatalogGroup(catalog = catalog, key = "transcription", target = GatewayTalkSetupTarget.DICTATION),
)
}
private fun parseTalkCatalogGroup(
catalog: JsonObject,
key: String,
target: GatewayTalkSetupTarget,
): GatewayTalkSetupState {
val group =
catalog[key].asObjectOrNull()
?: return GatewayTalkSetupState.Unverified(GatewayTalkSetupIssue.GroupMissing(target))
val providers =
(group["providers"] as? JsonArray)
?.mapNotNull(::parseTalkCatalogProvider)
.orEmpty()
val ready = (group["ready"] as? JsonPrimitive)?.booleanOrNull
val activeProviderId = group["activeProvider"].asStringOrNull()?.trim()?.takeIf(String::isNotEmpty)
if (providers.isEmpty()) {
return when {
ready == false -> GatewayTalkSetupState.NeedsSetup(GatewayTalkSetupIssue.NoProvider(target))
activeProviderId != null ->
GatewayTalkSetupState.Unverified(GatewayTalkSetupIssue.UnknownProvider(target, activeProviderId))
else -> GatewayTalkSetupState.Unverified(GatewayTalkSetupIssue.MissingReadiness(target))
}
}
if (activeProviderId == null) {
if (ready == false) {
return GatewayTalkSetupState.NeedsSetup(GatewayTalkSetupIssue.ConfigureProvider(target))
}
// Older Gateways can omit the selected provider and report alias-backed rows as unconfigured
// even though session startup resolves them. Only an explicit readiness result is authoritative.
return GatewayTalkSetupState.Unverified(GatewayTalkSetupIssue.MissingActiveProvider(target))
}
val selected =
// Match Gateway registry precedence: canonical ids win before alias fallback.
providers.firstOrNull { it.matchesId(activeProviderId) }
?: providers.firstOrNull { it.matchesAlias(activeProviderId) }
?: return if (ready == false) {
GatewayTalkSetupState.NeedsSetup(GatewayTalkSetupIssue.UnsupportedProvider(target))
} else {
GatewayTalkSetupState.Unverified(GatewayTalkSetupIssue.UnknownProvider(target, activeProviderId))
}
val provider = GatewayTalkProvider(id = selected.id, label = selected.label)
return when (ready) {
true -> GatewayTalkSetupState.Ready(provider)
false ->
GatewayTalkSetupState.NeedsSetup(
issue = GatewayTalkSetupIssue.ConfigureSelectedProvider(selected.label),
provider = provider,
)
null -> GatewayTalkSetupState.Unverified(GatewayTalkSetupIssue.MissingReadiness(target))
}
}
private data class TalkCatalogProvider(
val id: String,
val label: String,
val configured: Boolean,
val aliases: List<String>,
) {
fun matchesId(candidate: String): Boolean = id.equals(candidate, ignoreCase = true)
fun matchesAlias(candidate: String): Boolean = aliases.any { it.equals(candidate, ignoreCase = true) }
}
private fun parseTalkCatalogProvider(item: JsonElement): TalkCatalogProvider? {
val value = item.asObjectOrNull() ?: return null
val id = value["id"].asStringOrNull()?.trim()?.takeIf(String::isNotEmpty) ?: return null
val label = value["label"].asStringOrNull()?.trim()?.takeIf(String::isNotEmpty) ?: id
val aliases =
(value["aliases"] as? JsonArray)
?.mapNotNull { it.asStringOrNull()?.trim()?.takeIf(String::isNotEmpty) }
.orEmpty()
return TalkCatalogProvider(
id = id,
label = label,
configured = (value["configured"] as? JsonPrimitive)?.booleanOrNull == true,
aliases = aliases,
)
}
apps/android/app/src/main/java/ai/openclaw/app/MainViewModel.kt
+6
View File
@@ -127,6 +127,8 @@ class MainViewModel(
val modelAuthProviders: StateFlow<List<GatewayModelProviderSummary>> = runtimeState(initial = emptyList()) { it.modelAuthProviders }
val modelCatalogRefreshing: StateFlow<Boolean> = runtimeState(initial = false) { it.modelCatalogRefreshing }
val modelCatalogErrorText: StateFlow<String?> = runtimeState(initial = null) { it.modelCatalogErrorText }
val talkSetupReadiness: StateFlow<GatewayTalkSetupReadiness> =
runtimeState(initial = GatewayTalkSetupReadiness.unverified()) { it.talkSetupReadiness }
val gatewayDefaultAgentId: StateFlow<String?> = runtimeState(initial = null) { it.gatewayDefaultAgentId }
val gatewayAgents: StateFlow<List<GatewayAgentSummary>> = runtimeState(initial = emptyList()) { it.gatewayAgents }
val cronStatus: StateFlow<GatewayCronStatus> = runtimeState(initial = GatewayCronStatus(enabled = false, jobs = 0, nextWakeAtMs = null)) { it.cronStatus }
@@ -527,6 +529,10 @@ class MainViewModel(
ensureRuntime().refreshModelCatalog()
}
fun refreshTalkSetupReadiness() {
ensureRuntime().refreshTalkSetupReadiness()
}
fun refreshAgents() {
ensureRuntime().refreshAgents()
}
apps/android/app/src/main/java/ai/openclaw/app/NodeRuntime.kt
+24
View File
@@ -549,6 +549,8 @@ class NodeRuntime(
val modelCatalogRefreshing: StateFlow<Boolean> = _modelCatalogRefreshing.asStateFlow()
private val _modelCatalogErrorText = MutableStateFlow<String?>(null)
val modelCatalogErrorText: StateFlow<String?> = _modelCatalogErrorText.asStateFlow()
private val _talkSetupReadiness = MutableStateFlow(GatewayTalkSetupReadiness.unverified())
val talkSetupReadiness: StateFlow<GatewayTalkSetupReadiness> = _talkSetupReadiness.asStateFlow()
private val _gatewayDefaultAgentId = MutableStateFlow<String?>(null)
val gatewayDefaultAgentId: StateFlow<String?> = _gatewayDefaultAgentId.asStateFlow()
private val _gatewayAgents = MutableStateFlow<List<GatewayAgentSummary>>(emptyList())
@@ -667,6 +669,7 @@ class NodeRuntime(
_gatewayAgents.value = emptyList()
_modelCatalog.value = emptyList()
_modelAuthProviders.value = emptyList()
_talkSetupReadiness.value = GatewayTalkSetupReadiness.unverified()
_cronStatus.value = GatewayCronStatus(enabled = false, jobs = 0, nextWakeAtMs = null)
_cronJobs.value = emptyList()
_usageSummary.value = GatewayUsageSummary(updatedAtMs = null, providers = emptyList())
@@ -1052,6 +1055,7 @@ class NodeRuntime(
refreshBrandingFromGateway()
refreshAgentsFromGateway()
refreshModelCatalogFromGateway()
refreshTalkSetupReadinessFromGateway()
refreshCronFromGateway()
refreshUsageFromGateway()
refreshSkillsFromGateway()
@@ -1068,6 +1072,10 @@ class NodeRuntime(
}
}
fun refreshTalkSetupReadiness() {
scope.launch { refreshTalkSetupReadinessFromGateway() }
}
fun refreshAgents() {
scope.launch {
refreshAgentsFromGateway()
@@ -1501,6 +1509,8 @@ class NodeRuntime(
fun setVoiceScreenActive(active: Boolean) {
if (!active) {
stopManualVoiceSession()
} else {
refreshTalkSetupReadiness()
}
// Don't re-enable on active=true; mic toggle drives that
}
@@ -2322,6 +2332,20 @@ class NodeRuntime(
}
}
private suspend fun refreshTalkSetupReadinessFromGateway() {
if (!operatorConnected) {
_talkSetupReadiness.value = GatewayTalkSetupReadiness.unverified()
return
}
_talkSetupReadiness.value =
try {
val response = operatorSession.request("talk.catalog", "{}")
parseGatewayTalkSetupReadiness(json.parseToJsonElement(response).asObjectOrNull())
} catch (_: Throwable) {
GatewayTalkSetupReadiness.unverified(GatewayTalkSetupIssue.CatalogLoadFailed)
}
}
private suspend fun refreshCronFromGateway() {
_cronRefreshing.value = true
_cronErrorText.value = null
apps/android/app/src/main/java/ai/openclaw/app/ui/SettingsScreens.kt
+30 -27
View File
@@ -8,13 +8,18 @@ import ai.openclaw.app.GatewayConnectionDisplay
import ai.openclaw.app.GatewayConnectionProblem
import ai.openclaw.app.GatewayCronJobSummary
import ai.openclaw.app.GatewayExecApprovalSummary
import ai.openclaw.app.GatewayTalkSetupReadiness
import ai.openclaw.app.GatewayTalkSetupState
import ai.openclaw.app.GatewayUsageProviderSummary
import ai.openclaw.app.LocationMode
import ai.openclaw.app.MainViewModel
import ai.openclaw.app.NotificationPackageFilterMode
import ai.openclaw.app.SensitiveFeatureConfig
import ai.openclaw.app.chat.ChatPendingToolCall
import ai.openclaw.app.gatewayTalkSetupDescription
import ai.openclaw.app.gatewayTalkSetupStatusText
import ai.openclaw.app.hasPhotoReadPermission
import ai.openclaw.app.isReady
import ai.openclaw.app.loadAndroidLicenseNotices
import ai.openclaw.app.node.DeviceNotificationListenerService
import ai.openclaw.app.photoReadPermissionsForRequest
@@ -407,14 +412,16 @@ private fun VoiceSettingsScreen(
onBack: () -> Unit,
) {
val speakerEnabled by viewModel.speakerEnabled.collectAsState()
val micEnabled by viewModel.micEnabled.collectAsState()
val talkModeEnabled by viewModel.talkModeEnabled.collectAsState()
val isConnected by viewModel.isConnected.collectAsState()
val talkSetupReadiness by viewModel.talkSetupReadiness.collectAsState()
LaunchedEffect(isConnected) {
if (isConnected) viewModel.refreshTalkSetupReadiness()
}
SettingsDetailFrame(title = "Talk Provider Setup", subtitle = "Configure voice, transport, and playback.", icon = Icons.Default.Mic, onBack = onBack) {
Column(verticalArrangement = Arrangement.spacedBy(10.dp)) {
VoiceSetupPanel(
voiceActive = micEnabled || talkModeEnabled,
)
VoiceSetupPanel(talkSetupReadiness)
Text(text = "Audio Test", style = ClawTheme.type.section, color = ClawTheme.colors.text)
Text(text = "Check that OpenClaw can speak clearly on this phone.", style = ClawTheme.type.body, color = ClawTheme.colors.textMuted)
SettingsWaveformPanel(active = speakerEnabled, onClick = ::playVoiceSetupTone)
@@ -433,33 +440,29 @@ private fun VoiceSettingsScreen(
@Composable
private fun VoiceSetupPanel(
voiceActive: Boolean,
readiness: GatewayTalkSetupReadiness,
) {
Column(verticalArrangement = Arrangement.spacedBy(9.dp)) {
VoiceSetupActionRow(
title = "Realtime Provider",
subtitle = "Gateway voice relay",
icon = Icons.Default.GraphicEq,
statusText = if (voiceActive) "Live" else "Ready",
ready = true,
)
VoiceSetupActionRow(
title = "Voice",
subtitle = "Voice input",
icon = Icons.Default.Mic,
statusText = "Configured",
ready = true,
)
VoiceSetupActionRow(
title = "Transport",
subtitle = "Socket relay",
icon = Icons.Default.Bolt,
statusText = "Configured",
ready = true,
)
VoiceSetupReadinessRow(title = "Realtime Talk", state = readiness.realtimeTalk, icon = Icons.Default.GraphicEq)
VoiceSetupReadinessRow(title = "Dictation", state = readiness.dictation, icon = Icons.Default.Mic)
}
}
@Composable
private fun VoiceSetupReadinessRow(
title: String,
state: GatewayTalkSetupState,
icon: ImageVector,
) {
VoiceSetupActionRow(
title = title,
subtitle = gatewayTalkSetupDescription(state),
icon = icon,
statusText = gatewayTalkSetupStatusText(state),
ready = state.isReady,
)
}
@Composable
private fun VoiceSetupActionRow(
title: String,
apps/android/app/src/main/java/ai/openclaw/app/ui/VoiceScreen.kt
+69 -14
View File
@@ -1,8 +1,12 @@
package ai.openclaw.app.ui
import ai.openclaw.app.GatewayTalkSetupReadiness
import ai.openclaw.app.MainViewModel
import ai.openclaw.app.R
import ai.openclaw.app.VoiceCaptureMode
import ai.openclaw.app.gatewayTalkSetupDescription
import ai.openclaw.app.isReady
import ai.openclaw.app.requiresSetup
import ai.openclaw.app.ui.design.ClawPanel
import ai.openclaw.app.ui.design.ClawPlainIconButton
import ai.openclaw.app.ui.design.ClawPrimaryButton
@@ -102,6 +106,7 @@ fun VoiceScreen(
val talkModeSpeaking by viewModel.talkModeSpeaking.collectAsState()
val talkModeStatusText by viewModel.talkModeStatusText.collectAsState()
val talkModeConversation by viewModel.talkModeConversation.collectAsState()
val talkSetupReadiness by viewModel.talkSetupReadiness.collectAsState()
var pendingAction by remember { mutableStateOf<VoiceAction?>(null) }
var hasMicPermission by remember { mutableStateOf(context.hasRecordAudioPermission()) }
@@ -109,9 +114,20 @@ fun VoiceScreen(
rememberLauncherForActivityResult(ActivityResultContracts.RequestPermission()) { granted ->
hasMicPermission = granted
if (granted) {
// Gateway readiness can change while the system permission dialog is open.
when (pendingAction) {
VoiceAction.Talk -> viewModel.setTalkModeEnabled(true)
VoiceAction.Dictation -> viewModel.setMicEnabled(true)
VoiceAction.Talk ->
if (talkSetupReadiness.realtimeTalk.requiresSetup) {
onOpenVoiceSettings()
} else {
viewModel.setTalkModeEnabled(true)
}
VoiceAction.Dictation ->
if (talkSetupReadiness.dictation.requiresSetup) {
onOpenVoiceSettings()
} else {
viewModel.setMicEnabled(true)
}
null -> Unit
}
}
@@ -200,6 +216,7 @@ fun VoiceScreen(
micLiveTranscript = micLiveTranscript,
gatewayReady = gatewayReady,
voiceAttentionStatus = voiceAttentionStatus,
talkSetupReadiness = talkSetupReadiness,
onStartTalk = {
runVoiceAction(
action = VoiceAction.Talk,
@@ -224,12 +241,13 @@ fun VoiceScreen(
)
},
onConnectGateway = onOpenGatewaySettings,
onOpenVoiceSettings = onOpenVoiceSettings,
)
if (!hasMicPermission) {
VoicePermissionPanel(
onRequestPermission = {
pendingAction = VoiceAction.Talk
pendingAction = null
requestMicPermission.launch(Manifest.permission.RECORD_AUDIO)
},
)
@@ -599,10 +617,14 @@ private fun VoiceHero(
micLiveTranscript: String?,
gatewayReady: Boolean,
voiceAttentionStatus: String?,
talkSetupReadiness: GatewayTalkSetupReadiness,
onStartTalk: () -> Unit,
onStartDictation: () -> Unit,
onConnectGateway: () -> Unit,
onOpenVoiceSettings: () -> Unit,
) {
val talkNeedsSetup = gatewayReady && talkSetupReadiness.realtimeTalk.requiresSetup
val dictationNeedsSetup = gatewayReady && talkSetupReadiness.dictation.requiresSetup
Column(horizontalAlignment = Alignment.CenterHorizontally, verticalArrangement = Arrangement.spacedBy(9.dp)) {
VoiceOrb(
active = micEnabled || talkModeEnabled,
@@ -657,11 +679,11 @@ private fun VoiceHero(
subtitle =
when {
talkModeEnabled -> "Conversation is live"
gatewayReady -> "Natural conversation in real time"
gatewayReady -> gatewayTalkSetupDescription(talkSetupReadiness.realtimeTalk)
else -> "Connect gateway to start"
},
icon = if (talkModeEnabled) Icons.Default.PhoneDisabled else Icons.Default.RecordVoiceOver,
onClick = onStartTalk,
onClick = if (talkNeedsSetup) onOpenVoiceSettings else onStartTalk,
enabled = gatewayReady || talkModeEnabled,
)
VoiceModeRow(
@@ -669,31 +691,42 @@ private fun VoiceHero(
subtitle =
when {
micEnabled -> "Listening for one turn"
gatewayReady -> "Convert speech to text"
gatewayReady -> gatewayTalkSetupDescription(talkSetupReadiness.dictation)
else -> "Connect gateway to start"
},
icon = if (micEnabled) Icons.Default.MicOff else Icons.Default.TextFields,
onClick = onStartDictation,
onClick = if (dictationNeedsSetup) onOpenVoiceSettings else onStartDictation,
enabled = gatewayReady || micEnabled,
)
}
VoiceProviderCard(gatewayStatus = gatewayStatus, voiceAttentionStatus = voiceAttentionStatus)
VoiceProviderCard(
gatewayStatus = gatewayStatus,
voiceAttentionStatus = voiceAttentionStatus,
talkSetupReadiness = talkSetupReadiness,
)
VoicePrimaryAction(
text =
when {
talkModeEnabled -> "End Talk"
talkNeedsSetup -> "Set Up Talk"
gatewayReady -> "Start Talk"
else -> "Connect Gateway"
},
icon =
when {
talkModeEnabled -> Icons.Default.PhoneDisabled
talkNeedsSetup -> Icons.Default.Settings
gatewayReady -> Icons.Default.Phone
else -> Icons.Default.Cloud
},
onClick = if (gatewayReady || talkModeEnabled) onStartTalk else onConnectGateway,
onClick =
when {
talkModeEnabled || (gatewayReady && !talkNeedsSetup) -> onStartTalk
talkNeedsSetup -> onOpenVoiceSettings
else -> onConnectGateway
},
)
}
}
@@ -743,8 +776,17 @@ private fun VoiceModeRow(
private fun VoiceProviderCard(
gatewayStatus: String,
voiceAttentionStatus: String?,
talkSetupReadiness: GatewayTalkSetupReadiness,
) {
val ready = voiceAttentionStatus == null && gatewayStatus.isVoiceGatewayReady()
val ready =
voiceAttentionStatus == null &&
gatewayStatus.isVoiceGatewayReady() &&
talkSetupReadiness.realtimeTalk.isReady &&
talkSetupReadiness.dictation.isReady
val needsSetup =
voiceAttentionStatus == null &&
gatewayStatus.isVoiceGatewayReady() &&
(talkSetupReadiness.realtimeTalk.requiresSetup || talkSetupReadiness.dictation.requiresSetup)
Surface(
modifier = Modifier.fillMaxWidth().heightIn(min = 58.dp),
shape = RoundedCornerShape(ClawTheme.radii.panel),
@@ -769,13 +811,12 @@ private fun VoiceProviderCard(
}
}
Column(modifier = Modifier.weight(1f), verticalArrangement = Arrangement.spacedBy(2.dp)) {
Text(text = "Provider", style = ClawTheme.type.body, color = ClawTheme.colors.text, maxLines = 1)
Text(text = "Voice setup", style = ClawTheme.type.body, color = ClawTheme.colors.text, maxLines = 1)
Text(
text = voiceAttentionStatus ?: gatewayStatus.voiceGatewayLabel(),
text = voiceAttentionStatus ?: voiceSetupSummary(gatewayStatus, talkSetupReadiness),
style = ClawTheme.type.caption,
color = ClawTheme.colors.textMuted,
maxLines = 1,
overflow = TextOverflow.Ellipsis,
maxLines = 2,
)
}
Row(verticalAlignment = Alignment.CenterVertically, horizontalArrangement = Arrangement.spacedBy(7.dp)) {
@@ -787,6 +828,7 @@ private fun VoiceProviderCard(
.background(
when {
ready -> ClawTheme.colors.success
needsSetup -> ClawTheme.colors.warning
voiceAttentionStatus != null -> ClawTheme.colors.warning
else -> ClawTheme.colors.textSubtle
},
@@ -796,7 +838,9 @@ private fun VoiceProviderCard(
text =
when {
ready -> "Ready"
needsSetup -> "Setup"
voiceAttentionStatus != null -> "Attention"
gatewayStatus.isVoiceGatewayReady() -> "Unverified"
else -> "Offline"
},
style = ClawTheme.type.caption,
@@ -808,6 +852,17 @@ private fun VoiceProviderCard(
}
}
private fun voiceSetupSummary(
gatewayStatus: String,
readiness: GatewayTalkSetupReadiness,
): String {
if (!gatewayStatus.isVoiceGatewayReady()) return gatewayStatus.voiceGatewayLabel()
return listOf(
"Talk: ${gatewayTalkSetupDescription(readiness.realtimeTalk)}",
"Dictation: ${gatewayTalkSetupDescription(readiness.dictation)}",
).joinToString(" · ")
}
@Composable
private fun VoicePrimaryAction(
text: String,
apps/android/app/src/main/java/ai/openclaw/app/voice/TalkModeManager.kt
+14 -3
View File
@@ -695,6 +695,13 @@ class TalkModeManager internal constructor(
disableRealtimeModeAndNotifyOwner()
}
private fun realtimeCloseStatusText(reason: String?): String =
when (reason) {
null, "completed" -> "Off"
"error" -> "Talk failed: Realtime provider closed unexpectedly."
else -> "Talk failed: Realtime provider closed: $reason"
}
@SuppressLint("MissingPermission")
private fun startRealtimeCapture(sessionId: String) {
realtimeCaptureJob?.cancel()
@@ -837,11 +844,15 @@ class TalkModeManager internal constructor(
Log.w(tag, "realtime error: $message")
}
"close" -> {
Log.d(tag, "realtime close reason=${obj["reason"].asStringOrNull()}")
stopRealtimeRelay(closeSession = false)
val closeReason = obj["reason"].asStringOrNull()?.trim()?.takeIf(String::isNotEmpty)
val currentStatus = _statusText.value
val closeStatus =
if (currentStatus.startsWith("Talk failed:")) currentStatus else realtimeCloseStatusText(closeReason)
Log.d(tag, "realtime close reason=$closeReason")
stopRealtimeRelay(closeSession = false, preserveStatus = true)
if (_isEnabled.value) {
_isEnabled.value = false
_statusText.value = "Off"
_statusText.value = closeStatus
onStoppedByRelay()
}
}
apps/android/app/src/test/java/ai/openclaw/app/GatewayTalkSetupReadinessTest.kt
+218
View File
@@ -0,0 +1,218 @@
package ai.openclaw.app
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.jsonObject
import org.junit.Assert.assertEquals
import org.junit.Assert.assertTrue
import org.junit.Test
class GatewayTalkSetupReadinessTest {
@Test
fun mixedProviderStatesRemainDistinct() {
val readiness =
parseGatewayTalkSetupReadiness(
catalog(
realtime = providerGroup(id = "openai", label = "OpenAI Realtime", configured = false),
transcription = providerGroup(id = "deepgram", label = "Deepgram", configured = true),
),
)
val realtime = readiness.realtimeTalk as GatewayTalkSetupState.NeedsSetup
val dictation = readiness.dictation as GatewayTalkSetupState.Ready
assertEquals("OpenAI Realtime", realtime.provider?.label)
assertEquals("Deepgram", dictation.provider.label)
}
@Test
fun activeProviderAliasSelectsCanonicalCatalogEntry() {
val readiness =
parseGatewayTalkSetupReadiness(
catalog(
realtime = providerGroup(id = "google", label = "Google Live", configured = true),
transcription =
providerGroup(
id = "openai",
label = "OpenAI Realtime Transcription",
configured = true,
activeProvider = "openai-realtime",
aliases = listOf("openai-realtime"),
),
),
)
val dictation = readiness.dictation as GatewayTalkSetupState.Ready
assertEquals("openai", dictation.provider.id)
}
@Test
fun canonicalProviderIdWinsOverAnEarlierAliasCollision() {
val readiness =
parseGatewayTalkSetupReadiness(
json(
"""
{
"realtime": {
"ready": true,
"activeProvider": "google",
"providers": [
{"id":"bridge","aliases":["google"],"label":"Bridge","configured":false},
{"id":"google","label":"Google Live","configured":true}
]
},
"transcription": ${providerGroup(id = "deepgram", label = "Deepgram", configured = true)}
}
""".trimIndent(),
),
)
val realtime = readiness.realtimeTalk as GatewayTalkSetupState.Ready
assertEquals("google", realtime.provider.id)
}
@Test
fun missingActiveProviderStaysUnverifiedInsteadOfGuessingFromRowOrder() {
val readiness =
parseGatewayTalkSetupReadiness(
json(
"""
{
"realtime": {
"providers": [
{"id":"google","label":"Google Live","configured":false},
{"id":"openai","label":"OpenAI Realtime","configured":true}
]
},
"transcription": ${providerGroup(id = "deepgram", label = "Deepgram", configured = true)}
}
""".trimIndent(),
),
)
assertTrue(readiness.realtimeTalk is GatewayTalkSetupState.Unverified)
assertTrue(!readiness.realtimeTalk.requiresSetup)
}
@Test
fun authoritativeUnconfiguredProvidersRequireSetupWithoutAnActiveProvider() {
val readiness =
parseGatewayTalkSetupReadiness(
catalog(
realtime =
providerGroup(
id = "openai",
label = "OpenAI Realtime",
configured = false,
activeProvider = null,
ready = false,
),
transcription = providerGroup(id = "deepgram", label = "Deepgram", configured = true),
),
)
assertTrue(readiness.realtimeTalk is GatewayTalkSetupState.NeedsSetup)
assertTrue(readiness.realtimeTalk.requiresSetup)
}
@Test
fun olderCatalogRowStateStaysUnverified() {
val readiness =
parseGatewayTalkSetupReadiness(
catalog(
realtime =
providerGroup(
id = "openai",
label = "OpenAI Realtime",
configured = false,
ready = null,
),
transcription = providerGroup(id = "deepgram", label = "Deepgram", configured = true),
),
)
assertTrue(readiness.realtimeTalk is GatewayTalkSetupState.Unverified)
assertTrue(!readiness.realtimeTalk.requiresSetup)
}
@Test
fun unknownActiveProviderStaysUnverifiedInsteadOfBlockingStartup() {
val readiness =
parseGatewayTalkSetupReadiness(
catalog(
realtime = providerGroup(id = "google", label = "Google Live", configured = true, activeProvider = "future-alias"),
transcription = providerGroup(id = "deepgram", label = "Deepgram", configured = true),
),
)
assertTrue(readiness.realtimeTalk is GatewayTalkSetupState.Unverified)
assertTrue(!readiness.realtimeTalk.requiresSetup)
}
@Test
fun authoritativeUnknownActiveProviderRequiresSetup() {
val readiness =
parseGatewayTalkSetupReadiness(
catalog(
realtime =
providerGroup(
id = "google",
label = "Google Live",
configured = true,
activeProvider = "removed-provider",
ready = false,
),
transcription = providerGroup(id = "deepgram", label = "Deepgram", configured = true),
),
)
val realtime = readiness.realtimeTalk as GatewayTalkSetupState.NeedsSetup
assertEquals("Choose a supported Realtime Talk provider on the Gateway", gatewayTalkSetupDescription(realtime))
assertTrue(readiness.realtimeTalk.requiresSetup)
}
@Test
fun unknownActiveProviderWithEmptyRegistryStaysUnverified() {
val readiness =
parseGatewayTalkSetupReadiness(
json(
"""
{
"realtime": {"activeProvider":"custom-id","providers":[]},
"transcription": ${providerGroup(id = "deepgram", label = "Deepgram", configured = true)}
}
""".trimIndent(),
),
)
assertTrue(readiness.realtimeTalk is GatewayTalkSetupState.Unverified)
assertTrue(!readiness.realtimeTalk.requiresSetup)
}
@Test
fun missingCatalogIsUnverifiedForBothActions() {
val readiness = parseGatewayTalkSetupReadiness(null)
assertTrue(readiness.realtimeTalk is GatewayTalkSetupState.Unverified)
assertTrue(readiness.dictation is GatewayTalkSetupState.Unverified)
}
private fun catalog(
realtime: String,
transcription: String,
) = json("""{"realtime":$realtime,"transcription":$transcription}""")
private fun providerGroup(
id: String,
label: String,
configured: Boolean,
activeProvider: String? = id,
aliases: List<String> = emptyList(),
ready: Boolean? = configured,
): String {
val active = activeProvider?.let { "\"activeProvider\":\"$it\"," }.orEmpty()
val readiness = ready?.let { "\"ready\":$it," }.orEmpty()
val aliasJson = aliases.joinToString(prefix = "[", postfix = "]") { "\"$it\"" }
return """{$readiness$active"providers":[{"id":"$id","label":"$label","configured":$configured,"aliases":$aliasJson}]}"""
}
private fun json(value: String) = Json.parseToJsonElement(value).jsonObject
}
apps/android/app/src/test/java/ai/openclaw/app/voice/TalkModeManagerTest.kt
+37
View File
@@ -135,6 +135,43 @@ class TalkModeManagerTest {
assertTrue(realtimeToolRuns(manager).isEmpty())
}
@Test
fun realtimeCloseErrorDisablesTalkButKeepsFailureStatus() {
var stoppedByRelay = false
val manager = createManager(onStoppedByRelay = { stoppedByRelay = true })
setPrivateField(manager, "realtimeSessionId", "relay-1")
setMutableStateFlow(manager, "_isEnabled", true)
manager.handleGatewayEvent(
"talk.event",
"""{"relaySessionId":"relay-1","type":"close","reason":"error"}""",
)
assertFalse(manager.isEnabled.value)
assertTrue(stoppedByRelay)
assertEquals(
"Talk failed: Realtime provider closed unexpectedly.",
manager.statusText.value,
)
}
@Test
fun realtimeClosePreservesDetailedProviderFailure() {
val manager = createManager()
setPrivateField(manager, "realtimeSessionId", "relay-1")
setMutableStateFlow(manager, "_isEnabled", true)
setMutableStateFlow(manager, "_statusText", "Talk failed: Provider rejected the session.")
manager.handleGatewayEvent(
"talk.event",
"""{"relaySessionId":"relay-1","type":"close","reason":"error"}""",
)
assertEquals("Talk failed: Provider rejected the session.", manager.statusText.value)
}
@Test
fun realtimeTranscriptsPopulateVoiceConversation() {
val manager = createManager()
docs/gateway/protocol.md
+1 -1
View File
@@ -379,7 +379,7 @@ enumeration of `src/gateway/server-methods/*.ts`.
</Accordion>
<Accordion title="Talk and TTS">
- `talk.catalog` returns the read-only Talk provider catalog for speech, streaming transcription, and realtime voice. It includes provider ids, labels, configured state, exposed model/voice ids, canonical modes, transports, brain strategies, and realtime audio/capability flags without returning provider secrets or mutating global config.
- `talk.catalog` returns the read-only Talk provider catalog for speech, streaming transcription, and realtime voice. It includes canonical provider ids, registry aliases, labels, configured state, an optional group-level `ready` result, exposed model/voice ids, canonical modes, transports, brain strategies, and realtime audio/capability flags without returning provider secrets or mutating global config. Current Gateways set `ready` after applying runtime provider selection; clients should treat its absence as unverified for compatibility with older Gateways.
- `talk.config` returns the effective Talk config payload; `includeSecrets` requires `operator.talk.secrets` (or `operator.admin`).
- `talk.session.create` creates a Gateway-owned Talk session for `realtime/gateway-relay`, `transcription/gateway-relay`, or `stt-tts/managed-room`. For `stt-tts/managed-room`, `operator.write` callers that pass `sessionKey` must also pass `spawnedBy` for scoped session-key visibility; unscoped `sessionKey` creation and `brain: "direct-tools"` require `operator.admin`.
- `talk.session.join` validates a managed-room session token, emits `session.ready` or `session.replaced` events as needed, and returns room/session metadata plus recent Talk events without the plaintext token or stored token hash.
docs/nodes/talk.md
+1 -1
View File
@@ -119,7 +119,7 @@ Defaults:
- `realtime.brain`: `agent-consult` routes realtime tool calls through Gateway policy; `direct-tools` is legacy direct-tool compatibility behavior; `none` is for transcription or external orchestration.
- `realtime.consultRouting`: `provider-direct` preserves the provider's direct reply when it skips `openclaw_agent_consult`; `force-agent-consult` makes Gateway relay route finalized user transcripts through OpenClaw instead.
- `realtime.instructions`: appends provider-facing system instructions to OpenClaw's built-in realtime prompt. Use it for voice style and tone; OpenClaw keeps the default `openclaw_agent_consult` guidance.
- `talk.catalog` exposes each provider's valid modes, transports, brain strategies, realtime audio formats, and capability flags so first-party Talk clients can avoid unsupported combinations.
- `talk.catalog` exposes canonical provider ids and registry aliases alongside each provider's valid modes, transports, brain strategies, realtime audio formats, capability flags, and the runtime-selected readiness result. First-party Talk clients should use that catalog instead of maintaining provider aliases locally; an older Gateway that omits group readiness is unverified rather than definitively unconfigured.
- Streaming transcription providers are discovered through `talk.catalog.transcription`. The current Gateway relay uses the Voice Call streaming provider config until the dedicated Talk transcription config surface is added.
- `speechLocale`: optional BCP 47 locale id for on-device Talk speech recognition on iOS/macOS. Leave unset to use the device default.
- `outputFormat`: defaults to `pcm_44100` on macOS/iOS and `pcm_24000` on Android (set `mp3_*` to force MP3 streaming)
packages/gateway-protocol/src/index.test.ts
+27
View File
@@ -17,6 +17,7 @@ import {
validateNodePresenceAlivePayload,
validateTasksCancelParams,
validateTasksListParams,
validateTalkCatalogResult,
validateTalkConfigResult,
validateTalkEvent,
validateTalkClientCreateParams,
@@ -329,6 +330,32 @@ describe("validateTalkConfigResult", () => {
});
});
describe("validateTalkCatalogResult", () => {
it("accepts provider registry aliases", () => {
expect(
validateTalkCatalogResult({
modes: ["realtime"],
transports: ["gateway-relay"],
brains: ["agent-consult"],
speech: { providers: [] },
transcription: { providers: [] },
realtime: {
ready: true,
activeProvider: "google",
providers: [
{
id: "google",
aliases: ["gemini-live"],
label: "Google Live Voice",
configured: true,
},
],
},
}),
).toBe(true);
});
});
describe("validateTalkClientCreateParams", () => {
it("accepts provider, model, voice, mode, transport, and brain overrides", () => {
expect(
packages/gateway-protocol/src/schema/channels.ts
+2
View File
@@ -413,6 +413,7 @@ const TalkCatalogProviderSchema = Type.Object(
id: NonEmptyString,
label: NonEmptyString,
configured: Type.Boolean(),
aliases: Type.Optional(Type.Array(NonEmptyString)),
models: Type.Optional(Type.Array(Type.String())),
voices: Type.Optional(Type.Array(Type.String())),
defaultModel: Type.Optional(Type.String()),
@@ -455,6 +456,7 @@ const TalkCatalogProviderSchema = Type.Object(
/** Active provider plus all candidates for a Talk capability family. */
const TalkCatalogProviderGroupSchema = Type.Object(
{
ready: Type.Optional(Type.Boolean()),
activeProvider: Type.Optional(Type.String()),
providers: Type.Array(TalkCatalogProviderSchema),
},
src/gateway/server-methods/talk.test.ts
+311 -5
View File
@@ -17,9 +17,12 @@ const mocks = vi.hoisted(() => ({
resolveTtsConfig: vi.fn(() => ({ timeoutMs: 30_000 })),
synthesizeSpeech: vi.fn(),
canonicalizeRealtimeVoiceProviderId: vi.fn((providerId: string | undefined) =>
providerId?.trim().toLowerCase(),
providerId === "gemini-live" ? "google" : providerId?.trim().toLowerCase(),
),
listRealtimeVoiceProviders: vi.fn(() => []),
canonicalizeRealtimeTranscriptionProviderId: vi.fn((providerId: string | undefined) =>
providerId === "openai-realtime" ? "openai" : providerId?.trim().toLowerCase(),
),
listRealtimeTranscriptionProviders: vi.fn(() => []),
resolveConfiguredRealtimeVoiceProvider: vi.fn(),
createTalkRealtimeRelaySession: vi.fn(),
@@ -60,6 +63,7 @@ vi.mock("../../talk/provider-registry.js", () => ({
}));
vi.mock("../../realtime-transcription/provider-registry.js", () => ({
canonicalizeRealtimeTranscriptionProviderId: mocks.canonicalizeRealtimeTranscriptionProviderId,
listRealtimeTranscriptionProviders: mocks.listRealtimeTranscriptionProviders,
}));
@@ -170,6 +174,7 @@ describe("talk.catalog handler", () => {
{
id: "elevenlabs",
label: "ElevenLabs",
aliases: ["11labs"],
models: ["eleven_flash_v2_5"],
voices: ["voice-1"],
isConfigured: vi.fn(() => true),
@@ -180,10 +185,18 @@ describe("talk.catalog handler", () => {
{
id: "openai",
label: "OpenAI Realtime Transcription",
aliases: ["openai-realtime"],
defaultModel: "gpt-4o-transcribe",
resolveConfig: vi.fn(({ rawConfig }) => rawConfig),
isConfigured: vi.fn(({ providerConfig }) => providerConfig.apiKey === "stt-key"),
} as never,
{
id: "deepgram",
label: "Deepgram Realtime Transcription",
aliases: ["deepgram-realtime"],
resolveConfig: vi.fn(({ rawConfig }) => rawConfig),
isConfigured: vi.fn(({ providerConfig }) => providerConfig.apiKey === "deepgram-key"),
} as never,
]);
mocks.listRealtimeVoiceProviders.mockReturnValue([
{
@@ -191,7 +204,12 @@ describe("talk.catalog handler", () => {
label: "Google Live Voice",
defaultModel: "gemini-live",
resolveConfig: vi.fn(({ rawConfig }) => rawConfig),
isConfigured: vi.fn(({ providerConfig }) => providerConfig.apiKey === "live-key"),
isConfigured: vi.fn(
({ providerConfig }) =>
providerConfig.apiKey === "live-key" &&
providerConfig.project === "base" &&
providerConfig.model === "talk-model",
),
capabilities: {
transports: ["provider-websocket", "gateway-relay"],
inputAudioFormats: [{ encoding: "pcm16", sampleRateHz: 24000, channels: 1 }],
@@ -205,7 +223,18 @@ describe("talk.catalog handler", () => {
createBrowserSession: vi.fn(),
createBridge: vi.fn(),
} as never,
{
id: "openai",
label: "OpenAI Realtime",
resolveConfig: vi.fn(({ rawConfig }) => rawConfig),
isConfigured: vi.fn(({ providerConfig }) => providerConfig.apiKey === "openai-key"),
createBridge: vi.fn(),
} as never,
]);
mocks.resolveConfiguredRealtimeVoiceProvider.mockReturnValue({
provider: { id: "google" },
providerConfig: { apiKey: "live-key", project: "base", model: "talk-model" },
} as never);
const respond = vi.fn();
await talkHandlers["talk.catalog"]({
@@ -222,7 +251,10 @@ describe("talk.catalog handler", () => {
providers: { elevenlabs: { apiKey: "speech-key" } },
realtime: {
provider: "google",
providers: { google: { apiKey: "live-key" } },
providers: {
google: { apiKey: "live-key", project: "base" },
},
model: "talk-model",
},
},
plugins: {
@@ -230,8 +262,8 @@ describe("talk.catalog handler", () => {
"voice-call": {
config: {
streaming: {
provider: "openai",
providers: { openai: { apiKey: "stt-key" } },
provider: "openai-realtime",
providers: { "openai-realtime": { apiKey: "stt-key" } },
},
},
},
@@ -253,6 +285,7 @@ describe("talk.catalog handler", () => {
{
id: "elevenlabs",
label: "ElevenLabs",
aliases: ["11labs"],
configured: true,
modes: ["stt-tts"],
brains: ["agent-consult"],
@@ -262,20 +295,32 @@ describe("talk.catalog handler", () => {
],
},
transcription: {
ready: true,
activeProvider: "openai",
providers: [
{
id: "openai",
label: "OpenAI Realtime Transcription",
aliases: ["openai-realtime"],
configured: true,
modes: ["transcription"],
transports: ["gateway-relay"],
brains: ["none"],
defaultModel: "gpt-4o-transcribe",
},
{
id: "deepgram",
label: "Deepgram Realtime Transcription",
aliases: ["deepgram-realtime"],
configured: false,
modes: ["transcription"],
transports: ["gateway-relay"],
brains: ["none"],
},
],
},
realtime: {
ready: true,
activeProvider: "google",
providers: [
{
@@ -294,6 +339,14 @@ describe("talk.catalog handler", () => {
supportsVideoFrames: true,
supportsSessionResumption: true,
},
{
id: "openai",
label: "OpenAI Realtime",
configured: false,
modes: ["realtime"],
brains: ["agent-consult"],
supportsBrowserSession: false,
},
],
},
},
@@ -304,6 +357,259 @@ describe("talk.catalog handler", () => {
expect(responsePayload).not.toContain("stt-key");
expect(responsePayload).not.toContain("live-key");
});
it("reports the runtime-selected automatic providers instead of registry row order", async () => {
const transcriptionSlow = {
id: "transcription-slow",
label: "Transcription Slow",
autoSelectOrder: 20,
isConfigured: vi.fn(({ providerConfig }) => providerConfig.enabled === true),
};
const transcriptionFast = {
id: "transcription-fast",
label: "Transcription Fast",
models: ["transcribe-model"],
autoSelectOrder: 10,
isConfigured: vi.fn(
({ providerConfig }) =>
providerConfig.enabled === true && providerConfig.model === "transcribe-model",
),
};
const realtimeSlow = {
id: "realtime-slow",
label: "Realtime Slow",
autoSelectOrder: 20,
isConfigured: vi.fn(({ providerConfig }) => providerConfig.enabled === true),
createBridge: vi.fn(),
};
const realtimeFast = {
id: "realtime-fast",
label: "Realtime Fast",
autoSelectOrder: 10,
isConfigured: vi.fn(({ providerConfig }) => providerConfig.enabled === true),
createBridge: vi.fn(),
};
mocks.listRealtimeTranscriptionProviders.mockReturnValue([
transcriptionSlow,
transcriptionFast,
] as never);
mocks.listRealtimeVoiceProviders.mockReturnValue([realtimeSlow, realtimeFast] as never);
mocks.resolveConfiguredRealtimeVoiceProvider.mockReturnValue({
provider: realtimeFast,
providerConfig: { enabled: true },
} as never);
const respond = vi.fn();
await talkHandlers["talk.catalog"]({
req: { type: "req", id: "1", method: "talk.catalog" },
params: {},
client: { connect: { scopes: ["operator.read"] } } as never,
isWebchatConnect: () => false,
respond: respond as never,
context: {
getRuntimeConfig: () =>
({
agents: {
defaults: {
voiceModel: { primary: "transcription-fast/transcribe-model" },
},
},
talk: {
realtime: {
providers: {
"realtime-slow": { enabled: true },
"realtime-fast": { enabled: true },
},
},
},
plugins: {
entries: {
"voice-call": {
config: {
streaming: {
providers: {
"transcription-slow": { enabled: true },
"transcription-fast": { enabled: true },
},
},
},
},
},
},
}) as OpenClawConfig,
} as never,
});
expect(mockCallArg(respond, 0, 1)).toMatchObject({
transcription: {
ready: true,
activeProvider: "transcription-fast",
providers: [
{ id: "transcription-slow", configured: true },
{ id: "transcription-fast", configured: true },
],
},
realtime: { ready: true, activeProvider: "realtime-fast" },
});
});
it("reports the provider selected by runtime resolution when aliases collide", async () => {
const transcriptionAlias = {
id: "transcription-alias",
label: "Transcription Alias",
aliases: ["shared-transcription"],
isConfigured: vi.fn(() => true),
};
const transcriptionDirect = {
id: "shared-transcription",
label: "Transcription Direct",
isConfigured: vi.fn(() => true),
};
const realtimeAlias = {
id: "realtime-alias",
label: "Realtime Alias",
aliases: ["shared-realtime"],
isConfigured: vi.fn(() => true),
createBridge: vi.fn(),
};
const realtimeDirect = {
id: "shared-realtime",
label: "Realtime Direct",
isConfigured: vi.fn(() => true),
createBridge: vi.fn(),
};
mocks.listRealtimeTranscriptionProviders.mockReturnValue([
transcriptionAlias,
transcriptionDirect,
] as never);
mocks.listRealtimeVoiceProviders.mockReturnValue([realtimeAlias, realtimeDirect] as never);
mocks.canonicalizeRealtimeTranscriptionProviderId.mockReturnValueOnce("shared-transcription");
mocks.canonicalizeRealtimeVoiceProviderId.mockReturnValueOnce("shared-realtime");
mocks.resolveConfiguredRealtimeVoiceProvider.mockReturnValue({
provider: realtimeAlias,
providerConfig: { enabled: true },
} as never);
const respond = vi.fn();
await talkHandlers["talk.catalog"]({
req: { type: "req", id: "1", method: "talk.catalog" },
params: {},
client: { connect: { scopes: ["operator.read"] } } as never,
isWebchatConnect: () => false,
respond: respond as never,
context: {
getRuntimeConfig: () =>
({
talk: {
realtime: {
provider: "shared-realtime",
providers: { "shared-realtime": { enabled: true } },
},
},
plugins: {
entries: {
"voice-call": {
config: {
streaming: {
provider: "shared-transcription",
providers: { "shared-transcription": { enabled: true } },
},
},
},
},
},
}) as OpenClawConfig,
} as never,
});
expect(mockCallArg(respond, 0, 1)).toMatchObject({
transcription: { ready: true, activeProvider: "transcription-alias" },
realtime: { ready: true, activeProvider: "realtime-alias" },
});
});
it("reports an authoritative setup requirement when automatic selection fails", async () => {
mocks.listRealtimeTranscriptionProviders.mockReturnValue([
{
id: "transcription",
label: "Transcription",
isConfigured: vi.fn(() => false),
},
] as never);
mocks.listRealtimeVoiceProviders.mockReturnValue([
{
id: "realtime",
label: "Realtime",
isConfigured: vi.fn(() => false),
createBridge: vi.fn(),
},
] as never);
mocks.resolveConfiguredRealtimeVoiceProvider.mockImplementation(() => {
throw new Error("No realtime voice provider configured");
});
const respond = vi.fn();
await talkHandlers["talk.catalog"]({
req: { type: "req", id: "1", method: "talk.catalog" },
params: {},
client: { connect: { scopes: ["operator.read"] } } as never,
isWebchatConnect: () => false,
respond: respond as never,
context: { getRuntimeConfig: () => ({}) as OpenClawConfig } as never,
});
const catalog = mockCallArg(respond, 0, 1) as Record<string, Record<string, unknown>>;
expect(catalog.transcription).toMatchObject({ ready: false });
expect(catalog.transcription).not.toHaveProperty("activeProvider");
expect(catalog.realtime).toMatchObject({ ready: false });
expect(catalog.realtime).not.toHaveProperty("activeProvider");
});
it("validates explicitly selected providers before reporting readiness", async () => {
mocks.listRealtimeTranscriptionProviders.mockReturnValue([
{
id: "transcription",
label: "Transcription",
isConfigured: vi.fn(() => false),
},
] as never);
mocks.listRealtimeVoiceProviders.mockReturnValue([
{
id: "realtime",
label: "Realtime",
isConfigured: vi.fn(() => false),
createBridge: vi.fn(),
},
] as never);
mocks.resolveConfiguredRealtimeVoiceProvider.mockImplementation(() => {
throw new Error("Realtime provider is not configured");
});
const respond = vi.fn();
await talkHandlers["talk.catalog"]({
req: { type: "req", id: "1", method: "talk.catalog" },
params: {},
client: { connect: { scopes: ["operator.read"] } } as never,
isWebchatConnect: () => false,
respond: respond as never,
context: {
getRuntimeConfig: () =>
({
talk: { realtime: { provider: "realtime" } },
plugins: {
entries: {
"voice-call": { config: { streaming: { provider: "transcription" } } },
},
},
}) as OpenClawConfig,
} as never,
});
expect(mockCallArg(respond, 0, 1)).toMatchObject({
transcription: { ready: false, activeProvider: "transcription" },
realtime: { ready: false, activeProvider: "realtime" },
});
});
});
describe("talk.speak handler", () => {
src/gateway/server-methods/talk.ts
+90 -12
View File
@@ -19,6 +19,7 @@ import {
withSpeakerSelectionCompat,
withSpeakerSelectionFallbackCompat,
} from "../../../packages/speech-core/speaker.js";
import { getVoiceProviderConfig } from "../../../packages/speech-core/voice-models.js";
import { readConfigFileSnapshot } from "../../config/config.js";
import { redactConfigObject } from "../../config/redact-snapshot.js";
import {
@@ -32,11 +33,16 @@ import type {
TalkRealtimeConfig,
} from "../../config/types.gateway.js";
import type { OpenClawConfig, TtsConfig, TtsProviderConfigMap } from "../../config/types.js";
import { listRealtimeTranscriptionProviders } from "../../realtime-transcription/provider-registry.js";
import { resolveProviderRawConfig } from "../../plugin-sdk/provider-selection-runtime.js";
import {
canonicalizeRealtimeTranscriptionProviderId,
listRealtimeTranscriptionProviders,
} from "../../realtime-transcription/provider-registry.js";
import {
canonicalizeRealtimeVoiceProviderId,
listRealtimeVoiceProviders,
} from "../../talk/provider-registry.js";
import { resolveConfiguredRealtimeVoiceProvider } from "../../talk/provider-resolver.js";
import {
canonicalizeSpeechProviderId,
getSpeechProvider,
@@ -55,8 +61,9 @@ import { talkClientHandlers } from "./talk-client.js";
import { talkSessionHandlers } from "./talk-session.js";
import {
buildTalkRealtimeConfig,
buildTalkTranscriptionConfig,
configuredOrFalse,
getVoiceCallStreamingConfig,
resolveConfiguredRealtimeTranscriptionProvider,
} from "./talk-shared.js";
import type { GatewayRequestHandlers } from "./types.js";
@@ -71,6 +78,26 @@ type TalkSpeakErrorDetails = {
reason: TalkSpeakReason;
fallbackEligible: boolean;
};
function resolveCatalogProviderSelection(
configuredProvider: string | undefined,
resolveAutomaticProvider: () => string,
): { activeProvider?: string; ready: boolean } {
// Provider priority belongs to the runtime resolver; catalog consumers must not infer it from row order.
try {
const resolvedProvider = resolveAutomaticProvider();
return {
activeProvider: resolvedProvider,
ready: true,
};
} catch {
return {
...(configuredProvider ? { activeProvider: configuredProvider } : {}),
ready: false,
};
}
}
function canReadTalkSecrets(client: { connect?: { scopes?: string[] } } | null): boolean {
const scopes = Array.isArray(client?.connect?.scopes) ? client.connect.scopes : [];
return scopes.includes(ADMIN_SCOPE) || scopes.includes(TALK_SECRETS_SCOPE);
@@ -199,12 +226,30 @@ function buildTalkCatalog(config: OpenClawConfig) {
const ttsConfig = resolveTtsConfig(config);
const talkResolved = resolveActiveTalkProviderConfig(config.talk);
const activeSpeechProvider = canonicalizeSpeechProviderId(talkResolved?.provider, config);
const streamingConfig = getVoiceCallStreamingConfig(config);
const realtimeConfig = buildTalkRealtimeConfig(config);
const activeRealtimeProvider = canonicalizeRealtimeVoiceProviderId(
realtimeConfig.provider,
config,
const transcriptionConfig = buildTalkTranscriptionConfig(config);
const transcriptionSelection = resolveCatalogProviderSelection(
canonicalizeRealtimeTranscriptionProviderId(transcriptionConfig.provider, config),
() =>
resolveConfiguredRealtimeTranscriptionProvider({
config,
configuredProviderId: transcriptionConfig.provider,
providerConfigs: transcriptionConfig.providers,
defaultModel: transcriptionConfig.model,
}).provider.id,
);
const activeTranscriptionProvider = transcriptionSelection.activeProvider;
const realtimeConfig = buildTalkRealtimeConfig(config);
const realtimeSelection = resolveCatalogProviderSelection(
canonicalizeRealtimeVoiceProviderId(realtimeConfig.provider, config),
() =>
resolveConfiguredRealtimeVoiceProvider({
cfg: config,
configuredProviderId: realtimeConfig.provider,
providerConfigs: realtimeConfig.providers,
defaultModel: realtimeConfig.model,
}).provider.id,
);
const activeRealtimeProvider = realtimeSelection.activeProvider;
return {
modes: ["realtime", "stt-tts", "transcription"],
@@ -229,6 +274,9 @@ function buildTalkCatalog(config: OpenClawConfig) {
if (provider.models) {
entry.models = [...provider.models];
}
if (provider.aliases?.length) {
entry.aliases = [...provider.aliases];
}
if (provider.voices) {
entry.voices = [...provider.voices];
}
@@ -236,10 +284,22 @@ function buildTalkCatalog(config: OpenClawConfig) {
}),
},
transcription: {
...(streamingConfig.provider ? { activeProvider: streamingConfig.provider } : {}),
ready: transcriptionSelection.ready,
...(activeTranscriptionProvider ? { activeProvider: activeTranscriptionProvider } : {}),
providers: listRealtimeTranscriptionProviders(config).map((provider) => {
const rawConfig = streamingConfig.providers?.[provider.id] ?? {};
const providerConfig = provider.resolveConfig?.({ cfg: config, rawConfig }) ?? rawConfig;
const rawConfig = getVoiceProviderConfig({
providerConfigs: transcriptionConfig.providers,
provider,
configuredProviderId:
provider.id === activeTranscriptionProvider ? transcriptionConfig.provider : undefined,
});
const rawConfigWithModel =
transcriptionConfig.model && rawConfig.model === undefined
? { ...rawConfig, model: transcriptionConfig.model }
: rawConfig;
const providerConfig =
provider.resolveConfig?.({ cfg: config, rawConfig: rawConfigWithModel }) ??
rawConfigWithModel;
const entry: Record<string, unknown> = {
id: provider.id,
label: provider.label,
@@ -253,14 +313,29 @@ function buildTalkCatalog(config: OpenClawConfig) {
if (provider.defaultModel) {
entry.defaultModel = provider.defaultModel;
}
if (provider.aliases?.length) {
entry.aliases = [...provider.aliases];
}
return entry;
}),
},
realtime: {
ready: realtimeSelection.ready,
...(activeRealtimeProvider ? { activeProvider: activeRealtimeProvider } : {}),
providers: listRealtimeVoiceProviders(config).map((provider) => {
const rawConfig = realtimeConfig.providers?.[provider.id] ?? {};
const providerConfig = provider.resolveConfig?.({ cfg: config, rawConfig }) ?? rawConfig;
const rawConfig = resolveProviderRawConfig({
providerConfigs: realtimeConfig.providers ?? {},
providerId: provider.id,
configuredProviderId:
provider.id === activeRealtimeProvider ? realtimeConfig.provider : undefined,
});
const rawConfigWithModel =
realtimeConfig.model && rawConfig.model === undefined
? { ...rawConfig, model: realtimeConfig.model }
: rawConfig;
const providerConfig =
provider.resolveConfig?.({ cfg: config, rawConfig: rawConfigWithModel }) ??
rawConfigWithModel;
const capabilities = provider.capabilities;
const entry: Record<string, unknown> = {
id: provider.id,
@@ -277,6 +352,9 @@ function buildTalkCatalog(config: OpenClawConfig) {
if (provider.defaultModel) {
entry.defaultModel = provider.defaultModel;
}
if (provider.aliases?.length) {
entry.aliases = [...provider.aliases];
}
if (capabilities?.transports) {
entry.transports = [...capabilities.transports];
}
test/scripts/native-app-i18n.test.ts
+12
View File
@@ -48,6 +48,18 @@ describe("native app i18n inventory", () => {
expect(entries.some((entry) => entry.source === "Mute")).toBe(true);
expect(entries.some((entry) => entry.source === "Creating...")).toBe(true);
expect(entries.some((entry) => entry.source === "Permission required")).toBe(true);
expect(entries.some((entry) => entry.source === "Needs setup")).toBe(true);
expect(
entries.some(
(entry) =>
entry.source === "Choose a supported ${issue.target.title} provider on the Gateway",
),
).toBe(true);
expect(
entries.some(
(entry) => entry.source === "Talk failed: Realtime provider closed unexpectedly.",
),
).toBe(true);
expect(entries.some((entry) => entry.source === "Searching…")).toBe(true);
expect(entries.some((entry) => entry.source === "Run now")).toBe(true);
expect(entries.some((entry) => entry.source === "Loading chat")).toBe(true);