From ec9bc46e80ceb77b24bcc4105ce1786e364cd8d3 Mon Sep 17 00:00:00 2001 From: WhatsSkiLL Date: Mon, 20 Jul 2026 18:33:09 +0200 Subject: [PATCH] Wear: animate Talk avatar from playback audio (#111516) * Wear: animate Talk avatar from playback audio * fix(wear): preserve PCM mouth frame continuity --------- Co-authored-by: Colin --- .../java/ai/openclaw/wear/MainActivity.kt | 1 + .../openclaw/wear/WearRealtimeTalkClient.kt | 132 +++++- .../main/java/ai/openclaw/wear/WearScreens.kt | 135 +----- .../java/ai/openclaw/wear/WearTalkAvatar.kt | 409 ++++++++++++++++++ .../java/ai/openclaw/wear/WearViewModel.kt | 9 + .../ai/openclaw/wear/WearTalkAvatarTest.kt | 146 +++++++ 6 files changed, 713 insertions(+), 119 deletions(-) create mode 100644 apps/android/wear/src/main/java/ai/openclaw/wear/WearTalkAvatar.kt create mode 100644 apps/android/wear/src/test/java/ai/openclaw/wear/WearTalkAvatarTest.kt diff --git a/apps/android/wear/src/main/java/ai/openclaw/wear/MainActivity.kt b/apps/android/wear/src/main/java/ai/openclaw/wear/MainActivity.kt index 623b0e8f5a50..f40b2c96ae63 100644 --- a/apps/android/wear/src/main/java/ai/openclaw/wear/MainActivity.kt +++ b/apps/android/wear/src/main/java/ai/openclaw/wear/MainActivity.kt @@ -292,6 +292,7 @@ internal fun OpenClawWearApp( speaking = speaking, realtimeCapturing = state.realtimeCapturing, realtimePlaying = state.realtimePlaying, + realtimeMouthLevel = state.realtimeMouthLevel, realtimePlaybackFailed = state.realtimePlaybackFailed, realtimeThinkingOverride = realtimeThinkingTurnId != null, actionBusy = diff --git a/apps/android/wear/src/main/java/ai/openclaw/wear/WearRealtimeTalkClient.kt b/apps/android/wear/src/main/java/ai/openclaw/wear/WearRealtimeTalkClient.kt index ad29957d5ef2..49493cc92b0b 100644 --- a/apps/android/wear/src/main/java/ai/openclaw/wear/WearRealtimeTalkClient.kt +++ b/apps/android/wear/src/main/java/ai/openclaw/wear/WearRealtimeTalkClient.kt @@ -21,6 +21,8 @@ import kotlinx.coroutines.Job import kotlinx.coroutines.NonCancellable import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.cancel +import kotlinx.coroutines.channels.BufferOverflow +import kotlinx.coroutines.channels.Channel import kotlinx.coroutines.currentCoroutineContext import kotlinx.coroutines.delay import kotlinx.coroutines.flow.MutableStateFlow @@ -37,6 +39,8 @@ import java.io.OutputStream import java.util.Locale import kotlin.coroutines.resume import kotlin.coroutines.resumeWithException +import kotlin.math.ceil +import kotlin.math.sqrt internal class WearRealtimeTalkClient( context: Context, @@ -52,6 +56,8 @@ internal class WearRealtimeTalkClient( val isCapturing: StateFlow = _isCapturing private val _isPlaying = MutableStateFlow(false) val isPlaying: StateFlow = _isPlaying + private val _mouthLevel = MutableStateFlow(0f) + val mouthLevel: StateFlow = _mouthLevel private val _channelFailed = MutableStateFlow(false) val channelFailed: StateFlow = _channelFailed @@ -66,6 +72,9 @@ internal class WearRealtimeTalkClient( private var captureJob: Job? = null private var readJob: Job? = null private var playbackIdleJob: Job? = null + private var mouthJob: Job? = null + private var mouthFrames: Channel? = null + private val mouthLevelAccumulator = Pcm16MouthLevelAccumulator() private var audioTrack: AudioTrack? = null private var playbackEndsAtMillis = 0L @@ -267,6 +276,7 @@ internal class WearRealtimeTalkClient( pauseCaptureLocked() check(audioFocus.request()) } + val mouthLevels = mouthLevelAccumulator.append(bytes) val track = audioTrack ?: createAudioTrack(bytes.size).also { audioTrack = it } var written = 0 while (written < bytes.size) { @@ -277,6 +287,10 @@ internal class WearRealtimeTalkClient( check(written == bytes.size) if (track.playState != AudioTrack.PLAYSTATE_PLAYING) track.play() _isPlaying.value = true + if (mouthLevels.isNotEmpty()) { + val timeline = mouthTimelineLocked() + mouthLevels.forEach { level -> timeline.trySend(level) } + } val durationMillis = ((written / PCM_16_BYTES.toDouble()) / WearProtocol.REALTIME_AUDIO_SAMPLE_RATE_HZ * 1_000.0) .toLong() @@ -286,6 +300,28 @@ internal class WearRealtimeTalkClient( } } + private fun mouthTimelineLocked(): Channel { + mouthFrames?.let { return it } + val frames = + Channel( + capacity = MOUTH_QUEUE_CAPACITY, + onBufferOverflow = BufferOverflow.DROP_OLDEST, + ) + mouthFrames = frames + mouthJob = + scope.launch { + try { + for (level in frames) { + _mouthLevel.value = level + delay(MOUTH_FRAME_MILLIS.toLong()) + } + } finally { + if (mouthFrames === frames) _mouthLevel.value = 0f + } + } + return frames + } + private fun createAudioTrack(frameBytes: Int): AudioTrack { val minimumBuffer = AudioTrack.getMinBufferSize( @@ -312,12 +348,30 @@ internal class WearRealtimeTalkClient( private fun schedulePlaybackIdle() { playbackIdleJob?.cancel() + val scheduledPlaybackEndMillis = playbackEndsAtMillis + val finalFrameDurationMillis = mouthLevelAccumulator.pendingFrameDurationMillis() playbackIdleJob = scope.launch { - while (SystemClock.elapsedRealtime() < playbackEndsAtMillis) delay(20L) + if (finalFrameDurationMillis > 0L) { + val finalFrameStartsAtMillis = scheduledPlaybackEndMillis - finalFrameDurationMillis + // Emit the residual at its cumulative sample position; clear/reset below + // discards it only when the matching AudioTrack tail is also discarded. + while (SystemClock.elapsedRealtime() < finalFrameStartsAtMillis) delay(MOUTH_FRAME_MILLIS.toLong()) + synchronized(audioLock) { + if (playbackEndsAtMillis == scheduledPlaybackEndMillis) { + mouthLevelAccumulator.flush().forEach { level -> mouthTimelineLocked().trySend(level) } + } + } + } + while (SystemClock.elapsedRealtime() < scheduledPlaybackEndMillis) delay(MOUTH_FRAME_MILLIS.toLong()) delay(PLAYBACK_DRAIN_GRACE_MILLIS) synchronized(audioLock) { - if (SystemClock.elapsedRealtime() >= playbackEndsAtMillis) clearOutputLocked(resumeCapture = true) + if ( + playbackEndsAtMillis == scheduledPlaybackEndMillis && + SystemClock.elapsedRealtime() >= scheduledPlaybackEndMillis + ) { + clearOutputLocked(resumeCapture = true) + } } } } @@ -330,6 +384,13 @@ internal class WearRealtimeTalkClient( playbackIdleJob?.cancel() playbackIdleJob = null playbackEndsAtMillis = 0L + val activeMouthFrames = mouthFrames + mouthFrames = null + activeMouthFrames?.close() + mouthJob?.cancel() + mouthJob = null + mouthLevelAccumulator.reset() + _mouthLevel.value = 0f runCatching { audioTrack?.pause() audioTrack?.flush() @@ -415,11 +476,78 @@ internal class WearRealtimeTalkClient( const val CHANNEL_OPEN_ATTEMPTS = 2 const val CHANNEL_RETRY_DELAY_MILLIS = 250L const val ISO_639_1_LANGUAGE_LENGTH = 2 + const val MOUTH_QUEUE_CAPACITY = 256 const val PCM_16_BYTES = 2 const val PLAYBACK_DRAIN_GRACE_MILLIS = 120L } } +internal fun pcm16LeMouthLevels( + pcm: ByteArray, + sampleRateHz: Int = WearProtocol.REALTIME_AUDIO_SAMPLE_RATE_HZ, + frameMillis: Int = MOUTH_FRAME_MILLIS, +): List = + Pcm16MouthLevelAccumulator(sampleRateHz, frameMillis).run { + append(pcm) + flush() + } + +internal class Pcm16MouthLevelAccumulator( + private val sampleRateHz: Int = WearProtocol.REALTIME_AUDIO_SAMPLE_RATE_HZ, + frameMillis: Int = MOUTH_FRAME_MILLIS, +) { + private val samplesPerFrame: Int + private var squareSum = 0.0 + private var sampleCount = 0 + + init { + require(sampleRateHz > 0 && frameMillis > 0) + samplesPerFrame = (sampleRateHz * frameMillis / 1_000).coerceAtLeast(1) + } + + fun append(pcm: ByteArray): List { + require(pcm.size % PCM_BYTES_PER_SAMPLE == 0) + return buildList { + var byteIndex = 0 + while (byteIndex < pcm.size) { + val low = pcm[byteIndex].toInt() and 0xff + val high = pcm[byteIndex + 1].toInt() + val sample = ((high shl 8) or low).toShort().toInt() + val normalized = sample / 32_768.0 + squareSum += normalized * normalized + sampleCount += 1 + byteIndex += PCM_BYTES_PER_SAMPLE + if (sampleCount == samplesPerFrame) add(finishFrame()) + } + } + } + + fun flush(): List = if (sampleCount == 0) emptyList() else listOf(finishFrame()) + + fun reset() { + squareSum = 0.0 + sampleCount = 0 + } + + fun pendingFrameDurationMillis(): Long = + if (sampleCount == 0) { + 0L + } else { + ceil(sampleCount * 1_000.0 / sampleRateHz).toLong() + } + + private fun finishFrame(): Float { + val rms = sqrt(squareSum / sampleCount) + val gated = ((rms - RMS_NOISE_GATE) / RMS_SPEECH_RANGE).coerceIn(0.0, 1.0) + reset() + return sqrt(gated).toFloat() + } +} + +internal const val MOUTH_FRAME_MILLIS = 20 +private const val PCM_BYTES_PER_SAMPLE = 2 +private const val RMS_NOISE_GATE = 0.015 +private const val RMS_SPEECH_RANGE = 0.2 + private fun java.io.Closeable?.closeQuietly() { runCatching { this?.close() } } diff --git a/apps/android/wear/src/main/java/ai/openclaw/wear/WearScreens.kt b/apps/android/wear/src/main/java/ai/openclaw/wear/WearScreens.kt index 0f21e6c8dde4..5b73272798d6 100644 --- a/apps/android/wear/src/main/java/ai/openclaw/wear/WearScreens.kt +++ b/apps/android/wear/src/main/java/ai/openclaw/wear/WearScreens.kt @@ -95,6 +95,7 @@ internal fun OpenClawWearScreens( speaking: Boolean, realtimeCapturing: Boolean, realtimePlaying: Boolean, + realtimeMouthLevel: Float, realtimePlaybackFailed: Boolean, realtimeThinkingOverride: Boolean, actionBusy: Boolean, @@ -220,6 +221,7 @@ internal fun OpenClawWearScreens( speaking = speaking, realtimeCapturing = realtimeCapturing, realtimePlaying = realtimePlaying, + realtimeMouthLevel = realtimeMouthLevel, realtimePlaybackFailed = realtimePlaybackFailed, realtimeThinkingOverride = realtimeThinkingOverride, realtimeElapsedSeconds = realtimeElapsedSeconds, @@ -353,6 +355,7 @@ private fun VoicePage( speaking: Boolean, realtimeCapturing: Boolean, realtimePlaying: Boolean, + realtimeMouthLevel: Float, realtimePlaybackFailed: Boolean, realtimeThinkingOverride: Boolean, realtimeElapsedSeconds: Long, @@ -419,6 +422,7 @@ private fun VoicePage( speaking = speaking, realtimeCapturing = realtimeCapturing, realtimePlaying = realtimePlaying, + realtimeMouthLevel = realtimeMouthLevel, realtimePlaybackFailed = realtimePlaybackFailed, realtimeThinkingOverride = realtimeThinkingOverride, realtimeElapsedSeconds = realtimeElapsedSeconds, @@ -465,6 +469,7 @@ private fun VoiceHomeMode( speaking: Boolean, realtimeCapturing: Boolean, realtimePlaying: Boolean, + realtimeMouthLevel: Float, realtimePlaybackFailed: Boolean, realtimeThinkingOverride: Boolean, realtimeElapsedSeconds: Long, @@ -533,19 +538,7 @@ private fun VoiceHomeMode( state == RealtimeVoiceButtonState.ERROR -> colors.danger else -> colors.voiceAccent } - val containerColor = - when { - dictatePreview || state == RealtimeVoiceButtonState.IDLE -> colors.surfaceRaised - state == RealtimeVoiceButtonState.ERROR -> colors.danger.copy(alpha = 0.28f) - else -> colors.voiceAccentSoft - } - val contentColor = - when { - dictatePreview -> colors.voiceAccent - state == RealtimeVoiceButtonState.IDLE -> colors.text - state == RealtimeVoiceButtonState.ERROR -> colors.danger - else -> colors.voiceAccent - } + val avatarState = if (dictatePreview) RealtimeVoiceButtonState.LISTENING else state Box( modifier = Modifier @@ -586,14 +579,11 @@ private fun VoiceHomeMode( .offset(y = (-4).dp) .fillMaxWidth(), ) - VoiceOrb( - accent = accent, - containerColor = containerColor, - pulse = dictatePreview || state != RealtimeVoiceButtonState.IDLE, - size = 92.dp, + Box( modifier = Modifier .align(Alignment.Center) + .size(92.dp) .combinedClickable( enabled = !dictatePreview, role = Role.Button, @@ -601,22 +591,16 @@ private fun VoiceHomeMode( onDoubleClick = onOpenThread, onLongClick = startDictate, ), + contentAlignment = Alignment.Center, ) { - if ( - dictatePreview || - state == RealtimeVoiceButtonState.LISTENING || - state == RealtimeVoiceButtonState.SPEAKING - ) { - LiveWaveform( - color = colors.voiceAccent, - active = true, - ) - } else { - MicrophoneGlyph( - color = contentColor, - modifier = Modifier.size(38.dp), - ) - } + WearTalkAvatar( + state = avatarState, + mouthLevel = if (realtimePlaying) realtimeMouthLevel else 0f, + syntheticSpeech = ttsOnly, + accent = accent, + danger = colors.danger, + modifier = Modifier.fillMaxSize(), + ) } statusText?.let { status -> Text( @@ -969,48 +953,6 @@ internal fun nextWearThreadFollowForViewport( internal fun wearThreadFollowLatest(state: WearThreadFollowState): WearThreadFollowState = state.copy(followingLatest = true, hasNewContent = false) -@Composable -private fun VoiceOrb( - accent: Color, - containerColor: Color, - pulse: Boolean, - size: androidx.compose.ui.unit.Dp, - modifier: Modifier = Modifier, - content: @Composable () -> Unit, -) { - val ringAlpha = - if (pulse) { - val transition = rememberInfiniteTransition(label = "voice-orb-ring") - transition - .animateFloat( - initialValue = 0.4f, - targetValue = 1f, - animationSpec = - infiniteRepeatable( - animation = tween(durationMillis = 850), - repeatMode = RepeatMode.Reverse, - ), - label = "voice-orb-ring-alpha", - ).value - } else { - 1f - } - Box( - modifier = - modifier - .size(size) - .background(containerColor, CircleShape) - .border( - width = if (pulse) 3.dp else 1.dp, - color = accent.copy(alpha = ringAlpha), - shape = CircleShape, - ), - contentAlignment = Alignment.Center, - ) { - content() - } -} - @Composable private fun MicrophoneGlyph( color: Color, @@ -1056,47 +998,6 @@ private fun MicrophoneGlyph( } } -@Composable -private fun LiveWaveform( - color: Color, - active: Boolean, -) { - val transition = rememberInfiniteTransition(label = "live-waveform") - val phase by - transition.animateFloat( - initialValue = 0f, - targetValue = 1f, - animationSpec = - infiniteRepeatable( - animation = tween(durationMillis = 520), - repeatMode = RepeatMode.Reverse, - ), - label = "live-waveform-phase", - ) - val amplitude = if (active) phase else 0f - Row( - horizontalArrangement = Arrangement.spacedBy(3.dp), - verticalAlignment = Alignment.CenterVertically, - modifier = Modifier.height(28.dp), - ) { - listOf( - 7f + (8f * amplitude), - 10f + (13f * (1f - amplitude)), - 12f + (14f * amplitude), - 10f + (13f * (1f - amplitude)), - 7f + (8f * amplitude), - ).forEach { barHeight -> - Box( - modifier = - Modifier - .width(3.dp) - .height(barHeight.dp) - .background(color, RoundedCornerShape(2.dp)), - ) - } - } -} - private fun realtimeVoiceButtonState( realtimeTalk: WearRealtimeTalkSnapshot, ttsOnly: Boolean, @@ -1129,7 +1030,7 @@ private fun formatVoiceElapsedTime(totalSeconds: Long): String { return "$minutes:${seconds.toString().padStart(2, '0')}" } -private enum class RealtimeVoiceButtonState { +internal enum class RealtimeVoiceButtonState { IDLE, CONNECTING, LISTENING, diff --git a/apps/android/wear/src/main/java/ai/openclaw/wear/WearTalkAvatar.kt b/apps/android/wear/src/main/java/ai/openclaw/wear/WearTalkAvatar.kt new file mode 100644 index 000000000000..7c8db2d9ef41 --- /dev/null +++ b/apps/android/wear/src/main/java/ai/openclaw/wear/WearTalkAvatar.kt @@ -0,0 +1,409 @@ +package ai.openclaw.wear + +import android.provider.Settings +import androidx.compose.foundation.Canvas +import androidx.compose.runtime.Composable +import androidx.compose.runtime.LaunchedEffect +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableFloatStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.rememberUpdatedState +import androidx.compose.runtime.setValue +import androidx.compose.runtime.withFrameNanos +import androidx.compose.ui.Modifier +import androidx.compose.ui.geometry.Offset +import androidx.compose.ui.geometry.Size +import androidx.compose.ui.graphics.Brush +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.graphics.Path +import androidx.compose.ui.graphics.StrokeCap +import androidx.compose.ui.graphics.drawscope.DrawScope +import androidx.compose.ui.graphics.drawscope.Stroke +import androidx.compose.ui.graphics.drawscope.withTransform +import androidx.compose.ui.graphics.vector.PathParser +import androidx.compose.ui.platform.LocalContext +import kotlin.math.PI +import kotlin.math.cos +import kotlin.math.exp +import kotlin.math.max +import kotlin.math.sin + +// Canonical 120x120 mascot geometry from ui/public/favicon.svg. Parts stay +// separate so the original silhouette can react without substituting artwork. +private val BodyPath by lazy { + PathParser() + .parsePathString( + "M60 10 C30 10 15 35 15 55 C15 75 30 95 45 100 L45 110 L55 110 L55 100 " + + "C55 100 60 102 65 100 L65 110 L75 110 L75 100 C90 95 105 75 105 55 C105 35 90 10 60 10Z", + ).toPath() +} +private val LeftClawPath by lazy { + PathParser().parsePathString("M20 45 C5 40 0 50 5 60 C10 70 20 65 25 55 C28 48 25 45 20 45Z").toPath() +} +private val RightClawPath by lazy { + PathParser().parsePathString("M100 45 C115 40 120 50 115 60 C110 70 100 65 95 55 C92 48 95 45 100 45Z").toPath() +} +private val LeftAntennaPath by lazy { PathParser().parsePathString("M45 15 Q35 5 30 8").toPath() } +private val RightAntennaPath by lazy { PathParser().parsePathString("M75 15 Q85 5 90 8").toPath() } + +private val CoralBright = Color(0xFFFF4D4D) +private val CoralDark = Color(0xFF991B1B) +private val EyeDark = Color(0xFF050810) +private val EyeGlow = Color(0xFF00E5CC) +private val Tongue = Color(0xFFFF9EAE) +private val LeftClawPivot = Offset(26f, 53f) +private val RightClawPivot = Offset(94f, 53f) +private val LeftAntennaPivot = Offset(37.5f, 11f) +private val RightAntennaPivot = Offset(82.5f, 11f) +private val LeftEyeCenter = Offset(45f, 35f) +private val RightEyeCenter = Offset(75f, 35f) + +private data class WearAvatarPose( + val floatOffset: Float, + val bodyTilt: Float, + val bodyStretch: Float, + val antennaDegrees: Float, + val antennaDroop: Float, + val leftClawDegrees: Float, + val rightClawDegrees: Float, + val eyeOpenness: Float, + val gaze: Offset, + val mouthLevel: Float, + val haloPulse: Float, +) + +@Composable +internal fun WearTalkAvatar( + state: RealtimeVoiceButtonState, + mouthLevel: Float, + syntheticSpeech: Boolean, + accent: Color, + danger: Color, + modifier: Modifier = Modifier, +) { + val context = LocalContext.current + val animationsEnabled = + remember(context) { + Settings.Global.getFloat(context.contentResolver, Settings.Global.ANIMATOR_DURATION_SCALE, 1f) > 0f + } + val latestState by rememberUpdatedState(state) + val latestMouthLevel by rememberUpdatedState(mouthLevel) + val latestSyntheticSpeech by rememberUpdatedState(syntheticSpeech) + var animationSeconds by remember { mutableFloatStateOf(0f) } + var smoothedMouth by remember { mutableFloatStateOf(0f) } + + LaunchedEffect(animationsEnabled) { + if (!animationsEnabled) { + animationSeconds = 0f + smoothedMouth = 0f + return@LaunchedEffect + } + var lastFrameNanos = 0L + while (true) { + withFrameNanos { frameNanos -> + if (lastFrameNanos != 0L) { + val deltaSeconds = ((frameNanos - lastFrameNanos) / 1_000_000_000f).coerceIn(0f, 0.05f) + animationSeconds = (animationSeconds + deltaSeconds) % AVATAR_ANIMATION_CYCLE_SECONDS + val targetMouth = + if (latestState == RealtimeVoiceButtonState.SPEAKING) { + max( + latestMouthLevel.coerceIn(0f, 1f), + if (latestSyntheticSpeech) syntheticSpeechMouth(animationSeconds) else 0f, + ) + } else { + 0f + } + smoothedMouth = smoothAvatarMouth(smoothedMouth, targetMouth, deltaSeconds) + } + lastFrameNanos = frameNanos + } + } + } + + val staticMouth = + if (!animationsEnabled && state == RealtimeVoiceButtonState.SPEAKING) { + mouthLevel.coerceIn(0f, 1f) + } else { + smoothedMouth + } + val pose = avatarPoseAt(state, animationSeconds, staticMouth) + val stateColor = if (state == RealtimeVoiceButtonState.ERROR) danger else accent + + Canvas(modifier = modifier) { + val unit = size.minDimension + val center = Offset(size.width / 2f, size.height / 2f) + drawCircle( + color = stateColor.copy(alpha = 0.3f + (0.28f * pose.haloPulse)), + radius = unit * (0.455f + (0.012f * pose.haloPulse)), + center = center, + style = Stroke(width = unit * 0.025f), + ) + + val artScale = unit / CANONICAL_ART_BOX + val artLeft = center.x - ((CANONICAL_ART_SIZE * artScale) / 2f) + val artTop = center.y - ((CANONICAL_ART_SIZE * artScale) / 2f) + (unit * 0.025f) + withTransform({ translate(left = artLeft, top = artTop) }) { + withTransform({ scale(artScale, artScale, pivot = Offset.Zero) }) { + drawCanonicalAvatar(pose, state, animationSeconds) + } + } + } +} + +private fun DrawScope.drawCanonicalAvatar( + pose: WearAvatarPose, + state: RealtimeVoiceButtonState, + animationSeconds: Float, +) { + val stretchX = (1f + ((1f - pose.bodyStretch) * 0.5f)).coerceIn(0.96f, 1.04f) + withTransform({ translate(top = pose.floatOffset) }) { + withTransform({ + scale(stretchX, pose.bodyStretch, pivot = Offset(60f, 110f)) + rotate(pose.bodyTilt, pivot = Offset(60f, 60f)) + }) { + drawPath( + path = BodyPath, + brush = + Brush.linearGradient( + colors = listOf(CoralBright, CoralDark), + start = Offset(15f, 10f), + end = Offset(105f, 110f), + ), + ) + withTransform({ rotate(pose.leftClawDegrees, pivot = LeftClawPivot) }) { + drawPath( + path = LeftClawPath, + brush = + Brush.linearGradient( + colors = listOf(CoralBright, CoralDark), + start = Offset(3.125f, 43.67f), + end = Offset(26.197f, 65.451f), + ), + ) + } + withTransform({ rotate(pose.rightClawDegrees, pivot = RightClawPivot) }) { + drawPath( + path = RightClawPath, + brush = + Brush.linearGradient( + colors = listOf(CoralBright, CoralDark), + start = Offset(93.803f, 43.67f), + end = Offset(116.875f, 65.451f), + ), + ) + } + + val antennaStroke = Stroke(width = 2f, cap = StrokeCap.Round) + val wiggle = pose.antennaDegrees * (1f - pose.antennaDroop) + withTransform({ rotate((-pose.antennaDroop * 40f), pivot = Offset(45f, 15f)) }) { + withTransform({ rotate(wiggle, pivot = LeftAntennaPivot) }) { + drawPath(LeftAntennaPath, CoralBright, style = antennaStroke) + } + } + withTransform({ rotate((pose.antennaDroop * 40f), pivot = Offset(75f, 15f)) }) { + withTransform({ rotate(wiggle, pivot = RightAntennaPivot) }) { + drawPath(RightAntennaPath, CoralBright, style = antennaStroke) + } + } + + drawCanonicalEye(LeftEyeCenter, pose.eyeOpenness, pose.gaze) + drawCanonicalEye(RightEyeCenter, pose.eyeOpenness, pose.gaze) + drawCanonicalMouth(state, pose.mouthLevel, animationSeconds) + } + } +} + +private fun DrawScope.drawCanonicalEye( + center: Offset, + openness: Float, + gaze: Offset, +) { + val eyeHeight = max(1.2f, 12f * openness) + val eyeCenterY = center.y - 6f + ((12f - eyeHeight) * 0.65f) + (eyeHeight / 2f) + drawOval( + color = EyeDark, + topLeft = Offset(center.x - 6f, eyeCenterY - (eyeHeight / 2f)), + size = Size(12f, eyeHeight), + ) + if (openness <= 0.16f) return + + val pupil = + Offset( + x = center.x + (gaze.x * 2.7f), + y = center.y - 1f + (gaze.y * 2.1f), + ) + drawCircle( + color = EyeGlow, + radius = 2.1f, + center = pupil, + alpha = ((openness - 0.16f) / 0.84f).coerceIn(0f, 1f), + ) +} + +private fun DrawScope.drawCanonicalMouth( + state: RealtimeVoiceButtonState, + mouthLevel: Float, + animationSeconds: Float, +) { + if (state == RealtimeVoiceButtonState.ERROR) { + val frown = + Path().apply { + moveTo(52.5f, 54f) + quadraticTo(60f, 47f, 67.5f, 54f) + } + drawPath(frown, EyeDark, style = Stroke(width = 2.2f, cap = StrokeCap.Round)) + return + } + if (state != RealtimeVoiceButtonState.SPEAKING || mouthLevel <= 0.025f) return + + val vowelShape = 0.5f + (0.5f * sin(animationSeconds * 2f * PI.toFloat() / 0.31f)) + val radiusX = 2.2f + (mouthLevel * (4.7f + (1.6f * vowelShape))) + val radiusY = 1.1f + (mouthLevel * (6.5f - (1.3f * vowelShape))) + drawOval( + color = EyeDark, + topLeft = Offset(60f - radiusX, 52f - radiusY), + size = Size(radiusX * 2f, radiusY * 2f), + ) + if (mouthLevel > 0.48f) { + drawOval( + color = Tongue, + topLeft = Offset(60f - (radiusX * 0.55f), 52f + (radiusY * 0.24f)), + size = Size(radiusX * 1.1f, radiusY * 0.42f), + alpha = ((mouthLevel - 0.48f) / 0.52f).coerceIn(0f, 0.82f), + ) + } +} + +private fun avatarPoseAt( + state: RealtimeVoiceButtonState, + animationSeconds: Float, + mouthLevel: Float, +): WearAvatarPose { + val tau = 2f * PI.toFloat() + val breathing = sin(animationSeconds * tau / 3.8f) + var floatOffset = -2.6f * (1f - cos(animationSeconds * tau / 4.2f)) + var bodyTilt = 0.8f * sin(animationSeconds * tau / 6.4f) + var bodyStretch = 1f + (0.012f * breathing) + var antennaDegrees = -3f * sin(animationSeconds * tau / 2.1f) + var antennaDroop = 0f + var leftClawDegrees = 0f + var rightClawDegrees = 0f + var gaze = Offset(0.45f * sin(animationSeconds * tau / 7.5f), 0.2f * sin(animationSeconds * tau / 5.8f)) + var eyeOpenness = 1f - (0.96f * avatarBlinkClosure(animationSeconds)) + var haloPulse = 0.5f + (0.5f * sin(animationSeconds * tau / 2.4f)) + + when (state) { + RealtimeVoiceButtonState.IDLE -> Unit + RealtimeVoiceButtonState.CONNECTING -> { + val orbit = animationSeconds * tau / 1.65f + gaze = Offset(cos(orbit) * 1.05f, sin(orbit) * 0.82f) + bodyTilt = 2f * sin(animationSeconds * tau / 2.8f) + antennaDegrees = -7f * sin(animationSeconds * tau / 1.1f) + leftClawDegrees = 3f * sin(animationSeconds * tau / 1.4f) + rightClawDegrees = -leftClawDegrees + haloPulse = 0.5f + (0.5f * sin(animationSeconds * tau / 0.9f)) + } + RealtimeVoiceButtonState.LISTENING -> { + val attentivePulse = 0.5f + (0.5f * sin(animationSeconds * tau / 1.25f)) + gaze = Offset(0.2f * sin(animationSeconds * tau / 3.2f), 0.34f) + bodyStretch += 0.018f * attentivePulse + leftClawDegrees = 4f + (2f * attentivePulse) + rightClawDegrees = -leftClawDegrees + antennaDegrees = -4f * sin(animationSeconds * tau / 1.45f) + haloPulse = attentivePulse + } + RealtimeVoiceButtonState.THINKING -> { + val orbit = animationSeconds * tau / 2.15f + gaze = Offset(cos(orbit) * 1.15f, sin(orbit) * 0.92f) + bodyTilt = 2.8f * sin(animationSeconds * tau / 4.5f) + antennaDegrees = -7f * sin(animationSeconds * tau / 1.25f) + leftClawDegrees = 5f + (2f * sin(animationSeconds * tau / 2.7f)) + rightClawDegrees = -10f - (3f * sin(animationSeconds * tau / 2.2f)) + haloPulse = 0.5f + (0.5f * sin(animationSeconds * tau / 1.4f)) + } + RealtimeVoiceButtonState.SPEAKING -> { + val speechBeat = sin(animationSeconds * tau / 0.72f) + floatOffset -= mouthLevel * 2.2f + bodyStretch += (mouthLevel * 0.055f) + (speechBeat * 0.008f) + bodyTilt = 1.5f * sin(animationSeconds * tau / 2.1f) + antennaDegrees = -5f * sin(animationSeconds * tau / 0.95f) + leftClawDegrees = 4f + (mouthLevel * 10f) + (speechBeat * 2f) + rightClawDegrees = -leftClawDegrees + gaze = Offset(0.18f * sin(animationSeconds * tau / 2.6f), 0.12f) + haloPulse = (0.25f + (mouthLevel * 0.75f)).coerceIn(0f, 1f) + } + RealtimeVoiceButtonState.ERROR -> { + bodyTilt = 2.2f * sin(animationSeconds * tau / 0.42f) + antennaDroop = 0.72f + leftClawDegrees = -5f + rightClawDegrees = 5f + gaze = Offset(0f, 0.7f) + eyeOpenness *= 0.72f + haloPulse = 0.72f + (0.28f * sin(animationSeconds * tau / 0.8f)) + } + } + + return WearAvatarPose( + floatOffset = floatOffset, + bodyTilt = bodyTilt, + bodyStretch = bodyStretch.coerceIn(0.94f, 1.08f), + antennaDegrees = antennaDegrees, + antennaDroop = antennaDroop, + leftClawDegrees = leftClawDegrees, + rightClawDegrees = rightClawDegrees, + eyeOpenness = eyeOpenness.coerceIn(0.04f, 1f), + gaze = gaze, + mouthLevel = mouthLevel.coerceIn(0f, 1f), + haloPulse = haloPulse.coerceIn(0f, 1f), + ) +} + +internal fun smoothAvatarMouth( + current: Float, + target: Float, + deltaSeconds: Float, +): Float { + val safeCurrent = current.coerceIn(0f, 1f) + val safeTarget = target.coerceIn(0f, 1f) + val safeDelta = deltaSeconds.coerceIn(0f, 0.05f) + if (safeDelta == 0f) return safeCurrent + + val responseSeconds = if (safeTarget > safeCurrent) MOUTH_ATTACK_SECONDS else MOUTH_RELEASE_SECONDS + val blend = (1.0 - exp((-safeDelta / responseSeconds).toDouble())).toFloat() + return (safeCurrent + ((safeTarget - safeCurrent) * blend)).coerceIn(0f, 1f) +} + +private fun syntheticSpeechMouth(animationSeconds: Float): Float { + val tau = 2f * PI.toFloat() + val syllable = 0.5f + (0.5f * sin(animationSeconds * tau / 0.19f)) + val phrase = 0.68f + (0.32f * sin(animationSeconds * tau / 0.83f)) + return (0.1f + (0.72f * syllable * phrase)).coerceIn(0.08f, 0.86f) +} + +private fun avatarBlinkClosure(animationSeconds: Float): Float { + val phase = animationSeconds % BLINK_CYCLE_SECONDS + return when { + phase in FIRST_BLINK_START..FIRST_BLINK_END -> + smoothBell((phase - FIRST_BLINK_START) / (FIRST_BLINK_END - FIRST_BLINK_START)) + phase in SECOND_BLINK_START..SECOND_BLINK_END -> + smoothBell((phase - SECOND_BLINK_START) / (SECOND_BLINK_END - SECOND_BLINK_START)) + else -> 0f + } +} + +private fun smoothBell(value: Float): Float { + val mirrored = if (value < 0.5f) value * 2f else (1f - value) * 2f + val clamped = mirrored.coerceIn(0f, 1f) + return clamped * clamped * (3f - (2f * clamped)) +} + +private const val CANONICAL_ART_SIZE = 120f +private const val CANONICAL_ART_BOX = 126f +private const val AVATAR_ANIMATION_CYCLE_SECONDS = 60f +private const val MOUTH_ATTACK_SECONDS = 0.045f +private const val MOUTH_RELEASE_SECONDS = 0.11f +private const val BLINK_CYCLE_SECONDS = 5.4f +private const val FIRST_BLINK_START = 3.58f +private const val FIRST_BLINK_END = 3.76f +private const val SECOND_BLINK_START = 4.02f +private const val SECOND_BLINK_END = 4.17f diff --git a/apps/android/wear/src/main/java/ai/openclaw/wear/WearViewModel.kt b/apps/android/wear/src/main/java/ai/openclaw/wear/WearViewModel.kt index 5812011e8c9d..0cf66920a7df 100644 --- a/apps/android/wear/src/main/java/ai/openclaw/wear/WearViewModel.kt +++ b/apps/android/wear/src/main/java/ai/openclaw/wear/WearViewModel.kt @@ -40,6 +40,7 @@ internal data class WearUiState( val realtimeTalk: WearRealtimeTalkSnapshot = WearRealtimeTalkSnapshot(), val realtimeCapturing: Boolean = false, val realtimePlaying: Boolean = false, + val realtimeMouthLevel: Float = 0f, val realtimePlaybackFailed: Boolean = false, val talkBusy: Boolean = false, val controlBusy: Boolean = false, @@ -65,6 +66,7 @@ internal fun WearUiState.resetForPhoneChange(): WearUiState = realtimeTalk = WearRealtimeTalkSnapshot(), realtimeCapturing = false, realtimePlaying = false, + realtimeMouthLevel = 0f, realtimePlaybackFailed = false, talkBusy = false, controlBusy = false, @@ -92,6 +94,7 @@ internal fun WearUiState.switchSessionContext(session: WearSession): WearUiState selectedModelRef = session.modelRef, models = emptyList(), realtimeTalk = WearRealtimeTalkSnapshot(), + realtimeMouthLevel = 0f, talkBusy = false, error = null, ) @@ -152,6 +155,7 @@ internal class WearViewModel( realtimeTalk = WearRealtimeTalkSnapshot(), realtimeCapturing = false, realtimePlaying = false, + realtimeMouthLevel = 0f, talkBusy = false, error = "Watch audio link disconnected", ) @@ -169,6 +173,11 @@ internal class WearViewModel( mutableState.update { it.copy(realtimePlaying = playing) } } } + viewModelScope.launch { + realtimeTalkClient.mouthLevel.collect { level -> + mutableState.update { it.copy(realtimeMouthLevel = level) } + } + } refresh() } diff --git a/apps/android/wear/src/test/java/ai/openclaw/wear/WearTalkAvatarTest.kt b/apps/android/wear/src/test/java/ai/openclaw/wear/WearTalkAvatarTest.kt new file mode 100644 index 000000000000..1882fb0488f0 --- /dev/null +++ b/apps/android/wear/src/test/java/ai/openclaw/wear/WearTalkAvatarTest.kt @@ -0,0 +1,146 @@ +package ai.openclaw.wear + +import ai.openclaw.wear.shared.WearProtocol +import ai.openclaw.wear.shared.WearRpcMethod +import kotlinx.coroutines.channels.Channel +import kotlinx.serialization.json.JsonObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.RuntimeEnvironment +import org.robolectric.annotation.Config +import org.robolectric.shadows.ShadowSystemClock +import java.time.Duration + +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [35]) +class WearTalkAvatarTest { + @Test + fun silenceKeepsTheAvatarMouthClosed() { + val pcm = ByteArray(samplesForFrames(2) * 2) + + assertEquals(listOf(0f, 0f), pcm16LeMouthLevels(pcm)) + } + + @Test + fun outputPcmProducesOneBoundedMouthLevelPerPlaybackFrame() { + val pcm = pcm16Le(samplesForFrames(2), sample = 24_000) + + val levels = pcm16LeMouthLevels(pcm) + + assertEquals(2, levels.size) + assertTrue(levels.all { level -> level in 0f..1f }) + assertTrue(levels.all { level -> level > 0.9f }) + } + + @Test + fun finalPartialPlaybackFrameStillMovesTheMouth() { + val pcm = pcm16Le(samplesForFrames(1) + 12, sample = 12_000) + + val levels = pcm16LeMouthLevels(pcm) + + assertEquals(2, levels.size) + assertTrue(levels.last() > 0f) + } + + @Test + fun consecutiveMaximumSizeChunksPreserveCumulativeWindowsAndFlushTheFinalPartial() { + val chunks = + listOf( + pcm16Le(WearProtocol.MAX_REALTIME_AUDIO_FRAME_BYTES / 2, sample = 4_000), + pcm16Le(WearProtocol.MAX_REALTIME_AUDIO_FRAME_BYTES / 2, sample = 20_000), + pcm16Le(100, sample = 12_000), + ) + val client = realtimeTalkClient() + val queuedLevels = Channel(Channel.UNLIMITED) + client.setPrivateField("activeNodeId", "watch-a") + client.setPrivateField("mouthFrames", queuedLevels) + + try { + val writeOutput = WearRealtimeTalkClient::class.java.getDeclaredMethod("writeOutput", ByteArray::class.java) + writeOutput.isAccessible = true + chunks.forEach { chunk -> writeOutput.invoke(client, chunk) } + awaitPlaybackTeardown(client) + + val actualLevels = + buildList { + while (true) add(queuedLevels.tryReceive().getOrNull() ?: break) + } + assertEquals(pcm16LeMouthLevels(chunks.reduce(ByteArray::plus)), actualLevels) + } finally { + client.shutdown() + } + } + + @Test + fun mouthEnvelopeUsesFastAttackAndSoftReleaseWithoutOvershoot() { + val attack = smoothAvatarMouth(current = 0f, target = 1f, deltaSeconds = 0.02f) + val release = smoothAvatarMouth(current = 1f, target = 0f, deltaSeconds = 0.02f) + + assertTrue(attack in 0f..1f) + assertTrue(release in 0f..1f) + assertTrue(attack > 0f) + assertTrue(release > attack) + } + + @Test + fun mouthEnvelopeConvergesAcrossDisplayFrames() { + var level = 0f + repeat(30) { level = smoothAvatarMouth(level, target = 1f, deltaSeconds = 1f / 60f) } + + assertTrue(level > 0.99f) + + repeat(60) { level = smoothAvatarMouth(level, target = 0f, deltaSeconds = 1f / 60f) } + + assertTrue(level < 0.001f) + } + + private fun samplesForFrames(frameCount: Int): Int = WEAR_REALTIME_SAMPLE_RATE_HZ * MOUTH_FRAME_MILLIS / 1_000 * frameCount + + private fun pcm16Le( + sampleCount: Int, + sample: Int, + ): ByteArray = + ByteArray(sampleCount * 2).also { bytes -> + repeat(sampleCount) { index -> + bytes[index * 2] = (sample and 0xff).toByte() + bytes[(index * 2) + 1] = ((sample shr 8) and 0xff).toByte() + } + } + + private fun realtimeTalkClient(): WearRealtimeTalkClient { + val requester = + object : WearRpcRequester { + override suspend fun request( + method: WearRpcMethod, + params: JsonObject, + expectedNodeId: String?, + requirePreferredNode: Boolean, + ): WearRpcResult = error("Unexpected request: $method $params $expectedNodeId $requirePreferredNode") + } + return WearRealtimeTalkClient(RuntimeEnvironment.getApplication(), WearGatewayRepository(requester)) + } + + private fun awaitPlaybackTeardown(client: WearRealtimeTalkClient) { + ShadowSystemClock.advanceBy(Duration.ofSeconds(1L)) + val deadlineNanos = System.nanoTime() + 2_000_000_000L + while (client.isPlaying.value && System.nanoTime() < deadlineNanos) Thread.sleep(10L) + assertEquals(false, client.isPlaying.value) + } + + private fun Any.setPrivateField( + name: String, + value: Any, + ) { + javaClass.getDeclaredField(name).apply { + isAccessible = true + set(this@setPrivateField, value) + } + } + + private companion object { + const val WEAR_REALTIME_SAMPLE_RATE_HZ = 24_000 + } +}