diff --git a/AGENTS.md b/AGENTS.md index 759aab24094..c5f9fe6bd81 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -172,6 +172,7 @@ Skills own workflows; root owns hard policy and routing. - Autoreview staged/uncommitted diff: use `--mode uncommitted`; no `staged` mode. - If proof is blocked, say exactly what is missing and why. - Do not land related failing format/lint/type/build/tests. If unrelated on latest `origin/main`, say so with scoped proof. +- Landing PR onto red `main` (unrelated breakage blocks the merge gate): fix the breakage in the same landing PR; note it in the PR body; never land onto red or bypass the gate. Prefer the smallest correct fix (e.g. register a missing source file, add a dropped export). - Docs/changelog-only and CI/workflow metadata-only: `git diff --check` plus relevant docs/workflow sanity; escalate only if scripts/config/generated/package/runtime behavior changed. - Prompt snapshots: CI truth is Linux Node 24. If macOS local passes but CI drifts, reproduce/generate in Linux before rerun. diff --git a/apps/.i18n/apple-translation-contradictions.json b/apps/.i18n/apple-translation-contradictions.json index 573b5cce565..0a02d971822 100644 --- a/apps/.i18n/apple-translation-contradictions.json +++ b/apps/.i18n/apple-translation-contradictions.json @@ -6196,6 +6196,14 @@ "OpenClaw не може визначити поточний дозвіл iOS на доступ до геолокації." ] }, + { + "locale": "ko", + "source": "OpenClaw unavailable", + "translations": [ + "OpenClaw를 사용할 수 없습니다", + "OpenClaw을 사용할 수 없습니다" + ] + }, { "locale": "ar", "source": "Opens Settings / Gateway", diff --git a/apps/.i18n/native-source.json b/apps/.i18n/native-source.json index 7c7f62b617d..4907008a6ea 100644 --- a/apps/.i18n/native-source.json +++ b/apps/.i18n/native-source.json @@ -25763,7 +25763,7 @@ }, { "kind": "ui-localized-call", - "line": 608, + "line": 611, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Gateway relay ready", "surface": "apple", @@ -25771,7 +25771,7 @@ }, { "kind": "ui-localized-call", - "line": 610, + "line": 613, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Realtime Voice, Gateway relay ready", "surface": "apple", @@ -25779,7 +25779,7 @@ }, { "kind": "ui-localized-call", - "line": 612, + "line": 615, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Listening starts from this phone", "surface": "apple", @@ -25787,7 +25787,7 @@ }, { "kind": "ui-localized-call", - "line": 852, + "line": 861, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Off", "surface": "apple", @@ -25795,7 +25795,7 @@ }, { "kind": "ui-localized-call", - "line": 855, + "line": 864, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Not active", "surface": "apple", @@ -25803,7 +25803,7 @@ }, { "kind": "ui-localized-call", - "line": 1041, + "line": 1052, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Requesting permissions…", "surface": "apple", @@ -25811,7 +25811,7 @@ }, { "kind": "ui-localized-call", - "line": 1070, + "line": 1081, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Start failed: %@", "surface": "apple", @@ -25819,7 +25819,7 @@ }, { "kind": "ui-localized-call", - "line": 1326, + "line": 1337, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Microphone permission denied", "surface": "apple", @@ -25827,7 +25827,7 @@ }, { "kind": "ui-localized-call", - "line": 1336, + "line": 1347, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Speech recognition", "surface": "apple", @@ -25835,7 +25835,7 @@ }, { "kind": "ui-localized-call", - "line": 1467, + "line": 1478, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Offline", "surface": "apple", @@ -25843,7 +25843,7 @@ }, { "kind": "ui-localized-call", - "line": 1666, + "line": 1677, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Speech error: %@", "surface": "apple", @@ -25851,7 +25851,7 @@ }, { "kind": "ui-localized-call", - "line": 1981, + "line": 1992, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Gateway not connected", "surface": "apple", @@ -25859,7 +25859,7 @@ }, { "kind": "conditional-branch", - "line": 2025, + "line": 2036, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Aborted", "surface": "apple", @@ -25867,7 +25867,7 @@ }, { "kind": "conditional-branch", - "line": 2025, + "line": 2036, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Chat error", "surface": "apple", @@ -25875,7 +25875,7 @@ }, { "kind": "ui-localized-call", - "line": 2053, + "line": 2064, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Talk failed: %@", "surface": "apple", @@ -25883,7 +25883,7 @@ }, { "kind": "ui-localized-call", - "line": 2157, + "line": 2168, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "No reply", "surface": "apple", @@ -25891,7 +25891,7 @@ }, { "kind": "ui-localized-call", - "line": 2252, + "line": 2340, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Paused", "surface": "apple", @@ -25899,7 +25899,7 @@ }, { "kind": "ui-localized-call", - "line": 2664, + "line": 2869, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Generating voice…", "surface": "apple", @@ -25907,7 +25907,7 @@ }, { "kind": "ui-localized-call", - "line": 2802, + "line": 3007, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Speak failed: %@", "surface": "apple", @@ -25915,7 +25915,7 @@ }, { "kind": "ui-localized-call", - "line": 2901, + "line": 3106, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Speaking (System)…", "surface": "apple", @@ -25923,7 +25923,7 @@ }, { "kind": "ui-localized-call", - "line": 3782, + "line": 3987, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "iOS System Voice", "surface": "apple", @@ -25931,7 +25931,7 @@ }, { "kind": "ui-localized-call", - "line": 3784, + "line": 3989, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Realtime Voice", "surface": "apple", @@ -25939,7 +25939,7 @@ }, { "kind": "ui-localized-call", - "line": 3788, + "line": 3993, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Gateway Default", "surface": "apple", @@ -25947,7 +25947,7 @@ }, { "kind": "ui-localized-call", - "line": 3847, + "line": 4052, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Listening (Realtime)", "surface": "apple", @@ -25955,7 +25955,7 @@ }, { "kind": "ui-localized-call", - "line": 3849, + "line": 4054, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Thinking", "surface": "apple", @@ -25963,7 +25963,7 @@ }, { "kind": "ui-localized-call", - "line": 3851, + "line": 4056, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Thinking…", "surface": "apple", @@ -25971,7 +25971,7 @@ }, { "kind": "ui-localized-call", - "line": 3853, + "line": 4058, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Asking OpenClaw", "surface": "apple", @@ -25979,7 +25979,7 @@ }, { "kind": "ui-localized-call", - "line": 3855, + "line": 4060, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Still asking OpenClaw", "surface": "apple", @@ -25987,7 +25987,7 @@ }, { "kind": "ui-localized-call", - "line": 3857, + "line": 4062, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Updating OpenClaw", "surface": "apple", @@ -25995,7 +25995,7 @@ }, { "kind": "ui-localized-call", - "line": 3859, + "line": 4064, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Speaking", "surface": "apple", @@ -26003,7 +26003,7 @@ }, { "kind": "ui-localized-call", - "line": 3861, + "line": 4066, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Speaking…", "surface": "apple", @@ -26011,7 +26011,7 @@ }, { "kind": "ui-localized-call", - "line": 3863, + "line": 4068, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Connecting", "surface": "apple", @@ -26019,7 +26019,7 @@ }, { "kind": "ui-localized-call", - "line": 3865, + "line": 4070, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Connecting realtime…", "surface": "apple", @@ -26027,7 +26027,7 @@ }, { "kind": "ui-localized-call", - "line": 3867, + "line": 4072, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Waiting for realtime…", "surface": "apple", @@ -26035,7 +26035,7 @@ }, { "kind": "ui-localized-call", - "line": 3869, + "line": 4074, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Ready", "surface": "apple", @@ -26043,7 +26043,7 @@ }, { "kind": "ui-localized-call", - "line": 3871, + "line": 4076, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Reconnecting", "surface": "apple", @@ -26051,7 +26051,7 @@ }, { "kind": "ui-localized-call", - "line": 3873, + "line": 4078, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Reconnecting…", "surface": "apple", @@ -26059,7 +26059,7 @@ }, { "kind": "ui-localized-call", - "line": 3875, + "line": 4080, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Realtime failed before connecting", "surface": "apple", @@ -26067,7 +26067,7 @@ }, { "kind": "ui-localized-call", - "line": 3877, + "line": 4082, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Realtime disconnected", "surface": "apple", @@ -26075,7 +26075,7 @@ }, { "kind": "ui-localized-call", - "line": 3879, + "line": 4084, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "OpenClaw unavailable", "surface": "apple", @@ -26083,7 +26083,7 @@ }, { "kind": "ui-localized-call", - "line": 3919, + "line": 4124, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "iOS Speech + TTS", "surface": "apple", @@ -26091,7 +26091,7 @@ }, { "kind": "ui-localized-call", - "line": 3921, + "line": 4126, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Listening", "surface": "apple", @@ -26099,7 +26099,7 @@ }, { "kind": "ui-localized-call", - "line": 3925, + "line": 4130, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "iOS Speech fallback", "surface": "apple", @@ -26107,7 +26107,7 @@ }, { "kind": "ui-localized-call", - "line": 4186, + "line": 4391, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Gateway Relay", "surface": "apple", @@ -26115,7 +26115,7 @@ }, { "kind": "ui-localized-call", - "line": 4188, + "line": 4393, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Native WebRTC", "surface": "apple", @@ -26123,7 +26123,7 @@ }, { "kind": "ui-localized-call", - "line": 4189, + "line": 4394, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Native", "surface": "apple", @@ -26131,7 +26131,7 @@ }, { "kind": "ui-localized-call", - "line": 4250, + "line": 4455, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Gateway permission required", "surface": "apple", @@ -26139,7 +26139,7 @@ }, { "kind": "ui-localized-call", - "line": 4272, + "line": 4477, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Not loaded", "surface": "apple", @@ -26147,7 +26147,7 @@ }, { "kind": "ui-localized-call", - "line": 4290, + "line": 4495, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Approval requested", "surface": "apple", @@ -26155,7 +26155,7 @@ }, { "kind": "ui-localized-call", - "line": 4729, + "line": 4934, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Realtime unavailable", "surface": "apple", @@ -26163,7 +26163,7 @@ }, { "kind": "ui-localized-call", - "line": 4751, + "line": 4956, "path": "apps/ios/Sources/Voice/TalkModeManager.swift", "source": "Listening (PTT)", "surface": "apple", @@ -26171,7 +26171,7 @@ }, { "kind": "conditional-branch", - "line": 446, + "line": 616, "path": "apps/ios/Sources/Voice/TalkRealtimeWebRTCSession.swift", "source": "Asking OpenClaw", "surface": "apple", @@ -26179,12 +26179,28 @@ }, { "kind": "conditional-branch", - "line": 446, + "line": 616, "path": "apps/ios/Sources/Voice/TalkRealtimeWebRTCSession.swift", "source": "Updating OpenClaw", "surface": "apple", "id": "native.apple.d2db26b3bb266491" }, + { + "kind": "conditional-branch", + "line": 705, + "path": "apps/ios/Sources/Voice/TalkRealtimeWebRTCSession.swift", + "source": "Confirmation needed", + "surface": "apple", + "id": "native.apple.4b89dccea20c58fa" + }, + { + "kind": "conditional-branch", + "line": 705, + "path": "apps/ios/Sources/Voice/TalkRealtimeWebRTCSession.swift", + "source": "OpenClaw unavailable", + "surface": "apple", + "id": "native.apple.8a62f0e9416188e3" + }, { "kind": "ui-localized-call", "line": 47, diff --git a/apps/.i18n/native/ar.json b/apps/.i18n/native/ar.json index 5c942f43143..b14d67d39c8 100644 --- a/apps/.i18n/native/ar.json +++ b/apps/.i18n/native/ar.json @@ -16368,6 +16368,16 @@ "source": "Updating OpenClaw", "translated": "جارٍ تحديث OpenClaw" }, + { + "id": "native.apple.4b89dccea20c58fa", + "source": "Confirmation needed", + "translated": "التأكيد مطلوب" + }, + { + "id": "native.apple.8a62f0e9416188e3", + "source": "OpenClaw unavailable", + "translated": "OpenClaw غير متاح" + }, { "id": "native.apple.8a944b06087fe8ef", "source": "Microphone input format unavailable", diff --git a/apps/.i18n/native/de.json b/apps/.i18n/native/de.json index 2bed75b9202..5e7ccf8defe 100644 --- a/apps/.i18n/native/de.json +++ b/apps/.i18n/native/de.json @@ -16368,6 +16368,16 @@ "source": "Updating OpenClaw", "translated": "OpenClaw wird aktualisiert" }, + { + "id": "native.apple.4b89dccea20c58fa", + "source": "Confirmation needed", + "translated": "Bestätigung erforderlich" + }, + { + "id": "native.apple.8a62f0e9416188e3", + "source": "OpenClaw unavailable", + "translated": "OpenClaw nicht verfügbar" + }, { "id": "native.apple.8a944b06087fe8ef", "source": "Microphone input format unavailable", diff --git a/apps/.i18n/native/es.json b/apps/.i18n/native/es.json index 1b1668848b3..63c78071c5b 100644 --- a/apps/.i18n/native/es.json +++ b/apps/.i18n/native/es.json @@ -16368,6 +16368,16 @@ "source": "Updating OpenClaw", "translated": "Actualizando OpenClaw" }, + { + "id": "native.apple.4b89dccea20c58fa", + "source": "Confirmation needed", + "translated": "Se necesita confirmación" + }, + { + "id": "native.apple.8a62f0e9416188e3", + "source": "OpenClaw unavailable", + "translated": "OpenClaw no está disponible" + }, { "id": "native.apple.8a944b06087fe8ef", "source": "Microphone input format unavailable", diff --git a/apps/.i18n/native/fa.json b/apps/.i18n/native/fa.json index 8bfdaaac9f9..95eca533067 100644 --- a/apps/.i18n/native/fa.json +++ b/apps/.i18n/native/fa.json @@ -16368,6 +16368,16 @@ "source": "Updating OpenClaw", "translated": "در حال به‌روزرسانی OpenClaw" }, + { + "id": "native.apple.4b89dccea20c58fa", + "source": "Confirmation needed", + "translated": "نیاز به تأیید" + }, + { + "id": "native.apple.8a62f0e9416188e3", + "source": "OpenClaw unavailable", + "translated": "OpenClaw در دسترس نیست" + }, { "id": "native.apple.8a944b06087fe8ef", "source": "Microphone input format unavailable", diff --git a/apps/.i18n/native/fr.json b/apps/.i18n/native/fr.json index d4a0d7cf18c..02fca0d9d9e 100644 --- a/apps/.i18n/native/fr.json +++ b/apps/.i18n/native/fr.json @@ -16368,6 +16368,16 @@ "source": "Updating OpenClaw", "translated": "Mise à jour d’OpenClaw" }, + { + "id": "native.apple.4b89dccea20c58fa", + "source": "Confirmation needed", + "translated": "Confirmation requise" + }, + { + "id": "native.apple.8a62f0e9416188e3", + "source": "OpenClaw unavailable", + "translated": "OpenClaw indisponible" + }, { "id": "native.apple.8a944b06087fe8ef", "source": "Microphone input format unavailable", diff --git a/apps/.i18n/native/hi.json b/apps/.i18n/native/hi.json index f3362af7cdd..d48cffd0ce7 100644 --- a/apps/.i18n/native/hi.json +++ b/apps/.i18n/native/hi.json @@ -16368,6 +16368,16 @@ "source": "Updating OpenClaw", "translated": "OpenClaw अपडेट किया जा रहा है" }, + { + "id": "native.apple.4b89dccea20c58fa", + "source": "Confirmation needed", + "translated": "पुष्टि आवश्यक है" + }, + { + "id": "native.apple.8a62f0e9416188e3", + "source": "OpenClaw unavailable", + "translated": "OpenClaw उपलब्ध नहीं है" + }, { "id": "native.apple.8a944b06087fe8ef", "source": "Microphone input format unavailable", diff --git a/apps/.i18n/native/id.json b/apps/.i18n/native/id.json index f0f33bff1f3..b4c13482e72 100644 --- a/apps/.i18n/native/id.json +++ b/apps/.i18n/native/id.json @@ -16368,6 +16368,16 @@ "source": "Updating OpenClaw", "translated": "Memperbarui OpenClaw" }, + { + "id": "native.apple.4b89dccea20c58fa", + "source": "Confirmation needed", + "translated": "Konfirmasi diperlukan" + }, + { + "id": "native.apple.8a62f0e9416188e3", + "source": "OpenClaw unavailable", + "translated": "OpenClaw tidak tersedia" + }, { "id": "native.apple.8a944b06087fe8ef", "source": "Microphone input format unavailable", diff --git a/apps/.i18n/native/it.json b/apps/.i18n/native/it.json index a81730c5e0f..4bd77140938 100644 --- a/apps/.i18n/native/it.json +++ b/apps/.i18n/native/it.json @@ -16368,6 +16368,16 @@ "source": "Updating OpenClaw", "translated": "Aggiornamento di OpenClaw" }, + { + "id": "native.apple.4b89dccea20c58fa", + "source": "Confirmation needed", + "translated": "Conferma necessaria" + }, + { + "id": "native.apple.8a62f0e9416188e3", + "source": "OpenClaw unavailable", + "translated": "OpenClaw non disponibile" + }, { "id": "native.apple.8a944b06087fe8ef", "source": "Microphone input format unavailable", diff --git a/apps/.i18n/native/ja-JP.json b/apps/.i18n/native/ja-JP.json index 61c7c8c9b9e..d76aa3361b0 100644 --- a/apps/.i18n/native/ja-JP.json +++ b/apps/.i18n/native/ja-JP.json @@ -16368,6 +16368,16 @@ "source": "Updating OpenClaw", "translated": "OpenClaw を更新中" }, + { + "id": "native.apple.4b89dccea20c58fa", + "source": "Confirmation needed", + "translated": "確認が必要です" + }, + { + "id": "native.apple.8a62f0e9416188e3", + "source": "OpenClaw unavailable", + "translated": "OpenClawは利用できません" + }, { "id": "native.apple.8a944b06087fe8ef", "source": "Microphone input format unavailable", diff --git a/apps/.i18n/native/ko.json b/apps/.i18n/native/ko.json index dc18581441f..320b6fa7b4e 100644 --- a/apps/.i18n/native/ko.json +++ b/apps/.i18n/native/ko.json @@ -16368,6 +16368,16 @@ "source": "Updating OpenClaw", "translated": "OpenClaw 업데이트 중" }, + { + "id": "native.apple.4b89dccea20c58fa", + "source": "Confirmation needed", + "translated": "확인이 필요합니다" + }, + { + "id": "native.apple.8a62f0e9416188e3", + "source": "OpenClaw unavailable", + "translated": "OpenClaw을 사용할 수 없습니다" + }, { "id": "native.apple.8a944b06087fe8ef", "source": "Microphone input format unavailable", diff --git a/apps/.i18n/native/nl.json b/apps/.i18n/native/nl.json index baaa826838c..e9e4309f0c3 100644 --- a/apps/.i18n/native/nl.json +++ b/apps/.i18n/native/nl.json @@ -16368,6 +16368,16 @@ "source": "Updating OpenClaw", "translated": "OpenClaw bijwerken" }, + { + "id": "native.apple.4b89dccea20c58fa", + "source": "Confirmation needed", + "translated": "Bevestiging vereist" + }, + { + "id": "native.apple.8a62f0e9416188e3", + "source": "OpenClaw unavailable", + "translated": "OpenClaw niet beschikbaar" + }, { "id": "native.apple.8a944b06087fe8ef", "source": "Microphone input format unavailable", diff --git a/apps/.i18n/native/pl.json b/apps/.i18n/native/pl.json index fc67042cb80..be6263670d7 100644 --- a/apps/.i18n/native/pl.json +++ b/apps/.i18n/native/pl.json @@ -16368,6 +16368,16 @@ "source": "Updating OpenClaw", "translated": "Aktualizowanie OpenClaw" }, + { + "id": "native.apple.4b89dccea20c58fa", + "source": "Confirmation needed", + "translated": "Wymagane potwierdzenie" + }, + { + "id": "native.apple.8a62f0e9416188e3", + "source": "OpenClaw unavailable", + "translated": "OpenClaw jest niedostępny" + }, { "id": "native.apple.8a944b06087fe8ef", "source": "Microphone input format unavailable", diff --git a/apps/.i18n/native/pt-BR.json b/apps/.i18n/native/pt-BR.json index 02ca7a3396f..e83dc99a9c5 100644 --- a/apps/.i18n/native/pt-BR.json +++ b/apps/.i18n/native/pt-BR.json @@ -16368,6 +16368,16 @@ "source": "Updating OpenClaw", "translated": "Atualizando o OpenClaw" }, + { + "id": "native.apple.4b89dccea20c58fa", + "source": "Confirmation needed", + "translated": "Confirmação necessária" + }, + { + "id": "native.apple.8a62f0e9416188e3", + "source": "OpenClaw unavailable", + "translated": "OpenClaw indisponível" + }, { "id": "native.apple.8a944b06087fe8ef", "source": "Microphone input format unavailable", diff --git a/apps/.i18n/native/ru.json b/apps/.i18n/native/ru.json index d494fbe98b9..65473537219 100644 --- a/apps/.i18n/native/ru.json +++ b/apps/.i18n/native/ru.json @@ -16368,6 +16368,16 @@ "source": "Updating OpenClaw", "translated": "Обновление OpenClaw" }, + { + "id": "native.apple.4b89dccea20c58fa", + "source": "Confirmation needed", + "translated": "Требуется подтверждение" + }, + { + "id": "native.apple.8a62f0e9416188e3", + "source": "OpenClaw unavailable", + "translated": "OpenClaw недоступен" + }, { "id": "native.apple.8a944b06087fe8ef", "source": "Microphone input format unavailable", diff --git a/apps/.i18n/native/sv.json b/apps/.i18n/native/sv.json index ff82400d7ec..b37cdd3c1d3 100644 --- a/apps/.i18n/native/sv.json +++ b/apps/.i18n/native/sv.json @@ -16368,6 +16368,16 @@ "source": "Updating OpenClaw", "translated": "Uppdaterar OpenClaw" }, + { + "id": "native.apple.4b89dccea20c58fa", + "source": "Confirmation needed", + "translated": "Bekräftelse krävs" + }, + { + "id": "native.apple.8a62f0e9416188e3", + "source": "OpenClaw unavailable", + "translated": "OpenClaw är inte tillgängligt" + }, { "id": "native.apple.8a944b06087fe8ef", "source": "Microphone input format unavailable", diff --git a/apps/.i18n/native/th.json b/apps/.i18n/native/th.json index 265c425fdd0..f389d391e03 100644 --- a/apps/.i18n/native/th.json +++ b/apps/.i18n/native/th.json @@ -16368,6 +16368,16 @@ "source": "Updating OpenClaw", "translated": "กำลังอัปเดต OpenClaw" }, + { + "id": "native.apple.4b89dccea20c58fa", + "source": "Confirmation needed", + "translated": "ต้องยืนยัน" + }, + { + "id": "native.apple.8a62f0e9416188e3", + "source": "OpenClaw unavailable", + "translated": "OpenClaw ไม่พร้อมใช้งาน" + }, { "id": "native.apple.8a944b06087fe8ef", "source": "Microphone input format unavailable", diff --git a/apps/.i18n/native/tr.json b/apps/.i18n/native/tr.json index c762e483ea5..86f02a817fb 100644 --- a/apps/.i18n/native/tr.json +++ b/apps/.i18n/native/tr.json @@ -16368,6 +16368,16 @@ "source": "Updating OpenClaw", "translated": "OpenClaw güncelleniyor" }, + { + "id": "native.apple.4b89dccea20c58fa", + "source": "Confirmation needed", + "translated": "Onay gerekli" + }, + { + "id": "native.apple.8a62f0e9416188e3", + "source": "OpenClaw unavailable", + "translated": "OpenClaw kullanılamıyor" + }, { "id": "native.apple.8a944b06087fe8ef", "source": "Microphone input format unavailable", diff --git a/apps/.i18n/native/uk.json b/apps/.i18n/native/uk.json index 2d5f3087a96..d473718efce 100644 --- a/apps/.i18n/native/uk.json +++ b/apps/.i18n/native/uk.json @@ -16368,6 +16368,16 @@ "source": "Updating OpenClaw", "translated": "Оновлення OpenClaw" }, + { + "id": "native.apple.4b89dccea20c58fa", + "source": "Confirmation needed", + "translated": "Потрібне підтвердження" + }, + { + "id": "native.apple.8a62f0e9416188e3", + "source": "OpenClaw unavailable", + "translated": "OpenClaw недоступний" + }, { "id": "native.apple.8a944b06087fe8ef", "source": "Microphone input format unavailable", diff --git a/apps/.i18n/native/vi.json b/apps/.i18n/native/vi.json index 374432f69ad..49545d95be6 100644 --- a/apps/.i18n/native/vi.json +++ b/apps/.i18n/native/vi.json @@ -16368,6 +16368,16 @@ "source": "Updating OpenClaw", "translated": "Đang cập nhật OpenClaw" }, + { + "id": "native.apple.4b89dccea20c58fa", + "source": "Confirmation needed", + "translated": "Cần xác nhận" + }, + { + "id": "native.apple.8a62f0e9416188e3", + "source": "OpenClaw unavailable", + "translated": "OpenClaw không khả dụng" + }, { "id": "native.apple.8a944b06087fe8ef", "source": "Microphone input format unavailable", diff --git a/apps/.i18n/native/zh-CN.json b/apps/.i18n/native/zh-CN.json index 4bd490c6f33..0494f4ca7bb 100644 --- a/apps/.i18n/native/zh-CN.json +++ b/apps/.i18n/native/zh-CN.json @@ -16368,6 +16368,16 @@ "source": "Updating OpenClaw", "translated": "正在更新 OpenClaw" }, + { + "id": "native.apple.4b89dccea20c58fa", + "source": "Confirmation needed", + "translated": "需要确认" + }, + { + "id": "native.apple.8a62f0e9416188e3", + "source": "OpenClaw unavailable", + "translated": "OpenClaw 不可用" + }, { "id": "native.apple.8a944b06087fe8ef", "source": "Microphone input format unavailable", diff --git a/apps/.i18n/native/zh-TW.json b/apps/.i18n/native/zh-TW.json index d08f9799ca2..6e1decc2819 100644 --- a/apps/.i18n/native/zh-TW.json +++ b/apps/.i18n/native/zh-TW.json @@ -16368,6 +16368,16 @@ "source": "Updating OpenClaw", "translated": "正在更新 OpenClaw" }, + { + "id": "native.apple.4b89dccea20c58fa", + "source": "Confirmation needed", + "translated": "需要確認" + }, + { + "id": "native.apple.8a62f0e9416188e3", + "source": "OpenClaw unavailable", + "translated": "OpenClaw 無法使用" + }, { "id": "native.apple.8a944b06087fe8ef", "source": "Microphone input format unavailable", diff --git a/apps/ios/Resources/Localizable.xcstrings b/apps/ios/Resources/Localizable.xcstrings index 767c0181df5..4cc24eaa9bc 100644 --- a/apps/ios/Resources/Localizable.xcstrings +++ b/apps/ios/Resources/Localizable.xcstrings @@ -34681,6 +34681,142 @@ } } }, + "Confirmation needed": { + "localizations": { + "en": { + "stringUnit": { + "state": "translated", + "value": "Confirmation needed" + } + }, + "zh-CN": { + "stringUnit": { + "state": "translated", + "value": "需要确认" + } + }, + "zh-TW": { + "stringUnit": { + "state": "translated", + "value": "需要確認" + } + }, + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Confirmação necessária" + } + }, + "de": { + "stringUnit": { + "state": "translated", + "value": "Bestätigung erforderlich" + } + }, + "es": { + "stringUnit": { + "state": "translated", + "value": "Se necesita confirmación" + } + }, + "ja-JP": { + "stringUnit": { + "state": "translated", + "value": "確認が必要です" + } + }, + "ko": { + "stringUnit": { + "state": "translated", + "value": "확인이 필요합니다" + } + }, + "fr": { + "stringUnit": { + "state": "translated", + "value": "Confirmation requise" + } + }, + "hi": { + "stringUnit": { + "state": "translated", + "value": "पुष्टि आवश्यक है" + } + }, + "ar": { + "stringUnit": { + "state": "translated", + "value": "التأكيد مطلوب" + } + }, + "it": { + "stringUnit": { + "state": "translated", + "value": "Conferma necessaria" + } + }, + "tr": { + "stringUnit": { + "state": "translated", + "value": "Onay gerekli" + } + }, + "uk": { + "stringUnit": { + "state": "translated", + "value": "Потрібне підтвердження" + } + }, + "id": { + "stringUnit": { + "state": "translated", + "value": "Konfirmasi diperlukan" + } + }, + "pl": { + "stringUnit": { + "state": "translated", + "value": "Wymagane potwierdzenie" + } + }, + "th": { + "stringUnit": { + "state": "translated", + "value": "ต้องยืนยัน" + } + }, + "vi": { + "stringUnit": { + "state": "translated", + "value": "Cần xác nhận" + } + }, + "nl": { + "stringUnit": { + "state": "translated", + "value": "Bevestiging vereist" + } + }, + "fa": { + "stringUnit": { + "state": "translated", + "value": "نیاز به تأیید" + } + }, + "ru": { + "stringUnit": { + "state": "translated", + "value": "Требуется подтверждение" + } + }, + "sv": { + "stringUnit": { + "state": "translated", + "value": "Bekräftelse krävs" + } + } + } + }, "Connect": { "localizations": { "en": { diff --git a/apps/ios/Sources/Voice/TalkModeManager.swift b/apps/ios/Sources/Voice/TalkModeManager.swift index 0d4c0fe991b..81b06e0b03a 100644 --- a/apps/ios/Sources/Voice/TalkModeManager.swift +++ b/apps/ios/Sources/Voice/TalkModeManager.swift @@ -241,6 +241,9 @@ final class TalkModeManager: NSObject { private var recognitionGeneration: UInt64 = 0 private var silenceTask: Task? private var realtimeSession: TalkRealtimeWebRTCSession? + private var activeRealtimeVoiceSessionId: String? + private var realtimeVoiceSessionGeneration: UInt64 = 0 + private let realtimeTranscriptStore = TalkRealtimeTranscriptStore() private var realtimeSessionReadyAt: Date? private var rapidRealtimeRestartCount = 0 private var bypassRealtimeOnNextStart = false @@ -541,9 +544,7 @@ final class TalkModeManager: NSObject { } self.gatewayTalkActiveModeTitle = String(localized: "Not active") self.gatewayTalkActiveModeSubtitle = nil - self.realtimePrefetchGeneration &+= 1 - self.realtimePrefetchTask?.cancel() - self.realtimePrefetchTask = nil + self.cancelRealtimePrefetch() self.prefetchedRealtimeSession = nil } } @@ -561,6 +562,7 @@ final class TalkModeManager: NSObject { _ = self.cancelPushToTalk(captureId: captureId) } self.cancelFinishingPushToTalk() + self.cancelRealtimePrefetch() if hasTalkOwner { // Session identity owns every in-flight relay/capture generation, even while // Talk is disabled. Leaving one alive can publish work into the replacement session. @@ -570,6 +572,7 @@ final class TalkModeManager: NSObject { self.stopNativeCaptureAndDiscardTranscript() deactivateAudioSession() } + self.closeLogicalRealtimeVoiceSessions() self.mainSessionKey = trimmed if shouldRestartTalk, self.gatewayConnected, self.isEnabled { Task { await self.start() } @@ -806,6 +809,12 @@ final class TalkModeManager: NSObject { self.isStarting = false } + private func cancelRealtimePrefetch() { + self.realtimePrefetchGeneration &+= 1 + self.realtimePrefetchTask?.cancel() + self.realtimePrefetchTask = nil + } + private var talkProviderSelection: TalkModeProviderSelection { TalkModeProviderSelection.resolved( UserDefaults.standard.string(forKey: TalkModeProviderSelection.storageKey)) @@ -860,7 +869,9 @@ final class TalkModeManager: NSObject { self.silenceTask?.cancel() self.silenceTask = nil self.resetRealtimeRestartState() + self.cancelRealtimePrefetch() self.stopRealtimeSession() + self.closeLogicalRealtimeVoiceSessions() self.stopRecognition() self.stopSpeaking() self.lastInterruptedAtSeconds = nil @@ -2198,10 +2209,17 @@ final class TalkModeManager: NSObject { } guard self.isCurrentStartAttempt(attemptID) else { return .ignored } let prefetchedSession = self.consumePrefetchedRealtimeSession() - GatewayDiagnostics.log("talk.timeline realtime start attempt sessionKey=\(self.mainSessionKey)") + if let prefetchedSession { + self.activeRealtimeVoiceSessionId = prefetchedSession.voiceSessionId + } + let sessionKey = self.mainSessionKey + let voiceSessionGeneration = self.realtimeVoiceSessionGeneration + GatewayDiagnostics.log("talk.timeline realtime start attempt sessionKey=\(sessionKey)") let session = TalkRealtimeWebRTCSession( gateway: gateway, - sessionKey: mainSessionKey, + sessionKey: sessionKey, + voiceSessionId: self.activeRealtimeVoiceSessionId, + transcriptStore: self.realtimeTranscriptStore, delegate: self) self.realtimeSession = session // WebRTC owns the shared AVAudioSession internally; track the attempt @@ -2214,9 +2232,24 @@ final class TalkModeManager: NSObject { voice: self.realtimeVoiceId, prefetchedSession: prefetchedSession) guard self.realtimeSession === session, self.isCurrentStartAttempt(attemptID) else { - session.stop() + self.finishStaleRealtimeStart( + session, + gateway: gateway, + sessionKey: sessionKey, + voiceSessionGeneration: voiceSessionGeneration) return .ignored } + guard let voiceSessionId = session.voiceSessionId, + self.adoptRealtimeVoiceSessionId( + voiceSessionId, + gateway: gateway, + sessionKey: sessionKey) + else { + self.stopRealtimeSession() + return .unavailable(realtimeIssue( + message: "Gateway returned a conflicting realtime voice session", + phase: "start")) + } // WebRTC configures the shared session and may force speaker + built-in mic. // Apply the user's input last so the explicit microphone selection wins. Self.applyPreferredInput(self.preferredInputDeviceID, to: AVAudioSession.sharedInstance()) @@ -2227,9 +2260,19 @@ final class TalkModeManager: NSObject { return .started } catch { guard self.realtimeSession === session, self.isCurrentStartAttempt(attemptID) else { - session.stop() + self.finishStaleRealtimeStart( + session, + gateway: gateway, + sessionKey: sessionKey, + voiceSessionGeneration: voiceSessionGeneration) return .ignored } + if let voiceSessionId = session.voiceSessionId { + _ = self.adoptRealtimeVoiceSessionId( + voiceSessionId, + gateway: gateway, + sessionKey: sessionKey) + } self.stopRealtimeSession() let issue = realtimeIssue(from: error, phase: "start") GatewayDiagnostics @@ -2241,6 +2284,51 @@ final class TalkModeManager: NSObject { } } + private func finishStaleRealtimeStart( + _ session: TalkRealtimeWebRTCSession, + gateway: GatewayNodeSession, + sessionKey: String, + voiceSessionGeneration: UInt64) + { + if let voiceSessionId = session.voiceSessionId { + if self.activeRealtimeVoiceSessionId == nil, + self.realtimeSession == nil, + self.isEnabled, + self.mainSessionKey == sessionKey, + self.realtimeVoiceSessionGeneration == voiceSessionGeneration + { + // A transport restart keeps the logical voice session alive. + self.activeRealtimeVoiceSessionId = voiceSessionId + } else if self.activeRealtimeVoiceSessionId != voiceSessionId { + self.closeOrphanedRealtimeVoiceSession( + gateway: gateway, + sessionKey: sessionKey, + voiceSessionId: voiceSessionId) + } + } + session.stop() + } + + @discardableResult + private func adoptRealtimeVoiceSessionId( + _ voiceSessionId: String, + gateway: GatewayNodeSession, + sessionKey: String) -> Bool + { + if let activeRealtimeVoiceSessionId, activeRealtimeVoiceSessionId != voiceSessionId { + GatewayDiagnostics.log( + "talk realtime voice session mismatch active=\(activeRealtimeVoiceSessionId) " + + "returned=\(voiceSessionId)") + self.closeOrphanedRealtimeVoiceSession( + gateway: gateway, + sessionKey: sessionKey, + voiceSessionId: voiceSessionId) + return false + } + self.activeRealtimeVoiceSessionId = voiceSessionId + return true + } + private func startRealtimeRelayIfAvailable(attemptID: Int) async -> RealtimeStartResult { guard let gateway else { return .unavailable(realtimeIssue(message: "Gateway not connected", phase: "start")) @@ -2376,6 +2464,8 @@ final class TalkModeManager: NSObject { GatewayDiagnostics.log("talk.timeline realtime prefetch scheduled reason=\(reason)") self.realtimePrefetchGeneration &+= 1 let prefetchGeneration = self.realtimePrefetchGeneration + let sessionKey = self.mainSessionKey + let requestedVoiceSessionId = self.activeRealtimeVoiceSessionId self.realtimePrefetchTask = Task { @MainActor [weak self] in guard let self else { return } defer { @@ -2391,10 +2481,30 @@ final class TalkModeManager: NSObject { let session = try await self.createRealtimeClientSession( gateway: gateway, route: route, + sessionKey: sessionKey, + voiceSessionId: requestedVoiceSessionId, provider: self.realtimeProvider, model: self.realtimeModelId, voice: self.realtimeVoiceId) - guard !Task.isCancelled, shouldApply() else { return } + guard let voiceSessionId = session.voiceSessionId else { + throw NSError(domain: "TalkRealtimeVoiceSession", code: 2, userInfo: [ + NSLocalizedDescriptionKey: "Gateway did not return a realtime voice session", + ]) + } + guard !Task.isCancelled, shouldApply(), self.mainSessionKey == sessionKey else { + if self.activeRealtimeVoiceSessionId != voiceSessionId { + self.closeOrphanedRealtimeVoiceSession( + gateway: gateway, + sessionKey: sessionKey, + voiceSessionId: voiceSessionId) + } + return + } + guard self.adoptRealtimeVoiceSessionId( + voiceSessionId, + gateway: gateway, + sessionKey: sessionKey) + else { return } self.prefetchedRealtimeSession = session GatewayDiagnostics.log( "talk.timeline realtime prefetch ready elapsedMs=\(Self.elapsedMs(since: startedAt)) " @@ -2411,11 +2521,19 @@ final class TalkModeManager: NSObject { private func createRealtimeClientSession( gateway: GatewayNodeSession, route: GatewayNodeSessionRoute, + sessionKey: String, + voiceSessionId: String?, provider: String?, model: String?, voice: String?) async throws -> TalkRealtimeClientSession { - let params = TalkRealtimeClientCreateParams(provider: provider, model: model, voice: voice) + let params = TalkRealtimeClientCreateParams( + sessionKey: sessionKey, + voiceSessionId: voiceSessionId, + provider: provider, + model: model, + voice: voice, + capabilities: ["voice-transcript"]) let data = try JSONEncoder().encode(params) let json = String(data: data, encoding: .utf8) let res = try await gateway.request( @@ -2447,6 +2565,12 @@ final class TalkModeManager: NSObject { private func stopRealtimeSession() { let realtimeSession = self.realtimeSession + if self.activeRealtimeVoiceSessionId == nil, + let voiceSessionId = realtimeSession?.voiceSessionId + { + // Capture ownership before stopping the transport so close can drain its queue. + self.activeRealtimeVoiceSessionId = voiceSessionId + } let hadRealtimeOwner = realtimeSession != nil || self.realtimeRelaySession != nil || self.realtimeRelayStartGeneration != nil @@ -2467,6 +2591,87 @@ final class TalkModeManager: NSObject { } } + private func closeLogicalRealtimeVoiceSessions() { + // A close boundary invalidates every in-flight transport that captured the old owner. + self.realtimeVoiceSessionGeneration &+= 1 + let voiceSessionIds = Set([ + self.activeRealtimeVoiceSessionId, + self.prefetchedRealtimeSession?.voiceSessionId, + self.realtimeSession?.voiceSessionId, + ].compactMap(\.self)) + self.activeRealtimeVoiceSessionId = nil + self.prefetchedRealtimeSession = nil + guard !voiceSessionIds.isEmpty else { return } + let gateway = self.gateway + let sessionKey = self.mainSessionKey + let transcriptStore = self.realtimeTranscriptStore + Task { @MainActor in + defer { transcriptStore.remove(voiceSessionIds) } + for voiceSessionId in voiceSessionIds.sorted() { + await transcriptStore.flush(voiceSessionId: voiceSessionId) + } + for voiceSessionId in voiceSessionIds.sorted() { + guard let gateway else { + GatewayDiagnostics.log( + "talk voice session close FAILED voiceSessionId=\(voiceSessionId) error=gateway unavailable") + continue + } + do { + try await Self.closeRealtimeVoiceSession( + gateway: gateway, + sessionKey: sessionKey, + voiceSessionId: voiceSessionId) + } catch { + GatewayDiagnostics.log( + "talk voice session close FAILED voiceSessionId=\(voiceSessionId) " + + "error=\(error.localizedDescription)") + } + } + } + } + + private static func closeRealtimeVoiceSession( + gateway: GatewayNodeSession, + sessionKey: String, + voiceSessionId: String) async throws + { + let params = TalkRealtimeClientCloseParams( + sessionKey: sessionKey, + voiceSessionId: voiceSessionId) + let data = try JSONEncoder().encode(params) + guard let json = String(data: data, encoding: .utf8) else { + throw NSError(domain: "TalkRealtimeVoiceSession", code: 1, userInfo: [ + NSLocalizedDescriptionKey: "Failed to encode close request", + ]) + } + _ = try await gateway.request( + method: "talk.client.close", + paramsJSON: json, + timeoutSeconds: 10) + } + + private func closeOrphanedRealtimeVoiceSession( + gateway: GatewayNodeSession, + sessionKey: String, + voiceSessionId: String) + { + let transcriptStore = self.realtimeTranscriptStore + Task { @MainActor in + defer { transcriptStore.remove([voiceSessionId]) } + await transcriptStore.flush(voiceSessionId: voiceSessionId) + do { + try await Self.closeRealtimeVoiceSession( + gateway: gateway, + sessionKey: sessionKey, + voiceSessionId: voiceSessionId) + } catch { + GatewayDiagnostics.log( + "talk voice session close FAILED voiceSessionId=\(voiceSessionId) " + + "error=\(error.localizedDescription)") + } + } + } + private func buildPrompt(transcript: String) -> String { let interrupted = self.lastInterruptedAtSeconds self.lastInterruptedAtSeconds = nil diff --git a/apps/ios/Sources/Voice/TalkRealtimeClientSession.swift b/apps/ios/Sources/Voice/TalkRealtimeClientSession.swift index 998c55ba3e2..26abe556a2f 100644 --- a/apps/ios/Sources/Voice/TalkRealtimeClientSession.swift +++ b/apps/ios/Sources/Voice/TalkRealtimeClientSession.swift @@ -1,17 +1,21 @@ import Foundation struct TalkRealtimeClientCreateParams: Encodable { + var sessionKey: String? + var voiceSessionId: String? var mode = "realtime" var provider: String? var transport = "webrtc" var brain = "agent-consult" var model: String? var voice: String? + var capabilities: [String] } struct TalkRealtimeClientSession: Decodable { let provider: String let transport: String + let voiceSessionId: String? let clientSecret: String let offerUrl: String? let offerHeaders: [String: String]? @@ -24,6 +28,25 @@ struct TalkRealtimeClientSession: Decodable { } } +enum TalkRealtimeTranscriptRole: String, Encodable { + case user + case assistant +} + +struct TalkRealtimeTranscriptParams: Encodable { + let sessionKey: String + let voiceSessionId: String + let entryId: String + let role: TalkRealtimeTranscriptRole + let text: String + let timestamp: Double? +} + +struct TalkRealtimeClientCloseParams: Encodable { + let sessionKey: String + let voiceSessionId: String +} + struct TalkRealtimeToolCallResponse: Decodable { let runId: String? let idempotencyKey: String? diff --git a/apps/ios/Sources/Voice/TalkRealtimeWebRTCSession.swift b/apps/ios/Sources/Voice/TalkRealtimeWebRTCSession.swift index e0945c85663..8f5fc047e46 100644 --- a/apps/ios/Sources/Voice/TalkRealtimeWebRTCSession.swift +++ b/apps/ios/Sources/Voice/TalkRealtimeWebRTCSession.swift @@ -18,6 +18,114 @@ protocol TalkRealtimeWebRTCSessionDelegate: AnyObject { func realtimeSessionDidFinish(_ session: TalkRealtimeWebRTCSession) } +@MainActor +final class TalkRealtimeTranscriptWriteQueue { + typealias Persist = @MainActor (TalkRealtimeTranscriptParams) async throws -> Void + typealias FailureLog = @MainActor (String, Error) -> Void + + private let retryDelaysNanoseconds: [UInt64] + private var tail: Task? + private var generation = 0 + + init(retryDelaysNanoseconds: [UInt64] = [100_000_000, 250_000_000]) { + self.retryDelaysNanoseconds = retryDelaysNanoseconds + } + + func enqueue( + _ params: TalkRealtimeTranscriptParams, + persist: @escaping Persist, + failureLog: @escaping FailureLog) + { + let previous = self.tail + self.generation += 1 + let retryDelaysNanoseconds = Array(self.retryDelaysNanoseconds.prefix(2)) + self.tail = Task { @MainActor in + await previous?.value + var finalError: Error? + for attempt in 0...retryDelaysNanoseconds.count { + do { + try await persist(params) + return + } catch { + finalError = error + guard attempt < retryDelaysNanoseconds.count else { break } + do { + try await Task.sleep(nanoseconds: retryDelaysNanoseconds[attempt]) + } catch { + finalError = error + break + } + } + } + failureLog( + params.entryId, + finalError ?? NSError( + domain: "TalkRealtimeTranscript", + code: 1, + userInfo: [NSLocalizedDescriptionKey: "Transcript persistence failed"])) + } + } + + func flush() async { + while let tail = self.tail { + let generation = self.generation + await tail.value + if self.generation == generation { return } + } + } +} + +@MainActor +final class TalkRealtimeTranscriptStore { + private let retryDelaysNanoseconds: [UInt64] + private var lastEntryIdByVoiceSession: [String: Int] = [:] + private var queuesByVoiceSession: [String: TalkRealtimeTranscriptWriteQueue] = [:] + + init(retryDelaysNanoseconds: [UInt64] = [100_000_000, 250_000_000]) { + self.retryDelaysNanoseconds = retryDelaysNanoseconds + } + + @discardableResult + func enqueue( + sessionKey: String, + voiceSessionId: String, + role: TalkRealtimeTranscriptRole, + text: String, + timestamp: Double, + persist: @escaping TalkRealtimeTranscriptWriteQueue.Persist, + failureLog: @escaping TalkRealtimeTranscriptWriteQueue.FailureLog) -> String + { + let nextEntryId = (self.lastEntryIdByVoiceSession[voiceSessionId] ?? 0) + 1 + self.lastEntryIdByVoiceSession[voiceSessionId] = nextEntryId + let entryId = String(nextEntryId) + let queue = self.queuesByVoiceSession[voiceSessionId] ?? TalkRealtimeTranscriptWriteQueue( + retryDelaysNanoseconds: self.retryDelaysNanoseconds) + self.queuesByVoiceSession[voiceSessionId] = queue + queue.enqueue( + TalkRealtimeTranscriptParams( + sessionKey: sessionKey, + voiceSessionId: voiceSessionId, + entryId: entryId, + role: role, + text: text, + timestamp: timestamp), + persist: persist, + failureLog: failureLog) + return entryId + } + + func flush(voiceSessionId: String) async { + await self.queuesByVoiceSession[voiceSessionId]?.flush() + } + + func remove(_ voiceSessionIds: Set) { + for voiceSessionId in voiceSessionIds { + self.lastEntryIdByVoiceSession.removeValue(forKey: voiceSessionId) + self.queuesByVoiceSession.removeValue(forKey: voiceSessionId) + } + } +} + @MainActor final class TalkRealtimeWebRTCSession: NSObject { private static let logger = Logger(subsystem: "ai.openclawfoundation.app", category: "TalkRealtimeWebRTC") @@ -37,7 +145,9 @@ final class TalkRealtimeWebRTCSession: NSObject { private let gateway: GatewayNodeSession private let sessionKey: String + private let transcriptStore: TalkRealtimeTranscriptStore private weak var delegate: TalkRealtimeWebRTCSessionDelegate? + private var adoptedVoiceSessionId: String? private var factory: RTCPeerConnectionFactory? private var peerConnection: RTCPeerConnection? @@ -72,13 +182,25 @@ final class TalkRealtimeWebRTCSession: NSObject { let providerStarted: Bool? } - init(gateway: GatewayNodeSession, sessionKey: String, delegate: TalkRealtimeWebRTCSessionDelegate) { + init( + gateway: GatewayNodeSession, + sessionKey: String, + voiceSessionId: String? = nil, + transcriptStore: TalkRealtimeTranscriptStore, + delegate: TalkRealtimeWebRTCSessionDelegate) + { self.gateway = gateway self.sessionKey = sessionKey + self.adoptedVoiceSessionId = voiceSessionId + self.transcriptStore = transcriptStore self.delegate = delegate super.init() } + var voiceSessionId: String? { + self.adoptedVoiceSessionId + } + func start( provider: String?, model: String?, @@ -107,6 +229,18 @@ final class TalkRealtimeWebRTCSession: NSObject { } else { session = try await self.createClientSession(provider: provider, model: model, voice: voice) } + guard let returnedVoiceSessionId = session.voiceSessionId else { + throw NSError(domain: "TalkRealtimeWebRTC", code: 11, userInfo: [ + NSLocalizedDescriptionKey: "Gateway did not return a realtime voice session", + ]) + } + let requestedVoiceSessionId = self.adoptedVoiceSessionId + self.adoptedVoiceSessionId = returnedVoiceSessionId + if let requestedVoiceSessionId, requestedVoiceSessionId != returnedVoiceSessionId { + throw NSError(domain: "TalkRealtimeWebRTC", code: 10, userInfo: [ + NSLocalizedDescriptionKey: "Gateway returned a conflicting realtime voice session", + ]) + } let sessionModel = session.model ?? "unknown" let sessionVoice = session.voice ?? "unknown" Self.logger.info( @@ -284,7 +418,41 @@ final class TalkRealtimeWebRTCSession: NSObject { } } + private func recordFinalTranscript(role: TalkRealtimeTranscriptRole, text: String) { + guard let voiceSessionId = self.voiceSessionId else { return } + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + self.transcriptStore.enqueue( + sessionKey: self.sessionKey, + voiceSessionId: voiceSessionId, + role: role, + text: trimmed, + timestamp: Date().timeIntervalSince1970 * 1000, + persist: { [gateway] params in + let data = try JSONEncoder().encode(params) + guard let json = String(data: data, encoding: .utf8) else { + throw NSError(domain: "TalkRealtimeTranscript", code: 2, userInfo: [ + NSLocalizedDescriptionKey: "Failed to encode transcript request", + ]) + } + _ = try await gateway.request( + method: "talk.client.transcript", + paramsJSON: json, + timeoutSeconds: 5) + }, + failureLog: { entryId, error in + GatewayDiagnostics.log( + "talk transcript persist FAILED entryId=\(entryId) error=\(error.localizedDescription)") + }) + } + + func flushTranscriptWrites() async { + guard let voiceSessionId = self.voiceSessionId else { return } + await self.transcriptStore.flush(voiceSessionId: voiceSessionId) + } + private func handleRealtimeEvent(_ event: TalkRealtimeServerEvent) { + guard !self.stopped else { return } if !self.seenRealtimeEventTypes.contains(event.type) { self.seenRealtimeEventTypes.insert(event.type) self.trace("event first type=\(event.type)") @@ -306,6 +474,7 @@ final class TalkRealtimeWebRTCSession: NSObject { "conversation.item.input_audio_transcription.completed": if let text = event.transcript ?? event.text { self.delegate?.realtimeSession(self, didReceiveUserTranscript: text) + self.recordFinalTranscript(role: .user, text: text) } case "conversation.output_transcript.delta", "response.output_text.delta", @@ -324,6 +493,7 @@ final class TalkRealtimeWebRTCSession: NSObject { "response.output_audio_transcript.done": if let text = event.transcript ?? event.text { self.delegate?.realtimeSession(self, didReceiveAssistantTranscript: text) + self.recordFinalTranscript(role: .assistant, text: text) } case "response.function_call_arguments.delta": self.bufferToolDelta(event) @@ -473,12 +643,18 @@ final class TalkRealtimeWebRTCSession: NSObject { } do { let args = try Self.decodeJSONObject(argsJSON) - let params: [String: Any] = [ + await self.flushTranscriptWrites() + try Task.checkCancellation() + try self.checkNotStopped() + var params: [String: Any] = [ "sessionKey": sessionKey, "callId": callId, "name": Self.consultToolName, "args": args, ] + if let voiceSessionId = self.voiceSessionId { + params["voiceSessionId"] = voiceSessionId + } let historySince = Date().timeIntervalSince1970 let data = try JSONSerialization.data(withJSONObject: params) guard let json = String(data: data, encoding: .utf8) else { @@ -523,8 +699,11 @@ final class TalkRealtimeWebRTCSession: NSObject { if let runId = activeToolRunIds[callId] { await self.abortChatRun(runId: runId) } - self.delegate?.realtimeSession(self, didChangeStatus: "OpenClaw unavailable") - let fallbackMessage = [ + let confirmationInstruction = Self.voiceConfirmationInstruction(from: error) + self.delegate?.realtimeSession( + self, + didChangeStatus: confirmationInstruction == nil ? "OpenClaw unavailable" : "Confirmation needed") + let fallbackMessage = confirmationInstruction ?? [ "OpenClaw consult did not finish quickly enough.", "Give a brief spoken fallback from the realtime conversation", "and ask the user to try again if they need OpenClaw-specific context.", @@ -601,6 +780,28 @@ final class TalkRealtimeWebRTCSession: NSObject { return trimmed.isEmpty ? nil : trimmed } + static func voiceConfirmationInstruction(from error: Error) -> String? { + let messages: [String] = if let responseError = error as? GatewayResponseError { + [responseError.message, responseError.detailsReason].compactMap(\.self) + } else { + [error.localizedDescription] + } + let marker = "VOICE_CONFIRMATION_REQUIRED:" + for message in messages { + guard let markerRange = message.range(of: marker) else { continue } + let suffix = message[markerRange.upperBound...] + guard let confirmationId = suffix.split(whereSeparator: { $0.isWhitespace }).first, + !confirmationId.isEmpty + else { continue } + return [ + "\(marker)\(confirmationId) The requested action was not executed.", + "Ask the user for explicit spoken confirmation, then call openclaw_agent_consult again", + "with confirmationId \(confirmationId).", + ].joined(separator: " ") + } + return nil + } + private static func controlResultMessage(from data: Data) -> String? { guard let object = try? JSONSerialization.jsonObject(with: data), let record = object as? [String: Any] @@ -882,7 +1083,13 @@ extension TalkRealtimeWebRTCSession { { self.trace("gateway talk.client.create start") let startedAt = ProcessInfo.processInfo.systemUptime - let params = TalkRealtimeClientCreateParams(provider: provider, model: model, voice: voice) + let params = TalkRealtimeClientCreateParams( + sessionKey: self.sessionKey, + voiceSessionId: self.adoptedVoiceSessionId, + provider: provider, + model: model, + voice: voice, + capabilities: ["voice-transcript"]) let data = try JSONEncoder().encode(params) let json = String(data: data, encoding: .utf8) let res = try await gateway.request(method: "talk.client.create", paramsJSON: json, timeoutSeconds: 12) diff --git a/apps/ios/Tests/TalkModeConfigParsingTests.swift b/apps/ios/Tests/TalkModeConfigParsingTests.swift index 2110a7b8161..7fe96f92b63 100644 --- a/apps/ios/Tests/TalkModeConfigParsingTests.swift +++ b/apps/ios/Tests/TalkModeConfigParsingTests.swift @@ -6,6 +6,62 @@ import Testing @MainActor struct TalkModeManagerTests { + @Test func `encodes realtime client voice session identity`() throws { + let params = TalkRealtimeClientCreateParams( + sessionKey: "agent:main:main", + voiceSessionId: "voice-1", + provider: "openai", + model: "gpt-realtime-2", + voice: "marin", + capabilities: ["voice-transcript"]) + let object = try #require( + JSONSerialization.jsonObject(with: JSONEncoder().encode(params)) as? [String: Any]) + + #expect(object["sessionKey"] as? String == "agent:main:main") + #expect(object["voiceSessionId"] as? String == "voice-1") + #expect(object["capabilities"] as? [String] == ["voice-transcript"]) + } + + @Test func `decodes optional realtime client voice session id`() throws { + let session = try JSONDecoder().decode( + TalkRealtimeClientSession.self, + from: Data( + #"{"provider":"openai","transport":"webrtc","voiceSessionId":"voice-1","clientSecret":"secret"}"#.utf8)) + + #expect(session.voiceSessionId == "voice-1") + + let serverOwned = try JSONDecoder().decode( + TalkRealtimeClientSession.self, + from: Data(#"{"provider":"openai","transport":"gateway-relay","clientSecret":"secret"}"#.utf8)) + #expect(serverOwned.voiceSessionId == nil) + } + + @Test func `encodes transcript entry id as a decimal string`() throws { + let params = TalkRealtimeTranscriptParams( + sessionKey: "agent:main:main", + voiceSessionId: "voice-1", + entryId: "1", + role: .assistant, + text: "hello", + timestamp: 1234) + let object = try #require( + JSONSerialization.jsonObject(with: JSONEncoder().encode(params)) as? [String: Any]) + + #expect(object["entryId"] as? String == "1") + } + + @Test func `surfaces voice confirmation id for a follow up consult`() throws { + let error = GatewayResponseError( + method: "talk.client.toolCall", + code: "INVALID_REQUEST", + message: "VOICE_CONFIRMATION_REQUIRED:confirm_123 Ask the user to confirm.", + details: nil) + + let instruction = try #require(TalkRealtimeWebRTCSession.voiceConfirmationInstruction(from: error)) + #expect(instruction.contains("VOICE_CONFIRMATION_REQUIRED:confirm_123")) + #expect(instruction.contains("confirmationId confirm_123")) + } + @Test func `recognizes open AI maximum duration errors as terminal`() throws { let event = try JSONDecoder().decode( TalkRealtimeServerEvent.self, diff --git a/apps/ios/Tests/TalkRealtimeTranscriptWriteQueueTests.swift b/apps/ios/Tests/TalkRealtimeTranscriptWriteQueueTests.swift new file mode 100644 index 00000000000..f55af4b4e4c --- /dev/null +++ b/apps/ios/Tests/TalkRealtimeTranscriptWriteQueueTests.swift @@ -0,0 +1,81 @@ +import Foundation +import Testing +@testable import OpenClaw + +@MainActor +struct TalkRealtimeTranscriptWriteQueueTests { + private struct PersistError: Error {} + + @Test func `retries writes in order and continues after final failure`() async { + var attempts: [String] = [] + var failures: [String] = [] + let store = TalkRealtimeTranscriptStore(retryDelaysNanoseconds: [0, 0, 0, 0]) + + for _ in 1...3 { + store.enqueue( + sessionKey: "agent:main:main", + voiceSessionId: "voice-1", + role: .user, + text: "hello", + timestamp: 1234, + persist: { params in + attempts.append(params.entryId) + if params.entryId == "1", attempts.filter({ $0 == "1" }).count < 3 { + throw PersistError() + } + if params.entryId == "2" { + throw PersistError() + } + }, + failureLog: { entryId, _ in + failures.append(entryId) + }) + } + await store.flush(voiceSessionId: "voice-1") + + #expect(attempts == ["1", "1", "1", "2", "2", "2", "3"]) + #expect(failures == ["2"]) + } + + @Test func `shares ordering across transport replacements`() async { + var persisted: [String] = [] + let store = TalkRealtimeTranscriptStore(retryDelaysNanoseconds: []) + let persist: TalkRealtimeTranscriptWriteQueue.Persist = { params in + persisted.append("\(params.voiceSessionId):\(params.entryId)") + } + let failureLog: TalkRealtimeTranscriptWriteQueue.FailureLog = { _, _ in } + + let firstTransportEntry = store.enqueue( + sessionKey: "agent:main:main", + voiceSessionId: "voice-a", + role: .user, + text: "one", + timestamp: 1, + persist: persist, + failureLog: failureLog) + let replacementTransportEntry = store.enqueue( + sessionKey: "agent:main:main", + voiceSessionId: "voice-a", + role: .assistant, + text: "two", + timestamp: 2, + persist: persist, + failureLog: failureLog) + let otherVoiceSessionEntry = store.enqueue( + sessionKey: "agent:main:main", + voiceSessionId: "voice-b", + role: .user, + text: "other", + timestamp: 3, + persist: persist, + failureLog: failureLog) + await store.flush(voiceSessionId: "voice-a") + await store.flush(voiceSessionId: "voice-b") + + #expect(firstTransportEntry == "1") + #expect(replacementTransportEntry == "2") + #expect(otherVoiceSessionEntry == "1") + #expect(persisted.filter { $0.hasPrefix("voice-a:") } == ["voice-a:1", "voice-a:2"]) + #expect(persisted.filter { $0.hasPrefix("voice-b:") } == ["voice-b:1"]) + } +}