feat(talk): add Gemini Live video (#109719)

Stream bounded browser camera frames directly to Gemini Live and keep camera media off the Gateway. Add lifecycle, function-calling, fake-camera E2E, and live-smoke coverage.

Co-authored-by: shushushu <1064076525@qq.com>

docs/.generated/plugin-sdk-api-baseline.sha256

@@ -1,2 +1,2 @@
-f6306d8082be6dcde00435a02db2a5cf46f0983fccb8ed6a39a60f1931546fce  plugin-sdk-api-baseline.json
-7c11aba346798e6db5be30a9b61141d1904a09764ea029656fd0fca79929df61  plugin-sdk-api-baseline.jsonl
+4440586f2797bc47ac5085d962feaab844d921acaba4e3b2f754c76f70a4f1a5  plugin-sdk-api-baseline.json
+88ab286e2484b5178a5447cd880372c773a11f3bf3a73a54759aa5bff55f5957  plugin-sdk-api-baseline.jsonl

docs/web/control-ui.md

@@ -392,9 +392,9 @@ The macOS app keeps its native link-browser sidebar for links clicked in the das
 
     The Talk control itself is the microphone button in the composer toolbar. Its caret lists **System default** and every microphone exposed by the browser, including USB, Bluetooth, and virtual inputs. The selected device ID stays browser-local and is never sent to the Gateway; if that exact device disappears, Talk asks you to choose another input instead of silently recording from a different microphone. While Talk is live, the microphone button becomes a pill showing the live input-level meter; clicking it stops voice input, and hovering it reveals the stop glyph. Screen readers announce `Connecting voice input...`, `Listening...`, or `Asking OpenClaw...` while a realtime tool call is consulting the configured larger model through `talk.client.toolCall`. Stopping a running agent response stays a separate square **Stop** control next to the pill.
 
-    **Video Talk** is available for OpenAI Realtime WebRTC sessions. Click the camera button, allow camera and microphone access, and confirm the local preview. When the model needs visual context, `describe_view` captures one JPEG frame and sends it directly over the existing browser-to-OpenAI Realtime data channel; camera frames do not pass through the Gateway. Stopping Talk closes the preview and releases both media tracks. Google Live remains voice-only in the Control UI.
+    **Video Talk** is available for OpenAI Realtime WebRTC and Google Live browser sessions. Click the camera button, allow camera and microphone access, and confirm the local preview. OpenAI sends one bounded JPEG frame over its browser data channel when `describe_view` requests visual context. Google Live sends bounded JPEG frames directly from the browser to the provider at the supported maximum of one frame per second and answers `describe_view` function calls with the camera-stream state. Camera frames never pass through the Gateway. Stopping Talk closes the preview and releases both media tracks. See Google's [Live API capabilities](https://ai.google.dev/gemini-api/docs/live-api/capabilities#video) and [function-calling guide](https://ai.google.dev/gemini-api/docs/live-api/tools) for the provider wire contracts.
 
-    Maintainer live smoke: `OPENAI_API_KEY=... GEMINI_API_KEY=... node --import tsx scripts/dev/realtime-talk-live-smoke.ts` verifies the OpenAI backend WebSocket bridge, OpenAI browser WebRTC SDP exchange, Google Live constrained-token browser WebSocket setup, and the Gateway relay browser adapter with fake microphone media. The command prints provider status only and does not log secrets.
+    Maintainer live smoke: `OPENAI_API_KEY=... GEMINI_API_KEY=... node --import tsx scripts/dev/realtime-talk-live-smoke.ts` verifies the OpenAI backend WebSocket bridge, OpenAI browser WebRTC SDP exchange, Google Live constrained-token browser setup with a JPEG frame and `describe_view` function roundtrip, and the Gateway relay browser adapter with fake microphone media. The command prints provider status only and does not log secrets.
 
   </Accordion>
   <Accordion title="Stop and abort">

extensions/openai/realtime-voice-provider.test.ts

@@ -304,6 +304,7 @@ describe("buildOpenAIRealtimeVoiceProvider", () => {
       supportsBargeIn: true,
       handlesInputAudioBargeIn: true,
       supportsToolCalls: true,
+      supportsVideoFrames: true,
     });
   });
 

extensions/openai/realtime-voice-provider.ts

@@ -1560,6 +1560,7 @@ export function buildOpenAIRealtimeVoiceProvider(): RealtimeVoiceProviderPlugin
       supportsBargeIn: true,
       handlesInputAudioBargeIn: true,
       supportsToolCalls: true,
+      supportsVideoFrames: true,
     },
     resolveConfig: ({ rawConfig }) => normalizeProviderConfig(rawConfig),
     isConfigured: ({ cfg, providerConfig }) => {

scripts/dev/realtime-talk-live-smoke.ts

@@ -450,54 +450,55 @@ async function smokeOpenAIWebRtc(browser: Browser, apiKey: string): Promise<Smok
   }
 }
 
-async function createGoogleLiveToken(apiKey: string): Promise<string> {
-  const { GoogleGenAI, Modality } = await import("@google/genai");
-  const ai = new GoogleGenAI({
-    apiKey,
-    httpOptions: { apiVersion: "v1alpha" },
-  });
-  const now = Date.now();
-  const token = await ai.authTokens.create({
-    config: {
-      uses: 1,
-      expireTime: new Date(now + 30 * 60 * 1000).toISOString(),
-      newSessionExpireTime: new Date(now + 60 * 1000).toISOString(),
-      liveConnectConstraints: {
-        model: GOOGLE_REALTIME_MODEL,
-        config: {
-          responseModalities: [Modality.AUDIO],
-          speechConfig: {
-            voiceConfig: {
-              prebuiltVoiceConfig: { voiceName: GOOGLE_REALTIME_VOICE },
-            },
-          },
-          systemInstruction: "OpenClaw browser Talk live smoke.",
-          inputAudioTranscription: {},
-          outputAudioTranscription: {},
-        },
-      },
-    },
-  });
-  const name = token.name?.trim();
-  if (!name) {
-    throw new Error("Google Live auth token response did not include a token name");
-  }
-  return name;
-}
-
 async function smokeGoogleLiveBrowserWs(browser: Browser, apiKey: string): Promise<SmokeResult> {
   try {
-    const token = await createGoogleLiveToken(apiKey);
+    const { REALTIME_VOICE_DESCRIBE_VIEW_TOOL } =
+      await import("../../src/talk/describe-view-tool.ts");
+    const { buildGoogleRealtimeVoiceProvider } =
+      await import("../../extensions/google/realtime-voice-provider.ts");
+    const provider = buildGoogleRealtimeVoiceProvider();
+    const session = await provider.createBrowserSession?.({
+      cfg: {},
+      providerConfig: {
+        apiKey,
+        model: GOOGLE_REALTIME_MODEL,
+        voice: GOOGLE_REALTIME_VOICE,
+      },
+      model: GOOGLE_REALTIME_MODEL,
+      voice: GOOGLE_REALTIME_VOICE,
+      instructions:
+        "OpenClaw browser Video Talk live smoke. After receiving a visual frame and request, call describe_view exactly once.",
+      tools: [REALTIME_VOICE_DESCRIBE_VIEW_TOOL],
+    });
+    if (
+      !session ||
+      session.transport !== "provider-websocket" ||
+      session.protocol !== "google-live-bidi"
+    ) {
+      throw new Error("Google Live provider did not create a browser WebSocket session");
+    }
     const page = await browser.newPage();
     await page.evaluate("globalThis.__name = (fn) => fn");
     const result = await page.evaluate(
-      async ({ model, tokenName, websocketUrl }) => {
+      async ({
+        initialMessage,
+        tokenName,
+        websocketUrl,
+      }: {
+        initialMessage: unknown;
+        tokenName: string;
+        websocketUrl: string;
+      }) => {
         const debug: {
           opened: boolean;
           messages: string[];
           close?: { code: number; reason: string };
           error: boolean;
         } = { opened: false, messages: [], error: false };
+        let setupComplete = false;
+        let videoFrameSent = false;
+        let describeViewCalled = false;
+        let functionResponseSent = false;
         const dataToText = async (data: unknown): Promise<string> => {
           if (typeof data === "string") {
             return data;
@@ -520,27 +521,78 @@ async function smokeGoogleLiveBrowserWs(browser: Browser, apiKey: string): Promi
           );
           ws.addEventListener("open", () => {
             debug.opened = true;
-            ws.send(
-              JSON.stringify({
-                setup: {
-                  model: model.startsWith("models/") ? model : `models/${model}`,
-                  generationConfig: { responseModalities: ["AUDIO"] },
-                  inputAudioTranscription: {},
-                  outputAudioTranscription: {},
-                },
-              }),
-            );
+            ws.send(JSON.stringify(initialMessage));
           });
           ws.addEventListener("message", (event) => {
             void (async () => {
               const text = await dataToText(event.data);
               debug.messages.push(text.slice(0, 300));
-              const message = JSON.parse(text) as { setupComplete?: unknown };
-              if (!message.setupComplete) {
+              const message = JSON.parse(text) as {
+                setupComplete?: unknown;
+                serverContent?: unknown;
+                toolCall?: {
+                  functionCalls?: Array<{ id?: string; name?: string }>;
+                };
+              };
+              if (message.setupComplete) {
+                setupComplete = true;
+                const canvas = document.createElement("canvas");
+                canvas.width = 8;
+                canvas.height = 8;
+                const context = canvas.getContext("2d");
+                if (!context) {
+                  throw new Error("Google Live smoke could not create a camera fixture");
+                }
+                context.fillStyle = "#2f81f7";
+                context.fillRect(0, 0, canvas.width, canvas.height);
+                const frame = canvas.toDataURL("image/jpeg", 0.7).split(",")[1];
+                if (!frame) {
+                  throw new Error("Google Live smoke camera fixture was empty");
+                }
+                ws.send(
+                  JSON.stringify({
+                    realtimeInput: { video: { data: frame, mimeType: "image/jpeg" } },
+                  }),
+                );
+                videoFrameSent = true;
+                ws.send(
+                  JSON.stringify({
+                    realtimeInput: { text: "Call describe_view now for the visual frame." },
+                  }),
+                );
                 return;
               }
-              window.clearTimeout(timeout);
-              resolve({ setupComplete: true, readyState: ws.readyState });
+              const describeView = message.toolCall?.functionCalls?.find(
+                (call) => call.name === "describe_view" && call.id,
+              );
+              if (describeView?.id) {
+                describeViewCalled = true;
+                ws.send(
+                  JSON.stringify({
+                    toolResponse: {
+                      functionResponses: [
+                        {
+                          id: describeView.id,
+                          name: "describe_view",
+                          response: { ok: true, cameraStreamActive: true },
+                        },
+                      ],
+                    },
+                  }),
+                );
+                functionResponseSent = true;
+                return;
+              }
+              if (message.serverContent && functionResponseSent) {
+                window.clearTimeout(timeout);
+                resolve({
+                  setupComplete,
+                  videoFrameSent,
+                  describeViewCalled,
+                  functionResponseAccepted: true,
+                  readyState: ws.readyState,
+                });
+              }
             })().catch((error: unknown) => {
               window.clearTimeout(timeout);
               reject(toLintErrorObject(error, "Non-Error rejection"));
@@ -564,16 +616,26 @@ async function smokeGoogleLiveBrowserWs(browser: Browser, apiKey: string): Promi
         return value;
       },
       {
-        model: GOOGLE_REALTIME_MODEL,
-        tokenName: token,
-        websocketUrl: GOOGLE_LIVE_WS_URL,
+        initialMessage: session.initialMessage ?? { setup: {} },
+        tokenName: session.clientSecret,
+        websocketUrl: session.websocketUrl || GOOGLE_LIVE_WS_URL,
       },
     );
     await page.close();
     return {
       name: "google-live-browser-ws",
-      ok: result.setupComplete === true,
-      details: { model: GOOGLE_REALTIME_MODEL, setupComplete: result.setupComplete === true },
+      ok:
+        result.setupComplete === true &&
+        result.videoFrameSent === true &&
+        result.describeViewCalled === true &&
+        result.functionResponseAccepted === true,
+      details: {
+        model: GOOGLE_REALTIME_MODEL,
+        setupComplete: result.setupComplete === true,
+        videoFrameSent: result.videoFrameSent === true,
+        describeViewCalled: result.describeViewCalled === true,
+        functionResponseAccepted: result.functionResponseAccepted === true,
+      },
     };
   } catch (error) {
     return { name: "google-live-browser-ws", ok: false, details: { error: shortError(error) } };

src/gateway/server-methods/talk-client.ts

@@ -95,6 +95,7 @@ export const talkClientHandlers: GatewayRequestHandlers = {
       }
       const transport =
         normalizeOptionalLowercaseString(typedParams.transport) ?? realtimeConfig.transport;
+      const wantsCameraFrames = typedParams.capabilities?.includes("camera-frame") === true;
       if (transport === "managed-room") {
         respond(
           false,
@@ -112,7 +113,9 @@ export const talkClientHandlers: GatewayRequestHandlers = {
           undefined,
           errorShape(
             ErrorCodes.INVALID_REQUEST,
-            `talk.client.create is client-owned; use talk.session.create for gateway-relay`,
+            wantsCameraFrames
+              ? "gateway-relay does not support browser video frames"
+              : `talk.client.create is client-owned; use talk.session.create for gateway-relay`,
           ),
         );
         return;
@@ -125,17 +128,24 @@ export const talkClientHandlers: GatewayRequestHandlers = {
         defaultModel: realtimeConfig.model,
         noRegisteredProviderMessage: "No realtime voice provider registered",
       });
+      if (wantsCameraFrames && resolution.provider.capabilities?.supportsVideoFrames !== true) {
+        respond(
+          false,
+          undefined,
+          errorShape(
+            ErrorCodes.INVALID_REQUEST,
+            `Realtime provider ${resolution.provider.id} does not support browser video frames`,
+          ),
+        );
+        return;
+      }
       const launchOptions = buildRealtimeVoiceLaunchOptions({
         requested: typedParams,
         defaults: realtimeConfig,
       });
       if (resolution.provider.createBrowserSession && transport !== "gateway-relay") {
         const tools = [REALTIME_VOICE_AGENT_CONSULT_TOOL, REALTIME_VOICE_AGENT_CONTROL_TOOL];
-        if (
-          resolution.provider.id === "openai" &&
-          transport === "webrtc" &&
-          typedParams.capabilities?.includes("camera-frame")
-        ) {
+        if (wantsCameraFrames) {
           tools.push(REALTIME_VOICE_DESCRIBE_VIEW_TOOL);
         }
         const session = await resolution.provider.createBrowserSession({
@@ -146,6 +156,7 @@ export const talkClientHandlers: GatewayRequestHandlers = {
           ...launchOptions,
         });
         if (
+          (session.transport === "webrtc" || session.transport === "provider-websocket") &&
           !isUnsupportedBrowserWebRtcSession(session) &&
           (!transport || session.transport === transport)
         ) {

src/gateway/server-methods/talk.test.ts

@@ -2848,7 +2848,7 @@ describe("talk.client.create handler", () => {
     expectRespondOk(respond, { provider: "openai", transport: "webrtc" });
   });
 
-  it("adds describe_view only to an OpenAI WebRTC client with camera frames", async () => {
+  it("adds describe_view to camera clients whose provider supports video frames", async () => {
     const createBrowserSession = vi.fn(async (_input: unknown) => ({
       provider: "openai",
       transport: "webrtc" as const,
@@ -2857,6 +2857,7 @@ describe("talk.client.create handler", () => {
     const provider = {
       id: "openai",
       label: "OpenAI Realtime",
+      capabilities: { supportsVideoFrames: true },
       isConfigured: () => true,
       createBrowserSession,
       createBridge: vi.fn(),
@@ -2924,9 +2925,34 @@ describe("talk.client.create handler", () => {
       respond: respond as never,
       context: { getRuntimeConfig: () => ({}) as OpenClawConfig } as never,
     });
-    expect((mockCallArg(createBrowserSession) as Record<string, unknown>).tools).not.toContainEqual(
+    expect((mockCallArg(createBrowserSession) as Record<string, unknown>).tools).toContainEqual(
       expect.objectContaining({ name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME }),
     );
+
+    provider.capabilities.supportsVideoFrames = false;
+    createBrowserSession.mockClear();
+    respond.mockClear();
+    await expectDefined(
+      talkHandlers["talk.client.create"],
+      'talkHandlers["talk.client.create"] test invariant',
+    )({
+      req: { type: "req", id: "3", method: "talk.client.create" },
+      params: {
+        sessionKey: "main",
+        transport: "webrtc",
+        capabilities: ["camera-frame"],
+      },
+      client: { connId: "conn-1" } as never,
+      isWebchatConnect: () => false,
+      respond: respond as never,
+      context: { getRuntimeConfig: () => ({}) as OpenClawConfig } as never,
+    });
+    expect(createBrowserSession).not.toHaveBeenCalled();
+    expect(respond).toHaveBeenCalledWith(
+      false,
+      undefined,
+      expect.objectContaining({ message: expect.stringContaining("does not support") }),
+    );
   });
 
   it("uses agents.defaults.voiceModel as the realtime default model", async () => {
@@ -3311,6 +3337,78 @@ describe("talk.client.create handler", () => {
       message: "talk.client.create is client-owned; use talk.session.create for gateway-relay",
     });
     expect(mocks.resolveConfiguredRealtimeVoiceProvider).not.toHaveBeenCalled();
+
+    respond.mockClear();
+    await expectDefined(
+      talkHandlers["talk.client.create"],
+      'talkHandlers["talk.client.create"] test invariant',
+    )({
+      req: { type: "req", id: "2", method: "talk.client.create" },
+      params: {
+        sessionKey: "main",
+        mode: "realtime",
+        transport: "gateway-relay",
+        capabilities: ["camera-frame"],
+      },
+      client: { connId: "conn-1" } as never,
+      isWebchatConnect: () => false,
+      respond: respond as never,
+      context: { getRuntimeConfig: () => ({}) as OpenClawConfig } as never,
+    });
+
+    expectRespondError(respond, {
+      message: "gateway-relay does not support browser video frames",
+    });
+    expect(mocks.resolveConfiguredRealtimeVoiceProvider).not.toHaveBeenCalled();
+  });
+
+  it("rejects Gateway-owned sessions returned by a browser-session provider", async () => {
+    const createBrowserSession = vi.fn(async () => ({
+      provider: "custom",
+      transport: "gateway-relay" as const,
+      relaySessionId: "relay-1",
+      audio: {
+        inputEncoding: "pcm16" as const,
+        inputSampleRateHz: 24_000,
+        outputEncoding: "pcm16" as const,
+        outputSampleRateHz: 24_000,
+      },
+    }));
+    mocks.resolveConfiguredRealtimeVoiceProvider.mockReturnValue({
+      provider: {
+        id: "custom",
+        label: "Custom",
+        capabilities: {
+          transports: ["gateway-relay"],
+          inputAudioFormats: [],
+          outputAudioFormats: [],
+          supportsBrowserSession: true,
+          supportsVideoFrames: true,
+        },
+        isConfigured: () => true,
+        createBrowserSession,
+        createBridge: vi.fn(),
+      },
+      providerConfig: {},
+    });
+    const respond = vi.fn();
+
+    await expectDefined(
+      talkHandlers["talk.client.create"],
+      'talkHandlers["talk.client.create"] test invariant',
+    )({
+      req: { type: "req", id: "1", method: "talk.client.create" },
+      params: { sessionKey: "main", mode: "realtime", capabilities: ["camera-frame"] },
+      client: { connId: "conn-1" } as never,
+      isWebchatConnect: () => false,
+      respond: respond as never,
+      context: { getRuntimeConfig: () => ({}) as OpenClawConfig } as never,
+    });
+
+    expect(createBrowserSession).toHaveBeenCalledOnce();
+    expectRespondError(respond, {
+      message: 'Realtime provider "custom" does not support client-owned realtime sessions',
+    });
   });
 
   it("rejects realtime brains the client endpoint cannot run", async () => {

ui/src/e2e/browser-talk-start-stop.e2e.test.ts

@@ -574,9 +574,7 @@ describeControlUiE2e("Control UI browser Talk", () => {
       const request = await gateway.waitForRequest("talk.client.create");
       expect(request.params).toMatchObject({
         capabilities: ["camera-frame"],
-        provider: "openai",
         sessionKey: "main",
-        transport: "webrtc",
       });
       console.info("[video-talk-e2e] session=provider:openai,transport:webrtc");
       const preview = page.locator('video[aria-label="Camera preview"]');
@@ -672,15 +670,164 @@ describeControlUiE2e("Control UI browser Talk", () => {
     }
   });
 
+  it("starts Gemini Live Video Talk, streams a fake camera directly, and handles describe_view", async () => {
+    const context = await browser.newContext({ permissions: ["camera", "microphone"] });
+    const page = await context.newPage();
+    const gateway = await installMockGateway(page, {
+      methodResponses: {
+        "talk.client.create": {
+          provider: "google",
+          transport: "provider-websocket",
+          protocol: "google-live-bidi",
+          // Fake harness token, assembled so secret scanners do not flag it.
+          clientSecret: ["auth_tokens", "browser-video-e2e"].join("/"),
+          websocketUrl:
+            "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained",
+          audio: {
+            inputEncoding: "pcm16",
+            inputSampleRateHz: 16_000,
+            outputEncoding: "pcm16",
+            outputSampleRateHz: 24_000,
+          },
+        },
+      },
+    });
+    const googleLiveMessages: unknown[] = [];
+    let describeViewSent = false;
+    await page.routeWebSocket("wss://generativelanguage.googleapis.com/**", (ws) => {
+      ws.onMessage((message) => {
+        const parsed = JSON.parse(typeof message === "string" ? message : message.toString()) as {
+          setup?: unknown;
+          realtimeInput?: { video?: unknown };
+        };
+        googleLiveMessages.push(parsed);
+        if (parsed.setup) {
+          ws.send(JSON.stringify({ setupComplete: {} }));
+          return;
+        }
+        if (parsed.realtimeInput?.video && !describeViewSent) {
+          describeViewSent = true;
+          ws.send(
+            JSON.stringify({
+              toolCall: {
+                functionCalls: [{ id: "call-camera", name: "describe_view", args: {} }],
+              },
+            }),
+          );
+        }
+      });
+    });
+    await page.addInitScript(() => {
+      const getUserMedia = navigator.mediaDevices.getUserMedia.bind(navigator.mediaDevices);
+      Object.defineProperty(navigator.mediaDevices, "getUserMedia", {
+        configurable: true,
+        value: async (constraints: MediaStreamConstraints) => {
+          const stream = await getUserMedia(constraints);
+          (
+            window as Window & {
+              openclawGeminiVideoTalkTracks?: MediaStreamTrack[];
+            }
+          ).openclawGeminiVideoTalkTracks = [
+            ...((window as Window & { openclawGeminiVideoTalkTracks?: MediaStreamTrack[] })
+              .openclawGeminiVideoTalkTracks ?? []),
+            ...stream.getTracks(),
+          ];
+          return stream;
+        },
+      });
+    });
+
+    try {
+      await page.setViewportSize({ width: 1366, height: 900 });
+      await page.goto(`${server.baseUrl}chat`);
+      await page.getByRole("button", { name: "Start video talk" }).click();
+      const request = await gateway.waitForRequest("talk.client.create");
+      expect(request.params).toMatchObject({
+        capabilities: ["camera-frame"],
+        sessionKey: "main",
+      });
+      const preview = page.locator('video[aria-label="Camera preview"]');
+      await expect.poll(() => preview.isVisible()).toBe(true);
+      await expect
+        .poll(() => preview.evaluate((video) => (video as HTMLVideoElement).videoWidth))
+        .toBeGreaterThan(0);
+      await expect
+        .poll(() =>
+          googleLiveMessages.some(
+            (message) =>
+              typeof message === "object" &&
+              message !== null &&
+              "realtimeInput" in message &&
+              JSON.stringify(message).includes('"video"'),
+          ),
+        )
+        .toBe(true);
+      await expect
+        .poll(() =>
+          googleLiveMessages.some(
+            (message) =>
+              typeof message === "object" &&
+              message !== null &&
+              "toolResponse" in message &&
+              JSON.stringify(message).includes('"cameraStreamActive":true'),
+          ),
+        )
+        .toBe(true);
+      const videoMessage = googleLiveMessages.find(
+        (message) =>
+          typeof message === "object" &&
+          message !== null &&
+          "realtimeInput" in message &&
+          JSON.stringify(message).includes('"video"'),
+      );
+      expect(new TextEncoder().encode(JSON.stringify(videoMessage)).length).toBeLessThanOrEqual(
+        512 * 1024,
+      );
+      const talkRequests = (await gateway.getRequests()).filter((entry) =>
+        entry.method.startsWith("talk."),
+      );
+      expect(talkRequests.map((entry) => entry.method)).toEqual(["talk.client.create"]);
+      await captureVideoTalkProof(page, "05-gemini-live-camera-preview.png");
+      console.info(
+        "[video-talk-e2e] gemini=realtimeInput.video+functionResponse,gateway_frame_requests:0",
+      );
+
+      await page.getByRole("button", { name: "Stop voice input" }).click();
+      await expect.poll(() => preview.count()).toBe(0);
+      const trackStates = await page.evaluate(() =>
+        (
+          window as Window & {
+            openclawGeminiVideoTalkTracks?: MediaStreamTrack[];
+          }
+        ).openclawGeminiVideoTalkTracks?.map((track) => track.readyState),
+      );
+      expect(trackStates).toHaveLength(2);
+      expect(trackStates?.every((state) => state === "ended")).toBe(true);
+      console.info("[video-talk-e2e] gemini_stop=preview-removed,tracks:ended+ended");
+    } finally {
+      await context.close();
+    }
+  });
+
   it("shows actionable guidance when Video Talk camera permission is blocked", async () => {
     const context = await browser.newContext();
     const page = await context.newPage();
     const gateway = await installMockGateway(page, {
       methodResponses: {
         "talk.client.create": {
-          provider: "openai",
-          transport: "webrtc",
-          clientSecret: "test-client-secret",
+          provider: "google",
+          transport: "provider-websocket",
+          protocol: "google-live-bidi",
+          // Fake harness token, assembled so secret scanners do not flag it.
+          clientSecret: ["auth_tokens", "browser-video-denied"].join("/"),
+          websocketUrl:
+            "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained",
+          audio: {
+            inputEncoding: "pcm16",
+            inputSampleRateHz: 16_000,
+            outputEncoding: "pcm16",
+            outputSampleRateHz: 24_000,
+          },
         },
       },
     });

ui/src/pages/chat/chat-realtime.test.ts

@@ -67,7 +67,7 @@ describe("chat realtime actions", () => {
     expect(startSpy).toHaveBeenCalledOnce();
   });
 
-  it("launches video talk through WebRTC and owns the preview stream", async () => {
+  it("launches video talk through the configured provider and owns the preview stream", async () => {
     const state = createState();
 
     await state.toggleRealtimeTalk({ video: true });
@@ -75,8 +75,8 @@ describe("chat realtime actions", () => {
     const stream = {} as MediaStream;
     session.callbacks.onVideoStream?.(stream);
 
-    expect(session.options.provider).toBe("openai");
-    expect(session.options.transport).toBe("webrtc");
+    expect(session.options.provider).toBeUndefined();
+    expect(session.options.transport).toBeUndefined();
     expect(session.options.capabilities).toEqual(["camera-frame"]);
     expect(session.localOptions.videoEnabled).toBe(true);
     expect(state.realtimeTalkVideoStream).toBe(stream);

ui/src/pages/chat/chat-realtime.ts

@@ -133,9 +133,7 @@ export function attachChatRealtimeActions(state: ChatRealtimeState) {
           state.requestUpdate();
         },
       },
-      options.video
-        ? { provider: "openai", transport: "webrtc", capabilities: ["camera-frame"] }
-        : {},
+      options.video ? { capabilities: ["camera-frame"] } : {},
       { inputDeviceId, videoEnabled: options.video },
     );
     state.realtimeTalkSession = session;

ui/src/pages/chat/realtime-talk-google-live-video.test.ts

@@ -0,0 +1,280 @@
+// @vitest-environment jsdom
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import { REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME } from "../../../../src/talk/describe-view-tool.js";
+import { GoogleLiveRealtimeTalkTransport } from "./realtime-talk-google-live.ts";
+import type { RealtimeTalkCallbacks } from "./realtime-talk-shared.ts";
+
+class FakeGoogleLiveWebSocket extends EventTarget {
+  static OPEN = 1;
+  static instance: FakeGoogleLiveWebSocket | undefined;
+
+  readyState = FakeGoogleLiveWebSocket.OPEN;
+  readonly sent: unknown[] = [];
+  binaryType: BinaryType = "blob";
+
+  constructor(readonly url: string) {
+    super();
+    FakeGoogleLiveWebSocket.instance = this;
+  }
+
+  send(payload: string): void {
+    this.sent.push(JSON.parse(payload));
+  }
+
+  close(): void {
+    this.readyState = 3;
+  }
+
+  emitOpen(): void {
+    this.dispatchEvent(new Event("open"));
+  }
+
+  emitMessage(message: unknown): void {
+    this.dispatchEvent(new MessageEvent("message", { data: JSON.stringify(message) }));
+  }
+}
+
+class FakeAudioContext {
+  readonly currentTime = 0;
+  readonly destination = {};
+  readonly sampleRate: number;
+
+  constructor(options?: { sampleRate?: number }) {
+    this.sampleRate = options?.sampleRate ?? 24_000;
+  }
+
+  createMediaStreamSource() {
+    return { connect() {}, disconnect() {} };
+  }
+
+  createScriptProcessor() {
+    return { connect() {}, disconnect() {}, onaudioprocess: null };
+  }
+
+  createGain() {
+    return { connect() {}, disconnect() {}, gain: { value: 1 } };
+  }
+
+  async close(): Promise<void> {}
+}
+
+function createTransport(callbacks: RealtimeTalkCallbacks) {
+  return new GoogleLiveRealtimeTalkTransport(
+    {
+      provider: "google",
+      transport: "provider-websocket",
+      protocol: "google-live-bidi",
+      // Fake harness token, assembled so secret scanners do not flag it.
+      clientSecret: ["auth_tokens", "browser-video-test"].join("/"),
+      websocketUrl:
+        "wss://generativelanguage.googleapis.com/ws/google.ai.generativelanguage.v1alpha.GenerativeService.BidiGenerateContentConstrained",
+      audio: {
+        inputEncoding: "pcm16",
+        inputSampleRateHz: 16_000,
+        outputEncoding: "pcm16",
+        outputSampleRateHz: 24_000,
+      },
+    },
+    {
+      callbacks,
+      client: { request: vi.fn(), addEventListener: vi.fn() } as never,
+      sessionKey: "main",
+      videoEnabled: true,
+    },
+  );
+}
+
+describe("Google Live Video Talk", () => {
+  beforeEach(() => {
+    vi.useFakeTimers();
+    FakeGoogleLiveWebSocket.instance = undefined;
+    vi.stubGlobal("WebSocket", FakeGoogleLiveWebSocket);
+    vi.stubGlobal("AudioContext", FakeAudioContext);
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+    vi.unstubAllGlobals();
+    vi.restoreAllMocks();
+  });
+
+  it("streams bounded camera frames directly and answers describe_view calls", async () => {
+    const audioStop = vi.fn();
+    const videoStop = vi.fn();
+    const audioTrack = { stop: audioStop } as unknown as MediaStreamTrack;
+    const videoTrack = {
+      stop: videoStop,
+      readyState: "live",
+      enabled: true,
+      muted: false,
+    } as unknown as MediaStreamTrack;
+    const audio = {
+      getAudioTracks: () => [audioTrack],
+      getTracks: () => [audioTrack],
+    } as unknown as MediaStream;
+    const camera = {
+      getVideoTracks: () => [videoTrack],
+      getTracks: () => [videoTrack],
+    } as unknown as MediaStream;
+    class TestMediaStream {
+      constructor(readonly tracks: MediaStreamTrack[]) {}
+      getAudioTracks() {
+        return [audioTrack];
+      }
+      getVideoTracks() {
+        return [videoTrack];
+      }
+      getTracks() {
+        return this.tracks;
+      }
+    }
+    const getUserMedia = vi.fn().mockResolvedValueOnce(audio).mockResolvedValueOnce(camera);
+    vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } });
+    vi.stubGlobal("MediaStream", TestMediaStream);
+
+    const originalCreateElement = document.createElement.bind(document);
+    vi.spyOn(document, "createElement").mockImplementation((tagName: string) => {
+      const element = originalCreateElement(tagName);
+      if (element instanceof HTMLVideoElement) {
+        Object.defineProperties(element, {
+          readyState: { configurable: true, value: HTMLMediaElement.HAVE_CURRENT_DATA },
+          videoWidth: { configurable: true, value: 1280 },
+          videoHeight: { configurable: true, value: 720 },
+        });
+        vi.spyOn(element, "play").mockResolvedValue(undefined);
+      }
+      return element;
+    });
+    vi.spyOn(HTMLCanvasElement.prototype, "getContext").mockReturnValue({
+      drawImage: vi.fn(),
+    } as never);
+    vi.spyOn(HTMLCanvasElement.prototype, "toDataURL")
+      .mockReturnValueOnce(`data:image/jpeg;base64,${"x".repeat(600 * 1024)}`)
+      .mockReturnValue("data:image/jpeg;base64,gemini-camera-frame");
+    const onStatus = vi.fn();
+    const onVideoStream = vi.fn();
+    const transport = createTransport({ onStatus, onVideoStream });
+
+    await transport.start();
+    const ws = FakeGoogleLiveWebSocket.instance;
+    if (!ws) {
+      throw new Error("missing Google Live WebSocket");
+    }
+    ws.emitOpen();
+    ws.emitMessage({ setupComplete: {} });
+    await vi.advanceTimersByTimeAsync(0);
+
+    expect(ws.sent).toContainEqual({
+      realtimeInput: {
+        video: { data: "gemini-camera-frame", mimeType: "image/jpeg" },
+      },
+    });
+    for (const message of ws.sent) {
+      expect(new TextEncoder().encode(JSON.stringify(message)).length).toBeLessThanOrEqual(
+        512 * 1024,
+      );
+    }
+    ws.emitMessage({
+      toolCall: {
+        functionCalls: [
+          { id: "call-camera", name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME, args: {} },
+        ],
+      },
+    });
+    await Promise.resolve();
+    expect(ws.sent).toContainEqual({
+      toolResponse: {
+        functionResponses: [
+          {
+            id: "call-camera",
+            name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME,
+            response: { ok: true, cameraStreamActive: true },
+          },
+        ],
+      },
+    });
+    expect(getUserMedia).toHaveBeenNthCalledWith(1, {
+      audio: {
+        autoGainControl: true,
+        echoCancellation: true,
+        noiseSuppression: true,
+      },
+    });
+    expect(getUserMedia).toHaveBeenNthCalledWith(2, { video: true });
+    expect(onVideoStream).toHaveBeenCalledWith(expect.any(TestMediaStream));
+    expect(onStatus).toHaveBeenCalledWith("listening");
+
+    const countVideoMessages = () =>
+      ws.sent.filter((message) => JSON.stringify(message).includes('"video"')).length;
+    expect(countVideoMessages()).toBe(1);
+    await vi.advanceTimersByTimeAsync(999);
+    expect(countVideoMessages()).toBe(1);
+    await vi.advanceTimersByTimeAsync(1);
+    expect(countVideoMessages()).toBe(2);
+
+    (videoTrack as { readyState: MediaStreamTrackState }).readyState = "ended";
+    await vi.advanceTimersByTimeAsync(1_000);
+    expect(countVideoMessages()).toBe(2);
+    ws.emitMessage({
+      toolCall: {
+        functionCalls: [
+          { id: "call-ended-camera", name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME, args: {} },
+        ],
+      },
+    });
+    await Promise.resolve();
+    expect(ws.sent).toContainEqual({
+      toolResponse: {
+        functionResponses: [
+          {
+            id: "call-ended-camera",
+            name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME,
+            response: {
+              ok: false,
+              cameraStreamActive: false,
+              error: "Camera stream is unavailable",
+            },
+          },
+        ],
+      },
+    });
+
+    const sentBeforeStop = ws.sent.length;
+    transport.stop();
+    await vi.advanceTimersByTimeAsync(2_000);
+    expect(ws.sent).toHaveLength(sentBeforeStop);
+    expect(onVideoStream).toHaveBeenLastCalledWith(null);
+    expect(audioStop).toHaveBeenCalledOnce();
+    expect(videoStop).toHaveBeenCalledOnce();
+  });
+
+  it("releases acquired media when stopped during the camera prompt", async () => {
+    const audioStop = vi.fn();
+    const videoStop = vi.fn();
+    const audio = {
+      getAudioTracks: () => [{} as MediaStreamTrack],
+      getTracks: () => [{ stop: audioStop }],
+    } as unknown as MediaStream;
+    const camera = {
+      getVideoTracks: () => [{} as MediaStreamTrack],
+      getTracks: () => [{ stop: videoStop }],
+    } as unknown as MediaStream;
+    let resolveCamera: (stream: MediaStream) => void = () => undefined;
+    const cameraPending = new Promise<MediaStream>((resolve) => {
+      resolveCamera = resolve;
+    });
+    const getUserMedia = vi.fn().mockResolvedValueOnce(audio).mockReturnValueOnce(cameraPending);
+    vi.stubGlobal("navigator", { mediaDevices: { getUserMedia } });
+    const transport = createTransport({});
+
+    const start = transport.start();
+    await Promise.resolve();
+    transport.stop();
+    resolveCamera(camera);
+    await start;
+
+    expect(audioStop).toHaveBeenCalledOnce();
+    expect(videoStop).toHaveBeenCalledOnce();
+    expect(FakeGoogleLiveWebSocket.instance).toBeUndefined();
+  });
+});

ui/src/pages/chat/realtime-talk-google-live.ts

@@ -1,4 +1,5 @@
 // Control UI chat module implements realtime talk google live behavior.
+import { REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME } from "../../../../src/talk/describe-view-tool.js";
 import {
   base64ToBytes,
   bytesToBase64,
@@ -20,6 +21,10 @@ import {
   type RealtimeTalkTransport,
   type RealtimeTalkTransportContext,
 } from "./realtime-talk-shared.ts";
+import {
+  captureRealtimeTalkVideoFrame,
+  type RealtimeTalkVideoFrame,
+} from "./realtime-talk-video.ts";
 
 type GoogleLiveMessage = {
   setupComplete?: unknown;
@@ -53,6 +58,16 @@ type PendingFunctionCall = {
 const GOOGLE_LIVE_WEBSOCKET_HOST = "generativelanguage.googleapis.com";
 const GOOGLE_LIVE_WEBSOCKET_PATH =
   /^\/ws\/google\.ai\.generativelanguage\.v[0-9a-z]+\.GenerativeService\.BidiGenerateContent(?:Constrained)?$/;
+const GOOGLE_LIVE_VIDEO_FRAME_INTERVAL_MS = 1_000;
+const GOOGLE_LIVE_VIDEO_MESSAGE_MAX_BYTES = 512 * 1024;
+
+function googleLiveVideoMessage(frame: RealtimeTalkVideoFrame): unknown {
+  return {
+    realtimeInput: {
+      video: frame,
+    },
+  };
+}
 
 // Browser sessions can still pin a 2.5 model, whose text and tool-response wire
 // contract differs from the 3.1 default carried in new session metadata.
@@ -91,11 +106,16 @@ function buildGoogleLiveUrl(session: RealtimeTalkJsonPcmWebSocketSessionResult):
 export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport {
   private ws: WebSocket | null = null;
   private media: MediaStream | null = null;
+  private captureVideo: HTMLVideoElement | null = null;
   private inputContext: AudioContext | null = null;
   private outputContext: AudioContext | null = null;
   private inputMeter: RealtimeTalkMediaStreamMeter | null = null;
   private readonly inputPump = new RealtimeTalkPcmInputPump();
   private closed = false;
+  private mediaSetupController: AbortController | null = null;
+  private videoFramesActive = false;
+  private hasSentVideoFrame = false;
+  private videoFrameTimer: ReturnType<typeof globalThis.setTimeout> | null = null;
   private pendingCalls = new Map<string, PendingFunctionCall>();
   private readonly consultAbortControllers = new Set<AbortController>();
   private readonly outputQueue = new RealtimeTalkPcmOutputQueue();
@@ -117,20 +137,40 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport {
     }
     const wsUrl = buildGoogleLiveUrl(this.session);
     this.closed = false;
+    this.mediaSetupController?.abort();
+    const mediaSetupController = new AbortController();
+    this.mediaSetupController = mediaSetupController;
     let media: MediaStream;
     try {
-      media = await openRealtimeTalkInput(this.ctx.inputDeviceId);
+      media = await openRealtimeTalkInput(this.ctx.inputDeviceId, {
+        video: this.ctx.videoEnabled,
+        signal: mediaSetupController.signal,
+      });
     } catch (error) {
       if (this.closed) {
         return;
       }
       throw error;
+    } finally {
+      if (this.mediaSetupController === mediaSetupController) {
+        this.mediaSetupController = null;
+      }
     }
     if (this.closed) {
       media.getTracks().forEach((track) => track.stop());
       return;
     }
     this.media = media;
+    if (this.ctx.videoEnabled) {
+      const captureVideo = document.createElement("video");
+      captureVideo.autoplay = true;
+      captureVideo.muted = true;
+      captureVideo.playsInline = true;
+      captureVideo.srcObject = media;
+      this.captureVideo = captureVideo;
+      this.ctx.callbacks.onVideoStream?.(media);
+      void captureVideo.play().catch(() => undefined);
+    }
     this.inputContext = new AudioContext({ sampleRate: this.session.audio.inputSampleRateHz });
     this.outputContext = new AudioContext({ sampleRate: this.session.audio.outputSampleRateHz });
     if (this.ctx.callbacks.onInputLevel) {
@@ -166,6 +206,9 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport {
       this.emitTalkEvent({ type: "session.closed", final: true });
     }
     this.closed = true;
+    this.mediaSetupController?.abort();
+    this.mediaSetupController = null;
+    this.stopVideoFrames();
     for (const controller of this.consultAbortControllers) {
       controller.abort();
     }
@@ -176,6 +219,11 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport {
     this.inputMeter = null;
     this.media?.getTracks().forEach((track) => track.stop());
     this.media = null;
+    if (this.captureVideo) {
+      this.captureVideo.srcObject = null;
+      this.captureVideo = null;
+    }
+    this.ctx.callbacks.onVideoStream?.(null);
     this.stopOutput();
     void this.inputContext?.close();
     this.inputContext = null;
@@ -229,6 +277,7 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport {
     if (message.setupComplete) {
       this.ctx.callbacks.onStatus?.("listening");
       this.emitTalkEvent({ type: "session.ready" });
+      this.startVideoFrames();
     }
     const content = message.serverContent;
     if (content?.interrupted) {
@@ -344,6 +393,24 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport {
       });
       return;
     }
+    if (name === REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME) {
+      const active = this.videoFramesActive && this.hasSentVideoFrame && this.isCameraTrackUsable();
+      this.submitToolResult(callId, {
+        ok: active,
+        cameraStreamActive: active,
+        ...(!active ? { error: "Camera stream is unavailable" } : {}),
+      });
+      this.emitTalkEvent({
+        type: active ? "tool.result" : "tool.error",
+        callId,
+        final: true,
+        payload: {
+          name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME,
+          cameraStreamActive: active,
+        },
+      });
+      return;
+    }
     if (name !== REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME) {
       return;
     }
@@ -420,6 +487,77 @@ export class GoogleLiveRealtimeTalkTransport implements RealtimeTalkTransport {
     this.ctx.callbacks.onStatus?.("error", message);
   }
 
+  private startVideoFrames(): void {
+    if (!this.ctx.videoEnabled || !this.captureVideo || this.videoFramesActive || this.closed) {
+      return;
+    }
+    this.videoFramesActive = true;
+    this.scheduleVideoFrame(0);
+  }
+
+  private scheduleVideoFrame(delayMs: number): void {
+    if (!this.videoFramesActive || this.closed) {
+      return;
+    }
+    this.videoFrameTimer = globalThis.setTimeout(() => {
+      this.videoFrameTimer = null;
+      void this.sendVideoFrame();
+    }, delayMs);
+  }
+
+  private async sendVideoFrame(): Promise<void> {
+    if (!this.hasLiveCameraTrack()) {
+      this.stopVideoFrames();
+      return;
+    }
+    if (!this.isCameraTrackUsable()) {
+      this.scheduleVideoFrame(GOOGLE_LIVE_VIDEO_FRAME_INTERVAL_MS);
+      return;
+    }
+    try {
+      const frame = await captureRealtimeTalkVideoFrame(
+        this.captureVideo,
+        GOOGLE_LIVE_VIDEO_MESSAGE_MAX_BYTES,
+        googleLiveVideoMessage,
+      );
+      if (!this.videoFramesActive || this.closed) {
+        return;
+      }
+      if (!this.send(googleLiveVideoMessage(frame))) {
+        throw new Error("Google Live socket is not open");
+      }
+      this.hasSentVideoFrame = true;
+    } catch (error) {
+      if (!this.closed) {
+        this.videoFramesActive = false;
+        this.reportToolResultSubmissionError(error);
+      }
+      return;
+    }
+    this.scheduleVideoFrame(GOOGLE_LIVE_VIDEO_FRAME_INTERVAL_MS);
+  }
+
+  private stopVideoFrames(): void {
+    this.videoFramesActive = false;
+    this.hasSentVideoFrame = false;
+    if (this.videoFrameTimer !== null) {
+      globalThis.clearTimeout(this.videoFrameTimer);
+      this.videoFrameTimer = null;
+    }
+  }
+
+  private hasLiveCameraTrack(): boolean {
+    return this.media?.getVideoTracks().some((track) => track.readyState === "live") === true;
+  }
+
+  private isCameraTrackUsable(): boolean {
+    return (
+      this.media
+        ?.getVideoTracks()
+        .some((track) => track.readyState === "live" && track.enabled && !track.muted) === true
+    );
+  }
+
   private sendControlSpeechMessage(message: string): void {
     this.stopOutput();
     if (!isGemini31LiveModel(this.session.model)) {

ui/src/pages/chat/realtime-talk-input.ts

@@ -134,7 +134,14 @@ export async function openRealtimeTalkInput(
     return audio;
   }
 
-  const stopAudio = () => audio.getTracks().forEach((track) => track.stop());
+  let audioStopped = false;
+  const stopAudio = () => {
+    if (audioStopped) {
+      return;
+    }
+    audioStopped = true;
+    audio.getTracks().forEach((track) => track.stop());
+  };
   options.signal?.addEventListener("abort", stopAudio, { once: true });
   let camera: MediaStream | undefined;
   try {

ui/src/pages/chat/realtime-talk-video.ts

@@ -0,0 +1,81 @@
+// Control UI chat module owns bounded camera-frame capture for realtime Talk transports.
+
+export type RealtimeTalkVideoFrame = {
+  data: string;
+  mimeType: "image/jpeg";
+};
+
+const REALTIME_TALK_FRAME_MAX_ATTEMPTS = 8;
+
+export async function captureRealtimeTalkVideoFrame(
+  video: HTMLVideoElement | null,
+  maxMessageSize: number,
+  buildMessage: (frame: RealtimeTalkVideoFrame) => unknown,
+): Promise<RealtimeTalkVideoFrame> {
+  if (!video?.srcObject) {
+    throw new Error("Camera preview is unavailable");
+  }
+  if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {
+    await waitForRealtimeTalkVideoData(video);
+  }
+  if (
+    video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA ||
+    !video.videoWidth ||
+    !video.videoHeight
+  ) {
+    throw new Error("Camera frame has no image data");
+  }
+  let scale = Math.min(1, 1280 / video.videoWidth, 720 / video.videoHeight);
+  const canvas = document.createElement("canvas");
+  const context = canvas.getContext("2d");
+  if (!context) {
+    throw new Error("Camera frame capture is unavailable");
+  }
+  let quality = 0.8;
+  for (let attempt = 0; attempt < REALTIME_TALK_FRAME_MAX_ATTEMPTS; attempt += 1) {
+    canvas.width = Math.max(1, Math.round(video.videoWidth * scale));
+    canvas.height = Math.max(1, Math.round(video.videoHeight * scale));
+    context.drawImage(video, 0, 0, canvas.width, canvas.height);
+    const imageUrl = canvas.toDataURL("image/jpeg", quality);
+    const frame: RealtimeTalkVideoFrame = {
+      data: imageUrl.slice(imageUrl.indexOf(",") + 1),
+      mimeType: "image/jpeg",
+    };
+    const messageBytes = new TextEncoder().encode(JSON.stringify(buildMessage(frame))).length;
+    if (messageBytes <= maxMessageSize) {
+      return frame;
+    }
+    const reduction = Math.min(0.75, Math.sqrt(maxMessageSize / messageBytes) * 0.9);
+    scale *= reduction;
+    quality = Math.max(0.4, quality - 0.1);
+  }
+  throw new Error("Camera frame is too large for the Realtime connection");
+}
+
+function waitForRealtimeTalkVideoData(video: HTMLVideoElement): Promise<void> {
+  return new Promise((resolve, reject) => {
+    let timeout: ReturnType<typeof globalThis.setTimeout> | undefined;
+    const finish = (error?: Error) => {
+      if (timeout === undefined) {
+        return;
+      }
+      globalThis.clearTimeout(timeout);
+      timeout = undefined;
+      video.removeEventListener("loadeddata", onData);
+      if (error) {
+        reject(error);
+      } else {
+        resolve();
+      }
+    };
+    const onData = () => finish();
+    timeout = globalThis.setTimeout(
+      () => finish(new Error("Camera preview did not become ready")),
+      5_000,
+    );
+    video.addEventListener("loadeddata", onData);
+    if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
+      finish();
+    }
+  });
+}

ui/src/pages/chat/realtime-talk-webrtc.ts

@@ -14,6 +14,10 @@ import {
   type RealtimeTalkTransport,
   type RealtimeTalkTransportContext,
 } from "./realtime-talk-shared.ts";
+import {
+  captureRealtimeTalkVideoFrame,
+  type RealtimeTalkVideoFrame,
+} from "./realtime-talk-video.ts";
 
 type RealtimeServerEvent = {
   type?: string;
@@ -514,18 +518,12 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport {
       payload: { name: REALTIME_VOICE_DESCRIBE_VIEW_TOOL_NAME },
     });
     try {
-      const imageUrl = await captureRealtimeTalkVideoFrame(
+      const frame = await captureRealtimeTalkVideoFrame(
         this.captureVideo,
         realtimeTalkDataChannelMaxMessageSize(this.peer),
+        realtimeTalkImageEvent,
       );
-      this.send({
-        type: "conversation.item.create",
-        item: {
-          type: "message",
-          role: "user",
-          content: [{ type: "input_image", image_url: imageUrl }],
-        },
-      });
+      this.send(realtimeTalkImageEvent(frame));
       this.submitToolResult(callId, { ok: true, frameAttached: true });
       this.emitTalkEvent({
         type: "tool.result",
@@ -615,7 +613,6 @@ export class WebRtcSdpRealtimeTalkTransport implements RealtimeTalkTransport {
 }
 
 const REALTIME_TALK_DEFAULT_MAX_MESSAGE_SIZE = 64 * 1024;
-const REALTIME_TALK_FRAME_MAX_ATTEMPTS = 8;
 
 function realtimeTalkDataChannelMaxMessageSize(peer: RTCPeerConnection | null): number {
   const negotiated = peer?.sctp?.maxMessageSize;
@@ -624,83 +621,13 @@ function realtimeTalkDataChannelMaxMessageSize(peer: RTCPeerConnection | null):
     : REALTIME_TALK_DEFAULT_MAX_MESSAGE_SIZE;
 }
 
-function realtimeTalkImageEvent(imageUrl: string): unknown {
+function realtimeTalkImageEvent(frame: RealtimeTalkVideoFrame): unknown {
   return {
     type: "conversation.item.create",
     item: {
       type: "message",
       role: "user",
-      content: [{ type: "input_image", image_url: imageUrl }],
+      content: [{ type: "input_image", image_url: `data:${frame.mimeType};base64,${frame.data}` }],
     },
   };
 }
-
-async function captureRealtimeTalkVideoFrame(
-  video: HTMLVideoElement | null,
-  maxMessageSize: number,
-): Promise<string> {
-  if (!video?.srcObject) {
-    throw new Error("Camera preview is unavailable");
-  }
-  if (video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA) {
-    await waitForRealtimeTalkVideoData(video);
-  }
-  if (
-    video.readyState < HTMLMediaElement.HAVE_CURRENT_DATA ||
-    !video.videoWidth ||
-    !video.videoHeight
-  ) {
-    throw new Error("Camera frame has no image data");
-  }
-  let scale = Math.min(1, 1280 / video.videoWidth, 720 / video.videoHeight);
-  const canvas = document.createElement("canvas");
-  const context = canvas.getContext("2d");
-  if (!context) {
-    throw new Error("Camera frame capture is unavailable");
-  }
-  let quality = 0.8;
-  for (let attempt = 0; attempt < REALTIME_TALK_FRAME_MAX_ATTEMPTS; attempt += 1) {
-    canvas.width = Math.max(1, Math.round(video.videoWidth * scale));
-    canvas.height = Math.max(1, Math.round(video.videoHeight * scale));
-    context.drawImage(video, 0, 0, canvas.width, canvas.height);
-    const imageUrl = canvas.toDataURL("image/jpeg", quality);
-    const eventBytes = new TextEncoder().encode(
-      JSON.stringify(realtimeTalkImageEvent(imageUrl)),
-    ).length;
-    if (eventBytes <= maxMessageSize) {
-      return imageUrl;
-    }
-    const reduction = Math.min(0.75, Math.sqrt(maxMessageSize / eventBytes) * 0.9);
-    scale *= reduction;
-    quality = Math.max(0.4, quality - 0.1);
-  }
-  throw new Error("Camera frame is too large for the Realtime connection");
-}
-
-function waitForRealtimeTalkVideoData(video: HTMLVideoElement): Promise<void> {
-  return new Promise((resolve, reject) => {
-    let timeout: ReturnType<typeof globalThis.setTimeout> | undefined;
-    const finish = (error?: Error) => {
-      if (timeout === undefined) {
-        return;
-      }
-      globalThis.clearTimeout(timeout);
-      timeout = undefined;
-      video.removeEventListener("loadeddata", onData);
-      if (error) {
-        reject(error);
-      } else {
-        resolve();
-      }
-    };
-    const onData = () => finish();
-    timeout = globalThis.setTimeout(
-      () => finish(new Error("Camera preview did not become ready")),
-      5_000,
-    );
-    video.addEventListener("loadeddata", onData);
-    if (video.readyState >= HTMLMediaElement.HAVE_CURRENT_DATA) {
-      finish();
-    }
-  });
-}