feat: add live-validated Zoom meeting guest plugin (#111048)

* feat(zoom-meetings): add browser guest transport

* fix(zoom-meetings): type reusable refresh result

* test(meeting-bot): bind reusable refresh callback

* chore(zoom-meetings): use existing plugin label

* chore(zoom-meetings): defer changelog to release

* test(zoom-meetings): cover node setup through public host
This commit is contained in:
Peter Steinberger
2026-07-18 17:17:26 -07:00
committed by GitHub
parent 4e8f035912
commit b4187ced90
55 changed files with 8146 additions and 8 deletions
+6
View File
@@ -64,6 +64,12 @@
- "extensions/teams-meetings/**"
- "docs/plugins/teams-meetings.md"
- "docs/plugins/reference/teams-meetings.md"
"plugin: zoom":
- changed-files:
- any-glob-to-any-file:
- "extensions/zoom-meetings/**"
- "docs/plugins/zoom-meetings.md"
- "docs/plugins/reference/zoom-meetings.md"
"plugin: workboard":
- changed-files:
- any-glob-to-any-file:
+1
View File
@@ -1306,6 +1306,7 @@
"plugins/logbook",
"plugins/google-meet",
"plugins/teams-meetings",
"plugins/zoom-meetings",
"plugins/workboard",
"plugins/webhooks",
"plugins/admin-http-rpc",
+18
View File
@@ -7323,6 +7323,15 @@ Do not edit it by hand; run `pnpm docs:map:gen`.
- H2: Surface
- H2: Related docs
## plugins/reference/zoom-meetings.md
- Route: /plugins/reference/zoom-meetings
- Headings:
- H1: Zoom meetings plugin
- H2: Distribution
- H2: Surface
- H2: Related docs
## plugins/sdk-agent-harness.md
- Route: /plugins/sdk-agent-harness
@@ -7678,6 +7687,15 @@ Do not edit it by hand; run `pnpm docs:map:gen`.
- H2: Agent tool
- H2: Related
## plugins/zoom-meetings.md
- Route: /plugins/zoom-meetings
- Headings:
- H2: Setup
- H2: Modes
- H2: Guest join limits
- H2: Tool and gateway surface
## prose.md
- Route: /prose
+3 -1
View File
@@ -51,7 +51,7 @@ Each entry lists the package, distribution route, and description.
## Core npm package
68 plugins
69 plugins
- **[admin-http-rpc](/plugins/reference/admin-http-rpc)** (`@openclaw/admin-http-rpc`) - included in OpenClaw. OpenClaw admin HTTP RPC endpoint.
@@ -189,6 +189,8 @@ Each entry lists the package, distribution route, and description.
- **[xiaomi](/plugins/reference/xiaomi)** (`@openclaw/xiaomi-provider`) - included in OpenClaw. Adds Xiaomi, Xiaomi Token Plan model provider support to OpenClaw.
- **[zoom-meetings](/plugins/reference/zoom-meetings)** (`@openclaw/zoom-meetings`) - included in OpenClaw. Join Zoom meetings as a Chrome browser guest.
## Official external packages
72 plugins
+1 -1
View File
@@ -15,5 +15,5 @@ This page is generated from `extensions/*/package.json` and
pnpm plugins:inventory:gen
```
Use [Plugin inventory](/plugins/plugin-inventory) to browse all 141
Use [Plugin inventory](/plugins/plugin-inventory) to browse all 142
generated plugin reference pages by distribution, package, and description.
+23
View File
@@ -0,0 +1,23 @@
---
summary: "Join Zoom meetings as a Chrome browser guest."
read_when:
- You are installing, configuring, or auditing the zoom-meetings plugin
title: "Zoom meetings plugin"
---
# Zoom meetings plugin
Join Zoom meetings as a Chrome browser guest.
## Distribution
- Package: `@openclaw/zoom-meetings`
- Install route: included in OpenClaw
## Surface
contracts: `tools`
## Related docs
- [zoom-meetings](/plugins/zoom-meetings)
+67
View File
@@ -0,0 +1,67 @@
---
summary: "Zoom meetings plugin: join meetings as a Chrome browser guest"
read_when:
- You want an OpenClaw agent to join a Zoom meeting
- You are configuring Chrome, BlackHole, or SoX for Zoom meeting talk-back
title: "Zoom meetings plugin"
---
The `zoom-meetings` plugin joins Zoom meeting links as a guest through the Zoom Web App in the OpenClaw Chrome profile. It accepts meeting links under `zoom.us/j/...` and account subdomains such as `example.zoom.us/j/...`. It does not create meetings, dial in, use the Zoom Meeting SDK, or record meetings.
## Setup
Talk-back uses the same local audio prerequisites as the [Google Meet plugin](/plugins/google-meet): macOS, the `BlackHole 2ch` virtual audio device, and SoX.
```bash
brew install blackhole-2ch sox
sudo reboot
system_profiler SPAudioDataType | grep -i BlackHole
command -v sox
```
Enable the plugin, then check setup:
```json5
{
plugins: {
entries: {
"zoom-meetings": {
enabled: true,
config: {
defaultMode: "agent",
chrome: { guestName: "OpenClaw Agent" },
},
},
},
},
}
```
```bash
openclaw zoommeetings setup
openclaw zoommeetings join 'https://zoom.us/j/1234567890'
```
Use `chromeNode.node` to run Chrome, BlackHole, and SoX on a paired macOS node. The node must allow `zoommeetings.chrome` and `browser.proxy`.
## Modes
| Mode | Behavior |
| ------------ | --------------------------------------------------------------------------- |
| `agent` | Realtime transcription consults the configured OpenClaw agent; TTS replies. |
| `bidi` | A realtime voice model listens and replies directly. |
| `transcribe` | Observe-only join with live-caption transcript snapshots. |
Transcribe mode enables Zoom live captions after admission and captures the bounded caption display. The `transcript` action returns the caption buffer for the active OpenClaw meeting session.
## Guest join limits
The browser adapter chooses **Join from browser**, fills the guest name, turns the camera off, configures the microphone for the selected mode, and clicks **Join**. Zoom Web App runs under `app.zoom.us`; the plugin grants that origin microphone and speaker-selection permissions before navigation. In-call state uses Zoom's Leave control. Lobby, sign-in, passcode, CAPTCHA, and device-permission states return explicit manual-action reasons.
Zoom host and account policy can disable browser join, require authentication or email verification, show a CAPTCHA, or require host admission. Complete that step in the OpenClaw Chrome profile, then retry status or speech. The plugin does not bypass Zoom policy.
The Zoom Web App has been live-validated with an official Zoom test meeting for the app interstitial, iframe guest-name entry, prejoin microphone and camera controls, join, browser and macOS media permissions, in-call detection, live-caption enablement, and host-ended detection. Lobby and authentication states depend on host policy and retain text fallbacks when no stable DOM identifier is available.
## Tool and gateway surface
The `zoom_meetings` agent tool supports `join`, `leave`, `status`, `transcript`, and `speak`. Gateway methods use the `zoommeetings.*` prefix. The node command is `zoommeetings.chrome`.
+1 -1
View File
@@ -207,7 +207,7 @@ export class GoogleMeetRuntime {
refreshBrowserHealth: async (session, options) =>
await this.#refreshBrowserHealth(session, options),
refreshStatus: async (session) => await this.#refreshStatus(session),
refreshReusableSession: async (session) => {
refreshReusableSession: async (session, _request, _resolved) => {
if (session.transport === "twilio") {
await this.#refreshTwilioVoiceCallStatus(session);
}
+1 -1
View File
@@ -176,7 +176,7 @@ export class TeamsMeetingsRuntime {
await this.#refreshBrowserHealth(session, options),
refreshStatus: async (session) =>
await this.#sessions.refreshBrowserHealth(session, { force: true, readOnly: true }),
refreshReusableSession: async () => {},
refreshReusableSession: async (_session, _request, _resolved) => {},
ensureRealtimeBridge: async (session) => await this.#ensureRealtimeBridge(session),
captureTranscript: async (session, options) =>
await this.#captureTranscript(session, options),
+346
View File
@@ -0,0 +1,346 @@
import { ErrorCodes } from "openclaw/plugin-sdk/gateway-runtime";
import type { OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
import { createTestPluginApi } from "openclaw/plugin-sdk/plugin-test-api";
import { describe, expect, it, vi } from "vitest";
import plugin from "./index.js";
const MEETING_URL = "https://zoom.us/j/12345678901?pwd=owned";
type GatewayHandler = (options: {
client?: { internal?: { pluginRuntimeOwnerId?: string } };
params?: Record<string, unknown>;
respond(ok: boolean, payload?: unknown, error?: unknown): void;
}) => Promise<void>;
function authorizationHarness(options?: { browserError?: Error }) {
const methods = new Map<string, GatewayHandler>();
let tabOpen = false;
const gatewayRequest = vi.fn(async (_method: string, params: Record<string, unknown>) => {
if (options?.browserError) {
throw options.browserError;
}
if (params.path === "/tabs") {
return { tabs: tabOpen ? [{ targetId: "zoom-tab", title: "Zoom", url: MEETING_URL }] : [] };
}
if (params.path === "/tabs/open") {
tabOpen = true;
return { targetId: "zoom-tab", title: "Zoom", url: MEETING_URL };
}
if (params.path === "/tabs/focus") {
return { ok: true };
}
if (params.method === "DELETE" && params.path === "/tabs/zoom-tab") {
tabOpen = false;
return { ok: true };
}
if (params.path === "/act") {
const scriptValue = (params.body as { fn?: unknown } | undefined)?.fn;
const script = typeof scriptValue === "string" ? scriptValue : "";
return script.includes("leaveAction")
? { result: JSON.stringify({ departed: true, urlMatched: true }) }
: script.includes("expectedSessionId")
? {
result: JSON.stringify({
droppedLines: 0,
lines: [],
sessionMatched: true,
urlMatched: true,
}),
}
: {
result: JSON.stringify({
cameraOff: true,
inCall: true,
micMuted: true,
title: "Zoom",
url: MEETING_URL,
}),
};
}
throw new Error(`unexpected browser request ${String(params.path)}`);
});
const api = createTestPluginApi({
id: "zoom-meetings",
name: "Zoom meetings",
description: "test",
version: "0",
source: "test",
config: {},
pluginConfig: { defaultMode: "transcribe", chrome: { waitForInCallMs: 1 } },
runtime: {
gateway: { isAvailable: vi.fn(async () => true), request: gatewayRequest },
} as unknown as OpenClawPluginApi["runtime"],
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
registerGatewayMethod: (method: string, handler: unknown) =>
methods.set(method, handler as GatewayHandler),
});
plugin.register(api);
const invoke = async (
method: string,
params: Record<string, unknown>,
pluginRuntimeOwnerId?: string,
) => {
const handler = methods.get(method);
if (!handler) {
throw new Error(`missing handler ${method}`);
}
let response: { ok: boolean; payload?: unknown; error?: unknown } | undefined;
await handler({
params,
...(pluginRuntimeOwnerId ? { client: { internal: { pluginRuntimeOwnerId } } } : {}),
respond: (ok, payload, error) => {
response = { ok, payload, error };
},
});
return response;
};
const call = async (
method: string,
params: Record<string, unknown>,
pluginRuntimeOwnerId?: string,
) => {
const response = await invoke(method, params, pluginRuntimeOwnerId);
if (!response?.ok) {
throw new Error(`gateway call failed: ${JSON.stringify(response)}`);
}
return response.payload as Record<string, unknown>;
};
return { call, invoke };
}
describe("Zoom meetings plugin surface", () => {
it("registers the bounded gateway, tool, CLI, and node surfaces", () => {
const methods = new Map<string, unknown>();
const tools: Array<Record<string, unknown>> = [];
const cli: unknown[] = [];
const nodeCommands: unknown[] = [];
const policies: unknown[] = [];
const api = createTestPluginApi({
id: "zoom-meetings",
name: "Zoom meetings",
description: "test",
version: "0",
source: "test",
config: {},
pluginConfig: {},
runtime: {
gateway: { isAvailable: vi.fn(async () => false), request: vi.fn() },
} as unknown as OpenClawPluginApi["runtime"],
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
registerGatewayMethod: (method: string, handler: unknown) => methods.set(method, handler),
registerTool: (tool: unknown) => {
tools.push(
(typeof tool === "function"
? (tool as (context: Record<string, unknown>) => Record<string, unknown>)({})
: tool) as Record<string, unknown>,
);
},
registerCli: (_registrar: unknown, options: unknown) => cli.push(options),
registerNodeHostCommand: (command: unknown) => nodeCommands.push(command),
registerNodeInvokePolicy: (policy: unknown) => policies.push(policy),
});
plugin.register(api);
expect([...methods.keys()].toSorted()).toEqual(
[
"zoommeetings.join",
"zoommeetings.leave",
"zoommeetings.setup",
"zoommeetings.speak",
"zoommeetings.status",
"zoommeetings.testListen",
"zoommeetings.testSpeech",
"zoommeetings.transcript",
].toSorted(),
);
expect(tools.map((tool) => tool.name)).toEqual(["zoom_meetings"]);
expect(cli).toEqual([expect.objectContaining({ commands: ["zoommeetings"] })]);
expect(nodeCommands).toEqual([
expect.objectContaining({ command: "zoommeetings.chrome", cap: "zoom-meetings" }),
]);
expect(policies).toHaveLength(1);
});
it("does not expose the dangerous node surface when disabled", () => {
const nodeCommands: unknown[] = [];
const policies: unknown[] = [];
const api = createTestPluginApi({
id: "zoom-meetings",
name: "Zoom meetings",
description: "test",
version: "0",
source: "test",
config: {},
pluginConfig: { enabled: false },
runtime: {
gateway: { isAvailable: vi.fn(async () => false), request: vi.fn() },
} as unknown as OpenClawPluginApi["runtime"],
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
registerNodeHostCommand: (command: unknown) => nodeCommands.push(command),
registerNodeInvokePolicy: (policy: unknown) => policies.push(policy),
});
plugin.register(api);
expect(nodeCommands).toEqual([]);
expect(policies).toEqual([]);
});
it("routes main-agent tool calls through the ownership-attested runtime", async () => {
const gatewayRequest = vi.fn(async () => ({ found: true, sessions: [] }));
let tool:
| { execute: (id: string, params: unknown) => Promise<{ details: Record<string, unknown> }> }
| undefined;
const api = createTestPluginApi({
id: "zoom-meetings",
name: "Zoom meetings",
description: "test",
version: "0",
source: "test",
config: {},
pluginConfig: {},
runtime: {
gateway: { isAvailable: vi.fn(async () => true), request: gatewayRequest },
} as unknown as OpenClawPluginApi["runtime"],
logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn(), debug: vi.fn() },
registerTool: (registered: unknown) => {
tool = (
typeof registered === "function"
? (registered as (context: Record<string, unknown>) => typeof tool)({
agentId: "main",
sessionKey: "agent:main:main",
})
: registered
) as typeof tool;
},
});
plugin.register(api);
await tool?.execute("id", { action: "status" });
expect(gatewayRequest).toHaveBeenCalledWith(
"zoommeetings.status",
{
action: "status",
agentId: "main",
requesterSessionKey: "agent:main:main",
},
expect.objectContaining({ scopes: ["operator.admin"] }),
);
});
it("scopes trusted tool session operations to the invoking agent", async () => {
const { call } = authorizationHarness();
const joined = await call(
"zoommeetings.join",
{ agentId: "support", mode: "transcribe", url: MEETING_URL },
"zoom-meetings",
);
const sessionId = (joined.session as { id: string }).id;
expect(await call("zoommeetings.status", { agentId: "other" }, "zoom-meetings")).toMatchObject({
found: true,
sessions: [],
});
for (const method of ["status", "leave", "transcript", "speak"] as const) {
expect(
await call(
`zoommeetings.${method}`,
{ agentId: "other", message: "hello", sessionId },
"zoom-meetings",
),
).toMatchObject({ found: false });
}
expect(await call("zoommeetings.status", { agentId: "spoofed", sessionId })).toMatchObject({
found: true,
session: { agentId: "support", id: sessionId },
});
expect(
await call("zoommeetings.leave", { agentId: "support", sessionId }, "zoom-meetings"),
).toMatchObject({ found: true, session: { id: sessionId, state: "ended" } });
});
it.each([
["mode", "observe-only", "mode must be agent, bidi, or transcribe"],
["transport", "desktop", "transport must be chrome or chrome-node"],
])("rejects an explicit invalid %s", async (field, value, message) => {
const { invoke } = authorizationHarness();
const response = await invoke("zoommeetings.join", {
[field]: value,
url: MEETING_URL,
});
expect(response).toMatchObject({
error: { code: ErrorCodes.INVALID_REQUEST },
ok: false,
payload: { error: message },
});
});
it("rejects timeoutMs on normal join instead of silently ignoring it", async () => {
const { invoke } = authorizationHarness();
const response = await invoke("zoommeetings.join", {
mode: "transcribe",
timeoutMs: 1,
url: MEETING_URL,
});
expect(response).toMatchObject({
ok: false,
payload: { error: "timeoutMs is supported only by testSpeech or testListen" },
});
});
it("accepts timeoutMs on testListen and reports a bounded caption timeout", async () => {
const { invoke } = authorizationHarness();
const response = await invoke("zoommeetings.testListen", {
mode: "transcribe",
timeoutMs: 1,
url: MEETING_URL,
});
expect(response).toMatchObject({
ok: true,
payload: {
captioning: undefined,
createdSession: true,
listenTimedOut: true,
listenVerified: false,
},
});
});
it.each([
["zoommeetings.testSpeech", "transcribe", "test_speech requires mode: agent or bidi"],
["zoommeetings.testListen", "agent", "test_listen requires mode: transcribe"],
])(
"classifies invalid probe mode for %s as an invalid request",
async (method, mode, message) => {
const { invoke } = authorizationHarness();
const response = await invoke(method, { mode, timeoutMs: 1, url: MEETING_URL });
expect(response).toMatchObject({
error: { code: ErrorCodes.INVALID_REQUEST },
ok: false,
payload: { error: message },
});
},
);
it("classifies browser failures as unavailable, not invalid requests", async () => {
const { invoke } = authorizationHarness({ browserError: new Error("browser unavailable") });
const response = await invoke("zoommeetings.join", {
mode: "transcribe",
url: MEETING_URL,
});
expect(response).toMatchObject({
error: { code: ErrorCodes.UNAVAILABLE },
ok: false,
payload: { error: "browser unavailable" },
});
});
});
+456
View File
@@ -0,0 +1,456 @@
import {
readNonNegativeIntegerParam,
readPositiveIntegerParam,
} from "openclaw/plugin-sdk/channel-actions";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import {
callGatewayFromCli,
ErrorCodes,
errorShape,
type GatewayRequestHandlerOptions,
} from "openclaw/plugin-sdk/gateway-runtime";
import { createLazyRuntimeModule } from "openclaw/plugin-sdk/lazy-runtime";
import { definePluginEntry, type OpenClawPluginApi } from "openclaw/plugin-sdk/plugin-entry";
import { normalizeAgentId, parseAgentSessionKey } from "openclaw/plugin-sdk/routing";
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
import { jsonResult as json } from "openclaw/plugin-sdk/tool-results";
import { Type } from "typebox";
import {
resolveZoomMeetingsConfig,
resolveZoomMeetingsGatewayOperationTimeoutMs,
type ZoomMeetingsConfig,
type ZoomMeetingsMode,
type ZoomMeetingsTransport,
} from "./src/config.js";
import {
ZoomMeetingsInvalidRequestError,
zoomMeetingsInvalidRequest as invalidRequest,
} from "./src/errors.js";
import { handleZoomMeetingsNodeHostCommand } from "./src/node-host.js";
import { createZoomMeetingsNodeInvokePolicy } from "./src/node-invoke-policy.js";
import { ZoomMeetingsRuntime } from "./src/runtime.js";
import { ZOOM_MEETINGS_NODE_COMMAND } from "./src/transports/zoom-meetings-platform-constants.js";
import { normalizeZoomMeetingUrl } from "./src/transports/zoom-meetings-urls.js";
const loadZoomMeetingsCli = createLazyRuntimeModule(() => import("./src/cli.js"));
const zoomMeetingsConfigSchema = {
parse(value: unknown) {
return resolveZoomMeetingsConfig(value);
},
uiHints: {
defaultMode: {
label: "Default Mode",
help: "Agent consults OpenClaw, bidi uses direct realtime voice, and transcribe observes only.",
},
"chrome.browserProfile": { label: "Chrome Profile", advanced: true },
"chrome.guestName": { label: "Guest Name" },
"chrome.waitForInCallMs": { label: "Wait For In-Call (ms)", advanced: true },
"chrome.audioInputCommand": { label: "Audio Input Command", advanced: true },
"chrome.audioOutputCommand": { label: "Audio Output Command", advanced: true },
"chromeNode.node": {
label: "Chrome Node",
help: "Node id/name/IP that owns Chrome, BlackHole, and SoX.",
advanced: true,
},
"realtime.transcriptionProvider": { label: "Realtime Transcription Provider" },
"realtime.voiceProvider": { label: "Bidi Voice Provider" },
"realtime.model": { label: "Bidi Realtime Model", advanced: true },
"realtime.instructions": { label: "Realtime Instructions", advanced: true },
"realtime.introMessage": { label: "Realtime Intro Message" },
"realtime.agentId": { label: "Realtime Consult Agent", advanced: true },
"realtime.toolPolicy": { label: "Realtime Tool Policy", advanced: true },
},
};
const ZoomMeetingsToolSchema = Type.Object({
action: Type.String({ enum: ["join", "leave", "status", "transcript", "speak"] }),
url: Type.Optional(Type.String({ description: "Zoom meeting URL" })),
transport: Type.Optional(Type.String({ enum: ["chrome", "chrome-node"] })),
mode: Type.Optional(Type.String({ enum: ["agent", "bidi", "transcribe"] })),
sessionId: Type.Optional(Type.String({ description: "Zoom meeting session ID" })),
sinceIndex: Type.Optional(
Type.Integer({ minimum: 0, description: "Resume transcript from this index" }),
),
message: Type.Optional(Type.String({ description: "Instructions to speak" })),
});
function asRecord(value: unknown): Record<string, unknown> {
return value && typeof value === "object" && !Array.isArray(value)
? (value as Record<string, unknown>)
: {};
}
function normalizeTransport(value: unknown): ZoomMeetingsTransport | undefined {
if (value === undefined) {
return undefined;
}
if (value === "chrome" || value === "chrome-node") {
return value;
}
throw invalidRequest("transport must be chrome or chrome-node");
}
function normalizeMode(value: unknown): ZoomMeetingsMode | undefined {
if (value === undefined) {
return undefined;
}
if (value === "agent" || value === "bidi" || value === "transcribe") {
return value;
}
throw invalidRequest("mode must be agent, bidi, or transcribe");
}
function requireString(value: unknown, name: string): string {
const normalized = normalizeOptionalString(value);
if (!normalized) {
throw invalidRequest(`${name} required`);
}
return normalized;
}
function readSinceIndex(raw: Record<string, unknown>): number | undefined {
try {
return readNonNegativeIntegerParam(raw, "sinceIndex");
} catch (error) {
throw invalidRequest(formatErrorMessage(error));
}
}
function keepTrustedToolContext(
raw: Record<string, unknown>,
client: GatewayRequestHandlerOptions["client"],
): Record<string, unknown> {
const { agentId: rawAgentId, requesterSessionKey: rawRequesterSessionKey, ...rest } = raw;
if (client?.internal?.pluginRuntimeOwnerId !== "zoom-meetings") {
return rest;
}
const agentId = normalizeOptionalString(rawAgentId);
const requesterSessionKey = normalizeOptionalString(rawRequesterSessionKey);
return {
...rest,
...(agentId ? { agentId } : {}),
...(requesterSessionKey ? { requesterSessionKey } : {}),
};
}
function trustedToolAgentId(
raw: Record<string, unknown>,
client: GatewayRequestHandlerOptions["client"],
): string | undefined {
return normalizeOptionalString(keepTrustedToolContext(raw, client).agentId);
}
function joinRequest(raw: Record<string, unknown>, options?: { allowTimeout?: boolean }) {
if (!options?.allowTimeout && raw.timeoutMs !== undefined) {
throw invalidRequest("timeoutMs is supported only by testSpeech or testListen");
}
let url: string;
let timeoutMs: number | undefined;
try {
url = normalizeZoomMeetingUrl(requireString(raw.url, "url"));
timeoutMs = readPositiveIntegerParam(raw, "timeoutMs");
} catch (error) {
if (error instanceof ZoomMeetingsInvalidRequestError) {
throw error;
}
throw invalidRequest(formatErrorMessage(error));
}
return {
url,
transport: normalizeTransport(raw.transport),
mode: normalizeMode(raw.mode),
message: normalizeOptionalString(raw.message),
requesterSessionKey: normalizeOptionalString(raw.requesterSessionKey),
agentId: normalizeOptionalString(raw.agentId),
timeoutMs,
};
}
type ToolAction = "join" | "leave" | "status" | "transcript" | "speak";
function gatewayMethod(action: ToolAction): string {
return `zoommeetings.${action}`;
}
function readErrorDetails(error: unknown): unknown {
return error && typeof error === "object" && "details" in error
? (error as { details?: unknown }).details
: undefined;
}
async function callGatewayFromTool(params: {
action: ToolAction;
config: ZoomMeetingsConfig;
raw: Record<string, unknown>;
runtime?: OpenClawPluginApi["runtime"];
}) {
try {
if (params.runtime) {
return await params.runtime.gateway.request(gatewayMethod(params.action), params.raw, {
timeoutMs: resolveZoomMeetingsGatewayOperationTimeoutMs(params.config),
scopes: ["operator.admin"],
});
}
return await callGatewayFromCli(
gatewayMethod(params.action),
{
json: true,
timeout: String(resolveZoomMeetingsGatewayOperationTimeoutMs(params.config)),
},
params.raw,
{ progress: false, scopes: ["operator.admin"] },
);
} catch (error) {
const details = readErrorDetails(error);
if (details && typeof details === "object") {
return details;
}
throw error;
}
}
export default definePluginEntry({
id: "zoom-meetings",
name: "Zoom meetings",
description: "Join Zoom meetings as a Chrome browser guest",
configSchema: zoomMeetingsConfigSchema,
register(api: OpenClawPluginApi) {
const config = zoomMeetingsConfigSchema.parse(api.pluginConfig);
let runtime: ZoomMeetingsRuntime | undefined;
const ensureRuntime = async () => {
if (!config.enabled) {
throw new Error("Zoom meetings plugin disabled in plugin config");
}
runtime ??= new ZoomMeetingsRuntime({
config,
fullConfig: api.config,
runtime: api.runtime,
logger: api.logger,
});
return runtime;
};
const sendError = (
respond: GatewayRequestHandlerOptions["respond"],
error: unknown,
code: Parameters<typeof errorShape>[0] = ErrorCodes.UNAVAILABLE,
) => {
const payload = { error: formatErrorMessage(error) };
respond(false, payload, errorShape(code, payload.error, { details: payload }));
};
const sendRequestError = (respond: GatewayRequestHandlerOptions["respond"], error: unknown) =>
sendError(
respond,
error,
error instanceof ZoomMeetingsInvalidRequestError
? ErrorCodes.INVALID_REQUEST
: ErrorCodes.UNAVAILABLE,
);
api.registerGatewayMethod(
"zoommeetings.join",
async ({ params, client, respond }: GatewayRequestHandlerOptions) => {
try {
const raw = keepTrustedToolContext(asRecord(params), client);
respond(true, await (await ensureRuntime()).join(joinRequest(raw)));
} catch (error) {
sendRequestError(respond, error);
}
},
);
api.registerGatewayMethod(
"zoommeetings.leave",
async ({ params, client, respond }: GatewayRequestHandlerOptions) => {
try {
const raw = asRecord(params);
const agentId = trustedToolAgentId(raw, client);
const sessionId = requireString(raw.sessionId, "sessionId");
const rt = await ensureRuntime();
respond(
true,
agentId && !rt.ownsSession(agentId, sessionId)
? { found: false }
: await rt.leave(sessionId),
);
} catch (error) {
sendRequestError(respond, error);
}
},
);
api.registerGatewayMethod(
"zoommeetings.status",
async ({ params, client, respond }: GatewayRequestHandlerOptions) => {
try {
const raw = asRecord(params);
const agentId = trustedToolAgentId(raw, client);
const rt = await ensureRuntime();
respond(
true,
agentId
? await rt.statusForAgent(agentId, normalizeOptionalString(raw.sessionId))
: await rt.status(normalizeOptionalString(raw.sessionId)),
);
} catch (error) {
sendRequestError(respond, error);
}
},
);
api.registerGatewayMethod(
"zoommeetings.transcript",
async ({ params, client, respond }: GatewayRequestHandlerOptions) => {
try {
const raw = asRecord(params);
const sessionId = requireString(raw.sessionId, "sessionId");
const sinceIndex = readSinceIndex(raw);
const agentId = trustedToolAgentId(raw, client);
const rt = await ensureRuntime();
respond(
true,
agentId && !rt.ownsSession(agentId, sessionId)
? { found: false }
: await rt.transcript(sessionId, sinceIndex === undefined ? {} : { sinceIndex }),
);
} catch (error) {
sendRequestError(respond, error);
}
},
);
api.registerGatewayMethod(
"zoommeetings.speak",
async ({ params, client, respond }: GatewayRequestHandlerOptions) => {
try {
const raw = asRecord(params);
const sessionId = requireString(raw.sessionId, "sessionId");
const agentId = trustedToolAgentId(raw, client);
const rt = await ensureRuntime();
respond(
true,
agentId && !rt.ownsSession(agentId, sessionId)
? { found: false, spoken: false }
: await rt.speak(sessionId, normalizeOptionalString(raw.message)),
);
} catch (error) {
sendRequestError(respond, error);
}
},
);
api.registerGatewayMethod(
"zoommeetings.setup",
async ({ params, respond }: GatewayRequestHandlerOptions) => {
try {
respond(
true,
await (
await ensureRuntime()
).setupStatus({
mode: normalizeMode(params?.mode),
transport: normalizeTransport(params?.transport),
}),
);
} catch (error) {
sendRequestError(respond, error);
}
},
);
for (const [method, run] of [
[
"zoommeetings.testSpeech",
(rt: ZoomMeetingsRuntime, raw: Record<string, unknown>) =>
rt.testSpeech(joinRequest(raw, { allowTimeout: true })),
],
[
"zoommeetings.testListen",
(rt: ZoomMeetingsRuntime, raw: Record<string, unknown>) =>
rt.testListen(joinRequest(raw, { allowTimeout: true })),
],
] as const) {
api.registerGatewayMethod(
method,
async ({ params, client, respond }: GatewayRequestHandlerOptions) => {
try {
const raw = keepTrustedToolContext(asRecord(params), client);
respond(true, await run(await ensureRuntime(), raw));
} catch (error) {
sendRequestError(respond, error);
}
},
);
}
api.registerTool(
(toolContext) => ({
name: "zoom_meetings",
label: "Zoom meetings",
description:
"Join and manage Zoom meeting browser guests. Guest admission, tenant sign-in, and media permissions may require manual action in the OpenClaw Chrome profile.",
parameters: ZoomMeetingsToolSchema,
async execute(_toolCallId, params) {
const raw = asRecord(params);
const action = raw.action as ToolAction;
const requesterSessionKey = normalizeOptionalString(toolContext.sessionKey);
const contextAgentId =
toolContext.agentId ?? parseAgentSessionKey(requesterSessionKey)?.agentId;
const agentId = normalizeAgentId(contextAgentId);
try {
if (!(["join", "leave", "status", "transcript", "speak"] as const).includes(action)) {
throw new Error("unknown zoom_meetings action");
}
const useRuntime = await api.runtime.gateway.isAvailable();
if (!useRuntime) {
throw new Error("Zoom meeting tools require a Gateway-hosted agent run.");
}
return json(
await callGatewayFromTool({
action,
config,
raw: {
...raw,
...(requesterSessionKey ? { requesterSessionKey } : {}),
agentId,
},
runtime: api.runtime,
}),
);
} catch (error) {
return json({ error: formatErrorMessage(error) });
}
},
}),
{ name: "zoom_meetings" },
);
if (config.enabled) {
api.registerNodeHostCommand({
command: ZOOM_MEETINGS_NODE_COMMAND,
cap: "zoom-meetings",
dangerous: true,
handle: handleZoomMeetingsNodeHostCommand,
});
api.registerNodeInvokePolicy(createZoomMeetingsNodeInvokePolicy(config));
}
api.registerCli(
async ({ program }) => {
const cli = await loadZoomMeetingsCli();
cli.registerZoomMeetingsCli({ program, config });
},
{
commands: ["zoommeetings"],
descriptors: [
{
name: "zoommeetings",
description: "Join and manage Zoom meeting guests",
hasSubcommands: true,
},
],
},
);
},
});
@@ -0,0 +1,110 @@
{
"id": "zoom-meetings",
"name": "Zoom meetings",
"description": "Join Zoom meetings as a Chrome browser guest.",
"icon": "https://cdn.simpleicons.org/zoom",
"enabledByDefault": false,
"commandAliases": [{ "name": "zoommeetings" }],
"activation": {
"onStartup": true,
"onCommands": ["zoommeetings"],
"onCapabilities": ["tool"]
},
"contracts": {
"tools": ["zoom_meetings"]
},
"uiHints": {
"defaultMode": {
"label": "Default Mode",
"help": "Agent consults OpenClaw, bidi uses direct realtime voice, and transcribe observes only."
},
"chrome.browserProfile": { "label": "Chrome Profile", "advanced": true },
"chrome.guestName": { "label": "Guest Name" },
"chrome.waitForInCallMs": { "label": "Wait For In-Call (ms)", "advanced": true },
"chrome.audioInputCommand": { "label": "Audio Input Command", "advanced": true },
"chrome.audioOutputCommand": { "label": "Audio Output Command", "advanced": true },
"chromeNode.node": {
"label": "Chrome Node",
"help": "Node id/name/IP that owns Chrome, BlackHole, and SoX.",
"advanced": true
},
"realtime.transcriptionProvider": { "label": "Realtime Transcription Provider" },
"realtime.voiceProvider": { "label": "Bidi Voice Provider" },
"realtime.model": { "label": "Bidi Realtime Model", "advanced": true },
"realtime.instructions": { "label": "Realtime Instructions", "advanced": true },
"realtime.introMessage": { "label": "Realtime Intro Message" },
"realtime.agentId": { "label": "Realtime Consult Agent", "advanced": true },
"realtime.toolPolicy": { "label": "Realtime Tool Policy", "advanced": true }
},
"configSchema": {
"type": "object",
"additionalProperties": false,
"properties": {
"enabled": { "type": "boolean" },
"defaultMode": {
"type": "string",
"enum": ["agent", "bidi", "transcribe"],
"default": "agent"
},
"chrome": {
"type": "object",
"additionalProperties": false,
"properties": {
"audioBackend": {
"type": "string",
"enum": ["blackhole-2ch"],
"default": "blackhole-2ch"
},
"audioFormat": {
"type": "string",
"enum": ["pcm16-24khz", "g711-ulaw-8khz"],
"default": "pcm16-24khz"
},
"audioBufferBytes": { "type": "number", "minimum": 17 },
"launch": { "type": "boolean" },
"browserProfile": { "type": "string" },
"guestName": { "type": "string" },
"reuseExistingTab": { "type": "boolean" },
"autoJoin": { "type": "boolean" },
"joinTimeoutMs": { "type": "number", "minimum": 1 },
"waitForInCallMs": { "type": "number", "minimum": 1 },
"audioInputCommand": { "type": "array", "items": { "type": "string" } },
"audioOutputCommand": { "type": "array", "items": { "type": "string" } },
"bargeInInputCommand": { "type": "array", "items": { "type": "string" } },
"bargeInRmsThreshold": { "type": "number", "exclusiveMinimum": 0 },
"bargeInPeakThreshold": { "type": "number", "exclusiveMinimum": 0 },
"bargeInCooldownMs": { "type": "number", "minimum": 1 }
}
},
"chromeNode": {
"type": "object",
"additionalProperties": false,
"properties": {
"node": { "type": "string" }
}
},
"realtime": {
"type": "object",
"additionalProperties": false,
"properties": {
"strategy": { "type": "string", "enum": ["agent", "bidi"] },
"provider": { "type": "string" },
"transcriptionProvider": { "type": "string" },
"voiceProvider": { "type": "string" },
"model": { "type": "string" },
"instructions": { "type": "string" },
"introMessage": { "type": "string" },
"agentId": { "type": "string" },
"toolPolicy": {
"type": "string",
"enum": ["safe-read-only", "owner", "none"]
},
"providers": {
"type": "object",
"additionalProperties": { "type": "object" }
}
}
}
}
}
}
+33
View File
@@ -0,0 +1,33 @@
{
"name": "@openclaw/zoom-meetings",
"version": "2026.7.2",
"description": "OpenClaw Zoom browser meeting participant plugin.",
"private": true,
"type": "module",
"dependencies": {
"typebox": "1.3.3"
},
"devDependencies": {
"@openclaw/plugin-sdk": "workspace:*",
"openclaw": "workspace:*"
},
"peerDependencies": {
"openclaw": ">=2026.7.2"
},
"peerDependenciesMeta": {
"openclaw": {
"optional": true
}
},
"openclaw": {
"extensions": [
"./index.ts"
],
"compat": {
"pluginApi": ">=2026.7.2"
},
"build": {
"openclawVersion": "2026.7.2"
}
}
}
@@ -0,0 +1,93 @@
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import {
consultMeetingAgent,
handleMeetingRealtimeConsultToolCall,
resolveMeetingRealtimeTools,
type MeetingAgentConsultSurface,
} from "openclaw/plugin-sdk/meeting-runtime";
import type { PluginRuntime, RuntimeLogger } from "openclaw/plugin-sdk/plugin-runtime";
import type {
RealtimeVoiceBridgeSession,
RealtimeVoiceTool,
RealtimeVoiceToolCallEvent,
TalkEventInput,
} from "openclaw/plugin-sdk/realtime-voice";
import type { ZoomMeetingsConfig, ZoomMeetingsToolPolicy } from "./config.js";
const ZOOM_MEETINGS_CONSULT_SURFACE: MeetingAgentConsultSurface = {
id: "zoom-meetings",
provider: "zoom-meetings",
lane: "zoom-meetings",
surface: "a private Zoom meeting",
userLabel: "Participant",
assistantLabel: "Agent",
questionSourceLabel: "participant",
workingResponseLabel: "participant",
extraSystemPrompt: [
"You are a behind-the-scenes consultant for a live meeting voice agent.",
"Prioritize a fast, speakable answer over exhaustive investigation.",
"Use only bounded, task-relevant tool calls.",
"Never print secrets or dump environment variables.",
"Be accurate, brief, and speakable.",
].join(" "),
};
export function resolveZoomMeetingsRealtimeTools(
policy: ZoomMeetingsToolPolicy,
): RealtimeVoiceTool[] {
return resolveMeetingRealtimeTools(policy);
}
export async function consultOpenClawAgentForZoomMeeting(params: {
config: ZoomMeetingsConfig;
fullConfig: OpenClawConfig;
runtime: PluginRuntime;
logger: RuntimeLogger;
meetingSessionId: string;
requesterSessionKey?: string;
args: unknown;
transcript: Array<{ role: "user" | "assistant"; text: string }>;
}): Promise<{ text: string }> {
return await consultMeetingAgent({
surface: ZOOM_MEETINGS_CONSULT_SURFACE,
config: params.fullConfig,
runtime: params.runtime,
logger: params.logger,
agentId: params.config.realtime.agentId,
toolPolicy: params.config.realtime.toolPolicy,
meetingSessionId: params.meetingSessionId,
requesterSessionKey: params.requesterSessionKey,
args: params.args,
transcript: params.transcript,
});
}
export async function handleZoomMeetingsRealtimeConsultToolCall(params: {
strategy: string;
session: RealtimeVoiceBridgeSession;
event: RealtimeVoiceToolCallEvent;
config: ZoomMeetingsConfig;
fullConfig: OpenClawConfig;
runtime: PluginRuntime;
logger: RuntimeLogger;
meetingSessionId: string;
requesterSessionKey?: string;
transcript: Array<{ role: "user" | "assistant"; text: string }>;
onTalkEvent?: (event: TalkEventInput) => void;
}): Promise<void> {
await handleMeetingRealtimeConsultToolCall({
surface: ZOOM_MEETINGS_CONSULT_SURFACE,
strategy: params.strategy,
session: params.session,
event: params.event,
config: params.fullConfig,
runtime: params.runtime,
logger: params.logger,
agentId: params.config.realtime.agentId,
toolPolicy: params.config.realtime.toolPolicy,
meetingSessionId: params.meetingSessionId,
requesterSessionKey: params.requesterSessionKey,
transcript: params.transcript,
onTalkEvent: params.onTalkEvent,
});
}
+46
View File
@@ -0,0 +1,46 @@
import { Command } from "commander";
import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime";
import { describe, expect, it } from "vitest";
import { registerZoomMeetingsCli, resolveZoomMeetingsCliGatewayTimeoutMs } from "./cli.js";
import { resolveZoomMeetingsConfig } from "./config.js";
describe("Zoom meetings CLI", () => {
it("exposes the same bounded timeout on both live probes", () => {
const program = new Command();
registerZoomMeetingsCli({ program, config: resolveZoomMeetingsConfig({}) });
const root = program.commands.find((command) => command.name() === "zoommeetings");
for (const name of ["test-speech", "test-listen"]) {
const probe = root?.commands.find((command) => command.name() === name);
expect(probe?.options.map((option) => option.long)).toContain("--timeout-ms");
}
});
it("adds the post-join probe budget to the gateway deadline", () => {
const config = resolveZoomMeetingsConfig({});
expect(resolveZoomMeetingsCliGatewayTimeoutMs(config, { probe: false })).toBe(150_000);
expect(resolveZoomMeetingsCliGatewayTimeoutMs(config, { probe: true })).toBe(180_000);
expect(
resolveZoomMeetingsCliGatewayTimeoutMs(config, {
probe: true,
requestedTimeoutMs: 10_000,
}),
).toBe(160_000);
expect(
resolveZoomMeetingsCliGatewayTimeoutMs(
resolveZoomMeetingsConfig({
chrome: { joinTimeoutMs: 120_000, waitForInCallMs: 40_000 },
}),
{ probe: true },
),
).toBe(430_000);
expect(
resolveZoomMeetingsCliGatewayTimeoutMs(
resolveZoomMeetingsConfig({
chrome: { joinTimeoutMs: Number.MAX_VALUE, waitForInCallMs: Number.MAX_VALUE },
}),
{ probe: true, requestedTimeoutMs: Number.MAX_VALUE },
),
).toBe(MAX_TIMER_TIMEOUT_MS);
});
});
+180
View File
@@ -0,0 +1,180 @@
import type { Command } from "commander";
import { callGatewayFromCli } from "openclaw/plugin-sdk/gateway-runtime";
import {
addTimerTimeoutGraceMs,
parseStrictNonNegativeInteger,
} from "openclaw/plugin-sdk/number-runtime";
import type { ZoomMeetingsConfig, ZoomMeetingsMode, ZoomMeetingsTransport } from "./config.js";
import { resolveZoomMeetingsGatewayOperationTimeoutMs } from "./config.js";
import { resolveZoomMeetingsProbeTimeoutMs } from "./probe-timeout.js";
type JoinOptions = {
transport?: ZoomMeetingsTransport;
mode?: ZoomMeetingsMode;
message?: string;
timeoutMs?: string;
};
function print(value: unknown): void {
process.stdout.write(`${JSON.stringify(value, null, 2)}\n`);
}
function parseTimeout(value: string | undefined): number | undefined {
if (value === undefined) {
return undefined;
}
const parsed = parseStrictNonNegativeInteger(value);
if (parsed === undefined || parsed === 0) {
throw new Error("timeout-ms must be a positive integer");
}
return parsed;
}
export function resolveZoomMeetingsCliGatewayTimeoutMs(
config: ZoomMeetingsConfig,
options: { probe: boolean; requestedTimeoutMs?: number },
): number {
const operationTimeoutMs = resolveZoomMeetingsGatewayOperationTimeoutMs(config);
const probeTimeoutMs = options.probe
? resolveZoomMeetingsProbeTimeoutMs(options.requestedTimeoutMs, config.chrome.joinTimeoutMs)
: undefined;
return probeTimeoutMs === undefined
? operationTimeoutMs
: (addTimerTimeoutGraceMs(operationTimeoutMs, probeTimeoutMs) ?? 1);
}
async function call(params: {
config: ZoomMeetingsConfig;
method: string;
payload?: Record<string, unknown>;
}): Promise<void> {
const requestedTimeout =
typeof params.payload?.timeoutMs === "number" ? params.payload.timeoutMs : undefined;
const timeoutMs = resolveZoomMeetingsCliGatewayTimeoutMs(params.config, {
probe:
params.method === "zoommeetings.testSpeech" || params.method === "zoommeetings.testListen",
requestedTimeoutMs: requestedTimeout,
});
print(
await callGatewayFromCli(
params.method,
{
json: true,
timeout: String(timeoutMs),
},
params.payload,
{ progress: false, scopes: ["operator.admin"] },
),
);
}
function joinPayload(url: string, options: JoinOptions): Record<string, unknown> {
return {
url,
...(options.transport ? { transport: options.transport } : {}),
...(options.mode ? { mode: options.mode } : {}),
...(options.message ? { message: options.message } : {}),
...(options.timeoutMs ? { timeoutMs: parseTimeout(options.timeoutMs) } : {}),
};
}
function addJoinOptions(command: Command): Command {
return command
.option("--transport <transport>", "chrome or chrome-node")
.option("--mode <mode>", "agent, bidi, or transcribe")
.option("--message <text>", "instructions to speak after joining");
}
function addProbeOptions(command: Command): Command {
return addJoinOptions(command).option("--timeout-ms <ms>", "probe timeout in milliseconds");
}
export function registerZoomMeetingsCli(params: {
program: Command;
config: ZoomMeetingsConfig;
}): void {
const root = params.program
.command("zoommeetings")
.description("Join and manage Zoom meeting guests");
addJoinOptions(root.command("join <url>").description("join a Zoom meeting as a guest")).action(
async (url: string, options: JoinOptions) => {
await call({
config: params.config,
method: "zoommeetings.join",
payload: joinPayload(url, options),
});
},
);
root
.command("leave <session-id>")
.description("leave a Zoom meeting")
.action(async (sessionId: string) => {
await call({ config: params.config, method: "zoommeetings.leave", payload: { sessionId } });
});
root
.command("status [session-id]")
.description("show Zoom meeting session status")
.action(async (sessionId?: string) => {
await call({
config: params.config,
method: "zoommeetings.status",
payload: sessionId ? { sessionId } : {},
});
});
root
.command("transcript <session-id>")
.description("read the current transcript snapshot")
.option("--since-index <index>", "resume from a prior transcript index")
.action(async (sessionId: string, options: { sinceIndex?: string }) => {
const sinceIndex =
options.sinceIndex === undefined
? undefined
: parseStrictNonNegativeInteger(options.sinceIndex);
if (options.sinceIndex !== undefined && sinceIndex === undefined) {
throw new Error("since-index must be a non-negative integer");
}
await call({
config: params.config,
method: "zoommeetings.transcript",
payload: { sessionId, ...(sinceIndex === undefined ? {} : { sinceIndex }) },
});
});
root
.command("speak <session-id> [message]")
.description("speak through an active talk-back session")
.action(async (sessionId: string, message?: string) => {
await call({
config: params.config,
method: "zoommeetings.speak",
payload: { sessionId, ...(message ? { message } : {}) },
});
});
root
.command("setup")
.description("check Zoom meeting prerequisites")
.option("--transport <transport>", "chrome or chrome-node")
.option("--mode <mode>", "agent, bidi, or transcribe")
.action(async (options: { transport?: ZoomMeetingsTransport; mode?: ZoomMeetingsMode }) => {
await call({ config: params.config, method: "zoommeetings.setup", payload: options });
});
for (const [name, method, description] of [
["test-speech", "zoommeetings.testSpeech", "join and verify talk-back output"],
[
"test-listen",
"zoommeetings.testListen",
"join in transcribe mode and report caption support",
],
] as const) {
const command = root.command(`${name} <url>`).description(description);
addProbeOptions(command).action(async (url: string, options: JoinOptions) => {
await call({ config: params.config, method, payload: joinPayload(url, options) });
});
}
}
@@ -0,0 +1,68 @@
import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime";
import { describe, expect, it } from "vitest";
import {
resolveZoomMeetingsConfig,
resolveZoomMeetingsGatewayOperationTimeoutMs,
} from "./config.js";
describe("Zoom meetings config", () => {
it("allows the live Zoom web client enough time to reach prejoin and in-call UI", () => {
const config = resolveZoomMeetingsConfig({});
expect(config.chrome.waitForInCallMs).toBe(60_000);
expect(resolveZoomMeetingsGatewayOperationTimeoutMs(config)).toBe(150_000);
});
it("covers sequential browser launch and in-call waits", () => {
expect(
resolveZoomMeetingsGatewayOperationTimeoutMs(
resolveZoomMeetingsConfig({
chrome: { joinTimeoutMs: 120_000, waitForInCallMs: 40_000 },
}),
),
).toBe(310_000);
});
it("builds matching SoX commands from the selected audio format", () => {
const config = resolveZoomMeetingsConfig({ chrome: { audioBufferBytes: 2048 } });
expect(config.chrome.audioInputCommand).toContain("sox");
expect(config.chrome.audioInputCommand).toContain("2048");
expect(config.chrome.audioOutputCommand).toContain("BlackHole 2ch");
const g711 = resolveZoomMeetingsConfig({ chrome: { audioFormat: "g711-ulaw-8khz" } });
expect(g711.chrome.audioInputCommand).toContain("BlackHole 2ch");
expect(g711.chrome.audioOutputCommand).toContain("BlackHole 2ch");
expect(g711.chrome.audioInputCommand).toContain("mu-law");
});
it("preserves explicit command overrides and realtime passthrough", () => {
const config = resolveZoomMeetingsConfig({
defaultMode: "bidi",
chrome: { audioInputCommand: ["capture"], audioOutputCommand: ["play"] },
chromeNode: { node: "mac-node" },
realtime: {
voiceProvider: "google",
model: "voice-model",
providers: { google: { apiKey: "ref" } },
},
});
expect(config).toMatchObject({
defaultMode: "bidi",
chrome: { audioInputCommand: ["capture"], audioOutputCommand: ["play"] },
chromeNode: { node: "mac-node" },
realtime: {
voiceProvider: "google",
model: "voice-model",
providers: { google: { apiKey: "ref" } },
},
});
});
it("caps timer values and gateway grace", () => {
const config = resolveZoomMeetingsConfig({
chrome: { joinTimeoutMs: Number.MAX_VALUE, waitForInCallMs: Number.MAX_VALUE },
});
expect(config.chrome.joinTimeoutMs).toBe(MAX_TIMER_TIMEOUT_MS);
expect(config.chrome.waitForInCallMs).toBe(MAX_TIMER_TIMEOUT_MS);
expect(resolveZoomMeetingsGatewayOperationTimeoutMs(config)).toBe(MAX_TIMER_TIMEOUT_MS);
});
});
+238
View File
@@ -0,0 +1,238 @@
import { buildMeetingSoxAudioCommands } from "openclaw/plugin-sdk/meeting-runtime";
import {
addTimerTimeoutGraceMs,
resolvePositiveTimerTimeoutMs,
} from "openclaw/plugin-sdk/number-runtime";
import {
REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME,
resolveRealtimeVoiceAgentConsultToolPolicy,
type RealtimeVoiceAgentConsultToolPolicy,
} from "openclaw/plugin-sdk/realtime-voice";
import {
asRecord,
normalizeOptionalLowercaseString,
normalizeOptionalString,
normalizeOptionalTrimmedStringList,
} from "openclaw/plugin-sdk/string-coerce-runtime";
export type ZoomMeetingsMode = "agent" | "bidi" | "transcribe";
export type ZoomMeetingsTransport = "chrome" | "chrome-node";
export type ZoomMeetingsToolPolicy = RealtimeVoiceAgentConsultToolPolicy;
type ZoomMeetingsRealtimeStrategy = "agent" | "bidi";
type ZoomMeetingsAudioFormat = "pcm16-24khz" | "g711-ulaw-8khz";
export type ZoomMeetingsConfig = {
enabled: boolean;
defaultMode: ZoomMeetingsMode;
chrome: {
audioBackend: "blackhole-2ch";
audioFormat: ZoomMeetingsAudioFormat;
audioBufferBytes: number;
launch: boolean;
browserProfile?: string;
guestName: string;
reuseExistingTab: boolean;
autoJoin: boolean;
joinTimeoutMs: number;
waitForInCallMs: number;
audioInputCommand: string[];
audioOutputCommand: string[];
bargeInInputCommand?: string[];
bargeInRmsThreshold: number;
bargeInPeakThreshold: number;
bargeInCooldownMs: number;
};
chromeNode: {
node?: string;
};
realtime: {
strategy: ZoomMeetingsRealtimeStrategy;
provider?: string;
transcriptionProvider?: string;
voiceProvider?: string;
model?: string;
instructions?: string;
introMessage?: string;
agentId?: string;
toolPolicy: ZoomMeetingsToolPolicy;
providers: Record<string, Record<string, unknown>>;
};
};
export function resolveZoomMeetingsGatewayOperationTimeoutMs(config: ZoomMeetingsConfig): number {
return Math.max(
60_000,
addTimerTimeoutGraceMs(
config.chrome.joinTimeoutMs,
config.chrome.waitForInCallMs + config.chrome.joinTimeoutMs + 30_000,
) ?? 1,
);
}
const DEFAULT_AUDIO_BUFFER_BYTES = 4_096;
const DEFAULT_AUDIO_FORMAT: ZoomMeetingsAudioFormat = "pcm16-24khz";
function buildSoxCommands(format: ZoomMeetingsAudioFormat, bufferBytes: number) {
return buildMeetingSoxAudioCommands({
bufferBytes,
device: "BlackHole 2ch",
deviceType: "coreaudio",
format:
format === "g711-ulaw-8khz"
? { sampleRate: 8_000, channels: 1, encoding: "mu-law", bits: 8 }
: {
sampleRate: 24_000,
channels: 1,
encoding: "signed-integer",
bits: 16,
endian: "little",
},
});
}
const DEFAULT_SOX_COMMANDS = buildSoxCommands(DEFAULT_AUDIO_FORMAT, DEFAULT_AUDIO_BUFFER_BYTES);
export const DEFAULT_ZOOM_MEETINGS_AUDIO_INPUT_COMMAND = DEFAULT_SOX_COMMANDS.inputCommand;
export const DEFAULT_ZOOM_MEETINGS_AUDIO_OUTPUT_COMMAND = DEFAULT_SOX_COMMANDS.outputCommand;
const DEFAULT_REALTIME_INSTRUCTIONS = `You are joining a private Zoom meeting as an OpenClaw voice transport. Keep spoken replies brief and natural. In agent mode, wait for OpenClaw consult results and speak them exactly. In bidi mode, answer directly and call ${REALTIME_VOICE_AGENT_CONSULT_TOOL_NAME} for deeper reasoning, current information, or tools.`;
const DEFAULT_CONFIG: ZoomMeetingsConfig = {
enabled: true,
defaultMode: "agent",
chrome: {
audioBackend: "blackhole-2ch",
audioFormat: DEFAULT_AUDIO_FORMAT,
audioBufferBytes: DEFAULT_AUDIO_BUFFER_BYTES,
launch: true,
guestName: "OpenClaw Agent",
reuseExistingTab: true,
autoJoin: true,
joinTimeoutMs: 30_000,
waitForInCallMs: 60_000,
audioInputCommand: [...DEFAULT_ZOOM_MEETINGS_AUDIO_INPUT_COMMAND],
audioOutputCommand: [...DEFAULT_ZOOM_MEETINGS_AUDIO_OUTPUT_COMMAND],
bargeInRmsThreshold: 650,
bargeInPeakThreshold: 2_500,
bargeInCooldownMs: 900,
},
chromeNode: {},
realtime: {
strategy: "agent",
provider: "openai",
transcriptionProvider: "openai",
instructions: DEFAULT_REALTIME_INSTRUCTIONS,
introMessage: "Say exactly: I'm here and listening.",
toolPolicy: "safe-read-only",
providers: {},
},
};
function resolveBoolean(value: unknown, fallback: boolean): boolean {
return typeof value === "boolean" ? value : fallback;
}
function resolvePositiveNumber(value: unknown, fallback: number): number {
return typeof value === "number" && Number.isFinite(value) && value > 0 ? value : fallback;
}
function resolveTimer(value: unknown, fallback: number): number {
return resolvePositiveTimerTimeoutMs(resolvePositiveNumber(value, fallback), fallback);
}
function resolveMode(value: unknown): ZoomMeetingsMode {
const normalized = normalizeOptionalLowercaseString(value);
return normalized === "agent" || normalized === "bidi" || normalized === "transcribe"
? normalized
: DEFAULT_CONFIG.defaultMode;
}
function resolveAudioFormat(value: unknown): ZoomMeetingsAudioFormat {
const normalized = normalizeOptionalLowercaseString(value)?.replaceAll("_", "-");
return normalized === "g711-ulaw-8khz" ? normalized : DEFAULT_AUDIO_FORMAT;
}
function resolveProviders(value: unknown): Record<string, Record<string, unknown>> {
const providers: Record<string, Record<string, unknown>> = {};
for (const [key, entry] of Object.entries(asRecord(value))) {
const id = normalizeOptionalLowercaseString(key);
if (id) {
providers[id] = asRecord(entry);
}
}
return providers;
}
export function resolveZoomMeetingsConfig(input: unknown): ZoomMeetingsConfig {
const raw = asRecord(input);
const chrome = asRecord(raw.chrome);
const chromeNode = asRecord(raw.chromeNode);
const realtime = asRecord(raw.realtime);
const audioFormat = resolveAudioFormat(chrome.audioFormat);
const audioBufferBytes = Math.max(
17,
Math.trunc(resolvePositiveNumber(chrome.audioBufferBytes, DEFAULT_AUDIO_BUFFER_BYTES)),
);
const generatedCommands = buildSoxCommands(audioFormat, audioBufferBytes);
const provider = normalizeOptionalString(realtime.provider) ?? DEFAULT_CONFIG.realtime.provider;
return {
enabled: resolveBoolean(raw.enabled, DEFAULT_CONFIG.enabled),
defaultMode: resolveMode(raw.defaultMode),
chrome: {
audioBackend: "blackhole-2ch",
audioFormat,
audioBufferBytes,
launch: resolveBoolean(chrome.launch, DEFAULT_CONFIG.chrome.launch),
browserProfile: normalizeOptionalString(chrome.browserProfile),
guestName: normalizeOptionalString(chrome.guestName) ?? DEFAULT_CONFIG.chrome.guestName,
reuseExistingTab: resolveBoolean(
chrome.reuseExistingTab,
DEFAULT_CONFIG.chrome.reuseExistingTab,
),
autoJoin: resolveBoolean(chrome.autoJoin, DEFAULT_CONFIG.chrome.autoJoin),
joinTimeoutMs: resolveTimer(chrome.joinTimeoutMs, DEFAULT_CONFIG.chrome.joinTimeoutMs),
waitForInCallMs: resolveTimer(chrome.waitForInCallMs, DEFAULT_CONFIG.chrome.waitForInCallMs),
audioInputCommand:
normalizeOptionalTrimmedStringList(chrome.audioInputCommand) ??
generatedCommands.inputCommand,
audioOutputCommand:
normalizeOptionalTrimmedStringList(chrome.audioOutputCommand) ??
generatedCommands.outputCommand,
bargeInInputCommand: normalizeOptionalTrimmedStringList(chrome.bargeInInputCommand),
bargeInRmsThreshold: resolvePositiveNumber(
chrome.bargeInRmsThreshold,
DEFAULT_CONFIG.chrome.bargeInRmsThreshold,
),
bargeInPeakThreshold: resolvePositiveNumber(
chrome.bargeInPeakThreshold,
DEFAULT_CONFIG.chrome.bargeInPeakThreshold,
),
bargeInCooldownMs: resolveTimer(
chrome.bargeInCooldownMs,
DEFAULT_CONFIG.chrome.bargeInCooldownMs,
),
},
chromeNode: { node: normalizeOptionalString(chromeNode.node) },
realtime: {
strategy: normalizeOptionalLowercaseString(realtime.strategy) === "bidi" ? "bidi" : "agent",
provider,
transcriptionProvider:
normalizeOptionalString(realtime.transcriptionProvider) ??
DEFAULT_CONFIG.realtime.transcriptionProvider,
voiceProvider: normalizeOptionalString(realtime.voiceProvider),
model: normalizeOptionalString(realtime.model),
instructions:
normalizeOptionalString(realtime.instructions) ?? DEFAULT_CONFIG.realtime.instructions,
introMessage:
typeof realtime.introMessage === "string"
? realtime.introMessage.trim()
: DEFAULT_CONFIG.realtime.introMessage,
agentId: normalizeOptionalString(realtime.agentId),
toolPolicy: resolveRealtimeVoiceAgentConsultToolPolicy(
realtime.toolPolicy,
DEFAULT_CONFIG.realtime.toolPolicy,
),
providers: resolveProviders(realtime.providers),
},
};
}
+5
View File
@@ -0,0 +1,5 @@
export class ZoomMeetingsInvalidRequestError extends Error {}
export function zoomMeetingsInvalidRequest(message: string): ZoomMeetingsInvalidRequestError {
return new ZoomMeetingsInvalidRequestError(message);
}
@@ -0,0 +1,43 @@
import { afterEach, describe, expect, it, vi } from "vitest";
const childProcessMocks = vi.hoisted(() => ({ spawnSync: vi.fn() }));
vi.mock("node:child_process", () => ({ spawnSync: childProcessMocks.spawnSync }));
import { handleZoomMeetingsNodeHostCommand } from "./node-host.js";
afterEach(() => {
vi.useRealTimers();
vi.restoreAllMocks();
});
describe("Zoom meetings node setup", () => {
it("shares one timeout across the sequential device and command probes", async () => {
vi.useFakeTimers();
vi.setSystemTime(0);
vi.spyOn(process, "platform", "get").mockReturnValue("darwin");
childProcessMocks.spawnSync.mockImplementation(() => {
const call = childProcessMocks.spawnSync.mock.calls.length;
if (call === 1) {
vi.setSystemTime(6_000);
return { status: 0, stderr: "", stdout: "BlackHole 2ch" };
}
vi.setSystemTime(call === 2 ? 8_000 : 9_000);
return { status: 0, stderr: "", stdout: "" };
});
await handleZoomMeetingsNodeHostCommand(
JSON.stringify({
action: "setup",
audioInputCommand: ["sox"],
audioOutputCommand: ["play"],
}),
);
expect(
childProcessMocks.spawnSync.mock.calls.map(
(call) => (call[2] as { timeout?: number } | undefined)?.timeout,
),
).toEqual([10_000, 4_000, 2_000]);
});
});
+119
View File
@@ -0,0 +1,119 @@
import { spawnSync } from "node:child_process";
import { createMeetingNodeHost } from "openclaw/plugin-sdk/meeting-runtime";
import {
DEFAULT_ZOOM_MEETINGS_AUDIO_INPUT_COMMAND,
DEFAULT_ZOOM_MEETINGS_AUDIO_OUTPUT_COMMAND,
} from "./config.js";
import {
ZOOM_MEETINGS_SYSTEM_PROFILER_COMMAND,
outputMentionsBlackHole2ch,
} from "./transports/chrome-audio-device.js";
import { ZOOM_MEETINGS_PLATFORM_ADAPTER } from "./transports/zoom-meetings-platform-adapter.js";
import { ZOOM_MEETINGS_NODE_COMMAND } from "./transports/zoom-meetings-platform-constants.js";
function commandExists(command: string, timeoutMs: number): boolean {
const result = spawnSync("/bin/sh", ["-lc", 'command -v "$1" >/dev/null 2>&1', "sh", command], {
encoding: "utf8",
timeout: timeoutMs,
});
return result.status === 0;
}
function assertTalkBackPrerequisites(
timeoutMs: number,
commands: readonly (readonly string[])[] = [
DEFAULT_ZOOM_MEETINGS_AUDIO_INPUT_COMMAND,
DEFAULT_ZOOM_MEETINGS_AUDIO_OUTPUT_COMMAND,
],
) {
if (process.platform !== "darwin") {
throw new Error("Zoom meeting talk-back with BlackHole 2ch is macOS-only");
}
const deadline = Date.now() + timeoutMs;
const remainingMs = () => Math.max(1, deadline - Date.now());
const result = spawnSync(ZOOM_MEETINGS_SYSTEM_PROFILER_COMMAND, ["SPAudioDataType"], {
encoding: "utf8",
timeout: remainingMs(),
});
const stderr =
result.stderr ??
(result.error
? result.error instanceof Error
? result.error.message
: String(result.error)
: "");
const output = `${result.stdout ?? ""}\n${stderr}`;
if (
(typeof result.status === "number" ? result.status : result.error ? 1 : 0) !== 0 ||
!outputMentionsBlackHole2ch(output)
) {
throw new Error("BlackHole 2ch audio device not found on the node.");
}
for (const argv of commands) {
const command = argv[0];
if (!command || Date.now() >= deadline || !commandExists(command, remainingMs())) {
throw new Error(`Configured audio command not found on the node: ${command || "<empty>"}`);
}
}
}
function readCommand(params: Record<string, unknown>, name: string): string[] {
const value = params[name];
if (
!Array.isArray(value) ||
value.length === 0 ||
value.some((entry) => typeof entry !== "string")
) {
throw new Error(`${name} must be a non-empty string array.`);
}
return value as string[];
}
const zoomMeetingsNodeHost = createMeetingNodeHost({
commandName: ZOOM_MEETINGS_NODE_COMMAND,
displayName: "Zoom meetings",
browserLabel: "Zoom meeting",
bridgeIdPrefix: "zoom_meeting_node_",
defaultAudioInputCommand: DEFAULT_ZOOM_MEETINGS_AUDIO_INPUT_COMMAND,
defaultAudioOutputCommand: DEFAULT_ZOOM_MEETINGS_AUDIO_OUTPUT_COMMAND,
talkBackModes: new Set(["agent", "bidi"]),
agentMode: "agent",
normalizeUrl: (url) => ZOOM_MEETINGS_PLATFORM_ADAPTER.urls.validateAndNormalize(url),
normalizeMeetingKey: (url) => ZOOM_MEETINGS_PLATFORM_ADAPTER.urls.normalizeForReuse(url),
assertAudioAvailable: assertTalkBackPrerequisites,
browser: {
application: "Google Chrome",
buildProfileArgs: (profile) => ["--args", `--profile-directory=${profile}`],
openedStatus: "chrome-opened",
openedNotes: [
"Zoom page control is handled by OpenClaw browser automation when using chrome-node.",
],
},
});
export async function handleZoomMeetingsNodeHostCommand(
paramsJSON?: string | null,
): Promise<string> {
if (paramsJSON) {
let raw: unknown;
try {
raw = JSON.parse(paramsJSON) as unknown;
} catch {
throw new Error("Zoom meetings node host received malformed params JSON.");
}
const params =
raw && typeof raw === "object" && !Array.isArray(raw) ? (raw as Record<string, unknown>) : {};
if (params.action === "setup") {
const commands = [
readCommand(params, "audioInputCommand"),
readCommand(params, "audioOutputCommand"),
];
if (params.bargeInInputCommand !== undefined) {
commands.push(readCommand(params, "bargeInInputCommand"));
}
assertTalkBackPrerequisites(10_000, commands);
return JSON.stringify({ ok: true });
}
}
return await zoomMeetingsNodeHost.handleCommand(paramsJSON);
}
@@ -0,0 +1,39 @@
import type { OpenClawPluginNodeInvokePolicyContext } from "openclaw/plugin-sdk/plugin-entry";
import { describe, expect, it, vi } from "vitest";
import { resolveZoomMeetingsConfig } from "./config.js";
import { createZoomMeetingsNodeInvokePolicy } from "./node-invoke-policy.js";
describe("Zoom meetings node invoke policy", () => {
it("replaces setup probe commands with trusted configured commands", async () => {
const config = resolveZoomMeetingsConfig({
chrome: {
audioInputCommand: ["trusted-input", "--read"],
audioOutputCommand: ["trusted-output", "--write"],
bargeInInputCommand: ["trusted-barge-in"],
},
});
const invokeNode = vi.fn(async () => ({ ok: true as const }));
const policy = createZoomMeetingsNodeInvokePolicy(config);
await policy.handle({
command: "zoommeetings.chrome",
config: {},
invokeNode,
nodeId: "node-1",
params: {
action: "setup",
audioInputCommand: ["untrusted-input"],
audioOutputCommand: ["untrusted-output"],
},
} as OpenClawPluginNodeInvokePolicyContext);
expect(invokeNode).toHaveBeenCalledWith({
params: {
action: "setup",
audioInputCommand: ["trusted-input", "--read"],
audioOutputCommand: ["trusted-output", "--write"],
bargeInInputCommand: ["trusted-barge-in"],
},
});
});
});
@@ -0,0 +1,43 @@
import { createMeetingBrowserNodeInvokePolicy } from "openclaw/plugin-sdk/meeting-runtime";
import type {
OpenClawPluginNodeInvokePolicy,
OpenClawPluginNodeInvokePolicyContext,
} from "openclaw/plugin-sdk/plugin-entry";
import type { ZoomMeetingsConfig } from "./config.js";
import { ZOOM_MEETINGS_PLATFORM_ADAPTER } from "./transports/zoom-meetings-platform-adapter.js";
import { ZOOM_MEETINGS_NODE_COMMAND } from "./transports/zoom-meetings-platform-constants.js";
export function createZoomMeetingsNodeInvokePolicy(
config: ZoomMeetingsConfig,
): OpenClawPluginNodeInvokePolicy {
const base = createMeetingBrowserNodeInvokePolicy({
commandName: ZOOM_MEETINGS_NODE_COMMAND,
displayName: "Zoom meetings",
deniedCode: "ZOOM_MEETINGS_NODE_POLICY_DENIED",
supportedModes: new Set(["agent", "bidi", "transcribe"]),
normalizeUrl: (url) => ZOOM_MEETINGS_PLATFORM_ADAPTER.urls.validateAndNormalize(url),
start: config.chrome,
});
return {
...base,
async handle(ctx: OpenClawPluginNodeInvokePolicyContext) {
const params =
ctx.params && typeof ctx.params === "object" && !Array.isArray(ctx.params)
? (ctx.params as Record<string, unknown>)
: {};
if (ctx.command !== ZOOM_MEETINGS_NODE_COMMAND || params.action !== "setup") {
return await base.handle(ctx);
}
return await ctx.invokeNode({
params: {
action: "setup",
audioInputCommand: [...config.chrome.audioInputCommand],
audioOutputCommand: [...config.chrome.audioOutputCommand],
...(config.chrome.bargeInInputCommand
? { bargeInInputCommand: [...config.chrome.bargeInInputCommand] }
: {}),
},
});
},
};
}
@@ -0,0 +1,14 @@
import { zoomMeetingsInvalidRequest as invalidRequest } from "./errors.js";
export function resolveZoomMeetingsProbeTimeoutMs(
input: number | undefined,
fallback: number,
): number {
if (input === undefined) {
return Math.min(Math.max(fallback, 1), 120_000);
}
if (!Number.isFinite(input) || input <= 0) {
throw invalidRequest("timeoutMs must be a positive number");
}
return Math.min(Math.trunc(input), 120_000);
}
@@ -0,0 +1,169 @@
import type { PluginRuntime } from "openclaw/plugin-sdk/plugin-runtime";
import { describe, expect, it, vi } from "vitest";
import { resolveZoomMeetingsConfig } from "./config.js";
const realtimeMocks = vi.hoisted(() => ({
healths: [] as Array<{ bridgeClosed: boolean }>,
speak: vi.fn(),
startAgent: vi.fn(async () => {
const health = { bridgeClosed: false };
realtimeMocks.healths.push(health);
return {
getHealth: () => health,
providerId: "test",
speak: realtimeMocks.speak,
stop: vi.fn(async () => {}),
};
}),
}));
vi.mock("openclaw/plugin-sdk/meeting-runtime", async (importOriginal) => {
const original = await importOriginal<typeof import("openclaw/plugin-sdk/meeting-runtime")>();
return {
...original,
createNodeMeetingRealtimeAudioTransport: () => ({
clearOutput: vi.fn(async () => {}),
dispose: vi.fn(async () => {}),
onFatal: vi.fn(),
startInput: vi.fn(),
stop: vi.fn(async () => {}),
writeOutput: vi.fn(async () => {}),
}),
startMeetingAgentRealtimeEngine: realtimeMocks.startAgent,
};
});
import { ZoomMeetingsRuntime } from "./runtime.js";
const URL = "https://zoom.us/j/12345678903?pwd=node";
describe("Zoom meetings node realtime recovery", () => {
it("starts the node bridge after manual admission becomes route-ready", async () => {
let routeReady = false;
let tabOpen = false;
const invoke = vi.fn(async (request: Record<string, unknown>) => {
const params = (request.params as Record<string, unknown>) ?? {};
if (request.command === "browser.proxy") {
if (params.path === "/tabs") {
return {
payload: {
result: {
tabs: tabOpen ? [{ targetId: "zoom-tab", title: "Zoom", url: URL }] : [],
},
},
};
}
if (params.path === "/tabs/open") {
tabOpen = true;
return { payload: { result: { targetId: "zoom-tab", title: "Zoom", url: URL } } };
}
if (params.path === "/tabs/focus") {
return { payload: { result: { ok: true } } };
}
if (params.path === "/act") {
const scriptValue = (params.body as { fn?: unknown } | undefined)?.fn;
const script = typeof scriptValue === "string" ? scriptValue : "";
if (script.includes("leaveAction")) {
return {
payload: {
result: { result: JSON.stringify({ departed: true, urlMatched: true }) },
},
};
}
return {
payload: {
result: {
result: JSON.stringify(
routeReady
? {
audioInputRouted: true,
audioOutputRouted: true,
inCall: true,
micMuted: false,
url: URL,
}
: {
inCall: false,
manualActionMessage: "Waiting for admission",
manualActionReason: "zoom-admission-required",
manualActionRequired: true,
url: URL,
},
),
},
},
};
}
}
if (params.action === "start") {
return {
payload: {
audioBridge: { type: "node-command-pair" },
bridgeId: "bridge-1",
},
};
}
return { payload: { ok: true } };
});
const runtime = new ZoomMeetingsRuntime({
config: resolveZoomMeetingsConfig({
chrome: { waitForInCallMs: 1 },
chromeNode: { node: "node-1" },
realtime: { agentId: "consult" },
}),
fullConfig: {},
logger: { debug: vi.fn(), error: vi.fn(), info: vi.fn(), warn: vi.fn() },
runtime: {
nodes: {
invoke,
list: vi.fn(async () => ({
nodes: [
{
caps: ["browser"],
commands: ["browser.proxy", "zoommeetings.chrome"],
connected: true,
nodeId: "node-1",
},
],
})),
},
} as unknown as PluginRuntime,
});
const joined = await runtime.join({
agentId: "support",
message: undefined,
mode: "agent",
requesterSessionKey: "agent:support:session:caller",
transport: "chrome-node",
url: URL,
});
expect(joined.session.chrome?.audioBridge).toBeUndefined();
routeReady = true;
const spoken = await runtime.speak(joined.session.id, "hello");
expect(spoken.spoken).toBe(true);
expect(joined.session.agentId).toBe("support");
expect(realtimeMocks.startAgent).toHaveBeenCalledTimes(1);
expect(realtimeMocks.startAgent).toHaveBeenCalledWith(
expect.objectContaining({
config: expect.objectContaining({
realtime: expect.objectContaining({ agentId: "consult" }),
}),
requesterSessionKey: "agent:support:session:caller",
}),
);
expect(realtimeMocks.speak).toHaveBeenCalledWith("hello");
expect(joined.session.chrome?.audioBridge).toMatchObject({ type: "node-command-pair" });
realtimeMocks.healths[0]!.bridgeClosed = true;
await runtime.status(joined.session.id);
const recovered = await runtime.speak(joined.session.id, "again");
expect(recovered.spoken).toBe(true);
expect(realtimeMocks.startAgent).toHaveBeenCalledTimes(2);
expect(realtimeMocks.speak).toHaveBeenCalledWith("again");
expect(joined.session.chrome?.health?.bridgeClosed).toBe(false);
});
});
@@ -0,0 +1,188 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { resolveZoomMeetingsConfig } from "./config.js";
import {
testZoomMeetingListening,
testZoomMeetingSpeech,
type ZoomMeetingsProbeContext,
} from "./runtime-probes.js";
import type { ZoomMeetingsSession } from "./transports/types.js";
const URL = "https://zoom.us/j/12345678902?pwd=probe";
afterEach(() => {
vi.useRealTimers();
});
describe("Zoom meeting runtime probes", () => {
it("uses the per-request speech verification timeout", async () => {
vi.useFakeTimers();
vi.setSystemTime(0);
const session = {
agentId: "main",
chrome: { health: { inCall: true, lastOutputBytes: 0 } },
id: "zoom-1",
mode: "agent",
transport: "chrome",
} as ZoomMeetingsSession;
const refreshHealth = vi.fn();
const context = {
config: resolveZoomMeetingsConfig({ chrome: { joinTimeoutMs: 30_000 } }),
hasHealthHandle: () => true,
isReusable: () => false,
join: vi.fn(async () => ({ session, spoken: true })),
list: () => [],
refreshCaptionHealth: async () => {},
refreshHealth,
resolveAgentId: () => "main",
} satisfies ZoomMeetingsProbeContext;
const pending = testZoomMeetingSpeech(context, {
mode: "agent",
timeoutMs: 150,
url: URL,
});
await vi.advanceTimersByTimeAsync(200);
const result = await pending;
expect(result.speechOutputTimedOut).toBe(true);
expect(refreshHealth).toHaveBeenCalledTimes(2);
expect(Date.now()).toBe(200);
});
it("bounds a blocked caption refresh by the per-request listening timeout", async () => {
vi.useFakeTimers();
vi.setSystemTime(0);
const session = {
agentId: "main",
chrome: {
browserTab: { openedByPlugin: false, targetId: "manual-zoom-tab" },
health: { inCall: true },
launched: false,
},
id: "zoom-listen-1",
mode: "transcribe",
transport: "chrome",
} as ZoomMeetingsSession;
const refreshCaptionHealth = vi.fn(
(_session: ZoomMeetingsSession, _timeoutMs: number) => new Promise<void>(() => {}),
);
const context = {
config: resolveZoomMeetingsConfig({ chrome: { joinTimeoutMs: 30_000 } }),
hasHealthHandle: () => true,
isReusable: () => false,
join: vi.fn(async () => ({ session, spoken: false })),
list: () => [],
refreshCaptionHealth,
refreshHealth: () => {},
resolveAgentId: () => "main",
} satisfies ZoomMeetingsProbeContext;
const pending = testZoomMeetingListening(context, {
mode: "transcribe",
timeoutMs: 300,
url: URL,
});
await vi.advanceTimersByTimeAsync(350);
const result = await pending;
expect(result.listenTimedOut).toBe(true);
expect(refreshCaptionHealth).toHaveBeenCalledWith(session, 300);
expect(Date.now()).toBe(350);
expect(vi.getTimerCount()).toBe(0);
});
it("refreshes captions before a short listening timeout expires", async () => {
vi.useFakeTimers();
vi.setSystemTime(0);
const session = {
agentId: "main",
chrome: {
browserTab: { openedByPlugin: false, targetId: "manual-zoom-tab" },
health: { inCall: true },
launched: false,
},
id: "zoom-listen-2",
mode: "transcribe",
transport: "chrome",
} as ZoomMeetingsSession;
const refreshCaptionHealth = vi.fn(
async (_session: ZoomMeetingsSession, _timeoutMs: number) => {
session.chrome!.health = {
...session.chrome!.health,
lastCaptionText: "Caption already waiting",
manualActionRequired: true,
transcriptLines: 1,
};
},
);
const context = {
config: resolveZoomMeetingsConfig({ chrome: { joinTimeoutMs: 30_000 } }),
hasHealthHandle: () => true,
isReusable: () => false,
join: vi.fn(async () => ({ session, spoken: false })),
list: () => [],
refreshCaptionHealth,
refreshHealth: () => {},
resolveAgentId: () => "main",
} satisfies ZoomMeetingsProbeContext;
const result = await testZoomMeetingListening(context, {
mode: "transcribe",
timeoutMs: 100,
url: URL,
});
expect(result.listenVerified).toBe(true);
expect(result.manualActionRequired).toBe(true);
expect(refreshCaptionHealth).toHaveBeenCalledTimes(1);
expect(Date.now()).toBe(0);
expect(vi.getTimerCount()).toBe(0);
});
it("does not accept caption progress that arrives after the listening deadline", async () => {
vi.useFakeTimers();
vi.setSystemTime(0);
const session = {
agentId: "main",
chrome: {
browserTab: { openedByPlugin: true, targetId: "zoom-tab" },
health: { inCall: true },
launched: true,
},
id: "zoom-listen-late",
mode: "transcribe",
transport: "chrome",
} as ZoomMeetingsSession;
const context = {
config: resolveZoomMeetingsConfig({ chrome: { joinTimeoutMs: 30_000 } }),
hasHealthHandle: () => true,
isReusable: () => false,
join: vi.fn(async () => ({ session, spoken: false })),
list: () => [],
refreshCaptionHealth: async (_session: ZoomMeetingsSession, timeoutMs: number) => {
await new Promise<void>((resolve) => {
setTimeout(resolve, timeoutMs + 50);
});
session.chrome!.health = {
...session.chrome!.health,
lastCaptionText: "Too late",
transcriptLines: 1,
};
},
refreshHealth: () => {},
resolveAgentId: () => "main",
} satisfies ZoomMeetingsProbeContext;
const pending = testZoomMeetingListening(context, {
mode: "transcribe",
timeoutMs: 300,
url: URL,
});
await vi.advanceTimersByTimeAsync(400);
await expect(pending).resolves.toMatchObject({
listenTimedOut: true,
listenVerified: false,
});
});
});
@@ -0,0 +1,190 @@
import { sleep } from "openclaw/plugin-sdk/runtime-env";
import type { ZoomMeetingsConfig, ZoomMeetingsMode, ZoomMeetingsTransport } from "./config.js";
import { zoomMeetingsInvalidRequest as invalidRequest } from "./errors.js";
import { resolveZoomMeetingsProbeTimeoutMs } from "./probe-timeout.js";
import type {
ZoomMeetingsJoinRequest,
ZoomMeetingsJoinResult,
ZoomMeetingsSession,
} from "./transports/types.js";
export type ZoomMeetingsProbeContext = {
config: ZoomMeetingsConfig;
resolveAgentId(request: ZoomMeetingsJoinRequest): string;
list(): ZoomMeetingsSession[];
join(request: ZoomMeetingsJoinRequest): Promise<ZoomMeetingsJoinResult>;
isReusable(
session: ZoomMeetingsSession,
resolved: {
url: string;
transport: ZoomMeetingsTransport;
mode: ZoomMeetingsMode;
agentId: string;
},
): boolean;
hasHealthHandle(sessionId: string): boolean;
refreshHealth(sessionId: string): void;
refreshCaptionHealth(session: ZoomMeetingsSession, timeoutMs: number): Promise<void>;
};
function talkBackMode(mode: ZoomMeetingsMode): boolean {
return mode === "agent" || mode === "bidi";
}
export async function testZoomMeetingSpeech(
context: ZoomMeetingsProbeContext,
request: ZoomMeetingsJoinRequest,
) {
if (request.mode === "transcribe") {
throw invalidRequest("test_speech requires mode: agent or bidi");
}
const mode = talkBackMode(request.mode ?? context.config.defaultMode)
? (request.mode ?? context.config.defaultMode)
: "agent";
const resolved = {
url: request.url,
transport: request.transport ?? (context.config.chromeNode.node ? "chrome-node" : "chrome"),
mode,
agentId: context.resolveAgentId(request),
} satisfies {
url: string;
transport: ZoomMeetingsTransport;
mode: ZoomMeetingsMode;
agentId: string;
};
const beforeSessions = context.list();
const before = new Set(beforeSessions.map((session) => session.id));
const existing = beforeSessions.find((session) => context.isReusable(session, resolved));
const existingOutputBytes = existing?.chrome?.health?.lastOutputBytes ?? 0;
const result = await context.join({
...request,
...resolved,
message: request.message ?? "Say exactly: Zoom speech test complete.",
});
const startOutputBytes = existing?.id === result.session.id ? existingOutputBytes : 0;
let health = result.session.chrome?.health;
const shouldWait =
result.spoken === true &&
health?.manualActionRequired !== true &&
context.hasHealthHandle(result.session.id);
if (shouldWait && (health?.lastOutputBytes ?? 0) <= startOutputBytes) {
const deadline =
Date.now() +
resolveZoomMeetingsProbeTimeoutMs(request.timeoutMs, context.config.chrome.joinTimeoutMs);
while (Date.now() < deadline && (health?.lastOutputBytes ?? 0) <= startOutputBytes) {
await sleep(100);
context.refreshHealth(result.session.id);
health = result.session.chrome?.health;
}
}
const speechOutputVerified = (health?.lastOutputBytes ?? 0) > startOutputBytes;
return {
createdSession: !before.has(result.session.id),
inCall: health?.inCall,
manualActionRequired: health?.manualActionRequired,
manualActionReason: health?.manualActionReason,
manualActionMessage: health?.manualActionMessage,
spoken: result.spoken ?? false,
speechOutputVerified,
speechOutputTimedOut: shouldWait && !speechOutputVerified,
speechReady: health?.speechReady,
speechBlockedReason: health?.speechBlockedReason,
speechBlockedMessage: health?.speechBlockedMessage,
audioOutputActive: health?.audioOutputActive,
lastOutputBytes: health?.lastOutputBytes,
session: result.session,
};
}
export async function testZoomMeetingListening(
context: ZoomMeetingsProbeContext,
request: ZoomMeetingsJoinRequest,
) {
if (request.mode && request.mode !== "transcribe") {
throw invalidRequest("test_listen requires mode: transcribe");
}
const resolved = {
url: request.url,
transport: request.transport ?? (context.config.chromeNode.node ? "chrome-node" : "chrome"),
mode: "transcribe" as const,
agentId: context.resolveAgentId(request),
};
const beforeSessions = context.list();
const before = new Set(beforeSessions.map((session) => session.id));
const existing = beforeSessions.find((session) => context.isReusable(session, resolved));
const start = {
lines: existing?.chrome?.health?.transcriptLines ?? 0,
at: existing?.chrome?.health?.lastCaptionAt,
text: existing?.chrome?.health?.lastCaptionText,
};
const result = await context.join({ ...request, ...resolved, message: undefined });
let health = result.session.chrome?.health;
const advanced = () =>
(health?.transcriptLines ?? 0) > (existing?.id === result.session.id ? start.lines : 0) ||
Boolean(health?.lastCaptionAt && health.lastCaptionAt !== start.at) ||
Boolean(health?.lastCaptionText && health.lastCaptionText !== start.text);
const shouldWait =
health?.manualActionRequired !== true && Boolean(result.session.chrome?.browserTab?.targetId);
let listenVerified = advanced();
if (shouldWait && !listenVerified) {
const deadline =
Date.now() +
resolveZoomMeetingsProbeTimeoutMs(request.timeoutMs, context.config.chrome.joinTimeoutMs);
while (Date.now() < deadline) {
const remainingMs = deadline - Date.now();
if (remainingMs <= 0) {
break;
}
let deadlineTimer: ReturnType<typeof setTimeout> | undefined;
const deadlineReached = new Promise<boolean>((resolve) => {
deadlineTimer = setTimeout(() => resolve(false), remainingMs);
});
// Browser recovery receives this same remaining budget. The outer race
// keeps the probe wall-clock bounded while the inner deadline prevents
// the per-target browser act from lingering on its normal longer timeout.
const refreshed = await Promise.race([
context.refreshCaptionHealth(result.session, remainingMs).then(() => true),
deadlineReached,
]).finally(() => {
if (deadlineTimer !== undefined) {
clearTimeout(deadlineTimer);
}
});
if (!refreshed) {
break;
}
health = result.session.chrome?.health;
if (Date.now() >= deadline) {
break;
}
if (advanced()) {
listenVerified = true;
}
if (listenVerified || health?.manualActionRequired) {
break;
}
const retryDelayMs = deadline - Date.now();
if (retryDelayMs <= 0) {
break;
}
await sleep(Math.min(250, retryDelayMs));
}
}
return {
createdSession: !before.has(result.session.id),
inCall: health?.inCall,
manualActionRequired: health?.manualActionRequired,
manualActionReason: health?.manualActionReason,
manualActionMessage: health?.manualActionMessage,
listenVerified,
listenTimedOut: shouldWait && !listenVerified && health?.manualActionRequired !== true,
captioning: health?.captioning,
captionsEnabledAttempted: health?.captionsEnabledAttempted,
transcriptLines: health?.transcriptLines,
lastCaptionAt: health?.lastCaptionAt,
lastCaptionSpeaker: health?.lastCaptionSpeaker,
lastCaptionText: health?.lastCaptionText,
recentTranscript: health?.recentTranscript,
session: result.session,
};
}
@@ -0,0 +1,42 @@
import { randomUUID } from "node:crypto";
import type { ZoomMeetingsConfig, ZoomMeetingsMode, ZoomMeetingsTransport } from "./config.js";
import type { ZoomMeetingsSession } from "./transports/types.js";
export function createZoomMeetingsSession(params: {
config: ZoomMeetingsConfig;
resolved: {
url: string;
transport: ZoomMeetingsTransport;
mode: ZoomMeetingsMode;
agentId: string;
};
createdAt: string;
}): ZoomMeetingsSession {
const { config, createdAt, resolved } = params;
return {
id: `zoom_meeting_${randomUUID()}`,
...resolved,
state: "active",
createdAt,
updatedAt: createdAt,
participantIdentity:
resolved.transport === "chrome-node"
? "Zoom guest in Chrome on a paired node"
: "Zoom guest in the OpenClaw Chrome profile",
realtime: {
enabled: resolved.mode === "agent" || resolved.mode === "bidi",
strategy: resolved.mode === "bidi" ? "bidi" : "agent",
provider:
resolved.mode === "bidi"
? (config.realtime.voiceProvider ?? config.realtime.provider)
: undefined,
model: resolved.mode === "bidi" ? config.realtime.model : undefined,
transcriptionProvider:
resolved.mode === "agent"
? (config.realtime.transcriptionProvider ?? config.realtime.provider)
: undefined,
toolPolicy: config.realtime.toolPolicy,
},
notes: [],
};
}
@@ -0,0 +1,99 @@
import type { PluginRuntime } from "openclaw/plugin-sdk/plugin-runtime";
import { describe, expect, it, vi } from "vitest";
import { resolveZoomMeetingsConfig } from "./config.js";
import { getZoomMeetingsSetupStatus } from "./runtime-setup.js";
function runtimeWithNode(invoke: (params: Record<string, unknown>) => Promise<unknown>) {
return {
nodes: {
invoke: vi.fn(invoke),
list: vi.fn(async () => ({
nodes: [
{
caps: ["browser"],
commands: ["browser.proxy", "zoommeetings.chrome"],
connected: true,
displayName: "zoom-node",
nodeId: "node-1",
},
],
})),
},
} as unknown as PluginRuntime;
}
describe("Zoom meetings runtime setup", () => {
it("accepts fresh-tab launch when existing-tab reuse is disabled", async () => {
const status = await getZoomMeetingsSetupStatus({
config: resolveZoomMeetingsConfig({
defaultMode: "transcribe",
chrome: { launch: true, reuseExistingTab: false },
}),
fullConfig: {},
runtime: {} as PluginRuntime,
options: { mode: "transcribe", transport: "chrome" },
});
expect(status.checks).toContainEqual({
id: "guest-join",
message: "Guest name, auto-join, and a Chrome launch or reuse path are configured",
ok: true,
});
expect(status.ok).toBe(true);
});
it("probes remote talk-back prerequisites through the selected Chrome node", async () => {
const runtime = runtimeWithNode(async () => ({ ok: true }));
const config = resolveZoomMeetingsConfig({
chrome: {
audioInputCommand: ["custom-input", "--read"],
audioOutputCommand: ["custom-output", "--write"],
bargeInInputCommand: ["custom-barge-in"],
},
chromeNode: { node: "zoom-node" },
});
const status = await getZoomMeetingsSetupStatus({
config,
fullConfig: {},
runtime,
options: { mode: "agent", transport: "chrome-node" },
});
expect(runtime.nodes.invoke).toHaveBeenCalledWith({
command: "zoommeetings.chrome",
nodeId: "node-1",
params: {
action: "setup",
audioInputCommand: ["custom-input", "--read"],
audioOutputCommand: ["custom-output", "--write"],
bargeInInputCommand: ["custom-barge-in"],
},
timeoutMs: 12_000,
});
expect(status.checks).toContainEqual({
id: "chrome-node-audio-prerequisites",
message: "Remote macOS, BlackHole 2ch, and SoX prerequisites are ready",
ok: true,
});
expect(status.ok).toBe(true);
});
it("fails setup when the remote prerequisite probe fails", async () => {
const runtime = runtimeWithNode(async () => {
throw new Error("SoX audio command not found on the node.");
});
const status = await getZoomMeetingsSetupStatus({
config: resolveZoomMeetingsConfig({ chromeNode: { node: "zoom-node" } }),
fullConfig: {},
runtime,
options: { mode: "bidi", transport: "chrome-node" },
});
expect(status.ok).toBe(false);
expect(status.checks).toContainEqual({
id: "chrome-node-audio-prerequisites",
message: "SoX audio command not found on the node.",
ok: false,
});
});
});
@@ -0,0 +1,164 @@
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import {
addMeetingSetupCheck,
createMeetingSetupStatus,
resolveMeetingBrowserNodeInfo,
type MeetingSetupStatus,
} from "openclaw/plugin-sdk/meeting-runtime";
import type { PluginRuntime } from "openclaw/plugin-sdk/plugin-runtime";
import { uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
import type { ZoomMeetingsConfig, ZoomMeetingsMode, ZoomMeetingsTransport } from "./config.js";
import { assertBlackHole2chAvailable } from "./transports/chrome.js";
import { ZOOM_MEETINGS_BROWSER_NODE_ADAPTER } from "./transports/zoom-meetings-platform-constants.js";
function audioCommands(config: ZoomMeetingsConfig): string[] {
return uniqueStrings(
[
config.chrome.audioInputCommand[0],
config.chrome.audioOutputCommand[0],
config.chrome.bargeInInputCommand?.[0],
].filter((value): value is string => Boolean(value?.trim())),
);
}
async function commandExists(runtime: PluginRuntime, command: string): Promise<boolean> {
const result = await runtime.system.runCommandWithTimeout(
["/bin/sh", "-lc", 'command -v "$1" >/dev/null 2>&1', "sh", command],
{ timeoutMs: 5_000 },
);
return result.code === 0;
}
export async function getZoomMeetingsSetupStatus(params: {
config: ZoomMeetingsConfig;
fullConfig: OpenClawConfig;
runtime: PluginRuntime;
options?: { mode?: ZoomMeetingsMode; transport?: ZoomMeetingsTransport };
}): Promise<MeetingSetupStatus> {
const mode = params.options?.mode ?? params.config.defaultMode;
const transport =
params.options?.transport ?? (params.config.chromeNode.node ? "chrome-node" : "chrome");
const talkBack = mode === "agent" || mode === "bidi";
const guestJoinReady = Boolean(
params.config.chrome.guestName &&
params.config.chrome.autoJoin &&
(params.config.chrome.launch || params.config.chrome.reuseExistingTab),
);
let status = createMeetingSetupStatus([
{
id: "chrome-profile",
ok: true,
message: params.config.chrome.browserProfile
? `Chrome node profile configured: ${params.config.chrome.browserProfile}`
: "Local Chrome uses the configured OpenClaw browser profile",
},
{
id: "guest-join",
ok: guestJoinReady,
message: guestJoinReady
? "Guest name, auto-join, and a Chrome launch or reuse path are configured"
: "Set chrome.guestName, chrome.autoJoin, and either chrome.launch or chrome.reuseExistingTab for unattended guest joins",
},
{
id: "captions",
ok: true,
message:
mode === "transcribe"
? "Zoom live-caption capture is enabled and ready"
: "Caption scraping is not used by talk-back modes",
},
]);
if (transport === "chrome-node") {
try {
const node = await resolveMeetingBrowserNodeInfo({
runtime: params.runtime,
requestedNode: params.config.chromeNode.node,
adapter: ZOOM_MEETINGS_BROWSER_NODE_ADAPTER,
});
status = addMeetingSetupCheck(status, {
id: "chrome-node-connected",
ok: true,
message: `Connected Zoom meeting node ready: ${node.displayName ?? node.remoteIp ?? node.nodeId}`,
});
if (talkBack) {
if (!node.nodeId) {
throw new Error("Connected Zoom meetings node did not include a node id.");
}
await params.runtime.nodes.invoke({
nodeId: node.nodeId,
command: ZOOM_MEETINGS_BROWSER_NODE_ADAPTER.nodeCommandName,
params: {
action: "setup",
audioInputCommand: params.config.chrome.audioInputCommand,
audioOutputCommand: params.config.chrome.audioOutputCommand,
...(params.config.chrome.bargeInInputCommand
? { bargeInInputCommand: params.config.chrome.bargeInInputCommand }
: {}),
},
timeoutMs: 12_000,
});
status = addMeetingSetupCheck(status, {
id: "chrome-node-audio-prerequisites",
ok: true,
message: "Remote macOS, BlackHole 2ch, and SoX prerequisites are ready",
});
}
} catch (error) {
const connected = status.checks.some(
(check) => check.id === "chrome-node-connected" && check.ok,
);
status = addMeetingSetupCheck(status, {
id: connected ? "chrome-node-audio-prerequisites" : "chrome-node-connected",
ok: false,
message: formatErrorMessage(error),
});
}
}
if (!talkBack) {
return status;
}
status = addMeetingSetupCheck(status, {
id: "audio-bridge",
ok:
params.config.chrome.audioInputCommand.length > 0 &&
params.config.chrome.audioOutputCommand.length > 0,
message: `SoX command-pair audio bridge configured (${params.config.chrome.audioFormat})`,
});
if (transport === "chrome-node") {
return status;
}
try {
await assertBlackHole2chAvailable({
runtime: params.runtime,
timeoutMs: Math.min(params.config.chrome.joinTimeoutMs, 10_000),
});
status = addMeetingSetupCheck(status, {
id: "chrome-local-audio-device",
ok: true,
message: "BlackHole 2ch audio device found",
});
} catch (error) {
status = addMeetingSetupCheck(status, {
id: "chrome-local-audio-device",
ok: false,
message: formatErrorMessage(error),
});
}
const missing: string[] = [];
for (const command of audioCommands(params.config)) {
if (!(await commandExists(params.runtime, command).catch(() => false))) {
missing.push(command);
}
}
return addMeetingSetupCheck(status, {
id: "chrome-local-audio-commands",
ok: missing.length === 0,
message:
missing.length === 0
? "Configured Chrome audio commands are available"
: `Chrome audio commands missing: ${missing.join(", ")}`,
});
}
@@ -0,0 +1,640 @@
import type { PluginRuntime } from "openclaw/plugin-sdk/plugin-runtime";
import { describe, expect, it, vi } from "vitest";
import { resolveZoomMeetingsConfig } from "./config.js";
import { ZoomMeetingsRuntime } from "./runtime.js";
const URL = "https://zoom.us/j/12345678904?pwd=runtime";
const logger = {
info: vi.fn(),
warn: vi.fn(),
error: vi.fn(),
debug: vi.fn(),
};
function runtimeHarness(options?: {
inCall?: boolean;
meetingEnded?: boolean;
pendingReason?: "admission" | "passcode";
tabOpen?: boolean;
}) {
let tabOpen = options?.tabOpen ?? false;
let inCall = options?.inCall ?? true;
let meetingEnded = options?.meetingEnded ?? false;
let meetingEndedOnce = false;
const pendingReason = options?.pendingReason ?? "passcode";
let sessionConflict = false;
let tabListFailures = 0;
let targetId = "zoom-tab";
let tabUrl = URL;
const gatewayRequest = vi.fn(async (_method: string, params: Record<string, unknown>) => {
if (params.path === "/tabs") {
if (tabListFailures > 0) {
tabListFailures -= 1;
throw new Error("browser node unavailable");
}
return {
tabs: tabOpen ? [{ targetId, title: "Zoom call", url: tabUrl }] : [],
};
}
if (params.path === "/tabs/open") {
tabOpen = true;
const requestedUrl = (params.body as { url?: unknown } | undefined)?.url;
tabUrl = typeof requestedUrl === "string" ? requestedUrl : URL;
return { targetId, title: "Zoom call", url: tabUrl };
}
if (params.path === "/tabs/focus") {
return { ok: true };
}
if (params.path === "/act") {
const rawFn = (params.body as { fn?: unknown } | undefined)?.fn;
const fn = typeof rawFn === "string" ? rawFn : "";
if (fn.includes("leaveAction")) {
return {
result: JSON.stringify({ departed: true, sessionMatched: true, urlMatched: true }),
};
}
if (fn.includes("expectedSessionId")) {
return {
result: JSON.stringify({
urlMatched: true,
sessionMatched: true,
droppedLines: 0,
lines: sessionConflict ? [{ text: "Archived caption" }] : [],
}),
};
}
const reportedMeetingEnded = meetingEnded;
if (meetingEndedOnce) {
meetingEnded = false;
meetingEndedOnce = false;
}
return {
result: JSON.stringify({
inCall,
meetingEnded: reportedMeetingEnded,
micMuted: true,
cameraOff: true,
...(!inCall
? {
...(pendingReason === "admission" ? { lobbyWaiting: true } : {}),
manualActionRequired: true,
manualActionReason:
pendingReason === "admission"
? "zoom-admission-required"
: "zoom-passcode-required",
manualActionMessage:
pendingReason === "admission"
? "Waiting for host admission."
: "Enter the meeting passcode.",
}
: {}),
...(sessionConflict && fn.includes("const allowSessionAdoption = false")
? {
manualActionRequired: true,
manualActionReason: "zoom-session-conflict",
manualActionMessage: "This Zoom tab is owned by another active meeting session.",
}
: {}),
url: tabUrl,
title: "Zoom call",
}),
};
}
if (params.method === "DELETE" && params.path === `/tabs/${targetId}`) {
tabOpen = false;
return { ok: true };
}
throw new Error(`unexpected browser request ${String(params.method)} ${String(params.path)}`);
});
const runtime = {
gateway: {
isAvailable: vi.fn(async () => true),
request: gatewayRequest,
},
system: {
runCommandWithTimeout: vi.fn(async () => ({ code: 0, stdout: "", stderr: "" })),
},
} as unknown as PluginRuntime;
return {
runtime,
gatewayRequest,
closeTab() {
tabOpen = false;
},
setSessionConflict(value: boolean) {
sessionConflict = value;
},
setInCall(value: boolean) {
inCall = value;
},
setMeetingEnded(value: boolean, behavior?: { once?: boolean }) {
meetingEnded = value;
meetingEndedOnce = value && behavior?.once === true;
},
failNextTabLists(count = 1) {
tabListFailures = count;
},
setTargetId(value: string) {
targetId = value;
},
setTabUrl(value: string) {
tabUrl = value;
},
};
}
describe("Zoom meeting session flow", () => {
it("joins, reuses, reports, snapshots, speaks safely, and leaves through core", async () => {
const harness = runtimeHarness();
const runtime = new ZoomMeetingsRuntime({
config: resolveZoomMeetingsConfig({
defaultMode: "transcribe",
chrome: { waitForInCallMs: 1 },
}),
fullConfig: {},
runtime: harness.runtime,
logger,
});
const first = await runtime.join({ url: URL, mode: "transcribe" });
expect(first.session.chrome?.health).toMatchObject({ inCall: true, cameraOff: true });
const reused = await runtime.join({
url: `${URL.split("?")[0]}?context=%7b%22Tid%22%3a%22two%22%7d`,
mode: "transcribe",
});
expect(reused.session.id).toBe(first.session.id);
expect(runtime.list()).toHaveLength(1);
expect(await runtime.status(first.session.id)).toMatchObject({
found: true,
session: { id: first.session.id },
});
const transcriptStartCall = harness.gatewayRequest.mock.calls.length;
expect(await runtime.transcript(first.session.id)).toMatchObject({
found: true,
lines: [],
nextIndex: 0,
});
const transcriptActScripts = harness.gatewayRequest.mock.calls
.slice(transcriptStartCall)
.filter(([, params]) => params.path === "/act")
.map(([, params]) => {
const fn = (params.body as { fn?: unknown } | undefined)?.fn;
return typeof fn === "string" ? fn : "";
});
expect(transcriptActScripts).toHaveLength(2);
expect(transcriptActScripts[0]).toContain("const allowSessionAdoption = false");
expect(transcriptActScripts[0]).toContain("const captureCaptions = true");
expect(transcriptActScripts[1]).toContain("expectedSessionId");
expect(await runtime.speak(first.session.id, "hello")).toMatchObject({
found: true,
spoken: false,
});
Object.assign(first.session.chrome?.health ?? {}, {
audioInputActive: true,
audioInputRouted: true,
audioOutputActive: true,
audioOutputRouted: true,
captioning: true,
providerConnected: true,
realtimeReady: true,
});
expect(await runtime.leave(first.session.id)).toMatchObject({
found: true,
browserLeft: true,
session: {
state: "ended",
chrome: {
health: {
audioInputActive: false,
audioInputRouted: false,
audioOutputActive: false,
audioOutputRouted: false,
captioning: false,
inCall: false,
manualActionRequired: false,
providerConnected: false,
realtimeReady: false,
},
},
},
});
expect(harness.gatewayRequest).toHaveBeenCalledWith(
"browser.request",
expect.objectContaining({ path: "/tabs/open" }),
expect.objectContaining({ scopes: ["operator.admin"] }),
);
});
it("adopts the in-call page statefully after host admission", async () => {
const harness = runtimeHarness({ inCall: false, pendingReason: "admission" });
const runtime = new ZoomMeetingsRuntime({
config: resolveZoomMeetingsConfig({
defaultMode: "transcribe",
chrome: { waitForInCallMs: 1 },
}),
fullConfig: {},
runtime: harness.runtime,
logger,
});
const joined = await runtime.join({ url: URL, mode: "transcribe" });
harness.setInCall(true);
harness.gatewayRequest.mockClear();
const status = await runtime.status(joined.session.id);
const statusScript = harness.gatewayRequest.mock.calls
.filter(([, params]) => params.path === "/act")
.map(([, params]) => (params.body as { fn?: unknown } | undefined)?.fn)
.find((fn): fn is string => typeof fn === "string" && fn.includes("const readOnly"));
expect(statusScript).toContain("const readOnly = false");
expect(status.session?.chrome?.health).toMatchObject({ inCall: true });
});
it("reads an archived transcript without reclaiming a newer live tab owner", async () => {
const harness = runtimeHarness();
const runtime = new ZoomMeetingsRuntime({
config: resolveZoomMeetingsConfig({
defaultMode: "transcribe",
chrome: { waitForInCallMs: 1 },
}),
fullConfig: {},
runtime: harness.runtime,
logger,
});
const joined = await runtime.join({ url: URL, mode: "transcribe" });
harness.setSessionConflict(true);
harness.gatewayRequest.mockClear();
expect(await runtime.transcript(joined.session.id)).toMatchObject({
found: true,
lines: [{ text: "Archived caption" }],
});
const actScripts = harness.gatewayRequest.mock.calls
.filter(([, params]) => params.path === "/act")
.map(([, params]) => {
const fn = (params.body as { fn?: unknown } | undefined)?.fn;
return typeof fn === "string" ? fn : "";
});
expect(actScripts).toHaveLength(2);
expect(actScripts[0]).toContain("const allowSessionAdoption = false");
expect(actScripts[1]).toContain("expectedSessionId");
});
it("recovers and leaves a manually opened tab when Chrome launching is disabled", async () => {
const harness = runtimeHarness({ tabOpen: true });
const runtime = new ZoomMeetingsRuntime({
config: resolveZoomMeetingsConfig({
defaultMode: "transcribe",
chrome: { launch: false, waitForInCallMs: 1 },
}),
fullConfig: {},
runtime: harness.runtime,
logger,
});
const joined = await runtime.join({ url: URL, mode: "transcribe" });
expect(joined.session.chrome).toMatchObject({
browserTab: { openedByPlugin: false, targetId: "zoom-tab" },
launched: false,
});
expect(harness.gatewayRequest).not.toHaveBeenCalledWith(
"browser.request",
expect.objectContaining({ path: "/tabs/open" }),
expect.anything(),
);
expect(await runtime.leave(joined.session.id)).toMatchObject({
browserLeft: true,
session: { state: "ended" },
});
});
it("refreshes a recovered browser tab target", async () => {
const harness = runtimeHarness();
const runtime = new ZoomMeetingsRuntime({
config: resolveZoomMeetingsConfig({
defaultMode: "transcribe",
chrome: { waitForInCallMs: 1 },
}),
fullConfig: {},
runtime: harness.runtime,
logger,
});
const joined = await runtime.join({ url: URL, mode: "transcribe" });
harness.setTargetId("zoom-tab-replaced");
await runtime.status(joined.session.id);
expect(joined.session.chrome?.browserTab).toEqual({
openedByPlugin: false,
targetId: "zoom-tab-replaced",
});
harness.setTargetId("zoom-tab-replaced-again");
harness.gatewayRequest.mockClear();
await runtime.transcript(joined.session.id);
expect(joined.session.chrome?.browserTab).toEqual({
openedByPlugin: false,
targetId: "zoom-tab-replaced-again",
});
const transcriptRead = harness.gatewayRequest.mock.calls.find(([, params]) => {
const fn = (params.body as { fn?: unknown } | undefined)?.fn;
return params.path === "/act" && typeof fn === "string" && fn.includes("expectedSessionId");
});
expect(transcriptRead?.[1]).toMatchObject({
body: { targetId: "zoom-tab-replaced-again" },
});
});
it("recovers the tracked tab after Zoom rewrites the in-call URL", async () => {
const harness = runtimeHarness();
const runtime = new ZoomMeetingsRuntime({
config: resolveZoomMeetingsConfig({
defaultMode: "transcribe",
chrome: { waitForInCallMs: 1 },
}),
fullConfig: {},
runtime: harness.runtime,
logger,
});
const joined = await runtime.join({ url: URL, mode: "transcribe" });
harness.setTabUrl("https://zoom.us/");
harness.gatewayRequest.mockClear();
const status = await runtime.status(joined.session.id);
expect(status.session?.chrome?.health?.browserUrl).toBe("https://zoom.us/");
expect(harness.gatewayRequest).toHaveBeenCalledWith(
"browser.request",
expect.objectContaining({
path: "/act",
body: expect.objectContaining({ targetId: "zoom-tab" }),
}),
expect.objectContaining({ scopes: ["operator.admin"] }),
);
});
it("ends the session when the tracked Zoom tab disappears", async () => {
const harness = runtimeHarness();
const runtime = new ZoomMeetingsRuntime({
config: resolveZoomMeetingsConfig({
defaultMode: "transcribe",
chrome: { waitForInCallMs: 1 },
}),
fullConfig: {},
runtime: harness.runtime,
logger,
});
const joined = await runtime.join({ url: URL, mode: "transcribe" });
harness.closeTab();
const status = await runtime.status(joined.session.id);
expect(status.session).toMatchObject({
browserLeft: true,
state: "ended",
chrome: {
browserTab: undefined,
health: {
inCall: false,
manualActionReason: undefined,
manualActionRequired: false,
status: "browser-tab-missing",
},
},
});
});
it("opens a new session instead of reusing one whose Zoom tab disappeared", async () => {
const harness = runtimeHarness();
const runtime = new ZoomMeetingsRuntime({
config: resolveZoomMeetingsConfig({
defaultMode: "transcribe",
chrome: { waitForInCallMs: 1 },
}),
fullConfig: {},
runtime: harness.runtime,
logger,
});
const first = await runtime.join({ url: URL, mode: "transcribe" });
harness.closeTab();
const replacement = await runtime.join({ url: URL, mode: "transcribe" });
expect(first.session.state).toBe("ended");
expect(replacement.session.id).not.toBe(first.session.id);
expect(
harness.gatewayRequest.mock.calls.filter(([, params]) => params.path === "/tabs/open"),
).toHaveLength(2);
});
it("opens a new session when browser verification of a reusable tab fails", async () => {
const harness = runtimeHarness();
const runtime = new ZoomMeetingsRuntime({
config: resolveZoomMeetingsConfig({ defaultMode: "transcribe" }),
fullConfig: {},
runtime: harness.runtime,
logger,
});
const first = await runtime.join({ url: URL, mode: "transcribe" });
harness.failNextTabLists();
const replacement = await runtime.join({ url: URL, mode: "transcribe" });
expect(first.session.state).toBe("ended");
expect(replacement.session.id).not.toBe(first.session.id);
});
it("replaces a reusable session whose realtime bridge closed", async () => {
const harness = runtimeHarness();
const runtime = new ZoomMeetingsRuntime({
config: resolveZoomMeetingsConfig({
defaultMode: "transcribe",
chrome: { waitForInCallMs: 1 },
}),
fullConfig: {},
runtime: harness.runtime,
logger,
});
const first = await runtime.join({ url: URL, mode: "transcribe" });
Object.assign(first.session.chrome?.health ?? {}, { bridgeClosed: true });
const replacement = await runtime.join({ url: URL, mode: "transcribe" });
expect(first.session.state).toBe("ended");
expect(replacement.session.id).not.toBe(first.session.id);
expect(
harness.gatewayRequest.mock.calls.filter(([, params]) => params.path === "/tabs/open"),
).toHaveLength(2);
});
it("closes a host-ended tab before opening its replacement", async () => {
const harness = runtimeHarness();
const runtime = new ZoomMeetingsRuntime({
config: resolveZoomMeetingsConfig({
defaultMode: "transcribe",
chrome: { waitForInCallMs: 1 },
}),
fullConfig: {},
runtime: harness.runtime,
logger,
});
const first = await runtime.join({ url: URL, mode: "transcribe" });
harness.setInCall(false);
harness.setMeetingEnded(true, { once: true });
const replacement = await runtime.join({ url: URL, mode: "transcribe" });
expect(first.session.state).toBe("ended");
expect(replacement.session.id).not.toBe(first.session.id);
expect(
harness.gatewayRequest.mock.calls.filter(([, params]) => params.path === "/tabs/open"),
).toHaveLength(2);
});
it("rejects and closes the tab when the initial browser status is host-ended", async () => {
const harness = runtimeHarness({ inCall: false, meetingEnded: true });
const runtime = new ZoomMeetingsRuntime({
config: resolveZoomMeetingsConfig({
defaultMode: "transcribe",
chrome: { waitForInCallMs: 1 },
}),
fullConfig: {},
runtime: harness.runtime,
logger,
});
await expect(runtime.join({ url: URL, mode: "transcribe" })).rejects.toThrow(
"The Zoom meeting has already ended.",
);
expect(runtime.list()).toEqual([]);
expect(
harness.gatewayRequest.mock.calls.filter(
([, params]) => params.method === "DELETE" && params.path === "/tabs/zoom-tab",
),
).toHaveLength(1);
});
it("ends the active session when browser status confirms the host ended it", async () => {
const harness = runtimeHarness();
const runtime = new ZoomMeetingsRuntime({
config: resolveZoomMeetingsConfig({
defaultMode: "transcribe",
chrome: { waitForInCallMs: 1 },
}),
fullConfig: {},
runtime: harness.runtime,
logger,
});
const joined = await runtime.join({ url: URL, mode: "transcribe" });
harness.setInCall(false);
harness.setMeetingEnded(true);
const status = await runtime.status(joined.session.id);
expect(status.session).toMatchObject({
browserLeft: true,
chrome: { health: { inCall: false, meetingEnded: true } },
state: "ended",
});
});
it("restarts a failed join when the corrected invite changes the passcode", async () => {
const harness = runtimeHarness({ inCall: false });
const runtime = new ZoomMeetingsRuntime({
config: resolveZoomMeetingsConfig({
defaultMode: "transcribe",
chrome: { waitForInCallMs: 1 },
}),
fullConfig: {},
runtime: harness.runtime,
logger,
});
const first = await runtime.join({
mode: "transcribe",
url: "https://zoom.us/j/12345678904?pwd=old",
});
const corrected = await runtime.join({
mode: "transcribe",
url: "https://zoom.us/j/12345678904?pwd=correct",
});
expect(corrected.session.id).not.toBe(first.session.id);
expect(first.session.state).toBe("ended");
expect(
harness.gatewayRequest.mock.calls.filter(([, params]) => params.path === "/tabs/open"),
).toHaveLength(2);
});
it("serializes concurrent corrected passcodes under the meeting join lock", async () => {
const harness = runtimeHarness({ inCall: false });
const runtime = new ZoomMeetingsRuntime({
config: resolveZoomMeetingsConfig({
defaultMode: "transcribe",
chrome: { waitForInCallMs: 1 },
}),
fullConfig: {},
runtime: harness.runtime,
logger,
});
const first = await runtime.join({
mode: "transcribe",
url: "https://zoom.us/j/12345678904?pwd=old",
});
const [second, third] = await Promise.all([
runtime.join({
mode: "transcribe",
url: "https://zoom.us/j/12345678904?pwd=correct-one",
}),
runtime.join({
mode: "transcribe",
url: "https://zoom.us/j/12345678904?pwd=correct-two",
}),
]);
expect(first.session.state).toBe("ended");
expect(second.session.state).toBe("ended");
expect(third.session.state).toBe("active");
expect(new Set([first.session.id, second.session.id, third.session.id]).size).toBe(3);
expect(
harness.gatewayRequest.mock.calls.filter(([, params]) => params.path === "/tabs/open"),
).toHaveLength(3);
});
it("serializes cross-agent reassignment through the core join owner", async () => {
const harness = runtimeHarness({ inCall: false });
const runtime = new ZoomMeetingsRuntime({
config: resolveZoomMeetingsConfig({
defaultMode: "transcribe",
chrome: { waitForInCallMs: 1 },
}),
fullConfig: {},
runtime: harness.runtime,
logger,
});
const first = await runtime.join({
agentId: "support",
mode: "transcribe",
url: "https://zoom.us/j/12345678904?pwd=old",
});
const replacement = await runtime.join({
agentId: "main",
mode: "transcribe",
url: "https://zoom.us/j/12345678904?pwd=correct",
});
expect(first.session.state).toBe("ended");
expect(replacement.session.agentId).toBe("main");
expect(replacement.session.id).not.toBe(first.session.id);
expect(
harness.gatewayRequest.mock.calls.filter(([, params]) => params.path === "/tabs/open"),
).toHaveLength(2);
});
});
+629
View File
@@ -0,0 +1,629 @@
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import {
MeetingSessionRuntime,
type MeetingSessionRuntimeHandles,
type MeetingSessionRuntimeJoinContext,
} from "openclaw/plugin-sdk/meeting-runtime";
import type { PluginRuntime, RuntimeLogger } from "openclaw/plugin-sdk/plugin-runtime";
import { normalizeAgentId } from "openclaw/plugin-sdk/routing";
import type { ZoomMeetingsConfig, ZoomMeetingsMode, ZoomMeetingsTransport } from "./config.js";
import {
testZoomMeetingListening,
testZoomMeetingSpeech,
type ZoomMeetingsProbeContext,
} from "./runtime-probes.js";
import { createZoomMeetingsSession } from "./runtime-session.js";
import { getZoomMeetingsSetupStatus } from "./runtime-setup.js";
import {
launchZoomMeetingInChrome,
launchZoomMeetingOnNode,
leaveZoomMeetingInBrowser,
readZoomMeetingTranscript,
recoverCurrentZoomMeetingTab,
} from "./transports/chrome.js";
import type {
ZoomMeetingsBrowserTab,
ZoomMeetingsChromeHealth,
ZoomMeetingsJoinRequest,
ZoomMeetingsJoinResult,
ZoomMeetingsSession,
} from "./transports/types.js";
import {
ZOOM_MEETINGS_PLATFORM_ADAPTER,
isZoomMeetingsRealtimeRouteReady,
isZoomMeetingsTalkBackMode,
} from "./transports/zoom-meetings-platform-adapter.js";
import { hasSameZoomMeetingJoinCredential } from "./transports/zoom-meetings-urls.js";
type ManualActionReason = NonNullable<ZoomMeetingsChromeHealth["manualActionReason"]>;
type SpeechBlockedReason = NonNullable<ZoomMeetingsChromeHealth["speechBlockedReason"]>;
type SessionRuntime = MeetingSessionRuntime<
ZoomMeetingsSession,
ZoomMeetingsJoinRequest,
ZoomMeetingsTransport,
ZoomMeetingsMode,
ZoomMeetingsChromeHealth,
ZoomMeetingsBrowserTab,
ManualActionReason,
SpeechBlockedReason
>;
type JoinContext = MeetingSessionRuntimeJoinContext<
ZoomMeetingsSession,
ZoomMeetingsTransport,
ZoomMeetingsMode,
ZoomMeetingsChromeHealth,
ZoomMeetingsBrowserTab
>;
type LaunchResult =
| Awaited<ReturnType<typeof launchZoomMeetingInChrome>>
| Awaited<ReturnType<typeof launchZoomMeetingOnNode>>;
type AudioBridge = NonNullable<LaunchResult["audioBridge"]>;
function nowIso(): string {
return new Date().toISOString();
}
function resolveTransport(
request: ZoomMeetingsJoinRequest,
config: ZoomMeetingsConfig,
): ZoomMeetingsTransport {
return request.transport ?? (config.chromeNode.node ? "chrome-node" : "chrome");
}
function withSessionAgentConfig(config: ZoomMeetingsConfig, agentId: string): ZoomMeetingsConfig {
const consultAgentId = config.realtime.agentId ?? agentId;
return config.realtime.agentId === consultAgentId
? config
: { ...config, realtime: { ...config.realtime, agentId: consultAgentId } };
}
function noteSession(session: ZoomMeetingsSession, note: string): void {
session.notes = [...session.notes.filter((item) => item !== note), note];
}
function isAwaitingAdmission(session: ZoomMeetingsSession): boolean {
return (
session.chrome?.health?.lobbyWaiting === true ||
session.chrome?.health?.manualActionReason === "zoom-admission-required"
);
}
export class ZoomMeetingsRuntime {
readonly #sessions: SessionRuntime;
readonly #requesterSessionKeys = new Map<string, string>();
constructor(
private readonly params: {
config: ZoomMeetingsConfig;
fullConfig: OpenClawConfig;
runtime: PluginRuntime;
logger: RuntimeLogger;
},
) {
this.#sessions = new MeetingSessionRuntime({
logger: params.logger,
logScope: ZOOM_MEETINGS_PLATFORM_ADAPTER.logScope,
formatError: formatErrorMessage,
reuseExistingBrowserTab: params.config.chrome.reuseExistingTab,
waitForInCallMs: params.config.chrome.waitForInCallMs,
joinTimeoutMs: params.config.chrome.joinTimeoutMs,
defaultSpeechInstructions: params.config.realtime.introMessage,
transientSpeechBlockedReasons: new Set<SpeechBlockedReason>([
"not-in-call",
"browser-unverified",
"zoom-microphone-muted",
]),
messages: {
previousBrowserLeaveFailed:
"Could not leave the previous Zoom meeting tab before reassignment.",
reassignedSessionNote:
"Ended before the same Zoom meeting tab was reassigned to another agent.",
reusedSessionNote: "Reused existing active Zoom meeting session.",
replacementBrowserLeaveFailed:
"Could not leave the previous Zoom meeting tab before reassignment.",
speechBlockedFallback: "Realtime speech blocked until Zoom is ready.",
speech: {
audioBridgeUnavailable: "Realtime speech requires an active Chrome audio bridge.",
browserUnverified: "Zoom browser state has not been verified yet.",
manualActionFallback: "Resolve the Zoom browser prompt before asking OpenClaw to speak.",
microphoneMuted: "Turn on the OpenClaw Zoom microphone before asking OpenClaw to speak.",
microphoneMutedReason: "zoom-microphone-muted",
notInCall: "Zoom has not reported that the browser guest is in the call.",
notInCallReason: "not-in-call",
browserUnverifiedReason: "browser-unverified",
audioBridgeUnavailableReason: "audio-bridge-unavailable",
},
},
resolveJoin: (request) => ({
url: ZOOM_MEETINGS_PLATFORM_ADAPTER.urls.validateAndNormalize(request.url),
transport: resolveTransport(request, params.config),
mode: request.mode ?? params.config.defaultMode,
agentId: normalizeAgentId(request.agentId),
}),
createSession: ({ request, resolved, createdAt }) => {
const session = createZoomMeetingsSession({ config: params.config, resolved, createdAt });
if (request.requesterSessionKey) {
this.#requesterSessionKeys.set(session.id, request.requesterSessionKey);
}
return session;
},
resolveSpeechInstructions: (request) =>
request.message ?? params.config.realtime.introMessage,
isBrowserTransport: () => true,
isTalkBackMode: isZoomMeetingsTalkBackMode,
isTranscribeMode: (mode) => mode === "transcribe",
sameMeetingUrl: (left, right) =>
ZOOM_MEETINGS_PLATFORM_ADAPTER.urls.isSameMeeting(left, right),
normalizeMeetingUrlForReuse: (url) =>
ZOOM_MEETINGS_PLATFORM_ADAPTER.urls.normalizeForReuse(url),
getBrowser: (session) =>
session.chrome
? {
launched: session.chrome.launched,
nodeId: session.chrome.nodeId,
tab: session.chrome.browserTab,
health: session.chrome.health,
hasAudioBridge: Boolean(
session.chrome.audioBridge && session.chrome.health?.bridgeClosed !== true,
),
}
: undefined,
setBrowserTab: (session, tab) => {
if (session.chrome) {
session.chrome.browserTab = tab;
}
},
setBrowserHealth: (session, health) => {
if (session.chrome) {
session.chrome.health = health;
}
},
joinTransport: async ({ request, session, context }) =>
await this.#joinTransport(request, session, context),
releaseBrowserTab: async (session) => await this.#releaseBrowserTab(session),
refreshBrowserHealth: async (session, options) =>
await this.#refreshBrowserHealth(session, options),
refreshStatus: async (session) => {
await this.#sessions.refreshBrowserHealth(session, {
force: true,
readOnly: !isAwaitingAdmission(session),
});
const confirmedTabMissing = session.chrome?.health?.status === "browser-tab-missing";
if (session.state === "active" && confirmedTabMissing) {
session.browserLeft = true;
await this.#sessions.leave(session.id, { keepBrowserTab: true });
this.#requesterSessionKeys.delete(session.id);
} else if (session.state === "active" && session.chrome?.health?.meetingEnded === true) {
await this.leave(session.id);
}
},
refreshReusableSession: async (session, request) => {
await this.#sessions.refreshBrowserHealth(session, {
force: true,
readOnly: false,
});
const browser = session.chrome;
const health = browser?.health;
const staleSession =
!browser?.browserTab ||
health?.meetingEnded === true ||
health?.manualActionReason === "zoom-session-conflict" ||
health?.manualActionReason === "browser-control-unavailable" ||
health?.bridgeClosed === true;
const replacePendingJoin =
health?.inCall !== true &&
health?.manualActionReason === "zoom-passcode-required" &&
!hasSameZoomMeetingJoinCredential(session.url, request.url);
if (staleSession || replacePendingJoin) {
session.state = "ended";
session.updatedAt = nowIso();
noteSession(
session,
replacePendingJoin
? "Ended pending Zoom session after receiving a corrected meeting credential."
: "Ended stale Zoom session before opening a replacement.",
);
this.#requesterSessionKeys.delete(session.id);
return {
keepBrowserTab:
!replacePendingJoin && health?.meetingEnded !== true && health?.bridgeClosed !== true,
};
}
return undefined;
},
ensureRealtimeBridge: async (session) => await this.#ensureRealtimeBridge(session),
captureTranscript: async (session, options) =>
await this.#captureTranscript(session, options),
speakViaTransport: async () => undefined,
});
}
list(): ZoomMeetingsSession[] {
return this.#sessions.list();
}
ownsSession(agentId: string, sessionId: string): boolean {
return this.list().some((session) => session.id === sessionId && session.agentId === agentId);
}
async join(request: ZoomMeetingsJoinRequest): Promise<ZoomMeetingsJoinResult> {
try {
const url = ZOOM_MEETINGS_PLATFORM_ADAPTER.urls.validateAndNormalize(request.url);
const agentId = normalizeAgentId(request.agentId);
return await this.#sessions.join({ ...request, agentId, url });
} catch (error) {
const activeIds = new Set(this.list().map((session) => session.id));
for (const sessionId of this.#requesterSessionKeys.keys()) {
if (!activeIds.has(sessionId)) {
this.#requesterSessionKeys.delete(sessionId);
}
}
throw error;
}
}
async leave(sessionId: string) {
try {
return await this.#sessions.leave(sessionId);
} finally {
this.#requesterSessionKeys.delete(sessionId);
}
}
async status(sessionId?: string) {
return await this.#sessions.status(sessionId);
}
async statusForAgent(agentId: string, sessionId?: string) {
if (sessionId) {
return this.ownsSession(agentId, sessionId)
? await this.#sessions.status(sessionId)
: { found: false };
}
const sessions = this.list().filter((session) => session.agentId === agentId);
await Promise.all(sessions.map((session) => this.#sessions.status(session.id)));
return { found: true, sessions };
}
async transcript(sessionId: string, options: { sinceIndex?: number } = {}) {
return await this.#sessions.transcript(sessionId, options);
}
async speak(sessionId: string, instructions?: string) {
return await this.#sessions.speak(sessionId, instructions);
}
async setupStatus(options?: { mode?: ZoomMeetingsMode; transport?: ZoomMeetingsTransport }) {
return await getZoomMeetingsSetupStatus({
config: this.params.config,
fullConfig: this.params.fullConfig,
runtime: this.params.runtime,
options,
});
}
async testSpeech(request: ZoomMeetingsJoinRequest) {
return await testZoomMeetingSpeech(this.#probeContext(), request);
}
async testListen(request: ZoomMeetingsJoinRequest) {
return await testZoomMeetingListening(this.#probeContext(), request);
}
#probeContext(): ZoomMeetingsProbeContext {
return {
config: this.params.config,
resolveAgentId: (request) => normalizeAgentId(request.agentId),
list: () => this.list(),
join: async (request) => await this.join(request),
isReusable: (session, resolved) => this.#sessions.isReusableSession(session, resolved),
hasHealthHandle: (sessionId) => this.#sessions.hasHealthHandle(sessionId),
refreshHealth: (sessionId) => this.#sessions.refreshHealth(sessionId),
refreshCaptionHealth: async (session, timeoutMs) =>
await this.#refreshBrowserHealth(session, { timeoutMs }),
};
}
async #joinTransport(
request: ZoomMeetingsJoinRequest,
session: ZoomMeetingsSession,
context: JoinContext,
): Promise<{ delegatedSpoken?: boolean }> {
const config = withSessionAgentConfig(this.params.config, session.agentId);
const result: LaunchResult =
session.transport === "chrome-node"
? await launchZoomMeetingOnNode({
runtime: this.params.runtime,
config,
fullConfig: this.params.fullConfig,
meetingSessionId: session.id,
requesterSessionKey: request.requesterSessionKey,
mode: session.mode,
url: session.url,
logger: this.params.logger,
})
: await launchZoomMeetingInChrome({
runtime: this.params.runtime,
config,
fullConfig: this.params.fullConfig,
meetingSessionId: session.id,
requesterSessionKey: request.requesterSessionKey,
mode: session.mode,
url: session.url,
logger: this.params.logger,
});
const nodeId = "nodeId" in result ? result.nodeId : undefined;
const tab = context.inheritedBrowserTab({
session,
transport: session.transport,
nodeId,
meetingUrl: session.url,
tab: result.tab,
});
session.chrome = {
audioBackend: "blackhole-2ch",
launched: result.launched,
nodeId,
browserProfile: this.params.config.chrome.browserProfile,
browserTab: tab,
health: result.browser,
};
if (result.browser?.meetingEnded === true) {
throw new Error("The Zoom meeting has already ended.");
}
const handles = this.#attachAudioBridge(session, result.audioBridge);
if (handles) {
context.attachRuntimeHandles(session, handles);
}
session.notes.push(
result.audioBridge
? session.transport === "chrome-node"
? "Zoom guest joined in Chrome on the selected node with realtime audio through the node bridge."
: "Zoom guest joined in local Chrome with realtime audio through BlackHole 2ch and SoX."
: session.mode === "transcribe"
? "Zoom guest joined observe-only with live-caption transcript capture."
: "Zoom guest join is waiting for the browser to become ready before starting realtime audio.",
);
this.#sessions.refreshSpeechReadiness(session);
return {};
}
#attachAudioBridge(
session: ZoomMeetingsSession,
audioBridge: AudioBridge | undefined,
): MeetingSessionRuntimeHandles<ZoomMeetingsChromeHealth> | undefined {
if (!session.chrome || !audioBridge) {
return undefined;
}
session.chrome.audioBridge = {
type: audioBridge.type,
provider: audioBridge.providerId,
};
session.chrome.health = { ...session.chrome.health, bridgeClosed: false };
return {
stop: audioBridge.stop,
speak: audioBridge.speak,
getHealth: audioBridge.getHealth,
};
}
async #ensureRealtimeBridge(
session: ZoomMeetingsSession,
): Promise<MeetingSessionRuntimeHandles<ZoomMeetingsChromeHealth> | undefined> {
const bridgeClosed = session.chrome?.health?.bridgeClosed === true;
if (
!isZoomMeetingsTalkBackMode(session.mode) ||
session.state !== "active" ||
!session.chrome ||
(session.chrome.audioBridge && !bridgeClosed) ||
!isZoomMeetingsRealtimeRouteReady(session.mode, session.chrome.health)
) {
return undefined;
}
if (bridgeClosed) {
session.chrome.audioBridge = undefined;
}
const config = withSessionAgentConfig(this.params.config, session.agentId);
const recoveryConfig = {
...config,
chrome: { ...config.chrome, launch: false },
chromeNode: { node: session.chrome.nodeId ?? config.chromeNode.node },
};
const result =
session.transport === "chrome-node"
? await launchZoomMeetingOnNode({
runtime: this.params.runtime,
config: recoveryConfig,
fullConfig: this.params.fullConfig,
meetingSessionId: session.id,
requesterSessionKey: this.#requesterSessionKeys.get(session.id),
mode: session.mode,
trackedTargetId: session.chrome.browserTab?.targetId,
url: session.url,
logger: this.params.logger,
})
: await launchZoomMeetingInChrome({
runtime: this.params.runtime,
config: recoveryConfig,
fullConfig: this.params.fullConfig,
meetingSessionId: session.id,
requesterSessionKey: this.#requesterSessionKeys.get(session.id),
mode: session.mode,
trackedTargetId: session.chrome.browserTab?.targetId,
url: session.url,
logger: this.params.logger,
});
if (result.tab) {
const currentTab = session.chrome.browserTab;
session.chrome.browserTab = {
...result.tab,
openedByPlugin:
result.tab.targetId === currentTab?.targetId
? currentTab.openedByPlugin
: result.tab.openedByPlugin,
};
}
if (result.browser) {
session.chrome.health = { ...session.chrome.health, ...result.browser };
}
session.updatedAt = nowIso();
return this.#attachAudioBridge(session, result.audioBridge);
}
async #refreshBrowserHealth(
session: ZoomMeetingsSession,
options: { readOnly?: boolean; timeoutMs?: number } = {},
): Promise<void> {
try {
const result = await recoverCurrentZoomMeetingTab({
runtime: this.params.runtime,
config: this.params.config,
meetingSessionId: session.id,
mode: session.mode,
nodeId: session.chrome?.nodeId,
readOnly: options.readOnly,
trackedMeetingUrl: session.url,
trackedTargetId: session.chrome?.browserTab?.targetId,
transport: session.transport,
timeoutMs: options.timeoutMs,
url: session.url,
});
if (result.found && session.chrome) {
if (result.tab?.targetId) {
const currentTab = session.chrome.browserTab;
session.chrome.browserTab = {
targetId: result.tab.targetId,
openedByPlugin:
result.tab.targetId === currentTab?.targetId ? currentTab.openedByPlugin : false,
};
}
if (result.browser) {
session.chrome.health = { ...session.chrome.health, ...result.browser };
}
session.updatedAt = nowIso();
} else if (session.chrome) {
session.chrome.browserTab = undefined;
session.browserLeft = true;
session.chrome.health = {
...session.chrome.health,
inCall: false,
micMuted: undefined,
captioning: false,
audioInputRouted: false,
audioOutputRouted: false,
manualActionRequired: true,
manualActionReason: "browser-control-unavailable",
manualActionMessage: result.message,
status: "browser-tab-missing",
notes: [
...(session.chrome.health?.notes ?? []).filter((note) => note !== result.message),
result.message,
],
};
session.updatedAt = nowIso();
}
} catch (error) {
const message = `Zoom browser readiness refresh failed: ${formatErrorMessage(error)}`;
this.params.logger.debug?.(`${ZOOM_MEETINGS_PLATFORM_ADAPTER.logScope} ${message}`);
if (session.chrome) {
session.chrome.health = {
...session.chrome.health,
inCall: false,
micMuted: undefined,
captioning: false,
audioInputRouted: false,
audioOutputRouted: false,
manualActionRequired: true,
manualActionReason: "browser-control-unavailable",
manualActionMessage: message,
status: "browser-control",
notes: [
...(session.chrome.health?.notes ?? []).filter((note) => note !== message),
message,
],
};
session.updatedAt = nowIso();
}
}
}
async #captureTranscript(session: ZoomMeetingsSession, options: { finalize?: boolean } = {}) {
// Recovery permits caption setup but atomically refuses a different live
// session owner, so stale sessions read their archived page buffer instead.
await this.#sessions.refreshCaptionHealth(session);
const tab = session.chrome?.browserTab;
if (!tab) {
return undefined;
}
return await readZoomMeetingTranscript({
runtime: this.params.runtime,
config: this.params.config,
finalize: options.finalize,
meetingUrl: session.url,
meetingSessionId: session.id,
nodeId: session.chrome?.nodeId,
tab,
});
}
async #releaseBrowserTab(session: ZoomMeetingsSession): Promise<boolean | undefined> {
const tab = session.chrome?.browserTab;
if (!tab) {
noteSession(
session,
"No tracked Zoom meeting tab; leave the browser meeting manually if it is still active.",
);
session.browserLeft = false;
return false;
}
const shared = this.list().some(
(other) =>
other.id !== session.id &&
other.state === "active" &&
other.chrome?.browserTab?.targetId === tab.targetId &&
other.chrome?.nodeId === session.chrome?.nodeId,
);
if (shared) {
noteSession(session, "Kept the shared Zoom meeting tab open for another active session.");
return undefined;
}
try {
const result = await leaveZoomMeetingInBrowser({
runtime: this.params.runtime,
config: this.params.config,
meetingSessionId: session.id,
meetingUrl: session.url,
nodeId: session.chrome?.nodeId,
tab,
});
noteSession(session, result.note);
if (result.left && session.chrome) {
session.chrome.browserTab = undefined;
if (session.chrome.health) {
// MeetingSessionRuntime owns the canonical in-call/manual reset after this
// release reports success; this plugin clears only Zoom-specific health.
session.chrome.health = {
...session.chrome.health,
captioning: false,
audioInputRouted: false,
audioOutputRouted: false,
providerConnected: false,
realtimeReady: false,
audioInputActive: false,
audioOutputActive: false,
};
}
}
session.browserLeft = result.left;
return result.left;
} catch (error) {
noteSession(
session,
`Browser control could not leave the Zoom meeting tab: ${formatErrorMessage(error)}`,
);
session.browserLeft = false;
return false;
}
}
}
@@ -0,0 +1,5 @@
export const ZOOM_MEETINGS_SYSTEM_PROFILER_COMMAND = "/usr/sbin/system_profiler";
export function outputMentionsBlackHole2ch(output: string): boolean {
return /\bBlackHole\s+2ch\b/i.test(output);
}
@@ -0,0 +1,295 @@
import type { PluginRuntime } from "openclaw/plugin-sdk/plugin-runtime";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { resolveZoomMeetingsConfig } from "../config.js";
const engineMocks = vi.hoisted(() => ({
localDispose: vi.fn(async () => {}),
nodeDispose: vi.fn(async () => {}),
startAgent: vi.fn(),
}));
vi.mock("openclaw/plugin-sdk/meeting-runtime", async (importOriginal) => {
const original = await importOriginal<typeof import("openclaw/plugin-sdk/meeting-runtime")>();
const transport = (dispose: () => Promise<void>) => ({
clearOutput: vi.fn(async () => {}),
dispose,
onFatal: vi.fn(),
startInput: vi.fn(),
stop: dispose,
writeOutput: vi.fn(async () => {}),
});
return {
...original,
createLocalMeetingRealtimeAudioTransport: () => transport(engineMocks.localDispose),
createNodeMeetingRealtimeAudioTransport: () => transport(engineMocks.nodeDispose),
startMeetingAgentRealtimeEngine: engineMocks.startAgent,
};
});
import { launchZoomMeetingInChrome, launchZoomMeetingOnNode } from "./chrome.js";
const URL = "https://zoom.us/j/12345678905?pwd=rollback";
const logger = {
debug: vi.fn(),
error: vi.fn(),
info: vi.fn(),
warn: vi.fn(),
};
function browserResult(params: Record<string, unknown>, state: { tabOpen: boolean }) {
if (params.path === "/tabs") {
return {
tabs: state.tabOpen ? [{ targetId: "zoom-tab", title: "Zoom", url: URL }] : [],
};
}
if (params.path === "/tabs/open") {
state.tabOpen = true;
return { targetId: "zoom-tab", title: "Zoom", url: URL };
}
if (params.path === "/tabs/focus") {
return { ok: true };
}
if (params.path === "/act") {
const scriptValue = (params.body as { fn?: unknown } | undefined)?.fn;
const script = typeof scriptValue === "string" ? scriptValue : "";
return script.includes("leaveAction")
? { result: JSON.stringify({ departed: true, urlMatched: true }) }
: {
result: JSON.stringify({
audioInputRouted: true,
audioOutputRouted: true,
inCall: true,
micMuted: false,
url: URL,
}),
};
}
if (params.method === "DELETE" && params.path === "/tabs/zoom-tab") {
state.tabOpen = false;
return { ok: true };
}
throw new Error(
["Unexpected browser request:", String(params.method), String(params.path)].join(" "),
);
}
beforeEach(() => {
vi.clearAllMocks();
vi.spyOn(process, "platform", "get").mockReturnValue("darwin");
engineMocks.startAgent.mockRejectedValue(new Error("realtime startup failed"));
});
afterEach(() => {
vi.restoreAllMocks();
});
describe("Zoom meeting Chrome startup cleanup", () => {
it("keeps auto-join enabled when recovering an active meeting tab", async () => {
const state = { tabOpen: true };
const gatewayRequest = vi.fn(async (_method: string, params: Record<string, unknown>) =>
browserResult(params, state),
);
const runtime = {
gateway: { isAvailable: vi.fn(async () => true), request: gatewayRequest },
system: {
runCommandWithTimeout: vi.fn(async () => ({
code: 0,
stderr: "",
stdout: "BlackHole 2ch",
})),
},
} as unknown as PluginRuntime;
await expect(
launchZoomMeetingInChrome({
config: resolveZoomMeetingsConfig({
chrome: { launch: false, waitForInCallMs: 1 },
}),
fullConfig: {},
logger,
meetingSessionId: "session-1",
mode: "agent",
runtime,
trackedTargetId: "zoom-tab",
url: URL,
}),
).rejects.toThrow("realtime startup failed");
const evaluated = gatewayRequest.mock.calls.find(
([, params]) =>
params.path === "/act" &&
!(params.body as { fn?: string } | undefined)?.fn?.includes("leaveAction"),
);
expect((evaluated?.[1].body as { fn?: string } | undefined)?.fn).toContain(
"const autoJoin = true",
);
expect(
gatewayRequest.mock.calls.some(
([, params]) =>
params.method === "DELETE" ||
(params.body as { fn?: string } | undefined)?.fn?.includes("leaveAction"),
),
).toBe(false);
expect(state.tabOpen).toBe(true);
});
it("disposes local audio and leaves the browser when realtime startup fails", async () => {
const state = { tabOpen: false };
const gatewayRequest = vi.fn(async (_method: string, params: Record<string, unknown>) =>
browserResult(params, state),
);
const runtime = {
gateway: { isAvailable: vi.fn(async () => true), request: gatewayRequest },
system: {
runCommandWithTimeout: vi.fn(async () => ({
code: 0,
stderr: "",
stdout: "BlackHole 2ch",
})),
},
} as unknown as PluginRuntime;
await expect(
launchZoomMeetingInChrome({
config: resolveZoomMeetingsConfig({ chrome: { waitForInCallMs: 1 } }),
fullConfig: {},
logger,
meetingSessionId: "session-1",
mode: "agent",
runtime,
url: URL,
}),
).rejects.toThrow("realtime startup failed");
expect(engineMocks.localDispose).toHaveBeenCalled();
expect(gatewayRequest).toHaveBeenCalledWith(
"browser.request",
expect.objectContaining({ method: "DELETE", path: "/tabs/zoom-tab" }),
expect.anything(),
);
expect(state.tabOpen).toBe(false);
});
it("stops node audio and leaves the remote browser when realtime startup fails", async () => {
const state = { tabOpen: false };
const invoke = vi.fn(async (request: Record<string, unknown>) => {
if (request.command === "browser.proxy") {
return {
payload: {
result: browserResult((request.params as Record<string, unknown>) ?? {}, state),
},
};
}
const params = (request.params as Record<string, unknown>) ?? {};
if (params.action === "start") {
return { payload: { audioBridge: { type: "node-command-pair" }, bridgeId: "bridge-1" } };
}
return { payload: { ok: true } };
});
const runtime = {
nodes: {
invoke,
list: vi.fn(async () => ({
nodes: [
{
caps: ["browser"],
commands: ["browser.proxy", "zoommeetings.chrome"],
connected: true,
nodeId: "node-1",
},
],
})),
},
} as unknown as PluginRuntime;
await expect(
launchZoomMeetingOnNode({
config: resolveZoomMeetingsConfig({ chrome: { waitForInCallMs: 1 } }),
fullConfig: {},
logger,
meetingSessionId: "session-1",
mode: "agent",
runtime,
url: URL,
}),
).rejects.toThrow("realtime startup failed");
expect(engineMocks.nodeDispose).toHaveBeenCalled();
expect(
invoke.mock.calls.filter(
([request]) =>
request.command === "zoommeetings.chrome" &&
(request.params as Record<string, unknown>)?.action === "stopByUrl",
),
).toHaveLength(2);
expect(
invoke.mock.calls.some(
([request]) =>
request.command === "browser.proxy" &&
(request.params as Record<string, unknown>)?.method === "DELETE",
),
).toBe(true);
expect(state.tabOpen).toBe(false);
});
it("stops failed node audio recovery without leaving the active browser call", async () => {
const state = { tabOpen: true };
const invoke = vi.fn(async (request: Record<string, unknown>) => {
if (request.command === "browser.proxy") {
return {
payload: {
result: browserResult((request.params as Record<string, unknown>) ?? {}, state),
},
};
}
const params = (request.params as Record<string, unknown>) ?? {};
if (params.action === "start") {
return { payload: { audioBridge: { type: "node-command-pair" }, bridgeId: "bridge-1" } };
}
return { payload: { ok: true } };
});
const runtime = {
nodes: {
invoke,
list: vi.fn(async () => ({
nodes: [
{
caps: ["browser"],
commands: ["browser.proxy", "zoommeetings.chrome"],
connected: true,
nodeId: "node-1",
},
],
})),
},
} as unknown as PluginRuntime;
await expect(
launchZoomMeetingOnNode({
config: resolveZoomMeetingsConfig({
chrome: { launch: false, waitForInCallMs: 1 },
}),
fullConfig: {},
logger,
meetingSessionId: "session-1",
mode: "agent",
runtime,
trackedTargetId: "zoom-tab",
url: URL,
}),
).rejects.toThrow("realtime startup failed");
expect(engineMocks.nodeDispose).toHaveBeenCalled();
expect(
invoke.mock.calls.some(
([request]) =>
request.command === "browser.proxy" &&
((request.params as Record<string, unknown>)?.method === "DELETE" ||
((request.params as { body?: { fn?: string } }).body?.fn?.includes("leaveAction") ??
false)),
),
).toBe(false);
expect(state.tabOpen).toBe(true);
});
});
@@ -0,0 +1,620 @@
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
import {
callMeetingBrowserProxyOnNode,
createLocalMeetingRealtimeAudioTransport,
createNodeMeetingRealtimeAudioTransport,
leaveMeetingWithBrowser,
openMeetingWithBrowser,
readMeetingTranscriptWithBrowser,
recoverMeetingBrowserTab,
resolveLocalMeetingBrowserRequest,
resolveMeetingBrowserNode,
startMeetingAgentRealtimeEngine,
startMeetingRealtimeEngine,
type MeetingBrowserRequestCaller,
type MeetingAgentConsultParams,
type MeetingRealtimeAudioEngineHandle,
type MeetingRealtimeToolCallParams,
type MeetingRuntimePlatform,
} from "openclaw/plugin-sdk/meeting-runtime";
import { addTimerTimeoutGraceMs } from "openclaw/plugin-sdk/number-runtime";
import type { PluginRuntime, RuntimeLogger } from "openclaw/plugin-sdk/plugin-runtime";
import {
consultOpenClawAgentForZoomMeeting,
handleZoomMeetingsRealtimeConsultToolCall,
resolveZoomMeetingsRealtimeTools,
} from "../agent-consult.js";
import type { ZoomMeetingsConfig, ZoomMeetingsMode } from "../config.js";
import {
ZOOM_MEETINGS_SYSTEM_PROFILER_COMMAND,
outputMentionsBlackHole2ch,
} from "./chrome-audio-device.js";
import type {
ZoomMeetingsBrowserTab,
ZoomMeetingsChromeHealth,
ZoomMeetingsTranscriptSnapshot,
} from "./types.js";
import {
ZOOM_MEETINGS_PLATFORM_ADAPTER,
isZoomMeetingsRealtimeRouteReady,
isZoomMeetingsTalkBackMode,
} from "./zoom-meetings-platform-adapter.js";
import {
ZOOM_MEETINGS_BROWSER_NODE_ADAPTER,
ZOOM_MEETINGS_NODE_COMMAND,
} from "./zoom-meetings-platform-constants.js";
const ZOOM_MEETINGS_RUNTIME_PLATFORM = {
displayName: ZOOM_MEETINGS_PLATFORM_ADAPTER.displayName,
logScope: ZOOM_MEETINGS_PLATFORM_ADAPTER.logScope,
sessionIdPrefix: ZOOM_MEETINGS_PLATFORM_ADAPTER.id,
} satisfies MeetingRuntimePlatform;
type LocalAudioBridge = MeetingRealtimeAudioEngineHandle & {
type: "command-pair";
};
type NodeAudioBridge = MeetingRealtimeAudioEngineHandle & {
type: "node-command-pair";
nodeId: string;
bridgeId: string;
};
async function openOrRecoverZoomMeeting(params: {
callBrowser: MeetingBrowserRequestCaller;
config: ZoomMeetingsConfig;
meetingSessionId: string;
mode: ZoomMeetingsMode;
trackedTargetId?: string;
url: string;
locationLabel: string;
}) {
if (params.config.chrome.launch) {
return await openMeetingWithBrowser({
adapter: ZOOM_MEETINGS_PLATFORM_ADAPTER,
callBrowser: params.callBrowser,
config: params.config.chrome,
session: {
meetingSessionId: params.meetingSessionId,
mode: params.mode,
url: params.url,
},
});
}
const recovered = await recoverMeetingBrowserTab({
adapter: ZOOM_MEETINGS_PLATFORM_ADAPTER,
allowSessionAdoption: true,
autoJoin: params.config.chrome.autoJoin,
callBrowser: params.callBrowser,
config: params.config.chrome,
locationLabel: params.locationLabel,
meetingSessionId: params.meetingSessionId,
mode: params.mode,
requestedMeetingUrl: params.url,
trackedMeetingUrl: params.url,
trackedTargetId: params.trackedTargetId,
});
return {
launched: false,
browser: recovered.browser,
tab: recovered.targetId ? { targetId: recovered.targetId, openedByPlugin: false } : undefined,
};
}
async function rollbackZoomBrowserJoin(params: {
callBrowser: MeetingBrowserRequestCaller;
config: ZoomMeetingsConfig;
logger: RuntimeLogger;
meetingSessionId: string;
tab?: ZoomMeetingsBrowserTab;
url: string;
}) {
if (!params.tab) {
return;
}
const result = await leaveMeetingWithBrowser({
adapter: ZOOM_MEETINGS_PLATFORM_ADAPTER,
callBrowser: params.callBrowser,
launch: true,
meetingSessionId: params.meetingSessionId,
meetingUrl: params.url,
tab: params.tab,
timeoutMs: params.config.chrome.joinTimeoutMs,
}).catch((error: unknown) => ({
left: false,
note: error instanceof Error ? error.message : String(error),
}));
if (!result.left) {
params.logger.warn(
`${ZOOM_MEETINGS_RUNTIME_PLATFORM.logScope} browser rollback after realtime startup failure did not complete: ${result.note}`,
);
}
}
function realtimeBindings(params: {
config: ZoomMeetingsConfig;
fullConfig: OpenClawConfig;
runtime: PluginRuntime;
logger: RuntimeLogger;
}) {
return {
platform: ZOOM_MEETINGS_RUNTIME_PLATFORM,
consultAgent: (consult: MeetingAgentConsultParams) =>
consultOpenClawAgentForZoomMeeting({
config: params.config,
fullConfig: params.fullConfig,
runtime: params.runtime,
logger: params.logger,
...consult,
}),
tools: resolveZoomMeetingsRealtimeTools(params.config.realtime.toolPolicy),
handleToolCall: (call: MeetingRealtimeToolCallParams) =>
handleZoomMeetingsRealtimeConsultToolCall({
config: params.config,
fullConfig: params.fullConfig,
runtime: params.runtime,
logger: params.logger,
...call,
}),
};
}
export async function assertBlackHole2chAvailable(params: {
runtime: PluginRuntime;
timeoutMs: number;
}): Promise<void> {
if (process.platform !== "darwin") {
throw new Error("Zoom meeting talk-back with BlackHole 2ch is macOS-only");
}
const result = await params.runtime.system.runCommandWithTimeout(
[ZOOM_MEETINGS_SYSTEM_PROFILER_COMMAND, "SPAudioDataType"],
{ timeoutMs: params.timeoutMs },
);
const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}`;
if (result.code !== 0 || !outputMentionsBlackHole2ch(output)) {
const hint =
params.runtime.system.formatNativeDependencyHint?.({
packageName: "BlackHole 2ch",
downloadCommand: "brew install blackhole-2ch",
}) ?? "";
throw new Error(
["BlackHole 2ch audio device not found.", "Install BlackHole 2ch and SoX.", hint]
.filter(Boolean)
.join(" "),
);
}
}
async function startLocalAudioBridge(params: {
runtime: PluginRuntime;
config: ZoomMeetingsConfig;
fullConfig: OpenClawConfig;
meetingSessionId: string;
requesterSessionKey?: string;
mode: ZoomMeetingsMode;
logger: RuntimeLogger;
}): Promise<LocalAudioBridge | undefined> {
if (!isZoomMeetingsTalkBackMode(params.mode)) {
return undefined;
}
const transport = createLocalMeetingRealtimeAudioTransport({
inputCommand: params.config.chrome.audioInputCommand,
outputCommand: params.config.chrome.audioOutputCommand,
bargeInInputCommand: params.config.chrome.bargeInInputCommand,
bargeInRmsThreshold: params.config.chrome.bargeInRmsThreshold,
bargeInPeakThreshold: params.config.chrome.bargeInPeakThreshold,
bargeInCooldownMs: params.config.chrome.bargeInCooldownMs,
logger: params.logger,
logScope: ZOOM_MEETINGS_RUNTIME_PLATFORM.logScope,
});
const bindings = realtimeBindings(params);
try {
const engine =
params.mode === "agent"
? await startMeetingAgentRealtimeEngine({
config: params.config,
fullConfig: params.fullConfig,
runtime: params.runtime,
platform: bindings.platform,
meetingSessionId: params.meetingSessionId,
requesterSessionKey: params.requesterSessionKey,
transport,
logger: params.logger,
consultAgent: bindings.consultAgent,
})
: await startMeetingRealtimeEngine({
config: {
...params.config,
realtime: { ...params.config.realtime, strategy: "bidi" },
},
fullConfig: params.fullConfig,
runtime: params.runtime,
...bindings,
meetingSessionId: params.meetingSessionId,
requesterSessionKey: params.requesterSessionKey,
transport,
logger: params.logger,
});
return { type: "command-pair", ...engine };
} catch (error) {
await transport.dispose().catch(() => {});
throw error;
}
}
export async function launchZoomMeetingInChrome(params: {
runtime: PluginRuntime;
config: ZoomMeetingsConfig;
fullConfig: OpenClawConfig;
meetingSessionId: string;
requesterSessionKey?: string;
mode: ZoomMeetingsMode;
trackedTargetId?: string;
url: string;
logger: RuntimeLogger;
}): Promise<{
launched: boolean;
audioBridge?: LocalAudioBridge;
browser?: ZoomMeetingsChromeHealth;
tab?: ZoomMeetingsBrowserTab;
}> {
if (isZoomMeetingsTalkBackMode(params.mode)) {
await assertBlackHole2chAvailable({
runtime: params.runtime,
timeoutMs: Math.min(params.config.chrome.joinTimeoutMs, 10_000),
});
}
const callBrowser = await resolveLocalMeetingBrowserRequest(params.runtime);
const result = await openOrRecoverZoomMeeting({
callBrowser,
config: params.config,
locationLabel: "in local Chrome",
meetingSessionId: params.meetingSessionId,
mode: params.mode,
trackedTargetId: params.trackedTargetId,
url: params.url,
});
if (!isZoomMeetingsRealtimeRouteReady(params.mode, result.browser)) {
return result;
}
try {
return { ...result, audioBridge: await startLocalAudioBridge(params) };
} catch (error) {
// Recovery owns only the replacement audio bridge. The active browser call
// remains owned by the published session and must survive provider startup errors.
if (!params.trackedTargetId) {
await rollbackZoomBrowserJoin({
callBrowser,
config: params.config,
logger: params.logger,
meetingSessionId: params.meetingSessionId,
tab: result.tab,
url: params.url,
});
}
throw error;
}
}
async function resolveChromeNode(params: {
runtime: PluginRuntime;
requestedNode?: string;
}): Promise<string> {
return await resolveMeetingBrowserNode({
...params,
adapter: ZOOM_MEETINGS_BROWSER_NODE_ADAPTER,
});
}
async function callNodeBrowser(params: {
runtime: PluginRuntime;
nodeId: string;
method: "GET" | "POST" | "DELETE";
path: string;
body?: unknown;
timeoutMs: number;
}) {
return await callMeetingBrowserProxyOnNode({
...params,
adapter: ZOOM_MEETINGS_BROWSER_NODE_ADAPTER,
});
}
type ZoomMeetingsNodeStartResult = {
launched?: boolean;
bridgeId?: string;
audioBridge?: { type?: string };
browser?: ZoomMeetingsChromeHealth;
};
function parseNodeStartResult(raw: unknown): ZoomMeetingsNodeStartResult {
const value =
raw && typeof raw === "object" && "payload" in raw
? (raw as { payload?: unknown }).payload
: raw;
if (!value || typeof value !== "object") {
throw new Error("Zoom meeting node returned an invalid start result.");
}
return value as ZoomMeetingsNodeStartResult;
}
export async function launchZoomMeetingOnNode(params: {
runtime: PluginRuntime;
config: ZoomMeetingsConfig;
fullConfig: OpenClawConfig;
meetingSessionId: string;
requesterSessionKey?: string;
mode: ZoomMeetingsMode;
trackedTargetId?: string;
url: string;
logger: RuntimeLogger;
}): Promise<{
nodeId: string;
launched: boolean;
audioBridge?: NodeAudioBridge;
browser?: ZoomMeetingsChromeHealth;
tab?: ZoomMeetingsBrowserTab;
}> {
const nodeId = await resolveChromeNode({
runtime: params.runtime,
requestedNode: params.config.chromeNode.node,
});
try {
await params.runtime.nodes.invoke({
nodeId,
command: ZOOM_MEETINGS_NODE_COMMAND,
params: { action: "stopByUrl", url: params.url, mode: params.mode },
timeoutMs: 5_000,
});
} catch (error) {
params.logger.debug?.(
`${ZOOM_MEETINGS_RUNTIME_PLATFORM.logScope} node bridge cleanup ignored: ${
error instanceof Error ? error.message : String(error)
}`,
);
}
const callBrowser: MeetingBrowserRequestCaller = async (request) =>
await callNodeBrowser({
runtime: params.runtime,
nodeId,
method: request.method,
path: request.path,
body: request.body,
timeoutMs: request.timeoutMs,
});
const browser = await openOrRecoverZoomMeeting({
callBrowser,
config: params.config,
locationLabel: "on the selected Chrome node",
meetingSessionId: params.meetingSessionId,
mode: params.mode,
trackedTargetId: params.trackedTargetId,
url: params.url,
});
if (!isZoomMeetingsRealtimeRouteReady(params.mode, browser.browser)) {
return {
nodeId,
launched: browser.launched,
browser: browser.browser,
tab: browser.tab,
};
}
try {
const raw = await params.runtime.nodes.invoke({
nodeId,
command: ZOOM_MEETINGS_NODE_COMMAND,
params: {
action: "start",
url: params.url,
mode: params.mode,
launch: false,
browserProfile: params.config.chrome.browserProfile,
joinTimeoutMs: params.config.chrome.joinTimeoutMs,
audioInputCommand: params.config.chrome.audioInputCommand,
audioOutputCommand: params.config.chrome.audioOutputCommand,
},
timeoutMs: addTimerTimeoutGraceMs(params.config.chrome.joinTimeoutMs) ?? 1,
});
const result = parseNodeStartResult(raw);
if (result.audioBridge?.type !== "node-command-pair") {
return {
nodeId,
launched: browser.launched || result.launched === true,
browser: browser.browser ?? result.browser,
tab: browser.tab,
};
}
if (!result.bridgeId) {
throw new Error("Zoom meeting node did not return an audio bridge id.");
}
const transport = createNodeMeetingRealtimeAudioTransport({
runtime: params.runtime,
nodeId,
bridgeId: result.bridgeId,
logger: params.logger,
commandName: ZOOM_MEETINGS_NODE_COMMAND,
logScope: ZOOM_MEETINGS_RUNTIME_PLATFORM.logScope,
logPrefix: params.mode === "agent" ? "node agent" : "node",
});
const bindings = realtimeBindings(params);
let engine: MeetingRealtimeAudioEngineHandle;
try {
engine =
params.mode === "agent"
? await startMeetingAgentRealtimeEngine({
config: params.config,
fullConfig: params.fullConfig,
runtime: params.runtime,
platform: bindings.platform,
meetingSessionId: params.meetingSessionId,
requesterSessionKey: params.requesterSessionKey,
logPrefix: "node",
transport,
logger: params.logger,
consultAgent: bindings.consultAgent,
})
: await startMeetingRealtimeEngine({
config: {
...params.config,
realtime: { ...params.config.realtime, strategy: "bidi" },
},
fullConfig: params.fullConfig,
runtime: params.runtime,
...bindings,
meetingSessionId: params.meetingSessionId,
requesterSessionKey: params.requesterSessionKey,
logPrefix: "node",
talkSessionId: `zoom-meetings:${params.meetingSessionId}:${result.bridgeId}:node-realtime`,
talkContext: { nodeId, bridgeId: result.bridgeId },
transport,
logger: params.logger,
});
} catch (error) {
await transport.dispose().catch(() => {});
throw error;
}
return {
nodeId,
launched: browser.launched || result.launched === true,
audioBridge: {
type: "node-command-pair",
nodeId,
bridgeId: result.bridgeId,
...engine,
},
browser: browser.browser ?? result.browser,
tab: browser.tab,
};
} catch (error) {
await params.runtime.nodes
.invoke({
nodeId,
command: ZOOM_MEETINGS_NODE_COMMAND,
params: { action: "stopByUrl", url: params.url, mode: params.mode },
timeoutMs: 5_000,
})
.catch(() => {});
if (!params.trackedTargetId) {
await rollbackZoomBrowserJoin({
callBrowser,
config: params.config,
logger: params.logger,
meetingSessionId: params.meetingSessionId,
tab: browser.tab,
url: params.url,
});
}
throw error;
}
}
export async function recoverCurrentZoomMeetingTab(params: {
runtime: PluginRuntime;
config: ZoomMeetingsConfig;
meetingSessionId?: string;
mode: ZoomMeetingsMode;
nodeId?: string;
readOnly?: boolean;
trackedMeetingUrl?: string;
trackedTargetId?: string;
transport: "chrome" | "chrome-node";
timeoutMs?: number;
url?: string;
}) {
const nodeId =
params.transport === "chrome-node"
? (params.nodeId ??
(await resolveChromeNode({
runtime: params.runtime,
requestedNode: params.config.chromeNode.node,
})))
: undefined;
return {
transport: params.transport,
...(nodeId ? { nodeId } : {}),
...(await recoverMeetingBrowserTab({
adapter: ZOOM_MEETINGS_PLATFORM_ADAPTER,
callBrowser: nodeId
? async (request) =>
await callNodeBrowser({
runtime: params.runtime,
nodeId,
method: request.method,
path: request.path,
body: request.body,
timeoutMs: request.timeoutMs,
})
: await resolveLocalMeetingBrowserRequest(params.runtime),
config: params.config.chrome,
locationLabel: nodeId ? "on the selected Chrome node" : "in local Chrome",
meetingSessionId: params.meetingSessionId,
mode: params.mode,
readOnly: params.readOnly,
requestedMeetingUrl: params.url,
trackedMeetingUrl: params.trackedMeetingUrl,
trackedTargetId: params.trackedTargetId,
timeoutMs: params.timeoutMs,
})),
};
}
export async function leaveZoomMeetingInBrowser(params: {
runtime: PluginRuntime;
config: ZoomMeetingsConfig;
meetingSessionId: string;
meetingUrl: string;
nodeId?: string;
tab: ZoomMeetingsBrowserTab;
}) {
const nodeId = params.nodeId;
return await leaveMeetingWithBrowser({
adapter: ZOOM_MEETINGS_PLATFORM_ADAPTER,
callBrowser: nodeId
? async (request) =>
await callNodeBrowser({
runtime: params.runtime,
nodeId,
method: request.method,
path: request.path,
body: request.body,
timeoutMs: request.timeoutMs,
})
: await resolveLocalMeetingBrowserRequest(params.runtime),
launch: params.config.chrome.launch || !params.tab.openedByPlugin,
meetingSessionId: params.meetingSessionId,
meetingUrl: params.meetingUrl,
tab: params.tab,
timeoutMs: params.config.chrome.joinTimeoutMs,
});
}
export async function readZoomMeetingTranscript(params: {
runtime: PluginRuntime;
config: ZoomMeetingsConfig;
finalize?: boolean;
meetingUrl: string;
meetingSessionId: string;
nodeId?: string;
tab: ZoomMeetingsBrowserTab;
}): Promise<ZoomMeetingsTranscriptSnapshot> {
const nodeId = params.nodeId;
return await readMeetingTranscriptWithBrowser({
adapter: ZOOM_MEETINGS_PLATFORM_ADAPTER,
callBrowser: nodeId
? async (request) =>
await callNodeBrowser({
runtime: params.runtime,
nodeId,
method: request.method,
path: request.path,
body: request.body,
timeoutMs: request.timeoutMs,
})
: await resolveLocalMeetingBrowserRequest(params.runtime),
finalize: params.finalize === true,
meetingUrl: params.meetingUrl,
meetingSessionId: params.meetingSessionId,
tab: params.tab,
timeoutMs: Math.min(Math.max(1_000, params.config.chrome.joinTimeoutMs), 10_000),
});
}
@@ -0,0 +1,103 @@
import type {
MeetingBrowserHealth,
MeetingBrowserTab,
MeetingSessionRecord,
MeetingTranscriptSnapshot,
} from "openclaw/plugin-sdk/meeting-runtime";
import type { ZoomMeetingsMode, ZoomMeetingsTransport } from "../config.js";
export type ZoomMeetingsTranscriptSnapshot = MeetingTranscriptSnapshot;
export type ZoomMeetingsJoinRequest = {
url: string;
transport?: ZoomMeetingsTransport;
mode?: ZoomMeetingsMode;
message?: string;
requesterSessionKey?: string;
agentId?: string;
timeoutMs?: number;
};
type ZoomMeetingsManualActionReason =
| "zoom-login-required"
| "zoom-admission-required"
| "zoom-permission-required"
| "zoom-audio-choice-required"
| "zoom-camera-required"
| "zoom-microphone-required"
| "zoom-passcode-required"
| "zoom-captcha-required"
| "zoom-session-conflict"
| "browser-control-unavailable";
type ZoomMeetingsSpeechBlockedReason =
| ZoomMeetingsManualActionReason
| "not-in-call"
| "browser-unverified"
| "audio-bridge-unavailable"
| "zoom-microphone-muted";
export type ZoomMeetingsChromeHealth = MeetingBrowserHealth<
ZoomMeetingsManualActionReason,
ZoomMeetingsSpeechBlockedReason
> & {
inCall?: boolean;
meetingEnded?: boolean;
micMuted?: boolean;
cameraOff?: boolean;
lobbyWaiting?: boolean;
captionCaptureRequested?: boolean;
captioning?: boolean;
captionsEnabledAttempted?: boolean;
transcriptLines?: number;
lastCaptionAt?: string;
lastCaptionSpeaker?: string;
lastCaptionText?: string;
recentTranscript?: Array<{
at?: string;
speaker?: string;
text: string;
}>;
audioInputRouted?: boolean;
audioInputDeviceLabel?: string;
audioInputRouteError?: string;
audioOutputRouted?: boolean;
audioOutputDeviceLabel?: string;
audioOutputRouteError?: string;
audioOutputRouteRetryable?: boolean;
providerConnected?: boolean;
realtimeReady?: boolean;
audioInputActive?: boolean;
audioOutputActive?: boolean;
lastInputAt?: string;
lastOutputAt?: string;
lastInputBytes?: number;
lastOutputBytes?: number;
bridgeClosed?: boolean;
browserUrl?: string;
browserTitle?: string;
status?: string;
notes?: string[];
};
export type ZoomMeetingsBrowserTab = MeetingBrowserTab;
export type ZoomMeetingsSession = MeetingSessionRecord<ZoomMeetingsTransport, ZoomMeetingsMode> & {
chrome?: {
audioBackend: "blackhole-2ch";
launched: boolean;
nodeId?: string;
browserProfile?: string;
browserTab?: ZoomMeetingsBrowserTab;
audioBridge?: {
type: "command-pair" | "node-command-pair";
provider?: string;
};
health?: ZoomMeetingsChromeHealth;
};
};
export type ZoomMeetingsJoinResult = {
session: ZoomMeetingsSession;
spoken?: boolean;
};
@@ -0,0 +1,340 @@
import { ZOOM_MEETING_SELECTORS } from "./zoom-meetings-selectors.js";
import { zoomMeetingStatusCallSource } from "./zoom-meetings-status-call-source.js";
import { zoomMeetingStatusPreludeSource } from "./zoom-meetings-status-prejoin-source.js";
import { normalizeZoomMeetingUrlForReuse } from "./zoom-meetings-urls.js";
const ZOOM_MEETING_TRANSCRIPT_MAX_LINES = 500;
function pageIdentityFunctionSource(): string {
return `const meetingIdentity = (rawUrl) => {
try {
const parsed = new URL(rawUrl);
const host = parsed.hostname.toLowerCase();
if (
parsed.protocol !== "https:" ||
!(host === "zoom.us" || host.endsWith(".zoom.us"))
) return undefined;
const invitation = parsed.pathname.match(/^\\/j\\/(\\d{9,11})\\/?$/);
const webClient = parsed.pathname.match(/^\\/wc\\/(\\d{9,11})\\/join\\/?$/);
const meetingId = invitation?.[1] || webClient?.[1];
return meetingId ? "zoom:" + meetingId : undefined;
} catch {}
return undefined;
};`;
}
function zoomMeetingToggleStateFunctionSource(): string {
return `(input) => {
const pressed = String(input?.ariaPressed || "").toLowerCase();
if (pressed === "true") return "on";
if (pressed === "false") return "off";
const checked = String(input?.ariaChecked ?? input?.checked ?? "").toLowerCase();
if (checked === "true") return "on";
if (checked === "false") return "off";
const iconClass = String(input?.iconClass || "");
if (input?.kind === "camera" && /videooff/i.test(iconClass)) return "off";
if (input?.kind === "camera" && /videoon/i.test(iconClass)) return "on";
const value = String(input?.label || "").toLowerCase().replace(/\\s+/g, " ").trim();
if (!value) return undefined;
if (input?.kind === "camera") {
if (/\\bturn (?:your )?camera off\\b|\\bturn off (?:your )?camera\\b|\\bstop video\\b|\\bdisable (?:your )?(?:camera|video)\\b/.test(value)) return "on";
if (/\\bturn (?:your )?camera on\\b|\\bturn on (?:your )?camera\\b|\\bstart video\\b|\\benable (?:your )?(?:camera|video)\\b/.test(value)) return "off";
if (/\\b(?:camera|video) (?:is |currently )?(?:off|disabled)\\b/.test(value)) return "off";
if (/\\b(?:camera|video) (?:is |currently )?(?:on|enabled)\\b/.test(value)) return "on";
return undefined;
}
if (/^mute(?: mute)?$|\\bturn (?:your |my )?(?:microphone|mic) off\\b|\\bturn off (?:your |my )?(?:microphone|mic)\\b|\\bmute (?:your |my )?(?:microphone|mic)\\b|\\bdisable (?:your |my )?(?:microphone|mic)\\b/.test(value)) return "on";
if (/^unmute(?: unmute)?$|\\bturn (?:your |my )?(?:microphone|mic) on\\b|\\bturn on (?:your |my )?(?:microphone|mic)\\b|\\bunmute (?:your |my )?(?:microphone|mic)\\b|\\benable (?:your |my )?(?:microphone|mic)\\b/.test(value)) return "off";
if (/\\b(?:microphone|mic) (?:is |currently )?(?:off|muted|disabled)\\b/.test(value)) return "off";
if (/\\b(?:microphone|mic) (?:is |currently )?(?:on|unmuted|enabled)\\b/.test(value)) return "on";
return undefined;
}`;
}
export function zoomMeetingStatusScript(params: {
allowMicrophone: boolean;
allowSessionAdoption: boolean;
autoJoin: boolean;
captureCaptions: boolean;
guestName: string;
meetingSessionId?: string;
meetingUrl: string;
readOnly?: boolean;
waitForInCallMs: number;
}) {
const selectors = JSON.stringify(ZOOM_MEETING_SELECTORS);
const expectedIdentity = normalizeZoomMeetingUrlForReuse(params.meetingUrl);
const toggleStateFunction = zoomMeetingToggleStateFunctionSource();
return (
zoomMeetingStatusPreludeSource({
...params,
expectedIdentity,
pageIdentitySource: pageIdentityFunctionSource(),
selectors,
toggleStateFunction,
}) + zoomMeetingStatusCallSource()
);
}
export function zoomMeetingTranscriptScript(
meetingUrl: string,
meetingSessionId: string,
finalize: boolean,
) {
const expectedIdentity = normalizeZoomMeetingUrlForReuse(meetingUrl);
return `() => {
${pageIdentityFunctionSource()}
const expectedIdentity = ${JSON.stringify(expectedIdentity)};
const expectedSessionId = ${JSON.stringify(meetingSessionId)};
const currentIdentity = meetingIdentity(location.href);
const state = window.__openclawZoomMeeting;
const activeCaptions = window.__openclawZoomCaptions;
const archivedCaptions = window.__openclawZoomCaptionArchive?.[expectedSessionId];
const captions = activeCaptions &&
(!activeCaptions.sessionId || activeCaptions.sessionId === expectedSessionId)
? activeCaptions
: archivedCaptions;
// A same-session finalized buffer belongs to the departed call even if Zoom
// immediately navigated this tab into another meeting before transcript pickup.
const useFinalizedCaptions = Boolean(
captions?.finalized === true &&
captions?.identity === expectedIdentity &&
(!captions?.sessionId || captions.sessionId === expectedSessionId)
);
const effectiveIdentity = useFinalizedCaptions
? captions.identity
: currentIdentity || state?.identity || captions?.identity;
if (!expectedIdentity || effectiveIdentity !== expectedIdentity) {
return JSON.stringify({ urlMatched: false, droppedLines: 0, lines: [] });
}
if (!useFinalizedCaptions && state?.sessionId && state.sessionId !== expectedSessionId) {
return JSON.stringify({ urlMatched: true, sessionMatched: false, droppedLines: 0, lines: [] });
}
if (captions?.sessionId && captions.sessionId !== expectedSessionId) {
return JSON.stringify({ urlMatched: true, sessionMatched: false, droppedLines: 0, lines: [] });
}
if (${JSON.stringify(finalize)} && Array.isArray(captions?.visible) && captions.visible.length > 0) {
if (captions.settleTimer !== undefined) clearTimeout(captions.settleTimer);
captions.settleTimer = undefined;
captions.lines = Array.isArray(captions.lines) ? captions.lines : [];
captions.lines.push(...captions.visible.map((entry) => ({
at: entry.at,
speaker: entry.speaker,
text: entry.text,
})));
captions.visible = [];
const excess = captions.lines.length - ${ZOOM_MEETING_TRANSCRIPT_MAX_LINES};
if (excess > 0) {
captions.lines.splice(0, excess);
captions.droppedLines = (captions.droppedLines || 0) + excess;
}
}
if (${JSON.stringify(finalize)} && captions) {
if (captions.settleTimer !== undefined) clearTimeout(captions.settleTimer);
captions.settleTimer = undefined;
captions.observer?.disconnect?.();
captions.observer = undefined;
captions.observerInstalled = false;
captions.identity = expectedIdentity;
captions.finalized = true;
captions.finalizedAt = Date.now();
}
const allLines = [
...(Array.isArray(captions?.lines) ? captions.lines : []),
...(${JSON.stringify(finalize)} || !Array.isArray(captions?.visible) ? [] : captions.visible),
];
const visibleOverflow = Math.max(0, allLines.length - ${ZOOM_MEETING_TRANSCRIPT_MAX_LINES});
const lines = allLines.slice(-${ZOOM_MEETING_TRANSCRIPT_MAX_LINES});
const result = {
urlMatched: true,
sessionMatched: true,
epoch: typeof captions?.epoch === "string" ? captions.epoch : undefined,
droppedLines: (Number.isFinite(captions?.droppedLines)
? Math.max(0, Math.trunc(captions.droppedLines))
: 0) + visibleOverflow,
lines: lines.map((line) => ({
at: typeof line?.at === "string" ? line.at : undefined,
speaker: typeof line?.speaker === "string" ? line.speaker : undefined,
text: typeof line?.text === "string" ? line.text : "",
})).filter((line) => line.text),
};
return JSON.stringify(result);
}`;
}
export function zoomMeetingLeaveScript(params: {
leaveInitiated: boolean;
meetingSessionId: string;
meetingUrl: string;
}) {
const selectors = JSON.stringify(ZOOM_MEETING_SELECTORS);
const expectedIdentity = normalizeZoomMeetingUrlForReuse(params.meetingUrl);
return `() => {
${pageIdentityFunctionSource()}
const topDocument = globalThis.document;
const document = topDocument.querySelector("#webclient")?.contentDocument || topDocument;
const selectors = ${selectors};
const expectedIdentity = ${JSON.stringify(expectedIdentity)};
const expectedSessionId = ${JSON.stringify(params.meetingSessionId)};
const leaveInitiated = ${JSON.stringify(params.leaveInitiated)};
const currentIdentity = meetingIdentity(location.href);
const state = window.__openclawZoomMeeting;
const enforceSessionOwnership = Boolean(expectedSessionId);
if (enforceSessionOwnership && state?.sessionId && state.sessionId !== expectedSessionId) {
return JSON.stringify({ departed: false, sessionConflict: true, sessionMatched: false, urlMatched: true });
}
const sessionAdoptedFromUrl = Boolean(
enforceSessionOwnership &&
!state?.sessionId &&
currentIdentity === expectedIdentity &&
(!state?.identity || state.identity === expectedIdentity)
);
const sessionMatched = !enforceSessionOwnership ||
state?.sessionId === expectedSessionId ||
sessionAdoptedFromUrl;
const retainedLeaveOwnership = Boolean(!sessionMatched && leaveInitiated);
if (!sessionMatched && !retainedLeaveOwnership) {
return JSON.stringify({ departed: false, sessionMatched: false, urlMatched: true });
}
const retireOwnedAudioBridges = () => {
const entries = Array.isArray(window.__openclawZoomAudioOutputs)
? window.__openclawZoomAudioOutputs
: [];
const retained = [];
const activeSessionId = expectedSessionId || state?.sessionId;
for (const entry of entries) {
const ownedByActiveSession = Boolean(
!entry?.sessionId || (activeSessionId && entry.sessionId === activeSessionId)
);
if (!ownedByActiveSession) {
retained.push(entry);
continue;
}
const mediaSourceUrl = (element) => String(element?.currentSrc || element?.src || "");
const sources = Array.isArray(entry?.sources)
? entry.sources
: entry?.source
? [{ element: entry.source, muted: Boolean(entry.sourceMuted), stream: entry.stream, url: entry.sourceUrl }]
: [];
for (const source of sources) {
const element = source?.element;
const sourceMatches = source?.stream || element?.srcObject
? element?.srcObject === source?.stream
: Boolean(source?.url && mediaSourceUrl(element) === source.url);
const sourceIsEmpty = Boolean(element && !element.srcObject && !mediaSourceUrl(element));
if (!element) continue;
if (sourceIsEmpty) {
element.muted = true;
continue;
}
if (!sourceMatches) continue;
const detachedLiveSource = Boolean(
element.isConnected === false &&
element.srcObject?.getAudioTracks?.().some((track) => track.readyState === "live")
);
if (detachedLiveSource) {
element.muted = true;
element.pause?.();
element.srcObject = null;
} else {
element.muted = Boolean(source.muted);
}
}
entry?.bridge?.pause?.();
if (entry?.bridge) entry.bridge.srcObject = null;
entry?.bridge?.remove?.();
}
if (retained.length > 0) window.__openclawZoomAudioOutputs = retained;
else delete window.__openclawZoomAudioOutputs;
};
const first = (list) => {
for (const selector of list) {
const node = document.querySelector(selector);
if (!node) continue;
return node.matches?.("button") ? node : node.querySelector?.("button") || node.closest?.("button") || node;
}
return undefined;
};
const text = (node) => (node?.innerText || node?.textContent || "").trim();
const findTextButton = (pattern) => [...document.querySelectorAll("button")]
.find((button) => !button.disabled && pattern.test(text(button)));
const leave = first(selectors.leave);
const confirmation = first(selectors.leaveConfirmation) ||
findTextButton(/^leave meeting$/i);
const postCall = !leave && (
first(selectors.postCall) ||
[...document.querySelectorAll(".zm-modal-body-title")]
.find((node) => /meeting has been ended by host|you left the meeting|meeting has ended/i.test(text(node)))
);
const currentUrlMatches = Boolean(expectedIdentity && currentIdentity === expectedIdentity);
let webClientHome = false;
try {
const currentUrl = new URL(location.href);
webClientHome = !leave && currentUrl.hostname === "app.zoom.us" && /^\\/wc\\/?$/.test(currentUrl.pathname);
} catch {}
const preservedCallMatches = Boolean(
expectedIdentity &&
!currentIdentity &&
state?.identity === expectedIdentity &&
state?.inCallControl === leave &&
state?.inCallUrl === location.href &&
leave &&
leave.isConnected !== false
);
const pendingLeaveMatches = Boolean(
expectedIdentity &&
state?.identity === expectedIdentity &&
state?.leavePending === true &&
state?.inCallUrl === location.href &&
Date.now() - state?.leavePendingAt < 10_000
);
const rerenderPendingMatches = Boolean(
expectedIdentity &&
!currentIdentity &&
state?.identity === expectedIdentity &&
state?.inCallControl?.isConnected === false &&
state?.inCallUrl === location.href &&
Date.now() - state?.verifiedAt < 5_000 &&
!leave
);
const meetingIdentityMatches = Boolean(
currentUrlMatches || preservedCallMatches || pendingLeaveMatches || rerenderPendingMatches
);
// Zoom can replace the document between our Leave click and its post-call marker.
// Retain request ownership only while no identity or live-call control contradicts it.
const initiatedLeaveTransitionMatches = Boolean(
leaveInitiated &&
!currentIdentity &&
!leave &&
(!state?.identity || state.identity === expectedIdentity)
);
if ((postCall || webClientHome) && (meetingIdentityMatches || initiatedLeaveTransitionMatches)) {
retireOwnedAudioBridges();
if (sessionMatched) delete window.__openclawZoomMeeting;
return JSON.stringify({ departed: true, sessionMatched: true, urlMatched: true });
}
if (!meetingIdentityMatches && !initiatedLeaveTransitionMatches) {
return JSON.stringify({ departed: false, urlMatched: false });
}
if (!sessionMatched) {
return JSON.stringify({ departed: false, urlMatched: true });
}
if (confirmation) {
confirmation.click();
return JSON.stringify({ departed: false, leaveAction: "confirm", urlMatched: true });
}
if (leave) {
window.__openclawZoomMeeting = {
...state,
identity: expectedIdentity,
sessionId: expectedSessionId || state?.sessionId,
inCallControl: leave,
inCallUrl: location.href,
leavePending: true,
leavePendingAt: Date.now(),
};
leave.click();
return JSON.stringify({ departed: false, leaveAction: "leave", urlMatched: true });
}
return JSON.stringify({ departed: false, urlMatched: true });
}`;
}
@@ -0,0 +1,623 @@
import { runInNewContext } from "node:vm";
import { describe, expect, it, vi } from "vitest";
import { zoomMeetingLeaveScript, zoomMeetingStatusScript } from "./zoom-meetings-page-scripts.js";
import {
ZOOM_MEETINGS_PLATFORM_ADAPTER,
isZoomMeetingsRealtimeRouteReady,
} from "./zoom-meetings-platform-adapter.js";
const URL = "https://acme.zoom.us/j/12345678901?pwd=abc";
function pageControl(label: string) {
const click = vi.fn();
const control = {
disabled: false,
isConnected: true,
textContent: label,
click,
closest: () => control,
getAttribute: (name: string) => (name === "aria-label" ? label : null),
matches: (selector: string) => selector === "button",
querySelector: () => undefined,
querySelectorAll: () => [],
};
return control;
}
function guestInput(value: string) {
return {
...pageControl("Guest name"),
dispatchEvent: vi.fn(),
focus: vi.fn(),
placeholder: "Enter your name",
value,
};
}
function statusDocument(params: {
bodyText: string;
camera?: ReturnType<typeof pageControl>;
challenge?: ReturnType<typeof pageControl>;
devicePrompt?: ReturnType<typeof pageControl>;
guest?: ReturnType<typeof guestInput>;
join?: ReturnType<typeof pageControl>;
kind?: string;
leave?: ReturnType<typeof pageControl>;
microphone?: ReturnType<typeof pageControl>;
}) {
const controls = [
params.camera,
params.challenge,
params.devicePrompt,
params.join,
params.leave,
params.microphone,
].filter((control): control is ReturnType<typeof pageControl> => Boolean(control));
return {
body: { appendChild: () => {}, textContent: params.bodyText },
defaultView: {
Event: globalThis.Event,
HTMLInputElement: function HTMLInputElement() {
return undefined;
},
MutationObserver: function MutationObserver() {
return undefined;
},
},
title: "Zoom",
getElementById: () => undefined,
querySelector(selector: string) {
if (selector.includes("preview-join-button")) {
return params.join;
}
if (selector.includes("input-for-name")) {
return params.guest;
}
if (selector.includes('aria-label="Leave"')) {
return params.leave;
}
if (selector.includes("preview-video-control-button") || selector.includes("send-video")) {
return params.camera;
}
if (
selector.includes("preview-audio-control-button") ||
selector.includes("mute my microphone") ||
selector.includes("unmute my microphone")
) {
return params.microphone;
}
if (
params.kind === "passcode" &&
(selector.includes("password") || selector.includes("passcode"))
) {
return params.challenge;
}
if (
params.kind === "captcha" &&
(selector.includes("recaptcha") ||
selector.includes("captcha") ||
selector.includes("data-sitekey"))
) {
return params.challenge;
}
return undefined;
},
querySelectorAll: (selector: string) =>
selector === "button" || selector.includes('[role="button"]') ? controls : [],
};
}
async function runStatusFixture(params: {
allowMicrophone?: boolean;
currentUrl?: string;
document: ReturnType<typeof statusDocument>;
navigator?: Record<string, unknown>;
readOnly?: boolean;
window?: Record<string, unknown>;
}) {
const result = await runInNewContext(
`(${zoomMeetingStatusScript({
allowMicrophone: params.allowMicrophone ?? false,
allowSessionAdoption: true,
autoJoin: true,
captureCaptions: false,
guestName: "OpenClaw Agent",
meetingSessionId: "session-1",
meetingUrl: URL,
readOnly: params.readOnly,
waitForInCallMs: 60_000,
})})()`,
{
URL: globalThis.URL,
crypto: { randomUUID: () => "caption-id" },
document: params.document,
location: new globalThis.URL(params.currentUrl ?? URL),
navigator: params.navigator ?? {},
setTimeout,
clearTimeout,
window: params.window ?? {},
},
);
return JSON.parse(result) as Record<string, unknown>;
}
function status(reason: string) {
const health = ZOOM_MEETINGS_PLATFORM_ADAPTER.browser.parseStatus({
result: JSON.stringify({
inCall: false,
manualActionRequired: true,
manualActionReason: reason,
manualActionMessage: "manual action",
url: URL,
}),
});
if (!health) {
throw new Error("expected parsed health");
}
return health;
}
describe("Zoom meeting platform adapter", () => {
it("preserves host-ended state from the browser status", () => {
expect(
ZOOM_MEETINGS_PLATFORM_ADAPTER.browser.parseStatus({
result: JSON.stringify({ inCall: false, meetingEnded: true }),
}),
).toMatchObject({ inCall: false, meetingEnded: true });
});
it.each([
["zoom-login-required", "login-required"],
["zoom-admission-required", "admission-required"],
["zoom-passcode-required", "admission-required"],
["zoom-captcha-required", "admission-required"],
["zoom-permission-required", "permission-required"],
["zoom-audio-choice-required", "audio-choice-required"],
["zoom-session-conflict", "session-conflict"],
["browser-control-unavailable", "browser-control-unavailable"],
])("classifies %s as %s", (reason, category) => {
expect(ZOOM_MEETINGS_PLATFORM_ADAPTER.browser.classifyManualAction(status(reason))).toEqual({
category,
reason,
message: "manual action",
});
});
it("grants microphone and optional output routing on the Zoom Web App origin", () => {
expect(
ZOOM_MEETINGS_PLATFORM_ADAPTER.browser.permissions({
allowMicrophone: true,
meetingUrl: URL,
}),
).toEqual({
origin: "https://app.zoom.us",
permissions: ["audioCapture"],
optionalPermissions: ["speakerSelection"],
});
expect(
ZOOM_MEETINGS_PLATFORM_ADAPTER.browser.permissions({
allowMicrophone: false,
meetingUrl: URL,
}),
).toBeUndefined();
});
it("builds the live-validated guest, iframe, audio, leave, and caption controls", () => {
const script = zoomMeetingStatusScript({
allowMicrophone: true,
allowSessionAdoption: true,
autoJoin: true,
captureCaptions: true,
guestName: "OpenClaw Agent",
meetingSessionId: "session-1",
meetingUrl: URL,
waitForInCallMs: 60_000,
});
expect(script).toContain("#webclient");
expect(script).toContain("input#input-for-name");
expect(script).toContain("#preview-audio-control-button");
expect(script).toContain("#preview-video-control-button");
expect(script).toContain('aria-label=\\\"Leave\\\"');
expect(script).toContain("live-transcription-subtitle__box");
expect(script).toContain("live-transcription-subtitle__item");
expect(script).toContain("zmu-data-selector-item__icon");
expect(script).toContain("audio-option-menu__pop-menu");
expect(script).toContain("videooff");
expect(script).toContain("my )?(?:microphone|mic)");
expect(script).toContain("join from browser");
expect(script).toContain("host will let you in soon");
expect(script).toContain("setSinkId");
expect(script).toContain("BlackHole");
});
it("enables caption snapshots only in transcribe mode", () => {
expect(ZOOM_MEETINGS_PLATFORM_ADAPTER.browser.captions.enabled("transcribe")).toBe(true);
expect(ZOOM_MEETINGS_PLATFORM_ADAPTER.browser.captions.enabled("agent")).toBe(false);
});
it("requires verified bidirectional audio before realtime startup", () => {
expect(
isZoomMeetingsRealtimeRouteReady("agent", {
inCall: true,
micMuted: false,
audioInputRouted: true,
audioOutputRouted: true,
}),
).toBe(true);
expect(
isZoomMeetingsRealtimeRouteReady("agent", {
inCall: true,
micMuted: true,
audioInputRouted: true,
audioOutputRouted: true,
}),
).toBe(false);
});
it("recognizes the Zoom Web App home redirect as completed leave", () => {
const state = { identity: "zoom:12345678901", sessionId: "session-1" };
const document = {
body: { textContent: "Join Meeting" },
querySelector: () => undefined,
querySelectorAll: () => [],
};
const result = runInNewContext(
`(${zoomMeetingLeaveScript({
leaveInitiated: true,
meetingSessionId: "session-1",
meetingUrl: URL,
})})()`,
{
URL: globalThis.URL,
document,
location: new globalThis.URL("https://app.zoom.us/wc?ref_from=waffle_zwa"),
window: { __openclawZoomMeeting: state },
},
);
expect(JSON.parse(result)).toEqual({
departed: true,
sessionMatched: true,
urlMatched: true,
});
});
it("does not treat a live Web App home page or meeting text as post-call", () => {
const leave = pageControl("Leave");
const document = {
body: { textContent: "Someone said the meeting has ended" },
querySelector: (selector: string) =>
selector.includes('aria-label="Leave"') ? leave : undefined,
querySelectorAll: (selector: string) =>
selector === "button"
? [leave]
: selector.includes("main")
? [{ textContent: "Someone said the meeting has ended" }]
: [],
};
const result = runInNewContext(
`(${zoomMeetingLeaveScript({
leaveInitiated: false,
meetingSessionId: "session-1",
meetingUrl: URL,
})})()`,
{
URL: globalThis.URL,
document,
location: new globalThis.URL("https://app.zoom.us/wc"),
window: {
__openclawZoomMeeting: {
identity: "zoom:12345678901",
inCallControl: leave,
inCallUrl: "https://app.zoom.us/wc",
sessionId: "session-1",
},
},
},
);
expect(JSON.parse(result)).toMatchObject({ departed: false, leaveAction: "leave" });
expect(leave.click).toHaveBeenCalledOnce();
});
it("re-adopts a verified meeting URL after a full document reload before leaving", () => {
const leave = {
disabled: false,
isConnected: true,
textContent: "Leave",
click: vi.fn(),
closest: () => leave,
getAttribute: (name: string) => (name === "aria-label" ? "Leave" : null),
matches: (selector: string) => selector === "button",
querySelector: () => undefined,
};
const document = {
body: { textContent: "" },
querySelector: (selector: string) =>
selector.includes('aria-label="Leave"') ? leave : undefined,
querySelectorAll: (selector: string) => (selector === "button" ? [leave] : []),
};
const window: Record<string, unknown> = {};
const result = runInNewContext(
`(${zoomMeetingLeaveScript({
leaveInitiated: false,
meetingSessionId: "session-1",
meetingUrl: URL,
})})()`,
{
URL: globalThis.URL,
document,
location: new globalThis.URL(URL),
window,
},
);
expect(JSON.parse(result)).toMatchObject({ leaveAction: "leave", urlMatched: true });
expect(leave.click).toHaveBeenCalledOnce();
expect(window).toMatchObject({
__openclawZoomMeeting: { identity: "zoom:12345678901", sessionId: "session-1" },
});
});
it.each([
{
bodyText: "Enter meeting passcode",
kind: "passcode",
reason: "zoom-passcode-required",
},
{
bodyText: "Security check",
kind: "captcha",
reason: "zoom-captcha-required",
},
])("reports a $kind challenge before camera gating", async ({ bodyText, kind, reason }) => {
const join = pageControl("Join");
const challenge = pageControl(kind === "passcode" ? "Meeting passcode" : "CAPTCHA");
const document = statusDocument({ bodyText, challenge, join, kind });
const result = await runStatusFixture({ document });
expect(result).toMatchObject({
clickedJoin: false,
manualActionReason: reason,
manualActionRequired: true,
});
expect(join.click).not.toHaveBeenCalled();
});
it("replaces Zoom's remembered guest name before joining", async () => {
const guest = guestInput("Remembered Human");
const join = pageControl("Join");
const result = await runStatusFixture({
document: statusDocument({
bodyText: "",
camera: pageControl("Start Video"),
guest,
join,
microphone: pageControl("Unmute my microphone"),
}),
});
expect(guest.value).toBe("OpenClaw Agent");
expect(guest.dispatchEvent).toHaveBeenCalledTimes(2);
expect(join.click).toHaveBeenCalledOnce();
expect(result).toMatchObject({ clickedJoin: true, manualActionRequired: false });
});
it("persists Zoom's confirmed no-device state for observe-only joins", async () => {
const window: Record<string, unknown> = {};
const devicePrompt = pageControl("Continue without audio or video");
devicePrompt.click.mockImplementation(() => {
devicePrompt.disabled = true;
});
const first = await runStatusFixture({
document: statusDocument({
bodyText: "",
devicePrompt,
join: pageControl("Join"),
}),
window,
});
const second = await runStatusFixture({
document: statusDocument({ bodyText: "", join: pageControl("Join") }),
readOnly: true,
window,
});
expect(devicePrompt.click).toHaveBeenCalled();
expect(first.manualActionReason).toBeUndefined();
expect(first).toMatchObject({
cameraOff: true,
clickedJoin: true,
manualActionRequired: false,
micMuted: true,
});
expect(second).toMatchObject({
cameraOff: true,
manualActionRequired: false,
micMuted: true,
});
expect(window["__openclawZoomMeeting"]).toMatchObject({ devicesDisabled: true });
});
it("re-mutes an observe-only session after admission", async () => {
let microphoneLabel = "Mute my microphone";
const microphone = pageControl(microphoneLabel);
microphone.getAttribute = (name: string) => (name === "aria-label" ? microphoneLabel : null);
microphone.click.mockImplementation(() => {
microphoneLabel = "Unmute my microphone";
microphone.textContent = microphoneLabel;
});
const result = await runStatusFixture({
document: statusDocument({
bodyText: "",
camera: pageControl("Start Video"),
leave: pageControl("Leave"),
microphone,
}),
});
expect(result).toMatchObject({ inCall: true, micMuted: true, manualActionRequired: false });
expect(microphone.click).toHaveBeenCalledOnce();
});
it("fails closed when an in-call observe-only microphone cannot be verified", async () => {
const result = await runStatusFixture({
document: statusDocument({
bodyText: "",
camera: pageControl("Start Video"),
leave: pageControl("Leave"),
}),
});
expect(result).toMatchObject({
inCall: true,
manualActionReason: "zoom-microphone-required",
manualActionRequired: true,
});
});
it("turns off an adopted in-call camera", async () => {
let cameraLabel = "Stop Video";
const camera = pageControl(cameraLabel);
camera.getAttribute = (name: string) => (name === "aria-label" ? cameraLabel : null);
camera.click.mockImplementation(() => {
cameraLabel = "Start Video";
camera.textContent = cameraLabel;
});
const result = await runStatusFixture({
document: statusDocument({
bodyText: "",
camera,
leave: pageControl("Leave"),
microphone: pageControl("Unmute my microphone"),
}),
});
expect(result).toMatchObject({ cameraOff: true, inCall: true, manualActionRequired: false });
expect(camera.click).toHaveBeenCalledOnce();
});
it("fails closed when an adopted in-call camera cannot be verified", async () => {
const result = await runStatusFixture({
document: statusDocument({
bodyText: "",
leave: pageControl("Leave"),
microphone: pageControl("Unmute my microphone"),
}),
});
expect(result).toMatchObject({
inCall: true,
manualActionReason: "zoom-camera-required",
manualActionRequired: true,
});
});
it("marks a previously verified call ended after its control stays gone", async () => {
const result = await runStatusFixture({
currentUrl: "https://app.zoom.us/wc",
document: statusDocument({ bodyText: "" }),
window: {
__openclawZoomMeeting: {
identity: "zoom:12345678901",
inCallControl: { isConnected: false },
inCallControlLostAt: Date.now() - 6_000,
inCallUrl: "https://app.zoom.us/wc/12345678901/join",
sessionId: "session-1",
verifiedAt: Date.now() - 6_000,
},
},
});
expect(result).toMatchObject({ inCall: false, meetingEnded: true });
});
it("re-adopts a replacement Leave control after a late Zoom rerender", async () => {
const result = await runStatusFixture({
currentUrl: "https://app.zoom.us/wc/12345678901/join",
document: statusDocument({
bodyText: "",
camera: pageControl("Start Video"),
leave: pageControl("Leave"),
microphone: pageControl("Unmute my microphone"),
}),
readOnly: true,
window: {
__openclawZoomMeeting: {
identity: "zoom:12345678901",
inCallControl: { isConnected: false },
inCallUrl: "https://app.zoom.us/wc/12345678901/join",
sessionId: "session-1",
verifiedAt: Date.now() - 60_000,
},
},
});
expect(result).toMatchObject({ inCall: true, meetingEnded: false });
});
it("does not trust a cached BlackHole device after Zoom hides its current selection", async () => {
const meetingState = {
audioInputDeviceId: "blackhole-device",
identity: "zoom:12345678901",
sessionId: "session-1",
};
const result = await runStatusFixture({
allowMicrophone: true,
document: statusDocument({
bodyText: "",
camera: pageControl("Start Video"),
leave: pageControl("Leave"),
microphone: pageControl("Mute my microphone"),
}),
navigator: {
mediaDevices: {
enumerateDevices: vi.fn(async () => [
{ deviceId: "blackhole-device", kind: "audioinput", label: "BlackHole 2ch" },
]),
},
},
readOnly: true,
window: { __openclawZoomMeeting: meetingState },
});
expect(result).toMatchObject({
audioInputRouted: false,
manualActionReason: "zoom-audio-choice-required",
manualActionRequired: true,
});
expect(meetingState).not.toHaveProperty("audioInputDeviceId");
});
it("retains meeting ownership through an unbounded lobby wait", async () => {
const window: Record<string, unknown> = {};
const waiting = await runStatusFixture({
document: statusDocument({ bodyText: "The host will let you in soon" }),
window,
});
const marker = window["__openclawZoomMeeting"] as Record<string, unknown>;
marker.verifiedAt = 0;
const admitted = await runStatusFixture({
currentUrl: "https://app.zoom.us/wc",
document: statusDocument({
bodyText: "",
leave: pageControl("Leave"),
microphone: pageControl("Unmute my microphone"),
}),
window,
});
expect(waiting).toMatchObject({
lobbyWaiting: true,
manualActionReason: "zoom-admission-required",
});
expect(admitted).toMatchObject({ inCall: true, micMuted: true });
expect(window["__openclawZoomMeeting"]).toMatchObject({
awaitingAdmission: false,
identity: "zoom:12345678901",
sessionId: "session-1",
});
});
});
@@ -0,0 +1,345 @@
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import type {
MeetingBrowserJoinSession,
MeetingManualActionCategory,
MeetingPlatformAdapter,
} from "openclaw/plugin-sdk/meeting-runtime";
import type { ZoomMeetingsMode } from "../config.js";
import type { ZoomMeetingsChromeHealth, ZoomMeetingsTranscriptSnapshot } from "./types.js";
import {
zoomMeetingLeaveScript,
zoomMeetingStatusScript,
zoomMeetingTranscriptScript,
} from "./zoom-meetings-page-scripts.js";
import { ZOOM_MEETINGS_NODE_COMMAND } from "./zoom-meetings-platform-constants.js";
import {
isRecoverableZoomMeetingTab,
isSameZoomMeetingUrl,
normalizeZoomMeetingUrl,
normalizeZoomMeetingUrlForReuse,
} from "./zoom-meetings-urls.js";
function zoomMeetingOrigin(meetingUrl: string): string | undefined {
return normalizeZoomMeetingUrlForReuse(meetingUrl) ? "https://app.zoom.us" : undefined;
}
function parsePermissionGrantNotes(result: unknown): string[] {
const record = result && typeof result === "object" ? (result as Record<string, unknown>) : {};
const unsupportedPermissions = Array.isArray(record.unsupportedPermissions)
? record.unsupportedPermissions.filter((value): value is string => typeof value === "string")
: [];
const notes = ["Granted Zoom microphone permission through browser control."];
if (unsupportedPermissions.includes("speakerSelection")) {
notes.push("Chrome did not accept the optional Zoom speaker-selection permission.");
}
return notes;
}
export function isZoomMeetingsTalkBackMode(mode: ZoomMeetingsMode): boolean {
return mode === "agent" || mode === "bidi";
}
export function isZoomMeetingsRealtimeRouteReady(
mode: ZoomMeetingsMode,
health: ZoomMeetingsChromeHealth | undefined,
): boolean {
return (
isZoomMeetingsTalkBackMode(mode) &&
health?.inCall === true &&
health.micMuted === false &&
health.audioInputRouted === true &&
health.audioOutputRouted === true &&
health.manualActionRequired !== true
);
}
function parseBrowserStatus(result: unknown): ZoomMeetingsChromeHealth | undefined {
const record = result && typeof result === "object" ? (result as Record<string, unknown>) : {};
if (typeof record.result !== "string" || !record.result.trim()) {
return undefined;
}
let parsed: Record<string, unknown>;
try {
parsed = JSON.parse(record.result) as Record<string, unknown>;
} catch {
throw new Error("Zoom browser status JSON is malformed.");
}
return {
inCall: typeof parsed.inCall === "boolean" ? parsed.inCall : undefined,
meetingEnded: typeof parsed.meetingEnded === "boolean" ? parsed.meetingEnded : undefined,
micMuted: typeof parsed.micMuted === "boolean" ? parsed.micMuted : undefined,
cameraOff: typeof parsed.cameraOff === "boolean" ? parsed.cameraOff : undefined,
lobbyWaiting: typeof parsed.lobbyWaiting === "boolean" ? parsed.lobbyWaiting : undefined,
captionCaptureRequested:
typeof parsed.captionCaptureRequested === "boolean"
? parsed.captionCaptureRequested
: undefined,
captioning: typeof parsed.captioning === "boolean" ? parsed.captioning : undefined,
captionsEnabledAttempted:
typeof parsed.captionsEnabledAttempted === "boolean"
? parsed.captionsEnabledAttempted
: undefined,
transcriptLines:
typeof parsed.transcriptLines === "number" ? parsed.transcriptLines : undefined,
lastCaptionAt: typeof parsed.lastCaptionAt === "string" ? parsed.lastCaptionAt : undefined,
lastCaptionSpeaker:
typeof parsed.lastCaptionSpeaker === "string" ? parsed.lastCaptionSpeaker : undefined,
lastCaptionText:
typeof parsed.lastCaptionText === "string" ? parsed.lastCaptionText : undefined,
recentTranscript: Array.isArray(parsed.recentTranscript)
? parsed.recentTranscript.flatMap((value) => {
if (!value || typeof value !== "object") {
return [];
}
const line = value as { at?: unknown; speaker?: unknown; text?: unknown };
if (typeof line.text !== "string" || !line.text.trim()) {
return [];
}
return [
{
...(typeof line.at === "string" ? { at: line.at } : {}),
...(typeof line.speaker === "string" ? { speaker: line.speaker } : {}),
text: line.text,
},
];
})
: undefined,
audioInputRouted:
typeof parsed.audioInputRouted === "boolean" ? parsed.audioInputRouted : undefined,
audioInputDeviceLabel:
typeof parsed.audioInputDeviceLabel === "string" ? parsed.audioInputDeviceLabel : undefined,
audioInputRouteError:
typeof parsed.audioInputRouteError === "string" ? parsed.audioInputRouteError : undefined,
audioOutputRouted:
typeof parsed.audioOutputRouted === "boolean" ? parsed.audioOutputRouted : undefined,
audioOutputDeviceLabel:
typeof parsed.audioOutputDeviceLabel === "string" ? parsed.audioOutputDeviceLabel : undefined,
audioOutputRouteError:
typeof parsed.audioOutputRouteError === "string" ? parsed.audioOutputRouteError : undefined,
audioOutputRouteRetryable:
typeof parsed.audioOutputRouteRetryable === "boolean"
? parsed.audioOutputRouteRetryable
: undefined,
manualActionRequired:
typeof parsed.manualActionRequired === "boolean" ? parsed.manualActionRequired : undefined,
manualActionReason:
typeof parsed.manualActionReason === "string"
? (parsed.manualActionReason as ZoomMeetingsChromeHealth["manualActionReason"])
: undefined,
manualActionMessage:
typeof parsed.manualActionMessage === "string" ? parsed.manualActionMessage : undefined,
browserUrl: typeof parsed.url === "string" ? parsed.url : undefined,
browserTitle: typeof parsed.title === "string" ? parsed.title : undefined,
status: "browser-control",
notes: Array.isArray(parsed.notes)
? parsed.notes.filter((note): note is string => typeof note === "string")
: undefined,
};
}
function classifyManualAction(health: ZoomMeetingsChromeHealth) {
if (!health.manualActionRequired || !health.manualActionReason || !health.manualActionMessage) {
return undefined;
}
const category: MeetingManualActionCategory =
health.manualActionReason === "zoom-login-required"
? "login-required"
: health.manualActionReason === "zoom-admission-required"
? "admission-required"
: health.manualActionReason === "zoom-passcode-required" ||
health.manualActionReason === "zoom-captcha-required"
? "admission-required"
: health.manualActionReason === "zoom-permission-required"
? "permission-required"
: health.manualActionReason === "zoom-audio-choice-required"
? "audio-choice-required"
: health.manualActionReason === "zoom-session-conflict"
? "session-conflict"
: health.manualActionReason === "browser-control-unavailable"
? "browser-control-unavailable"
: "custom";
return {
category,
reason: health.manualActionReason,
message: health.manualActionMessage,
};
}
function parseLeaveResult(result: unknown): {
departed: boolean;
leaveAction?: "leave" | "confirm";
sessionConflict?: boolean;
sessionMatched?: boolean;
urlMatched?: boolean;
} {
const record = result && typeof result === "object" ? (result as Record<string, unknown>) : {};
if (typeof record.result !== "string" || !record.result.trim()) {
return { departed: false };
}
try {
const parsed = JSON.parse(record.result) as Record<string, unknown>;
const leaveAction =
parsed.leaveAction === "leave" || parsed.leaveAction === "confirm"
? parsed.leaveAction
: undefined;
return {
departed: parsed.departed === true,
...(leaveAction ? { leaveAction } : {}),
...(typeof parsed.sessionConflict === "boolean"
? { sessionConflict: parsed.sessionConflict }
: {}),
...(typeof parsed.sessionMatched === "boolean"
? { sessionMatched: parsed.sessionMatched }
: {}),
...(typeof parsed.urlMatched === "boolean" ? { urlMatched: parsed.urlMatched } : {}),
};
} catch {
return { departed: false };
}
}
function parseTranscript(
result: unknown,
): ZoomMeetingsTranscriptSnapshot & { sessionMatched?: boolean; urlMatched?: boolean } {
const record = result && typeof result === "object" ? (result as Record<string, unknown>) : {};
if (typeof record.result !== "string" || !record.result.trim()) {
return { droppedLines: 0, lines: [] };
}
let parsed: unknown;
try {
parsed = JSON.parse(record.result);
} catch {
throw new Error("Zoom transcript JSON is malformed.");
}
if (!parsed || typeof parsed !== "object") {
throw new Error("Zoom transcript payload is invalid.");
}
const payload = parsed as {
droppedLines?: unknown;
epoch?: unknown;
lines?: unknown;
sessionMatched?: unknown;
urlMatched?: unknown;
};
const droppedLines =
typeof payload.droppedLines === "number" && Number.isSafeInteger(payload.droppedLines)
? Math.max(0, payload.droppedLines)
: 0;
const lines = Array.isArray(payload.lines)
? payload.lines.flatMap((value) => {
if (!value || typeof value !== "object") {
return [];
}
const line = value as { at?: unknown; speaker?: unknown; text?: unknown };
if (typeof line.text !== "string" || !line.text.trim()) {
return [];
}
return [
{
...(typeof line.at === "string" ? { at: line.at } : {}),
...(typeof line.speaker === "string" ? { speaker: line.speaker } : {}),
text: line.text,
},
];
})
: [];
return {
droppedLines,
...(typeof payload.epoch === "string" ? { epoch: payload.epoch } : {}),
lines,
...(typeof payload.urlMatched === "boolean" ? { urlMatched: payload.urlMatched } : {}),
...(typeof payload.sessionMatched === "boolean"
? { sessionMatched: payload.sessionMatched }
: {}),
};
}
export const ZOOM_MEETINGS_PLATFORM_ADAPTER: MeetingPlatformAdapter<
MeetingBrowserJoinSession<ZoomMeetingsMode>,
ZoomMeetingsMode,
ZoomMeetingsChromeHealth,
ZoomMeetingsTranscriptSnapshot
> = {
id: "zoom-meetings",
displayName: "Zoom meetings",
browserLabel: "Zoom meeting",
logScope: "[zoom-meetings]",
nodeCommandName: ZOOM_MEETINGS_NODE_COMMAND,
nodeConfigPath: "plugins.entries.zoom-meetings.config.chromeNode.node",
urls: {
validateAndNormalize: normalizeZoomMeetingUrl,
normalizeForReuse: normalizeZoomMeetingUrlForReuse,
isSameMeeting: isSameZoomMeetingUrl,
buildJoinUrl: (session) => session.url,
accountHint: () => undefined,
isPreferredJoinUrl: (url) => Boolean(normalizeZoomMeetingUrlForReuse(url)),
isRecoverableTab: isRecoverableZoomMeetingTab,
localeAction: () => undefined,
},
browser: {
allowsMicrophone: isZoomMeetingsTalkBackMode,
buildStatusJoinScript: (params) =>
zoomMeetingStatusScript({
allowMicrophone: isZoomMeetingsTalkBackMode(params.mode),
allowSessionAdoption: params.allowSessionAdoption,
autoJoin: params.autoJoin,
captureCaptions: params.captureCaptions,
guestName: params.guestName,
meetingSessionId: params.meetingSessionId || undefined,
meetingUrl: params.url,
readOnly: params.readOnly,
waitForInCallMs: params.waitForInCallMs,
}),
parseStatus: parseBrowserStatus,
classifyManualAction,
shouldRetryJoinStatus: (health) =>
health.inCall === true &&
((health.manualActionReason === "zoom-audio-choice-required" &&
health.audioInputRouted === true &&
health.audioOutputRouteRetryable === true) ||
(health.manualActionRequired !== true &&
health.captionCaptureRequested === true &&
health.captioning !== true)),
browserControlUnavailable: () => ({
category: "browser-control-unavailable",
reason: "browser-control-unavailable",
message:
"Open the OpenClaw browser profile, finish the Zoom sign-in, admission, or permission prompt, then retry.",
}),
buildLeaveScript: (meetingUrl) =>
zoomMeetingLeaveScript({
leaveInitiated: false,
meetingSessionId: "",
meetingUrl,
}),
buildSessionLeaveScript: zoomMeetingLeaveScript,
parseLeaveResult,
captions: {
enabled: (mode) => mode === "transcribe",
buildTranscriptScript: ({ finalize, meetingSessionId, meetingUrl }) =>
zoomMeetingTranscriptScript(meetingUrl, meetingSessionId, finalize),
parseTranscript,
},
permissions: ({ allowMicrophone, meetingUrl }) => {
const origin = zoomMeetingOrigin(meetingUrl);
return allowMicrophone && origin
? {
origin,
permissions: ["audioCapture"],
optionalPermissions: ["speakerSelection"],
}
: undefined;
},
permissionNotes: ({ allowMicrophone, error, result }) => {
if (!allowMicrophone) {
return ["Observe-only mode does not request Zoom microphone access."];
}
if (error) {
return [
`Could not grant Zoom media permissions automatically: ${formatErrorMessage(error)}`,
];
}
return parsePermissionGrantNotes(result);
},
},
};
@@ -0,0 +1,7 @@
export const ZOOM_MEETINGS_NODE_COMMAND = "zoommeetings.chrome";
export const ZOOM_MEETINGS_BROWSER_NODE_ADAPTER = {
displayName: "Zoom meetings",
nodeCommandName: ZOOM_MEETINGS_NODE_COMMAND,
nodeConfigPath: "plugins.entries.zoom-meetings.config.chromeNode.node",
};
@@ -0,0 +1,73 @@
// Zoom Web App selectors validated against the live guest surface on 2026-07-18.
// Prefer Zoom-owned ids and accessibility labels; text remains the fallback where
// the app-launch and nested captions menus expose no stable product identifier.
export const ZOOM_MEETING_SELECTORS = {
continueInBrowser: [],
guestName: ["input#input-for-name"],
join: ["button.preview-join-button"],
microphone: [
"button#preview-audio-control-button",
'button[aria-label="mute my microphone" i]',
'button[aria-label="unmute my microphone" i]',
],
camera: ["button#preview-video-control-button", "button.send-video-container__btn"],
deviceSettings: ['button[aria-label="More audio controls" i]'],
microphoneDevice: ['[aria-label*="microphone" i][role="combobox"]'],
microphoneDeviceMenu: [".audio-option-menu__pop-menu", '[role="listbox"]', '[role="menu"]'],
selectedMicrophoneDevice: [
'a[role="button"][aria-label^="Select a microphone" i][aria-label$="selected" i]',
"option:checked",
'[role="option"][aria-selected="true"]',
'[role="menuitemradio"][aria-checked="true"]',
],
audioDeviceOptions: [
'a[role="button"][aria-label^="Select a microphone" i]',
"option",
'[role="option"]',
'[role="menuitemradio"]',
],
leave: ['button[aria-label="Leave" i]'],
leaveConfirmation: [
"button.leave-meeting-options__btn",
"button.zm-btn--danger",
'button[aria-label="Leave Meeting" i]',
],
postCall: [".meeting-ended", ".post-meeting", ".leave-meeting-page"],
lobby: [".waiting-room-container", '[class*="waiting-room"]', '[class*="waitingRoom"]'],
signIn: ['a[href*="/signin"]', 'button[aria-label*="sign in" i]'],
passcode: [
'input[type="password"]',
'input[id*="passcode" i]',
'input[name*="passcode" i]',
'input[aria-label*="passcode" i]',
],
captcha: [
'iframe[src*="recaptcha" i]',
'iframe[title*="captcha" i]',
".g-recaptcha",
"[data-sitekey]",
'[class*="captcha" i]',
],
permissionPrompt: [".pepc-permission-dialog"],
moreActions: ["button.more-button", ".footer-more-button button"],
captions: [
'a[aria-label*="Show Captions" i]',
'a[aria-label="Captions" i]',
'[role="button"][aria-label="Captions" i]',
],
captionsOff: ['a[aria-label*="Hide Captions" i]'],
captionRenderer: [".live-transcription-subtitle__box"],
captionContent: ["body"],
captionRows: ["#live-transcription-subtitle"],
captionAuthor: [
".zmu-data-selector-item__icon",
".live-transcription-subtitle__speaker",
'[class*="transcription"][class*="speaker"]',
],
captionText: [
".live-transcription-subtitle__item",
".live-transcription-subtitle__text",
'[class*="transcription"][class*="text"]',
".live-transcription-subtitle__box",
],
} as const;
@@ -0,0 +1,17 @@
export function zoomMeetingStatusAccessSource(): string {
return ` const passcodeInput = firstRaw(selectors.passcode);
const passcodeRequired = Boolean(passcodeInput) &&
/meeting passcode|enter (?:the )?passcode|invalid passcode|incorrect passcode/i.test(
pageText + " " + label(passcodeInput)
);
const captchaRequired = Boolean(firstRaw(selectors.captcha)) ||
/complete (?:the )?captcha|security check|verify (?:that )?you(?:'re| are) (?:a )?human/i.test(pageTextLower);
if (identityVerified && !inCall && passcodeRequired) {
controlManualActionReason = "zoom-passcode-required";
controlManualActionMessage = "Enter the Zoom meeting passcode in the OpenClaw browser profile, then retry joining.";
} else if (identityVerified && !inCall && captchaRequired) {
controlManualActionReason = "zoom-captcha-required";
controlManualActionMessage = "Complete Zoom's security check in the OpenClaw browser profile, then retry joining.";
}
`;
}
@@ -0,0 +1,669 @@
const ZOOM_MEETING_CAPTION_SETTLE_MS = 1_000;
const ZOOM_MEETING_TRANSCRIPT_MAX_LINES = 500;
export function zoomMeetingStatusCallSource(): string {
return ` let audioOutputRouted;
let audioOutputDeviceLabel;
let audioOutputRouteError;
let audioOutputRouteRetryable = false;
if (inCall && allowMicrophone && navigator.mediaDevices?.enumerateDevices) {
const media = [...document.querySelectorAll("audio, video")].filter(
(element) =>
typeof element.setSinkId === "function" &&
!String(element.id || "").startsWith("openclaw-zoom-audio-output-"),
);
if (media.length > 0) {
try {
const devices = await navigator.mediaDevices.enumerateDevices();
const output = devices.find((device) => device.kind === "audiooutput" && isBlackHole(device.label));
if (output?.deviceId) {
const routeErrors = [];
const liveStream = (element) =>
element.srcObject?.getAudioTracks?.().some((track) => track.readyState === "live")
? element.srcObject
: undefined;
const allBridgeEntries = Array.isArray(window.__openclawZoomAudioOutputs)
? window.__openclawZoomAudioOutputs
: [];
const retainedBridgeEntries = allBridgeEntries.filter((entry) => !bridgeOwnedBySession(entry));
const previousBridgeEntries = allBridgeEntries.filter(bridgeOwnedBySession);
const originalMuteBySource = new Map(previousBridgeEntries.flatMap((entry) =>
bridgeSources(entry).flatMap((source) =>
source?.element ? [[source.element, Boolean(source.muted)]] : []
)
));
const bridgedElements = new Set(previousBridgeEntries.flatMap((entry) =>
bridgeSources(entry).map((source) => source?.element).filter(Boolean)
));
const routeCandidates = media
.map((element) => ({ element, stream: liveStream(element) }))
// Zoom mutes local/self-view and intentionally suppressed playback. Preserve
// that product decision; only our own already-bridged source stays eligible.
.filter((entry) => !entry.element.muted || bridgedElements.has(entry.element));
// The self-view often exists before Zoom attaches remote playback. With the
// required output present, an all-filtered list is still a transient DOM state.
if (routeCandidates.length === 0) audioOutputRouteRetryable = true;
if (canMutateSession) {
for (const { element } of routeCandidates) {
if (!originalMuteBySource.has(element)) {
originalMuteBySource.set(element, Boolean(element.muted));
}
// Sink changes are asynchronous. Silence the physical output until either
// the source or its fallback bridge is confirmed on BlackHole.
element.muted = true;
}
}
const currentSources = new Set(routeCandidates.map((entry) => entry.element));
const bridgeEntries = previousBridgeEntries.filter((entry) =>
entry?.source &&
entry?.stream === liveStream(entry.source) &&
entry?.bridge?.isConnected &&
currentSources.has(entry.source)
);
const suspendedBySource = new Map();
for (const entry of previousBridgeEntries) {
if (bridgeEntries.includes(entry)) continue;
for (const source of bridgeSources(entry)) {
if (
!source?.element ||
source.muted ||
!bridgeSourceMatches(source.element, source)
) continue;
const sourceStillPresent = currentSources.has(source.element);
const detachedLiveSource = !sourceStillPresent && Boolean(liveStream(source.element));
if (!sourceStillPresent && !detachedLiveSource) continue;
suspendedBySource.set(source.element, {
detached: detachedLiveSource,
sessionId: entry.sessionId || sessionId,
source: source.element,
sourceMuted: false,
sourceUrl: mediaSourceUrl(source.element) || source.url,
stream: source.element.srcObject,
suspended: true,
});
}
}
if (canMutateSession) {
// One bridge owns one Zoom playback element. Stream or element replacement
// retires that bridge so it cannot keep playing or satisfy route verification.
previousBridgeEntries.filter((entry) => !bridgeEntries.includes(entry)).forEach((entry) => {
for (const source of bridgeSources(entry)) {
if (
!source?.element ||
suspendedBySource.has(source.element) ||
currentSources.has(source.element)
) continue;
restoreAudioBridgeSource(source);
}
// Reused current elements stay silent until this pass confirms their
// replacement source; unrelated exact sources were restored above.
retireAudioBridge(entry, false);
});
}
const routed = [];
for (const { element, stream } of routeCandidates) {
let entry = bridgeEntries.find((candidate) => candidate.source === element);
let elementRouted = element.sinkId === output.deviceId;
let directRouteError;
if (canMutateSession && !elementRouted) {
try {
await element.setSinkId(output.deviceId);
elementRouted = element.sinkId === output.deviceId;
} catch (error) {
directRouteError = {
message: error?.message || String(error),
retryable: error?.name === "AbortError",
};
}
}
if (elementRouted && entry && canMutateSession) {
const bridgedIndex = bridgeEntries.indexOf(entry);
if (bridgedIndex >= 0) {
const [bridged] = bridgeEntries.splice(bridgedIndex, 1);
retireAudioBridge(bridged);
entry = undefined;
}
}
// Direct sink routing is valid for src/MediaSource and pre-attachment elements.
// A live MediaStream is required only when the hidden bridge fallback is needed.
if (elementRouted) {
if (canMutateSession && originalMuteBySource.has(element)) {
element.muted = originalMuteBySource.get(element);
}
suspendedBySource.delete(element);
routed.push(true);
continue;
}
if (!stream) {
const hasLoadedPlaybackSource = Number(element.readyState) > 0;
routed.push(false);
if (hasLoadedPlaybackSource && directRouteError) routeErrors.push(directRouteError);
if (!hasLoadedPlaybackSource) audioOutputRouteRetryable = true;
if (canMutateSession && originalMuteBySource.get(element) === false) {
// Zoom may attach the remote MediaStream after creating its media element.
// Keep it silent until a later serialized status poll routes that source.
suspendedBySource.set(element, {
sessionId,
pending: true,
source: element,
sourceMuted: false,
sourceUrl: mediaSourceUrl(element),
stream: element.srcObject,
suspended: true,
});
}
continue;
}
if (!elementRouted && stream) {
if (!entry && canMutateSession) {
const bridge = document.createElement("audio");
bridge.id = "openclaw-zoom-audio-output-" + bridgeEntries.length;
bridge.autoplay = false;
bridge.hidden = true;
bridge.srcObject = stream;
document.body.appendChild(bridge);
entry = {
bridge,
playing: false,
sessionId,
source: element,
sourceMuted: originalMuteBySource.has(element)
? originalMuteBySource.get(element)
: Boolean(element.muted),
sourceUrl: mediaSourceUrl(element),
stream,
};
bridgeEntries.push(entry);
suspendedBySource.delete(element);
}
if (entry?.bridge) {
try {
if (canMutateSession) {
if (entry.bridge.sinkId !== output.deviceId) {
await entry.bridge.setSinkId(output.deviceId);
}
await entry.bridge.play();
entry.playing = true;
}
elementRouted =
entry.bridge.sinkId === output.deviceId && entry.playing === true;
if (elementRouted) {
suspendedBySource.delete(element);
if (canMutateSession && !entry.sourceMuted) element.muted = true;
}
} catch (error) {
entry.playing = false;
if (canMutateSession) retireAudioBridge(entry, false);
routeErrors.push({
message: error?.message || String(error),
retryable: error?.name === "AbortError",
});
}
}
}
routed.push(elementRouted);
}
if (canMutateSession) {
const nextBridgeEntries = [
...retainedBridgeEntries,
...bridgeEntries,
...suspendedBySource.values(),
];
if (nextBridgeEntries.length > 0) {
window.__openclawZoomAudioOutputs = nextBridgeEntries;
} else {
delete window.__openclawZoomAudioOutputs;
}
}
audioOutputRouted = routed.length > 0 && routed.every(Boolean);
if (canMutateSession && !audioOutputRouted) suspendOwnedAudioBridges();
if (audioOutputRouted && bridgeEntries.length > 0) {
notes.push("Routed Zoom remote audio to BlackHole 2ch through MediaStream bridges.");
}
audioOutputDeviceLabel = output.label || "BlackHole 2ch";
// An unloaded Zoom media element can reject setSinkId before its stream
// arrives. Keep that state retryable; loaded-source failures are terminal.
if (!audioOutputRouted && routed.length > 0 && routeErrors.length > 0) {
audioOutputRouteError = routeErrors[routeErrors.length - 1]?.message;
audioOutputRouteRetryable = routeErrors.every((error) => error.retryable === true);
}
} else {
audioOutputRouted = false;
if (canMutateSession) suspendOwnedAudioBridges();
notes.push("BlackHole 2ch speaker output was not visible to Zoom.");
}
} catch (error) {
audioOutputRouted = false;
audioOutputRouteError = error?.message || String(error);
if (canMutateSession) suspendOwnedAudioBridges();
}
if (!audioOutputRouted && audioOutputRouteError) {
notes.push("Could not route Zoom speaker output to BlackHole 2ch: " + audioOutputRouteError);
}
} else {
audioOutputRouted = false;
try {
const devices = await navigator.mediaDevices.enumerateDevices();
const output = devices.find(
(device) => device.kind === "audiooutput" && isBlackHole(device.label)
);
if (output?.deviceId) {
// Zoom can briefly remove every media element during an in-call rerender.
// Retry only after proving the required output still exists.
audioOutputRouteRetryable = true;
audioOutputDeviceLabel = output.label || "BlackHole 2ch";
} else {
notes.push("BlackHole 2ch speaker output was not visible to Zoom.");
}
} catch (error) {
audioOutputRouteError = error?.message || String(error);
notes.push("Could not inspect Zoom speaker outputs: " + audioOutputRouteError);
}
// Suspend ownership until the source returns; call teardown retires it.
if (canMutateSession) suspendOwnedAudioBridges();
}
} else if (inCall && allowMicrophone) {
audioOutputRouted = false;
if (canMutateSession) retireOwnedAudioBridges();
}
let captioning = false;
let captionsEnabledAttempted = false;
let transcriptLines = 0;
let lastCaptionAt;
let lastCaptionSpeaker;
let lastCaptionText;
let recentTranscript = [];
const captionState = (() => {
let active = window.__openclawZoomCaptions;
const activeOwnedByRequest = Boolean(
!active || (sessionId && (!active.sessionId || active.sessionId === sessionId))
);
if (!identityVerified) {
if (identityAwaitingRerender && activeOwnedByRequest) return active;
if (canMutateSession && activeOwnedByRequest) finalizeOwnedCaptions();
return undefined;
}
if (!activeOwnedByRequest) {
const replacedPriorOwner = Boolean(
canMutateSession &&
active?.sessionId &&
active.sessionId !== sessionId
);
if (replacedPriorOwner) {
if (priorMeeting?.sessionId === active.sessionId) {
active.identity ||= priorMeeting.identity;
}
finalizeCaptionState(active);
}
else if (!canMutateSession || !captureCaptions || active?.finalized !== true) return undefined;
archiveFinalizedCaptions(active);
if (active.settleTimer !== undefined) clearTimeout(active.settleTimer);
active.observer?.disconnect?.();
delete window.__openclawZoomCaptions;
active = undefined;
}
if (!captureCaptions) {
if (!canMutateSession) return undefined;
if (active?.settleTimer !== undefined) clearTimeout(active.settleTimer);
active?.observer?.disconnect?.();
if (active) delete window.__openclawZoomCaptions;
return undefined;
}
if (!inCall && !active) return undefined;
if (!active && !canMutateSession) return undefined;
if (!active) {
if (active?.settleTimer !== undefined) clearTimeout(active.settleTimer);
active?.observer?.disconnect?.();
window.__openclawZoomCaptions = {
sessionId,
identity: expectedIdentity,
epoch: crypto.randomUUID(),
enabledAttempted: false,
observerInstalled: false,
observer: undefined,
droppedLines: 0,
lines: [],
settled: [],
settleTimer: undefined,
visible: [],
};
}
return window.__openclawZoomCaptions;
})();
const normalizeCaption = (speaker, captionText) => {
if (!captionState) return undefined;
const clean = String(captionText || "").replace(/\\s+/g, " ").trim();
const cleanSpeaker = String(speaker || "").replace(/\\s+/g, " ").trim();
if (!clean) return undefined;
return { speaker: cleanSpeaker || undefined, text: clean };
};
const captionRowIdentity = (row) =>
// aria-posinset identifies the logical caption item across virtual-list
// rerenders. DOM ids and data indexes can belong to the recycled element.
["aria-posinset"]
.map((name) => {
const value = row?.getAttribute?.(name);
return typeof value === "string" && value.trim()
? name + ":" + value.trim()
: undefined;
})
.find(Boolean);
const sameCaptionUtterance = (prior, current) => {
if (prior.rowIdentity || current.rowIdentity) {
return Boolean(
prior.rowIdentity &&
current.rowIdentity &&
prior.rowIdentity === current.rowIdentity
);
}
if (prior.speaker && current.speaker && prior.speaker !== current.speaker) return false;
return prior.node === current.node;
};
const commitCaptionLines = (state, entries) => {
state.lines.push(...entries.map((entry) => {
entry.utteranceId ||= crypto.randomUUID();
return {
at: entry.at,
speaker: entry.speaker,
text: entry.text,
utteranceId: entry.utteranceId,
};
}));
const excess = state.lines.length - ${ZOOM_MEETING_TRANSCRIPT_MAX_LINES};
if (excess > 0) {
state.lines.splice(0, excess);
state.droppedLines = (state.droppedLines || 0) + excess;
}
};
const sameCaptionRow = (left, right) =>
right.rowIdentity
? left.rowIdentity === right.rowIdentity
: left.node === right.node;
const retainSettledCaptionLines = (state, entries) => {
const settled = [...state.settled];
for (const entry of entries) {
const priorIndex = settled.findIndex((candidate) => sameCaptionRow(candidate, entry));
if (priorIndex >= 0) settled.splice(priorIndex, 1, { ...entry });
else settled.push({ ...entry });
}
const retainedLineIds = new Set(state.lines.map((entry) => entry.utteranceId));
state.settled = settled.filter((entry) => retainedLineIds.has(entry.utteranceId));
};
const scheduleCaptionSettle = () => {
if (!captionState || captionState.visible.length === 0) return;
if (captionState.settleTimer !== undefined) clearTimeout(captionState.settleTimer);
const pendingState = captionState;
pendingState.settleTimer = setTimeout(() => {
if (window.__openclawZoomCaptions !== pendingState) return;
commitCaptionLines(pendingState, pendingState.visible);
retainSettledCaptionLines(pendingState, pendingState.visible);
pendingState.visible = [];
pendingState.settleTimer = undefined;
}, ${ZOOM_MEETING_CAPTION_SETTLE_MS});
};
const captionCaptureMatchesCurrentMeeting = () => {
if (
!captionState ||
captionState.finalized === true ||
window.__openclawZoomCaptions !== captionState
) return false;
const observedIdentity = meetingIdentity(location.href);
const observedMeeting = window.__openclawZoomMeeting;
const identityConflicts = Boolean(
observedIdentity && expectedIdentity && observedIdentity !== expectedIdentity
);
const sessionConflicts = Boolean(
observedMeeting?.sessionId && sessionId && observedMeeting.sessionId !== sessionId
);
if (identityConflicts || sessionConflicts) {
// The observer outlives Zoom SPA navigation. Freeze the old buffer before
// any caption nodes from the replacement meeting can be attributed to it.
finalizeOwnedCaptions();
return false;
}
if (observedIdentity === expectedIdentity) return true;
const observedMarkerAgeMs = Date.now() - (observedMeeting?.verifiedAt || 0);
const observedAwaitingRerender = Boolean(
!observedIdentity &&
observedMeeting?.identity === expectedIdentity &&
(!observedMeeting.sessionId || !sessionId || observedMeeting.sessionId === sessionId) &&
observedMeeting.inCallControl?.isConnected === false &&
observedMeeting.inCallUrl === location.href &&
observedMarkerAgeMs >= 0 &&
observedMarkerAgeMs < 5_000
);
if (observedAwaitingRerender) return true;
return Boolean(
observedMeeting?.identity === expectedIdentity &&
(!observedMeeting.sessionId || !sessionId || observedMeeting.sessionId === sessionId) &&
observedMeeting.inCallControl?.isConnected !== false &&
observedMeeting.inCallUrl === location.href
);
};
const scrapeCaptions = (mutations = []) => {
if (!captionCaptureMatchesCurrentMeeting()) return;
const content = firstRaw(selectors.captionContent);
const rows = content
? selectors.captionRows.flatMap((selector) => [...content.querySelectorAll(selector)])
: [];
captionState.settled = Array.isArray(captionState.settled) ? captionState.settled : [];
const removedNodes = mutations.flatMap((mutation) => [...(mutation.removedNodes || [])]);
const rowWasRemoved = (entry) => removedNodes.some((node) =>
node === entry.node || node?.contains?.(entry.node)
);
const removedVisible = captionState.visible.filter(rowWasRemoved);
if (removedVisible.length > 0) {
if (captionState.settleTimer !== undefined) clearTimeout(captionState.settleTimer);
captionState.settleTimer = undefined;
captionState.visible = captionState.visible.filter((entry) => !rowWasRemoved(entry));
commitCaptionLines(captionState, removedVisible);
retainSettledCaptionLines(captionState, removedVisible);
}
const retainedLineIds = new Set(captionState.lines.map((entry) => entry.utteranceId));
captionState.settled = captionState.settled.filter((entry) =>
entry.rowIdentity
? retainedLineIds.has(entry.utteranceId)
: !rowWasRemoved(entry) && rows.some((row) => sameCaptionRow(entry, {
node: row,
rowIdentity: captionRowIdentity(row),
}))
);
const parsedRows = rows.flatMap((row) => {
const speaker = text(firstWithin(row, selectors.captionAuthor));
const captionText = text(firstWithin(row, selectors.captionText));
const parsed = normalizeCaption(speaker, captionText);
if (!parsed) return [];
const current = { ...parsed, node: row, rowIdentity: captionRowIdentity(row) };
const settledIndex = captionState.settled.findIndex((entry) =>
sameCaptionRow(entry, current)
);
const settled = settledIndex >= 0 ? captionState.settled[settledIndex] : undefined;
if (
settled &&
settled.text === current.text &&
(settled.speaker || "") === (current.speaker || "")
) return [];
if (settled?.rowIdentity && settled.rowIdentity === current.rowIdentity) {
const committed = captionState.lines.find((entry) =>
entry.utteranceId === settled.utteranceId
);
if (committed) {
committed.speaker = current.speaker || committed.speaker;
committed.text = current.text;
}
captionState.settled.splice(settledIndex, 1, {
...settled,
...current,
speaker: current.speaker || settled.speaker,
});
return [];
}
if (settledIndex >= 0) captionState.settled.splice(settledIndex, 1);
return [current];
});
if (parsedRows.length === 0) {
if (captionState.visible.length > 0 && captionState.settleTimer === undefined) {
scheduleCaptionSettle();
}
return;
}
const unmatchedPrevious = [...captionState.visible];
const nextVisible = [];
const now = Date.now();
let captionChanged = false;
for (const row of parsedRows) {
const priorIndex = unmatchedPrevious.findIndex((candidate) =>
row.rowIdentity
? candidate.rowIdentity === row.rowIdentity
: candidate.node === row.node
);
const candidate = priorIndex >= 0 ? unmatchedPrevious[priorIndex] : undefined;
const prior = candidate && sameCaptionUtterance(candidate, row)
? unmatchedPrevious.splice(priorIndex, 1)[0]
: undefined;
if (prior) {
captionChanged ||=
prior.text !== row.text ||
prior.speaker !== row.speaker ||
prior.node !== row.node;
prior.speaker = row.speaker || prior.speaker;
prior.text = row.text;
prior.node = row.node;
prior.rowIdentity = row.rowIdentity || prior.rowIdentity;
prior.seenAt = now;
nextVisible.push(prior);
} else {
captionChanged = true;
nextVisible.push({
at: new Date().toISOString(),
node: row.node,
rowIdentity: row.rowIdentity,
seenAt: now,
speaker: row.speaker,
text: row.text,
});
}
}
captionChanged ||= unmatchedPrevious.length > 0;
commitCaptionLines(captionState, unmatchedPrevious);
retainSettledCaptionLines(captionState, unmatchedPrevious);
captionState.visible = nextVisible;
// Identity-less rows stay mutable while rendered; removal is their only
// reliable utterance boundary. Stable logical rows may settle on quiet.
if (
(captionChanged || captionState.settleTimer === undefined) &&
captionState.visible.every((entry) => entry.rowIdentity)
) {
scheduleCaptionSettle();
}
};
if (captionState) {
const captionsFinalized = captionState.finalized === true;
let captionsEnabledNow = captionsFinalized
? Boolean(captionState.enabledAttempted)
: Boolean(firstRaw(selectors.captionRenderer) || firstRaw(selectors.captionsOff));
if (!captionsFinalized && canMutateSession && inCall && !captionsEnabledNow) {
let captionButton = first(selectors.captions);
if (!captionButton) {
(first(selectors.moreActions) || findTextButton(/^more$/i))?.click?.();
await waitForUi();
captionButton = first(selectors.captions);
}
if (captionButton) {
const captionLabel = label(captionButton);
const alreadyEnabled = captionButton.getAttribute?.("aria-checked") === "true" ||
/hide (?:live )?captions|turn off captions/i.test(captionLabel) ||
Boolean(firstRaw(selectors.captionsOff));
if (!alreadyEnabled) {
captionButton.click();
await waitForUi();
const showCaptions = first(selectors.captions);
if (showCaptions && showCaptions !== captionButton && /show captions/i.test(label(showCaptions))) {
showCaptions.click();
await waitForUi();
}
const saveLanguage = findTextButton(/^save$/i);
if (saveLanguage && /caption language/i.test(text(document.body))) {
saveLanguage.click();
await waitForUi();
}
}
const currentCaptionButton = first(selectors.captions) || captionButton;
const currentLabel = label(currentCaptionButton);
captionsEnabledNow = currentCaptionButton.getAttribute?.("aria-checked") === "true" ||
/hide (?:live )?captions|turn off captions/i.test(currentLabel) ||
Boolean(firstRaw(selectors.captionRenderer)) ||
Boolean(firstRaw(selectors.captionsOff));
if (captionsEnabledNow && !alreadyEnabled) {
notes.push("Enabled Zoom live captions for transcript capture.");
}
}
}
if (!captionsFinalized && canMutateSession) captionState.enabledAttempted = captionsEnabledNow;
captionsEnabledAttempted = Boolean(captionState.enabledAttempted);
if (!captionsFinalized && canMutateSession && inCall && !captionState.observerInstalled) {
captionState.observerInstalled = true;
captionState.observer = new MutationObserver(scrapeCaptions);
captionState.observer.observe(document.body, {
childList: true,
subtree: true,
characterData: true,
});
notes.push("Installed Zoom live-caption observer.");
}
if (!captionsFinalized && canMutateSession && inCall) scrapeCaptions();
const allLines = [...captionState.lines, ...captionState.visible];
const lines = allLines.slice(-${ZOOM_MEETING_TRANSCRIPT_MAX_LINES});
const last = lines[lines.length - 1];
captioning = captionsEnabledNow;
transcriptLines = (captionState.droppedLines || 0) + allLines.length;
lastCaptionAt = last?.at;
lastCaptionSpeaker = last?.speaker;
lastCaptionText = last?.text;
recentTranscript = lines.slice(-5).map((entry) => ({
at: entry.at,
speaker: entry.speaker,
text: entry.text,
}));
}
if (inCall && allowMicrophone && !manualActionReason) {
if (audioInputRouted !== true || audioOutputRouted !== true) {
manualActionReason = "zoom-audio-choice-required";
manualActionMessage = "Verify BlackHole 2ch is selected as both the Zoom microphone and speaker before starting talk-back.";
} else if (micMuted !== false) {
manualActionReason = "zoom-microphone-required";
manualActionMessage = "Unmute the Zoom microphone and verify the microphone control shows it is on before starting talk-back.";
}
}
return JSON.stringify({
clickedContinueInBrowser: Boolean(continueInBrowser),
clickedJoin,
inCall,
meetingEnded,
micMuted,
cameraOff,
lobbyWaiting,
captionCaptureRequested: captureCaptions,
captioning,
captionsEnabledAttempted,
transcriptLines,
lastCaptionAt,
lastCaptionSpeaker,
lastCaptionText,
recentTranscript,
audioInputRouted,
audioInputDeviceLabel,
audioInputRouteError,
audioOutputRouted,
audioOutputDeviceLabel,
audioOutputRouteError,
audioOutputRouteRetryable,
manualActionRequired: Boolean(manualActionReason),
manualActionReason,
manualActionMessage,
title: document.title,
url: location.href,
notes,
});
}`;
}
@@ -0,0 +1,7 @@
export function zoomMeetingStatusPageSource(): string {
return ` const pageText = text(document.body);
const pageTextLower = pageText.toLowerCase();
const lobbyWaiting = Boolean(first(selectors.lobby)) ||
/host will let you in soon|waiting for the host to start|someone will let you in shortly|waiting for someone to let you in|when someone admits you|you.?re in the lobby|we.?ve let people in the meeting know you.?re waiting/i.test(pageTextLower);
`;
}
@@ -0,0 +1,695 @@
import { zoomMeetingStatusAccessSource } from "./zoom-meetings-status-access-source.js";
import { zoomMeetingStatusPageSource } from "./zoom-meetings-status-page-source.js";
import type { ZoomMeetingStatusPreludeParams } from "./zoom-meetings-status-types.js";
const ZOOM_MEETING_TRANSCRIPT_MAX_LINES = 500;
export function zoomMeetingStatusPreludeSource(params: ZoomMeetingStatusPreludeParams): string {
const selectors = params.selectors;
const expectedIdentity = params.expectedIdentity;
const toggleStateFunction = params.toggleStateFunction;
const pageIdentityFunctionSource = () => params.pageIdentitySource;
return `async () => {
${pageIdentityFunctionSource()}
const topDocument = globalThis.document;
const document = topDocument.querySelector("#webclient")?.contentDocument || topDocument;
const pageWindow = document.defaultView || globalThis;
const HTMLInputElement = pageWindow.HTMLInputElement || globalThis.HTMLInputElement;
const Event = pageWindow.Event || globalThis.Event;
const MutationObserver = pageWindow.MutationObserver || globalThis.MutationObserver;
const parseToggleState = ${toggleStateFunction};
const selectors = ${selectors};
const expectedIdentity = ${JSON.stringify(expectedIdentity)};
const allowMicrophone = ${JSON.stringify(params.allowMicrophone)};
const allowSessionAdoption = ${JSON.stringify(params.allowSessionAdoption)};
const autoJoin = ${JSON.stringify(params.autoJoin)};
const captureCaptions = ${JSON.stringify(params.captureCaptions)};
const readOnly = ${JSON.stringify(Boolean(params.readOnly))};
const sessionId = ${JSON.stringify(params.meetingSessionId)};
const identityRetentionMs = ${JSON.stringify(Math.max(30_000, params.waitForInCallMs))};
const text = (node) => (node?.innerText || node?.textContent || "").trim();
const label = (node) => [
node?.getAttribute?.("aria-label"),
node?.getAttribute?.("title"),
node?.getAttribute?.("data-tid"),
text(node),
].filter(Boolean).join(" ");
const clickable = (node) => node?.matches?.("button")
? node
: node?.querySelector?.("button") || node?.closest?.("button") || node;
const first = (list) => {
for (const selector of list) {
const node = document.querySelector(selector);
if (node) return clickable(node);
}
return undefined;
};
const firstRaw = (list) => {
for (const selector of list) {
const node = document.querySelector(selector);
if (node) return node;
}
return undefined;
};
const firstWithin = (root, list) => {
if (!root) return undefined;
for (const selector of list) {
if (root.matches?.(selector)) return root;
const node = root.querySelector?.(selector);
if (node) return node;
}
return undefined;
};
const findTextButton = (pattern) => [...document.querySelectorAll("button")]
.find((button) => !button.disabled && pattern.test(label(button)));
const findTextControl = (pattern) =>
[...document.querySelectorAll('button, a, [role="button"]')]
.find((control) => !control.disabled && pattern.test(label(control)));
const waitForUi = () => new Promise((resolve) => setTimeout(resolve, 120));
const bridgeOwnedBySession = (entry) => Boolean(
sessionId && (!entry?.sessionId || entry.sessionId === sessionId)
);
const mediaSourceUrl = (element) => String(element?.currentSrc || element?.src || "");
const bridgeSources = (entry) => Array.isArray(entry?.sources)
? entry.sources
: entry?.source
? [{ element: entry.source, muted: Boolean(entry.sourceMuted), pending: Boolean(entry.pending), stream: entry.stream, url: entry.sourceUrl }]
: [];
const bridgeSourceMatches = (element, source) => {
if (!element) return false;
if (source?.pending && mediaSourceIsEmpty(element) && !source.stream && !source.url) return true;
if (source?.stream || element.srcObject) return element.srcObject === source?.stream;
const currentUrl = mediaSourceUrl(element);
return Boolean(source?.url && currentUrl && source.url === currentUrl);
};
const mediaSourceIsEmpty = (element) => Boolean(
element && !element.srcObject && !mediaSourceUrl(element)
);
const restoreAudioBridgeSource = (source) => {
const element = source?.element;
// An empty element may receive a replacement source after cleanup. Keep it
// silent because there is no source identity that is safe to restore.
if (mediaSourceIsEmpty(element)) {
element.muted = true;
return;
}
// Zoom reuses media elements across source changes. Restore only the exact
// source this bridge muted.
if (!bridgeSourceMatches(element, source)) return;
const detachedLiveSource = Boolean(
element.isConnected === false &&
element.srcObject?.getAudioTracks?.().some((track) => track.readyState === "live")
);
if (detachedLiveSource) {
element.muted = true;
element.pause?.();
element.srcObject = null;
return;
}
element.muted = Boolean(source.muted);
};
const restoreAudioBridgeSources = (entry) => {
bridgeSources(entry).forEach(restoreAudioBridgeSource);
};
const retireAudioBridge = (entry, restoreSources = true) => {
if (restoreSources) restoreAudioBridgeSources(entry);
entry?.bridge?.pause?.();
if (entry?.bridge) entry.bridge.srcObject = null;
entry?.bridge?.remove?.();
};
const retireOwnedAudioBridges = (restoreSources = true) => {
const entries = Array.isArray(window.__openclawZoomAudioOutputs)
? window.__openclawZoomAudioOutputs
: [];
const retained = [];
for (const entry of entries) {
if (!bridgeOwnedBySession(entry)) {
retained.push(entry);
continue;
}
retireAudioBridge(entry, restoreSources);
}
if (retained.length > 0) window.__openclawZoomAudioOutputs = retained;
else delete window.__openclawZoomAudioOutputs;
};
const adoptAudioBridgeSourcesForSession = () => {
const entries = Array.isArray(window.__openclawZoomAudioOutputs)
? window.__openclawZoomAudioOutputs
: [];
const suspendedBySource = new Map();
for (const entry of entries) {
for (const source of bridgeSources(entry)) {
if (!source?.element || suspendedBySource.has(source.element)) continue;
if (!bridgeSourceMatches(source.element, source)) {
restoreAudioBridgeSource(source);
continue;
}
suspendedBySource.set(source.element, {
sessionId,
source: source.element,
sourceMuted: Boolean(source.muted),
sourceUrl: mediaSourceUrl(source.element) || source.url,
stream: source.element.srcObject,
suspended: true,
});
}
retireAudioBridge(entry, false);
}
const suspended = [...suspendedBySource.values()];
if (suspended.length > 0) window.__openclawZoomAudioOutputs = suspended;
else delete window.__openclawZoomAudioOutputs;
};
const suspendOwnedAudioBridges = () => {
const entries = Array.isArray(window.__openclawZoomAudioOutputs)
? window.__openclawZoomAudioOutputs
: [];
const retained = [];
const suspendedBySource = new Map();
for (const entry of entries) {
if (!bridgeOwnedBySession(entry)) {
retained.push(entry);
continue;
}
// This pending entry owns the muted element until a later serialized
// status poll sees and routes the attached playback source.
if (
entry?.pending &&
bridgeSources(entry).some((source) => bridgeSourceMatches(source?.element, source))
) {
retained.push(entry);
continue;
}
for (const source of bridgeSources(entry)) {
if (!source?.element || suspendedBySource.has(source.element)) continue;
if (!bridgeSourceMatches(source.element, source)) {
restoreAudioBridgeSource(source);
continue;
}
suspendedBySource.set(source.element, {
sessionId: entry.sessionId || sessionId,
source: source.element,
sourceMuted: Boolean(source.muted),
sourceUrl: source.url,
stream: source.element.srcObject,
suspended: true,
});
}
retireAudioBridge(entry, false);
}
const next = [...retained, ...suspendedBySource.values()];
if (next.length > 0) window.__openclawZoomAudioOutputs = next;
else delete window.__openclawZoomAudioOutputs;
};
const retireOwnedCaptions = () => {
const active = window.__openclawZoomCaptions;
const owned = Boolean(
active && sessionId && (!active.sessionId || active.sessionId === sessionId)
);
if (!owned) return;
if (active.settleTimer !== undefined) clearTimeout(active.settleTimer);
active.observer?.disconnect?.();
delete window.__openclawZoomCaptions;
};
const finalizeCaptionState = (active) => {
if (!active) return;
if (active.settleTimer !== undefined) clearTimeout(active.settleTimer);
active.settleTimer = undefined;
active.observer?.disconnect?.();
active.observer = undefined;
active.observerInstalled = false;
active.lines = Array.isArray(active.lines) ? active.lines : [];
if (Array.isArray(active.visible) && active.visible.length > 0) {
active.lines.push(...active.visible.map((entry) => ({
at: entry.at,
speaker: entry.speaker,
text: entry.text,
})));
active.visible = [];
}
const excess = active.lines.length - ${ZOOM_MEETING_TRANSCRIPT_MAX_LINES};
if (excess > 0) {
active.lines.splice(0, excess);
active.droppedLines = (active.droppedLines || 0) + excess;
}
active.finalized = true;
active.finalizedAt = Date.now();
};
const archiveFinalizedCaptions = (active) => {
if (active?.finalized !== true || !active.sessionId) return;
const archive = window.__openclawZoomCaptionArchive &&
typeof window.__openclawZoomCaptionArchive === "object"
? window.__openclawZoomCaptionArchive
: {};
archive[active.sessionId] = active;
const retained = Object.entries(archive)
.sort((left, right) => Number(right[1]?.finalizedAt || 0) - Number(left[1]?.finalizedAt || 0))
.slice(0, 4);
window.__openclawZoomCaptionArchive = Object.fromEntries(retained);
};
const finalizeOwnedCaptions = () => {
const active = window.__openclawZoomCaptions;
const owned = Boolean(
active && sessionId && (!active.sessionId || active.sessionId === sessionId)
);
if (owned) {
active.identity ||= priorMeeting?.identity || expectedIdentity;
finalizeCaptionState(active);
}
};
const toggleState = (node, kind) => parseToggleState({
kind,
ariaPressed: node?.getAttribute?.("aria-pressed"),
ariaChecked: node?.getAttribute?.("aria-checked"),
checked: typeof node?.checked === "boolean" ? node.checked : undefined,
iconClass: node?.querySelector?.("svg")?.getAttribute?.("class"),
label: label(node),
});
const notes = [];
const currentIdentity = meetingIdentity(location.href);
const priorMeeting = window.__openclawZoomMeeting;
if (expectedIdentity && currentIdentity && currentIdentity !== expectedIdentity) {
// A confirmed SPA transition must stop resources still owned by this
// request, while preserving any newer session already committed to the tab.
retireOwnedAudioBridges();
finalizeOwnedCaptions();
const requestOwnsMeeting = Boolean(
priorMeeting &&
sessionId &&
(!priorMeeting.sessionId || priorMeeting.sessionId === sessionId)
);
if (requestOwnsMeeting) delete window.__openclawZoomMeeting;
return JSON.stringify({
inCall: false,
manualActionRequired: true,
manualActionReason: "zoom-session-conflict",
manualActionMessage: "The tracked Zoom tab now shows a different meeting. Return to the requested meeting link, then retry.",
title: document.title,
url: location.href,
notes,
});
}
const meetingOwnerConflict = Boolean(
priorMeeting?.sessionId && priorMeeting.sessionId !== sessionId
);
const captionOwnerConflict = Boolean(
window.__openclawZoomCaptions?.sessionId &&
window.__openclawZoomCaptions.sessionId !== sessionId
);
const committedOwnerConflict = meetingOwnerConflict || captionOwnerConflict;
const canRepairCaptionOwner = Boolean(
!meetingOwnerConflict && priorMeeting?.sessionId === sessionId
);
const canMutateSession = Boolean(
!readOnly &&
sessionId &&
(!committedOwnerConflict || canRepairCaptionOwner || allowSessionAdoption)
);
const identityMatchedUrl = Boolean(expectedIdentity && currentIdentity === expectedIdentity);
const identityVerifiedBeforeCall = identityMatchedUrl;
const continueInBrowser = first(selectors.continueInBrowser) ||
findTextButton(/join from browser|continue on this browser|join on the web|use the web app|continue without the app/i);
if (canMutateSession && identityVerifiedBeforeCall && continueInBrowser) {
continueInBrowser.click();
notes.push("Continued to the Zoom web client.");
await waitForUi();
}
const guestInput = first(selectors.guestName) || [...document.querySelectorAll("input")].find((input) =>
/enter your name|type your name|your name|display name/i.test(label(input) + " " + (input.placeholder || ""))
);
if (canMutateSession && identityVerifiedBeforeCall && autoJoin && guestInput && guestInput.value !== ${JSON.stringify(params.guestName)}) {
const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")?.set;
guestInput.focus();
if (setter) setter.call(guestInput, ${JSON.stringify(params.guestName)});
else guestInput.value = ${JSON.stringify(params.guestName)};
guestInput.dispatchEvent(new Event("input", { bubbles: true }));
guestInput.dispatchEvent(new Event("change", { bubbles: true }));
}
const leave = first(selectors.leave);
let continueWithoutDevices = findTextControl(/\\bcontinue without (?:audio or video|microphone(?: and camera)?)\\b/i);
let dismissedDevicePrompt = false;
if (
canMutateSession &&
identityVerifiedBeforeCall &&
!leave &&
autoJoin &&
!allowMicrophone &&
continueWithoutDevices
) {
continueWithoutDevices.click();
dismissedDevicePrompt = true;
notes.push("Continued past the Zoom device prompt in observe-only mode.");
await waitForUi();
continueWithoutDevices = findTextControl(
/\\bcontinue without (?:audio or video|microphone(?: and camera)?)\\b/i
);
if (continueWithoutDevices) {
continueWithoutDevices.click();
await waitForUi();
}
} else if (
canMutateSession &&
identityVerifiedBeforeCall &&
!leave &&
autoJoin &&
allowMicrophone
) {
const useMicrophone = document.querySelector('usermedia.pepc-permission-dialog__permission-button[type*="microphone"]');
if (useMicrophone) {
useMicrophone.click();
notes.push("Requested Zoom microphone access from the prejoin prompt.");
await waitForUi();
}
}
${zoomMeetingStatusPageSource()}
const devicesDisabled = Boolean(!allowMicrophone && (dismissedDevicePrompt || (priorMeeting?.identity === expectedIdentity && (!sessionId || priorMeeting?.sessionId === sessionId) && priorMeeting?.devicesDisabled === true)));
// Zoom replaces the meeting URL after admission; retain only an adopted in-call control.
// Lobby ownership remains durable because host admission has no bounded wait.
const markerAgeMs = Date.now() - (priorMeeting?.verifiedAt || 0);
const inCallControlDisconnected = Boolean(!currentIdentity && priorMeeting?.identity === expectedIdentity && priorMeeting?.inCallControl?.isConnected === false);
if (inCallControlDisconnected && !leave) priorMeeting.inCallControlLostAt ||= Date.now();
const inCallControlLossAgeMs = Date.now() - (priorMeeting?.inCallControlLostAt || Date.now());
const identityAdoptedInCall = Boolean(
!currentIdentity &&
priorMeeting?.identity === expectedIdentity &&
!priorMeeting?.inCallControl &&
(
priorMeeting?.awaitingAdmission === true ||
(markerAgeMs >= 0 && markerAgeMs < identityRetentionMs)
) &&
leave &&
leave.isConnected !== false
);
const identityRerenderedInCall = Boolean(
inCallControlDisconnected &&
priorMeeting.inCallControl !== leave &&
priorMeeting?.inCallUrl === location.href &&
leave &&
leave.isConnected !== false
);
const identityAwaitingRerender = Boolean(
inCallControlDisconnected &&
inCallControlLossAgeMs >= 0 &&
inCallControlLossAgeMs < 5_000 &&
!leave
);
const identityPreservedInCall = Boolean(
!currentIdentity &&
priorMeeting?.identity === expectedIdentity &&
leave &&
leave.isConnected !== false &&
(
identityAdoptedInCall ||
identityRerenderedInCall ||
(
priorMeeting?.inCallControl === leave &&
priorMeeting?.inCallUrl === location.href
)
)
);
const identityVerified = identityVerifiedBeforeCall || identityPreservedInCall;
const meetingEnded = Boolean(
[...document.querySelectorAll(".zm-modal-body-title")].some((node) =>
/meeting (?:has been ended by host|has ended)/i.test(text(node))
) ||
(
inCallControlDisconnected &&
inCallControlLossAgeMs >= 5_000 &&
!leave
)
);
const inCall = Boolean(identityVerified && leave && !meetingEnded);
if (canMutateSession && identityVerified && meetingOwnerConflict) {
// The tab can survive a Zoom SPA meeting/session change. Old hidden bridges
// must stop, while their muted source streams remain eligible for the new owner.
adoptAudioBridgeSourcesForSession();
}
if (canMutateSession && !inCall && !identityAwaitingRerender) retireOwnedAudioBridges();
if (canMutateSession && (identityVerifiedBeforeCall || identityPreservedInCall)) {
window.__openclawZoomMeeting = {
...(priorMeeting?.identity === expectedIdentity && !meetingOwnerConflict ? priorMeeting : {}),
identity: expectedIdentity,
sessionId: sessionId || priorMeeting?.sessionId,
verifiedAt: Date.now(),
awaitingAdmission: !inCall && lobbyWaiting,
devicesDisabled,
...(inCall ? { inCallControl: leave, inCallControlLostAt: undefined, inCallUrl: location.href } : {}),
};
} else if (
canMutateSession &&
!currentIdentity &&
priorMeeting &&
!identityAwaitingRerender &&
(
priorMeeting.inCallControl ||
(priorMeeting.awaitingAdmission !== true && markerAgeMs >= identityRetentionMs)
)
) {
delete window.__openclawZoomMeeting;
}
const microphone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i);
let microphoneState = identityVerified ? (toggleState(microphone, "microphone") || (devicesDisabled ? "off" : undefined)) : undefined;
const camera = first(selectors.camera) || findTextButton(/camera|video/i);
let cameraState = identityVerified ? (toggleState(camera, "camera") || (devicesDisabled ? "off" : undefined)) : undefined;
let controlManualActionReason;
let controlManualActionMessage;
${zoomMeetingStatusAccessSource()}
if (
canMutateSession &&
identityVerified &&
camera &&
cameraState === "on" &&
!controlManualActionReason
) {
camera.click();
await waitForUi();
const continueWithoutCamera = findTextControl(/\\bcontinue without camera\\b/i);
if (continueWithoutCamera) {
clickable(continueWithoutCamera)?.click?.();
await waitForUi();
}
const currentCamera = first(selectors.camera) || findTextButton(/camera|video/i);
cameraState = toggleState(currentCamera, "camera");
if (cameraState === "off") {
notes.push(inCall ? "Turned the Zoom camera off after admission." : "Turned the Zoom camera off before joining.");
}
}
const join = first(selectors.join) ||
findTextButton(/^\\s*(join|join now|ask to join|join meeting)\\s*$/i);
if (
identityVerified &&
(inCall || join) &&
cameraState !== "off" &&
!controlManualActionReason
) {
controlManualActionReason = "zoom-camera-required";
controlManualActionMessage = inCall
? "Turn the Zoom camera off and verify the in-call camera control shows it is off."
: "Turn the Zoom camera off and verify the camera control shows it is off, then retry joining.";
}
const isBlackHole = (value) =>
/^blackhole 2ch(?: \\(virtual\\))?$/i.test(String(value || "").replace(/\\s+/g, " ").trim());
const isBlackHoleNode = (node) => [
node?.getAttribute?.("aria-label"),
node?.getAttribute?.("title"),
node?.label,
node?.value,
text(node),
].some(isBlackHole);
const microphoneDeviceRoots = () => {
// Consumer in-call controls expose the listbox itself, without the prejoin
// selected-device button/combobox wrapper.
const control = firstRaw(selectors.microphoneDevice) || firstRaw(selectors.microphoneDeviceMenu);
if (!control) return { control, roots: [] };
const roots = [control];
const scope = control.closest?.('[data-tid="device-settings-microphone"]');
if (scope && !roots.includes(scope)) roots.push(scope);
const listboxId = control.getAttribute?.("aria-controls");
const listbox = listboxId ? document.getElementById?.(listboxId) : undefined;
if (listbox && !roots.includes(listbox)) roots.push(listbox);
const liveMenu = firstRaw(selectors.microphoneDeviceMenu);
if (liveMenu && !roots.includes(liveMenu)) roots.push(liveMenu);
return { control, roots };
};
const selectedMicrophoneLabel = () => {
const { control, roots } = microphoneDeviceRoots();
const selectedOption = control?.selectedOptions?.[0];
if (selectedOption && isBlackHoleNode(selectedOption)) {
return label(selectedOption) || selectedOption.value;
}
if (control && isBlackHoleNode(control)) return label(control) || control.value;
for (const root of roots) {
const selected = firstWithin(root, selectors.selectedMicrophoneDevice);
if (selected && isBlackHoleNode(selected)) {
return label(selected) || selected.value;
}
}
return undefined;
};
let audioInputRouted;
let audioInputDeviceLabel;
let audioInputRouteError;
const ensureVirtualAudioInput = async () => {
const preparedInput = window.__openclawZoomMeeting;
if (preparedInput?.identity === expectedIdentity && (!sessionId || preparedInput?.sessionId === sessionId)) {
delete preparedInput.audioInputDeviceId;
}
if (!navigator.mediaDevices?.enumerateDevices) return false;
try {
const devices = await navigator.mediaDevices.enumerateDevices();
const input = devices.find((device) => device.kind === "audioinput" && isBlackHole(device.label));
if (!input?.deviceId) return false;
audioInputDeviceLabel = input.label || "BlackHole 2ch";
// Zoom hides the selected-device control after admission. Reopen the in-call audio
// options and verify the current selection before unmuting; installed devices alone
// do not prove which microphone Zoom is using.
let selected = Boolean(selectedMicrophoneLabel());
if (!selected && canMutateSession) {
const settings = first(selectors.deviceSettings);
if (settings) {
settings.click();
await waitForUi();
}
const { control } = microphoneDeviceRoots();
if (control?.tagName?.toLowerCase() === "select") {
const options = [...control.options];
const option = options.find(isBlackHoleNode);
if (option) {
control.value = option.value;
control.dispatchEvent(new Event("change", { bubbles: true }));
await waitForUi();
}
} else if (control) {
clickable(control)?.click?.();
await waitForUi();
}
const choices = microphoneDeviceRoots().roots.flatMap((root) =>
selectors.audioDeviceOptions.flatMap((selector) => [
...(root.querySelectorAll?.(selector) || []),
])
);
const choice = choices.find(isBlackHoleNode);
if (choice && choice.getAttribute?.("aria-selected") !== "true") {
clickable(choice)?.click?.();
await waitForUi();
}
selected = Boolean(selectedMicrophoneLabel());
}
return selected;
} catch (error) {
audioInputRouteError = error?.message || String(error);
return false;
}
};
if (identityVerified && !inCall && allowMicrophone && microphone) {
audioInputRouted = await ensureVirtualAudioInput();
if (!audioInputRouted) {
if (canMutateSession && microphoneState === "on") {
microphone.click();
await waitForUi();
const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i);
microphoneState = toggleState(currentMicrophone, "microphone");
}
notes.push("BlackHole input will be selected from Zoom's in-call audio controls.");
} else if (canMutateSession && microphoneState === "off") {
microphone.click();
await waitForUi();
const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i);
microphoneState = toggleState(currentMicrophone, "microphone");
if (microphoneState === "on") {
notes.push("Unmuted the Zoom microphone after verifying BlackHole 2ch input.");
}
}
} else if (canMutateSession && identityVerified && !allowMicrophone && microphoneState === "on") {
microphone.click();
await waitForUi();
const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i);
microphoneState = toggleState(currentMicrophone, "microphone");
if (microphoneState === "off") {
notes.push("Muted the Zoom microphone for observe-only mode.");
}
}
if (identityVerified && inCall && allowMicrophone) {
if (!selectedMicrophoneLabel() && canMutateSession && microphoneState === "on") {
microphone?.click();
await waitForUi();
const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i);
microphoneState = toggleState(currentMicrophone, "microphone");
}
audioInputRouted = await ensureVirtualAudioInput();
if (audioInputRouted && canMutateSession && microphoneState === "off") {
microphone?.click();
await waitForUi();
const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i);
microphoneState = toggleState(currentMicrophone, "microphone");
} else if (!audioInputRouted && canMutateSession && microphoneState === "on") {
microphone?.click();
await waitForUi();
const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i);
microphoneState = toggleState(currentMicrophone, "microphone");
if (microphoneState === "off") {
notes.push("Muted the Zoom microphone because BlackHole 2ch input could not be reverified.");
}
}
}
if (
identityVerified &&
(inCall || join) &&
!allowMicrophone &&
microphoneState !== "off" &&
!controlManualActionReason
) {
controlManualActionReason = "zoom-microphone-required";
controlManualActionMessage = inCall
? "Mute the Zoom microphone and verify it stays muted for observe-only mode."
: "Mute the Zoom microphone and verify the microphone control shows it is off, then retry joining.";
}
const micMuted = microphoneState === "off" ? true : microphoneState === "on" ? false : undefined;
const cameraOff = cameraState === "off" ? true : cameraState === "on" ? false : undefined;
const signInControl = first(selectors.signIn);
const tenantLoginRequired =
/authorized attendees only|meeting is for authorized attendees|sign in to join|verify your email|enter the code sent to/i.test(pageTextLower);
const loginRequired = tenantLoginRequired ||
(Boolean(signInControl) && !guestInput && !join && /sign in to (?:join|continue)|sign in to your account/i.test(pageTextLower));
let microphonePermissionState;
if (allowMicrophone && navigator.permissions?.query) {
try {
microphonePermissionState = (await navigator.permissions.query({ name: "microphone" })).state;
} catch {}
}
const devicePermissionPrompt = !dismissedDevicePrompt && Boolean(
first(selectors.permissionPrompt) || continueWithoutDevices
);
// Zoom shows the same no-audio/video warning when only camera access is denied.
// A granted microphone plus the verified BlackHole input is sufficient for talk-back.
const permissionRequired = devicePermissionPrompt &&
(!allowMicrophone || microphonePermissionState !== "granted");
let manualActionReason;
let manualActionMessage;
if (committedOwnerConflict && !canMutateSession) {
manualActionReason = "zoom-session-conflict";
manualActionMessage = "This Zoom tab is owned by another active meeting session.";
} else if (!inCall && loginRequired) {
manualActionReason = "zoom-login-required";
manualActionMessage = tenantLoginRequired
? "This Zoom tenant requires sign-in or email verification. Complete it in the OpenClaw browser profile, then retry."
: "Sign in to Zoom in the OpenClaw browser profile, then retry the meeting join.";
} else if (!inCall && lobbyWaiting) {
manualActionReason = "zoom-admission-required";
manualActionMessage = "Admit the OpenClaw guest from the Zoom lobby, then retry speech.";
} else if (!inCall && permissionRequired) {
manualActionReason = "zoom-permission-required";
manualActionMessage = allowMicrophone
? "Allow microphone permission for Zoom in the OpenClaw browser profile, then retry."
: "Dismiss the Zoom device-permission prompt or continue without devices, then retry.";
} else if (controlManualActionReason) {
manualActionReason = controlManualActionReason;
manualActionMessage = controlManualActionMessage;
}
let clickedJoin = false;
if (canMutateSession && identityVerified && autoJoin && !inCall && join && !join.disabled && !manualActionReason) {
join.click();
clickedJoin = true;
notes.push("Clicked the Zoom guest join button.");
}
`;
}
@@ -0,0 +1,14 @@
export type ZoomMeetingStatusPreludeParams = {
allowMicrophone: boolean;
allowSessionAdoption: boolean;
autoJoin: boolean;
captureCaptions: boolean;
expectedIdentity?: string;
guestName: string;
meetingSessionId?: string;
pageIdentitySource: string;
readOnly?: boolean;
selectors: string;
toggleStateFunction: string;
waitForInCallMs: number;
};
@@ -0,0 +1,61 @@
import { describe, expect, it } from "vitest";
import {
hasSameZoomMeetingJoinCredential,
isRecoverableZoomMeetingTab,
isSameZoomMeetingUrl,
normalizeZoomMeetingUrl,
normalizeZoomMeetingUrlForReuse,
} from "./zoom-meetings-urls.js";
describe("Zoom meeting URL normalization", () => {
it.each([
["https://zoom.us/j/123456789?pwd=abc", "zoom:123456789"],
["https://acme.zoom.us/j/12345678901/", "zoom:12345678901"],
["https://app.zoom.us/wc/12345678901/join?from=pwa&wpk=opaque", "zoom:12345678901"],
])("extracts a stable identity from %s", (url, expected) => {
expect(normalizeZoomMeetingUrlForReuse(url)).toBe(expected);
});
it("compares the invitation and web-client forms as one meeting", () => {
expect(
isSameZoomMeetingUrl(
"https://acme.zoom.us/j/12345678901?pwd=one",
"https://app.zoom.us/wc/12345678901/join?from=pwa",
),
).toBe(true);
});
it("distinguishes invite credentials without rejecting the admitted web-client URL", () => {
const oldInvite = "https://zoom.us/j/12345678901?pwd=old";
const correctedInvite = "https://zoom.us/j/12345678901?pwd=correct";
const webClient = "https://app.zoom.us/wc/12345678901/join";
expect(isSameZoomMeetingUrl(oldInvite, correctedInvite)).toBe(true);
expect(hasSameZoomMeetingJoinCredential(oldInvite, correctedInvite)).toBe(false);
expect(isRecoverableZoomMeetingTab({ url: oldInvite }, correctedInvite)).toBe(false);
expect(isRecoverableZoomMeetingTab({ url: webClient }, correctedInvite)).toBe(true);
});
it.each([
"https://zoom.us/",
"https://zoom.us/j/12345678",
"https://zoom.us/j/123456789012",
"https://zoom.us/wc/12345678901/join",
"https://app.zoom.us/wc/not-a-meeting/join",
"http://zoom.us/j/12345678901",
"https://zoom.us:8443/j/12345678901",
"https://evil.example/j/12345678901",
"https://zoom.us.evil.example/j/12345678901",
])("rejects non-meeting Zoom input: %s", (url) => {
expect(normalizeZoomMeetingUrlForReuse(url)).toBeUndefined();
expect(() => normalizeZoomMeetingUrl(url)).toThrow(
"Zoom meeting URL must use https://<account>.zoom.us/j/<meeting-id>",
);
});
it("preserves passcode parameters and removes fragments", () => {
const normalized = normalizeZoomMeetingUrl("https://zoom.us/j/12345678901?pwd=abc#success");
expect(normalized).toContain("pwd=abc");
expect(normalized).not.toContain("#success");
});
});
@@ -0,0 +1,111 @@
import type { MeetingBrowserCandidateTab } from "openclaw/plugin-sdk/meeting-runtime";
type ZoomMeetingIdentity = {
kind: "invitation" | "web-client";
meetingId: string;
passcode?: string;
};
function isZoomHostname(hostname: string): boolean {
return hostname === "zoom.us" || hostname.endsWith(".zoom.us");
}
function parseZoomMeetingIdentity(url: string | undefined): ZoomMeetingIdentity | undefined {
if (!url) {
return undefined;
}
try {
const parsed = new URL(url);
if (
parsed.protocol !== "https:" ||
parsed.port ||
parsed.username ||
parsed.password ||
!isZoomHostname(parsed.hostname.toLowerCase())
) {
return undefined;
}
const invitation = parsed.pathname.match(/^\/j\/(\d{9,11})\/?$/);
const webClient =
parsed.hostname.toLowerCase() === "app.zoom.us"
? parsed.pathname.match(/^\/wc\/(\d{9,11})\/join\/?$/)
: undefined;
const meetingId = invitation?.[1] ?? webClient?.[1];
return meetingId
? {
kind: invitation ? "invitation" : "web-client",
meetingId,
passcode: parsed.searchParams.get("pwd") || undefined,
}
: undefined;
} catch {
return undefined;
}
}
export function normalizeZoomMeetingUrl(input: unknown): string {
if (typeof input !== "string" || !input.trim()) {
throw new Error("Zoom meeting URL is required");
}
const value = input.trim();
if (!parseZoomMeetingIdentity(value)) {
throw new Error("Zoom meeting URL must use https://<account>.zoom.us/j/<meeting-id>");
}
const parsed = new URL(value);
parsed.hash = "";
return parsed.toString();
}
export function normalizeZoomMeetingUrlForReuse(url: string | undefined): string | undefined {
const identity = parseZoomMeetingIdentity(url);
return identity ? `zoom:${identity.meetingId}` : undefined;
}
export function isSameZoomMeetingUrl(left: string | undefined, right: string | undefined): boolean {
const normalizedLeft = normalizeZoomMeetingUrlForReuse(left);
const normalizedRight = normalizeZoomMeetingUrlForReuse(right);
return Boolean(normalizedLeft && normalizedRight && normalizedLeft === normalizedRight);
}
export function hasSameZoomMeetingJoinCredential(
left: string | undefined,
right: string | undefined,
): boolean {
const leftIdentity = parseZoomMeetingIdentity(left);
const rightIdentity = parseZoomMeetingIdentity(right);
return Boolean(
leftIdentity &&
rightIdentity &&
leftIdentity.meetingId === rightIdentity.meetingId &&
leftIdentity.passcode === rightIdentity.passcode,
);
}
export function isRecoverableZoomMeetingTab(
tab: MeetingBrowserCandidateTab,
url?: string,
): boolean {
if (url) {
const tabIdentity = parseZoomMeetingIdentity(tab.url);
const requestedIdentity = parseZoomMeetingIdentity(url);
if (
!tabIdentity ||
!requestedIdentity ||
tabIdentity.meetingId !== requestedIdentity.meetingId
) {
return false;
}
return tabIdentity.kind !== "invitation" || requestedIdentity.kind !== "invitation"
? true
: tabIdentity.passcode === requestedIdentity.passcode;
}
if (normalizeZoomMeetingUrlForReuse(tab.url)) {
return true;
}
try {
const hostname = new URL(tab.url ?? "").hostname.toLowerCase();
return isZoomHostname(hostname) && /sign in|verification|zoom/i.test(tab.title ?? "");
} catch {
return false;
}
}
+16
View File
@@ -0,0 +1,16 @@
{
"extends": "../tsconfig.package-boundary.base.json",
"compilerOptions": {
"rootDir": "."
},
"include": ["./*.ts", "./src/**/*.ts"],
"exclude": [
"./**/*.test.ts",
"./dist/**",
"./node_modules/**",
"./src/test-support/**",
"./src/**/*test-helpers.ts",
"./src/**/*test-harness.ts",
"./src/**/*test-support.ts"
]
}
+13
View File
@@ -2030,6 +2030,19 @@ importers:
specifier: workspace:*
version: link:../..
extensions/zoom-meetings:
dependencies:
typebox:
specifier: 1.3.3
version: 1.3.3
devDependencies:
'@openclaw/plugin-sdk':
specifier: workspace:*
version: link:../../packages/plugin-sdk
openclaw:
specifier: workspace:*
version: link:../..
packages/acp-core:
dependencies:
'@openclaw/normalization-core':
@@ -128,6 +128,9 @@ function humanizeId(value) {
if (value === "teams-meetings") {
return "Microsoft Teams meetings";
}
if (value === "zoom-meetings") {
return "Zoom meetings";
}
const names = new Map([
["acpx", "ACPx"],
["ai", "AI"],
+1 -1
View File
@@ -119,7 +119,7 @@ function selectReusableTab<
url: string;
}) {
const matches = params.tabs.filter((tab) =>
params.adapter.urls.isSameMeeting(tab.url, params.url),
params.adapter.urls.isRecoverableTab(tab, params.url),
);
const accountHint = params.adapter.urls.accountHint(params.url);
const tab = matches.find(
+41 -1
View File
@@ -27,6 +27,11 @@ type TestJoinContext = MeetingSessionRuntimeJoinContext<
function createTestRuntime(params: {
talkBack?: boolean;
refreshReusableSession?(
session: TestSession,
request: TestRequest,
resolved: { agentId: string; mode: TestMode; transport: TestTransport; url: string },
): Promise<{ keepBrowserTab: boolean } | void>;
joinTransport(input: {
request: TestRequest;
session: TestSession;
@@ -120,7 +125,8 @@ function createTestRuntime(params: {
releaseBrowserTab: (session) => params.releaseBrowserTab(session),
refreshBrowserHealth: async () => {},
refreshStatus: async () => {},
refreshReusableSession: async () => {},
refreshReusableSession: async (session, request, resolved) =>
await params.refreshReusableSession?.(session, request, resolved),
ensureRealtimeBridge: async () => undefined,
captureTranscript: async () => undefined,
speakViaTransport: async () => undefined,
@@ -129,6 +135,40 @@ function createTestRuntime(params: {
}
describe("MeetingSessionRuntime failed joins", () => {
it("cleans an externally ended reusable session before replacing it", async () => {
const stop = vi.fn(async () => {});
const releaseBrowserTab = vi.fn(async () => true);
const joinTransport = vi.fn(
async ({ session, context }: { session: TestSession; context: TestJoinContext }) => {
session.browser = {
launched: true,
tab: { targetId: session.id, openedByPlugin: true },
};
context.attachRuntimeHandles(session, { stop });
return {};
},
);
const { runtime } = createTestRuntime({
joinTransport,
refreshReusableSession: async (session) => {
session.state = "ended";
},
releaseBrowserTab,
});
const first = await runtime.join({ url: "https://meeting.example/room", agentId: "main" });
const replacement = await runtime.join({
url: "https://meeting.example/room",
agentId: "main",
});
expect(first.session.state).toBe("ended");
expect(replacement.session.id).not.toBe(first.session.id);
expect(stop).toHaveBeenCalledOnce();
expect(releaseBrowserTab).not.toHaveBeenCalled();
expect(joinTransport).toHaveBeenCalledTimes(2);
});
it("stops attached transport handles and releases the partial browser tab", async () => {
const joinError = new Error("transport setup failed");
const stop = vi.fn(async () => {});
+11 -2
View File
@@ -83,7 +83,11 @@ export type MeetingSessionRuntimeOptions<
options?: { force?: boolean; readOnly?: boolean },
): Promise<void>;
refreshStatus(session: TSession): Promise<void>;
refreshReusableSession(session: TSession): Promise<void>;
refreshReusableSession(
session: TSession,
request: TRequest,
resolved: MeetingResolvedJoin<TTransport, TMode>,
): Promise<{ keepBrowserTab: boolean } | void>;
ensureRealtimeBridge(
session: TSession,
): Promise<MeetingSessionRuntimeHandles<THealth> | undefined>;
@@ -411,8 +415,13 @@ export class MeetingSessionRuntime<
}
let reusable = activeSessions.find((session) => this.isReusableSession(session, resolved));
if (reusable) {
await this.options.refreshReusableSession(reusable);
const refreshResult = await this.options.refreshReusableSession(reusable, request, resolved);
if (reusable.state !== "active") {
// The refresh hook runs inside the join lock, so it marks stale sessions
// ended and lets this owner perform cleanup without recursive lock entry.
await this.#leaveSession(reusable, {
keepBrowserTab: refreshResult?.keepBrowserTab ?? true,
});
reusable = undefined;
}
}
@@ -63,6 +63,7 @@ const EXPECTED_BUNDLED_STARTUP_PLUGIN_IDS = [
"voice-call",
"webhooks",
"workboard",
"zoom-meetings",
] as const;
const EXPECTED_EMPTY_CONFIG_GATEWAY_STARTUP_PLUGIN_IDS = [
"acpx",