furyhawk/openclaw
1
0
Fork 0
mirror of https://github.com/openclaw/openclaw.git synced 2026-07-21 10:16:44 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(teams-meetings): join and transcribe live meetings reliably (#110615)

Browse Source 
* fix(teams-meetings): validate live web flows

* chore(plugin-sdk): update meeting runtime baseline

* test(teams-meetings): align live validation fixtures

* fix(teams-meetings): harden live browser validation

* fix(teams-meetings): preserve live transition lifecycles

* fix(meeting-bot): pass wait budget during recovery

* test(teams-meetings): satisfy extension lint

* fix(teams-meetings): preserve browser session ownership

* fix(teams-meetings): retire audio bridges after leave

* fix(teams-meetings): harden leave and audio ownership

* fix(teams-meetings): preserve leave transition ownership

* fix(teams-meetings): serialize leave cleanup safely

* fix(teams-meetings): guard live transition cleanup

* fix(teams-meetings): preserve adapter boundaries

* fix(teams-meetings): await caption readiness

* fix(teams-meetings): surface caption blockers

* fix(teams-meetings): retry transient media playback

* chore(teams-meetings): defer release notes

* refactor(meeting-bot): isolate navigation error helper

* test(teams-meetings): keep page control helper private

* fix(meeting-bot): bound caption lifecycle cleanup

* chore(plugin-sdk): refresh API baseline

* test(teams-meetings): split caption ownership coverage

* fix(teams-meetings): retry pending audio routing

* fix(teams-meetings): retain leave transition ownership

* fix(teams-meetings): refresh transcript target

* fix(teams-meetings): bound audio routing retries

* fix(teams-meetings): retry pending remote audio

* fix(teams-meetings): harden media source ownership
This commit is contained in:
Peter Steinberger
2026-07-18 15:57:22 +01:00
committed by GitHub
parent 33151cbc40
commit d7b0bacf10
41 changed files with 6318 additions and 887 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/.generated/plugin-sdk-api-baseline.sha256
+2 -2
View File
@@ -158,7 +158,7 @@ a5f59c9acbcaa3f82247bf806eb5ba08032373fb853719f0ec9457690f16fc70 module/media-m
7a5a1de743fff9d5a86515b501ffd88e7fa49970c9f46a5153816e2ebaef7bb0 module/media-store
c5e3eb1a584f4b8126d9d6c177a840ec9103671e8d1242634ee67db9b5b9e573 module/media-understanding
c0ffaed532578cf33493992e1ff806b2268b8e3774a92edbaede5cf5bda162a6 module/media-understanding-runtime
927014fec9d2e9384c696db06138a2e0f6a9cb1e33b9ec4bb1a8dba84adf3520 module/meeting-runtime
075b7a2783cf22c3210cfce630b815c2e3b7e85c1f5606f0dde4730d47242365 module/meeting-runtime
bdcdb7db74523c557b6bcaa98a47210ad90f16bc97f9143227d03108c7fd8c23 module/memory-core
6ae3c62fd51b36439cd3f22286f2e52c6bb27bc85de648650b805c3b7603a487 module/memory-core-engine-runtime
2bc363010e81c6990c30fd6743e3ffa37320ec39b91abfc222302dc812d8bd33 module/memory-core-host-embedding-registry
@@ -170,7 +170,7 @@ bbc06f2bb23d9e4a3445744711f691de9f902270e61c9740fd746f222d10fccb module/memory-
fe87e7d794bbbdaed3c834b218a787e98c0e69b26b0fac582b0668b8effab9b4 module/memory-core-host-multimodal
746e155bb06b398749a6a1bfdc1b435b1ec1941dca3854554a912577731218dd module/memory-core-host-query
e2377479799d52d8f67dcccf8f499f28a63948b0c03fa6043fc67e476a22ec0d module/memory-core-host-runtime-cli
deb9953dbae4323348197f7aa90de855446d89db6abaac8434104464b16341f5 module/memory-core-host-runtime-core
813b7f8b864e3e0497970f03a543801ff3f69cda2f80aecb26e947ef0e22534e module/memory-core-host-runtime-core
f8eb835f0769eba8324033141ecddb9f45bdb3d3241d2a5959fe8426ba6a2280 module/memory-core-host-runtime-files
ebbf937b2258487f773b8a6fbd6c648feea96399cae5fbea18ec7d29e027523d module/memory-core-host-secret
16bdd82751898ffa98157fd27c86fa0eb9ea415fb8ed703eb4e9b170f0a9501b module/memory-core-host-status
docs/plugins/sdk-migration.md
+5 -1
View File
@@ -923,7 +923,11 @@ use `openclaw/plugin-sdk/meeting-runtime` for session, browser, audio, node-host
agent-consult, and voice-call mechanics, then implement `MeetingPlatformAdapter`
for URL rules, DOM scripts, manual-action mapping, captions, creation, and dial-in
plans. Platform REST APIs, OAuth, artifacts, selectors, and wire names remain in
the plugin.
the plugin. Browser permission plans receive the requested meeting URL so each
platform can grant only its exact supported origins. Session runtimes must also
normalize platform-specific live health after confirmed browser departure;
historical transcript fields may remain, but caption and audio readiness must
not stay active after leave.
All bundled surfaces run on the shared controller: browser relay,
managed-room handoff, voice-call realtime, voice-call streaming STT, Google
docs/plugins/teams-meetings.md
+4 -4
View File
@@ -50,17 +50,17 @@ Use `chromeNode.node` to run Chrome, BlackHole, and SoX on a paired macOS node.
| ------------ | --------------------------------------------------------------------------- |
| `agent` | Realtime transcription consults the configured OpenClaw agent; TTS replies. |
| `bidi` | A realtime voice model listens and replies directly. |
| `transcribe` | Observe-only join. Caption snapshots are currently empty. |
| `transcribe` | Observe-only join with live-caption transcript snapshots. |
Caption scraping is disabled by default. Teams live-caption DOM selectors need validation across live work and consumer tenants before the plugin can promise transcript capture.
Transcribe mode enables Teams live captions after admission and captures speaker-attributed caption rows. The `transcript` action returns the bounded caption buffer for the active OpenClaw meeting session.
## Guest join limits
The browser adapter dismisses the app interstitial, fills the guest name, turns the camera off, configures the microphone for the selected mode, and clicks the join button. In-call state uses the hang-up control; lobby, tenant sign-in, and device-permission states return explicit manual-action reasons.
The browser adapter dismisses the app interstitial, fills the guest name, turns the camera off, configures the microphone for the selected mode, and clicks the join button. In-call state uses the hang-up control; lobby, tenant sign-in, and device-permission states return explicit manual-action reasons. Consumer meeting launcher redirects and the `BlackHole 2ch (Virtual)` labels shown by Chrome are supported.
Teams tenant policy can require sign-in, email verification, or organizer admission. Complete that step in the OpenClaw Chrome profile, then retry status or speech. The plugin does not bypass tenant policy.
All in-page selectors are best-effort pending live validation. Work and consumer Teams UI changes can require selector updates. Validate these flows before unattended use: app interstitial, guest-name entry, prejoin microphone/camera toggles, join, lobby admission, tenant sign-in/email verification, media permissions, in-call detection, BlackHole output routing, leave confirmation, and consumer-meeting behavior.
The consumer Teams web client has been live-validated for the app interstitial, guest-name entry, prejoin microphone/camera toggles, join, lobby admission, media permissions, in-call detection, live captions, BlackHole input/output routing, leave, and post-call detection. Work tenants can impose different sign-in, email-verification, admission, and leave-confirmation policy; complete any reported manual action in the OpenClaw Chrome profile.
## Tool and gateway surface
extensions/google-meet/index.test.ts
+9
View File
@@ -703,6 +703,7 @@ async function captureMeetLeaveScript() {
throw new Error(`unexpected browser request path ${String(request.path)}`);
}),
config,
meetingSessionId: "session-1",
meetingUrl: "https://meet.google.com/abc-defg-hij",
tab: { targetId: "local-meet-tab", openedByPlugin: false },
});
@@ -6250,6 +6251,7 @@ describe("google-meet plugin", () => {
runtime: expect.any(Object),
nodeId: "meet-node",
config: expect.any(Object),
meetingSessionId: expect.any(String),
meetingUrl: "https://meet.google.com/drf-ihtb-pad",
tab: { targetId: "created-meet-tab-a", openedByPlugin: true },
});
@@ -6257,6 +6259,7 @@ describe("google-meet plugin", () => {
runtime: expect.any(Object),
nodeId: "meet-node",
config: expect.any(Object),
meetingSessionId: expect.any(String),
meetingUrl: "https://meet.google.com/qwe-rtyu-iop",
tab: { targetId: "created-meet-tab-b", openedByPlugin: true },
});
@@ -6484,6 +6487,7 @@ describe("google-meet plugin", () => {
expect(leaveChromeMeet).toHaveBeenCalledWith({
runtime: expect.any(Object),
config: expect.any(Object),
meetingSessionId: expect.any(String),
meetingUrl: "https://meet.google.com/abc-defg-hij",
tab: { targetId: "shared-meet-tab", openedByPlugin: true },
});
@@ -6547,6 +6551,7 @@ describe("google-meet plugin", () => {
expect(leaveChromeMeet).toHaveBeenCalledWith({
runtime: expect.any(Object),
config: expect.any(Object),
meetingSessionId: expect.any(String),
meetingUrl: "https://meet.google.com/abc-defg-hij",
tab: { targetId: "old-meet-tab", openedByPlugin: true },
});
@@ -6608,6 +6613,7 @@ describe("google-meet plugin", () => {
expect(leaveChromeMeet).toHaveBeenCalledWith({
runtime: expect.any(Object),
config: expect.any(Object),
meetingSessionId: expect.any(String),
meetingUrl: "https://meet.google.com/abc-defg-hij",
tab: { targetId: "retained-meet-tab", openedByPlugin: true },
});
@@ -6677,12 +6683,14 @@ describe("google-meet plugin", () => {
expect(leaveChromeMeet).toHaveBeenNthCalledWith(2, {
runtime: expect.any(Object),
config: expect.any(Object),
meetingSessionId: expect.any(String),
meetingUrl: "https://meet.google.com/abc-defg-hij",
tab: { targetId: "replacement-meet-tab", openedByPlugin: true },
});
expect(leaveChromeMeet).toHaveBeenLastCalledWith({
runtime: expect.any(Object),
config: expect.any(Object),
meetingSessionId: expect.any(String),
meetingUrl: "https://meet.google.com/abc-defg-hij",
tab: { targetId: "retained-meet-tab", openedByPlugin: true },
});
@@ -6866,6 +6874,7 @@ describe("google-meet plugin", () => {
expect(leaveChromeMeet).toHaveBeenCalledWith({
runtime: expect.any(Object),
config: expect.any(Object),
meetingSessionId: expect.any(String),
meetingUrl: "https://meet.google.com/abc-defg-hij",
tab: { targetId: "meet-tab", openedByPlugin: true },
});
extensions/google-meet/src/runtime.ts
+13
View File
@@ -693,12 +693,14 @@ export class GoogleMeetRuntime {
runtime: this.params.runtime,
nodeId: session.chrome?.nodeId,
config: this.params.config,
meetingSessionId: session.id,
meetingUrl: session.url,
tab,
})
: await leaveChromeMeet({
runtime: this.params.runtime,
config: this.params.config,
meetingSessionId: session.id,
meetingUrl: session.url,
tab,
});
@@ -713,6 +715,17 @@ export class GoogleMeetRuntime {
}
if (session.chrome && left) {
session.chrome.browserTab = undefined;
if (session.chrome.health) {
session.chrome.health = {
...session.chrome.health,
captioning: false,
audioOutputRouted: false,
providerConnected: false,
realtimeReady: false,
audioInputActive: false,
audioOutputActive: false,
};
}
}
session.browserLeft = left;
return left;
extensions/google-meet/src/transports/chrome.ts
+4
View File
@@ -285,6 +285,7 @@ function parseNodeStartResult(raw: unknown): {
export async function leaveChromeMeet(params: {
runtime: PluginRuntime;
config: GoogleMeetConfig;
meetingSessionId: string;
meetingUrl: string;
tab: GoogleMeetBrowserTab;
}): Promise<{ left: boolean; note: string }> {
@@ -292,6 +293,7 @@ export async function leaveChromeMeet(params: {
adapter: GOOGLE_MEET_PLATFORM_ADAPTER,
callBrowser: await resolveLocalMeetingBrowserRequest(params.runtime),
launch: params.config.chrome.launch,
meetingSessionId: params.meetingSessionId,
meetingUrl: params.meetingUrl,
tab: params.tab,
timeoutMs: params.config.chrome.joinTimeoutMs,
@@ -356,6 +358,7 @@ export async function leaveChromeMeetOnNode(params: {
runtime: PluginRuntime;
nodeId?: string;
config: GoogleMeetConfig;
meetingSessionId: string;
meetingUrl: string;
tab: GoogleMeetBrowserTab;
}): Promise<{ left: boolean; note: string }> {
@@ -377,6 +380,7 @@ export async function leaveChromeMeetOnNode(params: {
timeoutMs: request.timeoutMs,
}),
launch: params.config.chrome.launch,
meetingSessionId: params.meetingSessionId,
meetingUrl: params.meetingUrl,
tab: params.tab,
timeoutMs: params.config.chrome.joinTimeoutMs,
extensions/teams-meetings/index.test.ts
+9 -4
View File
@@ -222,11 +222,11 @@ describe("Microsoft Teams meetings plugin surface", () => {
expect(response).toMatchObject({
ok: false,
payload: { error: "timeoutMs is supported only by testSpeech" },
payload: { error: "timeoutMs is supported only by testSpeech or testListen" },
});
});
it("rejects timeoutMs on testListen while captions are disabled", async () => {
it("accepts timeoutMs on testListen and reports a bounded caption timeout", async () => {
const { invoke } = authorizationHarness();
const response = await invoke("teamsmeetings.testListen", {
mode: "transcribe",
@@ -235,8 +235,13 @@ describe("Microsoft Teams meetings plugin surface", () => {
});
expect(response).toMatchObject({
ok: false,
payload: { error: "timeoutMs is supported only by testSpeech" },
ok: true,
payload: {
captioning: undefined,
createdSession: true,
listenTimedOut: true,
listenVerified: false,
},
});
});
extensions/teams-meetings/index.ts
+3 -2
View File
@@ -140,7 +140,7 @@ function trustedToolAgentId(
function joinRequest(raw: Record<string, unknown>, options?: { allowTimeout?: boolean }) {
if (!options?.allowTimeout && raw.timeoutMs !== undefined) {
throw invalidRequest("timeoutMs is supported only by testSpeech");
throw invalidRequest("timeoutMs is supported only by testSpeech or testListen");
}
let url: string;
let timeoutMs: number | undefined;
@@ -365,7 +365,8 @@ export default definePluginEntry({
],
[
"teamsmeetings.testListen",
(rt: TeamsMeetingsRuntime, raw: Record<string, unknown>) => rt.testListen(joinRequest(raw)),
(rt: TeamsMeetingsRuntime, raw: Record<string, unknown>) =>
rt.testListen(joinRequest(raw, { allowTimeout: true })),
],
] as const) {
api.registerGatewayMethod(
extensions/teams-meetings/src/config.test.ts
+4
View File
@@ -6,6 +6,10 @@ import {
} from "./config.js";
describe("Microsoft Teams meetings config", () => {
it("allows the live Teams web client enough time to reach prejoin and in-call UI", () => {
expect(resolveTeamsMeetingsConfig({}).chrome.waitForInCallMs).toBe(60_000);
});
it("builds matching SoX commands from the selected audio format", () => {
const config = resolveTeamsMeetingsConfig({ chrome: { audioBufferBytes: 2048 } });
expect(config.chrome.audioInputCommand).toContain("sox");
extensions/teams-meetings/src/config.ts
+1 -1
View File
@@ -105,7 +105,7 @@ const DEFAULT_CONFIG: TeamsMeetingsConfig = {
reuseExistingTab: true,
autoJoin: true,
joinTimeoutMs: 30_000,
waitForInCallMs: 20_000,
waitForInCallMs: 60_000,
audioInputCommand: [...DEFAULT_TEAMS_MEETINGS_AUDIO_INPUT_COMMAND],
audioOutputCommand: [...DEFAULT_TEAMS_MEETINGS_AUDIO_OUTPUT_COMMAND],
bargeInRmsThreshold: 650,
extensions/teams-meetings/src/runtime-probes.test.ts
+131 -1
View File
@@ -1,6 +1,10 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import { resolveTeamsMeetingsConfig } from "./config.js";
import { testTeamsMeetingSpeech, type TeamsMeetingsProbeContext } from "./runtime-probes.js";
import {
testTeamsMeetingListening,
testTeamsMeetingSpeech,
type TeamsMeetingsProbeContext,
} from "./runtime-probes.js";
import type { TeamsMeetingsSession } from "./transports/types.js";
const URL = "https://teams.microsoft.com/l/meetup-join/19%3ameeting_probe%40thread.v2/0";
@@ -27,6 +31,7 @@ describe("Microsoft Teams meeting runtime probes", () => {
isReusable: () => false,
join: vi.fn(async () => ({ session, spoken: true })),
list: () => [],
refreshCaptionHealth: async () => {},
refreshHealth,
resolveAgentId: () => "main",
} satisfies TeamsMeetingsProbeContext;
@@ -43,4 +48,129 @@ describe("Microsoft Teams meeting runtime probes", () => {
expect(refreshHealth).toHaveBeenCalledTimes(2);
expect(Date.now()).toBe(200);
});
it("bounds a blocked caption refresh by the per-request listening timeout", async () => {
vi.useFakeTimers();
vi.setSystemTime(0);
const session = {
agentId: "main",
chrome: { health: { inCall: true }, launched: true },
id: "teams-listen-1",
mode: "transcribe",
transport: "chrome",
} as TeamsMeetingsSession;
const refreshCaptionHealth = vi.fn(
(_session: TeamsMeetingsSession, _timeoutMs: number) => new Promise<void>(() => {}),
);
const context = {
config: resolveTeamsMeetingsConfig({ chrome: { joinTimeoutMs: 30_000 } }),
hasHealthHandle: () => true,
isReusable: () => false,
join: vi.fn(async () => ({ session, spoken: false })),
list: () => [],
refreshCaptionHealth,
refreshHealth: () => {},
resolveAgentId: () => "main",
} satisfies TeamsMeetingsProbeContext;
const pending = testTeamsMeetingListening(context, {
mode: "transcribe",
timeoutMs: 300,
url: URL,
});
await vi.advanceTimersByTimeAsync(350);
const result = await pending;
expect(result.listenTimedOut).toBe(true);
expect(refreshCaptionHealth).toHaveBeenCalledWith(session, 300);
expect(Date.now()).toBe(350);
expect(vi.getTimerCount()).toBe(0);
});
it("refreshes captions before a short listening timeout expires", async () => {
vi.useFakeTimers();
vi.setSystemTime(0);
const session = {
agentId: "main",
chrome: { health: { inCall: true }, launched: true },
id: "teams-listen-2",
mode: "transcribe",
transport: "chrome",
} as TeamsMeetingsSession;
const refreshCaptionHealth = vi.fn(
async (_session: TeamsMeetingsSession, _timeoutMs: number) => {
session.chrome!.health = {
...session.chrome!.health,
lastCaptionText: "Caption already waiting",
manualActionRequired: true,
transcriptLines: 1,
};
},
);
const context = {
config: resolveTeamsMeetingsConfig({ chrome: { joinTimeoutMs: 30_000 } }),
hasHealthHandle: () => true,
isReusable: () => false,
join: vi.fn(async () => ({ session, spoken: false })),
list: () => [],
refreshCaptionHealth,
refreshHealth: () => {},
resolveAgentId: () => "main",
} satisfies TeamsMeetingsProbeContext;
const result = await testTeamsMeetingListening(context, {
mode: "transcribe",
timeoutMs: 100,
url: URL,
});
expect(result.listenVerified).toBe(true);
expect(result.manualActionRequired).toBe(true);
expect(refreshCaptionHealth).toHaveBeenCalledTimes(1);
expect(Date.now()).toBe(0);
expect(vi.getTimerCount()).toBe(0);
});
it("does not accept caption progress that arrives after the listening deadline", async () => {
vi.useFakeTimers();
vi.setSystemTime(0);
const session = {
agentId: "main",
chrome: { health: { inCall: true }, launched: true },
id: "teams-listen-late",
mode: "transcribe",
transport: "chrome",
} as TeamsMeetingsSession;
const context = {
config: resolveTeamsMeetingsConfig({ chrome: { joinTimeoutMs: 30_000 } }),
hasHealthHandle: () => true,
isReusable: () => false,
join: vi.fn(async () => ({ session, spoken: false })),
list: () => [],
refreshCaptionHealth: async (_session: TeamsMeetingsSession, timeoutMs: number) => {
await new Promise<void>((resolve) => {
setTimeout(resolve, timeoutMs + 50);
});
session.chrome!.health = {
...session.chrome!.health,
lastCaptionText: "Too late",
transcriptLines: 1,
};
},
refreshHealth: () => {},
resolveAgentId: () => "main",
} satisfies TeamsMeetingsProbeContext;
const pending = testTeamsMeetingListening(context, {
mode: "transcribe",
timeoutMs: 300,
url: URL,
});
await vi.advanceTimersByTimeAsync(400);
await expect(pending).resolves.toMatchObject({
listenTimedOut: true,
listenVerified: false,
});
});
});
extensions/teams-meetings/src/runtime-probes.ts
+76 -10
View File
@@ -22,6 +22,7 @@ export type TeamsMeetingsProbeContext = {
): boolean;
hasHealthHandle(sessionId: string): boolean;
refreshHealth(sessionId: string): void;
refreshCaptionHealth(session: TeamsMeetingsSession, timeoutMs: number): Promise<void>;
};
function talkBackMode(mode: TeamsMeetingsMode): boolean {
@@ -106,25 +107,90 @@ export async function testTeamsMeetingListening(
context: TeamsMeetingsProbeContext,
request: TeamsMeetingsJoinRequest,
) {
if (request.timeoutMs !== undefined) {
throw new Error("timeoutMs is not supported while Teams caption scraping is disabled");
}
if (request.mode && request.mode !== "transcribe") {
throw new Error("test_listen requires mode: transcribe");
}
const before = new Set(context.list().map((session) => session.id));
const result = await context.join({ ...request, mode: "transcribe", message: undefined });
const health = result.session.chrome?.health;
const resolved = {
url: request.url,
transport: request.transport ?? (context.config.chromeNode.node ? "chrome-node" : "chrome"),
mode: "transcribe" as const,
agentId: context.resolveAgentId(request),
};
const beforeSessions = context.list();
const before = new Set(beforeSessions.map((session) => session.id));
const existing = beforeSessions.find((session) => context.isReusable(session, resolved));
const start = {
lines: existing?.chrome?.health?.transcriptLines ?? 0,
at: existing?.chrome?.health?.lastCaptionAt,
text: existing?.chrome?.health?.lastCaptionText,
};
const result = await context.join({ ...request, ...resolved, message: undefined });
let health = result.session.chrome?.health;
const advanced = () =>
(health?.transcriptLines ?? 0) > (existing?.id === result.session.id ? start.lines : 0) ||
Boolean(health?.lastCaptionAt && health.lastCaptionAt !== start.at) ||
Boolean(health?.lastCaptionText && health.lastCaptionText !== start.text);
const shouldWait =
health?.manualActionRequired !== true && Boolean(result.session.chrome?.launched);
let listenVerified = advanced();
if (shouldWait && !listenVerified) {
const deadline =
Date.now() + resolveProbeTimeoutMs(request.timeoutMs, context.config.chrome.joinTimeoutMs);
while (Date.now() < deadline) {
const remainingMs = deadline - Date.now();
if (remainingMs <= 0) {
break;
}
let deadlineTimer: ReturnType<typeof setTimeout> | undefined;
const deadlineReached = new Promise<boolean>((resolve) => {
deadlineTimer = setTimeout(() => resolve(false), remainingMs);
});
// Browser recovery receives this same remaining budget. The outer race
// keeps the probe wall-clock bounded while the inner deadline prevents
// the per-target browser act from lingering on its normal longer timeout.
const refreshed = await Promise.race([
context.refreshCaptionHealth(result.session, remainingMs).then(() => true),
deadlineReached,
]).finally(() => {
if (deadlineTimer !== undefined) {
clearTimeout(deadlineTimer);
}
});
if (!refreshed) {
break;
}
health = result.session.chrome?.health;
if (Date.now() >= deadline) {
break;
}
if (advanced()) {
listenVerified = true;
}
if (listenVerified || health?.manualActionRequired) {
break;
}
const retryDelayMs = deadline - Date.now();
if (retryDelayMs <= 0) {
break;
}
await sleep(Math.min(250, retryDelayMs));
}
}
return {
createdSession: !before.has(result.session.id),
inCall: health?.inCall,
manualActionRequired: health?.manualActionRequired,
manualActionReason: health?.manualActionReason,
manualActionMessage: health?.manualActionMessage,
listenVerified: false,
listenTimedOut: false,
captionScrapingEnabled: false,
note: "Teams caption scraping is disabled pending live selector validation.",
listenVerified,
listenTimedOut: shouldWait && !listenVerified && health?.manualActionRequired !== true,
captioning: health?.captioning,
captionsEnabledAttempted: health?.captionsEnabledAttempted,
transcriptLines: health?.transcriptLines,
lastCaptionAt: health?.lastCaptionAt,
lastCaptionSpeaker: health?.lastCaptionSpeaker,
lastCaptionText: health?.lastCaptionText,
recentTranscript: health?.recentTranscript,
session: result.session,
};
}
extensions/teams-meetings/src/runtime.test.ts
+98 -5
View File
@@ -15,6 +15,7 @@ const logger = {
function runtimeHarness(options?: { tabOpen?: boolean }) {
let tabOpen = options?.tabOpen ?? false;
let sessionConflict = false;
let targetId = "teams-tab";
let tabUrl = URL;
const gatewayRequest = vi.fn(async (_method: string, params: Record<string, unknown>) => {
@@ -34,24 +35,33 @@ function runtimeHarness(options?: { tabOpen?: boolean }) {
if (params.path === "/act") {
const rawFn = (params.body as { fn?: unknown } | undefined)?.fn;
const fn = typeof rawFn === "string" ? rawFn : "";
if (fn.includes("leaveAction")) {
return {
result: JSON.stringify({ departed: true, sessionMatched: true, urlMatched: true }),
};
}
if (fn.includes("expectedSessionId")) {
return {
result: JSON.stringify({
urlMatched: true,
sessionMatched: true,
droppedLines: 0,
lines: [],
lines: sessionConflict ? [{ text: "Archived caption" }] : [],
}),
};
}
if (fn.includes("leaveAction")) {
return { result: JSON.stringify({ departed: true, urlMatched: true }) };
}
return {
result: JSON.stringify({
inCall: true,
micMuted: true,
cameraOff: true,
...(sessionConflict && fn.includes("const allowSessionAdoption = false")
? {
manualActionRequired: true,
manualActionReason: "teams-session-conflict",
manualActionMessage: "This Teams tab is owned by another active meeting session.",
}
: {}),
url: tabUrl,
title: "Teams call",
}),
@@ -75,6 +85,9 @@ function runtimeHarness(options?: { tabOpen?: boolean }) {
return {
runtime,
gatewayRequest,
setSessionConflict(value: boolean) {
sessionConflict = value;
},
setTargetId(value: string) {
targetId = value;
},
@@ -111,19 +124,53 @@ describe("Microsoft Teams meeting session flow", () => {
found: true,
session: { id: first.session.id },
});
const transcriptStartCall = harness.gatewayRequest.mock.calls.length;
expect(await runtime.transcript(first.session.id)).toMatchObject({
found: true,
lines: [],
nextIndex: 0,
});
const transcriptActScripts = harness.gatewayRequest.mock.calls
.slice(transcriptStartCall)
.filter(([, params]) => params.path === "/act")
.map(([, params]) => {
const fn = (params.body as { fn?: unknown } | undefined)?.fn;
return typeof fn === "string" ? fn : "";
});
expect(transcriptActScripts).toHaveLength(2);
expect(transcriptActScripts[0]).toContain("const allowSessionAdoption = false");
expect(transcriptActScripts[0]).toContain("const captureCaptions = true");
expect(transcriptActScripts[1]).toContain("expectedSessionId");
expect(await runtime.speak(first.session.id, "hello")).toMatchObject({
found: true,
spoken: false,
});
Object.assign(first.session.chrome?.health ?? {}, {
audioInputActive: true,
audioInputRouted: true,
audioOutputActive: true,
audioOutputRouted: true,
captioning: true,
providerConnected: true,
realtimeReady: true,
});
expect(await runtime.leave(first.session.id)).toMatchObject({
found: true,
browserLeft: true,
session: { state: "ended" },
session: {
state: "ended",
chrome: {
health: {
audioInputActive: false,
audioInputRouted: false,
audioOutputActive: false,
audioOutputRouted: false,
captioning: false,
providerConnected: false,
realtimeReady: false,
},
},
},
});
expect(harness.gatewayRequest).toHaveBeenCalledWith(
"browser.request",
@@ -132,6 +179,36 @@ describe("Microsoft Teams meeting session flow", () => {
);
});
it("reads an archived transcript without reclaiming a newer live tab owner", async () => {
const harness = runtimeHarness();
const runtime = new TeamsMeetingsRuntime({
config: resolveTeamsMeetingsConfig({
defaultMode: "transcribe",
chrome: { waitForInCallMs: 1 },
}),
fullConfig: {},
runtime: harness.runtime,
logger,
});
const joined = await runtime.join({ url: URL, mode: "transcribe" });
harness.setSessionConflict(true);
harness.gatewayRequest.mockClear();
expect(await runtime.transcript(joined.session.id)).toMatchObject({
found: true,
lines: [{ text: "Archived caption" }],
});
const actScripts = harness.gatewayRequest.mock.calls
.filter(([, params]) => params.path === "/act")
.map(([, params]) => {
const fn = (params.body as { fn?: unknown } | undefined)?.fn;
return typeof fn === "string" ? fn : "";
});
expect(actScripts).toHaveLength(2);
expect(actScripts[0]).toContain("const allowSessionAdoption = false");
expect(actScripts[1]).toContain("expectedSessionId");
});
it("recovers and leaves a manually opened tab when Chrome launching is disabled", async () => {
const harness = runtimeHarness({ tabOpen: true });
const runtime = new TeamsMeetingsRuntime({
@@ -180,6 +257,22 @@ describe("Microsoft Teams meeting session flow", () => {
openedByPlugin: false,
targetId: "teams-tab-replaced",
});
harness.setTargetId("teams-tab-replaced-again");
harness.gatewayRequest.mockClear();
await runtime.transcript(joined.session.id);
expect(joined.session.chrome?.browserTab).toEqual({
openedByPlugin: false,
targetId: "teams-tab-replaced-again",
});
const transcriptRead = harness.gatewayRequest.mock.calls.find(([, params]) => {
const fn = (params.body as { fn?: unknown } | undefined)?.fn;
return params.path === "/act" && typeof fn === "string" && fn.includes("expectedSessionId");
});
expect(transcriptRead?.[1]).toMatchObject({
body: { targetId: "teams-tab-replaced-again" },
});
});
it("recovers the tracked tab after Teams rewrites the in-call URL", async () => {
extensions/teams-meetings/src/runtime.ts
+22 -2
View File
@@ -264,6 +264,8 @@ export class TeamsMeetingsRuntime {
isReusable: (session, resolved) => this.#sessions.isReusableSession(session, resolved),
hasHealthHandle: (sessionId) => this.#sessions.hasHealthHandle(sessionId),
refreshHealth: (sessionId) => this.#sessions.refreshHealth(sessionId),
refreshCaptionHealth: async (session, timeoutMs) =>
await this.#refreshBrowserHealth(session, { timeoutMs }),
};
}
@@ -321,7 +323,7 @@ export class TeamsMeetingsRuntime {
? "Teams guest joined in Chrome on the selected node with realtime audio through the node bridge."
: "Teams guest joined in local Chrome with realtime audio through BlackHole 2ch and SoX."
: session.mode === "transcribe"
? "Teams guest joined observe-only; caption snapshots remain empty pending live selector validation."
? "Teams guest joined observe-only with live-caption transcript capture."
: "Teams guest join is waiting for the browser to become ready before starting realtime audio.",
);
this.#sessions.refreshSpeechReadiness(session);
@@ -407,18 +409,20 @@ export class TeamsMeetingsRuntime {
async #refreshBrowserHealth(
session: TeamsMeetingsSession,
options: { readOnly?: boolean } = {},
options: { readOnly?: boolean; timeoutMs?: number } = {},
): Promise<void> {
try {
const result = await recoverCurrentTeamsMeetingTab({
runtime: this.params.runtime,
config: this.params.config,
meetingSessionId: session.id,
mode: session.mode,
nodeId: session.chrome?.nodeId,
readOnly: options.readOnly,
trackedMeetingUrl: session.url,
trackedTargetId: session.chrome?.browserTab?.targetId,
transport: session.transport,
timeoutMs: options.timeoutMs,
url: session.url,
});
if (result.found && session.chrome) {
@@ -443,6 +447,9 @@ export class TeamsMeetingsRuntime {
}
async #captureTranscript(session: TeamsMeetingsSession, options: { finalize?: boolean } = {}) {
// Recovery permits caption setup but atomically refuses a different live
// session owner, so stale sessions read their archived page buffer instead.
await this.#sessions.refreshCaptionHealth(session);
const tab = session.chrome?.browserTab;
if (!tab) {
return undefined;
@@ -483,6 +490,7 @@ export class TeamsMeetingsRuntime {
const result = await leaveTeamsMeetingInBrowser({
runtime: this.params.runtime,
config: this.params.config,
meetingSessionId: session.id,
meetingUrl: session.url,
nodeId: session.chrome?.nodeId,
tab,
@@ -490,6 +498,18 @@ export class TeamsMeetingsRuntime {
noteSession(session, result.note);
if (result.left && session.chrome) {
session.chrome.browserTab = undefined;
if (session.chrome.health) {
session.chrome.health = {
...session.chrome.health,
captioning: false,
audioInputRouted: false,
audioOutputRouted: false,
providerConnected: false,
realtimeReady: false,
audioInputActive: false,
audioOutputActive: false,
};
}
}
session.browserLeft = result.left;
return result.left;
extensions/teams-meetings/src/transports/chrome.test.ts
+41
View File
@@ -85,6 +85,47 @@ afterEach(() => {
});
describe("Microsoft Teams meeting Chrome startup cleanup", () => {
it("keeps auto-join enabled when recovering an active meeting tab", async () => {
const state = { tabOpen: true };
const gatewayRequest = vi.fn(async (_method: string, params: Record<string, unknown>) =>
browserResult(params, state),
);
const runtime = {
gateway: { isAvailable: vi.fn(async () => true), request: gatewayRequest },
system: {
runCommandWithTimeout: vi.fn(async () => ({
code: 0,
stderr: "",
stdout: "BlackHole 2ch",
})),
},
} as unknown as PluginRuntime;
await expect(
launchTeamsMeetingInChrome({
config: resolveTeamsMeetingsConfig({
chrome: { launch: false, waitForInCallMs: 1 },
}),
fullConfig: {},
logger,
meetingSessionId: "session-1",
mode: "agent",
runtime,
trackedTargetId: "teams-tab",
url: URL,
}),
).rejects.toThrow("realtime startup failed");
const evaluated = gatewayRequest.mock.calls.find(
([, params]) =>
params.path === "/act" &&
!(params.body as { fn?: string } | undefined)?.fn?.includes("leaveAction"),
);
expect((evaluated?.[1].body as { fn?: string } | undefined)?.fn).toContain(
"const autoJoin = true",
);
});
it("disposes local audio and leaves the browser when realtime startup fails", async () => {
const state = { tabOpen: false };
const gatewayRequest = vi.fn(async (_method: string, params: Record<string, unknown>) =>
extensions/teams-meetings/src/transports/chrome.ts
+13
View File
@@ -83,9 +83,12 @@ async function openOrRecoverTeamsMeeting(params: {
}
const recovered = await recoverMeetingBrowserTab({
adapter: TEAMS_MEETINGS_PLATFORM_ADAPTER,
allowSessionAdoption: true,
autoJoin: params.config.chrome.autoJoin,
callBrowser: params.callBrowser,
config: params.config.chrome,
locationLabel: params.locationLabel,
meetingSessionId: params.meetingSessionId,
mode: params.mode,
requestedMeetingUrl: params.url,
trackedMeetingUrl: params.url,
@@ -102,6 +105,7 @@ async function rollbackTeamsBrowserJoin(params: {
callBrowser: MeetingBrowserRequestCaller;
config: TeamsMeetingsConfig;
logger: RuntimeLogger;
meetingSessionId: string;
tab?: TeamsMeetingsBrowserTab;
url: string;
}) {
@@ -112,6 +116,7 @@ async function rollbackTeamsBrowserJoin(params: {
adapter: TEAMS_MEETINGS_PLATFORM_ADAPTER,
callBrowser: params.callBrowser,
launch: true,
meetingSessionId: params.meetingSessionId,
meetingUrl: params.url,
tab: params.tab,
timeoutMs: params.config.chrome.joinTimeoutMs,
@@ -279,6 +284,7 @@ export async function launchTeamsMeetingInChrome(params: {
callBrowser,
config: params.config,
logger: params.logger,
meetingSessionId: params.meetingSessionId,
tab: result.tab,
url: params.url,
});
@@ -488,6 +494,7 @@ export async function launchTeamsMeetingOnNode(params: {
callBrowser,
config: params.config,
logger: params.logger,
meetingSessionId: params.meetingSessionId,
tab: browser.tab,
url: params.url,
});
@@ -498,12 +505,14 @@ export async function launchTeamsMeetingOnNode(params: {
export async function recoverCurrentTeamsMeetingTab(params: {
runtime: PluginRuntime;
config: TeamsMeetingsConfig;
meetingSessionId?: string;
mode: TeamsMeetingsMode;
nodeId?: string;
readOnly?: boolean;
trackedMeetingUrl?: string;
trackedTargetId?: string;
transport: "chrome" | "chrome-node";
timeoutMs?: number;
url?: string;
}) {
const nodeId =
@@ -532,11 +541,13 @@ export async function recoverCurrentTeamsMeetingTab(params: {
: await resolveLocalMeetingBrowserRequest(params.runtime),
config: params.config.chrome,
locationLabel: nodeId ? "on the selected Chrome node" : "in local Chrome",
meetingSessionId: params.meetingSessionId,
mode: params.mode,
readOnly: params.readOnly,
requestedMeetingUrl: params.url,
trackedMeetingUrl: params.trackedMeetingUrl,
trackedTargetId: params.trackedTargetId,
timeoutMs: params.timeoutMs,
})),
};
}
@@ -544,6 +555,7 @@ export async function recoverCurrentTeamsMeetingTab(params: {
export async function leaveTeamsMeetingInBrowser(params: {
runtime: PluginRuntime;
config: TeamsMeetingsConfig;
meetingSessionId: string;
meetingUrl: string;
nodeId?: string;
tab: TeamsMeetingsBrowserTab;
@@ -563,6 +575,7 @@ export async function leaveTeamsMeetingInBrowser(params: {
})
: await resolveLocalMeetingBrowserRequest(params.runtime),
launch: params.config.chrome.launch || !params.tab.openedByPlugin,
meetingSessionId: params.meetingSessionId,
meetingUrl: params.meetingUrl,
tab: params.tab,
timeoutMs: params.config.chrome.joinTimeoutMs,
extensions/teams-meetings/src/transports/teams-meetings-audio-routing.test.ts
+1025
View File
File diff suppressed because it is too large Load Diff
extensions/teams-meetings/src/transports/teams-meetings-caption-capture.test.ts
+992
View File
@@ -0,0 +1,992 @@
import { runInNewContext } from "node:vm";
import { afterEach, describe, expect, it, vi } from "vitest";
import {
teamsMeetingStatusScript,
teamsMeetingTranscriptScript,
} from "./teams-meetings-page-scripts.js";
import { TEAMS_MEETINGS_PLATFORM_ADAPTER } from "./teams-meetings-platform-adapter.js";
import {
URL,
CONSUMER_URL,
MEETING_STATE_KEY,
control,
captionRow,
runStatusScript,
} from "./teams-meetings-platform-adapter.test-helpers.js";
afterEach(() => {
vi.useRealTimers();
});
describe("Microsoft Teams meeting captions and permissions", () => {
it("builds the guest join script from centralized stable selectors and text fallbacks", () => {
const script = teamsMeetingStatusScript({
allowMicrophone: true,
allowSessionAdoption: true,
autoJoin: true,
captureCaptions: true,
guestName: "OpenClaw Guest",
meetingSessionId: "session-1",
meetingUrl: URL,
waitForInCallMs: 60_000,
});
expect(script).toContain('data-tid=\\"prejoin-display-name-input\\"');
expect(script).toContain('data-tid=\\"call-hangup\\"');
expect(script).toContain("continue on this browser");
expect(script).toContain("someone will let you in shortly");
expect(script).toContain("setSinkId");
expect(script).toContain("BlackHole");
});
it("enables live captions and captures the validated Teams caption row DOM", async () => {
const leave = control({ label: "Leave" });
const { result, window } = await runStatusScript({
allowMicrophone: false,
captionRows: [captionRow("OpenClaw QA", "Copper lantern validates Teams captions seven.")],
captureCaptions: true,
leave,
});
expect(result).toMatchObject({
captioning: true,
captionsEnabledAttempted: true,
inCall: true,
lastCaptionSpeaker: "OpenClaw QA",
lastCaptionText: "Copper lantern validates Teams captions seven.",
transcriptLines: 1,
});
const readTranscript = runInNewContext(
`(${teamsMeetingTranscriptScript(URL, "session-1", false)})`,
{
URL: globalThis.URL,
clearTimeout,
document: {},
location: new globalThis.URL(URL),
window,
},
) as () => string;
expect(JSON.parse(readTranscript())).toMatchObject({
droppedLines: 0,
epoch: "teams-caption-epoch",
lines: [
{
speaker: "OpenClaw QA",
text: "Copper lantern validates Teams captions seven.",
},
],
sessionMatched: true,
urlMatched: true,
});
expect(window["__openclawTeamsCaptions"]).toMatchObject({
identity: "teams-work:19:meeting_test@thread.v2",
});
});
it("retries an unverified live-caption activation", async () => {
const first = await runStatusScript({
allowMicrophone: false,
captionClickIgnored: true,
captionsInitiallyOn: false,
captionRows: [captionRow("OpenClaw QA", "Retry captions")],
captureCaptions: true,
leave: control({ label: "Leave" }),
});
expect(first.captionButton.clicks).toBe(1);
expect(first.result).toMatchObject({
captioning: false,
captionsEnabledAttempted: false,
});
const second = await runStatusScript({
allowMicrophone: false,
captionsInitiallyOn: false,
captionRows: [captionRow("OpenClaw QA", "Retry captions")],
captureCaptions: true,
leave: control({ label: "Leave" }),
priorCaptions: first.window["__openclawTeamsCaptions"],
priorMeeting: first.window[MEETING_STATE_KEY] as Record<string, unknown>,
});
expect(second.captionButton.clicks).toBe(1);
expect(second.result).toMatchObject({
captioning: true,
captionsEnabledAttempted: true,
});
});
it("preserves valid one-character caption lines", async () => {
const { result } = await runStatusScript({
allowMicrophone: false,
captionRows: [captionRow("OpenClaw QA", "I")],
captureCaptions: true,
leave: control({ label: "Leave" }),
});
expect(result).toMatchObject({
lastCaptionText: "I",
recentTranscript: [{ speaker: "OpenClaw QA", text: "I" }],
transcriptLines: 1,
});
});
it("bounds visible and committed caption rows together", async () => {
const { result, window } = await runStatusScript({
allowMicrophone: false,
captionRows: Array.from({ length: 505 }, (_, index) =>
captionRow("OpenClaw QA", `Bounded caption ${index}`),
),
captureCaptions: true,
leave: control({ label: "Leave" }),
});
const state = window["__openclawTeamsCaptions"] as {
droppedLines: number;
lines: unknown[];
visible: unknown[];
};
expect(state.lines).toHaveLength(0);
expect(state.visible).toHaveLength(505);
expect(state.droppedLines).toBe(0);
expect(result.transcriptLines).toBe(505);
const readTranscript = runInNewContext(
`(${teamsMeetingTranscriptScript(URL, "session-1", false)})`,
{
URL: globalThis.URL,
clearTimeout,
document: {},
location: new globalThis.URL(URL),
window,
},
) as () => string;
const transcript = JSON.parse(readTranscript()) as { droppedLines: number; lines: unknown[] };
expect(transcript.lines).toHaveLength(500);
expect(transcript.droppedLines).toBe(5);
});
it("keeps repeated utterances from distinct caption rows", async () => {
const first = await runStatusScript({
allowMicrophone: false,
captionRows: [captionRow("OpenClaw QA", "Yes")],
captureCaptions: true,
leave: control({ label: "Leave" }),
});
const second = await runStatusScript({
allowMicrophone: false,
captionRows: [captionRow("OpenClaw QA", "Yes")],
captureCaptions: true,
leave: control({ label: "Leave" }),
priorCaptions: first.window["__openclawTeamsCaptions"],
priorMeeting: first.window[MEETING_STATE_KEY] as Record<string, unknown>,
});
expect(second.result.transcriptLines).toBe(2);
});
it("keeps the latest caption when Teams shortens a provisional row", async () => {
vi.useFakeTimers();
const row = captionRow("OpenClaw QA", "We should leave today");
const first = await runStatusScript({
allowMicrophone: false,
captionsInitiallyOn: true,
captionRows: [row],
captureCaptions: true,
leave: control({ label: "Leave" }),
});
await vi.advanceTimersByTimeAsync(1_000);
const caption = row.querySelector('[data-tid="closed-caption-text"]');
if (!caption) {
throw new Error("expected caption text control");
}
caption.textContent = "We should leave";
const second = await runStatusScript({
allowMicrophone: false,
captionRows: [row],
captureCaptions: true,
leave: control({ label: "Leave" }),
priorCaptions: first.window["__openclawTeamsCaptions"],
});
expect(second.result.lastCaptionText).toBe("We should leave");
expect(second.result.recentTranscript).toMatchObject([
{ speaker: "OpenClaw QA", text: "We should leave" },
]);
});
it("keeps a mid-sentence caption correction in the same row lifecycle", async () => {
const row = captionRow("OpenClaw QA", "I like cats");
const first = await runStatusScript({
allowMicrophone: false,
captionRows: [row],
captureCaptions: true,
leave: control({ label: "Leave" }),
});
const caption = row.querySelector('[data-tid="closed-caption-text"]');
if (!caption) {
throw new Error("expected caption text control");
}
caption.textContent = "I liked cats";
const second = await runStatusScript({
allowMicrophone: false,
captionRows: [row],
captureCaptions: true,
leave: control({ label: "Leave" }),
priorCaptions: first.window["__openclawTeamsCaptions"],
});
expect(second.result.transcriptLines).toBe(1);
expect(second.result.lastCaptionText).toBe("I liked cats");
});
it("keeps one utterance when Teams replaces a logically identical caption row", async () => {
const first = await runStatusScript({
allowMicrophone: false,
captionRows: [captionRow("OpenClaw QA", "Logical row", "8")],
captureCaptions: true,
leave: control({ label: "Leave" }),
});
const second = await runStatusScript({
allowMicrophone: false,
captionRows: [captionRow("OpenClaw QA", "Logical row replacement", "8")],
captureCaptions: true,
leave: control({ label: "Leave" }),
priorCaptions: first.window["__openclawTeamsCaptions"],
});
expect(second.result.transcriptLines).toBe(1);
expect(second.result.lastCaptionText).toBe("Logical row replacement");
});
it("updates a settled logical row in place when Teams corrects it late", async () => {
vi.useFakeTimers();
const first = await runStatusScript({
allowMicrophone: false,
captionsInitiallyOn: true,
captionRows: [captionRow("OpenClaw QA", "Late logical", "9")],
captureCaptions: true,
leave: control({ label: "Leave" }),
});
await vi.advanceTimersByTimeAsync(1_000);
const second = await runStatusScript({
allowMicrophone: false,
captionRows: [captionRow("OpenClaw QA", "Late logical correction", "9")],
captureCaptions: true,
leave: control({ label: "Leave" }),
priorCaptions: first.window["__openclawTeamsCaptions"],
});
expect(second.result.transcriptLines).toBe(1);
expect(second.result.recentTranscript).toMatchObject([{ text: "Late logical correction" }]);
});
it("deduplicates a settled logical row after temporary DOM removal", async () => {
vi.useFakeTimers();
const first = await runStatusScript({
allowMicrophone: false,
captionsInitiallyOn: true,
captionRows: [captionRow("OpenClaw QA", "Virtual row return", "13")],
captureCaptions: true,
leave: control({ label: "Leave" }),
});
await vi.advanceTimersByTimeAsync(1_000);
const missing = await runStatusScript({
allowMicrophone: false,
captionRows: [],
captureCaptions: true,
leave: control({ label: "Leave" }),
priorCaptions: first.window["__openclawTeamsCaptions"],
});
const returned = await runStatusScript({
allowMicrophone: false,
captionRows: [captionRow("OpenClaw QA", "Virtual row return", "13")],
captureCaptions: true,
leave: control({ label: "Leave" }),
priorCaptions: missing.window["__openclawTeamsCaptions"],
});
expect(returned.result.transcriptLines).toBe(1);
expect(returned.result.recentTranscript).toMatchObject([{ text: "Virtual row return" }]);
});
it("commits an utterance when Teams recycles the same virtual-list row", async () => {
vi.useFakeTimers();
const row = captionRow("OpenClaw QA", "First recycled-row utterance");
const first = await runStatusScript({
allowMicrophone: false,
captionsInitiallyOn: true,
captionRows: [row],
captureCaptions: true,
leave: control({ label: "Leave" }),
});
const disappeared = await runStatusScript({
allowMicrophone: false,
captionRows: [],
captureCaptions: true,
leave: control({ label: "Leave" }),
priorCaptions: first.window["__openclawTeamsCaptions"],
});
await vi.advanceTimersByTimeAsync(1_000);
const caption = row.querySelector('[data-tid="closed-caption-text"]');
if (!caption) {
throw new Error("expected caption text control");
}
caption.textContent = "Completely different second utterance";
const second = await runStatusScript({
allowMicrophone: false,
captionRows: [row],
captureCaptions: true,
leave: control({ label: "Leave" }),
priorCaptions: disappeared.window["__openclawTeamsCaptions"],
priorMeeting: first.window[MEETING_STATE_KEY] as Record<string, unknown>,
});
expect(second.result.transcriptLines).toBe(2);
expect(second.result.recentTranscript).toMatchObject([
{ text: "First recycled-row utterance" },
{ text: "Completely different second utterance" },
]);
});
it("commits a removed row before Teams rapidly reuses its DOM node", async () => {
const row = captionRow("OpenClaw QA", "Rapid first utterance");
const first = await runStatusScript({
allowMicrophone: false,
captionRows: [row],
captureCaptions: true,
leave: control({ label: "Leave" }),
});
const caption = row.querySelector('[data-tid="closed-caption-text"]');
if (!caption) {
throw new Error("expected caption text control");
}
caption.textContent = "Rapid second utterance";
first.triggerCaptionMutation(undefined, row);
const second = await runStatusScript({
allowMicrophone: false,
captionRows: [row],
captureCaptions: true,
leave: control({ label: "Leave" }),
priorCaptions: first.window["__openclawTeamsCaptions"],
});
expect(second.result.transcriptLines).toBe(2);
expect(second.result.recentTranscript).toMatchObject([
{ text: "Rapid first utterance" },
{ text: "Rapid second utterance" },
]);
});
it("does not merge recycled rows that only share a text prefix", async () => {
vi.useFakeTimers();
const row = captionRow("OpenClaw QA", "Thank you");
const first = await runStatusScript({
allowMicrophone: false,
captionsInitiallyOn: true,
captionRows: [row],
captureCaptions: true,
leave: control({ label: "Leave" }),
});
const disappeared = await runStatusScript({
allowMicrophone: false,
captionRows: [],
captureCaptions: true,
leave: control({ label: "Leave" }),
priorCaptions: first.window["__openclawTeamsCaptions"],
});
await vi.advanceTimersByTimeAsync(1_000);
const caption = row.querySelector('[data-tid="closed-caption-text"]');
if (!caption) {
throw new Error("expected caption text control");
}
caption.textContent = "Thank you everyone";
const second = await runStatusScript({
allowMicrophone: false,
captionRows: [row],
captureCaptions: true,
leave: control({ label: "Leave" }),
priorCaptions: disappeared.window["__openclawTeamsCaptions"],
});
expect(second.result.recentTranscript).toMatchObject([
{ text: "Thank you" },
{ text: "Thank you everyone" },
]);
});
it("retains settled markers for older rows while newer captions settle", async () => {
vi.useFakeTimers();
const firstRow = captionRow("OpenClaw QA", "First settled row", "11");
const secondRow = captionRow("OpenClaw QA", "Second settled row", "12");
const first = await runStatusScript({
allowMicrophone: false,
captionsInitiallyOn: true,
captionRows: [firstRow],
captureCaptions: true,
leave: control({ label: "Leave" }),
});
await vi.advanceTimersByTimeAsync(1_000);
const second = await runStatusScript({
allowMicrophone: false,
captionRows: [firstRow, secondRow],
captureCaptions: true,
leave: control({ label: "Leave" }),
priorCaptions: first.window["__openclawTeamsCaptions"],
});
await vi.advanceTimersByTimeAsync(1_000);
const third = await runStatusScript({
allowMicrophone: false,
captionRows: [firstRow, secondRow],
captureCaptions: true,
leave: control({ label: "Leave" }),
priorCaptions: second.window["__openclawTeamsCaptions"],
});
expect(third.result.transcriptLines).toBe(2);
expect(third.result.recentTranscript).toMatchObject([
{ text: "First settled row" },
{ text: "Second settled row" },
]);
});
it("updates a caption when its speaker label arrives late", async () => {
const row = captionRow("", "Late attribution");
const first = await runStatusScript({
allowMicrophone: false,
captionRows: [row],
captureCaptions: true,
leave: control({ label: "Leave" }),
});
const author = row.querySelector('[data-tid="author"]');
if (!author) {
throw new Error("expected caption author control");
}
author.textContent = "OpenClaw QA";
const second = await runStatusScript({
allowMicrophone: false,
captionRows: [row],
captureCaptions: true,
leave: control({ label: "Leave" }),
priorCaptions: first.window["__openclawTeamsCaptions"],
});
expect(second.result.transcriptLines).toBe(1);
expect(second.result.recentTranscript).toMatchObject([
{ speaker: "OpenClaw QA", text: "Late attribution" },
]);
});
it("updates a corrected speaker on the same logical row before settlement", async () => {
const first = await runStatusScript({
allowMicrophone: false,
captionRows: [captionRow("OpenClaw Q", "Stable speaker correction", "10")],
captureCaptions: true,
leave: control({ label: "Leave" }),
});
const second = await runStatusScript({
allowMicrophone: false,
captionRows: [captionRow("OpenClaw QA", "Stable speaker correction", "10")],
captureCaptions: true,
leave: control({ label: "Leave" }),
priorCaptions: first.window["__openclawTeamsCaptions"],
});
expect(second.result.transcriptLines).toBe(1);
expect(second.result.recentTranscript).toMatchObject([
{ speaker: "OpenClaw QA", text: "Stable speaker correction" },
]);
});
it("disconnects stale caption capture outside transcribe mode", async () => {
let disconnects = 0;
const { window } = await runStatusScript({
allowMicrophone: false,
captureCaptions: false,
leave: control({ label: "Leave" }),
priorCaptions: {
observer: { disconnect: () => (disconnects += 1) },
observerInstalled: true,
},
});
expect(disconnects).toBe(1);
expect(window).not.toHaveProperty("__openclawTeamsCaptions");
});
it("does not clear captions owned by another tab or meeting session", async () => {
let disconnects = 0;
const active = {
observer: { disconnect: () => (disconnects += 1) },
observerInstalled: true,
sessionId: "another-session",
};
const wrongTab = await runStatusScript({
allowMicrophone: false,
allowSessionAdoption: false,
captureCaptions: false,
currentUrl: "https://teams.live.com/v2/",
leave: control({ label: "Leave" }),
priorCaptions: active,
});
const wrongSession = await runStatusScript({
allowMicrophone: false,
allowSessionAdoption: false,
captureCaptions: true,
leave: control({ label: "Leave" }),
priorCaptions: active,
});
expect(disconnects).toBe(0);
expect(wrongTab.window["__openclawTeamsCaptions"]).toBe(active);
expect(wrongSession.window["__openclawTeamsCaptions"]).toBe(active);
});
it("finalizes requested-session capture after a confirmed SPA meeting transition", async () => {
let disconnects = 0;
const captions = {
droppedLines: 0,
lines: [],
observer: { disconnect: () => (disconnects += 1) },
observerInstalled: true,
sessionId: "session-1",
visible: [{ text: "Last caption before navigation" }],
};
const { window } = await runStatusScript({
allowMicrophone: false,
currentUrl: CONSUMER_URL,
priorCaptions: captions,
priorMeeting: {
identity: "teams-work:19:meeting_test@thread.v2",
sessionId: "session-1",
},
readOnly: true,
});
expect(disconnects).toBe(1);
expect(captions).toMatchObject({
finalized: true,
lines: [{ text: "Last caption before navigation" }],
});
expect(window[MEETING_STATE_KEY]).toBeUndefined();
});
it("finalizes same-session captions when meeting identity is lost", async () => {
let disconnects = 0;
const currentUrl = "https://teams.live.com/v2/";
const { window } = await runStatusScript({
allowMicrophone: false,
captureCaptions: true,
currentUrl,
leave: control({ label: "Leave" }),
priorCaptions: {
droppedLines: 0,
epoch: "caption-epoch",
lines: [],
observer: { disconnect: () => (disconnects += 1) },
observerInstalled: true,
sessionId: "session-1",
visible: [
{
at: "2026-07-17T12:00:00.000Z",
speaker: "OpenClaw QA",
text: "Preserve call-end captions",
},
],
},
});
const captions = window["__openclawTeamsCaptions"] as Record<string, unknown>;
const readTranscript = runInNewContext(
`(${teamsMeetingTranscriptScript(URL, "session-1", true)})`,
{
URL: globalThis.URL,
clearTimeout,
document: {},
location: new globalThis.URL(currentUrl),
window,
},
) as () => string;
const transcript = JSON.parse(readTranscript()) as { lines: Array<{ text: string }> };
expect(disconnects).toBe(1);
expect(captions.finalized).toBe(true);
expect(transcript.lines).toMatchObject([{ text: "Preserve call-end captions" }]);
expect(JSON.parse(readTranscript())).toMatchObject({
lines: [{ text: "Preserve call-end captions" }],
});
expect(window["__openclawTeamsCaptions"]).toBe(captions);
});
it("finalizes caption capture before an SPA navigation can mix meetings", async () => {
const params = {
allowMicrophone: false,
captionRows: [captionRow("OpenClaw QA", "Meeting A caption")],
captureCaptions: true,
leave: control({ label: "Leave" }),
};
const page = await runStatusScript(params);
params.captionRows = [captionRow("OpenClaw QA", "Meeting B caption")];
page.triggerCaptionMutation(CONSUMER_URL);
const captions = page.window["__openclawTeamsCaptions"] as {
finalized?: boolean;
lines?: Array<{ text: string }>;
visible?: Array<{ text: string }>;
};
expect(page.captionObserverDisconnects()).toBe(1);
expect(captions.finalized).toBe(true);
expect([...(captions.lines ?? []), ...(captions.visible ?? [])]).toMatchObject([
{ text: "Meeting A caption" },
]);
});
it("returns finalized captions after the tab navigates into another meeting", () => {
const window = {
__openclawTeamsCaptions: {
droppedLines: 0,
finalized: true,
identity: "teams-work:19:meeting_test@thread.v2",
lines: [{ text: "Finalized before navigation" }],
sessionId: "session-1",
visible: [],
},
__openclawTeamsMeeting: {
identity: "teams-consumer:9326458712345:abc",
sessionId: "session-2",
},
};
const readTranscript = runInNewContext(
`(${teamsMeetingTranscriptScript(URL, "session-1", true)})`,
{
URL: globalThis.URL,
clearTimeout,
location: new globalThis.URL(CONSUMER_URL),
window,
},
) as () => string;
expect(JSON.parse(readTranscript())).toMatchObject({
urlMatched: true,
sessionMatched: true,
lines: [{ text: "Finalized before navigation" }],
});
expect(JSON.parse(readTranscript())).toMatchObject({
lines: [{ text: "Finalized before navigation" }],
});
expect(window["__openclawTeamsCaptions"]).toBeDefined();
});
it("preserves same-session captions during the in-call rerender window", async () => {
let disconnects = 0;
const staleControl = control({ label: "Leave" });
staleControl.isConnected = false;
const active = {
droppedLines: 0,
enabledAttempted: true,
lines: [],
observer: { disconnect: () => (disconnects += 1) },
observerInstalled: true,
sessionId: "session-1",
visible: [],
};
const currentUrl = "https://teams.live.com/v2/";
const { window } = await runStatusScript({
allowMicrophone: false,
captureCaptions: true,
currentUrl,
priorCaptions: active,
priorMeeting: {
identity: "teams-work:19:meeting_test@thread.v2",
inCallControl: staleControl,
inCallUrl: currentUrl,
sessionId: "session-1",
verifiedAt: Date.now(),
},
});
expect(disconnects).toBe(0);
expect(window["__openclawTeamsCaptions"]).toBe(active);
});
it("keeps the live caption observer during a bounded in-call control rerender", async () => {
const leave = control({ label: "Leave" });
const currentUrl = "https://teams.live.com/v2/";
const page = await runStatusScript({
allowMicrophone: false,
captionRows: [captionRow("OpenClaw QA", "Caption before control rerender")],
captureCaptions: true,
currentUrl,
leave,
priorMeeting: {
identity: "teams-work:19:meeting_test@thread.v2",
inCallControl: leave,
inCallUrl: currentUrl,
sessionId: "session-1",
verifiedAt: Date.now(),
},
});
leave.isConnected = false;
page.triggerCaptionMutation();
expect(page.captionObserverDisconnects()).toBe(0);
expect(page.window["__openclawTeamsCaptions"]).not.toMatchObject({ finalized: true });
});
it("replaces finalized captions for a new verified session", async () => {
const old = {
droppedLines: 0,
epoch: "old-epoch",
finalized: true,
lines: [{ text: "Old session" }],
observerInstalled: false,
sessionId: "old-session",
visible: [],
};
const { window } = await runStatusScript({
allowMicrophone: false,
captionRows: [captionRow("OpenClaw QA", "New session")],
captureCaptions: true,
leave: control({ label: "Leave" }),
priorCaptions: old,
});
const current = window["__openclawTeamsCaptions"] as Record<string, unknown>;
expect(current).not.toBe(old);
expect(current.sessionId).toBe("session-1");
expect(current.epoch).toBe("teams-caption-epoch");
});
it("atomically refuses to replace a newer live owner during recovery", async () => {
let disconnects = 0;
const leave = control({ label: "Leave" });
const priorMeeting = {
identity: "teams-work:19:meeting_test@thread.v2",
inCallControl: leave,
inCallUrl: URL,
sessionId: "newer-session",
verifiedAt: Date.now(),
};
const priorCaptions = {
droppedLines: 0,
lines: [{ text: "Newer live caption" }],
observer: { disconnect: () => (disconnects += 1) },
observerInstalled: true,
sessionId: "newer-session",
visible: [],
};
const { result, window } = await runStatusScript({
allowMicrophone: false,
allowSessionAdoption: false,
captureCaptions: true,
leave,
priorCaptions,
priorMeeting,
});
expect(result).toMatchObject({
manualActionReason: "teams-session-conflict",
manualActionRequired: true,
});
expect(window[MEETING_STATE_KEY]).toBe(priorMeeting);
expect(window["__openclawTeamsCaptions"]).toBe(priorCaptions);
expect(disconnects).toBe(0);
});
it("treats a missing recovery session ID as foreign to committed page state", async () => {
let disconnects = 0;
const leave = control({ label: "Leave" });
const priorMeeting = {
identity: "teams-work:19:meeting_test@thread.v2",
inCallControl: leave,
inCallUrl: URL,
sessionId: "active-session",
verifiedAt: Date.now(),
};
const priorCaptions = {
droppedLines: 0,
lines: [{ text: "Active session caption" }],
observer: { disconnect: () => (disconnects += 1) },
observerInstalled: true,
sessionId: "active-session",
visible: [],
};
const { result, window } = await runStatusScript({
allowMicrophone: false,
allowSessionAdoption: false,
captureCaptions: true,
leave,
meetingSessionId: "",
priorCaptions,
priorMeeting,
});
expect(result).toMatchObject({
manualActionReason: "teams-session-conflict",
manualActionRequired: true,
});
expect(window[MEETING_STATE_KEY]).toBe(priorMeeting);
expect(window["__openclawTeamsCaptions"]).toBe(priorCaptions);
expect(disconnects).toBe(0);
});
it("repairs a stale caption owner for the committed meeting session", async () => {
let disconnects = 0;
const staleCaptions = {
droppedLines: 0,
lines: [{ text: "Stale caption owner" }],
observer: { disconnect: () => (disconnects += 1) },
observerInstalled: true,
sessionId: "stale-session",
visible: [],
};
const { result, window } = await runStatusScript({
allowMicrophone: false,
allowSessionAdoption: false,
captureCaptions: true,
leave: control({ label: "Leave" }),
priorCaptions: staleCaptions,
priorMeeting: {
identity: "teams-work:19:meeting_test@thread.v2",
sessionId: "session-1",
},
});
expect(result.manualActionRequired).toBe(false);
expect(disconnects).toBe(1);
expect(staleCaptions).toMatchObject({ finalized: true });
expect(window["__openclawTeamsCaptions"]).toMatchObject({ sessionId: "session-1" });
});
it("disconnects caption capture when finalizing a transcript", async () => {
const first = await runStatusScript({
allowMicrophone: false,
captionRows: [captionRow("OpenClaw QA", "Final caption")],
captureCaptions: true,
leave: control({ label: "Leave" }),
});
let disconnects = 0;
const captions = first.window["__openclawTeamsCaptions"] as Record<string, unknown>;
captions.observer = { disconnect: () => (disconnects += 1) };
captions.observerInstalled = true;
const finalize = runInNewContext(`(${teamsMeetingTranscriptScript(URL, "session-1", true)})`, {
URL: globalThis.URL,
clearTimeout,
document: {},
location: new globalThis.URL(URL),
window: first.window,
}) as () => string;
finalize();
expect(disconnects).toBe(1);
expect(captions.observerInstalled).toBe(false);
expect(captions.identity).toBe("teams-work:19:meeting_test@thread.v2");
expect(typeof captions.finalizedAt).toBe("number");
const refreshed = await runStatusScript({
allowMicrophone: false,
captionRows: [captionRow("OpenClaw QA", "Late caption")],
captureCaptions: true,
leave: control({ label: "Leave" }),
priorCaptions: captions,
});
expect(refreshed.window["__openclawTeamsCaptions"]).toBe(captions);
expect(captions.observerInstalled).toBe(false);
expect(captions.lines).toMatchObject([{ text: "Final caption" }]);
});
it("enables caption capture only for transcribe mode and parses snapshots", () => {
expect(TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.captions.enabled("agent")).toBe(false);
expect(TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.captions.enabled("bidi")).toBe(false);
expect(TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.captions.enabled("transcribe")).toBe(true);
expect(
TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.captions.parseTranscript({
result: JSON.stringify({
droppedLines: 2,
epoch: "caption-epoch",
urlMatched: true,
sessionMatched: true,
lines: [
{
at: "2026-07-17T12:00:00.000Z",
speaker: "OpenClaw QA",
text: "Copper lantern validates Teams captions seven.",
},
],
}),
}),
).toEqual({
droppedLines: 2,
epoch: "caption-epoch",
lines: [
{
at: "2026-07-17T12:00:00.000Z",
speaker: "OpenClaw QA",
text: "Copper lantern validates Teams captions seven.",
},
],
urlMatched: true,
sessionMatched: true,
});
});
it("grants media permissions only to the exact Teams meeting origin", () => {
expect(
TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.permissions({
allowMicrophone: true,
meetingUrl: URL,
}),
).toEqual({
origin: "https://teams.microsoft.com",
permissions: ["audioCapture"],
optionalPermissions: ["speakerSelection"],
});
expect(
TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.permissions({
allowMicrophone: true,
meetingUrl: "https://teams.live.com/meet/123?p=abc",
}),
).toMatchObject({ origin: "https://teams.live.com" });
expect(
TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.permissions({
allowMicrophone: true,
meetingUrl: "https://evil.example/meet/123",
}),
).toBeUndefined();
expect(
TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.permissions({
allowMicrophone: true,
meetingUrl: "https://teams.microsoft.com:8443/l/meetup-join/test",
}),
).toBeUndefined();
expect(
TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.permissions({
allowMicrophone: true,
meetingUrl: "https://teams.live.com:444/meet/123",
}),
).toBeUndefined();
expect(
TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.permissionNotes({ allowMicrophone: true }),
).toContain("Granted Teams microphone permission through browser control.");
});
it("parses leave steps and malformed status", () => {
expect(
TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.parseLeaveResult({
result: JSON.stringify({
departed: false,
leaveAction: "confirm",
sessionMatched: true,
urlMatched: true,
}),
}),
).toEqual({
departed: false,
leaveAction: "confirm",
sessionMatched: true,
urlMatched: true,
});
expect(() =>
TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.parseStatus({ result: "not-json" }),
).toThrow("Microsoft Teams browser status JSON is malformed.");
});
});
extensions/teams-meetings/src/transports/teams-meetings-caption-ownership.test.ts
+102
View File
@@ -0,0 +1,102 @@
import { runInNewContext } from "node:vm";
import { describe, expect, it } from "vitest";
import { teamsMeetingTranscriptScript } from "./teams-meetings-page-scripts.js";
import {
CONSUMER_URL,
URL,
control,
runStatusScript,
} from "./teams-meetings-platform-adapter.test-helpers.js";
describe("Microsoft Teams meeting caption ownership", () => {
it("rotates live captions when the verified tab adopts a new session", async () => {
let disconnects = 0;
const old = {
droppedLines: 0,
lines: [],
observer: { disconnect: () => (disconnects += 1) },
observerInstalled: true,
sessionId: "old-session",
visible: [{ text: "Old live caption" }],
};
const { window } = await runStatusScript({
allowMicrophone: false,
captureCaptions: true,
leave: control({ label: "Leave" }),
priorCaptions: old,
priorMeeting: {
identity: "teams-work:19:meeting_test@thread.v2",
sessionId: "old-session",
},
});
const current = window["__openclawTeamsCaptions"] as Record<string, unknown>;
expect(disconnects).toBe(1);
expect(old).toMatchObject({ finalized: true, lines: [{ text: "Old live caption" }] });
expect(current).not.toBe(old);
expect(current.sessionId).toBe("session-1");
expect(window["__openclawTeamsCaptionArchive"]).toMatchObject({
"old-session": old,
});
delete window["__openclawTeamsCaptions"];
const readOldTranscript = runInNewContext(
`(${teamsMeetingTranscriptScript(URL, "old-session", false)})`,
{
URL: globalThis.URL,
clearTimeout,
location: new globalThis.URL(URL),
window,
},
) as () => string;
expect(JSON.parse(readOldTranscript())).toMatchObject({
sessionMatched: true,
lines: [{ text: "Old live caption" }],
});
});
it("archives a prior meeting caption buffer without rewriting its identity", async () => {
const old = {
droppedLines: 0,
identity: "teams-work:19:meeting_test@thread.v2",
lines: [],
observerInstalled: true,
sessionId: "session-a",
visible: [{ text: "Meeting A caption" }],
};
const { window } = await runStatusScript({
allowMicrophone: false,
captureCaptions: true,
currentUrl: CONSUMER_URL,
leave: control({ label: "Leave" }),
meetingSessionId: "session-b",
meetingUrl: CONSUMER_URL,
priorCaptions: old,
priorMeeting: {
identity: "teams-work:19:meeting_test@thread.v2",
sessionId: "session-a",
},
});
expect(old).toMatchObject({
finalized: true,
identity: "teams-work:19:meeting_test@thread.v2",
lines: [{ text: "Meeting A caption" }],
});
delete window["__openclawTeamsCaptions"];
const readOldTranscript = runInNewContext(
`(${teamsMeetingTranscriptScript(URL, "session-a", false)})`,
{
URL: globalThis.URL,
clearTimeout,
location: new globalThis.URL(CONSUMER_URL),
window,
},
) as () => string;
expect(JSON.parse(readOldTranscript())).toMatchObject({
sessionMatched: true,
urlMatched: true,
lines: [{ text: "Meeting A caption" }],
});
});
});
extensions/teams-meetings/src/transports/teams-meetings-page-scripts.ts
+199 -446
View File
@@ -1,5 +1,8 @@
import { TEAMS_MEETING_SELECTORS } from "./teams-meetings-selectors.js";
import { teamsMeetingStatusCallSource } from "./teams-meetings-status-call-source.js";
import { teamsMeetingStatusPreludeSource } from "./teams-meetings-status-prejoin-source.js";
import { normalizeTeamsMeetingUrlForReuse } from "./teams-meetings-urls.js";
const TEAMS_MEETING_TRANSCRIPT_MAX_LINES = 500;
function pageIdentityFunctionSource(): string {
return `const meetingIdentity = (rawUrl) => {
@@ -16,12 +19,33 @@ function pageIdentityFunctionSource(): string {
: undefined;
}
if (host === "teams.live.com") {
const match = parsed.pathname.match(/^\\/meet\\/([^/]+)\\/?$/i);
const launcherTarget = parsed.pathname.toLowerCase() === "/dl/launcher/launcher.html"
? parsed.searchParams.get("url")
: undefined;
const launcherMatch = launcherTarget?.match(/^\\/_#\\/meet\\/([^/?#]+)(?:\\?(.+))?$/i);
let lightMeeting;
if (parsed.pathname.toLowerCase() === "/light-meetings/launch") {
try {
const coordinates = parsed.searchParams.get("coords");
const decoded = coordinates && coordinates.length <= 16_384
? JSON.parse(atob(coordinates))
: undefined;
if (decoded && typeof decoded === "object") lightMeeting = decoded;
} catch {}
}
const match = parsed.pathname.match(/^\\/meet\\/([^/]+)\\/?$/i) || launcherMatch ||
(typeof lightMeeting?.meetingCode === "string"
? [undefined, lightMeeting.meetingCode]
: undefined);
if (!match?.[1]) return undefined;
const code = decodeURIComponent(match[1]);
const password = parsed.searchParams.get("p");
const passcode = launcherMatch
? new URLSearchParams(launcherMatch[2] || "").get("p")
: typeof lightMeeting?.passcode === "string"
? lightMeeting.passcode
: parsed.searchParams.get("p");
return /^[a-z0-9_-]+$/i.test(code)
? "teams-consumer:" + code.toLowerCase() + ":p:" + encodeURIComponent(password || "")
? "teams-consumer:" + code.toLowerCase() + ":p:" + encodeURIComponent(passcode || "")
: undefined;
}
} catch {}
@@ -34,6 +58,9 @@ function teamsMeetingToggleStateFunctionSource(): string {
const pressed = String(input?.ariaPressed || "").toLowerCase();
if (pressed === "true") return "on";
if (pressed === "false") return "off";
const checked = String(input?.ariaChecked ?? input?.checked ?? "").toLowerCase();
if (checked === "true") return "on";
if (checked === "false") return "off";
const value = String(input?.label || "").toLowerCase().replace(/\\s+/g, " ").trim();
if (!value) return undefined;
if (input?.kind === "camera") {
@@ -53,447 +80,34 @@ function teamsMeetingToggleStateFunctionSource(): string {
export function teamsMeetingStatusScript(params: {
allowMicrophone: boolean;
allowSessionAdoption: boolean;
autoJoin: boolean;
captureCaptions: boolean;
guestName: string;
meetingSessionId?: string;
meetingUrl: string;
readOnly?: boolean;
waitForInCallMs: number;
}) {
const selectors = JSON.stringify(TEAMS_MEETING_SELECTORS);
const expectedIdentity = normalizeTeamsMeetingUrlForReuse(params.meetingUrl);
const toggleStateFunction = teamsMeetingToggleStateFunctionSource();
return `async () => {
${pageIdentityFunctionSource()}
const parseToggleState = ${toggleStateFunction};
const selectors = ${selectors};
const expectedIdentity = ${JSON.stringify(expectedIdentity)};
const allowMicrophone = ${JSON.stringify(params.allowMicrophone)};
const autoJoin = ${JSON.stringify(params.autoJoin)};
const readOnly = ${JSON.stringify(Boolean(params.readOnly))};
const sessionId = ${JSON.stringify(params.meetingSessionId)};
const text = (node) => (node?.innerText || node?.textContent || "").trim();
const label = (node) => [
node?.getAttribute?.("aria-label"),
node?.getAttribute?.("title"),
node?.getAttribute?.("data-tid"),
text(node),
].filter(Boolean).join(" ");
const clickable = (node) => node?.matches?.("button")
? node
: node?.querySelector?.("button") || node?.closest?.("button") || node;
const first = (list) => {
for (const selector of list) {
const node = document.querySelector(selector);
if (node) return clickable(node);
}
return undefined;
};
const firstRaw = (list) => {
for (const selector of list) {
const node = document.querySelector(selector);
if (node) return node;
}
return undefined;
};
const firstWithin = (root, list) => {
if (!root) return undefined;
for (const selector of list) {
if (root.matches?.(selector)) return root;
const node = root.querySelector?.(selector);
if (node) return node;
}
return undefined;
};
const buttons = [...document.querySelectorAll("button")];
const findTextButton = (pattern) => buttons.find((button) => !button.disabled && pattern.test(label(button)));
const waitForUi = () => new Promise((resolve) => setTimeout(resolve, 120));
const toggleState = (node, kind) => parseToggleState({
kind,
ariaPressed: node?.getAttribute?.("aria-pressed"),
label: label(node),
});
const notes = [];
const currentIdentity = meetingIdentity(location.href);
const priorMeeting = window.__openclawTeamsMeeting;
if (expectedIdentity && currentIdentity && currentIdentity !== expectedIdentity) {
delete window.__openclawTeamsMeeting;
return JSON.stringify({
inCall: false,
manualActionRequired: true,
manualActionReason: "teams-session-conflict",
manualActionMessage: "The tracked Teams tab now shows a different meeting. Return to the requested meeting link, then retry.",
title: document.title,
url: location.href,
notes,
});
}
const identityMatchedUrl = Boolean(expectedIdentity && currentIdentity === expectedIdentity);
const continueInBrowser = first(selectors.continueInBrowser) ||
findTextButton(/continue on this browser|join on the web|use the web app|continue without the app/i);
if (!readOnly && identityMatchedUrl && continueInBrowser) {
continueInBrowser.click();
notes.push("Continued to the Teams web client.");
await waitForUi();
}
const guestInput = first(selectors.guestName) || [...document.querySelectorAll("input")].find((input) =>
/enter your name|type your name|your name|display name/i.test(label(input) + " " + (input.placeholder || ""))
return (
teamsMeetingStatusPreludeSource({
...params,
expectedIdentity,
pageIdentitySource: pageIdentityFunctionSource(),
selectors,
toggleStateFunction,
}) + teamsMeetingStatusCallSource()
);
if (!readOnly && identityMatchedUrl && autoJoin && guestInput && !guestInput.value) {
const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")?.set;
guestInput.focus();
if (setter) setter.call(guestInput, ${JSON.stringify(params.guestName)});
else guestInput.value = ${JSON.stringify(params.guestName)};
guestInput.dispatchEvent(new Event("input", { bubbles: true }));
guestInput.dispatchEvent(new Event("change", { bubbles: true }));
}
const leave = first(selectors.leave);
// Teams replaces the meeting URL after admission. Preserve identity only
// while adopting the first in-call control or retaining that exact control.
const markerAgeMs = Date.now() - (priorMeeting?.verifiedAt || 0);
const identityAdoptedInCall = Boolean(
!currentIdentity &&
priorMeeting?.identity === expectedIdentity &&
!priorMeeting?.inCallControl &&
markerAgeMs >= 0 &&
markerAgeMs < 30_000 &&
leave &&
leave.isConnected !== false
);
const identityRerenderedInCall = Boolean(
!currentIdentity &&
priorMeeting?.identity === expectedIdentity &&
priorMeeting?.inCallControl &&
priorMeeting.inCallControl !== leave &&
priorMeeting.inCallControl.isConnected === false &&
priorMeeting?.inCallUrl === location.href &&
markerAgeMs >= 0 &&
markerAgeMs < 5_000 &&
leave &&
leave.isConnected !== false
);
const identityAwaitingRerender = Boolean(
!currentIdentity &&
priorMeeting?.identity === expectedIdentity &&
priorMeeting?.inCallControl &&
priorMeeting.inCallControl.isConnected === false &&
priorMeeting?.inCallUrl === location.href &&
markerAgeMs >= 0 &&
markerAgeMs < 5_000 &&
!leave
);
const identityPreservedInCall = Boolean(
!currentIdentity &&
priorMeeting?.identity === expectedIdentity &&
leave &&
leave.isConnected !== false &&
(
identityAdoptedInCall ||
identityRerenderedInCall ||
(
priorMeeting?.inCallControl === leave &&
priorMeeting?.inCallUrl === location.href
)
)
);
const identityVerified = identityMatchedUrl || identityPreservedInCall;
const inCall = Boolean(identityVerified && leave);
if (identityMatchedUrl || identityPreservedInCall) {
window.__openclawTeamsMeeting = {
...(priorMeeting?.identity === expectedIdentity ? priorMeeting : {}),
identity: expectedIdentity,
sessionId: sessionId || priorMeeting?.sessionId,
verifiedAt: Date.now(),
...(inCall ? { inCallControl: leave, inCallUrl: location.href } : {}),
};
} else if (
!currentIdentity &&
priorMeeting &&
!identityAwaitingRerender &&
(priorMeeting.inCallControl || markerAgeMs >= 30_000)
) {
delete window.__openclawTeamsMeeting;
}
const microphone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i);
let microphoneState = identityVerified ? toggleState(microphone, "microphone") : undefined;
const camera = first(selectors.camera) || findTextButton(/camera|video/i);
let cameraState = identityVerified ? toggleState(camera, "camera") : undefined;
let controlManualActionReason;
let controlManualActionMessage;
if (!readOnly && identityVerified && !inCall && camera && cameraState === "on") {
camera.click();
await waitForUi();
const currentCamera = first(selectors.camera) || findTextButton(/camera|video/i);
cameraState = toggleState(currentCamera, "camera");
if (cameraState === "off") {
notes.push("Turned the Teams camera off before joining.");
}
}
const join = first(selectors.join) || findTextButton(/^\\s*(join now|ask to join|join meeting)\\s*$/i);
if (identityVerified && !inCall && join && cameraState !== "off") {
controlManualActionReason = "teams-camera-required";
controlManualActionMessage = "Turn the Teams camera off and verify the camera control shows it is off, then retry joining.";
}
const isBlackHole = (value) =>
String(value || "").replace(/\\s+/g, " ").trim().toLowerCase() === "blackhole 2ch";
const isBlackHoleNode = (node) => [
node?.getAttribute?.("aria-label"),
node?.getAttribute?.("title"),
node?.label,
node?.value,
text(node),
].some(isBlackHole);
const microphoneDeviceRoots = () => {
const control = firstRaw(selectors.microphoneDevice);
if (!control) return { control, roots: [] };
const roots = [control];
const scope = control.closest?.('[data-tid="device-settings-microphone"]');
if (scope && !roots.includes(scope)) roots.push(scope);
const listboxId = control.getAttribute?.("aria-controls");
const listbox = listboxId ? document.getElementById?.(listboxId) : undefined;
if (listbox && !roots.includes(listbox)) roots.push(listbox);
return { control, roots };
};
const selectedMicrophoneLabel = () => {
const { control, roots } = microphoneDeviceRoots();
const selectedOption = control?.selectedOptions?.[0];
if (selectedOption && isBlackHoleNode(selectedOption)) {
return label(selectedOption) || selectedOption.value;
}
if (control && isBlackHoleNode(control)) return label(control) || control.value;
for (const root of roots) {
const selected = firstWithin(root, selectors.selectedMicrophoneDevice);
if (selected && isBlackHoleNode(selected)) {
return label(selected) || selected.value;
}
}
return undefined;
};
let audioInputRouted;
let audioInputDeviceLabel;
let audioInputRouteError;
const ensureVirtualAudioInput = async () => {
if (!navigator.mediaDevices?.enumerateDevices) return false;
try {
const devices = await navigator.mediaDevices.enumerateDevices();
const input = devices.find((device) => device.kind === "audioinput" && isBlackHole(device.label));
if (!input?.deviceId) return false;
audioInputDeviceLabel = input.label || "BlackHole 2ch";
let selected = Boolean(selectedMicrophoneLabel());
if (!selected && !readOnly) {
const settings = first(selectors.deviceSettings);
if (settings) {
settings.click();
await waitForUi();
}
const { control } = microphoneDeviceRoots();
if (control?.tagName?.toLowerCase() === "select") {
const options = [...control.options];
const option = options.find(isBlackHoleNode);
if (option) {
control.value = option.value;
control.dispatchEvent(new Event("change", { bubbles: true }));
await waitForUi();
}
} else if (control) {
clickable(control)?.click?.();
await waitForUi();
}
const choices = microphoneDeviceRoots().roots.flatMap((root) =>
selectors.audioDeviceOptions.flatMap((selector) => [
...(root.querySelectorAll?.(selector) || []),
])
);
const choice = choices.find(isBlackHoleNode);
if (choice && choice.getAttribute?.("aria-selected") !== "true") {
clickable(choice)?.click?.();
await waitForUi();
}
selected = Boolean(selectedMicrophoneLabel());
}
if (selected && window.__openclawTeamsMeeting?.identity === expectedIdentity) {
window.__openclawTeamsMeeting.audioInputDeviceId = input.deviceId;
}
return selected;
} catch (error) {
audioInputRouteError = error?.message || String(error);
return false;
}
};
if (identityVerified && !inCall && allowMicrophone && microphone) {
audioInputRouted = await ensureVirtualAudioInput();
if (!audioInputRouted) {
if (!readOnly && microphoneState === "on") {
microphone.click();
await waitForUi();
const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i);
microphoneState = toggleState(currentMicrophone, "microphone");
}
controlManualActionReason = "teams-audio-choice-required";
controlManualActionMessage = "Select BlackHole 2ch as the Teams microphone and verify it is selected before enabling talk-back.";
} else if (!readOnly && microphoneState === "off") {
microphone.click();
await waitForUi();
const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i);
microphoneState = toggleState(currentMicrophone, "microphone");
if (microphoneState === "on") {
notes.push("Unmuted the Teams microphone after verifying BlackHole 2ch input.");
}
}
if (audioInputRouted && microphoneState !== "on") {
controlManualActionReason = "teams-microphone-required";
controlManualActionMessage = "Unmute the Teams microphone and verify the microphone control shows it is on, then retry joining.";
}
} else if (!readOnly && identityVerified && !inCall && !allowMicrophone && microphoneState === "on") {
microphone.click();
await waitForUi();
const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i);
microphoneState = toggleState(currentMicrophone, "microphone");
if (microphoneState === "off") {
notes.push("Muted the Teams microphone for observe-only mode.");
}
}
if (identityVerified && inCall && allowMicrophone) {
if (!selectedMicrophoneLabel() && !readOnly && microphoneState === "on") {
microphone?.click();
await waitForUi();
const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i);
microphoneState = toggleState(currentMicrophone, "microphone");
}
audioInputRouted = await ensureVirtualAudioInput();
if (audioInputRouted && !readOnly && microphoneState === "off") {
microphone?.click();
await waitForUi();
const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i);
microphoneState = toggleState(currentMicrophone, "microphone");
} else if (!audioInputRouted && !readOnly && microphoneState === "on") {
microphone?.click();
await waitForUi();
const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i);
microphoneState = toggleState(currentMicrophone, "microphone");
if (microphoneState === "off") {
notes.push("Muted the Teams microphone because BlackHole 2ch input could not be reverified.");
}
}
}
if (identityVerified && !inCall && join && !allowMicrophone && microphoneState !== "off") {
controlManualActionReason = "teams-microphone-required";
controlManualActionMessage = "Mute the Teams microphone and verify the microphone control shows it is off, then retry joining.";
}
if (identityVerified && !inCall && join && allowMicrophone && !controlManualActionReason) {
if (!microphone) {
controlManualActionReason = "teams-microphone-required";
controlManualActionMessage = "Open Teams device settings and verify the microphone control before enabling talk-back.";
} else if (audioInputRouted !== true) {
controlManualActionReason = "teams-audio-choice-required";
controlManualActionMessage = "Select BlackHole 2ch as the Teams microphone and verify it is selected before enabling talk-back.";
} else if (microphoneState !== "on") {
controlManualActionReason = "teams-microphone-required";
controlManualActionMessage = "Unmute the Teams microphone and verify the microphone control shows it is on, then retry joining.";
}
}
const micMuted = microphoneState === "off" ? true : microphoneState === "on" ? false : undefined;
const cameraOff = cameraState === "off" ? true : cameraState === "on" ? false : undefined;
const pageText = text(document.body);
const pageTextLower = pageText.toLowerCase();
const lobbyWaiting = Boolean(first(selectors.lobby)) ||
/someone will let you in shortly|waiting for someone to let you in|when someone admits you|you.?re in the lobby|we.?ve let people in the meeting know you.?re waiting/i.test(pageTextLower);
const signInControl = first(selectors.signIn);
const hostname = location.hostname.toLowerCase();
const tenantLoginRequired =
/only people with a work or school account|sign in with an account from this organization|anonymous users (?:can.?t|cannot) join|verify your email|enter the code sent to/i.test(pageTextLower);
const loginRequired = hostname === "login.microsoftonline.com" ||
hostname.endsWith(".microsoftonline.com") ||
tenantLoginRequired ||
(Boolean(signInControl) && !guestInput && !join && /sign in to (?:join|continue)|sign in to your account/i.test(pageTextLower));
const permissionRequired = Boolean(first(selectors.permissionPrompt));
let manualActionReason;
let manualActionMessage;
if (!inCall && loginRequired) {
manualActionReason = "teams-login-required";
manualActionMessage = tenantLoginRequired
? "This Teams tenant requires sign-in or email verification. Complete it in the OpenClaw browser profile, then retry."
: "Sign in to Microsoft Teams in the OpenClaw browser profile, then retry the meeting join.";
} else if (!inCall && lobbyWaiting) {
manualActionReason = "teams-admission-required";
manualActionMessage = "Admit the OpenClaw guest from the Microsoft Teams lobby, then retry speech.";
} else if (!inCall && permissionRequired) {
manualActionReason = "teams-permission-required";
manualActionMessage = allowMicrophone
? "Allow microphone and camera permissions for Teams in the OpenClaw browser profile, then retry."
: "Dismiss the Teams device-permission prompt or continue without devices, then retry.";
} else if (!inCall && controlManualActionReason) {
manualActionReason = controlManualActionReason;
manualActionMessage = controlManualActionMessage;
}
let clickedJoin = false;
if (!readOnly && identityVerified && autoJoin && !inCall && join && !join.disabled && !manualActionReason) {
join.click();
clickedJoin = true;
notes.push("Clicked the Teams guest join button.");
}
let audioOutputRouted;
let audioOutputDeviceLabel;
let audioOutputRouteError;
if (inCall && allowMicrophone && navigator.mediaDevices?.enumerateDevices) {
const media = [...document.querySelectorAll("audio, video")].filter((element) =>
typeof element.setSinkId === "function"
);
if (media.length > 0) {
try {
const devices = await navigator.mediaDevices.enumerateDevices();
const output = devices.find((device) => device.kind === "audiooutput" && isBlackHole(device.label));
if (output?.deviceId) {
for (const element of media) {
if (!readOnly && element.sinkId !== output.deviceId) await element.setSinkId(output.deviceId);
}
audioOutputRouted = media.some((element) => element.sinkId === output.deviceId);
audioOutputDeviceLabel = output.label || "BlackHole 2ch";
} else {
audioOutputRouted = false;
notes.push("BlackHole 2ch speaker output was not visible to Teams.");
}
} catch (error) {
audioOutputRouted = false;
audioOutputRouteError = error?.message || String(error);
notes.push("Could not route Teams speaker output to BlackHole 2ch: " + audioOutputRouteError);
}
} else {
audioOutputRouted = false;
}
}
if (inCall && allowMicrophone) {
if (audioInputRouted !== true || audioOutputRouted !== true) {
manualActionReason = "teams-audio-choice-required";
manualActionMessage = "Verify BlackHole 2ch is selected as both the Teams microphone and speaker before starting talk-back.";
} else if (micMuted !== false) {
manualActionReason = "teams-microphone-required";
manualActionMessage = "Unmute the Teams microphone and verify the microphone control shows it is on before starting talk-back.";
}
}
return JSON.stringify({
clickedContinueInBrowser: Boolean(continueInBrowser),
clickedJoin,
inCall,
micMuted,
cameraOff,
lobbyWaiting,
audioInputRouted,
audioInputDeviceLabel,
audioInputRouteError,
audioOutputRouted,
audioOutputDeviceLabel,
audioOutputRouteError,
manualActionRequired: Boolean(manualActionReason),
manualActionReason,
manualActionMessage,
title: document.title,
url: location.href,
notes,
});
}`;
}
export function teamsMeetingTranscriptScript(meetingUrl: string, meetingSessionId: string) {
export function teamsMeetingTranscriptScript(
meetingUrl: string,
meetingSessionId: string,
finalize: boolean,
) {
const expectedIdentity = normalizeTeamsMeetingUrlForReuse(meetingUrl);
return `() => {
${pageIdentityFunctionSource()}
@@ -501,26 +115,155 @@ export function teamsMeetingTranscriptScript(meetingUrl: string, meetingSessionI
const expectedSessionId = ${JSON.stringify(meetingSessionId)};
const currentIdentity = meetingIdentity(location.href);
const state = window.__openclawTeamsMeeting;
const effectiveIdentity = currentIdentity || state?.identity;
const activeCaptions = window.__openclawTeamsCaptions;
const archivedCaptions = window.__openclawTeamsCaptionArchive?.[expectedSessionId];
const captions = activeCaptions &&
(!activeCaptions.sessionId || activeCaptions.sessionId === expectedSessionId)
? activeCaptions
: archivedCaptions;
// A same-session finalized buffer belongs to the departed call even if Teams
// immediately navigated this tab into another meeting before transcript pickup.
const useFinalizedCaptions = Boolean(
captions?.finalized === true &&
captions?.identity === expectedIdentity &&
(!captions?.sessionId || captions.sessionId === expectedSessionId)
);
const effectiveIdentity = useFinalizedCaptions
? captions.identity
: currentIdentity || state?.identity || captions?.identity;
if (!expectedIdentity || effectiveIdentity !== expectedIdentity) {
return JSON.stringify({ urlMatched: false, droppedLines: 0, lines: [] });
}
if (state?.sessionId && state.sessionId !== expectedSessionId) {
if (!useFinalizedCaptions && state?.sessionId && state.sessionId !== expectedSessionId) {
return JSON.stringify({ urlMatched: true, sessionMatched: false, droppedLines: 0, lines: [] });
}
return JSON.stringify({ urlMatched: true, sessionMatched: true, droppedLines: 0, lines: [] });
if (captions?.sessionId && captions.sessionId !== expectedSessionId) {
return JSON.stringify({ urlMatched: true, sessionMatched: false, droppedLines: 0, lines: [] });
}
if (${JSON.stringify(finalize)} && Array.isArray(captions?.visible) && captions.visible.length > 0) {
if (captions.settleTimer !== undefined) clearTimeout(captions.settleTimer);
captions.settleTimer = undefined;
captions.lines = Array.isArray(captions.lines) ? captions.lines : [];
captions.lines.push(...captions.visible.map((entry) => ({
at: entry.at,
speaker: entry.speaker,
text: entry.text,
})));
captions.visible = [];
const excess = captions.lines.length - ${TEAMS_MEETING_TRANSCRIPT_MAX_LINES};
if (excess > 0) {
captions.lines.splice(0, excess);
captions.droppedLines = (captions.droppedLines || 0) + excess;
}
}
if (${JSON.stringify(finalize)} && captions) {
if (captions.settleTimer !== undefined) clearTimeout(captions.settleTimer);
captions.settleTimer = undefined;
captions.observer?.disconnect?.();
captions.observer = undefined;
captions.observerInstalled = false;
captions.identity = expectedIdentity;
captions.finalized = true;
captions.finalizedAt = Date.now();
}
const allLines = [
...(Array.isArray(captions?.lines) ? captions.lines : []),
...(${JSON.stringify(finalize)} || !Array.isArray(captions?.visible) ? [] : captions.visible),
];
const visibleOverflow = Math.max(0, allLines.length - ${TEAMS_MEETING_TRANSCRIPT_MAX_LINES});
const lines = allLines.slice(-${TEAMS_MEETING_TRANSCRIPT_MAX_LINES});
const result = {
urlMatched: true,
sessionMatched: true,
epoch: typeof captions?.epoch === "string" ? captions.epoch : undefined,
droppedLines: (Number.isFinite(captions?.droppedLines)
? Math.max(0, Math.trunc(captions.droppedLines))
: 0) + visibleOverflow,
lines: lines.map((line) => ({
at: typeof line?.at === "string" ? line.at : undefined,
speaker: typeof line?.speaker === "string" ? line.speaker : undefined,
text: typeof line?.text === "string" ? line.text : "",
})).filter((line) => line.text),
};
return JSON.stringify(result);
}`;
}
export function teamsMeetingLeaveScript(meetingUrl: string) {
export function teamsMeetingLeaveScript(params: {
leaveInitiated: boolean;
meetingSessionId: string;
meetingUrl: string;
}) {
const selectors = JSON.stringify(TEAMS_MEETING_SELECTORS);
const expectedIdentity = normalizeTeamsMeetingUrlForReuse(meetingUrl);
const expectedIdentity = normalizeTeamsMeetingUrlForReuse(params.meetingUrl);
return `() => {
${pageIdentityFunctionSource()}
const selectors = ${selectors};
const expectedIdentity = ${JSON.stringify(expectedIdentity)};
const expectedSessionId = ${JSON.stringify(params.meetingSessionId)};
const leaveInitiated = ${JSON.stringify(params.leaveInitiated)};
const currentIdentity = meetingIdentity(location.href);
const state = window.__openclawTeamsMeeting;
const enforceSessionOwnership = Boolean(expectedSessionId);
if (enforceSessionOwnership && state?.sessionId && state.sessionId !== expectedSessionId) {
return JSON.stringify({ departed: false, sessionConflict: true, sessionMatched: false, urlMatched: true });
}
const sessionMatched = !enforceSessionOwnership || state?.sessionId === expectedSessionId;
const retainedLeaveOwnership = Boolean(!sessionMatched && leaveInitiated);
if (!sessionMatched && !retainedLeaveOwnership) {
return JSON.stringify({ departed: false, sessionMatched: false, urlMatched: true });
}
const retireOwnedAudioBridges = () => {
const entries = Array.isArray(window.__openclawTeamsAudioOutputs)
? window.__openclawTeamsAudioOutputs
: [];
const retained = [];
const activeSessionId = expectedSessionId || state?.sessionId;
for (const entry of entries) {
const ownedByActiveSession = Boolean(
!entry?.sessionId || (activeSessionId && entry.sessionId === activeSessionId)
);
if (!ownedByActiveSession) {
retained.push(entry);
continue;
}
const mediaSourceUrl = (element) => String(element?.currentSrc || element?.src || "");
const sources = Array.isArray(entry?.sources)
? entry.sources
: entry?.source
? [{ element: entry.source, muted: Boolean(entry.sourceMuted), stream: entry.stream, url: entry.sourceUrl }]
: [];
for (const source of sources) {
const element = source?.element;
const sourceMatches = source?.stream || element?.srcObject
? element?.srcObject === source?.stream
: Boolean(source?.url && mediaSourceUrl(element) === source.url);
const sourceIsEmpty = Boolean(element && !element.srcObject && !mediaSourceUrl(element));
if (!element) continue;
if (sourceIsEmpty) {
element.muted = true;
continue;
}
if (!sourceMatches) continue;
const detachedLiveSource = Boolean(
element.isConnected === false &&
element.srcObject?.getAudioTracks?.().some((track) => track.readyState === "live")
);
if (detachedLiveSource) {
element.muted = true;
element.pause?.();
element.srcObject = null;
} else {
element.muted = Boolean(source.muted);
}
}
entry?.bridge?.pause?.();
if (entry?.bridge) entry.bridge.srcObject = null;
entry?.bridge?.remove?.();
}
if (retained.length > 0) window.__openclawTeamsAudioOutputs = retained;
else delete window.__openclawTeamsAudioOutputs;
};
const first = (list) => {
for (const selector of list) {
const node = document.querySelector(selector);
@@ -558,17 +301,27 @@ export function teamsMeetingLeaveScript(meetingUrl: string) {
Date.now() - state?.verifiedAt < 5_000 &&
!leave
);
if (
!currentUrlMatches &&
!preservedCallMatches &&
!pendingLeaveMatches &&
!rerenderPendingMatches
) {
const meetingIdentityMatches = Boolean(
currentUrlMatches || preservedCallMatches || pendingLeaveMatches || rerenderPendingMatches
);
// Teams can replace the document between our Leave click and its post-call marker.
// Retain request ownership only while no identity or live-call control contradicts it.
const initiatedLeaveTransitionMatches = Boolean(
leaveInitiated &&
!currentIdentity &&
!leave &&
(!state?.identity || state.identity === expectedIdentity)
);
if (postCall && (meetingIdentityMatches || initiatedLeaveTransitionMatches)) {
retireOwnedAudioBridges();
if (sessionMatched) delete window.__openclawTeamsMeeting;
return JSON.stringify({ departed: true, sessionMatched: true, urlMatched: true });
}
if (!meetingIdentityMatches && !initiatedLeaveTransitionMatches) {
return JSON.stringify({ departed: false, urlMatched: false });
}
if (postCall) {
delete window.__openclawTeamsMeeting;
return JSON.stringify({ departed: true, urlMatched: true });
if (!sessionMatched) {
return JSON.stringify({ departed: false, urlMatched: true });
}
if (confirmation) {
confirmation.click();
extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.test-helpers.ts
+395
View File
@@ -0,0 +1,395 @@
import { runInNewContext } from "node:vm";
import {
teamsMeetingLeaveScript,
teamsMeetingStatusScript,
} from "./teams-meetings-page-scripts.js";
import { TEAMS_MEETINGS_PLATFORM_ADAPTER } from "./teams-meetings-platform-adapter.js";
export const URL =
"https://teams.microsoft.com/l/meetup-join/19%3ameeting_test%40thread.v2/0?context=%7b%7d";
export const CONSUMER_URL = "https://teams.live.com/meet/9326458712345?p=abc";
export const MEETING_STATE_KEY = "__openclawTeamsMeeting";
export function consumerLightMeetingUrl(meetingCode: string, passcode: string) {
const coordinates = btoa(JSON.stringify({ meetingCode, passcode }));
return `https://teams.live.com/light-meetings/launch?coords=${encodeURIComponent(coordinates)}`;
}
export function status(manualActionReason: string, manualActionMessage = "manual action") {
const health = TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.parseStatus({
result: JSON.stringify({
inCall: false,
manualActionRequired: true,
manualActionReason,
manualActionMessage,
url: URL,
}),
});
if (!health) {
throw new Error("expected parsed health");
}
return health;
}
type PageControl = {
checked?: boolean;
disabled?: boolean;
clicks: number;
isConnected: boolean;
click(): void;
closest(selector?: string): PageControl;
getAttribute(name: string): string | null;
matches(selector: string): boolean;
querySelector(selector?: string): PageControl | undefined;
querySelectorAll?(selector: string): PageControl[];
setPressed(pressed: boolean): void;
textContent: string;
};
export type PageMedia = {
autoplay?: boolean;
currentSrc?: string;
hidden?: boolean;
id?: string;
isConnected?: boolean;
muted?: boolean;
readyState?: number;
sinkId: string;
src?: string;
srcObject?: { getAudioTracks(): Array<{ readyState: string }> };
play?(): Promise<void>;
pause?(): void;
remove?(): void;
setSinkId(value: string): Promise<void>;
};
export function control(params: {
checked?: boolean;
label: string;
text?: string;
pressed?: boolean;
onClick?: (control: PageControl) => void;
}): PageControl {
const attributes = new Map<string, string>([["aria-label", params.label]]);
if (params.pressed !== undefined) {
attributes.set("aria-pressed", String(params.pressed));
}
const node: PageControl = {
...(params.checked === undefined ? {} : { checked: params.checked }),
clicks: 0,
isConnected: true,
textContent: params.text ?? "",
click() {
node.clicks += 1;
params.onClick?.(node);
},
closest: () => node,
getAttribute: (name) => attributes.get(name) ?? null,
matches: (selector) => selector === "button",
querySelector: () => undefined,
setPressed: (pressed) => attributes.set("aria-pressed", String(pressed)),
};
return node;
}
export function captionRow(
speaker: string,
captionText: string,
rowIdentity?: string,
): PageControl {
const author = control({ label: "", text: speaker });
const content = control({ label: "", text: captionText });
const row = control({ label: "" });
const getAttribute = row.getAttribute.bind(row);
row.getAttribute = (name) =>
name === "aria-posinset" && rowIdentity ? rowIdentity : getAttribute(name);
row.querySelector = (selector) => {
if (selector?.includes('data-tid="author"')) {
return author;
}
if (selector?.includes('data-tid="closed-caption-text"')) {
return content;
}
return undefined;
};
return row;
}
export async function runStatusScript(params: {
allowMicrophone: boolean;
allowSessionAdoption?: boolean;
autoJoin?: boolean;
bodyText?: string;
currentUrl?: string;
meetingUrl?: string;
microphone?: PageControl;
camera?: PageControl;
captionClickIgnored?: boolean;
captionsInitiallyOn?: boolean;
captionRows?: PageControl[];
captureCaptions?: boolean;
bridgeMedia?: PageMedia | PageMedia[];
continueWithoutDevices?: PageControl;
deviceSettings?: PageControl;
join?: PageControl;
leave?: PageControl;
microphoneDevice?: PageControl;
microphoneDeviceAfterSettings?: PageControl;
microphoneDeviceMenuAfterSettings?: PageControl;
microphonePermissionState?: "denied" | "granted" | "prompt";
permissionPrompt?: PageControl;
priorAudioOutputs?: unknown[];
priorCaptions?: unknown;
priorMeeting?: Record<string, unknown>;
readOnly?: boolean;
waitForInCallMs?: number;
globalSelectedOption?: PageControl;
media?: PageMedia[];
meetingSessionId?: string;
devices?: Array<{ deviceId: string; kind: string; label: string }>;
}) {
const currentUrl = params.currentUrl ?? URL;
const location = new globalThis.URL(currentUrl);
const controls = [
params.microphone,
params.camera,
params.continueWithoutDevices,
params.deviceSettings,
params.join,
params.leave,
].filter((entry): entry is PageControl => Boolean(entry));
let captionMenuOpen = false;
let captionsOn = params.captionsInitiallyOn ?? Boolean(params.priorCaptions);
const moreActions = control({
label: "More",
onClick: () => {
captionMenuOpen = true;
},
});
const captionButton = control({
label: "Captions Show live captions",
onClick: () => {
if (!params.captionClickIgnored) {
captionsOn = true;
}
},
});
const captionContent = control({ label: "", text: "" });
captionContent.querySelectorAll = () => params.captionRows ?? [];
const captionRenderer = control({ label: "Live Captions" });
const body = {
textContent: params.bodyText ?? "",
appendChild(node: PageMedia) {
node.isConnected = true;
},
};
let bridgeMediaIndex = 0;
const document = {
body,
createElement() {
if (!params.bridgeMedia) {
throw new Error("unexpected media bridge");
}
if (!Array.isArray(params.bridgeMedia)) {
return params.bridgeMedia;
}
const bridge = params.bridgeMedia[bridgeMediaIndex++];
if (!bridge) {
throw new Error("missing media bridge fixture");
}
return bridge;
},
title: "Teams",
getElementById() {
return undefined;
},
querySelector(selector: string) {
if (selector.includes("toggle-mute")) {
return params.microphone;
}
if (selector.includes("toggle-video")) {
return params.camera;
}
if (selector.includes("prejoin-join-button")) {
return params.join;
}
if (selector.includes("call-hangup")) {
return params.leave;
}
if (selector.includes("hangup-button")) {
return params.leave;
}
if (selector.includes("More")) {
return params.captionRows ? moreActions : undefined;
}
if (selector.includes("closed-captions-button") || selector.includes("title*=")) {
return params.captionRows && captionMenuOpen ? captionButton : undefined;
}
if (selector.includes("closed-caption-renderer-wrapper")) {
return params.captionRows && captionsOn ? captionRenderer : undefined;
}
if (selector.includes("closed-caption-v2-virtual-list-content")) {
return params.captionRows && captionsOn ? captionContent : undefined;
}
if (
selector.includes("audio-button-configure") ||
selector.includes("Open audio options") ||
selector.includes("device-settings-button")
) {
return params.deviceSettings;
}
if (
selector.includes("selected-microphone-display") ||
selector.includes("microphone-select") ||
selector.includes("audio-device-input") ||
selector.includes("device-settings-microphone")
) {
return params.deviceSettings?.clicks
? (params.microphoneDeviceAfterSettings ?? params.microphoneDevice)
: params.microphoneDevice;
}
if (selector.includes("microphone-settings")) {
return params.deviceSettings?.clicks
? (params.microphoneDeviceMenuAfterSettings ?? params.microphoneDeviceAfterSettings)
: undefined;
}
if (selector.includes("permission-prompt") || selector.includes("permission-error")) {
return params.permissionPrompt;
}
if (selector === '[role="option"][aria-selected="true"]') {
return params.globalSelectedOption;
}
return undefined;
},
querySelectorAll(selector: string) {
if (selector === "button") {
return controls;
}
if (selector === "audio, video") {
return params.media ?? [];
}
if (selector.includes('[role="option"]')) {
return params.globalSelectedOption ? [params.globalSelectedOption] : [];
}
return [];
},
};
const window: Record<string, unknown> = {};
let captionObserverCallback:
| ((records?: Array<{ removedNodes: PageControl[] }>) => void)
| undefined;
let captionObserverDisconnects = 0;
if (params.priorMeeting) {
window[MEETING_STATE_KEY] = params.priorMeeting;
}
if (params.priorAudioOutputs) {
window["__openclawTeamsAudioOutputs"] = params.priorAudioOutputs;
}
if (params.priorCaptions) {
window["__openclawTeamsCaptions"] = params.priorCaptions;
}
const script = teamsMeetingStatusScript({
allowMicrophone: params.allowMicrophone,
allowSessionAdoption: params.allowSessionAdoption ?? true,
autoJoin: params.autoJoin ?? true,
captureCaptions: params.captureCaptions ?? false,
guestName: "OpenClaw Guest",
meetingSessionId: params.meetingSessionId === undefined ? "session-1" : params.meetingSessionId,
meetingUrl: params.meetingUrl ?? URL,
readOnly: params.readOnly,
waitForInCallMs: params.waitForInCallMs ?? 60_000,
});
const run = runInNewContext(`(${script})`, {
Event: globalThis.Event,
HTMLInputElement: function HTMLInputElement() {},
MutationObserver: class MutationObserver {
constructor(callback: (records?: Array<{ removedNodes: PageControl[] }>) => void) {
captionObserverCallback = callback;
}
disconnect() {
captionObserverDisconnects += 1;
}
observe() {}
},
URL: globalThis.URL,
atob: globalThis.atob,
crypto: { randomUUID: () => "teams-caption-epoch" },
document,
location,
navigator: {
mediaDevices: {
enumerateDevices: async () => params.devices ?? [],
},
permissions: {
query: async () => ({ state: params.microphonePermissionState ?? "prompt" }),
},
},
clearTimeout,
setTimeout,
window,
}) as () => Promise<string>;
return {
captionButton,
captionObserverDisconnects: () => captionObserverDisconnects,
triggerCaptionMutation(nextUrl?: string, removedNode?: PageControl) {
if (nextUrl) {
location.href = nextUrl;
}
captionObserverCallback?.(removedNode ? [{ removedNodes: [removedNode] }] : []);
},
result: JSON.parse(await run()) as Record<string, unknown>,
window,
};
}
export function runLeaveScript(params: {
bodyText?: string;
currentUrl?: string;
leave?: PageControl;
leaveInitiated?: boolean;
meetingSessionId?: string;
omitMeetingState?: boolean;
postCall?: PageControl;
priorAudioOutputs?: unknown[];
priorMeeting?: Record<string, unknown>;
}) {
const currentUrl = params.currentUrl ?? URL;
const location = new globalThis.URL(currentUrl);
const document = {
body: { textContent: params.bodyText ?? "" },
querySelector(selector: string) {
if (selector.includes("call-hangup")) {
return params.leave;
}
if (
selector.includes("call-ended-screen") ||
selector.includes("post-call-screen") ||
selector.includes("prejoin-rejoin-button")
) {
return params.postCall;
}
return undefined;
},
};
const window: Record<string, unknown> = {};
if (!params.omitMeetingState) {
window[MEETING_STATE_KEY] = {
sessionId: params.meetingSessionId ?? "session-1",
...params.priorMeeting,
};
}
if (params.priorAudioOutputs) {
window["__openclawTeamsAudioOutputs"] = params.priorAudioOutputs;
}
const run = runInNewContext(
`(${teamsMeetingLeaveScript({ leaveInitiated: params.leaveInitiated ?? false, meetingSessionId: params.meetingSessionId ?? "session-1", meetingUrl: URL })})`,
{
URL: globalThis.URL,
document,
location,
window,
},
) as () => string;
return { result: JSON.parse(run()) as Record<string, unknown>, window };
}
extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.test.ts
+651 -253
View File
@@ -1,208 +1,18 @@
import { runInNewContext } from "node:vm";
import { describe, expect, it } from "vitest";
import {
teamsMeetingLeaveScript,
teamsMeetingStatusScript,
} from "./teams-meetings-page-scripts.js";
import { describe, expect, it, vi } from "vitest";
import {
TEAMS_MEETINGS_PLATFORM_ADAPTER,
isTeamsMeetingsRealtimeRouteReady,
} from "./teams-meetings-platform-adapter.js";
const URL =
"https://teams.microsoft.com/l/meetup-join/19%3ameeting_test%40thread.v2/0?context=%7b%7d";
const MEETING_STATE_KEY = "__openclawTeamsMeeting";
function status(manualActionReason: string, manualActionMessage = "manual action") {
const health = TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.parseStatus({
result: JSON.stringify({
inCall: false,
manualActionRequired: true,
manualActionReason,
manualActionMessage,
url: URL,
}),
});
if (!health) {
throw new Error("expected parsed health");
}
return health;
}
type PageControl = {
disabled?: boolean;
clicks: number;
isConnected: boolean;
click(): void;
closest(selector?: string): PageControl;
getAttribute(name: string): string | null;
matches(selector: string): boolean;
querySelector(): undefined;
querySelectorAll?(selector: string): PageControl[];
setPressed(pressed: boolean): void;
textContent: string;
};
function control(params: {
label: string;
pressed?: boolean;
onClick?: (control: PageControl) => void;
}): PageControl {
const attributes = new Map<string, string>([["aria-label", params.label]]);
if (params.pressed !== undefined) {
attributes.set("aria-pressed", String(params.pressed));
}
const node: PageControl = {
clicks: 0,
isConnected: true,
textContent: "",
click() {
node.clicks += 1;
params.onClick?.(node);
},
closest: () => node,
getAttribute: (name) => attributes.get(name) ?? null,
matches: (selector) => selector === "button",
querySelector: () => undefined,
setPressed: (pressed) => attributes.set("aria-pressed", String(pressed)),
};
return node;
}
async function runStatusScript(params: {
allowMicrophone: boolean;
bodyText?: string;
currentUrl?: string;
microphone?: PageControl;
camera?: PageControl;
join?: PageControl;
leave?: PageControl;
microphoneDevice?: PageControl;
permissionPrompt?: PageControl;
priorMeeting?: Record<string, unknown>;
readOnly?: boolean;
globalSelectedOption?: PageControl;
media?: Array<{ sinkId: string; setSinkId(value: string): Promise<void> }>;
devices?: Array<{ deviceId: string; kind: string; label: string }>;
}) {
const currentUrl = params.currentUrl ?? URL;
const location = new globalThis.URL(currentUrl);
const controls = [params.microphone, params.camera, params.join, params.leave].filter(
(entry): entry is PageControl => Boolean(entry),
);
const document = {
body: { textContent: params.bodyText ?? "" },
title: "Teams",
getElementById() {
return undefined;
},
querySelector(selector: string) {
if (selector.includes("toggle-mute")) {
return params.microphone;
}
if (selector.includes("toggle-video")) {
return params.camera;
}
if (selector.includes("prejoin-join-button")) {
return params.join;
}
if (selector.includes("call-hangup")) {
return params.leave;
}
if (
selector.includes("microphone-select") ||
selector.includes("audio-device-input") ||
selector.includes("device-settings-microphone")
) {
return params.microphoneDevice;
}
if (selector.includes("permission-prompt") || selector.includes("permission-error")) {
return params.permissionPrompt;
}
if (selector === '[role="option"][aria-selected="true"]') {
return params.globalSelectedOption;
}
return undefined;
},
querySelectorAll(selector: string) {
if (selector === "button") {
return controls;
}
if (selector === "audio, video") {
return params.media ?? [];
}
if (selector.includes('[role="option"]')) {
return params.globalSelectedOption ? [params.globalSelectedOption] : [];
}
return [];
},
};
const window: Record<string, unknown> = {};
if (params.priorMeeting) {
window[MEETING_STATE_KEY] = params.priorMeeting;
}
const script = teamsMeetingStatusScript({
allowMicrophone: params.allowMicrophone,
autoJoin: true,
guestName: "OpenClaw Guest",
meetingSessionId: "session-1",
meetingUrl: URL,
readOnly: params.readOnly,
});
const run = runInNewContext(`(${script})`, {
Event: globalThis.Event,
HTMLInputElement: function HTMLInputElement() {},
URL: globalThis.URL,
document,
location,
navigator: {
mediaDevices: {
enumerateDevices: async () => params.devices ?? [],
},
},
setTimeout,
window,
}) as () => Promise<string>;
return { result: JSON.parse(await run()) as Record<string, unknown>, window };
}
function runLeaveScript(params: {
bodyText?: string;
currentUrl?: string;
leave?: PageControl;
postCall?: PageControl;
priorMeeting?: Record<string, unknown>;
}) {
const currentUrl = params.currentUrl ?? URL;
const location = new globalThis.URL(currentUrl);
const document = {
body: { textContent: params.bodyText ?? "" },
querySelector(selector: string) {
if (selector.includes("call-hangup")) {
return params.leave;
}
if (
selector.includes("call-ended-screen") ||
selector.includes("post-call-screen") ||
selector.includes("prejoin-rejoin-button")
) {
return params.postCall;
}
return undefined;
},
};
const window: Record<string, unknown> = {};
if (params.priorMeeting) {
window[MEETING_STATE_KEY] = params.priorMeeting;
}
const run = runInNewContext(`(${teamsMeetingLeaveScript(URL)})`, {
URL: globalThis.URL,
document,
location,
window,
}) as () => string;
return { result: JSON.parse(run()) as Record<string, unknown>, window };
}
import {
CONSUMER_URL,
MEETING_STATE_KEY,
consumerLightMeetingUrl,
status,
control,
runStatusScript,
runLeaveScript,
type PageMedia,
} from "./teams-meetings-platform-adapter.test-helpers.js";
describe("Microsoft Teams meeting platform adapter", () => {
it.each([
@@ -220,6 +30,56 @@ describe("Microsoft Teams meeting platform adapter", () => {
});
});
it("retries transient in-call audio routing while Teams renders its media controls", () => {
const pending = { ...status("teams-audio-choice-required"), inCall: true };
expect(TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.shouldRetryJoinStatus?.(pending)).toBe(false);
expect(
TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.shouldRetryJoinStatus?.({
...pending,
audioInputRouted: true,
audioOutputRouteRetryable: true,
}),
).toBe(true);
expect(
TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.shouldRetryJoinStatus?.({
...pending,
audioOutputRouteError: "sink failed",
}),
).toBe(false);
expect(
TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.shouldRetryJoinStatus?.({
...pending,
audioInputRouted: true,
audioOutputRouteError: "play interrupted",
audioOutputRouteRetryable: true,
}),
).toBe(true);
});
it("retries transcribe join readiness until live captions are enabled", () => {
const pending = {
captionCaptureRequested: true,
captioning: false,
inCall: true,
};
expect(TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.shouldRetryJoinStatus?.(pending)).toBe(true);
expect(
TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.shouldRetryJoinStatus?.({
...pending,
captioning: true,
}),
).toBe(false);
expect(
TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.shouldRetryJoinStatus?.({
...pending,
manualActionReason: "teams-session-conflict",
manualActionRequired: true,
}),
).toBe(false);
});
it.each([
["camera", "Turn camera off", undefined, "on"],
["camera", "Turn camera on", undefined, "off"],
@@ -248,6 +108,21 @@ describe("Microsoft Teams meeting platform adapter", () => {
},
);
it.each([
["camera", true, false],
["camera", false, true],
["microphone", true, false],
["microphone", false, true],
])("reads the live %s switch checked=%s", async (kind, checked, expectedOff) => {
const target = control({ checked, label: kind === "camera" ? "Camera" : "Microphone" });
const { result } = await runStatusScript({
allowMicrophone: false,
...(kind === "camera" ? { camera: target } : { microphone: target }),
readOnly: true,
});
expect(kind === "camera" ? result.cameraOff : result.micMuted).toBe(expectedOff);
});
it("re-reads camera and microphone state after toggling before joining", async () => {
const camera = control({
label: "Turn camera off",
@@ -274,6 +149,54 @@ describe("Microsoft Teams meeting platform adapter", () => {
expect(join.clicks).toBe(1);
});
it("allows non-adopting recovery to continue for the current page owner", async () => {
const join = control({ label: "Join now" });
const { result } = await runStatusScript({
allowMicrophone: false,
allowSessionAdoption: false,
camera: control({ label: "Turn camera on", pressed: false }),
join,
microphone: control({ label: "Turn microphone on", pressed: false }),
priorMeeting: {
identity: "teams-work:19:meeting_test@thread.v2",
sessionId: "session-1",
},
});
expect(join.clicks).toBe(1);
expect(result).toMatchObject({ clickedJoin: true, manualActionRequired: false });
});
it("preserves a newer owner for a different meeting identity", async () => {
const priorMeeting = {
identity: "teams-consumer:9326458712345:p:abc",
sessionId: "consumer-session",
};
const { result, window } = await runStatusScript({
allowMicrophone: false,
allowSessionAdoption: true,
currentUrl: CONSUMER_URL,
priorMeeting,
});
expect(result).toMatchObject({ manualActionReason: "teams-session-conflict" });
expect(window[MEETING_STATE_KEY]).toBe(priorMeeting);
});
it("does not mask an in-call session conflict with talkback audio readiness", async () => {
const { result } = await runStatusScript({
allowMicrophone: true,
allowSessionAdoption: false,
leave: control({ label: "Leave" }),
priorMeeting: {
identity: "teams-work:19:meeting_test@thread.v2",
sessionId: "session-2",
},
});
expect(result).toMatchObject({ manualActionReason: "teams-session-conflict" });
});
it("does not unmute or join until BlackHole is visibly selected as the Teams input", async () => {
const camera = control({ label: "Turn camera on", pressed: false });
const microphone = control({ label: "Turn microphone on", pressed: false });
@@ -351,6 +274,34 @@ describe("Microsoft Teams meeting platform adapter", () => {
expect(window).not.toHaveProperty("__openclawTeamsMeeting");
});
it("verifies the consumer prejoin redirect from its encoded meeting coordinates", async () => {
const { result, window } = await runStatusScript({
allowMicrophone: false,
currentUrl: consumerLightMeetingUrl("9326458712345", "abc"),
meetingUrl: CONSUMER_URL,
});
expect(result.manualActionRequired).toBe(false);
expect(window[MEETING_STATE_KEY]).toMatchObject({
identity: "teams-consumer:9326458712345:p:abc",
sessionId: "session-1",
});
});
it("rejects consumer prejoin coordinates for a different meeting", async () => {
const { result, window } = await runStatusScript({
allowMicrophone: false,
currentUrl: consumerLightMeetingUrl("1111111111111", "other"),
meetingUrl: CONSUMER_URL,
});
expect(result).toMatchObject({
inCall: false,
manualActionReason: "teams-session-conflict",
});
expect(window).not.toHaveProperty(MEETING_STATE_KEY);
});
it("preserves a verified identity only across an in-call URL transition", async () => {
const leave = control({ label: "Leave" });
const inCallUrl = "https://teams.microsoft.com/v2/";
@@ -389,6 +340,27 @@ describe("Microsoft Teams meeting platform adapter", () => {
});
});
it("retains prejoin identity for the configured in-call wait", async () => {
const leave = control({ label: "Leave" });
const admitted = await runStatusScript({
allowMicrophone: false,
currentUrl: "https://teams.microsoft.com/v2/",
leave,
priorMeeting: {
identity: "teams-work:19:meeting_test@thread.v2",
sessionId: "session-1",
verifiedAt: Date.now() - 45_000,
},
waitForInCallMs: 60_000,
});
expect(admitted.result.inCall).toBe(true);
expect(admitted.window[MEETING_STATE_KEY]).toMatchObject({
identity: "teams-work:19:meeting_test@thread.v2",
inCallControl: leave,
});
});
it("re-adopts a replaced hang-up control only within the bounded rerender window", async () => {
const previousLeave = control({ label: "Leave" });
previousLeave.isConnected = false;
@@ -413,12 +385,22 @@ describe("Microsoft Teams meeting platform adapter", () => {
});
});
it("does not trust a stale identity marker to leave a different SPA call", () => {
it("does not trust a stale identity marker or post-call screen from another SPA call", () => {
const staleLeave = control({ label: "Leave old call" });
const currentLeave = control({ label: "Leave current call" });
const { result } = runLeaveScript({
const bridge = { pause: vi.fn(), remove: vi.fn(), srcObject: {} };
const { result, window } = runLeaveScript({
currentUrl: "https://teams.microsoft.com/v2/",
leave: currentLeave,
postCall: control({ label: "Rejoin" }),
priorAudioOutputs: [
{
bridge,
sessionId: "session-1",
source: { muted: true },
sourceMuted: false,
},
],
priorMeeting: {
identity: "teams-work:19:meeting_test@thread.v2",
inCallControl: staleLeave,
@@ -428,6 +410,254 @@ describe("Microsoft Teams meeting platform adapter", () => {
expect(result).toEqual({ departed: false, urlMatched: false });
expect(currentLeave.clicks).toBe(0);
expect(bridge.pause).not.toHaveBeenCalled();
expect(window["__openclawTeamsAudioOutputs"]).toHaveLength(1);
});
it("does not use initiated-leave proof to act on a replacement SPA call", () => {
const staleLeave = control({ label: "Leave old call" });
const currentLeave = control({ label: "Leave current call" });
const { result } = runLeaveScript({
currentUrl: "https://teams.microsoft.com/v2/",
leave: currentLeave,
leaveInitiated: true,
postCall: control({ label: "Rejoin" }),
priorMeeting: {
identity: "teams-work:19:meeting_test@thread.v2",
inCallControl: staleLeave,
inCallUrl: "https://teams.microsoft.com/v2/",
},
});
expect(result).toEqual({ departed: false, urlMatched: false });
expect(currentLeave.clicks).toBe(0);
});
it("does not leave a call owned by a newer OpenClaw session", () => {
const leave = control({ label: "Leave" });
const inCallUrl = "https://teams.microsoft.com/v2/";
const { result } = runLeaveScript({
currentUrl: inCallUrl,
leave,
meetingSessionId: "session-1",
priorMeeting: {
identity: "teams-work:19:meeting_test@thread.v2",
inCallControl: leave,
inCallUrl,
sessionId: "session-2",
},
});
expect(result).toEqual({
departed: false,
sessionConflict: true,
sessionMatched: false,
urlMatched: true,
});
expect(leave.clicks).toBe(0);
});
it("does not claim departure when page session ownership is missing", () => {
const leave = control({ label: "Leave" });
const { result } = runLeaveScript({
leave,
priorMeeting: {
identity: "teams-work:19:meeting_test@thread.v2",
sessionId: "",
},
});
expect(result).toEqual({ departed: false, sessionMatched: false, urlMatched: true });
expect(leave.clicks).toBe(0);
});
it("keeps the required ID-less leave callback functional for a matching meeting", () => {
const leave = control({ label: "Leave" });
const { result } = runLeaveScript({
leave,
meetingSessionId: "",
priorMeeting: {
identity: "teams-work:19:meeting_test@thread.v2",
sessionId: "session-1",
},
});
expect(result).toEqual({ departed: false, leaveAction: "leave", urlMatched: true });
expect(leave.clicks).toBe(1);
});
it("accepts post-call proof after an initiated leave replaces the document", () => {
const pending = runLeaveScript({
currentUrl: "https://teams.microsoft.com/v2/",
leaveInitiated: true,
omitMeetingState: true,
});
expect(pending.result).toEqual({ departed: false, urlMatched: true });
const { result } = runLeaveScript({
leaveInitiated: true,
omitMeetingState: true,
postCall: control({ label: "Rejoin" }),
});
expect(result).toEqual({ departed: true, sessionMatched: true, urlMatched: true });
});
it("retires only the departing session's audio bridges", () => {
const source = { currentSrc: "blob:https://teams.live.com/original", muted: true };
const bridge = { pause: vi.fn(), remove: vi.fn(), srcObject: {} };
const detachedSource = {
isConnected: false,
muted: true,
pause: vi.fn(),
srcObject: { getAudioTracks: () => [{ readyState: "live" }] },
};
const detachedBridge = { pause: vi.fn(), remove: vi.fn(), srcObject: {} };
const foreignBridge = { sessionId: "session-2" };
const { result, window } = runLeaveScript({
postCall: control({ label: "Rejoin" }),
priorAudioOutputs: [
{
bridge,
sessionId: "session-1",
source,
sourceMuted: false,
sourceUrl: source.currentSrc,
},
{
bridge: detachedBridge,
sessionId: "session-1",
source: detachedSource,
sourceMuted: false,
stream: detachedSource.srcObject,
},
foreignBridge,
],
priorMeeting: {
identity: "teams-work:19:meeting_test@thread.v2",
sessionId: "session-1",
},
});
expect(result).toEqual({ departed: true, sessionMatched: true, urlMatched: true });
expect(source.muted).toBe(false);
expect(bridge.pause).toHaveBeenCalledOnce();
expect(bridge.remove).toHaveBeenCalledOnce();
expect(bridge.srcObject).toBeNull();
expect(detachedSource.muted).toBe(true);
expect(detachedSource.pause).toHaveBeenCalledOnce();
expect(detachedSource.srcObject).toBeNull();
expect(detachedBridge.remove).toHaveBeenCalledOnce();
expect(window["__openclawTeamsAudioOutputs"]).toEqual([foreignBridge]);
});
it("does not unmute a replacement stream during leave cleanup", () => {
const bridgedStream = { getAudioTracks: () => [{ readyState: "live" }] };
const replacementStream = { getAudioTracks: () => [{ readyState: "live" }] };
const source = { muted: true, srcObject: replacementStream };
const bridge = { pause: vi.fn(), remove: vi.fn(), srcObject: bridgedStream };
const { result } = runLeaveScript({
postCall: control({ label: "Rejoin" }),
priorAudioOutputs: [
{
bridge,
sessionId: "session-1",
source,
sourceMuted: false,
stream: bridgedStream,
},
],
priorMeeting: {
identity: "teams-work:19:meeting_test@thread.v2",
sessionId: "session-1",
},
});
expect(result).toEqual({ departed: true, sessionMatched: true, urlMatched: true });
expect(source.muted).toBe(true);
expect(source.srcObject).toBe(replacementStream);
expect(bridge.pause).toHaveBeenCalledOnce();
expect(bridge.remove).toHaveBeenCalledOnce();
});
it("does not unmute a reused URL-backed element during leave cleanup", () => {
const source = { currentSrc: "blob:https://teams.live.com/replacement", muted: true };
const bridge = { pause: vi.fn(), remove: vi.fn(), srcObject: null };
const { result } = runLeaveScript({
postCall: control({ label: "Rejoin" }),
priorAudioOutputs: [
{
bridge,
sessionId: "session-1",
source,
sourceMuted: false,
sourceUrl: "blob:https://teams.live.com/original",
},
],
priorMeeting: {
identity: "teams-work:19:meeting_test@thread.v2",
sessionId: "session-1",
},
});
expect(result).toEqual({ departed: true, sessionMatched: true, urlMatched: true });
expect(source.muted).toBe(true);
expect(bridge.pause).toHaveBeenCalledOnce();
expect(bridge.remove).toHaveBeenCalledOnce();
});
it("restores pending and legacy sources-array entries during leave cleanup", () => {
const pending = { muted: true };
const legacy = { currentSrc: "blob:https://teams.live.com/legacy", muted: true };
const { result } = runLeaveScript({
postCall: control({ label: "Rejoin" }),
priorAudioOutputs: [
{
sessionId: "session-1",
sources: [
{ element: pending, muted: false, pending: true },
{ element: legacy, muted: false },
],
},
],
priorMeeting: {
identity: "teams-work:19:meeting_test@thread.v2",
sessionId: "session-1",
},
});
expect(result).toEqual({ departed: true, sessionMatched: true, urlMatched: true });
expect(pending.muted).toBe(true);
expect(legacy.muted).toBe(true);
});
it("retires the page-owned audio bridge from the required URL-only leave callback", () => {
const stream = { getAudioTracks: () => [{ readyState: "live" }] };
const source = { muted: true, srcObject: stream };
const bridge = { pause: vi.fn(), remove: vi.fn(), srcObject: stream };
const { result, window } = runLeaveScript({
meetingSessionId: "",
postCall: control({ label: "Rejoin" }),
priorAudioOutputs: [
{
bridge,
sessionId: "session-1",
source,
sourceMuted: false,
stream,
},
],
priorMeeting: {
identity: "teams-work:19:meeting_test@thread.v2",
sessionId: "session-1",
},
});
expect(result).toEqual({ departed: true, sessionMatched: true, urlMatched: true });
expect(source.muted).toBe(false);
expect(bridge.pause).toHaveBeenCalledOnce();
expect(bridge.remove).toHaveBeenCalledOnce();
expect(window).not.toHaveProperty("__openclawTeamsAudioOutputs");
});
it.each([
@@ -460,6 +690,52 @@ describe("Microsoft Teams meeting platform adapter", () => {
});
});
it("does not report a prompt that it just dismissed while Teams removes the DOM", async () => {
const continueWithoutDevices = control({ label: "Continue without audio or video" });
const { result } = await runStatusScript({
allowMicrophone: false,
continueWithoutDevices,
permissionPrompt: control({ label: "Device permission prompt" }),
});
expect(result).toMatchObject({ manualActionRequired: false });
expect(continueWithoutDevices.clicks).toBe(1);
});
it("does not treat the live camera troubleshooting banner as a media permission block", async () => {
const join = control({ label: "Join now" });
const { result } = await runStatusScript({
allowMicrophone: true,
bodyText: "Your camera is turned off\nGo to your device settings to troubleshoot",
camera: control({ checked: false, label: "Camera" }),
devices: [{ deviceId: "blackhole-input", kind: "audioinput", label: "BlackHole 2ch" }],
join,
microphone: control({ checked: true, label: "Microphone" }),
microphoneDevice: control({ label: "BlackHole 2ch" }),
});
expect(result).toMatchObject({ clickedJoin: true, manualActionRequired: false });
expect(join.clicks).toBe(1);
});
it("does not treat the camera-only no-devices warning as a microphone block", async () => {
const join = control({ label: "Join now" });
const { result } = await runStatusScript({
allowMicrophone: true,
camera: control({ checked: false, label: "Camera" }),
continueWithoutDevices: control({ label: "Continue without audio or video" }),
devices: [{ deviceId: "blackhole-input", kind: "audioinput", label: "BlackHole 2ch" }],
join,
microphone: control({ checked: true, label: "Microphone" }),
microphoneDevice: control({ label: "BlackHole 2ch (Virtual)" }),
microphonePermissionState: "granted",
permissionPrompt: control({ label: "Camera permission warning" }),
});
expect(result).toMatchObject({ clickedJoin: true, manualActionRequired: false });
expect(join.clicks).toBe(1);
});
it.each(["meeting ended", "call ended — rejoin"])(
"does not infer departure from page-wide text: %s",
(bodyText) => {
@@ -499,6 +775,7 @@ describe("Microsoft Teams meeting platform adapter", () => {
const microphone = control({ label: "Turn microphone off", pressed: true });
const media = {
sinkId: "",
srcObject: { getAudioTracks: () => [{ readyState: "live" }] },
async setSinkId(value: string) {
media.sinkId = value;
},
@@ -529,18 +806,65 @@ describe("Microsoft Teams meeting platform adapter", () => {
expect(media.sinkId).toBe("blackhole-output");
});
it("mutes an in-call physical microphone when Teams resets the prejoin selection", async () => {
const microphone = control({
label: "Turn microphone off",
pressed: true,
onClick: (node) => node.setPressed(false),
it("reports the prepared session input during read-only status inspection", async () => {
const media: PageMedia = { sinkId: "blackhole-output", async setSinkId() {} };
const { result } = await runStatusScript({
allowMicrophone: true,
devices: [
{ deviceId: "blackhole-input", kind: "audioinput", label: "BlackHole 2ch" },
{ deviceId: "blackhole-output", kind: "audiooutput", label: "BlackHole 2ch" },
],
leave: control({ label: "Leave" }),
media: [media],
microphone: control({ label: "Turn microphone off", pressed: true }),
priorMeeting: {
audioInputDeviceId: "blackhole-input",
identity: "teams-work:19:meeting_test@thread.v2",
sessionId: "session-1",
},
readOnly: true,
});
expect(result).toMatchObject({
audioInputRouted: true,
audioOutputRouted: true,
manualActionRequired: false,
});
});
it("routes a directly playable media element before its MediaStream is attached", async () => {
const media: PageMedia = {
sinkId: "",
async setSinkId(value) {
media.sinkId = value;
},
};
const { result } = await runStatusScript({
allowMicrophone: true,
devices: [
{ deviceId: "blackhole-input", kind: "audioinput", label: "BlackHole 2ch" },
{ deviceId: "blackhole-output", kind: "audiooutput", label: "BlackHole 2ch" },
],
leave: control({ label: "Leave" }),
media: [media],
microphone: control({ label: "Turn microphone off", pressed: true }),
microphoneDevice: control({ label: "BlackHole 2ch" }),
priorMeeting: { identity: "teams-work:19:meeting_test@thread.v2" },
});
expect(result.audioOutputRouted).toBe(true);
expect(media.sinkId).toBe("blackhole-output");
});
it("reopens in-call audio options to reverify the BlackHole input", async () => {
const deviceSettings = control({ label: "Open audio options" });
const { result } = await runStatusScript({
allowMicrophone: true,
devices: [{ deviceId: "blackhole-input", kind: "audioinput", label: "BlackHole 2ch" }],
deviceSettings,
leave: control({ label: "Leave" }),
microphone,
microphoneDevice: control({ label: "MacBook Pro Microphone" }),
microphone: control({ label: "Turn microphone off", pressed: true }),
microphoneDeviceAfterSettings: control({ label: "BlackHole 2ch (Virtual)" }),
priorMeeting: {
audioInputDeviceId: "blackhole-input",
identity: "teams-work:19:meeting_test@thread.v2",
@@ -548,60 +872,134 @@ describe("Microsoft Teams meeting platform adapter", () => {
});
expect(result).toMatchObject({
audioInputRouted: false,
manualActionReason: "teams-audio-choice-required",
micMuted: true,
audioInputRouted: true,
inCall: true,
micMuted: false,
});
expect(microphone.clicks).toBe(1);
expect(deviceSettings.clicks).toBe(1);
});
it("builds the guest join script from centralized stable selectors and text fallbacks", () => {
const script = teamsMeetingStatusScript({
it("reads the selected in-call microphone from the live consumer listbox", async () => {
const deviceSettings = control({ label: "Open audio options" });
const selected = control({ label: "BlackHole 2ch (Virtual)" });
selected.getAttribute = (name) =>
name === "aria-selected" ? "true" : name === "aria-label" ? "BlackHole 2ch (Virtual)" : null;
const microphoneMenu = control({ label: "Microphone devices" });
microphoneMenu.querySelector = (selector) =>
selector?.includes('aria-selected="true"') ? selected : undefined;
const { result } = await runStatusScript({
allowMicrophone: true,
autoJoin: true,
guestName: "OpenClaw Guest",
meetingSessionId: "session-1",
meetingUrl: URL,
devices: [{ deviceId: "blackhole-input", kind: "audioinput", label: "BlackHole 2ch" }],
deviceSettings,
leave: control({ label: "Leave" }),
microphone: control({ label: "Turn microphone off", pressed: true }),
microphoneDeviceMenuAfterSettings: microphoneMenu,
priorMeeting: { identity: "teams-work:19:meeting_test@thread.v2" },
});
expect(script).toContain('data-tid=\\"prejoin-display-name-input\\"');
expect(script).toContain('data-tid=\\"call-hangup\\"');
expect(script).toContain("continue on this browser");
expect(script).toContain("someone will let you in shortly");
expect(script).toContain("setSinkId");
expect(script).toContain("BlackHole");
expect(result).toMatchObject({ audioInputRouted: true, inCall: true, micMuted: false });
expect(deviceSettings.clicks).toBe(1);
});
it("keeps caption capture disabled and returns empty snapshots", () => {
expect(TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.captions.enabled("transcribe")).toBe(false);
expect(
TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.captions.parseTranscript({
result: JSON.stringify({ urlMatched: true, sessionMatched: true, lines: ["ignored"] }),
}),
).toEqual({
droppedLines: 0,
lines: [],
urlMatched: true,
sessionMatched: true,
it("does not choose the audio-less fallback in talkback modes", async () => {
const continueWithoutDevices = control({
label: "Continue without audio or video",
});
await runStatusScript({
allowMicrophone: true,
continueWithoutDevices,
microphone: control({ label: "Turn microphone on", pressed: false }),
});
expect(continueWithoutDevices.clicks).toBe(0);
await runStatusScript({
allowMicrophone: false,
continueWithoutDevices,
microphone: control({ label: "Turn microphone on", pressed: false }),
});
expect(continueWithoutDevices.clicks).toBe(1);
await runStatusScript({
allowMicrophone: false,
autoJoin: false,
continueWithoutDevices,
microphone: control({ label: "Turn microphone on", pressed: false }),
});
expect(continueWithoutDevices.clicks).toBe(1);
});
it("does not guess a permission origin across work and consumer Teams", () => {
expect(
TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.permissions({ allowMicrophone: true }),
).toBeUndefined();
expect(
TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.permissionNotes({ allowMicrophone: true }),
).toContain("Teams media permissions are handled in the browser when prompted.");
});
it("bridges a live Teams MediaStream when its unloaded audio element rejects setSinkId", async () => {
const source: PageMedia = {
muted: false,
sinkId: "built-in-output",
srcObject: { getAudioTracks: () => [{ readyState: "live" }] },
async setSinkId() {
throw new DOMException("The element has no supported source.", "AbortError");
},
};
const routingOrder: string[] = [];
const bridge: PageMedia = {
isConnected: false,
sinkId: "",
async play() {
expect(source.muted).toBe(true);
expect(bridge.sinkId).toBe("blackhole-output");
routingOrder.push("play");
},
async setSinkId(value) {
expect(source.muted).toBe(true);
routingOrder.push("sink");
bridge.sinkId = value;
},
};
const { result, window } = await runStatusScript({
allowMicrophone: true,
bridgeMedia: bridge,
devices: [
{ deviceId: "blackhole-input", kind: "audioinput", label: "BlackHole 2ch" },
{ deviceId: "blackhole-output", kind: "audiooutput", label: "BlackHole 2ch" },
],
leave: control({ label: "Leave" }),
media: [source],
microphone: control({ label: "Turn microphone off", pressed: true }),
microphoneDevice: control({ label: "BlackHole 2ch" }),
priorMeeting: {
audioInputDeviceId: "blackhole-input",
identity: "teams-work:19:meeting_test@thread.v2",
},
});
it("parses leave steps and malformed status", () => {
expect(bridge.autoplay).toBe(false);
expect(routingOrder).toEqual(["sink", "play"]);
expect(result).toMatchObject({
audioInputRouted: true,
audioOutputRouted: true,
manualActionRequired: false,
});
expect(bridge.sinkId).toBe("blackhole-output");
expect(source.muted).toBe(true);
expect(window).toHaveProperty("__openclawTeamsAudioOutputs");
expect((window["__openclawTeamsAudioOutputs"] as Array<{ bridge: PageMedia }>)[0]?.bridge).toBe(
bridge,
);
const repeated = await runStatusScript({
allowMicrophone: true,
bridgeMedia: bridge,
devices: [
{ deviceId: "blackhole-input", kind: "audioinput", label: "BlackHole 2ch" },
{ deviceId: "blackhole-output", kind: "audiooutput", label: "BlackHole 2ch" },
],
leave: control({ label: "Leave" }),
media: [source, bridge],
microphone: control({ label: "Turn microphone off", pressed: true }),
microphoneDevice: control({ label: "BlackHole 2ch" }),
priorAudioOutputs: window["__openclawTeamsAudioOutputs"] as unknown[],
priorMeeting: window[MEETING_STATE_KEY] as Record<string, unknown>,
});
expect(repeated.result.audioOutputRouted).toBe(true);
expect(
TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.parseLeaveResult({
result: JSON.stringify({ departed: false, leaveAction: "confirm", urlMatched: true }),
}),
).toEqual({ departed: false, leaveAction: "confirm", urlMatched: true });
expect(() =>
TEAMS_MEETINGS_PLATFORM_ADAPTER.browser.parseStatus({ result: "not-json" }),
).toThrow("Microsoft Teams browser status JSON is malformed.");
(repeated.window["__openclawTeamsAudioOutputs"] as Array<{ bridge: PageMedia }>)[0]?.bridge,
).toBe(bridge);
});
});
extensions/teams-meetings/src/transports/teams-meetings-platform-adapter.ts
+158 -22
View File
@@ -1,3 +1,4 @@
import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
import type {
MeetingBrowserJoinSession,
MeetingManualActionCategory,
@@ -18,6 +19,29 @@ import {
} from "./teams-meetings-urls.js";
import type { TeamsMeetingsChromeHealth, TeamsMeetingsTranscriptSnapshot } from "./types.js";
function teamsMeetingOrigin(meetingUrl: string): string | undefined {
try {
const origin = new URL(meetingUrl).origin;
return origin === "https://teams.microsoft.com" || origin === "https://teams.live.com"
? origin
: undefined;
} catch {
return undefined;
}
}
function parsePermissionGrantNotes(result: unknown): string[] {
const record = result && typeof result === "object" ? (result as Record<string, unknown>) : {};
const unsupportedPermissions = Array.isArray(record.unsupportedPermissions)
? record.unsupportedPermissions.filter((value): value is string => typeof value === "string")
: [];
const notes = ["Granted Teams microphone permission through browser control."];
if (unsupportedPermissions.includes("speakerSelection")) {
notes.push("Chrome did not accept the optional Teams speaker-selection permission.");
}
return notes;
}
export function isTeamsMeetingsTalkBackMode(mode: TeamsMeetingsMode): boolean {
return mode === "agent" || mode === "bidi";
}
@@ -52,6 +76,40 @@ function parseBrowserStatus(result: unknown): TeamsMeetingsChromeHealth | undefi
micMuted: typeof parsed.micMuted === "boolean" ? parsed.micMuted : undefined,
cameraOff: typeof parsed.cameraOff === "boolean" ? parsed.cameraOff : undefined,
lobbyWaiting: typeof parsed.lobbyWaiting === "boolean" ? parsed.lobbyWaiting : undefined,
captionCaptureRequested:
typeof parsed.captionCaptureRequested === "boolean"
? parsed.captionCaptureRequested
: undefined,
captioning: typeof parsed.captioning === "boolean" ? parsed.captioning : undefined,
captionsEnabledAttempted:
typeof parsed.captionsEnabledAttempted === "boolean"
? parsed.captionsEnabledAttempted
: undefined,
transcriptLines:
typeof parsed.transcriptLines === "number" ? parsed.transcriptLines : undefined,
lastCaptionAt: typeof parsed.lastCaptionAt === "string" ? parsed.lastCaptionAt : undefined,
lastCaptionSpeaker:
typeof parsed.lastCaptionSpeaker === "string" ? parsed.lastCaptionSpeaker : undefined,
lastCaptionText:
typeof parsed.lastCaptionText === "string" ? parsed.lastCaptionText : undefined,
recentTranscript: Array.isArray(parsed.recentTranscript)
? parsed.recentTranscript.flatMap((value) => {
if (!value || typeof value !== "object") {
return [];
}
const line = value as { at?: unknown; speaker?: unknown; text?: unknown };
if (typeof line.text !== "string" || !line.text.trim()) {
return [];
}
return [
{
...(typeof line.at === "string" ? { at: line.at } : {}),
...(typeof line.speaker === "string" ? { speaker: line.speaker } : {}),
text: line.text,
},
];
})
: undefined,
audioInputRouted:
typeof parsed.audioInputRouted === "boolean" ? parsed.audioInputRouted : undefined,
audioInputDeviceLabel:
@@ -64,6 +122,10 @@ function parseBrowserStatus(result: unknown): TeamsMeetingsChromeHealth | undefi
typeof parsed.audioOutputDeviceLabel === "string" ? parsed.audioOutputDeviceLabel : undefined,
audioOutputRouteError:
typeof parsed.audioOutputRouteError === "string" ? parsed.audioOutputRouteError : undefined,
audioOutputRouteRetryable:
typeof parsed.audioOutputRouteRetryable === "boolean"
? parsed.audioOutputRouteRetryable
: undefined,
manualActionRequired:
typeof parsed.manualActionRequired === "boolean" ? parsed.manualActionRequired : undefined,
manualActionReason:
@@ -109,6 +171,8 @@ function classifyManualAction(health: TeamsMeetingsChromeHealth) {
function parseLeaveResult(result: unknown): {
departed: boolean;
leaveAction?: "leave" | "confirm";
sessionConflict?: boolean;
sessionMatched?: boolean;
urlMatched?: boolean;
} {
const record = result && typeof result === "object" ? (result as Record<string, unknown>) : {};
@@ -124,6 +188,12 @@ function parseLeaveResult(result: unknown): {
return {
departed: parsed.departed === true,
...(leaveAction ? { leaveAction } : {}),
...(typeof parsed.sessionConflict === "boolean"
? { sessionConflict: parsed.sessionConflict }
: {}),
...(typeof parsed.sessionMatched === "boolean"
? { sessionMatched: parsed.sessionMatched }
: {}),
...(typeof parsed.urlMatched === "boolean" ? { urlMatched: parsed.urlMatched } : {}),
};
} catch {
@@ -131,24 +201,60 @@ function parseLeaveResult(result: unknown): {
}
}
function parseTranscript(result: unknown) {
function parseTranscript(
result: unknown,
): TeamsMeetingsTranscriptSnapshot & { sessionMatched?: boolean; urlMatched?: boolean } {
const record = result && typeof result === "object" ? (result as Record<string, unknown>) : {};
if (typeof record.result !== "string" || !record.result.trim()) {
return { droppedLines: 0, lines: [] };
}
let parsed: unknown;
try {
const parsed = JSON.parse(record.result) as Record<string, unknown>;
return {
droppedLines: 0,
lines: [],
...(typeof parsed.urlMatched === "boolean" ? { urlMatched: parsed.urlMatched } : {}),
...(typeof parsed.sessionMatched === "boolean"
? { sessionMatched: parsed.sessionMatched }
: {}),
};
parsed = JSON.parse(record.result);
} catch {
throw new Error("Microsoft Teams transcript JSON is malformed.");
}
if (!parsed || typeof parsed !== "object") {
throw new Error("Microsoft Teams transcript payload is invalid.");
}
const payload = parsed as {
droppedLines?: unknown;
epoch?: unknown;
lines?: unknown;
sessionMatched?: unknown;
urlMatched?: unknown;
};
const droppedLines =
typeof payload.droppedLines === "number" && Number.isSafeInteger(payload.droppedLines)
? Math.max(0, payload.droppedLines)
: 0;
const lines = Array.isArray(payload.lines)
? payload.lines.flatMap((value) => {
if (!value || typeof value !== "object") {
return [];
}
const line = value as { at?: unknown; speaker?: unknown; text?: unknown };
if (typeof line.text !== "string" || !line.text.trim()) {
return [];
}
return [
{
...(typeof line.at === "string" ? { at: line.at } : {}),
...(typeof line.speaker === "string" ? { speaker: line.speaker } : {}),
text: line.text,
},
];
})
: [];
return {
droppedLines,
...(typeof payload.epoch === "string" ? { epoch: payload.epoch } : {}),
lines,
...(typeof payload.urlMatched === "boolean" ? { urlMatched: payload.urlMatched } : {}),
...(typeof payload.sessionMatched === "boolean"
? { sessionMatched: payload.sessionMatched }
: {}),
};
}
export const TEAMS_MEETINGS_PLATFORM_ADAPTER: MeetingPlatformAdapter<
@@ -178,35 +284,65 @@ export const TEAMS_MEETINGS_PLATFORM_ADAPTER: MeetingPlatformAdapter<
buildStatusJoinScript: (params) =>
teamsMeetingStatusScript({
allowMicrophone: isTeamsMeetingsTalkBackMode(params.mode),
allowSessionAdoption: params.allowSessionAdoption,
autoJoin: params.autoJoin,
captureCaptions: params.captureCaptions,
guestName: params.guestName,
meetingSessionId: params.meetingSessionId || undefined,
meetingUrl: params.url,
readOnly: params.readOnly,
waitForInCallMs: params.waitForInCallMs,
}),
parseStatus: parseBrowserStatus,
classifyManualAction,
shouldRetryJoinStatus: (health) =>
health.inCall === true &&
((health.manualActionReason === "teams-audio-choice-required" &&
health.audioInputRouted === true &&
health.audioOutputRouteRetryable === true) ||
(health.manualActionRequired !== true &&
health.captionCaptureRequested === true &&
health.captioning !== true)),
browserControlUnavailable: () => ({
category: "browser-control-unavailable",
reason: "browser-control-unavailable",
message:
"Open the OpenClaw browser profile, finish the Teams sign-in, admission, or permission prompt, then retry.",
}),
buildLeaveScript: teamsMeetingLeaveScript,
buildLeaveScript: (meetingUrl) =>
teamsMeetingLeaveScript({
leaveInitiated: false,
meetingSessionId: "",
meetingUrl,
}),
buildSessionLeaveScript: teamsMeetingLeaveScript,
parseLeaveResult,
captions: {
// Teams caption DOM is not enabled until a live tenant flow validates stable selectors.
enabled: () => false,
buildTranscriptScript: ({ meetingSessionId, meetingUrl }) =>
teamsMeetingTranscriptScript(meetingUrl, meetingSessionId),
enabled: (mode) => mode === "transcribe",
buildTranscriptScript: ({ finalize, meetingSessionId, meetingUrl }) =>
teamsMeetingTranscriptScript(meetingUrl, meetingSessionId, finalize),
parseTranscript,
},
// The core permission hook has no meeting URL, so it cannot select between
// teams.microsoft.com and teams.live.com without guessing an origin.
permissions: () => undefined,
permissionNotes: ({ allowMicrophone }) =>
allowMicrophone
? ["Teams media permissions are handled in the browser when prompted."]
: ["Observe-only mode does not request Teams microphone access."],
permissions: ({ allowMicrophone, meetingUrl }) => {
const origin = teamsMeetingOrigin(meetingUrl);
return allowMicrophone && origin
? {
origin,
permissions: ["audioCapture"],
optionalPermissions: ["speakerSelection"],
}
: undefined;
},
permissionNotes: ({ allowMicrophone, error, result }) => {
if (!allowMicrophone) {
return ["Observe-only mode does not request Teams microphone access."];
}
if (error) {
return [
`Could not grant Teams media permissions automatically: ${formatErrorMessage(error)}`,
];
}
return parsePermissionGrantNotes(result);
},
},
};
extensions/teams-meetings/src/transports/teams-meetings-selectors.ts
+23 -2
View File
@@ -1,5 +1,5 @@
// Best-effort Teams web selectors. data-tid values are preferred because Teams
// uses them extensively; every flow still needs validation against a live tenant.
// Teams web selectors validated against live guest surfaces.
// Prefer product-owned ids/data-tids; text remains a fallback for localized drift.
export const TEAMS_MEETING_SELECTORS = {
continueInBrowser: [
'[data-tid="joinOnWeb"]',
@@ -20,27 +20,36 @@ export const TEAMS_MEETING_SELECTORS = {
microphone: [
'button[data-tid="toggle-mute"]',
'[data-tid="toggle-mute"] button',
'input[data-tid="toggle-mute"]',
'[data-tid="toggle-mute"][role="switch"]',
'button[data-tid="microphone-button"]',
],
camera: [
'button[data-tid="toggle-video"]',
'[data-tid="toggle-video"] button',
'input[data-tid="toggle-video"]',
'[data-tid="toggle-video"][role="switch"]',
'button[data-tid="camera-button"]',
],
deviceSettings: [
"button#audio-button-configure",
'button[aria-label="Open audio options"]',
'button[data-tid="prejoin-device-settings-button"]',
'button[data-tid="device-settings-button"]',
'button[data-tid="audio-device-settings-button"]',
],
microphoneDevice: [
'button[data-tid="selected-microphone-display"]',
'[data-tid="microphone-select"]',
'[data-tid="audio-device-input"]',
'[data-tid="device-settings-microphone"] [role="combobox"]',
'select[data-tid="microphone-select"]',
],
microphoneDeviceMenu: ['[data-tid="microphone-settings"][role="listbox"]'],
selectedMicrophoneDevice: ["option:checked", '[role="option"][aria-selected="true"]'],
audioDeviceOptions: ["option", '[role="option"]'],
leave: [
"button#hangup-button",
'button[data-tid="call-hangup"]',
'[data-tid="call-hangup"] button',
'button[data-tid="hangup-button"]',
@@ -52,6 +61,7 @@ export const TEAMS_MEETING_SELECTORS = {
'button[data-tid="leave-call-confirm"]',
],
postCall: [
'button[data-tid="anon-meeting-end-screen-rejoin-button"]',
'[data-tid="call-ended-screen"]',
'[data-tid="post-call-screen"]',
'button[data-tid="prejoin-rejoin-button"]',
@@ -67,4 +77,15 @@ export const TEAMS_MEETING_SELECTORS = {
'[data-tid="media-permission-prompt"]',
'[data-tid="browser-permission-error"]',
],
moreActions: ['button[aria-label="More"]'],
captions: ["button#closed-captions-button", '[role="menuitem"][title*="captions" i]'],
captionsOff: [
'button[data-tid="closed-captions-turn-off-button"]',
"button#captions-panel-dismiss-button",
],
captionRenderer: ['[data-tid="closed-caption-renderer-wrapper"]'],
captionContent: ['[data-tid="closed-caption-v2-virtual-list-content"]'],
captionRows: ['div[role="log"]'],
captionAuthor: ['[data-tid="author"]'],
captionText: ['[data-tid="closed-caption-text"]'],
} as const;
extensions/teams-meetings/src/transports/teams-meetings-status-call-source.ts
+657
View File
@@ -0,0 +1,657 @@
const TEAMS_MEETING_CAPTION_SETTLE_MS = 1_000;
const TEAMS_MEETING_TRANSCRIPT_MAX_LINES = 500;
export function teamsMeetingStatusCallSource(): string {
return ` let audioOutputRouted;
let audioOutputDeviceLabel;
let audioOutputRouteError;
let audioOutputRouteRetryable = false;
if (inCall && allowMicrophone && navigator.mediaDevices?.enumerateDevices) {
const media = [...document.querySelectorAll("audio, video")].filter(
(element) =>
typeof element.setSinkId === "function" &&
!String(element.id || "").startsWith("openclaw-teams-audio-output-"),
);
if (media.length > 0) {
try {
const devices = await navigator.mediaDevices.enumerateDevices();
const output = devices.find((device) => device.kind === "audiooutput" && isBlackHole(device.label));
if (output?.deviceId) {
const routeErrors = [];
const liveStream = (element) =>
element.srcObject?.getAudioTracks?.().some((track) => track.readyState === "live")
? element.srcObject
: undefined;
const allBridgeEntries = Array.isArray(window.__openclawTeamsAudioOutputs)
? window.__openclawTeamsAudioOutputs
: [];
const retainedBridgeEntries = allBridgeEntries.filter((entry) => !bridgeOwnedBySession(entry));
const previousBridgeEntries = allBridgeEntries.filter(bridgeOwnedBySession);
const originalMuteBySource = new Map(previousBridgeEntries.flatMap((entry) =>
bridgeSources(entry).flatMap((source) =>
source?.element ? [[source.element, Boolean(source.muted)]] : []
)
));
const bridgedElements = new Set(previousBridgeEntries.flatMap((entry) =>
bridgeSources(entry).map((source) => source?.element).filter(Boolean)
));
const routeCandidates = media
.map((element) => ({ element, stream: liveStream(element) }))
// Teams mutes local/self-view and intentionally suppressed playback. Preserve
// that product decision; only our own already-bridged source stays eligible.
.filter((entry) => !entry.element.muted || bridgedElements.has(entry.element));
// The self-view often exists before Teams attaches remote playback. With the
// required output present, an all-filtered list is still a transient DOM state.
if (routeCandidates.length === 0) audioOutputRouteRetryable = true;
if (canMutateSession) {
for (const { element } of routeCandidates) {
if (!originalMuteBySource.has(element)) {
originalMuteBySource.set(element, Boolean(element.muted));
}
// Sink changes are asynchronous. Silence the physical output until either
// the source or its fallback bridge is confirmed on BlackHole.
element.muted = true;
}
}
const currentSources = new Set(routeCandidates.map((entry) => entry.element));
const bridgeEntries = previousBridgeEntries.filter((entry) =>
entry?.source &&
entry?.stream === liveStream(entry.source) &&
entry?.bridge?.isConnected &&
currentSources.has(entry.source)
);
const suspendedBySource = new Map();
for (const entry of previousBridgeEntries) {
if (bridgeEntries.includes(entry)) continue;
for (const source of bridgeSources(entry)) {
if (
!source?.element ||
source.muted ||
!bridgeSourceMatches(source.element, source)
) continue;
const sourceStillPresent = currentSources.has(source.element);
const detachedLiveSource = !sourceStillPresent && Boolean(liveStream(source.element));
if (!sourceStillPresent && !detachedLiveSource) continue;
suspendedBySource.set(source.element, {
detached: detachedLiveSource,
sessionId: entry.sessionId || sessionId,
source: source.element,
sourceMuted: false,
sourceUrl: mediaSourceUrl(source.element) || source.url,
stream: source.element.srcObject,
suspended: true,
});
}
}
if (canMutateSession) {
// One bridge owns one Teams playback element. Stream or element replacement
// retires that bridge so it cannot keep playing or satisfy route verification.
previousBridgeEntries.filter((entry) => !bridgeEntries.includes(entry)).forEach((entry) => {
for (const source of bridgeSources(entry)) {
if (
!source?.element ||
suspendedBySource.has(source.element) ||
currentSources.has(source.element)
) continue;
restoreAudioBridgeSource(source);
}
// Reused current elements stay silent until this pass confirms their
// replacement source; unrelated exact sources were restored above.
retireAudioBridge(entry, false);
});
}
const routed = [];
for (const { element, stream } of routeCandidates) {
let entry = bridgeEntries.find((candidate) => candidate.source === element);
let elementRouted = element.sinkId === output.deviceId;
let directRouteError;
if (canMutateSession && !elementRouted) {
try {
await element.setSinkId(output.deviceId);
elementRouted = element.sinkId === output.deviceId;
} catch (error) {
directRouteError = {
message: error?.message || String(error),
retryable: error?.name === "AbortError",
};
}
}
if (elementRouted && entry && canMutateSession) {
const bridgedIndex = bridgeEntries.indexOf(entry);
if (bridgedIndex >= 0) {
const [bridged] = bridgeEntries.splice(bridgedIndex, 1);
retireAudioBridge(bridged);
entry = undefined;
}
}
// Direct sink routing is valid for src/MediaSource and pre-attachment elements.
// A live MediaStream is required only when the hidden bridge fallback is needed.
if (elementRouted) {
if (canMutateSession && originalMuteBySource.has(element)) {
element.muted = originalMuteBySource.get(element);
}
suspendedBySource.delete(element);
routed.push(true);
continue;
}
if (!stream) {
const hasLoadedPlaybackSource = Number(element.readyState) > 0;
routed.push(false);
if (hasLoadedPlaybackSource && directRouteError) routeErrors.push(directRouteError);
if (!hasLoadedPlaybackSource) audioOutputRouteRetryable = true;
if (canMutateSession && originalMuteBySource.get(element) === false) {
// Teams may attach the remote MediaStream after creating its media element.
// Keep it silent until a later serialized status poll routes that source.
suspendedBySource.set(element, {
sessionId,
pending: true,
source: element,
sourceMuted: false,
sourceUrl: mediaSourceUrl(element),
stream: element.srcObject,
suspended: true,
});
}
continue;
}
if (!elementRouted && stream) {
if (!entry && canMutateSession) {
const bridge = document.createElement("audio");
bridge.id = "openclaw-teams-audio-output-" + bridgeEntries.length;
bridge.autoplay = false;
bridge.hidden = true;
bridge.srcObject = stream;
document.body.appendChild(bridge);
entry = {
bridge,
playing: false,
sessionId,
source: element,
sourceMuted: originalMuteBySource.has(element)
? originalMuteBySource.get(element)
: Boolean(element.muted),
sourceUrl: mediaSourceUrl(element),
stream,
};
bridgeEntries.push(entry);
suspendedBySource.delete(element);
}
if (entry?.bridge) {
try {
if (canMutateSession) {
if (entry.bridge.sinkId !== output.deviceId) {
await entry.bridge.setSinkId(output.deviceId);
}
await entry.bridge.play();
entry.playing = true;
}
elementRouted =
entry.bridge.sinkId === output.deviceId && entry.playing === true;
if (elementRouted) {
suspendedBySource.delete(element);
if (canMutateSession && !entry.sourceMuted) element.muted = true;
}
} catch (error) {
entry.playing = false;
if (canMutateSession) retireAudioBridge(entry, false);
routeErrors.push({
message: error?.message || String(error),
retryable: error?.name === "AbortError",
});
}
}
}
routed.push(elementRouted);
}
if (canMutateSession) {
const nextBridgeEntries = [
...retainedBridgeEntries,
...bridgeEntries,
...suspendedBySource.values(),
];
if (nextBridgeEntries.length > 0) {
window.__openclawTeamsAudioOutputs = nextBridgeEntries;
} else {
delete window.__openclawTeamsAudioOutputs;
}
}
audioOutputRouted = routed.length > 0 && routed.every(Boolean);
if (canMutateSession && !audioOutputRouted) suspendOwnedAudioBridges();
if (audioOutputRouted && bridgeEntries.length > 0) {
notes.push("Routed Teams remote audio to BlackHole 2ch through MediaStream bridges.");
}
audioOutputDeviceLabel = output.label || "BlackHole 2ch";
// An unloaded Teams media element can reject setSinkId before its stream
// arrives. Keep that state retryable; loaded-source failures are terminal.
if (!audioOutputRouted && routed.length > 0 && routeErrors.length > 0) {
audioOutputRouteError = routeErrors[routeErrors.length - 1]?.message;
audioOutputRouteRetryable = routeErrors.every((error) => error.retryable === true);
}
} else {
audioOutputRouted = false;
if (canMutateSession) suspendOwnedAudioBridges();
notes.push("BlackHole 2ch speaker output was not visible to Teams.");
}
} catch (error) {
audioOutputRouted = false;
audioOutputRouteError = error?.message || String(error);
if (canMutateSession) suspendOwnedAudioBridges();
}
if (!audioOutputRouted && audioOutputRouteError) {
notes.push("Could not route Teams speaker output to BlackHole 2ch: " + audioOutputRouteError);
}
} else {
audioOutputRouted = false;
try {
const devices = await navigator.mediaDevices.enumerateDevices();
const output = devices.find(
(device) => device.kind === "audiooutput" && isBlackHole(device.label)
);
if (output?.deviceId) {
// Teams can briefly remove every media element during an in-call rerender.
// Retry only after proving the required output still exists.
audioOutputRouteRetryable = true;
audioOutputDeviceLabel = output.label || "BlackHole 2ch";
} else {
notes.push("BlackHole 2ch speaker output was not visible to Teams.");
}
} catch (error) {
audioOutputRouteError = error?.message || String(error);
notes.push("Could not inspect Teams speaker outputs: " + audioOutputRouteError);
}
// Suspend ownership until the source returns; call teardown retires it.
if (canMutateSession) suspendOwnedAudioBridges();
}
} else if (inCall && allowMicrophone) {
audioOutputRouted = false;
if (canMutateSession) retireOwnedAudioBridges();
}
let captioning = false;
let captionsEnabledAttempted = false;
let transcriptLines = 0;
let lastCaptionAt;
let lastCaptionSpeaker;
let lastCaptionText;
let recentTranscript = [];
const captionState = (() => {
let active = window.__openclawTeamsCaptions;
const activeOwnedByRequest = Boolean(
!active || (sessionId && (!active.sessionId || active.sessionId === sessionId))
);
if (!identityVerified) {
if (identityAwaitingRerender && activeOwnedByRequest) return active;
if (canMutateSession && activeOwnedByRequest) finalizeOwnedCaptions();
return undefined;
}
if (!activeOwnedByRequest) {
const replacedPriorOwner = Boolean(
canMutateSession &&
active?.sessionId &&
active.sessionId !== sessionId
);
if (replacedPriorOwner) {
if (priorMeeting?.sessionId === active.sessionId) {
active.identity ||= priorMeeting.identity;
}
finalizeCaptionState(active);
}
else if (!canMutateSession || !captureCaptions || active?.finalized !== true) return undefined;
archiveFinalizedCaptions(active);
if (active.settleTimer !== undefined) clearTimeout(active.settleTimer);
active.observer?.disconnect?.();
delete window.__openclawTeamsCaptions;
active = undefined;
}
if (!captureCaptions) {
if (!canMutateSession) return undefined;
if (active?.settleTimer !== undefined) clearTimeout(active.settleTimer);
active?.observer?.disconnect?.();
if (active) delete window.__openclawTeamsCaptions;
return undefined;
}
if (!inCall && !active) return undefined;
if (!active && !canMutateSession) return undefined;
if (!active) {
if (active?.settleTimer !== undefined) clearTimeout(active.settleTimer);
active?.observer?.disconnect?.();
window.__openclawTeamsCaptions = {
sessionId,
identity: expectedIdentity,
epoch: crypto.randomUUID(),
enabledAttempted: false,
observerInstalled: false,
observer: undefined,
droppedLines: 0,
lines: [],
settled: [],
settleTimer: undefined,
visible: [],
};
}
return window.__openclawTeamsCaptions;
})();
const normalizeCaption = (speaker, captionText) => {
if (!captionState) return undefined;
const clean = String(captionText || "").replace(/\\s+/g, " ").trim();
const cleanSpeaker = String(speaker || "").replace(/\\s+/g, " ").trim();
if (!clean) return undefined;
return { speaker: cleanSpeaker || undefined, text: clean };
};
const captionRowIdentity = (row) =>
// aria-posinset identifies the logical caption item across virtual-list
// rerenders. DOM ids and data indexes can belong to the recycled element.
["aria-posinset"]
.map((name) => {
const value = row?.getAttribute?.(name);
return typeof value === "string" && value.trim()
? name + ":" + value.trim()
: undefined;
})
.find(Boolean);
const sameCaptionUtterance = (prior, current) => {
if (prior.rowIdentity || current.rowIdentity) {
return Boolean(
prior.rowIdentity &&
current.rowIdentity &&
prior.rowIdentity === current.rowIdentity
);
}
if (prior.speaker && current.speaker && prior.speaker !== current.speaker) return false;
return prior.node === current.node;
};
const commitCaptionLines = (state, entries) => {
state.lines.push(...entries.map((entry) => {
entry.utteranceId ||= crypto.randomUUID();
return {
at: entry.at,
speaker: entry.speaker,
text: entry.text,
utteranceId: entry.utteranceId,
};
}));
const excess = state.lines.length - ${TEAMS_MEETING_TRANSCRIPT_MAX_LINES};
if (excess > 0) {
state.lines.splice(0, excess);
state.droppedLines = (state.droppedLines || 0) + excess;
}
};
const sameCaptionRow = (left, right) =>
right.rowIdentity
? left.rowIdentity === right.rowIdentity
: left.node === right.node;
const retainSettledCaptionLines = (state, entries) => {
const settled = [...state.settled];
for (const entry of entries) {
const priorIndex = settled.findIndex((candidate) => sameCaptionRow(candidate, entry));
if (priorIndex >= 0) settled.splice(priorIndex, 1, { ...entry });
else settled.push({ ...entry });
}
const retainedLineIds = new Set(state.lines.map((entry) => entry.utteranceId));
state.settled = settled.filter((entry) => retainedLineIds.has(entry.utteranceId));
};
const scheduleCaptionSettle = () => {
if (!captionState || captionState.visible.length === 0) return;
if (captionState.settleTimer !== undefined) clearTimeout(captionState.settleTimer);
const pendingState = captionState;
pendingState.settleTimer = setTimeout(() => {
if (window.__openclawTeamsCaptions !== pendingState) return;
commitCaptionLines(pendingState, pendingState.visible);
retainSettledCaptionLines(pendingState, pendingState.visible);
pendingState.visible = [];
pendingState.settleTimer = undefined;
}, ${TEAMS_MEETING_CAPTION_SETTLE_MS});
};
const captionCaptureMatchesCurrentMeeting = () => {
if (
!captionState ||
captionState.finalized === true ||
window.__openclawTeamsCaptions !== captionState
) return false;
const observedIdentity = meetingIdentity(location.href);
const observedMeeting = window.__openclawTeamsMeeting;
const identityConflicts = Boolean(
observedIdentity && expectedIdentity && observedIdentity !== expectedIdentity
);
const sessionConflicts = Boolean(
observedMeeting?.sessionId && sessionId && observedMeeting.sessionId !== sessionId
);
if (identityConflicts || sessionConflicts) {
// The observer outlives Teams SPA navigation. Freeze the old buffer before
// any caption nodes from the replacement meeting can be attributed to it.
finalizeOwnedCaptions();
return false;
}
if (observedIdentity === expectedIdentity) return true;
const observedMarkerAgeMs = Date.now() - (observedMeeting?.verifiedAt || 0);
const observedAwaitingRerender = Boolean(
!observedIdentity &&
observedMeeting?.identity === expectedIdentity &&
(!observedMeeting.sessionId || !sessionId || observedMeeting.sessionId === sessionId) &&
observedMeeting.inCallControl?.isConnected === false &&
observedMeeting.inCallUrl === location.href &&
observedMarkerAgeMs >= 0 &&
observedMarkerAgeMs < 5_000
);
if (observedAwaitingRerender) return true;
return Boolean(
observedMeeting?.identity === expectedIdentity &&
(!observedMeeting.sessionId || !sessionId || observedMeeting.sessionId === sessionId) &&
observedMeeting.inCallControl?.isConnected !== false &&
observedMeeting.inCallUrl === location.href
);
};
const scrapeCaptions = (mutations = []) => {
if (!captionCaptureMatchesCurrentMeeting()) return;
const content = firstRaw(selectors.captionContent);
const rows = content
? selectors.captionRows.flatMap((selector) => [...content.querySelectorAll(selector)])
: [];
captionState.settled = Array.isArray(captionState.settled) ? captionState.settled : [];
const removedNodes = mutations.flatMap((mutation) => [...(mutation.removedNodes || [])]);
const rowWasRemoved = (entry) => removedNodes.some((node) =>
node === entry.node || node?.contains?.(entry.node)
);
const removedVisible = captionState.visible.filter(rowWasRemoved);
if (removedVisible.length > 0) {
if (captionState.settleTimer !== undefined) clearTimeout(captionState.settleTimer);
captionState.settleTimer = undefined;
captionState.visible = captionState.visible.filter((entry) => !rowWasRemoved(entry));
commitCaptionLines(captionState, removedVisible);
retainSettledCaptionLines(captionState, removedVisible);
}
const retainedLineIds = new Set(captionState.lines.map((entry) => entry.utteranceId));
captionState.settled = captionState.settled.filter((entry) =>
entry.rowIdentity
? retainedLineIds.has(entry.utteranceId)
: !rowWasRemoved(entry) && rows.some((row) => sameCaptionRow(entry, {
node: row,
rowIdentity: captionRowIdentity(row),
}))
);
const parsedRows = rows.flatMap((row) => {
const speaker = text(firstWithin(row, selectors.captionAuthor));
const captionText = text(firstWithin(row, selectors.captionText));
const parsed = normalizeCaption(speaker, captionText);
if (!parsed) return [];
const current = { ...parsed, node: row, rowIdentity: captionRowIdentity(row) };
const settledIndex = captionState.settled.findIndex((entry) =>
sameCaptionRow(entry, current)
);
const settled = settledIndex >= 0 ? captionState.settled[settledIndex] : undefined;
if (
settled &&
settled.text === current.text &&
(settled.speaker || "") === (current.speaker || "")
) return [];
if (settled?.rowIdentity && settled.rowIdentity === current.rowIdentity) {
const committed = captionState.lines.find((entry) =>
entry.utteranceId === settled.utteranceId
);
if (committed) {
committed.speaker = current.speaker || committed.speaker;
committed.text = current.text;
}
captionState.settled.splice(settledIndex, 1, {
...settled,
...current,
speaker: current.speaker || settled.speaker,
});
return [];
}
if (settledIndex >= 0) captionState.settled.splice(settledIndex, 1);
return [current];
});
if (parsedRows.length === 0) {
if (captionState.visible.length > 0 && captionState.settleTimer === undefined) {
scheduleCaptionSettle();
}
return;
}
const unmatchedPrevious = [...captionState.visible];
const nextVisible = [];
const now = Date.now();
let captionChanged = false;
for (const row of parsedRows) {
const priorIndex = unmatchedPrevious.findIndex((candidate) =>
row.rowIdentity
? candidate.rowIdentity === row.rowIdentity
: candidate.node === row.node
);
const candidate = priorIndex >= 0 ? unmatchedPrevious[priorIndex] : undefined;
const prior = candidate && sameCaptionUtterance(candidate, row)
? unmatchedPrevious.splice(priorIndex, 1)[0]
: undefined;
if (prior) {
captionChanged ||=
prior.text !== row.text ||
prior.speaker !== row.speaker ||
prior.node !== row.node;
prior.speaker = row.speaker || prior.speaker;
prior.text = row.text;
prior.node = row.node;
prior.rowIdentity = row.rowIdentity || prior.rowIdentity;
prior.seenAt = now;
nextVisible.push(prior);
} else {
captionChanged = true;
nextVisible.push({
at: new Date().toISOString(),
node: row.node,
rowIdentity: row.rowIdentity,
seenAt: now,
speaker: row.speaker,
text: row.text,
});
}
}
captionChanged ||= unmatchedPrevious.length > 0;
commitCaptionLines(captionState, unmatchedPrevious);
retainSettledCaptionLines(captionState, unmatchedPrevious);
captionState.visible = nextVisible;
// Identity-less rows stay mutable while rendered; removal is their only
// reliable utterance boundary. Stable logical rows may settle on quiet.
if (
(captionChanged || captionState.settleTimer === undefined) &&
captionState.visible.every((entry) => entry.rowIdentity)
) {
scheduleCaptionSettle();
}
};
if (captionState) {
const captionsFinalized = captionState.finalized === true;
let captionsEnabledNow = captionsFinalized
? Boolean(captionState.enabledAttempted)
: Boolean(firstRaw(selectors.captionRenderer) || firstRaw(selectors.captionsOff));
if (!captionsFinalized && canMutateSession && inCall && !captionsEnabledNow) {
let captionButton = first(selectors.captions);
if (!captionButton) {
first(selectors.moreActions)?.click?.();
await waitForUi();
captionButton = first(selectors.captions);
}
if (captionButton) {
const captionLabel = label(captionButton);
const alreadyEnabled = captionButton.getAttribute?.("aria-checked") === "true" ||
/hide live captions|turn off captions/i.test(captionLabel) ||
Boolean(firstRaw(selectors.captionsOff));
if (!alreadyEnabled) {
captionButton.click();
await waitForUi();
}
const currentLabel = label(captionButton);
captionsEnabledNow = captionButton.getAttribute?.("aria-checked") === "true" ||
/hide live captions|turn off captions/i.test(currentLabel) ||
Boolean(firstRaw(selectors.captionRenderer)) ||
Boolean(firstRaw(selectors.captionsOff));
if (captionsEnabledNow && !alreadyEnabled) {
notes.push("Enabled Teams live captions for transcript capture.");
}
}
}
if (!captionsFinalized && canMutateSession) captionState.enabledAttempted = captionsEnabledNow;
captionsEnabledAttempted = Boolean(captionState.enabledAttempted);
if (!captionsFinalized && canMutateSession && inCall && !captionState.observerInstalled) {
captionState.observerInstalled = true;
captionState.observer = new MutationObserver(scrapeCaptions);
captionState.observer.observe(document.body, {
childList: true,
subtree: true,
characterData: true,
});
notes.push("Installed Teams live-caption observer.");
}
if (!captionsFinalized && canMutateSession && inCall) scrapeCaptions();
const allLines = [...captionState.lines, ...captionState.visible];
const lines = allLines.slice(-${TEAMS_MEETING_TRANSCRIPT_MAX_LINES});
const last = lines[lines.length - 1];
captioning = captionsEnabledNow;
transcriptLines = (captionState.droppedLines || 0) + allLines.length;
lastCaptionAt = last?.at;
lastCaptionSpeaker = last?.speaker;
lastCaptionText = last?.text;
recentTranscript = lines.slice(-5).map((entry) => ({
at: entry.at,
speaker: entry.speaker,
text: entry.text,
}));
}
if (inCall && allowMicrophone && !manualActionReason) {
if (audioInputRouted !== true || audioOutputRouted !== true) {
manualActionReason = "teams-audio-choice-required";
manualActionMessage = "Verify BlackHole 2ch is selected as both the Teams microphone and speaker before starting talk-back.";
} else if (micMuted !== false) {
manualActionReason = "teams-microphone-required";
manualActionMessage = "Unmute the Teams microphone and verify the microphone control shows it is on before starting talk-back.";
}
}
return JSON.stringify({
clickedContinueInBrowser: Boolean(continueInBrowser),
clickedJoin,
inCall,
micMuted,
cameraOff,
lobbyWaiting,
captionCaptureRequested: captureCaptions,
captioning,
captionsEnabledAttempted,
transcriptLines,
lastCaptionAt,
lastCaptionSpeaker,
lastCaptionText,
recentTranscript,
audioInputRouted,
audioInputDeviceLabel,
audioInputRouteError,
audioOutputRouted,
audioOutputDeviceLabel,
audioOutputRouteError,
audioOutputRouteRetryable,
manualActionRequired: Boolean(manualActionReason),
manualActionReason,
manualActionMessage,
title: document.title,
url: location.href,
notes,
});
}`;
}
extensions/teams-meetings/src/transports/teams-meetings-status-prejoin-source.ts
+665
View File
@@ -0,0 +1,665 @@
type TeamsMeetingStatusPreludeParams = {
allowMicrophone: boolean;
allowSessionAdoption: boolean;
autoJoin: boolean;
captureCaptions: boolean;
expectedIdentity?: string;
guestName: string;
meetingSessionId?: string;
pageIdentitySource: string;
readOnly?: boolean;
selectors: string;
toggleStateFunction: string;
waitForInCallMs: number;
};
const TEAMS_MEETING_TRANSCRIPT_MAX_LINES = 500;
export function teamsMeetingStatusPreludeSource(params: TeamsMeetingStatusPreludeParams): string {
const selectors = params.selectors;
const expectedIdentity = params.expectedIdentity;
const toggleStateFunction = params.toggleStateFunction;
const pageIdentityFunctionSource = () => params.pageIdentitySource;
return `async () => {
${pageIdentityFunctionSource()}
const parseToggleState = ${toggleStateFunction};
const selectors = ${selectors};
const expectedIdentity = ${JSON.stringify(expectedIdentity)};
const allowMicrophone = ${JSON.stringify(params.allowMicrophone)};
const allowSessionAdoption = ${JSON.stringify(params.allowSessionAdoption)};
const autoJoin = ${JSON.stringify(params.autoJoin)};
const captureCaptions = ${JSON.stringify(params.captureCaptions)};
const readOnly = ${JSON.stringify(Boolean(params.readOnly))};
const sessionId = ${JSON.stringify(params.meetingSessionId)};
const identityRetentionMs = ${JSON.stringify(Math.max(30_000, params.waitForInCallMs))};
const text = (node) => (node?.innerText || node?.textContent || "").trim();
const label = (node) => [
node?.getAttribute?.("aria-label"),
node?.getAttribute?.("title"),
node?.getAttribute?.("data-tid"),
text(node),
].filter(Boolean).join(" ");
const clickable = (node) => node?.matches?.("button")
? node
: node?.querySelector?.("button") || node?.closest?.("button") || node;
const first = (list) => {
for (const selector of list) {
const node = document.querySelector(selector);
if (node) return clickable(node);
}
return undefined;
};
const firstRaw = (list) => {
for (const selector of list) {
const node = document.querySelector(selector);
if (node) return node;
}
return undefined;
};
const firstWithin = (root, list) => {
if (!root) return undefined;
for (const selector of list) {
if (root.matches?.(selector)) return root;
const node = root.querySelector?.(selector);
if (node) return node;
}
return undefined;
};
const buttons = [...document.querySelectorAll("button")];
const findTextButton = (pattern) => buttons.find((button) => !button.disabled && pattern.test(label(button)));
const waitForUi = () => new Promise((resolve) => setTimeout(resolve, 120));
const bridgeOwnedBySession = (entry) => Boolean(
sessionId && (!entry?.sessionId || entry.sessionId === sessionId)
);
const mediaSourceUrl = (element) => String(element?.currentSrc || element?.src || "");
const bridgeSources = (entry) => Array.isArray(entry?.sources)
? entry.sources
: entry?.source
? [{ element: entry.source, muted: Boolean(entry.sourceMuted), pending: Boolean(entry.pending), stream: entry.stream, url: entry.sourceUrl }]
: [];
const bridgeSourceMatches = (element, source) => {
if (!element) return false;
if (source?.pending && mediaSourceIsEmpty(element) && !source.stream && !source.url) return true;
if (source?.stream || element.srcObject) return element.srcObject === source?.stream;
const currentUrl = mediaSourceUrl(element);
return Boolean(source?.url && currentUrl && source.url === currentUrl);
};
const mediaSourceIsEmpty = (element) => Boolean(
element && !element.srcObject && !mediaSourceUrl(element)
);
const restoreAudioBridgeSource = (source) => {
const element = source?.element;
// An empty element may receive a replacement source after cleanup. Keep it
// silent because there is no source identity that is safe to restore.
if (mediaSourceIsEmpty(element)) {
element.muted = true;
return;
}
// Teams reuses media elements across source changes. Restore only the exact
// source this bridge muted.
if (!bridgeSourceMatches(element, source)) return;
const detachedLiveSource = Boolean(
element.isConnected === false &&
element.srcObject?.getAudioTracks?.().some((track) => track.readyState === "live")
);
if (detachedLiveSource) {
element.muted = true;
element.pause?.();
element.srcObject = null;
return;
}
element.muted = Boolean(source.muted);
};
const restoreAudioBridgeSources = (entry) => {
bridgeSources(entry).forEach(restoreAudioBridgeSource);
};
const retireAudioBridge = (entry, restoreSources = true) => {
if (restoreSources) restoreAudioBridgeSources(entry);
entry?.bridge?.pause?.();
if (entry?.bridge) entry.bridge.srcObject = null;
entry?.bridge?.remove?.();
};
const retireOwnedAudioBridges = (restoreSources = true) => {
const entries = Array.isArray(window.__openclawTeamsAudioOutputs)
? window.__openclawTeamsAudioOutputs
: [];
const retained = [];
for (const entry of entries) {
if (!bridgeOwnedBySession(entry)) {
retained.push(entry);
continue;
}
retireAudioBridge(entry, restoreSources);
}
if (retained.length > 0) window.__openclawTeamsAudioOutputs = retained;
else delete window.__openclawTeamsAudioOutputs;
};
const adoptAudioBridgeSourcesForSession = () => {
const entries = Array.isArray(window.__openclawTeamsAudioOutputs)
? window.__openclawTeamsAudioOutputs
: [];
const suspendedBySource = new Map();
for (const entry of entries) {
for (const source of bridgeSources(entry)) {
if (!source?.element || suspendedBySource.has(source.element)) continue;
if (!bridgeSourceMatches(source.element, source)) {
restoreAudioBridgeSource(source);
continue;
}
suspendedBySource.set(source.element, {
sessionId,
source: source.element,
sourceMuted: Boolean(source.muted),
sourceUrl: mediaSourceUrl(source.element) || source.url,
stream: source.element.srcObject,
suspended: true,
});
}
retireAudioBridge(entry, false);
}
const suspended = [...suspendedBySource.values()];
if (suspended.length > 0) window.__openclawTeamsAudioOutputs = suspended;
else delete window.__openclawTeamsAudioOutputs;
};
const suspendOwnedAudioBridges = () => {
const entries = Array.isArray(window.__openclawTeamsAudioOutputs)
? window.__openclawTeamsAudioOutputs
: [];
const retained = [];
const suspendedBySource = new Map();
for (const entry of entries) {
if (!bridgeOwnedBySession(entry)) {
retained.push(entry);
continue;
}
// This pending entry owns the muted element until a later serialized
// status poll sees and routes the attached playback source.
if (
entry?.pending &&
bridgeSources(entry).some((source) => bridgeSourceMatches(source?.element, source))
) {
retained.push(entry);
continue;
}
for (const source of bridgeSources(entry)) {
if (!source?.element || suspendedBySource.has(source.element)) continue;
if (!bridgeSourceMatches(source.element, source)) {
restoreAudioBridgeSource(source);
continue;
}
suspendedBySource.set(source.element, {
sessionId: entry.sessionId || sessionId,
source: source.element,
sourceMuted: Boolean(source.muted),
sourceUrl: source.url,
stream: source.element.srcObject,
suspended: true,
});
}
retireAudioBridge(entry, false);
}
const next = [...retained, ...suspendedBySource.values()];
if (next.length > 0) window.__openclawTeamsAudioOutputs = next;
else delete window.__openclawTeamsAudioOutputs;
};
const retireOwnedCaptions = () => {
const active = window.__openclawTeamsCaptions;
const owned = Boolean(
active && sessionId && (!active.sessionId || active.sessionId === sessionId)
);
if (!owned) return;
if (active.settleTimer !== undefined) clearTimeout(active.settleTimer);
active.observer?.disconnect?.();
delete window.__openclawTeamsCaptions;
};
const finalizeCaptionState = (active) => {
if (!active) return;
if (active.settleTimer !== undefined) clearTimeout(active.settleTimer);
active.settleTimer = undefined;
active.observer?.disconnect?.();
active.observer = undefined;
active.observerInstalled = false;
active.lines = Array.isArray(active.lines) ? active.lines : [];
if (Array.isArray(active.visible) && active.visible.length > 0) {
active.lines.push(...active.visible.map((entry) => ({
at: entry.at,
speaker: entry.speaker,
text: entry.text,
})));
active.visible = [];
}
const excess = active.lines.length - ${TEAMS_MEETING_TRANSCRIPT_MAX_LINES};
if (excess > 0) {
active.lines.splice(0, excess);
active.droppedLines = (active.droppedLines || 0) + excess;
}
active.finalized = true;
active.finalizedAt = Date.now();
};
const archiveFinalizedCaptions = (active) => {
if (active?.finalized !== true || !active.sessionId) return;
const archive = window.__openclawTeamsCaptionArchive &&
typeof window.__openclawTeamsCaptionArchive === "object"
? window.__openclawTeamsCaptionArchive
: {};
archive[active.sessionId] = active;
const retained = Object.entries(archive)
.sort((left, right) => Number(right[1]?.finalizedAt || 0) - Number(left[1]?.finalizedAt || 0))
.slice(0, 4);
window.__openclawTeamsCaptionArchive = Object.fromEntries(retained);
};
const finalizeOwnedCaptions = () => {
const active = window.__openclawTeamsCaptions;
const owned = Boolean(
active && sessionId && (!active.sessionId || active.sessionId === sessionId)
);
if (owned) {
active.identity ||= priorMeeting?.identity || expectedIdentity;
finalizeCaptionState(active);
}
};
const toggleState = (node, kind) => parseToggleState({
kind,
ariaPressed: node?.getAttribute?.("aria-pressed"),
ariaChecked: node?.getAttribute?.("aria-checked"),
checked: typeof node?.checked === "boolean" ? node.checked : undefined,
label: label(node),
});
const notes = [];
const currentIdentity = meetingIdentity(location.href);
const priorMeeting = window.__openclawTeamsMeeting;
if (expectedIdentity && currentIdentity && currentIdentity !== expectedIdentity) {
// A confirmed SPA transition must stop resources still owned by this
// request, while preserving any newer session already committed to the tab.
retireOwnedAudioBridges();
finalizeOwnedCaptions();
const requestOwnsMeeting = Boolean(
priorMeeting &&
sessionId &&
(!priorMeeting.sessionId || priorMeeting.sessionId === sessionId)
);
if (requestOwnsMeeting) delete window.__openclawTeamsMeeting;
return JSON.stringify({
inCall: false,
manualActionRequired: true,
manualActionReason: "teams-session-conflict",
manualActionMessage: "The tracked Teams tab now shows a different meeting. Return to the requested meeting link, then retry.",
title: document.title,
url: location.href,
notes,
});
}
const meetingOwnerConflict = Boolean(
priorMeeting?.sessionId && priorMeeting.sessionId !== sessionId
);
const captionOwnerConflict = Boolean(
window.__openclawTeamsCaptions?.sessionId &&
window.__openclawTeamsCaptions.sessionId !== sessionId
);
const committedOwnerConflict = meetingOwnerConflict || captionOwnerConflict;
const canRepairCaptionOwner = Boolean(
!meetingOwnerConflict && priorMeeting?.sessionId === sessionId
);
const canMutateSession = Boolean(
!readOnly &&
sessionId &&
(!committedOwnerConflict || canRepairCaptionOwner || allowSessionAdoption)
);
const identityMatchedUrl = Boolean(expectedIdentity && currentIdentity === expectedIdentity);
const identityVerifiedBeforeCall = identityMatchedUrl;
const continueInBrowser = first(selectors.continueInBrowser) ||
findTextButton(/continue on this browser|join on the web|use the web app|continue without the app/i);
if (canMutateSession && identityVerifiedBeforeCall && continueInBrowser) {
continueInBrowser.click();
notes.push("Continued to the Teams web client.");
await waitForUi();
}
const guestInput = first(selectors.guestName) || [...document.querySelectorAll("input")].find((input) =>
/enter your name|type your name|your name|display name/i.test(label(input) + " " + (input.placeholder || ""))
);
if (canMutateSession && identityVerifiedBeforeCall && autoJoin && guestInput && !guestInput.value) {
const setter = Object.getOwnPropertyDescriptor(HTMLInputElement.prototype, "value")?.set;
guestInput.focus();
if (setter) setter.call(guestInput, ${JSON.stringify(params.guestName)});
else guestInput.value = ${JSON.stringify(params.guestName)};
guestInput.dispatchEvent(new Event("input", { bubbles: true }));
guestInput.dispatchEvent(new Event("change", { bubbles: true }));
}
const leave = first(selectors.leave);
const continueWithoutDevices = findTextButton(/^continue without audio or video$/i);
let dismissedDevicePrompt = false;
if (
canMutateSession &&
identityVerifiedBeforeCall &&
!leave &&
autoJoin &&
!allowMicrophone &&
continueWithoutDevices
) {
continueWithoutDevices.click();
dismissedDevicePrompt = true;
notes.push("Dismissed the Teams device prompt; selected audio is verified separately.");
await waitForUi();
}
// Teams replaces the meeting URL after admission. Preserve identity only
// while adopting the first in-call control or retaining that exact control.
const markerAgeMs = Date.now() - (priorMeeting?.verifiedAt || 0);
const identityAdoptedInCall = Boolean(
!currentIdentity &&
priorMeeting?.identity === expectedIdentity &&
!priorMeeting?.inCallControl &&
markerAgeMs >= 0 &&
markerAgeMs < identityRetentionMs &&
leave &&
leave.isConnected !== false
);
const identityRerenderedInCall = Boolean(
!currentIdentity &&
priorMeeting?.identity === expectedIdentity &&
priorMeeting?.inCallControl &&
priorMeeting.inCallControl !== leave &&
priorMeeting.inCallControl.isConnected === false &&
priorMeeting?.inCallUrl === location.href &&
markerAgeMs >= 0 &&
markerAgeMs < 5_000 &&
leave &&
leave.isConnected !== false
);
const identityAwaitingRerender = Boolean(
!currentIdentity &&
priorMeeting?.identity === expectedIdentity &&
priorMeeting?.inCallControl &&
priorMeeting.inCallControl.isConnected === false &&
priorMeeting?.inCallUrl === location.href &&
markerAgeMs >= 0 &&
markerAgeMs < 5_000 &&
!leave
);
const identityPreservedInCall = Boolean(
!currentIdentity &&
priorMeeting?.identity === expectedIdentity &&
leave &&
leave.isConnected !== false &&
(
identityAdoptedInCall ||
identityRerenderedInCall ||
(
priorMeeting?.inCallControl === leave &&
priorMeeting?.inCallUrl === location.href
)
)
);
const identityVerified = identityVerifiedBeforeCall || identityPreservedInCall;
const inCall = Boolean(identityVerified && leave);
if (canMutateSession && identityVerified && meetingOwnerConflict) {
// The tab can survive a Teams SPA meeting/session change. Old hidden bridges
// must stop, while their muted source streams remain eligible for the new owner.
adoptAudioBridgeSourcesForSession();
}
if (canMutateSession && !inCall && !identityAwaitingRerender) retireOwnedAudioBridges();
if (canMutateSession && (identityVerifiedBeforeCall || identityPreservedInCall)) {
window.__openclawTeamsMeeting = {
...(priorMeeting?.identity === expectedIdentity && !meetingOwnerConflict ? priorMeeting : {}),
identity: expectedIdentity,
sessionId: sessionId || priorMeeting?.sessionId,
verifiedAt: Date.now(),
...(inCall ? { inCallControl: leave, inCallUrl: location.href } : {}),
};
} else if (
canMutateSession &&
!currentIdentity &&
priorMeeting &&
!identityAwaitingRerender &&
(priorMeeting.inCallControl || markerAgeMs >= identityRetentionMs)
) {
delete window.__openclawTeamsMeeting;
}
const microphone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i);
let microphoneState = identityVerified ? toggleState(microphone, "microphone") : undefined;
const camera = first(selectors.camera) || findTextButton(/camera|video/i);
let cameraState = identityVerified ? toggleState(camera, "camera") : undefined;
let controlManualActionReason;
let controlManualActionMessage;
if (canMutateSession && identityVerified && !inCall && camera && cameraState === "on") {
camera.click();
await waitForUi();
const currentCamera = first(selectors.camera) || findTextButton(/camera|video/i);
cameraState = toggleState(currentCamera, "camera");
if (cameraState === "off") {
notes.push("Turned the Teams camera off before joining.");
}
}
const join = first(selectors.join) || findTextButton(/^\\s*(join now|ask to join|join meeting)\\s*$/i);
if (identityVerified && !inCall && join && cameraState !== "off") {
controlManualActionReason = "teams-camera-required";
controlManualActionMessage = "Turn the Teams camera off and verify the camera control shows it is off, then retry joining.";
}
const isBlackHole = (value) =>
/^blackhole 2ch(?: \\(virtual\\))?$/i.test(String(value || "").replace(/\\s+/g, " ").trim());
const isBlackHoleNode = (node) => [
node?.getAttribute?.("aria-label"),
node?.getAttribute?.("title"),
node?.label,
node?.value,
text(node),
].some(isBlackHole);
const microphoneDeviceRoots = () => {
// Consumer in-call controls expose the listbox itself, without the prejoin
// selected-device button/combobox wrapper.
const control = firstRaw(selectors.microphoneDevice) || firstRaw(selectors.microphoneDeviceMenu);
if (!control) return { control, roots: [] };
const roots = [control];
const scope = control.closest?.('[data-tid="device-settings-microphone"]');
if (scope && !roots.includes(scope)) roots.push(scope);
const listboxId = control.getAttribute?.("aria-controls");
const listbox = listboxId ? document.getElementById?.(listboxId) : undefined;
if (listbox && !roots.includes(listbox)) roots.push(listbox);
const liveMenu = firstRaw(selectors.microphoneDeviceMenu);
if (liveMenu && !roots.includes(liveMenu)) roots.push(liveMenu);
return { control, roots };
};
const selectedMicrophoneLabel = () => {
const { control, roots } = microphoneDeviceRoots();
const selectedOption = control?.selectedOptions?.[0];
if (selectedOption && isBlackHoleNode(selectedOption)) {
return label(selectedOption) || selectedOption.value;
}
if (control && isBlackHoleNode(control)) return label(control) || control.value;
for (const root of roots) {
const selected = firstWithin(root, selectors.selectedMicrophoneDevice);
if (selected && isBlackHoleNode(selected)) {
return label(selected) || selected.value;
}
}
return undefined;
};
let audioInputRouted;
let audioInputDeviceLabel;
let audioInputRouteError;
const ensureVirtualAudioInput = async () => {
if (!navigator.mediaDevices?.enumerateDevices) return false;
try {
const devices = await navigator.mediaDevices.enumerateDevices();
const input = devices.find((device) => device.kind === "audioinput" && isBlackHole(device.label));
if (!input?.deviceId) return false;
audioInputDeviceLabel = input.label || "BlackHole 2ch";
// Teams hides the selected-device control after admission. Reopen the in-call audio
// options and verify the current selection before unmuting; installed devices alone
// do not prove which microphone Teams is using.
const preparedInput = window.__openclawTeamsMeeting;
const preparedSelection = Boolean(
readOnly &&
preparedInput?.identity === expectedIdentity &&
(!sessionId || preparedInput?.sessionId === sessionId) &&
preparedInput?.audioInputDeviceId === input.deviceId
);
let selected = Boolean(selectedMicrophoneLabel()) || preparedSelection;
if (!selected && canMutateSession) {
const settings = first(selectors.deviceSettings);
if (settings) {
settings.click();
await waitForUi();
}
const { control } = microphoneDeviceRoots();
if (control?.tagName?.toLowerCase() === "select") {
const options = [...control.options];
const option = options.find(isBlackHoleNode);
if (option) {
control.value = option.value;
control.dispatchEvent(new Event("change", { bubbles: true }));
await waitForUi();
}
} else if (control) {
clickable(control)?.click?.();
await waitForUi();
}
const choices = microphoneDeviceRoots().roots.flatMap((root) =>
selectors.audioDeviceOptions.flatMap((selector) => [
...(root.querySelectorAll?.(selector) || []),
])
);
const choice = choices.find(isBlackHoleNode);
if (choice && choice.getAttribute?.("aria-selected") !== "true") {
clickable(choice)?.click?.();
await waitForUi();
}
selected = Boolean(selectedMicrophoneLabel());
}
if (selected && window.__openclawTeamsMeeting?.identity === expectedIdentity) {
window.__openclawTeamsMeeting.audioInputDeviceId = input.deviceId;
}
return selected;
} catch (error) {
audioInputRouteError = error?.message || String(error);
return false;
}
};
if (identityVerified && !inCall && allowMicrophone && microphone) {
audioInputRouted = await ensureVirtualAudioInput();
if (!audioInputRouted) {
if (canMutateSession && microphoneState === "on") {
microphone.click();
await waitForUi();
const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i);
microphoneState = toggleState(currentMicrophone, "microphone");
}
controlManualActionReason = "teams-audio-choice-required";
controlManualActionMessage = "Select BlackHole 2ch as the Teams microphone and verify it is selected before enabling talk-back.";
} else if (canMutateSession && microphoneState === "off") {
microphone.click();
await waitForUi();
const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i);
microphoneState = toggleState(currentMicrophone, "microphone");
if (microphoneState === "on") {
notes.push("Unmuted the Teams microphone after verifying BlackHole 2ch input.");
}
}
if (audioInputRouted && microphoneState !== "on") {
controlManualActionReason = "teams-microphone-required";
controlManualActionMessage = "Unmute the Teams microphone and verify the microphone control shows it is on, then retry joining.";
}
} else if (canMutateSession && identityVerified && !inCall && !allowMicrophone && microphoneState === "on") {
microphone.click();
await waitForUi();
const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i);
microphoneState = toggleState(currentMicrophone, "microphone");
if (microphoneState === "off") {
notes.push("Muted the Teams microphone for observe-only mode.");
}
}
if (identityVerified && inCall && allowMicrophone) {
if (!selectedMicrophoneLabel() && canMutateSession && microphoneState === "on") {
microphone?.click();
await waitForUi();
const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i);
microphoneState = toggleState(currentMicrophone, "microphone");
}
audioInputRouted = await ensureVirtualAudioInput();
if (audioInputRouted && canMutateSession && microphoneState === "off") {
microphone?.click();
await waitForUi();
const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i);
microphoneState = toggleState(currentMicrophone, "microphone");
} else if (!audioInputRouted && canMutateSession && microphoneState === "on") {
microphone?.click();
await waitForUi();
const currentMicrophone = first(selectors.microphone) || findTextButton(/mute|unmute|microphone/i);
microphoneState = toggleState(currentMicrophone, "microphone");
if (microphoneState === "off") {
notes.push("Muted the Teams microphone because BlackHole 2ch input could not be reverified.");
}
}
}
if (identityVerified && !inCall && join && !allowMicrophone && microphoneState !== "off") {
controlManualActionReason = "teams-microphone-required";
controlManualActionMessage = "Mute the Teams microphone and verify the microphone control shows it is off, then retry joining.";
}
if (identityVerified && !inCall && join && allowMicrophone && !controlManualActionReason) {
if (!microphone) {
controlManualActionReason = "teams-microphone-required";
controlManualActionMessage = "Open Teams device settings and verify the microphone control before enabling talk-back.";
} else if (audioInputRouted !== true) {
controlManualActionReason = "teams-audio-choice-required";
controlManualActionMessage = "Select BlackHole 2ch as the Teams microphone and verify it is selected before enabling talk-back.";
} else if (microphoneState !== "on") {
controlManualActionReason = "teams-microphone-required";
controlManualActionMessage = "Unmute the Teams microphone and verify the microphone control shows it is on, then retry joining.";
}
}
const micMuted = microphoneState === "off" ? true : microphoneState === "on" ? false : undefined;
const cameraOff = cameraState === "off" ? true : cameraState === "on" ? false : undefined;
const pageText = text(document.body);
const pageTextLower = pageText.toLowerCase();
const lobbyWaiting = Boolean(first(selectors.lobby)) ||
/someone will let you in shortly|waiting for someone to let you in|when someone admits you|you.?re in the lobby|we.?ve let people in the meeting know you.?re waiting/i.test(pageTextLower);
const signInControl = first(selectors.signIn);
const hostname = location.hostname.toLowerCase();
const tenantLoginRequired =
/only people with a work or school account|sign in with an account from this organization|anonymous users (?:can.?t|cannot) join|verify your email|enter the code sent to/i.test(pageTextLower);
const loginRequired = hostname === "login.microsoftonline.com" ||
hostname.endsWith(".microsoftonline.com") ||
tenantLoginRequired ||
(Boolean(signInControl) && !guestInput && !join && /sign in to (?:join|continue)|sign in to your account/i.test(pageTextLower));
let microphonePermissionState;
if (allowMicrophone && navigator.permissions?.query) {
try {
microphonePermissionState = (await navigator.permissions.query({ name: "microphone" })).state;
} catch {}
}
const devicePermissionPrompt = !dismissedDevicePrompt && Boolean(
first(selectors.permissionPrompt) || continueWithoutDevices
);
// Teams shows the same no-audio/video warning when only camera access is denied.
// A granted microphone plus the verified BlackHole input is sufficient for talk-back.
const permissionRequired = devicePermissionPrompt &&
(!allowMicrophone || microphonePermissionState !== "granted");
let manualActionReason;
let manualActionMessage;
if (committedOwnerConflict && !canMutateSession) {
manualActionReason = "teams-session-conflict";
manualActionMessage = "This Teams tab is owned by another active meeting session.";
} else if (!inCall && loginRequired) {
manualActionReason = "teams-login-required";
manualActionMessage = tenantLoginRequired
? "This Teams tenant requires sign-in or email verification. Complete it in the OpenClaw browser profile, then retry."
: "Sign in to Microsoft Teams in the OpenClaw browser profile, then retry the meeting join.";
} else if (!inCall && lobbyWaiting) {
manualActionReason = "teams-admission-required";
manualActionMessage = "Admit the OpenClaw guest from the Microsoft Teams lobby, then retry speech.";
} else if (!inCall && permissionRequired) {
manualActionReason = "teams-permission-required";
manualActionMessage = allowMicrophone
? "Allow microphone permission for Teams in the OpenClaw browser profile, then retry."
: "Dismiss the Teams device-permission prompt or continue without devices, then retry.";
} else if (!inCall && controlManualActionReason) {
manualActionReason = controlManualActionReason;
manualActionMessage = controlManualActionMessage;
}
let clickedJoin = false;
if (canMutateSession && identityVerified && autoJoin && !inCall && join && !join.disabled && !manualActionReason) {
join.click();
clickedJoin = true;
notes.push("Clicked the Teams guest join button.");
}
`;
}
extensions/teams-meetings/src/transports/teams-meetings-urls.test.ts
+15 -1
View File
@@ -7,11 +7,22 @@ import {
const WORK_URL =
"https://teams.microsoft.com/l/meetup-join/19%3ameeting_NzQ1ZDBjYzItZDAxNS00N2YxLTg4Y2EtYjQ1N2I4NDg2Njli%40thread.v2/0?context=%7b%22Tid%22%3a%22tenant%22%7d";
const CONSUMER_COORDINATES = Buffer.from(
JSON.stringify({ meetingCode: "9326458712345", passcode: "abc" }),
).toString("base64");
describe("Microsoft Teams meeting URL normalization", () => {
it.each([
[WORK_URL, "teams-work:19:meeting_NzQ1ZDBjYzItZDAxNS00N2YxLTg4Y2EtYjQ1N2I4NDg2Njli@thread.v2"],
["https://teams.live.com/meet/9326458712345?p=abc", "teams-consumer:9326458712345:p:abc"],
[
"https://teams.live.com/dl/launcher/launcher.html?url=%2F_%23%2Fmeet%2F9326458712345%3Fp%3Dabc",
"teams-consumer:9326458712345:p:abc",
],
[
`https://teams.live.com/light-meetings/launch?coords=${encodeURIComponent(CONSUMER_COORDINATES)}`,
"teams-consumer:9326458712345:p:abc",
],
])("extracts a stable identity from %s", (url, expected) => {
expect(normalizeTeamsMeetingUrlForReuse(url)).toBe(expected);
});
@@ -43,7 +54,7 @@ describe("Microsoft Teams meeting URL normalization", () => {
false,
],
["https://teams.live.com/meet/abc-123?p=one", "https://teams.live.com/meet/abc-123", false],
])("compares consumer meeting password identity: %s / %s", (left, right, expected) => {
])("compares consumer meeting passcode identity: %s / %s", (left, right, expected) => {
expect(isSameTeamsMeetingUrl(left, right)).toBe(expected);
});
@@ -52,6 +63,9 @@ describe("Microsoft Teams meeting URL normalization", () => {
"https://teams.microsoft.com/v2/",
"https://teams.microsoft.com/l/channel/19%3Achannel%40thread.tacv2",
"https://teams.live.com/",
"https://teams.live.com/dl/launcher/launcher.html?url=https%3A%2F%2Fevil.example%2Fmeet%2Fabc",
"https://teams.live.com/dl/launcher/launcher.html?url=%2F_%23%2Fmeet%2Fabc%2Fextra",
"https://teams.live.com/light-meetings/launch?coords=not-json",
"http://teams.live.com/meet/abc",
"https://example.com/l/meetup-join/19%3ameeting_x%40thread.v2/0",
])("rejects non-meeting Teams input: %s", (url) => {
extensions/teams-meetings/src/transports/teams-meetings-urls.ts
+32 -3
View File
@@ -24,7 +24,32 @@ function parseTeamsMeetingIdentity(url: string | undefined): TeamsMeetingIdentit
return { kind: "work", key: threadId };
}
if (hostname === "teams.live.com") {
const match = parsed.pathname.match(/^\/meet\/([^/]+)\/?$/i);
const launcherTarget =
parsed.pathname.toLowerCase() === "/dl/launcher/launcher.html"
? parsed.searchParams.get("url")
: undefined;
const launcherMatch = launcherTarget?.match(/^\/_#\/meet\/([^/?#]+)(?:\?(.+))?$/i);
let lightMeeting: { meetingCode?: unknown; passcode?: unknown } | undefined;
if (parsed.pathname.toLowerCase() === "/light-meetings/launch") {
try {
const coordinates = parsed.searchParams.get("coords");
const decoded =
coordinates && coordinates.length <= 16_384
? JSON.parse(Buffer.from(coordinates, "base64").toString("utf8"))
: undefined;
if (decoded && typeof decoded === "object") {
lightMeeting = decoded as { meetingCode?: unknown; passcode?: unknown };
}
} catch {
return undefined;
}
}
const match =
parsed.pathname.match(/^\/meet\/([^/]+)\/?$/i) ??
launcherMatch ??
(typeof lightMeeting?.meetingCode === "string"
? ([undefined, lightMeeting.meetingCode] as const)
: undefined);
if (!match?.[1]) {
return undefined;
}
@@ -32,10 +57,14 @@ function parseTeamsMeetingIdentity(url: string | undefined): TeamsMeetingIdentit
if (!/^[a-z0-9_-]+$/i.test(meetCode)) {
return undefined;
}
const password = parsed.searchParams.get("p");
const passcode = launcherMatch
? new URLSearchParams(launcherMatch[2] ?? "").get("p")
: typeof lightMeeting?.passcode === "string"
? lightMeeting.passcode
: parsed.searchParams.get("p");
return {
kind: "consumer",
key: `${meetCode.toLowerCase()}:p:${encodeURIComponent(password ?? "")}`,
key: `${meetCode.toLowerCase()}:p:${encodeURIComponent(passcode ?? "")}`,
};
}
} catch {
extensions/teams-meetings/src/transports/types.ts
+13
View File
@@ -43,12 +43,25 @@ export type TeamsMeetingsChromeHealth = MeetingBrowserHealth<
micMuted?: boolean;
cameraOff?: boolean;
lobbyWaiting?: boolean;
captionCaptureRequested?: boolean;
captioning?: boolean;
captionsEnabledAttempted?: boolean;
transcriptLines?: number;
lastCaptionAt?: string;
lastCaptionSpeaker?: string;
lastCaptionText?: string;
recentTranscript?: Array<{
at?: string;
speaker?: string;
text: string;
}>;
audioInputRouted?: boolean;
audioInputDeviceLabel?: string;
audioInputRouteError?: string;
audioOutputRouted?: boolean;
audioOutputDeviceLabel?: string;
audioOutputRouteError?: string;
audioOutputRouteRetryable?: boolean;
providerConnected?: boolean;
realtimeReady?: boolean;
audioInputActive?: boolean;
scripts/lib/plugin-sdk-declaration-budget.mjs
+3 -3
View File
@@ -1,9 +1,9 @@
// Raised for the plugins.uninstall/catalog actions and current session/plugin protocol surfaces;
// Raised for meeting-browser origin permissions and current browser-session contracts;
// the cap exists to force a conscious decision on published declaration growth.
export const MAX_PUBLIC_PLUGIN_SDK_DECLARATION_BYTES = 5_200_000;
export const MAX_PUBLIC_PLUGIN_SDK_DECLARATION_BYTES = 5_250_000;
// Private-only entrypoints reshape chunks reachable from public roots but are never published.
// Bound that topology overhead without counting local-only declarations as package surface.
export const MAX_PRIVATE_QA_PUBLIC_PLUGIN_SDK_DECLARATION_BYTES = 5_225_000;
export const MAX_PRIVATE_QA_PUBLIC_PLUGIN_SDK_DECLARATION_BYTES = 5_275_000;
// Rolldown can repartition equivalent declaration chunks between clean builds; measured spread is
// about 29 KiB across hosts. Keep one 64 KiB scheduling window above the intentional size ratchet.
export const PLUGIN_SDK_DECLARATION_OUTPUT_VARIANCE_BYTES = 64 * 1024;
src/meeting-bot/browser-act-lock.test.ts
+119
View File
@@ -0,0 +1,119 @@
import { describe, expect, it, vi } from "vitest";
import { runMeetingBrowserAct } from "./browser-act-lock.js";
describe("meeting browser act lock", () => {
it("serializes async evaluations for one browser target", async () => {
let active = 0;
let maxActive = 0;
let releaseFirst: (() => void) | undefined;
const firstGate = new Promise<void>((resolve) => {
releaseFirst = resolve;
});
const events: string[] = [];
const run = async (name: string, gate?: Promise<void>) =>
await runMeetingBrowserAct({
deadline: Date.now() + 10_000,
targetId: "target-1",
operation: async () => {
active += 1;
maxActive = Math.max(maxActive, active);
events.push(`start-${name}`);
await gate;
events.push(`end-${name}`);
active -= 1;
},
});
const first = run("first", firstGate);
const second = run("second");
await Promise.resolve();
expect(events).toEqual(["start-first"]);
releaseFirst?.();
await Promise.all([first, second]);
expect(maxActive).toBe(1);
expect(events).toEqual(["start-first", "end-first", "start-second", "end-second"]);
});
it("releases the target after a failed evaluation", async () => {
await expect(
runMeetingBrowserAct({
deadline: Date.now() + 10_000,
targetId: "target-failure",
operation: async () => {
throw new Error("evaluation failed");
},
}),
).rejects.toThrow("evaluation failed");
await expect(
runMeetingBrowserAct({
deadline: Date.now() + 10_000,
targetId: "target-failure",
operation: async () => "recovered",
}),
).resolves.toBe("recovered");
});
it("does not start an evaluation after its queue deadline", async () => {
vi.useFakeTimers();
let releaseFirst: (() => void) | undefined;
const firstGate = new Promise<void>((resolve) => {
releaseFirst = resolve;
});
const first = runMeetingBrowserAct({
deadline: Date.now() + 10_000,
targetId: "target-deadline",
operation: async () => await firstGate,
});
let expiredStarted = false;
const expired = runMeetingBrowserAct({
deadline: Date.now() + 1_000,
targetId: "target-deadline",
operation: async () => {
expiredStarted = true;
},
});
const expiredResult = expect(expired).rejects.toThrow(
"timed out waiting for browser tab control",
);
await vi.advanceTimersByTimeAsync(1_000);
await expiredResult;
expect(expiredStarted).toBe(false);
const third = runMeetingBrowserAct({
deadline: Date.now() + 10_000,
targetId: "target-deadline",
operation: async () => "recovered",
});
releaseFirst?.();
await first;
await expect(third).resolves.toBe("recovered");
expect(expiredStarted).toBe(false);
vi.useRealTimers();
});
it("does not time out after the evaluation acquires the target", async () => {
vi.useFakeTimers();
let release: (() => void) | undefined;
const gate = new Promise<void>((resolve) => {
release = resolve;
});
let settled = false;
const running = runMeetingBrowserAct({
deadline: Date.now() + 1_000,
targetId: "target-running",
operation: async () => await gate,
}).finally(() => {
settled = true;
});
await Promise.resolve();
await vi.advanceTimersByTimeAsync(1_000);
expect(settled).toBe(false);
release?.();
await expect(running).resolves.toBeUndefined();
vi.useRealTimers();
});
});
src/meeting-bot/browser-act-lock.ts
+50
View File
@@ -0,0 +1,50 @@
import { MeetingSessionJoinLock } from "./session-join-lock.js";
const browserActLock = new MeetingSessionJoinLock();
const BROWSER_ACT_TIMEOUT_MESSAGE =
"Meeting browser operation timed out waiting for browser tab control.";
// Browser evaluate calls can await page APIs and interleave in one tab. Keep
// ownership, audio, caption, transcript, and leave mutations process-serialized.
export async function runMeetingBrowserAct<T>(params: {
deadline: number;
operation: (remainingMs: number) => Promise<T>;
targetId: string;
}): Promise<T> {
const waitMs = Math.floor(params.deadline - Date.now());
if (waitMs <= 0) {
throw new Error(BROWSER_ACT_TIMEOUT_MESSAGE);
}
let acquired = false;
let markAcquired: (() => void) | undefined;
const acquisition = new Promise<void>((resolve) => {
markAcquired = resolve;
});
let timeout: ReturnType<typeof setTimeout> | undefined;
const queued = browserActLock.run(params.targetId, async () => {
const remainingMs = Math.floor(params.deadline - Date.now());
if (remainingMs <= 0) {
throw new Error(BROWSER_ACT_TIMEOUT_MESSAGE);
}
acquired = true;
clearTimeout(timeout);
markAcquired?.();
return await params.operation(remainingMs);
});
// The acquisition race may return before this queued no-op reaches the lock.
// Keep its eventual deadline rejection observed without masking caller errors.
void queued.catch(() => undefined);
const expired = new Promise<never>((_resolve, reject) => {
timeout = setTimeout(() => {
if (!acquired) {
reject(new Error(BROWSER_ACT_TIMEOUT_MESSAGE));
}
}, waitMs);
});
try {
await Promise.race([acquisition, expired]);
} finally {
clearTimeout(timeout);
}
return await queued;
}
src/meeting-bot/browser-controller.test.ts
+188
View File
@@ -0,0 +1,188 @@
import { describe, expect, it } from "vitest";
import { openMeetingWithBrowser, recoverMeetingBrowserTab } from "./browser-controller.js";
import { isMeetingBrowserTransientNavigationError } from "./browser-navigation-errors.js";
describe("meeting browser navigation errors", () => {
it.each([
"page.evaluate: Execution context was destroyed, most likely because of a navigation.",
"Protocol error: Cannot find context with specified id",
])("retries expected navigation races: %s", (message) => {
expect(isMeetingBrowserTransientNavigationError(new Error(message))).toBe(true);
});
it("does not retry unrelated browser-control failures", () => {
expect(isMeetingBrowserTransientNavigationError(new Error("browser unavailable"))).toBe(false);
});
});
describe("meeting browser join readiness", () => {
it("retries a platform-owned transient in-call status", async () => {
const adoptionAttempts: boolean[] = [];
let evaluationAttempts = 0;
const result = await openMeetingWithBrowser({
adapter: {
browserLabel: "Test meeting",
urls: {
accountHint: () => undefined,
buildJoinUrl: (session) => session.url,
isPreferredJoinUrl: () => true,
isRecoverableTab: () => true,
isSameMeeting: () => true,
localeAction: () => undefined,
normalizeForReuse: () => "test-meeting",
validateAndNormalize: (input) => String(input),
},
browser: {
allowsMicrophone: () => true,
browserControlUnavailable: () => ({
category: "browser-control-unavailable",
reason: "browser-unavailable",
message: "Browser unavailable.",
}),
buildLeaveScript: () => "",
buildStatusJoinScript: (params) => {
adoptionAttempts.push(params.allowSessionAdoption);
return "() => '{}'";
},
captions: {
buildTranscriptScript: () => "",
enabled: () => false,
parseTranscript: () => ({ droppedLines: 0, lines: [] }),
},
classifyManualAction: (health) =>
health.manualActionRequired
? { category: "audio-choice-required", reason: "audio-choice", message: "Wait." }
: undefined,
parseLeaveResult: () => ({ departed: false }),
parseStatus: () =>
evaluationAttempts === 1
? {
inCall: true,
manualActionRequired: true,
manualActionReason: "audio-choice",
micMuted: false,
}
: { inCall: true, manualActionRequired: false, micMuted: false },
permissions: () => undefined,
permissionNotes: () => [],
shouldRetryJoinStatus: (health) => health.manualActionReason === "audio-choice",
},
},
callBrowser: async (request) => {
if (request.path === "/tabs") {
return { tabs: [{ targetId: "target-1", url: "https://meet.test/meeting" }] };
}
if (request.path === "/act") {
evaluationAttempts += 1;
}
return {};
},
config: {
launch: true,
reuseExistingTab: true,
autoJoin: true,
guestName: "OpenClaw QA",
joinTimeoutMs: 1_000,
waitForInCallMs: 1_000,
},
session: {
meetingSessionId: "session-1",
mode: "agent",
url: "https://meet.test/meeting",
},
});
expect(evaluationAttempts).toBe(2);
expect(adoptionAttempts).toEqual([true, false]);
expect(result.browser).toMatchObject({
inCall: true,
manualActionRequired: false,
micMuted: false,
});
});
});
describe("meeting browser recovery", () => {
it("retries status inspection when auto-join navigation destroys the page context", async () => {
const adoptionAttempts: boolean[] = [];
let evaluationAttempts = 0;
const evaluationTimeouts: number[] = [];
const result = await recoverMeetingBrowserTab({
adapter: {
browserLabel: "Test meeting",
urls: {
accountHint: () => undefined,
buildJoinUrl: (session) => session.url,
isPreferredJoinUrl: () => true,
isRecoverableTab: () => true,
isSameMeeting: () => true,
localeAction: () => undefined,
normalizeForReuse: () => "test-meeting",
validateAndNormalize: (input) => String(input),
},
browser: {
allowsMicrophone: () => false,
browserControlUnavailable: () => ({
category: "browser-control-unavailable",
reason: "browser-unavailable",
message: "Browser unavailable.",
}),
buildLeaveScript: () => "",
buildStatusJoinScript: (params) => {
adoptionAttempts.push(params.allowSessionAdoption);
return "() => '{}'";
},
captions: {
buildTranscriptScript: () => "",
enabled: () => false,
parseTranscript: () => ({ droppedLines: 0, lines: [] }),
},
classifyManualAction: () => undefined,
parseLeaveResult: () => ({ departed: false }),
parseStatus: () => ({ status: "browser-control", inCall: true }),
permissions: () => undefined,
permissionNotes: () => [],
},
},
allowSessionAdoption: true,
autoJoin: true,
callBrowser: async (request) => {
if (request.path === "/tabs") {
return { tabs: [{ targetId: "target-1", url: "https://meet.test/meeting" }] };
}
if (request.path === "/act") {
evaluationAttempts += 1;
evaluationTimeouts.push(request.timeoutMs);
if (evaluationAttempts === 1) {
throw new Error("page.evaluate: Execution context was destroyed because of navigation");
}
}
return {};
},
config: {
launch: true,
reuseExistingTab: true,
autoJoin: true,
guestName: "OpenClaw QA",
joinTimeoutMs: 2_000,
waitForInCallMs: 2_000,
},
locationLabel: "for testing",
meetingSessionId: "session-1",
mode: "listen",
requestedMeetingUrl: "https://meet.test/meeting",
timeoutMs: 500,
trackedMeetingUrl: "https://meet.test/meeting",
trackedTargetId: "target-1",
});
expect(evaluationAttempts).toBe(2);
expect(adoptionAttempts).toEqual([true, false]);
expect(evaluationTimeouts[0]).toBeLessThanOrEqual(500);
expect(evaluationTimeouts[1]).toBeLessThan(evaluationTimeouts[0] ?? 0);
expect(result.browser).toMatchObject({
inCall: true,
notes: ["Test meeting navigated while recovering; retrying browser inspection."],
});
});
});
src/meeting-bot/browser-controller.ts
+146 -53
View File
@@ -1,4 +1,6 @@
import { uniqueStrings } from "@openclaw/normalization-core/string-normalization";
import { runMeetingBrowserAct } from "./browser-act-lock.js";
import { isMeetingBrowserTransientNavigationError } from "./browser-navigation-errors.js";
import { asMeetingBrowserTabs, readMeetingBrowserTab } from "./browser-request.js";
import type {
MeetingBrowserJoinSession,
@@ -68,11 +70,13 @@ async function prepareMeetingBrowserTab<
adapter: BrowserAdapter<Session, Mode, Health, Transcript>;
allowMicrophone: boolean;
callBrowser: MeetingBrowserRequestCaller;
meetingUrl: string;
targetId: string;
timeoutMs: number;
}): Promise<string[]> {
const plan = params.adapter.browser.permissions({
allowMicrophone: params.allowMicrophone,
meetingUrl: params.meetingUrl,
});
if (!plan) {
return params.adapter.browser.permissionNotes({
@@ -212,6 +216,7 @@ export async function openMeetingWithBrowser<
adapter: params.adapter,
allowMicrophone,
callBrowser: params.callBrowser,
meetingUrl: params.session.url,
targetId,
timeoutMs,
});
@@ -222,22 +227,33 @@ export async function openMeetingWithBrowser<
browserTitle: tab?.title,
notes: permissionNotes,
} as unknown as Health;
let allowSessionAdoption = true;
do {
try {
const evaluated = await params.callBrowser({
method: "POST",
path: "/act",
body: {
kind: "evaluate",
targetId,
fn: params.adapter.browser.buildStatusJoinScript({
...params.session,
autoJoin: params.config.autoJoin,
captureCaptions: params.adapter.browser.captions.enabled(params.session.mode),
guestName: params.config.guestName,
const adoptSession = allowSessionAdoption;
allowSessionAdoption = false;
const actionTimeoutMs = Math.min(timeoutMs, 10_000);
const evaluated = await runMeetingBrowserAct({
deadline: Date.now() + actionTimeoutMs,
targetId,
operation: async (remainingMs) =>
await params.callBrowser({
method: "POST",
path: "/act",
body: {
kind: "evaluate",
targetId,
fn: params.adapter.browser.buildStatusJoinScript({
...params.session,
allowSessionAdoption: adoptSession,
autoJoin: params.config.autoJoin,
captureCaptions: params.adapter.browser.captions.enabled(params.session.mode),
guestName: params.config.guestName,
waitForInCallMs: params.config.waitForInCallMs,
}),
},
timeoutMs: remainingMs,
}),
},
timeoutMs: Math.min(timeoutMs, 10_000),
});
browser = mergeBrowserNotes(
params.adapter.browser.parseStatus(evaluated) ?? browser,
@@ -247,28 +263,44 @@ export async function openMeetingWithBrowser<
? params.adapter.browser.classifyManualAction(browser)
: undefined;
browser = applyMeetingManualAction(browser, manual);
if (browser?.inCall === true && (!allowMicrophone || browser.micMuted !== true)) {
// Some web clients render their in-call media controls after admission. Let the
// platform retry those transient states before treating them as user action.
const shouldRetry = browser
? params.adapter.browser.shouldRetryJoinStatus?.(browser) === true
: false;
if (
!shouldRetry &&
browser?.inCall === true &&
browser.manualActionRequired !== true &&
(!allowMicrophone || browser.micMuted !== true)
) {
return { launched: true, browser, tab: tabIdentity };
}
if (browser?.manualActionRequired === true) {
if (!shouldRetry && browser?.manualActionRequired === true) {
return { launched: true, browser, tab: tabIdentity };
}
} catch (error) {
const manual = params.adapter.browser.browserControlUnavailable(error);
browser = {
...browser,
inCall: false,
manualActionRequired: true,
manualActionReason: manual.reason,
manualActionMessage: manual.message,
notes: [
...permissionNotes,
`Browser control could not inspect or auto-join ${params.adapter.browserLabel}: ${
error instanceof Error ? error.message : String(error)
}`,
],
} as unknown as Health;
break;
if (isMeetingBrowserTransientNavigationError(error) && Date.now() < deadline) {
browser = mergeBrowserNotes(browser, [
`${params.adapter.browserLabel} navigated while joining; retrying browser inspection.`,
]);
} else {
const manual = params.adapter.browser.browserControlUnavailable(error);
browser = {
...browser,
inCall: false,
manualActionRequired: true,
manualActionReason: manual.reason,
manualActionMessage: manual.message,
notes: [
...permissionNotes,
`Browser control could not inspect or auto-join ${params.adapter.browserLabel}: ${
error instanceof Error ? error.message : String(error)
}`,
],
} as unknown as Health;
break;
}
}
const remainingWaitMs = deadline - Date.now();
if (remainingWaitMs > 0) {
@@ -326,21 +358,30 @@ async function inspectRecoverableTab<
Transcript extends MeetingTranscriptSnapshot,
>(params: {
adapter: BrowserAdapter<Session, Mode, Health, Transcript>;
allowSessionAdoption?: boolean;
autoJoin?: boolean;
callBrowser: MeetingBrowserRequestCaller;
config: MeetingBrowserControllerConfig;
meetingSessionId?: string;
mode: Mode;
readOnly?: boolean;
requestedMeetingUrl: string | undefined;
tab: MeetingBrowserCandidateTab;
targetId: string;
deadline?: number;
timeoutMs: number;
}) {
const allowMicrophone = params.adapter.browser.allowsMicrophone(params.mode);
const focusTimeoutMs =
params.deadline === undefined ? params.timeoutMs : Math.floor(params.deadline - Date.now());
if (focusTimeoutMs <= 0) {
throw new Error("Meeting browser recovery timed out.");
}
await params.callBrowser({
method: "POST",
path: "/tabs/focus",
body: { targetId: params.targetId },
timeoutMs: Math.min(params.timeoutMs, 5_000),
timeoutMs: Math.min(focusTimeoutMs, 5_000),
});
const localeAction = params.adapter.urls.localeAction(params.tab);
if (localeAction) {
@@ -365,27 +406,63 @@ async function inspectRecoverableTab<
adapter: params.adapter,
allowMicrophone,
callBrowser: params.callBrowser,
meetingUrl: params.requestedMeetingUrl ?? params.tab.url ?? "",
targetId: params.targetId,
timeoutMs: params.timeoutMs,
timeoutMs:
params.deadline === undefined
? params.timeoutMs
: Math.max(1, Math.floor(params.deadline - Date.now())),
});
const evaluated = await params.callBrowser({
method: "POST",
path: "/act",
body: {
kind: "evaluate",
targetId: params.targetId,
fn: params.adapter.browser.buildStatusJoinScript({
meetingSessionId: "",
mode: params.mode,
url: params.requestedMeetingUrl ?? params.tab.url ?? "",
autoJoin: false,
captureCaptions: params.adapter.browser.captions.enabled(params.mode),
guestName: params.config.guestName,
readOnly: params.readOnly,
}),
},
timeoutMs: Math.min(params.timeoutMs, 10_000),
});
const navigationNotes: string[] = [];
const inspectionDeadline = params.deadline ?? Date.now() + Math.min(params.timeoutMs, 10_000);
let allowSessionAdoption = params.allowSessionAdoption ?? false;
let evaluated: unknown;
for (;;) {
try {
const adoptSession = allowSessionAdoption;
allowSessionAdoption = false;
evaluated = await runMeetingBrowserAct({
deadline: inspectionDeadline,
targetId: params.targetId,
operation: async (remainingMs) =>
await params.callBrowser({
method: "POST",
path: "/act",
body: {
kind: "evaluate",
targetId: params.targetId,
fn: params.adapter.browser.buildStatusJoinScript({
allowSessionAdoption: adoptSession,
meetingSessionId: params.meetingSessionId ?? "",
mode: params.mode,
url: params.requestedMeetingUrl ?? params.tab.url ?? "",
autoJoin: params.autoJoin ?? false,
captureCaptions: params.adapter.browser.captions.enabled(params.mode),
guestName: params.config.guestName,
readOnly: params.readOnly,
waitForInCallMs: params.config.waitForInCallMs,
}),
},
timeoutMs: remainingMs,
}),
});
break;
} catch (error) {
const remainingMs = inspectionDeadline - Date.now();
if (!isMeetingBrowserTransientNavigationError(error) || remainingMs <= 0) {
throw error;
}
navigationNotes.push(
`${params.adapter.browserLabel} navigated while recovering; retrying browser inspection.`,
);
await new Promise<void>((resolve) => {
setTimeout(resolve, Math.min(250, remainingMs));
});
if (Date.now() >= inspectionDeadline) {
throw error;
}
}
}
const browser = mergeBrowserNotes(
params.adapter.browser.parseStatus(evaluated) ??
({
@@ -393,7 +470,7 @@ async function inspectRecoverableTab<
browserUrl: params.tab.url,
browserTitle: params.tab.title,
} as unknown as Health),
permissionNotes,
[...permissionNotes, ...navigationNotes],
);
const manual: MeetingManualAction | undefined = browser
? params.adapter.browser.classifyManualAction(browser)
@@ -424,12 +501,16 @@ export async function recoverMeetingBrowserTab<
Transcript extends MeetingTranscriptSnapshot,
>(params: {
adapter: BrowserAdapter<Session, Mode, Health, Transcript>;
allowSessionAdoption?: boolean;
autoJoin?: boolean;
callBrowser: MeetingBrowserRequestCaller;
config: MeetingBrowserControllerConfig;
locationLabel: string;
meetingSessionId?: string;
mode: Mode;
requestedMeetingUrl: string | undefined;
readOnly?: boolean;
timeoutMs?: number;
trackedMeetingUrl: string | undefined;
trackedTargetId: string | undefined;
}): Promise<{
@@ -439,12 +520,20 @@ export async function recoverMeetingBrowserTab<
browser?: Health;
message: string;
}> {
const timeoutMs = Math.max(1_000, params.config.joinTimeoutMs);
const configuredTimeoutMs = Math.max(1_000, params.config.joinTimeoutMs);
const timeoutMs =
params.timeoutMs === undefined
? configuredTimeoutMs
: Math.max(1, Math.min(configuredTimeoutMs, params.timeoutMs));
const deadline = params.timeoutMs === undefined ? undefined : Date.now() + timeoutMs;
const tabs = asMeetingBrowserTabs(
await params.callBrowser({
method: "GET",
path: "/tabs",
timeoutMs: Math.min(timeoutMs, 5_000),
timeoutMs:
deadline === undefined
? Math.min(timeoutMs, 5_000)
: Math.min(Math.max(1, Math.floor(deadline - Date.now())), 5_000),
}),
);
const trackedCandidate = params.trackedTargetId
@@ -488,8 +577,12 @@ export async function recoverMeetingBrowserTab<
}
return await inspectRecoverableTab({
adapter: params.adapter,
allowSessionAdoption: params.allowSessionAdoption,
autoJoin: params.autoJoin,
callBrowser: params.callBrowser,
config: params.config,
...(deadline === undefined ? {} : { deadline }),
meetingSessionId: params.meetingSessionId,
mode: params.mode,
readOnly: params.readOnly,
requestedMeetingUrl: params.requestedMeetingUrl,
src/meeting-bot/browser-navigation-errors.ts
+6
View File
@@ -0,0 +1,6 @@
export function isMeetingBrowserTransientNavigationError(error: unknown): boolean {
const message = error instanceof Error ? error.message : String(error);
return /execution context was destroyed.*navigation|cannot find context with specified id/i.test(
message,
);
}
src/meeting-bot/browser-session-control.test.ts
+206
View File
@@ -0,0 +1,206 @@
import { describe, expect, it, vi } from "vitest";
import { runMeetingBrowserAct } from "./browser-act-lock.js";
import { leaveMeetingWithBrowser } from "./browser-session-control.js";
describe("meeting browser leave ownership", () => {
async function leaveWithStep(
step: {
departed: boolean;
sessionConflict?: boolean;
sessionMatched?: boolean;
urlMatched?: boolean;
},
openedByPlugin = true,
) {
const buildLeaveScript = vi.fn(() => "() => '{}'");
const deletedTabs: string[] = [];
const result = await leaveMeetingWithBrowser({
adapter: {
browserLabel: "Test meeting",
browser: {
buildLeaveScript,
parseLeaveResult: () => step,
},
} as never,
callBrowser: async (request) => {
if (request.path === "/tabs") {
return { tabs: [{ targetId: "target-1", url: "https://meet.test/meeting" }] };
}
if (request.path === "/act") {
return { result: "{}" };
}
if (request.method === "DELETE") {
deletedTabs.push(request.path);
return {};
}
throw new Error(`Unexpected browser request: ${request.method} ${request.path}`);
},
launch: true,
meetingSessionId: "session-1",
meetingUrl: "https://meet.test/meeting",
tab: { targetId: "target-1", openedByPlugin },
timeoutMs: 1_000,
});
expect(buildLeaveScript).toHaveBeenCalledWith("https://meet.test/meeting");
expect(deletedTabs).toEqual([]);
return result;
}
it("does not close a tab whose page belongs to another session", async () => {
const result = await leaveWithStep({
departed: false,
sessionConflict: true,
sessionMatched: false,
urlMatched: true,
});
expect(result).toEqual({
left: true,
note: "Test meeting tab belongs to another OpenClaw meeting session; left its current call untouched.",
});
});
it("does not report success when page ownership is unverified", async () => {
const result = await leaveWithStep({
departed: false,
sessionMatched: false,
urlMatched: true,
});
expect(result).toEqual({
left: false,
note: "Browser control could not verify that the Test meeting tab still belongs to this OpenClaw meeting session.",
});
});
it("reports departure while keeping a reused tab open", async () => {
const result = await leaveWithStep({ departed: true, urlMatched: true }, false);
expect(result).toEqual({
left: true,
note: "Clicked Test meeting's Leave call button; kept the reused browser tab open.",
});
});
it("carries initiated-leave evidence into the next page evaluation", async () => {
const leaveInitiated: boolean[] = [];
let evaluation = 0;
const deletedTabs: string[] = [];
let markDeleteStarted: (() => void) | undefined;
const deleteStarted = new Promise<void>((resolve) => {
markDeleteStarted = resolve;
});
let releaseDelete: (() => void) | undefined;
const deleteGate = new Promise<void>((resolve) => {
releaseDelete = resolve;
});
const leaving = leaveMeetingWithBrowser({
adapter: {
browserLabel: "Test meeting",
browser: {
buildLeaveScript: () => "() => '{}'",
buildSessionLeaveScript: (context: { leaveInitiated: boolean }) => {
leaveInitiated.push(context.leaveInitiated);
return "() => '{}'";
},
parseLeaveResult: () =>
evaluation === 1
? { departed: false, leaveAction: "leave", urlMatched: true }
: evaluation === 2
? { departed: false, sessionMatched: false, urlMatched: true }
: { departed: true, urlMatched: false },
},
} as never,
callBrowser: async (request) => {
if (request.path === "/tabs") {
return { tabs: [{ targetId: "target-1", url: "https://meet.test/meeting" }] };
}
if (request.path === "/act") {
evaluation += 1;
return { result: "{}" };
}
if (request.method === "DELETE") {
deletedTabs.push(request.path);
markDeleteStarted?.();
await deleteGate;
return {};
}
throw new Error(`Unexpected browser request: ${request.method} ${request.path}`);
},
launch: true,
meetingSessionId: "session-1",
meetingUrl: "https://meet.test/meeting",
tab: { targetId: "target-1", openedByPlugin: true },
timeoutMs: 1_000,
});
await deleteStarted;
let concurrentStarted = false;
const concurrent = runMeetingBrowserAct({
deadline: Date.now() + 1_000,
targetId: "target-1",
operation: async () => {
concurrentStarted = true;
},
});
await Promise.resolve();
expect(concurrentStarted).toBe(false);
releaseDelete?.();
const result = await leaving;
await concurrent;
expect(leaveInitiated).toEqual([false, true, true]);
expect(deletedTabs).toEqual(["/tabs/target-1"]);
expect(concurrentStarted).toBe(true);
expect(result.left).toBe(true);
});
it("reserves enough time to close a plugin tab after leave polling expires", async () => {
vi.useFakeTimers();
try {
let evaluation = 0;
const deletedTabs: string[] = [];
const leaving = leaveMeetingWithBrowser({
adapter: {
browserLabel: "Test meeting",
browser: {
buildLeaveScript: () => "() => '{}'",
parseLeaveResult: () =>
evaluation === 1
? { departed: false, leaveAction: "leave", urlMatched: true }
: { departed: false, urlMatched: true },
},
} as never,
callBrowser: async (request) => {
if (request.path === "/tabs") {
return { tabs: [{ targetId: "target-1", url: "https://meet.test/meeting" }] };
}
if (request.path === "/act") {
evaluation += 1;
return { result: "{}" };
}
if (request.method === "DELETE") {
deletedTabs.push(request.path);
return {};
}
throw new Error(`Unexpected browser request: ${request.method} ${request.path}`);
},
launch: true,
meetingSessionId: "session-1",
meetingUrl: "https://meet.test/meeting",
tab: { targetId: "target-1", openedByPlugin: true },
timeoutMs: 1_000,
});
await vi.advanceTimersByTimeAsync(1_000);
const result = await leaving;
expect(evaluation).toBeGreaterThan(1);
expect(deletedTabs).toEqual(["/tabs/target-1"]);
expect(result).toEqual({
left: true,
note: "Clicked Test meeting's Leave call button and closed the Test meeting tab.",
});
} finally {
vi.useRealTimers();
}
});
});
src/meeting-bot/browser-session-control.ts
+116 -37
View File
@@ -1,3 +1,4 @@
import { runMeetingBrowserAct } from "./browser-act-lock.js";
import { asMeetingBrowserTabs } from "./browser-request.js";
import type { MeetingBrowserRequestCaller, MeetingPlatformAdapter } from "./platform-adapter.js";
import type {
@@ -21,6 +22,7 @@ async function leaveMeetingInPage<
>(params: {
adapter: BrowserAdapter<Session, Mode, Health, Transcript>;
callBrowser: MeetingBrowserRequestCaller;
meetingSessionId?: string;
meetingUrl: string;
targetId: string;
timeoutMs: number;
@@ -28,30 +30,59 @@ async function leaveMeetingInPage<
departed: boolean;
clickedLeave: boolean;
clickedConfirmation: boolean;
ownershipRetained?: boolean;
sessionConflict?: boolean;
sessionMatched?: boolean;
urlMatched?: boolean;
}> {
const deadline = Date.now() + params.timeoutMs;
let clickedLeave = false;
let clickedConfirmation = false;
let ownershipRetained = false;
do {
const remainingMs = Math.floor(deadline - Date.now());
if (remainingMs <= 0) {
throw new Error("Meeting browser leave timed out.");
}
const evaluated = await params.callBrowser({
method: "POST",
path: "/act",
body: {
kind: "evaluate",
targetId: params.targetId,
fn: params.adapter.browser.buildLeaveScript(params.meetingUrl),
fn:
params.adapter.browser.buildSessionLeaveScript?.({
leaveInitiated: clickedLeave,
meetingSessionId: params.meetingSessionId ?? "",
meetingUrl: params.meetingUrl,
}) ?? params.adapter.browser.buildLeaveScript(params.meetingUrl),
},
timeoutMs: params.timeoutMs,
timeoutMs: remainingMs,
});
const step = params.adapter.browser.parseLeaveResult(evaluated);
clickedLeave ||= step.leaveAction === "leave";
clickedConfirmation ||= step.leaveAction === "confirm";
if (step.sessionMatched === false) {
const stepOwnershipRetained = clickedLeave && step.sessionConflict !== true;
if (step.departed || !stepOwnershipRetained) {
return {
departed: stepOwnershipRetained ? step.departed : false,
clickedLeave,
clickedConfirmation,
ownershipRetained: stepOwnershipRetained,
sessionConflict: step.sessionConflict,
sessionMatched: false,
urlMatched: step.urlMatched,
};
}
ownershipRetained = true;
}
if (step.departed || step.urlMatched !== true) {
return {
departed: step.departed,
clickedLeave,
clickedConfirmation,
...(ownershipRetained && step.sessionConflict !== true ? { ownershipRetained: true } : {}),
urlMatched: step.urlMatched,
};
}
@@ -64,7 +95,13 @@ async function leaveMeetingInPage<
});
}
} while (Date.now() < deadline);
return { departed: false, clickedLeave, clickedConfirmation, urlMatched: true };
return {
departed: false,
clickedLeave,
clickedConfirmation,
...(ownershipRetained ? { ownershipRetained: true, sessionMatched: false } : {}),
urlMatched: true,
};
}
// Leaving acts on the persisted tab identity. Reused tabs remain user-owned;
@@ -78,6 +115,7 @@ export async function leaveMeetingWithBrowser<
adapter: BrowserAdapter<Session, Mode, Health, Transcript>;
callBrowser: MeetingBrowserRequestCaller;
launch: boolean;
meetingSessionId?: string;
meetingUrl: string;
tab: MeetingBrowserTab;
timeoutMs: number;
@@ -102,14 +140,45 @@ export async function leaveMeetingWithBrowser<
};
}
let leaveResult: Awaited<ReturnType<typeof leaveMeetingInPage>>;
let tabClosed = false;
try {
leaveResult = await leaveMeetingInPage({
adapter: params.adapter,
callBrowser: params.callBrowser,
meetingUrl: params.meetingUrl,
const locked = await runMeetingBrowserAct({
deadline: Date.now() + timeoutMs,
targetId,
timeoutMs,
operation: async (remainingMs) => {
const operationDeadline = Date.now() + remainingMs;
const closeReserveMs = openedByPlugin
? Math.min(1_000, Math.max(250, Math.floor(remainingMs / 4)))
: 0;
const result = await leaveMeetingInPage({
adapter: params.adapter,
callBrowser: params.callBrowser,
meetingSessionId: params.meetingSessionId,
meetingUrl: params.meetingUrl,
targetId,
timeoutMs: Math.max(1, remainingMs - closeReserveMs),
});
const canCloseTrackedTab =
openedByPlugin &&
(result.urlMatched === true || result.departed) &&
(result.sessionMatched !== false || result.ownershipRetained === true);
if (!canCloseTrackedTab) {
return { leaveResult: result, tabClosed: false };
}
const closeTimeoutMs = Math.floor(operationDeadline - Date.now());
if (closeTimeoutMs <= 0) {
throw new Error("Meeting browser leave timed out before the tab could close.");
}
await params.callBrowser({
method: "DELETE",
path: `/tabs/${targetId}`,
timeoutMs: closeTimeoutMs,
});
return { leaveResult: result, tabClosed: true };
},
});
leaveResult = locked.leaveResult;
tabClosed = locked.tabClosed;
} catch (error) {
return {
left: false,
@@ -124,33 +193,38 @@ export async function leaveMeetingWithBrowser<
note: `${params.adapter.browserLabel} tab moved away from this session; left its current page untouched.`,
};
}
if (leaveResult.urlMatched !== true) {
// A verified Leave click happened under this target lock. A later document
// can lose its page marker without transferring ownership.
if (leaveResult.sessionMatched === false && leaveResult.ownershipRetained !== true) {
if (leaveResult.sessionConflict !== true) {
return {
left: false,
note: `Browser control could not verify that the ${params.adapter.browserLabel} tab still belongs to this OpenClaw meeting session.`,
};
}
return {
left: true,
note: `${params.adapter.browserLabel} tab belongs to another OpenClaw meeting session; left its current call untouched.`,
};
}
if (leaveResult.urlMatched !== true && !leaveResult.departed) {
return {
left: false,
note: "Browser control could not verify that the tracked tab still showed this meeting.",
};
}
const { clickedLeave, departed } = leaveResult;
if (!openedByPlugin) {
return {
left: departed,
note: departed
return {
left: openedByPlugin ? tabClosed : departed,
note: openedByPlugin
? clickedLeave
? `Clicked ${params.adapter.browserLabel}'s Leave call button and closed the ${params.adapter.browserLabel} tab.`
: `Closed the ${params.adapter.browserLabel} tab to leave the meeting (Leave call button was not found).`
: departed
? `Clicked ${params.adapter.browserLabel}'s Leave call button; kept the reused browser tab open.`
: clickedLeave
? `Clicked ${params.adapter.browserLabel}'s Leave call button, but could not verify departure; leave it manually.`
: `Could not find ${params.adapter.browserLabel}'s Leave call button in the reused browser tab; leave it manually.`,
};
}
await params.callBrowser({
method: "DELETE",
path: `/tabs/${targetId}`,
timeoutMs,
});
return {
left: true,
note: clickedLeave
? `Clicked ${params.adapter.browserLabel}'s Leave call button and closed the ${params.adapter.browserLabel} tab.`
: `Closed the ${params.adapter.browserLabel} tab to leave the meeting (Leave call button was not found).`,
};
} catch (error) {
return {
@@ -176,19 +250,24 @@ export async function readMeetingTranscriptWithBrowser<
tab: MeetingBrowserTab;
timeoutMs: number;
}): Promise<Transcript> {
const result = await params.callBrowser({
method: "POST",
path: "/act",
body: {
kind: "evaluate",
targetId: params.tab.targetId,
fn: params.adapter.browser.captions.buildTranscriptScript({
finalize: params.finalize,
meetingSessionId: params.meetingSessionId,
meetingUrl: params.meetingUrl,
const result = await runMeetingBrowserAct({
deadline: Date.now() + Math.max(1, params.timeoutMs),
targetId: params.tab.targetId,
operation: async (remainingMs) =>
await params.callBrowser({
method: "POST",
path: "/act",
body: {
kind: "evaluate",
targetId: params.tab.targetId,
fn: params.adapter.browser.captions.buildTranscriptScript({
finalize: params.finalize,
meetingSessionId: params.meetingSessionId,
meetingUrl: params.meetingUrl,
}),
},
timeoutMs: remainingMs,
}),
},
timeoutMs: params.timeoutMs,
});
const snapshot = params.adapter.browser.captions.parseTranscript(result);
if (snapshot.urlMatched === false) {
src/meeting-bot/platform-adapter.ts
+14 -1
View File
@@ -37,15 +37,19 @@ export type MeetingBrowserJoinSession<Mode extends string> = {
export type MeetingBrowserStatusScriptParams<Mode extends string> =
MeetingBrowserJoinSession<Mode> & {
allowSessionAdoption: boolean;
autoJoin: boolean;
captureCaptions: boolean;
guestName: string;
readOnly?: boolean;
waitForInCallMs: number;
};
export type MeetingBrowserLeaveStep = {
departed: boolean;
leaveAction?: "leave" | "confirm";
sessionConflict?: boolean;
sessionMatched?: boolean;
urlMatched?: boolean;
};
@@ -64,8 +68,14 @@ type MeetingBrowserAdapter<
buildStatusJoinScript(params: MeetingBrowserStatusScriptParams<Mode>): string;
parseStatus(result: unknown): Health | undefined;
classifyManualAction(health: Health): MeetingManualAction | undefined;
shouldRetryJoinStatus?(health: Health): boolean;
browserControlUnavailable(error: unknown): MeetingManualAction;
buildLeaveScript(meetingUrl: string): string;
buildSessionLeaveScript?(params: {
leaveInitiated: boolean;
meetingSessionId: string;
meetingUrl: string;
}): string;
parseLeaveResult(result: unknown): MeetingBrowserLeaveStep;
captions: {
enabled(mode: Mode): boolean;
@@ -79,7 +89,10 @@ type MeetingBrowserAdapter<
urlMatched?: boolean;
};
};
permissions(params: { allowMicrophone: boolean }): MeetingBrowserPermissionPlan | undefined;
permissions(params: {
allowMicrophone: boolean;
meetingUrl: string;
}): MeetingBrowserPermissionPlan | undefined;
permissionNotes(params: {
allowMicrophone: boolean;
error?: unknown;
src/meeting-bot/session-runtime-types.ts
+39
View File
@@ -0,0 +1,39 @@
import type {
MeetingBrowserHealth,
MeetingBrowserTab,
MeetingSessionRecord,
} from "./session-types.js";
export type MeetingSessionRuntimeHandles<THealth extends MeetingBrowserHealth> = {
stop?: () => Promise<void>;
speak?: (instructions?: string) => void;
getHealth?: () => Partial<THealth>;
};
export type MeetingBrowserSessionView<
THealth extends MeetingBrowserHealth,
TTab extends MeetingBrowserTab,
> = {
launched: boolean;
nodeId?: string;
tab?: TTab;
health?: THealth;
hasAudioBridge: boolean;
};
export type MeetingSessionRuntimeJoinContext<
TSession extends MeetingSessionRecord<TTransport, TMode>,
TTransport extends string,
TMode extends string,
THealth extends MeetingBrowserHealth,
TTab extends MeetingBrowserTab,
> = {
attachRuntimeHandles(session: TSession, handles: MeetingSessionRuntimeHandles<THealth>): void;
inheritedBrowserTab(params: {
session: TSession;
transport: TTransport;
nodeId?: string;
meetingUrl: string;
tab?: TTab;
}): TTab | undefined;
};
src/meeting-bot/session-runtime.test.ts
+45
View File
@@ -286,6 +286,51 @@ describe("MeetingSessionRuntime failed joins", () => {
});
describe("MeetingSessionRuntime leave cleanup", () => {
it("clears stale in-call health after confirmed browser departure", async () => {
const { runtime } = createTestRuntime({
releaseBrowserTab: async () => true,
joinTransport: async ({ session }) => {
session.browser = {
launched: true,
health: {
inCall: true,
micMuted: false,
manualActionMessage: "old action",
manualActionReason: "old-action",
manualActionRequired: true,
speechReady: true,
speechBlockedMessage: "old speech block",
speechBlockedReason: "old-speech-block",
},
tab: { targetId: "leave-tab", openedByPlugin: true },
};
return {};
},
});
const { session } = await runtime.join({
url: "https://meeting.example/room",
agentId: "main",
});
await expect(runtime.leave(session.id)).resolves.toMatchObject({
browserLeft: true,
session: {
browser: {
health: {
inCall: false,
manualActionRequired: false,
speechReady: false,
},
},
},
});
expect(session.browser?.health?.manualActionReason).toBeUndefined();
expect(session.browser?.health?.manualActionMessage).toBeUndefined();
expect(session.browser?.health?.micMuted).toBeUndefined();
expect(session.browser?.health?.speechBlockedReason).toBeUndefined();
expect(session.browser?.health?.speechBlockedMessage).toBeUndefined();
});
it("retries a failed transport stop without repeating settled browser cleanup", async () => {
const stopError = new Error("transport stop failed");
const stop = vi
src/meeting-bot/session-runtime.ts
+24 -34
View File
@@ -1,6 +1,11 @@
import type { RuntimeLogger } from "../plugins/runtime/types.js";
import { MeetingSessionCleanupTracker } from "./session-cleanup-tracker.js";
import { MeetingSessionJoinLock } from "./session-join-lock.js";
import type {
MeetingBrowserSessionView,
MeetingSessionRuntimeHandles,
MeetingSessionRuntimeJoinContext,
} from "./session-runtime-types.js";
import { MeetingSessionTranscriptStore } from "./session-transcript-store.js";
import type {
MeetingBrowserHealth,
@@ -9,40 +14,11 @@ import type {
MeetingSessionRecord,
MeetingTranscriptSnapshot,
} from "./session-types.js";
export type MeetingSessionRuntimeHandles<THealth extends MeetingBrowserHealth> = {
stop?: () => Promise<void>;
speak?: (instructions?: string) => void;
getHealth?: () => Partial<THealth>;
};
export type MeetingBrowserSessionView<
THealth extends MeetingBrowserHealth,
TTab extends MeetingBrowserTab,
> = {
launched: boolean;
nodeId?: string;
tab?: TTab;
health?: THealth;
hasAudioBridge: boolean;
};
export type MeetingSessionRuntimeJoinContext<
TSession extends MeetingSessionRecord<TTransport, TMode>,
TTransport extends string,
TMode extends string,
THealth extends MeetingBrowserHealth,
TTab extends MeetingBrowserTab,
> = {
attachRuntimeHandles(session: TSession, handles: MeetingSessionRuntimeHandles<THealth>): void;
inheritedBrowserTab(params: {
session: TSession;
transport: TTransport;
nodeId?: string;
meetingUrl: string;
tab?: TTab;
}): TTab | undefined;
};
export type {
MeetingBrowserSessionView,
MeetingSessionRuntimeHandles,
MeetingSessionRuntimeJoinContext,
} from "./session-runtime-types.js";
export type MeetingSessionRuntimeMessages<TSpeechBlockedReason extends string> = {
previousBrowserLeaveFailed: string;
@@ -550,6 +526,20 @@ export class MeetingSessionRuntime<
releaseBrowser: async () => await this.options.releaseBrowserTab(session),
});
session.browserLeft = cleanup.browserLeft;
const browser = this.options.getBrowser(session);
if (cleanup.browserLeft === true && browser?.health) {
this.options.setBrowserHealth(session, {
...browser.health,
inCall: false,
micMuted: undefined,
manualActionRequired: false,
manualActionReason: undefined,
manualActionMessage: undefined,
speechReady: false,
speechBlockedReason: undefined,
speechBlockedMessage: undefined,
} as THealth);
}
if (cleanup.stopSettled && stop && this.#sessionStops.get(session.id) === stop) {
this.#sessionStops.delete(session.id);
}