mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 10:16:44 +00:00
refactor(speech-core): share TTS request pre-resolution across discord and voice-call (#109640)
This commit is contained in:
Peter Steinberger
@@ -22,6 +22,7 @@ const {
|
||||
agentCommandMock,
|
||||
resolveRealtimeBootstrapContextInstructionsMock,
|
||||
transcribeAudioFileMock,
|
||||
prepareTtsRequestMock,
|
||||
textToSpeechStreamMock,
|
||||
textToSpeechMock,
|
||||
logVerboseMock,
|
||||
@@ -149,6 +150,15 @@ const {
|
||||
(...args: unknown[]) => Promise<string | undefined>
|
||||
>(async () => undefined),
|
||||
transcribeAudioFileMock: vi.fn(async () => ({ text: "hello from voice" })),
|
||||
prepareTtsRequestMock: vi.fn(async ({ cfg, text }: { cfg: unknown; text: string }) => ({
|
||||
cfg,
|
||||
directives: {
|
||||
cleanedText: text,
|
||||
hasDirective: false,
|
||||
overrides: {},
|
||||
warnings: [],
|
||||
},
|
||||
})),
|
||||
textToSpeechStreamMock: vi.fn(
|
||||
async (): Promise<unknown> => ({ success: false, error: "stream unavailable" }),
|
||||
),
|
||||
@@ -219,13 +229,7 @@ vi.mock("openclaw/plugin-sdk/agent-runtime", async () => {
|
||||
return {
|
||||
...actual,
|
||||
agentCommandFromIngress: agentCommandMock,
|
||||
getTtsProvider: vi.fn(() => "openai"),
|
||||
resolveAgentDir: vi.fn(() => "/tmp/openclaw-agent"),
|
||||
resolveTtsConfig: vi.fn(() => ({
|
||||
modelOverrides: {},
|
||||
providerConfigs: {},
|
||||
})),
|
||||
resolveTtsPrefsPath: vi.fn(() => "/tmp/openclaw-tts.json"),
|
||||
};
|
||||
});
|
||||
|
||||
@@ -286,6 +290,7 @@ vi.mock("../runtime.js", () => ({
|
||||
transcribeAudioFile: transcribeAudioFileMock,
|
||||
},
|
||||
tts: {
|
||||
prepareTtsRequest: prepareTtsRequestMock,
|
||||
textToSpeechStream: textToSpeechStreamMock,
|
||||
textToSpeech: textToSpeechMock,
|
||||
},
|
||||
@@ -371,6 +376,18 @@ describe("DiscordVoiceManager", () => {
|
||||
resolveRealtimeBootstrapContextInstructionsMock.mockResolvedValue(undefined);
|
||||
transcribeAudioFileMock.mockReset();
|
||||
transcribeAudioFileMock.mockResolvedValue({ text: "hello from voice" });
|
||||
prepareTtsRequestMock.mockReset();
|
||||
prepareTtsRequestMock.mockImplementation(
|
||||
async ({ cfg, text }: { cfg: unknown; text: string }) => ({
|
||||
cfg,
|
||||
directives: {
|
||||
cleanedText: text,
|
||||
hasDirective: false,
|
||||
overrides: {},
|
||||
warnings: [],
|
||||
},
|
||||
}),
|
||||
);
|
||||
textToSpeechStreamMock.mockReset();
|
||||
textToSpeechStreamMock.mockResolvedValue({ success: false, error: "stream unavailable" });
|
||||
textToSpeechMock.mockReset();
|
||||
@@ -6612,6 +6629,9 @@ describe("DiscordVoiceManager", () => {
|
||||
expect(commandArgs?.messageProvider).toBe("discord-voice");
|
||||
expect(commandArgs?.message).toContain("Do not call the tts tool");
|
||||
expect(commandArgs?.message).toContain("repair obvious transcription artifacts");
|
||||
expect(prepareTtsRequestMock).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ text: "hello back" }),
|
||||
);
|
||||
expect(lastTtsArgs().channel).toBe("discord");
|
||||
expect(lastTtsArgs().text).toBe("hello back");
|
||||
});
|
||||
|
||||
@@ -1,14 +1,7 @@
|
||||
// Discord plugin module implements tts behavior.
|
||||
import {
|
||||
getTtsProvider,
|
||||
resolveAgentDir,
|
||||
resolveTtsConfig,
|
||||
resolveTtsPrefsPath,
|
||||
type ResolvedTtsConfig,
|
||||
} from "openclaw/plugin-sdk/agent-runtime";
|
||||
import { resolveAgentDir } from "openclaw/plugin-sdk/agent-runtime";
|
||||
import type { OpenClawConfig, TtsConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { parseTtsDirectives } from "openclaw/plugin-sdk/speech";
|
||||
import { normalizeOptionalString, uniqueStrings } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { normalizeOptionalString } from "openclaw/plugin-sdk/string-coerce-runtime";
|
||||
import { getDiscordRuntime } from "../runtime.js";
|
||||
import { sanitizeVoiceReplyTextForSpeech } from "./sanitize.js";
|
||||
|
||||
@@ -34,58 +27,6 @@ type VoiceReplyAudioResult =
|
||||
error?: string;
|
||||
};
|
||||
|
||||
function mergeTtsConfig(base: TtsConfig, override?: TtsConfig): TtsConfig {
|
||||
if (!override) {
|
||||
return base;
|
||||
}
|
||||
const baseProviders = base.providers ?? {};
|
||||
const overrideProviders = override.providers ?? {};
|
||||
const mergedProviders = Object.fromEntries(
|
||||
uniqueStrings([...Object.keys(baseProviders), ...Object.keys(overrideProviders)]).map(
|
||||
(providerId) => {
|
||||
const baseProvider = baseProviders[providerId] ?? {};
|
||||
const overrideProvider = overrideProviders[providerId] ?? {};
|
||||
return [
|
||||
providerId,
|
||||
{
|
||||
...baseProvider,
|
||||
...overrideProvider,
|
||||
},
|
||||
];
|
||||
},
|
||||
),
|
||||
);
|
||||
return {
|
||||
...base,
|
||||
...override,
|
||||
modelOverrides: {
|
||||
...base.modelOverrides,
|
||||
...override.modelOverrides,
|
||||
},
|
||||
...(Object.keys(mergedProviders).length === 0 ? {} : { providers: mergedProviders }),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveVoiceTtsConfig(params: { cfg: OpenClawConfig; override?: TtsConfig }): {
|
||||
cfg: OpenClawConfig;
|
||||
resolved: ResolvedTtsConfig;
|
||||
} {
|
||||
if (!params.override) {
|
||||
return { cfg: params.cfg, resolved: resolveTtsConfig(params.cfg) };
|
||||
}
|
||||
const base = params.cfg.messages?.tts ?? {};
|
||||
const merged = mergeTtsConfig(base, params.override);
|
||||
const messages = params.cfg.messages ?? {};
|
||||
const cfg = {
|
||||
...params.cfg,
|
||||
messages: {
|
||||
...messages,
|
||||
tts: merged,
|
||||
},
|
||||
};
|
||||
return { cfg, resolved: resolveTtsConfig(cfg) };
|
||||
}
|
||||
|
||||
export async function transcribeVoiceAudio(params: {
|
||||
cfg: OpenClawConfig;
|
||||
agentId: string;
|
||||
@@ -106,25 +47,21 @@ export async function synthesizeVoiceReplyAudio(params: {
|
||||
replyText: string;
|
||||
speakerLabel: string;
|
||||
}): Promise<VoiceReplyAudioResult> {
|
||||
const { cfg: ttsCfg, resolved: ttsConfig } = resolveVoiceTtsConfig({
|
||||
const runtime = getDiscordRuntime();
|
||||
const prepared = await runtime.tts.prepareTtsRequest({
|
||||
cfg: params.cfg,
|
||||
override: params.override,
|
||||
text: params.replyText,
|
||||
});
|
||||
const directive = parseTtsDirectives(params.replyText, ttsConfig.modelOverrides, {
|
||||
cfg: ttsCfg,
|
||||
providerConfigs: ttsConfig.providerConfigs,
|
||||
preferredProviderId: getTtsProvider(ttsConfig, resolveTtsPrefsPath(ttsConfig)),
|
||||
});
|
||||
const directive = prepared.directives;
|
||||
const rawSpeakText = directive.overrides.ttsText ?? directive.cleanedText.trim();
|
||||
const speakText = sanitizeVoiceReplyTextForSpeech(rawSpeakText, params.speakerLabel);
|
||||
if (!speakText) {
|
||||
return { status: "empty" };
|
||||
}
|
||||
|
||||
const runtime = getDiscordRuntime();
|
||||
const streamResult = await runtime.tts.textToSpeechStream?.({
|
||||
text: speakText,
|
||||
cfg: ttsCfg,
|
||||
cfg: prepared.cfg,
|
||||
channel: "discord",
|
||||
overrides: directive.overrides,
|
||||
disableFallback: true,
|
||||
@@ -141,7 +78,7 @@ export async function synthesizeVoiceReplyAudio(params: {
|
||||
|
||||
const result = await runtime.tts.textToSpeech({
|
||||
text: speakText,
|
||||
cfg: ttsCfg,
|
||||
cfg: prepared.cfg,
|
||||
channel: "discord",
|
||||
overrides: directive.overrides,
|
||||
});
|
||||
|
||||
@@ -521,8 +521,8 @@ export async function createVoiceCallRuntime(params: {
|
||||
const twilioProvider = provider as TwilioProvider;
|
||||
if (ttsRuntime?.textToSpeechTelephony) {
|
||||
try {
|
||||
const ttsProvider = createTelephonyTtsProvider({
|
||||
coreConfig,
|
||||
const ttsProvider = await createTelephonyTtsProvider({
|
||||
coreConfig: cfg,
|
||||
ttsOverride: config.tts,
|
||||
runtime: ttsRuntime,
|
||||
logger: log,
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
// Voice Call tests cover telephony tts plugin behavior.
|
||||
import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { VoiceCallTtsConfig } from "./config.js";
|
||||
import type { CoreConfig } from "./core-bridge.js";
|
||||
import { createTelephonyTtsProvider } from "./telephony-tts.js";
|
||||
import { createTelephonyTtsProvider, type TelephonyTtsRuntime } from "./telephony-tts.js";
|
||||
|
||||
function createCoreConfig(): CoreConfig {
|
||||
function createCoreConfig(): OpenClawConfig {
|
||||
const tts: VoiceCallTtsConfig = {
|
||||
provider: "openai",
|
||||
providers: {
|
||||
@@ -18,98 +18,82 @@ function createCoreConfig(): CoreConfig {
|
||||
return { messages: { tts } };
|
||||
}
|
||||
|
||||
function requireMergedTtsConfig(mergedConfig: CoreConfig | undefined) {
|
||||
const tts = mergedConfig?.messages?.tts;
|
||||
if (!tts) {
|
||||
throw new Error("telephony TTS runtime did not receive merged TTS config");
|
||||
}
|
||||
return tts as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function requireOpenAIProviderConfig(tts: Record<string, unknown>): Record<string, unknown> {
|
||||
const providers =
|
||||
tts.providers && typeof tts.providers === "object"
|
||||
? (tts.providers as Record<string, unknown>)
|
||||
: null;
|
||||
const openai = providers?.openai;
|
||||
if (!openai || typeof openai !== "object") {
|
||||
throw new Error("merged TTS config did not preserve providers.openai");
|
||||
}
|
||||
return openai as Record<string, unknown>;
|
||||
}
|
||||
|
||||
async function mergeOverride(override: unknown): Promise<Record<string, unknown>> {
|
||||
let mergedConfig: CoreConfig | undefined;
|
||||
const provider = createTelephonyTtsProvider({
|
||||
coreConfig: createCoreConfig(),
|
||||
ttsOverride: override as VoiceCallTtsConfig,
|
||||
runtime: {
|
||||
textToSpeechTelephony: async ({ cfg }) => {
|
||||
mergedConfig = cfg;
|
||||
return {
|
||||
success: true,
|
||||
audioBuffer: Buffer.alloc(2),
|
||||
sampleRate: 8000,
|
||||
};
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await provider.synthesizeForTelephony("hello");
|
||||
return requireMergedTtsConfig(mergedConfig);
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
delete (Object.prototype as Record<string, unknown>).polluted;
|
||||
const passthroughPreparation: TelephonyTtsRuntime["prepareTtsRequest"] = async ({ cfg, text }) => ({
|
||||
cfg,
|
||||
directives: {
|
||||
cleanedText: text,
|
||||
hasDirective: false,
|
||||
overrides: {},
|
||||
warnings: [],
|
||||
},
|
||||
});
|
||||
|
||||
describe("createTelephonyTtsProvider deepMerge hardening", () => {
|
||||
it("merges safe nested overrides", async () => {
|
||||
const tts = await mergeOverride({
|
||||
providers: { openai: { voice: "coral" } },
|
||||
function createRuntime(
|
||||
textToSpeechTelephony: TelephonyTtsRuntime["textToSpeechTelephony"],
|
||||
prepareTtsRequest: TelephonyTtsRuntime["prepareTtsRequest"] = passthroughPreparation,
|
||||
): TelephonyTtsRuntime {
|
||||
return { prepareTtsRequest, textToSpeechTelephony };
|
||||
}
|
||||
|
||||
describe("createTelephonyTtsProvider", () => {
|
||||
it("uses shared preparation for the surface override and request text", async () => {
|
||||
const effectiveConfig: OpenClawConfig = {
|
||||
messages: { tts: { provider: "openai", timeoutMs: 15_000 } },
|
||||
};
|
||||
const prepareTtsRequest = vi.fn<TelephonyTtsRuntime["prepareTtsRequest"]>(
|
||||
async ({ cfg, override, text }) => ({
|
||||
cfg: override ? effectiveConfig : cfg,
|
||||
directives: {
|
||||
cleanedText: text,
|
||||
hasDirective: false,
|
||||
overrides: {},
|
||||
warnings: [],
|
||||
},
|
||||
}),
|
||||
);
|
||||
const textToSpeechTelephony = vi.fn(async () => ({
|
||||
success: true,
|
||||
audioBuffer: Buffer.alloc(2),
|
||||
sampleRate: 8000,
|
||||
}));
|
||||
const override: VoiceCallTtsConfig = { timeoutMs: 15_000 };
|
||||
const provider = await createTelephonyTtsProvider({
|
||||
coreConfig: createCoreConfig(),
|
||||
ttsOverride: override,
|
||||
runtime: createRuntime(textToSpeechTelephony, prepareTtsRequest),
|
||||
});
|
||||
const openai = requireOpenAIProviderConfig(tts);
|
||||
|
||||
expect(openai.voice).toBe("coral");
|
||||
expect(openai.model).toBe("gpt-4o-mini-tts");
|
||||
});
|
||||
await provider.synthesizeForTelephony("hello");
|
||||
|
||||
it("blocks top-level __proto__ keys", async () => {
|
||||
const tts = await mergeOverride(
|
||||
JSON.parse('{"__proto__":{"polluted":"top"},"providers":{"openai":{"voice":"coral"}}}'),
|
||||
);
|
||||
const openai = requireOpenAIProviderConfig(tts);
|
||||
|
||||
expect((Object.prototype as Record<string, unknown>).polluted).toBeUndefined();
|
||||
expect(tts.polluted).toBeUndefined();
|
||||
expect(openai.voice).toBe("coral");
|
||||
});
|
||||
|
||||
it("blocks nested __proto__ keys", async () => {
|
||||
const tts = await mergeOverride(
|
||||
JSON.parse('{"providers":{"openai":{"model":"safe","__proto__":{"polluted":"nested"}}}}'),
|
||||
);
|
||||
const openai = requireOpenAIProviderConfig(tts);
|
||||
|
||||
expect((Object.prototype as Record<string, unknown>).polluted).toBeUndefined();
|
||||
expect(openai.polluted).toBeUndefined();
|
||||
expect(openai.model).toBe("safe");
|
||||
expect(provider.synthesisTimeoutMs).toBe(15_000);
|
||||
expect(prepareTtsRequest).toHaveBeenNthCalledWith(1, {
|
||||
cfg: createCoreConfig(),
|
||||
override,
|
||||
text: "",
|
||||
});
|
||||
expect(prepareTtsRequest).toHaveBeenNthCalledWith(2, {
|
||||
cfg: effectiveConfig,
|
||||
text: "hello",
|
||||
});
|
||||
expect(textToSpeechTelephony).toHaveBeenCalledWith({
|
||||
text: "hello",
|
||||
cfg: effectiveConfig,
|
||||
overrides: {},
|
||||
});
|
||||
});
|
||||
|
||||
it("logs fallback metadata when telephony TTS uses a fallback provider", async () => {
|
||||
const warn = vi.fn();
|
||||
const provider = createTelephonyTtsProvider({
|
||||
const provider = await createTelephonyTtsProvider({
|
||||
coreConfig: createCoreConfig(),
|
||||
runtime: {
|
||||
textToSpeechTelephony: async () => ({
|
||||
success: true,
|
||||
audioBuffer: Buffer.alloc(2),
|
||||
sampleRate: 8000,
|
||||
provider: "microsoft",
|
||||
fallbackFrom: "elevenlabs",
|
||||
attemptedProviders: ["elevenlabs", "microsoft"],
|
||||
}),
|
||||
},
|
||||
runtime: createRuntime(async () => ({
|
||||
success: true,
|
||||
audioBuffer: Buffer.alloc(2),
|
||||
sampleRate: 8000,
|
||||
provider: "microsoft",
|
||||
fallbackFrom: "elevenlabs",
|
||||
attemptedProviders: ["elevenlabs", "microsoft"],
|
||||
})),
|
||||
logger: { warn },
|
||||
});
|
||||
|
||||
@@ -119,95 +103,100 @@ describe("createTelephonyTtsProvider deepMerge hardening", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("strips telephony TTS directive tags before synthesis", async () => {
|
||||
let requestText: string | undefined;
|
||||
const provider = createTelephonyTtsProvider({
|
||||
it("uses prepared directive-stripped text for synthesis", async () => {
|
||||
const textToSpeechTelephony = vi.fn(async () => ({
|
||||
success: true,
|
||||
audioBuffer: Buffer.alloc(2),
|
||||
sampleRate: 8000,
|
||||
}));
|
||||
const provider = await createTelephonyTtsProvider({
|
||||
coreConfig: createCoreConfig(),
|
||||
runtime: {
|
||||
textToSpeechTelephony: async ({ text }) => {
|
||||
requestText = text;
|
||||
return {
|
||||
success: true,
|
||||
audioBuffer: Buffer.alloc(2),
|
||||
sampleRate: 8000,
|
||||
};
|
||||
runtime: createRuntime(textToSpeechTelephony, async ({ cfg, text }) => ({
|
||||
cfg,
|
||||
directives: {
|
||||
cleanedText: text ? "Hello caller" : "",
|
||||
hasDirective: text.length > 0,
|
||||
overrides: {},
|
||||
warnings: [],
|
||||
},
|
||||
},
|
||||
})),
|
||||
});
|
||||
|
||||
await provider.synthesizeForTelephony("[[tts]]Hello caller[[/tts]]");
|
||||
|
||||
expect(requestText).toBe("Hello caller");
|
||||
expect(textToSpeechTelephony).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ text: "Hello caller" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("uses hidden telephony TTS directive text for synthesis", async () => {
|
||||
let requestText: string | undefined;
|
||||
let requestOverrides: unknown;
|
||||
const provider = createTelephonyTtsProvider({
|
||||
it("uses prepared hidden directive text and overrides for synthesis", async () => {
|
||||
const textToSpeechTelephony = vi.fn(async () => ({
|
||||
success: true,
|
||||
audioBuffer: Buffer.alloc(2),
|
||||
sampleRate: 8000,
|
||||
}));
|
||||
const provider = await createTelephonyTtsProvider({
|
||||
coreConfig: createCoreConfig(),
|
||||
runtime: {
|
||||
textToSpeechTelephony: async ({ text, overrides }) => {
|
||||
requestText = text;
|
||||
requestOverrides = overrides;
|
||||
return {
|
||||
success: true,
|
||||
audioBuffer: Buffer.alloc(2),
|
||||
sampleRate: 8000,
|
||||
};
|
||||
runtime: createRuntime(textToSpeechTelephony, async ({ cfg, text }) => ({
|
||||
cfg,
|
||||
directives: {
|
||||
cleanedText: text ? "Visible text " : "",
|
||||
ttsText: text ? "Speak this instead" : undefined,
|
||||
hasDirective: text.length > 0,
|
||||
overrides: text ? { ttsText: "Speak this instead" } : {},
|
||||
warnings: [],
|
||||
},
|
||||
},
|
||||
})),
|
||||
});
|
||||
|
||||
await provider.synthesizeForTelephony(
|
||||
"Visible text [[tts:text]]Speak this instead[[/tts:text]]",
|
||||
);
|
||||
|
||||
expect(requestText).toBe("Speak this instead");
|
||||
expect(requestOverrides).toStrictEqual({ ttsText: "Speak this instead" });
|
||||
expect(textToSpeechTelephony).toHaveBeenCalledWith(
|
||||
expect.objectContaining({
|
||||
text: "Speak this instead",
|
||||
overrides: { ttsText: "Speak this instead" },
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("exposes configured timeoutMs as synthesisTimeoutMs", () => {
|
||||
const provider = createTelephonyTtsProvider({
|
||||
it("exposes configured timeoutMs as synthesisTimeoutMs", async () => {
|
||||
const provider = await createTelephonyTtsProvider({
|
||||
coreConfig: { messages: { tts: { provider: "openai", timeoutMs: 15000 } } },
|
||||
runtime: {
|
||||
textToSpeechTelephony: async () => ({
|
||||
success: true,
|
||||
audioBuffer: Buffer.alloc(2),
|
||||
sampleRate: 8000,
|
||||
}),
|
||||
},
|
||||
runtime: createRuntime(async () => ({
|
||||
success: true,
|
||||
audioBuffer: Buffer.alloc(2),
|
||||
sampleRate: 8000,
|
||||
})),
|
||||
});
|
||||
|
||||
expect(provider.synthesisTimeoutMs).toBe(15000);
|
||||
});
|
||||
|
||||
it("clamps oversized configured timeoutMs", () => {
|
||||
const provider = createTelephonyTtsProvider({
|
||||
it("clamps oversized configured timeoutMs", async () => {
|
||||
const provider = await createTelephonyTtsProvider({
|
||||
coreConfig: {
|
||||
messages: { tts: { provider: "openai", timeoutMs: Number.MAX_SAFE_INTEGER } },
|
||||
},
|
||||
runtime: {
|
||||
textToSpeechTelephony: async () => ({
|
||||
success: true,
|
||||
audioBuffer: Buffer.alloc(2),
|
||||
sampleRate: 8000,
|
||||
}),
|
||||
},
|
||||
runtime: createRuntime(async () => ({
|
||||
success: true,
|
||||
audioBuffer: Buffer.alloc(2),
|
||||
sampleRate: 8000,
|
||||
})),
|
||||
});
|
||||
|
||||
expect(provider.synthesisTimeoutMs).toBe(MAX_TIMER_TIMEOUT_MS);
|
||||
});
|
||||
|
||||
it("keeps the telephony timeout default when timeoutMs is not configured", () => {
|
||||
const provider = createTelephonyTtsProvider({
|
||||
it("keeps the telephony timeout default when timeoutMs is not configured", async () => {
|
||||
const provider = await createTelephonyTtsProvider({
|
||||
coreConfig: createCoreConfig(),
|
||||
runtime: {
|
||||
textToSpeechTelephony: async () => ({
|
||||
success: true,
|
||||
audioBuffer: Buffer.alloc(2),
|
||||
sampleRate: 8000,
|
||||
}),
|
||||
},
|
||||
runtime: createRuntime(async () => ({
|
||||
success: true,
|
||||
audioBuffer: Buffer.alloc(2),
|
||||
sampleRate: 8000,
|
||||
})),
|
||||
});
|
||||
|
||||
expect(provider.synthesisTimeoutMs).toBe(8000);
|
||||
|
||||
@@ -1,23 +1,25 @@
|
||||
// Voice Call plugin module implements telephony tts behavior.
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { resolveTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime";
|
||||
import { mergeDeep } from "openclaw/plugin-sdk/plugin-config-runtime";
|
||||
import {
|
||||
parseTtsDirectives,
|
||||
type SpeechModelOverridePolicy,
|
||||
type SpeechProviderConfig,
|
||||
type TtsDirectiveOverrides,
|
||||
} from "openclaw/plugin-sdk/speech";
|
||||
import type { TtsDirectiveOverrides, TtsDirectiveParseResult } from "openclaw/plugin-sdk/speech";
|
||||
import type { VoiceCallTtsConfig } from "./config.js";
|
||||
import type { CoreConfig } from "./core-bridge.js";
|
||||
import { convertPcmToMulaw8k } from "./telephony-audio.js";
|
||||
|
||||
// Telephony TTS adapter that applies voice-call overrides and emits 8kHz mulaw audio.
|
||||
|
||||
/** Core runtime TTS API used by the telephony adapter. */
|
||||
export type TelephonyTtsRuntime = {
|
||||
prepareTtsRequest: (params: {
|
||||
cfg: OpenClawConfig;
|
||||
override?: VoiceCallTtsConfig;
|
||||
text: string;
|
||||
}) => Promise<{
|
||||
cfg: OpenClawConfig;
|
||||
directives: TtsDirectiveParseResult;
|
||||
}>;
|
||||
textToSpeechTelephony: (params: {
|
||||
text: string;
|
||||
cfg: CoreConfig;
|
||||
cfg: OpenClawConfig;
|
||||
prefsPath?: string;
|
||||
overrides?: TtsDirectiveOverrides;
|
||||
}) => Promise<{
|
||||
@@ -40,48 +42,34 @@ export type TelephonyTtsProvider = {
|
||||
/** Default timeout for one telephony synthesis request. */
|
||||
export const TELEPHONY_DEFAULT_TTS_TIMEOUT_MS = 8000;
|
||||
|
||||
/** Voice-call override policy for inline TTS model directives. */
|
||||
type TelephonyModelOverrideConfig = {
|
||||
enabled?: boolean;
|
||||
allowText?: boolean;
|
||||
allowProvider?: boolean;
|
||||
allowVoice?: boolean;
|
||||
allowModelId?: boolean;
|
||||
allowVoiceSettings?: boolean;
|
||||
allowNormalization?: boolean;
|
||||
allowSeed?: boolean;
|
||||
};
|
||||
|
||||
/** Create a TTS provider that honors voice-call overrides and converts PCM to mulaw. */
|
||||
export function createTelephonyTtsProvider(params: {
|
||||
coreConfig: CoreConfig;
|
||||
export async function createTelephonyTtsProvider(params: {
|
||||
coreConfig: OpenClawConfig;
|
||||
ttsOverride?: VoiceCallTtsConfig;
|
||||
runtime: TelephonyTtsRuntime;
|
||||
logger?: {
|
||||
warn?: (message: string) => void;
|
||||
};
|
||||
}): TelephonyTtsProvider {
|
||||
}): Promise<TelephonyTtsProvider> {
|
||||
const { coreConfig, ttsOverride, runtime, logger } = params;
|
||||
const mergedConfig = applyTtsOverride(coreConfig, ttsOverride);
|
||||
const ttsConfig = mergedConfig.messages?.tts;
|
||||
const modelOverrides = resolveTelephonyModelOverridePolicy(
|
||||
readTelephonyModelOverrides(ttsConfig),
|
||||
);
|
||||
const providerConfigs = collectTelephonyProviderConfigs(ttsConfig);
|
||||
const activeProvider = normalizeProviderId(ttsConfig?.provider);
|
||||
const preparedConfig = await runtime.prepareTtsRequest({
|
||||
cfg: coreConfig,
|
||||
override: ttsOverride,
|
||||
text: "",
|
||||
});
|
||||
const synthesisTimeoutMs = resolveTimerTimeoutMs(
|
||||
mergedConfig.messages?.tts?.timeoutMs,
|
||||
preparedConfig.cfg.messages?.tts?.timeoutMs,
|
||||
TELEPHONY_DEFAULT_TTS_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
return {
|
||||
synthesisTimeoutMs,
|
||||
synthesizeForTelephony: async (text: string) => {
|
||||
const directives = parseTtsDirectives(text, modelOverrides, {
|
||||
cfg: mergedConfig,
|
||||
providerConfigs,
|
||||
preferredProviderId: activeProvider,
|
||||
const prepared = await runtime.prepareTtsRequest({
|
||||
cfg: preparedConfig.cfg,
|
||||
text,
|
||||
});
|
||||
const directives = prepared.directives;
|
||||
if (directives.warnings.length > 0) {
|
||||
logger?.warn?.(
|
||||
`[voice-call] Ignored telephony TTS directive overrides (${directives.warnings.join("; ")})`,
|
||||
@@ -92,7 +80,7 @@ export function createTelephonyTtsProvider(params: {
|
||||
: text;
|
||||
const result = await runtime.textToSpeechTelephony({
|
||||
text: cleanText,
|
||||
cfg: mergedConfig,
|
||||
cfg: prepared.cfg,
|
||||
overrides: directives.overrides,
|
||||
});
|
||||
|
||||
@@ -114,140 +102,3 @@ export function createTelephonyTtsProvider(params: {
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Apply voice-call TTS overrides to core config without mutating the original object. */
|
||||
function applyTtsOverride(coreConfig: CoreConfig, override?: VoiceCallTtsConfig): CoreConfig {
|
||||
if (!override) {
|
||||
return coreConfig;
|
||||
}
|
||||
|
||||
const base = coreConfig.messages?.tts;
|
||||
const merged = mergeTtsConfig(base, override);
|
||||
if (!merged) {
|
||||
return coreConfig;
|
||||
}
|
||||
|
||||
return {
|
||||
...coreConfig,
|
||||
messages: {
|
||||
...coreConfig.messages,
|
||||
tts: merged,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Merge core and voice-call TTS config, keeping undefined override fields out. */
|
||||
function mergeTtsConfig(
|
||||
base?: VoiceCallTtsConfig,
|
||||
override?: VoiceCallTtsConfig,
|
||||
): VoiceCallTtsConfig | undefined {
|
||||
if (!base && !override) {
|
||||
return undefined;
|
||||
}
|
||||
if (!override) {
|
||||
return base;
|
||||
}
|
||||
if (!base) {
|
||||
return override;
|
||||
}
|
||||
return mergeDeep(base, override) as VoiceCallTtsConfig;
|
||||
}
|
||||
|
||||
/** Resolve directive override policy for telephony synthesis. */
|
||||
function resolveTelephonyModelOverridePolicy(
|
||||
overrides: TelephonyModelOverrideConfig | undefined,
|
||||
): SpeechModelOverridePolicy {
|
||||
const enabled = overrides?.enabled ?? true;
|
||||
if (!enabled) {
|
||||
return {
|
||||
enabled: false,
|
||||
allowText: false,
|
||||
allowProvider: false,
|
||||
allowVoice: false,
|
||||
allowModelId: false,
|
||||
allowVoiceSettings: false,
|
||||
allowNormalization: false,
|
||||
allowSeed: false,
|
||||
};
|
||||
}
|
||||
const allow = (value: boolean | undefined, defaultValue = true) => value ?? defaultValue;
|
||||
return {
|
||||
enabled: true,
|
||||
allowText: allow(overrides?.allowText),
|
||||
allowProvider: allow(overrides?.allowProvider, false),
|
||||
allowVoice: allow(overrides?.allowVoice),
|
||||
allowModelId: allow(overrides?.allowModelId),
|
||||
allowVoiceSettings: allow(overrides?.allowVoiceSettings),
|
||||
allowNormalization: allow(overrides?.allowNormalization),
|
||||
allowSeed: allow(overrides?.allowSeed),
|
||||
};
|
||||
}
|
||||
|
||||
/** Read model override policy from TTS config when present. */
|
||||
function readTelephonyModelOverrides(
|
||||
ttsConfig: VoiceCallTtsConfig | undefined,
|
||||
): TelephonyModelOverrideConfig | undefined {
|
||||
const value = (ttsConfig as Record<string, unknown> | undefined)?.modelOverrides;
|
||||
return value && typeof value === "object" && !Array.isArray(value)
|
||||
? (value as TelephonyModelOverrideConfig)
|
||||
: undefined;
|
||||
}
|
||||
|
||||
/** Normalize provider ids for config lookup. */
|
||||
function normalizeProviderId(value: unknown): string | undefined {
|
||||
return typeof value === "string" ? value.trim().toLowerCase() || undefined : undefined;
|
||||
}
|
||||
|
||||
/** Coerce provider config objects while rejecting arrays and primitives. */
|
||||
function asProviderConfig(value: unknown): SpeechProviderConfig {
|
||||
return value && typeof value === "object" && !Array.isArray(value)
|
||||
? (value as SpeechProviderConfig)
|
||||
: {};
|
||||
}
|
||||
|
||||
/** Collect named provider configs from canonical and legacy TTS config shapes. */
|
||||
function collectTelephonyProviderConfigs(
|
||||
ttsConfig: VoiceCallTtsConfig | undefined,
|
||||
): Record<string, SpeechProviderConfig> {
|
||||
if (!ttsConfig) {
|
||||
return {};
|
||||
}
|
||||
const entries: Record<string, SpeechProviderConfig> = {};
|
||||
const rawProviders =
|
||||
ttsConfig.providers &&
|
||||
typeof ttsConfig.providers === "object" &&
|
||||
!Array.isArray(ttsConfig.providers)
|
||||
? (ttsConfig.providers as Record<string, unknown>)
|
||||
: {};
|
||||
for (const [providerId, value] of Object.entries(rawProviders)) {
|
||||
const normalized = normalizeProviderId(providerId) ?? providerId;
|
||||
entries[normalized] = asProviderConfig(value);
|
||||
}
|
||||
const reservedKeys = new Set([
|
||||
"auto",
|
||||
"enabled",
|
||||
"maxTextLength",
|
||||
"mode",
|
||||
"modelOverrides",
|
||||
"persona",
|
||||
"personas",
|
||||
"prefsPath",
|
||||
"provider",
|
||||
"providers",
|
||||
"summaryModel",
|
||||
"timeoutMs",
|
||||
]);
|
||||
for (const [key, value] of Object.entries(ttsConfig as Record<string, unknown>)) {
|
||||
if (
|
||||
reservedKeys.has(key) ||
|
||||
typeof value !== "object" ||
|
||||
value === null ||
|
||||
Array.isArray(value)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
const normalized = normalizeProviderId(key) ?? key;
|
||||
entries[normalized] ??= asProviderConfig(value);
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@ export {
|
||||
isTtsProviderConfigured,
|
||||
listSpeechVoices,
|
||||
maybeApplyTtsToPayload,
|
||||
prepareTtsRequest,
|
||||
resolveExplicitTtsOverrides,
|
||||
resolveTtsProviderOrder,
|
||||
setLastTtsAttempt,
|
||||
@@ -41,6 +42,7 @@ export {
|
||||
testApi,
|
||||
type TtsDirectiveOverrides,
|
||||
type TtsDirectiveParseResult,
|
||||
type PreparedTtsRequest,
|
||||
type TtsResult,
|
||||
type TtsSynthesisResult,
|
||||
type TtsSynthesisStreamResult,
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
import { realpathSync, rmSync } from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import type { OpenClawConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import type { OpenClawConfig, TtsConfig } from "openclaw/plugin-sdk/config-contracts";
|
||||
import { MAX_TIMER_TIMEOUT_MS } from "openclaw/plugin-sdk/number-runtime";
|
||||
import type { ReplyPayload } from "openclaw/plugin-sdk/reply-payload";
|
||||
import {
|
||||
@@ -117,6 +117,7 @@ const {
|
||||
getTtsProvider,
|
||||
listSpeechVoices,
|
||||
maybeApplyTtsToPayload,
|
||||
prepareTtsRequest,
|
||||
resolveTtsConfig,
|
||||
setSummarizationEnabled,
|
||||
setTtsMaxLength,
|
||||
@@ -240,6 +241,7 @@ async function expectTtsPayloadResult(params: {
|
||||
describe("speech-core native voice-note routing", () => {
|
||||
afterEach(() => {
|
||||
clearRuntimeConfigSnapshot();
|
||||
delete (Object.prototype as Record<string, unknown>).polluted;
|
||||
synthesizeMock.mockClear();
|
||||
prepareSynthesisMock.mockClear();
|
||||
transcodeAudioBufferMock.mockClear();
|
||||
@@ -267,6 +269,84 @@ describe("speech-core native voice-note routing", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("prepares deep-merged surface config and directive inputs", () => {
|
||||
const cfg: OpenClawConfig = {
|
||||
messages: {
|
||||
tts: {
|
||||
provider: "mock",
|
||||
modelOverrides: { allowProvider: false },
|
||||
providers: {
|
||||
mock: {
|
||||
model: "base-model",
|
||||
voiceSettings: { stability: 0.4 },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const prepared = prepareTtsRequest({
|
||||
cfg,
|
||||
override: {
|
||||
modelOverrides: { allowProvider: true },
|
||||
providers: {
|
||||
mock: {
|
||||
voice: "surface-voice",
|
||||
voiceSettings: { speed: 1.1 },
|
||||
},
|
||||
},
|
||||
},
|
||||
text: "Hello [[tts:text]]Speak this instead[[/tts:text]] caller",
|
||||
});
|
||||
|
||||
expect(prepared.cfg).not.toBe(cfg);
|
||||
expect(prepared.cfg.messages?.tts?.providers?.mock).toEqual({
|
||||
model: "base-model",
|
||||
voice: "surface-voice",
|
||||
voiceSettings: { stability: 0.4, speed: 1.1 },
|
||||
});
|
||||
expect(prepared.cfg.messages?.tts?.modelOverrides?.allowProvider).toBe(true);
|
||||
expect(prepared.directives).toEqual({
|
||||
cleanedText: "Hello caller",
|
||||
hasDirective: true,
|
||||
overrides: {
|
||||
ttsText: "Speak this instead",
|
||||
},
|
||||
ttsText: "Speak this instead",
|
||||
warnings: [],
|
||||
});
|
||||
expect(cfg.messages?.tts?.providers?.mock).toEqual({
|
||||
model: "base-model",
|
||||
voiceSettings: { stability: 0.4 },
|
||||
});
|
||||
});
|
||||
|
||||
it("sanitizes blocked override keys while preparing TTS config", () => {
|
||||
const prepared = prepareTtsRequest({
|
||||
cfg: {
|
||||
messages: {
|
||||
tts: {
|
||||
provider: "mock",
|
||||
providers: { mock: { model: "base-model" } },
|
||||
},
|
||||
},
|
||||
},
|
||||
override: JSON.parse(
|
||||
'{"__proto__":{"polluted":"top"},"providers":{"mock":{"voice":"safe","__proto__":{"polluted":"nested"}}}}',
|
||||
) as TtsConfig,
|
||||
text: "[[tts:text]]Speak this instead[[/tts:text]]",
|
||||
});
|
||||
|
||||
expect((Object.prototype as Record<string, unknown>).polluted).toBeUndefined();
|
||||
expect(prepared.cfg.messages?.tts).not.toHaveProperty("polluted");
|
||||
expect(prepared.cfg.messages?.tts?.providers?.mock).toEqual({
|
||||
model: "base-model",
|
||||
voice: "safe",
|
||||
});
|
||||
expect(prepared.directives.cleanedText).toBe("");
|
||||
expect(prepared.directives.ttsText).toBe("Speak this instead");
|
||||
});
|
||||
|
||||
it("marks Discord auto TTS replies as native voice messages", async () => {
|
||||
await expectTtsPayloadResult({
|
||||
channel: "discord",
|
||||
|
||||
@@ -10,6 +10,7 @@ import { formatErrorMessage } from "openclaw/plugin-sdk/error-runtime";
|
||||
import { redactSensitiveText } from "openclaw/plugin-sdk/logging-core";
|
||||
import { transcodeAudioBuffer } from "openclaw/plugin-sdk/media-runtime";
|
||||
import { clampTimerTimeoutMs } from "openclaw/plugin-sdk/number-runtime";
|
||||
import { mergeDeep } from "openclaw/plugin-sdk/plugin-config-runtime";
|
||||
import {
|
||||
markReplyPayloadAsTtsSupplement,
|
||||
resolveSendableOutboundReplyParts,
|
||||
@@ -467,6 +468,35 @@ export function getTtsProvider(config: ResolvedTtsConfig, prefsPath: string): Tt
|
||||
return config.provider;
|
||||
}
|
||||
|
||||
export type PreparedTtsRequest = {
|
||||
cfg: OpenClawConfig;
|
||||
directives: TtsDirectiveParseResult;
|
||||
};
|
||||
|
||||
/** Merge a surface TTS override and resolve its inline synthesis directives. */
|
||||
export function prepareTtsRequest(params: {
|
||||
cfg: OpenClawConfig;
|
||||
override?: TtsConfig;
|
||||
text: string;
|
||||
}): PreparedTtsRequest {
|
||||
const cfg = params.override
|
||||
? {
|
||||
...params.cfg,
|
||||
messages: {
|
||||
...params.cfg.messages,
|
||||
tts: mergeDeep(params.cfg.messages?.tts ?? {}, params.override) as TtsConfig,
|
||||
},
|
||||
}
|
||||
: params.cfg;
|
||||
const config = resolveTtsConfig(cfg);
|
||||
const directives = parseTtsDirectives(params.text, config.modelOverrides, {
|
||||
cfg,
|
||||
providerConfigs: config.providerConfigs,
|
||||
preferredProviderId: getTtsProvider(config, resolveTtsPrefsPath(config)),
|
||||
});
|
||||
return { cfg, directives };
|
||||
}
|
||||
|
||||
export function resolveExplicitTtsOverrides(params: {
|
||||
cfg: OpenClawConfig;
|
||||
prefsPath?: string;
|
||||
|
||||
@@ -534,6 +534,7 @@ export function createPluginRuntimeMock(overrides: DeepPartial<PluginRuntime> =
|
||||
resizeToJpeg: vi.fn() as unknown as PluginRuntime["media"]["resizeToJpeg"],
|
||||
},
|
||||
tts: {
|
||||
prepareTtsRequest: vi.fn() as unknown as PluginRuntime["tts"]["prepareTtsRequest"],
|
||||
textToSpeech: vi.fn() as unknown as PluginRuntime["tts"]["textToSpeech"],
|
||||
textToSpeechStream: vi.fn() as unknown as PluginRuntime["tts"]["textToSpeechStream"],
|
||||
textToSpeechTelephony: vi.fn() as unknown as PluginRuntime["tts"]["textToSpeechTelephony"],
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
// TTS runtime types define plugin-facing text-to-speech synthesis hooks and results.
|
||||
import type { OpenClawConfig } from "../config/types.openclaw.js";
|
||||
import type { ResolvedTtsPersona, TtsAutoMode, TtsProvider } from "../config/types.tts.js";
|
||||
import type {
|
||||
ResolvedTtsPersona,
|
||||
TtsAutoMode,
|
||||
TtsConfig,
|
||||
TtsProvider,
|
||||
} from "../config/types.tts.js";
|
||||
import type {
|
||||
SpeechProviderConfig,
|
||||
SpeechVoiceOption,
|
||||
@@ -94,6 +99,22 @@ export type TtsRequestParams = {
|
||||
accountId?: string;
|
||||
};
|
||||
|
||||
/** Inputs for surface-specific config merge and directive pre-resolution. */
|
||||
export type PrepareTtsRequestParams = {
|
||||
cfg: OpenClawConfig;
|
||||
override?: TtsConfig;
|
||||
text: string;
|
||||
};
|
||||
|
||||
/** Effective synthesis inputs returned before choosing file, stream, or telephony output. */
|
||||
export type PreparedTtsRequest = {
|
||||
cfg: OpenClawConfig;
|
||||
directives: TtsDirectiveParseResult;
|
||||
};
|
||||
|
||||
/** Shared surface-specific TTS request preparation contract. */
|
||||
export type PrepareTtsRequest = (params: PrepareTtsRequestParams) => Promise<PreparedTtsRequest>;
|
||||
|
||||
/** Telephony-specific synthesis request where output format is constrained by the caller. */
|
||||
export type TtsTelephonyRequestParams = {
|
||||
text: string;
|
||||
|
||||
@@ -37,6 +37,7 @@ import { createRuntimeTasks } from "./runtime-tasks.js";
|
||||
import type { CreatePluginRuntimeOptions, PluginRuntime } from "./types.js";
|
||||
|
||||
const loadTtsRuntime = createLazyRuntimeModule(() => import("../../plugin-sdk/tts-runtime.js"));
|
||||
const loadTtsRequestRuntime = createLazyRuntimeModule(() => import("./runtime-tts-request.js"));
|
||||
const loadMediaUnderstandingRuntime = createLazyRuntimeModule(
|
||||
() => import("../../media-understanding/runtime.js"),
|
||||
);
|
||||
@@ -62,7 +63,9 @@ function createRuntimeGateway(): PluginRuntime["gateway"] {
|
||||
|
||||
function createRuntimeTts(): PluginRuntime["tts"] {
|
||||
const bindTtsRuntime = createLazyRuntimeMethodBinder(loadTtsRuntime);
|
||||
const bindTtsRequestRuntime = createLazyRuntimeMethodBinder(loadTtsRequestRuntime);
|
||||
return {
|
||||
prepareTtsRequest: bindTtsRequestRuntime((runtime) => runtime.prepareTtsRequest),
|
||||
textToSpeech: bindTtsRuntime((runtime) => runtime.textToSpeech),
|
||||
textToSpeechStream: bindTtsRuntime((runtime) => runtime.textToSpeechStream),
|
||||
textToSpeechTelephony: bindTtsRuntime((runtime) => runtime.textToSpeechTelephony),
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
// Lazy runtime bridge for speech-core request pre-resolution.
|
||||
export { prepareTtsRequest } from "../../../packages/speech-core/runtime-api.js";
|
||||
@@ -8,6 +8,7 @@ import type { LogLevel } from "../../logging/levels.js";
|
||||
import type { MediaUnderstandingRuntime } from "../../media-understanding/runtime-types.js";
|
||||
import type {
|
||||
ListSpeechVoices,
|
||||
PrepareTtsRequest,
|
||||
TextToSpeech,
|
||||
TextToSpeechStream,
|
||||
TextToSpeechTelephony,
|
||||
@@ -361,6 +362,7 @@ export type PluginRuntimeCore = {
|
||||
resizeToJpeg: typeof import("../../media/media-services.js").resizeToJpeg;
|
||||
};
|
||||
tts: {
|
||||
prepareTtsRequest: PrepareTtsRequest;
|
||||
textToSpeech: TextToSpeech;
|
||||
textToSpeechStream: TextToSpeechStream;
|
||||
textToSpeechTelephony: TextToSpeechTelephony;
|
||||
|
||||
Reference in New Issue
Block a user