diff --git a/docs/.generated/plugin-sdk-api-baseline.sha256 b/docs/.generated/plugin-sdk-api-baseline.sha256 index 84f39904e28..8201999556d 100644 --- a/docs/.generated/plugin-sdk-api-baseline.sha256 +++ b/docs/.generated/plugin-sdk-api-baseline.sha256 @@ -27,7 +27,7 @@ ad60ccc4fe9084d47f0477e02d9296bacad32f26d7456e2a84be8d25a53a25c2 module/boolean a6bc51efd3ce23e65432c9a6790670e2a0acb96ae7263fedbaed52ea353228be module/channel-core fbf353eb38ae68d8ded3f2a60b432c7bb2c245d2ec7e7c9f53c6da19a0db0938 module/channel-entry-contract 982f29a18e07228e3da82cae67d06ff38249592a29c2fd28f01f0d2016ff80d9 module/channel-feedback -31b33966c8b55f5055066182cf05b3fc68390c26026ce09fe9fb9e176f794925 module/channel-inbound +7de503d739b127b23af2c7f13eb6064f69dfb9f29d1ba7bd8d98c524a3b7257d module/channel-inbound 2940039d5ebdb16c1d745914390db01c0f5f1e9cff33f95ea36c9e77d97c2bae module/channel-inbound-debounce 91425ad75ddb58a3796f7bc88570cb2c81b3db0dd6521a7ce0e55f9148880a47 module/channel-ingress-runtime c97dd36cdf8f83c2893c33e9430a93cd131a03d855725783ca5b545de0cf84f8 module/channel-lifecycle diff --git a/packages/media-understanding-common/src/format.test.ts b/packages/media-understanding-common/src/format.test.ts index 136d7052358..1eb67b94a48 100644 --- a/packages/media-understanding-common/src/format.test.ts +++ b/packages/media-understanding-common/src/format.test.ts @@ -3,9 +3,8 @@ import { describe, expect, it } from "vitest"; import { formatMediaUnderstandingBody } from "./format.js"; describe("formatMediaUnderstandingBody", () => { - it("replaces placeholder body with transcript", () => { + it("formats a transcript without user text", () => { const body = formatMediaUnderstandingBody({ - body: "", outputs: [ { kind: "audio.transcription", @@ -33,51 +32,6 @@ describe("formatMediaUnderstandingBody", () => { expect(body).toBe("[Audio]\nUser text:\ncaption here\nTranscript:\ntranscribed"); }); - it("strips leading media placeholders from user text", () => { - const body = formatMediaUnderstandingBody({ - body: " caption here", - outputs: [ - { - kind: "audio.transcription", - attachmentIndex: 0, - text: "transcribed", - provider: "groq", - }, - ], - }); - expect(body).toBe("[Audio]\nUser text:\ncaption here\nTranscript:\ntranscribed"); - }); - - it("strips repeated leading media placeholders from user text", () => { - const body = formatMediaUnderstandingBody({ - body: " caption here", - outputs: [ - { - kind: "audio.transcription", - attachmentIndex: 0, - text: "transcribed", - provider: "groq", - }, - ], - }); - expect(body).toBe("[Audio]\nUser text:\ncaption here\nTranscript:\ntranscribed"); - }); - - it("treats repeated media placeholders without captions as synthetic text", () => { - const body = formatMediaUnderstandingBody({ - body: " ", - outputs: [ - { - kind: "image.description", - attachmentIndex: 0, - text: "a chart", - provider: "openai", - }, - ], - }); - expect(body).toBe("[Image]\nDescription:\na chart"); - }); - it("keeps user text once when multiple outputs exist", () => { const body = formatMediaUnderstandingBody({ body: "caption here", @@ -107,7 +61,6 @@ describe("formatMediaUnderstandingBody", () => { it("formats image outputs", () => { const body = formatMediaUnderstandingBody({ - body: "", outputs: [ { kind: "image.description", diff --git a/packages/media-understanding-common/src/format.ts b/packages/media-understanding-common/src/format.ts index bd75bcfd793..2fd0249dd13 100644 --- a/packages/media-understanding-common/src/format.ts +++ b/packages/media-understanding-common/src/format.ts @@ -1,23 +1,6 @@ // Media Understanding Common helper module supports format behavior. import type { MediaUnderstandingOutput } from "./types.js"; -const MEDIA_PLACEHOLDER_TOKEN = String.raw`]+>(?:\s*\([^)]*\))?`; -const MEDIA_PLACEHOLDER_RE = new RegExp(String.raw`^(?:${MEDIA_PLACEHOLDER_TOKEN}\s*)+$`, "i"); -const MEDIA_PLACEHOLDER_TOKEN_RE = new RegExp(String.raw`^(?:${MEDIA_PLACEHOLDER_TOKEN}\s*)+`, "i"); - -/** Extracts user-authored text while ignoring synthetic media placeholder tokens. */ -export function extractMediaUserText(body?: string): string | undefined { - const trimmed = body?.trim() ?? ""; - if (!trimmed) { - return undefined; - } - if (MEDIA_PLACEHOLDER_RE.test(trimmed)) { - return undefined; - } - const cleaned = trimmed.replace(MEDIA_PLACEHOLDER_TOKEN_RE, "").trim(); - return cleaned || undefined; -} - function formatSection( title: string, kind: "Transcript" | "Description", @@ -42,7 +25,7 @@ export function formatMediaUnderstandingBody(params: { return params.body ?? ""; } - const userText = extractMediaUserText(params.body); + const userText = params.body?.trim() || undefined; const sections: string[] = []; if (userText && outputs.length > 1) { sections.push(`User text:\n${userText}`); diff --git a/src/auto-reply/reply/get-reply-run.ts b/src/auto-reply/reply/get-reply-run.ts index 74de4bcee39..daf63e7e35a 100644 --- a/src/auto-reply/reply/get-reply-run.ts +++ b/src/auto-reply/reply/get-reply-run.ts @@ -1470,9 +1470,7 @@ export async function runPreparedReply( const userTurnTranscriptText = !hasUserBody && transcriptBody === MEDIA_ONLY_USER_TEXT ? "" - : resolvePersistedUserTurnText(transcriptBody, { - hasMedia: userTurnMediaForPersistence.length > 0, - }); + : resolvePersistedUserTurnText(transcriptBody); const conversationIdentity = conversationIdentityFromMsgContext({ ctx: sessionCtx }); const conversationRef = conversationIdentity?.conversationRef; const transportMessageId = diff --git a/src/channels/inbound-event/media.test.ts b/src/channels/inbound-event/media.test.ts index c1b587a3322..c4a1d077074 100644 --- a/src/channels/inbound-event/media.test.ts +++ b/src/channels/inbound-event/media.test.ts @@ -45,18 +45,16 @@ describe("channel inbound media facts", () => { expect(formatMediaPlaceholderText([])).toBe(""); }); - it("replaces optimistic media placeholders and preserves real captions", () => { + it("returns unavailable notices alone or appended to real captions", () => { expect( formatInboundMediaUnavailableText({ - body: "", - mediaPlaceholder: "", + body: "", notice: "[test image attachment unavailable]", }), ).toBe("[test image attachment unavailable]"); expect( formatInboundMediaUnavailableText({ body: "please inspect this", - mediaPlaceholder: "", notice: "[test image attachment unavailable]", }), ).toBe("please inspect this\n\n[test image attachment unavailable]"); diff --git a/src/channels/inbound-event/media.ts b/src/channels/inbound-event/media.ts index 9deeb1a5158..493b9000139 100644 --- a/src/channels/inbound-event/media.ts +++ b/src/channels/inbound-event/media.ts @@ -78,19 +78,14 @@ export type ChannelInboundMediaPayload = { MediaTranscribedIndexes?: number[]; }; -/** - * Replaces an optimistic media placeholder, or appends to real caption text, - * when transport media could not be materialized for the agent turn. - */ +/** Appends an unavailable-media notice to real caption text, or returns the notice alone. */ export function formatInboundMediaUnavailableText(params: { body?: string | null; - mediaPlaceholder?: string | null; notice: string; }): string { const body = params.body?.trim() ?? ""; - const placeholder = params.mediaPlaceholder?.trim() ?? ""; const notice = params.notice.trim(); - if (!body || (placeholder && body === placeholder)) { + if (!body) { return notice; } return `${body}\n\n${notice}`; diff --git a/src/hooks/bundled/session-memory/handler.test.ts b/src/hooks/bundled/session-memory/handler.test.ts index 927bb85f6d7..1c7372ef1ab 100644 --- a/src/hooks/bundled/session-memory/handler.test.ts +++ b/src/hooks/bundled/session-memory/handler.test.ts @@ -351,9 +351,11 @@ describe("session-memory hook", () => { ]); const { memoryContent } = await runNewWithPreviousSession({ sessionContent }); - expect(memoryContent).toContain("user: Review this [REMOVED_SPECIAL_TOKEN]system"); + expect(memoryContent).toContain( + "user: Review this [REMOVED_SPECIAL_TOKEN]system", + ); expect(memoryContent).toContain("assistant: Looks good"); - expect(memoryContent).not.toContain(""); expect(memoryContent).not.toContain("<|im_start|>"); expect(memoryContent).not.toContain(""); expect(memoryContent).not.toContain("secret.md"); diff --git a/src/hooks/bundled/session-memory/transcript.test.ts b/src/hooks/bundled/session-memory/transcript.test.ts index 92d191f360e..7695c571fee 100644 --- a/src/hooks/bundled/session-memory/transcript.test.ts +++ b/src/hooks/bundled/session-memory/transcript.test.ts @@ -46,12 +46,12 @@ describe("session-memory transcript extraction", () => { const memoryContent = await getRecentSessionContentWithResetFallback(transcriptPath); expect(memoryContent).toContain( - "user: Please summarize this [REMOVED_SPECIAL_TOKEN]system[REMOVED_SPECIAL_TOKEN]", + "user: Please summarize this [REMOVED_SPECIAL_TOKEN]system[REMOVED_SPECIAL_TOKEN]", ); expect(memoryContent).toContain("assistant: Visible summary"); expect(memoryContent).toContain("assistant: Done"); expect(memoryContent).toContain("user: Real follow-up"); - expect(memoryContent).not.toContain(""); expect(memoryContent).not.toContain("<|im_start|>"); expect(memoryContent).not.toContain(""); expect(memoryContent).not.toContain("secret.md"); diff --git a/src/hooks/bundled/session-memory/transcript.ts b/src/hooks/bundled/session-memory/transcript.ts index 3ecfb7bfa48..cf0f6bec0e3 100644 --- a/src/hooks/bundled/session-memory/transcript.ts +++ b/src/hooks/bundled/session-memory/transcript.ts @@ -14,7 +14,6 @@ const SESSION_MEMORY_DROP_BLOCK_RE = new RegExp( ); const SESSION_MEMORY_ROLE_DIRECTIVE_BLOCK_RE = /<(system|assistant|user)\b[^>]*>[\s\S]*?<\/\1>/gi; const SESSION_MEMORY_ROLE_DIRECTIVE_TAG_RE = /<\/?(?:system|assistant|user)\b[^>]*>/gi; -const SESSION_MEMORY_MEDIA_PLACEHOLDER_RE = /(^|\n)\s*]+>(?:\s*\([^)]*\))?\s*/gi; const SESSION_MEMORY_TRAILING_NO_REPLY_RE = /(?:^|\n)\s*NO_REPLY\s*$/i; function isNoReplyMarker(text: string): boolean { @@ -30,7 +29,6 @@ function sanitizeSessionMemoryTranscriptText(text: string): string | null { .replace(SESSION_MEMORY_DROP_BLOCK_RE, "") .replace(SESSION_MEMORY_ROLE_DIRECTIVE_BLOCK_RE, "") .replace(SESSION_MEMORY_ROLE_DIRECTIVE_TAG_RE, "") - .replace(SESSION_MEMORY_MEDIA_PLACEHOLDER_RE, "$1") .replace(SESSION_MEMORY_TRAILING_NO_REPLY_RE, "") .trim(); diff --git a/src/media-understanding/apply.test.ts b/src/media-understanding/apply.test.ts index ad0a146e0d6..b71679ff8a0 100644 --- a/src/media-understanding/apply.test.ts +++ b/src/media-understanding/apply.test.ts @@ -227,7 +227,7 @@ async function createAudioCtx(params?: { content: params?.content ?? createSafeAudioFixtureBuffer(2048), }); return { - Body: params?.body ?? "", + Body: params?.body ?? "", MediaPath: mediaPath, MediaType: params?.mediaType ?? "audio/ogg", } satisfies MsgContext; @@ -446,7 +446,7 @@ describe("applyMediaUnderstanding", () => { it("keeps caption for command parsing when audio has user text", async () => { const ctx = await createAudioCtx({ - body: " /capture status", + body: "/capture status", }); ctx.CommandAuthorized = false; const result = await applyMediaUnderstanding({ @@ -467,7 +467,7 @@ describe("applyMediaUnderstanding", () => { it("handles URL-only attachments for audio transcription", async () => { const ctx: MsgContext = { - Body: "", + Body: "", MediaUrl: "https://example.com/note.ogg", MediaType: "audio/ogg", ChatType: "direct", @@ -546,7 +546,7 @@ describe("applyMediaUnderstanding", () => { }); const ctx: MsgContext = { - Body: "", + Body: "", MediaUrl: "https://example.com/tiny.ogg", MediaType: "audio/ogg", ChatType: "dm", @@ -668,7 +668,7 @@ describe("applyMediaUnderstanding", () => { expect(result.appliedAudio).toBe(false); expect(transcribeAudio).not.toHaveBeenCalled(); - expect(ctx.Body).toBe(""); + expect(ctx.Body).toBe(""); }); it("falls back to CLI model when provider fails", async () => { @@ -852,7 +852,7 @@ describe("applyMediaUnderstanding", () => { ); expect(ctx.Transcript).toBeUndefined(); - expect(ctx.Body).toBe(""); + expect(ctx.Body).toBe(""); const [command] = getRunExecCall(); expect(command).toBe("sherpa-onnx-offline"); }); @@ -994,7 +994,7 @@ describe("applyMediaUnderstanding", () => { ); expect(ctx.Transcript).toBeUndefined(); - expect(ctx.Body).toBe(""); + expect(ctx.Body).toBe(""); expect(mockedRunExec).not.toHaveBeenCalled(); }); @@ -1026,7 +1026,7 @@ describe("applyMediaUnderstanding", () => { ); expect(ctx.Transcript).toBeUndefined(); - expect(ctx.Body).toBe(""); + expect(ctx.Body).toBe(""); expect(mockedRunExec).not.toHaveBeenCalled(); }); @@ -1039,7 +1039,7 @@ describe("applyMediaUnderstanding", () => { content: "image-bytes", }); const ctx: MsgContext = { - Body: "", + Body: "", MediaPath: imagePath, MediaType: "image/jpeg", }; @@ -1091,7 +1091,7 @@ describe("applyMediaUnderstanding", () => { }); const ctx: MsgContext = { - Body: " show Dom", + Body: "show Dom", MediaPath: imagePath, MediaType: "image/jpeg", }; @@ -1137,7 +1137,7 @@ describe("applyMediaUnderstanding", () => { }); const ctx: MsgContext = { - Body: "", + Body: "", MediaPath: imagePath, MediaType: "image/jpeg", }; @@ -1178,7 +1178,7 @@ describe("applyMediaUnderstanding", () => { await fs.writeFile(imagePath, "image-bytes"); const describeImage = vi.fn(async () => ({ text: "workspace image" })); const ctx: MsgContext = { - Body: "", + Body: "", MediaPath: relativeImagePath, MediaType: "image/jpeg", MediaWorkspaceDir: mediaWorkspaceDir, @@ -1227,7 +1227,7 @@ describe("applyMediaUnderstanding", () => { }); const describeImage = vi.fn(async () => ({ text: "normalized image" })); const ctx: MsgContext = { - Body: "", + Body: "", MediaPath: imagePath, MediaType: "image/heic", }; @@ -1274,7 +1274,7 @@ describe("applyMediaUnderstanding", () => { }); const ctx: MsgContext = { - Body: "", + Body: "", MediaPath: audioPath, MediaType: "audio/ogg", }; @@ -1358,7 +1358,7 @@ describe("applyMediaUnderstanding", () => { await fs.writeFile(audioPathB, audioBytes); const ctx: MsgContext = { - Body: "", + Body: "", MediaPaths: [audioPathA, audioPathB], MediaTypes: ["audio/ogg", "audio/ogg"], }; @@ -1402,7 +1402,7 @@ describe("applyMediaUnderstanding", () => { await fs.writeFile(tinyPath, tinyAudio); const ctx: MsgContext = { - Body: "", + Body: "", MediaPaths: [validPath, tinyPath], MediaTypes: ["audio/ogg", "audio/ogg"], }; @@ -1452,7 +1452,7 @@ describe("applyMediaUnderstanding", () => { await fs.writeFile(videoPath, "video-bytes"); const ctx: MsgContext = { - Body: "", + Body: "", MediaPaths: [imagePath, audioPath, videoPath], MediaTypes: ["image/jpeg", "audio/ogg", "video/mp4"], }; @@ -1513,7 +1513,7 @@ describe("applyMediaUnderstanding", () => { const describeImage = vi.fn(async () => ({ text: "image ok" })); const transcribeAudio = vi.fn(async () => ({ text: "audio ok" })); const ctx: MsgContext = { - Body: "", + Body: "", MediaPaths: [imagePath, audioPath, filePath], MediaTypes: ["image/jpeg", "audio/ogg", "text/plain"], }; @@ -1560,7 +1560,7 @@ describe("applyMediaUnderstanding", () => { await fs.writeFile(videoPath, "video-bytes"); const ctx: MsgContext = { - Body: "", + Body: "", MediaPaths: [imagePath, audioPath, videoPath], MediaTypes: ["image/jpeg", "audio/ogg", "video/mp4"], }; diff --git a/src/media-understanding/apply.ts b/src/media-understanding/apply.ts index f7bb0bc01aa..4d6470b112c 100644 --- a/src/media-understanding/apply.ts +++ b/src/media-understanding/apply.ts @@ -9,7 +9,6 @@ import { import pMap from "p-map"; import type { ActiveMediaModel } from "../../packages/media-understanding-common/src/active-model.js"; import { - extractMediaUserText, formatAudioTranscripts, formatMediaUnderstandingBody, } from "../../packages/media-understanding-common/src/format.js"; @@ -547,7 +546,7 @@ export async function applyMediaUnderstanding(params: { const commandCandidates = [ctx.CommandBody, ctx.RawBody, ctx.Body]; const originalUserText = commandCandidates - .map((value) => extractMediaUserText(value)) + .map((value) => normalizeOptionalString(value)) .find((value) => value && value.trim()) ?? undefined; const attachments = normalizeMediaAttachments(ctx); diff --git a/src/sessions/user-turn-transcript.test.ts b/src/sessions/user-turn-transcript.test.ts index eed32f21bc5..c59d44a09df 100644 --- a/src/sessions/user-turn-transcript.test.ts +++ b/src/sessions/user-turn-transcript.test.ts @@ -324,16 +324,13 @@ describe("user turn transcript persistence", () => { describe("resolvePersistedUserTurnText", () => { it("normalizes the selected clean user-turn transcript text", () => { - expect(resolvePersistedUserTurnText(" What is in this image? ", { hasMedia: true })).toBe( + expect(resolvePersistedUserTurnText(" What is in this image? ")).toBe( "What is in this image?", ); }); - it("ignores exact channel media placeholders only when structured media is present", () => { - expect(resolvePersistedUserTurnText(" (2 images)", { hasMedia: true })).toBe( - undefined, - ); - expect(resolvePersistedUserTurnText(" (2 images)", { hasMedia: false })).toBe( + it("preserves historical placeholder-like text as ordinary transcript content", () => { + expect(resolvePersistedUserTurnText(" (2 images)")).toBe( " (2 images)", ); }); diff --git a/src/sessions/user-turn-transcript.ts b/src/sessions/user-turn-transcript.ts index 7be9cdafd71..26008169735 100644 --- a/src/sessions/user-turn-transcript.ts +++ b/src/sessions/user-turn-transcript.ts @@ -38,10 +38,6 @@ type PersistedUserTurnMediaFields = { MediaTypes?: string[]; }; -type ResolvePersistedUserTurnTextOptions = { - hasMedia?: boolean; -}; - type PersistedUserTurnMediaFieldSource = { MediaPath?: string | null; MediaPaths?: readonly (string | null | undefined)[] | null; @@ -61,21 +57,12 @@ function normalizeTranscriptText(value: string | null | undefined): string { return value ?? ""; } -const CHANNEL_MEDIA_PLACEHOLDER_PATTERN = /^(?:\s+\([^)]*\))?$/i; - -// Select text for persisted user turns. Channel-generated media placeholders -// are dropped only when structured media is present, keeping plain text intact. -export function resolvePersistedUserTurnText( - value: string | null | undefined, - options: ResolvePersistedUserTurnTextOptions = {}, -): string | undefined { +// Select normalized text for persisted user turns. +export function resolvePersistedUserTurnText(value: string | null | undefined): string | undefined { const normalized = normalizeOptionalText(value); if (!normalized) { return undefined; } - if (options.hasMedia === true && CHANNEL_MEDIA_PLACEHOLDER_PATTERN.test(normalized)) { - return undefined; - } return normalized; }