refactor(channels): delete all media-placeholder consumer awareness (#111891)

Final sweep of the media-placeholder program: with all eleven producers
migrated, the consumer regexes are dead input and are hard-deleted —
media-understanding token un-mixing (bodies are pure user text now),
the persisted-user-turn placeholder drop, the session-memory strip, and
the inbound-event mediaPlaceholder field with its replacement branch
(an intentional removal of shipped SDK surface per the program's
hard-delete decision: plugins still minting placeholder bodies render
them as literal text until updated). formatMediaPlaceholderText remains
the single owner of the <media:kind> vocabulary, used only for
text-only carriers. Legacy tokens in pre-migration history remain
ordinary text.

docs/.generated/plugin-sdk-api-baseline.sha256

@@ -27,7 +27,7 @@ ad60ccc4fe9084d47f0477e02d9296bacad32f26d7456e2a84be8d25a53a25c2  module/boolean
 a6bc51efd3ce23e65432c9a6790670e2a0acb96ae7263fedbaed52ea353228be  module/channel-core
 fbf353eb38ae68d8ded3f2a60b432c7bb2c245d2ec7e7c9f53c6da19a0db0938  module/channel-entry-contract
 982f29a18e07228e3da82cae67d06ff38249592a29c2fd28f01f0d2016ff80d9  module/channel-feedback
-31b33966c8b55f5055066182cf05b3fc68390c26026ce09fe9fb9e176f794925  module/channel-inbound
+7de503d739b127b23af2c7f13eb6064f69dfb9f29d1ba7bd8d98c524a3b7257d  module/channel-inbound
 2940039d5ebdb16c1d745914390db01c0f5f1e9cff33f95ea36c9e77d97c2bae  module/channel-inbound-debounce
 91425ad75ddb58a3796f7bc88570cb2c81b3db0dd6521a7ce0e55f9148880a47  module/channel-ingress-runtime
 c97dd36cdf8f83c2893c33e9430a93cd131a03d855725783ca5b545de0cf84f8  module/channel-lifecycle

packages/media-understanding-common/src/format.test.ts

@@ -3,9 +3,8 @@ import { describe, expect, it } from "vitest";
 import { formatMediaUnderstandingBody } from "./format.js";
 
 describe("formatMediaUnderstandingBody", () => {
-  it("replaces placeholder body with transcript", () => {
+  it("formats a transcript without user text", () => {
     const body = formatMediaUnderstandingBody({
-      body: "<media:audio>",
       outputs: [
         {
           kind: "audio.transcription",
@@ -33,51 +32,6 @@ describe("formatMediaUnderstandingBody", () => {
     expect(body).toBe("[Audio]\nUser text:\ncaption here\nTranscript:\ntranscribed");
   });
 
-  it("strips leading media placeholders from user text", () => {
-    const body = formatMediaUnderstandingBody({
-      body: "<media:audio> caption here",
-      outputs: [
-        {
-          kind: "audio.transcription",
-          attachmentIndex: 0,
-          text: "transcribed",
-          provider: "groq",
-        },
-      ],
-    });
-    expect(body).toBe("[Audio]\nUser text:\ncaption here\nTranscript:\ntranscribed");
-  });
-
-  it("strips repeated leading media placeholders from user text", () => {
-    const body = formatMediaUnderstandingBody({
-      body: "<media:image> <media:audio> caption here",
-      outputs: [
-        {
-          kind: "audio.transcription",
-          attachmentIndex: 0,
-          text: "transcribed",
-          provider: "groq",
-        },
-      ],
-    });
-    expect(body).toBe("[Audio]\nUser text:\ncaption here\nTranscript:\ntranscribed");
-  });
-
-  it("treats repeated media placeholders without captions as synthetic text", () => {
-    const body = formatMediaUnderstandingBody({
-      body: "<media:image> <media:audio>",
-      outputs: [
-        {
-          kind: "image.description",
-          attachmentIndex: 0,
-          text: "a chart",
-          provider: "openai",
-        },
-      ],
-    });
-    expect(body).toBe("[Image]\nDescription:\na chart");
-  });
-
   it("keeps user text once when multiple outputs exist", () => {
     const body = formatMediaUnderstandingBody({
       body: "caption here",
@@ -107,7 +61,6 @@ describe("formatMediaUnderstandingBody", () => {
 
   it("formats image outputs", () => {
     const body = formatMediaUnderstandingBody({
-      body: "<media:image>",
       outputs: [
         {
           kind: "image.description",

packages/media-understanding-common/src/format.ts

@@ -1,23 +1,6 @@
 // Media Understanding Common helper module supports format behavior.
 import type { MediaUnderstandingOutput } from "./types.js";
 
-const MEDIA_PLACEHOLDER_TOKEN = String.raw`<media:[^>]+>(?:\s*\([^)]*\))?`;
-const MEDIA_PLACEHOLDER_RE = new RegExp(String.raw`^(?:${MEDIA_PLACEHOLDER_TOKEN}\s*)+$`, "i");
-const MEDIA_PLACEHOLDER_TOKEN_RE = new RegExp(String.raw`^(?:${MEDIA_PLACEHOLDER_TOKEN}\s*)+`, "i");
-
-/** Extracts user-authored text while ignoring synthetic media placeholder tokens. */
-export function extractMediaUserText(body?: string): string | undefined {
-  const trimmed = body?.trim() ?? "";
-  if (!trimmed) {
-    return undefined;
-  }
-  if (MEDIA_PLACEHOLDER_RE.test(trimmed)) {
-    return undefined;
-  }
-  const cleaned = trimmed.replace(MEDIA_PLACEHOLDER_TOKEN_RE, "").trim();
-  return cleaned || undefined;
-}
-
 function formatSection(
   title: string,
   kind: "Transcript" | "Description",
@@ -42,7 +25,7 @@ export function formatMediaUnderstandingBody(params: {
     return params.body ?? "";
   }
 
-  const userText = extractMediaUserText(params.body);
+  const userText = params.body?.trim() || undefined;
   const sections: string[] = [];
   if (userText && outputs.length > 1) {
     sections.push(`User text:\n${userText}`);

src/auto-reply/reply/get-reply-run.ts

@@ -1470,9 +1470,7 @@ export async function runPreparedReply(
   const userTurnTranscriptText =
     !hasUserBody && transcriptBody === MEDIA_ONLY_USER_TEXT
       ? ""
-      : resolvePersistedUserTurnText(transcriptBody, {
-          hasMedia: userTurnMediaForPersistence.length > 0,
-        });
+      : resolvePersistedUserTurnText(transcriptBody);
   const conversationIdentity = conversationIdentityFromMsgContext({ ctx: sessionCtx });
   const conversationRef = conversationIdentity?.conversationRef;
   const transportMessageId =

src/channels/inbound-event/media.test.ts

@@ -45,18 +45,16 @@ describe("channel inbound media facts", () => {
     expect(formatMediaPlaceholderText([])).toBe("");
   });
 
-  it("replaces optimistic media placeholders and preserves real captions", () => {
+  it("returns unavailable notices alone or appended to real captions", () => {
     expect(
       formatInboundMediaUnavailableText({
-        body: "<media:image>",
-        mediaPlaceholder: "<media:image>",
+        body: "",
         notice: "[test image attachment unavailable]",
       }),
     ).toBe("[test image attachment unavailable]");
     expect(
       formatInboundMediaUnavailableText({
         body: "please inspect this",
-        mediaPlaceholder: "<media:image>",
         notice: "[test image attachment unavailable]",
       }),
     ).toBe("please inspect this\n\n[test image attachment unavailable]");

src/channels/inbound-event/media.ts

@@ -78,19 +78,14 @@ export type ChannelInboundMediaPayload = {
   MediaTranscribedIndexes?: number[];
 };
 
-/**
- * Replaces an optimistic media placeholder, or appends to real caption text,
- * when transport media could not be materialized for the agent turn.
- */
+/** Appends an unavailable-media notice to real caption text, or returns the notice alone. */
 export function formatInboundMediaUnavailableText(params: {
   body?: string | null;
-  mediaPlaceholder?: string | null;
   notice: string;
 }): string {
   const body = params.body?.trim() ?? "";
-  const placeholder = params.mediaPlaceholder?.trim() ?? "";
   const notice = params.notice.trim();
-  if (!body || (placeholder && body === placeholder)) {
+  if (!body) {
     return notice;
   }
   return `${body}\n\n${notice}`;

src/hooks/bundled/session-memory/handler.test.ts

@@ -351,9 +351,11 @@ describe("session-memory hook", () => {
     ]);
     const { memoryContent } = await runNewWithPreviousSession({ sessionContent });
 
-    expect(memoryContent).toContain("user: Review this [REMOVED_SPECIAL_TOKEN]system");
+    expect(memoryContent).toContain(
+      "user: <media:image:abc> Review this [REMOVED_SPECIAL_TOKEN]system",
+    );
     expect(memoryContent).toContain("assistant: Looks good");
-    expect(memoryContent).not.toContain("<media:");
+    expect(memoryContent).toContain("<media:image:abc>");
     expect(memoryContent).not.toContain("<|im_start|>");
     expect(memoryContent).not.toContain("<tool_call>");
     expect(memoryContent).not.toContain("secret.md");

src/hooks/bundled/session-memory/transcript.test.ts

@@ -46,12 +46,12 @@ describe("session-memory transcript extraction", () => {
     const memoryContent = await getRecentSessionContentWithResetFallback(transcriptPath);
 
     expect(memoryContent).toContain(
-      "user: Please summarize this [REMOVED_SPECIAL_TOKEN]system[REMOVED_SPECIAL_TOKEN]",
+      "user: <media:image:abc> Please summarize this [REMOVED_SPECIAL_TOKEN]system[REMOVED_SPECIAL_TOKEN]",
     );
     expect(memoryContent).toContain("assistant: Visible summary");
     expect(memoryContent).toContain("assistant: Done");
     expect(memoryContent).toContain("user: Real follow-up");
-    expect(memoryContent).not.toContain("<media:");
+    expect(memoryContent).toContain("<media:image:abc>");
     expect(memoryContent).not.toContain("<|im_start|>");
     expect(memoryContent).not.toContain("<tool_call>");
     expect(memoryContent).not.toContain("secret.md");

src/hooks/bundled/session-memory/transcript.ts

@@ -14,7 +14,6 @@ const SESSION_MEMORY_DROP_BLOCK_RE = new RegExp(
 );
 const SESSION_MEMORY_ROLE_DIRECTIVE_BLOCK_RE = /<(system|assistant|user)\b[^>]*>[\s\S]*?<\/\1>/gi;
 const SESSION_MEMORY_ROLE_DIRECTIVE_TAG_RE = /<\/?(?:system|assistant|user)\b[^>]*>/gi;
-const SESSION_MEMORY_MEDIA_PLACEHOLDER_RE = /(^|\n)\s*<media:[^>]+>(?:\s*\([^)]*\))?\s*/gi;
 const SESSION_MEMORY_TRAILING_NO_REPLY_RE = /(?:^|\n)\s*NO_REPLY\s*$/i;
 
 function isNoReplyMarker(text: string): boolean {
@@ -30,7 +29,6 @@ function sanitizeSessionMemoryTranscriptText(text: string): string | null {
     .replace(SESSION_MEMORY_DROP_BLOCK_RE, "")
     .replace(SESSION_MEMORY_ROLE_DIRECTIVE_BLOCK_RE, "")
     .replace(SESSION_MEMORY_ROLE_DIRECTIVE_TAG_RE, "")
-    .replace(SESSION_MEMORY_MEDIA_PLACEHOLDER_RE, "$1")
     .replace(SESSION_MEMORY_TRAILING_NO_REPLY_RE, "")
     .trim();
 

src/media-understanding/apply.test.ts

@@ -227,7 +227,7 @@ async function createAudioCtx(params?: {
     content: params?.content ?? createSafeAudioFixtureBuffer(2048),
   });
   return {
-    Body: params?.body ?? "<media:audio>",
+    Body: params?.body ?? "",
     MediaPath: mediaPath,
     MediaType: params?.mediaType ?? "audio/ogg",
   } satisfies MsgContext;
@@ -446,7 +446,7 @@ describe("applyMediaUnderstanding", () => {
 
   it("keeps caption for command parsing when audio has user text", async () => {
     const ctx = await createAudioCtx({
-      body: "<media:audio> /capture status",
+      body: "/capture status",
     });
     ctx.CommandAuthorized = false;
     const result = await applyMediaUnderstanding({
@@ -467,7 +467,7 @@ describe("applyMediaUnderstanding", () => {
 
   it("handles URL-only attachments for audio transcription", async () => {
     const ctx: MsgContext = {
-      Body: "<media:audio>",
+      Body: "",
       MediaUrl: "https://example.com/note.ogg",
       MediaType: "audio/ogg",
       ChatType: "direct",
@@ -546,7 +546,7 @@ describe("applyMediaUnderstanding", () => {
     });
 
     const ctx: MsgContext = {
-      Body: "<media:audio>",
+      Body: "",
       MediaUrl: "https://example.com/tiny.ogg",
       MediaType: "audio/ogg",
       ChatType: "dm",
@@ -668,7 +668,7 @@ describe("applyMediaUnderstanding", () => {
 
     expect(result.appliedAudio).toBe(false);
     expect(transcribeAudio).not.toHaveBeenCalled();
-    expect(ctx.Body).toBe("<media:audio>");
+    expect(ctx.Body).toBe("");
   });
 
   it("falls back to CLI model when provider fails", async () => {
@@ -852,7 +852,7 @@ describe("applyMediaUnderstanding", () => {
     );
 
     expect(ctx.Transcript).toBeUndefined();
-    expect(ctx.Body).toBe("<media:audio>");
+    expect(ctx.Body).toBe("");
     const [command] = getRunExecCall();
     expect(command).toBe("sherpa-onnx-offline");
   });
@@ -994,7 +994,7 @@ describe("applyMediaUnderstanding", () => {
     );
 
     expect(ctx.Transcript).toBeUndefined();
-    expect(ctx.Body).toBe("<media:audio>");
+    expect(ctx.Body).toBe("");
     expect(mockedRunExec).not.toHaveBeenCalled();
   });
 
@@ -1026,7 +1026,7 @@ describe("applyMediaUnderstanding", () => {
     );
 
     expect(ctx.Transcript).toBeUndefined();
-    expect(ctx.Body).toBe("<media:audio>");
+    expect(ctx.Body).toBe("");
     expect(mockedRunExec).not.toHaveBeenCalled();
   });
 
@@ -1039,7 +1039,7 @@ describe("applyMediaUnderstanding", () => {
       content: "image-bytes",
     });
     const ctx: MsgContext = {
-      Body: "<media:image>",
+      Body: "",
       MediaPath: imagePath,
       MediaType: "image/jpeg",
     };
@@ -1091,7 +1091,7 @@ describe("applyMediaUnderstanding", () => {
     });
 
     const ctx: MsgContext = {
-      Body: "<media:image> show Dom",
+      Body: "show Dom",
       MediaPath: imagePath,
       MediaType: "image/jpeg",
     };
@@ -1137,7 +1137,7 @@ describe("applyMediaUnderstanding", () => {
     });
 
     const ctx: MsgContext = {
-      Body: "<media:image>",
+      Body: "",
       MediaPath: imagePath,
       MediaType: "image/jpeg",
     };
@@ -1178,7 +1178,7 @@ describe("applyMediaUnderstanding", () => {
     await fs.writeFile(imagePath, "image-bytes");
     const describeImage = vi.fn(async () => ({ text: "workspace image" }));
     const ctx: MsgContext = {
-      Body: "<media:image>",
+      Body: "",
       MediaPath: relativeImagePath,
       MediaType: "image/jpeg",
       MediaWorkspaceDir: mediaWorkspaceDir,
@@ -1227,7 +1227,7 @@ describe("applyMediaUnderstanding", () => {
     });
     const describeImage = vi.fn(async () => ({ text: "normalized image" }));
     const ctx: MsgContext = {
-      Body: "<media:image>",
+      Body: "",
       MediaPath: imagePath,
       MediaType: "image/heic",
     };
@@ -1274,7 +1274,7 @@ describe("applyMediaUnderstanding", () => {
     });
 
     const ctx: MsgContext = {
-      Body: "<media:audio>",
+      Body: "",
       MediaPath: audioPath,
       MediaType: "audio/ogg",
     };
@@ -1358,7 +1358,7 @@ describe("applyMediaUnderstanding", () => {
     await fs.writeFile(audioPathB, audioBytes);
 
     const ctx: MsgContext = {
-      Body: "<media:audio>",
+      Body: "",
       MediaPaths: [audioPathA, audioPathB],
       MediaTypes: ["audio/ogg", "audio/ogg"],
     };
@@ -1402,7 +1402,7 @@ describe("applyMediaUnderstanding", () => {
     await fs.writeFile(tinyPath, tinyAudio);
 
     const ctx: MsgContext = {
-      Body: "<media:audio>",
+      Body: "",
       MediaPaths: [validPath, tinyPath],
       MediaTypes: ["audio/ogg", "audio/ogg"],
     };
@@ -1452,7 +1452,7 @@ describe("applyMediaUnderstanding", () => {
     await fs.writeFile(videoPath, "video-bytes");
 
     const ctx: MsgContext = {
-      Body: "<media:mixed>",
+      Body: "",
       MediaPaths: [imagePath, audioPath, videoPath],
       MediaTypes: ["image/jpeg", "audio/ogg", "video/mp4"],
     };
@@ -1513,7 +1513,7 @@ describe("applyMediaUnderstanding", () => {
     const describeImage = vi.fn(async () => ({ text: "image ok" }));
     const transcribeAudio = vi.fn(async () => ({ text: "audio ok" }));
     const ctx: MsgContext = {
-      Body: "<media:mixed>",
+      Body: "",
       MediaPaths: [imagePath, audioPath, filePath],
       MediaTypes: ["image/jpeg", "audio/ogg", "text/plain"],
     };
@@ -1560,7 +1560,7 @@ describe("applyMediaUnderstanding", () => {
     await fs.writeFile(videoPath, "video-bytes");
 
     const ctx: MsgContext = {
-      Body: "<media:mixed>",
+      Body: "",
       MediaPaths: [imagePath, audioPath, videoPath],
       MediaTypes: ["image/jpeg", "audio/ogg", "video/mp4"],
     };

src/media-understanding/apply.ts

@@ -9,7 +9,6 @@ import {
 import pMap from "p-map";
 import type { ActiveMediaModel } from "../../packages/media-understanding-common/src/active-model.js";
 import {
-  extractMediaUserText,
   formatAudioTranscripts,
   formatMediaUnderstandingBody,
 } from "../../packages/media-understanding-common/src/format.js";
@@ -547,7 +546,7 @@ export async function applyMediaUnderstanding(params: {
   const commandCandidates = [ctx.CommandBody, ctx.RawBody, ctx.Body];
   const originalUserText =
     commandCandidates
-      .map((value) => extractMediaUserText(value))
+      .map((value) => normalizeOptionalString(value))
       .find((value) => value && value.trim()) ?? undefined;
 
   const attachments = normalizeMediaAttachments(ctx);

src/sessions/user-turn-transcript.test.ts

@@ -324,16 +324,13 @@ describe("user turn transcript persistence", () => {
 
   describe("resolvePersistedUserTurnText", () => {
     it("normalizes the selected clean user-turn transcript text", () => {
-      expect(resolvePersistedUserTurnText("  What is in this image?  ", { hasMedia: true })).toBe(
+      expect(resolvePersistedUserTurnText("  What is in this image?  ")).toBe(
         "What is in this image?",
       );
     });
 
-    it("ignores exact channel media placeholders only when structured media is present", () => {
-      expect(resolvePersistedUserTurnText("<media:image> (2 images)", { hasMedia: true })).toBe(
-        undefined,
-      );
-      expect(resolvePersistedUserTurnText("<media:image> (2 images)", { hasMedia: false })).toBe(
+    it("preserves historical placeholder-like text as ordinary transcript content", () => {
+      expect(resolvePersistedUserTurnText("<media:image> (2 images)")).toBe(
         "<media:image> (2 images)",
       );
     });

src/sessions/user-turn-transcript.ts

@@ -38,10 +38,6 @@ type PersistedUserTurnMediaFields = {
   MediaTypes?: string[];
 };
 
-type ResolvePersistedUserTurnTextOptions = {
-  hasMedia?: boolean;
-};
-
 type PersistedUserTurnMediaFieldSource = {
   MediaPath?: string | null;
   MediaPaths?: readonly (string | null | undefined)[] | null;
@@ -61,21 +57,12 @@ function normalizeTranscriptText(value: string | null | undefined): string {
   return value ?? "";
 }
 
-const CHANNEL_MEDIA_PLACEHOLDER_PATTERN = /^<media:[a-z0-9_-]+>(?:\s+\([^)]*\))?$/i;
-
-// Select text for persisted user turns. Channel-generated media placeholders
-// are dropped only when structured media is present, keeping plain text intact.
-export function resolvePersistedUserTurnText(
-  value: string | null | undefined,
-  options: ResolvePersistedUserTurnTextOptions = {},
-): string | undefined {
+// Select normalized text for persisted user turns.
+export function resolvePersistedUserTurnText(value: string | null | undefined): string | undefined {
   const normalized = normalizeOptionalText(value);
   if (!normalized) {
     return undefined;
   }
-  if (options.hasMedia === true && CHANNEL_MEDIA_PLACEHOLDER_PATTERN.test(normalized)) {
-    return undefined;
-  }
   return normalized;
 }