diff --git a/src/auto-reply/reply/current-turn-images.test.ts b/src/auto-reply/reply/current-turn-images.test.ts index 6133ae55a5b..3448c23b0a3 100644 --- a/src/auto-reply/reply/current-turn-images.test.ts +++ b/src/auto-reply/reply/current-turn-images.test.ts @@ -61,6 +61,59 @@ describe("resolveCurrentTurnImages", () => { }); }); + it("preserves the full order when only inline image payloads are present", async () => { + const inlineImage = { + type: "image" as const, + data: Buffer.from("inline").toString("base64"), + mimeType: "image/png", + }; + + const result = await resolveCurrentTurnImages({ + ctx: { Body: "compare these" } satisfies MsgContext, + cfg: {} as OpenClawConfig, + images: [inlineImage], + imageOrder: ["offloaded", "inline", "offloaded"], + }); + + expect(result).toEqual({ + images: [inlineImage], + imageOrder: ["offloaded", "inline", "offloaded"], + }); + }); + + it("preserves all-offloaded image order without inline payloads", async () => { + const result = await resolveCurrentTurnImages({ + ctx: { Body: "compare these" } satisfies MsgContext, + cfg: {} as OpenClawConfig, + images: [], + imageOrder: ["offloaded", "offloaded"], + }); + + expect(result).toEqual({ + imageOrder: ["offloaded", "offloaded"], + }); + }); + + it("preserves interleaved offloaded slots around inline image payloads", async () => { + const inlineImages = ["first", "second"].map((data) => ({ + type: "image" as const, + data: Buffer.from(data).toString("base64"), + mimeType: "image/png", + })); + + const result = await resolveCurrentTurnImages({ + ctx: { Body: "compare these" } satisfies MsgContext, + cfg: {} as OpenClawConfig, + images: inlineImages, + imageOrder: ["inline", "offloaded", "inline"], + }); + + expect(result).toEqual({ + images: inlineImages, + imageOrder: ["inline", "offloaded", "inline"], + }); + }); + it("appends extracted PDF page images without dropping current image attachments", async () => { await withTempDir({ prefix: "openclaw-current-turn-pdf-images-" }, async (base) => { const imagePath = path.join(base, "photo.png"); diff --git a/src/auto-reply/reply/current-turn-images.ts b/src/auto-reply/reply/current-turn-images.ts index bb22a36c6be..7d0aa53b412 100644 --- a/src/auto-reply/reply/current-turn-images.ts +++ b/src/auto-reply/reply/current-turn-images.ts @@ -20,7 +20,7 @@ type CurrentImageAttachment = { }; type OrderedTurnImage = { - image: ImageContent; + image?: ImageContent; imageOrder: PromptImageOrderEntry; sourceIndex?: number; sequence: number; @@ -105,13 +105,32 @@ function appendOrderedImages(params: { imageOrder?: PromptImageOrderEntry[]; sourceIndex?: number; }) { - if (!params.images || params.images.length === 0) { + const images = params.images ?? []; + if (!params.imageOrder || params.imageOrder.length === 0) { + for (const image of images) { + params.entries.push({ + image, + imageOrder: "inline", + sourceIndex: params.sourceIndex, + sequence: params.entries.length, + }); + } return; } - for (const [index, image] of params.images.entries()) { + + let inlineIndex = 0; + for (const imageOrder of params.imageOrder) { params.entries.push({ - image, - imageOrder: params.imageOrder?.[index] ?? "inline", + image: imageOrder === "inline" ? images[inlineIndex++] : undefined, + imageOrder, + sourceIndex: params.sourceIndex, + sequence: params.entries.length, + }); + } + while (inlineIndex < images.length) { + params.entries.push({ + image: images[inlineIndex++], + imageOrder: "inline", sourceIndex: params.sourceIndex, sequence: params.entries.length, }); @@ -134,8 +153,9 @@ function resolveMergedTurnImages(entries: OrderedTurnImage[]): { } return left.sequence - right.sequence; }); + const images = merged.flatMap((entry) => (entry.image ? [entry.image] : [])); return { - images: merged.map((entry) => entry.image), + ...(images.length > 0 ? { images } : {}), imageOrder: merged.map((entry) => entry.imageOrder), }; }