Compare commits

...
1 Commits
Author SHA1 Message Date
Aiden Cline 2c3b65a04e fix(session): respect model media capabilities 2026-08-08 04:47:13 +00:00
3 changed files with 68 additions and 1 deletions
@@ -431,6 +431,7 @@ function unsupportedParts(msgs: ModelMessage[], model: Provider.Model): ModelMes
const modality = mimeToModality(mime)
if (!modality) return part
if (model.capabilities.input[modality]) return part
if ((modality === "image" || modality === "pdf") && model.capabilities.attachment) return part
const name = filename ? `"${filename}"` : modality
return {
@@ -145,6 +145,10 @@ export const toModelMessagesEffect = Effect.fnUntraced(function* (
// Only apply this workaround if the model actually supports that media input -
// otherwise unsupportedParts() will turn it into a user-visible error.
const supportsMediaInToolResult = (attachment: { mime: string }) => {
if (attachment.mime.startsWith("image/") && !model.capabilities.attachment && !model.capabilities.input.image)
return false
if (attachment.mime === "application/pdf" && !model.capabilities.attachment && !model.capabilities.input.pdf)
return false
if (model.api.npm === "@ai-sdk/anthropic") return true
if (model.api.npm === "@ai-sdk/openai") return true
if (model.api.npm === "@ai-sdk/amazon-bedrock/mantle") return true
@@ -322,6 +322,16 @@ describe("session.message-v2.toModelMessage", () => {
test("converts assistant tool completion into tool-call + tool-result messages with attachments", async () => {
const userID = "m-user"
const assistantID = "m-assistant"
const imageModel: Provider.Model = {
...model,
capabilities: {
...model.capabilities,
input: {
...model.capabilities.input,
image: true,
},
},
}
const input: SessionV1.WithParts[] = [
{
@@ -371,7 +381,7 @@ describe("session.message-v2.toModelMessage", () => {
},
]
expect(await MessageV2.toModelMessages(input, model)).toStrictEqual([
expect(await MessageV2.toModelMessages(input, imageModel)).toStrictEqual([
{
role: "user",
content: [{ type: "text", text: "run tool" }],
@@ -409,6 +419,58 @@ describe("session.message-v2.toModelMessage", () => {
],
},
])
const unsupported = await MessageV2.toModelMessages(input, model)
expect(unsupported).toMatchObject([
{ role: "user" },
{ role: "assistant" },
{
role: "tool",
content: [{ output: { type: "text", value: "ok" } }],
},
{
role: "user",
content: [
{ type: "text", text: MessageV2.SYNTHETIC_ATTACHMENT_PROMPT },
{ type: "file", mediaType: "image/png" },
],
},
])
expect(ProviderTransform.message(unsupported, model, {})).toMatchObject([
{ role: "user" },
{ role: "assistant" },
{ role: "tool" },
{
role: "user",
content: [
{ type: "text", text: MessageV2.SYNTHETIC_ATTACHMENT_PROMPT },
{
type: "text",
text: 'ERROR: Cannot read "attachment.png" (this model does not support image input). Inform the user.',
},
],
},
])
const legacyModel: Provider.Model = {
...model,
api: { ...model.api, npm: "@ai-sdk/openai-compatible" },
capabilities: { ...model.capabilities, attachment: true },
}
expect(
ProviderTransform.message(await MessageV2.toModelMessages(input, legacyModel), legacyModel, {}),
).toMatchObject([
{ role: "user" },
{ role: "assistant" },
{ role: "tool" },
{
role: "user",
content: [
{ type: "text", text: MessageV2.SYNTHETIC_ATTACHMENT_PROMPT },
{ type: "file", mediaType: "image/png" },
],
},
])
})
test("preserves jpeg tool-result media for anthropic models", async () => {