Compare commits

...
Author SHA1 Message Date
Ryan Vogel 50dde6a646 fix(core): restore webfetch image output 2026-07-21 14:05:22 +00:00
3 changed files with 96 additions and 13 deletions
+37 -5
View File
@@ -36,6 +36,14 @@ const Output = Schema.Struct({
contentType: Schema.String,
format: Input.fields.format,
output: Schema.String,
data: Schema.String.pipe(Schema.optional),
mime: Schema.String.pipe(Schema.optional),
})
const StructuredOutput = Schema.Struct({
url: Output.fields.url,
contentType: Output.fields.contentType,
format: Output.fields.format,
output: Output.fields.output,
})
type Format = (typeof Input.Type)["format"]
@@ -104,7 +112,9 @@ const isTextualMime = (mime: string) =>
mime === "application/xml" ||
mime.endsWith("+xml") ||
mime === "application/javascript" ||
mime === "application/x-javascript"
mime === "application/x-javascript" ||
mime === "image/svg+xml" ||
mime === "image/vnd.fastbidsheet"
const convert = (content: string, contentType: string, format: Format) => {
if (!contentType.includes("text/html")) return content
if (format === "markdown") return convertHTMLToMarkdown(content)
@@ -126,7 +136,21 @@ export const Plugin = {
description,
input: Input,
output: Output,
toModelOutput: ({ output }) => [{ type: "text", text: output.output }],
structured: StructuredOutput,
// Image base64 belongs in the native file content only so it is not persisted twice.
toStructuredOutput: ({ output }) => ({
url: output.url,
contentType: output.contentType,
format: output.format,
output: output.output,
}),
toModelOutput: ({ input, output }) => {
if (output.data === undefined || output.mime === undefined) return [{ type: "text", text: output.output }]
return [
{ type: "text", text: output.output },
{ type: "file", data: output.data, mime: output.mime, name: input.url },
]
},
execute: (input, context) =>
Effect.gen(function* () {
yield* Effect.try({
@@ -150,9 +174,7 @@ export const Plugin = {
)
const contentType = response.headers["content-type"] || ""
const mime = mimeFrom(contentType)
if (isImageAttachment(mime))
return yield* Effect.fail(new Error(`Unsupported fetched image content type: ${mime}`))
if (!isTextualMime(mime))
if (!isTextualMime(mime) && !isImageAttachment(mime))
return yield* Effect.fail(new Error(`Unsupported fetched file content type: ${mime}`))
return { body: yield* collectBody(response), contentType }
}).pipe(
@@ -161,6 +183,16 @@ export const Plugin = {
orElse: () => Effect.fail(new Error("Request timed out")),
}),
)
const mime = mimeFrom(contentType)
if (isImageAttachment(mime))
return {
url: input.url,
contentType,
format: input.format,
output: "Image fetched successfully",
data: Buffer.from(body).toString("base64"),
mime,
}
const content = new TextDecoder().decode(body)
const output = yield* Effect.try({
try: () => convert(content, contentType, input.format),
+52 -5
View File
@@ -234,14 +234,41 @@ describe("WebFetchTool registration", () => {
}),
)
it.effect("keeps images and files unsupported until typed settlement can carry attachments", () =>
it.effect("returns images as native media and keeps other files unsupported", () =>
Effect.gen(function* () {
reset()
const registry = yield* ToolRegistry.Service
respond = () => Effect.succeed(new Response("png", { headers: { "content-type": "image/png" } }))
expect(yield* executeTool(registry, call({ url: "https://1.1.1.1/image", format: "html" }))).toEqual({
type: "error",
value: "Unable to fetch https://1.1.1.1/image",
respond = () => Effect.succeed(new Response("png", { headers: { "content-type": "IMAGE/PNG; charset=binary" } }))
expect(yield* settleTool(registry, call({ url: "https://1.1.1.1/image", format: "html" }))).toEqual({
result: {
type: "content",
value: [
{ type: "text", text: "Image fetched successfully" },
{
type: "file",
uri: "data:image/png;base64,cG5n",
mime: "image/png",
name: "https://1.1.1.1/image",
},
],
},
output: {
structured: {
url: "https://1.1.1.1/image",
contentType: "IMAGE/PNG; charset=binary",
format: "html",
output: "Image fetched successfully",
},
content: [
{ type: "text", text: "Image fetched successfully" },
{
type: "file",
uri: "data:image/png;base64,cG5n",
mime: "image/png",
name: "https://1.1.1.1/image",
},
],
},
})
respond = () => Effect.succeed(new Response("pdf", { headers: { "content-type": "application/pdf" } }))
@@ -252,6 +279,26 @@ describe("WebFetchTool registration", () => {
}),
)
it.effect("keeps SVG and FastBid sheets as text instead of image media", () =>
Effect.gen(function* () {
reset()
const registry = yield* ToolRegistry.Service
respond = () =>
Effect.succeed(new Response("<svg><text>hello</text></svg>", { headers: { "content-type": "image/svg+xml" } }))
expect(yield* executeTool(registry, call({ url: "https://1.1.1.1/image.svg", format: "html" }))).toEqual({
type: "text",
value: "<svg><text>hello</text></svg>",
})
respond = () =>
Effect.succeed(new Response("table Sheet {}", { headers: { "content-type": "image/vnd.fastbidsheet" } }))
expect(yield* executeTool(registry, call({ url: "https://1.1.1.1/sheet.fbs", format: "text" }))).toEqual({
type: "text",
value: "table Sheet {}",
})
}),
)
it.effect("retries Cloudflare challenges with an honest user agent", () =>
Effect.gen(function* () {
reset()
+7 -3
View File
@@ -17,6 +17,10 @@ and other binary prompt attachments are not currently included in the model
request. Some clients may let you select a PDF, but V2 does not yet make that
PDF visible to the model.
The built-in `webfetch` tool returns HTTP image responses as image media when
the response uses an `image/*` content type. SVG and FastBid sheet responses
remain text.
<Warning>
Use a model that supports image input before attaching an image. OpenCode
passes supported image media to the selected provider, but the provider and
@@ -133,9 +137,9 @@ All fields are optional:
<Note>
In the current V2 runtime, these settings apply to image media produced by
the built-in `read` tool. Images attached directly through the TUI, desktop,
web, CLI, or API bypass this normalization. Resize direct attachments before
adding them if the provider requires smaller media.
built-in tools such as `read` and `webfetch`. Images attached directly
through the TUI, desktop, web, CLI, or API bypass this normalization. Resize
direct attachments before adding them if the provider requires smaller media.
</Note>
The `read` tool recognizes PNG, JPEG, GIF, and WebP by their contents and will