Compare commits

...
Author SHA1 Message Date
Aiden Cline 1c339219c1 fix(core): share web search limits with OpenCode Web Search
OpenCode Web Search, the default search provider for Console-connected
users, made its own request with no size cap and discarded the body of
failed responses. It now uses the shared response handling, and OpenCode
API error bodies ({ _tag, message }) supply the failure explanation.

Reading a failed response for its explanation is bounded to one second
so a stalled error body still fails promptly.

Also formats the earlier changes with Prettier and tidies the tests.
2026-09-23 12:57:33 -05:00
Aiden Cline a7f3ba9a2f fix(core): make web search limits non-fatal and report real errors
Oversized provider responses failed the whole search: WebSearchMcp.call
rejected bodies over 256 KB, and a single 424 KB Firecrawl result was
enough. The tool then replaced the cause with a generic message.

All providers now read at most 1 MiB, parse a cut-off body as partial
JSON, and keep the results that arrived complete. WebSearch.query caps
each result's content at 4,000 characters so one page cannot crowd out
the rest of the bounded tool output.

Web search errors now carry readable messages, including the provider's
own explanation for non-2xx responses, MCP isError results, and
JSON-RPC errors, which previously surfaced as no results.
2026-09-23 11:44:55 -05:00
14 changed files with 607 additions and 185 deletions
+21 -15
View File
@@ -1,4 +1,4 @@
import { Duration, Effect, Equal, Schema, Semaphore, Stream } from "effect"
import { Duration, Effect, Equal, Option, Schema, Semaphore, Stream } from "effect"
import type { Scope } from "effect"
import type { IntegrationOAuthMethodRegistration } from "@opencode/plugin/effect/integration"
import { define } from "@opencode/plugin/effect/plugin"
@@ -11,6 +11,7 @@ import { IntegrationConnection } from "../../integration/connection.js"
import { ManagedPolicy } from "../../managed-policy.js"
import { Provider } from "../../provider.js"
import { WebSearch } from "../../websearch.js"
import { WebSearchResponse } from "../websearch/response.js"
import { ConfigPolicy } from "@opencode/schema/config/policy"
import { ConfigProvider } from "@opencode/schema/config/provider"
import { Money } from "@opencode/schema/money"
@@ -45,6 +46,9 @@ const TokenPending = Schema.Struct({ error: Schema.String })
const DeviceToken = Schema.Union([Token, TokenPending])
const User = Schema.Struct({ id: Schema.String, email: Schema.String })
const Org = Schema.Struct({ id: Schema.String, name: Schema.String })
const decodeWebSearchResponse = Schema.decodeUnknownOption(
Schema.Struct({ providerID: WebSearch.ID, results: Schema.Array(Schema.Unknown) }),
)
function oauth(http: HttpClient.HttpClient) {
return {
@@ -313,25 +317,27 @@ export const OpencodePlugin = define<HttpClient.HttpClient | Bus.Service | Manag
providerID: descriptor.providerID,
}),
)
const response = yield* HttpClient.withScope(HttpClient.filterStatusOk(http))
.execute(request)
.pipe(
Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }),
Effect.flatMap(HttpClientResponse.schemaBodyJson(WebSearch.Response)),
Effect.scoped,
Effect.timeoutOrElse({
duration: Duration.seconds(25),
orElse: () => Effect.fail(new Error("OpenCode web search request timed out")),
}),
)
if (response.providerID !== descriptor.providerID) {
const body = yield* WebSearchResponse.execute(http, request).pipe(
Effect.provideService(FetchHttpClient.RequestInit, { redirect: "error" }),
Effect.scoped,
Effect.timeoutOrElse({
duration: Duration.seconds(25),
orElse: () => Effect.fail(new Error("OpenCode web search request timed out")),
}),
)
const response = WebSearchResponse.json(body.text, body.truncated).pipe(
Option.flatMap(decodeWebSearchResponse),
)
if (Option.isNone(response))
return yield* Effect.fail(new Error("OpenCode web search returned an invalid response"))
if (response.value.providerID !== descriptor.providerID) {
return yield* Effect.fail(
new Error(
`OpenCode web search returned provider ${response.providerID} instead of ${descriptor.providerID}`,
`OpenCode web search returned provider ${response.value.providerID} instead of ${descriptor.providerID}`,
),
)
}
return response.results
return WebSearchResponse.items(WebSearch.Result, response.value.results, body.truncated)
}),
})
editor.default.set(descriptor.providerID)
+7 -5
View File
@@ -47,23 +47,25 @@ export const Plugin = define<HttpClient.HttpClient | Scope.Scope>({
const credential = connection ? yield* ctx.integration.connection.resolve(connection) : undefined
const url = new URL(endpoint)
if (credential?.type === "key") url.searchParams.set("exaApiKey", credential.key)
const result = yield* WebSearchMcp.call(
const response = yield* WebSearchMcp.call(
http,
url.toString(),
"web_search_exa",
{ input: McpInput, output: McpOutput },
{ query: input.query, numResults: 8 },
)
const content = result?.content.find((item) => item.text)
return content ? parseResults(content.text) : []
const content = response.result?.content.find((item) => item.text)
return content ? parseResults(content.text, response.truncated) : []
}),
})
})
}),
})
function parseResults(text: string) {
return text.split(/\n\n---\n\n/).flatMap((block) => {
function parseResults(text: string, truncated: boolean) {
const blocks = text.split(/\n\n---\n\n/)
// The final block of a truncated response may end mid-line, including inside its URL.
return (truncated ? blocks.slice(0, -1) : blocks).flatMap((block) => {
const url = block.match(/^URL:\s*(.+)$/m)?.[1]?.trim()
if (!url) return []
const title = block.match(/^Title:\s*(.+)$/m)?.[1]?.trim()
+22 -25
View File
@@ -5,6 +5,7 @@ import { Effect, Option, Schema, Scope } from "effect"
import { HttpClient } from "effect/unstable/http"
import { App } from "../../app.js"
import { WebSearchMcp } from "./mcp.js"
import { WebSearchResponse } from "./response.js"
export const endpoint = "https://mcp.firecrawl.dev/v2/mcp"
@@ -17,21 +18,15 @@ const McpOutput = Schema.Struct({
content: Schema.Array(Schema.Struct({ type: Schema.Literal("text"), text: Schema.String })),
})
const SearchResponse = Schema.fromJsonString(
Schema.Struct({
success: Schema.Boolean,
data: Schema.Struct({
web: Schema.Array(
Schema.Struct({
url: Schema.String,
title: Schema.NullOr(Schema.String).pipe(Schema.optional),
description: Schema.NullOr(Schema.String).pipe(Schema.optional),
}),
),
}),
}),
const decodeSearchResponse = Schema.decodeUnknownOption(
Schema.Struct({ data: Schema.Struct({ web: Schema.Array(Schema.Unknown) }) }),
)
const decodeSearchResponse = Schema.decodeUnknownOption(SearchResponse)
const SearchResult = Schema.Struct({
url: Schema.String,
title: Schema.NullOr(Schema.String).pipe(Schema.optional),
description: Schema.NullOr(Schema.String).pipe(Schema.optional),
})
export const Plugin = define<HttpClient.HttpClient | Scope.Scope>({
id: "opencode.websearch.firecrawl",
@@ -56,7 +51,7 @@ export const Plugin = define<HttpClient.HttpClient | Scope.Scope>({
Effect.gen(function* () {
const connection = yield* ctx.integration.connection.active("firecrawl")
const credential = connection ? yield* ctx.integration.connection.resolve(connection) : undefined
const result = yield* WebSearchMcp.call(
const response = yield* WebSearchMcp.call(
http,
endpoint,
"firecrawl_search",
@@ -67,16 +62,18 @@ export const Plugin = define<HttpClient.HttpClient | Scope.Scope>({
...(credential?.type === "key" ? { Authorization: `Bearer ${credential.key}` } : {}),
},
)
const content = result?.content.find((item) => item.text)
const response = content ? Option.getOrUndefined(decodeSearchResponse(content.text)) : undefined
return (
response?.data.web.map((item) => ({
url: item.url,
...(item.title ? { title: item.title } : {}),
...(item.description ? { content: item.description } : {}),
time: {},
})) ?? []
)
const content = response.result?.content.find((item) => item.text)
const search = content
? Option.getOrUndefined(
WebSearchResponse.json(content.text, response.truncated).pipe(Option.flatMap(decodeSearchResponse)),
)
: undefined
return WebSearchResponse.items(SearchResult, search?.data.web ?? [], response.truncated).map((item) => ({
url: item.url,
...(item.title ? { title: item.title } : {}),
...(item.description ? { content: item.description } : {}),
time: {},
}))
}),
})
})
+16 -25
View File
@@ -1,26 +1,22 @@
export * as WebSearchMcp from "./mcp.js"
import { Duration, Effect, Schema } from "effect"
import { Duration, Effect, Option, Schema } from "effect"
import { HttpClient, HttpClientRequest } from "effect/unstable/http"
import { collectBoundedResponseBody } from "../../tool/http-body.js"
import { WebSearchResponse } from "./response.js"
export const MAX_RESPONSE_BYTES = 256 * 1024
export const parseResponse = <F extends Schema.Struct.Fields>(body: string, result: Schema.Struct<F>) => {
const decode = Schema.decodeUnknownEffect(Schema.fromJsonString(Schema.Struct({ result })))
const parse = (payload: string) => {
const trimmed = payload.trim()
if (!trimmed.startsWith("{")) return Effect.undefined
return decode(trimmed).pipe(Effect.map((response) => response.result))
}
export const parseResponse = <F extends Schema.Struct.Fields>(
body: { readonly text: string; readonly truncated: boolean },
tool: string,
result: Schema.Struct<F>,
) => {
const decode = Schema.decodeUnknownEffect(Schema.Struct({ result }))
return Effect.gen(function* () {
const trimmed = body.trim()
const direct = trimmed ? yield* parse(trimmed) : undefined
if (direct) return direct
for (const line of body.split("\n")) {
if (!line.startsWith("data: ")) continue
const data = yield* parse(line.substring(6))
if (data) return data
for (const payload of WebSearchResponse.payloads(body.text)) {
const message = WebSearchResponse.json(payload, body.truncated)
if (Option.isNone(message)) return yield* Effect.fail(new Error(`${tool} returned invalid JSON`))
const failure = WebSearchResponse.failure(message.value)
if (failure !== undefined) return yield* Effect.fail(new Error(failure))
return (yield* decode(message.value)).result
}
})
}
@@ -52,13 +48,8 @@ export const call = <F extends Schema.Struct.Fields, R extends Schema.Struct.Fie
}),
)
return yield* Effect.gen(function* () {
const response = yield* HttpClient.withScope(HttpClient.filterStatusOk(http)).execute(request)
const body = yield* collectBoundedResponseBody(
response,
MAX_RESPONSE_BYTES,
() => new Error(`${tool} response exceeded ${MAX_RESPONSE_BYTES} bytes`),
)
return yield* parseResponse(body.toString("utf8"), schema.output)
const body = yield* WebSearchResponse.execute(http, request)
return { result: yield* parseResponse(body, tool, schema.output), truncated: body.truncated }
}).pipe(
Effect.scoped,
Effect.timeoutOrElse({
+31 -42
View File
@@ -1,10 +1,11 @@
export * as WebSearchParallel from "./parallel.js"
import { define } from "@opencode/plugin/effect/plugin"
import { Effect, Schema, Scope } from "effect"
import { Effect, Option, Schema, Scope } from "effect"
import { HttpClient } from "effect/unstable/http"
import { App } from "../../app.js"
import { WebSearchMcp } from "./mcp.js"
import { WebSearchResponse } from "./response.js"
export const endpoint = "https://search.parallel.ai/mcp"
@@ -14,38 +15,19 @@ const McpInput = Schema.Struct({
model_name: Schema.String.check(Schema.isMaxLength(100)).pipe(Schema.optional),
})
const SearchResponse = Schema.Struct({
search_id: Schema.String,
results: Schema.Array(
Schema.Struct({
url: Schema.String,
title: Schema.NullOr(Schema.String).pipe(Schema.optional),
publish_date: Schema.NullOr(Schema.String).pipe(Schema.optional),
excerpts: Schema.Array(Schema.String),
}),
),
warnings: Schema.NullOr(
Schema.Array(
Schema.Struct({
type: Schema.Literals(["spec_validation_warning", "input_validation_warning", "warning"]),
message: Schema.String,
detail: Schema.NullOr(Schema.Record(Schema.String, Schema.Json)).pipe(Schema.optional),
}),
),
).pipe(Schema.optional),
usage: Schema.NullOr(
Schema.Array(
Schema.Struct({
name: Schema.String,
count: Schema.Int,
}),
),
).pipe(Schema.optional),
session_id: Schema.String,
})
const McpOutput = Schema.Struct({
content: Schema.Array(Schema.Struct({ type: Schema.Literal("text"), text: Schema.String })),
structuredContent: SearchResponse,
// Parallel sends this after a text copy of the same JSON, so a truncated response can lose it.
structuredContent: Schema.Unknown.pipe(Schema.optional),
})
const decodeSearchResponse = Schema.decodeUnknownOption(Schema.Struct({ results: Schema.Array(Schema.Unknown) }))
const SearchResult = Schema.Struct({
url: Schema.String,
title: Schema.NullOr(Schema.String).pipe(Schema.optional),
publish_date: Schema.NullOr(Schema.String).pipe(Schema.optional),
excerpts: Schema.Array(Schema.String),
})
export const Plugin = define<HttpClient.HttpClient | Scope.Scope>({
@@ -71,7 +53,7 @@ export const Plugin = define<HttpClient.HttpClient | Scope.Scope>({
Effect.gen(function* () {
const connection = yield* ctx.integration.connection.active("parallel")
const credential = connection ? yield* ctx.integration.connection.resolve(connection) : undefined
const result = yield* WebSearchMcp.call(
const response = yield* WebSearchMcp.call(
http,
endpoint,
"web_search",
@@ -85,17 +67,24 @@ export const Plugin = define<HttpClient.HttpClient | Scope.Scope>({
...(credential?.type === "key" ? { Authorization: `Bearer ${credential.key}` } : {}),
},
)
return (
result?.structuredContent.results.map((item) => {
const published = item.publish_date ? Date.parse(item.publish_date) : undefined
return {
url: item.url,
...(item.title ? { title: item.title } : {}),
...(item.excerpts.length ? { content: item.excerpts.join("\n\n") } : {}),
time: { ...(published !== undefined && Number.isFinite(published) ? { published } : {}) },
}
}) ?? []
const content = response.result?.content.find((item) => item.text)
const search = Option.getOrUndefined(
decodeSearchResponse(
response.result?.structuredContent ??
(content
? Option.getOrUndefined(WebSearchResponse.json(content.text, response.truncated))
: undefined),
),
)
return WebSearchResponse.items(SearchResult, search?.results ?? [], response.truncated).map((item) => {
const published = item.publish_date ? Date.parse(item.publish_date) : undefined
return {
url: item.url,
...(item.title ? { title: item.title } : {}),
...(item.excerpts.length ? { content: item.excerpts.join("\n\n") } : {}),
time: { ...(published !== undefined && Number.isFinite(published) ? { published } : {}) },
}
})
}),
})
})
@@ -0,0 +1,88 @@
export * as WebSearchResponse from "./response.js"
import { parseJSON } from "@opencode/ai/protocols/utils/partial-json"
import { Duration, Effect, Option, Schema, Stream } from "effect"
import { HttpClient, HttpClientError, type HttpClientRequest, type HttpClientResponse } from "effect/unstable/http"
export const MAX_BYTES = 1024 * 1024
// Keeps the status for rate-limit failover and adds the provider's own explanation of the failure.
export const execute = (http: HttpClient.HttpClient, request: HttpClientRequest.HttpClientRequest) =>
Effect.gen(function* () {
const response = yield* HttpClient.withScope(http).execute(request)
if (response.status >= 200 && response.status < 300) return yield* read(response)
return yield* new HttpClientError.HttpClientError({
reason: new HttpClientError.StatusCodeError({
request,
response,
description: yield* read(response).pipe(
Effect.map((body) =>
payloads(body.text)
.flatMap((payload) => Option.toArray(json(payload, body.truncated)))
.map(failure)
.find((message) => message !== undefined),
),
// The explanation is optional; a slow or broken body must not delay the failure.
Effect.timeoutOrElse({ duration: Duration.seconds(1), orElse: () => Effect.undefined }),
Effect.orElseSucceed(() => undefined),
),
}),
})
})
// Stops reading at MAX_BYTES instead of failing; parsers keep the results that arrived complete.
const read = (response: HttpClientResponse.HttpClientResponse) =>
Effect.gen(function* () {
let size = 0
const chunks = yield* response.stream.pipe(
Stream.takeUntil((chunk) => (size += chunk.byteLength) > MAX_BYTES),
Stream.runCollect,
)
return { text: Buffer.concat(chunks).subarray(0, MAX_BYTES).toString("utf8"), truncated: size > MAX_BYTES }
})
// A JSON body, or the data lines of a server-sent event stream.
export const payloads = (text: string) =>
[text, ...text.split("\n").flatMap((line) => (line.startsWith("data: ") ? [line.slice(6)] : []))]
.map((payload) => payload.trim())
.filter((payload) => payload.startsWith("{"))
const decodeJson = Schema.decodeUnknownOption(Schema.fromJsonString(Schema.Unknown))
const decodePartialJson = Option.liftThrowable(parseJSON)
export const json = (text: string, truncated: boolean) => (truncated ? decodePartialJson(text) : decodeJson(text))
const decodeFailure = Schema.decodeUnknownOption(
Schema.Union([
Schema.Struct({ error: Schema.Struct({ message: Schema.String }) }),
Schema.Struct({
result: Schema.Struct({
isError: Schema.Literal(true),
content: Schema.Array(Schema.Struct({ text: Schema.String })),
}),
}),
Schema.Struct({ detail: Schema.Struct({ error: Schema.String }) }),
Schema.Struct({ _tag: Schema.String, message: Schema.String }),
]),
)
// JSON-RPC errors, MCP tool errors, Tavily's error detail, and OpenCode API errors.
export const failure = (value: unknown) =>
Option.getOrUndefined(
Option.map(decodeFailure(value), (decoded) => {
if ("error" in decoded) return decoded.error.message
if ("result" in decoded) return decoded.result.content.map((item) => item.text).join("\n")
if ("detail" in decoded) return decoded.detail.error
return decoded.message
}),
)
// The last item of a truncated response may be cut mid-value, so it is dropped rather than trusted.
export const items = <S extends Schema.ConstraintDecoder<unknown>>(
schema: S,
values: readonly unknown[],
truncated: boolean,
) => {
const decode = Schema.decodeUnknownOption(schema)
return (truncated ? values.slice(0, -1) : values).flatMap((value) => Option.toArray(decode(value)))
}
+19 -21
View File
@@ -1,9 +1,10 @@
export * as WebSearchTavily from "./tavily.js"
import { define } from "@opencode/plugin/effect/plugin"
import { Duration, Effect, Schema, Scope } from "effect"
import { HttpClient, HttpClientRequest, HttpClientResponse } from "effect/unstable/http"
import { Duration, Effect, Option, Schema, Scope } from "effect"
import { HttpClient, HttpClientRequest } from "effect/unstable/http"
import { App } from "../../app.js"
import { WebSearchResponse } from "./response.js"
export const endpoint = "https://api.tavily.com/search"
@@ -14,14 +15,12 @@ const SearchRequest = Schema.Struct({
max_results: Schema.Number,
})
const SearchResponse = Schema.Struct({
results: Schema.Array(
Schema.Struct({
title: Schema.String,
url: Schema.String,
content: Schema.String,
}),
),
const decodeSearchResponse = Schema.decodeUnknownOption(Schema.Struct({ results: Schema.Array(Schema.Unknown) }))
const SearchResult = Schema.Struct({
title: Schema.String,
url: Schema.String,
content: Schema.String,
})
export const Plugin = define<HttpClient.HttpClient | Scope.Scope>({
@@ -63,17 +62,16 @@ export const Plugin = define<HttpClient.HttpClient | Scope.Scope>({
max_results: 8,
}),
)
const response = yield* HttpClient.withScope(HttpClient.filterStatusOk(http))
.execute(request)
.pipe(
Effect.flatMap(HttpClientResponse.schemaBodyJson(SearchResponse)),
Effect.scoped,
Effect.timeoutOrElse({
duration: Duration.seconds(25),
orElse: () => Effect.fail(new Error("Tavily web search request timed out")),
}),
)
return response.results.map((item) => ({
const body = yield* WebSearchResponse.execute(http, request).pipe(
Effect.scoped,
Effect.timeoutOrElse({
duration: Duration.seconds(25),
orElse: () => Effect.fail(new Error("Tavily web search request timed out")),
}),
)
const search = WebSearchResponse.json(body.text, body.truncated).pipe(Option.flatMap(decodeSearchResponse))
if (Option.isNone(search)) return yield* Effect.fail(new Error("Tavily returned an invalid response"))
return WebSearchResponse.items(SearchResult, search.value.results, body.truncated).map((item) => ({
url: item.url,
title: item.title,
...(item.content ? { content: item.content } : {}),
+21 -23
View File
@@ -5,6 +5,7 @@ import { Effect, Option, Schema, Scope } from "effect"
import { HttpClient } from "effect/unstable/http"
import { App } from "../../app.js"
import { WebSearchMcp } from "./mcp.js"
import { WebSearchResponse } from "./response.js"
export const endpoint = "https://agent.tinyfish.ai/mcp"
@@ -16,18 +17,13 @@ const McpOutput = Schema.Struct({
content: Schema.Array(Schema.Struct({ type: Schema.Literal("text"), text: Schema.String })),
})
const SearchResponse = Schema.fromJsonString(
Schema.Struct({
results: Schema.Array(
Schema.Struct({
url: Schema.String,
title: Schema.String,
snippet: Schema.String,
}),
),
}),
)
const decodeSearchResponse = Schema.decodeUnknownOption(SearchResponse)
const decodeSearchResponse = Schema.decodeUnknownOption(Schema.Struct({ results: Schema.Array(Schema.Unknown) }))
const SearchResult = Schema.Struct({
url: Schema.String,
title: Schema.String,
snippet: Schema.String,
})
export const Plugin = define<HttpClient.HttpClient | Scope.Scope>({
id: "opencode.websearch.tinyfish",
@@ -52,7 +48,7 @@ export const Plugin = define<HttpClient.HttpClient | Scope.Scope>({
Effect.gen(function* () {
const connection = yield* ctx.integration.connection.active("tinyfish")
const credential = connection ? yield* ctx.integration.connection.resolve(connection) : undefined
const result = yield* WebSearchMcp.call(
const response = yield* WebSearchMcp.call(
http,
endpoint,
"search",
@@ -65,16 +61,18 @@ export const Plugin = define<HttpClient.HttpClient | Scope.Scope>({
: { "X-TinyFish-Access-Mode": "keyless" }),
},
)
const content = result?.content.find((item) => item.text)
const response = content ? Option.getOrUndefined(decodeSearchResponse(content.text)) : undefined
return (
response?.results.map((item) => ({
url: item.url,
title: item.title,
...(item.snippet ? { content: item.snippet } : {}),
time: {},
})) ?? []
)
const content = response.result?.content.find((item) => item.text)
const search = content
? Option.getOrUndefined(
WebSearchResponse.json(content.text, response.truncated).pipe(Option.flatMap(decodeSearchResponse)),
)
: undefined
return WebSearchResponse.items(SearchResult, search?.results ?? [], response.truncated).map((item) => ({
url: item.url,
title: item.title,
...(item.snippet ? { content: item.snippet } : {}),
time: {},
}))
}),
})
})
+15 -16
View File
@@ -4,7 +4,6 @@ import type { Context } from "@opencode/plugin/effect/plugin"
import type { SessionHooks } from "@opencode/plugin/effect/session"
import { ToolFailure } from "@opencode/ai"
import { Effect, Schema, Semaphore } from "effect"
import { HttpClientError } from "effect/unstable/http"
import { Form } from "../../form.js"
import { Permission } from "../../permission.js"
import { WebSearch } from "../../websearch.js"
@@ -12,10 +11,6 @@ import { WebSearch } from "../../websearch.js"
export const name = "websearch"
export const NO_RESULTS = "No search results found. Please try a different query."
const providerSelectionLock = Semaphore.makeUnsafe(1)
const httpErrors = new Map([
[429, "Web search rate limited (HTTP 429)"],
[401, "Web search authentication failed (HTTP 401)"],
])
export const description = `Search the web using the user's selected search integration. Use this for current information beyond knowledge cutoff.
@@ -164,17 +159,21 @@ export const Plugin = {
return { output, content, metadata: { provider: output.provider } }
}).pipe(
Effect.mapError((error) => {
const fallback = `Unable to search the web for ${input.query}`
if (!Schema.is(WebSearch.RequestError)(error)) return new ToolFailure({ message: fallback, error })
const status = HttpClientError.isHttpClientError(error.cause) ? error.cause.response?.status : undefined
return new ToolFailure({
message:
status === undefined
? fallback
: (httpErrors.get(status) ?? `Web search request failed (HTTP ${status})`),
error,
metadata: { provider: error.providerID },
})
const message = `Unable to search the web for ${input.query}`
// A cause becomes the model-visible error, so web search failures are described here instead;
// other causes, such as permission errors, keep their own session error.
if (error instanceof WebSearch.RequestError)
return new ToolFailure({
message: `${message} (${error.providerID}): ${error.message}`,
metadata: { provider: error.providerID },
})
if (
error instanceof WebSearch.ProviderRequiredError ||
error instanceof WebSearch.ProviderNotFoundError ||
error instanceof WebSearch.DisabledError
)
return new ToolFailure({ message: `${message}: ${error.message}` })
return new ToolFailure({ message, error })
}),
),
}),
+48 -5
View File
@@ -29,6 +29,8 @@ export const Response = WebSearch.Response
export type Response = WebSearch.Response
export const ProviderKey = "websearch:provider"
// Keeps one oversized page from crowding the other results out of the bounded tool output.
export const MAX_RESULT_CONTENT_LENGTH = 4_000
export const Selection = Schema.Union([ID, Schema.Literal("random"), Schema.Literal(false)])
export type Selection = typeof Selection.Type
@@ -39,18 +41,45 @@ export interface ProviderImplementation extends Provider {
export class ProviderRequiredError extends Schema.TaggedError<ProviderRequiredError>()(
"WebSearch.ProviderRequired",
{},
) {}
) {
override get message() {
return "No web search provider is selected"
}
}
export class ProviderNotFoundError extends Schema.TaggedError<ProviderNotFoundError>()("WebSearch.ProviderNotFound", {
providerID: ID,
}) {}
}) {
override get message() {
return `Web search provider not found: ${this.providerID}`
}
}
export class DisabledError extends Schema.TaggedError<DisabledError>()("WebSearch.Disabled", {}) {}
export class DisabledError extends Schema.TaggedError<DisabledError>()("WebSearch.Disabled", {}) {
override get message() {
return "Web search is disabled"
}
}
export class RequestError extends Schema.TaggedError<RequestError>()("WebSearch.Request", {
providerID: ID,
cause: Schema.Defect(),
}) {}
}) {
// HTTP client messages include the request URL, which can carry a provider credential.
override get message() {
const cause = this.cause
if (!HttpClientError.isHttpClientError(cause))
return cause instanceof Error && cause.message ? cause.message : "Request failed"
const status = cause.response?.status
if (status !== undefined && cause.reason.description) return `HTTP ${status}: ${cause.reason.description}`
if (status === 429) return "Rate limited (HTTP 429)"
if (status === 401) return "Authentication failed (HTTP 401)"
if (status !== undefined) return `Request failed (HTTP ${status})`
return cause.cause instanceof Error && cause.cause.message
? `Request failed: ${cause.cause.message}`
: "Request failed"
}
}
export type Error = ProviderRequiredError | ProviderNotFoundError | DisabledError | RequestError
@@ -195,7 +224,13 @@ const layer = Layer.effect(
const result = yield* provider
.execute({ query: input.query })
.pipe(Effect.flatMap(decodeResults), Effect.result)
if (result._tag === "Success") return new Response({ providerID: provider.id, results: result.success })
if (result._tag === "Success")
return new Response({
providerID: provider.id,
results: result.success.map((item) =>
item.content === undefined ? item : { ...item, content: limitContent(item.content) },
),
})
const cause = result.failure
const error = new RequestError({ providerID: provider.id, cause })
if (choice !== "random" || !HttpClientError.isHttpClientError(cause) || cause.response?.status !== 429)
@@ -212,6 +247,14 @@ const layer = Layer.effect(
}),
)
function limitContent(content: string) {
if (content.length <= MAX_RESULT_CONTENT_LENGTH) return content
const code = content.charCodeAt(MAX_RESULT_CONTENT_LENGTH - 1)
// Do not split a surrogate pair; a lone surrogate is invalid text for model requests.
const end = code >= 0xd800 && code <= 0xdbff ? MAX_RESULT_CONTENT_LENGTH - 1 : MAX_RESULT_CONTENT_LENGTH
return `${content.slice(0, end)}\n[truncated]`
}
function cooldownMillis(value: string | undefined, now: number) {
if (!value?.trim()) return 60_000
const seconds = Number(value)
@@ -17,6 +17,7 @@ import { ModelResolver } from "@opencode/core/model-resolver"
import { Plugin } from "@opencode/core/plugin"
import { PluginHost } from "@opencode/core/plugin/host"
import { OpencodePlugin } from "@opencode/core/plugin/provider/opencode"
import { WebSearchResponse } from "@opencode/core/plugin/websearch/response"
import { Provider } from "@opencode/core/provider"
import { WebSearch } from "@opencode/core/websearch"
import { withEnv } from "../fixture/env"
@@ -805,7 +806,13 @@ describe("OpencodePlugin", () => {
body?: unknown
}> = []
const gate = Promise.withResolvers<void>()
const state = { advertised: true, providerID: "opencode", waitForConfig: false }
const state = {
advertised: true,
providerID: "opencode",
waitForConfig: false,
unavailable: false,
oversized: false,
}
const server = Bun.serve({
port: 0,
fetch: async (request) => {
@@ -832,6 +839,11 @@ describe("OpencodePlugin", () => {
})
}
if (path === "/api/websearch" || path === "/other/api/websearch") {
if (state.unavailable)
return Response.json(
{ _tag: "ServiceUnavailableError", message: "Web search request failed: firecrawl" },
{ status: 503 },
)
return Response.json({
providerID: state.providerID,
results: [
@@ -841,6 +853,16 @@ describe("OpencodePlugin", () => {
content: "Open source AI coding agent.",
time: { published: 1_700_000_000_000 },
},
...(state.oversized
? [
{
url: "https://huge.example.com",
content: "x".repeat(2 * WebSearchResponse.MAX_BYTES),
time: {},
},
{ url: "https://after.example.com", content: "after", time: {} },
]
: []),
],
})
}
@@ -927,6 +949,18 @@ describe("OpencodePlugin", () => {
value: account("replacement"),
})
state.oversized = true
expect((yield* websearch.query({ query: "oversized" })).results.map((result) => result.url)).toEqual([
"https://github.com/anomalyco/opencode",
])
state.oversized = false
state.unavailable = true
expect((yield* websearch.query({ query: "unavailable" }).pipe(Effect.flip)).message).toBe(
"HTTP 503: Web search request failed: firecrawl",
)
state.unavailable = false
state.providerID = "unexpected"
expect((yield* websearch.query({ query: "wrong provider" }).pipe(Effect.flip))._tag).toBe("WebSearch.Request")
+195
View File
@@ -5,6 +5,7 @@ import { WebSearch } from "@opencode/core/websearch"
import { WebSearchExa } from "@opencode/core/plugin/websearch/exa"
import { WebSearchFirecrawl } from "@opencode/core/plugin/websearch/firecrawl"
import { WebSearchParallel } from "@opencode/core/plugin/websearch/parallel"
import { WebSearchResponse } from "@opencode/core/plugin/websearch/response"
import { WebSearchTavily } from "@opencode/core/plugin/websearch/tavily"
import { WebSearchTinyFish } from "@opencode/core/plugin/websearch/tinyfish"
import { host, integrationHost, webSearchHost } from "./host"
@@ -29,6 +30,8 @@ beforeEach(() => {
})
const it = webSearchIntegrationTest
const sseMessage = (message: object) =>
`event: message\ndata: ${JSON.stringify({ jsonrpc: "2.0", id: 1, ...message })}\n\n`
describe("built-in web search providers", () => {
;[
@@ -93,6 +96,198 @@ describe("built-in web search providers", () => {
}),
)
describe("responses larger than the size cap", () => {
const huge = "x".repeat(2 * WebSearchResponse.MAX_BYTES)
const mcpText = (text: string) => sseMessage({ result: { content: [{ type: "text", text }] } })
const exaBlock = (url: string, title: string, content: string) =>
`Title: ${title}\nURL: ${url}\nPublished: N/A\nAuthor: N/A\nHighlights:\n${content}`
const parallelSearch = {
search_id: "search_1",
results: [
{ url: "https://effect.website", title: "Effect", publish_date: null, excerpts: ["Effect documentation"] },
{ url: "https://huge.example.com", title: "Huge", publish_date: null, excerpts: [huge] },
{ url: "https://after.example.com", title: "After", publish_date: null, excerpts: ["after"] },
],
session_id: "ses_parallel",
}
;[
{
plugin: WebSearchExa.Plugin,
providerID: WebSearch.ID.make("exa"),
body: mcpText(
[
exaBlock("https://effect.website", "Effect", "Effect documentation"),
exaBlock("https://huge.example.com", "Huge", huge),
exaBlock("https://after.example.com", "After", "after"),
].join("\n\n---\n\n"),
),
},
{
plugin: WebSearchFirecrawl.Plugin,
providerID: WebSearch.ID.make("firecrawl"),
body: mcpText(
JSON.stringify({
success: true,
data: {
web: [
{ url: "https://effect.website", title: "Effect", description: "Effect documentation" },
{ url: "https://huge.example.com", title: "Huge", description: huge },
{ url: "https://after.example.com", title: "After", description: "after" },
],
},
}),
),
},
{
plugin: WebSearchParallel.Plugin,
providerID: WebSearch.ID.make("parallel"),
body: JSON.stringify({
jsonrpc: "2.0",
id: 1,
result: {
content: [{ type: "text", text: JSON.stringify(parallelSearch) }],
structuredContent: parallelSearch,
},
}),
},
{
plugin: WebSearchTavily.Plugin,
providerID: WebSearch.ID.make("tavily"),
body: JSON.stringify({
results: [
{ title: "Effect", url: "https://effect.website", content: "Effect documentation" },
{ title: "Huge", url: "https://huge.example.com", content: huge },
{ title: "After", url: "https://after.example.com", content: "after" },
],
}),
},
{
plugin: WebSearchTinyFish.Plugin,
providerID: WebSearch.ID.make("tinyfish"),
body: mcpText(
JSON.stringify({
results: [
{ title: "Effect", url: "https://effect.website", snippet: "Effect documentation" },
{ title: "Huge", url: "https://huge.example.com", snippet: huge },
{ title: "After", url: "https://after.example.com", snippet: "after" },
],
}),
),
},
].forEach((provider) => {
it.effect(`keeps the complete results from ${provider.plugin.id}`, () =>
Effect.gen(function* () {
resetWebSearchFixture(provider.body)
const integrations = yield* Integration.Service
const websearch = yield* WebSearch.Service
yield* provider.plugin.effect(
host({ integration: integrationHost(integrations), websearch: webSearchHost(websearch) }),
)
expect(yield* websearch.query({ query: "effect", providerID: provider.providerID })).toEqual(
new WebSearch.Response({
providerID: provider.providerID,
results: [{ url: "https://effect.website", title: "Effect", content: "Effect documentation", time: {} }],
}),
)
}),
)
})
})
it.effect("reports MCP tool errors instead of returning no results", () =>
Effect.gen(function* () {
const integrations = yield* Integration.Service
const websearch = yield* WebSearch.Service
yield* WebSearchFirecrawl.Plugin.effect(
host({ integration: integrationHost(integrations), websearch: webSearchHost(websearch) }),
)
const search = websearch.query({ query: "effect", providerID: WebSearch.ID.make("firecrawl") }).pipe(Effect.flip)
resetWebSearchFixture(
sseMessage({ result: { isError: true, content: [{ type: "text", text: "Rate limit exceeded" }] } }),
)
expect((yield* search).message).toBe("Rate limit exceeded")
resetWebSearchFixture(sseMessage({ error: { code: -32602, message: "Invalid arguments" } }))
expect((yield* search).message).toBe("Invalid arguments")
}),
)
it.effect("reports the provider's explanation for HTTP failures", () =>
Effect.gen(function* () {
const integrations = yield* Integration.Service
const websearch = yield* WebSearch.Service
yield* WebSearchTinyFish.Plugin.effect(
host({ integration: integrationHost(integrations), websearch: webSearchHost(websearch) }),
)
yield* WebSearchTavily.Plugin.effect(
host({ integration: integrationHost(integrations), websearch: webSearchHost(websearch) }),
)
const quota =
"Free daily Search quota used (50/50). Sign up for continued access: https://agent.tinyfish.ai/sign-up"
resetWebSearchFixture(JSON.stringify({ jsonrpc: "2.0", error: { code: -31001, message: quota }, id: 1 }), 401)
const tinyfish = yield* websearch
.query({ query: "effect", providerID: WebSearch.ID.make("tinyfish") })
.pipe(Effect.flip)
expect(tinyfish.message).toBe(`HTTP 401: ${quota}`)
resetWebSearchFixture(JSON.stringify({ detail: { error: "Unauthorized: missing or invalid API key." } }), 401)
const tavily = yield* websearch
.query({ query: "effect", providerID: WebSearch.ID.make("tavily") })
.pipe(Effect.flip)
expect(tavily.message).toBe("HTTP 401: Unauthorized: missing or invalid API key.")
}),
)
it.effect("limits oversized Firecrawl results instead of failing the search", () =>
Effect.gen(function* () {
const thread = "comment ".repeat(64 * 1024)
resetWebSearchFixture(
`event: message\ndata: ${JSON.stringify({
jsonrpc: "2.0",
id: 1,
result: {
content: [
{
type: "text",
text: JSON.stringify({
success: true,
data: {
web: [
{ url: "https://news.ycombinator.com/item?id=1", title: "Thread", description: thread },
{ url: "https://effect.website", title: "Effect", description: "Effect documentation" },
],
},
}),
},
],
},
})}\n\n`,
)
const integrations = yield* Integration.Service
const websearch = yield* WebSearch.Service
yield* WebSearchFirecrawl.Plugin.effect(
host({ integration: integrationHost(integrations), websearch: webSearchHost(websearch) }),
)
expect(yield* websearch.query({ query: "opencode", providerID: WebSearch.ID.make("firecrawl") })).toEqual(
new WebSearch.Response({
providerID: WebSearch.ID.make("firecrawl"),
results: [
{
url: "https://news.ycombinator.com/item?id=1",
title: "Thread",
content: `${thread.slice(0, WebSearch.MAX_RESULT_CONTENT_LENGTH)}\n[truncated]`,
time: {},
},
{ url: "https://effect.website", title: "Effect", content: "Effect documentation", time: {} },
],
}),
)
}),
)
it.effect("registers Exa with its MCP schema", () =>
Effect.gen(function* () {
const integrations = yield* Integration.Service
+61 -7
View File
@@ -1,6 +1,6 @@
import { describe, expect } from "bun:test"
import { Context, Effect, Layer } from "effect"
import type { HttpClientError } from "effect/unstable/http"
import { HttpClientError, HttpClientRequest } from "effect/unstable/http"
import { AppNodeBuilder } from "@opencode/core/effect/app-node-builder"
import { LayerNode } from "@opencode/util/effect/layer-node"
import { Permission } from "@opencode/core/permission"
@@ -44,7 +44,7 @@ class Fixture {
formResponse: Form.TerminalState = { status: "cancelled" }
formResponses: Form.TerminalState[] = []
formWait = Effect.void
error: HttpClientError.HttpClientError | undefined
error: HttpClientError.HttpClientError | Error | undefined
results: readonly WebSearch.Result[] = [
{ url: "https://example.com", title: "Search results", content: "search results", time: {} },
]
@@ -62,7 +62,7 @@ const setup = Effect.gen(function* () {
execute: () =>
Effect.gen(function* () {
fixture.events.push("query")
if (fixture.error) return yield* fixture.error
if (fixture.error) return yield* Effect.fail(fixture.error)
return fixture.results
}),
}),
@@ -418,7 +418,7 @@ describe("WebSearchTool registration", () => {
.pipe(Effect.flip)
expect(toSessionError(error)).toEqual({
type: "tool.execution",
message: "Web search rate limited (HTTP 429)",
message: `Unable to search the web for ${query} (${error.metadata?.provider}): Rate limited (HTTP 429)`,
})
expect(error.metadata).toMatchObject({ provider: expect.stringMatching(/^(exa|parallel)$/) })
}),
@@ -437,9 +437,9 @@ describe("WebSearchTool registration", () => {
yield* Effect.forEach(
[
{ status: 403, message: "Web search request failed (HTTP 403)" },
{ status: 429, message: "Web search rate limited (HTTP 429)" },
{ status: 401, message: "Web search authentication failed (HTTP 401)" },
{ status: 403, message: "Unable to search the web for effect (exa): Request failed (HTTP 403)" },
{ status: 429, message: "Unable to search the web for effect (exa): Rate limited (HTTP 429)" },
{ status: 401, message: "Unable to search the web for effect (exa): Authentication failed (HTTP 401)" },
],
({ status, message }, index) =>
Effect.gen(function* () {
@@ -469,4 +469,58 @@ describe("WebSearchTool registration", () => {
)
}),
)
it.effect("reports the underlying reason for provider failures", () =>
Effect.gen(function* () {
const fixture = yield* setup
const tools = yield* fixture.registry.snapshot()
yield* fixture.websearch.select(WebSearch.ID.make("exa"))
yield* Effect.forEach(
[
{
error: new Error("web_search_exa request timed out"),
message: "Unable to search the web for effect (exa): web_search_exa request timed out",
},
{
error: new HttpClientError.HttpClientError({
reason: new HttpClientError.TransportError({
request: HttpClientRequest.post("https://mcp.exa.ai/mcp?exaApiKey=secret"),
cause: new TypeError("Unable to connect. Is the computer able to access the url?"),
}),
}),
message:
"Unable to search the web for effect (exa): Request failed: Unable to connect. Is the computer able to access the url?",
},
],
(item, index) =>
Effect.gen(function* () {
fixture.error = item.error
const error = yield* tools
.execute({
sessionID,
...toolIdentity,
call: { type: "tool-call", id: `call-reason-${index}`, name: "websearch", input: { query: "effect" } },
})
.pipe(Effect.flip)
expect(toSessionError(error)).toEqual({ type: "tool.execution", message: item.message })
expect(error.metadata).toEqual({ provider: "exa" })
}),
{ discard: true },
)
yield* fixture.websearch.select(false)
const disabled = yield* tools
.execute({
sessionID,
...toolIdentity,
call: { type: "tool-call", id: "call-disabled", name: "websearch", input: { query: "effect" } },
})
.pipe(Effect.flip)
expect(toSessionError(disabled)).toEqual({
type: "tool.execution",
message: "Unable to search the web for effect: Web search is disabled",
})
}),
)
})
+28
View File
@@ -86,6 +86,34 @@ describe("WebSearch", () => {
}),
)
it.effect("limits each result's content for every provider", () =>
Effect.gen(function* () {
const websearch = yield* WebSearch.Service
const providerID = WebSearch.ID.make("large")
const limit = WebSearch.MAX_RESULT_CONTENT_LENGTH
yield* websearch.transform((editor) => {
editor.add({
id: providerID,
name: "Large",
execute: () =>
Effect.succeed([
{ url: "https://large.example.com", content: "b".repeat(limit + 1), time: {} },
{ url: "https://emoji.example.com", content: `${"a".repeat(limit - 1)}😀 after the limit`, time: {} },
{ url: "https://exact.example.com", content: "c".repeat(limit), time: {} },
{ url: "https://empty.example.com", time: {} },
]),
})
})
expect((yield* websearch.query({ query: "large", providerID })).results).toEqual([
{ url: "https://large.example.com", content: `${"b".repeat(limit)}\n[truncated]`, time: {} },
{ url: "https://emoji.example.com", content: `${"a".repeat(limit - 1)}\n[truncated]`, time: {} },
{ url: "https://exact.example.com", content: "c".repeat(limit), time: {} },
{ url: "https://empty.example.com", time: {} },
])
}),
)
it.effect("requires a provider when no default is set", () =>
Effect.gen(function* () {
yield* register("exa")