feat(codemode): add Web extension with policy-bound fetch

packages/codemode/README.md

@@ -97,6 +97,26 @@ input; `onToolCallEnd` observes settled outcomes and duration. Both hooks return
 `Values.RegExp`, `Values.Map`, `Values.Set`, and `Values.Promise`. The interpreter recognizes these by class; a
 program's `new URL(...)` is a `Values.URL` wrapping the host `URL`. `Values.isValue` narrows to the data-like kinds.
 
+### `Extension.make` and `Web.make`
+
+An extension is a set of named host functions installed as globals. Arguments arrive as copies, results return as
+copies, and a function inside a result is callable the same way. Extension calls are not tool calls and are not shown
+in the tool catalog, so hosts describe them in their own instructions.
+
+`Web.make` is the built-in outbound HTTP extension. Nothing is reachable unless a host lists it:
+
+```ts
+const runtime = CodeMode.make({
+  extensions: [Web.make({ allow: ["https://api.example.com"], methods: ["GET", "POST"], maxBodyBytes: 1_048_576 })],
+})
+// program: const res = await fetch("https://api.example.com/users"); return await res.json()
+```
+
+`fetch(url, init?)` accepts `method`, `headers`, and a `string`, `Uint8Array`, or `URLSearchParams` body, and
+resolves to `{ url, status, statusText, ok, redirected, headers: { get, has, entries }, text(), json(), bytes() }`.
+Every redirect hop is checked against `allow`, bodies larger than `maxBodyBytes` are refused, and `timeoutMs` bounds
+the whole request. `Web.signature` is the model-facing signature for host instructions.
+
 ### OpenAPI tools
 
 `OpenAPI.fromSpec` converts an OpenAPI 3.x document into one tool per supported operation. Dotted `operationId` values

packages/codemode/interpreter-support.md

@@ -474,6 +474,21 @@ Nothing is exposed unless a host provides it; extension calls are not tool calls
 - [ ] Host classes. Stateful host objects are expressed as closures; a declared method table would be the next
       step if `new X()` in a program is ever needed.
 
+### Web
+
+`Web.make({ allow, methods, maxBodyBytes, timeoutMs })` is an extension exposing `fetch`. It is unavailable unless a
+host provides it.
+
+- [x] `fetch(url, init?)` with `method`, `headers` (record or pairs), and a `string`, `Uint8Array`, or
+      `URLSearchParams` body. The response is a plain object: `url`, `status`, `statusText`, `ok`, `redirected`,
+      `headers.get/has/entries`, and `text()`, `json()`, `bytes()` over the fully read body.
+- [x] Policy is enforced before any request: origins outside `allow` (or `"*"`), methods outside `methods` (default
+      `GET`, `HEAD`), non-http(s) URLs, and unsupported `init` keys (`signal`, `credentials`, ...) throw a
+      `TypeError` naming the problem. Redirects are followed by hand with each hop checked against `allow`, at most
+      five hops; 303 and 301/302-after-POST switch to GET like browsers. Bodies over `maxBodyBytes` throw a
+      `RangeError`; `timeoutMs` bounds the whole request.
+- [ ] `Headers`, `Response`, and `Request` as runtime types; streaming bodies; `AbortSignal`; `FormData`; `Blob`.
+
 ## Errors and diagnostics
 
 - [x] `Error`, `TypeError`, `RangeError`, `SyntaxError`, `ReferenceError`, `EvalError`, and `URIError`, callable with

packages/codemode/src/index.ts

@@ -2,6 +2,7 @@ export * as CodeMode from "./codemode.js"
 export * as Extension from "./extension.js"
 export * as Namespace from "./namespace.js"
 export * as Tool from "./tool.js"
+export * as Web from "./web.js"
 export * as OpenAPI from "./openapi/index.js"
 export { searchSignature, toolExpression } from "./codemode.js"
 export { ToolError, toolError } from "./tool-error.js"

packages/codemode/src/web.ts

@@ -0,0 +1,140 @@
+export * as Web from "./web.js"
+
+import { Extension } from "./extension.js"
+
+/** Outbound HTTP for programs. Nothing is reachable unless a host lists it. */
+export type Options = {
+  /** Origins the program may request, such as `"https://api.example.com"`; `"*"` allows every origin. */
+  readonly allow: ReadonlyArray<string>
+  /** Request methods the program may use. Default: `GET` and `HEAD`. */
+  readonly methods?: ReadonlyArray<string>
+  /** Largest response body accepted, in bytes. Default: 1 MiB. */
+  readonly maxBodyBytes?: number
+  /** Time allowed for a request, including redirects and reading the body. Default: 30 seconds. */
+  readonly timeoutMs?: number
+}
+
+const MAX_REDIRECTS = 5
+
+/** The model-facing signature of the `fetch` global, for hosts to include in their instructions. */
+export const signature = `fetch(url: string | URL, init?: { method?: string; headers?: Record<string, string> | Array<[string, string]>; body?: string | Uint8Array | URLSearchParams }): Promise<{ url: string; status: number; statusText: string; ok: boolean; redirected: boolean; headers: { get(name: string): string | null; has(name: string): boolean; entries(): Array<[string, string]> }; text(): Promise<string>; json(): Promise<unknown>; bytes(): Promise<Uint8Array> }>`
+
+export const make = (options: Options): Extension => {
+  const methods = new Set((options.methods ?? ["GET", "HEAD"]).map((method) => method.toUpperCase()))
+  const maxBodyBytes = options.maxBodyBytes ?? 1024 * 1024
+  const timeoutMs = options.timeoutMs ?? 30_000
+
+  const checkUrl = (url: URL, what: string) => {
+    if (url.protocol !== "http:" && url.protocol !== "https:") {
+      throw new TypeError(`fetch: ${what} ${url.href} must use http or https.`)
+    }
+    if (options.allow.includes("*") || options.allow.includes(url.origin)) return
+    throw new TypeError(`fetch: ${what} ${url.href} is not an allowed origin. Allowed: ${options.allow.join(", ")}.`)
+  }
+
+  // Redirects are followed by hand so every hop is checked against the allow list before it is requested.
+  const request = async (
+    url: URL,
+    method: string,
+    headers: Headers,
+    body: string | Uint8Array<ArrayBuffer> | URLSearchParams | undefined,
+    signal: AbortSignal,
+    hop: number,
+  ): Promise<Response> => {
+    const response = await globalThis
+      .fetch(url, { method, headers, body, signal, redirect: "manual" })
+      .catch((cause: unknown) => {
+        if (signal.aborted) throw new Error(`fetch: request to ${url.href} timed out after ${timeoutMs}ms.`)
+        throw new TypeError(`fetch: request to ${url.href} failed: ${cause instanceof Error ? cause.message : cause}`)
+      })
+    const location = response.headers.get("location")
+    if (response.status < 300 || response.status > 399 || location === null) return response
+    if (hop === MAX_REDIRECTS) throw new TypeError(`fetch: ${url.href} redirected more than ${MAX_REDIRECTS} times.`)
+    const next = new URL(location, url)
+    checkUrl(next, "redirect to")
+    // Like browsers: 303 always switches to GET, 301/302 do so for POST, 307/308 keep the method and body.
+    const toGet = response.status === 303 || ((response.status === 301 || response.status === 302) && method === "POST")
+    return request(next, toGet ? "GET" : method, headers, toGet ? undefined : body, signal, hop + 1)
+  }
+
+  const readBody = async (response: Response, url: URL): Promise<Uint8Array> => {
+    const tooLarge = () => new RangeError(`fetch: response from ${url.href} exceeds ${maxBodyBytes} bytes.`)
+    if (Number(response.headers.get("content-length")) > maxBodyBytes) throw tooLarge()
+    if (response.body === null) return new Uint8Array()
+    const chunks: Array<Uint8Array> = []
+    let total = 0
+    for await (const chunk of response.body) {
+      total += chunk.byteLength
+      if (total > maxBodyBytes) throw tooLarge()
+      chunks.push(chunk)
+    }
+    const bytes = new Uint8Array(total)
+    let offset = 0
+    for (const chunk of chunks) {
+      bytes.set(chunk, offset)
+      offset += chunk.byteLength
+    }
+    return bytes
+  }
+
+  const fetch = async (input: unknown, init: unknown = {}) => {
+    if (typeof input !== "string" && !(input instanceof URL)) {
+      throw new TypeError("fetch: the first argument must be a URL string or URL.")
+    }
+    const url = URL.parse(String(input))
+    if (url === null) throw new TypeError(`fetch: ${JSON.stringify(String(input))} is not a valid URL.`)
+    checkUrl(url, "request to")
+    if (init === null || typeof init !== "object" || Array.isArray(init)) {
+      throw new TypeError("fetch: init must be an object with method, headers, and body.")
+    }
+    for (const key of Object.keys(init)) {
+      if (key !== "method" && key !== "headers" && key !== "body") {
+        throw new TypeError(`fetch: init.${key} is not supported here; only method, headers, and body are.`)
+      }
+    }
+    const given = init as { method?: unknown; headers?: unknown; body?: unknown }
+    const method = given.method === undefined ? "GET" : String(given.method).toUpperCase()
+    if (!methods.has(method)) {
+      throw new TypeError(`fetch: method ${method} is not allowed. Allowed: ${[...methods].join(", ")}.`)
+    }
+    if (given.headers !== undefined && (given.headers === null || typeof given.headers !== "object")) {
+      throw new TypeError("fetch: init.headers must be a { name: value } object or an array of [name, value] pairs.")
+    }
+    const raw = given.body
+    if (
+      raw !== undefined &&
+      typeof raw !== "string" &&
+      !(raw instanceof Uint8Array) &&
+      !(raw instanceof URLSearchParams)
+    ) {
+      throw new TypeError("fetch: init.body must be a string, Uint8Array, or URLSearchParams.")
+    }
+    const body = raw instanceof Uint8Array ? Uint8Array.from(raw) : raw
+    const headers = new Headers(given.headers as Record<string, string> | Array<[string, string]> | undefined)
+    const signal = AbortSignal.timeout(timeoutMs)
+    const response = await request(url, method, headers, body, signal, 0)
+    const bytes = await readBody(response, url).catch((cause: unknown) => {
+      if (signal.aborted) throw new Error(`fetch: request to ${url.href} timed out after ${timeoutMs}ms.`)
+      throw cause
+    })
+    const received = Object.fromEntries(response.headers)
+    const text = () => new TextDecoder().decode(bytes)
+    return {
+      url: response.url,
+      status: response.status,
+      statusText: response.statusText,
+      ok: response.ok,
+      redirected: response.url !== url.href,
+      headers: {
+        get: (name: string) => received[String(name).toLowerCase()] ?? null,
+        has: (name: string) => String(name).toLowerCase() in received,
+        entries: () => Object.entries(received),
+      },
+      text: async () => text(),
+      json: async () => JSON.parse(text()) as unknown,
+      bytes: async () => bytes,
+    }
+  }
+
+  return Extension.make({ name: "web", globals: { fetch } })
+}

packages/codemode/test/web.test.ts

@@ -0,0 +1,221 @@
+import { afterAll, describe, expect, test } from "bun:test"
+import { Effect } from "effect"
+import { CodeMode, Web } from "../src/index.js"
+
+const seen: Array<{ method: string; path: string; headers: Record<string, string>; body: string }> = []
+const other = Bun.serve({
+  port: 0,
+  fetch: (request) => {
+    seen.push({ method: request.method, path: new URL(request.url).pathname, headers: {}, body: "" })
+    return new Response("other")
+  },
+})
+const server = Bun.serve({
+  port: 0,
+  fetch: async (request) => {
+    const url = new URL(request.url)
+    seen.push({
+      method: request.method,
+      path: url.pathname,
+      headers: Object.fromEntries(request.headers),
+      body: await request.text(),
+    })
+    switch (url.pathname) {
+      case "/json":
+        return Response.json({ hello: "world" }, { headers: { "X-Custom": "yes" } })
+      case "/text":
+        return new Response("plain text", { status: 201, statusText: "Created" })
+      case "/bytes":
+        return new Response(new Uint8Array([1, 2, 3]))
+      case "/bad-json":
+        return new Response("{oops", { headers: { "content-type": "application/json" } })
+      case "/redirect":
+        return Response.redirect(`${url.origin}/json`, 302)
+      case "/redirect-post":
+        return Response.redirect(`${url.origin}/text`, 307)
+      case "/redirect-away":
+        return Response.redirect(`${other.url.origin}/leaked`, 302)
+      case "/redirect-loop":
+        return Response.redirect(`${url.origin}/redirect-loop`, 302)
+      case "/big":
+        return new Response("x".repeat(2048))
+      case "/big-chunked": {
+        const stream = new ReadableStream({
+          start(controller) {
+            for (let i = 0; i < 4; i++) controller.enqueue(new Uint8Array(1024))
+            controller.close()
+          },
+        })
+        return new Response(stream)
+      }
+      case "/slow":
+        await Bun.sleep(300)
+        return new Response("late")
+      default:
+        return new Response("not found", { status: 404 })
+    }
+  },
+})
+afterAll(() => {
+  server.stop(true)
+  other.stop(true)
+})
+
+const origin = server.url.origin
+const runtime = CodeMode.make({ extensions: [Web.make({ allow: [origin], methods: ["GET", "POST"] })] })
+
+const value = async (code: string, target = runtime) => {
+  const result = await Effect.runPromise(target.execute(code))
+  if (!result.ok) throw new Error(`expected success, got ${result.error.kind}: ${result.error.message}`)
+  return result.value
+}
+
+const failure = async (code: string, target = runtime) => {
+  const result = await Effect.runPromise(target.execute(code))
+  if (result.ok) throw new Error(`expected failure, got value ${JSON.stringify(result.value)}`)
+  return result.error.message
+}
+
+describe("fetch", () => {
+  test("GET json with status, headers, and body readers", async () => {
+    expect(
+      await value(`
+        const res = await fetch("${origin}/json")
+        return [res.status, res.ok, res.redirected, res.url, res.headers.get("X-Custom"), res.headers.has("content-type"), await res.json(), await res.text()]
+      `),
+    ).toEqual([200, true, false, `${origin}/json`, "yes", true, { hello: "world" }, '{"hello":"world"}'])
+  })
+
+  test("status text, bytes, and header entries", async () => {
+    expect(
+      await value(`
+        const res = await fetch(new URL("${origin}/text"))
+        const bytes = await (await fetch("${origin}/bytes")).bytes()
+        return [res.status, res.statusText, await res.text(), [...bytes], bytes instanceof Uint8Array, res.headers.entries().some(([name]) => name === "content-type")]
+      `),
+    ).toEqual([201, "Created", "plain text", [1, 2, 3], true, true])
+  })
+
+  test("POST with headers and each body kind", async () => {
+    seen.length = 0
+    await value(`
+      await fetch("${origin}/text", { method: "post", headers: { "X-A": "1" }, body: "hello" })
+      await fetch("${origin}/text", { method: "POST", headers: [["X-B", "2"]], body: new Uint8Array([104, 105]) })
+      await fetch("${origin}/text", { method: "POST", body: new URLSearchParams({ q: "x y" }) })
+    `)
+    expect(seen.map((request) => [request.method, request.body])).toEqual([
+      ["POST", "hello"],
+      ["POST", "hi"],
+      ["POST", "q=x+y"],
+    ])
+    expect(seen[0].headers["x-a"]).toBe("1")
+    expect(seen[1].headers["x-b"]).toBe("2")
+    expect(seen[2].headers["content-type"]).toContain("application/x-www-form-urlencoded")
+  })
+
+  test("a JSON parse failure is a catchable SyntaxError", async () => {
+    expect(
+      await value(`
+        try { await (await fetch("${origin}/bad-json")).json() } catch (e) { return [e instanceof SyntaxError, e.message.length > 0] }
+      `),
+    ).toEqual([true, true])
+  })
+
+  test("a non-2xx response is returned, not thrown", async () => {
+    expect(await value(`const res = await fetch("${origin}/missing"); return [res.ok, res.status]`)).toEqual([
+      false,
+      404,
+    ])
+  })
+
+  test("fetch is not a tool call", async () => {
+    const limited = CodeMode.make({ extensions: [Web.make({ allow: [origin] })], limits: { maxToolCalls: 0 } })
+    const result = await Effect.runPromise(limited.execute(`return (await fetch("${origin}/json")).status`))
+    expect(result.ok).toBe(true)
+    expect(result.toolCalls).toEqual([])
+  })
+})
+
+describe("policy", () => {
+  test("origins outside the allow list are refused before any request is made", async () => {
+    seen.length = 0
+    expect(await failure(`await fetch("${other.url.origin}/leaked")`)).toContain(
+      `request to ${other.url.origin}/leaked is not an allowed origin. Allowed: ${origin}.`,
+    )
+    expect(await failure(`await fetch("file:///etc/passwd")`)).toContain("must use http or https")
+    expect(await failure(`await fetch("not a url")`)).toContain('"not a url" is not a valid URL')
+    expect(await failure(`await fetch(42)`)).toContain("first argument must be a URL string or URL")
+    expect(seen).toEqual([])
+  })
+
+  test("* allows every origin", async () => {
+    const open = CodeMode.make({ extensions: [Web.make({ allow: ["*"] })] })
+    expect(await value(`return (await fetch("${other.url.origin}/")).status`, open)).toBe(200)
+  })
+
+  test("methods default to GET and HEAD", async () => {
+    const readOnly = CodeMode.make({ extensions: [Web.make({ allow: [origin] })] })
+    expect(await failure(`await fetch("${origin}/text", { method: "POST" })`, readOnly)).toContain(
+      "method POST is not allowed. Allowed: GET, HEAD.",
+    )
+    expect(await failure(`await fetch("${origin}/text", { method: "DELETE" })`)).toContain(
+      "method DELETE is not allowed. Allowed: GET, POST.",
+    )
+  })
+
+  test("init keys and shapes that mean nothing here are rejected by name", async () => {
+    expect(await failure(`await fetch("${origin}/json", { signal: 1 })`)).toContain(
+      "init.signal is not supported here; only method, headers, and body are.",
+    )
+    expect(await failure(`await fetch("${origin}/json", { credentials: "include" })`)).toContain("init.credentials")
+    expect(await failure(`await fetch("${origin}/json", "GET")`)).toContain("init must be an object")
+    expect(await failure(`await fetch("${origin}/json", { headers: "X: 1" })`)).toContain("init.headers must be")
+    expect(await failure(`await fetch("${origin}/json", { method: "POST", body: { a: 1 } })`)).toContain(
+      "init.body must be a string, Uint8Array, or URLSearchParams.",
+    )
+  })
+
+  test("redirects are followed within the allow list and reported", async () => {
+    expect(
+      await value(`const res = await fetch("${origin}/redirect"); return [res.redirected, res.url, await res.json()]`),
+    ).toEqual([true, `${origin}/json`, { hello: "world" }])
+    seen.length = 0
+    expect(
+      await value(
+        `const res = await fetch("${origin}/redirect-post", { method: "POST", body: "keep" }); return res.status`,
+      ),
+    ).toBe(201)
+    expect(seen.map((request) => [request.path, request.method, request.body])).toEqual([
+      ["/redirect-post", "POST", "keep"],
+      ["/text", "POST", "keep"],
+    ])
+  })
+
+  test("a redirect to a disallowed origin is refused and never requested", async () => {
+    seen.length = 0
+    expect(await failure(`await fetch("${origin}/redirect-away")`)).toContain(
+      `redirect to ${other.url.origin}/leaked is not an allowed origin`,
+    )
+    expect(seen.map((request) => request.path)).toEqual(["/redirect-away"])
+  })
+
+  test("redirect loops stop", async () => {
+    expect(await failure(`await fetch("${origin}/redirect-loop")`)).toContain("redirected more than 5 times")
+  })
+
+  test("oversized bodies are refused, declared or streamed", async () => {
+    const small = CodeMode.make({ extensions: [Web.make({ allow: [origin], maxBodyBytes: 1024 })] })
+    expect(await failure(`await fetch("${origin}/big")`, small)).toContain("exceeds 1024 bytes")
+    expect(await failure(`await fetch("${origin}/big-chunked")`, small)).toContain("exceeds 1024 bytes")
+    expect(await value(`return (await fetch("${origin}/json")).status`, small)).toBe(200)
+  })
+
+  test("slow requests time out", async () => {
+    const quick = CodeMode.make({ extensions: [Web.make({ allow: [origin], timeoutMs: 50 })] })
+    expect(await failure(`await fetch("${origin}/slow")`, quick)).toContain("timed out after 50ms")
+  })
+})
+
+test("the signature names the global and its shape", () => {
+  expect(Web.signature).toStartWith("fetch(url: string | URL, init?:")
+})