feat(ai): send Gemini url media as fileData

packages/ai/README.md

@@ -478,7 +478,7 @@ call rather than being dropped, so check this table before swapping only the `mo
 | Provider              | `n` | `size`    | `aspectRatio` | `seed` | `format` | `images`                         | `mask`              |
 | --------------------- | --- | --------- | ------------- | ------ | -------- | -------------------------------- | ------------------- |
 | OpenAI                | ✓¹  | ✓         | ✗             | ✗      | ✓        | ✓                                | ✓                   |
-| Google (Gemini)       | 1   | ✗         | ✓             | ✓      | ✗        | ✓ (no public URLs)               | ✗                   |
+| Google (Gemini)       | 1   | ✗         | ✓             | ✓      | ✗        | ✓ (URLs need a media type)       | ✗                   |
 | xAI                   | ✓   | ✗         | ✓             | ✗      | ✗        | ✓                                | ✗                   |
 | Z.ai                  | ✗   | ✓         | ✗             | ✗      | ✗        | ✗                                | ✗                   |
 | Meta                  | ✓   | ✓ (hint)  | ✗             | ✗      | ✓        | ✓                                | ✗                   |
@@ -535,8 +535,9 @@ const inpainted = Image.generate({
 
 On multipart requests, `http.body` can override option fields but not structural `model`, `prompt`, `image[]`,
 or `mask` fields, and the transport owns the multipart `Content-Type` boundary. For JSON requests, `http.body`
-remains the final raw-native overlay. Gemini does not fetch public HTTP URLs, and hosted Z.ai image generation does
-not accept image inputs. These cases fail with a typed `AIError` before network I/O.
+remains the final raw-native overlay. Gemini fetches public or pre-signed `Media.url` inputs itself but needs their
+`mediaType`, and hosted Z.ai image generation does not accept image inputs. These cases fail with a typed `AIError`
+before network I/O.
 
 Provider-native image options belong to each request. Raw `http.body` fields have final precedence over them:
 

packages/ai/docs/media-design.md

@@ -330,7 +330,7 @@ Settled rules:
 | Provider | Kind | Audio input | `timestamps` | `diarize` | Unsupported | Usage |
 |---|---|---|---|---|---|---|
 | OpenAI | stream (`stream: true` in `stream` mode; `whisper-1` ignores `stream`, so it emits only `finish`) | multipart `file` (inline only) | `whisper-1` (`verbose_json`); diarize model: `segment` | `gpt-4o-transcribe-diarize` (`diarized_json`) | `speakers`; `prompt` on the diarize model | `tokens` or `seconds` |
-| Gemini | stream (`generateContent` / `streamGenerateContent`) | `inlineData` or Gemini Files `fileData` | `audioTranscriptionConfig.wordTimestamp` | `audioTranscriptionConfig.diarization` | `prompt`, `speakers` | `tokens` |
+| Gemini | stream (`generateContent` / `streamGenerateContent`) | `inlineData`, or `fileData` for Gemini Files refs and public/pre-signed URLs (declared `mediaType` required) | `audioTranscriptionConfig.wordTimestamp` | `audioTranscriptionConfig.diarization` | `prompt`, `speakers` | `tokens` |
 | Deepgram | inline | raw body, or JSON `{ url }` | words always; `segment` → `utterances` | `diarize_model=latest` + `utterances` | `prompt`, `speakers` | `seconds` (`metadata.duration`) |
 | AssemblyAI | queued (upload → submit → poll) | `/v2/upload` then `audio_url`, or a URL | words always; `segment` → `speaker_labels` | `speaker_labels` | — | `seconds` (`audio_duration`) |
 

packages/ai/src/protocols/gemini.ts

@@ -79,7 +79,7 @@ const GeminiInlineDataPart = Schema.Struct({
 })
 type GeminiInlineDataPart = Schema.Schema.Type<typeof GeminiInlineDataPart>
 
-/** Gemini Files API reference; the only remote input Gemini accepts. */
+/** Gemini Files API reference or a public / pre-signed http(s) URL that Gemini fetches itself. */
 const GeminiFileDataPart = Schema.Struct({
   fileData: Schema.Struct({
     mimeType: Schema.String,

packages/ai/src/protocols/utils/gemini-generate-content.ts

@@ -132,7 +132,9 @@ export const path = (model: string, mode: MediaProtocol.Mode) =>
 export const frames = (bytes: Stream.Stream<Uint8Array, AIError>, mode: MediaProtocol.Mode) =>
   mode === "stream" ? Framing.sse.frame(bytes) : Framing.document.frame(bytes)
 
-// Gemini does not fetch public URLs; inline payloads and Gemini Files references are the accepted inputs.
+// Gemini fetches public or pre-signed http(s) URLs itself as `fileData`, alongside Gemini Files references. A URL has
+// no bytes to sniff, so its media type must be declared. URLs that need transient download headers (Veo outputs)
+// cannot be fetched by Gemini and fall through to the inline requirement so callers materialize them first.
 export const mediaPart = (
   route: string,
   asset: Media.Asset,
@@ -143,6 +145,16 @@ export const mediaPart = (
 > => {
   const fileUri = MediaInput.refID(asset, PROVIDER)
   if (fileUri !== undefined) return Effect.succeed({ fileData: { mimeType: asset.mediaType, fileUri } })
+  const url = ProviderShared.mediaUrl(asset)
+  if (url !== undefined && asset.headers === undefined) {
+    if (asset.mediaType === "application/octet-stream")
+      return Effect.fail(
+        ProviderShared.invalidRequest(
+          `${route} needs the media type of a url source; pass mediaType to Media.url or materialize it first`,
+        ),
+      )
+    return Effect.succeed({ fileData: { mimeType: asset.mediaType, fileUri: url } })
+  }
   return ProviderShared.requireInlineMedia(route, asset).pipe(
     Effect.map((media) => ({ inlineData: { mimeType: media.mime, data: media.base64 } })),
   )

packages/ai/test/image.test.ts

@@ -355,6 +355,7 @@ describe("Image", () => {
         Media.bytes(Uint8Array.from([1, 2, 3]), "image/png"),
         Media.fromDataUrl("data:image/jpeg;base64,BAUG"),
         Media.ref("google", "https://generativelanguage.googleapis.com/v1beta/files/123", "image/webp"),
+        Media.url("https://example.test/photo.png", { mediaType: "image/png" }),
       ],
     }).pipe(
       Effect.provide(
@@ -371,6 +372,7 @@ describe("Image", () => {
                     fileUri: "https://generativelanguage.googleapis.com/v1beta/files/123",
                   },
                 },
+                { fileData: { mimeType: "image/png", fileUri: "https://example.test/photo.png" } },
               ])
               return Effect.succeed(
                 input.respond(

packages/ai/test/provider/gemini.test.ts

@@ -673,6 +673,54 @@ describe("Gemini route", () => {
     }),
   )
 
+  it.effect("sends url media as fileData for Gemini to fetch", () =>
+    Effect.gen(function* () {
+      const prepared = yield* compileRequest(
+        LLM.request({
+          model,
+          messages: [
+            Message.user([
+              { type: "text", text: "Summarize" },
+              {
+                type: "media",
+                media: Media.url("https://example.test/report.pdf", { mediaType: "application/pdf" }),
+              },
+            ]),
+          ],
+        }),
+      )
+      expect(prepared.body.contents).toEqual([
+        {
+          role: "user",
+          parts: [
+            { text: "Summarize" },
+            { fileData: { mimeType: "application/pdf", fileUri: "https://example.test/report.pdf" } },
+          ],
+        },
+      ])
+    }),
+  )
+
+  it.effect("rejects url media Gemini cannot fetch before sending", () =>
+    Effect.gen(function* () {
+      const errors = yield* Effect.forEach(
+        [
+          Media.url("https://example.test/unknown"),
+          Media.url("https://example.test/video.mp4", { mediaType: "video/mp4", headers: { "x-goog-api-key": "k" } }),
+        ],
+        (media) =>
+          compileRequest(LLM.request({ model, messages: [Message.user({ type: "media", media })] })).pipe(Effect.flip),
+      )
+      expect(errors.map((error) => [error.reason._tag, error.message])).toEqual([
+        [
+          "InvalidRequest",
+          "Gemini needs the media type of a url source; pass mediaType to Media.url or materialize it first",
+        ],
+        ["InvalidRequest", "Gemini requires inline media (bytes or base64); url sources must be materialized first"],
+      ])
+    }),
+  )
+
   it.effect("keeps tools and sends function calling mode NONE", () =>
     Effect.gen(function* () {
       const prepared = yield* compileRequest(

packages/ai/test/provider/google-vertex.test.ts

@@ -1,7 +1,7 @@
 import { describe, expect, test } from "bun:test"
 import { Effect } from "effect"
 import { HttpClientRequest } from "effect/unstable/http"
-import { LanguageModel, LLM, Message, ToolCallPart } from "../../src/index.js"
+import { LanguageModel, LLM, Media, Message, ToolCallPart } from "../../src/index.js"
 import { GoogleVertex, GoogleVertexChat, GoogleVertexMessages, GoogleVertexResponses } from "../../src/providers.js"
 import { LLMClient } from "../../src/route.js"
 import { compileRequest } from "../../src/route/client.js"
@@ -75,6 +75,30 @@ describe("Google Vertex providers", () => {
     }),
   )
 
+  it.effect("sends url media to Vertex Gemini as fileData", () =>
+    Effect.gen(function* () {
+      const prepared = yield* compileRequest(
+        LLM.request({
+          model: GoogleVertex.configure({ accessToken: "vertex-token", project: "vertex-project" }).model(
+            "gemini-3.5-flash",
+          ),
+          messages: [
+            Message.user({
+              type: "media",
+              media: Media.url("https://example.test/clip.mp3", { mediaType: "audio/mpeg" }),
+            }),
+          ],
+        }),
+      )
+
+      expect(prepared.body).toMatchObject({
+        contents: [
+          { role: "user", parts: [{ fileData: { mimeType: "audio/mpeg", fileUri: "https://example.test/clip.mp3" } }] },
+        ],
+      })
+    }),
+  )
+
   it.effect("maps service tiers to the Vertex shared PayGo header", () =>
     Effect.gen(function* () {
       const response = yield* LLMClient.generate(

packages/ai/test/transcription.test.ts

@@ -48,6 +48,7 @@ describe("Transcription", () => {
             model: Google.configure({ apiKey: "test" }).transcription("gemini-3.6-flash"),
             audio,
           }),
+          Transcription.generate({ model: google, audio: Media.url("https://a.test/x") }),
         ].map((effect) => Effect.flip(effect)),
       )
       expect(errors.map((error) => [error.reason._tag, "operation" in error.reason && error.reason.operation])).toEqual(
@@ -63,6 +64,7 @@ describe("Transcription", () => {
           ["InvalidRequest", false],
           ["InvalidRequest", false],
           ["UnsupportedOperation", "transcription.model"],
+          ["InvalidRequest", false],
         ],
       )
     }).pipe(Effect.provide(layer(() => Effect.die("an unsupported request reached the network")))),
@@ -272,6 +274,32 @@ describe("Transcription", () => {
     ),
   )
 
+  it.effect("sends url audio to Gemini as fileData for Gemini to fetch", () =>
+    Effect.gen(function* () {
+      const result = yield* Transcription.generate({
+        model: google,
+        audio: Media.url("https://a.test/call.mp3", { mediaType: "audio/mpeg" }),
+      })
+      expect(result.text).toBe("Hello")
+    }).pipe(
+      Effect.provide(
+        layer((input) => {
+          expect(JSON.parse(input.text).contents).toEqual([
+            { role: "user", parts: [{ fileData: { mimeType: "audio/mpeg", fileUri: "https://a.test/call.mp3" } }] },
+          ])
+          return Effect.succeed(
+            input.respond(
+              JSON.stringify({
+                candidates: [{ content: { parts: [{ audioTranscription: { text: "Hello" } }] }, finishReason: "STOP" }],
+              }),
+              { headers: { "content-type": "application/json" } },
+            ),
+          )
+        }),
+      ),
+    ),
+  )
+
   it.effect("surfaces Gemini transcripts that ended without STOP instead of returning them as complete", () =>
     Effect.gen(function* () {
       const document = (finishReason?: string) =>