diff --git a/packages/ai/README.md b/packages/ai/README.md index e78dee0153..f72a78e0c2 100644 --- a/packages/ai/README.md +++ b/packages/ai/README.md @@ -45,6 +45,37 @@ const program = Effect.gen(function* () { }) ``` +Google's current Gemini image models use the same direct API: + +```ts +import { Google } from "@opencode-ai/ai/providers" + +const googleProgram = Effect.gen(function* () { + const response = yield* Image.generate({ + model: Google.configure({ + apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY, + image: { + providerOptions: { imageSize: "2K", thinkingLevel: "HIGH", includeThoughts: true }, + }, + }).image("gemini-3.1-flash-image"), + prompt: "A robot tending a rooftop garden", + aspectRatio: "16:9", + seed: 42, + providerOptions: { google: { imageSize: "2K" } }, + }) + + return response.images +}) +``` + +`Google.configure(...).image(...)` is limited to Gemini `generateContent` image models (the Nano Banana family, +including `gemini-2.5-flash-image`, `gemini-3-pro-image`, and `gemini-3.1-flash-image` plus their preview +variants). Imagen model IDs use a different API and are rejected locally. Google provider options are `imageSize`, +`thinkingLevel` (`MINIMAL`, `LOW`, `MEDIUM`, or `HIGH`), and `includeThoughts`; request-level options override the +configured defaults. The API does not support a requested image count or exact pixel dimensions, so `count` and +`size` are rejected; use `aspectRatio` and the provider-specific `imageSize` tier instead. This route performs one +direct `Image.generate` request and does not provide conversational image continuation. + Conversational image generation remains part of the LLM interaction. OpenAI Responses exposes it through its hosted image tool: ```ts diff --git a/packages/ai/src/protocols/google-images.ts b/packages/ai/src/protocols/google-images.ts index 51dea5ebf9..62eb16ca4d 100644 --- a/packages/ai/src/protocols/google-images.ts +++ b/packages/ai/src/protocols/google-images.ts @@ -9,15 +9,25 @@ import { type ImageRoute, } from "../image" import { Auth, type Definition as AuthDefinition } from "../route/auth" -import { InvalidProviderOutputReason, LLMError, Usage, mergeHttpOptions, mergeJsonRecords } from "../schema" +import { + InvalidProviderOutputReason, + LLMError, + Usage, + mergeHttpOptions, + mergeJsonRecords, + type ProviderMetadata, +} from "../schema" import { ProviderShared } from "./shared" const ADAPTER = "google-images" export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta" +export const GoogleThinkingLevel = Schema.Literals(["MINIMAL", "LOW", "MEDIUM", "HIGH"]) +export type GoogleThinkingLevel = Schema.Schema.Type + export interface GoogleImageOptions { readonly imageSize?: string - readonly thinkingLevel?: string + readonly thinkingLevel?: GoogleThinkingLevel readonly includeThoughts?: boolean } @@ -39,7 +49,7 @@ const GoogleImageBody = Schema.Struct({ seed: Schema.optional(Schema.Number), thinkingConfig: Schema.optional( Schema.Struct({ - thinkingLevel: Schema.optional(Schema.String), + thinkingLevel: Schema.optional(GoogleThinkingLevel), includeThoughts: Schema.optional(Schema.Boolean), }), ), @@ -47,15 +57,18 @@ const GoogleImageBody = Schema.Struct({ }) export type GoogleImageBody = Schema.Schema.Type -const GoogleUsage = Schema.Struct({ - cachedContentTokenCount: Schema.optional(Schema.Number), - thoughtsTokenCount: Schema.optional(Schema.Number), - promptTokenCount: Schema.optional(Schema.Number), - candidatesTokenCount: Schema.optional(Schema.Number), - totalTokenCount: Schema.optional(Schema.Number), - promptTokensDetails: Schema.optional(Schema.Unknown), - candidatesTokensDetails: Schema.optional(Schema.Unknown), -}) +const GoogleUsage = Schema.StructWithRest( + Schema.Struct({ + cachedContentTokenCount: Schema.optional(Schema.Number), + thoughtsTokenCount: Schema.optional(Schema.Number), + promptTokenCount: Schema.optional(Schema.Number), + candidatesTokenCount: Schema.optional(Schema.Number), + totalTokenCount: Schema.optional(Schema.Number), + promptTokensDetails: Schema.optional(Schema.Unknown), + candidatesTokensDetails: Schema.optional(Schema.Unknown), + }), + [Schema.Record(Schema.String, Schema.Unknown)], +) const GoogleImageResponse = Schema.Struct({ candidates: Schema.optional( @@ -80,6 +93,7 @@ const GoogleImageResponse = Schema.Struct({ }), ), finishReason: Schema.optional(Schema.String), + finishMessage: Schema.optional(Schema.String), safetyRatings: Schema.optional(Schema.Unknown), citationMetadata: Schema.optional(Schema.Unknown), groundingMetadata: Schema.optional(Schema.Unknown), @@ -126,11 +140,11 @@ const body = (request: ImageRequest): GoogleImageBody => { } } -const invalidOutput = (message: string) => +const invalidOutput = (message: string, providerMetadata?: ProviderMetadata) => new LLMError({ module: ADAPTER, method: "generate", - reason: new InvalidProviderOutputReason({ message, route: ADAPTER }), + reason: new InvalidProviderOutputReason({ message, route: ADAPTER, providerMetadata }), }) const applyQuery = (url: string, query: Record | undefined) => { @@ -159,6 +173,10 @@ export const model = (input: ModelInput) => { const route: ImageRoute = { id: ADAPTER, generate: Effect.fn("GoogleImages.generate")(function* (request: ImageRequest, execute) { + if (/^imagen(?:-|$)/i.test(request.model.id)) + return yield* ProviderShared.invalidRequest( + `Google Images uses Gemini generateContent and does not support Imagen model ID ${request.model.id}`, + ) if (request.count !== undefined) return yield* ProviderShared.invalidRequest("Google Images does not support the common count option") if (request.size !== undefined) @@ -192,9 +210,34 @@ export const model = (input: ModelInput) => { Effect.mapError(() => invalidOutput("Google Images returned an invalid response")), ) const candidates = decoded.candidates ?? [] + const candidateMetadata = candidates.map((candidate, candidateIndex) => ({ + index: candidate.index ?? candidateIndex, + finishReason: candidate.finishReason, + finishMessage: candidate.finishMessage, + safetyRatings: candidate.safetyRatings, + citationMetadata: candidate.citationMetadata, + groundingMetadata: candidate.groundingMetadata, + parts: (candidate.content?.parts ?? []).map((part) => + part.inlineData === undefined + ? { + type: "text", + text: part.text, + thought: part.thought, + thoughtSignature: part.thoughtSignature, + } + : { + type: "inlineData", + mediaType: part.inlineData.mimeType, + thought: part.thought, + thoughtSignature: part.thoughtSignature, + }, + ), + })) const encoded = candidates.flatMap((candidate, candidateIndex) => (candidate.content?.parts ?? []).flatMap((part, partIndex) => - part.inlineData === undefined ? [] : [{ candidate, candidateIndex, partIndex, inlineData: part.inlineData }], + part.inlineData === undefined || part.thought === true + ? [] + : [{ candidate, candidateIndex, partIndex, inlineData: part.inlineData }], ), ) const images = yield* Effect.forEach(encoded, (item) => @@ -224,7 +267,22 @@ export const model = (input: ModelInput) => { ), ), ) - if (images.length === 0) return yield* invalidOutput("Google Images returned no images") + if (images.length === 0) { + const finishReasons = candidates.flatMap((candidate) => + candidate.finishReason === undefined ? [] : [candidate.finishReason], + ) + return yield* invalidOutput( + `Google Images returned no final images${ + finishReasons.length === 0 ? "" : ` (finish reasons: ${finishReasons.join(", ")})` + }; inspect reason.providerMetadata.google for prompt feedback and candidate details`, + { + google: { + promptFeedback: decoded.promptFeedback, + candidates: candidateMetadata, + }, + }, + ) + } const usage = decoded.usageMetadata const outputTokens = usage?.candidatesTokenCount === undefined @@ -252,9 +310,7 @@ export const model = (input: ModelInput) => { modelVersion: decoded.modelVersion, responseId: decoded.responseId, promptFeedback: decoded.promptFeedback, - text: candidates.flatMap((candidate) => - (candidate.content?.parts ?? []).flatMap((part) => (part.text === undefined ? [] : [part.text])), - ), + candidates: candidateMetadata, }, }, }) diff --git a/packages/ai/test/image.test.ts b/packages/ai/test/image.test.ts index fba4577383..6c7214dc0c 100644 --- a/packages/ai/test/image.test.ts +++ b/packages/ai/test/image.test.ts @@ -102,7 +102,7 @@ describe("Image", () => { headers: { "x-default": "yes" }, http: { body: { labels: { deployment: "test" } }, query: { api: "v1" } }, image: { - providerOptions: { imageSize: "2K", thinkingLevel: "High", includeThoughts: true }, + providerOptions: { imageSize: "2K", thinkingLevel: "HIGH", includeThoughts: true }, }, }).image("gemini-3.1-flash-image"), prompt: "A robot tending a rooftop garden", @@ -124,18 +124,65 @@ describe("Image", () => { expect(response.images.map((image) => image.mediaType)).toEqual(["image/png", "image/jpeg", "image/webp"]) expect(response.images[0].providerMetadata).toMatchObject({ google: { thoughtSignature: "signature-1" } }) expect(response.images[1].providerMetadata).toMatchObject({ - google: { candidateIndex: 0, partIndex: 2, finishReason: "STOP" }, + google: { candidateIndex: 0, partIndex: 3, finishReason: "STOP" }, }) expect(response.images[2].providerMetadata).toMatchObject({ google: { candidateIndex: 7, partIndex: 0 } }) expect(response.usage?.inputTokens).toBe(5) expect(response.usage?.outputTokens).toBe(10) expect(response.usage?.reasoningTokens).toBe(3) + expect(response.usage?.providerMetadata).toMatchObject({ google: { serviceTier: "STANDARD" } }) expect(response.providerMetadata).toEqual({ google: { modelVersion: "gemini-3.1-flash-image", responseId: "response-1", promptFeedback: undefined, - text: ["planning"], + candidates: [ + { + index: 0, + finishReason: "STOP", + finishMessage: undefined, + safetyRatings: [{ category: "safe" }], + citationMetadata: undefined, + groundingMetadata: undefined, + parts: [ + { + type: "inlineData", + mediaType: "image/png", + thought: undefined, + thoughtSignature: "signature-1", + }, + { type: "text", text: "planning", thought: true, thoughtSignature: "text-signature" }, + { + type: "inlineData", + mediaType: "image/png", + thought: true, + thoughtSignature: "draft-signature", + }, + { + type: "inlineData", + mediaType: "image/jpeg", + thought: undefined, + thoughtSignature: undefined, + }, + ], + }, + { + index: 7, + finishReason: undefined, + finishMessage: undefined, + safetyRatings: undefined, + citationMetadata: undefined, + groundingMetadata: undefined, + parts: [ + { + type: "inlineData", + mediaType: "image/webp", + thought: undefined, + thoughtSignature: undefined, + }, + ], + }, + ], }, }) }).pipe( @@ -157,7 +204,7 @@ describe("Image", () => { responseModalities: ["IMAGE"], imageConfig: { aspectRatio: "16:9", imageSize: "2K" }, seed: 42, - thinkingConfig: { thinkingLevel: "High", includeThoughts: true }, + thinkingConfig: { thinkingLevel: "HIGH", includeThoughts: true }, }, labels: { deployment: "test" }, safetySettings: [], @@ -172,7 +219,12 @@ describe("Image", () => { inlineData: { mimeType: "image/png", data: "AQID" }, thoughtSignature: "signature-1", }, - { text: "planning", thought: true }, + { text: "planning", thought: true, thoughtSignature: "text-signature" }, + { + inlineData: { mimeType: "image/png", data: "CgsM" }, + thought: true, + thoughtSignature: "draft-signature", + }, { inlineData: { mimeType: "image/jpeg", data: "BAUG" } }, ], }, @@ -189,6 +241,7 @@ describe("Image", () => { candidatesTokenCount: 7, thoughtsTokenCount: 3, totalTokenCount: 15, + serviceTier: "STANDARD", }, modelVersion: "gemini-3.1-flash-image", responseId: "response-1", @@ -223,4 +276,82 @@ describe("Image", () => { ), ), ) + + it.effect("rejects Imagen model IDs locally", () => + Image.generate({ + model: Google.configure({ apiKey: "test" }).image("imagen-4.0-generate-001"), + prompt: "A robot tending a rooftop garden", + }).pipe( + Effect.flip, + Effect.tap((error) => + Effect.sync(() => { + expect(error.reason._tag).toBe("InvalidRequest") + expect(error.message).toContain("does not support Imagen model ID") + }), + ), + Effect.provide( + ImageClient.layer.pipe( + Layer.provide(dynamicResponse(() => Effect.die("invalid model should not reach the provider"))), + ), + ), + ), + ) + + it.effect("includes Google diagnostics when no final image is returned", () => + Image.generate({ + model: Google.configure({ apiKey: "test", baseURL: "https://generativelanguage.test/v1beta" }).image( + "gemini-3.1-flash-image", + ), + prompt: "A robot tending a rooftop garden", + }).pipe( + Effect.flip, + Effect.tap((error) => + Effect.sync(() => { + expect(error.reason._tag).toBe("InvalidProviderOutput") + if (error.reason._tag !== "InvalidProviderOutput") return + expect(error.reason.message).toContain("finish reasons: IMAGE_SAFETY") + expect(error.reason.providerMetadata).toEqual({ + google: { + promptFeedback: { blockReason: "SAFETY" }, + candidates: [ + { + index: 0, + finishReason: "IMAGE_SAFETY", + finishMessage: "The generated image was blocked by safety filters.", + safetyRatings: [{ category: "HARM_CATEGORY_DANGEROUS_CONTENT", blocked: true }], + citationMetadata: undefined, + groundingMetadata: undefined, + parts: [{ type: "text", text: "blocked", thought: false, thoughtSignature: undefined }], + }, + ], + }, + }) + }), + ), + Effect.provide( + ImageClient.layer.pipe( + Layer.provide( + dynamicResponse((input) => + Effect.succeed( + input.respond( + JSON.stringify({ + candidates: [ + { + content: { parts: [{ text: "blocked", thought: false }] }, + finishReason: "IMAGE_SAFETY", + finishMessage: "The generated image was blocked by safety filters.", + safetyRatings: [{ category: "HARM_CATEGORY_DANGEROUS_CONTENT", blocked: true }], + }, + ], + promptFeedback: { blockReason: "SAFETY" }, + }), + { headers: { "content-type": "application/json" } }, + ), + ), + ), + ), + ), + ), + ), + ) })