furyhawk/opencode
1
0
Fork 0
mirror of https://github.com/anomalyco/opencode.git synced 2026-07-21 02:06:00 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ai): address Google image audit findings

Browse Source
This commit is contained in:
Aiden Cline
2026-07-19 16:15:33 +00:00
parent 5969025f98
commit af6b036b22
3 changed files with 242 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/ai/README.md
+31
View File
@@ -45,6 +45,37 @@ const program = Effect.gen(function* () {
})
```
Google's current Gemini image models use the same direct API:
```ts
import { Google } from "@opencode-ai/ai/providers"
const googleProgram = Effect.gen(function* () {
const response = yield* Image.generate({
model: Google.configure({
apiKey: process.env.GOOGLE_GENERATIVE_AI_API_KEY,
image: {
providerOptions: { imageSize: "2K", thinkingLevel: "HIGH", includeThoughts: true },
},
}).image("gemini-3.1-flash-image"),
prompt: "A robot tending a rooftop garden",
aspectRatio: "16:9",
seed: 42,
providerOptions: { google: { imageSize: "2K" } },
})
return response.images
})
```
`Google.configure(...).image(...)` is limited to Gemini `generateContent` image models (the Nano Banana family,
including `gemini-2.5-flash-image`, `gemini-3-pro-image`, and `gemini-3.1-flash-image` plus their preview
variants). Imagen model IDs use a different API and are rejected locally. Google provider options are `imageSize`,
`thinkingLevel` (`MINIMAL`, `LOW`, `MEDIUM`, or `HIGH`), and `includeThoughts`; request-level options override the
configured defaults. The API does not support a requested image count or exact pixel dimensions, so `count` and
`size` are rejected; use `aspectRatio` and the provider-specific `imageSize` tier instead. This route performs one
direct `Image.generate` request and does not provide conversational image continuation.
Conversational image generation remains part of the LLM interaction. OpenAI Responses exposes it through its hosted image tool:
```ts
packages/ai/src/protocols/google-images.ts
+75 -19
View File
@@ -9,15 +9,25 @@ import {
type ImageRoute,
} from "../image"
import { Auth, type Definition as AuthDefinition } from "../route/auth"
import { InvalidProviderOutputReason, LLMError, Usage, mergeHttpOptions, mergeJsonRecords } from "../schema"
import {
InvalidProviderOutputReason,
LLMError,
Usage,
mergeHttpOptions,
mergeJsonRecords,
type ProviderMetadata,
} from "../schema"
import { ProviderShared } from "./shared"
const ADAPTER = "google-images"
export const DEFAULT_BASE_URL = "https://generativelanguage.googleapis.com/v1beta"
export const GoogleThinkingLevel = Schema.Literals(["MINIMAL", "LOW", "MEDIUM", "HIGH"])
export type GoogleThinkingLevel = Schema.Schema.Type<typeof GoogleThinkingLevel>
export interface GoogleImageOptions {
readonly imageSize?: string
readonly thinkingLevel?: string
readonly thinkingLevel?: GoogleThinkingLevel
readonly includeThoughts?: boolean
}
@@ -39,7 +49,7 @@ const GoogleImageBody = Schema.Struct({
seed: Schema.optional(Schema.Number),
thinkingConfig: Schema.optional(
Schema.Struct({
thinkingLevel: Schema.optional(Schema.String),
thinkingLevel: Schema.optional(GoogleThinkingLevel),
includeThoughts: Schema.optional(Schema.Boolean),
}),
),
@@ -47,15 +57,18 @@ const GoogleImageBody = Schema.Struct({
})
export type GoogleImageBody = Schema.Schema.Type<typeof GoogleImageBody>
const GoogleUsage = Schema.Struct({
cachedContentTokenCount: Schema.optional(Schema.Number),
thoughtsTokenCount: Schema.optional(Schema.Number),
promptTokenCount: Schema.optional(Schema.Number),
candidatesTokenCount: Schema.optional(Schema.Number),
totalTokenCount: Schema.optional(Schema.Number),
promptTokensDetails: Schema.optional(Schema.Unknown),
candidatesTokensDetails: Schema.optional(Schema.Unknown),
})
const GoogleUsage = Schema.StructWithRest(
Schema.Struct({
cachedContentTokenCount: Schema.optional(Schema.Number),
thoughtsTokenCount: Schema.optional(Schema.Number),
promptTokenCount: Schema.optional(Schema.Number),
candidatesTokenCount: Schema.optional(Schema.Number),
totalTokenCount: Schema.optional(Schema.Number),
promptTokensDetails: Schema.optional(Schema.Unknown),
candidatesTokensDetails: Schema.optional(Schema.Unknown),
}),
[Schema.Record(Schema.String, Schema.Unknown)],
)
const GoogleImageResponse = Schema.Struct({
candidates: Schema.optional(
@@ -80,6 +93,7 @@ const GoogleImageResponse = Schema.Struct({
}),
),
finishReason: Schema.optional(Schema.String),
finishMessage: Schema.optional(Schema.String),
safetyRatings: Schema.optional(Schema.Unknown),
citationMetadata: Schema.optional(Schema.Unknown),
groundingMetadata: Schema.optional(Schema.Unknown),
@@ -126,11 +140,11 @@ const body = (request: ImageRequest): GoogleImageBody => {
}
}
const invalidOutput = (message: string) =>
const invalidOutput = (message: string, providerMetadata?: ProviderMetadata) =>
new LLMError({
module: ADAPTER,
method: "generate",
reason: new InvalidProviderOutputReason({ message, route: ADAPTER }),
reason: new InvalidProviderOutputReason({ message, route: ADAPTER, providerMetadata }),
})
const applyQuery = (url: string, query: Record<string, string> | undefined) => {
@@ -159,6 +173,10 @@ export const model = (input: ModelInput) => {
const route: ImageRoute = {
id: ADAPTER,
generate: Effect.fn("GoogleImages.generate")(function* (request: ImageRequest, execute) {
if (/^imagen(?:-|$)/i.test(request.model.id))
return yield* ProviderShared.invalidRequest(
`Google Images uses Gemini generateContent and does not support Imagen model ID ${request.model.id}`,
)
if (request.count !== undefined)
return yield* ProviderShared.invalidRequest("Google Images does not support the common count option")
if (request.size !== undefined)
@@ -192,9 +210,34 @@ export const model = (input: ModelInput) => {
Effect.mapError(() => invalidOutput("Google Images returned an invalid response")),
)
const candidates = decoded.candidates ?? []
const candidateMetadata = candidates.map((candidate, candidateIndex) => ({
index: candidate.index ?? candidateIndex,
finishReason: candidate.finishReason,
finishMessage: candidate.finishMessage,
safetyRatings: candidate.safetyRatings,
citationMetadata: candidate.citationMetadata,
groundingMetadata: candidate.groundingMetadata,
parts: (candidate.content?.parts ?? []).map((part) =>
part.inlineData === undefined
? {
type: "text",
text: part.text,
thought: part.thought,
thoughtSignature: part.thoughtSignature,
}
: {
type: "inlineData",
mediaType: part.inlineData.mimeType,
thought: part.thought,
thoughtSignature: part.thoughtSignature,
},
),
}))
const encoded = candidates.flatMap((candidate, candidateIndex) =>
(candidate.content?.parts ?? []).flatMap((part, partIndex) =>
part.inlineData === undefined ? [] : [{ candidate, candidateIndex, partIndex, inlineData: part.inlineData }],
part.inlineData === undefined || part.thought === true
? []
: [{ candidate, candidateIndex, partIndex, inlineData: part.inlineData }],
),
)
const images = yield* Effect.forEach(encoded, (item) =>
@@ -224,7 +267,22 @@ export const model = (input: ModelInput) => {
),
),
)
if (images.length === 0) return yield* invalidOutput("Google Images returned no images")
if (images.length === 0) {
const finishReasons = candidates.flatMap((candidate) =>
candidate.finishReason === undefined ? [] : [candidate.finishReason],
)
return yield* invalidOutput(
`Google Images returned no final images${
finishReasons.length === 0 ? "" : ` (finish reasons: ${finishReasons.join(", ")})`
}; inspect reason.providerMetadata.google for prompt feedback and candidate details`,
{
google: {
promptFeedback: decoded.promptFeedback,
candidates: candidateMetadata,
},
},
)
}
const usage = decoded.usageMetadata
const outputTokens =
usage?.candidatesTokenCount === undefined
@@ -252,9 +310,7 @@ export const model = (input: ModelInput) => {
modelVersion: decoded.modelVersion,
responseId: decoded.responseId,
promptFeedback: decoded.promptFeedback,
text: candidates.flatMap((candidate) =>
(candidate.content?.parts ?? []).flatMap((part) => (part.text === undefined ? [] : [part.text])),
),
candidates: candidateMetadata,
},
},
})
packages/ai/test/image.test.ts
+136 -5
View File
@@ -102,7 +102,7 @@ describe("Image", () => {
headers: { "x-default": "yes" },
http: { body: { labels: { deployment: "test" } }, query: { api: "v1" } },
image: {
providerOptions: { imageSize: "2K", thinkingLevel: "High", includeThoughts: true },
providerOptions: { imageSize: "2K", thinkingLevel: "HIGH", includeThoughts: true },
},
}).image("gemini-3.1-flash-image"),
prompt: "A robot tending a rooftop garden",
@@ -124,18 +124,65 @@ describe("Image", () => {
expect(response.images.map((image) => image.mediaType)).toEqual(["image/png", "image/jpeg", "image/webp"])
expect(response.images[0].providerMetadata).toMatchObject({ google: { thoughtSignature: "signature-1" } })
expect(response.images[1].providerMetadata).toMatchObject({
google: { candidateIndex: 0, partIndex: 2, finishReason: "STOP" },
google: { candidateIndex: 0, partIndex: 3, finishReason: "STOP" },
})
expect(response.images[2].providerMetadata).toMatchObject({ google: { candidateIndex: 7, partIndex: 0 } })
expect(response.usage?.inputTokens).toBe(5)
expect(response.usage?.outputTokens).toBe(10)
expect(response.usage?.reasoningTokens).toBe(3)
expect(response.usage?.providerMetadata).toMatchObject({ google: { serviceTier: "STANDARD" } })
expect(response.providerMetadata).toEqual({
google: {
modelVersion: "gemini-3.1-flash-image",
responseId: "response-1",
promptFeedback: undefined,
text: ["planning"],
candidates: [
{
index: 0,
finishReason: "STOP",
finishMessage: undefined,
safetyRatings: [{ category: "safe" }],
citationMetadata: undefined,
groundingMetadata: undefined,
parts: [
{
type: "inlineData",
mediaType: "image/png",
thought: undefined,
thoughtSignature: "signature-1",
},
{ type: "text", text: "planning", thought: true, thoughtSignature: "text-signature" },
{
type: "inlineData",
mediaType: "image/png",
thought: true,
thoughtSignature: "draft-signature",
},
{
type: "inlineData",
mediaType: "image/jpeg",
thought: undefined,
thoughtSignature: undefined,
},
],
},
{
index: 7,
finishReason: undefined,
finishMessage: undefined,
safetyRatings: undefined,
citationMetadata: undefined,
groundingMetadata: undefined,
parts: [
{
type: "inlineData",
mediaType: "image/webp",
thought: undefined,
thoughtSignature: undefined,
},
],
},
],
},
})
}).pipe(
@@ -157,7 +204,7 @@ describe("Image", () => {
responseModalities: ["IMAGE"],
imageConfig: { aspectRatio: "16:9", imageSize: "2K" },
seed: 42,
thinkingConfig: { thinkingLevel: "High", includeThoughts: true },
thinkingConfig: { thinkingLevel: "HIGH", includeThoughts: true },
},
labels: { deployment: "test" },
safetySettings: [],
@@ -172,7 +219,12 @@ describe("Image", () => {
inlineData: { mimeType: "image/png", data: "AQID" },
thoughtSignature: "signature-1",
},
{ text: "planning", thought: true },
{ text: "planning", thought: true, thoughtSignature: "text-signature" },
{
inlineData: { mimeType: "image/png", data: "CgsM" },
thought: true,
thoughtSignature: "draft-signature",
},
{ inlineData: { mimeType: "image/jpeg", data: "BAUG" } },
],
},
@@ -189,6 +241,7 @@ describe("Image", () => {
candidatesTokenCount: 7,
thoughtsTokenCount: 3,
totalTokenCount: 15,
serviceTier: "STANDARD",
},
modelVersion: "gemini-3.1-flash-image",
responseId: "response-1",
@@ -223,4 +276,82 @@ describe("Image", () => {
),
),
)
it.effect("rejects Imagen model IDs locally", () =>
Image.generate({
model: Google.configure({ apiKey: "test" }).image("imagen-4.0-generate-001"),
prompt: "A robot tending a rooftop garden",
}).pipe(
Effect.flip,
Effect.tap((error) =>
Effect.sync(() => {
expect(error.reason._tag).toBe("InvalidRequest")
expect(error.message).toContain("does not support Imagen model ID")
}),
),
Effect.provide(
ImageClient.layer.pipe(
Layer.provide(dynamicResponse(() => Effect.die("invalid model should not reach the provider"))),
),
),
),
)
it.effect("includes Google diagnostics when no final image is returned", () =>
Image.generate({
model: Google.configure({ apiKey: "test", baseURL: "https://generativelanguage.test/v1beta" }).image(
"gemini-3.1-flash-image",
),
prompt: "A robot tending a rooftop garden",
}).pipe(
Effect.flip,
Effect.tap((error) =>
Effect.sync(() => {
expect(error.reason._tag).toBe("InvalidProviderOutput")
if (error.reason._tag !== "InvalidProviderOutput") return
expect(error.reason.message).toContain("finish reasons: IMAGE_SAFETY")
expect(error.reason.providerMetadata).toEqual({
google: {
promptFeedback: { blockReason: "SAFETY" },
candidates: [
{
index: 0,
finishReason: "IMAGE_SAFETY",
finishMessage: "The generated image was blocked by safety filters.",
safetyRatings: [{ category: "HARM_CATEGORY_DANGEROUS_CONTENT", blocked: true }],
citationMetadata: undefined,
groundingMetadata: undefined,
parts: [{ type: "text", text: "blocked", thought: false, thoughtSignature: undefined }],
},
],
},
})
}),
),
Effect.provide(
ImageClient.layer.pipe(
Layer.provide(
dynamicResponse((input) =>
Effect.succeed(
input.respond(
JSON.stringify({
candidates: [
{
content: { parts: [{ text: "blocked", thought: false }] },
finishReason: "IMAGE_SAFETY",
finishMessage: "The generated image was blocked by safety filters.",
safetyRatings: [{ category: "HARM_CATEGORY_DANGEROUS_CONTENT", blocked: true }],
},
],
promptFeedback: { blockReason: "SAFETY" },
}),
{ headers: { "content-type": "application/json" } },
),
),
),
),
),
),
),
)
})