fix(google): restore image tool results for prefixed Gemini 2 models (#102382)

* fix(google): handle prefixed Gemini 2 image fallback

* test(google): cover prefixed Gemini 3 image responses

* docs(changelog): note prefixed Gemini fallback

Co-authored-by: 李兰 0668001394 <li.lan3@xydigit.com>

---------

Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
LiLan0125
2026-07-09 15:09:05 +01:00
committed by GitHub
co-authored by Peter Steinberger
parent 19f42a27e5
commit db0f6f09dc
5 changed files with 80 additions and 12 deletions
+1
View File
@@ -24,6 +24,7 @@ Docs: https://docs.openclaw.ai
### Fixes
- **Google Chat request deadlines:** bound control calls to 30 seconds while giving media transfers size-aware total budgets and a separate 30-second stalled-body guard, preventing hung Chat API requests without breaking large attachment uploads. (#102227) Thanks @hugenshen.
- **Google Gemini prefixed model IDs:** recognize `google/gemini-*` and `models/gemini-*` when selecting multimodal function-response behavior, preserving the Gemini 2 image fallback without regressing Gemini 3 inline image responses. (#102382) Thanks @LiLan0125.
- **Browser actions on Node 24:** keep browser request cancellation bound to the client and response lifetime instead of Node 24.16+'s prematurely aborted body-stream signal, preventing valid POST actions from failing after JSON parsing. Thanks @obviyus and @vincentkoc.
- **SecretRef model credentials:** keep resolved provider secrets behind process-local sentinels through auth storage, stream setup, SDK configuration, and managed local-provider probing, then inject plaintext only at the final network or provider-plugin boundary while retaining exact-value log redaction. (#102008, #102009)
- **Lean local model shell access:** keep `exec` directly visible beside the default structured Tool Search controls so coding-tuned local models can use their shell fallback instead of searching for missing domain tools. (#87587) Thanks @vincentkoc.
+38 -5
View File
@@ -2658,13 +2658,19 @@ describe("google transport stream", () => {
});
it.each([
["image first", ["screenshot", "weather"]],
["image last", ["weather", "screenshot"]],
["bare Gemini 2.5 image first", "gemini-2.5-flash", ["screenshot", "weather"]],
["bare Gemini 2.5 image last", "gemini-2.5-flash", ["weather", "screenshot"]],
[
"provider-prefixed Gemini 2.5 image first",
"google/gemini-2.5-pro",
["screenshot", "weather"],
],
["models-prefixed Gemini 2.5 image last", "models/gemini-2.5-pro", ["weather", "screenshot"]],
] as const)(
"keeps parallel function responses immediate and retains the deferred %s result",
(_label, resultOrder) => {
"keeps parallel function responses immediate and retains the deferred result for %s",
(_label, modelId, resultOrder) => {
const params = buildGoogleGenerativeAiParams(
buildGeminiModel({ id: "gemini-2.5-flash", input: ["text", "image"] }),
buildGeminiModel({ id: modelId, input: ["text", "image"] }),
{
messages: [
{ role: "user", content: "Screenshot the page and check the weather.", timestamp: 0 },
@@ -2700,6 +2706,33 @@ describe("google transport stream", () => {
},
);
it.each(["google/gemini-3.1-pro-preview", "models/gemini-3.1-pro-preview"])(
"keeps image parts inside function responses for prefixed Gemini 3 model %s",
(modelId) => {
const params = buildGoogleGenerativeAiParams(
buildGeminiModel({ id: modelId, input: ["text", "image"] }),
{
messages: [
{ role: "user", content: "Take a screenshot.", timestamp: 0 },
googleToolCallAssistantTurn({
model: modelId,
name: "screenshot",
args: {},
}),
googleToolResultMessage("screenshot"),
],
} as never,
);
const functionResponse = (params.contents[2] as GoogleTestContentTurn).parts[0]
?.functionResponse as { parts?: unknown };
expect(params.contents.map((content) => content.role)).toEqual(["user", "model", "user"]);
expect(functionResponse.parts).toEqual([
{ inlineData: { mimeType: "image/png", data: "png-bytes" } },
]);
},
);
it.each([
["gemini-2.5-flash-lite", "minimal", 512],
["gemini-2.5-flash-lite", "low", 2048],
+1 -1
View File
@@ -165,7 +165,7 @@ function requiresToolCallThoughtSignature(modelId: string): boolean {
}
function supportsMultimodalFunctionResponse(modelId: string): boolean {
const match = normalizeLowercaseStringOrEmpty(modelId).match(/^gemini(?:-live)?-(\d+)/);
const match = normalizeLowercaseStringOrEmpty(modelId).match(/(?:^|\/)gemini(?:-live)?-(\d+)/);
if (!match) {
return true;
}
@@ -35,12 +35,18 @@ function functionResponseNames(parts: readonly Part[] | undefined): string[] {
describe("google-shared convertMessages — parallel tool results with an image (Gemini < 3)", () => {
it.each([
["image first", ["screenshot", "weather"]],
["image last", ["weather", "screenshot"]],
["bare Gemini 2.5 image first", "gemini-2.5-flash", ["screenshot", "weather"]],
["bare Gemini 2.5 image last", "gemini-2.5-flash", ["weather", "screenshot"]],
[
"provider-prefixed Gemini 2.5 image first",
"google/gemini-2.5-pro",
["screenshot", "weather"],
],
["models-prefixed Gemini 2.5 image last", "models/gemini-2.5-pro", ["weather", "screenshot"]],
] as const)(
"keeps the response run immediate and retains a deferred %s result",
(_label, resultOrder) => {
const model = makeVisionModel("gemini-2.5-flash");
"keeps the response run immediate and retains a deferred result for %s",
(_label, modelId, resultOrder) => {
const model = makeVisionModel(modelId);
const toolResults = {
screenshot: {
role: "toolResult",
@@ -84,4 +90,32 @@ describe("google-shared convertMessages — parallel tool results with an image
expect(contents.slice(3).some((c) => countFunctionResponses(c.parts) > 0)).toBe(false);
},
);
it.each(["google/gemini-3.1-pro-preview", "models/gemini-3.1-pro-preview"])(
"keeps image parts inside function responses for prefixed Gemini 3 model %s",
(modelId) => {
const model = makeVisionModel(modelId);
const contents = convertMessagesForTest(model, {
messages: [
{ role: "user", content: "Take a screenshot." },
makeGoogleAssistantMessage(model.id, [
{ type: "toolCall", id: "call_1", name: "screenshot", arguments: {} },
]),
{
role: "toolResult",
toolCallId: "call_1",
toolName: "screenshot",
content: [{ type: "image", mimeType: "image/png", data: "AAAA" }],
isError: false,
timestamp: 0,
},
],
} as unknown as Context);
expect(contents.map((content) => content.role)).toEqual(["user", "model", "user"]);
expect(contents[2]?.parts?.[0]?.functionResponse?.parts).toEqual([
{ inlineData: { mimeType: "image/png", data: "AAAA" } },
]);
},
);
});
+1 -1
View File
@@ -140,7 +140,7 @@ export function requiresToolCallId(modelId: string): boolean {
}
function getGeminiMajorVersion(modelId: string): number | undefined {
const match = modelId.toLowerCase().match(/^gemini(?:-live)?-(\d+)/);
const match = modelId.toLowerCase().match(/(?:^|\/)gemini(?:-live)?-(\d+)/);
if (!match) {
return undefined;
}