mirror of
https://github.com/openclaw/openclaw.git
synced 2026-07-21 10:16:44 +00:00
fix(google): restore image tool results for prefixed Gemini 2 models (#102382)
* fix(google): handle prefixed Gemini 2 image fallback * test(google): cover prefixed Gemini 3 image responses * docs(changelog): note prefixed Gemini fallback Co-authored-by: 李兰 0668001394 <li.lan3@xydigit.com> --------- Co-authored-by: Peter Steinberger <steipete@gmail.com>
This commit is contained in:
co-authored by
Peter Steinberger
parent
19f42a27e5
commit
db0f6f09dc
@@ -24,6 +24,7 @@ Docs: https://docs.openclaw.ai
|
||||
### Fixes
|
||||
|
||||
- **Google Chat request deadlines:** bound control calls to 30 seconds while giving media transfers size-aware total budgets and a separate 30-second stalled-body guard, preventing hung Chat API requests without breaking large attachment uploads. (#102227) Thanks @hugenshen.
|
||||
- **Google Gemini prefixed model IDs:** recognize `google/gemini-*` and `models/gemini-*` when selecting multimodal function-response behavior, preserving the Gemini 2 image fallback without regressing Gemini 3 inline image responses. (#102382) Thanks @LiLan0125.
|
||||
- **Browser actions on Node 24:** keep browser request cancellation bound to the client and response lifetime instead of Node 24.16+'s prematurely aborted body-stream signal, preventing valid POST actions from failing after JSON parsing. Thanks @obviyus and @vincentkoc.
|
||||
- **SecretRef model credentials:** keep resolved provider secrets behind process-local sentinels through auth storage, stream setup, SDK configuration, and managed local-provider probing, then inject plaintext only at the final network or provider-plugin boundary while retaining exact-value log redaction. (#102008, #102009)
|
||||
- **Lean local model shell access:** keep `exec` directly visible beside the default structured Tool Search controls so coding-tuned local models can use their shell fallback instead of searching for missing domain tools. (#87587) Thanks @vincentkoc.
|
||||
|
||||
@@ -2658,13 +2658,19 @@ describe("google transport stream", () => {
|
||||
});
|
||||
|
||||
it.each([
|
||||
["image first", ["screenshot", "weather"]],
|
||||
["image last", ["weather", "screenshot"]],
|
||||
["bare Gemini 2.5 image first", "gemini-2.5-flash", ["screenshot", "weather"]],
|
||||
["bare Gemini 2.5 image last", "gemini-2.5-flash", ["weather", "screenshot"]],
|
||||
[
|
||||
"provider-prefixed Gemini 2.5 image first",
|
||||
"google/gemini-2.5-pro",
|
||||
["screenshot", "weather"],
|
||||
],
|
||||
["models-prefixed Gemini 2.5 image last", "models/gemini-2.5-pro", ["weather", "screenshot"]],
|
||||
] as const)(
|
||||
"keeps parallel function responses immediate and retains the deferred %s result",
|
||||
(_label, resultOrder) => {
|
||||
"keeps parallel function responses immediate and retains the deferred result for %s",
|
||||
(_label, modelId, resultOrder) => {
|
||||
const params = buildGoogleGenerativeAiParams(
|
||||
buildGeminiModel({ id: "gemini-2.5-flash", input: ["text", "image"] }),
|
||||
buildGeminiModel({ id: modelId, input: ["text", "image"] }),
|
||||
{
|
||||
messages: [
|
||||
{ role: "user", content: "Screenshot the page and check the weather.", timestamp: 0 },
|
||||
@@ -2700,6 +2706,33 @@ describe("google transport stream", () => {
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["google/gemini-3.1-pro-preview", "models/gemini-3.1-pro-preview"])(
|
||||
"keeps image parts inside function responses for prefixed Gemini 3 model %s",
|
||||
(modelId) => {
|
||||
const params = buildGoogleGenerativeAiParams(
|
||||
buildGeminiModel({ id: modelId, input: ["text", "image"] }),
|
||||
{
|
||||
messages: [
|
||||
{ role: "user", content: "Take a screenshot.", timestamp: 0 },
|
||||
googleToolCallAssistantTurn({
|
||||
model: modelId,
|
||||
name: "screenshot",
|
||||
args: {},
|
||||
}),
|
||||
googleToolResultMessage("screenshot"),
|
||||
],
|
||||
} as never,
|
||||
);
|
||||
|
||||
const functionResponse = (params.contents[2] as GoogleTestContentTurn).parts[0]
|
||||
?.functionResponse as { parts?: unknown };
|
||||
expect(params.contents.map((content) => content.role)).toEqual(["user", "model", "user"]);
|
||||
expect(functionResponse.parts).toEqual([
|
||||
{ inlineData: { mimeType: "image/png", data: "png-bytes" } },
|
||||
]);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
["gemini-2.5-flash-lite", "minimal", 512],
|
||||
["gemini-2.5-flash-lite", "low", 2048],
|
||||
|
||||
@@ -165,7 +165,7 @@ function requiresToolCallThoughtSignature(modelId: string): boolean {
|
||||
}
|
||||
|
||||
function supportsMultimodalFunctionResponse(modelId: string): boolean {
|
||||
const match = normalizeLowercaseStringOrEmpty(modelId).match(/^gemini(?:-live)?-(\d+)/);
|
||||
const match = normalizeLowercaseStringOrEmpty(modelId).match(/(?:^|\/)gemini(?:-live)?-(\d+)/);
|
||||
if (!match) {
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -35,12 +35,18 @@ function functionResponseNames(parts: readonly Part[] | undefined): string[] {
|
||||
|
||||
describe("google-shared convertMessages — parallel tool results with an image (Gemini < 3)", () => {
|
||||
it.each([
|
||||
["image first", ["screenshot", "weather"]],
|
||||
["image last", ["weather", "screenshot"]],
|
||||
["bare Gemini 2.5 image first", "gemini-2.5-flash", ["screenshot", "weather"]],
|
||||
["bare Gemini 2.5 image last", "gemini-2.5-flash", ["weather", "screenshot"]],
|
||||
[
|
||||
"provider-prefixed Gemini 2.5 image first",
|
||||
"google/gemini-2.5-pro",
|
||||
["screenshot", "weather"],
|
||||
],
|
||||
["models-prefixed Gemini 2.5 image last", "models/gemini-2.5-pro", ["weather", "screenshot"]],
|
||||
] as const)(
|
||||
"keeps the response run immediate and retains a deferred %s result",
|
||||
(_label, resultOrder) => {
|
||||
const model = makeVisionModel("gemini-2.5-flash");
|
||||
"keeps the response run immediate and retains a deferred result for %s",
|
||||
(_label, modelId, resultOrder) => {
|
||||
const model = makeVisionModel(modelId);
|
||||
const toolResults = {
|
||||
screenshot: {
|
||||
role: "toolResult",
|
||||
@@ -84,4 +90,32 @@ describe("google-shared convertMessages — parallel tool results with an image
|
||||
expect(contents.slice(3).some((c) => countFunctionResponses(c.parts) > 0)).toBe(false);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["google/gemini-3.1-pro-preview", "models/gemini-3.1-pro-preview"])(
|
||||
"keeps image parts inside function responses for prefixed Gemini 3 model %s",
|
||||
(modelId) => {
|
||||
const model = makeVisionModel(modelId);
|
||||
const contents = convertMessagesForTest(model, {
|
||||
messages: [
|
||||
{ role: "user", content: "Take a screenshot." },
|
||||
makeGoogleAssistantMessage(model.id, [
|
||||
{ type: "toolCall", id: "call_1", name: "screenshot", arguments: {} },
|
||||
]),
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: "call_1",
|
||||
toolName: "screenshot",
|
||||
content: [{ type: "image", mimeType: "image/png", data: "AAAA" }],
|
||||
isError: false,
|
||||
timestamp: 0,
|
||||
},
|
||||
],
|
||||
} as unknown as Context);
|
||||
|
||||
expect(contents.map((content) => content.role)).toEqual(["user", "model", "user"]);
|
||||
expect(contents[2]?.parts?.[0]?.functionResponse?.parts).toEqual([
|
||||
{ inlineData: { mimeType: "image/png", data: "AAAA" } },
|
||||
]);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
@@ -140,7 +140,7 @@ export function requiresToolCallId(modelId: string): boolean {
|
||||
}
|
||||
|
||||
function getGeminiMajorVersion(modelId: string): number | undefined {
|
||||
const match = modelId.toLowerCase().match(/^gemini(?:-live)?-(\d+)/);
|
||||
const match = modelId.toLowerCase().match(/(?:^|\/)gemini(?:-live)?-(\d+)/);
|
||||
if (!match) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user