I have a code execution tool which can return both text or images. I want to send it back to gemini to understand both text and image contents. here i am getting error.
ClientError: 400 Bad Request. {‘message’: ‘{\n “error”: {\n “code”: 400,\n “message”: “Multimodal function responses are not supported for this model.”,\n “status”: “INVALID_ARGUMENT”\n }\n}\n’, ‘status’: ‘Bad Request’}
My tool converstion code:
def custom_tool_message_to_gemini(message: CustomFunctionExecutionResultMessage) → types.Content:
"""Convert CustomFunctionExecutionResultMessage to Gemini Content with proper multiple image handling."""
parts: List\[types.Part\] = \[\]
for result in message.content:
if result.is_error:
parts.append(
types.Part(
function_response=types.FunctionResponse(
id=result.call_id,
name=result.name,
response={"error": str(result.content)}
)
)
)
else:
\# Process all content items and consolidate into a single FunctionResponse
response_parts: List\[types.FunctionResponsePart\] = \[\]
text_outputs: List\[str\] = \[\]
for item in result.content:
if isinstance(item, str):
text_outputs.append(item)
elif isinstance(item, Image):
\# Handle Image content (base64 encoded)
try:
response_parts.append(
types.FunctionResponsePart(
inline_data=types.FunctionResponseBlob(
data=base64.b64decode(item.to_base64()),
mime_type=get_mime_type_from_image(item)
)
)
)
except Exception as e:
text_outputs.append(f"Failed to process image: {str(e)}")
else:
\# Future response types can be handled here
raise ValueError(f"Unsupported tool response content type: {type(item)}")
\# Create consolidated response with both text and rich content
response_data = {}
if text_outputs:
response_data\["output"\] = " ".join(text_outputs)
parts.append(
types.Part(
function_response=types.FunctionResponse(
id=result.call_id,
name=result.name,
response=response_data,
parts=response_parts if response_parts else None
)
)
)
return types.Content(role="user", parts=parts)
a temporary work around for me was to send image as direct types.Part inline data not sending as function response.
let me know if i am any thing wrong in managing Image (because i see function reponse part accept blob but model is throwing error).
I have used 2.5 flash and pro models