Google VertexAI `with_structured_output` throws `StatusCode.INVALID_ARGUMENT`

Gemini API
1

I hit the following error:

self = <google.api_core.grpc_helpers_async._WrappedUnaryStreamCall object at 0x7208104dc590>

    async def wait_for_connection(self):
        try:
>           await self._call.wait_for_connection()

/home/khteh/.local/share/virtualenvs/rag-agent-YeW3dxEa/lib/python3.12/site-packages/google/api_core/grpc_helpers_async.py:77: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
/home/khteh/.local/share/virtualenvs/rag-agent-YeW3dxEa/lib/python3.12/site-packages/grpc/aio/_call.py:659: in wait_for_connection
    await self._raise_for_status()
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <_AioCall of RPC that terminated with:
	status = Unable to submit request because at least one contents field is requi...ntents field is required. Learn more: https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/gemini"}"
>

    async def _raise_for_status(self) -> None:
        if self._cython_call.is_locally_cancelled():
            raise asyncio.CancelledError()
        code = await self.code()
        if code != grpc.StatusCode.OK:
>           raise _create_rpc_error(
                await self.initial_metadata(), await self._cython_call.status()
            )
E           grpc.aio._call.AioRpcError: <AioRpcError of RPC that terminated with:
E           	status = StatusCode.INVALID_ARGUMENT
E           	details = "Unable to submit request because at least one contents field is required. Learn more: https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/gemini"
E           	debug_error_string = "UNKNOWN:Error received from peer ipv4:74.125.68.95:443 {created_time:"2025-03-21T19:02:12.96775249+08:00", grpc_status:3, grpc_message:"Unable to submit request because at least one contents field is required. Learn more: https://cloud.google.com/vertex-ai/generative-ai/docs/model-reference/gemini"}"
E           >

/home/khteh/.local/share/virtualenvs/rag-agent-YeW3dxEa/lib/python3.12/site-packages/grpc/aio/_call.py:272: AioRpcError

The code snippet:

class EmailModel(BaseModel):
    entity_name: str | None = Field(
        default=None,
        description="""The name of the entity sending the notice (if present
        in the message)""",
    )

llm = init_chat_model("gemini-2.0-flash", model_provider="google_vertexai", streaming=True)
email_parser_prompt = ChatPromptTemplate.from_messages(
        [
            (
                "system",
                """
                Parse the date of notice, sending entity name, sending entity
                phone, sending entity email, project id, site location,
                violation type, required changes, compliance deadline, and
                maximum potential fine from the message. If any of the fields
                aren't present, don't populate them. Try to cast dates into
                the YYYY-mm-dd format. Don't populate fields if they're not
                present in the message.

                Here's the notice message:

                {message}
                """,
            )
        ]
    )
email_parser_chain = (
    email_parser_prompt | lm.with_structured_output(EmailModel)
)

I have tried the following but to no avail:

await email_parser_chain.ainvoke({"message": email})
await email_parser_chain.ainvoke({"message": [{"role": "system", "content": email}]}