Handle gemini audio input (#10739)

* fix(vertex_ai/gemini/transformation.py): handle gemini audio data translation

Fixes #10070

* feat(vertex_ai/gemini/transformation.py): Handle audio format param translation

Fixes #10070

* fix: fix linting error

* test: update test

* fix: fix linting error

Author: krrishdholakia

litellm/llms/vertex_ai/gemini/transformation.py

@@ -16,6 +16,7 @@
     _get_image_mime_type_from_url,
 )
 from litellm.litellm_core_utils.prompt_templates.factory import (
+    convert_generic_image_chunk_to_openai_image_obj,
     convert_to_anthropic_image_obj,
     convert_to_gemini_tool_call_invoke,
     convert_to_gemini_tool_call_result,
@@ -45,6 +46,7 @@
     ToolConfig,
     Tools,
 )
+from litellm.types.utils import GenericImageParsingChunk
 
 from ..common_utils import (
     _check_text_in_content,
@@ -154,10 +156,26 @@ def _gemini_convert_messages_with_history(  # noqa: PLR0915
                             _parts.append(_part)
                         elif element["type"] == "input_audio":
                             audio_element = cast(ChatCompletionAudioObject, element)
-                            if audio_element["input_audio"].get("data") is not None:
+                            audio_data = audio_element["input_audio"].get("data")
+                            audio_format = audio_element["input_audio"].get("format")
+                            if audio_data is not None and audio_format is not None:
+                                audio_format_modified = (
+                                    "audio/" + audio_format
+                                    if audio_format.startswith("audio/") is False
+                                    else audio_format
+                                )  # Gemini expects audio/wav, audio/mp3, etc.
+                                openai_image_str = (
+                                    convert_generic_image_chunk_to_openai_image_obj(
+                                        image_chunk=GenericImageParsingChunk(
+                                            type="base64",
+                                            media_type=audio_format_modified,
+                                            data=audio_data,
+                                        )
+                                    )
+                                )
                                 _part = _process_gemini_image(
-                                    image_url=audio_element["input_audio"]["data"],
-                                    format=audio_element["input_audio"].get("format"),
+                                    image_url=openai_image_str,
+                                    format=audio_format_modified,
                                 )
                                 _parts.append(_part)
                         elif element["type"] == "file":

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -398,6 +398,19 @@ def _map_thinking_param(
 
         return params
 
+    def map_response_modalities(self, value: list) -> list:
+        response_modalities = []
+        for modality in value:
+            if modality == "text":
+                response_modalities.append("TEXT")
+            elif modality == "image":
+                response_modalities.append("IMAGE")
+            elif modality == "audio":
+                response_modalities.append("AUDIO")
+            else:
+                response_modalities.append("MODALITY_UNSPECIFIED")
+        return response_modalities
+
     def map_openai_params(
         self,
         non_default_params: Dict,
@@ -465,14 +478,7 @@ def map_openai_params(
                     cast(AnthropicThinkingParam, value)
                 )
             elif param == "modalities" and isinstance(value, list):
-                response_modalities = []
-                for modality in value:
-                    if modality == "text":
-                        response_modalities.append("TEXT")
-                    elif modality == "image":
-                        response_modalities.append("IMAGE")
-                    else:
-                        response_modalities.append("MODALITY_UNSPECIFIED")
+                response_modalities = self.map_response_modalities(value)
                 optional_params["responseModalities"] = response_modalities
 
         if litellm.vertex_ai_safety_settings is not None:

tests/llm_translation/base_llm_unit_tests.py

@@ -525,6 +525,45 @@ class EventsList(BaseModel):
         except litellm.InternalServerError:
             pytest.skip("Model is overloaded")
 
+    @pytest.mark.flaky(retries=6, delay=1)
+    def test_audio_input(self):
+        """
+        Test that audio input is supported by the LLM API
+        """
+        from litellm.utils import supports_audio_input
+        litellm._turn_on_debug()
+        base_completion_call_args = self.get_base_completion_call_args()
+        if not supports_audio_input(base_completion_call_args["model"], None):
+            pytest.skip(
+                f"Model={base_completion_call_args['model']} does not support audio input"
+            )
+
+        url = "https://openaiassets.blob.core.windows.net/$web/API/docs/audio/alloy.wav"
+        response = httpx.get(url)
+        response.raise_for_status()
+        wav_data = response.content
+        encoded_string = base64.b64encode(wav_data).decode("utf-8")
+
+        completion = self.completion_function(
+            **base_completion_call_args,
+            messages=[
+                {
+                    "role": "user",
+                    "content": [
+                        {"type": "text", "text": "What is in this recording?"},
+                        {
+                            "type": "input_audio",
+                            "input_audio": {"data": encoded_string, "format": "wav"},
+                        },
+                    ],
+                },
+            ],
+        )
+
+        print(completion.choices[0].message)
+
+
+            
     @pytest.mark.flaky(retries=6, delay=1)
     def test_json_response_format_stream(self):
         """
@@ -979,7 +1018,7 @@ async def test_completion_cost(self):
         assert response._hidden_params["response_cost"] > 0
 
     @pytest.mark.parametrize("input_type", ["input_audio", "audio_url"])
-    @pytest.mark.parametrize("format_specified", [True, False])
+    @pytest.mark.parametrize("format_specified", [True])
     def test_supports_audio_input(self, input_type, format_specified):
         from litellm.utils import return_raw_request, supports_audio_input
         from litellm.types.utils import CallTypes
@@ -1010,16 +1049,10 @@ def test_supports_audio_input(self, input_type, format_specified):
         test_file_id = "gs://bucket/file.wav"
 
         if input_type == "input_audio":
-            if format_specified:
-                audio_content.append({
-                    "type": "input_audio",
-                    "input_audio": {"data": encoded_string, "format": audio_format},
-                })
-            else:
-                audio_content.append({
-                    "type": "input_audio",
-                    "input_audio": {"data": encoded_string},
-                })
+            audio_content.append({
+                "type": "input_audio",
+                "input_audio": {"data": encoded_string, "format": audio_format},
+            })
         elif input_type == "audio_url":
             audio_content.append(
                 {