Multi modal eval fix

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_utils.py

@@ -90,24 +90,25 @@ def _store_multimodal_content(messages, tmpdir: str):
 
     # traverse all messages and replace base64 image data with new file name.
     for message in messages:
-        for content in message.get("content", []):
-            if content.get("type") == "image_url":
-                image_url = content.get("image_url")
-                if image_url and "url" in image_url and image_url["url"].startswith("data:image/jpg;base64,"):
-                    # Extract the base64 string
-                    base64image = image_url["url"].replace("data:image/jpg;base64,", "")
-
-                    # Generate a unique filename
-                    image_file_name = f"{str(uuid.uuid4())}.jpg"
-                    image_url["url"] = f"images/{image_file_name}"  # Replace the base64 URL with the file path
-
-                    # Decode the base64 string to binary image data
-                    image_data_binary = base64.b64decode(base64image)
-
-                    # Write the binary image data to the file
-                    image_file_path = os.path.join(images_folder_path, image_file_name)
-                    with open(image_file_path, "wb") as f:
-                        f.write(image_data_binary)
+        if isinstance(message.get("content", []), list):
+            for content in message.get("content", []):
+                if content.get("type") == "image_url":
+                    image_url = content.get("image_url")
+                    if image_url and "url" in image_url and image_url["url"].startswith("data:image/jpg;base64,"):
+                        # Extract the base64 string
+                        base64image = image_url["url"].replace("data:image/jpg;base64,", "")
+
+                        # Generate a unique filename
+                        image_file_name = f"{str(uuid.uuid4())}.jpg"
+                        image_url["url"] = f"images/{image_file_name}"  # Replace the base64 URL with the file path
+
+                        # Decode the base64 string to binary image data
+                        image_data_binary = base64.b64decode(base64image)
+
+                        # Write the binary image data to the file
+                        image_file_path = os.path.join(images_folder_path, image_file_name)
+                        with open(image_file_path, "wb") as f:
+                            f.write(image_data_binary)
 
 
 def _log_metrics_and_instance_results(

sdk/evaluation/azure-ai-evaluation/tests/unittests/data/generated_conv_image_urls.jsonl

@@ -0,0 +1,2 @@
+{"messages":[{"role":"system","content":[{"type":"text","text":"This is a nature boardwalk at the University of Wisconsin-Madison."}]},{"role":"user","content":[{"type":"text","text":"Can you describe this image?"},{"type":"image_url","image_url":{"url":"https://cdn.britannica.com/68/178268-050-5B4E7FB6/Tom-Cruise-2013.jpg"}}]}]}
+{"messages":[{"role":"system","content":[{"type":"text","text":"This is a nature boardwalk at the University of Wisconsin-Madison."}]},{"role":"user","content":[{"type":"text","text":"Can you describe this image?"},{"type":"image_url","image_url":{"url":"https://cdn.britannica.com/68/178268-050-5B4E7FB6/Tom-Cruise-2013.jpg"}}]}]}

sdk/evaluation/azure-ai-evaluation/tests/unittests/test_eval_run.py

@@ -4,8 +4,10 @@
 import time
 from unittest.mock import MagicMock, patch
 from uuid import uuid4
-
+import tempfile
 import jwt
+import pandas as pd
+import pathlib
 import pytest
 from promptflow.azure._utils._token_cache import ArmTokenCache
 
@@ -303,6 +305,30 @@ def test_wrong_artifact_path(
             assert len(caplog.records) == 1
             assert expected_error in caplog.records[0].message
 
+    def test_store_multi_modal_no_images(self, token_mock, caplog):
+        data_path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data")
+        data_file = os.path.join(data_path, "generated_qa_chat_conv.jsonl")
+        data_convo = pd.read_json(data_file, lines=True)
+        with tempfile.TemporaryDirectory() as tmpdir:
+            for value in data_convo["messages"]:
+                ev_utils._store_multimodal_content(value, tmpdir)
+
+    def test_store_multi_modal_image_urls(self, token_mock, caplog):
+        data_path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data")
+        data_file = os.path.join(data_path, "generated_conv_image_urls.jsonl")
+        data_convo = pd.read_json(data_file, lines=True)
+        with tempfile.TemporaryDirectory() as tmpdir:
+            for value in data_convo["messages"]:
+                ev_utils._store_multimodal_content(value, tmpdir)
+
+    def test_store_multi_modal_images(self, token_mock, caplog):
+        data_path = os.path.join(pathlib.Path(__file__).parent.resolve(), "data")
+        data_file = os.path.join(data_path, "generated_conv_images.jsonl")
+        data_convo = pd.read_json(data_file, lines=True)
+        with tempfile.TemporaryDirectory() as tmpdir:
+            for value in data_convo["messages"]:
+                ev_utils._store_multimodal_content(value, tmpdir)
+
     def test_log_metrics_and_instance_results_logs_error(self, token_mock, caplog):
         """Test that we are logging the error when there is no trace destination."""
         logger = logging.getLogger(ev_utils.__name__)