fix: Add E5 model test case in test CLI

docs/mmteb/points/958.jsonl

@@ -0,0 +1,2 @@
+{"GitHub": "isaac-chung", "Bug fixes": 2}
+{"GitHub": "KennethEnevoldsen", "Review PR": 2}

mteb/evaluation/evaluators/ClassificationEvaluator.py

@@ -91,7 +91,7 @@ def __init__(
         y_train,
         sentences_test,
         y_test,
-        task_name: str,
+        task_name: str | None = None,
         k: int = 1,
         batch_size: int = 32,
         limit: int | None = None,

mteb/evaluation/evaluators/SummarizationEvaluator.py

@@ -19,7 +19,7 @@
 class SummarizationEvaluator(Evaluator):
     def __init__(
         self,
-        task_name: str | None,
+        task_name: str | None = None,
         human_summaries=None,
         machine_summaries=None,
         texts=None,

tests/test_cli.py

@@ -4,6 +4,7 @@
 from argparse import Namespace
 from pathlib import Path
 
+import pytest
 import yaml
 
 from mteb.cli import create_meta
@@ -18,17 +19,33 @@ def test_available_tasks():
     ), "Sample task Banking77Classification task not found in available tasks"
 
 
+run_task_fixures = [
+    (
+        "average_word_embeddings_komninos",
+        "BornholmBitextMining",
+        "21eec43590414cb8e3a6f654857abed0483ae36e",
+    ),
+    (
+        "intfloat/multilingual-e5-small",
+        "BornholmBitextMining",
+        "e4ce9877abf3edfe10b0d82785e83bdcb973e22e",
+    ),
+]
+
+
+@pytest.mark.parametrize("model_name,task_name,model_revision", run_task_fixures)
 def test_run_task(
-    model_name: str = "average_word_embeddings_komninos",
-    task_name="BornholmBitextMining",
-    model_revision="21eec43590414cb8e3a6f654857abed0483ae36e",
+    model_name: str,
+    task_name: str,
+    model_revision: str,
 ):
     command = f"mteb run -m {model_name} -t {task_name} --verbosity 3 --output_folder tests/results/test_model --model_revision {model_revision}"
     result = subprocess.run(command, shell=True, capture_output=True, text=True)
     assert result.returncode == 0, "Command failed"
 
+    model_name_as_path = model_name.replace("/", "__").replace(" ", "_")
     results_path = Path(
-        f"tests/results/test_model/average_word_embeddings_komninos/{model_revision}"
+        f"tests/results/test_model/{model_name_as_path}/{model_revision}"
     )
     assert results_path.exists(), "Output folder not created"
     assert "model_meta.json" in [