Add llm as judge mt-bench dataset and metrics

.github/workflows/catalog_consistency.yml

@@ -12,13 +12,15 @@ jobs:
      runs-on: ubuntu-latest
      env:
        OS: ubuntu-latest
+       GENAI_KEY: "dummy"
+       UNITXT_ALLOW_PASSING_DATA_TO_REMOTE_API: "True"
 
      steps:
      - uses: actions/checkout@v4
 
      - uses: actions/setup-python@v5
        with:
-         python-version: '3.8'
+         python-version: '3.9'
          cache: 'pip' # caching pip dependencies
      - run: pip install -r requirements/base.rqr
      - run: pip install -r requirements/tests.rqr

.github/workflows/catalog_preparation.yml

@@ -12,13 +12,15 @@ jobs:
      runs-on: ubuntu-latest
      env:
        OS: ubuntu-latest
+       GENAI_KEY: "dummy"
+       UNITXT_ALLOW_PASSING_DATA_TO_REMOTE_API: "True"
 
      steps:
      - uses: actions/checkout@v4
 
      - uses: actions/setup-python@v5
        with:
-         python-version: '3.8'
+         python-version: '3.9'
          cache: 'pip' # caching pip dependencies
      - run: pip install -r requirements/base.rqr
      - run: pip install -r requirements/tests.rqr

.github/workflows/library_tests.yml

@@ -18,7 +18,7 @@ jobs:
 
     - uses: actions/setup-python@v5
       with:
-        python-version: '3.8'
+        python-version: '3.9'
         cache: 'pip' # caching pip dependencies
     - run: pip install -r requirements/base.rqr
     - run: pip install -r requirements/tests.rqr

prepare/cards/dynamic_cards_for_llm_judges/llm_as_judge_metrics.py

@@ -0,0 +1,15 @@
+from unitxt.blocks import TaskCard
+from unitxt.catalog import add_to_catalog
+
+tasks = [
+    "tasks.response_assessment.rating.single_turn",
+    "tasks.response_assessment.rating.single_turn_with_reference",
+]
+for task in tasks:
+    card = TaskCard(loader=None, preprocess_steps=[], task=task)
+    sub_task = ".".join(task.split(".")[-2:])
+    add_to_catalog(
+        card,
+        f"cards.dynamic_cards_for_llm_judges.{sub_task}",
+        overwrite=True,
+    )

prepare/cards/mt_bench/generation/english_single_turn.py

@@ -0,0 +1,42 @@
+from unitxt.blocks import (
+    TaskCard,
+)
+from unitxt.catalog import add_to_catalog
+from unitxt.loaders import LoadHF
+from unitxt.operators import (
+    AddFields,
+    CopyFields,
+    RenameFields,
+)
+from unitxt.splitters import RenameSplits
+from unitxt.test_utils.card import test_card
+
+card = TaskCard(
+    loader=LoadHF(path="dim/mt_bench_en", split="train"),
+    preprocess_steps=[
+        RenameSplits({"train": "test"}),
+        CopyFields(field_to_field={"turns/0": "turns"}),
+        RenameFields(
+            field_to_field={
+                "turns": "input",
+                "category": "group",
+            }
+        ),
+        AddFields(
+            fields={
+                "output": "None",
+                "type_of_input": "question",
+                "type_of_output": "answer",
+            }
+        ),
+    ],
+    task="tasks.generation",
+    templates=["templates.empty"],
+)
+
+test_card(card, demos_taken_from="test", strict=False)
+add_to_catalog(
+    card,
+    "cards.mt_bench.generation.english_single_turn",
+    overwrite=True,
+)

prepare/cards/mt_bench/generation/japanese_single_turn.py

@@ -0,0 +1,42 @@
+from unitxt.blocks import (
+    TaskCard,
+)
+from unitxt.catalog import add_to_catalog
+from unitxt.loaders import LoadHF
+from unitxt.operators import (
+    AddFields,
+    CopyFields,
+    RenameFields,
+)
+from unitxt.splitters import RenameSplits
+from unitxt.test_utils.card import test_card
+
+card = TaskCard(
+    loader=LoadHF(path="shi3z/MTbenchJapanese", split="train"),
+    preprocess_steps=[
+        RenameSplits({"train": "test"}),
+        CopyFields(field_to_field={"turns/0": "turns"}),
+        RenameFields(
+            field_to_field={
+                "turns": "input",
+                "category": "group",
+            }
+        ),
+        AddFields(
+            fields={
+                "output": "None",
+                "type_of_input": "question",
+                "type_of_output": "answer",
+            }
+        ),
+    ],
+    task="tasks.generation",
+    templates=["templates.empty"],
+)
+
+test_card(card, demos_taken_from="test", strict=False)
+add_to_catalog(
+    card,
+    "cards.mt_bench.generation.japanese_single_turn",
+    overwrite=True,
+)

prepare/cards/mt_bench/response_assessment/pairwise_comparison/multi_turn_gpt4_judgement.py

@@ -0,0 +1,62 @@
+from unitxt.blocks import (
+    TaskCard,
+)
+from unitxt.catalog import add_to_catalog
+from unitxt.loaders import LoadHF
+from unitxt.operators import (
+    FilterByCondition,
+    InterleaveListsToDialogOperator,
+    MapInstanceValues,
+    RenameFields,
+)
+from unitxt.processors import LiteralEval
+from unitxt.splitters import RenameSplits
+from unitxt.test_utils.card import test_card
+
+card = TaskCard(
+    loader=LoadHF(
+        path="OfirArviv/mt_bench_pairwise_comparison_gpt4_judgments", split="train"
+    ),
+    preprocess_steps=[
+        RenameSplits({"train": "test"}),
+        FilterByCondition(values={"turn": 2}, condition="eq"),
+        FilterByCondition(values={"reference": "[]"}, condition="eq"),
+        FilterByCondition(
+            values={"winner": ["model_1", "tie", "model_2"]}, condition="in"
+        ),
+        MapInstanceValues(
+            mappers={
+                "winner": {"model_1": "choice_a", "model_2": "choice_b", "tie": "tie"}
+            }
+        ),
+        RenameFields(
+            field_to_field={
+                "category": "group",
+            }
+        ),
+        LiteralEval("model_input", to_field="model_input"),
+        LiteralEval("model_1_output", to_field="model_1_output"),
+        LiteralEval("model_2_output", to_field="model_2_output"),
+        InterleaveListsToDialogOperator(
+            user_turns_field="model_input",
+            assistant_turns_field="model_1_output",
+            to_field="dialog_a",
+        ),
+        InterleaveListsToDialogOperator(
+            user_turns_field="model_input",
+            assistant_turns_field="model_2_output",
+            to_field="dialog_b",
+        ),
+    ],
+    task="tasks.response_assessment.pairwise_comparison.multi_turn",
+    templates=[
+        "templates.response_assessment.pairwise_comparison.mt_bench_multi_turn_with_shuffle"
+    ],
+)
+
+test_card(card, demos_taken_from="test", strict=False)
+add_to_catalog(
+    card,
+    "cards.mt_bench.response_assessment.pairwise_comparison.multi_turn_gpt4_judgement",
+    overwrite=True,
+)

prepare/cards/mt_bench/response_assessment/pairwise_comparison/multi_turn_with_reference_gpt4_judgement.py

@@ -0,0 +1,64 @@
+from unitxt.blocks import (
+    TaskCard,
+)
+from unitxt.catalog import add_to_catalog
+from unitxt.loaders import LoadHF
+from unitxt.operators import (
+    FilterByCondition,
+    InterleaveListsToDialogOperator,
+    MapInstanceValues,
+    RenameFields,
+)
+from unitxt.processors import LiteralEval
+from unitxt.splitters import RenameSplits
+from unitxt.test_utils.card import test_card
+
+card = TaskCard(
+    loader=LoadHF(
+        path="OfirArviv/mt_bench_pairwise_comparison_gpt4_judgments", split="train"
+    ),
+    preprocess_steps=[
+        RenameSplits({"train": "test"}),
+        FilterByCondition(values={"turn": 2}, condition="eq"),
+        FilterByCondition(values={"reference": "[]"}, condition="ne"),
+        FilterByCondition(
+            values={"winner": ["model_1", "tie", "model_2"]}, condition="in"
+        ),
+        MapInstanceValues(
+            mappers={
+                "winner": {"model_1": "choice_a", "model_2": "choice_b", "tie": "tie"}
+            }
+        ),
+        RenameFields(field_to_field={"category": "group"}),
+        LiteralEval("model_input", to_field="model_input"),
+        LiteralEval("model_1_output", to_field="model_1_output"),
+        LiteralEval("model_2_output", to_field="model_2_output"),
+        LiteralEval("reference", to_field="reference"),
+        InterleaveListsToDialogOperator(
+            user_turns_field="model_input",
+            assistant_turns_field="model_1_output",
+            to_field="dialog_a",
+        ),
+        InterleaveListsToDialogOperator(
+            user_turns_field="model_input",
+            assistant_turns_field="model_2_output",
+            to_field="dialog_b",
+        ),
+        InterleaveListsToDialogOperator(
+            user_turns_field="model_input",
+            assistant_turns_field="reference",
+            to_field="reference_dialog",
+        ),
+    ],
+    task="tasks.response_assessment.pairwise_comparison.multi_turn_with_reference",
+    templates=[
+        "templates.response_assessment.pairwise_comparison.mt_bench_multi_turn_with_reference_with_shuffle"
+    ],
+)
+
+test_card(card, demos_taken_from="test", strict=False, loader_limit=1000)
+add_to_catalog(
+    card,
+    "cards.mt_bench.response_assessment.pairwise_comparison.multi_turn_with_reference_gpt4_judgement",
+    overwrite=True,
+)

prepare/cards/mt_bench/response_assessment/pairwise_comparison/single_turn_gpt4_judgement.py

@@ -0,0 +1,58 @@
+from unitxt.blocks import (
+    TaskCard,
+)
+from unitxt.catalog import add_to_catalog
+from unitxt.loaders import LoadHF
+from unitxt.operators import (
+    CopyFields,
+    FilterByCondition,
+    MapInstanceValues,
+    RenameFields,
+)
+from unitxt.processors import LiteralEval
+from unitxt.splitters import RenameSplits
+from unitxt.test_utils.card import test_card
+
+card = TaskCard(
+    loader=LoadHF(
+        path="OfirArviv/mt_bench_pairwise_comparison_gpt4_judgments", split="train"
+    ),
+    preprocess_steps=[
+        RenameSplits({"train": "test"}),
+        FilterByCondition(values={"turn": 1}, condition="eq"),
+        FilterByCondition(values={"reference": "[]"}, condition="eq"),
+        FilterByCondition(
+            values={"winner": ["model_1", "tie", "model_2"]}, condition="in"
+        ),
+        MapInstanceValues(
+            mappers={
+                "winner": {"model_1": "choice_a", "model_2": "choice_b", "tie": "tie"}
+            }
+        ),
+        RenameFields(
+            field_to_field={
+                "model_input": "question",
+                "model_1_output": "answer_a",
+                "model_2_output": "answer_b",
+                "category": "group",
+            }
+        ),
+        LiteralEval("question", to_field="question"),
+        CopyFields(field_to_field={"question/0": "question"}),
+        LiteralEval("answer_a", to_field="answer_a"),
+        CopyFields(field_to_field={"answer_a/0": "answer_a"}),
+        LiteralEval("answer_b", to_field="answer_b"),
+        CopyFields(field_to_field={"answer_b/0": "answer_b"}),
+    ],
+    task="tasks.response_assessment.pairwise_comparison.single_turn",
+    templates=[
+        "templates.response_assessment.pairwise_comparison.mt_bench_single_turn_with_shuffle"
+    ],
+)
+
+test_card(card, demos_taken_from="test", strict=False)
+add_to_catalog(
+    card,
+    "cards.mt_bench.response_assessment.pairwise_comparison.single_turn_gpt4_judgement",
+    overwrite=True,
+)

prepare/cards/mt_bench/response_assessment/pairwise_comparison/single_turn_with_reference_gpt4_judgement.py

@@ -0,0 +1,61 @@
+from unitxt.blocks import (
+    TaskCard,
+)
+from unitxt.catalog import add_to_catalog
+from unitxt.loaders import LoadHF
+from unitxt.operators import (
+    CopyFields,
+    FilterByCondition,
+    MapInstanceValues,
+    RenameFields,
+)
+from unitxt.processors import LiteralEval
+from unitxt.splitters import RenameSplits
+from unitxt.test_utils.card import test_card
+
+card = TaskCard(
+    loader=LoadHF(
+        path="OfirArviv/mt_bench_pairwise_comparison_gpt4_judgments", split="train"
+    ),
+    preprocess_steps=[
+        RenameSplits({"train": "test"}),
+        FilterByCondition(values={"turn": 1}, condition="eq"),
+        FilterByCondition(values={"reference": "[]"}, condition="ne"),
+        FilterByCondition(
+            values={"winner": ["model_1", "tie", "model_2"]}, condition="in"
+        ),
+        MapInstanceValues(
+            mappers={
+                "winner": {"model_1": "choice_a", "model_2": "choice_b", "tie": "tie"}
+            }
+        ),
+        RenameFields(
+            field_to_field={
+                "model_input": "question",
+                "model_1_output": "answer_a",
+                "model_2_output": "answer_b",
+                "reference": "reference_answer",
+                "category": "group",
+            }
+        ),
+        LiteralEval("question", to_field="question"),
+        CopyFields(field_to_field={"question/0": "question"}),
+        LiteralEval("answer_a", to_field="answer_a"),
+        CopyFields(field_to_field={"answer_a/0": "answer_a"}),
+        LiteralEval("answer_b", to_field="answer_b"),
+        CopyFields(field_to_field={"answer_b/0": "answer_b"}),
+        LiteralEval("reference_answer", to_field="reference_answer"),
+        CopyFields(field_to_field={"reference_answer/0": "reference_answer"}),
+    ],
+    task="tasks.response_assessment.pairwise_comparison.single_turn_with_reference",
+    templates=[
+        "templates.response_assessment.pairwise_comparison.mt_bench_single_turn_with_reference_with_shuffle"
+    ],
+)
+
+test_card(card, demos_taken_from="test", strict=False, loader_limit=1000)
+add_to_catalog(
+    card,
+    "cards.mt_bench.response_assessment.pairwise_comparison.single_turn_with_reference_gpt4_judgement",
+    overwrite=True,
+)