Add match_closest_option post processor for multiple choice qa

prepare/cards/arc.py

@@ -20,7 +20,7 @@
         task="tasks.qa.multiple_choice.with_topic",
         templates="templates.qa.multiple_choice.with_topic.all",
     )
-    test_card(card)
+    test_card(card, strict=False)
     add_to_catalog(
         card, f'cards.ai2_arc.{subtask.replace("-", "_").lower()}', overwrite=True
     )

prepare/cards/belebele.py

@@ -160,5 +160,5 @@
     )
 
     if lang == language_codes[0]:
-        test_card(card, demos_taken_from="test")
+        test_card(card, demos_taken_from="test", strict=False)
     add_to_catalog(card, f"cards.belebele.{lang.lower()}", overwrite=True)

prepare/cards/boolq.py

@@ -62,5 +62,5 @@
     templates="templates.qa.multiple_choice.with_context.all",
 )
 
-test_card(card, demos_taken_from="test")
+test_card(card, demos_taken_from="test", strict=False)
 add_to_catalog(card, "cards.boolq.multiple_choice", overwrite=True)

prepare/cards/copa.py

@@ -29,5 +29,5 @@
     templates="templates.qa.multiple_choice.with_context.all",
 )
 
-test_card(card)
+test_card(card, strict=False)
 add_to_catalog(card, "cards.copa", overwrite=True)

prepare/cards/hellaswag.py

@@ -17,7 +17,7 @@
         RenameFields(field_to_field={"label": "answer"}),
         CastFields(fields={"answer": "int"}),
     ],
-    task="tasks.completion.multiple_choice.standard",
+    task="tasks.completion.multiple_choice",
     templates="templates.completion.multiple_choice.all",
 )
 # We disable strict checking because garbage predictions (when using the post processor

prepare/cards/mmlu.py

@@ -76,7 +76,7 @@ def main():
             templates="templates.qa.multiple_choice.with_topic.all",
         )
         if i == 0:
-            test_card(card)
+            test_card(card, strict=False)
         add_to_catalog(card, f"cards.mmlu.{subtask}", overwrite=True)
 
 

prepare/cards/openbookqa.py

@@ -18,5 +18,5 @@
     task="tasks.qa.multiple_choice.open",
     templates="templates.qa.multiple_choice.open.all",
 )
-test_card(card)
+test_card(card, strict=False)
 add_to_catalog(card, "cards.openbook_qa", overwrite=True)

prepare/cards/piqa.py

@@ -14,5 +14,5 @@
     task="tasks.qa.multiple_choice.open",
     templates="templates.qa.multiple_choice.open.all",
 )
-test_card(card)
+test_card(card, strict=False)
 add_to_catalog(card, "cards.piqa", overwrite=True)

prepare/cards/race.py

@@ -20,5 +20,5 @@
         templates="templates.qa.multiple_choice.with_context.all",
     )
     if subset == "middle":
-        test_card(card)
+        test_card(card, strict=False)
     add_to_catalog(card, f"cards.race_{subset}", overwrite=True)

prepare/cards/sciq.py

@@ -26,5 +26,5 @@
     task="tasks.qa.multiple_choice.with_context",
     templates="templates.qa.multiple_choice.with_context.all",
 )
-test_card(card)
+test_card(card, strict=False)
 add_to_catalog(card, "cards.sciq", overwrite=True)

prepare/cards/winogrande.py

@@ -19,7 +19,7 @@
         templates="templates.qa.multiple_choice.open.all",
     )
     if subtask == "debiased":
-        test_card(card, demos_taken_from="test")
+        test_card(card, demos_taken_from="test", strict=False)
     add_to_catalog(
         card, f"cards.winogrande.{subtask.replace('-', '_')}", overwrite=True
     )

prepare/cards/xwinogrande.py

@@ -18,5 +18,5 @@
         templates="templates.qa.multiple_choice.open.all",
     )
     if lang == "pt":
-        test_card(card, demos_taken_from="test")
+        test_card(card, demos_taken_from="test", strict=False)
     add_to_catalog(card, f"cards.xwinogrande.{lang}", overwrite=True)

prepare/processors/processors.py

@@ -9,6 +9,7 @@
     GetStringAfter,
     LowerCase,
     LowerCaseTillPunc,
+    MatchClosestOption,
     StanceToProCon,
     StringOrNotString,
     StrToFloatFormat,
@@ -237,3 +238,20 @@
     "processors.remove_none_from_list",
     overwrite=True,
 )
+
+
+add_to_catalog(
+    SequentialOperator(
+        steps=[
+            MatchClosestOption(
+                field="prediction",
+            ),
+            MatchClosestOption(
+                field="references",
+                process_every_value=True,
+            ),
+        ]
+    ),
+    "processors.match_closest_option",
+    overwrite=True,
+)

prepare/tasks/completion/standard.py → prepare/tasks/completion/multiple_choice.py

@@ -7,6 +7,6 @@
         outputs=["answer", "choices"],
         metrics=["metrics.accuracy"],
     ),
-    "tasks.completion.multiple_choice.standard",
+    "tasks.completion.multiple_choice",
     overwrite=True,
 )

prepare/templates/completion/multiple_choice/templates.py

@@ -1,15 +1,13 @@
 from src.unitxt.catalog import add_to_catalog
 from src.unitxt.templates import MultipleChoiceTemplate, TemplatesList
 
-input_format = "{context}"
 add_to_catalog(
     MultipleChoiceTemplate(
-        input_format=input_format,
+        input_format="{context}",
         target_field="answer",
         choices_seperator="",
         source_choice_format="{choice_text}",
-        target_choice_format=" {choice_text}",
-        postprocessors=["processors.first_character"],
+        target_choice_format="{choice_text}",
     ),
     "templates.completion.multiple_choice.simple",
     overwrite=True,

prepare/templates/qa/multiple_choice/templates.py

@@ -234,12 +234,30 @@
         target_prefix="Answer:\n",
         target_field="answer",
         choices_seperator="\n",
-        postprocessors=["processors.first_character"],
+        postprocessors=["processors.to_string_stripped", "processors.first_character"],
     ),
     "templates.qa.multiple_choice.title",
     overwrite=True,
 )
 
+add_to_catalog(
+    MultipleChoiceTemplate(
+        instruction="Answer the multiple choice Question based on the Choices (choose from {numerals}).",
+        input_format="Question:\n{question}\nChoices:\n{choices}",
+        target_prefix="Answer:\n",
+        target_field="answer",
+        choices_seperator="\n",
+        target_choice_format="{choice_numeral}. {choice_text}",
+        postprocessors=[
+            "processors.take_first_non_empty_line",
+            "processors.match_closest_option",
+        ],
+    ),
+    "templates.qa.multiple_choice.match",
+    overwrite=True,
+)
+
+
 input_format = "Context: {context}\nQuestion: {question}\nChoices:\n{choices}\nAnswer:"
 add_to_catalog(
     MultipleChoiceTemplate(
@@ -259,13 +277,31 @@
         target_prefix="Answer:\n",
         target_field="answer",
         choices_seperator="\n",
-        postprocessors=["processors.first_character"],
+        postprocessors=["processors.to_string_stripped", "processors.first_character"],
         title_fields=["context_type"],
     ),
     "templates.qa.multiple_choice.with_context.title",
     overwrite=True,
 )
 
+add_to_catalog(
+    MultipleChoiceTemplate(
+        instruction="Answer the multiple choice Question from one of the Choices (choose from {numerals}) based on the {context_type}.",
+        input_format="{context_type}:\n{context}\nQuestion:\n{question}\nChoices:\n{choices}",
+        target_prefix="Answer:\n",
+        target_field="answer",
+        choices_seperator="\n",
+        target_choice_format="{choice_numeral}. {choice_text}",
+        postprocessors=[
+            "processors.take_first_non_empty_line",
+            "processors.match_closest_option",
+        ],
+        title_fields=["context_type"],
+    ),
+    "templates.qa.multiple_choice.with_context.match",
+    overwrite=True,
+)
+
 # fm_eval
 
 input_format = "The following are multiple choice questions (with answers) about {topic}.\n\nQuestion: {question}\nChoose from {numerals}\nAnswers:\n{choices}\nAnswer:"
@@ -299,26 +335,61 @@
         target_prefix="Answer:\n",
         target_field="answer",
         choices_seperator="\n",
-        postprocessors=["processors.first_character"],
+        postprocessors=["processors.to_string_stripped", "processors.first_character"],
         title_fields=["context_type"],
     ),
     "templates.qa.multiple_choice.with_context.with_topic.title",
     overwrite=True,
 )
 
+add_to_catalog(
+    MultipleChoiceTemplate(
+        instruction="Answer the multiple choice Question about {topic} from one of the Choices (choose from {numerals}) based on the {context_type}.",
+        input_format="{context_type}:\n{context}\nQuestion:\n{question}\nChoices:\n{choices}",
+        target_prefix="Answer:\n",
+        target_field="answer",
+        choices_seperator="\n",
+        target_choice_format="{choice_numeral}. {choice_text}",
+        postprocessors=[
+            "processors.take_first_non_empty_line",
+            "processors.match_closest_option",
+        ],
+        title_fields=["context_type"],
+    ),
+    "templates.qa.multiple_choice.with_context.with_topic.match",
+    overwrite=True,
+)
+
 add_to_catalog(
     MultipleChoiceTemplate(
         instruction="Answer the multiple choice Question about {topic} from one of the Choices (choose from {numerals}).",
         input_format="Question:\n{question}\nChoices:\n{choices}",
         target_prefix="Answer:\n",
         target_field="answer",
         choices_seperator="\n",
-        postprocessors=["processors.first_character"],
+        postprocessors=["processors.to_string_stripped", "processors.first_character"],
     ),
     "templates.qa.multiple_choice.with_topic.title",
     overwrite=True,
 )
 
+add_to_catalog(
+    MultipleChoiceTemplate(
+        instruction="Answer the multiple choice Question about {topic} from one of the Choices (choose from {numerals}).",
+        input_format="Question:\n{question}\nChoices:\n{choices}",
+        target_prefix="Answer:\n",
+        target_field="answer",
+        choices_seperator="\n",
+        target_choice_format="{choice_numeral}. {choice_text}",
+        postprocessors=[
+            "processors.take_first_non_empty_line",
+            "processors.match_closest_option",
+        ],
+    ),
+    "templates.qa.multiple_choice.with_topic.match",
+    overwrite=True,
+)
+
 
 def remove_duplicates(input_list):
     seen = set()
@@ -342,6 +413,7 @@ def remove_duplicates(input_list):
                     )[0].items
                 ],
                 "templates.qa.multiple_choice.with_context.title",
+                "templates.qa.multiple_choice.with_context.match",
             ]
         )
     ),
@@ -363,6 +435,7 @@ def remove_duplicates(input_list):
                     )[0].items
                 ],
                 "templates.qa.multiple_choice.with_context.with_topic.title",
+                "templates.qa.multiple_choice.with_context.with_topic.match",
             ]
         )
     ),
@@ -385,6 +458,7 @@ def remove_duplicates(input_list):
                     )[0].items
                 ],
                 "templates.qa.multiple_choice.with_topic.title",
+                "templates.qa.multiple_choice.with_topic.match",
             ]
         )
     ),

src/unitxt/catalog/cards/hellaswag.json

@@ -27,6 +27,6 @@
             }
         }
     ],
-    "task": "tasks.completion.multiple_choice.standard",
+    "task": "tasks.completion.multiple_choice",
     "templates": "templates.completion.multiple_choice.all"
 }

src/unitxt/catalog/processors/match_closest_option.json

@@ -0,0 +1,14 @@
+{
+    "type": "sequential_operator",
+    "steps": [
+        {
+            "type": "match_closest_option",
+            "field": "prediction"
+        },
+        {
+            "type": "match_closest_option",
+            "field": "references",
+            "process_every_value": true
+        }
+    ]
+}

src/unitxt/catalog/templates/qa/multiple_choice/match.json

@@ -0,0 +1,13 @@
+{
+    "type": "multiple_choice_template",
+    "instruction": "Answer the multiple choice Question based on the Choices (choose from {numerals}).",
+    "input_format": "Question:\n{question}\nChoices:\n{choices}",
+    "target_prefix": "Answer:\n",
+    "target_field": "answer",
+    "choices_seperator": "\n",
+    "target_choice_format": "{choice_numeral}. {choice_text}",
+    "postprocessors": [
+        "processors.take_first_non_empty_line",
+        "processors.match_closest_option"
+    ]
+}

src/unitxt/catalog/templates/qa/multiple_choice/title.json

@@ -6,6 +6,7 @@
     "target_field": "answer",
     "choices_seperator": "\n",
     "postprocessors": [
+        "processors.to_string_stripped",
         "processors.first_character"
     ]
 }

src/unitxt/catalog/templates/qa/multiple_choice/with_context/all.json

@@ -5,6 +5,7 @@
         "templates.qa.multiple_choice.with_context.no_intro.helm",
         "templates.qa.multiple_choice.with_context.no_intro.mmlu",
         "templates.qa.multiple_choice.with_context.no_intro.lm_eval_harness",
-        "templates.qa.multiple_choice.with_context.title"
+        "templates.qa.multiple_choice.with_context.title",
+        "templates.qa.multiple_choice.with_context.match"
     ]
 }

src/unitxt/catalog/templates/qa/multiple_choice/with_context/match.json

@@ -0,0 +1,16 @@
+{
+    "type": "multiple_choice_template",
+    "instruction": "Answer the multiple choice Question from one of the Choices (choose from {numerals}) based on the {context_type}.",
+    "input_format": "{context_type}:\n{context}\nQuestion:\n{question}\nChoices:\n{choices}",
+    "target_prefix": "Answer:\n",
+    "target_field": "answer",
+    "choices_seperator": "\n",
+    "target_choice_format": "{choice_numeral}. {choice_text}",
+    "postprocessors": [
+        "processors.take_first_non_empty_line",
+        "processors.match_closest_option"
+    ],
+    "title_fields": [
+        "context_type"
+    ]
+}

src/unitxt/catalog/templates/qa/multiple_choice/with_context/title.json

@@ -6,6 +6,7 @@
     "target_field": "answer",
     "choices_seperator": "\n",
     "postprocessors": [
+        "processors.to_string_stripped",
         "processors.first_character"
     ],
     "title_fields": [

src/unitxt/catalog/templates/qa/multiple_choice/with_context/with_topic/all.json

@@ -5,6 +5,7 @@
         "templates.qa.multiple_choice.with_context.with_topic.mmlu",
         "templates.qa.multiple_choice.with_context.with_topic.helm",
         "templates.qa.multiple_choice.with_context.with_topic.lm_eval_harness",
-        "templates.qa.multiple_choice.with_context.with_topic.title"
+        "templates.qa.multiple_choice.with_context.with_topic.title",
+        "templates.qa.multiple_choice.with_context.with_topic.match"
     ]
 }