From ad1388a63a3f9ef94ba780fd26f8d3ff6b706d33 Mon Sep 17 00:00:00 2001 From: Kangmin Tan Date: Fri, 28 Jan 2022 11:35:41 -0800 Subject: [PATCH 1/2] add templates for riddle_sense dataset Use almost identical templates as commonsense_qa because the two datasets are very similar. --- .../templates/riddle_sense/templates.yaml | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 promptsource/templates/riddle_sense/templates.yaml diff --git a/promptsource/templates/riddle_sense/templates.yaml b/promptsource/templates/riddle_sense/templates.yaml new file mode 100644 index 000000000..c247db951 --- /dev/null +++ b/promptsource/templates/riddle_sense/templates.yaml @@ -0,0 +1,119 @@ +dataset: riddle_sense +templates: + 3df2405b-4a0e-4fcf-8600-b7f4843b945a: !Template + answer_choices: null + id: 3df2405b-4a0e-4fcf-8600-b7f4843b945a + jinja: '{% if answerKey != "" %} + + Use the following options to predict a possible question for them: + + + {% for i in range(choices[''text'']|length) %} + + - {{choices[''text''][i]}} + + {% endfor %} ||| + + {{question}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + - ROUGE + original_task: false + name: answer_to_question + reference: '' + 8b44338c-f635-47fc-86fb-3135ef2a76ae: !Template + answer_choices: '{{choices.text | join("|||")}}' + id: 8b44338c-f635-47fc-86fb-3135ef2a76ae + jinja: '{% if answerKey != "" %} + + Given the options below, select the most suitable answer for the following question: + + {{question}} + + Options: + + - {{answer_choices | join("\n- ")}}||| + + {{answer_choices[choices["label"].index(answerKey)]}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: most_suitable_answer + reference: '' + ac002734-5b1f-4478-9ffc-f16d9ca2f70e: !Template + answer_choices: '{{choices.text | join("|||")}}' + id: ac002734-5b1f-4478-9ffc-f16d9ca2f70e + jinja: '{% if answerKey != "" %} + + Answer the following question: + + {{question}} ||| + + {{ answer_choices[choices[''label''].index(answerKey)] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: answer_given_question_without_options + reference: '' + bf3f7bd2-91c6-455c-b9f7-42ce265fa2db: !Template + answer_choices: A ||| B ||| C ||| D ||| E + id: bf3f7bd2-91c6-455c-b9f7-42ce265fa2db + jinja: '{% if answerKey != "" %} + + Given the following options, what do you think is the correct answer to the + question below: + + + {{question}} + + + Options: + + {% for letter, t in zip(answer_choices, choices.text) %} + + - {{letter}}: {{t}} + + {% endfor %} ||| + + {{answerKey}} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: question_to_answer_index + reference: '' + bf7d7cbc-aa05-4aca-97ff-29eb34502019: !Template + answer_choices: '{{choices.text | join("|||")}}' + id: bf7d7cbc-aa05-4aca-97ff-29eb34502019 + jinja: '{% if answerKey != "" %} + + {{question}} + + + - {{answer_choices | join("\n- ")}} ||| + + {{answer_choices[choices[''label''].index(answerKey)] }} + + {% endif %}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: true + name: question_answering + reference: '' From 6c62a512972f7866bb64c570b058b20484b2fa90 Mon Sep 17 00:00:00 2001 From: Victor Sanh Date: Fri, 4 Feb 2022 19:12:19 +0000 Subject: [PATCH 2/2] moving `{% if answerKey != "" %}` to the target --- .../templates/riddle_sense/templates.yaml | 35 +++++++++---------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/promptsource/templates/riddle_sense/templates.yaml b/promptsource/templates/riddle_sense/templates.yaml index c247db951..773dccb69 100644 --- a/promptsource/templates/riddle_sense/templates.yaml +++ b/promptsource/templates/riddle_sense/templates.yaml @@ -3,9 +3,7 @@ templates: 3df2405b-4a0e-4fcf-8600-b7f4843b945a: !Template answer_choices: null id: 3df2405b-4a0e-4fcf-8600-b7f4843b945a - jinja: '{% if answerKey != "" %} - - Use the following options to predict a possible question for them: + jinja: 'Use the following options to predict a possible question for them: {% for i in range(choices[''text'']|length) %} @@ -14,9 +12,7 @@ templates: {% endfor %} ||| - {{question}} - - {% endif %}' + {{question}}' metadata: !TemplateMetadata choices_in_prompt: false metrics: @@ -28,9 +24,8 @@ templates: 8b44338c-f635-47fc-86fb-3135ef2a76ae: !Template answer_choices: '{{choices.text | join("|||")}}' id: 8b44338c-f635-47fc-86fb-3135ef2a76ae - jinja: '{% if answerKey != "" %} - - Given the options below, select the most suitable answer for the following question: + jinja: 'Given the options below, select the most suitable answer for the following + question: {{question}} @@ -38,6 +33,8 @@ templates: - {{answer_choices | join("\n- ")}}||| + {% if answerKey != "" %} + {{answer_choices[choices["label"].index(answerKey)]}} {% endif %}' @@ -51,12 +48,12 @@ templates: ac002734-5b1f-4478-9ffc-f16d9ca2f70e: !Template answer_choices: '{{choices.text | join("|||")}}' id: ac002734-5b1f-4478-9ffc-f16d9ca2f70e - jinja: '{% if answerKey != "" %} - - Answer the following question: + jinja: 'Answer the following question: {{question}} ||| + {% if answerKey != "" %} + {{ answer_choices[choices[''label''].index(answerKey)] }} {% endif %}' @@ -70,10 +67,8 @@ templates: bf3f7bd2-91c6-455c-b9f7-42ce265fa2db: !Template answer_choices: A ||| B ||| C ||| D ||| E id: bf3f7bd2-91c6-455c-b9f7-42ce265fa2db - jinja: '{% if answerKey != "" %} - - Given the following options, what do you think is the correct answer to the - question below: + jinja: 'Given the following options, what do you think is the correct answer to + the question below: {{question}} @@ -87,6 +82,8 @@ templates: {% endfor %} ||| + {% if answerKey != "" %} + {{answerKey}} {% endif %}' @@ -100,13 +97,13 @@ templates: bf7d7cbc-aa05-4aca-97ff-29eb34502019: !Template answer_choices: '{{choices.text | join("|||")}}' id: bf7d7cbc-aa05-4aca-97ff-29eb34502019 - jinja: '{% if answerKey != "" %} - - {{question}} + jinja: '{{question}} - {{answer_choices | join("\n- ")}} ||| + {% if answerKey != "" %} + {{answer_choices[choices[''label''].index(answerKey)] }} {% endif %}'