diff --git a/promptsource/templates/onestop_english/templates.yaml b/promptsource/templates/onestop_english/templates.yaml index 2755e8a2c..4ade30f88 100644 --- a/promptsource/templates/onestop_english/templates.yaml +++ b/promptsource/templates/onestop_english/templates.yaml @@ -17,7 +17,8 @@ templates: {{ answer_choices [label] }}' metadata: !TemplateMetadata choices_in_prompt: true - metrics: [] + metrics: + - Accuracy original_task: true name: esl_context reference: '' @@ -28,7 +29,7 @@ templates: How would you rate the difficulty level of the passage above for automatic readability - assessment? {{"Elementary"}}, {{"Intermediate"}} or {{"Advanced"}} level? + assessment? {{"Elementary"}}, {{"Intermediate"}} or {{"Advanced"}} level? ||| @@ -37,10 +38,36 @@ templates: {{ answer_choices [label] }}' metadata: !TemplateMetadata choices_in_prompt: true - metrics: [] + metrics: + - Accuracy original_task: true name: ara_context reference: '' + b0b076b8-b72b-4b71-8dd0-0b20877e001c: !Template + answer_choices: Elementary ||| Intermediate ||| Advanced + id: b0b076b8-b72b-4b71-8dd0-0b20877e001c + jinja: 'Look at the first three sentences of a story shown here: {{text.strip().split(''\n'')[:3]|join("\n")}} + + + + Based on these three sentences alone, can you predict whether the entire story + is on the {{"Elementary"}}, {{"Intermediate"}} or {{"Advanced"}} level? + + + + + + ||| + + + {{ answer_choices [label] }}' + metadata: !TemplateMetadata + choices_in_prompt: true + metrics: + - Accuracy + original_task: false + name: determine_reading_level_from_the_first_three_sentences + reference: '' de75ccb8-c0ba-4510-abf8-649b42019cd5: !Template answer_choices: Elementary ||| Intermediate ||| Advanced id: de75ccb8-c0ba-4510-abf8-649b42019cd5 @@ -60,7 +87,8 @@ templates: {{ answer_choices [label] }}' metadata: !TemplateMetadata choices_in_prompt: true - metrics: [] + metrics: + - Accuracy original_task: true name: esl_variation reference: '' @@ -80,7 +108,8 @@ templates: {{ answer_choices [label] }}' metadata: !TemplateMetadata choices_in_prompt: true - metrics: [] + metrics: + - Accuracy original_task: true name: assess reference: '' @@ -88,12 +117,13 @@ templates: answer_choices: Elementary ||| Intermediate ||| Advanced id: fdc96a76-6415-437b-bf61-ef6d1d1b8645 jinja: "Consider the following text passage: \n\n\"{{text}}\"\n\nHow would you\ - \ assess the difficulty level for the task of generating a simplified version\ - \ of the text passage above? Would it be at the {{\"Elementary\"}}, {{\"Intermediate\"\ - }} or {{\"Advanced\"}} level?\n\n|||\n\n{{ answer_choices [label] }}" + \ assess the difficulty level for the task of simplifying the text passage above?\ + \ Would it be at the {{\"Elementary\"}}, {{\"Intermediate\"}} or {{\"Advanced\"\ + }} level?\n\n|||\n\n{{ answer_choices [label] }}" metadata: !TemplateMetadata choices_in_prompt: true - metrics: [] + metrics: + - Accuracy original_task: true name: ats reference: '' diff --git a/promptsource/templates/scientific_papers/pubmed/templates.yaml b/promptsource/templates/scientific_papers/pubmed/templates.yaml index 4ae878482..f5207fd86 100644 --- a/promptsource/templates/scientific_papers/pubmed/templates.yaml +++ b/promptsource/templates/scientific_papers/pubmed/templates.yaml @@ -12,14 +12,17 @@ templates: {{ section_names.strip().split(''\n'')|join('', '')}}' metadata: !TemplateMetadata choices_in_prompt: null - metrics: [] + metrics: + - BLEU + - ROUGE + - Other original_task: false name: get_section_names_from_abstract reference: Given the abstract, get the section names. 7ef28be4-382c-46ee-9fd5-1c4d83aab433: !Template answer_choices: null id: 7ef28be4-382c-46ee-9fd5-1c4d83aab433 - jinja: 'Prepare the first three sentences of a scientific article for the following + jinja: 'Write the first three sentences of a scientific article for the following abstract: {{abstract}} @@ -28,41 +31,62 @@ templates: {{article.strip().split(''\n'')[:3]|join("\n")}}' metadata: !TemplateMetadata - choices_in_prompt: null - metrics: [] + choices_in_prompt: false + metrics: + - BLEU + - ROUGE original_task: false name: generate_first_three_sentences_of_an_article_from_abstract reference: Given the abstract, generate first three sentences of the article. a34723df-0c10-4553-8323-99c4cfb53544: !Template answer_choices: null id: a34723df-0c10-4553-8323-99c4cfb53544 - jinja: 'What would be the first line of scientific article for the following abstract: - {{ abstract }} + jinja: 'What would be the first line of a scientific article for the following + abstract: {{ abstract }} ||| {{ article.strip().split(''\n'')[0] }}' metadata: !TemplateMetadata - choices_in_prompt: null - metrics: [] + choices_in_prompt: false + metrics: + - BLEU + - ROUGE original_task: false name: generate_first_line_from_abstract reference: Given the abstract, generate the first line of the article. a8992de1-443f-4f6a-983a-99e10a34b328: !Template answer_choices: null id: a8992de1-443f-4f6a-983a-99e10a34b328 - jinja: 'Write the first 100 words for a scientific article with following abstract - : {{ abstract }} + jinja: 'Write the first 100 words for a scientific article with the following + abstract : {{ abstract }} ||| {{ article.strip().split('' '')[:100] |join('' '')}}' metadata: !TemplateMetadata - choices_in_prompt: null - metrics: [] + choices_in_prompt: false + metrics: + - BLEU + - ROUGE original_task: false name: generate_first_100_words_for_article reference: Given the abstract, generate the first 100 words of the article. + e5a41fa1-3e81-4c74-ad73-6aab5e4855c7: !Template + answer_choices: null + id: e5a41fa1-3e81-4c74-ad73-6aab5e4855c7 + jinja: "Suppose that you have an abstract for a scientific paper: {{abstract}}\ + \ \nand you have already written the first three sentences of the full article:\ + \ {{article.strip().split('\\n')[:3]}}. \nPlease generate the next two sentences\ + \ of the article.\n|||\n{{article.strip().split('\\n')[3:5]|join(\"\\n\")}}" + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - BLEU + - ROUGE + original_task: false + name: write_sentences_4_and_5_from_sentences_1_2_3_and_abstract + reference: '' ff694788-2e84-49e7-8df3-2665b8c687f1: !Template answer_choices: null id: ff694788-2e84-49e7-8df3-2665b8c687f1 @@ -74,7 +98,9 @@ templates: {{ abstract.strip().split(''\n'')[0]}}' metadata: !TemplateMetadata choices_in_prompt: null - metrics: [] + metrics: + - BLEU + - ROUGE original_task: false name: generate_first_line_abstract_from_first_three_lines_article reference: Given the first three lines of an article, write the first line of