diff --git a/promptsource/app.py b/promptsource/app.py index 236992036..945bd72d7 100644 --- a/promptsource/app.py +++ b/promptsource/app.py @@ -229,7 +229,7 @@ def get_infos(d_name): "Dataset", dataset_list, key="dataset_select", - index=12, # AG_NEWS + index=4, # AG_NEWS help="Select the dataset to work on.", ) diff --git a/promptsource/seqio_tasks/experiment_D4.csv b/promptsource/seqio_tasks/experiment_D4.csv index 4c0061a99..71c8216cc 100644 --- a/promptsource/seqio_tasks/experiment_D4.csv +++ b/promptsource/seqio_tasks/experiment_D4.csv @@ -83,6 +83,7 @@ imdb,,sentiment,cls,,,TRUE,TRUE,,25000,25000,TRUE,,25000,,,,,,imdb,cls/sentiment rotten_tomatoes,,sentiment,cls,,,TRUE,TRUE,,8530,8530,TRUE,,8530,,,,,,rotten tomatoes,cls/sentiment analysis,,,,Pang and Lee 2005 yelp_review_full,,sentiment,cls,no dev set,,TRUE,TRUE,,650000,500000,TRUE,,500000,,,,,,yelp review full,other/regression,,,,Zhang et al. 2015; (link) lambada,,story_completion,gen,revisit: story or cloze or coref? trivial cloze prompt; training set is just unlabeled corpora; GPT task,GPT,,,TRUE,0,0,,TRUE,0,accuracy;perplexity;median_rank,,https://arxiv.org/pdf/1606.06031.pdf,,,,,,,, +craffel/openai_lambada,,story_completion,gen,revisit: story or cloze or coref? trivial cloze prompt; training set is just unlabeled corpora; GPT task,GPT,,,TRUE,0,0,,TRUE,0,accuracy;perplexity;median_rank,,https://arxiv.org/pdf/1606.06031.pdf,,,,,,,, story_cloze,2016,story_completion,cls,todo: custom loading; swag like?,GPT,,,TRUE,,0,,TRUE,0,accuracy,,https://arxiv.org/pdf/1604.01696.pdf,,,,,,,, hellaswag,,story_completion,cls,,GPT,,,TRUE,39905,0,TRUE,,39905,accuracy,,https://arxiv.org/pdf/1905.07830.pdf,,,hellaswag,qa/multiple-choice qa,,,,Zellers et al. 2019 common_gen,,structure_to_text,gen,,,TRUE,TRUE,,67389,67389,TRUE,,67389,,,,,,common gen,other,,,,Lin et al. 2020b diff --git a/promptsource/templates.py b/promptsource/templates.py index 9f7261fbf..3e5786545 100644 --- a/promptsource/templates.py +++ b/promptsource/templates.py @@ -25,7 +25,7 @@ # These are users whose datasets should be included in the results returned by # filter_english_datasets (regardless of their metadata) -INCLUDED_USERS = {"Zaid"} +INCLUDED_USERS = {"Zaid", "craffel"} def highlight(input): diff --git a/promptsource/templates/craffel/openai_lambada/templates.yaml b/promptsource/templates/craffel/openai_lambada/templates.yaml new file mode 100644 index 000000000..a9f65446a --- /dev/null +++ b/promptsource/templates/craffel/openai_lambada/templates.yaml @@ -0,0 +1,78 @@ +dataset: craffel/openai_lambada +templates: + 1ee5ddef-fffb-4b73-a2f7-f600ffac63cb: !Template + answer_choices: null + answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}' + id: 1ee5ddef-fffb-4b73-a2f7-f600ffac63cb + jinja: '{{ text.split()[:-1] | join('' '') }}... + + + What comes after the ellipses? ||| {{ text.split()[-1] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: ellipses + reference: '' + 4f08e9d4-bcff-4bc0-9902-87c497625d17: !Template + answer_choices: null + answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}' + id: 4f08e9d4-bcff-4bc0-9902-87c497625d17 + jinja: 'Fill in the blank: + + + {{ text.split()[:-1] | join('' '') }} ____. ||| {{ text.split()[-1] + }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: GPT-3 style + reference: Brown et al. + 507de732-8298-4971-bac3-7d768d511a31: !Template + answer_choices: null + answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}' + id: 507de732-8298-4971-bac3-7d768d511a31 + jinja: '{{ text.split()[:-1] | join('' '') }} ____. + + + Fill in the ____: ||| {{ text.split()[-1] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: fill in the ____ + reference: '' + 774b4349-0524-4a34-881b-b344f8f5c34e: !Template + answer_choices: null + answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}' + id: 774b4349-0524-4a34-881b-b344f8f5c34e + jinja: 'This story got cut short. What comes next? + + + {{ text.split()[:-1] | join('' '') }} ||| {{ text.split()[-1] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: what comes next + reference: '' + ef072a60-252e-4c52-aa8a-4152bb4dd83c: !Template + answer_choices: null + answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}' + id: ef072a60-252e-4c52-aa8a-4152bb4dd83c + jinja: 'Please predict the next word after the following chunk of text. + + + {{ text.split()[:-1] | join('' '') }} ||| {{ text.split()[-1] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: please next word + reference: '' diff --git a/promptsource/templates/lambada/templates.yaml b/promptsource/templates/lambada/templates.yaml index f57cc4a1d..d3c2765fb 100644 --- a/promptsource/templates/lambada/templates.yaml +++ b/promptsource/templates/lambada/templates.yaml @@ -1,15 +1,78 @@ dataset: lambada templates: + 3747e80a-4182-44eb-944b-dee40095bb17: !Template + answer_choices: null + answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}' + id: 3747e80a-4182-44eb-944b-dee40095bb17 + jinja: 'Please predict the next word after the following chunk of text. + + + {{ text.split()[:-1] | join('' '') }} ||| {{ text.split()[-1] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: please next word + reference: '' + 506765b8-17c0-4946-bbb0-b28288caacb3: !Template + answer_choices: null + answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}' + id: 506765b8-17c0-4946-bbb0-b28288caacb3 + jinja: '{{ text.split()[:-1] | join('' '') }} ____. + + + Fill in the ____: ||| {{ text.split()[-1] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: fill in the ____ + reference: '' + 948664d5-2ea2-4245-b656-9283948dd5cd: !Template + answer_choices: null + answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}' + id: 948664d5-2ea2-4245-b656-9283948dd5cd + jinja: '{{ text.split()[:-1] | join('' '') }}... + + + What comes after the ellipses? ||| {{ text.split()[-1] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: ellipses + reference: '' + acfe374c-60ce-4354-b285-e7b0717cffe5: !Template + answer_choices: null + answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}' + id: acfe374c-60ce-4354-b285-e7b0717cffe5 + jinja: 'This story got cut short. What comes next? + + + {{ text.split()[:-1] | join('' '') }} ||| {{ text.split()[-1] }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Accuracy + original_task: true + name: what comes next + reference: '' d5707bd9-d3cc-4535-b4c1-5c2aee8cb8c7: !Template answer_choices: null - answer_choices_key: null + answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}' id: d5707bd9-d3cc-4535-b4c1-5c2aee8cb8c7 - jinja: '{{ text.split('' '')[:-1] | join('' '') }} ||| {{ text.split('' '')[-1] + jinja: 'Fill in the blank: + + + {{ text.split()[:-1] | join('' '') }} ____. ||| {{ text.split()[-1] }}' metadata: !TemplateMetadata choices_in_prompt: false metrics: - Accuracy original_task: true - name: predict the last word - reference: '' + name: GPT-3 style + reference: Brown et al. diff --git a/test/test_templates.py b/test/test_templates.py index 730fb37b0..c8a7bc919 100644 --- a/test/test_templates.py +++ b/test/test_templates.py @@ -67,8 +67,10 @@ def test_dataset(dataset): else: raise e - features = builder_instance.info.features.keys() - features = set([feature.replace("-", "_") for feature in features]) + has_features = builder_instance.info.features is not None + if has_features: + features = builder_instance.info.features.keys() + features = set([feature.replace("-", "_") for feature in features]) # Initializes sets for checking uniqueness among templates template_name_set = set() @@ -89,7 +91,7 @@ def test_dataset(dataset): variables = meta.find_undeclared_variables(parse) for variable in variables: - if variable not in features and variable != "answer_choices": + if has_features and variable not in features and variable != "answer_choices": raise ValueError(f"Template for dataset {dataset_name}/{subset_name} " f"with uuid {template.get_id()} has unrecognized variable {variable}.")