Add more prompts to LAMBADA

promptsource/app.py

@@ -229,7 +229,7 @@ def get_infos(d_name):
         "Dataset",
         dataset_list,
         key="dataset_select",
-        index=12,  # AG_NEWS
+        index=4,  # AG_NEWS
         help="Select the dataset to work on.",
     )
 

promptsource/seqio_tasks/experiment_D4.csv

@@ -83,6 +83,7 @@ imdb,,sentiment,cls,,,TRUE,TRUE,,25000,25000,TRUE,,25000,,,,,,imdb,cls/sentiment
 rotten_tomatoes,,sentiment,cls,,,TRUE,TRUE,,8530,8530,TRUE,,8530,,,,,,rotten tomatoes,cls/sentiment analysis,,,,Pang and Lee 2005
 yelp_review_full,,sentiment,cls,no dev set,,TRUE,TRUE,,650000,500000,TRUE,,500000,,,,,,yelp review full,other/regression,,,,Zhang et al. 2015; (link)
 lambada,,story_completion,gen,revisit: story or cloze or coref? trivial cloze prompt; training set is just unlabeled corpora; GPT task,GPT,,,TRUE,0,0,,TRUE,0,accuracy;perplexity;median_rank,,https://arxiv.org/pdf/1606.06031.pdf,,,,,,,,
+craffel/openai_lambada,,story_completion,gen,revisit: story or cloze or coref? trivial cloze prompt; training set is just unlabeled corpora; GPT task,GPT,,,TRUE,0,0,,TRUE,0,accuracy;perplexity;median_rank,,https://arxiv.org/pdf/1606.06031.pdf,,,,,,,,
 story_cloze,2016,story_completion,cls,todo: custom loading; swag like?,GPT,,,TRUE,,0,,TRUE,0,accuracy,,https://arxiv.org/pdf/1604.01696.pdf,,,,,,,,
 hellaswag,,story_completion,cls,,GPT,,,TRUE,39905,0,TRUE,,39905,accuracy,,https://arxiv.org/pdf/1905.07830.pdf,,,hellaswag,qa/multiple-choice qa,,,,Zellers et al. 2019
 common_gen,,structure_to_text,gen,,,TRUE,TRUE,,67389,67389,TRUE,,67389,,,,,,common gen,other,,,,Lin et al. 2020b

promptsource/templates.py

@@ -25,7 +25,7 @@
 
 # These are users whose datasets should be included in the results returned by
 # filter_english_datasets (regardless of their metadata)
-INCLUDED_USERS = {"Zaid"}
+INCLUDED_USERS = {"Zaid", "craffel"}
 
 
 def highlight(input):

promptsource/templates/craffel/openai_lambada/templates.yaml

@@ -0,0 +1,78 @@
+dataset: craffel/openai_lambada
+templates:
+  1ee5ddef-fffb-4b73-a2f7-f600ffac63cb: !Template
+    answer_choices: null
+    answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}'
+    id: 1ee5ddef-fffb-4b73-a2f7-f600ffac63cb
+    jinja: '{{ text.split()[:-1] | join('' '') }}...
+
+
+      What comes after the ellipses? ||| {{ text.split()[-1] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
+      original_task: true
+    name: ellipses
+    reference: ''
+  4f08e9d4-bcff-4bc0-9902-87c497625d17: !Template
+    answer_choices: null
+    answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}'
+    id: 4f08e9d4-bcff-4bc0-9902-87c497625d17
+    jinja: 'Fill in the blank:
+
+
+      {{ text.split()[:-1] | join('' '') }} ____. ||| {{ text.split()[-1]
+      }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
+      original_task: true
+    name: GPT-3 style
+    reference: Brown et al.
+  507de732-8298-4971-bac3-7d768d511a31: !Template
+    answer_choices: null
+    answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}'
+    id: 507de732-8298-4971-bac3-7d768d511a31
+    jinja: '{{ text.split()[:-1] | join('' '') }} ____.
+
+
+      Fill in the ____: ||| {{ text.split()[-1] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
+      original_task: true
+    name: fill in the ____
+    reference: ''
+  774b4349-0524-4a34-881b-b344f8f5c34e: !Template
+    answer_choices: null
+    answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}'
+    id: 774b4349-0524-4a34-881b-b344f8f5c34e
+    jinja: 'This story got cut short. What comes next?
+
+
+      {{ text.split()[:-1] | join('' '') }} ||| {{ text.split()[-1] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
+      original_task: true
+    name: what comes next
+    reference: ''
+  ef072a60-252e-4c52-aa8a-4152bb4dd83c: !Template
+    answer_choices: null
+    answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}'
+    id: ef072a60-252e-4c52-aa8a-4152bb4dd83c
+    jinja: 'Please predict the next word after the following chunk of text.
+
+
+      {{ text.split()[:-1] | join('' '') }} ||| {{ text.split()[-1] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
+      original_task: true
+    name: please next word
+    reference: ''

promptsource/templates/lambada/templates.yaml

@@ -1,15 +1,78 @@
 dataset: lambada
 templates:
+  3747e80a-4182-44eb-944b-dee40095bb17: !Template
+    answer_choices: null
+    answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}'
+    id: 3747e80a-4182-44eb-944b-dee40095bb17
+    jinja: 'Please predict the next word after the following chunk of text.
+
+
+      {{ text.split()[:-1] | join('' '') }} ||| {{ text.split()[-1] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
+      original_task: true
+    name: please next word
+    reference: ''
+  506765b8-17c0-4946-bbb0-b28288caacb3: !Template
+    answer_choices: null
+    answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}'
+    id: 506765b8-17c0-4946-bbb0-b28288caacb3
+    jinja: '{{ text.split()[:-1] | join('' '') }} ____.
+
+
+      Fill in the ____: ||| {{ text.split()[-1] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
+      original_task: true
+    name: fill in the ____
+    reference: ''
+  948664d5-2ea2-4245-b656-9283948dd5cd: !Template
+    answer_choices: null
+    answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}'
+    id: 948664d5-2ea2-4245-b656-9283948dd5cd
+    jinja: '{{ text.split()[:-1] | join('' '') }}...
+
+
+      What comes after the ellipses? ||| {{ text.split()[-1] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
+      original_task: true
+    name: ellipses
+    reference: ''
+  acfe374c-60ce-4354-b285-e7b0717cffe5: !Template
+    answer_choices: null
+    answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}'
+    id: acfe374c-60ce-4354-b285-e7b0717cffe5
+    jinja: 'This story got cut short. What comes next?
+
+
+      {{ text.split()[:-1] | join('' '') }} ||| {{ text.split()[-1] }}'
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
+      original_task: true
+    name: what comes next
+    reference: ''
   d5707bd9-d3cc-4535-b4c1-5c2aee8cb8c7: !Template
     answer_choices: null
-    answer_choices_key: null
+    answer_choices_key: '{{ text.split()[:-1] | unique | join('' ||| '') }}'
     id: d5707bd9-d3cc-4535-b4c1-5c2aee8cb8c7
-    jinja: '{{ text.split('' '')[:-1] | join('' '') }} ||| {{ text.split('' '')[-1]
+    jinja: 'Fill in the blank:
+
+
+      {{ text.split()[:-1] | join('' '') }} ____. ||| {{ text.split()[-1]
       }}'
     metadata: !TemplateMetadata
       choices_in_prompt: false
       metrics:
       - Accuracy
       original_task: true
-    name: predict the last word
-    reference: ''
+    name: GPT-3 style
+    reference: Brown et al.

test/test_templates.py

@@ -67,8 +67,10 @@ def test_dataset(dataset):
             else:
                 raise e
 
-    features = builder_instance.info.features.keys()
-    features = set([feature.replace("-", "_") for feature in features])
+    has_features = builder_instance.info.features is not None
+    if has_features:
+        features = builder_instance.info.features.keys()
+        features = set([feature.replace("-", "_") for feature in features])
 
     # Initializes sets for checking uniqueness among templates
     template_name_set = set()
@@ -89,7 +91,7 @@ def test_dataset(dataset):
 
         variables = meta.find_undeclared_variables(parse)
         for variable in variables:
-            if variable not in features and variable != "answer_choices":
+            if has_features and variable not in features and variable != "answer_choices":
                 raise ValueError(f"Template for dataset {dataset_name}/{subset_name} "
                                  f"with uuid {template.get_id()} has unrecognized variable {variable}.")