Updated the existing prompts

promptsource/templates/billsum/templates.yaml

@@ -1,104 +1,137 @@
 dataset: billsum
 templates:
-  3ac01292-4a54-4546-b4e6-c225ae114213: !Template
+  0938c6e4-dbaf-43d8-8d8f-4bc62489ae74: !Template
     answer_choices: null
-    id: 3ac01292-4a54-4546-b4e6-c225ae114213
-    jinja: 'Summarize: {{text}}|||
+    id: 0938c6e4-dbaf-43d8-8d8f-4bc62489ae74
+    jinja: 'Given the title: "{{title}}" and the summary of a bill: {{summary}}.
 
-      Title: {{title}}
+      Write this bill based on the title and summary.
 
-      Summary: {{summary}}'
+      |||
+
+      {{text}}'
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
-      original_task: null
-    name: 'Summarize: (text-> title,summary)'
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      - ROUGE
+      original_task: false
+    name: 'Write a bill: (title, summary->text)'
     reference: ''
   3c790ac3-0557-47a9-9b71-1cb435f15629: !Template
     answer_choices: null
     id: 3c790ac3-0557-47a9-9b71-1cb435f15629
-    jinja: 'Summarize this bill: {{text}} |||
-
-      {{title}}'
+    jinja: "Given a state bill: {{text}}. \nPlease write the title of this bill in\
+      \ one sentence.\n|||\n{{title}}"
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
-      original_task: null
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      - ROUGE
+      original_task: false
     name: 'Summarize this bill in one sentence: (text-> title)'
     reference: ''
   438192e5-d67a-4098-9d82-a9fe892f6be2: !Template
     answer_choices: null
     id: 438192e5-d67a-4098-9d82-a9fe892f6be2
-    jinja: 'Write a bill: {{summary}} |||
+    jinja: 'Given a summary of a bill: {{summary}}.
+
+      Write this bill.
+
+      |||
 
       {{text}}'
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
-      original_task: null
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      - ROUGE
+      original_task: false
     name: 'Write a bill: (summary-> text)'
     reference: ''
   4891a8e7-258c-41e2-80d3-0c1a054acb07: !Template
     answer_choices: null
     id: 4891a8e7-258c-41e2-80d3-0c1a054acb07
-    jinja: 'Write a bill: {{title}} |||
+    jinja: 'Given a title: "{{title}}" of a bill.
+
+      Write this bill based on this title.
+
+      |||
 
       {{text}}'
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
-      original_task: null
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      - ROUGE
+      original_task: false
     name: 'Write a bill: (title-> text)'
     reference: ''
   550fa161-af4e-4430-9844-ce7dad587733: !Template
     answer_choices: null
     id: 550fa161-af4e-4430-9844-ce7dad587733
-    jinja: 'Summarize this bill: {{text}} |||
+    jinja: 'Given this bill: {{text}}.
+
+      Write a summary of this bill.
+
+      |||
 
       {{summary}}'
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
-      original_task: null
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      - ROUGE
+      original_task: true
     name: 'Summarize this bill: (text-> summary)'
     reference: ''
   5d2404b9-63ff-406e-977d-eda6afb5c689: !Template
     answer_choices: null
     id: 5d2404b9-63ff-406e-977d-eda6afb5c689
-    jinja: '{{summary}}
-
-      ===
+    jinja: 'Given a summary: {{summary}}, we want to generate a title from this summary.
 
-      Generate title from summary:
+      |||
 
-      |||{{title}}'
+      {{title}}'
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
-      original_task: null
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      - ROUGE
+      original_task: false
     name: Generate title from summary
     reference: ''
   6a439a80-4924-49e9-b5ae-f661683b399f: !Template
     answer_choices: null
     id: 6a439a80-4924-49e9-b5ae-f661683b399f
-    jinja: 'Summarize: {{text}}
+    jinja: 'Summarize this US bill: {{text}}
+
+      |||
 
-      |||{{summary}}'
+      {{summary}}'
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
-      original_task: null
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      - ROUGE
+      original_task: true
     name: 'Summarize: (text -> summary )'
     reference: ''
   ea9f0376-6cec-450c-b258-89f479cb9f6d: !Template
     answer_choices: null
     id: ea9f0376-6cec-450c-b258-89f479cb9f6d
-    jinja: 'Summarize: {{summary}}
+    jinja: 'Given a summary of a bill: {{summary}}.
+
+      Please write the title of this summary.
 
-      |||{{title}}'
+      |||
+
+      {{title}}'
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
-      original_task: null
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      - ROUGE
+      original_task: false
     name: 'Summarize: (summary -> title)'
     reference: ''

promptsource/templates/pubmed_qa/pqa_labeled/templates.yaml

@@ -1,102 +1,113 @@
 dataset: pubmed_qa
 subset: pqa_labeled
 templates:
+  00584766-2415-4d10-ab76-bf86058faa07: !Template
+    answer_choices: null
+    id: 00584766-2415-4d10-ab76-bf86058faa07
+    jinja: "Given a research abstract: {{ context.contexts | join(\", \") }}\nAnd\
+      \ given keywords: {{context.meshes | join(\", \")}}.\n \nWhat is the question\
+      \ answered by the above research abstract? \n|||\n{{question}} "
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      - ROUGE
+      original_task: false
+    name: Generate Question Title with meshes
+    reference: ''
   00f58886-e04a-4efb-bf41-cfcbd00a5e7d: !Template
     answer_choices: null
     id: 00f58886-e04a-4efb-bf41-cfcbd00a5e7d
-    jinja: "\"{{ context.contexts | join(\", \") }}\"\n\nAnswer the following question.\n\
-      \nQ: \"{{question}}\" ||| \n{{long_answer}}\n"
+    jinja: "Given a PubMed abstract: {{ context.contexts | join(\", \") }}\n\nAnswer\
+      \ this question: \"{{question}}\" \n||| \n{{long_answer}}\n"
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
-      original_task: null
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      - ROUGE
+      - Other
+      original_task: false
     name: Question Answering (Long)
     reference: Provide a long/verbose answer to the provided question
   0b630e04-02a8-46d6-b164-a41cd34042ff: !Template
     answer_choices: null
     id: 0b630e04-02a8-46d6-b164-a41cd34042ff
-    jinja: '"{{ context.contexts | join(", ") }}"
-
-
-      What is the main question answered by the above research abstract? |||
-
-      {{question}} '
+    jinja: "Given a research abstract: {{ context.contexts | join(\", \") }}\n\nWhat\
+      \ is the question answered by the above research abstract? \n|||\n{{question}} "
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
-      original_task: null
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      - ROUGE
+      original_task: false
     name: 'Generate Question Title '
     reference: Given abstract, generate title (which is in the form of a question)
   1e0a77f8-0eb4-40a1-814d-8a111df66e5e: !Template
-    answer_choices: null
+    answer_choices: 'Yes ||| No ||| Maybe'
     id: 1e0a77f8-0eb4-40a1-814d-8a111df66e5e
-    jinja: "Q: \"{{ question  }}\" \n\nA: \"{{ long_answer }}\"\n\nSummarize the above\
-      \ answer as: YES, NO or MAYBE ? |||\n{{final_decision}}"
+    jinja: "Question: \"{{ question }}\" \n\nAnswer: \"{{ long_answer }}\"\n\nSummarize\
+      \ the above answer as YES, NO, or MAYBE? \n|||\n{{final_decision}}"
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
-      original_task: null
+      choices_in_prompt: true
+      metrics:
+      - Accuracy
+      original_task: false
     name: Long Answer to Final Decision
     reference: Given a question, the full text of the relevant answer, summarize a
       yes/no/maybe answer
   21240f74-530a-47b7-a5d9-a6a13083b72e: !Template
-    answer_choices: null
+    answer_choices: '{{context.labels | join("|||")}}'
     id: 21240f74-530a-47b7-a5d9-a6a13083b72e
-    jinja: '{% set n_sections = context.contexts | length %}
-
-      {% set choice = range(0, n_sections) | random %}
-
-
-      "{{ context.contexts[choice] }}"
-
-
-      In a research article, the above text would most likely be found in which section:  {{
-      context.labels[:-1] | join(", ") }} or {{ context.labels[-1] }} ? |||
-
-
-      {{ context.labels[choice] }}
-
-      '
+    jinja: "{% set n_sections = context.contexts | length %}\n{% set choice = range(0,\
+      \ n_sections) | random %}\n\n\"{{ context.contexts[choice] }}\"\n\nThe above\
+      \ text would most likely be found in which section of a biomedical paper:  {{\
+      \ context.labels[:-1] | join(\", \") }} or {{ context.labels[-1] }} ? \n|||\n\
+      {{ context.labels[choice] }}\n"
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
-      original_task: null
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
+      original_task: false
     name: Context Section Type
     reference: Assign text in an abstract to specific paper section headers
   45cb344c-bb36-492a-ace0-7cfc897e127a: !Template
     answer_choices: null
     id: 45cb344c-bb36-492a-ace0-7cfc897e127a
-    jinja: '"{{ context.contexts | join(", ") }}"
-
-
-      What are the MeSH terms for this PubMed abstract? |||
-
-      {{ context.meshes | join(", ") }}'
+    jinja: "Given a PubMed abstract:{{ context.contexts | join(\", \") }}\n\nWhat\
+      \ are the MeSH (Medical Subject Headings) terms for this? \n|||\n{{ context.meshes | join(\", \") }}"
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
-      original_task: null
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      - ROUGE
+      - Other
+      original_task: false
     name: Medical Subject Headings
     reference: 'Predict the set of MeSH terms for a given PubMed abstract '
-  91d481e5-fac6-4532-b013-5ac1235b6e1a: !Template
+  48ee58bb-6a4a-4667-9d9c-69618408c6ce: !Template
     answer_choices: null
+    id: 48ee58bb-6a4a-4667-9d9c-69618408c6ce
+    jinja: "Given a research abstract: {{ context.contexts | join(\", \") }}\nAnd\
+      \ given long answer of a question: {{long_answer}}.\n \nWhat is the question asked by this research paper?\
+      \ \n|||\n{{question}} "
+    metadata: !TemplateMetadata
+      choices_in_prompt: false
+      metrics:
+      - BLEU
+      - ROUGE
+      original_task: false
+    name: Generate Question Title with long answer
+    reference: ''
+  91d481e5-fac6-4532-b013-5ac1235b6e1a: !Template
+    answer_choices: 'Yes ||| No ||| Maybe'
     id: 91d481e5-fac6-4532-b013-5ac1235b6e1a
-    jinja: '"{{ context.contexts | join(", ") }}"
-
-
-      Answer the following question as YES, NO, MAYBE
-
-
-      Q: "{{question}}"
-
-
-      A: ||| {{final_decision}}
-
-
-      '
+    jinja: "Given a PubMed abstract: {{ context.contexts | join(\", \") }}\n\nAnswer\
+      \ the question: \"{{question}}\" as YES, NO, MAYBE.\n||| \n{{final_decision}}\n\
+      \n"
     metadata: !TemplateMetadata
-      choices_in_prompt: null
-      metrics: []
+      choices_in_prompt: false
+      metrics:
+      - Accuracy
       original_task: true
     name: Question Answering (Short)
     reference: Answer the following question using the provided abstract text