diff --git a/promptsource/templates/wino_bias/type1_anti/templates.yaml b/promptsource/templates/wino_bias/type1_anti/templates.yaml index 5626dc013..304363ede 100644 --- a/promptsource/templates/wino_bias/type1_anti/templates.yaml +++ b/promptsource/templates/wino_bias/type1_anti/templates.yaml @@ -1,15 +1,40 @@ dataset: wino_bias subset: type1_anti templates: - 51578f5e-4913-41d9-b109-054b24a2f0b5: !Template + 4faa9623-6d11-47d1-8d6e-bb41af088cff: !Template answer_choices: null answer_choices_key: null - id: 51578f5e-4913-41d9-b109-054b24a2f0b5 - jinja: 'What does "{{tokens[coreference_clusters[2] | int : coreference_clusters[3] - | int + 1] | join(" ")}}" refer to in the following sentence? + id: 4faa9623-6d11-47d1-8d6e-bb41af088cff + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} - {{tokens | join(" ")}} ||| {{tokens[coreference_clusters[0] | int : coreference_clusters[1] - | int + 1] | join(" ")}}' + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} + + {{tokens | join(" ")}} + + In the previous sentence, the pronoun "{{ pronoun }}" can be replaced with ||| + {{ referent }}' + metadata: !TemplateMetadata + choices_in_prompt: false + metrics: + - Other + original_task: true + name: replaced with + reference: '' + 5e5c9f7b-2c07-42d7-baf2-925e91a5fb9b: !Template + answer_choices: null + answer_choices_key: null + id: 5e5c9f7b-2c07-42d7-baf2-925e91a5fb9b + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} + + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} + + What does "{{ pronoun }}" refer to in the following sentence? + + {{tokens | join(" ")}} ||| {{referent}}' metadata: !TemplateMetadata _do_eval: false _do_train: false @@ -19,15 +44,19 @@ templates: original_task: true name: refers_to reference: '' - 51ae7473-969e-4219-b213-ff857f8fc487: !Template + 5ea6715b-20b2-4f10-8122-54ed3af54763: !Template answer_choices: null answer_choices_key: null - id: 51ae7473-969e-4219-b213-ff857f8fc487 - jinja: '{{tokens | join(" ")}} + id: 5ea6715b-20b2-4f10-8122-54ed3af54763 + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} + + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} - What is the correct referent for "{{tokens[coreference_clusters[2] | int : coreference_clusters[3] - | int + 1] | join(" ")}}" in the above sentence: ||| {{tokens[coreference_clusters[0] - | int : coreference_clusters[1] | int + 1] | join(" ")}}' + In the sentence below, what does "{{pronoun}}" represent? + + {{tokens | join(" ")}} ||| {{referent}}' metadata: !TemplateMetadata _do_eval: false _do_train: false @@ -35,63 +64,102 @@ templates: metrics: - Other original_task: true - name: referent + name: represent reference: '' - 6abfe0fb-1f40-44ae-bc1f-78e4f6628398: !Template + 8d5eedf2-de08-41fb-a584-7f35df315fd3: !Template answer_choices: null answer_choices_key: null - id: 6abfe0fb-1f40-44ae-bc1f-78e4f6628398 - jinja: 'Coreference occurs in a text when two or more expressions refer to the - same person or thing. Consider the following sentence: "{{tokens | join(" ")}}" + id: 8d5eedf2-de08-41fb-a584-7f35df315fd3 + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} + + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} - What is the coreferent of {{tokens[coreference_clusters[2] | int : coreference_clusters[3] - | int + 1] | join(" ")}}? + {{tokens | join(" ")}} - ||| {{tokens[coreference_clusters[0] | int : coreference_clusters[1] | int + - 1] | join(" ")}}' + In the passage above, the pronoun "{{ pronoun }}" refers to ||| {{ referent + }}' metadata: !TemplateMetadata - _do_eval: false - _do_train: false + _do_eval: true + _do_train: true choices_in_prompt: false metrics: - Other original_task: true - name: explanation_coreference + name: the pronoun refers to + reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." + d102cd81-e0d1-46bf-9e7d-a620328ad3bf: !Template + answer_choices: null + answer_choices_key: null + id: d102cd81-e0d1-46bf-9e7d-a620328ad3bf + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} + + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} + + {{tokens | join(" ")}} + + Here, what does "{{ pronoun }}" stand for? ||| {{ referent }}' + metadata: !TemplateMetadata + _do_eval: true + _do_train: true + choices_in_prompt: false + metrics: + - Other + original_task: true + name: What does p stand for reference: '' - b25cd2ed-554c-43ff-9060-b2590a8f733c: !Template + d355811f-eb29-4e6e-9d57-299eea1d96e1: !Template answer_choices: null answer_choices_key: null - id: b25cd2ed-554c-43ff-9060-b2590a8f733c - jinja: 'In the sentence "The pan did not fit in the suitcase because it was too - big", "it" refers to the pan. + id: d355811f-eb29-4e6e-9d57-299eea1d96e1 + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} + + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} + + {{tokens | join(" ")}} - For the sentence: "{{tokens | join(" ")}}", what does "{{tokens[coreference_clusters[2] - | int : coreference_clusters[3] | int + 1] | join(" ")}}" refer to? ||| {{tokens[coreference_clusters[0] - | int : coreference_clusters[1] | int + 1] | join(" ")}}' + Here, by "{{ pronoun }}" they mean ||| {{ referent }}' metadata: !TemplateMetadata - _do_eval: false - _do_train: false choices_in_prompt: false metrics: - Other original_task: true - name: example_coreference + name: by p they mean reference: '' - d743c10b-7afc-48d4-af12-a23bc7c89866: !Template + f4bdb35d-ccb0-4482-a47e-603f8566301e: !Template answer_choices: null answer_choices_key: null - id: d743c10b-7afc-48d4-af12-a23bc7c89866 - jinja: 'In the sentence below, what does "{{tokens[coreference_clusters[2] | int - : coreference_clusters[3] | int + 1] | join(" ")}}" represent? + id: f4bdb35d-ccb0-4482-a47e-603f8566301e + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} - {{tokens | join(" ")}} ||| {{tokens[coreference_clusters[0] | int : coreference_clusters[1] - | int + 1] | join(" ")}}' + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} + + {{tokens | join(" ")}} + + {% if pronoun.lower() == "they" or pronoun.lower() == "them" %} + + Question: Who or what are "{{ pronoun }}"? + + {% else %} + + Question: Who or what is "{{ pronoun }}"? + + {% endif %} + + Answer: ||| {{ referent }}' metadata: !TemplateMetadata - _do_eval: false - _do_train: false + _do_eval: true + _do_train: true choices_in_prompt: false metrics: - Other original_task: true - name: represent + name: Who or what is/are reference: '' diff --git a/promptsource/templates/wino_bias/type1_pro/templates.yaml b/promptsource/templates/wino_bias/type1_pro/templates.yaml index 90c9f52f2..112a387cd 100644 --- a/promptsource/templates/wino_bias/type1_pro/templates.yaml +++ b/promptsource/templates/wino_bias/type1_pro/templates.yaml @@ -1,15 +1,19 @@ dataset: wino_bias subset: type1_pro templates: - 0cc669d0-da87-411d-9e1c-8e102da63c7d: !Template + 13b2dbe4-abf3-4b09-b7cb-459224881800: !Template answer_choices: null answer_choices_key: null - id: 0cc669d0-da87-411d-9e1c-8e102da63c7d - jinja: '{{tokens | join(" ")}} + id: 13b2dbe4-abf3-4b09-b7cb-459224881800 + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} + + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} + + What does "{{ pronoun }}" refer to in the following sentence? - What is the correct referent for "{{tokens[coreference_clusters[2] | int : coreference_clusters[3] | int + 1] | join(" ")}}" - in the above sentence: - ||| {{tokens[coreference_clusters[0] | int : coreference_clusters[1] | int + 1] | join(" ")}}' + {{tokens | join(" ")}} ||| {{referent}}' metadata: !TemplateMetadata _do_eval: false _do_train: false @@ -17,17 +21,21 @@ templates: metrics: - Other original_task: true - name: referent + name: refers_to reference: '' - 13b2dbe4-abf3-4b09-b7cb-459224881800: !Template + 13b2dbe4-abf3-4b09-b7cb-459224881801: !Template answer_choices: null answer_choices_key: null - id: 13b2dbe4-abf3-4b09-b7cb-459224881800 - jinja: 'What does "{{tokens[coreference_clusters[2] | int : coreference_clusters[3] | int + 1] | join(" ")}}" refer - to in the following sentence? + id: 13b2dbe4-abf3-4b09-b7cb-459224881801 + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} + + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} + + In the sentence below, what does "{{pronoun}}" represent? - {{tokens | join(" ")}} - ||| {{tokens[coreference_clusters[0] | int : coreference_clusters[1] | int + 1] | join(" ")}}' + {{tokens | join(" ")}} ||| {{referent}}' metadata: !TemplateMetadata _do_eval: false _do_train: false @@ -35,58 +43,123 @@ templates: metrics: - Other original_task: true - name: refers_to + name: represent reference: '' - 5f874d91-c3aa-4437-90d0-4319904030a3: !Template + 143449f6-350a-44ef-ab4d-857841eadaf8: !Template answer_choices: null answer_choices_key: null - id: 5f874d91-c3aa-4437-90d0-4319904030a3 - jinja: 'In the sentence "The pan did not fit in the suitcase because it was too big", "it" refers to the pan. + id: 143449f6-350a-44ef-ab4d-857841eadaf8 + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} + + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} - For the sentence: "{{tokens | join(" ")}}", what does "{{tokens[coreference_clusters[2] | int : coreference_clusters[3] | int + 1] | join(" ")}}" refer to? - ||| {{tokens[coreference_clusters[0] | int : coreference_clusters[1] | int + 1] | join(" ")}}' + {{tokens | join(" ")}} + + In the previous sentence, the pronoun "{{ pronoun }}" can be replaced with ||| + {{ referent }}' metadata: !TemplateMetadata - _do_eval: false - _do_train: false choices_in_prompt: false metrics: - - Other + - Other original_task: true - name: example_coreference + name: replaced with reference: '' - 13b2dbe4-abf3-4b09-b7cb-459224881801: !Template + 18004871-0d0c-4f59-976c-53becd04c98f: !Template answer_choices: null answer_choices_key: null - id: 13b2dbe4-abf3-4b09-b7cb-459224881801 - jinja: 'In the sentence below, what does "{{tokens[coreference_clusters[2] | int : coreference_clusters[3] | int + 1] | join(" ")}}" represent? + id: 18004871-0d0c-4f59-976c-53becd04c98f + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} + + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} - {{tokens | join(" ")}} - ||| {{tokens[coreference_clusters[0] | int : coreference_clusters[1] | int + 1] | join(" ")}}' + {{tokens | join(" ")}} + + {% if pronoun.lower() == "they" or pronoun.lower() == "them" %} + + Question: Who or what are "{{ pronoun }}"? + + {% else %} + + Question: Who or what is "{{ pronoun }}"? + + {% endif %} + + Answer: ||| {{ referent }}' metadata: !TemplateMetadata - _do_eval: false - _do_train: false + _do_eval: true + _do_train: true choices_in_prompt: false metrics: - - Other + - Other original_task: true - name: represent + name: Who or what is/are reference: '' - 5f874d91-c3aa-4437-90d0-4319904030a2: !Template + 1ab4e47e-bb58-47c4-8148-fcfaf4a75785: !Template answer_choices: null answer_choices_key: null - id: 5f874d91-c3aa-4437-90d0-4319904030a2 - jinja: 'Coreference occurs in a text when two or more expressions refer to the same person or thing. - Consider the following sentence: "{{tokens | join(" ")}}" + id: 1ab4e47e-bb58-47c4-8148-fcfaf4a75785 + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} + + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} - What is the coreferent of {{tokens[coreference_clusters[2] | int : coreference_clusters[3] | int + 1] | join(" ")}}? + {{tokens | join(" ")}} - ||| {{tokens[coreference_clusters[0] | int : coreference_clusters[1] | int + 1] | join(" ")}}' + Here, what does "{{ pronoun }}" stand for? ||| {{ referent }}' + metadata: !TemplateMetadata + _do_eval: true + _do_train: true + choices_in_prompt: false + metrics: + - Other + original_task: true + name: What does p stand for + reference: '' + 97fb69f9-34d6-4fb2-bb60-75679c4a25c1: !Template + answer_choices: null + answer_choices_key: null + id: 97fb69f9-34d6-4fb2-bb60-75679c4a25c1 + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} + + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} + + {{tokens | join(" ")}} + + In the passage above, the pronoun "{{ pronoun }}" refers to ||| {{ referent + }}' + metadata: !TemplateMetadata + _do_eval: true + _do_train: true + choices_in_prompt: false + metrics: + - Other + original_task: true + name: the pronoun refers to + reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." + e5ac51e8-beaf-4cf9-a7fe-20d8cc2b1d0a: !Template + answer_choices: null + answer_choices_key: null + id: e5ac51e8-beaf-4cf9-a7fe-20d8cc2b1d0a + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} + + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} + + {{tokens | join(" ")}} + + Here, by "{{ pronoun }}" they mean ||| {{ referent }}' metadata: !TemplateMetadata - _do_eval: false - _do_train: false choices_in_prompt: false metrics: - Other original_task: true - name: explanation_coreference + name: by p they mean reference: '' diff --git a/promptsource/templates/wino_bias/type2_anti/templates.yaml b/promptsource/templates/wino_bias/type2_anti/templates.yaml index d16450604..d2e10f3b8 100644 --- a/promptsource/templates/wino_bias/type2_anti/templates.yaml +++ b/promptsource/templates/wino_bias/type2_anti/templates.yaml @@ -1,18 +1,19 @@ dataset: wino_bias subset: type2_anti templates: - 4961e233-d738-4503-80f0-c5e8812aa2b7: !Template + 3cdaa371-affb-48da-ba8f-f3dcb574fdcc: !Template answer_choices: null answer_choices_key: null - id: 4961e233-d738-4503-80f0-c5e8812aa2b7 - jinja: 'Coreference occurs in a text when two or more expressions refer to the - same person or thing. Consider the following sentence: "{{tokens | join(" ")}}" + id: 3cdaa371-affb-48da-ba8f-f3dcb574fdcc + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} - What is the coreferent of {{tokens[coreference_clusters[2] | int : coreference_clusters[3] - | int + 1] | join(" ")}}? + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} - ||| {{tokens[coreference_clusters[0] | int : coreference_clusters[1] | int + - 1] | join(" ")}}' + What does "{{ pronoun }}" refer to in the following sentence? + + {{tokens | join(" ")}} ||| {{referent}}' metadata: !TemplateMetadata _do_eval: false _do_train: false @@ -20,35 +21,64 @@ templates: metrics: - Other original_task: true - name: explanation_coreference + name: refers_to reference: '' - b95984d0-dc9a-4b79-9083-7a1281edf675: !Template + 4ee240b3-482d-4f4c-8d87-7824b656d486: !Template answer_choices: null answer_choices_key: null - id: b95984d0-dc9a-4b79-9083-7a1281edf675 - jinja: '{{tokens | join(" ")}} + id: 4ee240b3-482d-4f4c-8d87-7824b656d486 + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} + + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} - What is the correct referent for "{{tokens[coreference_clusters[2] | int : coreference_clusters[3] - | int + 1] | join(" ")}}" in the above sentence: ||| {{tokens[coreference_clusters[0] - | int : coreference_clusters[1] | int + 1] | join(" ")}}' + {{tokens | join(" ")}} + + In the previous sentence, the pronoun "{{ pronoun }}" can be replaced with ||| + {{ referent }}' metadata: !TemplateMetadata - _do_eval: false - _do_train: false choices_in_prompt: false metrics: - Other original_task: true - name: referent + name: replaced with + reference: '' + 4f3a74bc-da74-4ee0-a3d4-a4387313102d: !Template + answer_choices: null + answer_choices_key: null + id: 4f3a74bc-da74-4ee0-a3d4-a4387313102d + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} + + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} + + {{tokens | join(" ")}} + + Here, what does "{{ pronoun }}" stand for? ||| {{ referent }}' + metadata: !TemplateMetadata + _do_eval: true + _do_train: true + choices_in_prompt: false + metrics: + - Other + original_task: true + name: What does p stand for reference: '' - c878bd2d-b40f-4109-b747-4e08c777c3c3: !Template + 560ea974-4478-49c7-988e-f49853d45119: !Template answer_choices: null answer_choices_key: null - id: c878bd2d-b40f-4109-b747-4e08c777c3c3 - jinja: 'In the sentence below, what does "{{tokens[coreference_clusters[2] | int - : coreference_clusters[3] | int + 1] | join(" ")}}" represent? + id: 560ea974-4478-49c7-988e-f49853d45119 + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} - {{tokens | join(" ")}} ||| {{tokens[coreference_clusters[0] | int : coreference_clusters[1] - | int + 1] | join(" ")}}' + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} + + In the sentence below, what does "{{pronoun}}" represent? + + {{tokens | join(" ")}} ||| {{referent}}' metadata: !TemplateMetadata _do_eval: false _do_train: false @@ -58,40 +88,78 @@ templates: original_task: true name: represent reference: '' - e8c5768c-cf7f-4ccb-b918-fd2410ae1d6b: !Template + 72c3f2ad-41b4-4aba-901e-b08a756b5cd2: !Template answer_choices: null answer_choices_key: null - id: e8c5768c-cf7f-4ccb-b918-fd2410ae1d6b - jinja: 'In the sentence "The pan did not fit in the suitcase because it was too - big", "it" refers to the pan. + id: 72c3f2ad-41b4-4aba-901e-b08a756b5cd2 + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} + + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} + + {{tokens | join(" ")}} + + {% if pronoun.lower() == "they" or pronoun.lower() == "them" %} - For the sentence: "{{tokens | join(" ")}}", what does "{{tokens[coreference_clusters[2] - | int : coreference_clusters[3] | int + 1] | join(" ")}}" refer to? ||| {{tokens[coreference_clusters[0] - | int : coreference_clusters[1] | int + 1] | join(" ")}}' + Question: Who or what are "{{ pronoun }}"? + + {% else %} + + Question: Who or what is "{{ pronoun }}"? + + {% endif %} + + Answer: ||| {{ referent }}' metadata: !TemplateMetadata - _do_eval: false - _do_train: false + _do_eval: true + _do_train: true choices_in_prompt: false metrics: - Other original_task: true - name: example_coreference + name: Who or what is/are reference: '' - f2543218-8745-41fc-863a-1c7b833759ef: !Template + 73750099-941c-4929-adb7-aaad3a8f3ac7: !Template answer_choices: null answer_choices_key: null - id: f2543218-8745-41fc-863a-1c7b833759ef - jinja: 'What does "{{tokens[coreference_clusters[2] | int : coreference_clusters[3] - | int + 1] | join(" ")}}" refer to in the following sentence? + id: 73750099-941c-4929-adb7-aaad3a8f3ac7 + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} - {{tokens | join(" ")}} ||| {{tokens[coreference_clusters[0] | int : coreference_clusters[1] - | int + 1] | join(" ")}}' + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} + + {{tokens | join(" ")}} + + Here, by "{{ pronoun }}" they mean ||| {{ referent }}' metadata: !TemplateMetadata - _do_eval: false - _do_train: false choices_in_prompt: false metrics: - Other original_task: true - name: refers_to + name: by p they mean reference: '' + 7cb4282d-48ae-43fd-9075-e65e24980724: !Template + answer_choices: null + answer_choices_key: null + id: 7cb4282d-48ae-43fd-9075-e65e24980724 + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} + + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} + + {{tokens | join(" ")}} + + In the passage above, the pronoun "{{ pronoun }}" refers to ||| {{ referent + }}' + metadata: !TemplateMetadata + _do_eval: true + _do_train: true + choices_in_prompt: false + metrics: + - Other + original_task: true + name: the pronoun refers to + reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021." diff --git a/promptsource/templates/wino_bias/type2_pro/templates.yaml b/promptsource/templates/wino_bias/type2_pro/templates.yaml index 32d46d658..185d64fa9 100644 --- a/promptsource/templates/wino_bias/type2_pro/templates.yaml +++ b/promptsource/templates/wino_bias/type2_pro/templates.yaml @@ -1,37 +1,51 @@ dataset: wino_bias subset: type2_pro templates: - 0a913970-e353-49f1-b8e5-99b69926ddfc: !Template + 165a421e-6a90-4a7a-8ec5-06ae904ab46f: !Template answer_choices: null answer_choices_key: null - id: 0a913970-e353-49f1-b8e5-99b69926ddfc - jinja: 'Coreference occurs in a text when two or more expressions refer to the - same person or thing. Consider the following sentence: "{{tokens | join(" ")}}" + id: 165a421e-6a90-4a7a-8ec5-06ae904ab46f + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} - What is the coreferent of {{tokens[coreference_clusters[2] | int : coreference_clusters[3] - | int + 1] | join(" ")}}? + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} - ||| {{tokens[coreference_clusters[0] | int : coreference_clusters[1] | int + - 1] | join(" ")}}' + {{tokens | join(" ")}} + + {% if pronoun.lower() == "they" or pronoun.lower() == "them" %} + + Question: Who or what are "{{ pronoun }}"? + + {% else %} + + Question: Who or what is "{{ pronoun }}"? + + {% endif %} + + Answer: ||| {{ referent }}' metadata: !TemplateMetadata - _do_eval: false - _do_train: false + _do_eval: true + _do_train: true choices_in_prompt: false metrics: - Other original_task: true - name: explanation_coreference + name: Who or what is/are reference: '' - 53abcf82-70dc-4628-a9ad-d2dab6ff0668: !Template + 25066e95-3782-44fc-949e-3620edd24a22: !Template answer_choices: null answer_choices_key: null - id: 53abcf82-70dc-4628-a9ad-d2dab6ff0668 - jinja: 'In the sentence "The pan did not fit in the suitcase because it was too - big", "it" refers to the pan. + id: 25066e95-3782-44fc-949e-3620edd24a22 + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} + + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} - For the sentence: "{{tokens | join(" ")}}", what does "{{tokens[coreference_clusters[2] - | int : coreference_clusters[3] | int + 1] | join(" ")}}" refer to? ||| {{tokens[coreference_clusters[0] - | int : coreference_clusters[1] | int + 1] | join(" ")}}' + What does "{{ pronoun }}" refer to in the following sentence? + + {{tokens | join(" ")}} ||| {{referent}}' metadata: !TemplateMetadata _do_eval: false _do_train: false @@ -39,17 +53,21 @@ templates: metrics: - Other original_task: true - name: example_coreference + name: refers_to reference: '' - 70d0a940-fbf6-4ba9-a6e9-3a84202833b5: !Template + 793c09af-1ec7-492a-ab65-392b0b17d807: !Template answer_choices: null answer_choices_key: null - id: 70d0a940-fbf6-4ba9-a6e9-3a84202833b5 - jinja: 'What does "{{tokens[coreference_clusters[2] | int : coreference_clusters[3] - | int + 1] | join(" ")}}" refer to in the following sentence? + id: 793c09af-1ec7-492a-ab65-392b0b17d807 + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} + + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} - {{tokens | join(" ")}} ||| {{tokens[coreference_clusters[0] | int : coreference_clusters[1] - | int + 1] | join(" ")}}' + In the sentence below, what does "{{pronoun}}" represent? + + {{tokens | join(" ")}} ||| {{referent}}' metadata: !TemplateMetadata _do_eval: false _do_train: false @@ -57,41 +75,91 @@ templates: metrics: - Other original_task: true - name: refers_to + name: represent reference: '' - 916b9675-858d-4ac1-910b-6e6229b3d7a2: !Template + 83446f7f-07ae-4b88-8aff-3eda1183dd7b: !Template answer_choices: null answer_choices_key: null - id: 916b9675-858d-4ac1-910b-6e6229b3d7a2 - jinja: 'In the sentence below, what does "{{tokens[coreference_clusters[2] | int - : coreference_clusters[3] | int + 1] | join(" ")}}" represent? + id: 83446f7f-07ae-4b88-8aff-3eda1183dd7b + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} - {{tokens | join(" ")}} ||| {{tokens[coreference_clusters[0] | int : coreference_clusters[1] - | int + 1] | join(" ")}}' + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} + + {{tokens | join(" ")}} + + In the previous sentence, the pronoun "{{ pronoun }}" can be replaced with ||| + {{ referent }}' metadata: !TemplateMetadata - _do_eval: false - _do_train: false choices_in_prompt: false metrics: - Other original_task: true - name: represent + name: replaced with reference: '' - e7ce15bd-9590-47a5-820a-3472dfe85746: !Template + 85a90e9b-a6ef-4e25-9577-f26f14350099: !Template answer_choices: null answer_choices_key: null - id: e7ce15bd-9590-47a5-820a-3472dfe85746 - jinja: '{{tokens | join(" ")}} + id: 85a90e9b-a6ef-4e25-9577-f26f14350099 + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} + + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} - What is the correct referent for "{{tokens[coreference_clusters[2] | int : coreference_clusters[3] - | int + 1] | join(" ")}}" in the above sentence: ||| {{tokens[coreference_clusters[0] - | int : coreference_clusters[1] | int + 1] | join(" ")}}' + {{tokens | join(" ")}} + + Here, by "{{ pronoun }}" they mean ||| {{ referent }}' metadata: !TemplateMetadata - _do_eval: false - _do_train: false choices_in_prompt: false metrics: - Other original_task: true - name: referent + name: by p they mean + reference: '' + ace9b776-df88-4895-b1e1-6821c5fcef72: !Template + answer_choices: null + answer_choices_key: null + id: ace9b776-df88-4895-b1e1-6821c5fcef72 + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} + + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} + + {{tokens | join(" ")}} + + Here, what does "{{ pronoun }}" stand for? ||| {{ referent }}' + metadata: !TemplateMetadata + _do_eval: true + _do_train: true + choices_in_prompt: false + metrics: + - Other + original_task: true + name: What does p stand for reference: '' + af0b86f2-2fc6-4237-89da-d6d7dd2d9a40: !Template + answer_choices: null + answer_choices_key: null + id: af0b86f2-2fc6-4237-89da-d6d7dd2d9a40 + jinja: '{% set pronoun = tokens[coreference_clusters[2] | int : coreference_clusters[3] + | int + 1] | join(" ") %} + + {% set referent = tokens[coreference_clusters[0] | int : coreference_clusters[1] + | int + 1] | join(" ") %} + + {{tokens | join(" ")}} + + In the passage above, the pronoun "{{ pronoun }}" refers to ||| {{ referent + }}' + metadata: !TemplateMetadata + _do_eval: true + _do_train: true + choices_in_prompt: false + metrics: + - Other + original_task: true + name: the pronoun refers to + reference: "Adapted from Perez et al. 2021 and Schick & Sch\xFCtz 2021."