Paraphrase Challenges

prompting/agent.py

@@ -87,10 +87,16 @@ def create_challenge(self) -> str:
         cleaner = None
         if hasattr(self.task, "cleaning_pipeline"):
             cleaner = CleanerPipeline(cleaning_pipeline=self.task.cleaning_pipeline)
-
-        self.challenge = super().query(
-            message="Ask a question related to your goal", cleaner=cleaner
-        )
+        if self.task.challenge_type == "inference":
+            self.challenge = super().query(
+                message="Ask a question related to your goal", cleaner=cleaner
+            )
+        elif self.task.challenge_type == 'paraphrase':
+            self.challenge = self.task.challenge_template.next(self.task.query)
+        elif self.task.challenge_type == 'query':
+            self.challenge = self.task.query
+        else:
+            bt.logging.error(f"Task {self.task.name} has challenge type of: {self.task.challenge_type} which is not supported.")
         self.challenge = self.task.format_challenge(self.challenge)
         self.challenge_time = time.time() - t0
 

prompting/rewards/__init__.py

@@ -10,4 +10,5 @@
 from .rouge import RougeRewardModel
 from .float_diff import FloatDiffModel
 from .date import DateRewardModel
+from .ordinal import OrdinalRewardModel
 from .pipeline import RewardPipeline, REWARD_MODELS

prompting/rewards/ordinal.py

@@ -0,0 +1,50 @@
+import time
+import torch
+from typing import List
+from prompting.rewards import BaseRewardModel, BatchRewardOutput
+
+
+class OrdinalRewardModel(BaseRewardModel):
+    @property
+    def name(self) -> str:
+        return "category_distance"
+
+    def __init__(self, **kwargs):
+        super().__init__()
+        #TODO: Expand to allow for more than 3 classes (Must also adjust dataset/review.py)
+        self.sentiments = [
+            "casual",
+            "basic",
+            "silly",
+            "random",
+            "thoughtful",
+            "serious",
+            "rushed",
+        ]
+        #NOTE: These sentimens are not the same as the sentiments defined in the dataset/review.py file. These are the subtopic
+
+
+    def reward(self, reference: str, completions: List[str]) -> BatchRewardOutput:
+        """Compute difference scores given a completion and reference pair."""
+        rewards = []
+        timings = []
+        classes = self.sentiments
+        for completion in completions:
+            t0 = time.time()
+
+            # Check if exactly one answer can be found in the completion
+            if sum(option in completion for option in classes) == 1:
+                reward = abs(classes.index(reference) - classes.index(completion))
+            else:
+                reward = 0 
+            timings.append(time.time() - t0)
+            rewards.append(reward)
+
+        output = BatchRewardOutput(
+            rewards=torch.FloatTensor(rewards),
+            timings=torch.FloatTensor(timings),
+            extra_info={
+                "type": "math",
+            },
+        )
+        return output

prompting/rewards/pipeline.py

@@ -8,6 +8,7 @@
     RelevanceRewardModel,
     FloatDiffModel,
     DateRewardModel,
+    OrdinalRewardModel,
 )
 
 REWARD_MODELS = {
@@ -16,6 +17,7 @@
     "diff": DiffRewardModel,
     "float_diff": FloatDiffModel,
     "date": DateRewardModel,
+    "ordinal": OrdinalRewardModel,
 }
 
 

prompting/task_registry.py

@@ -1,5 +1,5 @@
-from .tasks import Task, MockTask, SummarizationTask, QuestionAnsweringTask, DebuggingTask, MathTask, DateQuestionAnsweringTask, GenericInstructionTask
-from .tools import MockDataset, WikiDataset, HFCodingDataset, StackOverflowDataset, MathDataset, WikiDateDataset, GenericInstructionDataset
+from .tasks import Task, MockTask, SummarizationTask, QuestionAnsweringTask, DebuggingTask, MathTask, DateQuestionAnsweringTask, GenericInstructionTask, SentimentAnalysisTask
+from .tools import MockDataset, WikiDataset, HFCodingDataset, StackOverflowDataset, MathDataset, WikiDateDataset, GenericInstructionDataset, ReviewDataset
 
 # TODO: Expand this to include extra information beyond just the task and dataset names
 summarization_task, summarization_dataset = SummarizationTask.name, [WikiDataset.name]
@@ -8,12 +8,14 @@
 math_task, math_dataset = MathTask.name, [MathDataset.name]
 date_qa_task, date_qa_dataset = DateQuestionAnsweringTask.name, [WikiDateDataset.name]
 generic_instruction_task, generic_instruction_dataset = GenericInstructionTask.name, [GenericInstructionDataset.name]
+sentiment_analysis_task, sentiment_analysis_dataset = SentimentAnalysisTask.name, [ReviewDataset.name]
 
 TASK_REGISTRY = {
     summarization_task: summarization_dataset,
     qa_task: qa_dataset,
     #debugging_task: debugging_dataset,
     math_task: math_dataset,
     date_qa_task: date_qa_dataset,
-    generic_instruction_task: generic_instruction_dataset
+    generic_instruction_task: generic_instruction_dataset,
+    sentiment_analysis_task: sentiment_analysis_dataset,
 }

prompting/tasks/__init__.py

@@ -6,6 +6,7 @@
 from .generic_instruction import GenericInstructionTask
 from .math import MathTask
 from .mock import MockTask
+from .sentiment import SentimentAnalysisTask
 
 
 TASKS = {
@@ -15,4 +16,5 @@
     #DebuggingTask.name: DebuggingTask,
     GenericInstructionTask.name: GenericInstructionTask,
     MathTask.name: MathTask,
+    SentimentAnalysisTask.name: SentimentAnalysisTask,
 }

prompting/tasks/challenge_templates/__init__.py

@@ -0,0 +1,3 @@
+from .base import ChallengeTemplate
+from .math import MathChallengeTemplate
+from .sentiment import SentimentChallengeTemplate

prompting/tasks/challenge_templates/base.py

@@ -0,0 +1,17 @@
+import random
+from abc import ABC
+from typing import List
+
+
+class ChallengeTemplate(ABC):
+    templates: List[str] = ["This is a template with {query}! <end>"]
+    fields: dict = {"query": ["This is a placeholder for the query"]}
+
+    def next(self, query: str):
+        self.fields["query"] = [query]
+        return self.get_template().format(
+            **{field: random.choice(entries) for field, entries in self.fields.items()}
+        )
+
+    def get_template(self):
+        return random.choice(self.templates)

prompting/tasks/challenge_templates/math.py

@@ -0,0 +1,184 @@
+import random
+from .base import ChallengeTemplate
+
+
+class MathChallengeTemplate(ChallengeTemplate):
+    def __init__(self):
+        super().__init__()
+        self.templates = [
+            "{greeting}{greeting_punctuation}{query}{request}<end>",
+            "{query}<end>{greeting}{greeting_punctuation}{request}",
+            "{greeting}{greeting_punctuation}{query}<end>{request}",
+            "{query}{request}<end>{greeting}{greeting_punctuation}",
+        ]
+        self.fields = {
+            "greeting": [
+                "Hello",
+                "Hi",
+                "Hey",
+                "Yo",
+                "What's up",
+                "Howdy",
+                "Hola",
+                "Bonjour",
+                "G'day",
+                "Good morning",
+                "Good afternoon",
+                "Good evening",
+                "Greetings",
+                "Sup",
+                "Hi there",
+                "Hey there",
+                "Morning",
+                "Afternoon",
+                "Evening",
+                "Salutations",
+                "Hey, what's going on",
+                "Howdy",
+                "Sup",
+                "Sup yo",
+                "Yo",
+                "Yo yo",
+                "Greetings",
+                "Greetings to you",
+                "Hello",
+                "Hello friend",
+                "Hello to you",
+                "Hey",
+                "Hey there",
+                "",
+            ],
+            "greeting_punctuation": [
+                "!",
+                "! ",
+                "!  ",
+                "!\n",
+                ",",
+                ", ",
+                ",  ",
+                ",\n",
+                ".",
+                ". ",
+                ".  ",
+                ".\n",
+                "",
+                " ",
+                "  ",
+                "\n",
+                "...",
+                "... ",
+                "...  ",
+                "...\n",
+                "",
+            ],
+            "request": [
+                "Can you assist me, please?",
+                "Could you lend me a hand?",
+                "Would you mind helping me out?",
+                "I could use some assistance.",
+                "Do you have a moment to help me?",
+                "I'm in need of some help.",
+                "Could you give me a hand with this?",
+                "Would you be willing to help me?",
+                "Can you offer me some guidance?",
+                "I'm struggling a bit, could you help?",
+                "I could really use your expertise.",
+                "Would you mind showing me how to do this?",
+                "Can you lend me your expertise for a moment?",
+                "I'm having trouble, could you assist?",
+                "Would you be able to lend me a hand?",
+                "Can you offer me some assistance?",
+                "I'm stuck, could you help me out?",
+                "Could you assist me with this problem?",
+                "Would you be so kind as to help me?",
+                "Can you offer me some help, please?",
+                "Solve",
+                "Could you spare a moment to help me?",
+                "Would you mind giving me some assistance?",
+                "Can you help me understand this better?",
+                "I need your help with something.",
+                "Could you offer me some support, please?",
+                "Would you be willing to give me a hand?",
+                "Can you show me how to do this?",
+                "I'm having difficulty, could you help me?",
+                "Could you assist me with this issue?",
+                "Would you mind helping me with this task?",
+                "Can you provide some help, please?",
+                "I'm in a bit of a bind, could you help?",
+                "Could you lend me a hand with this problem?",
+                "Would you be able to offer me some guidance?",
+                "Can you help me out with this, please?",
+                "I'm having trouble understanding, could you help?",
+                "Could you offer me some assistance, please?",
+                "Would you mind assisting me with this?",
+                "Can you give me some advice?",
+                "I could use your help with this.",
+                "Could you spare some time to help me?",
+                "Would you be willing to lend me a hand?",
+                "Can you help me solve this problem?",
+                "I'm struggling to figure this out, could you help?",
+                "Could you provide me with some assistance?",
+                "Would you mind showing me what to do?",
+                "Can you assist me in resolving this issue?",
+                "I could really use your help.",
+                "Could you help me out with this task?",
+                "Would you be so kind as to give me a hand?",
+                "Can you help me with this problem, please?",
+                "I'm stuck on this, could you assist?",
+                "Could you lend me a hand with this, please?",
+                "Would you be able to provide me with some guidance?",
+                "Can you offer me some assistance with this?",
+                "I'm having difficulty understanding, could you help me?",
+                "Could you assist me with this problem, please?",
+                "Would you mind giving me a hand with this?",
+                "Can you show me how to do this, please?",
+                "I'm struggling with this, could you help me out?",
+                "Could you offer me some help with this?",
+                "Would you be willing to help me with this, please?",
+                "Can you provide me with some support, please?",
+                "I'm in a bit of a bind, could you assist me?",
+                "Could you lend me your expertise?",
+                "Would you be able to spare a moment to help me?",
+                "Can you help me out with this problem, please?",
+                "I'm having trouble with this, could you help me out?",
+                "Could you assist me with this task, please?",
+                "Would you mind offering me some assistance?",
+                "Can you assist me with this issue, please?",
+                "I could use some assistance with this, could you help?",
+                "Could you give me a hand with this issue, please?",
+                "Would you be so kind as to lend me a hand?",
+                "Can you provide me with some assistance on this?",
+                "I'm having difficulty with this task, could you help?",
+                "Could you offer me some help on this, please?",
+                "Would you mind helping me with this problem?",
+                "Can you lend me a hand with this, please?",
+                "I'm stuck on this problem, could you help?",
+                "Could you show me how to do this, please?",
+                "Would you be willing to assist me with this?",
+                "Can you help me with this task, please?",
+                "I'm struggling with this problem, could you assist?",
+                "Could you give me some guidance on this, please?",
+                "Would you mind giving me some help with this?",
+                "Can you help me with this issue, please?",
+                "I could use your help with this problem.",
+                "Could you spare some time to help me out?",
+                "Would you be able to lend me your expertise?",
+                "Can you offer me some assistance with this problem?",
+                "I'm in need of some help with this, could you assist?",
+                "Could you assist me with this problem, please?",
+                "Can you help me out with this issue, please?",
+                "I'm having trouble with this task, could you help?",
+                "Could you lend me a hand with this problem, please?",
+                "Would you be willing to give me some assistance?",
+                "Can you provide me with some help, please?",
+                "I'm stuck on this issue, could you help me?",
+                "Could you show me what to do, please?",
+                "Would you mind helping me with this task, please?",
+                "Can you lend me your expertise for a moment, please?",
+                "I'm struggling with this issue, could you assist me?",
+                "Could you give me a hand with this problem, please?",
+                "Would you be so kind as to offer me some assistance?",
+                "Can you help me understand this, please?",
+                "I could use your help figuring this out.",
+            ],
+        }