diff --git a/prompting/agent.py b/prompting/agent.py index 34ed4d092..b4b67255b 100644 --- a/prompting/agent.py +++ b/prompting/agent.py @@ -87,10 +87,16 @@ def create_challenge(self) -> str: cleaner = None if hasattr(self.task, "cleaning_pipeline"): cleaner = CleanerPipeline(cleaning_pipeline=self.task.cleaning_pipeline) - - self.challenge = super().query( - message="Ask a question related to your goal", cleaner=cleaner - ) + if self.task.challenge_type == "inference": + self.challenge = super().query( + message="Ask a question related to your goal", cleaner=cleaner + ) + elif self.task.challenge_type == 'paraphrase': + self.challenge = self.task.challenge_template.next(self.task.query) + elif self.task.challenge_type == 'query': + self.challenge = self.task.query + else: + bt.logging.error(f"Task {self.task.name} has challenge type of: {self.task.challenge_type} which is not supported.") self.challenge = self.task.format_challenge(self.challenge) self.challenge_time = time.time() - t0 diff --git a/prompting/rewards/__init__.py b/prompting/rewards/__init__.py index 4df8159d9..51cab779a 100644 --- a/prompting/rewards/__init__.py +++ b/prompting/rewards/__init__.py @@ -10,4 +10,5 @@ from .rouge import RougeRewardModel from .float_diff import FloatDiffModel from .date import DateRewardModel +from .ordinal import OrdinalRewardModel from .pipeline import RewardPipeline, REWARD_MODELS diff --git a/prompting/rewards/ordinal.py b/prompting/rewards/ordinal.py new file mode 100644 index 000000000..d8cfd433b --- /dev/null +++ b/prompting/rewards/ordinal.py @@ -0,0 +1,50 @@ +import time +import torch +from typing import List +from prompting.rewards import BaseRewardModel, BatchRewardOutput + + +class OrdinalRewardModel(BaseRewardModel): + @property + def name(self) -> str: + return "category_distance" + + def __init__(self, **kwargs): + super().__init__() + #TODO: Expand to allow for more than 3 classes (Must also adjust dataset/review.py) + self.sentiments = [ + "casual", + "basic", + "silly", + "random", + "thoughtful", + "serious", + "rushed", + ] + #NOTE: These sentimens are not the same as the sentiments defined in the dataset/review.py file. These are the subtopic + + + def reward(self, reference: str, completions: List[str]) -> BatchRewardOutput: + """Compute difference scores given a completion and reference pair.""" + rewards = [] + timings = [] + classes = self.sentiments + for completion in completions: + t0 = time.time() + + # Check if exactly one answer can be found in the completion + if sum(option in completion for option in classes) == 1: + reward = abs(classes.index(reference) - classes.index(completion)) + else: + reward = 0 + timings.append(time.time() - t0) + rewards.append(reward) + + output = BatchRewardOutput( + rewards=torch.FloatTensor(rewards), + timings=torch.FloatTensor(timings), + extra_info={ + "type": "math", + }, + ) + return output \ No newline at end of file diff --git a/prompting/rewards/pipeline.py b/prompting/rewards/pipeline.py index 1623e3e39..244ec0b3e 100644 --- a/prompting/rewards/pipeline.py +++ b/prompting/rewards/pipeline.py @@ -8,6 +8,7 @@ RelevanceRewardModel, FloatDiffModel, DateRewardModel, + OrdinalRewardModel, ) REWARD_MODELS = { @@ -16,6 +17,7 @@ "diff": DiffRewardModel, "float_diff": FloatDiffModel, "date": DateRewardModel, + "ordinal": OrdinalRewardModel, } diff --git a/prompting/task_registry.py b/prompting/task_registry.py index 2e006694f..de12c105e 100644 --- a/prompting/task_registry.py +++ b/prompting/task_registry.py @@ -1,5 +1,5 @@ -from .tasks import Task, MockTask, SummarizationTask, QuestionAnsweringTask, DebuggingTask, MathTask, DateQuestionAnsweringTask, GenericInstructionTask -from .tools import MockDataset, WikiDataset, HFCodingDataset, StackOverflowDataset, MathDataset, WikiDateDataset, GenericInstructionDataset +from .tasks import Task, MockTask, SummarizationTask, QuestionAnsweringTask, DebuggingTask, MathTask, DateQuestionAnsweringTask, GenericInstructionTask, SentimentAnalysisTask +from .tools import MockDataset, WikiDataset, HFCodingDataset, StackOverflowDataset, MathDataset, WikiDateDataset, GenericInstructionDataset, ReviewDataset # TODO: Expand this to include extra information beyond just the task and dataset names summarization_task, summarization_dataset = SummarizationTask.name, [WikiDataset.name] @@ -8,6 +8,7 @@ math_task, math_dataset = MathTask.name, [MathDataset.name] date_qa_task, date_qa_dataset = DateQuestionAnsweringTask.name, [WikiDateDataset.name] generic_instruction_task, generic_instruction_dataset = GenericInstructionTask.name, [GenericInstructionDataset.name] +sentiment_analysis_task, sentiment_analysis_dataset = SentimentAnalysisTask.name, [ReviewDataset.name] TASK_REGISTRY = { summarization_task: summarization_dataset, @@ -15,5 +16,6 @@ #debugging_task: debugging_dataset, math_task: math_dataset, date_qa_task: date_qa_dataset, - generic_instruction_task: generic_instruction_dataset + generic_instruction_task: generic_instruction_dataset, + sentiment_analysis_task: sentiment_analysis_dataset, } \ No newline at end of file diff --git a/prompting/tasks/__init__.py b/prompting/tasks/__init__.py index 01292ded9..04d0282b3 100644 --- a/prompting/tasks/__init__.py +++ b/prompting/tasks/__init__.py @@ -6,6 +6,7 @@ from .generic_instruction import GenericInstructionTask from .math import MathTask from .mock import MockTask +from .sentiment import SentimentAnalysisTask TASKS = { @@ -15,4 +16,5 @@ #DebuggingTask.name: DebuggingTask, GenericInstructionTask.name: GenericInstructionTask, MathTask.name: MathTask, + SentimentAnalysisTask.name: SentimentAnalysisTask, } diff --git a/prompting/tasks/challenge_templates/__init__.py b/prompting/tasks/challenge_templates/__init__.py new file mode 100644 index 000000000..d92a27498 --- /dev/null +++ b/prompting/tasks/challenge_templates/__init__.py @@ -0,0 +1,3 @@ +from .base import ChallengeTemplate +from .math import MathChallengeTemplate +from .sentiment import SentimentChallengeTemplate diff --git a/prompting/tasks/challenge_templates/base.py b/prompting/tasks/challenge_templates/base.py new file mode 100644 index 000000000..6c8fc7ef1 --- /dev/null +++ b/prompting/tasks/challenge_templates/base.py @@ -0,0 +1,17 @@ +import random +from abc import ABC +from typing import List + + +class ChallengeTemplate(ABC): + templates: List[str] = ["This is a template with {query}! "] + fields: dict = {"query": ["This is a placeholder for the query"]} + + def next(self, query: str): + self.fields["query"] = [query] + return self.get_template().format( + **{field: random.choice(entries) for field, entries in self.fields.items()} + ) + + def get_template(self): + return random.choice(self.templates) diff --git a/prompting/tasks/challenge_templates/math.py b/prompting/tasks/challenge_templates/math.py new file mode 100644 index 000000000..ae4769be8 --- /dev/null +++ b/prompting/tasks/challenge_templates/math.py @@ -0,0 +1,184 @@ +import random +from .base import ChallengeTemplate + + +class MathChallengeTemplate(ChallengeTemplate): + def __init__(self): + super().__init__() + self.templates = [ + "{greeting}{greeting_punctuation}{query}{request}", + "{query}{greeting}{greeting_punctuation}{request}", + "{greeting}{greeting_punctuation}{query}{request}", + "{query}{request}{greeting}{greeting_punctuation}", + ] + self.fields = { + "greeting": [ + "Hello", + "Hi", + "Hey", + "Yo", + "What's up", + "Howdy", + "Hola", + "Bonjour", + "G'day", + "Good morning", + "Good afternoon", + "Good evening", + "Greetings", + "Sup", + "Hi there", + "Hey there", + "Morning", + "Afternoon", + "Evening", + "Salutations", + "Hey, what's going on", + "Howdy", + "Sup", + "Sup yo", + "Yo", + "Yo yo", + "Greetings", + "Greetings to you", + "Hello", + "Hello friend", + "Hello to you", + "Hey", + "Hey there", + "", + ], + "greeting_punctuation": [ + "!", + "! ", + "! ", + "!\n", + ",", + ", ", + ", ", + ",\n", + ".", + ". ", + ". ", + ".\n", + "", + " ", + " ", + "\n", + "...", + "... ", + "... ", + "...\n", + "", + ], + "request": [ + "Can you assist me, please?", + "Could you lend me a hand?", + "Would you mind helping me out?", + "I could use some assistance.", + "Do you have a moment to help me?", + "I'm in need of some help.", + "Could you give me a hand with this?", + "Would you be willing to help me?", + "Can you offer me some guidance?", + "I'm struggling a bit, could you help?", + "I could really use your expertise.", + "Would you mind showing me how to do this?", + "Can you lend me your expertise for a moment?", + "I'm having trouble, could you assist?", + "Would you be able to lend me a hand?", + "Can you offer me some assistance?", + "I'm stuck, could you help me out?", + "Could you assist me with this problem?", + "Would you be so kind as to help me?", + "Can you offer me some help, please?", + "Solve", + "Could you spare a moment to help me?", + "Would you mind giving me some assistance?", + "Can you help me understand this better?", + "I need your help with something.", + "Could you offer me some support, please?", + "Would you be willing to give me a hand?", + "Can you show me how to do this?", + "I'm having difficulty, could you help me?", + "Could you assist me with this issue?", + "Would you mind helping me with this task?", + "Can you provide some help, please?", + "I'm in a bit of a bind, could you help?", + "Could you lend me a hand with this problem?", + "Would you be able to offer me some guidance?", + "Can you help me out with this, please?", + "I'm having trouble understanding, could you help?", + "Could you offer me some assistance, please?", + "Would you mind assisting me with this?", + "Can you give me some advice?", + "I could use your help with this.", + "Could you spare some time to help me?", + "Would you be willing to lend me a hand?", + "Can you help me solve this problem?", + "I'm struggling to figure this out, could you help?", + "Could you provide me with some assistance?", + "Would you mind showing me what to do?", + "Can you assist me in resolving this issue?", + "I could really use your help.", + "Could you help me out with this task?", + "Would you be so kind as to give me a hand?", + "Can you help me with this problem, please?", + "I'm stuck on this, could you assist?", + "Could you lend me a hand with this, please?", + "Would you be able to provide me with some guidance?", + "Can you offer me some assistance with this?", + "I'm having difficulty understanding, could you help me?", + "Could you assist me with this problem, please?", + "Would you mind giving me a hand with this?", + "Can you show me how to do this, please?", + "I'm struggling with this, could you help me out?", + "Could you offer me some help with this?", + "Would you be willing to help me with this, please?", + "Can you provide me with some support, please?", + "I'm in a bit of a bind, could you assist me?", + "Could you lend me your expertise?", + "Would you be able to spare a moment to help me?", + "Can you help me out with this problem, please?", + "I'm having trouble with this, could you help me out?", + "Could you assist me with this task, please?", + "Would you mind offering me some assistance?", + "Can you assist me with this issue, please?", + "I could use some assistance with this, could you help?", + "Could you give me a hand with this issue, please?", + "Would you be so kind as to lend me a hand?", + "Can you provide me with some assistance on this?", + "I'm having difficulty with this task, could you help?", + "Could you offer me some help on this, please?", + "Would you mind helping me with this problem?", + "Can you lend me a hand with this, please?", + "I'm stuck on this problem, could you help?", + "Could you show me how to do this, please?", + "Would you be willing to assist me with this?", + "Can you help me with this task, please?", + "I'm struggling with this problem, could you assist?", + "Could you give me some guidance on this, please?", + "Would you mind giving me some help with this?", + "Can you help me with this issue, please?", + "I could use your help with this problem.", + "Could you spare some time to help me out?", + "Would you be able to lend me your expertise?", + "Can you offer me some assistance with this problem?", + "I'm in need of some help with this, could you assist?", + "Could you assist me with this problem, please?", + "Can you help me out with this issue, please?", + "I'm having trouble with this task, could you help?", + "Could you lend me a hand with this problem, please?", + "Would you be willing to give me some assistance?", + "Can you provide me with some help, please?", + "I'm stuck on this issue, could you help me?", + "Could you show me what to do, please?", + "Would you mind helping me with this task, please?", + "Can you lend me your expertise for a moment, please?", + "I'm struggling with this issue, could you assist me?", + "Could you give me a hand with this problem, please?", + "Would you be so kind as to offer me some assistance?", + "Can you help me understand this, please?", + "I could use your help figuring this out.", + ], + } diff --git a/prompting/tasks/challenge_templates/sentiment.py b/prompting/tasks/challenge_templates/sentiment.py new file mode 100644 index 000000000..47da111a7 --- /dev/null +++ b/prompting/tasks/challenge_templates/sentiment.py @@ -0,0 +1,179 @@ +import random +from .base import ChallengeTemplate + + +class SentimentChallengeTemplate(ChallengeTemplate): + def __init__(self): + super().__init__() + self.templates = [ + "{greeting}{greeting_punctuation}{request}{query}", + "{request}{query}{greeting}{greeting_punctuation}", + ] + self.fields = { + "greeting": [ + "Hello", + "Hi", + "Hey", + "Yo", + "What's up", + "Howdy", + "Hola", + "Bonjour", + "G'day", + "Good morning", + "Good afternoon", + "Good evening", + "Greetings", + "Sup", + "Hi there", + "Hey there", + "Morning", + "Afternoon", + "Evening", + "Salutations", + "Hey, what's going on", + "Howdy", + "Sup", + "Sup yo", + "Yo", + "Yo yo", + "Greetings", + "Greetings to you", + "Hello", + "Hello friend", + "Hello to you", + "Hey", + "Hey there", + "", + ], + "greeting_punctuation": [ + "!", + "! ", + "! ", + "!\n", + ",", + ", ", + ", ", + ",\n", + ".", + ". ", + ". ", + ".\n", + "", + " ", + " ", + "\n", + "...", + "... ", + "... ", + "...\n", + "", + ], + "request": [ + "Please analyze the sentiment of this review.", + "I'd like to know the sentiment of this text.", + "Can you determine the sentiment of this review?", + "Analyze the sentiment of this review, please.", + "Evaluate the sentiment of this text.", + "What's the sentiment of this review?", + "I'm curious about the sentiment of this text.", + "Please provide the sentiment analysis for this review.", + "Could you analyze the sentiment of this text?", + "Assess the sentiment of this review.", + "I'd like the sentiment of this text analyzed.", + "Can you determine the sentiment of this text?", + "Analyze this review's sentiment, please.", + "What's the sentiment of this text?", + "I'm interested in the sentiment analysis of this review.", + "Please evaluate the sentiment of this text.", + "Could you provide the sentiment analysis for this review?", + "Determine the sentiment of this text.", + "I'd appreciate it if you could analyze the sentiment of this review.", + "Can you analyze the sentiment of this text?", + "Please assess the sentiment of this review.", + "What's the sentiment of this review, please?", + "I'm curious about the sentiment of this review.", + "Could you analyze the sentiment of this review, please?", + "Provide the sentiment analysis for this text.", + "Evaluate this review's sentiment.", + "What's the sentiment of this review?", + "Can you analyze the sentiment of this review?", + "Please assess the sentiment of this text.", + "Could you provide the sentiment analysis of this review?", + "Analyze this text's sentiment, please.", + "I'd like to know the sentiment of this review.", + "Determine the sentiment of this review, please.", + "Analyze the sentiment of this text, please.", + "Can you evaluate the sentiment of this review?", + "What's the sentiment of this text, please?", + "I'm interested in the sentiment analysis of this text.", + "Please analyze the sentiment of this review text.", + "Assess this text's sentiment, please.", + "Provide the sentiment analysis for this text.", + "Evaluate this text's sentiment, please.", + "What's the sentiment of this text?", + "Can you determine the sentiment of this text, please?", + "I'd like to know the sentiment of this review text.", + "Determine the sentiment of this text, please.", + "Analyze this review text's sentiment.", + "Please assess the sentiment of this text, please.", + "I'm curious about the sentiment of this text.", + "Could you analyze the sentiment of this review text, please?", + "Provide the sentiment analysis of this text.", + "Evaluate the sentiment of this review text, please.", + "What's the sentiment of this review text, please?", + "Can you evaluate the sentiment of this review text?", + "I'm interested in the sentiment analysis of this review text.", + "Please analyze the sentiment of this review text.", + "Assess this review text's sentiment, please.", + "Determine the sentiment of this review text.", + "Analyze the sentiment of this review text, please.", + "What's the sentiment of this review text?", + "Can you determine the sentiment of this review text, please?", + "I'd like to know the sentiment of this text, please.", + "Determine the sentiment of this text, please.", + "Analyze this text's sentiment.", + "Please assess the sentiment of this text, please.", + "I'm curious about the sentiment of this text.", + "Could you analyze the sentiment of this text, please?", + "Provide the sentiment analysis of this text.", + "Evaluate the sentiment of this text, please.", + "What's the sentiment of this text, please?", + "Can you evaluate the sentiment of this text?", + "I'm interested in the sentiment analysis of this text.", + "Please analyze the sentiment of this text.", + "Assess this text's sentiment, please.", + "Determine the sentiment of this text.", + "Analyze the sentiment of this text, please.", + "What's the sentiment of this text?", + "Can you determine the sentiment of this text, please?", + "I'd like to know the sentiment of this text, please.", + "Determine the sentiment of this text, please.", + "Analyze this text's sentiment.", + "Please assess the sentiment of this text, please.", + "I'm curious about the sentiment of this text.", + "Could you analyze the sentiment of this text, please?", + "Provide the sentiment analysis of this text.", + "Evaluate the sentiment of this text, please.", + "What's the sentiment of this text, please?", + "Can you evaluate the sentiment of this text?", + "I'm interested in the sentiment analysis of this text.", + "Please analyze the sentiment of this text.", + "Assess this text's sentiment, please.", + "Determine the sentiment of this text.", + "Analyze the sentiment of this text, please.", + "What's the sentiment of this text?", + "Can you determine the sentiment of this text, please?", + "I'd like to know the sentiment of this text, please.", + "Determine the sentiment of this text, please.", + "Analyze this text's sentiment.", + "Please assess the sentiment of this text, please.", + "I'm curious about the sentiment of this text.", + "Could you analyze the sentiment of this text, please?", + "Provide the sentiment analysis of this text.", + "Evaluate the sentiment of this text, please.", + "What's the sentiment of this text, please?", + "Can you evaluate the sentiment of this text?", + "I'm interested in the sentiment analysis of this text.", + ], + } diff --git a/prompting/tasks/generic_instruction.py b/prompting/tasks/generic_instruction.py index d5b706a68..66c6c8f3c 100644 --- a/prompting/tasks/generic_instruction.py +++ b/prompting/tasks/generic_instruction.py @@ -13,10 +13,10 @@ class GenericInstructionTask(Task): - challenge_type = 'query' name = "generic" desc = "get help on answering a general instruction" goal = "to get the answer to the following instruction" + challenge_type = 'query' reward_definition = [ dict(name="rouge", ngram="rouge-1", metric="f", weight=0.75), diff --git a/prompting/tasks/math.py b/prompting/tasks/math.py index 97d43921e..c1abf208e 100644 --- a/prompting/tasks/math.py +++ b/prompting/tasks/math.py @@ -2,6 +2,7 @@ import bittensor as bt from dataclasses import dataclass from prompting.tasks import Task +from .challenge_templates import MathChallengeTemplate @dataclass @@ -17,15 +18,13 @@ class MathTask(Task): static_reference = True static_query = True + challenge_type = 'paraphrase' + challenge_template = MathChallengeTemplate() def __init__(self, llm_pipeline, context, create_reference=True): self.context = context - self.query = ( - "How can I solve the following problem, " - + context.content - + "? Make sure to include the whole problem when you ask your question." - ) + self.query = context.content self.reference = context.extra["solution"] self.topic = context.title self.subtopic = context.topic diff --git a/prompting/tasks/sentiment.py b/prompting/tasks/sentiment.py new file mode 100644 index 000000000..60635d59a --- /dev/null +++ b/prompting/tasks/sentiment.py @@ -0,0 +1,37 @@ +from prompting.tasks import Task +from .challenge_templates import SentimentChallengeTemplate + +QUERY_PROMPT_TEMPLATE = """\ +You are a review-generating expert, focusing on making highly reaslistic revies. Your response contains only the review, nothing more, nothing less. You will adhere to a word limit of 250 words. Ask a specific question about the following context: +{context} +""" + + +class SentimentAnalysisTask(Task): + name = "sentiment" + desc = "get help analyzing the sentiment of a review" + goal = "to get the sentiment to the following review" + challenge_type = 'paraphrase' + challenge_type = 'paraphrase' + challenge_template = SentimentChallengeTemplate() + + reward_definition = [ + dict(name="ordinal", weight=1.0), + ] + penalty_definition = [] + cleaning_pipeline = [] + + static_reference = True + + def __init__(self, llm_pipeline, context, create_reference=True): + self.context = context + self.query_prompt = QUERY_PROMPT_TEMPLATE.format(context=context.content) + self.query = self.generate_query(llm_pipeline) + self.reference = context.subtopic + + self.topic = context.title + self.subtopic = context.topic + self.tags = context.tags + + def format_challenge(self, challenge) -> str: + return challenge.format(context = self.query) \ No newline at end of file diff --git a/prompting/tasks/task.py b/prompting/tasks/task.py index cfa03a3d8..d837aac46 100644 --- a/prompting/tasks/task.py +++ b/prompting/tasks/task.py @@ -49,6 +49,7 @@ class Task(ABC): query_system_prompt = "" query_prompt = "" cleaner = None + challenge_type = 'inference' def __str__(self): return f"{self.__class__.__name__}(name={self.name!r}, desc={self.desc!r}, goal={self.goal!r}, query={self.query!r}, reference={self.reference!r}, topic={self.topic!r}, subtopic={self.subtopic!r}, tags={self.tags!r})" diff --git a/prompting/tools/__init__.py b/prompting/tools/__init__.py index c296ce17c..1722a9d84 100644 --- a/prompting/tools/__init__.py +++ b/prompting/tools/__init__.py @@ -8,6 +8,7 @@ WikiDateDataset, MathDataset, GenericInstructionDataset, + ReviewDataset, ) from .selector import Selector @@ -18,6 +19,7 @@ MathDataset.name: MathDataset, WikiDateDataset.name: WikiDateDataset, GenericInstructionDataset.name: GenericInstructionDataset, + ReviewDataset.name: ReviewDataset } diff --git a/prompting/tools/datasets/__init__.py b/prompting/tools/datasets/__init__.py index 948fead7d..b1187ee60 100644 --- a/prompting/tools/datasets/__init__.py +++ b/prompting/tools/datasets/__init__.py @@ -4,4 +4,5 @@ from .math import MathDataset from .mock import MockDataset from .wiki import WikiDataset, WikiDateDataset -from .generic_instruction import GenericInstructionDataset \ No newline at end of file +from .generic_instruction import GenericInstructionDataset +from .review import ReviewDataset \ No newline at end of file diff --git a/prompting/tools/datasets/review.py b/prompting/tools/datasets/review.py new file mode 100644 index 000000000..cfadbcf8e --- /dev/null +++ b/prompting/tools/datasets/review.py @@ -0,0 +1,38 @@ +import random +import functools + +import bittensor as bt +from typing import Dict, Union, List, Tuple +from .base import TemplateDataset + + +class ReviewDataset(TemplateDataset): + "Review dataset, which creates LLM prompts for writing reviews." + name = 'review' + SENTIMENTS = ["positive", "neutral", "negative"] + # TODO: Expand the params to create a larger dataset, while maintaining the same quality. + query_template = "Create a {topic} review of a {title}. The review must be of {subtopic} sentiment." + params = dict( + topic=[ + "short", + "long", + "medium length", + "twitter", + "amazon", + "terribly written", + "hilarious", + ], + title=[ + "movie", + "book", + "restaurant", + "hotel", + "product", + "service", + "car", + "company", + "live event", + ], + subtopic=SENTIMENTS, + ) + \ No newline at end of file