From 882a345c70a87be5ed9ee34f0537b58234dc82a6 Mon Sep 17 00:00:00 2001 From: steffencruz Date: Wed, 17 Apr 2024 20:43:16 +0000 Subject: [PATCH 1/9] Initial addition of sentiment --- prompting/task_registry.py | 5 +-- prompting/tasks/__init__.py | 2 ++ prompting/tasks/sentiment.py | 38 ++++++++++++++++++++++ prompting/tools/__init__.py | 2 ++ prompting/tools/datasets/__init__.py | 3 +- prompting/tools/datasets/review.py | 48 ++++++++++++++++++++++++++++ 6 files changed, 95 insertions(+), 3 deletions(-) create mode 100644 prompting/tasks/sentiment.py create mode 100644 prompting/tools/datasets/review.py diff --git a/prompting/task_registry.py b/prompting/task_registry.py index 22ea3b19f..bc649acec 100644 --- a/prompting/task_registry.py +++ b/prompting/task_registry.py @@ -1,5 +1,5 @@ -from .tasks import Task, MockTask, SummarizationTask, QuestionAnsweringTask, DebuggingTask, MathTask, DateQuestionAnsweringTask, GenericInstructionTask -from .tools import MockDataset, WikiDataset, HFCodingDataset, StackOverflowDataset, MathDataset, WikiDateDataset, GenericInstructionDataset +from .tasks import Task, MockTask, SummarizationTask, QuestionAnsweringTask, DebuggingTask, MathTask, DateQuestionAnsweringTask, GenericInstructionTask, SentimentAnalysisTask +from .tools import MockDataset, WikiDataset, HFCodingDataset, StackOverflowDataset, MathDataset, WikiDateDataset, GenericInstructionDataset, ReviewDataset # TODO: Expand this to include extra information beyond just the task and dataset names mock_task, mock_dataset = MockTask.name, [MockDataset.name] @@ -9,6 +9,7 @@ math_task, math_dataset = MathTask.name, [MathDataset.name] date_qa_task, date_qa_dataset = DateQuestionAnsweringTask.name, [WikiDateDataset.name] generic_instruction_task, generic_instruction_dataset = GenericInstructionTask.name, [GenericInstructionDataset.name] +sentiment_analysis_task, sentiment_analysis_dataset = SentimentAnalysisTask.name, [ReviewDataset.name] TASK_REGISTRY = { mock_task: mock_dataset, diff --git a/prompting/tasks/__init__.py b/prompting/tasks/__init__.py index 81e8e373c..8743d1eb7 100644 --- a/prompting/tasks/__init__.py +++ b/prompting/tasks/__init__.py @@ -6,6 +6,7 @@ from .generic_instruction import GenericInstructionTask from .math import MathTask from .mock import MockTask +from .sentiment import SentimentAnalysisTask TASKS = { @@ -16,4 +17,5 @@ DebuggingTask.name: DebuggingTask, GenericInstructionTask.name: GenericInstructionTask, MathTask.name: MathTask, + SentimentAnalysisTask.name: SentimentAnalysisTask, } diff --git a/prompting/tasks/sentiment.py b/prompting/tasks/sentiment.py new file mode 100644 index 000000000..47b8b1e73 --- /dev/null +++ b/prompting/tasks/sentiment.py @@ -0,0 +1,38 @@ +from prompting.tasks import Task + +QUERY_PROMPT_TEMPLATE = """\ +You are a review-generating expert, focusing on making highly reaslistic revies. Your response contains only the review, nothing more, nothing less. You will adhere to a word limit of 250 words. Ask a specific question about the following context: +{context} +""" + + +class SentimentAnalysisTask(Task): + important_field = ["{context"] + name = "sentiment" + desc = "get help analyzing the sentiment of a review" + goal = "to get the sentiment to the following review" + challenge_type = 'paraphrase' + challenge_template = "What is the sentiment of the following review: {context}" + + + reward_definition = [ + dict(name="ordinal", weight=1.0), + ] + penalty_definition = [] + cleaning_pipeline = [] + + static_reference = True + + def __init__(self, llm_pipeline, context, create_reference=True): + self.context = context + + self.query_prompt = QUERY_PROMPT_TEMPLATE.format(context=context.content) + self.query = self.generate_query(llm_pipeline) + self.reference = context.subtopic + + self.topic = context.title + self.subtopic = context.topic + self.tags = context.tags + + def format_challenge(self, challenge) -> str: + return challenge.format(context = self.query) \ No newline at end of file diff --git a/prompting/tools/__init__.py b/prompting/tools/__init__.py index 8ee07bc0d..6c9827f3c 100644 --- a/prompting/tools/__init__.py +++ b/prompting/tools/__init__.py @@ -8,6 +8,7 @@ WikiDateDataset, MathDataset, GenericInstructionDataset, + ReviewDataset, ) from .selector import Selector @@ -19,6 +20,7 @@ MathDataset.name: MathDataset, WikiDateDataset.name: WikiDateDataset, GenericInstructionDataset.name: GenericInstructionDataset, + ReviewDataset.name: ReviewDataset } diff --git a/prompting/tools/datasets/__init__.py b/prompting/tools/datasets/__init__.py index 948fead7d..b1187ee60 100644 --- a/prompting/tools/datasets/__init__.py +++ b/prompting/tools/datasets/__init__.py @@ -4,4 +4,5 @@ from .math import MathDataset from .mock import MockDataset from .wiki import WikiDataset, WikiDateDataset -from .generic_instruction import GenericInstructionDataset \ No newline at end of file +from .generic_instruction import GenericInstructionDataset +from .review import ReviewDataset \ No newline at end of file diff --git a/prompting/tools/datasets/review.py b/prompting/tools/datasets/review.py new file mode 100644 index 000000000..528dfdbf8 --- /dev/null +++ b/prompting/tools/datasets/review.py @@ -0,0 +1,48 @@ +import random +import functools + +import bittensor as bt +from typing import Dict, Union, List, Tuple +from .base import TemplateDataset + + +class ReviewDataset(TemplateDataset): + "Review dataset, which creates LLM prompts for writing reviews." + name = 'review' + SENTIMENTS = ["positive", "neutral", "negative"] + # TODO: filter nonsense combinations of params + + query_template = "Create a {topic} review of a {title} in the style of {mood} person in a {subtopic} tone. The review must be of {sentiment} sentiment." + params = dict( + topic=[ + "short", + "long", + "medium length", + "twitter", + "amazon", + "terribly written", + "hilarious", + ], + mood=["angry", "sad", "amused", "bored", "indifferent", "shocked", "terse"], #TODO: Remove + subtopic=[ + "casual", + "basic", + "silly", + "random", + "thoughtful", + "serious", + "rushed", + ], + title=[ + "movie", + "book", + "restaurant", + "hotel", + "product", + "service", + "car", + "company", + "live event", + ], + sentiment=SENTIMENTS, + ) \ No newline at end of file From 6799b40cf5a6f39969f133a56e38eac8c4df2907 Mon Sep 17 00:00:00 2001 From: steffencruz Date: Wed, 17 Apr 2024 21:03:12 +0000 Subject: [PATCH 2/9] Update task registry --- prompting/task_registry.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/prompting/task_registry.py b/prompting/task_registry.py index bc649acec..e6a9d9bb1 100644 --- a/prompting/task_registry.py +++ b/prompting/task_registry.py @@ -18,5 +18,6 @@ debugging_task: debugging_dataset, math_task: math_dataset, date_qa_task: date_qa_dataset, - generic_instruction_task: generic_instruction_dataset + generic_instruction_task: generic_instruction_dataset, + sentiment_analysis_task: sentiment_analysis_dataset, } \ No newline at end of file From 4628a6f0f1690bb2a3e0d180a234c48059c15d97 Mon Sep 17 00:00:00 2001 From: steffencruz Date: Wed, 17 Apr 2024 22:33:09 +0000 Subject: [PATCH 3/9] Create ordinal reward model --- prompting/rewards/ordinal.py | 50 ++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 prompting/rewards/ordinal.py diff --git a/prompting/rewards/ordinal.py b/prompting/rewards/ordinal.py new file mode 100644 index 000000000..d8cfd433b --- /dev/null +++ b/prompting/rewards/ordinal.py @@ -0,0 +1,50 @@ +import time +import torch +from typing import List +from prompting.rewards import BaseRewardModel, BatchRewardOutput + + +class OrdinalRewardModel(BaseRewardModel): + @property + def name(self) -> str: + return "category_distance" + + def __init__(self, **kwargs): + super().__init__() + #TODO: Expand to allow for more than 3 classes (Must also adjust dataset/review.py) + self.sentiments = [ + "casual", + "basic", + "silly", + "random", + "thoughtful", + "serious", + "rushed", + ] + #NOTE: These sentimens are not the same as the sentiments defined in the dataset/review.py file. These are the subtopic + + + def reward(self, reference: str, completions: List[str]) -> BatchRewardOutput: + """Compute difference scores given a completion and reference pair.""" + rewards = [] + timings = [] + classes = self.sentiments + for completion in completions: + t0 = time.time() + + # Check if exactly one answer can be found in the completion + if sum(option in completion for option in classes) == 1: + reward = abs(classes.index(reference) - classes.index(completion)) + else: + reward = 0 + timings.append(time.time() - t0) + rewards.append(reward) + + output = BatchRewardOutput( + rewards=torch.FloatTensor(rewards), + timings=torch.FloatTensor(timings), + extra_info={ + "type": "math", + }, + ) + return output \ No newline at end of file From 131f382acc488c5d971e618844cc0e518a4612f5 Mon Sep 17 00:00:00 2001 From: steffencruz Date: Thu, 18 Apr 2024 19:41:12 +0000 Subject: [PATCH 4/9] Fix ordinal scoring --- prompting/rewards/__init__.py | 1 + prompting/rewards/pipeline.py | 2 ++ prompting/tools/datasets/review.py | 24 ++++++++++++------------ 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/prompting/rewards/__init__.py b/prompting/rewards/__init__.py index 4df8159d9..51cab779a 100644 --- a/prompting/rewards/__init__.py +++ b/prompting/rewards/__init__.py @@ -10,4 +10,5 @@ from .rouge import RougeRewardModel from .float_diff import FloatDiffModel from .date import DateRewardModel +from .ordinal import OrdinalRewardModel from .pipeline import RewardPipeline, REWARD_MODELS diff --git a/prompting/rewards/pipeline.py b/prompting/rewards/pipeline.py index 1623e3e39..244ec0b3e 100644 --- a/prompting/rewards/pipeline.py +++ b/prompting/rewards/pipeline.py @@ -8,6 +8,7 @@ RelevanceRewardModel, FloatDiffModel, DateRewardModel, + OrdinalRewardModel, ) REWARD_MODELS = { @@ -16,6 +17,7 @@ "diff": DiffRewardModel, "float_diff": FloatDiffModel, "date": DateRewardModel, + "ordinal": OrdinalRewardModel, } diff --git a/prompting/tools/datasets/review.py b/prompting/tools/datasets/review.py index 528dfdbf8..5c9300d91 100644 --- a/prompting/tools/datasets/review.py +++ b/prompting/tools/datasets/review.py @@ -11,8 +11,8 @@ class ReviewDataset(TemplateDataset): name = 'review' SENTIMENTS = ["positive", "neutral", "negative"] # TODO: filter nonsense combinations of params - - query_template = "Create a {topic} review of a {title} in the style of {mood} person in a {subtopic} tone. The review must be of {sentiment} sentiment." + # "Create a {topic} review of a {title} in the style of {mood} person in a {subtopic} tone. The review must be of {sentiment} sentiment." + query_template = "Create a {topic} review of a {title}. The review must be of {sentiment} sentiment." params = dict( topic=[ "short", @@ -23,16 +23,16 @@ class ReviewDataset(TemplateDataset): "terribly written", "hilarious", ], - mood=["angry", "sad", "amused", "bored", "indifferent", "shocked", "terse"], #TODO: Remove - subtopic=[ - "casual", - "basic", - "silly", - "random", - "thoughtful", - "serious", - "rushed", - ], + #mood=["angry", "sad", "amused", "bored", "indifferent", "shocked", "terse"], #TODO: Remove + # subtopic=[ + # "casual", + # "basic", + # "silly", + # "random", + # "thoughtful", + # "serious", + # "rushed", + # ], title=[ "movie", "book", From 0a0a0f726aca7ea85ce4db5d662202da07ce01e4 Mon Sep 17 00:00:00 2001 From: bkb2135 Date: Tue, 30 Apr 2024 16:32:05 +0000 Subject: [PATCH 5/9] Add templates --- prompting/agent.py | 10 +- .../tasks/challenge_templates/__init__.py | 3 + prompting/tasks/challenge_templates/base.py | 17 ++ prompting/tasks/challenge_templates/math.py | 184 ++++++++++++++++++ .../tasks/challenge_templates/sentiment.py | 179 +++++++++++++++++ prompting/tasks/math.py | 3 + prompting/tasks/task.py | 1 + 7 files changed, 393 insertions(+), 4 deletions(-) create mode 100644 prompting/tasks/challenge_templates/__init__.py create mode 100644 prompting/tasks/challenge_templates/base.py create mode 100644 prompting/tasks/challenge_templates/math.py create mode 100644 prompting/tasks/challenge_templates/sentiment.py diff --git a/prompting/agent.py b/prompting/agent.py index 34ed4d092..e5b0fe1a5 100644 --- a/prompting/agent.py +++ b/prompting/agent.py @@ -87,10 +87,12 @@ def create_challenge(self) -> str: cleaner = None if hasattr(self.task, "cleaning_pipeline"): cleaner = CleanerPipeline(cleaning_pipeline=self.task.cleaning_pipeline) - - self.challenge = super().query( - message="Ask a question related to your goal", cleaner=cleaner - ) + if self.task.challenge_type == "inference": + self.challenge = super().query( + message="Ask a question related to your goal", cleaner=cleaner + ) + elif self.task.challenge_type == 'paraphrase': + self.challenge = self.task.challenge_template.next(self.task.query) self.challenge = self.task.format_challenge(self.challenge) self.challenge_time = time.time() - t0 diff --git a/prompting/tasks/challenge_templates/__init__.py b/prompting/tasks/challenge_templates/__init__.py new file mode 100644 index 000000000..d92a27498 --- /dev/null +++ b/prompting/tasks/challenge_templates/__init__.py @@ -0,0 +1,3 @@ +from .base import ChallengeTemplate +from .math import MathChallengeTemplate +from .sentiment import SentimentChallengeTemplate diff --git a/prompting/tasks/challenge_templates/base.py b/prompting/tasks/challenge_templates/base.py new file mode 100644 index 000000000..6c8fc7ef1 --- /dev/null +++ b/prompting/tasks/challenge_templates/base.py @@ -0,0 +1,17 @@ +import random +from abc import ABC +from typing import List + + +class ChallengeTemplate(ABC): + templates: List[str] = ["This is a template with {query}! "] + fields: dict = {"query": ["This is a placeholder for the query"]} + + def next(self, query: str): + self.fields["query"] = [query] + return self.get_template().format( + **{field: random.choice(entries) for field, entries in self.fields.items()} + ) + + def get_template(self): + return random.choice(self.templates) diff --git a/prompting/tasks/challenge_templates/math.py b/prompting/tasks/challenge_templates/math.py new file mode 100644 index 000000000..ae4769be8 --- /dev/null +++ b/prompting/tasks/challenge_templates/math.py @@ -0,0 +1,184 @@ +import random +from .base import ChallengeTemplate + + +class MathChallengeTemplate(ChallengeTemplate): + def __init__(self): + super().__init__() + self.templates = [ + "{greeting}{greeting_punctuation}{query}{request}", + "{query}{greeting}{greeting_punctuation}{request}", + "{greeting}{greeting_punctuation}{query}{request}", + "{query}{request}{greeting}{greeting_punctuation}", + ] + self.fields = { + "greeting": [ + "Hello", + "Hi", + "Hey", + "Yo", + "What's up", + "Howdy", + "Hola", + "Bonjour", + "G'day", + "Good morning", + "Good afternoon", + "Good evening", + "Greetings", + "Sup", + "Hi there", + "Hey there", + "Morning", + "Afternoon", + "Evening", + "Salutations", + "Hey, what's going on", + "Howdy", + "Sup", + "Sup yo", + "Yo", + "Yo yo", + "Greetings", + "Greetings to you", + "Hello", + "Hello friend", + "Hello to you", + "Hey", + "Hey there", + "", + ], + "greeting_punctuation": [ + "!", + "! ", + "! ", + "!\n", + ",", + ", ", + ", ", + ",\n", + ".", + ". ", + ". ", + ".\n", + "", + " ", + " ", + "\n", + "...", + "... ", + "... ", + "...\n", + "", + ], + "request": [ + "Can you assist me, please?", + "Could you lend me a hand?", + "Would you mind helping me out?", + "I could use some assistance.", + "Do you have a moment to help me?", + "I'm in need of some help.", + "Could you give me a hand with this?", + "Would you be willing to help me?", + "Can you offer me some guidance?", + "I'm struggling a bit, could you help?", + "I could really use your expertise.", + "Would you mind showing me how to do this?", + "Can you lend me your expertise for a moment?", + "I'm having trouble, could you assist?", + "Would you be able to lend me a hand?", + "Can you offer me some assistance?", + "I'm stuck, could you help me out?", + "Could you assist me with this problem?", + "Would you be so kind as to help me?", + "Can you offer me some help, please?", + "Solve", + "Could you spare a moment to help me?", + "Would you mind giving me some assistance?", + "Can you help me understand this better?", + "I need your help with something.", + "Could you offer me some support, please?", + "Would you be willing to give me a hand?", + "Can you show me how to do this?", + "I'm having difficulty, could you help me?", + "Could you assist me with this issue?", + "Would you mind helping me with this task?", + "Can you provide some help, please?", + "I'm in a bit of a bind, could you help?", + "Could you lend me a hand with this problem?", + "Would you be able to offer me some guidance?", + "Can you help me out with this, please?", + "I'm having trouble understanding, could you help?", + "Could you offer me some assistance, please?", + "Would you mind assisting me with this?", + "Can you give me some advice?", + "I could use your help with this.", + "Could you spare some time to help me?", + "Would you be willing to lend me a hand?", + "Can you help me solve this problem?", + "I'm struggling to figure this out, could you help?", + "Could you provide me with some assistance?", + "Would you mind showing me what to do?", + "Can you assist me in resolving this issue?", + "I could really use your help.", + "Could you help me out with this task?", + "Would you be so kind as to give me a hand?", + "Can you help me with this problem, please?", + "I'm stuck on this, could you assist?", + "Could you lend me a hand with this, please?", + "Would you be able to provide me with some guidance?", + "Can you offer me some assistance with this?", + "I'm having difficulty understanding, could you help me?", + "Could you assist me with this problem, please?", + "Would you mind giving me a hand with this?", + "Can you show me how to do this, please?", + "I'm struggling with this, could you help me out?", + "Could you offer me some help with this?", + "Would you be willing to help me with this, please?", + "Can you provide me with some support, please?", + "I'm in a bit of a bind, could you assist me?", + "Could you lend me your expertise?", + "Would you be able to spare a moment to help me?", + "Can you help me out with this problem, please?", + "I'm having trouble with this, could you help me out?", + "Could you assist me with this task, please?", + "Would you mind offering me some assistance?", + "Can you assist me with this issue, please?", + "I could use some assistance with this, could you help?", + "Could you give me a hand with this issue, please?", + "Would you be so kind as to lend me a hand?", + "Can you provide me with some assistance on this?", + "I'm having difficulty with this task, could you help?", + "Could you offer me some help on this, please?", + "Would you mind helping me with this problem?", + "Can you lend me a hand with this, please?", + "I'm stuck on this problem, could you help?", + "Could you show me how to do this, please?", + "Would you be willing to assist me with this?", + "Can you help me with this task, please?", + "I'm struggling with this problem, could you assist?", + "Could you give me some guidance on this, please?", + "Would you mind giving me some help with this?", + "Can you help me with this issue, please?", + "I could use your help with this problem.", + "Could you spare some time to help me out?", + "Would you be able to lend me your expertise?", + "Can you offer me some assistance with this problem?", + "I'm in need of some help with this, could you assist?", + "Could you assist me with this problem, please?", + "Can you help me out with this issue, please?", + "I'm having trouble with this task, could you help?", + "Could you lend me a hand with this problem, please?", + "Would you be willing to give me some assistance?", + "Can you provide me with some help, please?", + "I'm stuck on this issue, could you help me?", + "Could you show me what to do, please?", + "Would you mind helping me with this task, please?", + "Can you lend me your expertise for a moment, please?", + "I'm struggling with this issue, could you assist me?", + "Could you give me a hand with this problem, please?", + "Would you be so kind as to offer me some assistance?", + "Can you help me understand this, please?", + "I could use your help figuring this out.", + ], + } diff --git a/prompting/tasks/challenge_templates/sentiment.py b/prompting/tasks/challenge_templates/sentiment.py new file mode 100644 index 000000000..47da111a7 --- /dev/null +++ b/prompting/tasks/challenge_templates/sentiment.py @@ -0,0 +1,179 @@ +import random +from .base import ChallengeTemplate + + +class SentimentChallengeTemplate(ChallengeTemplate): + def __init__(self): + super().__init__() + self.templates = [ + "{greeting}{greeting_punctuation}{request}{query}", + "{request}{query}{greeting}{greeting_punctuation}", + ] + self.fields = { + "greeting": [ + "Hello", + "Hi", + "Hey", + "Yo", + "What's up", + "Howdy", + "Hola", + "Bonjour", + "G'day", + "Good morning", + "Good afternoon", + "Good evening", + "Greetings", + "Sup", + "Hi there", + "Hey there", + "Morning", + "Afternoon", + "Evening", + "Salutations", + "Hey, what's going on", + "Howdy", + "Sup", + "Sup yo", + "Yo", + "Yo yo", + "Greetings", + "Greetings to you", + "Hello", + "Hello friend", + "Hello to you", + "Hey", + "Hey there", + "", + ], + "greeting_punctuation": [ + "!", + "! ", + "! ", + "!\n", + ",", + ", ", + ", ", + ",\n", + ".", + ". ", + ". ", + ".\n", + "", + " ", + " ", + "\n", + "...", + "... ", + "... ", + "...\n", + "", + ], + "request": [ + "Please analyze the sentiment of this review.", + "I'd like to know the sentiment of this text.", + "Can you determine the sentiment of this review?", + "Analyze the sentiment of this review, please.", + "Evaluate the sentiment of this text.", + "What's the sentiment of this review?", + "I'm curious about the sentiment of this text.", + "Please provide the sentiment analysis for this review.", + "Could you analyze the sentiment of this text?", + "Assess the sentiment of this review.", + "I'd like the sentiment of this text analyzed.", + "Can you determine the sentiment of this text?", + "Analyze this review's sentiment, please.", + "What's the sentiment of this text?", + "I'm interested in the sentiment analysis of this review.", + "Please evaluate the sentiment of this text.", + "Could you provide the sentiment analysis for this review?", + "Determine the sentiment of this text.", + "I'd appreciate it if you could analyze the sentiment of this review.", + "Can you analyze the sentiment of this text?", + "Please assess the sentiment of this review.", + "What's the sentiment of this review, please?", + "I'm curious about the sentiment of this review.", + "Could you analyze the sentiment of this review, please?", + "Provide the sentiment analysis for this text.", + "Evaluate this review's sentiment.", + "What's the sentiment of this review?", + "Can you analyze the sentiment of this review?", + "Please assess the sentiment of this text.", + "Could you provide the sentiment analysis of this review?", + "Analyze this text's sentiment, please.", + "I'd like to know the sentiment of this review.", + "Determine the sentiment of this review, please.", + "Analyze the sentiment of this text, please.", + "Can you evaluate the sentiment of this review?", + "What's the sentiment of this text, please?", + "I'm interested in the sentiment analysis of this text.", + "Please analyze the sentiment of this review text.", + "Assess this text's sentiment, please.", + "Provide the sentiment analysis for this text.", + "Evaluate this text's sentiment, please.", + "What's the sentiment of this text?", + "Can you determine the sentiment of this text, please?", + "I'd like to know the sentiment of this review text.", + "Determine the sentiment of this text, please.", + "Analyze this review text's sentiment.", + "Please assess the sentiment of this text, please.", + "I'm curious about the sentiment of this text.", + "Could you analyze the sentiment of this review text, please?", + "Provide the sentiment analysis of this text.", + "Evaluate the sentiment of this review text, please.", + "What's the sentiment of this review text, please?", + "Can you evaluate the sentiment of this review text?", + "I'm interested in the sentiment analysis of this review text.", + "Please analyze the sentiment of this review text.", + "Assess this review text's sentiment, please.", + "Determine the sentiment of this review text.", + "Analyze the sentiment of this review text, please.", + "What's the sentiment of this review text?", + "Can you determine the sentiment of this review text, please?", + "I'd like to know the sentiment of this text, please.", + "Determine the sentiment of this text, please.", + "Analyze this text's sentiment.", + "Please assess the sentiment of this text, please.", + "I'm curious about the sentiment of this text.", + "Could you analyze the sentiment of this text, please?", + "Provide the sentiment analysis of this text.", + "Evaluate the sentiment of this text, please.", + "What's the sentiment of this text, please?", + "Can you evaluate the sentiment of this text?", + "I'm interested in the sentiment analysis of this text.", + "Please analyze the sentiment of this text.", + "Assess this text's sentiment, please.", + "Determine the sentiment of this text.", + "Analyze the sentiment of this text, please.", + "What's the sentiment of this text?", + "Can you determine the sentiment of this text, please?", + "I'd like to know the sentiment of this text, please.", + "Determine the sentiment of this text, please.", + "Analyze this text's sentiment.", + "Please assess the sentiment of this text, please.", + "I'm curious about the sentiment of this text.", + "Could you analyze the sentiment of this text, please?", + "Provide the sentiment analysis of this text.", + "Evaluate the sentiment of this text, please.", + "What's the sentiment of this text, please?", + "Can you evaluate the sentiment of this text?", + "I'm interested in the sentiment analysis of this text.", + "Please analyze the sentiment of this text.", + "Assess this text's sentiment, please.", + "Determine the sentiment of this text.", + "Analyze the sentiment of this text, please.", + "What's the sentiment of this text?", + "Can you determine the sentiment of this text, please?", + "I'd like to know the sentiment of this text, please.", + "Determine the sentiment of this text, please.", + "Analyze this text's sentiment.", + "Please assess the sentiment of this text, please.", + "I'm curious about the sentiment of this text.", + "Could you analyze the sentiment of this text, please?", + "Provide the sentiment analysis of this text.", + "Evaluate the sentiment of this text, please.", + "What's the sentiment of this text, please?", + "Can you evaluate the sentiment of this text?", + "I'm interested in the sentiment analysis of this text.", + ], + } diff --git a/prompting/tasks/math.py b/prompting/tasks/math.py index 97d43921e..ba769f77a 100644 --- a/prompting/tasks/math.py +++ b/prompting/tasks/math.py @@ -2,6 +2,7 @@ import bittensor as bt from dataclasses import dataclass from prompting.tasks import Task +from .challenge_templates import MathChallengeTemplate @dataclass @@ -17,6 +18,8 @@ class MathTask(Task): static_reference = True static_query = True + challenge_type = 'paraphrase' + challenge_template = MathChallengeTemplate() def __init__(self, llm_pipeline, context, create_reference=True): self.context = context diff --git a/prompting/tasks/task.py b/prompting/tasks/task.py index cfa03a3d8..d837aac46 100644 --- a/prompting/tasks/task.py +++ b/prompting/tasks/task.py @@ -49,6 +49,7 @@ class Task(ABC): query_system_prompt = "" query_prompt = "" cleaner = None + challenge_type = 'inference' def __str__(self): return f"{self.__class__.__name__}(name={self.name!r}, desc={self.desc!r}, goal={self.goal!r}, query={self.query!r}, reference={self.reference!r}, topic={self.topic!r}, subtopic={self.subtopic!r}, tags={self.tags!r})" From a64fd0a51ee7e4a6ad8bf252af46778891cdc0d3 Mon Sep 17 00:00:00 2001 From: bkb2135 Date: Tue, 30 Apr 2024 16:38:49 +0000 Subject: [PATCH 6/9] Change the math query to just the problem --- prompting/tasks/math.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/prompting/tasks/math.py b/prompting/tasks/math.py index ba769f77a..c1abf208e 100644 --- a/prompting/tasks/math.py +++ b/prompting/tasks/math.py @@ -24,11 +24,7 @@ class MathTask(Task): def __init__(self, llm_pipeline, context, create_reference=True): self.context = context - self.query = ( - "How can I solve the following problem, " - + context.content - + "? Make sure to include the whole problem when you ask your question." - ) + self.query = context.content self.reference = context.extra["solution"] self.topic = context.title self.subtopic = context.topic From 011d1406539a8125578224ee6e3cd8a65199a913 Mon Sep 17 00:00:00 2001 From: p-ferreira Date: Tue, 30 Apr 2024 17:47:57 +0000 Subject: [PATCH 7/9] Set subtopic to be the sentiment --- prompting/tasks/sentiment.py | 1 - prompting/tools/datasets/review.py | 7 ++++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/prompting/tasks/sentiment.py b/prompting/tasks/sentiment.py index 47b8b1e73..7b4b58945 100644 --- a/prompting/tasks/sentiment.py +++ b/prompting/tasks/sentiment.py @@ -25,7 +25,6 @@ class SentimentAnalysisTask(Task): def __init__(self, llm_pipeline, context, create_reference=True): self.context = context - self.query_prompt = QUERY_PROMPT_TEMPLATE.format(context=context.content) self.query = self.generate_query(llm_pipeline) self.reference = context.subtopic diff --git a/prompting/tools/datasets/review.py b/prompting/tools/datasets/review.py index 5c9300d91..41e512971 100644 --- a/prompting/tools/datasets/review.py +++ b/prompting/tools/datasets/review.py @@ -12,7 +12,7 @@ class ReviewDataset(TemplateDataset): SENTIMENTS = ["positive", "neutral", "negative"] # TODO: filter nonsense combinations of params # "Create a {topic} review of a {title} in the style of {mood} person in a {subtopic} tone. The review must be of {sentiment} sentiment." - query_template = "Create a {topic} review of a {title}. The review must be of {sentiment} sentiment." + query_template = "Create a {topic} review of a {title}. The review must be of {subtopic} sentiment." params = dict( topic=[ "short", @@ -44,5 +44,6 @@ class ReviewDataset(TemplateDataset): "company", "live event", ], - sentiment=SENTIMENTS, - ) \ No newline at end of file + subtopic=SENTIMENTS, + ) + \ No newline at end of file From 2a5c4b59fbc89dd785f07ecc4af5cb50e07b6483 Mon Sep 17 00:00:00 2001 From: p-ferreira Date: Tue, 30 Apr 2024 20:29:33 +0000 Subject: [PATCH 8/9] Update Generic --- prompting/agent.py | 4 ++++ prompting/tasks/generic_instruction.py | 2 +- prompting/tasks/sentiment.py | 5 +++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/prompting/agent.py b/prompting/agent.py index e5b0fe1a5..b4b67255b 100644 --- a/prompting/agent.py +++ b/prompting/agent.py @@ -93,6 +93,10 @@ def create_challenge(self) -> str: ) elif self.task.challenge_type == 'paraphrase': self.challenge = self.task.challenge_template.next(self.task.query) + elif self.task.challenge_type == 'query': + self.challenge = self.task.query + else: + bt.logging.error(f"Task {self.task.name} has challenge type of: {self.task.challenge_type} which is not supported.") self.challenge = self.task.format_challenge(self.challenge) self.challenge_time = time.time() - t0 diff --git a/prompting/tasks/generic_instruction.py b/prompting/tasks/generic_instruction.py index d5b706a68..66c6c8f3c 100644 --- a/prompting/tasks/generic_instruction.py +++ b/prompting/tasks/generic_instruction.py @@ -13,10 +13,10 @@ class GenericInstructionTask(Task): - challenge_type = 'query' name = "generic" desc = "get help on answering a general instruction" goal = "to get the answer to the following instruction" + challenge_type = 'query' reward_definition = [ dict(name="rouge", ngram="rouge-1", metric="f", weight=0.75), diff --git a/prompting/tasks/sentiment.py b/prompting/tasks/sentiment.py index 47b8b1e73..6f96853ed 100644 --- a/prompting/tasks/sentiment.py +++ b/prompting/tasks/sentiment.py @@ -1,4 +1,5 @@ from prompting.tasks import Task +from .challenge_templates import SentimentChallengeTemplate QUERY_PROMPT_TEMPLATE = """\ You are a review-generating expert, focusing on making highly reaslistic revies. Your response contains only the review, nothing more, nothing less. You will adhere to a word limit of 250 words. Ask a specific question about the following context: @@ -12,8 +13,8 @@ class SentimentAnalysisTask(Task): desc = "get help analyzing the sentiment of a review" goal = "to get the sentiment to the following review" challenge_type = 'paraphrase' - challenge_template = "What is the sentiment of the following review: {context}" - + challenge_type = 'paraphrase' + challenge_template = SentimentChallengeTemplate() reward_definition = [ dict(name="ordinal", weight=1.0), From 119a356685d0d41c720f2964a264be8a074ef55e Mon Sep 17 00:00:00 2001 From: p-ferreira Date: Tue, 30 Apr 2024 20:39:20 +0000 Subject: [PATCH 9/9] Remove Typos --- prompting/tasks/sentiment.py | 1 - prompting/tools/datasets/review.py | 13 +------------ 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/prompting/tasks/sentiment.py b/prompting/tasks/sentiment.py index 9b78ed32f..60635d59a 100644 --- a/prompting/tasks/sentiment.py +++ b/prompting/tasks/sentiment.py @@ -8,7 +8,6 @@ class SentimentAnalysisTask(Task): - important_field = ["{context"] name = "sentiment" desc = "get help analyzing the sentiment of a review" goal = "to get the sentiment to the following review" diff --git a/prompting/tools/datasets/review.py b/prompting/tools/datasets/review.py index 41e512971..cfadbcf8e 100644 --- a/prompting/tools/datasets/review.py +++ b/prompting/tools/datasets/review.py @@ -10,8 +10,7 @@ class ReviewDataset(TemplateDataset): "Review dataset, which creates LLM prompts for writing reviews." name = 'review' SENTIMENTS = ["positive", "neutral", "negative"] - # TODO: filter nonsense combinations of params - # "Create a {topic} review of a {title} in the style of {mood} person in a {subtopic} tone. The review must be of {sentiment} sentiment." + # TODO: Expand the params to create a larger dataset, while maintaining the same quality. query_template = "Create a {topic} review of a {title}. The review must be of {subtopic} sentiment." params = dict( topic=[ @@ -23,16 +22,6 @@ class ReviewDataset(TemplateDataset): "terribly written", "hilarious", ], - #mood=["angry", "sad", "amused", "bored", "indifferent", "shocked", "terse"], #TODO: Remove - # subtopic=[ - # "casual", - # "basic", - # "silly", - # "random", - # "thoughtful", - # "serious", - # "rushed", - # ], title=[ "movie", "book",