From 246a4823bdb9362bce069eda98a220216e0727aa Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Fri, 14 Jun 2024 11:51:09 +0000 Subject: [PATCH 01/57] Add draft organic dataset, task, validator axon --- prompting/base/validator.py | 58 +++++++++----- prompting/forward.py | 1 + prompting/task_registry.py | 28 ++++++- prompting/tasks/__init__.py | 2 + prompting/tasks/organic.py | 108 +++++++++++++++++++++++++++ prompting/tools/__init__.py | 5 +- prompting/tools/datasets/__init__.py | 3 +- prompting/tools/datasets/organic.py | 50 +++++++++++++ prompting/utils/config.py | 2 +- 9 files changed, 230 insertions(+), 27 deletions(-) create mode 100644 prompting/tasks/organic.py create mode 100644 prompting/tools/datasets/organic.py diff --git a/prompting/base/validator.py b/prompting/base/validator.py index 93240ad5b..5c3fe87a9 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -28,6 +28,7 @@ from prompting.base.neuron import BaseNeuron from prompting.mock import MockDendrite +from prompting.protocol import StreamPromptingSynapse from prompting.utils.config import add_validator_args from prompting.utils.exceptions import MaxRetryError @@ -64,11 +65,6 @@ def __init__(self, config=None): # Init sync with the network. Updates the metagraph. self.sync() - # Serve axon to enable external connections. - if not self.config.neuron.axon_off: - self.serve_axon() - else: - bt.logging.warning("axon off, not serving ip to chain.") # Create asyncio event loop to manage async tasks. self.loop = asyncio.get_event_loop() @@ -78,24 +74,45 @@ def __init__(self, config=None): self.is_running: bool = False self.thread: threading.Thread = None self.lock = asyncio.Lock() + # Serve axon to enable external connections. + # if not self.config.neuron.axon_off: + # self.serve_axon() + # else: + # bt.logging.warning("axon off, not serving ip to chain.") + + # def blacklist_prompt(self, synapse: StreamPromptingSynapse) -> Tuple[bool, str]: + # blacklist = self.base_blacklist(synapse, PROMPT_BLACKLIST_STAKE) + # bt.logging.info(blacklist[1]) + # return blacklist + + async def _handle_organic(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: + # TODO: Wrap into OrganicTask. + with self.lock: + bt.logging.info(f"Organic handle: {synapse}") + return synapse def serve_axon(self): """Serve axon to enable external connections.""" - - bt.logging.info("serving ip to chain...") - try: - self.axon = bt.axon(wallet=self.wallet, config=self.config) - - try: - self.subtensor.serve_axon( - netuid=self.config.netuid, - axon=self.axon, - ) - except Exception as e: - bt.logging.error(f"Failed to serve Axon with exception: {e}") - - except Exception as e: - bt.logging.error(f"Failed to create Axon initialize with exception: {e}") + bt.logging.info("Serving IP to chain...") + # try: + self.axon = bt.axon(wallet=self.wallet, config=self.config) + self.axon.attach( + forward_fn=self._handle_organic, + blacklist_fn=None, + priority_fn=None, + ) + # self.subtensor.serve_axon( + # netuid=self.config.netuid, + # axon=self.axon, + # ) + self.axon.serve(netuid=self.config.netuid, subtensor=self.subtensor) + self.axon.start() + validator_uid = self.metagraph.hotkeys.index( + self.wallet.hotkey.ss58_address + ) + bt.logging.info(f"Running validator on uid: {validator_uid}") + # except Exception as e: + # bt.logging.error(f"Failed to create Axon initialize with exception: {e}") def run(self): """ @@ -120,6 +137,7 @@ def run(self): # Check that validator is registered on the network. self.sync() + self.serve_axon() if not self.config.neuron.axon_off: bt.logging.info( f"Running validator {self.axon} on network: {self.config.subtensor.chain_endpoint} with netuid: {self.config.netuid}" diff --git a/prompting/forward.py b/prompting/forward.py index c6f0ad268..06fb0fce7 100644 --- a/prompting/forward.py +++ b/prompting/forward.py @@ -58,6 +58,7 @@ async def process_stream(uid: int, async_iterator: Awaitable, tokenizer: Tokeniz start_time = time.time() try: + chunk = None async for chunk in async_iterator: # most important loop, as this is where we acquire the final synapse. if isinstance(chunk, str): accumulated_chunks.append(chunk) diff --git a/prompting/task_registry.py b/prompting/task_registry.py index f111a29d9..41e7c4e2d 100644 --- a/prompting/task_registry.py +++ b/prompting/task_registry.py @@ -1,7 +1,30 @@ -from .tasks import Task, MockTask, SummarizationTask, QuestionAnsweringTask, DebuggingTask, MathTask, DateQuestionAnsweringTask, GenericInstructionTask, SentimentAnalysisTask, TranslationTask -from .tools import MockDataset, WikiDataset, HFCodingDataset, StackOverflowDataset, MathDataset, WikiDateDataset, GenericInstructionDataset, ReviewDataset +from .tasks import ( + Task, + MockTask, + OrganicTask, + SummarizationTask, + QuestionAnsweringTask, + DebuggingTask, + MathTask, + DateQuestionAnsweringTask, + GenericInstructionTask, + SentimentAnalysisTask, + TranslationTask +) +from .tools import ( + MockDataset, + OrganicDataset, + WikiDataset, + HFCodingDataset, + StackOverflowDataset, + MathDataset, + WikiDateDataset, + GenericInstructionDataset, + ReviewDataset +) # TODO: Expand this to include extra information beyond just the task and dataset names +organic_task, organic_dataset = OrganicTask.name, [OrganicDataset.name] summarization_task, summarization_dataset = SummarizationTask.name, [WikiDataset.name] qa_task, qa_dataset = QuestionAnsweringTask.name, [WikiDataset.name] #debugging_task, debugging_dataset = DebuggingTask.name, [HFCodingDataset.name] @@ -12,6 +35,7 @@ sentiment_analysis_task, sentiment_analysis_dataset = SentimentAnalysisTask.name, [ReviewDataset.name] TASK_REGISTRY = { + organic_task: organic_dataset, summarization_task: summarization_dataset, qa_task: qa_dataset, #debugging_task: debugging_dataset, diff --git a/prompting/tasks/__init__.py b/prompting/tasks/__init__.py index 6794c97ca..a9c18603c 100644 --- a/prompting/tasks/__init__.py +++ b/prompting/tasks/__init__.py @@ -8,8 +8,10 @@ from .translate import TranslationTask, TranslationPipeline from .mock import MockTask from .sentiment import SentimentAnalysisTask +from .organic import OrganicTask TASKS = { + OrganicTask.name: OrganicTask, QuestionAnsweringTask.name: QuestionAnsweringTask, DateQuestionAnsweringTask.name: DateQuestionAnsweringTask, SummarizationTask.name: SummarizationTask, diff --git a/prompting/tasks/organic.py b/prompting/tasks/organic.py new file mode 100644 index 000000000..1c08a0f60 --- /dev/null +++ b/prompting/tasks/organic.py @@ -0,0 +1,108 @@ +import bittensor as bt +from dataclasses import dataclass +from prompting.tasks import Task + +# TODO: introduce criteria for the query and reference answer (length, layout, etc.) and make these arguments + +# Used to instruct the LLM to provide a good query when given a context +QUERY_SYSTEM_PROMPT = """\ +You are a question-generating expert, focusing on delivering comprehensive and accurate questions with depth and clarity. The questions you generate should be based on the context that is provided. +You will maintain a neutral tone in your questions. +You will adhere to a word limit of 50 words for each question. +""" + +# Used to obtain the query (which is a question about the context) +QUERY_PROMPT_TEMPLATE = """\ +Ask a specific question about the following context: + +#Context: +{context} +""" + +# Used to obtain the query (which is a followup question about the context) +# TODO: we may not need the entire conversation history - we can sample a subset of it (first k messages, last k messages, etc.) +FOLLOWUP_PROMPT_TEMPLATE = """ +Compose a single, specific question to continue the dialogue below. Adopt the persona of the original user, including their communication style and objectives. The question should be based on the previous exchanges and must not be answerable with a simple yes or no. + +The question should require detailed knowledge of the conversation history for a correct response, emphasizing requests for clarification or additional details (e.g., 'What specific steps did you take?' or 'How did that situation resolve?'). Avoid referring to the subject by name and instead use indirect pronouns or descriptions (e.g., 'he,' 'she,' 'it'). Avoid answering the question yourself and refrain from providing new information not already discussed. + +# Context: +{context} + +# Conversation History: +{history} +""" + +# Used to obtain reference answer +REFERENCE_PROMPT_TEMPLATE = """\ +Answer the question you will receive in detail, utilizing the following context. + +#Context: +{context} + +# Question: +{question} +""" + +# TODO: We also need a special followup reference prompt (or just merge both) +# Used to obtain reference answer +FOLLOWUP_REFERENCE_PROMPT_TEMPLATE = """\ +Answer the question you will receive in detail, utilizing the following context and conversation history as required. + +#Context: +{context} + +# Conversation History: +{history} + +# Question: +{question} +""" + +@dataclass +class OrganicTask(Task): + name = "organic" + desc = "get help on answering a question" + goal = "to get the answer to the following question" + + reward_definition = [ + dict(name="rouge", ngram="rouge-1", metric="f", weight=0.5), + dict(name="relevance", weight=0.5), + ] + penalty_definition = [ + dict(name="rouge", ngram="rouge-1", metric="f", weight=0.5), + ] + + cleaning_pipeline = [ + dict(name="remove_quotes"), + dict(name="prune_ending"), + dict(name="remove_roles"), + dict(name="remove_post_question_text"), + ] + + def __init__(self, llm_pipeline, context, create_reference=True, history=None): + self.context = context + + self.query_system_prompt = QUERY_SYSTEM_PROMPT + if history: + self.query_prompt = FOLLOWUP_PROMPT_TEMPLATE.format(context=context.content, history=history) + bt.logging.warning(f'Using history!!\n{history=}\n\n{context=}\n\n{self.query_prompt=}') + else: + self.query_prompt = QUERY_PROMPT_TEMPLATE.format(context=context.content) + + self.query = self.generate_query(llm_pipeline) + + if history: + self.reference_prompt = FOLLOWUP_REFERENCE_PROMPT_TEMPLATE.format( + context=context.content, question=self.query, history=history + ) + else: + self.reference_prompt = REFERENCE_PROMPT_TEMPLATE.format( + context=context.content, question=self.query + ) + if create_reference: + self.reference = self.generate_reference(llm_pipeline) + + self.topic = context.title + self.subtopic = context.topic + self.tags = context.tags diff --git a/prompting/tools/__init__.py b/prompting/tools/__init__.py index 82e3713d9..151f5ef8a 100644 --- a/prompting/tools/__init__.py +++ b/prompting/tools/__init__.py @@ -1,6 +1,7 @@ from .datasets import ( Dataset, MockDataset, + OrganicDataset, HFCodingDataset, WikiDataset, StackOverflowDataset, @@ -13,6 +14,7 @@ DATASETS = { #HFCodingDataset.name: HFCodingDataset, + OrganicDataset.name: OrganicDataset, WikiDataset.name: WikiDataset, #StackOverflowDataset.name: StackOverflowDataset, MathDataset.name: MathDataset, @@ -20,6 +22,3 @@ GenericInstructionDataset.name: GenericInstructionDataset, ReviewDataset.name: ReviewDataset } - - - \ No newline at end of file diff --git a/prompting/tools/datasets/__init__.py b/prompting/tools/datasets/__init__.py index 3bdda191b..21942d91a 100644 --- a/prompting/tools/datasets/__init__.py +++ b/prompting/tools/datasets/__init__.py @@ -4,4 +4,5 @@ from .mock import MockDataset from .wiki import WikiDataset, WikiDateDataset from .generic_instruction import GenericInstructionDataset -from .review import ReviewDataset \ No newline at end of file +from .review import ReviewDataset +from .organic import OrganicDataset diff --git a/prompting/tools/datasets/organic.py b/prompting/tools/datasets/organic.py new file mode 100644 index 000000000..35cc63cd2 --- /dev/null +++ b/prompting/tools/datasets/organic.py @@ -0,0 +1,50 @@ +# The MIT License (MIT) +# Copyright © 2024 Yuma Rao +# Copyright © 2023 Opentensor Foundation + +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the “Software”), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. + +# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. + +from typing import Any +from .base import Dataset +from ..selector import Selector + + +class OrganicDataset(Dataset): + name = "organic" + + def __init__( + self, + ): + """Collects organic prompts and/or generated synthetic prompts + + Args: + """ + pass + + def get( + self, + name: str, + **kwargs, + ) -> dict[str, Any]: + """""" + return {} + + def search(self, name) -> dict[str, Any]: + """""" + return {} + + def random(self) -> dict[str, Any]: + """""" + return {} diff --git a/prompting/utils/config.py b/prompting/utils/config.py index 4ffa0ddd4..787b4ca73 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -300,7 +300,7 @@ def add_validator_args(cls, parser): type=float, nargs="+", help="The probability of sampling each task.", - default=[1.0 / len(TASKS)] * len(TASKS), + default=[0.5] + [0.5 / (len(TASKS) - 1)] * (len(TASKS) - 1), ) parser.add_argument( From 8786155582cdb0fbe4d2edcfa4e918aa313d40fe Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Wed, 19 Jun 2024 07:01:25 +0000 Subject: [PATCH 02/57] [WIP] Add architecture, rewards, task, dataset etc --- connecting_vali.ipynb | 380 ++++++++++++++++++++ prompting/agent.py | 2 +- prompting/base/validator.py | 22 +- prompting/forward.py | 27 +- prompting/llms/vllm_llm.py | 20 +- prompting/shared/context.py | 2 + prompting/tasks/__init__.py | 2 +- prompting/tasks/organic.py | 108 ------ prompting/tasks/organic_task.py | 82 +++++ prompting/tasks/qa.py | 2 +- prompting/tools/datasets/__init__.py | 2 +- prompting/tools/datasets/organic.py | 50 --- prompting/tools/datasets/organic_dataset.py | 71 ++++ prompting/utils/config.py | 2 +- 14 files changed, 579 insertions(+), 193 deletions(-) create mode 100644 connecting_vali.ipynb delete mode 100644 prompting/tasks/organic.py create mode 100644 prompting/tasks/organic_task.py delete mode 100644 prompting/tools/datasets/organic.py create mode 100644 prompting/tools/datasets/organic_dataset.py diff --git a/connecting_vali.ipynb b/connecting_vali.ipynb new file mode 100644 index 000000000..2f2242e28 --- /dev/null +++ b/connecting_vali.ipynb @@ -0,0 +1,380 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# https://docs.bittensor.com/subnets/register-validate-mine#check-the-permit-status\n", + "\"\"\"\n", + "Except in Subnet 1, all subnets have 256 UID slots per subnet. Of these 256 UID slots,\n", + "a subnet can have a maximum of 64 subnet validator UIDs and 192 (i.e., 256-64) UIDs for subnet miners.\n", + "\"\"\"\n", + "\n", + "import bittensor as bt\n", + "import asyncio\n", + "\n", + "NET_UID = 61\n", + "NETWORK = \"test\"\n", + "WALLET = \"validator\"" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Hotkey 5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr is registered. UID: 166\n" + ] + } + ], + "source": [ + "# Initialize your wallet\n", + "wallet = bt.wallet(name=WALLET)\n", + "\n", + "# Initialize the subtensor\n", + "subtensor = bt.subtensor(network=NETWORK)\n", + "\n", + "# Fetch the metagraph\n", + "subnet = subtensor.metagraph(netuid=NET_UID)\n", + "\n", + "# Replace below with your SS58 hotkey \n", + "hotkey = wallet.hotkey.ss58_address\n", + "my_uid = subnet.hotkeys.index(wallet.hotkey.ss58_address)\n", + "sub = bt.subtensor(NETWORK)\n", + "mg = sub.metagraph(NET_UID)\n", + "if hotkey not in mg.hotkeys:\n", + " print(f\"Hotkey {hotkey} deregistered\")\n", + "else:\n", + " print(f\"Hotkey {hotkey} is registered. UID: {my_uid}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "204" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "len(subnet.validator_permit)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Active UIDs (total: 204): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203]\n", + "My hotkey is active: True\n" + ] + } + ], + "source": [ + "# Extract active UIDs\n", + "active_uids = subnet.uids.tolist()\n", + "\n", + "# Print the active UIDs\n", + "print(f\"Active UIDs (total: {len(active_uids)}): {active_uids}\")\n", + "print(f\"My hotkey is active: {my_uid in active_uids}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "False" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "subnet.validator_permit[my_uid]" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(204,)" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "subnet.validator_permit.shape # 1000+ tao" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/workspace/venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n", + "2024-06-18 13:30:49,105\tINFO util.py:154 -- Missing packages: ['ipywidgets']. Run `pip install -U ipywidgets`, then restart the notebook server for rich notebook output.\n", + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "INFO:bittensor: - Created DendriteResponseEvent:\n", + " DendriteResponseEvent(uids=[166], completions=[''], timings=[17], status_messages=['Timedout after 17.0 seconds.'], status_codes=[408]) - \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2024-06-18 13:31:25.474 | INFO | - Created DendriteResponseEvent:\n", + " DendriteResponseEvent(uids=[166], completions=[''], timings=[17], status_messages=['Timedout after 17.0 seconds.'], status_codes=[408]) - \n" + ] + } + ], + "source": [ + "# Sample validators.\n", + "# Define your request payload (e.g., a text prompting request)\n", + "# import time\n", + "from prompting.protocol import StreamPromptingSynapse\n", + "# request_payload = bt.synapse.TextPrompting(\n", + "# text=\"Hello, can you process this request?\"\n", + "# )\n", + "# request_payload\n", + "\n", + "# Get the list of uids to query for this step.\n", + "# uids = get_random_uids(self, k=k, exclude=exclude or []).to(self.device)\n", + "# uids = get_random_uids(self, k=k, exclude=exclude or []).to(self.device)\n", + "# uids_cpu = uids.cpu().tolist()\n", + "\n", + "uids = [my_uid]\n", + "axons = [subnet.axons[uid] for uid in uids]\n", + "\n", + "# # Directly call dendrite and process responses in parallel\n", + "dendrite = bt.dendrite(wallet=wallet)\n", + "roles = [\"user\"]\n", + "messages = [\"Capital of Zimbabwe?\"]\n", + "timeout = 17\n", + "streams_responses = await dendrite(\n", + " axons=axons,\n", + " synapse=StreamPromptingSynapse(roles=roles, messages=messages),\n", + " timeout=timeout,\n", + " deserialize=False,\n", + " streaming=True,\n", + ")\n", + "# Prepare the task for handling stream responses\n", + "from prompting.dendrite import DendriteResponseEvent\n", + "from prompting.forward import handle_response\n", + "from transformers import AutoTokenizer\n", + "\n", + "stream_results_dict = dict(zip(uids, streams_responses))\n", + "model_name = \"casperhansen/llama-3-8b-instruct-awq\"\n", + "tokenizer = AutoTokenizer.from_pretrained(model_name)\n", + "handle_stream_responses_task = asyncio.create_task(handle_response(stream_results_dict, tokenizer))\n", + "stream_results = await handle_stream_responses_task\n", + "# if not agent.task.static_reference:\n", + "# reference_generation_task = generate_reference(agent)\n", + "# _, stream_results = await asyncio.gather(\n", + "# reference_generation_task, handle_stream_responses_task\n", + "# )\n", + "# else:\n", + "# stream_results = await handle_stream_responses_task\n", + "\n", + "# log_stream_results(stream_results)\n", + "\n", + "# Encapsulate the responses in a response event (dataclass)\n", + "response_event = DendriteResponseEvent(\n", + " stream_results=stream_results, uids=uids, timeout=timeout\n", + ")\n", + "\n", + "bt.logging.info(f\"Created DendriteResponseEvent:\\n {response_event}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "INFO:bittensor: - Created DendriteResponseEvent:\n", + " DendriteResponseEvent(uids=[166], completions=[''], timings=[0], status_messages=['Service at 194.68.245.74:8091/StreamPromptingSynapse unavailable.'], status_codes=[503]) - \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2024-06-17 11:39:42.019 | INFO | - Created DendriteResponseEvent:\n", + " DendriteResponseEvent(uids=[166], completions=[''], timings=[0], status_messages=['Service at 194.68.245.74:8091/StreamPromptingSynapse unavailable.'], status_codes=[503]) - \n" + ] + } + ], + "source": [ + "\n", + "# Reward the responses and get the reward result (dataclass)\n", + "# This contains a list of RewardEvents but can be exported as a dict (column-wise) for logging etc\n", + "# reward_result = RewardResult(\n", + "# self.reward_pipeline,\n", + "# agent=agent,\n", + "# response_event=response_event,\n", + "# device=self.device,\n", + "# )\n", + "# bt.logging.info(f\"Created RewardResult:\\n {reward_result}\")\n", + "\n", + "# best_response = response_event.completions[reward_result.rewards.argmax()]" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[SynapseStreamResult(exception=UnboundLocalError(\"local variable 'chunk' referenced before assignment\"), uid=166, accumulated_chunks=[], accumulated_chunks_timings=[], tokens_per_chunk=[], synapse=StreamPromptingSynapse(required_hash_fields=['messages'], roles=['user'], messages=['failure'], completion=''))]" + ] + }, + "execution_count": 53, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "stream_results" + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "DendriteResponseEvent(uids=[166], completions=[''], timings=[0], status_messages=['Service at 194.68.245.74:8091/StreamPromptingSynapse unavailable.'], status_codes=[503])" + ] + }, + "execution_count": 44, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response_event" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[]" + ] + }, + "execution_count": 35, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "streams_responses" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Function to send a request to a single validator\n", + "async def send_request_to_validator(dendrite, uid):\n", + " axon_info = subnet.axons[uid.item()]\n", + " response = await dendrite.query(\n", + " axons=[axon_info],\n", + " synapse=request_payload,\n", + " timeout=17 # Adjust timeout as needed\n", + " )\n", + " return response\n", + "\n", + "# Function to send requests to all validators\n", + "async def send_requests_to_all_validators(dendrite, active_uids):\n", + " tasks = [send_request_to_validator(dendrite, uid) for uid in active_uids]\n", + " responses = await asyncio.gather(*tasks, return_exceptions=True)\n", + " return responses\n", + "\n", + "# Initialize Dendrite\n", + "dendrite = bt.dendrite(wallet=wallet)\n", + "\n", + "# Run the function to send requests\n", + "responses = asyncio.run(send_requests_to_all_validators(dendrite, active_uids))\n", + "\n", + "# Process responses\n", + "for response in responses:\n", + " if isinstance(response, Exception):\n", + " print(f\"Request failed with exception: {response}\")\n", + " else:\n", + " print(f\"Received response: {response}\")\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/prompting/agent.py b/prompting/agent.py index b4b67255b..974387bc1 100644 --- a/prompting/agent.py +++ b/prompting/agent.py @@ -42,7 +42,7 @@ def finished(self): """This is a roleplaying game where you are impersonating {mood} human user with a specific persona. As a human, you are using AI assistant to {desc} related to {topic} ({subtopic}) in a {tone} tone. You don't need to greet the assistant or be polite, unless this is part of your persona. The spelling and grammar of your messages should also reflect your persona. Your singular focus is to use the assistant to {goal}: {query} - """ + """ ) def __init__( diff --git a/prompting/base/validator.py b/prompting/base/validator.py index 5c3fe87a9..50f818b90 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -29,6 +29,7 @@ from prompting.base.neuron import BaseNeuron from prompting.mock import MockDendrite from prompting.protocol import StreamPromptingSynapse +from prompting.tools.datasets.organic_dataset import OrganicDataset from prompting.utils.config import add_validator_args from prompting.utils.exceptions import MaxRetryError @@ -86,33 +87,22 @@ def __init__(self, config=None): # return blacklist async def _handle_organic(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: - # TODO: Wrap into OrganicTask. - with self.lock: - bt.logging.info(f"Organic handle: {synapse}") + OrganicDataset().add(synapse) + bt.logging.info(f"Organic handle: {synapse}") return synapse def serve_axon(self): - """Serve axon to enable external connections.""" - bt.logging.info("Serving IP to chain...") - # try: + """Serve axon to enable external connections""" + validator_uid = self.metagraph.hotkeys.index(self.wallet.hotkey.ss58_address) + bt.logging.info(f"Serving validator IP of UID {validator_uid} to chain...") self.axon = bt.axon(wallet=self.wallet, config=self.config) self.axon.attach( forward_fn=self._handle_organic, blacklist_fn=None, priority_fn=None, ) - # self.subtensor.serve_axon( - # netuid=self.config.netuid, - # axon=self.axon, - # ) self.axon.serve(netuid=self.config.netuid, subtensor=self.subtensor) self.axon.start() - validator_uid = self.metagraph.hotkeys.index( - self.wallet.hotkey.ss58_address - ) - bt.logging.info(f"Running validator on uid: {validator_uid}") - # except Exception as e: - # bt.logging.error(f"Failed to create Axon initialize with exception: {e}") def run(self): """ diff --git a/prompting/forward.py b/prompting/forward.py index 06fb0fce7..e89fecfe7 100644 --- a/prompting/forward.py +++ b/prompting/forward.py @@ -28,7 +28,7 @@ from prompting.conversation import create_task from prompting.protocol import StreamPromptingSynapse from prompting.rewards import RewardResult -from prompting.tasks import QuestionAnsweringTask +from prompting.tasks import QuestionAnsweringTask, organic_task from prompting.utils.uids import get_random_uids from prompting.utils.logging import log_event from prompting.utils.misc import async_log, serialize_exception_to_string @@ -36,11 +36,6 @@ from prompting.utils.uids import get_random_uids from dataclasses import dataclass -@async_log -async def generate_reference(agent): - loop = asyncio.get_running_loop() - result = await loop.run_in_executor(None, agent.task.generate_reference, agent.llm_pipeline) - return result @async_log async def execute_dendrite_call(dendrite_call): @@ -285,14 +280,20 @@ async def forward(self): # Create random agent with task, topic, profile... bt.logging.info(f"🤖 Creating agent for {task_name} task... ") - agent = HumanAgent( - task=task, llm_pipeline=self.llm_pipeline, begin_conversation=True - ) turn = 0 exclude_uids = [] - roles = ['user'] - messages = [agent.challenge] + agent = HumanAgent( + task=task, llm_pipeline=self.llm_pipeline, begin_conversation=True + ) + if task_name == organic_task.TASK_NAME: + # Organic prompts with conversational history. + roles = task.roles + messages = task.messages + else: + # Benchmarking tasks. + roles = ['user'] + messages = [agent.challenge] while True: # Note: The try catch is a safe clause to ensure that the forward loop continues even if an error occurs in run_step. # To be reconsidered in the next version. @@ -318,8 +319,8 @@ async def forward(self): roles.append("assistant") messages.append(accepted_answer) - # 50% chance of single turn conversation, 25% of two turns, 12.5% chance of 3 turns, 6.25% chance of 4 turns, 3.63% chance of 5... - if random.random()<0.5 or turn>=1: + # 50% chance of single turn conversation, 25% of two turns. + if random.random() < 0.5 or turn >= 1 or task_name == organic_task.TASK_NAME: break history = '\n'.join([f"{role}: {message}" for role, message in zip(roles, messages)]) diff --git a/prompting/llms/vllm_llm.py b/prompting/llms/vllm_llm.py index 85ee7b974..5e8d78705 100644 --- a/prompting/llms/vllm_llm.py +++ b/prompting/llms/vllm_llm.py @@ -18,7 +18,7 @@ import time import torch import bittensor as bt -from typing import List, Dict +from typing import List, Dict, Optional, Any from vllm import LLM, SamplingParams from prompting.cleaners.cleaner import CleanerPipeline from prompting.llms import BasePipeline, BaseLLM @@ -112,6 +112,24 @@ def __init__( "end": "<|start_header_id|>assistant<|end_header_id|>", } + def query_conversation( + self, + messages: list[str], + roles: list[str], + cleaner: Optional[CleanerPipeline] = None, + ): + """Query LLM with the given lists of conversation history and roles.""" + assert len(messages) == len(roles), "Length of messages and roles must be the same" + inputs: list[dict[str, Any]] = [{"content": self.system_prompt, "role": "system"}] + for role, message in zip(roles, messages): + inputs.append({"content": message, "role": role}) + + t0 = time.perf_counter() + response = self.forward(messages=inputs) + response = self.clean_response(cleaner, response) + self.times.extend((0, time.perf_counter() - t0)) + return response + def query( self, message: str, diff --git a/prompting/shared/context.py b/prompting/shared/context.py index a9918c725..c4e96f816 100644 --- a/prompting/shared/context.py +++ b/prompting/shared/context.py @@ -15,3 +15,5 @@ class Context: tags: List[str] = None extra: dict = None # additional non-essential information stats: dict = None # retrieval stats such as fetch time, number of tries, etc. + messages: list[str] = None + roles: list[str] = None diff --git a/prompting/tasks/__init__.py b/prompting/tasks/__init__.py index a9c18603c..9443480c3 100644 --- a/prompting/tasks/__init__.py +++ b/prompting/tasks/__init__.py @@ -8,7 +8,7 @@ from .translate import TranslationTask, TranslationPipeline from .mock import MockTask from .sentiment import SentimentAnalysisTask -from .organic import OrganicTask +from .organic_task import OrganicTask TASKS = { OrganicTask.name: OrganicTask, diff --git a/prompting/tasks/organic.py b/prompting/tasks/organic.py deleted file mode 100644 index 1c08a0f60..000000000 --- a/prompting/tasks/organic.py +++ /dev/null @@ -1,108 +0,0 @@ -import bittensor as bt -from dataclasses import dataclass -from prompting.tasks import Task - -# TODO: introduce criteria for the query and reference answer (length, layout, etc.) and make these arguments - -# Used to instruct the LLM to provide a good query when given a context -QUERY_SYSTEM_PROMPT = """\ -You are a question-generating expert, focusing on delivering comprehensive and accurate questions with depth and clarity. The questions you generate should be based on the context that is provided. -You will maintain a neutral tone in your questions. -You will adhere to a word limit of 50 words for each question. -""" - -# Used to obtain the query (which is a question about the context) -QUERY_PROMPT_TEMPLATE = """\ -Ask a specific question about the following context: - -#Context: -{context} -""" - -# Used to obtain the query (which is a followup question about the context) -# TODO: we may not need the entire conversation history - we can sample a subset of it (first k messages, last k messages, etc.) -FOLLOWUP_PROMPT_TEMPLATE = """ -Compose a single, specific question to continue the dialogue below. Adopt the persona of the original user, including their communication style and objectives. The question should be based on the previous exchanges and must not be answerable with a simple yes or no. - -The question should require detailed knowledge of the conversation history for a correct response, emphasizing requests for clarification or additional details (e.g., 'What specific steps did you take?' or 'How did that situation resolve?'). Avoid referring to the subject by name and instead use indirect pronouns or descriptions (e.g., 'he,' 'she,' 'it'). Avoid answering the question yourself and refrain from providing new information not already discussed. - -# Context: -{context} - -# Conversation History: -{history} -""" - -# Used to obtain reference answer -REFERENCE_PROMPT_TEMPLATE = """\ -Answer the question you will receive in detail, utilizing the following context. - -#Context: -{context} - -# Question: -{question} -""" - -# TODO: We also need a special followup reference prompt (or just merge both) -# Used to obtain reference answer -FOLLOWUP_REFERENCE_PROMPT_TEMPLATE = """\ -Answer the question you will receive in detail, utilizing the following context and conversation history as required. - -#Context: -{context} - -# Conversation History: -{history} - -# Question: -{question} -""" - -@dataclass -class OrganicTask(Task): - name = "organic" - desc = "get help on answering a question" - goal = "to get the answer to the following question" - - reward_definition = [ - dict(name="rouge", ngram="rouge-1", metric="f", weight=0.5), - dict(name="relevance", weight=0.5), - ] - penalty_definition = [ - dict(name="rouge", ngram="rouge-1", metric="f", weight=0.5), - ] - - cleaning_pipeline = [ - dict(name="remove_quotes"), - dict(name="prune_ending"), - dict(name="remove_roles"), - dict(name="remove_post_question_text"), - ] - - def __init__(self, llm_pipeline, context, create_reference=True, history=None): - self.context = context - - self.query_system_prompt = QUERY_SYSTEM_PROMPT - if history: - self.query_prompt = FOLLOWUP_PROMPT_TEMPLATE.format(context=context.content, history=history) - bt.logging.warning(f'Using history!!\n{history=}\n\n{context=}\n\n{self.query_prompt=}') - else: - self.query_prompt = QUERY_PROMPT_TEMPLATE.format(context=context.content) - - self.query = self.generate_query(llm_pipeline) - - if history: - self.reference_prompt = FOLLOWUP_REFERENCE_PROMPT_TEMPLATE.format( - context=context.content, question=self.query, history=history - ) - else: - self.reference_prompt = REFERENCE_PROMPT_TEMPLATE.format( - context=context.content, question=self.query - ) - if create_reference: - self.reference = self.generate_reference(llm_pipeline) - - self.topic = context.title - self.subtopic = context.topic - self.tags = context.tags diff --git a/prompting/tasks/organic_task.py b/prompting/tasks/organic_task.py new file mode 100644 index 000000000..5da765144 --- /dev/null +++ b/prompting/tasks/organic_task.py @@ -0,0 +1,82 @@ +import time +from typing import Any +import bittensor as bt +from dataclasses import dataclass +from prompting.cleaners.cleaner import CleanerPipeline +from prompting.llms.base_llm import BasePipeline +from prompting.llms.vllm_llm import vLLM_LLM +from prompting.shared.context import Context +from prompting.tasks import Task +from transformers import Pipeline + +from prompting.tasks.task import make_system_prompt + +# QUERY_SYSTEM_PROMPT = "" +TASK_NAME = "organic" + + +@dataclass +class OrganicTask(Task): + name = TASK_NAME + desc = "get help on answering a question" + goal = "to get the answer to the following question" + # Use challenge as a query. + challenge_type = "query" + + reward_definition = [ + # dict(name="rouge", ngram="rouge-1", metric="f", weight=0.5), + dict(name="relevance", weight=1.0), + ] + penalty_definition = [ + dict(name="relevance", weight=1.0), + ] + + cleaning_pipeline = [ + dict(name="remove_quotes"), + dict(name="prune_ending"), + dict(name="remove_roles"), + dict(name="remove_post_question_text"), + ] + + def __init__(self, llm_pipeline: Pipeline, context: Context, create_reference: bool = True): + self.context = context + self.query = context.content + self.reference_prompt = context.content + self.messages = context.messages + self.roles = context.roles + self.topic = context.title + self.subtopic = context.topic + self.tags = context.tags + if create_reference: + self.reference = self.generate_reference(llm_pipeline) + + def generate( + self, + system: str, + messages: list[str], + roles: list[str], + pipeline: BasePipeline, + clean=True + ) -> str: + """Use the LLM to generate a response to a prompt""" + cleaner = CleanerPipeline(cleaning_pipeline=self.cleaning_pipeline) if clean else None + return vLLM_LLM(pipeline, system_prompt=system).query_conversation( + messages=messages, roles=roles, cleaner=cleaner) + + def generate_reference(self, pipeline: BasePipeline, clean=True) -> str: + """Generates a reference answer to be used for scoring miner completions""" + t0 = time.perf_counter() + if not self.static_reference: + if not self.clean_reference: + clean = False + bt.logging.info("🤖 Generating reference...") + self.reference = self.generate( + system=make_system_prompt(), + messages=self.messages, + roles=self.roles, + pipeline=pipeline, + clean=clean, + ) + + self.reference_time = time.perf_counter() - t0 + return self.reference diff --git a/prompting/tasks/qa.py b/prompting/tasks/qa.py index 2fa293087..e8e32e3b0 100644 --- a/prompting/tasks/qa.py +++ b/prompting/tasks/qa.py @@ -88,7 +88,7 @@ def __init__(self, llm_pipeline, context, create_reference=True, history=None): self.query_prompt = FOLLOWUP_PROMPT_TEMPLATE.format(context=context.content, history=history) bt.logging.warning(f'Using history!!\n{history=}\n\n{context=}\n\n{self.query_prompt=}') else: - self.query_prompt = QUERY_PROMPT_TEMPLATE.format(context=context.content) + self.query_prompt = QUERY_PROMPT_TEMPLATE.format(context=context.content) self.query = self.generate_query(llm_pipeline) diff --git a/prompting/tools/datasets/__init__.py b/prompting/tools/datasets/__init__.py index 21942d91a..9e17c71d1 100644 --- a/prompting/tools/datasets/__init__.py +++ b/prompting/tools/datasets/__init__.py @@ -5,4 +5,4 @@ from .wiki import WikiDataset, WikiDateDataset from .generic_instruction import GenericInstructionDataset from .review import ReviewDataset -from .organic import OrganicDataset +from .organic_dataset import OrganicDataset diff --git a/prompting/tools/datasets/organic.py b/prompting/tools/datasets/organic.py deleted file mode 100644 index 35cc63cd2..000000000 --- a/prompting/tools/datasets/organic.py +++ /dev/null @@ -1,50 +0,0 @@ -# The MIT License (MIT) -# Copyright © 2024 Yuma Rao -# Copyright © 2023 Opentensor Foundation - -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the “Software”), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. - -# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -from typing import Any -from .base import Dataset -from ..selector import Selector - - -class OrganicDataset(Dataset): - name = "organic" - - def __init__( - self, - ): - """Collects organic prompts and/or generated synthetic prompts - - Args: - """ - pass - - def get( - self, - name: str, - **kwargs, - ) -> dict[str, Any]: - """""" - return {} - - def search(self, name) -> dict[str, Any]: - """""" - return {} - - def random(self) -> dict[str, Any]: - """""" - return {} diff --git a/prompting/tools/datasets/organic_dataset.py b/prompting/tools/datasets/organic_dataset.py new file mode 100644 index 000000000..1d0b7b86d --- /dev/null +++ b/prompting/tools/datasets/organic_dataset.py @@ -0,0 +1,71 @@ +from typing import Any, Optional + +from prompting.protocol import StreamPromptingSynapse +from prompting.tools.datasets.base import Dataset +import threading + +from prompting.tools.selector import Selector +# import sqlite3 + + +class OrganicDataset(Dataset): + """Organic dataset singleton""" + name = "organic" + + _instance = None + _lock = threading.Lock() + _queue: list[StreamPromptingSynapse] = [] + + def __new__(cls, *args, **kwargs): + if not cls._instance: + with cls._lock: + if not cls._instance: + cls._instance = super(OrganicDataset, cls).__new__(cls, *args, **kwargs) + return cls._instance + + @classmethod + def add(cls, synapse: StreamPromptingSynapse): + with cls._lock: + cls._queue.append(synapse) + + @classmethod + def random(cls, selector: Optional[Selector]) -> dict[str, Any]: + with cls._lock: + if cls._queue: + synapse = cls._queue.pop(0) + organic_source = "organic" + else: + # TODO: Pop synthetic data. + synapse = StreamPromptingSynapse(messages=["Capital of Australia?"], roles=["user"]) + organic_source = "synthetic" + return { + "title": "Prompt", + "topic": "", + "subtopic": "", + "content": synapse.messages[-1], + "internal_links": [], + "external_links": [], + "source": organic_source, + "messages": synapse.messages, + "roles": synapse.roles, + "extra": {"date": None}, + } + + def get(self, name: str, **kwargs) -> dict[str, Any]: + """""" + raise NotImplementedError + + def search(self, name) -> dict[str, Any]: + """""" + raise NotImplementedError + + +if __name__ == "__main__": + dataset1 = OrganicDataset() + dataset2 = OrganicDataset() + dataset1.add(StreamPromptingSynapse(messages=["Capital of Australia?"], roles=["user"])) + dataset1.add(StreamPromptingSynapse(messages=["Capital of South Korea?"], roles=["user"])) + print(dataset2.random()["synapse"].messages) + print(dataset2.random()["synapse"].messages) + # This will raise an IndexError as the queue is empty + # print(dataset2.random().messages) \ No newline at end of file diff --git a/prompting/utils/config.py b/prompting/utils/config.py index 71956d741..473a68dc7 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -300,7 +300,7 @@ def add_validator_args(cls, parser): type=float, nargs="+", help="The probability of sampling each task.", - default=[0.5] + [0.5 / (len(TASKS) - 1)] * (len(TASKS) - 1), + default=[0.7] + [0.3 / (len(TASKS) - 1)] * (len(TASKS) - 1), ) parser.add_argument( From f36ed99b2d999bf4a30209e28191fd6e6e10b145 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Wed, 19 Jun 2024 07:18:12 +0000 Subject: [PATCH 03/57] Update draft notebook --- connecting_vali.ipynb | 68 ++++--------------------------------------- 1 file changed, 6 insertions(+), 62 deletions(-) diff --git a/connecting_vali.ipynb b/connecting_vali.ipynb index 2f2242e28..d7c9aee27 100644 --- a/connecting_vali.ipynb +++ b/connecting_vali.ipynb @@ -58,33 +58,15 @@ "cell_type": "code", "execution_count": 3, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "204" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "len(subnet.validator_permit)" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Active UIDs (total: 204): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203]\n", - "My hotkey is active: True\n" + "My hotkey is active: True\n", + "Is val permit: False\n", + "1000+ TAO:\n" ] } ], @@ -94,47 +76,9 @@ "\n", "# Print the active UIDs\n", "print(f\"Active UIDs (total: {len(active_uids)}): {active_uids}\")\n", - "print(f\"My hotkey is active: {my_uid in active_uids}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "False" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "subnet.validator_permit[my_uid]" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(204,)" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "subnet.validator_permit.shape # 1000+ tao" + "print(f\"My hotkey is active: {my_uid in active_uids}\")\n", + "print(f\"Is val permit: {subnet.validator_permit[my_uid]}\")\n", + "print(f\"1000+ TAO:\")" ] }, { From 02d2de29d8b194523e5a46f9a1bc99e551c8ef04 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Wed, 19 Jun 2024 10:28:12 +0000 Subject: [PATCH 04/57] Add minor organic changes --- connecting_vali.ipynb | 127 +++----------------- prompting/base/validator.py | 43 ++++++- prompting/tools/datasets/organic_dataset.py | 5 +- prompting/utils/config.py | 2 +- 4 files changed, 64 insertions(+), 113 deletions(-) diff --git a/connecting_vali.ipynb b/connecting_vali.ipynb index d7c9aee27..dcb7a014a 100644 --- a/connecting_vali.ipynb +++ b/connecting_vali.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -15,21 +15,30 @@ "import bittensor as bt\n", "import asyncio\n", "\n", - "NET_UID = 61\n", + "# NET_UID = 61\n", + "NET_UID = 170\n", "NETWORK = \"test\"\n", "WALLET = \"validator\"" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 31, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:bittensor: - Connected to test network and wss://test.finney.opentensor.ai:443/. - \n", + "INFO:bittensor: - Connected to test network and wss://test.finney.opentensor.ai:443/. - \n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "Hotkey 5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr is registered. UID: 166\n" + "Hotkey 5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr is registered. UID: 0\n" ] } ], @@ -56,16 +65,16 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 32, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Active UIDs (total: 204): [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203]\n", + "Active UIDs (total: 2): [0, 1]\n", "My hotkey is active: True\n", - "Is val permit: False\n", + "Is val permit: True\n", "1000+ TAO:\n" ] } @@ -83,27 +92,16 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 36, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "/workspace/venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n", - "2024-06-18 13:30:49,105\tINFO util.py:154 -- Missing packages: ['ipywidgets']. Run `pip install -U ipywidgets`, then restart the notebook server for rich notebook output.\n", "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", "INFO:bittensor: - Created DendriteResponseEvent:\n", - " DendriteResponseEvent(uids=[166], completions=[''], timings=[17], status_messages=['Timedout after 17.0 seconds.'], status_codes=[408]) - \n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2024-06-18 13:31:25.474 | INFO | - Created DendriteResponseEvent:\n", - " DendriteResponseEvent(uids=[166], completions=[''], timings=[17], status_messages=['Timedout after 17.0 seconds.'], status_codes=[408]) - \n" + " DendriteResponseEvent(uids=[0], completions=['{\"name\":\"StreamPromptingSynapse\",\"timeout\":40.0,\"total_size\":4036,\"header_size\":0,\"dendrite\":{\"status_code\":null,\"status_message\":null,\"process_time\":null,\"ip\":\"194.68.245.74\",\"port\":null,\"version\":701,\"nonce\":16747099962016130,\"uuid\":\"e28789d2-2e1a-11ef-9141-0242c0a80902\",\"hotkey\":\"5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr\",\"signature\":\"0xa4dd17e6360645f9eb86b4ce4eeb975985a23cc492b91de4f72f657690426d1b39782578940f417516b71680e2495664e23150cbda9efff3cf4d2ef8f8f2fe8c\"},\"axon\":{\"status_code\":null,\"status_message\":null,\"process_time\":null,\"ip\":\"194.68.245.74\",\"port\":8091,\"version\":null,\"nonce\":null,\"uuid\":null,\"hotkey\":\"5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr\",\"signature\":null},\"computed_body_hash\":\"\",\"required_hash_fields\":[\"messages\"],\"roles\":[\"user\"],\"messages\":[\"Organic prompt query. Question: Capital of Zimbabwe?\"],\"completion\":\"\"}'], timings=[6.793562173843384], status_messages=['Success'], status_codes=[200]) - \n" ] } ], @@ -128,8 +126,8 @@ "# # Directly call dendrite and process responses in parallel\n", "dendrite = bt.dendrite(wallet=wallet)\n", "roles = [\"user\"]\n", - "messages = [\"Capital of Zimbabwe?\"]\n", - "timeout = 17\n", + "messages = [\"Organic prompt query. Question: Capital of Zimbabwe?\"]\n", + "timeout = 40\n", "streams_responses = await dendrite(\n", " axons=axons,\n", " synapse=StreamPromptingSynapse(roles=roles, messages=messages),\n", @@ -147,17 +145,6 @@ "tokenizer = AutoTokenizer.from_pretrained(model_name)\n", "handle_stream_responses_task = asyncio.create_task(handle_response(stream_results_dict, tokenizer))\n", "stream_results = await handle_stream_responses_task\n", - "# if not agent.task.static_reference:\n", - "# reference_generation_task = generate_reference(agent)\n", - "# _, stream_results = await asyncio.gather(\n", - "# reference_generation_task, handle_stream_responses_task\n", - "# )\n", - "# else:\n", - "# stream_results = await handle_stream_responses_task\n", - "\n", - "# log_stream_results(stream_results)\n", - "\n", - "# Encapsulate the responses in a response event (dataclass)\n", "response_event = DendriteResponseEvent(\n", " stream_results=stream_results, uids=uids, timeout=timeout\n", ")\n", @@ -165,44 +152,6 @@ "bt.logging.info(f\"Created DendriteResponseEvent:\\n {response_event}\")" ] }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", - "INFO:bittensor: - Created DendriteResponseEvent:\n", - " DendriteResponseEvent(uids=[166], completions=[''], timings=[0], status_messages=['Service at 194.68.245.74:8091/StreamPromptingSynapse unavailable.'], status_codes=[503]) - \n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2024-06-17 11:39:42.019 | INFO | - Created DendriteResponseEvent:\n", - " DendriteResponseEvent(uids=[166], completions=[''], timings=[0], status_messages=['Service at 194.68.245.74:8091/StreamPromptingSynapse unavailable.'], status_codes=[503]) - \n" - ] - } - ], - "source": [ - "\n", - "# Reward the responses and get the reward result (dataclass)\n", - "# This contains a list of RewardEvents but can be exported as a dict (column-wise) for logging etc\n", - "# reward_result = RewardResult(\n", - "# self.reward_pipeline,\n", - "# agent=agent,\n", - "# response_event=response_event,\n", - "# device=self.device,\n", - "# )\n", - "# bt.logging.info(f\"Created RewardResult:\\n {reward_result}\")\n", - "\n", - "# best_response = response_event.completions[reward_result.rewards.argmax()]" - ] - }, { "cell_type": "code", "execution_count": 53, @@ -262,42 +211,6 @@ "source": [ "streams_responses" ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Function to send a request to a single validator\n", - "async def send_request_to_validator(dendrite, uid):\n", - " axon_info = subnet.axons[uid.item()]\n", - " response = await dendrite.query(\n", - " axons=[axon_info],\n", - " synapse=request_payload,\n", - " timeout=17 # Adjust timeout as needed\n", - " )\n", - " return response\n", - "\n", - "# Function to send requests to all validators\n", - "async def send_requests_to_all_validators(dendrite, active_uids):\n", - " tasks = [send_request_to_validator(dendrite, uid) for uid in active_uids]\n", - " responses = await asyncio.gather(*tasks, return_exceptions=True)\n", - " return responses\n", - "\n", - "# Initialize Dendrite\n", - "dendrite = bt.dendrite(wallet=wallet)\n", - "\n", - "# Run the function to send requests\n", - "responses = asyncio.run(send_requests_to_all_validators(dendrite, active_uids))\n", - "\n", - "# Process responses\n", - "for response in responses:\n", - " if isinstance(response, Exception):\n", - " print(f\"Request failed with exception: {response}\")\n", - " else:\n", - " print(f\"Received response: {response}\")\n" - ] } ], "metadata": { diff --git a/prompting/base/validator.py b/prompting/base/validator.py index 50f818b90..339b26677 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -15,6 +15,7 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. +from functools import partial import sys import copy import torch @@ -75,9 +76,10 @@ def __init__(self, config=None): self.is_running: bool = False self.thread: threading.Thread = None self.lock = asyncio.Lock() + self._organic_dataset = OrganicDataset() # Serve axon to enable external connections. # if not self.config.neuron.axon_off: - # self.serve_axon() + # self.serve_axon() # else: # bt.logging.warning("axon off, not serving ip to chain.") @@ -86,8 +88,45 @@ def __init__(self, config=None): # bt.logging.info(blacklist[1]) # return blacklist + # async def _handle_organic(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: + # bt.logging.info(f"Received {synapse}") + + # from starlette.types import Send + # async def _prompt(synapse, send: Send): + # bt.logging.info( + # f"Sending {synapse} request to uid: {synapse.uid}, " + # ) + # async def handle_response(responses): + # for resp in responses: + # async for chunk in resp: + # if isinstance(chunk, str): + # await send( + # { + # "type": "http.response.body", + # "body": chunk.encode("utf-8"), + # "more_body": True, + # } + # ) + # bt.logging.info(f"Streamed text: {chunk}") + # await send({"type": "http.response.body", "body": b'', "more_body": False}) + + # axon = self.metagraph.axons[synapse.uid] + # responses = self.dendrite.query( + # axons=[axon], + # synapse=synapse, + # deserialize=False, + # timeout=synapse.timeout, + # streaming=True, + # ) + # return await handle_response(responses) + # + # token_streamer = partial(_prompt, synapse) + # response = synapse.create_streaming_response(token_streamer) + # OrganicDataset().add(response) + # return response + async def _handle_organic(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: - OrganicDataset().add(synapse) + self._organic_dataset.add(synapse) bt.logging.info(f"Organic handle: {synapse}") return synapse diff --git a/prompting/tools/datasets/organic_dataset.py b/prompting/tools/datasets/organic_dataset.py index 1d0b7b86d..906555394 100644 --- a/prompting/tools/datasets/organic_dataset.py +++ b/prompting/tools/datasets/organic_dataset.py @@ -5,7 +5,6 @@ import threading from prompting.tools.selector import Selector -# import sqlite3 class OrganicDataset(Dataset): @@ -35,8 +34,8 @@ def random(cls, selector: Optional[Selector]) -> dict[str, Any]: synapse = cls._queue.pop(0) organic_source = "organic" else: - # TODO: Pop synthetic data. - synapse = StreamPromptingSynapse(messages=["Capital of Australia?"], roles=["user"]) + # TODO: Get synthetic data. + synapse = StreamPromptingSynapse(messages=["Synthetic organic: Capital of Australia?"], roles=["user"]) organic_source = "synthetic" return { "title": "Prompt", diff --git a/prompting/utils/config.py b/prompting/utils/config.py index 473a68dc7..3bfb465d8 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -307,7 +307,7 @@ def add_validator_args(cls, parser): "--neuron.timeout", type=float, help="The timeout for each forward call in seconds.", - default=17, + default=120, ) parser.add_argument( From d9927d3789d1a562a3a015c6ca1c85f65853dc17 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Wed, 19 Jun 2024 20:54:22 +0000 Subject: [PATCH 05/57] Add WIP miners response --- connecting_vali.ipynb | 10 ++-- prompting/base/validator.py | 59 ++++++++------------- prompting/forward.py | 39 +++++++++----- prompting/tools/datasets/organic_dataset.py | 2 - prompting/utils/uids.py | 35 +++++++++++- 5 files changed, 88 insertions(+), 57 deletions(-) diff --git a/connecting_vali.ipynb b/connecting_vali.ipynb index dcb7a014a..3efd691aa 100644 --- a/connecting_vali.ipynb +++ b/connecting_vali.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 30, + "execution_count": 39, "metadata": {}, "outputs": [], "source": [ @@ -23,7 +23,7 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 40, "metadata": {}, "outputs": [ { @@ -65,7 +65,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 41, "metadata": {}, "outputs": [ { @@ -92,7 +92,7 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 42, "metadata": {}, "outputs": [ { @@ -101,7 +101,7 @@ "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", "INFO:bittensor: - Created DendriteResponseEvent:\n", - " DendriteResponseEvent(uids=[0], completions=['{\"name\":\"StreamPromptingSynapse\",\"timeout\":40.0,\"total_size\":4036,\"header_size\":0,\"dendrite\":{\"status_code\":null,\"status_message\":null,\"process_time\":null,\"ip\":\"194.68.245.74\",\"port\":null,\"version\":701,\"nonce\":16747099962016130,\"uuid\":\"e28789d2-2e1a-11ef-9141-0242c0a80902\",\"hotkey\":\"5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr\",\"signature\":\"0xa4dd17e6360645f9eb86b4ce4eeb975985a23cc492b91de4f72f657690426d1b39782578940f417516b71680e2495664e23150cbda9efff3cf4d2ef8f8f2fe8c\"},\"axon\":{\"status_code\":null,\"status_message\":null,\"process_time\":null,\"ip\":\"194.68.245.74\",\"port\":8091,\"version\":null,\"nonce\":null,\"uuid\":null,\"hotkey\":\"5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr\",\"signature\":null},\"computed_body_hash\":\"\",\"required_hash_fields\":[\"messages\"],\"roles\":[\"user\"],\"messages\":[\"Organic prompt query. Question: Capital of Zimbabwe?\"],\"completion\":\"\"}'], timings=[6.793562173843384], status_messages=['Success'], status_codes=[200]) - \n" + " DendriteResponseEvent(uids=[0], completions=['{\"name\":\"StreamPromptingSynapse\",\"timeout\":40.0,\"total_size\":4036,\"header_size\":0,\"dendrite\":{\"status_code\":null,\"status_message\":null,\"process_time\":null,\"ip\":\"194.68.245.74\",\"port\":null,\"version\":701,\"nonce\":16748188472635226,\"uuid\":\"6b47886a-2e1d-11ef-9141-0242c0a80902\",\"hotkey\":\"5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr\",\"signature\":\"0x0003bf86a3f219bca84df126263784401d524d846ea3834c7e88e21fcb676137fd8de8c3990a5e27e42211e5d5146175eeea06beb4712b7f040d6486e831b58f\"},\"axon\":{\"status_code\":null,\"status_message\":null,\"process_time\":null,\"ip\":\"194.68.245.74\",\"port\":8091,\"version\":null,\"nonce\":null,\"uuid\":null,\"hotkey\":\"5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr\",\"signature\":null},\"computed_body_hash\":\"\",\"required_hash_fields\":[\"messages\"],\"roles\":[\"user\"],\"messages\":[\"Organic prompt query. Question: Capital of Zimbabwe?\"],\"completion\":\"\"}'], timings=[11.840179920196533], status_messages=['Success'], status_codes=[200]) - \n" ] } ], diff --git a/prompting/base/validator.py b/prompting/base/validator.py index 339b26677..9416c1ab1 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -16,6 +16,7 @@ # DEALINGS IN THE SOFTWARE. from functools import partial +import random import sys import copy import torch @@ -28,11 +29,14 @@ from traceback import print_exception from prompting.base.neuron import BaseNeuron +from prompting.forward import query_miners from prompting.mock import MockDendrite from prompting.protocol import StreamPromptingSynapse from prompting.tools.datasets.organic_dataset import OrganicDataset from prompting.utils.config import add_validator_args from prompting.utils.exceptions import MaxRetryError +from prompting.utils.uids import get_random_uids, get_uids +from prompting.utils.uids import check_uid_availability class BaseValidatorNeuron(BaseNeuron): @@ -88,47 +92,28 @@ def __init__(self, config=None): # bt.logging.info(blacklist[1]) # return blacklist - # async def _handle_organic(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: - # bt.logging.info(f"Received {synapse}") - - # from starlette.types import Send - # async def _prompt(synapse, send: Send): - # bt.logging.info( - # f"Sending {synapse} request to uid: {synapse.uid}, " - # ) - # async def handle_response(responses): - # for resp in responses: - # async for chunk in resp: - # if isinstance(chunk, str): - # await send( - # { - # "type": "http.response.body", - # "body": chunk.encode("utf-8"), - # "more_body": True, - # } - # ) - # bt.logging.info(f"Streamed text: {chunk}") - # await send({"type": "http.response.body", "body": b'', "more_body": False}) - - # axon = self.metagraph.axons[synapse.uid] - # responses = self.dendrite.query( - # axons=[axon], - # synapse=synapse, - # deserialize=False, - # timeout=synapse.timeout, - # streaming=True, - # ) - # return await handle_response(responses) - # - # token_streamer = partial(_prompt, synapse) - # response = synapse.create_streaming_response(token_streamer) - # OrganicDataset().add(response) - # return response + async def _miners_organic_query( + self, + synapse: StreamPromptingSynapse, + sampling_mode: str, + exclude: List[int] = [] + ) -> StreamPromptingSynapse: + # Get the list of uids to query for this step. + uids = get_uids(sampling_mode=sampling_mode, k=self.config.neuron.sample_size, exclude=exclude) + streams_responses = query_miners(self, synapse.roles, synapse.messages, uids, self.config.neuron.timeout) + uid_stream_dict = dict(zip(uids, streams_responses)) + random_uid, random_stream = random.choice(list(uid_stream_dict.items())) + return random_stream + # Creates a streamer from the selected stream + # streamer = AsyncResponseDataStreamer(async_iterator=random_stream, selected_uid=random_uid) + # response = await streamer.stream(params.request) + # return response async def _handle_organic(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: self._organic_dataset.add(synapse) bt.logging.info(f"Organic handle: {synapse}") - return synapse + response = await _miners_organic_query(self, synapse, "random", []) + return response def serve_axon(self): """Serve axon to enable external connections""" diff --git a/prompting/forward.py b/prompting/forward.py index 766bb680d..196b3f76f 100644 --- a/prompting/forward.py +++ b/prompting/forward.py @@ -23,6 +23,8 @@ import numpy as np import bittensor as bt from typing import List, Dict, Awaitable + +import torch from prompting.agent import HumanAgent from prompting.dendrite import DendriteResponseEvent, SynapseStreamResult from prompting.conversation import create_task @@ -36,7 +38,7 @@ from prompting.utils.uids import get_random_uids from dataclasses import dataclass -SINGLE_TURN_TASKS = ['sentiment', 'translation'] +SINGLE_TURN_TASKS = ('sentiment', 'translation', organic_task.TASK_NAME) @async_log @@ -156,6 +158,16 @@ def log_stream_results(stream_results: List[SynapseStreamResult]): f"Failed response for uid {failed_response.uid}: {formatted_exception}" ) +async def query_miners(self, roles: List[str], messages: List[str], uids: torch.LongTensor, timeout: float): + axons = [self.metagraph.axons[uid] for uid in uids] + # Directly call dendrite and process responses in parallel + return await self.dendrite( + axons=axons, + synapse=StreamPromptingSynapse(roles=roles, messages=messages), + timeout=timeout, + deserialize=False, + streaming=True, + ) async def run_step( self, agent: HumanAgent, roles: List[str], messages: List[str], k: int, timeout: float, exclude: list = None @@ -182,17 +194,20 @@ async def run_step( # Get the list of uids to query for this step. uids = get_random_uids(self, k=k, exclude=exclude or []).to(self.device) uids_cpu = uids.cpu().tolist() + streams_responses = query_miners(self, roles, messages, uids, timeout) + # uids = get_random_uids(self, k=k, exclude=exclude or []).to(self.device) + # uids_cpu = uids.cpu().tolist() - axons = [self.metagraph.axons[uid] for uid in uids] + # axons = [self.metagraph.axons[uid] for uid in uids] - # Directly call dendrite and process responses in parallel - streams_responses = await self.dendrite( - axons=axons, - synapse=StreamPromptingSynapse(roles=roles, messages=messages), - timeout=timeout, - deserialize=False, - streaming=True, - ) + # # Directly call dendrite and process responses in parallel + # streams_responses = await self.dendrite( + # axons=axons, + # synapse=StreamPromptingSynapse(roles=roles, messages=messages), + # timeout=timeout, + # deserialize=False, + # streaming=True, + # ) # Prepare the task for handling stream responses stream_results_dict = dict(zip(uids_cpu, streams_responses)) @@ -240,7 +255,7 @@ async def run_step( "best": best_response, "block": self.block, "step": self.step, - "step_time": time.time() - start_time, + "step_time": time.time() - start_time, **agent.__state_dict__(full=self.config.neuron.log_full), **reward_result.__state_dict__(full=self.config.neuron.log_full), **response_event.__state_dict__(), @@ -322,7 +337,7 @@ async def forward(self): messages.append(accepted_answer) # 50% chance of single turn conversation, 25% of two turns. - if random.random() < 0.5 or turn >= 1 or task_name == organic_task.TASK_NAME: + if random.random() < 0.5 or turn >= 1: break if task.name in SINGLE_TURN_TASKS: diff --git a/prompting/tools/datasets/organic_dataset.py b/prompting/tools/datasets/organic_dataset.py index 906555394..131afde9e 100644 --- a/prompting/tools/datasets/organic_dataset.py +++ b/prompting/tools/datasets/organic_dataset.py @@ -51,11 +51,9 @@ def random(cls, selector: Optional[Selector]) -> dict[str, Any]: } def get(self, name: str, **kwargs) -> dict[str, Any]: - """""" raise NotImplementedError def search(self, name) -> dict[str, Any]: - """""" raise NotImplementedError diff --git a/prompting/utils/uids.py b/prompting/utils/uids.py index 15f8aceee..d3f2694a4 100644 --- a/prompting/utils/uids.py +++ b/prompting/utils/uids.py @@ -1,7 +1,7 @@ import torch import random import bittensor as bt -from typing import List +from typing import List, Union def check_uid_availability( @@ -89,3 +89,36 @@ def get_random_uids(self, k: int, exclude: List[int] = None) -> torch.LongTensor return torch.tensor(random.sample(candidate_uids, k)) else: raise ValueError(f"No eligible uids were found. Cannot return {k} uids") + + +def get_top_incentive_uids(self, k: int, vpermit_tao_limit: int) -> List[int]: + metagraph = self.validator.metagraph + miners_uids = list(map(int, filter(lambda uid: check_uid_availability(metagraph, uid, vpermit_tao_limit), + metagraph.uids))) + + # Builds a dictionary of uids and their corresponding incentives. + all_miners_incentives = { + "miners_uids": miners_uids, + "incentives": list(map(lambda uid: metagraph.I[uid], miners_uids)) + } + + # Zip the uids and their corresponding incentives into a list of tuples. + uid_incentive_pairs = list(zip(all_miners_incentives["miners_uids"], all_miners_incentives["incentives"])) + + # Sort the list of tuples by the incentive value in descending order. + uid_incentive_pairs_sorted = sorted(uid_incentive_pairs, key=lambda x: x[1], reverse=True) + + # Extract the top uids. + top_k_uids = [uid for uid, incentive in uid_incentive_pairs_sorted[:k]] + + return top_k_uids + + +def get_uids(self, sampling_mode: str, k: int, exclude: List[int] = []) -> Union[list[int], torch.LongTensor]: + if sampling_mode == "random": + uids = get_random_uids(self.validator, k=k, exclude=exclude or []).tolist() + return uids + if sampling_mode == "top_incentive": + vpermit_tao_limit = self.validator.config.neuron.vpermit_tao_limit + top_uids = get_top_incentive_uids(self, k=k, vpermit_tao_limit=vpermit_tao_limit) + return top_uids From 785afb63ee7cbf2f139df49f4341f8c761703aa3 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Wed, 26 Jun 2024 11:04:00 +0000 Subject: [PATCH 06/57] Finish end-to-end organic communication --- connecting_vali.ipynb | 177 +++++++++++--------- neurons/miners/huggingface/miner.py | 2 +- prompting/base/validator.py | 72 ++++---- prompting/forward.py | 6 +- prompting/llms/hf.py | 2 +- prompting/tools/datasets/organic_dataset.py | 3 +- prompting/utils/config.py | 7 + prompting/utils/uids.py | 12 +- 8 files changed, 151 insertions(+), 130 deletions(-) diff --git a/connecting_vali.ipynb b/connecting_vali.ipynb index 3efd691aa..a69ea2223 100644 --- a/connecting_vali.ipynb +++ b/connecting_vali.ipynb @@ -2,38 +2,9 @@ "cells": [ { "cell_type": "code", - "execution_count": 39, - "metadata": {}, - "outputs": [], - "source": [ - "# https://docs.bittensor.com/subnets/register-validate-mine#check-the-permit-status\n", - "\"\"\"\n", - "Except in Subnet 1, all subnets have 256 UID slots per subnet. Of these 256 UID slots,\n", - "a subnet can have a maximum of 64 subnet validator UIDs and 192 (i.e., 256-64) UIDs for subnet miners.\n", - "\"\"\"\n", - "\n", - "import bittensor as bt\n", - "import asyncio\n", - "\n", - "# NET_UID = 61\n", - "NET_UID = 170\n", - "NETWORK = \"test\"\n", - "WALLET = \"validator\"" - ] - }, - { - "cell_type": "code", - "execution_count": 40, + "execution_count": 2, "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:bittensor: - Connected to test network and wss://test.finney.opentensor.ai:443/. - \n", - "INFO:bittensor: - Connected to test network and wss://test.finney.opentensor.ai:443/. - \n" - ] - }, { "name": "stdout", "output_type": "stream", @@ -43,16 +14,18 @@ } ], "source": [ - "# Initialize your wallet\n", - "wallet = bt.wallet(name=WALLET)\n", + "# https://docs.bittensor.com/subnets/register-validate-mine#check-the-permit-status\n", + "import bittensor as bt\n", + "import asyncio\n", "\n", - "# Initialize the subtensor\n", + "# NET_UID = 61\n", + "NET_UID = 170\n", + "NETWORK = \"test\"\n", + "WALLET = \"validator\"\n", + "wallet = bt.wallet(name=WALLET)\n", "subtensor = bt.subtensor(network=NETWORK)\n", - "\n", - "# Fetch the metagraph\n", "subnet = subtensor.metagraph(netuid=NET_UID)\n", "\n", - "# Replace below with your SS58 hotkey \n", "hotkey = wallet.hotkey.ss58_address\n", "my_uid = subnet.hotkeys.index(wallet.hotkey.ss58_address)\n", "sub = bt.subtensor(NETWORK)\n", @@ -60,30 +33,8 @@ "if hotkey not in mg.hotkeys:\n", " print(f\"Hotkey {hotkey} deregistered\")\n", "else:\n", - " print(f\"Hotkey {hotkey} is registered. UID: {my_uid}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 41, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Active UIDs (total: 2): [0, 1]\n", - "My hotkey is active: True\n", - "Is val permit: True\n", - "1000+ TAO:\n" - ] - } - ], - "source": [ - "# Extract active UIDs\n", + " print(f\"Hotkey {hotkey} is registered. UID: {my_uid}\")\n", "active_uids = subnet.uids.tolist()\n", - "\n", - "# Print the active UIDs\n", "print(f\"Active UIDs (total: {len(active_uids)}): {active_uids}\")\n", "print(f\"My hotkey is active: {my_uid in active_uids}\")\n", "print(f\"Is val permit: {subnet.validator_permit[my_uid]}\")\n", @@ -92,7 +43,7 @@ }, { "cell_type": "code", - "execution_count": 42, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -100,34 +51,29 @@ "output_type": "stream", "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", - "INFO:bittensor: - Created DendriteResponseEvent:\n", - " DendriteResponseEvent(uids=[0], completions=['{\"name\":\"StreamPromptingSynapse\",\"timeout\":40.0,\"total_size\":4036,\"header_size\":0,\"dendrite\":{\"status_code\":null,\"status_message\":null,\"process_time\":null,\"ip\":\"194.68.245.74\",\"port\":null,\"version\":701,\"nonce\":16748188472635226,\"uuid\":\"6b47886a-2e1d-11ef-9141-0242c0a80902\",\"hotkey\":\"5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr\",\"signature\":\"0x0003bf86a3f219bca84df126263784401d524d846ea3834c7e88e21fcb676137fd8de8c3990a5e27e42211e5d5146175eeea06beb4712b7f040d6486e831b58f\"},\"axon\":{\"status_code\":null,\"status_message\":null,\"process_time\":null,\"ip\":\"194.68.245.74\",\"port\":8091,\"version\":null,\"nonce\":null,\"uuid\":null,\"hotkey\":\"5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr\",\"signature\":null},\"computed_body_hash\":\"\",\"required_hash_fields\":[\"messages\"],\"roles\":[\"user\"],\"messages\":[\"Organic prompt query. Question: Capital of Zimbabwe?\"],\"completion\":\"\"}'], timings=[11.840179920196533], status_messages=['Success'], status_codes=[200]) - \n" + "INFO:bittensor: - Response:\n", + " [SynapseStreamResult(exception=None, uid=0, accumulated_chunks=['The capital of Zimbabwe is Harare.'], accumulated_chunks_timings=[6.252922296524048], tokens_per_chunk=[8], synapse=StreamPromptingSynapse(required_hash_fields=['messages'], roles=['user'], messages=['Organic prompt query. Question: Capital of Zimbabwe?'], completion='The capital of Zimbabwe is Harare.'))] - \n" ] } ], "source": [ "# Sample validators.\n", - "# Define your request payload (e.g., a text prompting request)\n", - "# import time\n", + "# TODO: Stress test: how many queries\n", "from prompting.protocol import StreamPromptingSynapse\n", - "# request_payload = bt.synapse.TextPrompting(\n", - "# text=\"Hello, can you process this request?\"\n", - "# )\n", - "# request_payload\n", - "\n", - "# Get the list of uids to query for this step.\n", - "# uids = get_random_uids(self, k=k, exclude=exclude or []).to(self.device)\n", - "# uids = get_random_uids(self, k=k, exclude=exclude or []).to(self.device)\n", - "# uids_cpu = uids.cpu().tolist()\n", + "from prompting.forward import handle_response\n", + "from transformers import AutoTokenizer\n", "\n", "uids = [my_uid]\n", "axons = [subnet.axons[uid] for uid in uids]\n", "\n", - "# # Directly call dendrite and process responses in parallel\n", + "# Directly call dendrite and process responses in parallel\n", "dendrite = bt.dendrite(wallet=wallet)\n", "roles = [\"user\"]\n", "messages = [\"Organic prompt query. Question: Capital of Zimbabwe?\"]\n", "timeout = 40\n", + "model_name = \"casperhansen/llama-3-8b-instruct-awq\"\n", + "tokenizer = AutoTokenizer.from_pretrained(model_name)\n", + "\n", "streams_responses = await dendrite(\n", " axons=axons,\n", " synapse=StreamPromptingSynapse(roles=roles, messages=messages),\n", @@ -135,21 +81,86 @@ " deserialize=False,\n", " streaming=True,\n", ")\n", - "# Prepare the task for handling stream responses\n", - "from prompting.dendrite import DendriteResponseEvent\n", + "\n", + "stream_results_dict = dict(zip(uids, streams_responses))\n", + "handle_stream_responses_task = asyncio.create_task(handle_response(stream_results_dict, tokenizer))\n", + "stream_results = await handle_stream_responses_task\n", + "\n", + "bt.logging.info(f\"Response:\\n {stream_results[0].synapse.completion}\")" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" + ] + } + ], + "source": [ + "# TODO: Stress test: how many queries\n", + "from prompting.protocol import StreamPromptingSynapse\n", "from prompting.forward import handle_response\n", "from transformers import AutoTokenizer\n", "\n", - "stream_results_dict = dict(zip(uids, streams_responses))\n", + "uids = [my_uid]\n", + "axons = [subnet.axons[uid] for uid in uids]\n", + "\n", + "# Directly call dendrite and process responses in parallel\n", + "dendrite = bt.dendrite(wallet=wallet)\n", + "roles = []\n", + "messages = []\n", + "timeout = 40\n", "model_name = \"casperhansen/llama-3-8b-instruct-awq\"\n", "tokenizer = AutoTokenizer.from_pretrained(model_name)\n", - "handle_stream_responses_task = asyncio.create_task(handle_response(stream_results_dict, tokenizer))\n", - "stream_results = await handle_stream_responses_task\n", - "response_event = DendriteResponseEvent(\n", - " stream_results=stream_results, uids=uids, timeout=timeout\n", - ")\n", + "while True:\n", + " message = input()\n", + " if message == \"exit()\":\n", + " break\n", + " roles.append(\"user\")\n", + " messages.append(message)\n", + " streams_responses = await dendrite(\n", + " axons=axons,\n", + " synapse=StreamPromptingSynapse(roles=roles, messages=messages),\n", + " timeout=timeout,\n", + " deserialize=False,\n", + " streaming=True,\n", + " )\n", + " stream_results_dict = dict(zip(uids, streams_responses))\n", + " handle_stream_responses_task = asyncio.create_task(handle_response(stream_results_dict, tokenizer))\n", + " stream_results = await handle_stream_responses_task\n", + " response = stream_results[0].synapse.completion\n", "\n", - "bt.logging.info(f\"Created DendriteResponseEvent:\\n {response_event}\")" + " print(f\"User: {message}\")\n", + " print(f\"Assistant {response}\")\n", + "\n", + " messages.append(response)\n", + " roles.append(\"assistant\")" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'The capital of Zimbabwe is Harare.'" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "stream_results[0].synapse.completion" ] }, { diff --git a/neurons/miners/huggingface/miner.py b/neurons/miners/huggingface/miner.py index 6c33730d1..5053dce15 100644 --- a/neurons/miners/huggingface/miner.py +++ b/neurons/miners/huggingface/miner.py @@ -16,7 +16,7 @@ # DEALINGS IN THE SOFTWARE. import time import bittensor as bt -from prompting.miners import HuggingFaceMiner +from prompting.miners.hf_miner import HuggingFaceMiner from deprecated import deprecated diff --git a/prompting/base/validator.py b/prompting/base/validator.py index 9416c1ab1..8c0415673 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -15,7 +15,6 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. -from functools import partial import random import sys import copy @@ -23,9 +22,12 @@ import asyncio import argparse import threading +from starlette.types import Send +from functools import partial + import bittensor as bt -from typing import List +from typing import AsyncGenerator, List from traceback import print_exception from prompting.base.neuron import BaseNeuron @@ -36,7 +38,6 @@ from prompting.utils.config import add_validator_args from prompting.utils.exceptions import MaxRetryError from prompting.utils.uids import get_random_uids, get_uids -from prompting.utils.uids import check_uid_availability class BaseValidatorNeuron(BaseNeuron): @@ -71,7 +72,6 @@ def __init__(self, config=None): # Init sync with the network. Updates the metagraph. self.sync() - # Create asyncio event loop to manage async tasks. self.loop = asyncio.get_event_loop() @@ -81,39 +81,34 @@ def __init__(self, config=None): self.thread: threading.Thread = None self.lock = asyncio.Lock() self._organic_dataset = OrganicDataset() - # Serve axon to enable external connections. - # if not self.config.neuron.axon_off: - # self.serve_axon() - # else: - # bt.logging.warning("axon off, not serving ip to chain.") - - # def blacklist_prompt(self, synapse: StreamPromptingSynapse) -> Tuple[bool, str]: - # blacklist = self.base_blacklist(synapse, PROMPT_BLACKLIST_STAKE) - # bt.logging.info(blacklist[1]) - # return blacklist - - async def _miners_organic_query( - self, - synapse: StreamPromptingSynapse, - sampling_mode: str, - exclude: List[int] = [] - ) -> StreamPromptingSynapse: - # Get the list of uids to query for this step. - uids = get_uids(sampling_mode=sampling_mode, k=self.config.neuron.sample_size, exclude=exclude) - streams_responses = query_miners(self, synapse.roles, synapse.messages, uids, self.config.neuron.timeout) - uid_stream_dict = dict(zip(uids, streams_responses)) - random_uid, random_stream = random.choice(list(uid_stream_dict.items())) - return random_stream - # Creates a streamer from the selected stream - # streamer = AsyncResponseDataStreamer(async_iterator=random_stream, selected_uid=random_uid) - # response = await streamer.stream(params.request) - # return response async def _handle_organic(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: - self._organic_dataset.add(synapse) bt.logging.info(f"Organic handle: {synapse}") - response = await _miners_organic_query(self, synapse, "random", []) - return response + + async def _handle_response(responses, send: Send): + for resp in responses: + async for chunk in resp: + if isinstance(chunk, str): + await send( + { + "type": "http.response.body", + "body": chunk.encode("utf-8"), + "more_body": True, + } + ) + bt.logging.info(f"Streamed text: {chunk}") + await send({"type": "http.response.body", "body": b"", "more_body": False}) + + async def _prompt(synapse, send: Send): + uids = get_uids(self, sampling_mode="random", k=self.config.neuron.organic_size, exclude=[]) + bt.logging.info(f"Sending {synapse} request to UIDs: {uids}") + responses = await query_miners(self, synapse.roles, synapse.messages, uids, self.config.neuron.timeout) + return await _handle_response(responses, send) + + token_streamer = partial(_prompt, synapse) + streaming_response = synapse.create_streaming_response(token_streamer) + self._organic_dataset.add({"synapse": synapse, "response": streaming_response}) + return streaming_response def serve_axon(self): """Serve axon to enable external connections""" @@ -170,10 +165,11 @@ def run(self): forward_timeout = self.config.neuron.forward_max_time try: - task = self.loop.create_task(self.forward()) - self.loop.run_until_complete( - asyncio.wait_for(task, timeout=forward_timeout) - ) + pass + # task = self.loop.create_task(self.forward()) + # self.loop.run_until_complete( + # asyncio.wait_for(task, timeout=forward_timeout) + # ) except torch.cuda.OutOfMemoryError as e: bt.logging.error(f"Out of memory error: {e}") continue diff --git a/prompting/forward.py b/prompting/forward.py index 196b3f76f..7f2074406 100644 --- a/prompting/forward.py +++ b/prompting/forward.py @@ -169,6 +169,7 @@ async def query_miners(self, roles: List[str], messages: List[str], uids: torch. streaming=True, ) + async def run_step( self, agent: HumanAgent, roles: List[str], messages: List[str], k: int, timeout: float, exclude: list = None ): @@ -194,7 +195,8 @@ async def run_step( # Get the list of uids to query for this step. uids = get_random_uids(self, k=k, exclude=exclude or []).to(self.device) uids_cpu = uids.cpu().tolist() - streams_responses = query_miners(self, roles, messages, uids, timeout) + # TODO: if organic and response is ready + streams_responses = await query_miners(self, roles, messages, uids, timeout) # uids = get_random_uids(self, k=k, exclude=exclude or []).to(self.device) # uids_cpu = uids.cpu().tolist() @@ -224,6 +226,7 @@ async def run_step( log_stream_results(stream_results) + # TODO: Create separate thread for consuming organic prompts, and return reward. # Encapsulate the responses in a response event (dataclass) response_event = DendriteResponseEvent( stream_results=stream_results, uids=uids, timeout=timeout @@ -232,6 +235,7 @@ async def run_step( bt.logging.info(f"Created DendriteResponseEvent:\n {response_event}") # Reward the responses and get the reward result (dataclass) # This contains a list of RewardEvents but can be exported as a dict (column-wise) for logging etc + bt.logging.info(f"Response from miners: {stream_results}") reward_result = RewardResult( self.reward_pipeline, agent=agent, diff --git a/prompting/llms/hf.py b/prompting/llms/hf.py index d2a095405..8a2da3cc1 100644 --- a/prompting/llms/hf.py +++ b/prompting/llms/hf.py @@ -211,7 +211,7 @@ def _make_prompt(self, messages: List[Dict[str, str]]): # return self.llm_pipeline.tokenizer.apply_chat_template( # messages, tokenize=False, add_generation_prompt=True # ) - return self.llm_pipeline.tokenizer.tokenizer.apply_chat_template( + return self.llm_pipeline.tokenizer.apply_chat_template( messages, tokenize=False, add_generation_prompt=True ) diff --git a/prompting/tools/datasets/organic_dataset.py b/prompting/tools/datasets/organic_dataset.py index 131afde9e..ff0769577 100644 --- a/prompting/tools/datasets/organic_dataset.py +++ b/prompting/tools/datasets/organic_dataset.py @@ -31,7 +31,8 @@ def add(cls, synapse: StreamPromptingSynapse): def random(cls, selector: Optional[Selector]) -> dict[str, Any]: with cls._lock: if cls._queue: - synapse = cls._queue.pop(0) + response = cls._queue.pop(0) + synapse = response["synapse"] organic_source = "organic" else: # TODO: Get synthetic data. diff --git a/prompting/utils/config.py b/prompting/utils/config.py index 3bfb465d8..a5aa4ccdf 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -331,6 +331,13 @@ def add_validator_args(cls, parser): default=100, ) + parser.add_argument( + "--neuron.organic_size", + type=int, + help="The number of miners to organic query in a single step.", + default=5, + ) + parser.add_argument( "--neuron.disable_set_weights", action="store_true", diff --git a/prompting/utils/uids.py b/prompting/utils/uids.py index d3f2694a4..a2c43d1ae 100644 --- a/prompting/utils/uids.py +++ b/prompting/utils/uids.py @@ -3,6 +3,8 @@ import bittensor as bt from typing import List, Union +from prompting.base.neuron import BaseNeuron + def check_uid_availability( metagraph: "bt.metagraph.Metagraph", @@ -43,7 +45,7 @@ def check_uid_availability( return True -def get_random_uids(self, k: int, exclude: List[int] = None) -> torch.LongTensor: +def get_random_uids(self: BaseNeuron, k: int, exclude: List[int] = None) -> torch.LongTensor: """Returns k available random uids from the metagraph. Args: k (int): Number of uids to return. @@ -92,7 +94,7 @@ def get_random_uids(self, k: int, exclude: List[int] = None) -> torch.LongTensor def get_top_incentive_uids(self, k: int, vpermit_tao_limit: int) -> List[int]: - metagraph = self.validator.metagraph + metagraph = self.metagraph miners_uids = list(map(int, filter(lambda uid: check_uid_availability(metagraph, uid, vpermit_tao_limit), metagraph.uids))) @@ -114,11 +116,11 @@ def get_top_incentive_uids(self, k: int, vpermit_tao_limit: int) -> List[int]: return top_k_uids -def get_uids(self, sampling_mode: str, k: int, exclude: List[int] = []) -> Union[list[int], torch.LongTensor]: +def get_uids(self: BaseNeuron, sampling_mode: str, k: int, exclude: List[int] = []) -> Union[list[int], torch.LongTensor]: if sampling_mode == "random": - uids = get_random_uids(self.validator, k=k, exclude=exclude or []).tolist() + uids = get_random_uids(self, k=k, exclude=exclude or []).tolist() return uids if sampling_mode == "top_incentive": - vpermit_tao_limit = self.validator.config.neuron.vpermit_tao_limit + vpermit_tao_limit = self.config.neuron.vpermit_tao_limit top_uids = get_top_incentive_uids(self, k=k, vpermit_tao_limit=vpermit_tao_limit) return top_uids From b4f29fe5c74e9768595e58b18e7750e91adc4ba1 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Wed, 26 Jun 2024 11:05:56 +0000 Subject: [PATCH 07/57] Remove commented code --- prompting/forward.py | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/prompting/forward.py b/prompting/forward.py index 7f2074406..de4bba926 100644 --- a/prompting/forward.py +++ b/prompting/forward.py @@ -56,7 +56,7 @@ async def process_stream(uid: int, async_iterator: Awaitable, tokenizer: Tokeniz accumulated_tokens_per_chunk = [] start_time = time.time() - try: + try: chunk = None async for chunk in async_iterator: # most important loop, as this is where we acquire the final synapse. if isinstance(chunk, str): @@ -197,19 +197,6 @@ async def run_step( uids_cpu = uids.cpu().tolist() # TODO: if organic and response is ready streams_responses = await query_miners(self, roles, messages, uids, timeout) - # uids = get_random_uids(self, k=k, exclude=exclude or []).to(self.device) - # uids_cpu = uids.cpu().tolist() - - # axons = [self.metagraph.axons[uid] for uid in uids] - - # # Directly call dendrite and process responses in parallel - # streams_responses = await self.dendrite( - # axons=axons, - # synapse=StreamPromptingSynapse(roles=roles, messages=messages), - # timeout=timeout, - # deserialize=False, - # streaming=True, - # ) # Prepare the task for handling stream responses stream_results_dict = dict(zip(uids_cpu, streams_responses)) From d56573264f94e0e682590a728c4797cef18bacfc Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Thu, 27 Jun 2024 20:18:51 +0000 Subject: [PATCH 08/57] WIP Organic module refactor --- connecting_vali.ipynb | 12 +- prompting/base/validator.py | 58 +---- prompting/forward.py | 6 +- prompting/organic/__init__.py | 0 .../datasets => organic}/organic_dataset.py | 3 + prompting/{tasks => organic}/organic_task.py | 3 + prompting/organic/organic_weight_setter.py | 205 ++++++++++++++++++ prompting/tasks/__init__.py | 2 +- prompting/tools/datasets/__init__.py | 2 +- 9 files changed, 237 insertions(+), 54 deletions(-) create mode 100644 prompting/organic/__init__.py rename prompting/{tools/datasets => organic}/organic_dataset.py (97%) rename prompting/{tasks => organic}/organic_task.py (97%) create mode 100644 prompting/organic/organic_weight_setter.py diff --git a/connecting_vali.ipynb b/connecting_vali.ipynb index a69ea2223..c2641571f 100644 --- a/connecting_vali.ipynb +++ b/connecting_vali.ipynb @@ -91,7 +91,7 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 21, "metadata": {}, "outputs": [ { @@ -100,6 +100,14 @@ "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "User: Hey my name is John, and I live in Zimbabwe\n", + "Assistant Hello, John! It's nice to meet you. How can I assist you today?\n" + ] } ], "source": [ @@ -120,7 +128,7 @@ "tokenizer = AutoTokenizer.from_pretrained(model_name)\n", "while True:\n", " message = input()\n", - " if message == \"exit()\":\n", + " if message == \"exit\":\n", " break\n", " roles.append(\"user\")\n", " messages.append(message)\n", diff --git a/prompting/base/validator.py b/prompting/base/validator.py index 8c0415673..bb37de4dc 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -22,8 +22,6 @@ import asyncio import argparse import threading -from starlette.types import Send -from functools import partial import bittensor as bt @@ -31,13 +29,11 @@ from traceback import print_exception from prompting.base.neuron import BaseNeuron -from prompting.forward import query_miners from prompting.mock import MockDendrite -from prompting.protocol import StreamPromptingSynapse -from prompting.tools.datasets.organic_dataset import OrganicDataset +from prompting.organic.organic_dataset import OrganicDataset from prompting.utils.config import add_validator_args from prompting.utils.exceptions import MaxRetryError -from prompting.utils.uids import get_random_uids, get_uids +from prompting.organic.organic_weight_setter import OrganicWeightSetter class BaseValidatorNeuron(BaseNeuron): @@ -80,48 +76,16 @@ def __init__(self, config=None): self.is_running: bool = False self.thread: threading.Thread = None self.lock = asyncio.Lock() - self._organic_dataset = OrganicDataset() - - async def _handle_organic(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: - bt.logging.info(f"Organic handle: {synapse}") - - async def _handle_response(responses, send: Send): - for resp in responses: - async for chunk in resp: - if isinstance(chunk, str): - await send( - { - "type": "http.response.body", - "body": chunk.encode("utf-8"), - "more_body": True, - } - ) - bt.logging.info(f"Streamed text: {chunk}") - await send({"type": "http.response.body", "body": b"", "more_body": False}) - - async def _prompt(synapse, send: Send): - uids = get_uids(self, sampling_mode="random", k=self.config.neuron.organic_size, exclude=[]) - bt.logging.info(f"Sending {synapse} request to UIDs: {uids}") - responses = await query_miners(self, synapse.roles, synapse.messages, uids, self.config.neuron.timeout) - return await _handle_response(responses, send) - - token_streamer = partial(_prompt, synapse) - streaming_response = synapse.create_streaming_response(token_streamer) - self._organic_dataset.add({"synapse": synapse, "response": streaming_response}) - return streaming_response - - def serve_axon(self): + self._serve_axon() + self._organic_weight_setter = OrganicWeightSetter(validator=self, axon=self._axon, loop=self.loop) + + def _serve_axon(self): """Serve axon to enable external connections""" - validator_uid = self.metagraph.hotkeys.index(self.wallet.hotkey.ss58_address) + validator_uid = self._val.metagraph.hotkeys.index(self._val.wallet.hotkey.ss58_address) bt.logging.info(f"Serving validator IP of UID {validator_uid} to chain...") - self.axon = bt.axon(wallet=self.wallet, config=self.config) - self.axon.attach( - forward_fn=self._handle_organic, - blacklist_fn=None, - priority_fn=None, - ) - self.axon.serve(netuid=self.config.netuid, subtensor=self.subtensor) - self.axon.start() + self._axon = bt.axon(wallet=self._val.wallet, config=self._val.config) + self._axon.serve(netuid=self._val.config.netuid, subtensor=self._val.subtensor) + self._axon.start() def run(self): """ @@ -146,7 +110,7 @@ def run(self): # Check that validator is registered on the network. self.sync() - self.serve_axon() + self._organic_weight_setter.start_task() if not self.config.neuron.axon_off: bt.logging.info( f"Running validator {self.axon} on network: {self.config.subtensor.chain_endpoint} with netuid: {self.config.netuid}" diff --git a/prompting/forward.py b/prompting/forward.py index de4bba926..259c11e51 100644 --- a/prompting/forward.py +++ b/prompting/forward.py @@ -28,15 +28,15 @@ from prompting.agent import HumanAgent from prompting.dendrite import DendriteResponseEvent, SynapseStreamResult from prompting.conversation import create_task +from prompting.organic import organic_task from prompting.protocol import StreamPromptingSynapse from prompting.rewards import RewardResult -from prompting.tasks import QuestionAnsweringTask, organic_task +from prompting.tasks import QuestionAnsweringTask from prompting.utils.uids import get_random_uids from prompting.utils.logging import log_event from prompting.utils.misc import async_log, serialize_exception_to_string from transformers import PreTrainedTokenizerFast as Tokenizer from prompting.utils.uids import get_random_uids -from dataclasses import dataclass SINGLE_TURN_TASKS = ('sentiment', 'translation', organic_task.TASK_NAME) @@ -74,7 +74,7 @@ async def process_stream(uid: int, async_iterator: Awaitable, tokenizer: Tokeniz raise ValueError( f"Something went wrong with miner uid {uid}, Synapse is not StreamPromptingSynapse." ) - except Exception as e: + except Exception as e: exception = e traceback_details = traceback.format_exc() bt.logging.error( diff --git a/prompting/organic/__init__.py b/prompting/organic/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/prompting/tools/datasets/organic_dataset.py b/prompting/organic/organic_dataset.py similarity index 97% rename from prompting/tools/datasets/organic_dataset.py rename to prompting/organic/organic_dataset.py index ff0769577..1a411ed2c 100644 --- a/prompting/tools/datasets/organic_dataset.py +++ b/prompting/organic/organic_dataset.py @@ -1,3 +1,6 @@ +""" +TODO (dbobrenko): Decouple code from prompting module. +""" from typing import Any, Optional from prompting.protocol import StreamPromptingSynapse diff --git a/prompting/tasks/organic_task.py b/prompting/organic/organic_task.py similarity index 97% rename from prompting/tasks/organic_task.py rename to prompting/organic/organic_task.py index 5da765144..d48d5f91f 100644 --- a/prompting/tasks/organic_task.py +++ b/prompting/organic/organic_task.py @@ -1,3 +1,6 @@ +""" +TODO (dbobrenko): Decouple code from prompting module. +""" import time from typing import Any import bittensor as bt diff --git a/prompting/organic/organic_weight_setter.py b/prompting/organic/organic_weight_setter.py new file mode 100644 index 000000000..f7c32a9a0 --- /dev/null +++ b/prompting/organic/organic_weight_setter.py @@ -0,0 +1,205 @@ +import sys +import time +import bittensor as bt +from prompting.agent import HumanAgent +from prompting.base.neuron import BaseNeuron +from prompting.conversation import create_task +from prompting.dendrite import DendriteResponseEvent +from prompting.organic import organic_task +from prompting.protocol import StreamPromptingSynapse + +import asyncio +from starlette.types import Send +from functools import partial + +import bittensor as bt + +from prompting.forward import handle_response, log_stream_results, query_miners +from prompting.protocol import StreamPromptingSynapse +from prompting.organic.organic_dataset import OrganicDataset +from prompting.rewards.reward import RewardResult +from prompting.utils.logging import log_event +from prompting.utils.uids import get_random_uids, get_uids + + +class OrganicWeightSetter: + def __init__(self, validator: BaseNeuron, axon: bt.axon, loop: asyncio.AbstractEventLoop): + self._val = validator + self._loop = loop + self._axon = axon + self._organic_dataset = OrganicDataset() + + def start_task(self): + self._axon.attach( + forward_fn=self._handle_organic, + blacklist_fn=None, + priority_fn=None, + ) + self.loop.create_task(self._weight_setter()) + + async def _weight_setter(self): + # TODO (dbobrenko): Get rid of HumanAgent dependency. + while True: + timer_start = time.perf_counter() + task_name = organic_task.TASK_NAME + try: + task = create_task( + llm_pipeline=self.llm_pipeline, + translation_pipeline=self.translation_pipeline, + task_name=task_name, + create_reference=False, + ) + except Exception as e: + bt.logging.error(f"Failed to create {task_name} task. {sys.exc_info()}. Skipping to next task.") + continue + agent = HumanAgent(task=task, llm_pipeline=self.llm_pipeline, begin_conversation=True) + # sample = self._organic_dataset.random() + roles = task.roles + messages = task.messages + event = await self._run_step( + self, + agent, + roles=roles, + messages=messages, + k=self.config.neuron.sample_size, + timeout=self.config.neuron.timeout, + exclude=None, + ) + + # Adds forward time to event and logs it to wandb + event["forward_time"] = time.perf_counter() - timer_start + log_event(self, event) + + async def run_step( + self, agent: HumanAgent, roles: list[str], messages: list[str], k: int, timeout: float, exclude: list = None + ): + """Executes a single step of the agent, which consists of: + - Getting a list of uids to query + - Querying the network + - Rewarding the network + - Updating the scores + - Logging the event + + Args: + agent (HumanAgent): The agent to run the step for. + roles (List[str]): The roles for the synapse. + messages (List[str]): The messages for the synapse. + k (int): The number of uids to query. + timeout (float): The timeout for the queries. + exclude (list, optional): The list of uids to exclude from the query. Defaults to []. + """ + bt.logging.debug("run_step", agent.task.name) + + # Record event start time. + start_time = time.time() + # Get the list of uids to query for this step. + uids = get_random_uids(self, k=k, exclude=exclude or []).to(self.device) + uids_cpu = uids.cpu().tolist() + # TODO: if organic and response is ready + streams_responses = await query_miners(self, roles, messages, uids, timeout) + + # Prepare the task for handling stream responses + stream_results_dict = dict(zip(uids_cpu, streams_responses)) + tokenizer = self.llm_pipeline.tokenizer + handle_stream_responses_task = asyncio.create_task(handle_response(stream_results_dict, tokenizer)) + + # if not agent.task.static_reference: + # reference_generation_task = generate_reference(agent) + # _, stream_results = await asyncio.gather( + # reference_generation_task, handle_stream_responses_task + # ) + # else: + stream_results = await handle_stream_responses_task + + log_stream_results(stream_results) + + # TODO: Create separate thread for consuming organic prompts, and return reward. + # Encapsulate the responses in a response event (dataclass) + response_event = DendriteResponseEvent(stream_results=stream_results, uids=uids, timeout=timeout) + + bt.logging.info(f"Created DendriteResponseEvent:\n {response_event}") + # Reward the responses and get the reward result (dataclass) + # This contains a list of RewardEvents but can be exported as a dict (column-wise) for logging etc + bt.logging.info(f"Response from miners: {stream_results}") + reward_result = RewardResult( + self.reward_pipeline, + agent=agent, + response_event=response_event, + device=self.device, + ) + bt.logging.info(f"Created RewardResult:\n {reward_result}") + + best_response = response_event.completions[reward_result.rewards.argmax()] + + # The original idea was that the agent is 'satisfied' when it gets a good enough response (e.g. reward critera is met, such as ROUGE>threshold) + agent.update_progress( + top_reward=reward_result.rewards.max(), + top_response=best_response, + ) + + self.update_scores(reward_result.rewards, uids) + + # Log the step event. + event = { + "best": best_response, + "block": self.block, + "step": self.step, + "step_time": time.time() - start_time, + **agent.__state_dict__(full=self.config.neuron.log_full), + **reward_result.__state_dict__(full=self.config.neuron.log_full), + **response_event.__state_dict__(), + } + + return event + + async def _collect(self): + while True: + try: + if self.organic_scoring_tasks: + completed, _ = await asyncio.wait( + self.organic_scoring_tasks, + timeout=1, + return_when=asyncio.FIRST_COMPLETED + ) + for task in completed: + if task.exception(): + bt.logging.error(f"Error encountered in {OrganicWeightSetter.__name__} task") + else: + success, data = task.result() + if not success: + continue + self.total_scores += data[0] + self.organic_scoring_tasks.difference_update(completed) + else: + await asyncio.sleep(60) + except Exception as e: + bt.logging.error(f"Error encountered in {OrganicWeightSetter.__name__}: {e}") + await asyncio.sleep(10) + + async def _handle_organic(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: + bt.logging.info(f"Organic handle: {synapse}") + + uids = get_uids(self._val, sampling_mode="random", k=self._val.config.neuron.organic_size, exclude=[]) + token_streamer = partial(self._query_miner_uids, synapse, uids) + streaming_response = synapse.create_streaming_response(token_streamer) + self._organic_dataset.add({"synapse": synapse, "response": streaming_response, "uids": uids}) + return streaming_response + + async def _query_miner_uids(self, synapse, uids, send: Send): + bt.logging.info(f"Sending {synapse} request to UIDs: {uids}") + responses = await query_miners(self._val, synapse.roles, synapse.messages, uids, self._val.config.neuron.timeout) + return await self._stream_miner_responses(responses, send) + + async def _stream_miner_responses(self, responses, send: Send): + for resp in responses: + async for chunk in resp: + if isinstance(chunk, str): + await send( + { + "type": "http.response.body", + "body": chunk.encode("utf-8"), + "more_body": True, + } + ) + bt.logging.info(f"Streamed text: {chunk}") + await send({"type": "http.response.body", "body": b"", "more_body": False}) diff --git a/prompting/tasks/__init__.py b/prompting/tasks/__init__.py index 9443480c3..81ff6d87c 100644 --- a/prompting/tasks/__init__.py +++ b/prompting/tasks/__init__.py @@ -8,7 +8,7 @@ from .translate import TranslationTask, TranslationPipeline from .mock import MockTask from .sentiment import SentimentAnalysisTask -from .organic_task import OrganicTask +from ..organic.organic_task import OrganicTask TASKS = { OrganicTask.name: OrganicTask, diff --git a/prompting/tools/datasets/__init__.py b/prompting/tools/datasets/__init__.py index 9e17c71d1..031f55134 100644 --- a/prompting/tools/datasets/__init__.py +++ b/prompting/tools/datasets/__init__.py @@ -5,4 +5,4 @@ from .wiki import WikiDataset, WikiDateDataset from .generic_instruction import GenericInstructionDataset from .review import ReviewDataset -from .organic_dataset import OrganicDataset +from ...organic.organic_dataset import OrganicDataset From 7782d5ddf4be443a1dc2a73eee37001a05880095 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Fri, 28 Jun 2024 09:20:59 +0000 Subject: [PATCH 09/57] WIP-2 Organic thread refactor --- prompting/base/validator.py | 34 ++- prompting/forward.py | 1 + prompting/organic/organic_weight_setter.py | 305 ++++++++++++++++----- prompting/task_registry.py | 2 +- prompting/tasks/__init__.py | 2 +- prompting/tools/__init__.py | 2 +- prompting/utils/config.py | 2 +- 7 files changed, 262 insertions(+), 86 deletions(-) diff --git a/prompting/base/validator.py b/prompting/base/validator.py index bb37de4dc..70ad06017 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -22,6 +22,7 @@ import asyncio import argparse import threading +import concurrent.futures import bittensor as bt @@ -70,22 +71,33 @@ def __init__(self, config=None): # Create asyncio event loop to manage async tasks. self.loop = asyncio.get_event_loop() + self._executor = concurrent.futures.ThreadPoolExecutor() + self._executor.submit(self.start_asyncio_loop, self.loop) # Instantiate runners self.should_exit: bool = False self.is_running: bool = False self.thread: threading.Thread = None self.lock = asyncio.Lock() - self._serve_axon() - self._organic_weight_setter = OrganicWeightSetter(validator=self, axon=self._axon, loop=self.loop) - + # self._serve_axon() + # self._organic_weight_setter = OrganicWeightSetter(validator=self, axon=self.axon, loop=self.loop) + self._organic_weight_setter = OrganicWeightSetter(validator=self, axon=None, loop=self.loop) + self.axon = self._organic_weight_setter._axon + + @staticmethod + def start_asyncio_loop(loop): + try: + loop.run_forever() + except RuntimeError: + pass + def _serve_axon(self): """Serve axon to enable external connections""" - validator_uid = self._val.metagraph.hotkeys.index(self._val.wallet.hotkey.ss58_address) + validator_uid = self.metagraph.hotkeys.index(self.wallet.hotkey.ss58_address) bt.logging.info(f"Serving validator IP of UID {validator_uid} to chain...") - self._axon = bt.axon(wallet=self._val.wallet, config=self._val.config) - self._axon.serve(netuid=self._val.config.netuid, subtensor=self._val.subtensor) - self._axon.start() + self.axon = bt.axon(wallet=self.wallet, config=self.config) + self.axon.serve(netuid=self.config.netuid, subtensor=self.subtensor) + self.axon.start() def run(self): """ @@ -129,7 +141,13 @@ def run(self): forward_timeout = self.config.neuron.forward_max_time try: - pass + # pass + forward_future = asyncio.run_coroutine_threadsafe(self.forward(), self.loop) + try: + forward_future.result(timeout=forward_timeout) + except concurrent.futures.TimeoutError: + print("Forward task timed out") + # task = self.loop.create_task(self.forward()) # self.loop.run_until_complete( # asyncio.wait_for(task, timeout=forward_timeout) diff --git a/prompting/forward.py b/prompting/forward.py index 259c11e51..e037b8e32 100644 --- a/prompting/forward.py +++ b/prompting/forward.py @@ -158,6 +158,7 @@ def log_stream_results(stream_results: List[SynapseStreamResult]): f"Failed response for uid {failed_response.uid}: {formatted_exception}" ) + async def query_miners(self, roles: List[str], messages: List[str], uids: torch.LongTensor, timeout: float): axons = [self.metagraph.axons[uid] for uid in uids] # Directly call dendrite and process responses in parallel diff --git a/prompting/organic/organic_weight_setter.py b/prompting/organic/organic_weight_setter.py index f7c32a9a0..e0448542f 100644 --- a/prompting/organic/organic_weight_setter.py +++ b/prompting/organic/organic_weight_setter.py @@ -1,19 +1,18 @@ import sys import time import bittensor as bt -from prompting.agent import HumanAgent -from prompting.base.neuron import BaseNeuron -from prompting.conversation import create_task -from prompting.dendrite import DendriteResponseEvent -from prompting.organic import organic_task -from prompting.protocol import StreamPromptingSynapse - import asyncio -from starlette.types import Send from functools import partial +from typing import Awaitable, AsyncIterator +from starlette.types import Send import bittensor as bt +from prompting.agent import HumanAgent +from prompting.base.neuron import BaseNeuron +from prompting.dendrite import DendriteResponseEvent +from prompting.organic import organic_task +from prompting.protocol import StreamPromptingSynapse from prompting.forward import handle_response, log_stream_results, query_miners from prompting.protocol import StreamPromptingSynapse from prompting.organic.organic_dataset import OrganicDataset @@ -24,53 +23,83 @@ class OrganicWeightSetter: def __init__(self, validator: BaseNeuron, axon: bt.axon, loop: asyncio.AbstractEventLoop): + """Runs the organic weight setter task in separate threads. + + Creates 3 threads: + - Receiving organic requests through axon. + - Streaming response completions back to the caller through axon. + - Queue to incentivize miner's completions for organic or synthetic organic queries. + + Args: + validator (BaseNeuron): The validator to use. + axon (bt.axon): Served and started axon. + loop (asyncio.AbstractEventLoop): The loop to use. + """ + # TODO (dbobrenko): Decouple HumanAgent dependency. + # TODO (dbobrenko): Decouple OrganicTask dependency. + # TODO (dbobrenko): Decouple OrganicDataset dependency. + # TODO (dbobrenko): Decouple Validator dependecies: llm_pipeline, etc. self._val = validator self._loop = loop self._axon = axon self._organic_dataset = OrganicDataset() - + def start_task(self): + validator_uid = self._val.metagraph.hotkeys.index(self._val.wallet.hotkey.ss58_address) + bt.logging.info(f"Serving validator IP of UID {validator_uid} to chain...") + self._axon = bt.axon(wallet=self._val.wallet, config=self._val.config) self._axon.attach( forward_fn=self._handle_organic, blacklist_fn=None, priority_fn=None, ) - self.loop.create_task(self._weight_setter()) + self._axon.serve(netuid=self._val.config.netuid, subtensor=self._val.subtensor) + self._axon.start() + # try: + # asyncio.run_coroutine_threadsafe(self._weight_setter(), self._loop) + # bt.logging.info("Weight setter task started successfully.") + # except Exception as e: + # bt.logging.error(f"Failed to start weight setter task: {e}") + + # async def _weight_setter(self): + # bt.logging.info("Entered _weight_setter") + # while True: + # bt.logging.info("Weight setter loop iteration") + # await asyncio.sleep(1) # Simplified for testing async def _weight_setter(self): - # TODO (dbobrenko): Get rid of HumanAgent dependency. while True: timer_start = time.perf_counter() task_name = organic_task.TASK_NAME try: - task = create_task( - llm_pipeline=self.llm_pipeline, - translation_pipeline=self.translation_pipeline, - task_name=task_name, - create_reference=False, + task = organic_task.OrganicTask( + llm_pipeline=self._val.llm_pipeline, + context=self._organic_dataset.next(), + create_reference=True, ) except Exception as e: - bt.logging.error(f"Failed to create {task_name} task. {sys.exc_info()}. Skipping to next task.") + bt.logging.error(f"Failed to create {task_name} task. {sys.exc_info()}.") + await asyncio.sleep(1) continue - agent = HumanAgent(task=task, llm_pipeline=self.llm_pipeline, begin_conversation=True) + agent = HumanAgent(task=task, llm_pipeline=self._val.llm_pipeline, begin_conversation=True) # sample = self._organic_dataset.random() roles = task.roles messages = task.messages event = await self._run_step( - self, agent, roles=roles, messages=messages, - k=self.config.neuron.sample_size, - timeout=self.config.neuron.timeout, + k=self._val.config.neuron.sample_size, + timeout=self._val.config.neuron.timeout, exclude=None, ) - # Adds forward time to event and logs it to wandb + # Adds forward time to event and logs it to wandb. event["forward_time"] = time.perf_counter() - timer_start - log_event(self, event) + log_event(self._val, event) + await asyncio.sleep(1) - async def run_step( + async def _run_step( self, agent: HumanAgent, roles: list[str], messages: list[str], k: int, timeout: float, exclude: list = None ): """Executes a single step of the agent, which consists of: @@ -91,25 +120,17 @@ async def run_step( bt.logging.debug("run_step", agent.task.name) # Record event start time. - start_time = time.time() + start_time = time.perf_counter() # Get the list of uids to query for this step. - uids = get_random_uids(self, k=k, exclude=exclude or []).to(self.device) + uids = get_random_uids(self._val, k=k, exclude=exclude or []).to(self._val.device) uids_cpu = uids.cpu().tolist() # TODO: if organic and response is ready - streams_responses = await query_miners(self, roles, messages, uids, timeout) + streams_responses = await query_miners(self._val, roles, messages, uids, timeout) # Prepare the task for handling stream responses stream_results_dict = dict(zip(uids_cpu, streams_responses)) - tokenizer = self.llm_pipeline.tokenizer - handle_stream_responses_task = asyncio.create_task(handle_response(stream_results_dict, tokenizer)) - - # if not agent.task.static_reference: - # reference_generation_task = generate_reference(agent) - # _, stream_results = await asyncio.gather( - # reference_generation_task, handle_stream_responses_task - # ) - # else: - stream_results = await handle_stream_responses_task + tokenizer = self._val.llm_pipeline.tokenizer + stream_results = await handle_response(stream_results_dict, tokenizer) log_stream_results(stream_results) @@ -122,75 +143,55 @@ async def run_step( # This contains a list of RewardEvents but can be exported as a dict (column-wise) for logging etc bt.logging.info(f"Response from miners: {stream_results}") reward_result = RewardResult( - self.reward_pipeline, + self._val.reward_pipeline, agent=agent, response_event=response_event, - device=self.device, + device=self._val.device, ) bt.logging.info(f"Created RewardResult:\n {reward_result}") best_response = response_event.completions[reward_result.rewards.argmax()] - # The original idea was that the agent is 'satisfied' when it gets a good enough response (e.g. reward critera is met, such as ROUGE>threshold) + # The original idea was that the agent is 'satisfied' when it gets a good enough response + # (e.g. reward critera is met, such as ROUGE>threshold) agent.update_progress( top_reward=reward_result.rewards.max(), top_response=best_response, ) - self.update_scores(reward_result.rewards, uids) + self._val.update_scores(reward_result.rewards, uids) # Log the step event. event = { "best": best_response, - "block": self.block, - "step": self.step, - "step_time": time.time() - start_time, - **agent.__state_dict__(full=self.config.neuron.log_full), - **reward_result.__state_dict__(full=self.config.neuron.log_full), + "block": self._val.block, + "step": self._val.step, + "step_time": time.perf_counter() - start_time, + **agent.__state_dict__(full=self._val.config.neuron.log_full), + **reward_result.__state_dict__(full=self._val.config.neuron.log_full), **response_event.__state_dict__(), } return event - async def _collect(self): - while True: - try: - if self.organic_scoring_tasks: - completed, _ = await asyncio.wait( - self.organic_scoring_tasks, - timeout=1, - return_when=asyncio.FIRST_COMPLETED - ) - for task in completed: - if task.exception(): - bt.logging.error(f"Error encountered in {OrganicWeightSetter.__name__} task") - else: - success, data = task.result() - if not success: - continue - self.total_scores += data[0] - self.organic_scoring_tasks.difference_update(completed) - else: - await asyncio.sleep(60) - except Exception as e: - bt.logging.error(f"Error encountered in {OrganicWeightSetter.__name__}: {e}") - await asyncio.sleep(10) - async def _handle_organic(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: - bt.logging.info(f"Organic handle: {synapse}") + async def token_streamer(send: Send) -> Awaitable[None]: + return await self._query_miner_uids(synapse, uids, send) + # token_streamer = partial(self._query_miner_uids, synapse, uids) + + bt.logging.info(f"Organic handle: {synapse}") uids = get_uids(self._val, sampling_mode="random", k=self._val.config.neuron.organic_size, exclude=[]) - token_streamer = partial(self._query_miner_uids, synapse, uids) streaming_response = synapse.create_streaming_response(token_streamer) self._organic_dataset.add({"synapse": synapse, "response": streaming_response, "uids": uids}) return streaming_response - async def _query_miner_uids(self, synapse, uids, send: Send): + async def _query_miner_uids(self, synapse: StreamPromptingSynapse, uids, send: Send): bt.logging.info(f"Sending {synapse} request to UIDs: {uids}") responses = await query_miners(self._val, synapse.roles, synapse.messages, uids, self._val.config.neuron.timeout) return await self._stream_miner_responses(responses, send) - async def _stream_miner_responses(self, responses, send: Send): + async def _stream_miner_responses(self, responses: AsyncIterator, send: Send): for resp in responses: async for chunk in resp: if isinstance(chunk, str): @@ -203,3 +204,159 @@ async def _stream_miner_responses(self, responses, send: Send): ) bt.logging.info(f"Streamed text: {chunk}") await send({"type": "http.response.body", "body": b"", "more_body": False}) + + async def _handle_organic1(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: + async def _forward( + self, + synapse: StreamPromptingSynapse, + uids, + init_time: float, + timeout_threshold: float, + send: Send, + ): + buffer = [] + accumulated_chunks = [] + accumulated_chunks_timings = [] + messages = [] + temp_completion = "" # for wandb logging + timeout_reached = False + + try: + start_time = time.time() + + responses = await query_miners( + self._val, + synapse.roles, + synapse.messages, + uids, + self._val.config.neuron.timeout + ) + # system_prompt_message = [{"role": "system", "content": self.system_prompt}] + # synapse_messages = [{"role": role, "content": message} + # for role, message in zip(synapse.roles, synapse.messages)] + + # messages = system_prompt_message + synapse_messages + + # stream_response = self.model.chat.completions.create( + # model=self.config.neuron.model_id, + # messages=messages, + # temperature=self.config.neuron.temperature, + # max_tokens=self.config.neuron.max_tokens, + # stream=True + # ) + + async for chunk in responses: + if isinstance(chunk, list): + concatenated_chunks = "".join(chunk) + await send( + { + "type": "http.response.body", + "body": concatenated_chunks.encode("utf-8"), + "more_body": True, + } + ) + bt.logging.info(f"Streamed text: {chunk}") + if isinstance(chunk, str): + await send( + { + "type": "http.response.body", + "body": chunk.encode("utf-8"), + "more_body": True, + } + ) + bt.logging.info(f"Streamed text: {chunk}") + if chunk is not None and isinstance(chunk, StreamPromptingSynapse): + if len(self.accumulated_chunks) == 0: + self.accumulated_chunks.append(chunk.completion) + self.accumulated_chunks_timings.append(time.time() - start_time) + + self.finish_reason = "completed" + self.sequence_number += 1 + # Assuming the last chunk holds the last value yielded which should be a synapse with the completion filled + synapse = chunk + + await send( + { + "type": "http.response.body", + "body": synapse.completion.encode("utf-8"), + "more_body": True, + } + ) + + # chunk_content = chunk.choices[0].delta.content + # if chunk_content is None: + # bt.logging.info("OpenAI returned chunk content with None") + # continue + + # accumulated_chunks.append(chunk_content) + # accumulated_chunks_timings.append(time.time() - start_time) + + # buffer.append(chunk_content) + + # if time.time() - init_time > timeout_threshold: + # bt.logging.debug(f"⏰ Timeout reached, stopping streaming") + # timeout_reached = True + # break + + # if len(buffer) == self._val.config.neuron.streaming_batch_size: + # joined_buffer = "".join(buffer) + # temp_completion += joined_buffer + # bt.logging.debug(f"Streamed tokens: {joined_buffer}") + + # await send( + # { + # "type": "http.response.body", + # "body": joined_buffer.encode("utf-8"), + # "more_body": True, + # } + # ) + # buffer = [] + + # if ( + # buffer and not timeout_reached + # ): # Don't send the last buffer of data if timeout. + # joined_buffer = "".join(buffer) + # await send( + # { + # "type": "http.response.body", + # "body": joined_buffer.encode("utf-8"), + # "more_body": False, + # } + # ) + + except Exception as e: + bt.logging.error(f"Error in forward: {e}") + # # bt.logging.error(print_exception(type(e), e, e.__traceback__)) + # if self._val.config.neuron.stop_on_forward_exception: + # self.should_exit = True + + finally: + synapse_latency = time.time() - init_time + # # if self.config.wandb.on: + # # self.log_event( + # # synapse=synapse, + # # timing=synapse_latency, + # # messages=messages, + # # accumulated_chunks=accumulated_chunks, + # # accumulated_chunks_timings = accumulated_chunks_timings, + # # ) + + bt.logging.debug(f"📧 Message received from {synapse.dendrite.hotkey}, IP: {synapse.dendrite.ip}; \nForwarding synapse: {synapse}") + + init_time = time.time() + timeout_threshold = synapse.timeout + + uids = get_uids(self._val, sampling_mode="random", k=self._val.config.neuron.organic_size, exclude=[]) + token_streamer = partial( + _forward, + self, + synapse, + uids, + init_time, + timeout_threshold, + ) + + response = synapse.create_streaming_response(token_streamer) + + self._organic_dataset.add({"synapse": synapse, "response": response, "uids": uids}) + return response diff --git a/prompting/task_registry.py b/prompting/task_registry.py index 41e7c4e2d..6fd02d81a 100644 --- a/prompting/task_registry.py +++ b/prompting/task_registry.py @@ -35,7 +35,7 @@ sentiment_analysis_task, sentiment_analysis_dataset = SentimentAnalysisTask.name, [ReviewDataset.name] TASK_REGISTRY = { - organic_task: organic_dataset, + # organic_task: organic_dataset, summarization_task: summarization_dataset, qa_task: qa_dataset, #debugging_task: debugging_dataset, diff --git a/prompting/tasks/__init__.py b/prompting/tasks/__init__.py index 81ff6d87c..0de8c8c11 100644 --- a/prompting/tasks/__init__.py +++ b/prompting/tasks/__init__.py @@ -11,7 +11,7 @@ from ..organic.organic_task import OrganicTask TASKS = { - OrganicTask.name: OrganicTask, + # OrganicTask.name: OrganicTask, QuestionAnsweringTask.name: QuestionAnsweringTask, DateQuestionAnsweringTask.name: DateQuestionAnsweringTask, SummarizationTask.name: SummarizationTask, diff --git a/prompting/tools/__init__.py b/prompting/tools/__init__.py index 151f5ef8a..758ecd698 100644 --- a/prompting/tools/__init__.py +++ b/prompting/tools/__init__.py @@ -14,7 +14,7 @@ DATASETS = { #HFCodingDataset.name: HFCodingDataset, - OrganicDataset.name: OrganicDataset, + # OrganicDataset.name: OrganicDataset, WikiDataset.name: WikiDataset, #StackOverflowDataset.name: StackOverflowDataset, MathDataset.name: MathDataset, diff --git a/prompting/utils/config.py b/prompting/utils/config.py index a5aa4ccdf..4ee538e73 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -300,7 +300,7 @@ def add_validator_args(cls, parser): type=float, nargs="+", help="The probability of sampling each task.", - default=[0.7] + [0.3 / (len(TASKS) - 1)] * (len(TASKS) - 1), + default=[1 / len(TASKS)] * len(TASKS), ) parser.add_argument( From ec511321fb27b99a6b57e25cf90b2b62e22771b0 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Thu, 4 Jul 2024 18:39:29 +0000 Subject: [PATCH 10/57] WIP concurrent streams --- connecting_vali.ipynb | 277 +++++++++++++++------ prompting/base/validator.py | 13 +- prompting/forward.py | 55 +++- prompting/miners/openai_miner.py | 10 +- prompting/organic/organic_weight_setter.py | 211 +++++----------- prompting/utils/config.py | 5 +- 6 files changed, 337 insertions(+), 234 deletions(-) diff --git a/connecting_vali.ipynb b/connecting_vali.ipynb index c2641571f..7a7610a32 100644 --- a/connecting_vali.ipynb +++ b/connecting_vali.ipynb @@ -2,14 +2,26 @@ "cells": [ { "cell_type": "code", - "execution_count": 2, + "execution_count": 14, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:bittensor: - Connected to test network and wss://test.finney.opentensor.ai:443/. - \n", + "INFO:bittensor: - Connected to test network and wss://test.finney.opentensor.ai:443/. - \n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ - "Hotkey 5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr is registered. UID: 0\n" + "Hotkey 5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr is registered. UID: 0\n", + "Active UIDs (total: 2): [0, 1]\n", + "My hotkey is active: True\n", + "Is val permit: True\n", + "1000+ TAO:\n" ] } ], @@ -17,11 +29,15 @@ "# https://docs.bittensor.com/subnets/register-validate-mine#check-the-permit-status\n", "import bittensor as bt\n", "import asyncio\n", + "from prompting.protocol import StreamPromptingSynapse\n", + "from prompting.forward import handle_response\n", + "from transformers import AutoTokenizer\n", "\n", "# NET_UID = 61\n", "NET_UID = 170\n", "NETWORK = \"test\"\n", "WALLET = \"validator\"\n", + "# WALLET = \"miner\"\n", "wallet = bt.wallet(name=WALLET)\n", "subtensor = bt.subtensor(network=NETWORK)\n", "subnet = subtensor.metagraph(netuid=NET_UID)\n", @@ -43,7 +59,55 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", + "INFO:bittensor: - Response:\n", + " - \n" + ] + } + ], + "source": [ + "uids = [my_uid]\n", + "axons = [subnet.axons[uid] for uid in uids]\n", + "\n", + "# Directly call dendrite and process responses in parallel\n", + "dendrite = bt.dendrite(wallet=wallet)\n", + "roles = [\"user\"]\n", + "messages = [\"Organic prompt query. Question: Capital of Zimbabwe?\"]\n", + "timeout = 40\n", + "model_name = \"casperhansen/llama-3-8b-instruct-awq\"\n", + "tokenizer = AutoTokenizer.from_pretrained(model_name)\n", + "\n", + "streams_responses = await dendrite(\n", + " axons=axons,\n", + " synapse=StreamPromptingSynapse(roles=roles, messages=messages),\n", + " timeout=timeout,\n", + " deserialize=False,\n", + " streaming=True,\n", + ")\n", + "stream_results_dict = dict(zip(uids, streams_responses))\n", + "handle_stream_responses_task = asyncio.create_task(handle_response(stream_results_dict, tokenizer))\n", + "stream_results = await handle_stream_responses_task\n", + "\n", + "bt.logging.info(f\"Response:\\n {stream_results[0].synapse.completion}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "code", + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -52,16 +116,13 @@ "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", "INFO:bittensor: - Response:\n", - " [SynapseStreamResult(exception=None, uid=0, accumulated_chunks=['The capital of Zimbabwe is Harare.'], accumulated_chunks_timings=[6.252922296524048], tokens_per_chunk=[8], synapse=StreamPromptingSynapse(required_hash_fields=['messages'], roles=['user'], messages=['Organic prompt query. Question: Capital of Zimbabwe?'], completion='The capital of Zimbabwe is Harare.'))] - \n" + " - \n" ] } ], "source": [ "# Sample validators.\n", "# TODO: Stress test: how many queries\n", - "from prompting.protocol import StreamPromptingSynapse\n", - "from prompting.forward import handle_response\n", - "from transformers import AutoTokenizer\n", "\n", "uids = [my_uid]\n", "axons = [subnet.axons[uid] for uid in uids]\n", @@ -91,7 +152,69 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 63, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "ERROR:bittensor: - Error in generating reference or handling responses for uid 0: Something went wrong with miner uid 0, Synapse is not StreamPromptingSynapse.\n", + "Traceback (most recent call last):\n", + " File \"/workspace/prompting-organic/prompting/forward.py\", line 74, in process_stream\n", + " raise ValueError(\n", + "ValueError: Something went wrong with miner uid 0, Synapse is not StreamPromptingSynapse.\n", + " - \n" + ] + } + ], + "source": [ + "a = await asyncio.create_task(handle_response(stream_results_dict, tokenizer))" + ] + }, + { + "cell_type": "code", + "execution_count": 64, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[SynapseStreamResult(exception=ValueError('Something went wrong with miner uid 0, Synapse is not StreamPromptingSynapse.'), uid=0, accumulated_chunks=[], accumulated_chunks_timings=[], tokens_per_chunk=[], synapse=StreamPromptingSynapse(required_hash_fields=['messages'], roles=['user'], messages=['failure'], completion=''))]" + ] + }, + "execution_count": 64, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "a" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[SynapseStreamResult(exception=None, uid=0, accumulated_chunks=[], accumulated_chunks_timings=[], tokens_per_chunk=[], synapse=StreamPromptingSynapse(required_hash_fields=['messages'], roles=['user'], messages=['Organic prompt query. Question: Capital of Zimbabwe?'], completion=''))]" + ] + }, + "execution_count": 62, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "stream_results" + ] + }, + { + "cell_type": "code", + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -105,8 +228,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "User: Hey my name is John, and I live in Zimbabwe\n", - "Assistant Hello, John! It's nice to meet you. How can I assist you today?\n" + "User: Hey Im John\n", + "Assistant: Hi John! How can I assist you today?\n" ] } ], @@ -140,12 +263,11 @@ " streaming=True,\n", " )\n", " stream_results_dict = dict(zip(uids, streams_responses))\n", - " handle_stream_responses_task = asyncio.create_task(handle_response(stream_results_dict, tokenizer))\n", - " stream_results = await handle_stream_responses_task\n", + " stream_results = await asyncio.create_task(handle_response(stream_results_dict, tokenizer))\n", " response = stream_results[0].synapse.completion\n", "\n", " print(f\"User: {message}\")\n", - " print(f\"Assistant {response}\")\n", + " print(f\"Assistant: {response}\")\n", "\n", " messages.append(response)\n", " roles.append(\"assistant\")" @@ -153,83 +275,98 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "'The capital of Zimbabwe is Harare.'" - ] - }, - "execution_count": 13, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "stream_results[0].synapse.completion" ] }, { "cell_type": "code", - "execution_count": 53, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[SynapseStreamResult(exception=UnboundLocalError(\"local variable 'chunk' referenced before assignment\"), uid=166, accumulated_chunks=[], accumulated_chunks_timings=[], tokens_per_chunk=[], synapse=StreamPromptingSynapse(required_hash_fields=['messages'], roles=['user'], messages=['failure'], completion=''))]" - ] - }, - "execution_count": 53, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "stream_results" + "import asyncio\n", + "import concurrent.futures\n", + "import time\n", + "\n", + "async def val():\n", + " print(\"Val !\")\n", + " await asyncio.sleep(5)\n", + "\n", + "async def forward():\n", + " while True:\n", + " print(\"Forward\")\n", + " await asyncio.sleep(1)\n", + "\n", + "async def background_task():\n", + " while True:\n", + " print(\"Background\")\n", + " await val()\n", + " await asyncio.sleep(1)\n", + "\n", + "loop = asyncio.get_event_loop()\n", + "try:\n", + " loop.run_forever()\n", + "except RuntimeError:\n", + " pass\n", + "\n", + "task = loop.create_task(forward())\n", + "task2 = loop.create_task(background_task())\n", + "try:\n", + " await asyncio.wait_for(task, timeout=5)\n", + "except asyncio.TimeoutError:\n", + " print(\"Forward task timed out\")" ] }, { "cell_type": "code", - "execution_count": 44, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "DendriteResponseEvent(uids=[166], completions=[''], timings=[0], status_messages=['Service at 194.68.245.74:8091/StreamPromptingSynapse unavailable.'], status_codes=[503])" - ] - }, - "execution_count": 44, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "response_event" + "import asyncio\n", + "import concurrent.futures\n", + "import time\n", + "\n", + "# Asynchronous background task\n", + "async def background_task():\n", + " while True:\n", + " print(\"Background task is running\")\n", + " await asyncio.sleep(1)\n", + "\n", + "# Function to run the asyncio event loop in a separate thread\n", + "def start_asyncio_loop(loop):\n", + " asyncio.set_event_loop(loop)\n", + " loop.run_forever()\n", + "\n", + "def main():\n", + " # Create a new asyncio event loop\n", + " loop = asyncio.new_event_loop()\n", + "\n", + " # Start the event loop in a new thread\n", + " executor = concurrent.futures.ThreadPoolExecutor()\n", + " executor.submit(start_asyncio_loop, loop)\n", + "\n", + " # Schedule the asynchronous background task\n", + " asyncio.run_coroutine_threadsafe(background_task(), loop)\n", + "\n", + " # Synchronous while True loop in main\n", + " while True:\n", + " print(\"Main loop is running\")\n", + " time.sleep(1)\n", + "\n", + "main()" ] }, { "cell_type": "code", - "execution_count": 35, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[]" - ] - }, - "execution_count": 35, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "streams_responses" - ] + "outputs": [], + "source": [] } ], "metadata": { diff --git a/prompting/base/validator.py b/prompting/base/validator.py index 70ad06017..e2cb9006e 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -79,10 +79,8 @@ def __init__(self, config=None): self.is_running: bool = False self.thread: threading.Thread = None self.lock = asyncio.Lock() - # self._serve_axon() - # self._organic_weight_setter = OrganicWeightSetter(validator=self, axon=self.axon, loop=self.loop) - self._organic_weight_setter = OrganicWeightSetter(validator=self, axon=None, loop=self.loop) - self.axon = self._organic_weight_setter._axon + self._serve_axon() + self._organic_weight_setter = OrganicWeightSetter(validator=self, axon=self.axon, loop=self.loop) @staticmethod def start_asyncio_loop(loop): @@ -101,7 +99,7 @@ def _serve_axon(self): def run(self): """ - Initiates and manages the main loop for the miner on the Bittensor network. The main loop handles graceful shutdown on keyboard interrupts and logs unforeseen errors. + Initiates and manages the main lop for the miner on the Bittensor network. The main loop handles graceful shutdown on keyboard interrupts and logs unforeseen errors. This function performs the following primary tasks: 1. Check for registration on the Bittensor network. @@ -141,17 +139,12 @@ def run(self): forward_timeout = self.config.neuron.forward_max_time try: - # pass forward_future = asyncio.run_coroutine_threadsafe(self.forward(), self.loop) try: forward_future.result(timeout=forward_timeout) except concurrent.futures.TimeoutError: print("Forward task timed out") - # task = self.loop.create_task(self.forward()) - # self.loop.run_until_complete( - # asyncio.wait_for(task, timeout=forward_timeout) - # ) except torch.cuda.OutOfMemoryError as e: bt.logging.error(f"Out of memory error: {e}") continue diff --git a/prompting/forward.py b/prompting/forward.py index e037b8e32..e6b586547 100644 --- a/prompting/forward.py +++ b/prompting/forward.py @@ -159,6 +159,45 @@ def log_stream_results(stream_results: List[SynapseStreamResult]): ) +class QueryMinersManager: + _instance = None + + def __new__(cls, validator): + if cls._instance is None: + cls._instance = super(QueryMinersManager, cls).__new__(cls) + # Initialize attributes only once + cls._instance.current_priority = 0 # 0 for low, 1 for high + cls._instance.event = asyncio.Event() + cls._instance.metagraph = validator.metagraph + cls._instance.dendrite = validator.dendrite + cls._val = validator + cls._instance.lock = asyncio.Lock() + return cls._instance + + async def query_miners(self, priority, roles, messages, uids, timeout): + async with self.lock: + if priority > self.current_priority: + self.current_priority = priority + self.event.set() # Signal that a higher-priority task is starting + + # Prepare the axons and call the dendrite + axons = [self.metagraph.axons[uid] for uid in uids] + streams_responses = await self._val.dendrite( + axons=axons, + synapse=StreamPromptingSynapse(roles=roles, messages=messages), + timeout=timeout, + deserialize=False, + streaming=True, + ) + bt.logging.info(f"Querying miners with priority={priority}") + + if self.event.is_set() and priority == self.current_priority: + # Clear the event after processing the higher-priority task + self.event.clear() + + return streams_responses + + async def query_miners(self, roles: List[str], messages: List[str], uids: torch.LongTensor, timeout: float): axons = [self.metagraph.axons[uid] for uid in uids] # Directly call dendrite and process responses in parallel @@ -197,7 +236,8 @@ async def run_step( uids = get_random_uids(self, k=k, exclude=exclude or []).to(self.device) uids_cpu = uids.cpu().tolist() # TODO: if organic and response is ready - streams_responses = await query_miners(self, roles, messages, uids, timeout) + streams_responses = await QueryMinersManager(validator=self).query_miners(0, roles, messages, uids, timeout) + # streams_responses = await query_miners(self, roles, messages, uids, timeout) # Prepare the task for handling stream responses stream_results_dict = dict(zip(uids_cpu, streams_responses)) @@ -301,12 +341,13 @@ async def forward(self): messages = task.messages else: # Benchmarking tasks. - roles = ['user'] + roles = ["user"] messages = [agent.challenge] while True: # Note: The try catch is a safe clause to ensure that the forward loop continues even if an error occurs in run_step. # To be reconsidered in the next version. try: + continue # when run_step is called, the agent updates its progress event = await run_step( self, @@ -323,7 +364,7 @@ async def forward(self): event["turn"] = turn log_event(self, event) task.complete = True - + accepted_answer = event["best"] if random.random() < 0.5 else agent.task.reference roles.append("assistant") messages.append(accepted_answer) @@ -350,13 +391,17 @@ async def forward(self): except BaseException as e: unexpected_errors = serialize_exception_to_string(e) bt.logging.error( - f"Error in run_step: Skipping to next round. \n {unexpected_errors}" + f"Error in run_step: Skipping to next round.\n" + f"Task: {task_name}\nMessages: {messages}\nRoles: {roles}\nTurn: {turn}.\n" + f"{unexpected_errors}\n" ) event = {"unexpected_errors": unexpected_errors} log_event(self, event) - continue + await asyncio.sleep(1) + continue + await asyncio.sleep(1) del agent del task diff --git a/prompting/miners/openai_miner.py b/prompting/miners/openai_miner.py index 9afe0ea2d..047f256d8 100644 --- a/prompting/miners/openai_miner.py +++ b/prompting/miners/openai_miner.py @@ -70,8 +70,11 @@ def __init__(self, config=None): self.accumulated_prompt_tokens = 0 self.accumulated_completion_tokens = 0 self.accumulated_total_cost = 0 + bt.logging.info("STARTING MINER") def forward(self, synapse: StreamPromptingSynapse) -> Awaitable: + bt.logging.info(f"Received request with {synapse.messages}") + async def _forward( self, synapse: StreamPromptingSynapse, @@ -87,7 +90,7 @@ async def _forward( timeout_reached = False - try: + try: system_prompt_message = [{ 'role': 'system', 'content': self.system_prompt }] synapse_messages = [{'role': role, 'content': message} for role, message in zip(synapse.roles, synapse.messages)] @@ -106,7 +109,7 @@ async def _forward( chunk_content = chunk.choices[0].delta.content if chunk_content is None: - bt.logging.info("OpenAI returned chunk content with None") + # bt.logging.info("OpenAI returned chunk content with None") continue accumulated_chunks.append(chunk_content) @@ -119,8 +122,10 @@ async def _forward( timeout_reached = True break + # bt.logging.info(f"Streaming back request {synapse.messages} -- {buffer} / {self.config.neuron.streaming_batch_size}") if len(buffer) == self.config.neuron.streaming_batch_size: joined_buffer = "".join(buffer) + bt.logging.info(f"Streaming back request {synapse.messages} -- {joined_buffer}") temp_completion += joined_buffer bt.logging.debug(f"Streamed tokens: {joined_buffer}") @@ -137,6 +142,7 @@ async def _forward( buffer and not timeout_reached ): # Don't send the last buffer of data if timeout. joined_buffer = "".join(buffer) + bt.logging.info(f"On buffer end {synapse.messages} -- {joined_buffer}") await send( { "type": "http.response.body", diff --git a/prompting/organic/organic_weight_setter.py b/prompting/organic/organic_weight_setter.py index e0448542f..a1067325a 100644 --- a/prompting/organic/organic_weight_setter.py +++ b/prompting/organic/organic_weight_setter.py @@ -13,8 +13,7 @@ from prompting.dendrite import DendriteResponseEvent from prompting.organic import organic_task from prompting.protocol import StreamPromptingSynapse -from prompting.forward import handle_response, log_stream_results, query_miners -from prompting.protocol import StreamPromptingSynapse +from prompting.forward import QueryMinersManager, handle_response, log_stream_results, query_miners from prompting.organic.organic_dataset import OrganicDataset from prompting.rewards.reward import RewardResult from prompting.utils.logging import log_event @@ -45,27 +44,21 @@ def __init__(self, validator: BaseNeuron, axon: bt.axon, loop: asyncio.AbstractE self._organic_dataset = OrganicDataset() def start_task(self): - validator_uid = self._val.metagraph.hotkeys.index(self._val.wallet.hotkey.ss58_address) - bt.logging.info(f"Serving validator IP of UID {validator_uid} to chain...") - self._axon = bt.axon(wallet=self._val.wallet, config=self._val.config) + # validator_uid = self._val.metagraph.hotkeys.index(self._val.wallet.hotkey.ss58_address) + # bt.logging.info(f"Serving validator IP of UID {validator_uid} to chain...") + # self._axon = bt.axon(wallet=self._val.wallet, config=self._val.config) self._axon.attach( forward_fn=self._handle_organic, blacklist_fn=None, priority_fn=None, ) - self._axon.serve(netuid=self._val.config.netuid, subtensor=self._val.subtensor) - self._axon.start() - # try: - # asyncio.run_coroutine_threadsafe(self._weight_setter(), self._loop) - # bt.logging.info("Weight setter task started successfully.") - # except Exception as e: - # bt.logging.error(f"Failed to start weight setter task: {e}") - - # async def _weight_setter(self): - # bt.logging.info("Entered _weight_setter") - # while True: - # bt.logging.info("Weight setter loop iteration") - # await asyncio.sleep(1) # Simplified for testing + # self._axon.serve(netuid=self._val.config.netuid, subtensor=self._val.subtensor) + # self._axon.start() + try: + asyncio.run_coroutine_threadsafe(self._weight_setter(), self._loop) + bt.logging.info("Weight setter task started successfully.") + except Exception as e: + bt.logging.error(f"Failed to start weight setter task: {e}") async def _weight_setter(self): while True: @@ -125,7 +118,14 @@ async def _run_step( uids = get_random_uids(self._val, k=k, exclude=exclude or []).to(self._val.device) uids_cpu = uids.cpu().tolist() # TODO: if organic and response is ready - streams_responses = await query_miners(self._val, roles, messages, uids, timeout) + # streams_responses = await query_miners(self._val, roles, messages, uids, timeout) + streams_responses = await QueryMinersManager(validator=self._val).query_miners( + 0, + roles, + messages, + uids, + self._val.config.neuron.timeout + ) # Prepare the task for handling stream responses stream_results_dict = dict(zip(uids_cpu, streams_responses)) @@ -134,7 +134,6 @@ async def _run_step( log_stream_results(stream_results) - # TODO: Create separate thread for consuming organic prompts, and return reward. # Encapsulate the responses in a response event (dataclass) response_event = DendriteResponseEvent(stream_results=stream_results, uids=uids, timeout=timeout) @@ -171,41 +170,9 @@ async def _run_step( **reward_result.__state_dict__(full=self._val.config.neuron.log_full), **response_event.__state_dict__(), } - return event async def _handle_organic(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: - async def token_streamer(send: Send) -> Awaitable[None]: - return await self._query_miner_uids(synapse, uids, send) - - # token_streamer = partial(self._query_miner_uids, synapse, uids) - - bt.logging.info(f"Organic handle: {synapse}") - uids = get_uids(self._val, sampling_mode="random", k=self._val.config.neuron.organic_size, exclude=[]) - streaming_response = synapse.create_streaming_response(token_streamer) - self._organic_dataset.add({"synapse": synapse, "response": streaming_response, "uids": uids}) - return streaming_response - - async def _query_miner_uids(self, synapse: StreamPromptingSynapse, uids, send: Send): - bt.logging.info(f"Sending {synapse} request to UIDs: {uids}") - responses = await query_miners(self._val, synapse.roles, synapse.messages, uids, self._val.config.neuron.timeout) - return await self._stream_miner_responses(responses, send) - - async def _stream_miner_responses(self, responses: AsyncIterator, send: Send): - for resp in responses: - async for chunk in resp: - if isinstance(chunk, str): - await send( - { - "type": "http.response.body", - "body": chunk.encode("utf-8"), - "more_body": True, - } - ) - bt.logging.info(f"Streamed text: {chunk}") - await send({"type": "http.response.body", "body": b"", "more_body": False}) - - async def _handle_organic1(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: async def _forward( self, synapse: StreamPromptingSynapse, @@ -214,107 +181,59 @@ async def _forward( timeout_threshold: float, send: Send, ): - buffer = [] accumulated_chunks = [] - accumulated_chunks_timings = [] - messages = [] - temp_completion = "" # for wandb logging - timeout_reached = False + # accumulated_chunks_timings = [] + # messages = [] + # temp_completion = "" # for wandb logging + # timeout_reached = False try: - start_time = time.time() - - responses = await query_miners( - self._val, + # timer_start = time.perf_counter() + responses = await QueryMinersManager(validator=self._val).query_miners( + 1, synapse.roles, synapse.messages, uids, self._val.config.neuron.timeout ) - # system_prompt_message = [{"role": "system", "content": self.system_prompt}] - # synapse_messages = [{"role": role, "content": message} - # for role, message in zip(synapse.roles, synapse.messages)] - - # messages = system_prompt_message + synapse_messages - - # stream_response = self.model.chat.completions.create( - # model=self.config.neuron.model_id, - # messages=messages, - # temperature=self.config.neuron.temperature, - # max_tokens=self.config.neuron.max_tokens, - # stream=True - # ) - async for chunk in responses: - if isinstance(chunk, list): - concatenated_chunks = "".join(chunk) - await send( - { - "type": "http.response.body", - "body": concatenated_chunks.encode("utf-8"), - "more_body": True, - } - ) - bt.logging.info(f"Streamed text: {chunk}") - if isinstance(chunk, str): - await send( - { - "type": "http.response.body", - "body": chunk.encode("utf-8"), - "more_body": True, - } - ) - bt.logging.info(f"Streamed text: {chunk}") - if chunk is not None and isinstance(chunk, StreamPromptingSynapse): - if len(self.accumulated_chunks) == 0: - self.accumulated_chunks.append(chunk.completion) - self.accumulated_chunks_timings.append(time.time() - start_time) - - self.finish_reason = "completed" - self.sequence_number += 1 - # Assuming the last chunk holds the last value yielded which should be a synapse with the completion filled - synapse = chunk - - await send( - { - "type": "http.response.body", - "body": synapse.completion.encode("utf-8"), - "more_body": True, - } - ) - - # chunk_content = chunk.choices[0].delta.content - # if chunk_content is None: - # bt.logging.info("OpenAI returned chunk content with None") - # continue + for chunks in responses: + # await asyncio.sleep(1) + async for chunk in chunks: + if isinstance(chunk, str): + accumulated_chunks.append(chunk) + await send( + { + "type": "http.response.body", + "body": chunk.encode("utf-8"), + "more_body": True, + } + ) + bt.logging.info(f"Streamed text: {chunk}") + if chunk is not None and isinstance(chunk, StreamPromptingSynapse): + accumulated_chunks.append(chunk.completion) + # self.finish_reason = "completed" + # self.sequence_number += 1 + # Assuming the last chunk holds the last value yielded which should be a synapse with the completion filled + synapse = chunk + + if len(accumulated_chunks) == 0: + await send( + { + "type": "http.response.body", + "body": synapse.completion.encode("utf-8"), + "more_body": True, + } + ) # accumulated_chunks.append(chunk_content) - # accumulated_chunks_timings.append(time.time() - start_time) - + # accumulated_chunks_timings.append(time.perf_counter() - timer_start) # buffer.append(chunk_content) - # if time.time() - init_time > timeout_threshold: # bt.logging.debug(f"⏰ Timeout reached, stopping streaming") # timeout_reached = True # break - - # if len(buffer) == self._val.config.neuron.streaming_batch_size: - # joined_buffer = "".join(buffer) - # temp_completion += joined_buffer - # bt.logging.debug(f"Streamed tokens: {joined_buffer}") - - # await send( - # { - # "type": "http.response.body", - # "body": joined_buffer.encode("utf-8"), - # "more_body": True, - # } - # ) - # buffer = [] - - # if ( - # buffer and not timeout_reached - # ): # Don't send the last buffer of data if timeout. + # if (buffer and not timeout_reached): # Don't send the last buffer of data if timeout. # joined_buffer = "".join(buffer) # await send( # { @@ -332,14 +251,16 @@ async def _forward( finally: synapse_latency = time.time() - init_time - # # if self.config.wandb.on: - # # self.log_event( - # # synapse=synapse, - # # timing=synapse_latency, - # # messages=messages, - # # accumulated_chunks=accumulated_chunks, - # # accumulated_chunks_timings = accumulated_chunks_timings, - # # ) + print(synapse) + print("".join(accumulated_chunks)) + # if self.config.wandb.on: + # self.log_event( + # synapse=synapse, + # timing=synapse_latency, + # messages=messages, + # accumulated_chunks=accumulated_chunks, + # accumulated_chunks_timings = accumulated_chunks_timings, + # ) bt.logging.debug(f"📧 Message received from {synapse.dendrite.hotkey}, IP: {synapse.dendrite.ip}; \nForwarding synapse: {synapse}") diff --git a/prompting/utils/config.py b/prompting/utils/config.py index 4ee538e73..0511949ff 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -328,14 +328,15 @@ def add_validator_args(cls, parser): "--neuron.sample_size", type=int, help="The number of miners to query in a single step.", - default=100, + # default=100, + default=1, ) parser.add_argument( "--neuron.organic_size", type=int, help="The number of miners to organic query in a single step.", - default=5, + default=1, ) parser.add_argument( From 1b6c16b10ff0499fb10ee74fb85dfe56991c6dbe Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Fri, 5 Jul 2024 23:40:13 +0000 Subject: [PATCH 11/57] [WIP] Fix issue with dendride streaming --- connecting_vali.ipynb | 25 ++- prompting/base/validator.py | 52 +++-- prompting/forward.py | 9 +- prompting/llms/vllm_llm.py | 5 +- prompting/organic/organic_weight_setter.py | 236 ++++++++++++++------- prompting/utils/config.py | 2 +- prompting/validator.py | 6 +- 7 files changed, 227 insertions(+), 108 deletions(-) diff --git a/connecting_vali.ipynb b/connecting_vali.ipynb index 7a7610a32..583631e4d 100644 --- a/connecting_vali.ipynb +++ b/connecting_vali.ipynb @@ -2,14 +2,16 @@ "cells": [ { "cell_type": "code", - "execution_count": 14, + "execution_count": 1, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "INFO:bittensor: - Connected to test network and wss://test.finney.opentensor.ai:443/. - \n", + "/workspace/venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n", + "2024-07-05 10:59:13,583\tINFO util.py:154 -- Missing packages: ['ipywidgets']. Run `pip install -U ipywidgets`, then restart the notebook server for rich notebook output.\n", "INFO:bittensor: - Connected to test network and wss://test.finney.opentensor.ai:443/. - \n" ] }, @@ -17,6 +19,21 @@ "name": "stdout", "output_type": "stream", "text": [ + "2024-07-05 10:59:30.694 | INFO | - Connected to test network and wss://test.finney.opentensor.ai:443/. - \n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:bittensor: - Connected to test network and wss://test.finney.opentensor.ai:443/. - \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2024-07-05 10:59:33.646 | INFO | - Connected to test network and wss://test.finney.opentensor.ai:443/. - \n", "Hotkey 5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr is registered. UID: 0\n", "Active UIDs (total: 2): [0, 1]\n", "My hotkey is active: True\n", @@ -59,7 +76,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 38, "metadata": {}, "outputs": [ { @@ -68,7 +85,7 @@ "text": [ "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", "INFO:bittensor: - Response:\n", - " - \n" + " The capital of Zimbabwe is Harare. - \n" ] } ], diff --git a/prompting/base/validator.py b/prompting/base/validator.py index e2cb9006e..49bedc02a 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -18,11 +18,11 @@ import random import sys import copy +import time import torch import asyncio import argparse import threading -import concurrent.futures import bittensor as bt @@ -31,7 +31,6 @@ from prompting.base.neuron import BaseNeuron from prompting.mock import MockDendrite -from prompting.organic.organic_dataset import OrganicDataset from prompting.utils.config import add_validator_args from prompting.utils.exceptions import MaxRetryError from prompting.organic.organic_weight_setter import OrganicWeightSetter @@ -70,32 +69,33 @@ def __init__(self, config=None): self.sync() # Create asyncio event loop to manage async tasks. - self.loop = asyncio.get_event_loop() - self._executor = concurrent.futures.ThreadPoolExecutor() - self._executor.submit(self.start_asyncio_loop, self.loop) + # self.loop = asyncio.get_event_loop() + # self._executor = concurrent.futures.ThreadPoolExecutor() + # self._executor.submit(self.start_asyncio_loop, self.loop) # Instantiate runners self.should_exit: bool = False self.is_running: bool = False self.thread: threading.Thread = None self.lock = asyncio.Lock() - self._serve_axon() - self._organic_weight_setter = OrganicWeightSetter(validator=self, axon=self.axon, loop=self.loop) - - @staticmethod - def start_asyncio_loop(loop): - try: - loop.run_forever() - except RuntimeError: - pass + self.axon = None + # self._serve_axon() + # self.axon = bt.axon(wallet=self.wallet, config=self.config) + # self.axon.serve(netuid=self.config.netuid, subtensor=self.subtensor) + # self.axon.start() + + # @staticmethod + # def start_asyncio_loop(loop): + # try: + # loop.run_forever() + # except RuntimeError: + # pass def _serve_axon(self): """Serve axon to enable external connections""" validator_uid = self.metagraph.hotkeys.index(self.wallet.hotkey.ss58_address) bt.logging.info(f"Serving validator IP of UID {validator_uid} to chain...") self.axon = bt.axon(wallet=self.wallet, config=self.config) - self.axon.serve(netuid=self.config.netuid, subtensor=self.subtensor) - self.axon.start() def run(self): """ @@ -120,7 +120,6 @@ def run(self): # Check that validator is registered on the network. self.sync() - self._organic_weight_setter.start_task() if not self.config.neuron.axon_off: bt.logging.info( f"Running validator {self.axon} on network: {self.config.subtensor.chain_endpoint} with netuid: {self.config.netuid}" @@ -132,6 +131,9 @@ def run(self): bt.logging.info(f"Validator starting at block: {self.block}") + self.loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.loop) + # This loop maintains the validator's operations until intentionally stopped. try: while True: @@ -139,12 +141,18 @@ def run(self): forward_timeout = self.config.neuron.forward_max_time try: - forward_future = asyncio.run_coroutine_threadsafe(self.forward(), self.loop) - try: - forward_future.result(timeout=forward_timeout) - except concurrent.futures.TimeoutError: - print("Forward task timed out") + task = self.loop.create_task(self.forward()) + self.loop.run_until_complete( + asyncio.wait_for(task, timeout=forward_timeout) + ) + + # forward_future = asyncio.run_coroutine_threadsafe(self.forward(), self.loop) + # try: + # forward_future.result(timeout=forward_timeout) + # except concurrent.futures.TimeoutError: + # print("Forward task timed out") + time.sleep(5) except torch.cuda.OutOfMemoryError as e: bt.logging.error(f"Out of memory error: {e}") continue diff --git a/prompting/forward.py b/prompting/forward.py index e6b586547..c56a7ec6b 100644 --- a/prompting/forward.py +++ b/prompting/forward.py @@ -199,10 +199,8 @@ async def query_miners(self, priority, roles, messages, uids, timeout): async def query_miners(self, roles: List[str], messages: List[str], uids: torch.LongTensor, timeout: float): - axons = [self.metagraph.axons[uid] for uid in uids] - # Directly call dendrite and process responses in parallel return await self.dendrite( - axons=axons, + axons=[self.metagraph.axons[uid] for uid in uids], synapse=StreamPromptingSynapse(roles=roles, messages=messages), timeout=timeout, deserialize=False, @@ -347,7 +345,6 @@ async def forward(self): # Note: The try catch is a safe clause to ensure that the forward loop continues even if an error occurs in run_step. # To be reconsidered in the next version. try: - continue # when run_step is called, the agent updates its progress event = await run_step( self, @@ -400,8 +397,8 @@ async def forward(self): log_event(self, event) - await asyncio.sleep(1) + await asyncio.sleep(5) continue - await asyncio.sleep(1) + await asyncio.sleep(5) del agent del task diff --git a/prompting/llms/vllm_llm.py b/prompting/llms/vllm_llm.py index 5e8d78705..ed8f377cc 100644 --- a/prompting/llms/vllm_llm.py +++ b/prompting/llms/vllm_llm.py @@ -15,6 +15,7 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. import gc +import threading import time import torch import bittensor as bt @@ -87,6 +88,7 @@ def __call__(self, composed_prompt: str, **model_kwargs: Dict) -> str: class vLLM_LLM(BaseLLM): + _lock = threading.Lock() def __init__( self, llm_pipeline: BasePipeline, @@ -167,7 +169,8 @@ def _make_prompt(self, messages: List[Dict[str, str]]) -> str: def forward(self, messages: List[Dict[str, str]]): # make composed prompt from messages composed_prompt = self._make_prompt(messages) - response = self.llm_pipeline(composed_prompt, **self.model_kwargs) + with self._lock: + response = self.llm_pipeline(composed_prompt, **self.model_kwargs) bt.logging.info( f"{self.__class__.__name__} generated the following output:\n{response}" diff --git a/prompting/organic/organic_weight_setter.py b/prompting/organic/organic_weight_setter.py index a1067325a..3ec6a9559 100644 --- a/prompting/organic/organic_weight_setter.py +++ b/prompting/organic/organic_weight_setter.py @@ -1,12 +1,15 @@ import sys +import threading import time import bittensor as bt import asyncio from functools import partial from typing import Awaitable, AsyncIterator +from concurrent.futures import TimeoutError from starlette.types import Send import bittensor as bt +import torch from prompting.agent import HumanAgent from prompting.base.neuron import BaseNeuron @@ -21,7 +24,7 @@ class OrganicWeightSetter: - def __init__(self, validator: BaseNeuron, axon: bt.axon, loop: asyncio.AbstractEventLoop): + def __init__(self, validator: BaseNeuron, axon: bt.axon): """Runs the organic weight setter task in separate threads. Creates 3 threads: @@ -39,58 +42,115 @@ def __init__(self, validator: BaseNeuron, axon: bt.axon, loop: asyncio.AbstractE # TODO (dbobrenko): Decouple OrganicDataset dependency. # TODO (dbobrenko): Decouple Validator dependecies: llm_pipeline, etc. self._val = validator - self._loop = loop self._axon = axon + self.should_exit = False + self.is_running = False self._organic_dataset = OrganicDataset() def start_task(self): - # validator_uid = self._val.metagraph.hotkeys.index(self._val.wallet.hotkey.ss58_address) - # bt.logging.info(f"Serving validator IP of UID {validator_uid} to chain...") - # self._axon = bt.axon(wallet=self._val.wallet, config=self._val.config) + + try: + # asyncio.run_coroutine_threadsafe(self._weight_setter(), self._loop) + self.run_in_background_thread() + bt.logging.info("Weight setter task started successfully.") + except Exception as e: + bt.logging.error(f"Failed to start weight setter task: {e}") + + def run_in_background_thread(self): + """ + Starts the validator's operations in a background thread upon entering the context. + This method facilitates the use of the validator in a 'with' statement. + """ + if not self.is_running: + bt.logging.debug("Starting organic tasks in background thread.") + self.should_exit = False + self.thread = threading.Thread(target=self._weight_setter, daemon=True) + self.thread.start() + self.is_running = True + bt.logging.debug("Started") + + def stop_run_thread(self): + """ + Stops the validator's operations that are running in the background thread. + """ + if self.is_running: + bt.logging.debug("Stopping organic tasks in background thread.") + self.should_exit = True + self.thread.join(5) + self.is_running = False + bt.logging.debug("Stopped") + + async def simple_coroutine(self): + bt.logging.debug("Entered simple_coroutine") + await asyncio.sleep(1) + return "simple result" + + def _weight_setter(self): + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + + ### + self._axon = bt.axon(wallet=self._val.wallet, config=self._val.config) self._axon.attach( forward_fn=self._handle_organic, blacklist_fn=None, priority_fn=None, ) - # self._axon.serve(netuid=self._val.config.netuid, subtensor=self._val.subtensor) - # self._axon.start() + self._axon.serve(netuid=self._val.config.netuid, subtensor=self._val.subtensor) + self._axon.start() + + async def stop_event_loop(): + while not self.should_exit: + await asyncio.sleep(0.1) + loop.stop() + bt.logging.debug("Event loop stopped.") + + loop.create_task(stop_event_loop()) + + async def run_tasks_forever(): + while not self.should_exit: + timer_start = time.perf_counter() + task_name = organic_task.TASK_NAME + try: + task = organic_task.OrganicTask( + llm_pipeline=self._val.llm_pipeline, + context=self._organic_dataset.next(), + create_reference=True, + ) + except Exception as e: + bt.logging.error(f"Failed to create {task_name} task. {sys.exc_info()}.") + continue + agent = HumanAgent(task=task, llm_pipeline=self._val.llm_pipeline, begin_conversation=True) + roles = task.roles + messages = task.messages + try: + # Schedule the async task in the new event loop + # event = await self.simple_coroutine() + event = await asyncio.wait_for( + self._run_step( + agent=agent, + roles=roles, + messages=messages, + k=self._val.config.neuron.organic_size, + timeout=self._val.config.neuron.timeout, + exclude=None, + ), + timeout=30 + ) + + # Adds forward time to event and logs it to wandb. + event["forward_time"] = time.perf_counter() - timer_start + log_event(self._val, event) + except asyncio.TimeoutError: + bt.logging.error(f"Failed to run {task_name} task. {sys.exc_info()}.") + except TimeoutError: + bt.logging.error("Task timed out.") + try: - asyncio.run_coroutine_threadsafe(self._weight_setter(), self._loop) - bt.logging.info("Weight setter task started successfully.") - except Exception as e: - bt.logging.error(f"Failed to start weight setter task: {e}") + loop.run_until_complete(run_tasks_forever()) + finally: + loop.close() - async def _weight_setter(self): - while True: - timer_start = time.perf_counter() - task_name = organic_task.TASK_NAME - try: - task = organic_task.OrganicTask( - llm_pipeline=self._val.llm_pipeline, - context=self._organic_dataset.next(), - create_reference=True, - ) - except Exception as e: - bt.logging.error(f"Failed to create {task_name} task. {sys.exc_info()}.") - await asyncio.sleep(1) - continue - agent = HumanAgent(task=task, llm_pipeline=self._val.llm_pipeline, begin_conversation=True) - # sample = self._organic_dataset.random() - roles = task.roles - messages = task.messages - event = await self._run_step( - agent, - roles=roles, - messages=messages, - k=self._val.config.neuron.sample_size, - timeout=self._val.config.neuron.timeout, - exclude=None, - ) - - # Adds forward time to event and logs it to wandb. - event["forward_time"] = time.perf_counter() - timer_start - log_event(self._val, event) - await asyncio.sleep(1) async def _run_step( self, agent: HumanAgent, roles: list[str], messages: list[str], k: int, timeout: float, exclude: list = None @@ -119,8 +179,10 @@ async def _run_step( uids_cpu = uids.cpu().tolist() # TODO: if organic and response is ready # streams_responses = await query_miners(self._val, roles, messages, uids, timeout) - streams_responses = await QueryMinersManager(validator=self._val).query_miners( - 0, + bt.logging.info(f"Querying miners with organic reward prompts: {uids_cpu}") + # streams_responses = await QueryMinersManager(validator=self._val).query_miners( + streams_responses = await query_miners( + self._val, roles, messages, uids, @@ -137,17 +199,17 @@ async def _run_step( # Encapsulate the responses in a response event (dataclass) response_event = DendriteResponseEvent(stream_results=stream_results, uids=uids, timeout=timeout) - bt.logging.info(f"Created DendriteResponseEvent:\n {response_event}") + bt.logging.info(f"Created organic reward DendriteResponseEvent:\n {response_event}") # Reward the responses and get the reward result (dataclass) # This contains a list of RewardEvents but can be exported as a dict (column-wise) for logging etc - bt.logging.info(f"Response from miners: {stream_results}") + bt.logging.info(f"Organic reward response from miners: {stream_results}") reward_result = RewardResult( self._val.reward_pipeline, agent=agent, response_event=response_event, device=self._val.device, ) - bt.logging.info(f"Created RewardResult:\n {reward_result}") + bt.logging.info(f"Created organic reward RewardResult:\n {reward_result}") best_response = response_event.completions[reward_result.rewards.argmax()] @@ -173,6 +235,9 @@ async def _run_step( return event async def _handle_organic(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: + from transformers import AutoTokenizer + model_name = "casperhansen/llama-3-8b-instruct-awq" + tokenizer = AutoTokenizer.from_pretrained(model_name) async def _forward( self, synapse: StreamPromptingSynapse, @@ -186,17 +251,43 @@ async def _forward( # messages = [] # temp_completion = "" # for wandb logging # timeout_reached = False - try: # timer_start = time.perf_counter() - responses = await QueryMinersManager(validator=self._val).query_miners( - 1, - synapse.roles, - synapse.messages, - uids, - self._val.config.neuron.timeout + bt.logging.info(f"Querying miners with organic prompts: {uids}") + # uids_cpu = uids.cpu().tolist() + + responses = self._val.dendrite.query( + axons=[self._val.metagraph.axons[uid] for uid in uids], + # synapse=StreamPromptingSynapse(roles=synapse.roles, messages=synapse.messages), + synapse=synapse, + timeout=30, + deserialize=False, + streaming=False, + # Doesn't work + # streaming=True, ) + # responses = await query_miners( + # self._val, + # synapse.roles, + # synapse.messages, + # torch.tensor(uids).to(self._val.device), + # self._val.config.neuron.timeout + # ) + + # Prepare the task for handling stream responses + stream_results_dict = dict(zip(uids, responses)) + # tokenizer = self._val.llm_pipeline.tokenizer + # stream_results = await handle_response(stream_results_dict, tokenizer) + # bt.logging.info(f"ORGANIC Response:\n {stream_results[0].synapse.completion}") + # return await send( + # { + # "type": "http.response.body", + # "body": stream_results[0].synapse.completion.encode("utf-8"), + # "more_body": True, + # } + # ) + bt.logging.info(f"Awaiting miners with organic prompts: {uids}") for chunks in responses: # await asyncio.sleep(1) async for chunk in chunks: @@ -210,21 +301,21 @@ async def _forward( } ) bt.logging.info(f"Streamed text: {chunk}") - if chunk is not None and isinstance(chunk, StreamPromptingSynapse): - accumulated_chunks.append(chunk.completion) - # self.finish_reason = "completed" - # self.sequence_number += 1 - # Assuming the last chunk holds the last value yielded which should be a synapse with the completion filled - synapse = chunk + # if chunk is not None and isinstance(chunk, StreamPromptingSynapse): + # accumulated_chunks.append(chunk.completion) + # # self.finish_reason = "completed" + # # self.sequence_number += 1 + # # Assuming the last chunk holds the last value yielded which should be a synapse with the completion filled + # synapse = chunk - if len(accumulated_chunks) == 0: - await send( - { - "type": "http.response.body", - "body": synapse.completion.encode("utf-8"), - "more_body": True, - } - ) + # if len(accumulated_chunks) == 0: + # await send( + # { + # "type": "http.response.body", + # "body": synapse.completion.encode("utf-8"), + # "more_body": False, + # } + # ) # accumulated_chunks.append(chunk_content) # accumulated_chunks_timings.append(time.perf_counter() - timer_start) @@ -251,7 +342,7 @@ async def _forward( finally: synapse_latency = time.time() - init_time - print(synapse) + print(f"Organic response: {synapse}") print("".join(accumulated_chunks)) # if self.config.wandb.on: # self.log_event( @@ -277,7 +368,6 @@ async def _forward( timeout_threshold, ) - response = synapse.create_streaming_response(token_streamer) - - self._organic_dataset.add({"synapse": synapse, "response": response, "uids": uids}) - return response + # response = synapse.create_streaming_response(token_streamer) + # self._organic_dataset.add({"synapse": synapse, "response": response, "uids": uids}) + return synapse.create_streaming_response(token_streamer) diff --git a/prompting/utils/config.py b/prompting/utils/config.py index 0511949ff..0a6faa03c 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -125,7 +125,7 @@ def add_args(cls, parser): "--no_background_thread", action="store_true", help="If set, we dont run the neuron in a background thread.", - default=True, + default=False, ) parser.add_argument( diff --git a/prompting/validator.py b/prompting/validator.py index 030bbd44f..7ad5aebbc 100644 --- a/prompting/validator.py +++ b/prompting/validator.py @@ -2,6 +2,7 @@ from prompting.forward import forward from prompting.llms import vLLMPipeline from prompting.base.validator import BaseValidatorNeuron +from prompting.organic.organic_weight_setter import OrganicWeightSetter from prompting.rewards import RewardPipeline from prompting.tasks.translate import TranslationPipeline @@ -39,6 +40,8 @@ def __init__(self, config=None): self.reward_pipeline = RewardPipeline( selected_tasks=self.active_tasks, device=self.device ) + self._organic_weight_setter = OrganicWeightSetter(validator=self, axon=self.axon) + self._organic_weight_setter.start_task() async def forward(self): """ @@ -56,7 +59,8 @@ def __enter__(self): bt.logging.warning("Running validator in main thread.") self.run() else: - self.run_in_background_thread() + pass + # self.run_in_background_thread() return self From 7d26c26eb46a472219092105122f58b9300c71f8 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Tue, 9 Jul 2024 09:39:38 +0000 Subject: [PATCH 12/57] WIP --- connecting_vali.ipynb | 23 +- prompting/base/validator.py | 7 +- prompting/forward.py | 59 +--- prompting/organic/organic_dataset.py | 7 +- prompting/organic/organic_weight_setter.py | 302 ++++++++------------- 5 files changed, 122 insertions(+), 276 deletions(-) diff --git a/connecting_vali.ipynb b/connecting_vali.ipynb index 583631e4d..8578e6596 100644 --- a/connecting_vali.ipynb +++ b/connecting_vali.ipynb @@ -2,16 +2,14 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": 16, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "/workspace/venv/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n", - "2024-07-05 10:59:13,583\tINFO util.py:154 -- Missing packages: ['ipywidgets']. Run `pip install -U ipywidgets`, then restart the notebook server for rich notebook output.\n", + "INFO:bittensor: - Connected to test network and wss://test.finney.opentensor.ai:443/. - \n", "INFO:bittensor: - Connected to test network and wss://test.finney.opentensor.ai:443/. - \n" ] }, @@ -19,21 +17,6 @@ "name": "stdout", "output_type": "stream", "text": [ - "2024-07-05 10:59:30.694 | INFO | - Connected to test network and wss://test.finney.opentensor.ai:443/. - \n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:bittensor: - Connected to test network and wss://test.finney.opentensor.ai:443/. - \n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2024-07-05 10:59:33.646 | INFO | - Connected to test network and wss://test.finney.opentensor.ai:443/. - \n", "Hotkey 5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr is registered. UID: 0\n", "Active UIDs (total: 2): [0, 1]\n", "My hotkey is active: True\n", @@ -76,7 +59,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 32, "metadata": {}, "outputs": [ { diff --git a/prompting/base/validator.py b/prompting/base/validator.py index 49bedc02a..5c202140a 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -78,11 +78,10 @@ def __init__(self, config=None): self.is_running: bool = False self.thread: threading.Thread = None self.lock = asyncio.Lock() - self.axon = None # self._serve_axon() - # self.axon = bt.axon(wallet=self.wallet, config=self.config) - # self.axon.serve(netuid=self.config.netuid, subtensor=self.subtensor) - # self.axon.start() + self.axon = bt.axon(wallet=self.wallet, config=self.config) + self.axon.serve(netuid=self.config.netuid, subtensor=self.subtensor) + self.axon.start() # @staticmethod # def start_asyncio_loop(loop): diff --git a/prompting/forward.py b/prompting/forward.py index c56a7ec6b..6cb2a1328 100644 --- a/prompting/forward.py +++ b/prompting/forward.py @@ -159,55 +159,6 @@ def log_stream_results(stream_results: List[SynapseStreamResult]): ) -class QueryMinersManager: - _instance = None - - def __new__(cls, validator): - if cls._instance is None: - cls._instance = super(QueryMinersManager, cls).__new__(cls) - # Initialize attributes only once - cls._instance.current_priority = 0 # 0 for low, 1 for high - cls._instance.event = asyncio.Event() - cls._instance.metagraph = validator.metagraph - cls._instance.dendrite = validator.dendrite - cls._val = validator - cls._instance.lock = asyncio.Lock() - return cls._instance - - async def query_miners(self, priority, roles, messages, uids, timeout): - async with self.lock: - if priority > self.current_priority: - self.current_priority = priority - self.event.set() # Signal that a higher-priority task is starting - - # Prepare the axons and call the dendrite - axons = [self.metagraph.axons[uid] for uid in uids] - streams_responses = await self._val.dendrite( - axons=axons, - synapse=StreamPromptingSynapse(roles=roles, messages=messages), - timeout=timeout, - deserialize=False, - streaming=True, - ) - bt.logging.info(f"Querying miners with priority={priority}") - - if self.event.is_set() and priority == self.current_priority: - # Clear the event after processing the higher-priority task - self.event.clear() - - return streams_responses - - -async def query_miners(self, roles: List[str], messages: List[str], uids: torch.LongTensor, timeout: float): - return await self.dendrite( - axons=[self.metagraph.axons[uid] for uid in uids], - synapse=StreamPromptingSynapse(roles=roles, messages=messages), - timeout=timeout, - deserialize=False, - streaming=True, - ) - - async def run_step( self, agent: HumanAgent, roles: List[str], messages: List[str], k: int, timeout: float, exclude: list = None ): @@ -233,9 +184,13 @@ async def run_step( # Get the list of uids to query for this step. uids = get_random_uids(self, k=k, exclude=exclude or []).to(self.device) uids_cpu = uids.cpu().tolist() - # TODO: if organic and response is ready - streams_responses = await QueryMinersManager(validator=self).query_miners(0, roles, messages, uids, timeout) - # streams_responses = await query_miners(self, roles, messages, uids, timeout) + streams_responses = await self.dendrite( + axons=[self.metagraph.axons[uid] for uid in uids], + synapse=StreamPromptingSynapse(roles=roles, messages=messages), + timeout=timeout, + deserialize=False, + streaming=True, + ) # Prepare the task for handling stream responses stream_results_dict = dict(zip(uids_cpu, streams_responses)) diff --git a/prompting/organic/organic_dataset.py b/prompting/organic/organic_dataset.py index 1a411ed2c..c1533980c 100644 --- a/prompting/organic/organic_dataset.py +++ b/prompting/organic/organic_dataset.py @@ -1,6 +1,3 @@ -""" -TODO (dbobrenko): Decouple code from prompting module. -""" from typing import Any, Optional from prompting.protocol import StreamPromptingSynapse @@ -16,7 +13,7 @@ class OrganicDataset(Dataset): _instance = None _lock = threading.Lock() - _queue: list[StreamPromptingSynapse] = [] + _queue: list[dict] = [] def __new__(cls, *args, **kwargs): if not cls._instance: @@ -68,5 +65,3 @@ def search(self, name) -> dict[str, Any]: dataset1.add(StreamPromptingSynapse(messages=["Capital of South Korea?"], roles=["user"])) print(dataset2.random()["synapse"].messages) print(dataset2.random()["synapse"].messages) - # This will raise an IndexError as the queue is empty - # print(dataset2.random().messages) \ No newline at end of file diff --git a/prompting/organic/organic_weight_setter.py b/prompting/organic/organic_weight_setter.py index 3ec6a9559..026025520 100644 --- a/prompting/organic/organic_weight_setter.py +++ b/prompting/organic/organic_weight_setter.py @@ -16,7 +16,7 @@ from prompting.dendrite import DendriteResponseEvent from prompting.organic import organic_task from prompting.protocol import StreamPromptingSynapse -from prompting.forward import QueryMinersManager, handle_response, log_stream_results, query_miners +from prompting.forward import handle_response, log_stream_results from prompting.organic.organic_dataset import OrganicDataset from prompting.rewards.reward import RewardResult from prompting.utils.logging import log_event @@ -48,7 +48,6 @@ def __init__(self, validator: BaseNeuron, axon: bt.axon): self._organic_dataset = OrganicDataset() def start_task(self): - try: # asyncio.run_coroutine_threadsafe(self._weight_setter(), self._loop) self.run_in_background_thread() @@ -80,77 +79,61 @@ def stop_run_thread(self): self.is_running = False bt.logging.debug("Stopped") - async def simple_coroutine(self): - bt.logging.debug("Entered simple_coroutine") - await asyncio.sleep(1) - return "simple result" + async def _priority_fn(self, synapse: StreamPromptingSynapse) -> float: + # High priority for organic traffic. + return 1000000.0 def _weight_setter(self): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) - ### - self._axon = bt.axon(wallet=self._val.wallet, config=self._val.config) self._axon.attach( - forward_fn=self._handle_organic, + forward_fn=self.on_organic_entry, blacklist_fn=None, - priority_fn=None, + priority_fn=self._priority_fn, ) - self._axon.serve(netuid=self._val.config.netuid, subtensor=self._val.subtensor) - self._axon.start() - - async def stop_event_loop(): - while not self.should_exit: - await asyncio.sleep(0.1) - loop.stop() - bt.logging.debug("Event loop stopped.") - - loop.create_task(stop_event_loop()) - - async def run_tasks_forever(): - while not self.should_exit: - timer_start = time.perf_counter() - task_name = organic_task.TASK_NAME - try: - task = organic_task.OrganicTask( - llm_pipeline=self._val.llm_pipeline, - context=self._organic_dataset.next(), - create_reference=True, - ) - except Exception as e: - bt.logging.error(f"Failed to create {task_name} task. {sys.exc_info()}.") - continue - agent = HumanAgent(task=task, llm_pipeline=self._val.llm_pipeline, begin_conversation=True) - roles = task.roles - messages = task.messages - try: - # Schedule the async task in the new event loop - # event = await self.simple_coroutine() - event = await asyncio.wait_for( - self._run_step( - agent=agent, - roles=roles, - messages=messages, - k=self._val.config.neuron.organic_size, - timeout=self._val.config.neuron.timeout, - exclude=None, - ), - timeout=30 - ) - - # Adds forward time to event and logs it to wandb. - event["forward_time"] = time.perf_counter() - timer_start - log_event(self._val, event) - except asyncio.TimeoutError: - bt.logging.error(f"Failed to run {task_name} task. {sys.exc_info()}.") - except TimeoutError: - bt.logging.error("Task timed out.") try: - loop.run_until_complete(run_tasks_forever()) + loop.run_until_complete(self._run_tasks_loop()) finally: loop.close() + async def _run_tasks_loop(self): + while not self.should_exit: + timer_start = time.perf_counter() + task_name = organic_task.TASK_NAME + try: + task = organic_task.OrganicTask( + llm_pipeline=self._val.llm_pipeline, + context=self._organic_dataset.next(), + create_reference=True, + ) + except Exception as e: + bt.logging.error(f"Failed to create {task_name} task. {sys.exc_info()}.") + continue + agent = HumanAgent(task=task, llm_pipeline=self._val.llm_pipeline, begin_conversation=True) + roles = task.roles + messages = task.messages + try: + event = await asyncio.wait_for( + self._run_step( + agent=agent, + roles=roles, + messages=messages, + k=self._val.config.neuron.organic_size, + timeout=self._val.config.neuron.timeout, + exclude=None, + ), + timeout=30 + ) + + # Adds forward time to event and logs it to wandb. + event["forward_time"] = time.perf_counter() - timer_start + log_event(self._val, event) + except asyncio.TimeoutError: + bt.logging.error(f"Failed to run {task_name} task. {sys.exc_info()}.") + except TimeoutError: + bt.logging.error("Task timed out.") async def _run_step( self, agent: HumanAgent, roles: list[str], messages: list[str], k: int, timeout: float, exclude: list = None @@ -177,16 +160,14 @@ async def _run_step( # Get the list of uids to query for this step. uids = get_random_uids(self._val, k=k, exclude=exclude or []).to(self._val.device) uids_cpu = uids.cpu().tolist() - # TODO: if organic and response is ready - # streams_responses = await query_miners(self._val, roles, messages, uids, timeout) + bt.logging.info(f"Querying miners with organic reward prompts: {uids_cpu}") - # streams_responses = await QueryMinersManager(validator=self._val).query_miners( - streams_responses = await query_miners( - self._val, - roles, - messages, - uids, - self._val.config.neuron.timeout + streams_responses = self._val.dendrite.query( + axons=[self._val.metagraph.axons[uid] for uid in uids], + synapse=StreamPromptingSynapse(roles=roles, messages=messages), + timeout=self._val.config.neuron.timeout, + deserialize=False, + # streaming=True, ) # Prepare the task for handling stream responses @@ -234,140 +215,73 @@ async def _run_step( } return event - async def _handle_organic(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: - from transformers import AutoTokenizer - model_name = "casperhansen/llama-3-8b-instruct-awq" - tokenizer = AutoTokenizer.from_pretrained(model_name) - async def _forward( - self, - synapse: StreamPromptingSynapse, - uids, - init_time: float, - timeout_threshold: float, - send: Send, - ): - accumulated_chunks = [] - # accumulated_chunks_timings = [] - # messages = [] - # temp_completion = "" # for wandb logging - # timeout_reached = False - try: - # timer_start = time.perf_counter() - bt.logging.info(f"Querying miners with organic prompts: {uids}") - # uids_cpu = uids.cpu().tolist() - - responses = self._val.dendrite.query( - axons=[self._val.metagraph.axons[uid] for uid in uids], - # synapse=StreamPromptingSynapse(roles=synapse.roles, messages=synapse.messages), - synapse=synapse, - timeout=30, - deserialize=False, - streaming=False, - # Doesn't work - # streaming=True, - ) - # responses = await query_miners( - # self._val, - # synapse.roles, - # synapse.messages, - # torch.tensor(uids).to(self._val.device), - # self._val.config.neuron.timeout - # ) - - # Prepare the task for handling stream responses - stream_results_dict = dict(zip(uids, responses)) - # tokenizer = self._val.llm_pipeline.tokenizer - # stream_results = await handle_response(stream_results_dict, tokenizer) - # bt.logging.info(f"ORGANIC Response:\n {stream_results[0].synapse.completion}") - # return await send( - # { - # "type": "http.response.body", - # "body": stream_results[0].synapse.completion.encode("utf-8"), - # "more_body": True, - # } - # ) - - bt.logging.info(f"Awaiting miners with organic prompts: {uids}") - for chunks in responses: - # await asyncio.sleep(1) - async for chunk in chunks: - if isinstance(chunk, str): - accumulated_chunks.append(chunk) - await send( - { - "type": "http.response.body", - "body": chunk.encode("utf-8"), - "more_body": True, - } - ) - bt.logging.info(f"Streamed text: {chunk}") - # if chunk is not None and isinstance(chunk, StreamPromptingSynapse): - # accumulated_chunks.append(chunk.completion) - # # self.finish_reason = "completed" - # # self.sequence_number += 1 - # # Assuming the last chunk holds the last value yielded which should be a synapse with the completion filled - # synapse = chunk - - # if len(accumulated_chunks) == 0: - # await send( - # { - # "type": "http.response.body", - # "body": synapse.completion.encode("utf-8"), - # "more_body": False, - # } - # ) - - # accumulated_chunks.append(chunk_content) - # accumulated_chunks_timings.append(time.perf_counter() - timer_start) - # buffer.append(chunk_content) - # if time.time() - init_time > timeout_threshold: - # bt.logging.debug(f"⏰ Timeout reached, stopping streaming") - # timeout_reached = True - # break - # if (buffer and not timeout_reached): # Don't send the last buffer of data if timeout. - # joined_buffer = "".join(buffer) - # await send( - # { - # "type": "http.response.body", - # "body": joined_buffer.encode("utf-8"), - # "more_body": False, - # } - # ) - - except Exception as e: - bt.logging.error(f"Error in forward: {e}") - # # bt.logging.error(print_exception(type(e), e, e.__traceback__)) - # if self._val.config.neuron.stop_on_forward_exception: - # self.should_exit = True - - finally: - synapse_latency = time.time() - init_time - print(f"Organic response: {synapse}") - print("".join(accumulated_chunks)) - # if self.config.wandb.on: - # self.log_event( - # synapse=synapse, - # timing=synapse_latency, - # messages=messages, - # accumulated_chunks=accumulated_chunks, - # accumulated_chunks_timings = accumulated_chunks_timings, - # ) - - bt.logging.debug(f"📧 Message received from {synapse.dendrite.hotkey}, IP: {synapse.dendrite.ip}; \nForwarding synapse: {synapse}") + async def on_organic_entry(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: + bt.logging.debug(f"📧 Message received from {synapse.dendrite.hotkey}, " + f"IP: {synapse.dendrite.ip};\nForwarding synapse: {synapse}") init_time = time.time() timeout_threshold = synapse.timeout uids = get_uids(self._val, sampling_mode="random", k=self._val.config.neuron.organic_size, exclude=[]) + completions: dict[int, dict] = {} token_streamer = partial( - _forward, - self, + self._stream_miner_response, synapse, uids, init_time, timeout_threshold, + completions, ) - # response = synapse.create_streaming_response(token_streamer) - # self._organic_dataset.add({"synapse": synapse, "response": response, "uids": uids}) - return synapse.create_streaming_response(token_streamer) + response = synapse.create_streaming_response(token_streamer) + self._organic_dataset.add({"synapse": synapse, "response": response, "uids": uids, "completions": completions}) + return response + + async def _stream_miner_response( + self, + synapse: StreamPromptingSynapse, + uids: list[int], + init_time: float, + timeout_threshold: float, + completions: dict[int, dict], + send: Send, + ): + accumulated_chunks = [] + try: + # timer_start = time.perf_counter() + bt.logging.info(f"Querying miners with organic prompts: {uids}") + + responses = self._val.dendrite.query( + axons=[self._val.metagraph.axons[uid] for uid in uids], + synapse=synapse, + timeout=30, + deserialize=False, + streaming=True, + ) + + bt.logging.info(f"Awaiting miners with organic prompts: {uids}") + for uid, chunks in zip(uids, responses): + completions[uid] = {"chunks": [], "completed": False} + async for chunk in chunks: + if isinstance(chunk, str): + completions[uid]["chunks"].append(chunk) + accumulated_chunks.append(chunk) + await send( + { + "type": "http.response.body", + "body": chunk.encode("utf-8"), + "more_body": True, + } + ) + completions[uid]["completed"] = True + + + # except Exception as e: + # bt.logging.error(f"Error in forward: {e}") + # # bt.logging.error(print_exception(type(e), e, e.__traceback__)) + # if self._val.config.neuron.stop_on_forward_exception: + # self.should_exit = True + + finally: + synapse_latency = time.time() - init_time + bt.logging.info(f"Organic response: {''.join(accumulated_chunks)}") From 5c5078e33bea64f858d017eeb743a975adb5b98f Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Thu, 18 Jul 2024 10:56:00 +0000 Subject: [PATCH 13/57] SN1-109: Refactor organics to base organics framework --- connecting_vali.ipynb | 393 ------------------ prompting/agent.py | 20 +- prompting/base/neuron.py | 19 +- prompting/base/validator.py | 56 +-- prompting/forward.py | 22 +- prompting/organic/organic_dataset.py | 67 --- .../organic/organic_scoring_prompting.py | 225 ++++++++++ prompting/organic/organic_task.py | 84 +--- prompting/organic/organic_weight_setter.py | 287 ------------- prompting/organic/synth_organic_task.py | 16 + prompting/task_registry.py | 6 +- prompting/tasks/__init__.py | 2 +- prompting/tools/__init__.py | 2 +- prompting/tools/datasets/__init__.py | 2 +- prompting/utils/config.py | 34 +- prompting/utils/uids.py | 12 +- prompting/validator.py | 18 +- requirements.txt | 6 +- 18 files changed, 375 insertions(+), 896 deletions(-) delete mode 100644 connecting_vali.ipynb delete mode 100644 prompting/organic/organic_dataset.py create mode 100644 prompting/organic/organic_scoring_prompting.py delete mode 100644 prompting/organic/organic_weight_setter.py create mode 100644 prompting/organic/synth_organic_task.py diff --git a/connecting_vali.ipynb b/connecting_vali.ipynb deleted file mode 100644 index 8578e6596..000000000 --- a/connecting_vali.ipynb +++ /dev/null @@ -1,393 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:bittensor: - Connected to test network and wss://test.finney.opentensor.ai:443/. - \n", - "INFO:bittensor: - Connected to test network and wss://test.finney.opentensor.ai:443/. - \n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Hotkey 5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr is registered. UID: 0\n", - "Active UIDs (total: 2): [0, 1]\n", - "My hotkey is active: True\n", - "Is val permit: True\n", - "1000+ TAO:\n" - ] - } - ], - "source": [ - "# https://docs.bittensor.com/subnets/register-validate-mine#check-the-permit-status\n", - "import bittensor as bt\n", - "import asyncio\n", - "from prompting.protocol import StreamPromptingSynapse\n", - "from prompting.forward import handle_response\n", - "from transformers import AutoTokenizer\n", - "\n", - "# NET_UID = 61\n", - "NET_UID = 170\n", - "NETWORK = \"test\"\n", - "WALLET = \"validator\"\n", - "# WALLET = \"miner\"\n", - "wallet = bt.wallet(name=WALLET)\n", - "subtensor = bt.subtensor(network=NETWORK)\n", - "subnet = subtensor.metagraph(netuid=NET_UID)\n", - "\n", - "hotkey = wallet.hotkey.ss58_address\n", - "my_uid = subnet.hotkeys.index(wallet.hotkey.ss58_address)\n", - "sub = bt.subtensor(NETWORK)\n", - "mg = sub.metagraph(NET_UID)\n", - "if hotkey not in mg.hotkeys:\n", - " print(f\"Hotkey {hotkey} deregistered\")\n", - "else:\n", - " print(f\"Hotkey {hotkey} is registered. UID: {my_uid}\")\n", - "active_uids = subnet.uids.tolist()\n", - "print(f\"Active UIDs (total: {len(active_uids)}): {active_uids}\")\n", - "print(f\"My hotkey is active: {my_uid in active_uids}\")\n", - "print(f\"Is val permit: {subnet.validator_permit[my_uid]}\")\n", - "print(f\"1000+ TAO:\")" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", - "INFO:bittensor: - Response:\n", - " The capital of Zimbabwe is Harare. - \n" - ] - } - ], - "source": [ - "uids = [my_uid]\n", - "axons = [subnet.axons[uid] for uid in uids]\n", - "\n", - "# Directly call dendrite and process responses in parallel\n", - "dendrite = bt.dendrite(wallet=wallet)\n", - "roles = [\"user\"]\n", - "messages = [\"Organic prompt query. Question: Capital of Zimbabwe?\"]\n", - "timeout = 40\n", - "model_name = \"casperhansen/llama-3-8b-instruct-awq\"\n", - "tokenizer = AutoTokenizer.from_pretrained(model_name)\n", - "\n", - "streams_responses = await dendrite(\n", - " axons=axons,\n", - " synapse=StreamPromptingSynapse(roles=roles, messages=messages),\n", - " timeout=timeout,\n", - " deserialize=False,\n", - " streaming=True,\n", - ")\n", - "stream_results_dict = dict(zip(uids, streams_responses))\n", - "handle_stream_responses_task = asyncio.create_task(handle_response(stream_results_dict, tokenizer))\n", - "stream_results = await handle_stream_responses_task\n", - "\n", - "bt.logging.info(f\"Response:\\n {stream_results[0].synapse.completion}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n", - "INFO:bittensor: - Response:\n", - " - \n" - ] - } - ], - "source": [ - "# Sample validators.\n", - "# TODO: Stress test: how many queries\n", - "\n", - "uids = [my_uid]\n", - "axons = [subnet.axons[uid] for uid in uids]\n", - "\n", - "# Directly call dendrite and process responses in parallel\n", - "dendrite = bt.dendrite(wallet=wallet)\n", - "roles = [\"user\"]\n", - "messages = [\"Organic prompt query. Question: Capital of Zimbabwe?\"]\n", - "timeout = 40\n", - "model_name = \"casperhansen/llama-3-8b-instruct-awq\"\n", - "tokenizer = AutoTokenizer.from_pretrained(model_name)\n", - "\n", - "streams_responses = await dendrite(\n", - " axons=axons,\n", - " synapse=StreamPromptingSynapse(roles=roles, messages=messages),\n", - " timeout=timeout,\n", - " deserialize=False,\n", - " streaming=True,\n", - ")\n", - "\n", - "stream_results_dict = dict(zip(uids, streams_responses))\n", - "handle_stream_responses_task = asyncio.create_task(handle_response(stream_results_dict, tokenizer))\n", - "stream_results = await handle_stream_responses_task\n", - "\n", - "bt.logging.info(f\"Response:\\n {stream_results[0].synapse.completion}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 63, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "ERROR:bittensor: - Error in generating reference or handling responses for uid 0: Something went wrong with miner uid 0, Synapse is not StreamPromptingSynapse.\n", - "Traceback (most recent call last):\n", - " File \"/workspace/prompting-organic/prompting/forward.py\", line 74, in process_stream\n", - " raise ValueError(\n", - "ValueError: Something went wrong with miner uid 0, Synapse is not StreamPromptingSynapse.\n", - " - \n" - ] - } - ], - "source": [ - "a = await asyncio.create_task(handle_response(stream_results_dict, tokenizer))" - ] - }, - { - "cell_type": "code", - "execution_count": 64, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[SynapseStreamResult(exception=ValueError('Something went wrong with miner uid 0, Synapse is not StreamPromptingSynapse.'), uid=0, accumulated_chunks=[], accumulated_chunks_timings=[], tokens_per_chunk=[], synapse=StreamPromptingSynapse(required_hash_fields=['messages'], roles=['user'], messages=['failure'], completion=''))]" - ] - }, - "execution_count": 64, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "a" - ] - }, - { - "cell_type": "code", - "execution_count": 62, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[SynapseStreamResult(exception=None, uid=0, accumulated_chunks=[], accumulated_chunks_timings=[], tokens_per_chunk=[], synapse=StreamPromptingSynapse(required_hash_fields=['messages'], roles=['user'], messages=['Organic prompt query. Question: Capital of Zimbabwe?'], completion=''))]" - ] - }, - "execution_count": 62, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "stream_results" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "User: Hey Im John\n", - "Assistant: Hi John! How can I assist you today?\n" - ] - } - ], - "source": [ - "# TODO: Stress test: how many queries\n", - "from prompting.protocol import StreamPromptingSynapse\n", - "from prompting.forward import handle_response\n", - "from transformers import AutoTokenizer\n", - "\n", - "uids = [my_uid]\n", - "axons = [subnet.axons[uid] for uid in uids]\n", - "\n", - "# Directly call dendrite and process responses in parallel\n", - "dendrite = bt.dendrite(wallet=wallet)\n", - "roles = []\n", - "messages = []\n", - "timeout = 40\n", - "model_name = \"casperhansen/llama-3-8b-instruct-awq\"\n", - "tokenizer = AutoTokenizer.from_pretrained(model_name)\n", - "while True:\n", - " message = input()\n", - " if message == \"exit\":\n", - " break\n", - " roles.append(\"user\")\n", - " messages.append(message)\n", - " streams_responses = await dendrite(\n", - " axons=axons,\n", - " synapse=StreamPromptingSynapse(roles=roles, messages=messages),\n", - " timeout=timeout,\n", - " deserialize=False,\n", - " streaming=True,\n", - " )\n", - " stream_results_dict = dict(zip(uids, streams_responses))\n", - " stream_results = await asyncio.create_task(handle_response(stream_results_dict, tokenizer))\n", - " response = stream_results[0].synapse.completion\n", - "\n", - " print(f\"User: {message}\")\n", - " print(f\"Assistant: {response}\")\n", - "\n", - " messages.append(response)\n", - " roles.append(\"assistant\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "stream_results[0].synapse.completion" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import asyncio\n", - "import concurrent.futures\n", - "import time\n", - "\n", - "async def val():\n", - " print(\"Val !\")\n", - " await asyncio.sleep(5)\n", - "\n", - "async def forward():\n", - " while True:\n", - " print(\"Forward\")\n", - " await asyncio.sleep(1)\n", - "\n", - "async def background_task():\n", - " while True:\n", - " print(\"Background\")\n", - " await val()\n", - " await asyncio.sleep(1)\n", - "\n", - "loop = asyncio.get_event_loop()\n", - "try:\n", - " loop.run_forever()\n", - "except RuntimeError:\n", - " pass\n", - "\n", - "task = loop.create_task(forward())\n", - "task2 = loop.create_task(background_task())\n", - "try:\n", - " await asyncio.wait_for(task, timeout=5)\n", - "except asyncio.TimeoutError:\n", - " print(\"Forward task timed out\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import asyncio\n", - "import concurrent.futures\n", - "import time\n", - "\n", - "# Asynchronous background task\n", - "async def background_task():\n", - " while True:\n", - " print(\"Background task is running\")\n", - " await asyncio.sleep(1)\n", - "\n", - "# Function to run the asyncio event loop in a separate thread\n", - "def start_asyncio_loop(loop):\n", - " asyncio.set_event_loop(loop)\n", - " loop.run_forever()\n", - "\n", - "def main():\n", - " # Create a new asyncio event loop\n", - " loop = asyncio.new_event_loop()\n", - "\n", - " # Start the event loop in a new thread\n", - " executor = concurrent.futures.ThreadPoolExecutor()\n", - " executor.submit(start_asyncio_loop, loop)\n", - "\n", - " # Schedule the asynchronous background task\n", - " asyncio.run_coroutine_threadsafe(background_task(), loop)\n", - "\n", - " # Synchronous while True loop in main\n", - " while True:\n", - " print(\"Main loop is running\")\n", - " time.sleep(1)\n", - "\n", - "main()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - } - ], - "metadata": { - "kernelspec": { - "display_name": "venv", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/prompting/agent.py b/prompting/agent.py index 974387bc1..0f9093447 100644 --- a/prompting/agent.py +++ b/prompting/agent.py @@ -19,7 +19,7 @@ import bittensor as bt from dataclasses import asdict from prompting.tasks import Task -from prompting.llms import HuggingFaceLLM, vLLM_LLM +from prompting.llms import vLLM_LLM from prompting.cleaners.cleaner import CleanerPipeline from prompting.persona import Persona, create_persona @@ -52,10 +52,8 @@ def __init__( system_template: str = None, persona: Persona = None, begin_conversation=True, + system_prompt: str | None = None, ): - if persona is None: - persona = create_persona() - self.persona = persona self.task = task self.llm_pipeline = llm_pipeline @@ -63,11 +61,15 @@ def __init__( if system_template is not None: self.system_prompt_template = system_template - self.system_prompt = self.system_prompt_template.format( - mood=self.persona.mood, - tone=self.persona.tone, - **self.task.__state_dict__(), # Adds desc, subject, topic - ) + self.system_prompt = system_prompt + if self.system_prompt is None: + if self.persona is None: + self.persona = create_persona() + self.system_prompt = self.system_prompt_template.format( + mood=self.persona.mood, + tone=self.persona.tone, + **self.task.__state_dict__(), # Adds desc, subject, topic + ) super().__init__( llm_pipeline=llm_pipeline, diff --git a/prompting/base/neuron.py b/prompting/base/neuron.py index 80fe42260..203db53e6 100644 --- a/prompting/base/neuron.py +++ b/prompting/base/neuron.py @@ -17,6 +17,7 @@ import copy import sys +import threading import bittensor as bt @@ -100,6 +101,7 @@ def __init__(self, config=None): f"Running neuron on subnet: {self.config.netuid} with uid {self.uid} using network: {self.subtensor.chain_endpoint}" ) self.step = 0 + self._thread_lock = threading.Lock() @abstractmethod def forward(self, synapse: bt.Synapse) -> bt.Synapse: @@ -113,17 +115,18 @@ def sync(self): """ Wrapper for synchronizing the state of the network for the given miner or validator. """ - # Ensure miner or validator hotkey is still registered on the network. - self.check_registered() + with self._thread_lock: + # Ensure miner or validator hotkey is still registered on the network. + self.check_registered() - if self.should_sync_metagraph(): - self.resync_metagraph() + if self.should_sync_metagraph(): + self.resync_metagraph() - if self.should_set_weights(): - self.set_weights() + if self.should_set_weights(): + self.set_weights() - # Always save state. - self.save_state() + # Always save state. + self.save_state() def check_registered(self): # --- Check for registration. diff --git a/prompting/base/validator.py b/prompting/base/validator.py index 5c202140a..1c7255bbb 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -15,7 +15,6 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. -import random import sys import copy import time @@ -26,14 +25,13 @@ import bittensor as bt -from typing import AsyncGenerator, List +from typing import List, Optional from traceback import print_exception from prompting.base.neuron import BaseNeuron from prompting.mock import MockDendrite from prompting.utils.config import add_validator_args from prompting.utils.exceptions import MaxRetryError -from prompting.organic.organic_weight_setter import OrganicWeightSetter class BaseValidatorNeuron(BaseNeuron): @@ -77,24 +75,26 @@ def __init__(self, config=None): self.should_exit: bool = False self.is_running: bool = False self.thread: threading.Thread = None - self.lock = asyncio.Lock() - # self._serve_axon() - self.axon = bt.axon(wallet=self.wallet, config=self.config) - self.axon.serve(netuid=self.config.netuid, subtensor=self.subtensor) - self.axon.start() + self.axon: Optional[bt.axon] = None + self.loop = asyncio.get_event_loop() + # threading.Thread(target=self._start_loop, daemon=True).start() + self._serve_axon() - # @staticmethod - # def start_asyncio_loop(loop): - # try: - # loop.run_forever() - # except RuntimeError: - # pass + def _start_loop(self): + asyncio.set_event_loop(self.loop) + self.loop.run_forever() def _serve_axon(self): """Serve axon to enable external connections""" validator_uid = self.metagraph.hotkeys.index(self.wallet.hotkey.ss58_address) bt.logging.info(f"Serving validator IP of UID {validator_uid} to chain...") self.axon = bt.axon(wallet=self.wallet, config=self.config) + self.axon.serve(netuid=self.config.netuid, subtensor=self.subtensor) + self.axon.start() + + def _start_loop(self): + asyncio.set_event_loop(self.loop) + self.loop.run_forever() def run(self): """ @@ -129,29 +129,32 @@ def run(self): ) bt.logging.info(f"Validator starting at block: {self.block}") - - self.loop = asyncio.new_event_loop() - asyncio.set_event_loop(self.loop) + # self.loop = asyncio.new_event_loop() + # asyncio.set_event_loop(self.loop) # This loop maintains the validator's operations until intentionally stopped. try: + # self.loop = asyncio.get_event_loop() + # self.loop = asyncio.new_event_loop() + # asyncio.set_event_loop(self.loop) + # threading.Thread(target=self._start_loop, daemon=True).start() while True: bt.logging.info(f"step({self.step}) block({self.block})") forward_timeout = self.config.neuron.forward_max_time try: task = self.loop.create_task(self.forward()) - self.loop.run_until_complete( - asyncio.wait_for(task, timeout=forward_timeout) - ) + self.loop.run_until_complete(asyncio.wait_for(task, timeout=forward_timeout)) + # task = self.loop.create_task(self.forward()) + # task = self.loop.create_task(self.forward()) + # self.loop.run_until_complete( + # asyncio.wait_for(task, timeout=forward_timeout) + # ) # forward_future = asyncio.run_coroutine_threadsafe(self.forward(), self.loop) - # try: - # forward_future.result(timeout=forward_timeout) - # except concurrent.futures.TimeoutError: - # print("Forward task timed out") + # forward_future.result(timeout=forward_timeout) - time.sleep(5) + # time.sleep(5) except torch.cuda.OutOfMemoryError as e: bt.logging.error(f"Out of memory error: {e}") continue @@ -185,6 +188,9 @@ def run(self): bt.logging.debug(print_exception(type(err), err, err.__traceback__)) self.should_exit = True + # finally: + # self.loop.close() + def run_in_background_thread(self): """ Starts the validator's operations in a background thread upon entering the context. diff --git a/prompting/forward.py b/prompting/forward.py index 6cb2a1328..0ef1c8dd7 100644 --- a/prompting/forward.py +++ b/prompting/forward.py @@ -15,6 +15,7 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. +from concurrent.futures import ThreadPoolExecutor import sys import time import random @@ -24,7 +25,6 @@ import bittensor as bt from typing import List, Dict, Awaitable -import torch from prompting.agent import HumanAgent from prompting.dendrite import DendriteResponseEvent, SynapseStreamResult from prompting.conversation import create_task @@ -36,9 +36,8 @@ from prompting.utils.logging import log_event from prompting.utils.misc import async_log, serialize_exception_to_string from transformers import PreTrainedTokenizerFast as Tokenizer -from prompting.utils.uids import get_random_uids -SINGLE_TURN_TASKS = ('sentiment', 'translation', organic_task.TASK_NAME) +SINGLE_TURN_TASKS = ('sentiment', 'translation') @async_log @@ -158,6 +157,10 @@ def log_stream_results(stream_results: List[SynapseStreamResult]): f"Failed response for uid {failed_response.uid}: {formatted_exception}" ) +async def async_generate_reference(loop, llm_pipeline): + with ThreadPoolExecutor() as pool: + result = await loop.run_in_executor(pool, generate_reference, llm_pipeline) + return result async def run_step( self, agent: HumanAgent, roles: List[str], messages: List[str], k: int, timeout: float, exclude: list = None @@ -195,10 +198,11 @@ async def run_step( # Prepare the task for handling stream responses stream_results_dict = dict(zip(uids_cpu, streams_responses)) tokenizer = self.llm_pipeline.tokenizer - handle_stream_responses_task = asyncio.create_task(handle_response(stream_results_dict, tokenizer)) + handle_stream_responses_task = handle_response(stream_results_dict, tokenizer) if not agent.task.static_reference: reference_generation_task = generate_reference(agent) + # reference_generation_task = async_generate_reference(self.loop, agent.llm_pipeline) _, stream_results = await asyncio.gather( reference_generation_task, handle_stream_responses_task ) @@ -288,14 +292,8 @@ async def forward(self): agent = HumanAgent( task=task, llm_pipeline=self.llm_pipeline, begin_conversation=True ) - if task_name == organic_task.TASK_NAME: - # Organic prompts with conversational history. - roles = task.roles - messages = task.messages - else: - # Benchmarking tasks. - roles = ["user"] - messages = [agent.challenge] + roles = ["user"] + messages = [agent.challenge] while True: # Note: The try catch is a safe clause to ensure that the forward loop continues even if an error occurs in run_step. # To be reconsidered in the next version. diff --git a/prompting/organic/organic_dataset.py b/prompting/organic/organic_dataset.py deleted file mode 100644 index c1533980c..000000000 --- a/prompting/organic/organic_dataset.py +++ /dev/null @@ -1,67 +0,0 @@ -from typing import Any, Optional - -from prompting.protocol import StreamPromptingSynapse -from prompting.tools.datasets.base import Dataset -import threading - -from prompting.tools.selector import Selector - - -class OrganicDataset(Dataset): - """Organic dataset singleton""" - name = "organic" - - _instance = None - _lock = threading.Lock() - _queue: list[dict] = [] - - def __new__(cls, *args, **kwargs): - if not cls._instance: - with cls._lock: - if not cls._instance: - cls._instance = super(OrganicDataset, cls).__new__(cls, *args, **kwargs) - return cls._instance - - @classmethod - def add(cls, synapse: StreamPromptingSynapse): - with cls._lock: - cls._queue.append(synapse) - - @classmethod - def random(cls, selector: Optional[Selector]) -> dict[str, Any]: - with cls._lock: - if cls._queue: - response = cls._queue.pop(0) - synapse = response["synapse"] - organic_source = "organic" - else: - # TODO: Get synthetic data. - synapse = StreamPromptingSynapse(messages=["Synthetic organic: Capital of Australia?"], roles=["user"]) - organic_source = "synthetic" - return { - "title": "Prompt", - "topic": "", - "subtopic": "", - "content": synapse.messages[-1], - "internal_links": [], - "external_links": [], - "source": organic_source, - "messages": synapse.messages, - "roles": synapse.roles, - "extra": {"date": None}, - } - - def get(self, name: str, **kwargs) -> dict[str, Any]: - raise NotImplementedError - - def search(self, name) -> dict[str, Any]: - raise NotImplementedError - - -if __name__ == "__main__": - dataset1 = OrganicDataset() - dataset2 = OrganicDataset() - dataset1.add(StreamPromptingSynapse(messages=["Capital of Australia?"], roles=["user"])) - dataset1.add(StreamPromptingSynapse(messages=["Capital of South Korea?"], roles=["user"])) - print(dataset2.random()["synapse"].messages) - print(dataset2.random()["synapse"].messages) diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py new file mode 100644 index 000000000..2c4834532 --- /dev/null +++ b/prompting/organic/organic_scoring_prompting.py @@ -0,0 +1,225 @@ +import time +from functools import partial +from typing import Any, Literal, Sequence + +import bittensor as bt +import torch +from organic_scoring import OrganicScoringBase +from organic_scoring.synth_dataset import SynthDatasetBase +from prompting.agent import HumanAgent +from prompting.base.neuron import BaseNeuron +from prompting.cleaners.cleaner import CleanerPipeline +from prompting.dendrite import DendriteResponseEvent, SynapseStreamResult +from prompting.forward import handle_response +from prompting.llms.vllm_llm import vLLM_LLM +from prompting.organic import organic_task +from prompting.protocol import StreamPromptingSynapse +from prompting.rewards.reward import RewardResult +from prompting.tasks.task import make_system_prompt +from prompting.utils.logging import log_event +from prompting.utils.uids import get_random_uids, get_uids +from starlette.types import Send +from typing_extensions import override + + +class OrganicScoringPrompting(OrganicScoringBase): + def __init__( + self, + axon: bt.axon, + synth_dataset: SynthDatasetBase | Sequence[SynthDatasetBase], + trigger_frequency: float | int, + trigger: Literal["seconds", "steps"], + validator: BaseNeuron, + ): + super().__init__(axon=axon, synth_dataset=synth_dataset, trigger_frequency=trigger_frequency, trigger=trigger) + self._val = validator + + async def _priority_fn(self, synapse: StreamPromptingSynapse) -> float: + """Priority function for the axon""" + return 1000000.0 + + async def _blacklist_fn(self, synapse: StreamPromptingSynapse) -> tuple[bool, str]: + """Blacklist function for the axon""" + return False, "" + + @override + async def _on_organic_entry(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: + bt.logging.info(f"[Organic] Received from {synapse.dendrite.hotkey}, IP: {synapse.dendrite.ip}") + + uids = get_uids(self._val, sampling_mode="random", k=self._val.config.neuron.organic_size, exclude=[]) + uids_list = uids.cpu().tolist() + completions: dict[int, dict] = {} + token_streamer = partial( + self._stream_miner_response, + synapse, + uids_list, + completions, + ) + + streaming_response = synapse.create_streaming_response(token_streamer) + self._organic_queue.add({ + "roles": synapse.roles, + "messages": synapse.messages, + "organic": True, + "synapse": synapse, + "streaming_response": streaming_response, + "uids": uids_list, + "completions": completions, + }) + return streaming_response + + async def _stream_miner_response( + self, + synapse: StreamPromptingSynapse, + uids: list[int], + completions: dict[int, dict], + send: Send, + ): + bt.logging.info(f"[Organic] Querying miner UIDs: {uids}") + responses = self._val.dendrite.query( + axons=[self._val.metagraph.axons[uid] for uid in uids], + synapse=synapse, + timeout=self._val.config.neuron.organic_timeout, + deserialize=False, + streaming=True, + ) + + bt.logging.info(f"[Organic] Awaiting miner streams UIDs: {uids}") + for uid, chunks in zip(uids, responses): + accumulated_chunks: list[str] = [] + accumulated_chunks_timings: list[float] = [] + accumulated_tokens_per_chunk: list[int] = [] + completions[uid] = {"completed": False} + timer_start = time.perf_counter() + async for chunk in chunks: + if isinstance(chunk, str): + accumulated_chunks.append(chunk) + accumulated_chunks_timings.append(time.perf_counter() - timer_start) + accumulated_tokens_per_chunk.append(len(self._val.llm_pipeline.tokenizer.tokenize(chunk))) + await send( + { + "type": "http.response.body", + "body": chunk.encode("utf-8"), + "more_body": True, + } + ) + elif isinstance(chunk, StreamPromptingSynapse): + synapse = chunk + completions[uid]["accumulated_chunks"] = accumulated_chunks + completions[uid]["accumulated_chunks_timings"] = accumulated_chunks_timings + completions[uid]["accumulated_tokens_per_chunk"] = accumulated_tokens_per_chunk + completions[uid]["completed"] = True + completions[uid]["synapse"] = synapse + + async def _reuse_organic_response(self, sample: dict[str, Any]) -> dict[int, SynapseStreamResult]: + """Returns a dict where the keys are miner UIDs and the values are their corresponding streaming responses""" + if not sample.get("organic", False): + return None + uids_cpu = sample["uids"] + responses: dict[int, SynapseStreamResult] = {} + bt.logging.info(f"[Organic] Reusing miner responses for organic data, UIDs: {uids_cpu}") + for uid in uids_cpu: + response = SynapseStreamResult( + accumulated_chunks=sample["completions"][uid]["accumulated_chunks"], + accumulated_chunks_timings=sample["completions"][uid]["accumulated_chunks_timings"], + tokens_per_chunk=sample["completions"][uid]["accumulated_tokens_per_chunk"], + synapse=sample["completions"][uid]["synapse"], + uid=uid, + exception=None + ) + responses[uid] = response + return responses + + @override + async def _query_miners(self, sample: dict[str, Any]) -> dict[str, Any]: + if sample.get("organic", False): + responses = await self._reuse_organic_response(sample) + return responses + + # Get the list of uids to query. + uids = get_random_uids(self._val, k=self._val.config.neuron.organic_size, exclude=None).to(self._val.device) + uids_cpu = uids.cpu().tolist() + bt.logging.info(f"[Organic] Querying miners with synthetic data, UIDs: {uids_cpu}") + streams_responses = self._val.dendrite.query( + axons=[self._val.metagraph.axons[uid] for uid in uids_cpu], + synapse=StreamPromptingSynapse(roles=sample["roles"], messages=sample["messages"]), + timeout=self._val.config.neuron.timeout, + deserialize=False, + streaming=True, + ) + stream_results_dict = dict(zip(uids_cpu, streams_responses)) + responses = await handle_response(stream_results_dict, self._val.llm_pipeline.tokenizer) + return dict(zip(uids_cpu, responses)) + + @override + async def _generate_rewards( + self, + sample: dict[str, Any], + responses: dict[str, Any], + reference: dict[str, Any], + ) -> dict[str, Any]: + assert reference is not None + if sample.get("organic", False): + task = organic_task.OrganicTask(context=sample, reference=reference) + else: + task = organic_task.SynthOrganicTask(context=sample, reference=reference) + stream_results = list(responses.values()) + uids_list = list(responses.keys()) + uids = torch.tensor(uids_list) + timeout = self._val.config.neuron.timeout + response_event = DendriteResponseEvent(stream_results=stream_results, uids=uids, timeout=timeout) + + bt.logging.debug(f"[Organic] Miner stream results: {stream_results}") + + # Dummy HumanAgent used to reuse existing reward pipeline. + agent = HumanAgent( + task=task, + llm_pipeline=self._val.llm_pipeline, + begin_conversation=True, + system_prompt=make_system_prompt(), + ) + reward_result = RewardResult( + self._val.reward_pipeline, + agent=agent, + response_event=response_event, + device=self._val.device, + ) + bt.logging.info(f"[Organic] RewardResult: {reward_result}") + return { + "reward": reward_result, + "uids": uids_list, + "agent": agent + } + + @override + async def _set_weights(self, reward: dict[str, Any]): + uids = reward["uids"] + reward_result = reward["reward"] + self._val.update_scores(reward_result.rewards, uids) + self._val.sync() + + @override + async def _log_results( + self, + logs: dict[str, Any], + reference: dict[str, Any], + responses: dict[str, Any], + rewards: dict[str, Any], + sample: dict[str, Any], + *args, + **kwargs, + ): + logs["block"] = self._val.block, + logs["step"] = self._val.step, + logs.update(rewards["reward"].__state_dict__(full=self._val.config.neuron.log_full)) + log_event(self._val, logs) + return logs + + @override + async def _generate_reference(self, sample: dict[str, Any]) -> dict[str, Any]: + reference = vLLM_LLM(self._val.llm_pipeline, system_prompt=make_system_prompt()).query_conversation( + messages=sample["messages"], + roles=sample["roles"], + cleaner=CleanerPipeline(cleaning_pipeline=[]) + ) + return reference diff --git a/prompting/organic/organic_task.py b/prompting/organic/organic_task.py index d48d5f91f..12f4b51b0 100644 --- a/prompting/organic/organic_task.py +++ b/prompting/organic/organic_task.py @@ -1,85 +1,39 @@ -""" -TODO (dbobrenko): Decouple code from prompting module. -""" -import time -from typing import Any -import bittensor as bt from dataclasses import dataclass -from prompting.cleaners.cleaner import CleanerPipeline -from prompting.llms.base_llm import BasePipeline -from prompting.llms.vllm_llm import vLLM_LLM -from prompting.shared.context import Context -from prompting.tasks import Task -from transformers import Pipeline - -from prompting.tasks.task import make_system_prompt -# QUERY_SYSTEM_PROMPT = "" -TASK_NAME = "organic" +from prompting.tasks import Task @dataclass class OrganicTask(Task): - name = TASK_NAME - desc = "get help on answering a question" - goal = "to get the answer to the following question" + name = "organic" # Use challenge as a query. challenge_type = "query" reward_definition = [ # dict(name="rouge", ngram="rouge-1", metric="f", weight=0.5), - dict(name="relevance", weight=1.0), + dict(name="relevance", weight=2.0), ] + penalty_definition = [ - dict(name="relevance", weight=1.0), + dict(name="relevance", weight=2.0), ] - cleaning_pipeline = [ - dict(name="remove_quotes"), - dict(name="prune_ending"), - dict(name="remove_roles"), - dict(name="remove_post_question_text"), - ] + cleaning_pipeline = [] - def __init__(self, llm_pipeline: Pipeline, context: Context, create_reference: bool = True): + def __init__(self, context: dict, reference: str): self.context = context - self.query = context.content - self.reference_prompt = context.content - self.messages = context.messages - self.roles = context.roles - self.topic = context.title - self.subtopic = context.topic - self.tags = context.tags - if create_reference: - self.reference = self.generate_reference(llm_pipeline) + self.messages = context["messages"] + self.roles = context["roles"] + self.query = context["messages"][-1] + self.reference = reference - def generate( - self, - system: str, - messages: list[str], - roles: list[str], - pipeline: BasePipeline, - clean=True - ) -> str: - """Use the LLM to generate a response to a prompt""" - cleaner = CleanerPipeline(cleaning_pipeline=self.cleaning_pipeline) if clean else None - return vLLM_LLM(pipeline, system_prompt=system).query_conversation( - messages=messages, roles=roles, cleaner=cleaner) + def __str__(self): + return f"{self.__class__.__name__}(name={self.name!r}, query={self.query!r}, reference={self.reference!r})" - def generate_reference(self, pipeline: BasePipeline, clean=True) -> str: - """Generates a reference answer to be used for scoring miner completions""" - t0 = time.perf_counter() - if not self.static_reference: - if not self.clean_reference: - clean = False - bt.logging.info("🤖 Generating reference...") - self.reference = self.generate( - system=make_system_prompt(), - messages=self.messages, - roles=self.roles, - pipeline=pipeline, - clean=clean, - ) + def __repr__(self): + return str(self) - self.reference_time = time.perf_counter() - t0 - return self.reference + def __state_dict__(self, full=False): + # Disable any logs for organic queries. + state = {} + return state diff --git a/prompting/organic/organic_weight_setter.py b/prompting/organic/organic_weight_setter.py deleted file mode 100644 index 026025520..000000000 --- a/prompting/organic/organic_weight_setter.py +++ /dev/null @@ -1,287 +0,0 @@ -import sys -import threading -import time -import bittensor as bt -import asyncio -from functools import partial -from typing import Awaitable, AsyncIterator -from concurrent.futures import TimeoutError - -from starlette.types import Send -import bittensor as bt -import torch - -from prompting.agent import HumanAgent -from prompting.base.neuron import BaseNeuron -from prompting.dendrite import DendriteResponseEvent -from prompting.organic import organic_task -from prompting.protocol import StreamPromptingSynapse -from prompting.forward import handle_response, log_stream_results -from prompting.organic.organic_dataset import OrganicDataset -from prompting.rewards.reward import RewardResult -from prompting.utils.logging import log_event -from prompting.utils.uids import get_random_uids, get_uids - - -class OrganicWeightSetter: - def __init__(self, validator: BaseNeuron, axon: bt.axon): - """Runs the organic weight setter task in separate threads. - - Creates 3 threads: - - Receiving organic requests through axon. - - Streaming response completions back to the caller through axon. - - Queue to incentivize miner's completions for organic or synthetic organic queries. - - Args: - validator (BaseNeuron): The validator to use. - axon (bt.axon): Served and started axon. - loop (asyncio.AbstractEventLoop): The loop to use. - """ - # TODO (dbobrenko): Decouple HumanAgent dependency. - # TODO (dbobrenko): Decouple OrganicTask dependency. - # TODO (dbobrenko): Decouple OrganicDataset dependency. - # TODO (dbobrenko): Decouple Validator dependecies: llm_pipeline, etc. - self._val = validator - self._axon = axon - self.should_exit = False - self.is_running = False - self._organic_dataset = OrganicDataset() - - def start_task(self): - try: - # asyncio.run_coroutine_threadsafe(self._weight_setter(), self._loop) - self.run_in_background_thread() - bt.logging.info("Weight setter task started successfully.") - except Exception as e: - bt.logging.error(f"Failed to start weight setter task: {e}") - - def run_in_background_thread(self): - """ - Starts the validator's operations in a background thread upon entering the context. - This method facilitates the use of the validator in a 'with' statement. - """ - if not self.is_running: - bt.logging.debug("Starting organic tasks in background thread.") - self.should_exit = False - self.thread = threading.Thread(target=self._weight_setter, daemon=True) - self.thread.start() - self.is_running = True - bt.logging.debug("Started") - - def stop_run_thread(self): - """ - Stops the validator's operations that are running in the background thread. - """ - if self.is_running: - bt.logging.debug("Stopping organic tasks in background thread.") - self.should_exit = True - self.thread.join(5) - self.is_running = False - bt.logging.debug("Stopped") - - async def _priority_fn(self, synapse: StreamPromptingSynapse) -> float: - # High priority for organic traffic. - return 1000000.0 - - def _weight_setter(self): - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - - self._axon.attach( - forward_fn=self.on_organic_entry, - blacklist_fn=None, - priority_fn=self._priority_fn, - ) - - try: - loop.run_until_complete(self._run_tasks_loop()) - finally: - loop.close() - - async def _run_tasks_loop(self): - while not self.should_exit: - timer_start = time.perf_counter() - task_name = organic_task.TASK_NAME - try: - task = organic_task.OrganicTask( - llm_pipeline=self._val.llm_pipeline, - context=self._organic_dataset.next(), - create_reference=True, - ) - except Exception as e: - bt.logging.error(f"Failed to create {task_name} task. {sys.exc_info()}.") - continue - agent = HumanAgent(task=task, llm_pipeline=self._val.llm_pipeline, begin_conversation=True) - roles = task.roles - messages = task.messages - try: - event = await asyncio.wait_for( - self._run_step( - agent=agent, - roles=roles, - messages=messages, - k=self._val.config.neuron.organic_size, - timeout=self._val.config.neuron.timeout, - exclude=None, - ), - timeout=30 - ) - - # Adds forward time to event and logs it to wandb. - event["forward_time"] = time.perf_counter() - timer_start - log_event(self._val, event) - except asyncio.TimeoutError: - bt.logging.error(f"Failed to run {task_name} task. {sys.exc_info()}.") - except TimeoutError: - bt.logging.error("Task timed out.") - - async def _run_step( - self, agent: HumanAgent, roles: list[str], messages: list[str], k: int, timeout: float, exclude: list = None - ): - """Executes a single step of the agent, which consists of: - - Getting a list of uids to query - - Querying the network - - Rewarding the network - - Updating the scores - - Logging the event - - Args: - agent (HumanAgent): The agent to run the step for. - roles (List[str]): The roles for the synapse. - messages (List[str]): The messages for the synapse. - k (int): The number of uids to query. - timeout (float): The timeout for the queries. - exclude (list, optional): The list of uids to exclude from the query. Defaults to []. - """ - bt.logging.debug("run_step", agent.task.name) - - # Record event start time. - start_time = time.perf_counter() - # Get the list of uids to query for this step. - uids = get_random_uids(self._val, k=k, exclude=exclude or []).to(self._val.device) - uids_cpu = uids.cpu().tolist() - - bt.logging.info(f"Querying miners with organic reward prompts: {uids_cpu}") - streams_responses = self._val.dendrite.query( - axons=[self._val.metagraph.axons[uid] for uid in uids], - synapse=StreamPromptingSynapse(roles=roles, messages=messages), - timeout=self._val.config.neuron.timeout, - deserialize=False, - # streaming=True, - ) - - # Prepare the task for handling stream responses - stream_results_dict = dict(zip(uids_cpu, streams_responses)) - tokenizer = self._val.llm_pipeline.tokenizer - stream_results = await handle_response(stream_results_dict, tokenizer) - - log_stream_results(stream_results) - - # Encapsulate the responses in a response event (dataclass) - response_event = DendriteResponseEvent(stream_results=stream_results, uids=uids, timeout=timeout) - - bt.logging.info(f"Created organic reward DendriteResponseEvent:\n {response_event}") - # Reward the responses and get the reward result (dataclass) - # This contains a list of RewardEvents but can be exported as a dict (column-wise) for logging etc - bt.logging.info(f"Organic reward response from miners: {stream_results}") - reward_result = RewardResult( - self._val.reward_pipeline, - agent=agent, - response_event=response_event, - device=self._val.device, - ) - bt.logging.info(f"Created organic reward RewardResult:\n {reward_result}") - - best_response = response_event.completions[reward_result.rewards.argmax()] - - # The original idea was that the agent is 'satisfied' when it gets a good enough response - # (e.g. reward critera is met, such as ROUGE>threshold) - agent.update_progress( - top_reward=reward_result.rewards.max(), - top_response=best_response, - ) - - self._val.update_scores(reward_result.rewards, uids) - - # Log the step event. - event = { - "best": best_response, - "block": self._val.block, - "step": self._val.step, - "step_time": time.perf_counter() - start_time, - **agent.__state_dict__(full=self._val.config.neuron.log_full), - **reward_result.__state_dict__(full=self._val.config.neuron.log_full), - **response_event.__state_dict__(), - } - return event - - async def on_organic_entry(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: - bt.logging.debug(f"📧 Message received from {synapse.dendrite.hotkey}, " - f"IP: {synapse.dendrite.ip};\nForwarding synapse: {synapse}") - - init_time = time.time() - timeout_threshold = synapse.timeout - - uids = get_uids(self._val, sampling_mode="random", k=self._val.config.neuron.organic_size, exclude=[]) - completions: dict[int, dict] = {} - token_streamer = partial( - self._stream_miner_response, - synapse, - uids, - init_time, - timeout_threshold, - completions, - ) - - response = synapse.create_streaming_response(token_streamer) - self._organic_dataset.add({"synapse": synapse, "response": response, "uids": uids, "completions": completions}) - return response - - async def _stream_miner_response( - self, - synapse: StreamPromptingSynapse, - uids: list[int], - init_time: float, - timeout_threshold: float, - completions: dict[int, dict], - send: Send, - ): - accumulated_chunks = [] - try: - # timer_start = time.perf_counter() - bt.logging.info(f"Querying miners with organic prompts: {uids}") - - responses = self._val.dendrite.query( - axons=[self._val.metagraph.axons[uid] for uid in uids], - synapse=synapse, - timeout=30, - deserialize=False, - streaming=True, - ) - - bt.logging.info(f"Awaiting miners with organic prompts: {uids}") - for uid, chunks in zip(uids, responses): - completions[uid] = {"chunks": [], "completed": False} - async for chunk in chunks: - if isinstance(chunk, str): - completions[uid]["chunks"].append(chunk) - accumulated_chunks.append(chunk) - await send( - { - "type": "http.response.body", - "body": chunk.encode("utf-8"), - "more_body": True, - } - ) - completions[uid]["completed"] = True - - - # except Exception as e: - # bt.logging.error(f"Error in forward: {e}") - # # bt.logging.error(print_exception(type(e), e, e.__traceback__)) - # if self._val.config.neuron.stop_on_forward_exception: - # self.should_exit = True - - finally: - synapse_latency = time.time() - init_time - bt.logging.info(f"Organic response: {''.join(accumulated_chunks)}") diff --git a/prompting/organic/synth_organic_task.py b/prompting/organic/synth_organic_task.py new file mode 100644 index 000000000..b36d2c51d --- /dev/null +++ b/prompting/organic/synth_organic_task.py @@ -0,0 +1,16 @@ +from dataclasses import dataclass + +from prompting.organic.organic_task import OrganicTask + + +@dataclass +class SynthOrganicTask(OrganicTask): + name = "synthetic-organic" + + reward_definition = [ + # dict(name="rouge", ngram="rouge-1", metric="f", weight=0.5), + dict(name="relevance", weight=1.0), + ] + penalty_definition = [ + dict(name="relevance", weight=1.0), + ] diff --git a/prompting/task_registry.py b/prompting/task_registry.py index 6fd02d81a..03dfb819e 100644 --- a/prompting/task_registry.py +++ b/prompting/task_registry.py @@ -1,7 +1,7 @@ from .tasks import ( Task, MockTask, - OrganicTask, + # OrganicTask, SummarizationTask, QuestionAnsweringTask, DebuggingTask, @@ -13,7 +13,7 @@ ) from .tools import ( MockDataset, - OrganicDataset, + # OrganicDataset, WikiDataset, HFCodingDataset, StackOverflowDataset, @@ -24,7 +24,7 @@ ) # TODO: Expand this to include extra information beyond just the task and dataset names -organic_task, organic_dataset = OrganicTask.name, [OrganicDataset.name] +# organic_task, organic_dataset = OrganicTask.name, [OrganicDataset.name] summarization_task, summarization_dataset = SummarizationTask.name, [WikiDataset.name] qa_task, qa_dataset = QuestionAnsweringTask.name, [WikiDataset.name] #debugging_task, debugging_dataset = DebuggingTask.name, [HFCodingDataset.name] diff --git a/prompting/tasks/__init__.py b/prompting/tasks/__init__.py index 0de8c8c11..021b76aee 100644 --- a/prompting/tasks/__init__.py +++ b/prompting/tasks/__init__.py @@ -8,7 +8,7 @@ from .translate import TranslationTask, TranslationPipeline from .mock import MockTask from .sentiment import SentimentAnalysisTask -from ..organic.organic_task import OrganicTask +# from ..organic.organic_task import OrganicTask TASKS = { # OrganicTask.name: OrganicTask, diff --git a/prompting/tools/__init__.py b/prompting/tools/__init__.py index 758ecd698..96a93847c 100644 --- a/prompting/tools/__init__.py +++ b/prompting/tools/__init__.py @@ -1,7 +1,7 @@ from .datasets import ( Dataset, MockDataset, - OrganicDataset, + # OrganicDataset, HFCodingDataset, WikiDataset, StackOverflowDataset, diff --git a/prompting/tools/datasets/__init__.py b/prompting/tools/datasets/__init__.py index 031f55134..f449f6017 100644 --- a/prompting/tools/datasets/__init__.py +++ b/prompting/tools/datasets/__init__.py @@ -5,4 +5,4 @@ from .wiki import WikiDataset, WikiDateDataset from .generic_instruction import GenericInstructionDataset from .review import ReviewDataset -from ...organic.organic_dataset import OrganicDataset +# from ...organic.organic_dataset import OrganicDataset diff --git a/prompting/utils/config.py b/prompting/utils/config.py index 0a6faa03c..947f91900 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -125,7 +125,7 @@ def add_args(cls, parser): "--no_background_thread", action="store_true", help="If set, we dont run the neuron in a background thread.", - default=False, + default=True, ) parser.add_argument( @@ -307,7 +307,7 @@ def add_validator_args(cls, parser): "--neuron.timeout", type=float, help="The timeout for each forward call in seconds.", - default=120, + default=15, ) parser.add_argument( @@ -328,15 +328,35 @@ def add_validator_args(cls, parser): "--neuron.sample_size", type=int, help="The number of miners to query in a single step.", - # default=100, - default=1, + default=100, ) parser.add_argument( "--neuron.organic_size", type=int, help="The number of miners to organic query in a single step.", - default=1, + default=5, + ) + + parser.add_argument( + "--neuron.organic_timeout", + type=int, + help="Organic query timeout for each call in seconds.", + default=30, + ) + + parser.add_argument( + "--neuron.trigger_frequency", + type=int, + help="Organic query validation frequency (seconds or steps value).", + default=15, + ) + + parser.add_argument( + "--neuron.trigger", + type=str, + help="Organic query validation trigger mode (seconds or steps).", + default="seconds", ) parser.add_argument( @@ -381,14 +401,14 @@ def add_validator_args(cls, parser): "--wandb.project_name", type=str, help="The name of the project where you are sending the new run.", - default="alpha-validators", + default="prompting-validators", ) parser.add_argument( "--wandb.entity", type=str, help="The name of the project where you are sending the new run.", - default="opentensor-dev", + default="macrocosmos" ) parser.add_argument( diff --git a/prompting/utils/uids.py b/prompting/utils/uids.py index a2c43d1ae..2286fb1ed 100644 --- a/prompting/utils/uids.py +++ b/prompting/utils/uids.py @@ -93,7 +93,7 @@ def get_random_uids(self: BaseNeuron, k: int, exclude: List[int] = None) -> torc raise ValueError(f"No eligible uids were found. Cannot return {k} uids") -def get_top_incentive_uids(self, k: int, vpermit_tao_limit: int) -> List[int]: +def get_top_incentive_uids(self, k: int, vpermit_tao_limit: int) -> torch.LongTensor: metagraph = self.metagraph miners_uids = list(map(int, filter(lambda uid: check_uid_availability(metagraph, uid, vpermit_tao_limit), metagraph.uids))) @@ -113,14 +113,12 @@ def get_top_incentive_uids(self, k: int, vpermit_tao_limit: int) -> List[int]: # Extract the top uids. top_k_uids = [uid for uid, incentive in uid_incentive_pairs_sorted[:k]] - return top_k_uids + return torch.tensor(top_k_uids) -def get_uids(self: BaseNeuron, sampling_mode: str, k: int, exclude: List[int] = []) -> Union[list[int], torch.LongTensor]: +def get_uids(self: BaseNeuron, sampling_mode: str, k: int, exclude: List[int] = []) -> torch.LongTensor: if sampling_mode == "random": - uids = get_random_uids(self, k=k, exclude=exclude or []).tolist() - return uids + return get_random_uids(self, k=k, exclude=exclude or []) if sampling_mode == "top_incentive": vpermit_tao_limit = self.config.neuron.vpermit_tao_limit - top_uids = get_top_incentive_uids(self, k=k, vpermit_tao_limit=vpermit_tao_limit) - return top_uids + return get_top_incentive_uids(self, k=k, vpermit_tao_limit=vpermit_tao_limit) diff --git a/prompting/validator.py b/prompting/validator.py index 7ad5aebbc..9963bf7d4 100644 --- a/prompting/validator.py +++ b/prompting/validator.py @@ -1,8 +1,10 @@ import bittensor as bt +from organic_scoring.synth_dataset import SynthDatasetLmSysChat + from prompting.forward import forward from prompting.llms import vLLMPipeline from prompting.base.validator import BaseValidatorNeuron -from prompting.organic.organic_weight_setter import OrganicWeightSetter +from prompting.organic.organic_scoring_prompting import OrganicScoringPrompting from prompting.rewards import RewardPipeline from prompting.tasks.translate import TranslationPipeline @@ -40,8 +42,15 @@ def __init__(self, config=None): self.reward_pipeline = RewardPipeline( selected_tasks=self.active_tasks, device=self.device ) - self._organic_weight_setter = OrganicWeightSetter(validator=self, axon=self.axon) - self._organic_weight_setter.start_task() + self._organic_scoring = OrganicScoringPrompting( + axon=self.axon, + synth_dataset=SynthDatasetLmSysChat(), + trigger_frequency=self.config.neuron.trigger_frequency, + trigger=self.config.neuron.trigger, + validator=self, + ) + # self._organic_scoring.start_thread() + self.loop.create_task(self._organic_scoring.start_loop()) async def forward(self): """ @@ -59,8 +68,7 @@ def __enter__(self): bt.logging.warning("Running validator in main thread.") self.run() else: - pass - # self.run_in_background_thread() + self.run_in_background_thread() return self diff --git a/requirements.txt b/requirements.txt index 79e0965cf..1c93400f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,10 +25,6 @@ wikipedia_sections==2.0.0 vllm==0.4.3 loguru==0.7.2 argostranslate==1.9.6 -python-dotenv -wikipedia_sections -vllm -loguru -argostranslate transformers==4.41.2 autoawq==0.2.5 +git+https://github.com/macrocosm-os/organic-scoring.git@main From ba5641f5f915d4929b86dc6251d248051e2d5e6b Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Thu, 18 Jul 2024 11:41:29 +0000 Subject: [PATCH 14/57] SN1-131: Clean up the code --- neurons/miners/huggingface/miner.py | 2 +- prompting/base/validator.py | 35 +++++-------------------- prompting/forward.py | 26 +++++++----------- prompting/llms/hf.py | 2 +- prompting/llms/vllm_llm.py | 10 ++++--- prompting/miners/openai_miner.py | 10 ++----- prompting/organic/organic_task.py | 2 +- prompting/organic/synth_organic_task.py | 1 + prompting/shared/context.py | 2 -- prompting/task_registry.py | 28 ++------------------ prompting/tasks/__init__.py | 2 -- prompting/tools/__init__.py | 5 ++-- prompting/tools/datasets/__init__.py | 3 +-- prompting/utils/config.py | 4 +-- prompting/validator.py | 3 +-- 15 files changed, 39 insertions(+), 96 deletions(-) diff --git a/neurons/miners/huggingface/miner.py b/neurons/miners/huggingface/miner.py index 5053dce15..6c33730d1 100644 --- a/neurons/miners/huggingface/miner.py +++ b/neurons/miners/huggingface/miner.py @@ -16,7 +16,7 @@ # DEALINGS IN THE SOFTWARE. import time import bittensor as bt -from prompting.miners.hf_miner import HuggingFaceMiner +from prompting.miners import HuggingFaceMiner from deprecated import deprecated diff --git a/prompting/base/validator.py b/prompting/base/validator.py index 1c7255bbb..3e6ee6066 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -17,7 +17,6 @@ import sys import copy -import time import torch import asyncio import argparse @@ -66,19 +65,16 @@ def __init__(self, config=None): # Init sync with the network. Updates the metagraph. self.sync() + self.axon: Optional[bt.axon] = None + self._serve_axon() + # Create asyncio event loop to manage async tasks. - # self.loop = asyncio.get_event_loop() - # self._executor = concurrent.futures.ThreadPoolExecutor() - # self._executor.submit(self.start_asyncio_loop, self.loop) + self.loop = asyncio.get_event_loop() # Instantiate runners self.should_exit: bool = False self.is_running: bool = False self.thread: threading.Thread = None - self.axon: Optional[bt.axon] = None - self.loop = asyncio.get_event_loop() - # threading.Thread(target=self._start_loop, daemon=True).start() - self._serve_axon() def _start_loop(self): asyncio.set_event_loop(self.loop) @@ -129,32 +125,18 @@ def run(self): ) bt.logging.info(f"Validator starting at block: {self.block}") - # self.loop = asyncio.new_event_loop() - # asyncio.set_event_loop(self.loop) # This loop maintains the validator's operations until intentionally stopped. try: - # self.loop = asyncio.get_event_loop() - # self.loop = asyncio.new_event_loop() - # asyncio.set_event_loop(self.loop) - # threading.Thread(target=self._start_loop, daemon=True).start() while True: bt.logging.info(f"step({self.step}) block({self.block})") forward_timeout = self.config.neuron.forward_max_time try: task = self.loop.create_task(self.forward()) - self.loop.run_until_complete(asyncio.wait_for(task, timeout=forward_timeout)) - # task = self.loop.create_task(self.forward()) - # task = self.loop.create_task(self.forward()) - # self.loop.run_until_complete( - # asyncio.wait_for(task, timeout=forward_timeout) - # ) - - # forward_future = asyncio.run_coroutine_threadsafe(self.forward(), self.loop) - # forward_future.result(timeout=forward_timeout) - - # time.sleep(5) + self.loop.run_until_complete( + asyncio.wait_for(task, timeout=forward_timeout) + ) except torch.cuda.OutOfMemoryError as e: bt.logging.error(f"Out of memory error: {e}") continue @@ -188,9 +170,6 @@ def run(self): bt.logging.debug(print_exception(type(err), err, err.__traceback__)) self.should_exit = True - # finally: - # self.loop.close() - def run_in_background_thread(self): """ Starts the validator's operations in a background thread upon entering the context. diff --git a/prompting/forward.py b/prompting/forward.py index 0ef1c8dd7..9f0b6794c 100644 --- a/prompting/forward.py +++ b/prompting/forward.py @@ -15,7 +15,6 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. -from concurrent.futures import ThreadPoolExecutor import sys import time import random @@ -28,7 +27,6 @@ from prompting.agent import HumanAgent from prompting.dendrite import DendriteResponseEvent, SynapseStreamResult from prompting.conversation import create_task -from prompting.organic import organic_task from prompting.protocol import StreamPromptingSynapse from prompting.rewards import RewardResult from prompting.tasks import QuestionAnsweringTask @@ -157,10 +155,6 @@ def log_stream_results(stream_results: List[SynapseStreamResult]): f"Failed response for uid {failed_response.uid}: {formatted_exception}" ) -async def async_generate_reference(loop, llm_pipeline): - with ThreadPoolExecutor() as pool: - result = await loop.run_in_executor(pool, generate_reference, llm_pipeline) - return result async def run_step( self, agent: HumanAgent, roles: List[str], messages: List[str], k: int, timeout: float, exclude: list = None @@ -187,8 +181,12 @@ async def run_step( # Get the list of uids to query for this step. uids = get_random_uids(self, k=k, exclude=exclude or []).to(self.device) uids_cpu = uids.cpu().tolist() + + axons = [self.metagraph.axons[uid] for uid in uids] + + # Directly call dendrite and process responses in parallel streams_responses = await self.dendrite( - axons=[self.metagraph.axons[uid] for uid in uids], + axons=axons, synapse=StreamPromptingSynapse(roles=roles, messages=messages), timeout=timeout, deserialize=False, @@ -198,11 +196,10 @@ async def run_step( # Prepare the task for handling stream responses stream_results_dict = dict(zip(uids_cpu, streams_responses)) tokenizer = self.llm_pipeline.tokenizer - handle_stream_responses_task = handle_response(stream_results_dict, tokenizer) + handle_stream_responses_task = asyncio.create_task(handle_response(stream_results_dict, tokenizer)) if not agent.task.static_reference: reference_generation_task = generate_reference(agent) - # reference_generation_task = async_generate_reference(self.loop, agent.llm_pipeline) _, stream_results = await asyncio.gather( reference_generation_task, handle_stream_responses_task ) @@ -211,7 +208,6 @@ async def run_step( log_stream_results(stream_results) - # TODO: Create separate thread for consuming organic prompts, and return reward. # Encapsulate the responses in a response event (dataclass) response_event = DendriteResponseEvent( stream_results=stream_results, uids=uids, timeout=timeout @@ -220,7 +216,6 @@ async def run_step( bt.logging.info(f"Created DendriteResponseEvent:\n {response_event}") # Reward the responses and get the reward result (dataclass) # This contains a list of RewardEvents but can be exported as a dict (column-wise) for logging etc - bt.logging.info(f"Response from miners: {stream_results}") reward_result = RewardResult( self.reward_pipeline, agent=agent, @@ -286,12 +281,12 @@ async def forward(self): # Create random agent with task, topic, profile... bt.logging.info(f"🤖 Creating agent for {task_name} task... ") - - turn = 0 - exclude_uids = [] agent = HumanAgent( task=task, llm_pipeline=self.llm_pipeline, begin_conversation=True ) + + turn = 0 + exclude_uids = [] roles = ["user"] messages = [agent.challenge] while True: @@ -350,8 +345,7 @@ async def forward(self): log_event(self, event) - await asyncio.sleep(5) + await asyncio.sleep(1) continue - await asyncio.sleep(5) del agent del task diff --git a/prompting/llms/hf.py b/prompting/llms/hf.py index 8a2da3cc1..d2a095405 100644 --- a/prompting/llms/hf.py +++ b/prompting/llms/hf.py @@ -211,7 +211,7 @@ def _make_prompt(self, messages: List[Dict[str, str]]): # return self.llm_pipeline.tokenizer.apply_chat_template( # messages, tokenize=False, add_generation_prompt=True # ) - return self.llm_pipeline.tokenizer.apply_chat_template( + return self.llm_pipeline.tokenizer.tokenizer.apply_chat_template( messages, tokenize=False, add_generation_prompt=True ) diff --git a/prompting/llms/vllm_llm.py b/prompting/llms/vllm_llm.py index ed8f377cc..3a1eceafe 100644 --- a/prompting/llms/vllm_llm.py +++ b/prompting/llms/vllm_llm.py @@ -14,10 +14,8 @@ # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. -import gc import threading import time -import torch import bittensor as bt from typing import List, Dict, Optional, Any from vllm import LLM, SamplingParams @@ -120,7 +118,13 @@ def query_conversation( roles: list[str], cleaner: Optional[CleanerPipeline] = None, ): - """Query LLM with the given lists of conversation history and roles.""" + """Query LLM with the given lists of conversation history and roles + + Args: + messages (list[str]): List of messages in the conversation. + roles (list[str]): List of roles for each message. + cleaner (Optional[CleanerPipeline], optional): Cleaner pipeline to use, if any. + """ assert len(messages) == len(roles), "Length of messages and roles must be the same" inputs: list[dict[str, Any]] = [{"content": self.system_prompt, "role": "system"}] for role, message in zip(roles, messages): diff --git a/prompting/miners/openai_miner.py b/prompting/miners/openai_miner.py index 047f256d8..9afe0ea2d 100644 --- a/prompting/miners/openai_miner.py +++ b/prompting/miners/openai_miner.py @@ -70,11 +70,8 @@ def __init__(self, config=None): self.accumulated_prompt_tokens = 0 self.accumulated_completion_tokens = 0 self.accumulated_total_cost = 0 - bt.logging.info("STARTING MINER") def forward(self, synapse: StreamPromptingSynapse) -> Awaitable: - bt.logging.info(f"Received request with {synapse.messages}") - async def _forward( self, synapse: StreamPromptingSynapse, @@ -90,7 +87,7 @@ async def _forward( timeout_reached = False - try: + try: system_prompt_message = [{ 'role': 'system', 'content': self.system_prompt }] synapse_messages = [{'role': role, 'content': message} for role, message in zip(synapse.roles, synapse.messages)] @@ -109,7 +106,7 @@ async def _forward( chunk_content = chunk.choices[0].delta.content if chunk_content is None: - # bt.logging.info("OpenAI returned chunk content with None") + bt.logging.info("OpenAI returned chunk content with None") continue accumulated_chunks.append(chunk_content) @@ -122,10 +119,8 @@ async def _forward( timeout_reached = True break - # bt.logging.info(f"Streaming back request {synapse.messages} -- {buffer} / {self.config.neuron.streaming_batch_size}") if len(buffer) == self.config.neuron.streaming_batch_size: joined_buffer = "".join(buffer) - bt.logging.info(f"Streaming back request {synapse.messages} -- {joined_buffer}") temp_completion += joined_buffer bt.logging.debug(f"Streamed tokens: {joined_buffer}") @@ -142,7 +137,6 @@ async def _forward( buffer and not timeout_reached ): # Don't send the last buffer of data if timeout. joined_buffer = "".join(buffer) - bt.logging.info(f"On buffer end {synapse.messages} -- {joined_buffer}") await send( { "type": "http.response.body", diff --git a/prompting/organic/organic_task.py b/prompting/organic/organic_task.py index 12f4b51b0..0652f8dfe 100644 --- a/prompting/organic/organic_task.py +++ b/prompting/organic/organic_task.py @@ -5,6 +5,7 @@ @dataclass class OrganicTask(Task): + """Task with defined reward and penalty mechanisms for organic prompts.""" name = "organic" # Use challenge as a query. challenge_type = "query" @@ -22,7 +23,6 @@ class OrganicTask(Task): def __init__(self, context: dict, reference: str): self.context = context - self.messages = context["messages"] self.roles = context["roles"] self.query = context["messages"][-1] self.reference = reference diff --git a/prompting/organic/synth_organic_task.py b/prompting/organic/synth_organic_task.py index b36d2c51d..ed9b6fc2e 100644 --- a/prompting/organic/synth_organic_task.py +++ b/prompting/organic/synth_organic_task.py @@ -5,6 +5,7 @@ @dataclass class SynthOrganicTask(OrganicTask): + """Task with defined reward and penalty mechanisms for synthetic organic prompts.""" name = "synthetic-organic" reward_definition = [ diff --git a/prompting/shared/context.py b/prompting/shared/context.py index c4e96f816..a9918c725 100644 --- a/prompting/shared/context.py +++ b/prompting/shared/context.py @@ -15,5 +15,3 @@ class Context: tags: List[str] = None extra: dict = None # additional non-essential information stats: dict = None # retrieval stats such as fetch time, number of tries, etc. - messages: list[str] = None - roles: list[str] = None diff --git a/prompting/task_registry.py b/prompting/task_registry.py index 03dfb819e..f111a29d9 100644 --- a/prompting/task_registry.py +++ b/prompting/task_registry.py @@ -1,30 +1,7 @@ -from .tasks import ( - Task, - MockTask, - # OrganicTask, - SummarizationTask, - QuestionAnsweringTask, - DebuggingTask, - MathTask, - DateQuestionAnsweringTask, - GenericInstructionTask, - SentimentAnalysisTask, - TranslationTask -) -from .tools import ( - MockDataset, - # OrganicDataset, - WikiDataset, - HFCodingDataset, - StackOverflowDataset, - MathDataset, - WikiDateDataset, - GenericInstructionDataset, - ReviewDataset -) +from .tasks import Task, MockTask, SummarizationTask, QuestionAnsweringTask, DebuggingTask, MathTask, DateQuestionAnsweringTask, GenericInstructionTask, SentimentAnalysisTask, TranslationTask +from .tools import MockDataset, WikiDataset, HFCodingDataset, StackOverflowDataset, MathDataset, WikiDateDataset, GenericInstructionDataset, ReviewDataset # TODO: Expand this to include extra information beyond just the task and dataset names -# organic_task, organic_dataset = OrganicTask.name, [OrganicDataset.name] summarization_task, summarization_dataset = SummarizationTask.name, [WikiDataset.name] qa_task, qa_dataset = QuestionAnsweringTask.name, [WikiDataset.name] #debugging_task, debugging_dataset = DebuggingTask.name, [HFCodingDataset.name] @@ -35,7 +12,6 @@ sentiment_analysis_task, sentiment_analysis_dataset = SentimentAnalysisTask.name, [ReviewDataset.name] TASK_REGISTRY = { - # organic_task: organic_dataset, summarization_task: summarization_dataset, qa_task: qa_dataset, #debugging_task: debugging_dataset, diff --git a/prompting/tasks/__init__.py b/prompting/tasks/__init__.py index 021b76aee..6794c97ca 100644 --- a/prompting/tasks/__init__.py +++ b/prompting/tasks/__init__.py @@ -8,10 +8,8 @@ from .translate import TranslationTask, TranslationPipeline from .mock import MockTask from .sentiment import SentimentAnalysisTask -# from ..organic.organic_task import OrganicTask TASKS = { - # OrganicTask.name: OrganicTask, QuestionAnsweringTask.name: QuestionAnsweringTask, DateQuestionAnsweringTask.name: DateQuestionAnsweringTask, SummarizationTask.name: SummarizationTask, diff --git a/prompting/tools/__init__.py b/prompting/tools/__init__.py index 96a93847c..82e3713d9 100644 --- a/prompting/tools/__init__.py +++ b/prompting/tools/__init__.py @@ -1,7 +1,6 @@ from .datasets import ( Dataset, MockDataset, - # OrganicDataset, HFCodingDataset, WikiDataset, StackOverflowDataset, @@ -14,7 +13,6 @@ DATASETS = { #HFCodingDataset.name: HFCodingDataset, - # OrganicDataset.name: OrganicDataset, WikiDataset.name: WikiDataset, #StackOverflowDataset.name: StackOverflowDataset, MathDataset.name: MathDataset, @@ -22,3 +20,6 @@ GenericInstructionDataset.name: GenericInstructionDataset, ReviewDataset.name: ReviewDataset } + + + \ No newline at end of file diff --git a/prompting/tools/datasets/__init__.py b/prompting/tools/datasets/__init__.py index f449f6017..3bdda191b 100644 --- a/prompting/tools/datasets/__init__.py +++ b/prompting/tools/datasets/__init__.py @@ -4,5 +4,4 @@ from .mock import MockDataset from .wiki import WikiDataset, WikiDateDataset from .generic_instruction import GenericInstructionDataset -from .review import ReviewDataset -# from ...organic.organic_dataset import OrganicDataset +from .review import ReviewDataset \ No newline at end of file diff --git a/prompting/utils/config.py b/prompting/utils/config.py index 947f91900..c85075b2c 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -300,14 +300,14 @@ def add_validator_args(cls, parser): type=float, nargs="+", help="The probability of sampling each task.", - default=[1 / len(TASKS)] * len(TASKS), + default=[1.0 / len(TASKS)] * len(TASKS), ) parser.add_argument( "--neuron.timeout", type=float, help="The timeout for each forward call in seconds.", - default=15, + default=17, ) parser.add_argument( diff --git a/prompting/validator.py b/prompting/validator.py index 9963bf7d4..56ea6bddc 100644 --- a/prompting/validator.py +++ b/prompting/validator.py @@ -1,9 +1,9 @@ import bittensor as bt from organic_scoring.synth_dataset import SynthDatasetLmSysChat +from prompting.base.validator import BaseValidatorNeuron from prompting.forward import forward from prompting.llms import vLLMPipeline -from prompting.base.validator import BaseValidatorNeuron from prompting.organic.organic_scoring_prompting import OrganicScoringPrompting from prompting.rewards import RewardPipeline from prompting.tasks.translate import TranslationPipeline @@ -49,7 +49,6 @@ def __init__(self, config=None): trigger=self.config.neuron.trigger, validator=self, ) - # self._organic_scoring.start_thread() self.loop.create_task(self._organic_scoring.start_loop()) async def forward(self): From 262e5c51eb0a4c04ffc5bec953d10797b8dce5c6 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Thu, 18 Jul 2024 11:45:30 +0000 Subject: [PATCH 15/57] Add organic sampling method to config --- prompting/organic/organic_scoring_prompting.py | 6 +++++- prompting/utils/config.py | 8 ++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index 2c4834532..76e50ea01 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -46,7 +46,11 @@ async def _blacklist_fn(self, synapse: StreamPromptingSynapse) -> tuple[bool, st async def _on_organic_entry(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: bt.logging.info(f"[Organic] Received from {synapse.dendrite.hotkey}, IP: {synapse.dendrite.ip}") - uids = get_uids(self._val, sampling_mode="random", k=self._val.config.neuron.organic_size, exclude=[]) + uids = get_uids( + self._val, + sampling_mode=self._val.config.neuron.organic_sampling_mode, + k=self._val.config.neuron.organic_size, + exclude=[]) uids_list = uids.cpu().tolist() completions: dict[int, dict] = {} token_streamer = partial( diff --git a/prompting/utils/config.py b/prompting/utils/config.py index c85075b2c..a245cd2cb 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -338,6 +338,14 @@ def add_validator_args(cls, parser): default=5, ) + parser.add_argument( + "--neuron.organic_sampling_mode", + type=str, + help="The mode for sampling miners using organic queries. Options include 'random' for random selection, " + "'top_incentive' for selecting based on highest incentives.", + default="random", + ) + parser.add_argument( "--neuron.organic_timeout", type=int, From fcb653d20a85eb24b630bb53422b35d76f6b92b8 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Thu, 18 Jul 2024 11:52:27 +0000 Subject: [PATCH 16/57] Add axon disabled warning --- prompting/base/validator.py | 5 ++++- prompting/validator.py | 19 +++++++++++-------- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/prompting/base/validator.py b/prompting/base/validator.py index 3e6ee6066..b93d2f7d3 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -66,7 +66,10 @@ def __init__(self, config=None): self.sync() self.axon: Optional[bt.axon] = None - self._serve_axon() + if not self.config.neuron.axon_off: + self._serve_axon() + else: + bt.logging.warning("axon off, not serving ip to chain.") # Create asyncio event loop to manage async tasks. self.loop = asyncio.get_event_loop() diff --git a/prompting/validator.py b/prompting/validator.py index 56ea6bddc..56243152e 100644 --- a/prompting/validator.py +++ b/prompting/validator.py @@ -42,14 +42,17 @@ def __init__(self, config=None): self.reward_pipeline = RewardPipeline( selected_tasks=self.active_tasks, device=self.device ) - self._organic_scoring = OrganicScoringPrompting( - axon=self.axon, - synth_dataset=SynthDatasetLmSysChat(), - trigger_frequency=self.config.neuron.trigger_frequency, - trigger=self.config.neuron.trigger, - validator=self, - ) - self.loop.create_task(self._organic_scoring.start_loop()) + if self.axon is not None: + self._organic_scoring = OrganicScoringPrompting( + axon=self.axon, + synth_dataset=SynthDatasetLmSysChat(), + trigger_frequency=self.config.neuron.trigger_frequency, + trigger=self.config.neuron.trigger, + validator=self, + ) + self.loop.create_task(self._organic_scoring.start_loop()) + else: + bt.logging.warning("Axon is disabled. Organic scoring will not be enabled.") async def forward(self): """ From 19094e7f73c17f0ea99736273d144c1baa4ae866 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Thu, 18 Jul 2024 12:16:39 +0000 Subject: [PATCH 17/57] Fix OrganicScoring args --- .../organic/organic_scoring_prompting.py | 23 ++++-- prompting/utils/config.py | 80 ++++++++++--------- prompting/validator.py | 5 +- 3 files changed, 63 insertions(+), 45 deletions(-) diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index 76e50ea01..69d09150f 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -30,8 +30,17 @@ def __init__( trigger_frequency: float | int, trigger: Literal["seconds", "steps"], validator: BaseNeuron, + trigger_frequency_min: float | int = 2, + trigger_scaling_factor: float | int = 50, ): - super().__init__(axon=axon, synth_dataset=synth_dataset, trigger_frequency=trigger_frequency, trigger=trigger) + super().__init__( + axon=axon, + synth_dataset=synth_dataset, + trigger_frequency=trigger_frequency, + trigger=trigger, + trigger_frequency_min=trigger_frequency_min, + trigger_scaling_factor=trigger_scaling_factor, + ) self._val = validator async def _priority_fn(self, synapse: StreamPromptingSynapse) -> float: @@ -48,8 +57,8 @@ async def _on_organic_entry(self, synapse: StreamPromptingSynapse) -> StreamProm uids = get_uids( self._val, - sampling_mode=self._val.config.neuron.organic_sampling_mode, - k=self._val.config.neuron.organic_size, + sampling_mode=self._val.config.organic.sampling_mode, + k=self._val.config.organic.sample_size, exclude=[]) uids_list = uids.cpu().tolist() completions: dict[int, dict] = {} @@ -83,7 +92,7 @@ async def _stream_miner_response( responses = self._val.dendrite.query( axons=[self._val.metagraph.axons[uid] for uid in uids], synapse=synapse, - timeout=self._val.config.neuron.organic_timeout, + timeout=self._val.config.organic.timeout, deserialize=False, streaming=True, ) @@ -141,13 +150,13 @@ async def _query_miners(self, sample: dict[str, Any]) -> dict[str, Any]: return responses # Get the list of uids to query. - uids = get_random_uids(self._val, k=self._val.config.neuron.organic_size, exclude=None).to(self._val.device) + uids = get_random_uids(self._val, k=self._val.config.organic.sample_size, exclude=None).to(self._val.device) uids_cpu = uids.cpu().tolist() bt.logging.info(f"[Organic] Querying miners with synthetic data, UIDs: {uids_cpu}") streams_responses = self._val.dendrite.query( axons=[self._val.metagraph.axons[uid] for uid in uids_cpu], synapse=StreamPromptingSynapse(roles=sample["roles"], messages=sample["messages"]), - timeout=self._val.config.neuron.timeout, + timeout=self._val.organic.timeout, deserialize=False, streaming=True, ) @@ -170,7 +179,7 @@ async def _generate_rewards( stream_results = list(responses.values()) uids_list = list(responses.keys()) uids = torch.tensor(uids_list) - timeout = self._val.config.neuron.timeout + timeout = self._val.config.organic.timeout response_event = DendriteResponseEvent(stream_results=stream_results, uids=uids, timeout=timeout) bt.logging.debug(f"[Organic] Miner stream results: {stream_results}") diff --git a/prompting/utils/config.py b/prompting/utils/config.py index 9d602bd61..d80075506 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -331,42 +331,6 @@ def add_validator_args(cls, parser): default=100, ) - parser.add_argument( - "--neuron.organic_size", - type=int, - help="The number of miners to organic query in a single step.", - default=5, - ) - - parser.add_argument( - "--neuron.organic_sampling_mode", - type=str, - help="The mode for sampling miners using organic queries. Options include 'random' for random selection, " - "'top_incentive' for selecting based on highest incentives.", - default="random", - ) - - parser.add_argument( - "--neuron.organic_timeout", - type=int, - help="Organic query timeout for each call in seconds.", - default=30, - ) - - parser.add_argument( - "--neuron.trigger_frequency", - type=int, - help="Organic query validation frequency (seconds or steps value).", - default=15, - ) - - parser.add_argument( - "--neuron.trigger", - type=str, - help="Organic query validation trigger mode (seconds or steps).", - default="seconds", - ) - parser.add_argument( "--neuron.disable_set_weights", action="store_true", @@ -440,6 +404,50 @@ def add_validator_args(cls, parser): default=120, ) + parser.add_argument( + "--organic.sample_size", + type=int, + help="The number of miners to organic query in a single step.", + default=5, + ) + + parser.add_argument( + "--organic.sampling_mode", + type=str, + help="The mode for sampling miners using organic queries. Options include 'random' for random selection, " + "'top_incentive' for selecting based on highest incentives.", + default="random", + ) + + parser.add_argument( + "--organic.timeout", + type=int, + help="Organic query timeout for each call in seconds.", + default=30, + ) + + parser.add_argument( + "--organic.trigger_frequency", + type=float, + help="Organic query sampling frequency (seconds or steps value).", + default=15.0, + ) + + parser.add_argument( + "--organic.trigger_frequency_min", + type=float, + help="Minimum organic query sampling frequency (seconds or steps value).", + default=2.0, + ) + + parser.add_argument( + "--organic.trigger", + type=str, + help="Organic query validation trigger mode (seconds or steps).", + default="seconds", + ) + + def config(cls): """ diff --git a/prompting/validator.py b/prompting/validator.py index 56243152e..3da796691 100644 --- a/prompting/validator.py +++ b/prompting/validator.py @@ -46,8 +46,9 @@ def __init__(self, config=None): self._organic_scoring = OrganicScoringPrompting( axon=self.axon, synth_dataset=SynthDatasetLmSysChat(), - trigger_frequency=self.config.neuron.trigger_frequency, - trigger=self.config.neuron.trigger, + trigger_frequency=self.config.organic.trigger_frequency, + trigger_frequency_min=self.config.organic.trigger_frequency_min, + trigger=self.config.organic.trigger, validator=self, ) self.loop.create_task(self._organic_scoring.start_loop()) From abd352c3befdb44b38bc4bc3095ff32ea474b4b1 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Thu, 18 Jul 2024 12:18:37 +0000 Subject: [PATCH 18/57] Run isort, remove Optional typing --- prompting/base/validator.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/prompting/base/validator.py b/prompting/base/validator.py index b93d2f7d3..e2e6498ad 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -15,17 +15,15 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. -import sys -import copy -import torch -import asyncio import argparse +import asyncio +import copy +import sys import threading +from traceback import print_exception import bittensor as bt - -from typing import List, Optional -from traceback import print_exception +import torch from prompting.base.neuron import BaseNeuron from prompting.mock import MockDendrite @@ -65,7 +63,7 @@ def __init__(self, config=None): # Init sync with the network. Updates the metagraph. self.sync() - self.axon: Optional[bt.axon] = None + self.axon: bt.axon | None = None if not self.config.neuron.axon_off: self._serve_axon() else: @@ -311,7 +309,7 @@ def resync_metagraph(self): # Update the hotkeys. self.hotkeys = copy.deepcopy(self.metagraph.hotkeys) - def update_scores(self, rewards: torch.FloatTensor, uids: List[int]): + def update_scores(self, rewards: torch.FloatTensor, uids: list[int]): """Performs exponential moving average on the scores based on the rewards received from the miners.""" # Check if rewards contains NaN values. From 1bbcc1dcfc4ace6734cb9a0d6ed388a3195ed824 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Thu, 18 Jul 2024 13:21:18 +0000 Subject: [PATCH 19/57] Move Organic reward pipeline to a separate object --- prompting/base/validator.py | 20 +++++------- .../organic/organic_scoring_prompting.py | 31 +++++++++++++------ prompting/organic/organic_task.py | 13 ++++---- prompting/organic/synth_organic_task.py | 11 ++++++- prompting/rewards/pipeline.py | 21 +++++++------ prompting/utils/config.py | 12 +++---- prompting/validator.py | 6 ++-- 7 files changed, 65 insertions(+), 49 deletions(-) diff --git a/prompting/base/validator.py b/prompting/base/validator.py index e2e6498ad..d58111f17 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -15,15 +15,16 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. -import argparse -import asyncio -import copy import sys +import copy +import torch +import asyncio +import argparse import threading -from traceback import print_exception import bittensor as bt -import torch + +from traceback import print_exception from prompting.base.neuron import BaseNeuron from prompting.mock import MockDendrite @@ -76,10 +77,7 @@ def __init__(self, config=None): self.should_exit: bool = False self.is_running: bool = False self.thread: threading.Thread = None - - def _start_loop(self): - asyncio.set_event_loop(self.loop) - self.loop.run_forever() + self.lock = asyncio.Lock() def _serve_axon(self): """Serve axon to enable external connections""" @@ -89,10 +87,6 @@ def _serve_axon(self): self.axon.serve(netuid=self.config.netuid, subtensor=self.subtensor) self.axon.start() - def _start_loop(self): - asyncio.set_event_loop(self.loop) - self.loop.run_forever() - def run(self): """ Initiates and manages the main lop for the miner on the Bittensor network. The main loop handles graceful shutdown on keyboard interrupts and logs unforeseen errors. diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index 69d09150f..7752f936e 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -12,8 +12,10 @@ from prompting.dendrite import DendriteResponseEvent, SynapseStreamResult from prompting.forward import handle_response from prompting.llms.vllm_llm import vLLM_LLM -from prompting.organic import organic_task +from prompting.organic.organic_task import OrganicTask +from prompting.organic.synth_organic_task import SynthOrganicTask from prompting.protocol import StreamPromptingSynapse +from prompting.rewards.pipeline import RewardPipeline from prompting.rewards.reward import RewardResult from prompting.tasks.task import make_system_prompt from prompting.utils.logging import log_event @@ -42,6 +44,15 @@ def __init__( trigger_scaling_factor=trigger_scaling_factor, ) self._val = validator + # Organic scoring reward pipeline. + self._reward_pipeline = RewardPipeline( + selected_tasks=[OrganicTask.name, SynthOrganicTask.name], + device=self._val.device, + available_tasks={ + OrganicTask.name: OrganicTask, + SynthOrganicTask.name: SynthOrganicTask, + } + ) async def _priority_fn(self, synapse: StreamPromptingSynapse) -> float: """Priority function for the axon""" @@ -57,8 +68,8 @@ async def _on_organic_entry(self, synapse: StreamPromptingSynapse) -> StreamProm uids = get_uids( self._val, - sampling_mode=self._val.config.organic.sampling_mode, - k=self._val.config.organic.sample_size, + sampling_mode=self._val.config.neuron.organic_sampling_mode, + k=self._val.config.neuron.organic_sample_size, exclude=[]) uids_list = uids.cpu().tolist() completions: dict[int, dict] = {} @@ -92,7 +103,7 @@ async def _stream_miner_response( responses = self._val.dendrite.query( axons=[self._val.metagraph.axons[uid] for uid in uids], synapse=synapse, - timeout=self._val.config.organic.timeout, + timeout=self._val.config.neuron.organic_timeout, deserialize=False, streaming=True, ) @@ -150,13 +161,13 @@ async def _query_miners(self, sample: dict[str, Any]) -> dict[str, Any]: return responses # Get the list of uids to query. - uids = get_random_uids(self._val, k=self._val.config.organic.sample_size, exclude=None).to(self._val.device) + uids = get_random_uids(self._val, k=self._val.config.neuron.organic_sample_size, exclude=None).to(self._val.device) uids_cpu = uids.cpu().tolist() bt.logging.info(f"[Organic] Querying miners with synthetic data, UIDs: {uids_cpu}") streams_responses = self._val.dendrite.query( axons=[self._val.metagraph.axons[uid] for uid in uids_cpu], synapse=StreamPromptingSynapse(roles=sample["roles"], messages=sample["messages"]), - timeout=self._val.organic.timeout, + timeout=self._val.config.neuron.organic_timeout, deserialize=False, streaming=True, ) @@ -173,13 +184,13 @@ async def _generate_rewards( ) -> dict[str, Any]: assert reference is not None if sample.get("organic", False): - task = organic_task.OrganicTask(context=sample, reference=reference) + task = OrganicTask(context=sample, reference=reference) else: - task = organic_task.SynthOrganicTask(context=sample, reference=reference) + task = SynthOrganicTask(context=sample, reference=reference) stream_results = list(responses.values()) uids_list = list(responses.keys()) uids = torch.tensor(uids_list) - timeout = self._val.config.organic.timeout + timeout = self._val.config.neuron.organic_timeout response_event = DendriteResponseEvent(stream_results=stream_results, uids=uids, timeout=timeout) bt.logging.debug(f"[Organic] Miner stream results: {stream_results}") @@ -192,7 +203,7 @@ async def _generate_rewards( system_prompt=make_system_prompt(), ) reward_result = RewardResult( - self._val.reward_pipeline, + self._reward_pipeline, agent=agent, response_event=response_event, device=self._val.device, diff --git a/prompting/organic/organic_task.py b/prompting/organic/organic_task.py index 0652f8dfe..c7add3413 100644 --- a/prompting/organic/organic_task.py +++ b/prompting/organic/organic_task.py @@ -10,22 +10,21 @@ class OrganicTask(Task): # Use challenge as a query. challenge_type = "query" - reward_definition = [ - # dict(name="rouge", ngram="rouge-1", metric="f", weight=0.5), - dict(name="relevance", weight=2.0), - ] + reward_definition = [dict(name="relevance", weight=1.0)] - penalty_definition = [ - dict(name="relevance", weight=2.0), - ] + penalty_definition = [dict(name="relevance", weight=1.0)] cleaning_pipeline = [] def __init__(self, context: dict, reference: str): self.context = context + self.messages = context["messages"] self.roles = context["roles"] self.query = context["messages"][-1] + self.topic = "Organic" self.reference = reference + self.subtopic = "" + self.tags = [""] def __str__(self): return f"{self.__class__.__name__}(name={self.name!r}, query={self.query!r}, reference={self.reference!r})" diff --git a/prompting/organic/synth_organic_task.py b/prompting/organic/synth_organic_task.py index ed9b6fc2e..c2ba2da56 100644 --- a/prompting/organic/synth_organic_task.py +++ b/prompting/organic/synth_organic_task.py @@ -9,9 +9,18 @@ class SynthOrganicTask(OrganicTask): name = "synthetic-organic" reward_definition = [ - # dict(name="rouge", ngram="rouge-1", metric="f", weight=0.5), dict(name="relevance", weight=1.0), ] penalty_definition = [ dict(name="relevance", weight=1.0), ] + + def __init__(self, context: dict, reference: str): + self.context = context + self.messages = context["messages"] + self.roles = context["roles"] + self.query = context["messages"][-1] + self.topic = "Organic" + self.reference = reference + self.subtopic = "" + self.tags = [""] diff --git a/prompting/rewards/pipeline.py b/prompting/rewards/pipeline.py index 2b126c401..dd18f8cce 100644 --- a/prompting/rewards/pipeline.py +++ b/prompting/rewards/pipeline.py @@ -1,5 +1,4 @@ -from typing import List - +from typing import Type from prompting.tasks import TASKS from prompting.rewards import ( BaseRewardModel, @@ -11,6 +10,7 @@ OrdinalRewardModel, StreamingRewardModel ) +from prompting.tasks.task import Task REWARD_MODELS = { "rouge": RougeRewardModel, @@ -24,7 +24,10 @@ class RewardPipeline: - def __init__(self, selected_tasks: List[str], device): + def __init__(self, selected_tasks: list[str], device, available_tasks: dict[str, Type[Task]] | None = None): + self.available_tasks = available_tasks + if self.available_tasks is None: + self.available_tasks = TASKS self.selected_tasks = selected_tasks self.device = device self.validate_tasks() @@ -41,9 +44,9 @@ def __repr__(self): def validate_tasks(self): for task in self.selected_tasks: - if task not in TASKS: + if task not in self.available_tasks: raise ValueError( - f"Task {task} not supported. Please choose from {TASKS.keys()}" + f"Task {task} not supported. Please choose from {self.available_tasks.keys()}" ) # Check that the reward_definition and penalty_definition are lists of dictionaries whose weights sum to one self._check_weights(task, "reward_definition", expected_weight=1) @@ -52,7 +55,7 @@ def validate_tasks(self): def _check_weights(self, task, definition, expected_weight): total_weight = 0 - model_infos = getattr(TASKS[task], definition) + model_infos = getattr(self.available_tasks[task], definition) for model_info in model_infos: if not isinstance(model_info, dict): @@ -90,9 +93,9 @@ def load_reward_pipeline(self): active_reward_models = [] for task in self.selected_tasks: - active_reward_models += TASKS[task].reward_definition - active_reward_models += TASKS[task].penalty_definition - active_reward_models += TASKS[task].global_penalty_definition + active_reward_models += self.available_tasks[task].reward_definition + active_reward_models += self.available_tasks[task].penalty_definition + active_reward_models += self.available_tasks[task].global_penalty_definition # Instantiate only the required reward models reward_models = {} diff --git a/prompting/utils/config.py b/prompting/utils/config.py index d80075506..0267c80bf 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -405,14 +405,14 @@ def add_validator_args(cls, parser): ) parser.add_argument( - "--organic.sample_size", + "--neuron.organic_sample_size", type=int, help="The number of miners to organic query in a single step.", default=5, ) parser.add_argument( - "--organic.sampling_mode", + "--neuron.organic_sampling_mode", type=str, help="The mode for sampling miners using organic queries. Options include 'random' for random selection, " "'top_incentive' for selecting based on highest incentives.", @@ -420,28 +420,28 @@ def add_validator_args(cls, parser): ) parser.add_argument( - "--organic.timeout", + "--neuron.organic_timeout", type=int, help="Organic query timeout for each call in seconds.", default=30, ) parser.add_argument( - "--organic.trigger_frequency", + "--neuron.organic_trigger_frequency", type=float, help="Organic query sampling frequency (seconds or steps value).", default=15.0, ) parser.add_argument( - "--organic.trigger_frequency_min", + "--neuron.organic_trigger_frequency_min", type=float, help="Minimum organic query sampling frequency (seconds or steps value).", default=2.0, ) parser.add_argument( - "--organic.trigger", + "--neuron.organic_trigger", type=str, help="Organic query validation trigger mode (seconds or steps).", default="seconds", diff --git a/prompting/validator.py b/prompting/validator.py index 3da796691..926adb42c 100644 --- a/prompting/validator.py +++ b/prompting/validator.py @@ -46,9 +46,9 @@ def __init__(self, config=None): self._organic_scoring = OrganicScoringPrompting( axon=self.axon, synth_dataset=SynthDatasetLmSysChat(), - trigger_frequency=self.config.organic.trigger_frequency, - trigger_frequency_min=self.config.organic.trigger_frequency_min, - trigger=self.config.organic.trigger, + trigger_frequency=self.config.neuron.organic_trigger_frequency, + trigger_frequency_min=self.config.neuron.organic_trigger_frequency_min, + trigger=self.config.neuron.organic_trigger, validator=self, ) self.loop.create_task(self._organic_scoring.start_loop()) From 8e0f3e3a3fd3efd76a2dde0370582844260832a9 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Thu, 18 Jul 2024 21:19:37 +0000 Subject: [PATCH 20/57] Add whitelist hotkey for axon --- prompting/base/validator.py | 2 ++ prompting/organic/organic_scoring_prompting.py | 14 ++++++++++---- prompting/utils/config.py | 8 ++++++++ 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/prompting/base/validator.py b/prompting/base/validator.py index d58111f17..6ce134e3c 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -321,6 +321,8 @@ def update_scores(self, rewards: torch.FloatTensor, uids: list[int]): # Update scores with rewards produced by this step. # shape: [ metagraph.n ] + # TODO: if miner's organics are empty then apply additional penalty: + # scores = scores * self.config.organic_empty_penalty alpha = self.config.neuron.moving_average_alpha self.scores = alpha * step_rewards + (1 - alpha) * self.scores self.scores = (self.scores - self.config.neuron.decay_alpha).clamp(min=0) diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index 7752f936e..d6a3dd963 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -1,6 +1,6 @@ import time from functools import partial -from typing import Any, Literal, Sequence +from typing import Any, Literal, Sequence, Tuple import bittensor as bt import torch @@ -54,13 +54,17 @@ def __init__( } ) + @override async def _priority_fn(self, synapse: StreamPromptingSynapse) -> float: """Priority function for the axon""" return 1000000.0 - async def _blacklist_fn(self, synapse: StreamPromptingSynapse) -> tuple[bool, str]: + @override + async def _blacklist_fn(self, synapse: StreamPromptingSynapse) -> Tuple[bool, str]: """Blacklist function for the axon""" - return False, "" + # ! DO NOT CHANGE `Tuple` return type to `tuple`, it will break the code (bittensor internal signature checks). + # We expect the API to be run with one specific hotkey (e.g. OTF). + return synapse.dendrite.hotkey != self._val.config.neuron.organic_whitelist_hotkey, "" @override async def _on_organic_entry(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: @@ -219,8 +223,10 @@ async def _generate_rewards( async def _set_weights(self, reward: dict[str, Any]): uids = reward["uids"] reward_result = reward["reward"] + bt.logging.info(f"[Organic] Rewards for miner's UIDs: {dict(zip(uids, reward_result.rewards))}") self._val.update_scores(reward_result.rewards, uids) - self._val.sync() + # Sync is not needed as it's done in the benchmarks loop. + # self._val.sync() @override async def _log_results( diff --git a/prompting/utils/config.py b/prompting/utils/config.py index 0267c80bf..bbd74d54a 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -447,6 +447,14 @@ def add_validator_args(cls, parser): default="seconds", ) + parser.add_argument( + "--neuron.organic_whitelist_hotkey", + type=str, + help="Allow request from specific hotkey. Defaults to OTF hotkey.", + # OTF hotkey. + default="5F4tQyWrhfGVcNhoqeiNsR6KjD4wMZ2kfhLj4oHYuyHbZAc3", + ) + def config(cls): From 027d7061f0e841c3bfbf64c91f174fbf37fe9f22 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Fri, 19 Jul 2024 18:06:33 +0000 Subject: [PATCH 21/57] Add config flags to disable weight setting, docstring --- prompting/base/validator.py | 2 +- .../organic/organic_scoring_prompting.py | 37 ++++++++++++++++--- prompting/utils/config.py | 20 +++++++++- prompting/validator.py | 6 ++- 4 files changed, 54 insertions(+), 11 deletions(-) diff --git a/prompting/base/validator.py b/prompting/base/validator.py index 6ce134e3c..c26a313b2 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -321,7 +321,7 @@ def update_scores(self, rewards: torch.FloatTensor, uids: list[int]): # Update scores with rewards produced by this step. # shape: [ metagraph.n ] - # TODO: if miner's organics are empty then apply additional penalty: + # TODO: if miner's UID organics are empty then apply additional penalty: # scores = scores * self.config.organic_empty_penalty alpha = self.config.neuron.moving_average_alpha self.scores = alpha * step_rewards + (1 - alpha) * self.scores diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index d6a3dd963..217281d85 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -6,6 +6,9 @@ import torch from organic_scoring import OrganicScoringBase from organic_scoring.synth_dataset import SynthDatasetBase +from starlette.types import Send +from typing_extensions import override + from prompting.agent import HumanAgent from prompting.base.neuron import BaseNeuron from prompting.cleaners.cleaner import CleanerPipeline @@ -20,8 +23,6 @@ from prompting.tasks.task import make_system_prompt from prompting.utils.logging import log_event from prompting.utils.uids import get_random_uids, get_uids -from starlette.types import Send -from typing_extensions import override class OrganicScoringPrompting(OrganicScoringBase): @@ -35,6 +36,22 @@ def __init__( trigger_frequency_min: float | int = 2, trigger_scaling_factor: float | int = 50, ): + """Organic Scoring implementation + + Organic scoring runs in a separate `asyncio` task and is triggered by a timer or a step counter. + + Process Workflow: + 1. Trigger Check: Upon triggering the rewarding process, the system checks if the organic queue is empty. + If the queue is empty, synthetic datasets are used to bootstrap the organic scoring mechanism. + Otherwise, samples from the organic queue are utilized. + 2. Data Processing: The sampled data is concurrently passed to the `_query_miners` and `_generate_reference` + methods. + 3. Reward Generation: After receiving responses from miners and any reference data, the information + is processed by the `_generate_rewards` method. + 4. Weight Setting: The generated rewards are then applied through the `_set_weights` method. + 5. Logging: Finally, the results can be logged using the `_log_results` method, along with all relevant data + provided as arguments, and default time elapsed on each step of rewarding process. + """ super().__init__( axon=axon, synth_dataset=synth_dataset, @@ -68,6 +85,7 @@ async def _blacklist_fn(self, synapse: StreamPromptingSynapse) -> Tuple[bool, st @override async def _on_organic_entry(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: + """Organic query handle""" bt.logging.info(f"[Organic] Received from {synapse.dendrite.hotkey}, IP: {synapse.dendrite.ip}") uids = get_uids( @@ -103,6 +121,7 @@ async def _stream_miner_response( completions: dict[int, dict], send: Send, ): + """Stream back miner's responses""" bt.logging.info(f"[Organic] Querying miner UIDs: {uids}") responses = self._val.dendrite.query( axons=[self._val.metagraph.axons[uid] for uid in uids], @@ -140,7 +159,7 @@ async def _stream_miner_response( completions[uid]["synapse"] = synapse async def _reuse_organic_response(self, sample: dict[str, Any]) -> dict[int, SynapseStreamResult]: - """Returns a dict where the keys are miner UIDs and the values are their corresponding streaming responses""" + """Return a dict where the keys are miner UIDs and the values are their corresponding streaming responses""" if not sample.get("organic", False): return None uids_cpu = sample["uids"] @@ -160,6 +179,7 @@ async def _reuse_organic_response(self, sample: dict[str, Any]) -> dict[int, Syn @override async def _query_miners(self, sample: dict[str, Any]) -> dict[str, Any]: + """Query miners with the given synthetic or organic sample""" if sample.get("organic", False): responses = await self._reuse_organic_response(sample) return responses @@ -186,6 +206,7 @@ async def _generate_rewards( responses: dict[str, Any], reference: dict[str, Any], ) -> dict[str, Any]: + """Generate rewards for the given sample, responses, and reference""" assert reference is not None if sample.get("organic", False): task = OrganicTask(context=sample, reference=reference) @@ -221,12 +242,15 @@ async def _generate_rewards( @override async def _set_weights(self, reward: dict[str, Any]): + """Set weights based on the given reward""" uids = reward["uids"] reward_result = reward["reward"] bt.logging.info(f"[Organic] Rewards for miner's UIDs: {dict(zip(uids, reward_result.rewards))}") - self._val.update_scores(reward_result.rewards, uids) - # Sync is not needed as it's done in the benchmarks loop. - # self._val.sync() + bt.logging.info(f"[Organic] Weight setting enabled: {self._val.config.neuron.organic_weight_setting_enabled}") + if self._val.config.neuron.organic_weight_setting_enabled: + self._val.update_scores(reward_result.rewards, uids) + # Sync is not needed as it's done in the benchmarks loop. + # self._val.sync() @override async def _log_results( @@ -247,6 +271,7 @@ async def _log_results( @override async def _generate_reference(self, sample: dict[str, Any]) -> dict[str, Any]: + """Generate reference for the given organic or synthetic sample""" reference = vLLM_LLM(self._val.llm_pipeline, system_prompt=make_system_prompt()).query_conversation( messages=sample["messages"], roles=sample["roles"], diff --git a/prompting/utils/config.py b/prompting/utils/config.py index bbd74d54a..edf204200 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -419,6 +419,22 @@ def add_validator_args(cls, parser): default="random", ) + parser.add_argument( + "--neuron.organic_disabled", + action="store_true", + help="Set this flag to disable organic scoring.", + default=False, + ) + + + # TODO: Ser organic weight setting enabled by default after Aug 1, 2024. + parser.add_argument( + "--neuron.organic_weight_setting_enabled", + action="store_true", + help="Set this flag to enable organic scoring weight setting.", + default=False, + ) + parser.add_argument( "--neuron.organic_timeout", type=int, @@ -430,14 +446,14 @@ def add_validator_args(cls, parser): "--neuron.organic_trigger_frequency", type=float, help="Organic query sampling frequency (seconds or steps value).", - default=15.0, + default=30.0, ) parser.add_argument( "--neuron.organic_trigger_frequency_min", type=float, help="Minimum organic query sampling frequency (seconds or steps value).", - default=2.0, + default=5.0, ) parser.add_argument( diff --git a/prompting/validator.py b/prompting/validator.py index 926adb42c..2214e2811 100644 --- a/prompting/validator.py +++ b/prompting/validator.py @@ -42,7 +42,7 @@ def __init__(self, config=None): self.reward_pipeline = RewardPipeline( selected_tasks=self.active_tasks, device=self.device ) - if self.axon is not None: + if self.axon is not None and not self.config.neuron.organic_disabled: self._organic_scoring = OrganicScoringPrompting( axon=self.axon, synth_dataset=SynthDatasetLmSysChat(), @@ -53,7 +53,9 @@ def __init__(self, config=None): ) self.loop.create_task(self._organic_scoring.start_loop()) else: - bt.logging.warning("Axon is disabled. Organic scoring will not be enabled.") + bt.logging.warning( + "Organic scoring is not enabled. To enable, remove '--neuron.axon_off' and '--neuron.organic_disabled'" + ) async def forward(self): """ From e3b4e2403d6e341e3f8765fc354cb31c3082b5ce Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Fri, 19 Jul 2024 19:29:45 +0000 Subject: [PATCH 22/57] Set max_tokens to 1024 for organic reference --- prompting/organic/organic_scoring_prompting.py | 6 +++++- prompting/utils/config.py | 7 +++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index 217281d85..64bd1cf0d 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -272,7 +272,11 @@ async def _log_results( @override async def _generate_reference(self, sample: dict[str, Any]) -> dict[str, Any]: """Generate reference for the given organic or synthetic sample""" - reference = vLLM_LLM(self._val.llm_pipeline, system_prompt=make_system_prompt()).query_conversation( + reference = vLLM_LLM( + self._val.llm_pipeline, + system_prompt=make_system_prompt(), + max_new_tokens=self._val.config.neuron.organic_reference_max_tokens, + ).query_conversation( messages=sample["messages"], roles=sample["roles"], cleaner=CleanerPipeline(cleaning_pipeline=[]) diff --git a/prompting/utils/config.py b/prompting/utils/config.py index edf204200..a7b42bc51 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -442,6 +442,13 @@ def add_validator_args(cls, parser): default=30, ) + parser.add_argument( + "--neuron.organic_reference_max_tokens", + type=int, + help="Organic query timeout for each call in seconds.", + default=1024, + ) + parser.add_argument( "--neuron.organic_trigger_frequency", type=float, From 6e0fbd36e2b5b1bdb54d40a23fe1d7303ec6b141 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Sat, 20 Jul 2024 11:24:52 +0000 Subject: [PATCH 23/57] Wait for miners completion during reuse, add flags Wait for miners completion while reusing their responses; Add flags to disable miner's completion reuse; Correct punctuation in docstrings; Increase organic trigger seconds to 45; Increase organic reference max_tokens to 1024. --- .../organic/organic_scoring_prompting.py | 87 +++++++++++++------ prompting/utils/config.py | 13 ++- 2 files changed, 71 insertions(+), 29 deletions(-) diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index 64bd1cf0d..0bafd00df 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -1,6 +1,8 @@ +import asyncio +import json import time from functools import partial -from typing import Any, Literal, Sequence, Tuple +from typing import Any, AsyncGenerator, Literal, Sequence, Tuple import bittensor as bt import torch @@ -36,7 +38,7 @@ def __init__( trigger_frequency_min: float | int = 2, trigger_scaling_factor: float | int = 50, ): - """Organic Scoring implementation + """Organic Scoring implementation. Organic scoring runs in a separate `asyncio` task and is triggered by a timer or a step counter. @@ -73,19 +75,20 @@ def __init__( @override async def _priority_fn(self, synapse: StreamPromptingSynapse) -> float: - """Priority function for the axon""" + """Priority function for the axon.""" return 1000000.0 @override async def _blacklist_fn(self, synapse: StreamPromptingSynapse) -> Tuple[bool, str]: - """Blacklist function for the axon""" + """Blacklist function for the axon.""" # ! DO NOT CHANGE `Tuple` return type to `tuple`, it will break the code (bittensor internal signature checks). # We expect the API to be run with one specific hotkey (e.g. OTF). - return synapse.dendrite.hotkey != self._val.config.neuron.organic_whitelist_hotkey, "" + # return synapse.dendrite.hotkey != self._val.config.neuron.organic_whitelist_hotkey, "" + return synapse.dendrite.hotkey != "5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr", "" @override async def _on_organic_entry(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: - """Organic query handle""" + """Organic query handle.""" bt.logging.info(f"[Organic] Received from {synapse.dendrite.hotkey}, IP: {synapse.dendrite.ip}") uids = get_uids( @@ -121,7 +124,7 @@ async def _stream_miner_response( completions: dict[int, dict], send: Send, ): - """Stream back miner's responses""" + """Stream back miner's responses.""" bt.logging.info(f"[Organic] Querying miner UIDs: {uids}") responses = self._val.dendrite.query( axons=[self._val.metagraph.axons[uid] for uid in uids], @@ -131,11 +134,11 @@ async def _stream_miner_response( streaming=True, ) - bt.logging.info(f"[Organic] Awaiting miner streams UIDs: {uids}") - for uid, chunks in zip(uids, responses): + async def stream_miner_chunks(uid: int, chunks: AsyncGenerator): accumulated_chunks: list[str] = [] accumulated_chunks_timings: list[float] = [] accumulated_tokens_per_chunk: list[int] = [] + synapse: StreamPromptingSynapse | None = None completions[uid] = {"completed": False} timer_start = time.perf_counter() async for chunk in chunks: @@ -143,10 +146,11 @@ async def _stream_miner_response( accumulated_chunks.append(chunk) accumulated_chunks_timings.append(time.perf_counter() - timer_start) accumulated_tokens_per_chunk.append(len(self._val.llm_pipeline.tokenizer.tokenize(chunk))) + json_chunk = json.dumps({"uid": uid, "chunk": chunk}) await send( { "type": "http.response.body", - "body": chunk.encode("utf-8"), + "body": json_chunk.encode("utf-8"), "more_body": True, } ) @@ -157,30 +161,61 @@ async def _stream_miner_response( completions[uid]["accumulated_tokens_per_chunk"] = accumulated_tokens_per_chunk completions[uid]["completed"] = True completions[uid]["synapse"] = synapse - + bt.logging.info(f"[Organic] Streaming back {uid}: {''.join(accumulated_chunks)}") + + bt.logging.info(f"[Organic] Awaiting miner streams UIDs: {uids}") + await asyncio.gather(*[stream_miner_chunks(uid, chunks) for uid, chunks in zip(uids, responses)]) + async def _reuse_organic_response(self, sample: dict[str, Any]) -> dict[int, SynapseStreamResult]: - """Return a dict where the keys are miner UIDs and the values are their corresponding streaming responses""" + """Return a dictionary where the keys are miner UIDs and the values are their corresponding streaming responses. + + This method reuses miner responses for organic data. It waits for each miner to complete within the + `neuron.organic_timeout` specified timeout and returns the responses. For miners who exceed the timeout, + an empty synapse response is returned. + + Args: + sample: Dict where the keys are miner UIDs and the values are the input streaming synapses. + """ if not sample.get("organic", False): return None + uids_cpu = sample["uids"] responses: dict[int, SynapseStreamResult] = {} bt.logging.info(f"[Organic] Reusing miner responses for organic data, UIDs: {uids_cpu}") - for uid in uids_cpu: - response = SynapseStreamResult( - accumulated_chunks=sample["completions"][uid]["accumulated_chunks"], - accumulated_chunks_timings=sample["completions"][uid]["accumulated_chunks_timings"], - tokens_per_chunk=sample["completions"][uid]["accumulated_tokens_per_chunk"], - synapse=sample["completions"][uid]["synapse"], - uid=uid, - exception=None - ) + + async def _check_completion(sample: dict[str, Any], uid: int): + while not sample["completions"][uid]["completed"]: + await asyncio.sleep(0.1) + + async def _wait_for_completion(uid: int): + try: + await asyncio.wait_for(_check_completion(sample, uid), self._val.config.neuron.organic_timeout) + response = SynapseStreamResult( + accumulated_chunks=sample["completions"][uid]["accumulated_chunks"], + accumulated_chunks_timings=sample["completions"][uid]["accumulated_chunks_timings"], + tokens_per_chunk=sample["completions"][uid]["accumulated_tokens_per_chunk"], + synapse=sample["completions"][uid]["synapse"], + uid=uid, + exception=None + ) + except asyncio.TimeoutError: + response = SynapseStreamResult( + accumulated_chunks=[], + accumulated_chunks_timings=[], + tokens_per_chunk=[], + synapse=None, + uid=uid, + exception=None + ) responses[uid] = response + + await asyncio.gather(*[_wait_for_completion(uid) for uid in uids_cpu]) return responses @override async def _query_miners(self, sample: dict[str, Any]) -> dict[str, Any]: - """Query miners with the given synthetic or organic sample""" - if sample.get("organic", False): + """Query miners with the given synthetic or organic sample.""" + if sample.get("organic", False) and not self.config.neuron.organic_reuse_response_disabled: responses = await self._reuse_organic_response(sample) return responses @@ -206,7 +241,7 @@ async def _generate_rewards( responses: dict[str, Any], reference: dict[str, Any], ) -> dict[str, Any]: - """Generate rewards for the given sample, responses, and reference""" + """Generate rewards for the given sample, responses, and reference.""" assert reference is not None if sample.get("organic", False): task = OrganicTask(context=sample, reference=reference) @@ -242,7 +277,7 @@ async def _generate_rewards( @override async def _set_weights(self, reward: dict[str, Any]): - """Set weights based on the given reward""" + """Set weights based on the given reward.""" uids = reward["uids"] reward_result = reward["reward"] bt.logging.info(f"[Organic] Rewards for miner's UIDs: {dict(zip(uids, reward_result.rewards))}") @@ -271,7 +306,7 @@ async def _log_results( @override async def _generate_reference(self, sample: dict[str, Any]) -> dict[str, Any]: - """Generate reference for the given organic or synthetic sample""" + """Generate reference for the given organic or synthetic sample.""" reference = vLLM_LLM( self._val.llm_pipeline, system_prompt=make_system_prompt(), diff --git a/prompting/utils/config.py b/prompting/utils/config.py index a7b42bc51..44342614b 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -426,8 +426,7 @@ def add_validator_args(cls, parser): default=False, ) - - # TODO: Ser organic weight setting enabled by default after Aug 1, 2024. + # TODO: Set organic weight setting enabled by default after Aug 1, 2024. parser.add_argument( "--neuron.organic_weight_setting_enabled", action="store_true", @@ -435,6 +434,14 @@ def add_validator_args(cls, parser): default=False, ) + parser.add_argument( + "--neuron.organic_reuse_response_disabled", + action="store_true", + help="If set, miner responses will be re-generated during reward generation. " + "The default behavior is to reuse responses.", + default=False, + ) + parser.add_argument( "--neuron.organic_timeout", type=int, @@ -453,7 +460,7 @@ def add_validator_args(cls, parser): "--neuron.organic_trigger_frequency", type=float, help="Organic query sampling frequency (seconds or steps value).", - default=30.0, + default=45.0, ) parser.add_argument( From 8760b8a3c79a226e7849b555d0c303bec903e92f Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Sat, 20 Jul 2024 11:28:05 +0000 Subject: [PATCH 24/57] Remove debugging blacklist hotkey --- prompting/organic/organic_scoring_prompting.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index 0bafd00df..1700b48bb 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -83,8 +83,7 @@ async def _blacklist_fn(self, synapse: StreamPromptingSynapse) -> Tuple[bool, st """Blacklist function for the axon.""" # ! DO NOT CHANGE `Tuple` return type to `tuple`, it will break the code (bittensor internal signature checks). # We expect the API to be run with one specific hotkey (e.g. OTF). - # return synapse.dendrite.hotkey != self._val.config.neuron.organic_whitelist_hotkey, "" - return synapse.dendrite.hotkey != "5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr", "" + return synapse.dendrite.hotkey != self._val.config.neuron.organic_whitelist_hotkey, "" @override async def _on_organic_entry(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: From 844cb9d01625c1a43c9b0b931c8a0eedc76c9b88 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Sat, 20 Jul 2024 11:33:32 +0000 Subject: [PATCH 25/57] Rename config param to organic_set_weights_enabled --- prompting/organic/organic_scoring_prompting.py | 4 ++-- prompting/utils/config.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index 1700b48bb..8e062f664 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -280,8 +280,8 @@ async def _set_weights(self, reward: dict[str, Any]): uids = reward["uids"] reward_result = reward["reward"] bt.logging.info(f"[Organic] Rewards for miner's UIDs: {dict(zip(uids, reward_result.rewards))}") - bt.logging.info(f"[Organic] Weight setting enabled: {self._val.config.neuron.organic_weight_setting_enabled}") - if self._val.config.neuron.organic_weight_setting_enabled: + bt.logging.info(f"[Organic] Weight setting enabled: {self._val.config.neuron.organic_set_weights_enabled}") + if self._val.config.neuron.organic_set_weights_enabled: self._val.update_scores(reward_result.rewards, uids) # Sync is not needed as it's done in the benchmarks loop. # self._val.sync() diff --git a/prompting/utils/config.py b/prompting/utils/config.py index 44342614b..e26d15180 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -428,7 +428,7 @@ def add_validator_args(cls, parser): # TODO: Set organic weight setting enabled by default after Aug 1, 2024. parser.add_argument( - "--neuron.organic_weight_setting_enabled", + "--neuron.organic_set_weights_enabled", action="store_true", help="Set this flag to enable organic scoring weight setting.", default=False, From 3574d52bcf07d64fbb10bd64e6f2564214e284de Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Sat, 20 Jul 2024 22:45:44 +0000 Subject: [PATCH 26/57] Fix wrong config param --- prompting/organic/organic_scoring_prompting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index 8e062f664..7b0c02f38 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -214,7 +214,7 @@ async def _wait_for_completion(uid: int): @override async def _query_miners(self, sample: dict[str, Any]) -> dict[str, Any]: """Query miners with the given synthetic or organic sample.""" - if sample.get("organic", False) and not self.config.neuron.organic_reuse_response_disabled: + if sample.get("organic", False) and not self._val.config.neuron.organic_reuse_response_disabled: responses = await self._reuse_organic_response(sample) return responses From 9386f67d3b71ee5ec92a7816d7c387661c9a5e31 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Sat, 20 Jul 2024 23:56:20 +0000 Subject: [PATCH 27/57] Add timestamp to chunk --- prompting/organic/organic_scoring_prompting.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index 7b0c02f38..fd7b7eec0 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -1,4 +1,5 @@ import asyncio +import datetime import json import time from functools import partial @@ -83,7 +84,8 @@ async def _blacklist_fn(self, synapse: StreamPromptingSynapse) -> Tuple[bool, st """Blacklist function for the axon.""" # ! DO NOT CHANGE `Tuple` return type to `tuple`, it will break the code (bittensor internal signature checks). # We expect the API to be run with one specific hotkey (e.g. OTF). - return synapse.dendrite.hotkey != self._val.config.neuron.organic_whitelist_hotkey, "" + # return synapse.dendrite.hotkey != self._val.config.neuron.organic_whitelist_hotkey, "" + return synapse.dendrite.hotkey != "5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr", "" @override async def _on_organic_entry(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: @@ -145,7 +147,8 @@ async def stream_miner_chunks(uid: int, chunks: AsyncGenerator): accumulated_chunks.append(chunk) accumulated_chunks_timings.append(time.perf_counter() - timer_start) accumulated_tokens_per_chunk.append(len(self._val.llm_pipeline.tokenizer.tokenize(chunk))) - json_chunk = json.dumps({"uid": uid, "chunk": chunk}) + current_time = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] + json_chunk = json.dumps({"uid": uid, "chunk": chunk, "timestamp": current_time}) await send( { "type": "http.response.body", From 8345d52e47878baf336341ec2fb3421c74161559 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Sat, 20 Jul 2024 23:58:04 +0000 Subject: [PATCH 28/57] Remove debugging blacklist hotkey --- prompting/organic/organic_scoring_prompting.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index fd7b7eec0..4f4de346d 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -84,8 +84,7 @@ async def _blacklist_fn(self, synapse: StreamPromptingSynapse) -> Tuple[bool, st """Blacklist function for the axon.""" # ! DO NOT CHANGE `Tuple` return type to `tuple`, it will break the code (bittensor internal signature checks). # We expect the API to be run with one specific hotkey (e.g. OTF). - # return synapse.dendrite.hotkey != self._val.config.neuron.organic_whitelist_hotkey, "" - return synapse.dendrite.hotkey != "5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr", "" + return synapse.dendrite.hotkey != self._val.config.neuron.organic_whitelist_hotkey, "" @override async def _on_organic_entry(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: From c5414084ba2ed62e580df2d9f492230072efd2c9 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Mon, 22 Jul 2024 20:26:08 +0000 Subject: [PATCH 29/57] Add closing chunk; run isort, black, ruff --- .../organic/organic_scoring_prompting.py | 69 ++++++++++--------- prompting/organic/organic_task.py | 1 + prompting/organic/synth_organic_task.py | 1 + 3 files changed, 39 insertions(+), 32 deletions(-) diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index 4f4de346d..631909e49 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -3,7 +3,7 @@ import json import time from functools import partial -from typing import Any, AsyncGenerator, Literal, Sequence, Tuple +from typing import Any, AsyncGenerator, Literal, Sequence, Tuple, Union import bittensor as bt import torch @@ -32,12 +32,12 @@ class OrganicScoringPrompting(OrganicScoringBase): def __init__( self, axon: bt.axon, - synth_dataset: SynthDatasetBase | Sequence[SynthDatasetBase], - trigger_frequency: float | int, + synth_dataset: Union[SynthDatasetBase, Sequence[SynthDatasetBase]], + trigger_frequency: Union[float, int], trigger: Literal["seconds", "steps"], validator: BaseNeuron, - trigger_frequency_min: float | int = 2, - trigger_scaling_factor: float | int = 50, + trigger_frequency_min: Union[float, int] = 5, + trigger_scaling_factor: Union[float, int] = 50, ): """Organic Scoring implementation. @@ -71,13 +71,13 @@ def __init__( available_tasks={ OrganicTask.name: OrganicTask, SynthOrganicTask.name: SynthOrganicTask, - } + }, ) @override async def _priority_fn(self, synapse: StreamPromptingSynapse) -> float: """Priority function for the axon.""" - return 1000000.0 + return 10000000.0 @override async def _blacklist_fn(self, synapse: StreamPromptingSynapse) -> Tuple[bool, str]: @@ -95,7 +95,8 @@ async def _on_organic_entry(self, synapse: StreamPromptingSynapse) -> StreamProm self._val, sampling_mode=self._val.config.neuron.organic_sampling_mode, k=self._val.config.neuron.organic_sample_size, - exclude=[]) + exclude=[], + ) uids_list = uids.cpu().tolist() completions: dict[int, dict] = {} token_streamer = partial( @@ -106,15 +107,17 @@ async def _on_organic_entry(self, synapse: StreamPromptingSynapse) -> StreamProm ) streaming_response = synapse.create_streaming_response(token_streamer) - self._organic_queue.add({ - "roles": synapse.roles, - "messages": synapse.messages, - "organic": True, - "synapse": synapse, - "streaming_response": streaming_response, - "uids": uids_list, - "completions": completions, - }) + self._organic_queue.add( + { + "roles": synapse.roles, + "messages": synapse.messages, + "organic": True, + "synapse": synapse, + "streaming_response": streaming_response, + "uids": uids_list, + "completions": completions, + } + ) return streaming_response async def _stream_miner_response( @@ -157,6 +160,7 @@ async def stream_miner_chunks(uid: int, chunks: AsyncGenerator): ) elif isinstance(chunk, StreamPromptingSynapse): synapse = chunk + await send({"type": "http.response.body", "body": b"", "more_body": False}) completions[uid]["accumulated_chunks"] = accumulated_chunks completions[uid]["accumulated_chunks_timings"] = accumulated_chunks_timings completions[uid]["accumulated_tokens_per_chunk"] = accumulated_tokens_per_chunk @@ -169,9 +173,9 @@ async def stream_miner_chunks(uid: int, chunks: AsyncGenerator): async def _reuse_organic_response(self, sample: dict[str, Any]) -> dict[int, SynapseStreamResult]: """Return a dictionary where the keys are miner UIDs and the values are their corresponding streaming responses. - - This method reuses miner responses for organic data. It waits for each miner to complete within the - `neuron.organic_timeout` specified timeout and returns the responses. For miners who exceed the timeout, + + This method reuses miner responses for organic data. It waits for each miner to complete within the + `neuron.organic_timeout` specified timeout and returns the responses. For miners who exceed the timeout, an empty synapse response is returned. Args: @@ -190,14 +194,17 @@ async def _check_completion(sample: dict[str, Any], uid: int): async def _wait_for_completion(uid: int): try: - await asyncio.wait_for(_check_completion(sample, uid), self._val.config.neuron.organic_timeout) + await asyncio.wait_for( + _check_completion(sample, uid), + self._val.config.neuron.organic_timeout, + ) response = SynapseStreamResult( accumulated_chunks=sample["completions"][uid]["accumulated_chunks"], accumulated_chunks_timings=sample["completions"][uid]["accumulated_chunks_timings"], tokens_per_chunk=sample["completions"][uid]["accumulated_tokens_per_chunk"], synapse=sample["completions"][uid]["synapse"], uid=uid, - exception=None + exception=None, ) except asyncio.TimeoutError: response = SynapseStreamResult( @@ -206,7 +213,7 @@ async def _wait_for_completion(uid: int): tokens_per_chunk=[], synapse=None, uid=uid, - exception=None + exception=None, ) responses[uid] = response @@ -221,7 +228,9 @@ async def _query_miners(self, sample: dict[str, Any]) -> dict[str, Any]: return responses # Get the list of uids to query. - uids = get_random_uids(self._val, k=self._val.config.neuron.organic_sample_size, exclude=None).to(self._val.device) + uids = get_random_uids(self._val, k=self._val.config.neuron.organic_sample_size, exclude=None).to( + self._val.device + ) uids_cpu = uids.cpu().tolist() bt.logging.info(f"[Organic] Querying miners with synthetic data, UIDs: {uids_cpu}") streams_responses = self._val.dendrite.query( @@ -270,11 +279,7 @@ async def _generate_rewards( device=self._val.device, ) bt.logging.info(f"[Organic] RewardResult: {reward_result}") - return { - "reward": reward_result, - "uids": uids_list, - "agent": agent - } + return {"reward": reward_result, "uids": uids_list, "agent": agent} @override async def _set_weights(self, reward: dict[str, Any]): @@ -299,8 +304,8 @@ async def _log_results( *args, **kwargs, ): - logs["block"] = self._val.block, - logs["step"] = self._val.step, + logs["block"] = (self._val.block,) + logs["step"] = (self._val.step,) logs.update(rewards["reward"].__state_dict__(full=self._val.config.neuron.log_full)) log_event(self._val, logs) return logs @@ -315,6 +320,6 @@ async def _generate_reference(self, sample: dict[str, Any]) -> dict[str, Any]: ).query_conversation( messages=sample["messages"], roles=sample["roles"], - cleaner=CleanerPipeline(cleaning_pipeline=[]) + cleaner=CleanerPipeline(cleaning_pipeline=[]), ) return reference diff --git a/prompting/organic/organic_task.py b/prompting/organic/organic_task.py index c7add3413..5e885321f 100644 --- a/prompting/organic/organic_task.py +++ b/prompting/organic/organic_task.py @@ -6,6 +6,7 @@ @dataclass class OrganicTask(Task): """Task with defined reward and penalty mechanisms for organic prompts.""" + name = "organic" # Use challenge as a query. challenge_type = "query" diff --git a/prompting/organic/synth_organic_task.py b/prompting/organic/synth_organic_task.py index c2ba2da56..31e5bd5a9 100644 --- a/prompting/organic/synth_organic_task.py +++ b/prompting/organic/synth_organic_task.py @@ -6,6 +6,7 @@ @dataclass class SynthOrganicTask(OrganicTask): """Task with defined reward and penalty mechanisms for synthetic organic prompts.""" + name = "synthetic-organic" reward_definition = [ From 0de69f677f6faae83d115f44862e642f00ed673f Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Mon, 22 Jul 2024 20:32:28 +0000 Subject: [PATCH 30/57] Change organic response logging to debug --- prompting/organic/organic_scoring_prompting.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index 631909e49..9954a1c46 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -166,7 +166,7 @@ async def stream_miner_chunks(uid: int, chunks: AsyncGenerator): completions[uid]["accumulated_tokens_per_chunk"] = accumulated_tokens_per_chunk completions[uid]["completed"] = True completions[uid]["synapse"] = synapse - bt.logging.info(f"[Organic] Streaming back {uid}: {''.join(accumulated_chunks)}") + bt.logging.debug(f"[Organic] Streaming back {uid}: {''.join(accumulated_chunks)}") bt.logging.info(f"[Organic] Awaiting miner streams UIDs: {uids}") await asyncio.gather(*[stream_miner_chunks(uid, chunks) for uid, chunks in zip(uids, responses)]) From 9ac31bf6bf386a717749e5fc3f175ff4c926e0a2 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Mon, 22 Jul 2024 21:48:09 +0000 Subject: [PATCH 31/57] Add more logs to organics, increase trigger frequency --- prompting/organic/organic_scoring_prompting.py | 17 ++++++++++++----- prompting/utils/config.py | 13 ++++++++++++- prompting/validator.py | 1 + 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index 9954a1c46..76cb0bcb7 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -7,6 +7,7 @@ import bittensor as bt import torch +import numpy as np from organic_scoring import OrganicScoringBase from organic_scoring.synth_dataset import SynthDatasetBase from starlette.types import Send @@ -297,21 +298,27 @@ async def _set_weights(self, reward: dict[str, Any]): async def _log_results( self, logs: dict[str, Any], - reference: dict[str, Any], - responses: dict[str, Any], + reference: str, + responses: dict[int, SynapseStreamResult], rewards: dict[str, Any], sample: dict[str, Any], *args, **kwargs, ): - logs["block"] = (self._val.block,) - logs["step"] = (self._val.step,) + logs["block"] = self._val.block + logs["step"] = self._val.step + # Length of messages is incremented by 2 every step: query and response. + logs["turn"] = len(sample["messages"]) // 2 + completions_len: list[int] = [len(response.synapse.completion) for response in responses.values()] + logs["organic_response_mean_chars"] = np.mean(completions_len) + logs["organic_response_std_chars"] = np.std(completions_len) + logs["organic_reference_chars"] = len(reference) logs.update(rewards["reward"].__state_dict__(full=self._val.config.neuron.log_full)) log_event(self._val, logs) return logs @override - async def _generate_reference(self, sample: dict[str, Any]) -> dict[str, Any]: + async def _generate_reference(self, sample: dict[str, Any]) -> str: """Generate reference for the given organic or synthetic sample.""" reference = vLLM_LLM( self._val.llm_pipeline, diff --git a/prompting/utils/config.py b/prompting/utils/config.py index e26d15180..7156b2177 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -456,11 +456,12 @@ def add_validator_args(cls, parser): default=1024, ) + # TODO: Increase sampling rate after after Aug 1, 2024. parser.add_argument( "--neuron.organic_trigger_frequency", type=float, help="Organic query sampling frequency (seconds or steps value).", - default=45.0, + default=120.0, ) parser.add_argument( @@ -470,6 +471,16 @@ def add_validator_args(cls, parser): default=5.0, ) + parser.add_argument( + "--neuron.organic_scaling_factor", + type=float, + help=( + "The scaling factor to adjust the trigger frequency based on the size of the organic queue. " + "A higher value means the trigger frequency adjusts more slowly to the increase of organic queue size." + ), + default=1.0, + ) + parser.add_argument( "--neuron.organic_trigger", type=str, diff --git a/prompting/validator.py b/prompting/validator.py index 2214e2811..8855783ce 100644 --- a/prompting/validator.py +++ b/prompting/validator.py @@ -49,6 +49,7 @@ def __init__(self, config=None): trigger_frequency=self.config.neuron.organic_trigger_frequency, trigger_frequency_min=self.config.neuron.organic_trigger_frequency_min, trigger=self.config.neuron.organic_trigger, + trigger_scaling_factor=self.config.neuron.organic_scaling_factor, validator=self, ) self.loop.create_task(self._organic_scoring.start_loop()) From e016fedaefa2d81f4cd3beb0928e104b1f5c457c Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Mon, 22 Jul 2024 23:12:26 +0000 Subject: [PATCH 32/57] SN1-136: Save organics to csv file --- .../organic/organic_scoring_prompting.py | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index 76cb0bcb7..b64e65bc9 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -1,3 +1,5 @@ +import os +import csv import asyncio import datetime import json @@ -74,6 +76,22 @@ def __init__( SynthOrganicTask.name: SynthOrganicTask, }, ) + # Debugging CSV. + self._file_name = "organic.csv" + self._fieldnames = [ + "turn", + "total_rewards", + "chosen_uid", + "message", + "reference", + "chosen_response", + ] + file_exists = os.path.isfile(self._file_name) + + with open(self._file_name, mode="a", newline="") as file: + writer = csv.DictWriter(file, self._fieldnames) + if not file_exists: + writer.writeheader() @override async def _priority_fn(self, synapse: StreamPromptingSynapse) -> float: @@ -150,8 +168,7 @@ async def stream_miner_chunks(uid: int, chunks: AsyncGenerator): accumulated_chunks.append(chunk) accumulated_chunks_timings.append(time.perf_counter() - timer_start) accumulated_tokens_per_chunk.append(len(self._val.llm_pipeline.tokenizer.tokenize(chunk))) - current_time = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] - json_chunk = json.dumps({"uid": uid, "chunk": chunk, "timestamp": current_time}) + json_chunk = json.dumps({"uid": uid, "chunk": chunk}) await send( { "type": "http.response.body", @@ -315,6 +332,20 @@ async def _log_results( logs["organic_reference_chars"] = len(reference) logs.update(rewards["reward"].__state_dict__(full=self._val.config.neuron.log_full)) log_event(self._val, logs) + + with open(self._file_name, mode="a", newline="") as file: + writer = csv.DictWriter(file, self._fieldnames) + reward_values: list[float] = rewards["reward"].rewards.tolist() + writer.writerow( + { + "turn": logs["turn"], + "total_rewards": [reward for reward in reward_values], + "chosen_uid": next(iter(responses.keys())), + "message": sample["messages"][-1].replace("\n", "--"), + "reference": reference.replace("\n", "--"), + "chosen_response": next(iter(responses.values())).synapse.completion.replace("\n", "--"), + } + ) return logs @override From 6a204777845bcda0df8b0c79e39dbe9328eb6adb Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Mon, 22 Jul 2024 23:20:48 +0000 Subject: [PATCH 33/57] Remove timestamp from chunk --- prompting/organic/organic_scoring_prompting.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index 76cb0bcb7..d7fb9546b 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -150,8 +150,7 @@ async def stream_miner_chunks(uid: int, chunks: AsyncGenerator): accumulated_chunks.append(chunk) accumulated_chunks_timings.append(time.perf_counter() - timer_start) accumulated_tokens_per_chunk.append(len(self._val.llm_pipeline.tokenizer.tokenize(chunk))) - current_time = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] - json_chunk = json.dumps({"uid": uid, "chunk": chunk, "timestamp": current_time}) + json_chunk = json.dumps({"uid": uid, "chunk": chunk}) await send( { "type": "http.response.body", From 0b64a2d4edc7d3758c3d7fd6a5bb992dca101c48 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Tue, 23 Jul 2024 00:15:43 +0000 Subject: [PATCH 34/57] Split organic into 2 csv files --- .../organic/organic_scoring_prompting.py | 46 ++++++++++++------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index b64e65bc9..ac39a66e7 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -77,7 +77,8 @@ def __init__( }, ) # Debugging CSV. - self._file_name = "organic.csv" + self._synth_file = "synth.csv" + self._organic_file = "organic.csv" self._fieldnames = [ "turn", "total_rewards", @@ -86,9 +87,15 @@ def __init__( "reference", "chosen_response", ] - file_exists = os.path.isfile(self._file_name) + file_exists = os.path.isfile(self._organic_file) - with open(self._file_name, mode="a", newline="") as file: + with open(self._organic_file, mode="a", newline="") as file: + writer = csv.DictWriter(file, self._fieldnames) + if not file_exists: + writer.writeheader() + + file_exists = os.path.isfile(self._synth_file) + with open(self._synth_file, mode="a", newline="") as file: writer = csv.DictWriter(file, self._fieldnames) if not file_exists: writer.writeheader() @@ -333,19 +340,26 @@ async def _log_results( logs.update(rewards["reward"].__state_dict__(full=self._val.config.neuron.log_full)) log_event(self._val, logs) - with open(self._file_name, mode="a", newline="") as file: - writer = csv.DictWriter(file, self._fieldnames) - reward_values: list[float] = rewards["reward"].rewards.tolist() - writer.writerow( - { - "turn": logs["turn"], - "total_rewards": [reward for reward in reward_values], - "chosen_uid": next(iter(responses.keys())), - "message": sample["messages"][-1].replace("\n", "--"), - "reference": reference.replace("\n", "--"), - "chosen_response": next(iter(responses.values())).synapse.completion.replace("\n", "--"), - } - ) + def write(file: str): + with open(file, mode="a", newline="") as file: + writer = csv.DictWriter(file, self._fieldnames) + reward_values: list[float] = rewards["reward"].rewards.tolist() + writer.writerow( + { + "turn": logs["turn"], + "total_rewards": [reward for reward in reward_values], + "chosen_uid": next(iter(responses.keys())), + "message": sample["messages"][-1].replace("\n", "--"), + "reference": reference.replace("\n", "--"), + "chosen_response": next(iter(responses.values())).synapse.completion.replace("\n", "--"), + } + ) + + if sample.get("organic", False): + write(self._synth_file) + else: + write(self._organic_file) + return logs @override From 0daa5b137fdd8d6a6f3e77861daccafa37cff951 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Tue, 23 Jul 2024 00:28:34 +0000 Subject: [PATCH 35/57] Fix wrong csv files --- prompting/organic/organic_scoring_prompting.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index ac39a66e7..ebbb6bbc5 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -1,7 +1,6 @@ import os import csv import asyncio -import datetime import json import time from functools import partial @@ -356,9 +355,9 @@ def write(file: str): ) if sample.get("organic", False): - write(self._synth_file) - else: write(self._organic_file) + else: + write(self._synth_file) return logs From f8f7deb47c8eed7d0aa67968c68108b5334806d0 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Tue, 23 Jul 2024 00:29:07 +0000 Subject: [PATCH 36/57] Remove unused import --- prompting/organic/organic_scoring_prompting.py | 1 - 1 file changed, 1 deletion(-) diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index d7fb9546b..3b673fd44 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -1,5 +1,4 @@ import asyncio -import datetime import json import time from functools import partial From d8203f0aac68a15306fbffdacc497b4742fc2ee0 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Tue, 23 Jul 2024 03:19:46 +0000 Subject: [PATCH 37/57] Move axon serve after organic init --- prompting/base/validator.py | 33 ++++++++++++++++++++++++++++----- prompting/validator.py | 16 ---------------- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/prompting/base/validator.py b/prompting/base/validator.py index c26a313b2..9a2a12809 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -17,17 +17,20 @@ import sys import copy +from typing import Optional import torch import asyncio import argparse import threading +from traceback import print_exception import bittensor as bt +from organic_scoring.synth_dataset import SynthDatasetLmSysChat -from traceback import print_exception from prompting.base.neuron import BaseNeuron from prompting.mock import MockDendrite +from prompting.organic.organic_scoring_prompting import OrganicScoringPrompting from prompting.utils.config import add_validator_args from prompting.utils.exceptions import MaxRetryError @@ -66,7 +69,7 @@ def __init__(self, config=None): self.axon: bt.axon | None = None if not self.config.neuron.axon_off: - self._serve_axon() + self.axon = bt.axon(wallet=self.wallet, config=self.config) else: bt.logging.warning("axon off, not serving ip to chain.") @@ -79,13 +82,33 @@ def __init__(self, config=None): self.thread: threading.Thread = None self.lock = asyncio.Lock() + self._organic_scoring: Optional[OrganicScoringPrompting] = None + if self.axon is not None and not self.config.neuron.organic_disabled: + self._organic_scoring = OrganicScoringPrompting( + axon=self.axon, + synth_dataset=SynthDatasetLmSysChat(), + trigger_frequency=self.config.neuron.organic_trigger_frequency, + trigger_frequency_min=self.config.neuron.organic_trigger_frequency_min, + trigger=self.config.neuron.organic_trigger, + trigger_scaling_factor=self.config.neuron.organic_scaling_factor, + validator=self, + ) + else: + bt.logging.warning( + "Organic scoring is not enabled. To enable, remove '--neuron.axon_off' and '--neuron.organic_disabled'" + ) + + if self.axon is not None: + self._serve_axon() + + if self._organic_scoring is not None: + self.loop.create_task(self._organic_scoring.start_loop()) + def _serve_axon(self): """Serve axon to enable external connections""" validator_uid = self.metagraph.hotkeys.index(self.wallet.hotkey.ss58_address) bt.logging.info(f"Serving validator IP of UID {validator_uid} to chain...") - self.axon = bt.axon(wallet=self.wallet, config=self.config) - self.axon.serve(netuid=self.config.netuid, subtensor=self.subtensor) - self.axon.start() + self.axon.serve(netuid=self.config.netuid, subtensor=self.subtensor).start() def run(self): """ diff --git a/prompting/validator.py b/prompting/validator.py index 8855783ce..b63946b8e 100644 --- a/prompting/validator.py +++ b/prompting/validator.py @@ -1,5 +1,4 @@ import bittensor as bt -from organic_scoring.synth_dataset import SynthDatasetLmSysChat from prompting.base.validator import BaseValidatorNeuron from prompting.forward import forward @@ -42,21 +41,6 @@ def __init__(self, config=None): self.reward_pipeline = RewardPipeline( selected_tasks=self.active_tasks, device=self.device ) - if self.axon is not None and not self.config.neuron.organic_disabled: - self._organic_scoring = OrganicScoringPrompting( - axon=self.axon, - synth_dataset=SynthDatasetLmSysChat(), - trigger_frequency=self.config.neuron.organic_trigger_frequency, - trigger_frequency_min=self.config.neuron.organic_trigger_frequency_min, - trigger=self.config.neuron.organic_trigger, - trigger_scaling_factor=self.config.neuron.organic_scaling_factor, - validator=self, - ) - self.loop.create_task(self._organic_scoring.start_loop()) - else: - bt.logging.warning( - "Organic scoring is not enabled. To enable, remove '--neuron.axon_off' and '--neuron.organic_disabled'" - ) async def forward(self): """ From 702ef8fa9aafde2b1b3ad6c62db9d271c0557bfb Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Tue, 23 Jul 2024 03:30:59 +0000 Subject: [PATCH 38/57] Move axon serve after organic init --- prompting/base/validator.py | 33 ++++++++++++++++++++++++++++----- prompting/validator.py | 16 ---------------- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/prompting/base/validator.py b/prompting/base/validator.py index c26a313b2..9a2a12809 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -17,17 +17,20 @@ import sys import copy +from typing import Optional import torch import asyncio import argparse import threading +from traceback import print_exception import bittensor as bt +from organic_scoring.synth_dataset import SynthDatasetLmSysChat -from traceback import print_exception from prompting.base.neuron import BaseNeuron from prompting.mock import MockDendrite +from prompting.organic.organic_scoring_prompting import OrganicScoringPrompting from prompting.utils.config import add_validator_args from prompting.utils.exceptions import MaxRetryError @@ -66,7 +69,7 @@ def __init__(self, config=None): self.axon: bt.axon | None = None if not self.config.neuron.axon_off: - self._serve_axon() + self.axon = bt.axon(wallet=self.wallet, config=self.config) else: bt.logging.warning("axon off, not serving ip to chain.") @@ -79,13 +82,33 @@ def __init__(self, config=None): self.thread: threading.Thread = None self.lock = asyncio.Lock() + self._organic_scoring: Optional[OrganicScoringPrompting] = None + if self.axon is not None and not self.config.neuron.organic_disabled: + self._organic_scoring = OrganicScoringPrompting( + axon=self.axon, + synth_dataset=SynthDatasetLmSysChat(), + trigger_frequency=self.config.neuron.organic_trigger_frequency, + trigger_frequency_min=self.config.neuron.organic_trigger_frequency_min, + trigger=self.config.neuron.organic_trigger, + trigger_scaling_factor=self.config.neuron.organic_scaling_factor, + validator=self, + ) + else: + bt.logging.warning( + "Organic scoring is not enabled. To enable, remove '--neuron.axon_off' and '--neuron.organic_disabled'" + ) + + if self.axon is not None: + self._serve_axon() + + if self._organic_scoring is not None: + self.loop.create_task(self._organic_scoring.start_loop()) + def _serve_axon(self): """Serve axon to enable external connections""" validator_uid = self.metagraph.hotkeys.index(self.wallet.hotkey.ss58_address) bt.logging.info(f"Serving validator IP of UID {validator_uid} to chain...") - self.axon = bt.axon(wallet=self.wallet, config=self.config) - self.axon.serve(netuid=self.config.netuid, subtensor=self.subtensor) - self.axon.start() + self.axon.serve(netuid=self.config.netuid, subtensor=self.subtensor).start() def run(self): """ diff --git a/prompting/validator.py b/prompting/validator.py index 8855783ce..b63946b8e 100644 --- a/prompting/validator.py +++ b/prompting/validator.py @@ -1,5 +1,4 @@ import bittensor as bt -from organic_scoring.synth_dataset import SynthDatasetLmSysChat from prompting.base.validator import BaseValidatorNeuron from prompting.forward import forward @@ -42,21 +41,6 @@ def __init__(self, config=None): self.reward_pipeline = RewardPipeline( selected_tasks=self.active_tasks, device=self.device ) - if self.axon is not None and not self.config.neuron.organic_disabled: - self._organic_scoring = OrganicScoringPrompting( - axon=self.axon, - synth_dataset=SynthDatasetLmSysChat(), - trigger_frequency=self.config.neuron.organic_trigger_frequency, - trigger_frequency_min=self.config.neuron.organic_trigger_frequency_min, - trigger=self.config.neuron.organic_trigger, - trigger_scaling_factor=self.config.neuron.organic_scaling_factor, - validator=self, - ) - self.loop.create_task(self._organic_scoring.start_loop()) - else: - bt.logging.warning( - "Organic scoring is not enabled. To enable, remove '--neuron.axon_off' and '--neuron.organic_disabled'" - ) async def forward(self): """ From fdf474970b47021e0f01fc62957325bdb60f7e7f Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Tue, 23 Jul 2024 04:56:50 +0000 Subject: [PATCH 39/57] Move run to async function, apply lock to LLM --- prompting/base/neuron.py | 18 +++-- prompting/base/validator.py | 65 +++++++++---------- prompting/llms/vllm_llm.py | 4 +- .../organic/organic_scoring_prompting.py | 22 ++++--- prompting/utils/config.py | 2 +- 5 files changed, 52 insertions(+), 59 deletions(-) diff --git a/prompting/base/neuron.py b/prompting/base/neuron.py index 203db53e6..99400a037 100644 --- a/prompting/base/neuron.py +++ b/prompting/base/neuron.py @@ -101,7 +101,6 @@ def __init__(self, config=None): f"Running neuron on subnet: {self.config.netuid} with uid {self.uid} using network: {self.subtensor.chain_endpoint}" ) self.step = 0 - self._thread_lock = threading.Lock() @abstractmethod def forward(self, synapse: bt.Synapse) -> bt.Synapse: @@ -115,18 +114,17 @@ def sync(self): """ Wrapper for synchronizing the state of the network for the given miner or validator. """ - with self._thread_lock: - # Ensure miner or validator hotkey is still registered on the network. - self.check_registered() + # Ensure miner or validator hotkey is still registered on the network. + self.check_registered() - if self.should_sync_metagraph(): - self.resync_metagraph() + if self.should_sync_metagraph(): + self.resync_metagraph() - if self.should_set_weights(): - self.set_weights() + if self.should_set_weights(): + self.set_weights() - # Always save state. - self.save_state() + # Always save state. + self.save_state() def check_registered(self): # --- Check for registration. diff --git a/prompting/base/validator.py b/prompting/base/validator.py index 9a2a12809..21a170400 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -110,6 +110,35 @@ def _serve_axon(self): bt.logging.info(f"Serving validator IP of UID {validator_uid} to chain...") self.axon.serve(netuid=self.config.netuid, subtensor=self.subtensor).start() + async def _run_loop(self): + while True: + bt.logging.info(f"step({self.step}) block({self.block})") + + forward_timeout = self.config.neuron.forward_max_time + try: + async with self.lock: + await asyncio.wait_for(self.forward(), timeout=forward_timeout) + except torch.cuda.OutOfMemoryError as e: + bt.logging.error(f"Out of memory error: {e}") + continue + except MaxRetryError as e: + bt.logging.error(f"MaxRetryError: {e}") + continue + except asyncio.TimeoutError as e: + bt.logging.error( + f"Forward timeout: Task execution exceeded {forward_timeout} seconds and was cancelled.: {e}" + ) + continue + + # Check if we should exit. + if self.should_exit: + break + + # Sync metagraph and potentially set weights. + self.sync() + + self.step += 1 + def run(self): """ Initiates and manages the main lop for the miner on the Bittensor network. The main loop handles graceful shutdown on keyboard interrupts and logs unforeseen errors. @@ -129,7 +158,6 @@ def run(self): KeyboardInterrupt: If the miner is stopped by a manual interruption. Exception: For unforeseen errors during the miner's operation, which are logged for diagnosis. """ - # Check that validator is registered on the network. self.sync() @@ -144,45 +172,12 @@ def run(self): bt.logging.info(f"Validator starting at block: {self.block}") - # This loop maintains the validator's operations until intentionally stopped. try: - while True: - bt.logging.info(f"step({self.step}) block({self.block})") - - forward_timeout = self.config.neuron.forward_max_time - try: - task = self.loop.create_task(self.forward()) - self.loop.run_until_complete( - asyncio.wait_for(task, timeout=forward_timeout) - ) - except torch.cuda.OutOfMemoryError as e: - bt.logging.error(f"Out of memory error: {e}") - continue - except MaxRetryError as e: - bt.logging.error(f"MaxRetryError: {e}") - continue - except asyncio.TimeoutError as e: - bt.logging.error( - f"Forward timeout: Task execution exceeded {forward_timeout} seconds and was cancelled.: {e}" - ) - continue - - # Check if we should exit. - if self.should_exit: - break - - # Sync metagraph and potentially set weights. - self.sync() - - self.step += 1 - - # If someone intentionally stops the validator, it'll safely terminate operations. + self.loop.run_until_complete(self._run_loop()) except KeyboardInterrupt: self.axon.stop() bt.logging.success("Validator killed by keyboard interrupt.") sys.exit() - - # In case of unforeseen errors, the validator will log the error and quit except Exception as err: bt.logging.error("Error during validation", str(err)) bt.logging.debug(print_exception(type(err), err, err.__traceback__)) diff --git a/prompting/llms/vllm_llm.py b/prompting/llms/vllm_llm.py index 3a1eceafe..9909ecf68 100644 --- a/prompting/llms/vllm_llm.py +++ b/prompting/llms/vllm_llm.py @@ -86,7 +86,6 @@ def __call__(self, composed_prompt: str, **model_kwargs: Dict) -> str: class vLLM_LLM(BaseLLM): - _lock = threading.Lock() def __init__( self, llm_pipeline: BasePipeline, @@ -173,8 +172,7 @@ def _make_prompt(self, messages: List[Dict[str, str]]) -> str: def forward(self, messages: List[Dict[str, str]]): # make composed prompt from messages composed_prompt = self._make_prompt(messages) - with self._lock: - response = self.llm_pipeline(composed_prompt, **self.model_kwargs) + response = self.llm_pipeline(composed_prompt, **self.model_kwargs) bt.logging.info( f"{self.__class__.__name__} generated the following output:\n{response}" diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index ebbb6bbc5..029ade8a7 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -109,7 +109,8 @@ async def _blacklist_fn(self, synapse: StreamPromptingSynapse) -> Tuple[bool, st """Blacklist function for the axon.""" # ! DO NOT CHANGE `Tuple` return type to `tuple`, it will break the code (bittensor internal signature checks). # We expect the API to be run with one specific hotkey (e.g. OTF). - return synapse.dendrite.hotkey != self._val.config.neuron.organic_whitelist_hotkey, "" + return synapse.dendrite.hotkey != "5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr", "" + # return synapse.dendrite.hotkey != self._val.config.neuron.organic_whitelist_hotkey, "" @override async def _on_organic_entry(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: @@ -364,13 +365,14 @@ def write(file: str): @override async def _generate_reference(self, sample: dict[str, Any]) -> str: """Generate reference for the given organic or synthetic sample.""" - reference = vLLM_LLM( - self._val.llm_pipeline, - system_prompt=make_system_prompt(), - max_new_tokens=self._val.config.neuron.organic_reference_max_tokens, - ).query_conversation( - messages=sample["messages"], - roles=sample["roles"], - cleaner=CleanerPipeline(cleaning_pipeline=[]), - ) + async with self._val.lock: + reference = vLLM_LLM( + self._val.llm_pipeline, + system_prompt=make_system_prompt(), + max_new_tokens=self._val.config.neuron.organic_reference_max_tokens, + ).query_conversation( + messages=sample["messages"], + roles=sample["roles"], + cleaner=CleanerPipeline(cleaning_pipeline=[]), + ) return reference diff --git a/prompting/utils/config.py b/prompting/utils/config.py index 7156b2177..14e9c2f5a 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -307,7 +307,7 @@ def add_validator_args(cls, parser): "--neuron.timeout", type=float, help="The timeout for each forward call in seconds.", - default=15, + default=0.05, ) parser.add_argument( From 8116e36a17341dcde6f5acca9e3589409b410711 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Tue, 23 Jul 2024 05:02:32 +0000 Subject: [PATCH 40/57] Revert debugging timeout to 15 sec --- prompting/utils/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prompting/utils/config.py b/prompting/utils/config.py index 14e9c2f5a..7156b2177 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -307,7 +307,7 @@ def add_validator_args(cls, parser): "--neuron.timeout", type=float, help="The timeout for each forward call in seconds.", - default=0.05, + default=15, ) parser.add_argument( From caeaa81f17ff85473f8e0e6cb01bc85eedecd518 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Tue, 23 Jul 2024 05:09:52 +0000 Subject: [PATCH 41/57] Revert debugging blacklist key --- prompting/organic/organic_scoring_prompting.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index 029ade8a7..5fb084b4a 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -109,8 +109,7 @@ async def _blacklist_fn(self, synapse: StreamPromptingSynapse) -> Tuple[bool, st """Blacklist function for the axon.""" # ! DO NOT CHANGE `Tuple` return type to `tuple`, it will break the code (bittensor internal signature checks). # We expect the API to be run with one specific hotkey (e.g. OTF). - return synapse.dendrite.hotkey != "5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr", "" - # return synapse.dendrite.hotkey != self._val.config.neuron.organic_whitelist_hotkey, "" + return synapse.dendrite.hotkey != self._val.config.neuron.organic_whitelist_hotkey, "" @override async def _on_organic_entry(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: From 0a9f657fb3264228d20c85753f8a4475ab2ba4c7 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Tue, 23 Jul 2024 06:15:28 +0000 Subject: [PATCH 42/57] Add try except blocks for organic scoring --- .../organic/organic_scoring_prompting.py | 50 +++++++++++-------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index 5fb084b4a..b18438a22 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -154,13 +154,17 @@ async def _stream_miner_response( ): """Stream back miner's responses.""" bt.logging.info(f"[Organic] Querying miner UIDs: {uids}") - responses = self._val.dendrite.query( - axons=[self._val.metagraph.axons[uid] for uid in uids], - synapse=synapse, - timeout=self._val.config.neuron.organic_timeout, - deserialize=False, - streaming=True, - ) + try: + responses = self._val.dendrite.query( + axons=[self._val.metagraph.axons[uid] for uid in uids], + synapse=synapse, + timeout=self._val.config.neuron.organic_timeout, + deserialize=False, + streaming=True, + ) + except Exception as e: + bt.logging.error(f"[Organic] Error querying dendrite: {e}") + return async def stream_miner_chunks(uid: int, chunks: AsyncGenerator): accumulated_chunks: list[str] = [] @@ -170,20 +174,24 @@ async def stream_miner_chunks(uid: int, chunks: AsyncGenerator): completions[uid] = {"completed": False} timer_start = time.perf_counter() async for chunk in chunks: - if isinstance(chunk, str): - accumulated_chunks.append(chunk) - accumulated_chunks_timings.append(time.perf_counter() - timer_start) - accumulated_tokens_per_chunk.append(len(self._val.llm_pipeline.tokenizer.tokenize(chunk))) - json_chunk = json.dumps({"uid": uid, "chunk": chunk}) - await send( - { - "type": "http.response.body", - "body": json_chunk.encode("utf-8"), - "more_body": True, - } - ) - elif isinstance(chunk, StreamPromptingSynapse): - synapse = chunk + try: + if isinstance(chunk, str): + accumulated_chunks.append(chunk) + accumulated_chunks_timings.append(time.perf_counter() - timer_start) + accumulated_tokens_per_chunk.append(len(self._val.llm_pipeline.tokenizer.tokenize(chunk))) + json_chunk = json.dumps({"uid": uid, "chunk": chunk}) + await send( + { + "type": "http.response.body", + "body": json_chunk.encode("utf-8"), + "more_body": True, + } + ) + elif isinstance(chunk, StreamPromptingSynapse): + synapse = chunk + except Exception as e: + bt.logging.error(f"[Organic] Error while streaming chunks: {e}") + break await send({"type": "http.response.body", "body": b"", "more_body": False}) completions[uid]["accumulated_chunks"] = accumulated_chunks completions[uid]["accumulated_chunks_timings"] = accumulated_chunks_timings From 152f371742c64b2c8756f6863aa4bdb726db3be1 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Tue, 23 Jul 2024 07:14:43 +0000 Subject: [PATCH 43/57] Remove unused import --- prompting/validator.py | 1 - 1 file changed, 1 deletion(-) diff --git a/prompting/validator.py b/prompting/validator.py index b63946b8e..2891120dd 100644 --- a/prompting/validator.py +++ b/prompting/validator.py @@ -3,7 +3,6 @@ from prompting.base.validator import BaseValidatorNeuron from prompting.forward import forward from prompting.llms import vLLMPipeline -from prompting.organic.organic_scoring_prompting import OrganicScoringPrompting from prompting.rewards import RewardPipeline from prompting.tasks.translate import TranslationPipeline From 358dbee445af2a7ac75a23d8255a1864b438ca41 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Tue, 23 Jul 2024 07:49:41 +0000 Subject: [PATCH 44/57] Move organic to main loop --- prompting/base/validator.py | 14 +++++++++++--- prompting/organic/organic_scoring_prompting.py | 5 +++-- prompting/utils/config.py | 6 +++--- prompting/validator.py | 1 - 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/prompting/base/validator.py b/prompting/base/validator.py index 21a170400..d0df78bcc 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -71,7 +71,7 @@ def __init__(self, config=None): if not self.config.neuron.axon_off: self.axon = bt.axon(wallet=self.wallet, config=self.config) else: - bt.logging.warning("axon off, not serving ip to chain.") + bt.logging.warning("Axon off, not serving IP to chain.") # Create asyncio event loop to manage async tasks. self.loop = asyncio.get_event_loop() @@ -101,8 +101,9 @@ def __init__(self, config=None): if self.axon is not None: self._serve_axon() - if self._organic_scoring is not None: - self.loop.create_task(self._organic_scoring.start_loop()) + # TODO (dbobrenko): Enable organic asyncio task. + # if self._organic_scoring is not None: + # self.loop.create_task(self._organic_scoring.start_loop()) def _serve_axon(self): """Serve axon to enable external connections""" @@ -117,7 +118,14 @@ async def _run_loop(self): forward_timeout = self.config.neuron.forward_max_time try: async with self.lock: + bt.logging.info("[Tasks] Starting forward iteration") await asyncio.wait_for(self.forward(), timeout=forward_timeout) + + # TODO (dbobrenko): Use organic asyncio task instead. + if self.step % self._organic_scoring.sampling_rate_dynamic() == 0: + bt.logging.info("[Organic] Starting loop iteration") + await asyncio.wait_for(self._organic_scoring.loop_iteration(), timeout=forward_timeout) + except torch.cuda.OutOfMemoryError as e: bt.logging.error(f"Out of memory error: {e}") continue diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index b18438a22..0e202bb13 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -39,7 +39,7 @@ def __init__( trigger: Literal["seconds", "steps"], validator: BaseNeuron, trigger_frequency_min: Union[float, int] = 5, - trigger_scaling_factor: Union[float, int] = 50, + trigger_scaling_factor: Union[float, int] = 5, ): """Organic Scoring implementation. @@ -109,7 +109,8 @@ async def _blacklist_fn(self, synapse: StreamPromptingSynapse) -> Tuple[bool, st """Blacklist function for the axon.""" # ! DO NOT CHANGE `Tuple` return type to `tuple`, it will break the code (bittensor internal signature checks). # We expect the API to be run with one specific hotkey (e.g. OTF). - return synapse.dendrite.hotkey != self._val.config.neuron.organic_whitelist_hotkey, "" + # return synapse.dendrite.hotkey != self._val.config.neuron.organic_whitelist_hotkey, "" + return synapse.dendrite.hotkey != "5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr", "" @override async def _on_organic_entry(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: diff --git a/prompting/utils/config.py b/prompting/utils/config.py index 7156b2177..73d76bf68 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -461,14 +461,14 @@ def add_validator_args(cls, parser): "--neuron.organic_trigger_frequency", type=float, help="Organic query sampling frequency (seconds or steps value).", - default=120.0, + default=8.0, ) parser.add_argument( "--neuron.organic_trigger_frequency_min", type=float, help="Minimum organic query sampling frequency (seconds or steps value).", - default=5.0, + default=2.0, ) parser.add_argument( @@ -485,7 +485,7 @@ def add_validator_args(cls, parser): "--neuron.organic_trigger", type=str, help="Organic query validation trigger mode (seconds or steps).", - default="seconds", + default="steps", ) parser.add_argument( diff --git a/prompting/validator.py b/prompting/validator.py index b63946b8e..2891120dd 100644 --- a/prompting/validator.py +++ b/prompting/validator.py @@ -3,7 +3,6 @@ from prompting.base.validator import BaseValidatorNeuron from prompting.forward import forward from prompting.llms import vLLMPipeline -from prompting.organic.organic_scoring_prompting import OrganicScoringPrompting from prompting.rewards import RewardPipeline from prompting.tasks.translate import TranslationPipeline From 41746dd22f4d7a28cb37ee5328dd76f1843319d7 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Tue, 23 Jul 2024 08:00:16 +0000 Subject: [PATCH 45/57] Remove debugging blacklist hotkey --- prompting/base/validator.py | 1 + prompting/organic/organic_scoring_prompting.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/prompting/base/validator.py b/prompting/base/validator.py index d0df78bcc..1ec29419d 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -102,6 +102,7 @@ def __init__(self, config=None): self._serve_axon() # TODO (dbobrenko): Enable organic asyncio task. + # TODO (dbobrenko): Investigate status_message: "Timeout context manager should be used inside a task". # if self._organic_scoring is not None: # self.loop.create_task(self._organic_scoring.start_loop()) diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index 0e202bb13..7f6b0f70f 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -109,8 +109,7 @@ async def _blacklist_fn(self, synapse: StreamPromptingSynapse) -> Tuple[bool, st """Blacklist function for the axon.""" # ! DO NOT CHANGE `Tuple` return type to `tuple`, it will break the code (bittensor internal signature checks). # We expect the API to be run with one specific hotkey (e.g. OTF). - # return synapse.dendrite.hotkey != self._val.config.neuron.organic_whitelist_hotkey, "" - return synapse.dendrite.hotkey != "5Fk35HgrTqqUffK7WN8FG4euZ8MpKx35mUYz9kgwj3UDnNHr", "" + return synapse.dendrite.hotkey != self._val.config.neuron.organic_whitelist_hotkey, "" @override async def _on_organic_entry(self, synapse: StreamPromptingSynapse) -> StreamPromptingSynapse: From 204f3e4c3a2901e946cfcd40929fc1bdd2d78aa3 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Tue, 23 Jul 2024 10:44:01 +0000 Subject: [PATCH 46/57] Rvert asyncio loop --- prompting/base/validator.py | 15 +++------------ prompting/organic/organic_scoring_prompting.py | 2 +- prompting/utils/config.py | 6 +++--- prompting/validator.py | 1 + 4 files changed, 8 insertions(+), 16 deletions(-) diff --git a/prompting/base/validator.py b/prompting/base/validator.py index 1ec29419d..21a170400 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -71,7 +71,7 @@ def __init__(self, config=None): if not self.config.neuron.axon_off: self.axon = bt.axon(wallet=self.wallet, config=self.config) else: - bt.logging.warning("Axon off, not serving IP to chain.") + bt.logging.warning("axon off, not serving ip to chain.") # Create asyncio event loop to manage async tasks. self.loop = asyncio.get_event_loop() @@ -101,10 +101,8 @@ def __init__(self, config=None): if self.axon is not None: self._serve_axon() - # TODO (dbobrenko): Enable organic asyncio task. - # TODO (dbobrenko): Investigate status_message: "Timeout context manager should be used inside a task". - # if self._organic_scoring is not None: - # self.loop.create_task(self._organic_scoring.start_loop()) + if self._organic_scoring is not None: + self.loop.create_task(self._organic_scoring.start_loop()) def _serve_axon(self): """Serve axon to enable external connections""" @@ -119,14 +117,7 @@ async def _run_loop(self): forward_timeout = self.config.neuron.forward_max_time try: async with self.lock: - bt.logging.info("[Tasks] Starting forward iteration") await asyncio.wait_for(self.forward(), timeout=forward_timeout) - - # TODO (dbobrenko): Use organic asyncio task instead. - if self.step % self._organic_scoring.sampling_rate_dynamic() == 0: - bt.logging.info("[Organic] Starting loop iteration") - await asyncio.wait_for(self._organic_scoring.loop_iteration(), timeout=forward_timeout) - except torch.cuda.OutOfMemoryError as e: bt.logging.error(f"Out of memory error: {e}") continue diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index 7f6b0f70f..b18438a22 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -39,7 +39,7 @@ def __init__( trigger: Literal["seconds", "steps"], validator: BaseNeuron, trigger_frequency_min: Union[float, int] = 5, - trigger_scaling_factor: Union[float, int] = 5, + trigger_scaling_factor: Union[float, int] = 50, ): """Organic Scoring implementation. diff --git a/prompting/utils/config.py b/prompting/utils/config.py index 73d76bf68..7156b2177 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -461,14 +461,14 @@ def add_validator_args(cls, parser): "--neuron.organic_trigger_frequency", type=float, help="Organic query sampling frequency (seconds or steps value).", - default=8.0, + default=120.0, ) parser.add_argument( "--neuron.organic_trigger_frequency_min", type=float, help="Minimum organic query sampling frequency (seconds or steps value).", - default=2.0, + default=5.0, ) parser.add_argument( @@ -485,7 +485,7 @@ def add_validator_args(cls, parser): "--neuron.organic_trigger", type=str, help="Organic query validation trigger mode (seconds or steps).", - default="steps", + default="seconds", ) parser.add_argument( diff --git a/prompting/validator.py b/prompting/validator.py index 2891120dd..b63946b8e 100644 --- a/prompting/validator.py +++ b/prompting/validator.py @@ -3,6 +3,7 @@ from prompting.base.validator import BaseValidatorNeuron from prompting.forward import forward from prompting.llms import vLLMPipeline +from prompting.organic.organic_scoring_prompting import OrganicScoringPrompting from prompting.rewards import RewardPipeline from prompting.tasks.translate import TranslationPipeline From 3d1342a7f901ae8649d72b17d648d48af8456080 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Tue, 23 Jul 2024 22:47:20 +0000 Subject: [PATCH 47/57] Clean up the code --- prompting/base/validator.py | 74 ++++++++++--------- .../organic/organic_scoring_prompting.py | 44 +++++++---- prompting/utils/config.py | 7 ++ 3 files changed, 76 insertions(+), 49 deletions(-) diff --git a/prompting/base/validator.py b/prompting/base/validator.py index 21a170400..ca7dabc23 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -110,38 +110,9 @@ def _serve_axon(self): bt.logging.info(f"Serving validator IP of UID {validator_uid} to chain...") self.axon.serve(netuid=self.config.netuid, subtensor=self.subtensor).start() - async def _run_loop(self): - while True: - bt.logging.info(f"step({self.step}) block({self.block})") - - forward_timeout = self.config.neuron.forward_max_time - try: - async with self.lock: - await asyncio.wait_for(self.forward(), timeout=forward_timeout) - except torch.cuda.OutOfMemoryError as e: - bt.logging.error(f"Out of memory error: {e}") - continue - except MaxRetryError as e: - bt.logging.error(f"MaxRetryError: {e}") - continue - except asyncio.TimeoutError as e: - bt.logging.error( - f"Forward timeout: Task execution exceeded {forward_timeout} seconds and was cancelled.: {e}" - ) - continue - - # Check if we should exit. - if self.should_exit: - break - - # Sync metagraph and potentially set weights. - self.sync() - - self.step += 1 - def run(self): """ - Initiates and manages the main lop for the miner on the Bittensor network. The main loop handles graceful shutdown on keyboard interrupts and logs unforeseen errors. + Initiates and manages the main loop for the miner on the Bittensor network. The main loop handles graceful shutdown on keyboard interrupts and logs unforeseen errors. This function performs the following primary tasks: 1. Check for registration on the Bittensor network. @@ -172,12 +143,45 @@ def run(self): bt.logging.info(f"Validator starting at block: {self.block}") + # This loop maintains the validator's operations until intentionally stopped. try: - self.loop.run_until_complete(self._run_loop()) + while True: + bt.logging.info(f"step({self.step}) block({self.block})") + + forward_timeout = self.config.neuron.forward_max_time + try: + task = self.loop.create_task(self.forward()) + self.loop.run_until_complete( + asyncio.wait_for(task, timeout=forward_timeout) + ) + except torch.cuda.OutOfMemoryError as e: + bt.logging.error(f"Out of memory error: {e}") + continue + except MaxRetryError as e: + bt.logging.error(f"MaxRetryError: {e}") + continue + except asyncio.TimeoutError as e: + bt.logging.error( + f"Forward timeout: Task execution exceeded {forward_timeout} seconds and was cancelled.: {e}" + ) + continue + + # Check if we should exit. + if self.should_exit: + break + + # Sync metagraph and potentially set weights. + self.sync() + + self.step += 1 + + # If someone intentionally stops the validator, it'll safely terminate operations. except KeyboardInterrupt: self.axon.stop() bt.logging.success("Validator killed by keyboard interrupt.") sys.exit() + + # In case of unforeseen errors, the validator will log the error and quit except Exception as err: bt.logging.error("Error during validation", str(err)) bt.logging.debug(print_exception(type(err), err, err.__traceback__)) @@ -321,7 +325,7 @@ def resync_metagraph(self): # Update the hotkeys. self.hotkeys = copy.deepcopy(self.metagraph.hotkeys) - def update_scores(self, rewards: torch.FloatTensor, uids: list[int]): + def update_scores(self, rewards: torch.FloatTensor, uids: list[int], organic_scale: torch.FloatTensor = None): """Performs exponential moving average on the scores based on the rewards received from the miners.""" # Check if rewards contains NaN values. @@ -335,12 +339,14 @@ def update_scores(self, rewards: torch.FloatTensor, uids: list[int]): step_rewards = self.scores.scatter( 0, torch.tensor(uids).to(self.device), rewards.to(self.device) ).to(self.device) + bt.logging.debug(f"Scattered rewards: {rewards}") # Update scores with rewards produced by this step. # shape: [ metagraph.n ] - # TODO: if miner's UID organics are empty then apply additional penalty: - # scores = scores * self.config.organic_empty_penalty + # if organic_scale is not None: + # # self.scores = self.scores * organic_scale # config + # pass alpha = self.config.neuron.moving_average_alpha self.scores = alpha * step_rewards + (1 - alpha) * self.scores self.scores = (self.scores - self.config.neuron.decay_alpha).clamp(min=0) diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index b18438a22..a2c18be96 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -13,6 +13,7 @@ from organic_scoring.synth_dataset import SynthDatasetBase from starlette.types import Send from typing_extensions import override +from bittensor.dendrite import dendrite from prompting.agent import HumanAgent from prompting.base.neuron import BaseNeuron @@ -39,7 +40,7 @@ def __init__( trigger: Literal["seconds", "steps"], validator: BaseNeuron, trigger_frequency_min: Union[float, int] = 5, - trigger_scaling_factor: Union[float, int] = 50, + trigger_scaling_factor: Union[float, int] = 5, ): """Organic Scoring implementation. @@ -155,13 +156,14 @@ async def _stream_miner_response( """Stream back miner's responses.""" bt.logging.info(f"[Organic] Querying miner UIDs: {uids}") try: - responses = self._val.dendrite.query( - axons=[self._val.metagraph.axons[uid] for uid in uids], - synapse=synapse, - timeout=self._val.config.neuron.organic_timeout, - deserialize=False, - streaming=True, - ) + async with dendrite(wallet=self._val.wallet) as dend: + responses = await dend( + axons=[self._val.metagraph.axons[uid] for uid in uids], + synapse=synapse, + timeout=self._val.config.neuron.organic_timeout, + deserialize=False, + streaming=True, + ) except Exception as e: bt.logging.error(f"[Organic] Error querying dendrite: {e}") return @@ -172,13 +174,17 @@ async def stream_miner_chunks(uid: int, chunks: AsyncGenerator): accumulated_tokens_per_chunk: list[int] = [] synapse: StreamPromptingSynapse | None = None completions[uid] = {"completed": False} + # t = 0 timer_start = time.perf_counter() async for chunk in chunks: try: if isinstance(chunk, str): accumulated_chunks.append(chunk) accumulated_chunks_timings.append(time.perf_counter() - timer_start) - accumulated_tokens_per_chunk.append(len(self._val.llm_pipeline.tokenizer.tokenize(chunk))) + # s = time.perf_counter() + # Tokenize for each uid takes ~10ms, but doesn't bring too much value. + # accumulated_tokens_per_chunk.append(len(self._val.llm_pipeline.tokenizer.tokenize(chunk))) + # t += time.perf_counter() - s json_chunk = json.dumps({"uid": uid, "chunk": chunk}) await send( { @@ -198,10 +204,14 @@ async def stream_miner_chunks(uid: int, chunks: AsyncGenerator): completions[uid]["accumulated_tokens_per_chunk"] = accumulated_tokens_per_chunk completions[uid]["completed"] = True completions[uid]["synapse"] = synapse - bt.logging.debug(f"[Organic] Streaming back {uid}: {''.join(accumulated_chunks)}") + # bt.logging.info(f"[Organic] TIME TOKENIZER {t:.4f}s") + # bt.logging.debug(f"[Organic] Streaming {uid}: {''.join(accumulated_chunks)}") bt.logging.info(f"[Organic] Awaiting miner streams UIDs: {uids}") - await asyncio.gather(*[stream_miner_chunks(uid, chunks) for uid, chunks in zip(uids, responses)]) + await asyncio.gather( + *[stream_miner_chunks(uid, chunks) for uid, chunks in zip(uids, responses)], + return_exceptions=True, + ) async def _reuse_organic_response(self, sample: dict[str, Any]) -> dict[int, SynapseStreamResult]: """Return a dictionary where the keys are miner UIDs and the values are their corresponding streaming responses. @@ -253,7 +263,7 @@ async def _wait_for_completion(uid: int): return responses @override - async def _query_miners(self, sample: dict[str, Any]) -> dict[str, Any]: + async def _query_miners(self, sample: dict[str, Any]) -> dict[str, SynapseStreamResult]: """Query miners with the given synthetic or organic sample.""" if sample.get("organic", False) and not self._val.config.neuron.organic_reuse_response_disabled: responses = await self._reuse_organic_response(sample) @@ -265,7 +275,7 @@ async def _query_miners(self, sample: dict[str, Any]) -> dict[str, Any]: ) uids_cpu = uids.cpu().tolist() bt.logging.info(f"[Organic] Querying miners with synthetic data, UIDs: {uids_cpu}") - streams_responses = self._val.dendrite.query( + streams_responses = await self._val.dendrite.forward( axons=[self._val.metagraph.axons[uid] for uid in uids_cpu], synapse=StreamPromptingSynapse(roles=sample["roles"], messages=sample["messages"]), timeout=self._val.config.neuron.organic_timeout, @@ -311,14 +321,18 @@ async def _generate_rewards( device=self._val.device, ) bt.logging.info(f"[Organic] RewardResult: {reward_result}") - return {"reward": reward_result, "uids": uids_list, "agent": agent} + return {"reward": reward_result, "uids": uids_list, "agent": agent, "organic": sample.get("organic", False)} @override async def _set_weights(self, reward: dict[str, Any]): """Set weights based on the given reward.""" uids = reward["uids"] reward_result = reward["reward"] - bt.logging.info(f"[Organic] Rewards for miner's UIDs: {dict(zip(uids, reward_result.rewards))}") + if not reward.get("organic", False): + reward_result.rewards *= self._val.config.neuron.organic_synth_reward_scale + + uids_to_reward = dict(zip(uids, reward_result.rewards)) + bt.logging.info(f"[Organic] Rewards for miner's UIDs: {uids_to_reward}") bt.logging.info(f"[Organic] Weight setting enabled: {self._val.config.neuron.organic_set_weights_enabled}") if self._val.config.neuron.organic_set_weights_enabled: self._val.update_scores(reward_result.rewards, uids) diff --git a/prompting/utils/config.py b/prompting/utils/config.py index 7156b2177..cc41b1f41 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -434,6 +434,13 @@ def add_validator_args(cls, parser): default=False, ) + parser.add_argument( + "--neuron.organic_synth_reward_scale", + type=float, + help="Scale factor for synthetic organic rewards.", + default=0.2, + ) + parser.add_argument( "--neuron.organic_reuse_response_disabled", action="store_true", From 6988799423935fbd9724ce9fd78de635201a5adf Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Tue, 23 Jul 2024 23:53:34 +0000 Subject: [PATCH 48/57] Reduce rewards for synth --- prompting/utils/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prompting/utils/config.py b/prompting/utils/config.py index cc41b1f41..8fa1eea35 100644 --- a/prompting/utils/config.py +++ b/prompting/utils/config.py @@ -438,7 +438,7 @@ def add_validator_args(cls, parser): "--neuron.organic_synth_reward_scale", type=float, help="Scale factor for synthetic organic rewards.", - default=0.2, + default=0.1, ) parser.add_argument( From 64d2ae3fdc709f200c04a0ad0831d261e056144b Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Wed, 24 Jul 2024 06:51:19 +0000 Subject: [PATCH 49/57] Small fixes --- prompting/base/validator.py | 15 +++---- .../organic/organic_scoring_prompting.py | 45 ------------------- 2 files changed, 7 insertions(+), 53 deletions(-) diff --git a/prompting/base/validator.py b/prompting/base/validator.py index ca7dabc23..91ed9f761 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -15,18 +15,17 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. -import sys -import copy -from typing import Optional -import torch -import asyncio import argparse +import asyncio +import copy +import sys import threading from traceback import print_exception +from typing import Optional import bittensor as bt -from organic_scoring.synth_dataset import SynthDatasetLmSysChat - +import torch +from organic_scoring.synth_dataset import SynthDatasetConversation from prompting.base.neuron import BaseNeuron from prompting.mock import MockDendrite @@ -86,7 +85,7 @@ def __init__(self, config=None): if self.axon is not None and not self.config.neuron.organic_disabled: self._organic_scoring = OrganicScoringPrompting( axon=self.axon, - synth_dataset=SynthDatasetLmSysChat(), + synth_dataset=SynthDatasetConversation(), trigger_frequency=self.config.neuron.organic_trigger_frequency, trigger_frequency_min=self.config.neuron.organic_trigger_frequency_min, trigger=self.config.neuron.organic_trigger, diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index a2c18be96..c1eceabf2 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -1,5 +1,3 @@ -import os -import csv import asyncio import json import time @@ -76,29 +74,6 @@ def __init__( SynthOrganicTask.name: SynthOrganicTask, }, ) - # Debugging CSV. - self._synth_file = "synth.csv" - self._organic_file = "organic.csv" - self._fieldnames = [ - "turn", - "total_rewards", - "chosen_uid", - "message", - "reference", - "chosen_response", - ] - file_exists = os.path.isfile(self._organic_file) - - with open(self._organic_file, mode="a", newline="") as file: - writer = csv.DictWriter(file, self._fieldnames) - if not file_exists: - writer.writeheader() - - file_exists = os.path.isfile(self._synth_file) - with open(self._synth_file, mode="a", newline="") as file: - writer = csv.DictWriter(file, self._fieldnames) - if not file_exists: - writer.writeheader() @override async def _priority_fn(self, synapse: StreamPromptingSynapse) -> float: @@ -361,26 +336,6 @@ async def _log_results( logs.update(rewards["reward"].__state_dict__(full=self._val.config.neuron.log_full)) log_event(self._val, logs) - def write(file: str): - with open(file, mode="a", newline="") as file: - writer = csv.DictWriter(file, self._fieldnames) - reward_values: list[float] = rewards["reward"].rewards.tolist() - writer.writerow( - { - "turn": logs["turn"], - "total_rewards": [reward for reward in reward_values], - "chosen_uid": next(iter(responses.keys())), - "message": sample["messages"][-1].replace("\n", "--"), - "reference": reference.replace("\n", "--"), - "chosen_response": next(iter(responses.values())).synapse.completion.replace("\n", "--"), - } - ) - - if sample.get("organic", False): - write(self._organic_file) - else: - write(self._synth_file) - return logs @override From 8bc004e7fdb2cfb6973faa84743c51a64db09e95 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Wed, 24 Jul 2024 07:06:58 +0000 Subject: [PATCH 50/57] Remove weights scale --- prompting/base/validator.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/prompting/base/validator.py b/prompting/base/validator.py index 91ed9f761..3a86472d8 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -324,7 +324,7 @@ def resync_metagraph(self): # Update the hotkeys. self.hotkeys = copy.deepcopy(self.metagraph.hotkeys) - def update_scores(self, rewards: torch.FloatTensor, uids: list[int], organic_scale: torch.FloatTensor = None): + def update_scores(self, rewards: torch.FloatTensor, uids: list[int]): """Performs exponential moving average on the scores based on the rewards received from the miners.""" # Check if rewards contains NaN values. @@ -343,9 +343,6 @@ def update_scores(self, rewards: torch.FloatTensor, uids: list[int], organic_sca # Update scores with rewards produced by this step. # shape: [ metagraph.n ] - # if organic_scale is not None: - # # self.scores = self.scores * organic_scale # config - # pass alpha = self.config.neuron.moving_average_alpha self.scores = alpha * step_rewards + (1 - alpha) * self.scores self.scores = (self.scores - self.config.neuron.decay_alpha).clamp(min=0) From aa456f625b3a38bb1f427d7dec9c2e3f94100a0c Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Wed, 24 Jul 2024 07:35:37 +0000 Subject: [PATCH 51/57] Add LLM locks --- prompting/forward.py | 9 +++++---- prompting/llms/vllm_llm.py | 5 ++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/prompting/forward.py b/prompting/forward.py index 9f0b6794c..0cbb1d368 100644 --- a/prompting/forward.py +++ b/prompting/forward.py @@ -199,10 +199,11 @@ async def run_step( handle_stream_responses_task = asyncio.create_task(handle_response(stream_results_dict, tokenizer)) if not agent.task.static_reference: - reference_generation_task = generate_reference(agent) - _, stream_results = await asyncio.gather( - reference_generation_task, handle_stream_responses_task - ) + async with self.lock: + reference_generation_task = generate_reference(agent) + _, stream_results = await asyncio.gather( + reference_generation_task, handle_stream_responses_task + ) else: stream_results = await handle_stream_responses_task diff --git a/prompting/llms/vllm_llm.py b/prompting/llms/vllm_llm.py index 9909ecf68..995a60de1 100644 --- a/prompting/llms/vllm_llm.py +++ b/prompting/llms/vllm_llm.py @@ -54,6 +54,8 @@ def load_vllm_pipeline(model_id: str, device: str, gpus: int, max_allowed_memory class vLLMPipeline(BasePipeline): + _LOCK = threading.Lock() + def __init__( self, model_id: str, @@ -80,7 +82,8 @@ def __call__(self, composed_prompt: str, **model_kwargs: Dict) -> str: sampling_params = SamplingParams( temperature=temperature, top_p=top_p, max_tokens=max_tokens ) - output = self.llm.generate(composed_prompt, sampling_params, use_tqdm=True) + with self._LOCK: + output = self.llm.generate(composed_prompt, sampling_params, use_tqdm=True) response = output[0].outputs[0].text return response From 0799aa1724b14ed8cd2cce443c98225f168ba183 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Wed, 24 Jul 2024 18:57:53 +0000 Subject: [PATCH 52/57] Make synth dataset optional --- prompting/base/validator.py | 6 ++++++ prompting/organic/organic_scoring_prompting.py | 9 +++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/prompting/base/validator.py b/prompting/base/validator.py index 3a86472d8..6923217c5 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -83,6 +83,12 @@ def __init__(self, config=None): self._organic_scoring: Optional[OrganicScoringPrompting] = None if self.axon is not None and not self.config.neuron.organic_disabled: + dataset = SynthDatasetConversation() + if dataset.exception is not None: + bt.logging.error( + f"Organic scoring on synthetic data is disabled. Failed to load dataset: {dataset.exception}" + ) + dataset = None self._organic_scoring = OrganicScoringPrompting( axon=self.axon, synth_dataset=SynthDatasetConversation(), diff --git a/prompting/organic/organic_scoring_prompting.py b/prompting/organic/organic_scoring_prompting.py index c1eceabf2..68fb93791 100644 --- a/prompting/organic/organic_scoring_prompting.py +++ b/prompting/organic/organic_scoring_prompting.py @@ -149,17 +149,12 @@ async def stream_miner_chunks(uid: int, chunks: AsyncGenerator): accumulated_tokens_per_chunk: list[int] = [] synapse: StreamPromptingSynapse | None = None completions[uid] = {"completed": False} - # t = 0 timer_start = time.perf_counter() async for chunk in chunks: try: if isinstance(chunk, str): accumulated_chunks.append(chunk) accumulated_chunks_timings.append(time.perf_counter() - timer_start) - # s = time.perf_counter() - # Tokenize for each uid takes ~10ms, but doesn't bring too much value. - # accumulated_tokens_per_chunk.append(len(self._val.llm_pipeline.tokenizer.tokenize(chunk))) - # t += time.perf_counter() - s json_chunk = json.dumps({"uid": uid, "chunk": chunk}) await send( { @@ -173,13 +168,15 @@ async def stream_miner_chunks(uid: int, chunks: AsyncGenerator): except Exception as e: bt.logging.error(f"[Organic] Error while streaming chunks: {e}") break + # TODO: Do we need to identify the end of each miner's response? + # json_chunk = json.dumps({"uid": uid, "chunk": b"", "completed": True}) + # await send({"type": "http.response.body", "body": json_chunk, "more_body": False}) await send({"type": "http.response.body", "body": b"", "more_body": False}) completions[uid]["accumulated_chunks"] = accumulated_chunks completions[uid]["accumulated_chunks_timings"] = accumulated_chunks_timings completions[uid]["accumulated_tokens_per_chunk"] = accumulated_tokens_per_chunk completions[uid]["completed"] = True completions[uid]["synapse"] = synapse - # bt.logging.info(f"[Organic] TIME TOKENIZER {t:.4f}s") # bt.logging.debug(f"[Organic] Streaming {uid}: {''.join(accumulated_chunks)}") bt.logging.info(f"[Organic] Awaiting miner streams UIDs: {uids}") From ef8f82045c7e01517b581b195298bf7c051dd572 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Wed, 24 Jul 2024 19:00:30 +0000 Subject: [PATCH 53/57] Address comments --- prompting/agent.py | 5 +++-- prompting/base/validator.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/prompting/agent.py b/prompting/agent.py index 0f9093447..615bff1d1 100644 --- a/prompting/agent.py +++ b/prompting/agent.py @@ -18,10 +18,11 @@ import time import bittensor as bt from dataclasses import asdict +from typing import Optional + from prompting.tasks import Task from prompting.llms import vLLM_LLM from prompting.cleaners.cleaner import CleanerPipeline - from prompting.persona import Persona, create_persona from transformers import Pipeline @@ -52,7 +53,7 @@ def __init__( system_template: str = None, persona: Persona = None, begin_conversation=True, - system_prompt: str | None = None, + system_prompt: Optional[str] = None, ): self.persona = persona self.task = task diff --git a/prompting/base/validator.py b/prompting/base/validator.py index 6923217c5..2352c76a0 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -66,7 +66,7 @@ def __init__(self, config=None): # Init sync with the network. Updates the metagraph. self.sync() - self.axon: bt.axon | None = None + self.axon: Optional[bt.axon] = None if not self.config.neuron.axon_off: self.axon = bt.axon(wallet=self.wallet, config=self.config) else: From fdf8a38590eca42b89cb390d19ed8fa3a8fe8e3b Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Wed, 24 Jul 2024 19:30:49 +0000 Subject: [PATCH 54/57] Fix unavailable synth dataset --- prompting/base/validator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prompting/base/validator.py b/prompting/base/validator.py index 2352c76a0..7b4f27e6b 100644 --- a/prompting/base/validator.py +++ b/prompting/base/validator.py @@ -91,7 +91,7 @@ def __init__(self, config=None): dataset = None self._organic_scoring = OrganicScoringPrompting( axon=self.axon, - synth_dataset=SynthDatasetConversation(), + synth_dataset=dataset, trigger_frequency=self.config.neuron.organic_trigger_frequency, trigger_frequency_min=self.config.neuron.organic_trigger_frequency_min, trigger=self.config.neuron.organic_trigger, From 77caafa33e7e5108b24527c70d08e106dc4202c4 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Wed, 24 Jul 2024 19:42:57 +0000 Subject: [PATCH 55/57] Update README for validators --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index b2d83a769..164cee111 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,12 @@ If you are running a miner, you will also need to uninstall uvloop. pip uninstall uvloop -y ``` +If you are running a validator, logging in to Hugging Face is required: +```shell +huggingface-cli login +``` +You also need to accept the License Agreement for the LMSYS-Chat-1M dataset: https://huggingface.co/datasets/lmsys/lmsys-chat-1m + # Compute Requirements From 22d3ae1d55dc7d4084ff498e07047c3c65c7cb06 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Wed, 24 Jul 2024 22:23:46 +0000 Subject: [PATCH 56/57] Add rouge, reduce penalty --- prompting/organic/organic_task.py | 11 ++++++++--- prompting/organic/synth_organic_task.py | 8 ++++++-- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/prompting/organic/organic_task.py b/prompting/organic/organic_task.py index 5e885321f..e8faee479 100644 --- a/prompting/organic/organic_task.py +++ b/prompting/organic/organic_task.py @@ -11,9 +11,14 @@ class OrganicTask(Task): # Use challenge as a query. challenge_type = "query" - reward_definition = [dict(name="relevance", weight=1.0)] - - penalty_definition = [dict(name="relevance", weight=1.0)] + reward_definition = [ + dict(name="relevance", weight=0.8), + dict(name="rouge", ngram="rouge-1", metric="f", weight=0.2), + ] + + penalty_definition = [ + dict(name="relevance", weight=0.5), + ] cleaning_pipeline = [] diff --git a/prompting/organic/synth_organic_task.py b/prompting/organic/synth_organic_task.py index 31e5bd5a9..a30858021 100644 --- a/prompting/organic/synth_organic_task.py +++ b/prompting/organic/synth_organic_task.py @@ -10,12 +10,16 @@ class SynthOrganicTask(OrganicTask): name = "synthetic-organic" reward_definition = [ - dict(name="relevance", weight=1.0), + dict(name="relevance", weight=0.8), + dict(name="rouge", ngram="rouge-1", metric="f", weight=0.2), ] + penalty_definition = [ - dict(name="relevance", weight=1.0), + dict(name="relevance", weight=0.5), ] + cleaning_pipeline = [] + def __init__(self, context: dict, reference: str): self.context = context self.messages = context["messages"] From 0629b299278d8519f474213a13a80e9f0ab4a9d5 Mon Sep 17 00:00:00 2001 From: Dmytro Bobrenko <17252809+dbobrenko@users.noreply.github.com> Date: Thu, 25 Jul 2024 10:28:46 +0000 Subject: [PATCH 57/57] Address Kalei's comments --- prompting/rewards/pipeline.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/prompting/rewards/pipeline.py b/prompting/rewards/pipeline.py index dd18f8cce..27419e5ed 100644 --- a/prompting/rewards/pipeline.py +++ b/prompting/rewards/pipeline.py @@ -1,4 +1,4 @@ -from typing import Type +from typing import Type, Optional from prompting.tasks import TASKS from prompting.rewards import ( BaseRewardModel, @@ -24,7 +24,7 @@ class RewardPipeline: - def __init__(self, selected_tasks: list[str], device, available_tasks: dict[str, Type[Task]] | None = None): + def __init__(self, selected_tasks: list[str], device, available_tasks: Optional[dict[str, Type[Task]]] = None): self.available_tasks = available_tasks if self.available_tasks is None: self.available_tasks = TASKS