From 191cba3dc261229121a8cf00a1c23231fea612b5 Mon Sep 17 00:00:00 2001 From: eclipsevortex Date: Thu, 7 Mar 2024 18:00:47 +0000 Subject: [PATCH] fix: implement new incentive mechanism --- .gitignore | 2 + VERSION | 2 +- docker-compose.yml | 19 + neurons/miner.py | 127 ++-- neurons/validator.py | 34 +- scripts/docs/TESTNET.md | 11 +- scripts/redis/docker/Dockerfile | 21 + scripts/redis/docker/entrypoint.sh | 4 + scripts/redis/docker/redis_check.py | 19 + scripts/subtensor/setup.sh | 32 +- subnet/__init__.py | 2 +- subnet/constants.py | 73 +-- subnet/localisation.json | 246 ++++++++ subnet/miner/run.py | 237 +------ subnet/miner/utils.py | 270 +------- subnet/protocol.py | 11 +- subnet/shared/checks.py | 18 +- subnet/shared/utils.py | 108 ---- subnet/shared/weights.py | 13 +- subnet/validator/bonding.py | 87 +-- subnet/validator/challenge-2.py | 108 ++++ subnet/validator/challenge.py | 214 ++++--- subnet/validator/config.py | 50 -- subnet/validator/database.py | 937 +--------------------------- subnet/validator/encryption.py | 425 ------------- subnet/validator/forward.py | 90 +-- subnet/validator/key.py | 159 ++--- subnet/validator/localisation.py | 52 ++ subnet/validator/metric.py | 91 ++- subnet/validator/metrics.py | 99 ++- subnet/validator/network.py | 228 ------- subnet/validator/rebalance.py | 89 --- subnet/validator/reward.py | 2 +- subnet/validator/score.py | 97 +++ subnet/validator/ssh.py | 13 +- subnet/validator/state.py | 35 +- subnet/validator/subtensor copy.py | 104 +++ subnet/validator/subtensor-2.py | 116 ++++ subnet/validator/subtensor.py | 67 +- subnet/validator/utils.py | 798 +++-------------------- subnet/validator/weights.py | 48 +- 41 files changed, 1586 insertions(+), 3572 deletions(-) create mode 100644 docker-compose.yml create mode 100644 scripts/redis/docker/Dockerfile create mode 100755 scripts/redis/docker/entrypoint.sh create mode 100644 scripts/redis/docker/redis_check.py create mode 100644 subnet/localisation.json create mode 100644 subnet/validator/challenge-2.py delete mode 100644 subnet/validator/encryption.py create mode 100644 subnet/validator/localisation.py delete mode 100644 subnet/validator/network.py delete mode 100644 subnet/validator/rebalance.py create mode 100644 subnet/validator/score.py create mode 100644 subnet/validator/subtensor copy.py create mode 100644 subnet/validator/subtensor-2.py diff --git a/.gitignore b/.gitignore index ca327fe0..9229f2d5 100644 --- a/.gitignore +++ b/.gitignore @@ -154,6 +154,8 @@ dmypy.json # Cython debug symbols cython_debug/ +*.key + # PyCharm # JetBrains specific template is maintained in a separate JetBrains.gitignore that can # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore diff --git a/VERSION b/VERSION index 6e8bf73a..341cf11f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.1.0 +0.2.0 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..8c4df257 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,19 @@ +version: "3" + +services: + redis: + build: + context: . + dockerfile: ./scripts/redis/docker/Dockerfile + args: + - REDIS_PASSWORD=${REDIS_PASSWORD} + container_name: redis + restart: always + ports: + - 6379:6379 + volumes: + - cache:/data + +volumes: + cache: + driver: local \ No newline at end of file diff --git a/neurons/miner.py b/neurons/miner.py index 511a5a29..97f5def7 100644 --- a/neurons/miner.py +++ b/neurons/miner.py @@ -17,22 +17,21 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. -import os import sys import typing import time +import json import torch import asyncio import bittensor as bt import threading import traceback from urllib.parse import urlparse -from redis import asyncio as aioredis -from subnet.protocol import IsAlive, Key, Subtensor, Challenge +from subnet.protocol import IsAlive, Key, Subtensor, Challenge, Score from subnet.shared.key import generate_ssh_key, clean_ssh_key -from subnet.shared.checks import check_environment +from subnet.shared.checks import check_environment, check_registration from subnet.miner import run from subnet.miner.config import ( @@ -40,14 +39,8 @@ check_config, add_args, ) -from subnet.shared.utils import ( - get_redis_password, -) -from subnet.miner.utils import ( - update_storage_stats, - load_request_log, - get_purge_ttl_script_path, -) +from subnet.miner.utils import load_request_log + class Miner: @classmethod @@ -121,14 +114,7 @@ def __init__(self): bt.logging.debug("loading wallet") self.wallet = bt.wallet(config=self.config) self.wallet.create_if_non_existent() - if not self.config.wallet._mock: - if not self.subtensor.is_hotkey_registered_on_subnet( - hotkey_ss58=self.wallet.hotkey.ss58_address, netuid=self.config.netuid - ): - raise Exception( - f"Wallet not currently registered on netuid {self.config.netuid}, please first register wallet before running" - ) - + check_registration(self.subtensor, self.wallet, self.config.netuid) bt.logging.debug(f"wallet: {str(self.wallet)}") # Init metagraph. @@ -144,9 +130,8 @@ def __init__(self): ) bt.logging.info(f"Running miner on uid: {self.my_subnet_uid}") - # The axon handles request processing, allowing validators to send this process requests. - self.axon = bt.axon(wallet=self.wallet, config=self.config) + self.axon = bt.axon(wallet=self.wallet, config=self.config, external_ip=bt.net.get_external_ip()) bt.logging.info(f"Axon {self.axon}") # Attach determiners which functions are called when servicing a request. @@ -155,16 +140,29 @@ def __init__(self): forward_fn=self._is_alive, blacklist_fn=self.blacklist_isalive, ).attach( - forward_fn=self._generate_key, - blacklist_fn=self.blacklist_generate_key, - ).attach( - forward_fn=self._subtensor, - blacklist_fn=self.blacklist_subtensor, - ).attach( - forward_fn=self._challenge, - blacklist_fn=self.blacklist_challenge, + forward_fn=self._score, + blacklist_fn=self.blacklist_score, ) - + + # .attach( + # forward_fn=self._key, + # blacklist_fn=self.blacklist_key, + # ) + # .attach( + # forward_fn=self._subtensor, + # blacklist_fn=self.blacklist_subtensor, + # ).attach( + # forward_fn=self._key, + # blacklist_fn=self.blacklist_key, + # ) + # .attach( + # forward_fn=self._subtensor, + # blacklist_fn=self.blacklist_subtensor, + # ).attach( + # forward_fn=self._challenge, + # blacklist_fn=self.blacklist_challenge, + # ) + # Serve passes the axon information to the network + netuid we are hosting on. # This will auto-update if the axon port of external ip have changed. bt.logging.info( @@ -194,39 +192,39 @@ def __init__(self): self.requests_per_hour = [] self.average_requests_per_hour = 0 - # Init the miner's storage usage tracker - update_storage_stats(self) - self.rate_limiters = {} self.request_log = load_request_log(self.config.miner.request_log_path) - def _is_alive(self, synapse: IsAlive) -> IsAlive: bt.logging.info("I'm alive!") synapse.answer = "alive" return synapse - - + def blacklist_isalive(self, synapse: IsAlive) -> typing.Tuple[bool, str]: return False, synapse.dendrite.hotkey + def _score(self, synapse: Score) -> Score: + bt.logging.info(f"Availability score {synapse.availability}") + bt.logging.info(f"Latency score {synapse.latency}") + bt.logging.info(f"Reliability score {synapse.reliability}") + bt.logging.info(f"Distribution score {synapse.distribution}") + bt.logging.success(f"Score {synapse.score}") + return synapse - async def _generate_key( - self, synapse: Key - ) -> Key: - synapse_type="Save" if synapse.generate else "Clean" + def blacklist_score(self, synapse: Score) -> typing.Tuple[bool, str]: + return False, synapse.dendrite.hotkey + + async def _key(self, synapse: Key) -> Key: + synapse_type = "Save" if synapse.generate else "Clean" bt.logging.info(f"[Key/{synapse_type}] Synapse received") if synapse.generate: generate_ssh_key(synapse.validator_public_key) else: clean_ssh_key(synapse.validator_public_key) - bt.logging.info(f"[Key/{synapse_type}] Synapse proceed") + bt.logging.success(f"[Key/{synapse_type}] Synapse proceed") return synapse - - async def blacklist_generate_key( - self, synapse: Key - ) -> typing.Tuple[bool, str]: + async def blacklist_key(self, synapse: Key) -> typing.Tuple[bool, str]: if synapse.dendrite.hotkey not in self.metagraph.hotkeys: # Ignore requests from unrecognized entities. bt.logging.trace( @@ -239,21 +237,19 @@ async def blacklist_generate_key( ) return False, "Hotkey recognized!" - - async def _subtensor( - self, synapse: Subtensor - ) -> Subtensor: + async def _subtensor(self, synapse: Subtensor) -> Subtensor: bt.logging.info("[Subtensor] Synapse received") parsed_url = urlparse(self.subtensor.chain_endpoint) - ip = self.axon.external_ip if parsed_url.hostname == '127.0.0.1' else parsed_url.hostname + ip = ( + self.axon.external_ip + if parsed_url.hostname == "127.0.0.1" + else parsed_url.hostname + ) synapse.subtensor_ip = ip - bt.logging.info("[Subtensor] Synapse proceed") + bt.logging.success("[Subtensor] Synapse proceed") return synapse - - async def blacklist_subtensor( - self, synapse: Subtensor - ) -> typing.Tuple[bool, str]: + async def blacklist_subtensor(self, synapse: Subtensor) -> typing.Tuple[bool, str]: if synapse.dendrite.hotkey not in self.metagraph.hotkeys: # Ignore requests from unrecognized entities. bt.logging.trace( @@ -266,20 +262,14 @@ async def blacklist_subtensor( ) return False, "Hotkey recognized!" - - async def _challenge( - self, synapse: Challenge - ) -> Challenge: + async def _challenge(self, synapse: Challenge) -> Challenge: bt.logging.info("[Challenge] Synapse received") - block=self.subtensor.get_current_block() + block = self.subtensor.get_current_block() synapse.answer = f"{block}" bt.logging.info("[Challenge] Synapse proceed") return synapse - - async def blacklist_challenge( - self, synapse: Challenge - ) -> typing.Tuple[bool, str]: + async def blacklist_challenge(self, synapse: Challenge) -> typing.Tuple[bool, str]: if synapse.dendrite.hotkey not in self.metagraph.hotkeys: # Ignore requests from unrecognized entities. bt.logging.trace( @@ -292,7 +282,6 @@ async def blacklist_challenge( ) return False, "Hotkey recognized!" - def start_request_count_timer(self): """ Initializes and starts a timer for tracking the number of requests received by the miner in an hour. @@ -307,7 +296,6 @@ def start_request_count_timer(self): self.request_count_timer = threading.Timer(3600, self.reset_request_count) self.request_count_timer.start() - def reset_request_count(self): """ Logs the number of requests received in the last hour and resets the count. @@ -331,11 +319,9 @@ def reset_request_count(self): self.request_count = 0 self.start_request_count_timer() - def run(self): run(self) - def run_in_background_thread(self): """ Starts the miner's operations in a separate background thread. @@ -349,7 +335,6 @@ def run_in_background_thread(self): self.is_running = True bt.logging.debug("Started") - def stop_run_thread(self): """ Stops the miner's operations that are running in the background thread. @@ -361,7 +346,6 @@ def stop_run_thread(self): self.is_running = False bt.logging.debug("Stopped") - def __enter__(self): """ Starts the miner's operations in a background thread upon entering the context. @@ -369,7 +353,6 @@ def __enter__(self): """ self.run_in_background_thread() - def __exit__(self, exc_type, exc_value, traceback): """ Stops the miner's background operations upon exiting the context. diff --git a/neurons/validator.py b/neurons/validator.py index 801a5f50..975d51fd 100644 --- a/neurons/validator.py +++ b/neurons/validator.py @@ -28,14 +28,15 @@ from pprint import pformat from traceback import print_exception from substrateinterface.base import SubstrateInterface +# from dotenv import load_dotenv -from subnet.shared.checks import check_environment +from subnet.shared.checks import check_environment, check_registration from subnet.shared.utils import get_redis_password from subnet.shared.subtensor import get_current_block from subnet.shared.weights import should_set_weights from subnet.validator.config import config, check_config, add_args -from subnet.validator.encryption import setup_encryption_wallet +from subnet.validator.localisation import get_country, get_localisation from subnet.validator.forward import forward from subnet.validator.state import ( checkpoint, @@ -86,6 +87,9 @@ def __init__(self): self.check_config(self.config) bt.logging(config=self.config, logging_dir=self.config.neuron.full_path) + # Load env variables + # load_dotenv() + try: asyncio.run(check_environment(self.config.database.redis_conf_path)) except AssertionError as e: @@ -112,25 +116,11 @@ def __init__(self): self.wallet = bt.wallet(config=self.config) self.wallet.create_if_non_existent() - if not self.config.wallet._mock: - if not self.subtensor.is_hotkey_registered_on_subnet( - hotkey_ss58=self.wallet.hotkey.ss58_address, netuid=self.config.netuid - ): - raise Exception( - f"Wallet not currently registered on netuid {self.config.netuid}, please first register wallet before running" - ) + # Check registration + check_registration(self.subtensor, self.wallet, self.config.netuid) bt.logging.debug(f"wallet: {str(self.wallet)}") - # Setup dummy wallet for encryption purposes. No password needed. - self.encryption_wallet = setup_encryption_wallet( - wallet_name=self.config.encryption.wallet_name, - wallet_hotkey=self.config.encryption.hotkey, - password=self.config.encryption.password, - ) - self.encryption_wallet.coldkey # Unlock the coldkey. - bt.logging.info(f"loading encryption wallet {self.encryption_wallet}") - # Init metagraph. bt.logging.debug("loading metagraph") self.metagraph = bt.metagraph( @@ -171,6 +161,12 @@ def __init__(self): self.dendrite = bt.dendrite(wallet=self.wallet) bt.logging.debug(str(self.dendrite)) + # Get the validator country + self.country = get_country(self.dendrite.external_ip) + country_localisation = get_localisation(self.country) + country_name = country_localisation['country'] if country_localisation else 'None' + bt.logging.debug(F"Validator based in {country_name}") + # Init the event loop. self.loop = asyncio.get_event_loop() @@ -179,7 +175,7 @@ def __init__(self): # Start with 0 monitor pings # TODO: load this from disk instead of reset on restart - self.monitor_lookup = {uid: 0 for uid in self.metagraph.uids.tolist()} + # self.monitor_lookup = {uid: 0 for uid in self.metagraph.uids.tolist()} # Instantiate runners self.should_exit: bool = False diff --git a/scripts/docs/TESTNET.md b/scripts/docs/TESTNET.md index 5a3d001d..b9440dfc 100644 --- a/scripts/docs/TESTNET.md +++ b/scripts/docs/TESTNET.md @@ -7,6 +7,11 @@ Go to the home directory cd $HOME ``` +Install git +``` +apt-get update && apt-get install git +``` + Clone the SubVortex repository ``` git clone https://github.com/eclipsevortex/SubVortex.git @@ -23,6 +28,8 @@ $HOME/SubVortex/scripts/setup/setup.sh ``` # Subnet +This section is for owner who want to create a subnet if it has not been done yet + Install the subnet ``` $HOME/SubVortex/scripts/subnet/setup.sh @@ -110,7 +117,7 @@ Register the validator ``` btcli subnet register \ --netuid 92 \ - --subtensor.chain_endpoint ws://:9944 \ + --subtensor.network test \ --wallet.name validator \ --wallet.hotkey default ``` @@ -121,7 +128,7 @@ pm2 start neurons/validator.py \ --name validator-1 \ --interpreter python3 -- \ --netuid 92 \ - --subtensor.chain_endpoint ws://:9944 \ + --subtensor.network test \ --wallet.name validator \ --wallet.hotkey default \ --logging.debug diff --git a/scripts/redis/docker/Dockerfile b/scripts/redis/docker/Dockerfile new file mode 100644 index 00000000..924f8617 --- /dev/null +++ b/scripts/redis/docker/Dockerfile @@ -0,0 +1,21 @@ +FROM redis:7.2.4-alpine + +ARG REDIS_PASSWORD +ENV REDIS_PASSWORD=$REDIS_PASSWORD + +RUN mkdir /etc/redis/ && \ + mkdir /var/lib/redis/ && \ + mkdir /var/run/redis/ + +RUN chown -R redis:redis /etc/redis/ && \ + chown -R redis:redis /var/lib/redis/ && \ + chown redis:redis /var/run/redis + +# Copy the entrypoint script into the image +COPY ./scripts/redis/docker/entrypoint.sh /usr/local/bin/entrypoint.sh + +# Set the script as the entrypoint +ENTRYPOINT ["entrypoint.sh"] + +# Set the default command (which can be overridden) +CMD [] \ No newline at end of file diff --git a/scripts/redis/docker/entrypoint.sh b/scripts/redis/docker/entrypoint.sh new file mode 100755 index 00000000..38367141 --- /dev/null +++ b/scripts/redis/docker/entrypoint.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +# Start Redis with the password provided via REDIS_PASSWORD environment variable +exec redis-server --requirepass "$REDIS_PASSWORD" \ No newline at end of file diff --git a/scripts/redis/docker/redis_check.py b/scripts/redis/docker/redis_check.py new file mode 100644 index 00000000..aded585e --- /dev/null +++ b/scripts/redis/docker/redis_check.py @@ -0,0 +1,19 @@ +import redis +from dotenv import load_dotenv +import os + +# Load the .env file +load_dotenv() + +# Now you can use the environment variables +password = os.getenv('REDIS_PASSWORD') + +# Connect to Redis on localhost and the default port 6379 +r = redis.Redis(host='localhost', port=6379, db=0, password=password) + +# Set a key +r.set('mykey', 'myvalue') + +# Get the value of a key +value = r.get('mykey') +print(value) \ No newline at end of file diff --git a/scripts/subtensor/setup.sh b/scripts/subtensor/setup.sh index 513533e3..f55d13ef 100755 --- a/scripts/subtensor/setup.sh +++ b/scripts/subtensor/setup.sh @@ -19,6 +19,9 @@ install_dependencies() { # Function to install packages on Ubuntu/Debian install_ubuntu() { + # Update the list of packages + apt-get update + if [[ "$EXEC_TYPE" == "docker" ]]; then # Install docker ## Install Required Dependencies @@ -38,24 +41,20 @@ install_dependencies() { ## Add the user to the docker group sudo usermod -aG docker $USER echo -e '\e[32m[docker] Docker user created\e[0m' + + ## Apply the group membership (you may need to log out and log back in for the group to be recognized): + newgrp docker + echo -e '\e[32m[docker] Group membership applied\e[0m' # Install docker compose sudo apt-get install -y docker-compose echo -e '\e[32m[docker] Docker compose installed\e[0m' - - ## Apply changes - newgrp docker - echo -e '\e[32m[docker] Docker group created\e[0m' fi - # echo "Updating system packages..." - # sudo apt update - # echo "Installing required packages..." - # sudo apt install --assume-yes make build-essential clang libssl-dev llvm libudev-dev protobuf-compiler - + # Necessary libraries for Rust execution - apt-get update apt-get install -y curl build-essential protobuf-compiler clang git rm -rf /var/lib/apt/lists/* + echo -e '\e[32mRust dependencies installed\e[0m' } # Detect OS and call the appropriate function @@ -71,15 +70,21 @@ install_dependencies() { # Install rust and cargo # curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh curl https://sh.rustup.rs -sSf | sh -s -- -y + echo -e '\e[32mRust and Cargo installed\e[0m' # Update your shell's source to include Cargo's path source "$HOME/.cargo/env" + echo -e '\e[32mRust and Cargo added to the path\e[0m' } setup_environment() { + # Go to the root + cd $ROOT + # Clone subtensor and enter the directory if [ ! -d "subtensor" ]; then git clone https://github.com/opentensor/subtensor.git + echo -e '\e[32m[Subtensor] Repository cloned\e[0m' fi # Go the repository @@ -87,9 +92,11 @@ setup_environment() { # Get the latest version git pull + echo -e '\e[32m[Subtensor] Last version pulled\e[0m' # Setup rust ./scripts/init.sh + echo -e '\e[32m[Subtensor] Setup done\e[0m' # Go back cd $ROOT @@ -97,6 +104,9 @@ setup_environment() { # Install dependencies install_dependencies +echo -e '\e[32m[Subtensor] Dependencies installed\e[0m' + # Setup environment -setup_environment \ No newline at end of file +setup_environment +echo -e '\e[32m[Subtensor] Environment setup\e[0m' \ No newline at end of file diff --git a/subnet/__init__.py b/subnet/__init__.py index f785cb79..68269838 100644 --- a/subnet/__init__.py +++ b/subnet/__init__.py @@ -51,7 +51,7 @@ def __lt__(self, other): ) -__version__ = "0.1.0" +__version__ = "0.2.0" version = SubnetVersion.from_string(__version__) __spec_version__ = version.to_spec_version() diff --git a/subnet/constants.py b/subnet/constants.py index a164b55b..b6d2cb41 100644 --- a/subnet/constants.py +++ b/subnet/constants.py @@ -1,45 +1,32 @@ -# Failure mode negative rewards -STORE_FAILURE_REWARD = 0.0 +# Failure rewards +SUBTENSOR_FAILURE_REWARD = 0.0 +METRIC_FAILURE_REWARD = 0.0 CHALLENGE_FAILURE_REWARD = -0.01 -MONITOR_FAILURE_REWARD = -0.01 -RETRIEVAL_FAILURE_REWARD = -0.05 -# Constants for storage limits in bytes -STORAGE_LIMIT_SUPER_SAIYAN = 1024**6 * 1 # 1 EB -STORAGE_LIMIT_DIAMOND = 1024**5 * 1 # 1 PB -STORAGE_LIMIT_GOLD = 1024**4 * 100 # 100 TB -STORAGE_LIMIT_SILVER = 1024**4 * 10 # 10 TB -STORAGE_LIMIT_BRONZE = 1024**4 * 1 # 1 TB - -# Requirements for each tier. These must be maintained for a miner to remain in that tier. -SUPER_SAIYAN_STORE_SUCCESS_RATE = 0.999 # 1/1000 chance of failure -SUPER_SAIYAN_RETIREVAL_SUCCESS_RATE = 0.999 # 1/1000 chance of failure -SUPER_SAIYAN_CHALLENGE_SUCCESS_RATE = 0.999 # 1/1000 chance of failure - -DIAMOND_STORE_SUCCESS_RATE = 0.99 # 1/100 chance of failure -DIAMOND_RETRIEVAL_SUCCESS_RATE = 0.99 # 1/100 chance of failure -DIAMOND_CHALLENGE_SUCCESS_RATE = 0.99 # 1/100 chance of failure - -GOLD_STORE_SUCCESS_RATE = 0.975 # 1/50 chance of failure -GOLD_RETRIEVAL_SUCCESS_RATE = 0.975 # 1/50 chance of failure -GOLD_CHALLENGE_SUCCESS_RATE = 0.975 # 1/50 chance of failure - -SILVER_STORE_SUCCESS_RATE = 0.95 # 1/20 chance of failure -SILVER_RETRIEVAL_SUCCESS_RATE = 0.90 # 1/20 chance of failure -SILVER_CHALLENGE_SUCCESS_RATE = 0.95 # 1/20 chance of failure - -SUPER_SAIYAN_TIER_REWARD_FACTOR = 1.0 # Get 100% rewards -DIAMOND_TIER_REWARD_FACTOR = 0.888 # Get 88.8% rewards -GOLD_TIER_REWARD_FACTOR = 0.777 # Get 77.7% rewards -SILVER_TIER_REWARD_FACTOR = 0.555 # Get 55.5% rewards -BRONZE_TIER_REWARD_FACTOR = 0.444 # Get 44.4% rewards - -SUPER_SAIYAN_TIER_TOTAL_SUCCESSES = 10**5 # 100,000 -DIAMOND_TIER_TOTAL_SUCCESSES = 10**4 * 5 # 50,000 -GOLD_TIER_TOTAL_SUCCESSES = 10**3 * 5 # 5,000 -SILVER_TIER_TOTAL_SUCCESSES = 10**3 # 1,000 - -SUPER_SAIYAN_SUCCESS_RATE = 0.99 # 1/100 chance of failure -DIAMOND_SUCCESS_RATE = 0.975 # 1/50 chance of failure -GOLD_SUCCESS_RATE = 0.95 # 1/20 chance of failure -SILVER_SUCCESS_RATE = 0.90 # 1/10 chance of failure +# Failure rewards +AVAILABILITY_FAILURE_REWARD = 0.0 +LATENCY_FAILURE_REWARD = 0.0 +DISTRIBUTION_FAILURE_REWARD = 0.0 + +# Score weights +AVAILABILITY_WEIGHT = 1.0 +LATENCY_WEIGHT = 1.0 +RELIABILLITY_WEIGHT = 1.0 +DISTRIBUTION_WEIGHT = 1.0 + +# Success rewards +SUPER_SAIYAN_TIER_REWARD_FACTOR = 1.0 +DIAMOND_TIER_REWARD_FACTOR = 0.888 +GOLD_TIER_REWARD_FACTOR = 0.777 +SILVER_TIER_REWARD_FACTOR = 0.666 +BRONZE_TIER_REWARD_FACTOR = 0.555 + +SUPER_SAIYAN_TIER_TOTAL_SUCCESSES = 10**4 # 10,000 +DIAMOND_TIER_TOTAL_SUCCESSES = 10**3 * 5 # 5,000 +GOLD_TIER_TOTAL_SUCCESSES = 10**3 * 2 # 2,000 +SILVER_TIER_TOTAL_SUCCESSES = 10**3 # 1,000 + +SUPER_SAIYAN_WILSON_SCORE = 0.88 +DIAMOND_WILSON_SCORE = 0.77 +GOLD_WILSON_SCORE = 0.66 +SILVER_WILSON_SCORE = 0.55 diff --git a/subnet/localisation.json b/subnet/localisation.json new file mode 100644 index 00000000..5cd85848 --- /dev/null +++ b/subnet/localisation.json @@ -0,0 +1,246 @@ +{ + "AF": {"country": "Afghanistan", "latitude": 33.9391, "longitude": 67.7100}, + "AX": {"country": "Åland Islands", "latitude": 60.1785, "longitude": 19.9156}, + "AL": {"country": "Albania", "latitude": 41.1533, "longitude": 20.1683}, + "DZ": {"country": "Algeria", "latitude": 28.0339, "longitude": 1.6596}, + "AS": {"country": "American Samoa", "latitude": -14.2710, "longitude": -170.1322}, + "AD": {"country": "Andorra", "latitude": 42.5463, "longitude": 1.6016}, + "AO": {"country": "Angola", "latitude": -11.2027, "longitude": 17.8739}, + "AI": {"country": "Anguilla", "latitude": 18.2206, "longitude": -63.0686}, + "AQ": {"country": "Antarctica", "latitude": -90.0000, "longitude": 0.0000}, + "AG": {"country": "Antigua and Barbuda", "latitude": 17.0608, "longitude": -61.7964}, + "AR": {"country": "Argentina", "latitude": -38.4161, "longitude": -63.6167}, + "AM": {"country": "Armenia", "latitude": 40.0691, "longitude": 45.0382}, + "AW": {"country": "Aruba", "latitude": 12.5211, "longitude": -69.9683}, + "AU": {"country": "Australia", "latitude": -25.2744, "longitude": 133.7751}, + "AT": {"country": "Austria", "latitude": 47.5162, "longitude": 14.5501}, + "AZ": {"country": "Azerbaijan", "latitude": 40.1431, "longitude": 47.5769}, + "BS": {"country": "Bahamas", "latitude": 25.0343, "longitude": -77.3963}, + "BH": {"country": "Bahrain", "latitude": 26.0667, "longitude": 50.5577}, + "BD": {"country": "Bangladesh", "latitude": 23.6850, "longitude": 90.3563}, + "BB": {"country": "Barbados", "latitude": 13.1939, "longitude": -59.5432}, + "BY": {"country": "Belarus", "latitude": 53.7098, "longitude": 27.9534}, + "BE": {"country": "Belgium", "latitude": 50.5039, "longitude": 4.4699}, + "BZ": {"country": "Belize", "latitude": 17.1899, "longitude": -88.4976}, + "BJ": {"country": "Benin", "latitude": 9.3077, "longitude": 2.3158}, + "BM": {"country": "Bermuda", "latitude": 32.3078, "longitude": -64.7505}, + "BT": {"country": "Bhutan", "latitude": 27.5142, "longitude": 90.4336}, + "BO": {"country": "Bolivia", "latitude": -16.2902, "longitude": -63.5887}, + "BA": {"country": "Bosnia and Herzegovina", "latitude": 43.9159, "longitude": 17.6791}, + "BW": {"country": "Botswana", "latitude": -22.3285, "longitude": 24.6849}, + "BV": {"country": "Bouvet Island", "latitude": -54.4232, "longitude": 3.4132}, + "BR": {"country": "Brazil", "latitude": -14.2350, "longitude": -51.9253}, + "IO": {"country": "British Indian Ocean Territory", "latitude": -6.3432, "longitude": 71.8765}, + "BN": {"country": "Brunei Darussalam", "latitude": 4.5353, "longitude": 114.7277}, + "BG": {"country": "Bulgaria", "latitude": 42.7339, "longitude": 25.4858}, + "BF": {"country": "Burkina Faso", "latitude": 12.2383, "longitude": -1.5616}, + "BI": {"country": "Burundi", "latitude": -3.3731, "longitude": 29.9189}, + "KH": {"country": "Cambodia", "latitude": 12.5657, "longitude": 104.9910}, + "CM": {"country": "Cameroon", "latitude": 7.3697, "longitude": 12.3547}, + "CA": {"country": "Canada", "latitude": 56.1304, "longitude": -106.3468}, + "CV": {"country": "Cape Verde", "latitude": 16.5388, "longitude": -23.0418}, + "KY": {"country": "Cayman Islands", "latitude": 19.5135, "longitude": -80.5668}, + "CF": {"country": "Central African Republic", "latitude": 6.6111, "longitude": 20.9394}, + "TD": {"country": "Chad", "latitude": 15.4542, "longitude": 18.7322}, + "CL": {"country": "Chile", "latitude": -35.6751, "longitude": -71.5430}, + "CN": {"country": "China", "latitude": 35.8617, "longitude": 104.1954}, + "CX": {"country": "Christmas Island", "latitude": -10.4475, "longitude": 105.6904}, + "CC": {"country": "Cocos (Keeling) Islands", "latitude": -12.1642, "longitude": 96.8703}, + "CO": {"country": "Colombia", "latitude": 4.5709, "longitude": -74.2973}, + "KM": {"country": "Comoros", "latitude": -11.6455, "longitude": 43.3333}, + "CG": {"country": "Congo", "latitude": -0.2280, "longitude": 15.8277}, + "CD": {"country": "Congo, Democratic Republic of the", "latitude": -4.0383, "longitude": 21.7587}, + "CK": {"country": "Cook Islands", "latitude": -21.2367, "longitude": -159.7777}, + "CR": {"country": "Costa Rica", "latitude": 9.7489, "longitude": -83.7534}, + "CI": {"country": "Côte d'Ivoire", "latitude": 7.5400, "longitude": -5.5471}, + "HR": {"country": "Croatia", "latitude": 45.1000, "longitude": 15.2000}, + "CU": {"country": "Cuba", "latitude": 21.5218, "longitude": -77.7812}, + "CY": {"country": "Cyprus", "latitude": 35.1264, "longitude": 33.4299}, + "CZ": {"country": "Czech Republic", "latitude": 49.8175, "longitude": 15.4729}, + "DK": {"country": "Denmark", "latitude": 56.2639, "longitude": 9.5018}, + "DJ": {"country": "Djibouti", "latitude": 11.8251, "longitude": 42.5903}, + "DM": {"country": "Dominica", "latitude": 15.4150, "longitude": -61.3710}, + "DO": {"country": "Dominican Republic", "latitude": 18.7357, "longitude": -70.1627}, + "EC": {"country": "Ecuador", "latitude": -1.8312, "longitude": -78.1834}, + "EG": {"country": "Egypt", "latitude": 26.8206, "longitude": 30.8025}, + "SV": {"country": "El Salvador", "latitude": 13.7942, "longitude": -88.8965}, + "GQ": {"country": "Equatorial Guinea", "latitude": 1.6508, "longitude": 10.2679}, + "ER": {"country": "Eritrea", "latitude": 15.1794, "longitude": 39.7823}, + "EE": {"country": "Estonia", "latitude": 58.5953, "longitude": 25.0136}, + "ET": {"country": "Ethiopia", "latitude": 9.1450, "longitude": 40.4897}, + "FK": {"country": "Falkland Islands (Malvinas)", "latitude": -51.7963, "longitude": -59.5236}, + "FO": {"country": "Faroe Islands", "latitude": 61.8926, "longitude": -6.9118}, + "FJ": {"country": "Fiji", "latitude": -17.7134, "longitude": 178.0650}, + "FI": {"country": "Finland", "latitude": 61.9241, "longitude": 25.7482}, + "FR": {"country": "France", "latitude": 46.6034, "longitude": 1.8883}, + "GF": {"country": "French Guiana", "latitude": 3.9339, "longitude": -53.1258}, + "PF": {"country": "French Polynesia", "latitude": -17.6797, "longitude": -149.4068}, + "TF": {"country": "French Southern Territories", "latitude": -49.2804, "longitude": 69.3486}, + "GA": {"country": "Gabon", "latitude": -0.8037, "longitude": 11.6094}, + "GM": {"country": "Gambia", "latitude": 13.4432, "longitude": -15.3101}, + "GE": {"country": "Georgia", "latitude": 42.3154, "longitude": 43.3569}, + "DE": {"country": "Germany", "latitude": 51.1657, "longitude": 10.4515}, + "GH": {"country": "Ghana", "latitude": 7.9465, "longitude": -1.0232}, + "GI": {"country": "Gibraltar", "latitude": 36.1408, "longitude": -5.3536}, + "GR": {"country": "Greece", "latitude": 39.0742, "longitude": 21.8243}, + "GL": {"country": "Greenland", "latitude": 71.7069, "longitude": -42.6043}, + "GD": {"country": "Grenada", "latitude": 12.1165, "longitude": -61.6789}, + "GP": {"country": "Guadeloupe", "latitude": 16.2650, "longitude": -61.5510}, + "GU": {"country": "Guam", "latitude": 13.4443, "longitude": 144.7937}, + "GT": {"country": "Guatemala", "latitude": 15.7835, "longitude": -90.2308}, + "GG": {"country": "Guernsey", "latitude": 49.4657, "longitude": -2.5853}, + "GN": {"country": "Guinea", "latitude": 9.9456, "longitude": -9.6966}, + "GW": {"country": "Guinea-Bissau", "latitude": 11.8037, "longitude": -15.1804}, + "GY": {"country": "Guyana", "latitude": 4.8604, "longitude": -58.9302}, + "HT": {"country": "Haiti", "latitude": 18.9712, "longitude": -72.2852}, + "HM": {"country": "Heard Island and McDonald Islands", "latitude": -53.0818, "longitude": 73.5042}, + "VA": {"country": "Holy See (Vatican City State)", "latitude": 41.9029, "longitude": 12.4534}, + "HN": {"country": "Honduras", "latitude": 15.199999, "longitude": -86.241905}, + "HK": {"country": "Hong Kong", "latitude": 22.3193, "longitude": 114.1694}, + "HU": {"country": "Hungary", "latitude": 47.1625, "longitude": 19.5033}, + "IS": {"country": "Iceland", "latitude": 64.9631, "longitude": -19.0208}, + "IN": {"country": "India", "latitude": 20.5937, "longitude": 78.9629}, + "ID": {"country": "Indonesia", "latitude": -0.7893, "longitude": 113.9213}, + "IR": {"country": "Iran, Islamic Republic of", "latitude": 32.4279, "longitude": 53.6880}, + "IQ": {"country": "Iraq", "latitude": 33.2232, "longitude": 43.6793}, + "IE": {"country": "Ireland", "latitude": 53.1424, "longitude": -7.6921}, + "IM": {"country": "Isle of Man", "latitude": 54.2361, "longitude": -4.5481}, + "IL": {"country": "Israel", "latitude": 31.0461, "longitude": 34.8516}, + "IT": {"country": "Italy", "latitude": 41.8719, "longitude": 12.5674}, + "JM": {"country": "Jamaica", "latitude": 18.1096, "longitude": -77.2975}, + "JP": {"country": "Japan", "latitude": 36.2048, "longitude": 138.2529}, + "JE": {"country": "Jersey", "latitude": 49.2144, "longitude": -2.1313}, + "JO": {"country": "Jordan", "latitude": 30.5852, "longitude": 36.2384}, + "KZ": {"country": "Kazakhstan", "latitude": 48.0196, "longitude": 66.9237}, + "KE": {"country": "Kenya", "latitude": -0.0236, "longitude": 37.9062}, + "KI": {"country": "Kiribati", "latitude": 3.3704, "longitude": -168.7340}, + "KP": {"country": "Korea, Democratic People's Republic of", "latitude": 40.3399, "longitude": 127.5101}, + "KR": {"country": "Korea, Republic of", "latitude": 35.9078, "longitude": 127.7669}, + "KW": {"country": "Kuwait", "latitude": 29.3759, "longitude": 47.9774}, + "KG": {"country": "Kyrgyzstan", "latitude": 41.2044, "longitude": 74.7661}, + "LA": {"country": "Lao People's Democratic Republic", "latitude": 19.8563, "longitude": 102.4955}, + "LV": {"country": "Latvia", "latitude": 56.8796, "longitude": 24.6032}, + "LB": {"country": "Lebanon", "latitude": 33.8547, "longitude": 35.8623}, + "LS": {"country": "Lesotho", "latitude": -29.6099, "longitude": 28.2336}, + "LR": {"country": "Liberia", "latitude": 6.4281, "longitude": -9.4295}, + "LY": {"country": "Libyan Arab Jamahiriya", "latitude": 26.3351, "longitude": 17.2283}, + "LI": {"country": "Liechtenstein", "latitude": 47.1660, "longitude": 9.5554}, + "LT": {"country": "Lithuania", "latitude": 55.1694, "longitude": 23.8813}, + "LU": {"country": "Luxembourg", "latitude": 49.8153, "longitude": 6.1296}, + "MO": {"country": "Macao", "latitude": 22.1987, "longitude": 113.5439}, + "MK": {"country": "Macedonia, the Former Yugoslav Republic of", "latitude": 41.6086, "longitude": 21.7453}, + "MG": {"country": "Madagascar", "latitude": -18.7669, "longitude": 46.8691}, + "MW": {"country": "Malawi", "latitude": -13.2543, "longitude": 34.3015}, + "MY": {"country": "Malaysia", "latitude": 4.2105, "longitude": 101.9758}, + "MV": {"country": "Maldives", "latitude": 3.2028, "longitude": 73.2207}, + "ML": {"country": "Mali", "latitude": 17.5707, "longitude": -3.9962}, + "MT": {"country": "Malta", "latitude": 35.9375, "longitude": 14.3754}, + "MH": {"country": "Marshall Islands", "latitude": 7.1315, "longitude": 171.1845}, + "MQ": {"country": "Martinique", "latitude": 14.6415, "longitude": -61.0242}, + "MR": {"country": "Mauritania", "latitude": 21.0079, "longitude": -10.9408}, + "MU": {"country": "Mauritius", "latitude": -20.3484, "longitude": 57.5522}, + "YT": {"country": "Mayotte", "latitude": -12.8275, "longitude": 45.1662}, + "MX": {"country": "Mexico", "latitude": 23.6345, "longitude": -102.5528}, + "FM": {"country": "Micronesia, Federated States of", "latitude": 7.4256, "longitude": 150.5508}, + "MD": {"country": "Moldova, Republic of", "latitude": 47.4116, "longitude": 28.3699}, + "MC": {"country": "Monaco", "latitude": 43.7384, "longitude": 7.4246}, + "MN": {"country": "Mongolia", "latitude": 46.8625, "longitude": 103.8467}, + "ME": {"country": "Montenegro", "latitude": 42.7087, "longitude": 19.3744}, + "MS": {"country": "Montserrat", "latitude": 16.7425, "longitude": -62.1874}, + "MA": {"country": "Morocco", "latitude": 31.7917, "longitude": -7.0926}, + "MZ": {"country": "Mozambique", "latitude": -18.6657, "longitude": 35.5296}, + "MM": {"country": "Myanmar", "latitude": 21.9162, "longitude": 95.9560}, + "NA": {"country": "Namibia", "latitude": -22.9576, "longitude": 18.4904}, + "NR": {"country": "Nauru", "latitude": -0.5228, "longitude": 166.9315}, + "NP": {"country": "Nepal", "latitude": 28.3949, "longitude": 84.1240}, + "NL": {"country": "Netherlands", "latitude": 52.1326, "longitude": 5.2913}, + "AN": {"country": "Netherlands Antilles", "latitude": 12.2261, "longitude": -69.0601}, + "NC": {"country": "New Caledonia", "latitude": -20.9043, "longitude": 165.6180}, + "NZ": {"country": "New Zealand", "latitude": -40.9006, "longitude": 174.8860}, + "NI": {"country": "Nicaragua", "latitude": 12.8654, "longitude": -85.2072}, + "NE": {"country": "Niger", "latitude": 17.6078, "longitude": 8.0817}, + "NG": {"country": "Nigeria", "latitude": 9.0820, "longitude": 8.6753}, + "NU": {"country": "Niue", "latitude": -19.0544, "longitude": -169.8672}, + "NF": {"country": "Norfolk Island", "latitude": -29.0408, "longitude": 167.9547}, + "MP": {"country": "Northern Mariana Islands", "latitude": 17.3308, "longitude": 145.3847}, + "NO": {"country": "Norway", "latitude": 60.4720, "longitude": 8.4689}, + "OM": {"country": "Oman", "latitude": 21.4735, "longitude": 55.9754}, + "PK": {"country": "Pakistan", "latitude": 30.3753, "longitude": 69.3451}, + "PW": {"country": "Palau", "latitude": 7.5150, "longitude": 134.5825}, + "PS": {"country": "Palestinian Territory, Occupied", "latitude": 31.9522, "longitude": 35.2332}, + "PA": {"country": "Panama", "latitude": 8.5380, "longitude": -80.7821}, + "PG": {"country": "Papua New Guinea", "latitude": -6.3149, "longitude": 143.9555}, + "PY": {"country": "Paraguay", "latitude": -23.4425, "longitude": -58.4438}, + "PE": {"country": "Peru", "latitude": -9.1900, "longitude": -75.0152}, + "PH": {"country": "Philippines", "latitude": 12.8797, "longitude": 121.7740}, + "PN": {"country": "Pitcairn", "latitude": -24.3767, "longitude": -128.3243}, + "PL": {"country": "Poland", "latitude": 51.9194, "longitude": 19.1451}, + "PT": {"country": "Portugal", "latitude": 39.3999, "longitude": -8.2245}, + "PR": {"country": "Puerto Rico", "latitude": 18.2208, "longitude": -66.5901}, + "QA": {"country": "Qatar", "latitude": 25.3548, "longitude": 51.1839}, + "RE": {"country": "Réunion", "latitude": -21.1151, "longitude": 55.5364}, + "RO": {"country": "Romania", "latitude": 45.9432, "longitude": 24.9668}, + "RU": {"country": "Russian Federation", "latitude": 61.5240, "longitude": 105.3188}, + "RW": {"country": "Rwanda", "latitude": -1.9403, "longitude": 29.8739}, + "SH": {"country": "Saint Helena", "latitude": -24.1435, "longitude": -10.0307}, + "KN": {"country": "Saint Kitts and Nevis", "latitude": 17.3578, "longitude": -62.7829}, + "LC": {"country": "Saint Lucia", "latitude": 13.9094, "longitude": -60.9789}, + "PM": {"country": "Saint Pierre and Miquelon", "latitude": 46.8852, "longitude": -56.3159}, + "VC": {"country": "Saint Vincent and the Grenadines", "latitude": 12.9843, "longitude": -61.2872}, + "WS": {"country": "Samoa", "latitude": -13.7590, "longitude": -172.1046}, + "SM": {"country": "San Marino", "latitude": 43.9424, "longitude": 12.4578}, + "ST": {"country": "Sao Tome and Principe", "latitude": 0.1864, "longitude": 6.6131}, + "SA": {"country": "Saudi Arabia", "latitude": 23.8859, "longitude": 45.0792}, + "SN": {"country": "Senegal", "latitude": 14.4974, "longitude": -14.4524}, + "RS": {"country": "Serbia", "latitude": 44.0165, "longitude": 21.0059}, + "SC": {"country": "Seychelles", "latitude": -4.6796, "longitude": 55.4919}, + "SL": {"country": "Sierra Leone", "latitude": 8.4606, "longitude": -11.7799}, + "SG": {"country": "Singapore", "latitude": 1.3521, "longitude": 103.8198}, + "SK": {"country": "Slovakia", "latitude": 48.6690, "longitude": 19.6990}, + "SI": {"country": "Slovenia", "latitude": 46.1512, "longitude": 14.9955}, + "SB": {"country": "Solomon Islands", "latitude": -9.6457, "longitude": 160.1562}, + "SO": {"country": "Somalia", "latitude": 5.1521, "longitude": 46.1996}, + "ZA": {"country": "South Africa", "latitude": -30.5595, "longitude": 22.9375}, + "GS": {"country": "South Georgia and the South Sandwich Islands", "latitude": -54.4296, "longitude": -36.5879}, + "ES": {"country": "Spain", "latitude": 40.4637, "longitude": -3.7492}, + "LK": {"country": "Sri Lanka", "latitude": 7.8731, "longitude": 80.7718}, + "SD": {"country": "Sudan", "latitude": 12.8628, "longitude": 30.2176}, + "SR": {"country": "Suriname", "latitude": 3.9193, "longitude": -56.0278}, + "SJ": {"country": "Svalbard and Jan Mayen", "latitude": 77.5536, "longitude": 23.6703}, + "SZ": {"country": "Swaziland", "latitude": -26.5225, "longitude": 31.4659}, + "SE": {"country": "Sweden", "latitude": 60.1282, "longitude": 18.6435}, + "CH": {"country": "Switzerland", "latitude": 46.8182, "longitude": 8.2275}, + "SY": {"country": "Syrian Arab Republic", "latitude": 34.8021, "longitude": 38.9968}, + "TW": {"country": "Taiwan, Province of China", "latitude": 23.6978, "longitude": 120.9605}, + "TJ": {"country": "Tajikistan", "latitude": 38.8610, "longitude": 71.2761}, + "TZ": {"country": "Tanzania, United Republic of", "latitude": -6.3690, "longitude": 34.8888}, + "TH": {"country": "Thailand", "latitude": 15.8700, "longitude": 100.9925}, + "TL": {"country": "Timor-Leste", "latitude": -8.8742, "longitude": 125.7275}, + "TG": {"country": "Togo", "latitude": 8.6195, "longitude": 0.8248}, + "TK": {"country": "Tokelau", "latitude": -9.2002, "longitude": -171.8484}, + "TO": {"country": "Tonga", "latitude": -21.1787, "longitude": -175.1982}, + "TT": {"country": "Trinidad and Tobago", "latitude": 10.6918, "longitude": -61.2225}, + "TN": {"country": "Tunisia", "latitude": 33.8869, "longitude": 9.5375}, + "TR": {"country": "Turkey", "latitude": 38.9637, "longitude": 35.2433}, + "TM": {"country": "Turkmenistan", "latitude": 38.9697, "longitude": 59.5563}, + "TC": {"country": "Turks and Caicos Islands", "latitude": 21.6940, "longitude": -71.7979}, + "TV": {"country": "Tuvalu", "latitude": -7.1095, "longitude": 177.6493}, + "UG": {"country": "Uganda", "latitude": 1.3733, "longitude": 32.2903}, + "UA": {"country": "Ukraine", "latitude": 48.3794, "longitude": 31.1656}, + "AE": {"country": "United Arab Emirates", "latitude": 23.4241, "longitude": 53.8478}, + "GB": {"country": "United Kingdom", "latitude": 55.3781, "longitude": -3.4360}, + "US": {"country": "United States", "latitude": 37.0902, "longitude": -95.7129}, + "UM": {"country": "United States Minor Outlying Islands", "latitude": 19.2921, "longitude": 166.6181}, + "UY": {"country": "Uruguay", "latitude": -32.5228, "longitude": -55.7658}, + "UZ": {"country": "Uzbekistan", "latitude": 41.3775, "longitude": 64.5853}, + "VU": {"country": "Vanuatu", "latitude": -15.3767, "longitude": 166.9592}, + "VE": {"country": "Venezuela", "latitude": 6.4238, "longitude": -66.5897}, + "VN": {"country": "Viet Nam", "latitude": 14.0583, "longitude": 108.2772}, + "VG": {"country": "Virgin Islands, British", "latitude": 18.4207, "longitude": -64.6390}, + "VI": {"country": "Virgin Islands, U.S.", "latitude": 18.3358, "longitude": -64.8963}, + "WF": {"country": "Wallis and Futuna", "latitude": -13.7687, "longitude": -177.1561}, + "EH": {"country": "Western Sahara", "latitude": 24.2155, "longitude": -12.8858}, + "YE": {"country": "Yemen", "latitude": 15.5527, "longitude": 48.5164}, + "ZM": {"country": "Zambia", "latitude": -13.1339, "longitude": 27.8493}, + "ZW": {"country": "Zimbabwe", "latitude": -19.0154, "longitude": 29.1549} + } \ No newline at end of file diff --git a/subnet/miner/run.py b/subnet/miner/run.py index 200d25a7..0585a44d 100644 --- a/subnet/miner/run.py +++ b/subnet/miner/run.py @@ -1,120 +1,6 @@ -# The MIT License (MIT) -# Copyright © 2023 Yuma Rao -# Copyright © 2023 philanthrope - -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the “Software”), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. - -# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -import json import bittensor as bt - -from pprint import pformat from substrateinterface import SubstrateInterface -from scalecodec import ScaleBytes -from scalecodec.exceptions import RemainingScaleBytesNotEmptyException - -from .utils import update_storage_stats, run_async_in_sync_context - - -tagged_tx_queue_registry = { - "types": { - "TransactionTag": "Vec", - "TransactionPriority": "u64", - "TransactionLongevity": "u64", - "ValidTransaction": { - "type": "struct", - "type_mapping": [ - ["priority", "TransactionPriority"], - ["requires", "Vec"], - ["provides", "Vec"], - ["longevity", "TransactionLongevity"], - ["propagate", "bool"], - ], - }, - "TransactionValidity": "Result", - "TransactionSource": { - "type": "enum", - "value_list": ["InBlock", "Local", "External"], - }, - }, - "runtime_api": { - "TaggedTransactionQueue": { - "methods": { - "validate_transaction": { - "params": [ - { - "name": "source", - "type": "TransactionSource", - }, - { - "name": "tx", - "type": "Extrinsic", - }, - {"name": "block_hash", "type": "Hash"}, - ], - "type": "TransactionValidity", - }, - }, - } - }, -} - - -def runtime_call( - substrate: SubstrateInterface, api: str, method: str, params: list, block_hash: str -): - substrate.runtime_config.update_type_registry(tagged_tx_queue_registry) - runtime_call_def = substrate.runtime_config.type_registry["runtime_api"][api][ - "methods" - ][method] - # TODO: review is this variable is needed - runtime_api_types = substrate.runtime_config.type_registry["runtime_api"][api].get( - "types", {} - ) - - # Encode params - param_data = ScaleBytes(bytes()) - for idx, param in enumerate(runtime_call_def["params"]): - scale_obj = substrate.runtime_config.create_scale_object(param["type"]) - if type(params) is list: - param_data += scale_obj.encode(params[idx]) - else: - if param["name"] not in params: - raise ValueError(f"Runtime Call param '{param['name']}' is missing") - - param_data += scale_obj.encode(params[param["name"]]) - - # RPC request - result_data = substrate.rpc_request( - "state_call", [f"{api}_{method}", str(param_data), block_hash] - ) - - # Decode result - result_obj = substrate.runtime_config.create_scale_object(runtime_call_def["type"]) - try: - result_obj.decode( - ScaleBytes(result_data["result"]), - check_remaining=substrate.config.get("strict_scale_decode"), - ) - except RemainingScaleBytesNotEmptyException: - bt.logging.error(f"BytesNotEmptyException: result_data could not be decoded {result_data}") - result_obj = "Dry run failed. Could not decode result." - except Exception as e: - bt.logging.error(f"Exception: result_data could not be decoded {e} {result_data}") - result_obj = "Dry run failed. Could not decode result." - - return result_obj +from subnet.shared.checks import check_registration def run(self): @@ -152,130 +38,25 @@ def run(self): netuid = self.config.netuid # --- Check for registration. - if not self.subtensor.is_hotkey_registered( - netuid=netuid, - hotkey_ss58=self.wallet.hotkey.ss58_address, - ): - bt.logging.error( - f"Wallet: {self.wallet} is not registered on netuid {netuid}" - f"Please register the hotkey using `btcli subnets register` before trying again" - ) - exit() + check_registration(self.subtensor, self.wallet, netuid) tempo = block_handler_substrate.query( module="SubtensorModule", storage_function="Tempo", params=[netuid] ).value - last_extrinsic_hash = None - checked_extrinsics_count = 0 - should_retry = False - account_nonce = block_handler_substrate.get_account_nonce(self.wallet.hotkey.ss58_address) - def handler(obj, update_nr, subscription_id): current_block = obj["header"]["number"] - block_hash = block_handler_substrate.get_block_hash(current_block) bt.logging.debug(f"New block #{current_block}") + # --- Check for registration every 100 blocks (20 minutes). + if current_block % 100 == 0: + check_registration(self.subtensor, self.wallet, netuid) + bt.logging.debug( f"Blocks since epoch: {(current_block + netuid + 1) % (tempo + 1)}" ) - nonlocal last_extrinsic_hash, checked_extrinsics_count, should_retry, account_nonce - - if last_extrinsic_hash is not None: - try: - receipt = block_handler_substrate.retrieve_extrinsic_by_hash( - block_hash, last_extrinsic_hash - ) - bt.logging.trace( - f"Last set-weights call: {'Success' if receipt.is_success else format('Failure, reason: %s', receipt.error_message['name'] if receipt.error_message is not None else 'nil')}" - ) - - should_retry = False - last_extrinsic_hash = None - checked_extrinsics_count = 0 - except Exception: - checked_extrinsics_count += 1 - bt.logging.trace("An error occurred, extrinsic not found in block.") - finally: - if checked_extrinsics_count >= 20: - should_retry = True - last_extrinsic_hash = None - checked_extrinsics_count = 0 - - if ((current_block + netuid + 1) % (tempo + 1) == 0) or should_retry: - bt.logging.info("Saving request log") - try: - with open(self.config.miner.request_log_path, "w") as f: - json.dump(self.request_log, f) - except Exception as e: - bt.logging.warning(f"Unable to save request log to disk {e}") - - bt.logging.info( - f"New epoch started, setting weights at block {current_block}" - ) - with self.subtensor.substrate as substrate: - call = substrate.compose_call( - call_module="SubtensorModule", - call_function="set_weights", - call_params={ - "dests": [self.my_subnet_uid], - "weights": [65535], - "netuid": netuid, - "version_key": 1, - }, - ) - - # Period dictates how long the extrinsic will stay as part of waiting pool - extrinsic = substrate.create_signed_extrinsic( - call=call, keypair=self.wallet.hotkey, era={"period": 10}, nonce=account_nonce - ) - - dry_run = runtime_call( - substrate=substrate, - api="TaggedTransactionQueue", - method="validate_transaction", - params=["InBlock", extrinsic, block_hash], - block_hash=block_hash, - ) - bt.logging.debug(dry_run) - - try: - response = substrate.submit_extrinsic( - extrinsic, - wait_for_inclusion=False, - wait_for_finalization=False, - ) - - result_data = substrate.rpc_request("author_pendingExtrinsics", []) - for extrinsic_data in result_data["result"]: - extrinsic = substrate.runtime_config.create_scale_object( - "Extrinsic", metadata=substrate.metadata - ) - extrinsic.decode( - ScaleBytes(extrinsic_data), - check_remaining=substrate.config.get("strict_scale_decode"), - ) - - if extrinsic.value["extrinsic_hash"] == response.extrinsic_hash: - bt.logging.debug( - "Weights transaction is in the pending transaction pool" - ) - - last_extrinsic_hash = response.extrinsic_hash - should_retry = False - account_nonce = account_nonce + 1 - - except BaseException as e: - bt.logging.warning(f"Error while submitting set weights extrinsic: {e}. Retrying...") - should_retry = True - - # --- Update the miner storage information periodically. - if not should_retry: - update_storage_stats(self) - bt.logging.debug("Storage statistics updated...") - - if self.should_exit: - return True + if self.should_exit: + return True - block_handler_substrate.subscribe_block_headers(handler) + block_handler_substrate.subscribe_block_headers(handler) \ No newline at end of file diff --git a/subnet/miner/utils.py b/subnet/miner/utils.py index 6db5f8fe..1698d0da 100644 --- a/subnet/miner/utils.py +++ b/subnet/miner/utils.py @@ -18,127 +18,32 @@ import os import json -import time import shutil -import asyncio -import multiprocessing import bittensor as bt -from collections import deque -from ..shared.ecc import ( - ecc_point_to_hex, - hash_data, -) -from ..shared.merkle import ( - MerkleTree, -) - -def commit_data_with_seed(committer, data_chunks, n_chunks, seed): - """ - Commits chunks of data with a seed using a Merkle tree structure to create a proof of - integrity for each chunk. This function is used in environments where the integrity - and order of data need to be verifiable. - - Parameters: - - committer: The committing object, which should have a commit method. - - data_chunks (list): A list of data chunks to be committed. - - n_chunks (int): The number of chunks expected to be committed. - - seed: A seed value that is combined with data chunks before commitment. - - Returns: - - randomness (list): A list of randomness values associated with each data chunk's commitment. - - chunks (list): The list of original data chunks that were committed. - - points (list): A list of commitment points in hex format. - - merkle_tree (MerkleTree): A Merkle tree constructed from the commitment points. - - This function handles the conversion of commitment points to hex format and adds them to the - Merkle tree. The completed tree represents the combined commitments. - """ - merkle_tree = MerkleTree() - - # Commit each chunk of data - randomness, chunks, points = [None] * n_chunks, [None] * n_chunks, [None] * n_chunks - for index, chunk in enumerate(data_chunks): - c, m_val, r = committer.commit(chunk + str(seed).encode()) - c_hex = ecc_point_to_hex(c) - randomness[index] = r - chunks[index] = chunk - points[index] = c_hex - merkle_tree.add_leaf(c_hex) - - # Create the tree from the leaves - merkle_tree.make_tree() - return randomness, chunks, points, merkle_tree - - -def save_data_to_filesystem(data, directory, hotkey, filename): - """ - Saves data to the filesystem at the specified directory and filename. If the directory does - not exist, it is created. - - Parameters: - - data: The data to be saved. - - directory (str): The directory path where the data should be saved. - - hotkey (str): The hotkey associated with the data. - - filename (str): The name of the file to save the data in. - - Returns: - - file_path (str): The full path to the saved file. - - This function is useful for persisting data to the disk. - """ - # Ensure the directory exists - directory = os.path.join(os.path.expanduser(directory), hotkey) - os.makedirs(directory, exist_ok=True) - file_path = os.path.join(directory, filename) - with open(file_path, "wb") as file: - file.write(data) - return file_path - - -def load_from_filesystem(filepath): - """ - Loads data from a file in the filesystem. - - Parameters: - - filepath (str): The path to the file from which data is to be loaded. - - Returns: - - data: The data read from the file. - - This function is a straightforward utility for reading binary data from a file. - """ - with open(os.path.expanduser(filepath), "rb") as file: - data = file.read() - return data - - -def compute_subsequent_commitment(data, previous_seed, new_seed, verbose=False): +def load_request_log(request_log_path: str) -> dict: """ - Computes a new commitment based on provided data and a change from an old seed to a new seed. - This function is typically used in cryptographic operations to update commitments without - altering the underlying data. + Loads the request logger from disk if it exists. - Parameters: - - data: The original data for which the commitment is being updated. - - previous_seed: The seed used in the previous commitment. - - new_seed: The seed to be used for the new commitment. - - verbose (bool): If True, additional debug information will be printed. Defaults to False. + Args: + log_path (str): The path to the directory containing the request log. Returns: - - A tuple containing the new commitment and the proof of the old commitment. + Dict: The request log data, if it exists, or an empty dictionary. - If verbose is set to True, debug information about the types and contents of the parameters - will be printed to aid in debugging. + This method loads the request log from disk if it exists. If not, it returns an empty dictionary. """ - if verbose: - bt.logging.debug("IN COMPUTE SUBESEQUENT COMMITMENT") - bt.logging.debug("type of data :", type(data)) - bt.logging.debug("type of prev_seed:", type(previous_seed)) - bt.logging.debug("type of new_seed :", type(new_seed)) - proof = hash_data(data + previous_seed) - return hash_data(str(proof).encode("utf-8") + new_seed), proof + if os.path.exists(request_log_path): + try: + with open(request_log_path, "r") as f: + request_log = json.load(f) + except Exception as e: + bt.logging.error(f"Error loading request log: {e}. Resetting.") + request_log = {} + else: + request_log = {} + return request_log def get_disk_space_stats(path): @@ -214,146 +119,3 @@ def get_directory_size(path): if not os.path.islink(fp): total_size += os.path.getsize(fp) return total_size - - -def update_storage_stats(self): - """ - Updates the miner's storage statistics. - - This function updates the miner's storage statistics, including the free disk space, current storage usage, - and percent disk usage. It's useful for understanding the storage capacity and usage of the system where - the miner is running. - """ - - self.free_memory = get_free_disk_space() - bt.logging.info(f"Free memory: {self.free_memory} bytes") - self.current_storage_usage = get_directory_size(self.config.database.directory) - bt.logging.info(f"Miner storage usage: {self.current_storage_usage} bytes") - self.percent_disk_usage = self.current_storage_usage / (self.free_memory + self.current_storage_usage) - bt.logging.info(f"Miner % disk usage : {100 * self.percent_disk_usage:.3f}%") - - -def load_request_log(request_log_path: str) -> dict: - """ - Loads the request logger from disk if it exists. - - Args: - log_path (str): The path to the directory containing the request log. - - Returns: - Dict: The request log data, if it exists, or an empty dictionary. - - This method loads the request log from disk if it exists. If not, it returns an empty dictionary. - """ - if os.path.exists(request_log_path): - try: - with open(request_log_path, "r") as f: - request_log = json.load(f) - except Exception as e: - bt.logging.error(f"Error loading request log: {e}. Resetting.") - request_log = {} - else: - request_log = {} - return request_log - - -def log_request(synapse: "bt.Synapse", request_log: dict): - """ - Log the request and store the timestamp of each request. - - Args: - synapse (bt.Synapse): The synapse object with the request details. - request_log (dict): The dictionary to log request timestamps. - - The function logs the time of each request in the request log and the request type. - """ - current_time = time.time() - caller = synapse.dendrite.hotkey - if caller not in request_log: - request_log[caller] = [] - - request_log[caller].append((synapse.name, current_time)) - return request_log - - -class RateLimiter: - def __init__(self, max_requests, time_window): - self.max_requests = max_requests - self.time_window = time_window - self.requests = deque() - - def is_allowed(self, caller): - current_time = time.time() - while ( - self.requests - and current_time - self.requests[0]["timestamp"] > self.time_window - ): - self.requests.popleft() - - if len(self.requests) < self.max_requests: - self.requests.append({"caller": caller, "timestamp": current_time}) - return True - else: - return False - - -def get_purge_ttl_script_path(current_dir): - """ - Constructs and returns the path to the 'rebalance_deregistration.sh' script within a project directory. - - This function takes the root path of a project and appends the relative path to the 'rebalance_deregistration.sh' script. - It assumes that the script is located within the 'scripts' subdirectory of the given project root. - - Parameters: - project_root (str): The root path of the project directory. - - Returns: - str: The full path to the 'rebalance_deregistration.sh' script. - """ - project_root = os.path.join(current_dir, "..") - project_root = os.path.normpath(project_root) - script_path = os.path.join(project_root, "scripts", "run_ttl_purge.sh") - return script_path - - -def run_async_in_sync_context( - coroutine_function: callable, - loop: asyncio.unix_events._UnixSelectorEventLoop = None, - ttl: int = 100, - *args, -): - """ - Runs an asynchronous coroutine in a synchronous context using a separate process. - - This function is useful for running asynchronous coroutines in a synchronous context, such as in a synchronous - function or method. It creates a separate process to run the asynchronous coroutine and waits for it to complete - before returning control to the caller. - - Parameters: - - coroutine_function (callable): The asynchronous coroutine function to be run. - - loop (asyncio.unix_events._UnixSelectorEventLoop): The event loop to be used for running the coroutine. - - ttl (int): The time-to-live (TTL) for the process. If the process does not complete within this time, it will be terminated. - - *args: The arguments to be passed to the coroutine function. - - Usage: - ```python - async def async_function(): - await asyncio.sleep(1) - print("Async function completed") - - def sync_function(): - run_async_in_sync_context(async_function) - print("Sync function completed") - ``` - """ - if loop is None: - loop = asyncio.get_event_loop() - - def sync_wrapper(self): - async def run_async_coro(): - await asyncio.gather(coroutine_function(*args)) - loop.run_until_complete(run_async_coro()) - - process = multiprocessing.Process(target=sync_wrapper, args=args) - process.start() - process.join(timeout=ttl) diff --git a/subnet/protocol.py b/subnet/protocol.py index b7f34324..e3ee6dc3 100644 --- a/subnet/protocol.py +++ b/subnet/protocol.py @@ -20,6 +20,14 @@ import typing import bittensor as bt + +class Score(bt.Synapse): + availability: float + latency: float + reliability: float + distribution: float + score: float + class IsAlive(bt.Synapse): # Returns answer: typing.Optional[str] = None @@ -34,9 +42,6 @@ class Key(bt.Synapse): class Subtensor(bt.Synapse): - # Parameters - task: int - # Returns subtensor_ip: typing.Optional[str] = None diff --git a/subnet/shared/checks.py b/subnet/shared/checks.py index f5f3516f..1bd9b768 100644 --- a/subnet/shared/checks.py +++ b/subnet/shared/checks.py @@ -1,8 +1,6 @@ import subprocess -import time import asyncio -import re -import os +import bittensor as bt from redis import asyncio as aioredis @@ -142,3 +140,17 @@ def _get_redis_password(redis_conf_path): assert False, f"An error occurred: {e}" return None + + +def check_registration(subtensor, wallet, netuid): + if not subtensor.is_hotkey_registered( + netuid=netuid, + hotkey_ss58=wallet.hotkey.ss58_address, + ): + bt.logging.error( + f"Wallet: {wallet} is not registered on netuid {netuid}" + f"Please register the hotkey using `btcli subnets register` before trying again" + ) + exit() + + pass \ No newline at end of file diff --git a/subnet/shared/utils.py b/subnet/shared/utils.py index 7303fc2f..8f6e523f 100644 --- a/subnet/shared/utils.py +++ b/subnet/shared/utils.py @@ -17,118 +17,10 @@ # DEALINGS IN THE SOFTWARE. import os -import re -import json -import base64 import subprocess import bittensor as bt -from typing import List, Union -from redis import asyncio as aioredis -async def safe_key_search(database: aioredis.Redis, pattern: str) -> List[str]: - """ - Safely search for keys in the database that doesn't block. - `scan_iter` uses cursor under the hood. - """ - return [key async for key in database.scan_iter(pattern)] - - -def b64_encode(data: Union[bytes, str, List[str], List[bytes], dict]) -> str: - """ - Encodes the given data into a base64 string. If the data is a list or dictionary of bytes, it converts - the bytes into hexadecimal strings before encoding. - - Args: - data (list or dict): The data to be base64 encoded. Can be a list of bytes or a dictionary with bytes values. - - Returns: - str: The base64 encoded string of the input data. - - Raises: - TypeError: If the input is not a list, dict, or bytes. - """ - if isinstance(data, bytes): - data = data.hex() - if isinstance(data, list) and len(data) and isinstance(data[0], bytes): - data = [d.hex() for d in data] - if isinstance(data, dict) and isinstance(data[list(data.keys())[0]], bytes): - data = {k: v.hex() for k, v in data.items()} - return base64.b64encode(json.dumps(data).encode()).decode("utf-8") - - -def b64_decode(data: bytes, decode_hex: bool = False, encrypted: bool = False): - """ - Decodes a base64 string into a list or dictionary. If decode_hex is True, it converts any hexadecimal strings - within the data back into bytes. - - Args: - data (bytes or str): The base64 encoded data to be decoded. - decode_hex (bool): A flag to indicate whether to decode hex strings into bytes. Defaults to False. - - Returns: - list or dict: The decoded data. Returns a list if the original encoded data was a list, and a dict if it was a dict. - - Raises: - ValueError: If the input is not properly base64 encoded or if hex decoding fails. - """ - data = data.decode("utf-8") if isinstance(data, bytes) else data - decoded_data = json.loads( - base64.b64decode(data) if encrypted else base64.b64decode(data).decode("utf-8") - ) - if decode_hex: - try: - decoded_data = ( - [bytes.fromhex(d) for d in decoded_data] - if isinstance(decoded_data, list) - else {k: bytes.fromhex(v) for k, v in decoded_data.items()} - ) - except: # TODO: do not use bare except - pass - return decoded_data - - -def chunk_data(data: bytes, chunksize: int) -> List[bytes]: - """ - Generator function that chunks the given data into pieces of a specified size. - - Args: - data (bytes): The binary data to be chunked. - chunksize (int): The size of each chunk in bytes. - - Yields: - bytes: A chunk of the data with the size equal to 'chunksize' or the remaining size of data. - - Raises: - ValueError: If 'chunksize' is less than or equal to 0. - """ - for i in range(0, len(data), chunksize): - yield data[i : i + chunksize] - - -def get_redis_port(): - """ - Gets the port number of the Redis server. - - Returns: - str: The port number of the Redis server. - - Raises: - CalledProcessError: If the command to get the Redis service status fails. - """ - - try: - result = subprocess.check_output( - ["sudo", "systemctl", "status", "redis-server.service"], text=True - ) - match = re.search(r"(\d{1,3}\.){3}\d{1,3}:(\d+)", result) - if match: - return match.group(2) - else: - return "Redis server port not found in the service status." - except subprocess.CalledProcessError as e: - return "Failed to get Redis service status: " + str(e) - def get_redis_password( redis_password: str = None, redis_conf: str = "/etc/redis/redis.conf" diff --git a/subnet/shared/weights.py b/subnet/shared/weights.py index 40e3208d..817b606a 100644 --- a/subnet/shared/weights.py +++ b/subnet/shared/weights.py @@ -2,6 +2,7 @@ from bittensor import subtensor from bittensor import wallet from torch import Tensor +from typing import Tuple def should_wait_to_set_weights(current_block, last_epoch_block, tempo): @@ -31,7 +32,7 @@ def set_weights( version_key: int, wait_for_inclusion: bool = False, wait_for_finalization: bool = False, -) -> bool: +) -> Tuple[bool, str]: """ Sets the miner's weights on the Bittensor network. @@ -43,6 +44,7 @@ def set_weights( 1. Queries the Bittensor network for the total number of peers. 2. Sets a weight vector with a value of 1 for the current miner and 0 for all other peers. 3. Updates these weights on the Bittensor network using the `set_weights` method of the subtensor. + 4. Optionally logs the weight-setting operation to Weights & Biases (wandb) for monitoring. Args: subtensor (bt.subtensor): The Bittensor object managing the blockchain connection. @@ -51,6 +53,7 @@ def set_weights( uids (torch.Tensor): miners UIDs on the network. weights (torch.Tensor): weights to sent for UIDs on the network. metagraph (bt.metagraph): Bittensor metagraph. + wandb_on (bool, optional): Flag to determine if logging to Weights & Biases is enabled. Defaults to False. wait_for_inclusion (bool, optional): Wether to wait for the extrinsic to enter a block. wait_for_finalization (bool, optional): Wether to wait for the extrinsic to be finalized on the chain. @@ -66,7 +69,7 @@ def set_weights( """ try: # --- Set weights. - success = subtensor.set_weights( + success, message = subtensor.set_weights( wallet=wallet, netuid=netuid, uids=uids, @@ -75,8 +78,8 @@ def set_weights( wait_for_finalization=wait_for_finalization, version_key=version_key, ) - - return success + + return success, message except Exception as e: bt_logging.error(f"Failed to set weights on chain with exception: { e }") - return False + return False, message diff --git a/subnet/validator/bonding.py b/subnet/validator/bonding.py index e2f8bb8c..dc19254e 100644 --- a/subnet/validator/bonding.py +++ b/subnet/validator/bonding.py @@ -165,15 +165,12 @@ async def register_miner(ss58_address: str, database: aioredis.Redis): await database.hmset( f"stats:{ss58_address}", { - "store_attempts": 0, - "store_successes": 0, - "challenge_successes": 0, - "challenge_attempts": 0, - "retrieve_successes": 0, - "retrieve_attempts": 0, + "subtensor_successes": 0, + "subtensor_attempts": 0, + "metric_successes": 0, + "metric_attempts": 0, "total_successes": 0, "tier": "Bronze", - "storage_limit": STORAGE_LIMIT_BRONZE, }, ) @@ -200,59 +197,37 @@ async def update_statistics( # Update statistics in the stats hash stats_key = f"stats:{ss58_address}" - if task_type in ["store", "challenge", "retrieve"]: + if task_type in ["challenge"]: await database.hincrby(stats_key, f"{task_type}_attempts", 1) if success: await database.hincrby(stats_key, f"{task_type}_successes", 1) - # Transition retireval -> retrieve successes (legacy) - legacy_retrieve_successes = await database.hget(stats_key, "retrieval_successes") - if legacy_retrieve_successes is not None: - await database.hset( - stats_key, "retrieve_successes", int(legacy_retrieve_successes) - ) - await database.hdel(stats_key, "retrieval_successes") - - # Transition retireval -> retrieve attempts (legacy) - legacy_retrieve_attempts = await database.hget(stats_key, "retrieval_attempts") - if legacy_retrieve_attempts is not None: - await database.hset( - stats_key, "retrieve_attempts", int(legacy_retrieve_attempts) - ) - await database.hdel(stats_key, "retrieval_attempts") - # Update the total successes that we rollover every epoch if await database.hget(stats_key, "total_successes") is None: - store_successes = int(await database.hget(stats_key, "store_successes")) - challenge_successes = int(await database.hget(stats_key, "challenge_successes")) - retrieval_successes = int(await database.hget(stats_key, "retrieve_successes")) - total_successes = store_successes + retrieval_successes + challenge_successes + subtensor_successes = int(await database.hget(stats_key, "subtensor_successes")) + metric_successes = int(await database.hget(stats_key, "metric_successes")) + total_successes = subtensor_successes + metric_successes await database.hset(stats_key, "total_successes", total_successes) if success: await database.hincrby(stats_key, "total_successes", 1) -async def compute_tier(stats_key: str, database: aioredis.Redis, confidence=0.95): +async def compute_tier(stats_key: str, database: aioredis.Redis): if not await database.exists(stats_key): bt.logging.warning(f"Miner key {stats_key} is not registered!") return # Retrieve statistics from the database - challenge_successes = int( - await database.hget(stats_key, "challenge_successes") or 0 + subtensor_successes = int( + await database.hget(stats_key, "subtensor_successes") or 0 ) - challenge_attempts = int(await database.hget(stats_key, "challenge_attempts") or 0) - # retrieval_successes = int(await database.hget(stats_key, "retrieve_successes") or 0) - # retrieval_attempts = int(await database.hget(stats_key, "retrieve_attempts") or 0) - # store_successes = int(await database.hget(stats_key, "store_successes") or 0) - # store_attempts = int(await database.hget(stats_key, "store_attempts") or 0) + subtensor_attempts = int(await database.hget(stats_key, "subtensor_attempts") or 0) + metric_successes = int(await database.hget(stats_key, "metric_successes") or 0) + metric_attempts = int(await database.hget(stats_key, "metric_attempts") or 0) total_successes = int(await database.hget(stats_key, "total_successes") or 0) - total_current_attempts = challenge_attempts #+ retrieval_attempts + store_attempts - total_current_successes = challenge_successes - # ( - # challenge_successes + retrieval_successes + store_successes - # ) + total_current_attempts = subtensor_attempts + metric_attempts + total_current_successes = subtensor_successes + metric_successes # Compute the overall success rate across all tasks current_wilson_score = wilson_score_interval( @@ -262,27 +237,29 @@ async def compute_tier(stats_key: str, database: aioredis.Redis, confidence=0.95 f"Miner {stats_key} current total success rate: {current_wilson_score}" ) + # + # Use the lower bounds of the intervals to determine the tier if ( - current_wilson_score >= SUPER_SAIYAN_SUCCESS_RATE + current_wilson_score >= SUPER_SAIYAN_WILSON_SCORE and total_successes >= SUPER_SAIYAN_TIER_TOTAL_SUCCESSES ): bt.logging.trace(f"Setting {stats_key} to Super Saiyan tier.") tier = "Super Saiyan" elif ( - current_wilson_score >= DIAMOND_SUCCESS_RATE + current_wilson_score >= DIAMOND_WILSON_SCORE and total_successes >= DIAMOND_TIER_TOTAL_SUCCESSES ): bt.logging.trace(f"Setting {stats_key} to Diamond tier.") tier = "Diamond" elif ( - current_wilson_score >= GOLD_SUCCESS_RATE + current_wilson_score >= GOLD_WILSON_SCORE and total_successes >= GOLD_TIER_TOTAL_SUCCESSES ): bt.logging.trace(f"Setting {stats_key} to Gold tier.") tier = "Gold" elif ( - current_wilson_score >= SILVER_SUCCESS_RATE + current_wilson_score >= SILVER_WILSON_SCORE and total_successes >= SILVER_TIER_TOTAL_SUCCESSES ): bt.logging.trace(f"Setting {stats_key} to Silver tier.") @@ -294,26 +271,6 @@ async def compute_tier(stats_key: str, database: aioredis.Redis, confidence=0.95 # Update the tier in the database await database.hset(stats_key, "tier", tier) - # Update storage limit based on tier - if tier == "Super Saiyan": - storage_limit = STORAGE_LIMIT_SUPER_SAIYAN - elif tier == "Diamond": - storage_limit = STORAGE_LIMIT_DIAMOND - elif tier == "Gold": - storage_limit = STORAGE_LIMIT_GOLD - elif tier == "Silver": - storage_limit = STORAGE_LIMIT_SILVER - else: # Bronze - storage_limit = STORAGE_LIMIT_BRONZE - - current_limit = await database.hget(stats_key, "storage_limit") - bt.logging.trace(f"Current storage limit for {stats_key}: {current_limit}") - if current_limit.decode() != storage_limit: - await database.hset(stats_key, "storage_limit", storage_limit) - bt.logging.trace( - f"Storage limit for {stats_key} set from {current_limit} -> {storage_limit} bytes." - ) - async def compute_all_tiers(database: aioredis.Redis): # Iterate over all miners diff --git a/subnet/validator/challenge-2.py b/subnet/validator/challenge-2.py new file mode 100644 index 00000000..ae0708d2 --- /dev/null +++ b/subnet/validator/challenge-2.py @@ -0,0 +1,108 @@ +import time +import typing +import asyncio +import bittensor as bt + +from subnet import protocol + +from subnet.validator.utils import get_available_query_miners + + +async def handle_synapse(self, uid: int, subtensor_ip: str) -> typing.Tuple[bool, protocol.Challenge]: + # Get the axon + axon = self.metagraph.axons[uid] + + # Send the public ssh key to the miner + response = self.dendrite.query( + # Send the query to selected miner axons in the network. + axons=[axon], + # Construct a dummy query. This simply contains a single integer. + synapse=protocol.Challenge(), + # All responses have the deserialize function called on them before returning. + # You are encouraged to define your own deserialization function. + deserialize=True, + ) + + # Get the current block by requesting the miner subtensor + try: + # Create a subtensor with the ip return by the synapse + config = bt.subtensor.config() + config.subtensor.network = "local" + config.subtensor.chain_endpoint = f"ws://{subtensor_ip}:9944" + miner_subtensor = bt.subtensor(config) + + # Get the current block + current_block = miner_subtensor.get_current_block() + verified = current_block == response[0].answer + except Exception: + verified = False + + return verified, response + + +async def challenge_data(self): + start_time = time.time() + bt.logging.debug(f"[Challenge] Step starting") + + # Select the miners + uids = await get_available_query_miners(self, k=10) + bt.logging.debug(f"[Challenge] Available uids {uids}") + + # Send the challenge + tasks = [] + responses = [] + for idx, (uid) in enumerate(uids): + # Get the coldkey + axon = self.metagraph.axons[idx] + coldkey = axon.coldkey + + # Get the hotkey + hotkey = self.metagraph.hotkeys[uid] + + # Get the subs hash + subs_key = f"subs:{coldkey}:{hotkey}" + + # Get the subtensor ip + subtensor_ip = await self.database.hget(subs_key, "ip") + + tasks.append(asyncio.create_task(handle_synapse(self, uid, subtensor_ip))) + responses = await asyncio.gather(*tasks) + + remove_reward_idxs = [] + for idx, (uid, (verified, response)) in enumerate( + zip(uids, responses) + ): + if not verified: + if idx not in remove_reward_idxs: + remove_reward_idxs.append(idx) + bt.logging.warning(f"[Challenge][{uid}] The challenge could not be verified") + continue + + bt.logging.success(f"[Challenge][{uid}] The challenge is verified") + + # Get the coldkey + axon = self.metagraph.axons[idx] + coldkey = axon.coldkey + + # Get the hotkey + hotkey = self.metagraph.hotkeys[uid] + + # Update subtensor statistics in the subs hash + subs_key = f"subs:{coldkey}:{hotkey}" + + # Processing time download + process_time = response[0].dendrite.process_time + legacy_process_time = await self.database.hget(subs_key, "process_time") + if legacy_process_time is not None: + process_time = (float(legacy_process_time) + process_time) / 2 + + await self.database.hset(subs_key, "process_time", process_time) + bt.logging.info(f"[Challenge][{uid}] Process time {process_time}") + + + # Display step time + forward_time = time.time() - start_time + bt.logging.debug(f"[Challenge] Step finished in {forward_time:.2f}s") + + return remove_reward_idxs + diff --git a/subnet/validator/challenge.py b/subnet/validator/challenge.py index 794f906c..9750b18b 100644 --- a/subnet/validator/challenge.py +++ b/subnet/validator/challenge.py @@ -1,101 +1,175 @@ +import torch import time -import typing import asyncio import bittensor as bt from subnet import protocol - -from subnet.validator.utils import get_available_query_miners - - -async def handle_synapse(self, uid: int, subtensor_ip: str) -> typing.Tuple[bool, protocol.Challenge]: - # Get the axon - axon = self.metagraph.axons[uid] - - # Send the public ssh key to the miner - response = self.dendrite.query( - # Send the query to selected miner axons in the network. - axons=[axon], - # Construct a dummy query. This simply contains a single integer. - synapse=protocol.Challenge(), - # All responses have the deserialize function called on them before returning. - # You are encouraged to define your own deserialization function. - deserialize=True, - ) - - # Get the current block by requesting the miner subtensor +from subnet.constants import ( + AVAILABILITY_FAILURE_REWARD, + LATENCY_FAILURE_REWARD, + DISTRIBUTION_FAILURE_REWARD, + AVAILABILITY_WEIGHT, + LATENCY_WEIGHT, + RELIABILLITY_WEIGHT, + DISTRIBUTION_WEIGHT, +) +from subnet.shared.subtensor import get_current_block +from subnet.validator.utils import ping_and_retry_uids +from subnet.validator.localisation import get_country +from subnet.validator.bonding import update_statistics +from subnet.validator.score import ( + compute_reliability_score, + compute_latency_score, + compute_distribution_score, +) + + +CHALLENGE_NAME = "Challenge" + + +async def handle_synapse(self, uid: int): + # Get miner ip + ip = self.metagraph.axons[uid].ip + + # Get the country of the subtensor via a free api + country = get_country(ip) + bt.logging.debug(f"[{CHALLENGE_NAME}][{uid}] Subtensor country {country}") + + process_time = None try: # Create a subtensor with the ip return by the synapse config = bt.subtensor.config() config.subtensor.network = "local" - config.subtensor.chain_endpoint = f"ws://{subtensor_ip}:9944" - miner_subtensor = bt.subtensor(config) + config.subtensor.chain_endpoint = f"ws://{ip}:9944" + miner_subtensor = bt.subtensor(config, log_verbose=False) - # Get the current block - current_block = miner_subtensor.get_current_block() - verified = current_block == response[0].answer + # Start the timer + start_time = time.time() + + # Get the current block from the miner subtensor + miner_block = miner_subtensor.get_current_block() + + # Compute the process time + process_time = time.time() - start_time + + # Get the current block from the validator subtensor + validator_block = get_current_block(self.subtensor) + + # Check both blocks are the same + verified = miner_block == validator_block except Exception: verified = False - return verified, response + return verified, country, process_time -async def challenge_data(self): +async def challenge_data(self, keys: list): start_time = time.time() - bt.logging.debug(f"[Challenge] Starting") + bt.logging.debug(f"[{CHALLENGE_NAME}] Step starting") # Select the miners - uids = await get_available_query_miners(self, k=10) - bt.logging.debug(f"[Challenge] Available uids {uids}") + uids, _ = await ping_and_retry_uids(self, k=10) + bt.logging.debug(f"[{CHALLENGE_NAME}] Available uids {uids}") + + # Initialise the rewards object + rewards: torch.FloatTensor = torch.zeros(len(uids), dtype=torch.float32).to( + self.device + ) - # Send the challenge + # Execute the challenge tasks = [] responses = [] for idx, (uid) in enumerate(uids): - # Get the coldkey - axon = self.metagraph.axons[idx] - coldkey = axon.coldkey - - # Get the hotkey - hotkey = self.metagraph.hotkeys[uid] - - # Get the subs hash - subs_key = f"subs:{coldkey}:{hotkey}" - - # Get the subtensor ip - subtensor_ip = await self.database.hget(subs_key, "ip") - - tasks.append(asyncio.create_task(handle_synapse(self, uid, subtensor_ip))) + tasks.append(asyncio.create_task(handle_synapse(self, uid))) responses = await asyncio.gather(*tasks) - # Check the challenge and save the processing time - for idx, (uid, (verified, response)) in enumerate( - zip(uids, responses) - ): - if not verified: - # TODO: do we punished miner now, later or never? - continue - - # Get the coldkey - axon = self.metagraph.axons[idx] - coldkey = axon.coldkey - + # Compute the score + for idx, (uid, (verified)) in enumerate(zip(uids, responses)): # Get the hotkey hotkey = self.metagraph.hotkeys[uid] - # Update subtensor statistics in the subs hash - subs_key = f"subs:{coldkey}:{hotkey}" - - # Processing time download - process_time = response[0].dendrite.process_time - legacy_process_time = await self.database.hget(subs_key, "process_time") - if legacy_process_time is not None: - process_time = (float(legacy_process_time) + process_time) / 2 + # Update statistics + await update_statistics( + ss58_address=hotkey, + success=verified, + task_type="challenge", + database=self.database, + ) + + # Compute score for availability + availability_score = 1.0 if verified else AVAILABILITY_FAILURE_REWARD + bt.logging.debug( + f"[{CHALLENGE_NAME}][{uid}] Availability score {availability_score}" + ) + + # Compute score for latency + latency_score = ( + compute_latency_score(idx, uid, self.country, responses) + if verified + else LATENCY_FAILURE_REWARD + ) + bt.logging.debug(f"[{CHALLENGE_NAME}][{uid}] Latency score {latency_score}") + + # Compute score for reliability + reliability_score = await compute_reliability_score(self.database, hotkey) + bt.logging.debug( + f"[{CHALLENGE_NAME}][{uid}] Reliability score {reliability_score}" + ) + + # Compute score for distribution + distribution_score = ( + compute_distribution_score(idx, responses) + if responses[idx][2] is not None + else DISTRIBUTION_FAILURE_REWARD + ) + bt.logging.debug( + f"[{CHALLENGE_NAME}][{uid}] Distribution score {distribution_score}" + ) + + # Compute final score + rewards[idx] = ( + (AVAILABILITY_WEIGHT * availability_score) + + (LATENCY_WEIGHT * latency_score) + + (RELIABILLITY_WEIGHT * reliability_score) + + (DISTRIBUTION_WEIGHT * distribution_score) + ) / 4.0 + bt.logging.info(f"[{CHALLENGE_NAME}][{uid}] Final score {rewards[idx]}") + + # Send the score details to the miner + await self.dendrite( + axons=[self.metagraph.axons[uid]], + synapse=protocol.Score( + availability=availability_score, + latency=latency_score, + reliability=reliability_score, + distribution=distribution_score, + score=rewards[idx] + ), + deserialize=True, + timeout=5, + ) + + + # Compute forward pass rewards + scattered_rewards: torch.FloatTensor = ( + self.moving_averaged_scores.to(self.device) + .scatter( + 0, + torch.tensor(uids).to(self.device), + rewards.to(self.device), + ) + .to(self.device) + ) + bt.logging.trace(f"Scattered rewards: {scattered_rewards}") - await self.database.hset(subs_key, "process_time", process_time) - bt.logging.info(f"[Challenge] Download {process_time}") + # Update moving_averaged_scores with rewards produced by this step. + # alpha of 0.05 means that each new score replaces 5% of the weight of the previous weights + alpha: float = 0.05 + self.moving_averaged_scores = alpha * scattered_rewards + ( + 1 - alpha + ) * self.moving_averaged_scores.to(self.device) + bt.logging.trace(f"Updated moving avg scores: {self.moving_averaged_scores}") # Display step time forward_time = time.time() - start_time - bt.logging.debug(f"[Challenge] Step time {forward_time:.2f}s") - + bt.logging.debug(f"[{CHALLENGE_NAME}] Step finished in {forward_time:.2f}s") diff --git a/subnet/validator/config.py b/subnet/validator/config.py index 29c3cf30..93a75bd5 100644 --- a/subnet/validator/config.py +++ b/subnet/validator/config.py @@ -129,24 +129,6 @@ def add_args(cls, parser): help="Device to run the validator on.", default="cuda" if torch.cuda.is_available() else "cpu", ) - parser.add_argument( - "--neuron.curve", - default="P-256", - help="Curve for elliptic curve cryptography.", - choices=["P-256"], # TODO: expand this list - ) - parser.add_argument( - "--neuron.maxsize", - default=None, # Use lognormal random gaussian if None (2**16, # 64KB) - type=int, - help="Maximum size of random data to store.", - ) - parser.add_argument( - "--neuron.min_chunk_size", - default=256, - type=int, - help="Minimum chunk size of random data to challenge (bytes).", - ) parser.add_argument( "--neuron.disable_log_rewards", action="store_true", @@ -159,12 +141,6 @@ def add_args(cls, parser): help="The path to save subscription logs.", default="subscription_logs.txt", ) - parser.add_argument( - "--neuron.chunk_factor", - type=int, - help="The chunk factor to divide data.", - default=4, - ) parser.add_argument( "--neuron.num_concurrent_forwards", type=int, @@ -219,12 +195,6 @@ def add_args(cls, parser): help="If set, we will log responses. These can be LONG.", default=False, ) - parser.add_argument( - "--neuron.data_ttl", - type=int, - help="The number of blocks before data expires (seconds).", - default=60 * 60 * 24 * 30, # 30 days - ) parser.add_argument( "--neuron.profile", action="store_true", @@ -268,26 +238,6 @@ def add_args(cls, parser): "--mock", action="store_true", help="Mock all items.", default=False ) - # Encryption wallet - parser.add_argument( - "--encryption.wallet_name", - type=str, - help="The name of the wallet to use for encryption.", - default="core_storage_coldkey", - ) - parser.add_argument( - "--encryption.wallet_hotkey", - type=str, - help="The hotkey name of the wallet to use for encryption.", - default="core_storage_hotkey", - ) - parser.add_argument( - "--encryption.password", - type=str, - help="The password of the wallet to use for encryption.", - default="dummy_password", - ) - def config(cls): parser = argparse.ArgumentParser() diff --git a/subnet/validator/database.py b/subnet/validator/database.py index f2c9ac78..ab3943c3 100644 --- a/subnet/validator/database.py +++ b/subnet/validator/database.py @@ -17,265 +17,10 @@ # DEALINGS IN THE SOFTWARE. import json -import time -from redis import asyncio as aioredis -import asyncio import bittensor as bt -from typing import Dict, List, Any, Union, Optional - - -async def set_ttl_for_hash_and_hotkey( - data_hash: str, - ss58_address: str, - database: aioredis.Redis, - ttl: int = 60 * 60 * 24 * 30, -): - """ - Sets the TTL for a hash in Redis. - - Parameters: - data_hash (str): The key representing the hash. - database (aioredis.Redis): The Redis client instance. - ttl (int): The TTL in seconds. (Default 30 days) - """ - key = f"hotkey:{ss58_address}" - ttl_metadata = { - "generated": time.time(), # This is required to compare against later - "ttl": ttl, - } - ttl_metadata_json = json.dumps(ttl_metadata) - await database.hset(key, f"ttl:{data_hash}", ttl_metadata_json) - bt.logging.trace(f"Set TTL for {data_hash} to {ttl} seconds.") - - -async def get_ttl_for_hash_and_hotkey( - data_hash: str, ss58_address: str, database: aioredis.Redis -) -> int: - """ - Retrieves the TTL for a hash in Redis. - - Parameters: - data_hash (str): The key representing the hash. - database (aioredis.Redis): The Redis client instance. - - Returns: - The TTL in seconds, or None if not found. - """ - key = f"hotkey:{ss58_address}" - ttl_metadata = await database.hget(key, f"ttl:{data_hash}") - if ttl_metadata: - ttl_metadata = json.loads(ttl_metadata) - ttl = int(ttl_metadata["ttl"]) - return ttl - else: - return None - - -async def get_time_since_generation_for_hash_and_hotkey( - data_hash: str, ss58_address: str, database: aioredis.Redis -) -> Optional[int]: - """ - Retrieves the TTL for a hash in Redis. - - Parameters: - data_hash (str): The key representing the hash. - database (aioredis.Redis): The Redis client instance. - - Returns: - The TTL in seconds, or None if not found. - """ - key = f"hotkey:{ss58_address}" - ttl_metadata = await database.hget(key, f"ttl:{data_hash}") - if ttl_metadata: - ttl_metadata = json.loads(ttl_metadata) - generated = float(ttl_metadata["generated"]) - return time.time() - generated - else: - return None - - -async def is_ttl_expired_for_hash_and_hotkey( - data_hash: str, ss58_address: str, database: aioredis.Redis -) -> bool: - """ - Checks if the TTL for a hash has expired. - - Parameters: - data_hash (str): The key representing the hash. - database (aioredis.Redis): The Redis client instance. - - Returns: - True if the TTL has expired, False otherwise. - """ - key = f"hotkey:{ss58_address}" - elapsed = await get_time_since_generation_for_hash_and_hotkey( - data_hash, ss58_address, database - ) - ttl = await get_ttl_for_hash_and_hotkey(data_hash, ss58_address, database) - if elapsed > ttl: - return True - else: - return False - - -async def purge_expired_ttl_keys(database: aioredis.Redis): - """ - Purges all expired TTL keys from the database. - - Parameters: - database (aioredis.Redis): The Redis client instance. - """ - # Iterate over all hotkeys - async for hotkey in database.scan_iter("*"): - if not hotkey.startswith(b"hotkey:"): - continue - data_hashes = await database.hgetall(hotkey) - for data_hash in data_hashes: - data_hash = data_hash.decode("utf-8") - if data_hash.startswith("ttl:"): - hk = hotkey.decode("utf-8")[7:] - hs = data_hash[4:] - if await is_ttl_expired_for_hash_and_hotkey(hs, hk, database): - await remove_metadata_from_hotkey(hk, hs, database) - - -async def add_metadata_to_hotkey( - ss58_address: str, - data_hash: str, - metadata: Dict, - database: aioredis.Redis, - ttl: Optional[int] = None, -): - """ - Associates a data hash and its metadata with a hotkey in Redis. - - Parameters: - ss58_address (str): The primary key representing the hotkey. - data_hash (str): The subkey representing the data hash. - metadata (dict): The metadata to associate with the data hash. Includes the size of the data, the seed, - and the encryption payload. E.g. {'size': 123, 'seed': 456, 'encryption_payload': 'abc'}. - database (aioredis.Redis): The Redis client instance. - """ - # Serialize the metadata as a JSON string - metadata_json = json.dumps(metadata) - # Use HSET to associate the data hash with the hotkey - key = f"hotkey:{ss58_address}" - await database.hset(key, data_hash, metadata_json) - bt.logging.trace(f"Associated data hash {data_hash} with hotkey {ss58_address}.") - - if ttl: - await set_ttl_for_hash_and_hotkey(data_hash, ss58_address, database, ttl) - - -async def remove_metadata_from_hotkey( - ss58_address: str, data_hash: str, database: aioredis.Redis -): - """ - Removes a data hash and its metadata from a hotkey in Redis. - - Parameters: - ss58_address (str): The primary key representing the hotkey. - data_hash (str): The subkey representing the data hash. - database (aioredis.Redis): The Redis client instance. - """ - # Use HDEL to remove the data hash from the hotkey - key = f"hotkey:{ss58_address}" - await database.hdel(key, data_hash) - await database.hdel(key, f"ttl:{data_hash}") # delete the TTL as well - bt.logging.trace(f"Removed data hash {data_hash} from hotkey {ss58_address}.") - - -async def get_metadata_for_hotkey( - ss58_address: str, database: aioredis.Redis -) -> Dict[str, dict]: - """ - Retrieves all data hashes and their metadata for a given hotkey. - - Parameters: - ss58_address (str): The key representing the hotkey. - database (aioredis.Redis): The Redis client instance. - - Returns: - A dictionary where keys are data hashes and values are the associated metadata. - """ - # Fetch all fields (data hashes) and values (metadata) for the hotkey - all_data_hashes = await database.hgetall(f"hotkey:{ss58_address}") - - # Deserialize the metadata for each data hash - data_hash_metadata = { - data_hash.decode("utf-8"): json.loads(metadata.decode("utf-8")) - for data_hash, metadata in all_data_hashes.items() - if not data_hash.startswith(b"ttl:") - } - - # TODO: include the TTL in this metadata as a unified dict? - bt.logging.trace( - f"get_metadata_for_hotkey() # hashes found for hotkey {ss58_address}: {len(data_hash_metadata)}" - ) - return data_hash_metadata - - -async def get_hashes_for_hotkey( - ss58_address: str, database: aioredis.Redis -) -> List[str]: - """ - Retrieves all data hashes and their metadata for a given hotkey. - - Parameters: - ss58_address (str): The key representing the hotkey. - database (aioredis.Redis): The Redis client instance. - - Returns: - A dictionary where keys are data hashes and values are the associated metadata. - """ - # Fetch all fields (data hashes) and values (metadata) for the hotkey - all_data_hashes = await database.hgetall(f"hotkey:{ss58_address}") - - # Deserialize the metadata for each data hash - return [ - data_hash.decode("utf-8") for data_hash, metadata in all_data_hashes.items() - ] - - -async def remove_hashes_for_hotkey( - ss58_address: str, hashes: list, database: aioredis.Redis -) -> List[str]: - """ - Retrieves all data hashes and their metadata for a given hotkey. - - Parameters: - ss58_address (str): The key representing the hotkey. - database (aioredis.Redis): The Redis client instance. - - Returns: - A dictionary where keys are data hashes and values are the associated metadata. - """ - bt.logging.trace( - f"remove_hashes_for_hotkey() removing {len(hashes)} hashes from hotkey {ss58_address}" - ) - for _hash in hashes: - await remove_metadata_from_hotkey(ss58_address, _hash, database) - - -async def update_metadata_for_data_hash( - ss58_address: str, data_hash: str, new_metadata: dict, database: aioredis.Redis -): - """ - Updates the metadata for a specific data hash associated with a hotkey. - - Parameters: - ss58_address (str): The key representing the hotkey. - data_hash (str): The subkey representing the data hash to update. - new_metadata (dict): The new metadata to associate with the data hash. - database (aioredis.Redis): The Redis client instance. - """ - # Serialize the new metadata as a JSON string - new_metadata_json = json.dumps(new_metadata) - # Update the field in the hash with the new metadata - await database.hset(f"hotkey:{ss58_address}", data_hash, new_metadata_json) - bt.logging.trace( - f"Updated metadata for data hash {data_hash} under hotkey {ss58_address}." - ) +from typing import Any +from redis import asyncio as aioredis +from typing import Dict, Union, Optional async def get_metadata_for_hotkey_and_hash( @@ -307,91 +52,6 @@ async def get_metadata_for_hotkey_and_hash( return None -async def get_all_chunk_hashes(database: aioredis.Redis) -> Dict[str, List[str]]: - """ - Retrieves all chunk hashes and associated metadata from the Redis instance. - - Parameters: - database (aioredis.Redis): The Redis client instance. - - Returns: - A dictionary where keys are chunk hashes and values are lists of hotkeys associated with each chunk hash. - """ - # Initialize an empty dictionary to store the inverse map - chunk_hash_hotkeys = {} - - # Retrieve all hotkeys (assuming keys are named with a 'hotkey:' prefix) - async for hotkey in database.scan_iter("*"): - if not hotkey.startswith(b"hotkey:"): - continue - # Fetch all fields (data hashes) for the current hotkey - data_hashes = await database.hkeys(hotkey) - # Iterate over each data hash and append the hotkey to the corresponding list - for data_hash in data_hashes: - data_hash = data_hash.decode("utf-8") - if data_hash not in chunk_hash_hotkeys: - chunk_hash_hotkeys[data_hash] = [] - chunk_hash_hotkeys[data_hash].append(hotkey.decode("utf-8").split(":")[1]) - - return chunk_hash_hotkeys - - -async def get_all_full_hashes(database: aioredis.Redis) -> List[str]: - """ - Retrieves all data hashes and their corresponding hotkeys from the Redis instance. - - Parameters: - database (aioredis.Redis): The Redis client instance. - - Returns: - A dictionary where keys are data hashes and values are lists of hotkeys associated with each data hash. - """ - data_hashes = [] - keys = await database.scan_iter("*") - for key in keys: - if not key.startswith(b"file:"): - continue - data_hashes.append(key.decode("utf-8").split(":")[1]) - - return data_hashes - - -async def get_all_hotkeys_for_data_hash( - data_hash: str, database: aioredis.Redis, is_full_hash: bool = False -) -> List[str]: - """ - Fetch all hotkeys associated with a given hash, which can be a full file hash or a chunk hash. - - Parameters: - - data_hash (str): The hash value of the file or chunk. - - database (aioredis.Redis): An instance of the Redis database. - - is_full_hash (bool): A flag indicating if the hash_value is a full file hash. - - Returns: - - List[str]: A list of hotkeys associated with the hash. - Returns None if no hotkeys are found. - """ - all_hotkeys = set() - - if is_full_hash: - # Get all chunks for the full hash - chunks_info = await get_all_chunks_for_file(data_hash, database) - if chunks_info is None: - return None - # Aggregate hotkeys from all chunks - for chunk_info in chunks_info.values(): - all_hotkeys.update(chunk_info["hotkeys"]) - else: - # Fetch hotkeys for a single chunk hash - chunk_metadata = await database.hgetall(f"chunk:{data_hash}") - if chunk_metadata: - hotkeys = chunk_metadata.get(b"hotkeys") - if hotkeys: - all_hotkeys.update(hotkeys.decode().split(",")) - - return list(all_hotkeys) - - async def total_hotkey_storage( hotkey: str, database: aioredis.Redis, verbose: bool = False ) -> int: @@ -461,95 +121,6 @@ async def hotkey_at_capacity( return False -async def cache_hotkeys_capacity( - hotkeys: List[str], database: aioredis.Redis, verbose: bool = False -): - """ - Caches the capacity information for a list of hotkeys. - - Parameters: - hotkeys (list): List of hotkey strings to check. - database (aioredis.Redis): The Redis client instance. - - Returns: - dict: A dictionary with hotkeys as keys and a tuple of (total_storage, limit) as values. - """ - hotkeys_capacity = {} - - for hotkey in hotkeys: - # Get the total storage used by the hotkey - total_storage = await total_hotkey_storage(hotkey, database, verbose) - # Get the byte limit for the hotkey - byte_limit = await database.hget(f"stats:{hotkey}", "storage_limit") - - if byte_limit is None: - bt.logging.warning(f"Could not find storage limit for {hotkey}.") - limit = None - else: - try: - limit = int(byte_limit) - except Exception as e: - bt.logging.warning(f"Could not parse storage limit for {hotkey} | {e}.") - limit = None - - hotkeys_capacity[hotkey] = (total_storage, limit) - - return hotkeys_capacity - - -async def check_hotkeys_capacity(hotkeys_capacity, hotkey: str, verbose: bool = False): - """ - Checks if a hotkey is at capacity using the cached information. - - Parameters: - hotkeys_capacity (dict): Dictionary with cached capacity information. - hotkey (str): The key representing the hotkey. - - Returns: - True if the hotkey is at capacity, False otherwise. - """ - total_storage, limit = hotkeys_capacity.get(hotkey, (0, None)) - - if limit is None: - # Limit information not available or couldn't be parsed - return False - - if total_storage >= limit: - if verbose: - bt.logging.trace( - f"Hotkey {hotkey} is at max capacity {limit // 1024**3} GB." - ) - return True - else: - if verbose: - bt.logging.trace( - f"Hotkey {hotkey} has {(limit - total_storage) // 1024**3} GB free." - ) - return False - - -async def total_validator_storage(database: aioredis.Redis) -> int: - """ - Calculates the total storage used by all hotkeys in the database. - - Parameters: - database (aioredis.Redis): The Redis client instance. - - Returns: - The total storage used by all hotkeys in the database in bytes. - """ - total_storage = 0 - # Iterate over all hotkeys - async for hotkey in database.scan_iter("*"): - if not hotkey.startswith(b"hotkey:"): - continue - # Grab storage for that hotkey - total_storage += await total_hotkey_storage( - hotkey.decode().split(":")[1], database - ) - return total_storage - - async def get_miner_statistics(database: aioredis.Redis) -> Dict[str, Dict[str, str]]: """ Retrieves statistics for all miners in the database. @@ -569,505 +140,3 @@ async def get_miner_statistics(database: aioredis.Redis) -> Dict[str, Dict[str, stats[key.decode("utf-8").split(":")[-1]] = processed_stats return stats - - -async def get_single_miner_statistics( - ss58_address: str, database: aioredis.Redis -) -> Dict[str, Dict[str, str]]: - """ - Retrieves statistics for all miners in the database. - Parameters: - database (aioredis.Redis): The Redis client instance. - Returns: - A dictionary where keys are hotkeys and values are dictionaries containing the statistics for each hotkey. - """ - stats = await database.hgetall(f"stats:{ss58_address}") - return {k.decode("utf-8"): v.decode("utf-8") for k, v in stats.items()} - - -async def get_redis_db_size(database: aioredis.Redis) -> int: - """ - Calculates the total approximate size of all keys in a Redis database. - Parameters: - database (int): Redis database - Returns: - int: Total size of all keys in bytes - """ - total_size = 0 - async for key in await database.scan_iter("*"): - size = await database.execute_command("MEMORY USAGE", key) - if size: - total_size += size - return total_size - - -async def store_file_chunk_mapping_ordered( - full_hash: str, - chunk_hashes: List[str], - chunk_indices: List[str], - database: aioredis.Redis, - encryption_payload: Optional[Union[bytes, dict]] = None, -): - """ - Store an ordered mapping of file chunks in the database. - - This function takes a file's full hash and the hashes of its individual chunks, along with their - respective indices, and stores them in a sorted set in the Redis database. The order is preserved - based on the chunk index. - - Parameters: - - full_hash (str): The full hash of the file. - - chunk_hashes (List[str]): A list of hashes for the individual chunks of the file. - - chunk_indices (List[int]): A list of indices corresponding to each chunk hash. - - database (aioredis.Redis): An instance of the Redis database. - - encryption_payload (Optional[Union[bytes, dict]]): The encryption payload to store with the file. - """ - key = f"file:{full_hash}" - for chunk_index, chunk_hash in zip(chunk_indices, chunk_hashes): - await database.zadd(key, {chunk_hash: chunk_index}) - - # Store the encryption payload if provided - if encryption_payload: - if isinstance(encryption_payload, dict): - encryption_payload = json.dumps(encryption_payload) - await database.set(f"payload:{full_hash}", encryption_payload) - - -async def retrieve_encryption_payload( - full_hash: str, - database: aioredis.Redis, - return_dict: bool = False, -) -> Optional[Union[bytes, dict]]: - """ - Retrieve the encryption payload for a file. - - This function fetches the encryption payload for a file from the Redis database. - - Parameters: - - full_hash (str): The full hash of the file. - - database (aioredis.Redis): An instance of the Redis database. - - Returns: - - Optional[Union[bytes, dict]]: The encryption payload for the file. - """ - encryption_payload = await database.get(f"payload:{full_hash}") - if encryption_payload: - if return_dict: - return encryption_payload - try: - return json.loads(encryption_payload) - except json.JSONDecodeError: - return encryption_payload - else: - return None - - -async def get_all_chunks_for_file( - file_hash: str, database: aioredis.Redis -) -> Optional[Dict[int, Dict[str, Union[str, List[str], int]]]]: - """ - Retrieve all chunk hashes and their metadata for a given file hash. - - This function fetches the hashes and metadata of all chunks associated with a particular file hash. - The data is retrieved from a sorted set and returned in a dictionary with the chunk index as the key. - - Parameters: - - file_hash (str): The full hash of the file whose chunks are to be retrieved. - - database (aioredis.Redis): An instance of the Redis database. - - Returns: - - dict: A dictionary where keys are chunk indices, and values are dictionaries with chunk metadata. - Returns None if no chunks are found. - """ - file_chunks_key = f"file:{file_hash}" - chunk_hashes_with_index = await database.zrange( - file_chunks_key, 0, -1, withscores=True - ) - if not chunk_hashes_with_index: - return None - - chunks_info = {} - for chunk_hash_bytes, index in chunk_hashes_with_index: - chunk_hash = chunk_hash_bytes.decode() - chunk_metadata = await database.hgetall(f"chunk:{chunk_hash}") - if chunk_metadata: - chunks_info[int(index)] = { - "chunk_hash": chunk_hash, - "hotkeys": chunk_metadata[b"hotkeys"].decode().split(","), - "size": int(chunk_metadata[b"size"]), - } - return chunks_info - - -async def get_hotkeys_for_hash( - hash_value: str, database: aioredis.Redis, is_full_hash: bool = False -): - """ - Fetch all hotkeys associated with a given hash, which can be a full file hash or a chunk hash. - - Parameters: - - hash_value (str): The hash value of the file or chunk. - - database (aioredis.Redis): An instance of the Redis database. - - is_full_hash (bool): A flag indicating if the hash_value is a full file hash. - - Returns: - - List[str]: A list of hotkeys associated with the hash. - Returns None if no hotkeys are found. - """ - all_hotkeys = set() - - if is_full_hash: - # Get UIDs for all chunks under the full hash - chunks_info = get_all_chunks_for_file(hash_value, database) - if chunks_info is None: - return None - for chunk_info in chunks_info.values(): - all_hotkeys.update(chunk_info["hotkeys"]) - else: - # Get UIDs for a single chunk hash - chunk_metadata = await database.hgetall(f"chunk:{hash_value}") - if chunk_metadata: - hotkeys = chunk_metadata.get(b"hotkeys") - if hotkeys: - all_hotkeys.update(hotkeys.decode().split(",")) - - return list(all_hotkeys) - - -async def add_hotkey_to_chunk(chunk_hash: str, hotkey: str, database: aioredis.Redis): - """ - Add a hotkey to the metadata of a specific chunk. - - This function updates the chunk's metadata to include the given hotkey. If the hotkey is already - associated with the chunk, no changes are made. - - Parameters: - - chunk_hash (str): The hash of the chunk to which the hotkey is to be added. - - hotkey (str): The hotkey to add to the chunk's metadata. - - database (aioredis.Redis): An instance of the Redis database. - """ - chunk_metadata_key = f"chunk:{chunk_hash}" - - # Fetch existing UIDs for the chunk - existing_metadata = await database.hget(chunk_metadata_key, "hotkeys") - if existing_metadata: - existing_hotkeys = existing_metadata.decode().split(",") - - # Add new UID if it's not already in the list - if hotkey not in existing_hotkeys: - updated_hotkeys = existing_hotkeys + [hotkey] - await database.hset( - chunk_metadata_key, "hotkeys", ",".join(updated_hotkeys) - ) - print(f"UID {hotkey} added to chunk {chunk_hash}.") - else: - print(f"UID {hotkey} already exists for chunk {chunk_hash}.") - else: - # If no UIDs are associated with this chunk, create a new entry - await database.hmset(chunk_metadata_key, {"hotkeys": hotkey}) - print(f"UID {hotkey} set for new chunk {chunk_hash}.") - - -async def remove_hotkey_from_chunk( - chunk_hash: str, hotkey: str, database: aioredis.Redis, verbose: bool = False -): - """ - Remove a hotkey from the metadata of a specific chunk. - - This function updates the chunk's metadata to remove the given hotkey. If the hotkey is not - associated with the chunk, no changes are made. - - Parameters: - - chunk_hash (str): The hash of the chunk to which the hotkey is to be added. - - hotkey (str): The hotkey to add to the chunk's metadata. - - database (aioredis.Redis): An instance of the Redis database. - """ - chunk_metadata_key = f"chunk:{chunk_hash}" - - # Fetch existing UIDs for the chunk - existing_metadata = await database.hget(chunk_metadata_key, "hotkeys") - if existing_metadata: - existing_hotkeys = existing_metadata.decode().split(",") - - # Remove UID if it's in the list - if hotkey in existing_hotkeys: - existing_hotkeys.remove(hotkey) - await database.hset( - chunk_metadata_key, "hotkeys", ",".join(existing_hotkeys) - ) - if verbose: - bt.logging.trace(f"UID {hotkey} removed from chunk {chunk_hash}.") - else: - if verbose: - bt.logging.trace(f"UID {hotkey} does not exist for chunk {chunk_hash}.") - else: - if verbose: - bt.logging.trace(f"No UIDs associated with chunk {chunk_hash}.") - - -async def store_chunk_metadata( - full_hash: str, - chunk_hash: str, - hotkeys: List[str], - chunk_size: int, - database: aioredis.Redis, -): - """ - Store metadata for a specific file chunk. - - This function creates or updates the metadata for a chunk, including the associated hotkeys and chunk size. - - Parameters: - - full_hash (str): The full hash of the file that the chunk belongs to. - - chunk_hash (str): The hash of the chunk whose metadata is to be stored. - - hotkeys (List[str]): A list of hotkeys associated with the chunk. - - chunk_size (int): The size of the chunk in bytes. - - database (aioredis.Redis): An instance of the Redis database. - """ - chunk_metadata_key = f"chunk:{chunk_hash}" - existing_metadata = await database.hget(chunk_metadata_key, "hotkeys") - if existing_metadata: - existing_hotkeys = existing_metadata.decode().split(",") - hotkeys = set(existing_hotkeys + hotkeys) - metadata = {"hotkeys": ",".join(hotkeys), "size": chunk_size} - - await database.hmset(chunk_metadata_key, metadata) - - -async def get_ordered_metadata( - file_hash: str, database: aioredis.Redis -) -> List[Dict[str, Union[str, List[str], int]]]: - """ - Retrieve the metadata for all chunks of a file in the order of their indices. - - This function calls `get_all_chunks_for_file` to fetch all chunks' metadata and then sorts - them based on their indices to maintain the original file order. - - Parameters: - - file_hash (str): The full hash of the file whose ordered metadata is to be retrieved. - - database (aioredis.Redis): An instance of the Redis database. - - Returns: - - List[dict]: A list of metadata dictionaries for each chunk, ordered by their chunk index. - Returns None if no chunks are found. - """ - chunks_info = await get_all_chunks_for_file(file_hash, database) - if chunks_info is None: - return None - - ordered_chunks = sorted(chunks_info.items(), key=lambda x: x[0]) - return [chunk_info for _, chunk_info in ordered_chunks] - - -# Function to grab mutually exclusiv UIDs for a specific full_hash (get chunks of non-overlapping UIDs) -async def retrieve_mutually_exclusive_hotkeys_full_hash( - full_hash: str, database: aioredis.Redis -) -> Dict[str, List[str]]: - """ - Retrieve a list of mutually exclusive hotkeys for a specific full hash. - - This function retrieves the metadata for all chunks of a file and then sorts them based on their - indices to maintain the original file order. It then iterates over the chunks and adds the hotkeys - of each chunk to the dict of chunk hash <> mutually exclusive hotkey mappings if not already present. - - Parameters: - - full_hash (str): The full hash of the file whose ordered metadata is to be retrieved. - - database (aioredis.Redis): An instance of the Redis database. - - Returns: - - Dict[str, List[str]]: A dict of mutually exclusive hotkeys for each corresponding hash. - Returns None if no chunks are found. - """ - chunks_info = await get_all_chunks_for_file(full_hash, database) - if chunks_info is None: - return None - - ordered_chunks = sorted(chunks_info.items(), key=lambda x: x[0]) - mutually_exclusive_hotkeys = {} - for _, chunk_info in ordered_chunks: - if chunk_info["chunk_hash"] not in mutually_exclusive_hotkeys: - mutually_exclusive_hotkeys[chunk_info["chunk_hash"]] = [] - for hotkey in chunk_info["hotkeys"]: - if hotkey not in mutually_exclusive_hotkeys[chunk_info["chunk_hash"]]: - mutually_exclusive_hotkeys[chunk_info["chunk_hash"]].append(hotkey) - - return mutually_exclusive_hotkeys - - -async def check_hash_type(data_hash: str, database: aioredis.Redis) -> str: - """ - Determine if the data_hash is a full file hash, a chunk hash, or a standalone challenge hash. - - Parameters: - - data_hash (str): The data hash to check. - - database (aioredis.Redis): The Redis database client. - - Returns: - - str: A string indicating the type of hash ('full_file', 'chunk', or 'challenge'). - """ - is_full_file = await database.exists(f"file:{data_hash}") - if is_full_file: - return "full_file" - - is_chunk = await database.exists(f"chunk:{data_hash}") - if is_chunk: - return "chunk" - - return "challenge" - - -async def is_file_chunk(chunk_hash: str, database: aioredis.Redis) -> str: - """ - Determines if the given chunk_hash is part of a full file. - - Parameters: - - chunk_hash (str): The hash of the chunk to check. - - database (aioredis.Redis): The Redis database client. - - Returns: - - bool: True if the hash belongs to a full file, false otherwise (challenge data) - """ - async for key in database.scan_iter(match="chunk:*"): - if chunk_hash in key.decode(): - return True - return False - - -async def get_all_hashes_in_database(database: aioredis.Redis) -> List[str]: - """ - Retrieves all hashes from the Redis instance. - - Parameters: - database (aioredis.Redis): The Redis client instance. - - Returns: - A list of hashes. - """ - all_hashes = set() - - async for hotkey_key in database.scan_iter(match="hotkey:*"): - all_hashes.update(list(await database.hgetall(hotkey_key))) - - return list(all_hashes) - - -async def get_all_challenge_hashes(database: aioredis.Redis) -> List[str]: - """ - Retrieves all challenge hashes from the Redis instance. - - Parameters: - database (aioredis.Redis): The Redis client instance. - - Returns: - A list of challenge hashes. - """ - all_hashes = await get_all_hashes_in_database(database) - - challenge_hashes = [] - for h in all_hashes: - if await check_hash_type(h, database) == "challenge": - challenge_hashes.append(h) - - return challenge_hashes - - -async def get_challenges_for_hotkey(ss58_address: str, database: aioredis.Redis): - """ - Retrieves a list of challenge hashes associated with a specific hotkey. - - This function scans through all the hashes related to a given hotkey and filters out - those which are identified as challenge data. It's useful for identifying which - challenges a particular miner (identified by hotkey) is involved with. - - Parameters: - - ss58_address (str): The hotkey (miner identifier) whose challenge hashes are to be retrieved. - - database (aioredis.Redis): An instance of the Redis database used for data storage. - - Returns: - - List[str]: A list of challenge hashes associated with the given hotkey. - Returns an empty list if no challenge data is associated with the hotkey. - """ - hashes = list(await database.hgetall(f"hotkey:{ss58_address}")) - challenges = [] - for j, h in enumerate(hashes): - if await check_hash_type(h, database) == "challenge": - challenges.append(h) - - return challenges - - -async def purge_challenges_for_hotkey(ss58_address: str, database: aioredis.Redis): - """ - Purges (deletes) all challenge hashes associated with a specific hotkey. - - This function is used for housekeeping purposes in the database, allowing for the - removal of all challenge data related to a particular miner (hotkey). This can be - useful for clearing outdated or irrelevant challenge data from the database. - - Parameters: - - ss58_address (str): The hotkey (miner identifier) whose challenge hashes are to be purged. - - database (aioredis.Redis): An instance of the Redis database used for data storage. - """ - challenge_hashes = await get_challenges_for_hotkey(ss58_address, database) - bt.logging.trace(f"purging challenges for {ss58_address}...") - for ch in challenge_hashes: - await database.hdel(f"hotkey:{ss58_address}", ch) - - -async def purge_challenges_for_all_hotkeys(database: aioredis.Redis): - """ - Purges (deletes) all challenge hashes for every hotkey in the database. - - This function performs a comprehensive cleanup of the database by removing all - challenge-related data. It iterates over each hotkey in the database and - individually purges the challenge hashes associated with them. This is particularly - useful for global maintenance tasks where outdated or irrelevant challenge data - needs to be cleared from the entire database. For example, when a UID is replaced. - - Parameters: - - database (aioredis.Redis): An instance of the Redis database used for data storage. - """ - bt.logging.debug("purging challenges for ALL hotkeys...") - async for hotkey in database.scan_iter(match="hotkey:*"): - hotkey = hotkey.decode().split(":")[1] - await purge_challenges_for_hotkey(hotkey, database) - - -async def delete_file_from_database(file_hash: str, database: aioredis.Redis): - """ - Deletes all data related to a file from the database. - - This function is used for housekeeping purposes in the database, allowing for the - removal of all data related to a particular file. This can be useful for clearing - outdated or irrelevant data from the database. - - Parameters: - - file_hash (str): The hash of the file to be deleted. - - database (aioredis.Redis): An instance of the Redis database used for data storage. - """ - bt.logging.debug(f"deleting file {file_hash} from database...") - - chunk_data = await get_all_chunks_for_file(file_hash, database) - if chunk_data is None: - bt.logging.debug(f"file {file_hash} not found in database.") - return - - # Delete all chunk hashes - for idx, chunk_dict in chunk_data.items(): - chunk_hash = chunk_dict["chunk_hash"] - await database.delete(f"chunk:{chunk_hash}") - - # Test getting the chunk hash back - chunk_data = await get_all_chunks_for_file(file_hash, database) - if chunk_data == {}: - bt.logging.debug(f"all chunks deleted for file {file_hash}.") - await database.delete(f"file:{file_hash}") - - -def get_hash_keys(ss58_address, r): - """ - Filter out the ttl: hashes from the hotkey hashes and return the list of keys. - """ - return [ - key for key in r.hkeys(f"hotkey:{ss58_address}") if not key.startswith(b"ttl:") - ] diff --git a/subnet/validator/encryption.py b/subnet/validator/encryption.py deleted file mode 100644 index 4fe58026..00000000 --- a/subnet/validator/encryption.py +++ /dev/null @@ -1,425 +0,0 @@ -# The MIT License (MIT) -# Copyright © 2023 Yuma Rao -# Copyright © 2023 philanthrope - -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the “Software”), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. - -# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -import os -import json -import typing - -import bittensor as bt -from Crypto.Cipher import AES -from nacl import pwhash, secret -from nacl.encoding import HexEncoder -from nacl.utils import EncryptedMessage - -NACL_SALT = b"\x13q\x83\xdf\xf1Z\t\xbc\x9c\x90\xb5Q\x879\xe9\xb1" - - -def encrypt_aes(filename: typing.Union[bytes, str], key: bytes) -> bytes: - """ - Encrypt the data in the given filename using AES-GCM. - - Parameters: - - filename: str or bytes. If str, it's considered as a file name. If bytes, as the data itself. - - key: bytes. 16-byte (128-bit), 24-byte (192-bit), or 32-byte (256-bit) secret key. - - Returns: - - cipher_text: bytes. The encrypted data. - - nonce: bytes. The nonce used for the GCM mode. - - tag: bytes. The tag for authentication. - """ - - # If filename is a string, treat it as a file name and read the data - if isinstance(filename, str): - with open(filename, "rb") as file: - data = file.read() - else: - data = filename - - # Initialize AES-GCM cipher - cipher = AES.new(key, AES.MODE_GCM) - - # Encrypt the data - cipher_text, tag = cipher.encrypt_and_digest(data) - - return cipher_text, cipher.nonce, tag - - -def decrypt_aes(cipher_text: bytes, key: bytes, nonce: bytes, tag: bytes) -> bytes: - """ - Decrypt the data using AES-GCM. - - Parameters: - - cipher_text: bytes. The encrypted data. - - key: bytes. The secret key used for decryption. - - nonce: bytes. The nonce used in the GCM mode for encryption. - - tag: bytes. The tag for authentication. - - Returns: - - data: bytes. The decrypted data. - """ - - # Initialize AES-GCM cipher with the given key and nonce - cipher = AES.new(key, AES.MODE_GCM, nonce=nonce) - - # Decrypt the data and verify the tag - try: - data = cipher.decrypt_and_verify(cipher_text, tag) - except ValueError: - # This is raised if the tag does not match - raise ValueError("Incorrect decryption key or corrupted data.") - - return data - - -def encrypt_data_with_wallet(data: bytes, wallet) -> bytes: - """ - Encrypts the given data using a symmetric key derived from the wallet's coldkey public key. - - Args: - data (bytes): Data to be encrypted. - wallet (bt.wallet): Bittensor wallet object containing the coldkey. - - Returns: - bytes: Encrypted data. - - This function generates a symmetric key using the public key of the wallet's coldkey. - The generated key is used to encrypt the data using the NaCl secret box (XSalsa20-Poly1305). - The function is intended for encrypting arbitrary data securely using wallet-based keys. - """ - # Derive symmetric key from wallet's coldkey - password = wallet.coldkey.private_key.hex() - password_bytes = bytes(password, "utf-8") - kdf = pwhash.argon2i.kdf - key = kdf( - secret.SecretBox.KEY_SIZE, - password_bytes, - NACL_SALT, - opslimit=pwhash.argon2i.OPSLIMIT_SENSITIVE, - memlimit=pwhash.argon2i.MEMLIMIT_SENSITIVE, - ) - - # Encrypt the data - box = secret.SecretBox(key) - encrypted = box.encrypt(data) - return encrypted - - -def decrypt_data_with_coldkey_private_key( - encrypted_data: bytes, private_key: typing.Union[str, bytes] -) -> bytes: - """ - Decrypts the given encrypted data using a symmetric key derived from the wallet's coldkey public key. - - Args: - encrypted_data (bytes): Data to be decrypted. - private_key (bytes): The bittensor wallet private key (password) to decrypt the AES payload. - - Returns: - bytes: Decrypted data. - - Similar to the encryption function, this function derives a symmetric key from the wallet's coldkey public key. - It then uses this key to decrypt the given encrypted data. The function is primarily used for decrypting data - that was previously encrypted by the `encrypt_data_with_wallet` function. - """ - password_bytes = ( - bytes(private_key, "utf-8") if isinstance(private_key, str) else private_key - ) - - kdf = pwhash.argon2i.kdf - key = kdf( - secret.SecretBox.KEY_SIZE, - password_bytes, - NACL_SALT, - opslimit=pwhash.argon2i.OPSLIMIT_SENSITIVE, - memlimit=pwhash.argon2i.MEMLIMIT_SENSITIVE, - ) - - box = secret.SecretBox(key) - decrypted = box.decrypt(encrypted_data) - return decrypted - - -def decrypt_data_with_wallet(encrypted_data: bytes, wallet) -> bytes: - """ - Decrypts the given encrypted data using a symmetric key derived from the wallet's coldkey public key. - - Args: - encrypted_data (bytes): Data to be decrypted. - wallet (bt.wallet): Bittensor wallet object containing the coldkey. - - Returns: - bytes: Decrypted data. - - Similar to the encryption function, this function derives a symmetric key from the wallet's coldkey public key. - It then uses this key to decrypt the given encrypted data. The function is primarily used for decrypting data - that was previously encrypted by the `encrypt_data_with_wallet` function. - """ - # Derive symmetric key from wallet's coldkey - password = wallet.coldkey.private_key.hex() - password_bytes = bytes(password, "utf-8") - kdf = pwhash.argon2i.kdf - key = kdf( - secret.SecretBox.KEY_SIZE, - password_bytes, - NACL_SALT, - opslimit=pwhash.argon2i.OPSLIMIT_SENSITIVE, - memlimit=pwhash.argon2i.MEMLIMIT_SENSITIVE, - ) - - # Decrypt the data - box = secret.SecretBox(key) - decrypted = box.decrypt(encrypted_data) - return decrypted - - -def encrypt_data_with_aes_and_serialize( - data: bytes, wallet: bt.wallet -) -> typing.Tuple[bytes, bytes]: - """ - Decrypts the given encrypted data using a symmetric key derived from the wallet's coldkey public key. - - Args: - encrypted_data (bytes): Data to be decrypted. - wallet (bt.wallet): Bittensor wallet object containing the coldkey. - - Returns: - bytes: Decrypted data. - - Similar to the encryption function, this function derives a symmetric key from the wallet's coldkey public key. - It then uses this key to decrypt the given encrypted data. The function is primarily used for decrypting data - that was previously encrypted by the `encrypt_data_with_wallet` function. - """ - # Generate a random AES key - aes_key = os.urandom(32) # AES key for 256-bit encryption - - # Create AES cipher - cipher = AES.new(aes_key, AES.MODE_GCM) - nonce = cipher.nonce - - # Encrypt the data - encrypted_data, tag = cipher.encrypt_and_digest(data) - - # Serialize AES key, nonce, and tag - aes_info = { - "aes_key": aes_key.hex(), # Convert bytes to hex string for serialization - "nonce": nonce.hex(), - "tag": tag.hex(), - } - aes_info_str = json.dumps(aes_info) - - encrypted_msg: EncryptedMessage = encrypt_data_with_wallet( - aes_info_str.encode(), wallet - ) # Encrypt the serialized JSON string - - return encrypted_data, serialize_nacl_encrypted_message(encrypted_msg) - - -encrypt_data = encrypt_data_with_aes_and_serialize - - -def decrypt_data_and_deserialize( - encrypted_data: bytes, encryption_payload: bytes, wallet: bt.wallet -) -> bytes: - """ - Decrypts and deserializes the encrypted payload to extract the AES key, nonce, and tag, which are then used to - decrypt the given encrypted data. - - Args: - encrypted_data (bytes): AES encrypted data. - encryption_payload (bytes): Encrypted payload containing the AES key, nonce, and tag. - wallet (bt.wallet): Bittensor wallet object containing the coldkey. - - Returns: - bytes: Decrypted data. - - This function reverses the process performed by `encrypt_data_with_aes_and_serialize`. - It first decrypts the payload to extract the AES key, nonce, and tag, and then uses them to decrypt the data. - """ - - # Deserialize the encrypted payload to get the AES key, nonce, and tag in nacl.utils.EncryptedMessage format - encrypted_msg: EncryptedMessage = deserialize_nacl_encrypted_message( - encryption_payload - ) - - # Decrypt the payload to get the JSON string - decrypted_aes_info_str = decrypt_data_with_wallet(encrypted_msg, wallet) - - # Deserialize JSON string to get AES key, nonce, and tag - aes_info = json.loads(decrypted_aes_info_str) - aes_key = bytes.fromhex(aes_info["aes_key"]) - nonce = bytes.fromhex(aes_info["nonce"]) - tag = bytes.fromhex(aes_info["tag"]) - - # Decrypt data - cipher = AES.new(aes_key, AES.MODE_GCM, nonce=nonce) - decrypted_data = cipher.decrypt_and_verify(encrypted_data, tag) - - return decrypted_data - - -def decrypt_data_and_deserialize_with_coldkey_private_key( - encrypted_data: bytes, - encryption_payload: bytes, - private_key: typing.Union[str, bytes], -) -> bytes: - """ - Decrypts and deserializes the encrypted payload to extract the AES key, nonce, and tag, which are then used to - decrypt the given encrypted data. - - Args: - encrypted_data (bytes): AES encrypted data. - encryption_payload (bytes): Encrypted payload containing the AES key, nonce, and tag. - private_key (bytes): The bittensor wallet private key (password) to decrypt the AES payload. - - Returns: - bytes: Decrypted data. - - This function reverses the process performed by `encrypt_data_with_aes_and_serialize`. - It first decrypts the payload to extract the AES key, nonce, and tag, and then uses them to decrypt the data. - """ - - # Deserialize the encrypted payload to get the AES key, nonce, and tag in nacl.utils.EncryptedMessage format - encrypted_msg: EncryptedMessage = deserialize_nacl_encrypted_message( - encryption_payload - ) - - # Decrypt the payload to get the JSON string - decrypted_aes_info_str = decrypt_data_with_coldkey_private_key( - encrypted_msg, private_key - ) - - # Deserialize JSON string to get AES key, nonce, and tag - aes_info = json.loads(decrypted_aes_info_str) - aes_key = bytes.fromhex(aes_info["aes_key"]) - nonce = bytes.fromhex(aes_info["nonce"]) - tag = bytes.fromhex(aes_info["tag"]) - - # Decrypt data - cipher = AES.new(aes_key, AES.MODE_GCM, nonce=nonce) - decrypted_data = cipher.decrypt_and_verify(encrypted_data, tag) - - return decrypted_data - - -decrypt_data = decrypt_data_and_deserialize -decrypt_data_with_private_key = decrypt_data_and_deserialize_with_coldkey_private_key - - -def serialize_nacl_encrypted_message(encrypted_message: EncryptedMessage) -> str: - """ - Serializes an EncryptedMessage object to a JSON string. - - Args: - encrypted_message (EncryptedMessage): The EncryptedMessage object to serialize. - - Returns: - str: A JSON string representing the serialized object. - - This function takes an EncryptedMessage object, extracts its nonce and ciphertext, - and encodes them into a hex format. It then constructs a dictionary with these - values and serializes the dictionary into a JSON string. - """ - data = { - "nonce": HexEncoder.encode(encrypted_message.nonce).decode("utf-8"), - "ciphertext": HexEncoder.encode(encrypted_message.ciphertext).decode("utf-8"), - } - return json.dumps(data) - - -def deserialize_nacl_encrypted_message(serialized_data: str) -> EncryptedMessage: - """ - Deserializes a JSON string back into an EncryptedMessage object. - - Args: - serialized_data (str): The JSON string to deserialize. - - Returns: - EncryptedMessage: The reconstructed EncryptedMessage object. - - This function takes a JSON string representing a serialized EncryptedMessage object, - decodes it into a dictionary, and extracts the nonce and ciphertext. It then - reconstructs the EncryptedMessage object using the original nonce and ciphertext. - """ - data = json.loads(serialized_data) - nonce = HexEncoder.decode(data["nonce"].encode("utf-8")) - ciphertext = HexEncoder.decode(data["ciphertext"].encode("utf-8")) - combined = nonce + ciphertext - return EncryptedMessage._from_parts(nonce, ciphertext, combined) - - -def setup_encryption_wallet( - wallet_name="encryption", - wallet_hotkey="encryption", - password="dummy_password", - n_words=12, - use_encryption=False, - overwrite=False, -): - """ - Sets up a Bittensor wallet with coldkey and coldkeypub using a generated mnemonic. - - Args: - wallet_name (str): Name of the wallet. Default is 'encryption_coldkey'. - wallet_hotkey (str): Name of the hotkey. Default is 'encryption_hotkey'. - n_words (int): Number of words for the mnemonic. Default is 12. - password (str): Password used for encryption. Default is 'your_password'. - use_encryption (bool): Flag to determine if encryption should be used. Default is True. - overwrite (bool): Flag to determine if existing keys should be overwritten. Default is False. - - Returns: - bt.wallet: A Bittensor wallet object with coldkey and coldkeypub set. - """ - - # Init wallet - w = bt.wallet(wallet_name, wallet_hotkey) - - # Check if wallet exists on device - if w.coldkey_file.exists_on_device() or w.coldkeypub_file.exists_on_device(): - bt.logging.info(f"Wallet {w} already exists on device. Not overwriting wallet.") - return w - - # Generate mnemonic and create keypair - mnemonic = bt.Keypair.generate_mnemonic(n_words) - keypair = bt.Keypair.create_from_mnemonic(mnemonic) - - # Set coldkeypub - w._coldkeypub = bt.Keypair(ss58_address=keypair.ss58_address) - w.coldkeypub_file.set_keypair( - w._coldkeypub, encrypt=use_encryption, overwrite=overwrite, password=password - ) - - # Set coldkey - w._coldkey = keypair - w.coldkey_file.set_keypair( - w._coldkey, encrypt=use_encryption, overwrite=overwrite, password=password - ) - - # Write cold keyfile data to file with specified password - keyfile = w.coldkey_file - keyfile.make_dirs() - keyfile_data = bt.serialized_keypair_to_keyfile_data(keypair) - if use_encryption: - keyfile_data = bt.encrypt_keyfile_data(keyfile_data, password) - keyfile._write_keyfile_data_to_file(keyfile_data, overwrite=True) - - # Setup hotkey (dummy, but necessary) - mnemonic = bt.Keypair.generate_mnemonic(n_words) - keypair = bt.Keypair.create_from_mnemonic(mnemonic) - w.set_hotkey(keypair, encrypt=False, overwrite=True) - - return w diff --git a/subnet/validator/forward.py b/subnet/validator/forward.py index b16b1cf4..65fda0e3 100644 --- a/subnet/validator/forward.py +++ b/subnet/validator/forward.py @@ -18,23 +18,16 @@ import time import bittensor as bt +# from pprint import pformat -from pprint import pformat -from .network import monitor - -from subnet.validator.rebalance import rebalance_data -from subnet.validator.state import save_state, log_event -from subnet.validator.utils import get_current_epoch -from subnet.validator.bonding import compute_all_tiers -from subnet.validator.subtensor import subtensor_data +# from subnet.validator.bonding import compute_all_tiers from subnet.validator.challenge import challenge_data -from subnet.validator.metrics import metrics_data -from subnet.validator.metric import compute_metrics -from subnet.validator.database import ( - get_miner_statistics, - purge_expired_ttl_keys, - purge_challenges_for_all_hotkeys, -) +from subnet.validator.subtensor import subtensor_data +# from subnet.validator.key import generate_ssh_keys, clean_ssh_keys +# from subnet.validator.challenge import challenge_data +# from subnet.validator.metrics import metrics_data +# from subnet.validator.metric import compute_metrics +from subnet.validator.database import get_miner_statistics async def forward(self): @@ -43,57 +36,38 @@ async def forward(self): # Record forward time start = time.time() - # Send synapse to get Subtensor details - bt.logging.info("initiating subtensor") - await subtensor_data(self) + # Generate ssh key + # bt.logging.info("generate ssh keys") + # keys = await generate_ssh_keys(self) + keys = [] + + # Send synapse to get challenge + bt.logging.info("initiating challenge") + await challenge_data(self, keys) + + # Clean ssh key + # bt.logging.info("clean ssh keys") + # await clean_ssh_keys(self, keys) # Send synapse to get some metrics - bt.logging.info("initiating metrics") - await metrics_data(self) + # bt.logging.info("initiating metrics") + # await metrics_data(self) - # Send synapse to challenge the miner - bt.logging.info("initiating challenge") - await challenge_data(self) + # # Send synapse to challenge the miner + # bt.logging.info("initiating challenge") + # await challenge_data(self) # Compute the metrics - await compute_metrics(self) - - # Monitor every step - down_uids = await monitor(self) - if len(down_uids) > 0: - bt.logging.info(f"Downed uids marked for rebalance: {down_uids}") - await rebalance_data( - self, - k=2, # increase redundancy - dropped_hotkeys=[self.metagraph.hotkeys[uid] for uid in down_uids], - hotkey_replaced=False, # Don't delete challenge data (only in subscription handler) - ) - - # Purge all challenge data to start fresh and avoid requerying hotkeys with stale challenge data - current_epoch = get_current_epoch(self.subtensor, self.config.netuid) - bt.logging.info( - f"Current epoch: {current_epoch} | Last purged epoch: {self.last_purged_epoch}" - ) - if current_epoch % 10 == 0: - if self.last_purged_epoch < current_epoch: - bt.logging.info("initiating challenges purge") - await purge_challenges_for_all_hotkeys(self.database) - self.last_purged_epoch = current_epoch - save_state(self) - - # Purge expired TTL keys - if self.step % 60 == 0: - bt.logging.info("initiating TTL purge for expired keys") - await purge_expired_ttl_keys(self.database) + # await compute_metrics(self) # Compute tiers and stats - if self.step % 360 == 0 and self.step > 0: - bt.logging.info("initiating compute stats") - await compute_all_tiers(self.database) + # if self.step % 360 == 0 and self.step > 0: + # bt.logging.info("initiating compute stats") + # await compute_all_tiers(self.database) - # Update miner statistics and usage data. - stats = await get_miner_statistics(self.database) - bt.logging.debug(f"miner stats: {pformat(stats)}") + # # Update miner statistics and usage data. + # stats = await get_miner_statistics(self.database) + # bt.logging.debug(f"miner stats: {pformat(stats)}") # Display step time forward_time = time.time() - start diff --git a/subnet/validator/key.py b/subnet/validator/key.py index 3eba49d8..3cea198a 100644 --- a/subnet/validator/key.py +++ b/subnet/validator/key.py @@ -1,28 +1,28 @@ -import sys import time -import torch -import json -import paramiko import typing import asyncio +import paramiko import bittensor as bt from subnet import protocol from subnet.shared.key import generate_key -from subnet.validator.utils import get_available_query_miners from subnet.validator.ssh import check_connection +from subnet.validator.utils import get_available_query_miners -async def handle_synapse(self, uid: int) -> typing.Tuple[bool, protocol.Challenge]: - # Generate the public key - public_key, private_key = generate_key('validator') - bt.logging.debug("keys generated") +CHALLENGE_NAME = "Key" + +async def handle_generation_synapse( + self, uid: int, public_key: paramiko.RSAKey, private_key: paramiko.RSAKey +) -> typing.Tuple[bool]: # Get the axon axon = self.metagraph.axons[uid] - # Generate SSH key - response = self.dendrite.query( + bt.logging.debug(f"[{CHALLENGE_NAME}][{uid}] Generating Ssh keys") + + # Send the public ssh key to the miner + self.dendrite.query( # Send the query to selected miner axons in the network. axons=[axon], # Construct a dummy query. This simply contains a single integer. @@ -30,34 +30,27 @@ async def handle_synapse(self, uid: int) -> typing.Tuple[bool, protocol.Challeng # All responses have the deserialize function called on them before returning. # You are encouraged to define your own deserialization function. deserialize=True, + timeout=10, ) # Check the ssh connection works verified = check_connection(axon.ip, private_key) - bt.logging.debug("Ssh connection verified") - - # Execute something here + if verified: + bt.logging.debug(f"[{CHALLENGE_NAME}][{uid}] Ssh connexion is verified") + else: + bt.logging.warning(f"[{CHALLENGE_NAME}][{uid}] Ssh connexion is not verified") - ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.connect(axon.ip, username='root', pkey=private_key) + return verified - # Execute a command on the remote server - command_to_execute = 'speedtest-cli --json' - stdin, stdout, stderr = ssh.exec_command(command_to_execute) - # Get the command output - output = stdout.read().decode('utf-8') - error = stderr.read().decode('utf-8') - - # Print the output or error - if error is None: - subtensor_details = json.loads(output) +async def handle_cleaning_synapse(self, uid: int, public_key: paramiko.RSAKey): + # Get the axon + axon = self.metagraph.axons[uid] - + bt.logging.debug(f"[{CHALLENGE_NAME}][{uid}] Cleaning Ssh keys") - # Clean SSH key - response = self.dendrite.query( + # Send the public ssh key to the miner + self.dendrite.query( # Send the query to selected miner axons in the network. axons=[axon], # Construct a dummy query. This simply contains a single integer. @@ -65,82 +58,64 @@ async def handle_synapse(self, uid: int) -> typing.Tuple[bool, protocol.Challeng # All responses have the deserialize function called on them before returning. # You are encouraged to define your own deserialization function. deserialize=True, + timeout=10, ) - return verified, response + # TODO: check the connection is not possible anymore -async def generate_key_data(self): +async def generate_ssh_keys(self): start_time = time.time() + bt.logging.debug(f"[{CHALLENGE_NAME}] Step starting") # Select the miners uids = await get_available_query_miners(self, k=10) - bt.logging.debug(f"key uids {uids}") + bt.logging.debug(f"[{CHALLENGE_NAME}] Available uids {uids}") - # Generate SSH key - tasks = [] - responses = [] + # Generate the ssh keys + keys = [] for uid in uids: - tasks.append(asyncio.create_task(handle_synapse(self, uid))) - responses = await asyncio.gather(*tasks) - - # # Compute the rewards for the responses given the prompt. - # rewards: torch.FloatTensor = torch.zeros(len(responses), dtype=torch.float32).to( - # self.device - # ) - - # remove_reward_idxs = [] - # for idx, (uid, (verified, response)) in enumerate(zip(uids, responses)): - # # TODO: Check the result from miner equal the one the validator will get by requesting the subtensor itself - # success = True + public_key, private_key = generate_key(f"validator-{uid}") + keys.append((public_key, private_key)) + bt.logging.debug(f"[{CHALLENGE_NAME}] Ssh keys generated") - # # Get the hotkey - # hotkey = self.metagraph.hotkeys[uid] - - # # Update the challenge statistics - # # await update_statistics( - # # ss58_address=hotkey, - # # success=success, - # # task_type="challenge", - # # database=self.database, - # # ) - - # # # Apply reward for this challenge - # # tier_factor = await get_tier_factor(hotkey, self.database) - # # rewards[idx] = 1.0 * tier_factor - - # if len(responses) == 0: - # bt.logging.debug("Received zero hashes from miners, returning event early.") - # return - - # uids, responses = _filter_verified_responses(uids, responses) - # bt.logging.debug( - # f"challenge_data() full rewards: {rewards} | uids {uids} | uids to remove {remove_reward_idxs}" - # ) - - # # bt.logging.trace("Applying challenge rewards") - # # apply_reward_scores( - # # self, - # # uids=uids, - # # responses=responses, - # # rewards=rewards, - # # timeout=30, - # # ) - - -def _filter_verified_responses(uids, responses): - not_none_responses = [ - (uid, response[0]) - for (uid, (verified, response)) in zip(uids, responses) - if verified is not None - ] + # Request the miners to create the ssh key + tasks = [] + for idx, (uid) in enumerate(uids): + (public_key, private_key) = keys[idx] + tasks.append( + asyncio.create_task( + handle_generation_synapse(self, uid, public_key, private_key) + ) + ) + await asyncio.gather(*tasks) - if len(not_none_responses) == 0: - return (), () + # Display step time + forward_time = time.time() - start_time + bt.logging.debug(f"[{CHALLENGE_NAME}] Step finished in {forward_time:.2f}s") - uids, responses = zip(*not_none_responses) - return uids, responses + return keys +async def clean_ssh_keys(self, keys: list): + start_time = time.time() + bt.logging.debug(f"[{CHALLENGE_NAME}] Step starting") + # Select the miners + uids = await get_available_query_miners(self, k=10) + bt.logging.debug(f"[{CHALLENGE_NAME}] Available uids {uids}") + # Request miners to remove the ssh key + tasks = [] + for idx, (uid) in enumerate(uids): + (public_key, private_key) = keys[idx] + tasks.append( + asyncio.create_task(handle_cleaning_synapse(self, uid, public_key)) + ) + await asyncio.gather(*tasks) + + # Display step time + forward_time = time.time() - start_time + bt.logging.debug(f"[{CHALLENGE_NAME}] Step finished in {forward_time:.2f}s") + + return keys diff --git a/subnet/validator/localisation.py b/subnet/validator/localisation.py new file mode 100644 index 00000000..cc2df753 --- /dev/null +++ b/subnet/validator/localisation.py @@ -0,0 +1,52 @@ +import json +import os +import requests +from math import radians, sin, cos, sqrt, atan2 + +LOCLISATION_API="https://api.country.is" + + +def get_localisation(country_code: str): + ''' + Get the longitude and latitude of the country + ''' + current_dir = os.path.dirname(os.path.realpath(__file__)) + file_path = os.path.join(current_dir, '..', 'localisation.json') + + countries = {} + with open(file_path, 'r') as f: + countries = json.load(f) + + return countries.get(country_code) + + +def get_country(ip: str): + ''' + Get the country code of the ip + ''' + url = f"{LOCLISATION_API}/{ip}" + + response = requests.get(url) + + if response.status_code != 200: + return None + + data = response.json() + + return data['country'] + + +def compute_localisation_distance(lat1, lon1, lat2, lon2): + ''' + Compute the distance between two localisations using Haversine formula + ''' + # Convert latitude and longitude from degrees to radians + lat1, lon1, lat2, lon2 = map(radians, [lat1, lon1, lat2, lon2]) + + # Haversine formula + dlat = lat2 - lat1 + dlon = lon2 - lon1 + a = sin(dlat / 2)**2 + cos(lat1) * cos(lat2) * sin(dlon / 2)**2 + c = 2 * atan2(sqrt(a), sqrt(1 - a)) + distance = 6371 * c # Radius of the Earth in kilometers + return distance diff --git a/subnet/validator/metric.py b/subnet/validator/metric.py index 6c16c814..78b28ca3 100644 --- a/subnet/validator/metric.py +++ b/subnet/validator/metric.py @@ -2,8 +2,13 @@ import torch import bittensor as bt +from subnet.constants import METRIC_FAILURE_REWARD from subnet.validator.utils import get_available_query_miners -from subnet.validator.reward import apply_reward_scores +from subnet.validator.bonding import ( + register_miner, + miner_is_registered, + get_tier_factor, +) metric2_rewards = {1: 1, 2: 0.75, 3: 0.5, 4: 0.25} metric3_rewards: typing.List = [ @@ -12,9 +17,15 @@ {"min": 10, "max": 15, "value": 0.75}, {"min": 20, "value": 1}, ] +metric4_rewards: typing.List = [ + {"min": 0, "max": 1, "value": 1}, + {"min": 1, "max": 2, "value": 0.75}, + {"min": 2, "max": 3, "value": 0.5}, + {"min": 3, "value": 0}, +] -async def compute_metrics(self): +async def compute_rewards(self, verification): # Select the miners uids = await get_available_query_miners(self, k=10) bt.logging.debug(f"compute metrics uids {uids}") @@ -24,7 +35,6 @@ async def compute_metrics(self): self.device ) - keys=[] for idx, uid in enumerate(uids): axon = self.metagraph.axons[idx] @@ -34,31 +44,42 @@ async def compute_metrics(self): # Get the hotkey hotkey = self.metagraph.hotkeys[uid] - # Update subtensor statistics in the subs hash + # Subtensor info key subs_key = f"subs:{coldkey}:{hotkey}" # Get all the keys owned by the coldkey - keys = await self.database.keys(f"subs:{coldkey}:*") + keys_cached = await self.database.keys(f"subs:{coldkey}:*") + + # Check and see if this miner is registered. + if not await miner_is_registered(hotkey, self.database): + bt.logging.debug(f"[Metric][{uid}] Registering new miner {hotkey}...") + await register_miner(hotkey, self.database) + + # Miner statistics key + stats_key = f"stats:{hotkey}" - # Metric 1 - Ownership: Subtensor and miner have to be on the same machine subtensor_ip = await self.database.hget(subs_key, "ip") - metric1 = 1 * (1 if axon.ip == subtensor_ip else 0) - bt.logging.debug(f"[Metric 1] Ownership {metric1}") + + # # Metric 1 - Ownership: Subtensor and miner have to be on the same machine + # metric1 = 1 * (1 if axon.ip == subtensor_ip else 0) + # await self.database.hset(stats_key, f"ownership", metric1) + # bt.logging.debug(f"[Metric][{uid}][#1] Ownership {metric1}") # Metric 2 - Unicity: One subtensor linked to one miner miners = [] - for key in keys: + for key in keys_cached: ip = await self.database.hget(subs_key, "ip") if subtensor_ip == ip: miners.append(key) - + number_of_miners = len(miners) metric2 = 1 * (metric2_rewards.get(number_of_miners) or 0) - bt.logging.debug(f"[Metric 2] Unicity {metric2}") + await self.database.hset(stats_key, f"unicity", metric2) + bt.logging.debug(f"[Metric][{uid}][#2] Unicity {metric2}") # Metric 3 - Diversity: Maximise subtensors's timezone owned by a coldkey timezones = [] - for key in keys: + for key in keys_cached: timezone = await self.database.hget(key, "timezone") if timezone not in timezones: timezones.append(key) @@ -67,32 +88,36 @@ async def compute_metrics(self): metric3_reward = next( ( obj - for i, obj in enumerate(metric3_rewards) + for obj in metric3_rewards if obj["min"] <= number_of_timezone and (obj["max"] is None or number_of_timezone < obj["max"]) ), None, ) metric3 = 1 * (metric3_reward["value"] if metric3_reward is not None else 0) - bt.logging.debug(f"[Metric 3] Diversity {metric3}") + await self.database.hset(stats_key, f"diversity", metric3) + bt.logging.debug(f"[Metric][{uid}][#3] Diversity {metric3}") + + # Metric 4 - Latency: Maximise the best time + latency = await self.database.hget(subs_key, "latency") + metric4_reward = next( + ( + obj + for obj in metric3_rewards + if obj["min"] <= latency + and (obj["max"] is None or number_of_timezone < obj["max"]) + ), + None, + ) + metric4 = 1 * (metric4_reward["value"] if metric4_reward is not None else 0) + await self.database.hset(stats_key, f"latency", metric4) + bt.logging.debug(f"[Metric][{uid}][#4] Latency {metric4}") + + # Get the tier factor + tier_factor = await get_tier_factor(hotkey, self.database) # Apply reward for this challenge - rewards[idx] = (metric1 + metric2 + metric3) / 3 - bt.logging.debug(f"Rewards {rewards[idx]}") - - if len(keys) == 0: - return - - process_times = [] - for key in keys: - process_time = await self.database.hget(key, "process_time") - process_times.append(float(process_time)) - - bt.logging.trace("Applying challenge rewards") - apply_reward_scores( - self, - uids=uids, - process_times=process_times, - rewards=rewards, - timeout=30, - ) + rewards[idx] = ((metric2 + metric3 + metric4) / 3) * tier_factor if verification[idx] else METRIC_FAILURE_REWARD + bt.logging.debug(f"[Metric][{uid}] Rewards {rewards[idx]}") + + return rewards diff --git a/subnet/validator/metrics.py b/subnet/validator/metrics.py index bdeaa6ed..787c7fc6 100644 --- a/subnet/validator/metrics.py +++ b/subnet/validator/metrics.py @@ -3,14 +3,20 @@ import typing import asyncio import paramiko +import torch import bittensor as bt from subnet import protocol - +from subnet.constants import METRIC_FAILURE_REWARD from subnet.shared.key import generate_key - from subnet.validator.ssh import check_connection -from subnet.validator.utils import get_available_query_miners +from subnet.validator.utils import get_available_query_miners, remove_indices_from_tensor +from subnet.validator.reward import apply_reward_scores +from subnet.validator.metric import compute_rewards +from subnet.validator.bonding import update_statistics + + +CHALLENGE_NAME = "Metric" def execute_speed_test(self, uid: int, private_key: paramiko.RSAKey): @@ -24,7 +30,6 @@ def execute_speed_test(self, uid: int, private_key: paramiko.RSAKey): try: # Connect to the remote server ssh.connect(axon.ip, username="root", pkey=private_key) - bt.logging.debug(f"Ssh connection with {axon.ip}") # Execute a command on the remote server command_to_execute = "speedtest-cli --json" @@ -38,16 +43,18 @@ def execute_speed_test(self, uid: int, private_key: paramiko.RSAKey): # Check for errors in the stderr output if error: - bt.logging.error(f"[Metrics] Error executing speed test: {error}") + bt.logging.error(f"[{CHALLENGE_NAME}][{uid}] Error executing speed test: {error}") return None return json.loads(output) except paramiko.AuthenticationException: - bt.logging.error("[Metrics] Authentication failed, please verify your credentials") + bt.logging.error( + "[{CHALLENGE_NAME}][{uid}] Authentication failed, please verify your credentials" + ) except paramiko.SSHException as e: - bt.logging.error(f"[Metrics] Ssh connection error: {e}") + bt.logging.error(f"[{CHALLENGE_NAME}][{uid}] Ssh connection error: {e}") except Exception as e: - bt.logging.error(f"[Metrics] An error occurred: {e}") + bt.logging.error(f"[{CHALLENGE_NAME}][{uid}] An error occurred: {e}") finally: # Close the SSH connection ssh.close() @@ -55,12 +62,12 @@ def execute_speed_test(self, uid: int, private_key: paramiko.RSAKey): async def handle_generation_synapse( self, uid: int, public_key: paramiko.RSAKey, private_key: paramiko.RSAKey -) -> typing.Tuple[bool, protocol.Key]: +) -> typing.Tuple[bool]: # Get the axon axon = self.metagraph.axons[uid] # Send the public ssh key to the miner - response = self.dendrite.query( + self.dendrite.query( # Send the query to selected miner axons in the network. axons=[axon], # Construct a dummy query. This simply contains a single integer. @@ -72,22 +79,18 @@ async def handle_generation_synapse( # Check the ssh connection works verified = check_connection(axon.ip, private_key) - if verified: - bt.logging.info("[Metrics] Ssh connection verified") - else: - bt.logging.warning("[Metrics] Ssh connection is not verified") - return verified, response + return verified async def handle_cleaning_synapse( self, uid: int, public_key: paramiko.RSAKey -) -> typing.Tuple[bool, protocol.Key]: +): # Get the axon axon = self.metagraph.axons[uid] # Send the public ssh key to the miner - response = self.dendrite.query( + self.dendrite.query( # Send the query to selected miner axons in the network. axons=[axon], # Construct a dummy query. This simply contains a single integer. @@ -99,25 +102,23 @@ async def handle_cleaning_synapse( # TODO: check the connection is not possible anymore - return True, response - async def metrics_data(self): start_time = time.time() - bt.logging.debug(f"[Metrics] Starting") + bt.logging.debug(f"[{CHALLENGE_NAME}] Step starting") # Select the miners uids = await get_available_query_miners(self, k=10) - bt.logging.debug(f"[Metrics] Available uids {uids}") + bt.logging.debug(f"[{CHALLENGE_NAME}] Available uids {uids}") # Generate the ssh keys keys = [] for uid in uids: public_key, private_key = generate_key(f"validator-{uid}") - keys.append(( public_key, private_key )) - bt.logging.debug("[Metrics] Ssh keys generated") + keys.append((public_key, private_key)) + bt.logging.debug("[{CHALLENGE_NAME}] Ssh keys generated") - # Generate and send the ssh keys + # Request the miners to create the ssh key tasks = [] responses = [] for idx, (uid) in enumerate(uids): @@ -129,18 +130,29 @@ async def metrics_data(self): ) responses = await asyncio.gather(*tasks) - # Execute the speedtest-cli to get some metrics - for idx, (uid, (verified, response)) in enumerate(zip(uids, responses)): + rewards: torch.FloatTensor = torch.zeros(len(responses), dtype=torch.float32).to( + self.device + ) + verification = [False] * len(uids) + + for idx, (uid, (verified)) in enumerate(zip(uids, responses)): + verification[idx] = verified + if not verified: - # TODO: do we punished miner now, later or never? + message = f"[{CHALLENGE_NAME}][{uid}] The ssh connection could not be established" + bt.logging.warning(message) continue + message = f"[{CHALLENGE_NAME}][{uid}] The ssh connection is established" + bt.logging.debug(message) + # Get the ssh keys public_key, private_key = keys[idx] + # Execute speedtest-cli in a temporary ssh connection result = execute_speed_test(self, uid, private_key) if result is None: - bt.logging.warning("[Metrics] Speed test failed") + bt.logging.warning(f"[{CHALLENGE_NAME}][{uid}] Speed test failed") continue # Bandwidth - measured in Mbps @@ -164,6 +176,14 @@ async def metrics_data(self): # Get the hotkey hotkey = self.metagraph.hotkeys[uid] + # Update statistics + await update_statistics( + ss58_address=hotkey, + success=verified, + task_type="metric", + database=self.database, + ) + # Get the subs hash subs_key = f"subs:{coldkey}:{hotkey}" @@ -180,7 +200,7 @@ async def metrics_data(self): avg_download = (float(legacy_download) + download) / 2 await self.database.hset(subs_key, "download", avg_download) - bt.logging.info(f"[Metrics] Download {avg_download}") + bt.logging.info(f"[{CHALLENGE_NAME}][{uid}] Download {avg_download}") # Update upload avg_upload = upload @@ -189,7 +209,7 @@ async def metrics_data(self): avg_upload = (float(legacy_upload) + avg_upload) / 2 await self.database.hset(subs_key, "upload", avg_upload) - bt.logging.info(f"[Metrics] Upload {avg_upload}") + bt.logging.info(f"[{CHALLENGE_NAME}][{uid}] Upload {avg_upload}") # Update lantency avg_latency = ping @@ -198,9 +218,9 @@ async def metrics_data(self): avg_latency = (float(legacy_latency) + ping) / 2 await self.database.hset(subs_key, "latency", avg_latency) - bt.logging.info(f"[Metrics] Latency {avg_latency}") + bt.logging.info(f"[{CHALLENGE_NAME}][{uid}] Latency {avg_latency}") - # Clean the ssh keys + # Request miners to remove the ssh key tasks = [] for uid in uids: (public_key, private_key) = keys[idx] @@ -209,6 +229,19 @@ async def metrics_data(self): ) await asyncio.gather(*tasks) + # Compute rewards + rewards = compute_rewards(verification) + + # Apply rewards to the miners + bt.logging.trace(f"[{CHALLENGE_NAME}] Applying rewards") + apply_reward_scores( + self, + uids=uids, + rewards=rewards, + process_times=[None] * rewards, + timeout=5, + ) + # Display step time forward_time = time.time() - start_time - bt.logging.debug(f"[Metrics] Step time {forward_time:.2f}s") + bt.logging.debug(f"[{CHALLENGE_NAME}] Step finished in {forward_time:.2f}s") diff --git a/subnet/validator/network.py b/subnet/validator/network.py deleted file mode 100644 index 0f5c3429..00000000 --- a/subnet/validator/network.py +++ /dev/null @@ -1,228 +0,0 @@ -# The MIT License (MIT) -# Copyright © 2023 Yuma Rao -# Copyright © 2023 philanthrope - -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the “Software”), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. - -# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -import torch -import typing -import bittensor as bt - -from subnet.validator.utils import get_available_query_miners -from subnet.validator.bonding import update_statistics -from subnet.constants import MONITOR_FAILURE_REWARD - - -async def ping_uids(self, uids): - """ - Ping a list of UIDs to check their availability. - Returns a tuple with a list of successful UIDs and a list of failed UIDs. - """ - axons = [self.metagraph.axons[uid] for uid in uids] - try: - responses = await self.dendrite( - axons, - bt.Synapse(), - deserialize=False, - timeout=5, - ) - successful_uids = [ - uid - for uid, response in zip(uids, responses) - if response.dendrite.status_code == 200 - ] - failed_uids = [ - uid - for uid, response in zip(uids, responses) - if response.dendrite.status_code != 200 - ] - except Exception as e: - bt.logging.error(f"Dendrite ping failed: {e}") - successful_uids = [] - failed_uids = uids - bt.logging.debug("ping() successful uids:", successful_uids) - bt.logging.debug("ping() failed uids :", failed_uids) - return successful_uids, failed_uids - - -async def compute_and_ping_chunks(self, distributions): - """ - Asynchronously evaluates the availability of miners for the given chunk distributions by pinging them. - Rerolls the distribution to replace failed miners, ensuring exactly k successful miners are selected. - - Parameters: - distributions (list of dicts): A list of chunk distribution dictionaries, each containing - information about chunk indices and assigned miner UIDs. - - Returns: - list of dicts: The updated list of chunk distributions with exactly k successful miner UIDs. - - Note: - - This function is crucial for ensuring that data chunks are assigned to available and responsive miners. - - Pings miners based on their UIDs and updates the distributions accordingly. - - Logs the new set of UIDs and distributions for traceability. - """ - max_retries = 3 # Define the maximum number of retries - target_number_of_uids = len( - distributions[0]["uids"] - ) # Assuming k is the length of the uids in the first distribution - - for dist in distributions: - retries = 0 - successful_uids = set() - - while len(successful_uids) < target_number_of_uids and retries < max_retries: - # Ping all UIDs - current_successful_uids, _ = await ping_uids(self, dist["uids"]) - successful_uids.update(current_successful_uids) - - # If enough UIDs are successful, select the first k items - if len(successful_uids) >= target_number_of_uids: - dist["uids"] = tuple(sorted(successful_uids)[:target_number_of_uids]) - break - - # Reroll for k UIDs excluding the successful ones - new_uids = await get_available_query_miners( - self, k=target_number_of_uids, exclude=successful_uids - ) - bt.logging.trace("compute_and_ping_chunks() new uids:", new_uids) - - # Update the distribution with new UIDs - dist["uids"] = tuple(new_uids) - retries += 1 - - # Log if the maximum retries are reached without enough successful UIDs - if len(successful_uids) < target_number_of_uids: - bt.logging.warning( - f"compute_and_ping_chunks(): Insufficient successful UIDs for distribution: {dist}" - ) - - # Continue with your logic using the updated distributions - bt.logging.trace("new distributions:", distributions) - return distributions - - -async def reroll_distribution(self, distribution, failed_uids): - """ - Asynchronously rerolls a single data chunk distribution by replacing failed miner UIDs with new, available ones. - This is part of the error handling process in data distribution to ensure that each chunk is reliably stored. - - Parameters: - distribution (dict): The original chunk distribution dictionary, containing chunk information and miner UIDs. - failed_uids (list of int): List of UIDs that failed in the original distribution and need replacement. - - Returns: - dict: The updated chunk distribution with new miner UIDs replacing the failed ones. - - Note: - - This function is typically used when certain miners are unresponsive or unable to store the chunk. - - Ensures that each chunk has the required number of active miners for redundancy. - """ - # Get new UIDs to replace the failed ones - new_uids = await get_available_query_miners( - self, k=len(failed_uids), exclude=failed_uids - ) - # Ping miners to ensure they are available - new_uids, _ = await ping_uids(self, new_uids) - distribution["uids"] = new_uids - return distribution - - -async def ping_and_retry_uids( - self, k: int = None, max_retries: int = 3, exclude_uids: typing.List[int] = [] -): - """ - Fetch available uids to minimize waiting for timeouts if they're going to fail anyways... - """ - # Select initial subset of miners to query - uids = await get_available_query_miners(self, k=k or 4, exclude=exclude_uids) - bt.logging.debug("initial ping_and_retry() uids:", uids) - - retries = 0 - successful_uids = set() - failed_uids = set() - while len(successful_uids) < k and retries < max_retries: - # Ping all UIDs - current_successful_uids, current_failed_uids = await ping_uids(self, uids) - successful_uids.update(current_successful_uids) - failed_uids.update(current_failed_uids) - - # If enough UIDs are successful, select the first k items - if len(successful_uids) >= k: - uids = list(successful_uids)[:k] - break - - # Reroll for k UIDs excluding the successful ones - new_uids = await get_available_query_miners( - self, k=k, exclude=list(successful_uids.union(failed_uids)) - ) - bt.logging.debug(f"ping_and_retry() new uids: {new_uids}") - retries += 1 - - # Log if the maximum retries are reached without enough successful UIDs - if len(successful_uids) < k: - bt.logging.warning( - f"Insufficient successful UIDs for k: {k} Success UIDs {successful_uids} Failed UIDs: {failed_uids}" - ) - - return list(successful_uids)[:k], failed_uids - - -# Monitor all UIDs by ping and keep track of how many failures -async def monitor(self): - """ - Monitor all UIDs by ping and keep track of how many failures - occur. If a UID fails too many times, remove it from the - list of UIDs to ping. - """ - # Ping current subset of UIDs - query_uids = await get_available_query_miners(self, k=40) - bt.logging.debug(f"monitor() uids: {query_uids}") - _, failed_uids = await ping_uids(self, query_uids) - bt.logging.debug(f"monitor() failed uids: {failed_uids}") - - down_uids = [] - for uid in failed_uids: - self.monitor_lookup[uid] += 1 - if self.monitor_lookup[uid] > 5: - self.monitor_lookup[uid] = 0 - down_uids.append(uid) - bt.logging.debug(f"monitor() down uids: {down_uids}") - bt.logging.trace(f"monitor() monitor_lookup: {self.monitor_lookup}") - - if down_uids: - # Negatively reward - rewards = torch.zeros(len(down_uids), dtype=torch.float32).to(self.device) - - for i, uid in enumerate(down_uids): - await update_statistics( - ss58_address=self.metagraph.hotkeys[uid], - success=False, - task_type="monitor", - database=self.database, - ) - rewards[i] = MONITOR_FAILURE_REWARD - - bt.logging.debug(f"monitor() rewards: {rewards}") - scattered_rewards: torch.FloatTensor = self.moving_averaged_scores.scatter( - 0, torch.tensor(down_uids).to(self.device), rewards - ).to(self.device) - - alpha: float = 0.05 - self.moving_averaged_scores: torch.FloatTensor = alpha * scattered_rewards + ( - 1 - alpha - ) * self.moving_averaged_scores.to(self.device) - - return down_uids diff --git a/subnet/validator/rebalance.py b/subnet/validator/rebalance.py deleted file mode 100644 index 01728119..00000000 --- a/subnet/validator/rebalance.py +++ /dev/null @@ -1,89 +0,0 @@ -# The MIT License (MIT) -# Copyright © 2023 Yuma Rao -# Copyright © 2023 philanthrope - -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the “Software”), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, -# and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. - -# THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -# DEALINGS IN THE SOFTWARE. - -import typing -import bittensor as bt - -from subnet.validator.database import ( - get_metadata_for_hotkey, - get_ordered_metadata, - remove_hotkey_from_chunk, - purge_challenges_for_hotkey, -) -from subnet.validator.bonding import register_miner - - -async def rebalance_data_for_hotkey( - self, k: int, source_hotkey: str, hotkey_replaced: bool = False -): - """ - TODO: This might take a while, would be better to run in a separate process/thread - rather than block other validator duties? - - Get all data from a given miner/hotkey and rebalance it to other miners. - - (1) Get all data from a given miner/hotkey. - (2) Find out which chunks belong to full files, ignore the rest (challenges) - (3) Distribute the data that belongs to full files to other miners. - - """ - - metadata = await get_metadata_for_hotkey(source_hotkey, self.database) - - miner_hashes = list(metadata) - bt.logging.debug(f"Rebalancing miner hashes {miner_hashes[:5]}") - - if hotkey_replaced: - # Reset miner statistics - bt.logging.debug(f"Resetting statistics for hotkey {source_hotkey}") - await register_miner(source_hotkey, self.database) - # Update index for full and chunk hashes for retrieve - # Iterate through ordered metadata for all full hashses this miner had - bt.logging.debug(f"Removing all challenge metadata for hotkey {source_hotkey}") - async for file_key in self.database.scan_iter("file:*"): - file_key = file_key.decode("utf-8") - file_hash = file_key.split(":")[1] - # Get all ordered metadata for this file - ordered_metadata = await get_ordered_metadata(file_hash, self.database) - bt.logging.debug( - f"Length of removed ordered metadata: {len(ordered_metadata)} for hotkey: {source_hotkey}" - ) - for chunk_metadata in ordered_metadata: - # Remove the dropped miner from the chunk metadata - await remove_hotkey_from_chunk( - chunk_metadata, source_hotkey, self.database - ) - # Purge challenge hashes so new miner doesn't get hosed - bt.logging.debug(f"Purging all challenge hashes for hotkey {source_hotkey}") - await purge_challenges_for_hotkey(source_hotkey, self.database) - - -async def rebalance_data( - self, - k: int = 2, - dropped_hotkeys: typing.List[str] = [], - hotkey_replaced: bool = False, -): - bt.logging.debug(f"Rebalancing data for dropped hotkeys: {dropped_hotkeys}") - if isinstance(dropped_hotkeys, str): - dropped_hotkeys = [dropped_hotkeys] - - for hotkey in dropped_hotkeys: - await rebalance_data_for_hotkey( - self, k, hotkey, hotkey_replaced=hotkey_replaced - ) diff --git a/subnet/validator/reward.py b/subnet/validator/reward.py index 24a4fb29..ec050490 100644 --- a/subnet/validator/reward.py +++ b/subnet/validator/reward.py @@ -115,7 +115,7 @@ def scale_rewards( # Scale the rewards with normalized times time_scaled_rewards = torch.tensor( [ - rewards.to(device) * uid_to_normalized_time[uid] + rewards[i].to(device) * uid_to_normalized_time[uid] for i, uid in enumerate(uids) ] ) diff --git a/subnet/validator/score.py b/subnet/validator/score.py new file mode 100644 index 00000000..59fae098 --- /dev/null +++ b/subnet/validator/score.py @@ -0,0 +1,97 @@ +import numpy as np +import bittensor as bt + +from subnet.validator.bonding import wilson_score_interval +from subnet.validator.localisation import ( + compute_localisation_distance, + get_localisation, +) + +# Controls how quickly the tolerance decreases with distance. +SIGMA = 20 +# Longest distance between any two places on Earth is 20,010 kilometers +MAX_DISTANCE = 20010 + + +async def compute_reliability_score(database, hotkey: str): + stats_key = f"stats:{hotkey}" + + # Step 1: Retrieve statistics + challenge_successes = int( + await database.hget(stats_key, "challenge_successes") or 0 + ) + challenge_attempts = int(await database.hget(stats_key, "challenge_attempts") or 0) + + # Step 2: Normalization + normalized_score = wilson_score_interval(challenge_successes, challenge_attempts) + + return normalized_score + + +def compute_latency_score(idx, uid, validator_country, responses): + # Step 1: Get the localisation of the validator + validator_localisation = get_localisation(validator_country) + + # Step 2: Compute the miners process times by adding a tolerance + process_times = [] + for response in responses: + country = response[1] + process_time = response[2] + + distance = 0 + location = get_localisation(country) + if location is not None: + distance = compute_localisation_distance( + validator_localisation["latitude"], + validator_localisation["longitude"], + location["latitude"], + location["longitude"], + ) + + scaled_distance = distance / MAX_DISTANCE + tolerance = 1 - scaled_distance + + process_time = process_time * tolerance if process_time else 5 + process_times.append(process_time) + bt.logging.trace(f"[{uid}][Score][Latency] Process times {process_times}") + + + # Step 3: Baseline Latency Calculation + baseline_latency = np.mean(process_times) + bt.logging.trace(f"[{uid}][Score][Latency] Base latency {baseline_latency}") + + # Step 4: Relative Latency Score Calculation + relative_latency_scores = [] + for process_time in process_times: + relative_latency_score = 1 - (process_time / baseline_latency) + relative_latency_scores.append(relative_latency_score) + + # Step 5: Normalization + min_score = min(relative_latency_scores) + bt.logging.trace(f"[{uid}][Score][Latency] Minimum relative score {min_score}") + max_score = max(relative_latency_scores) + bt.logging.trace(f"[{uid}][Score][Latency] Maximum relative score {max_score}") + score = relative_latency_scores[idx] + bt.logging.trace(f"[{uid}][Score][Latency] Relative score {score}") + + normalized_scores = (score - min_score) / ( + max_score - min_score + ) + + return normalized_scores + + +def compute_distribution_score(idx, responses): + # Step 1: Country of the requested response + country = responses[idx][1] + + # Step 1: Country the number of miners in the country + count = 0 + for response in responses: + if response[1] == country: + count = count + 1 + + # Step 2: Compute the score + score = 1 / count + + return score diff --git a/subnet/validator/ssh.py b/subnet/validator/ssh.py index ceadb3b7..9aca7ca8 100644 --- a/subnet/validator/ssh.py +++ b/subnet/validator/ssh.py @@ -1,6 +1,11 @@ import paramiko +import bittensor as bt + def check_connection(ip, private_key): + ''' + Check the ssh connection with / is working + ''' ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) @@ -8,13 +13,13 @@ def check_connection(ip, private_key): ssh.connect(ip, username='root', pkey=private_key) return True except paramiko.AuthenticationException: - print("Authentication failed, please verify your credentials") + bt.logging.error("Authentication failed, please verify your credentials") except paramiko.SSHException as sshException: - print("Unable to establish SSH connection: %s" % sshException) + bt.logging.error("Unable to establish SSH connection: %s" % sshException) except paramiko.BadHostKeyException as badHostKeyException: - print("Unable to verify server's host key: %s" % badHostKeyException) + bt.logging.error("Unable to verify server's host key: %s" % badHostKeyException) except Exception as e: - print(e) + bt.logging.error(e) finally: # Close the SSH connection ssh.close() diff --git a/subnet/validator/state.py b/subnet/validator/state.py index 46400e41..8213c8a0 100644 --- a/subnet/validator/state.py +++ b/subnet/validator/state.py @@ -20,9 +20,6 @@ import torch import copy -from loguru import logger -from dataclasses import asdict - import subnet.validator as validator import bittensor as bt @@ -87,7 +84,7 @@ def save_state(self): neuron_state_dict = { "neuron_weights": self.moving_averaged_scores.to("cpu").tolist(), "last_purged_epoch": self.last_purged_epoch, - "monitor_lookup": self.monitor_lookup, + # "monitor_lookup": self.monitor_lookup, } torch.save(neuron_state_dict, f"{self.config.neuron.full_path}/model.torch") bt.logging.success( @@ -109,18 +106,18 @@ def load_state(self): neuron_weights = torch.tensor(state_dict["neuron_weights"]) self.last_purged_epoch = state_dict.get("last_purged_epoch", 0) bt.logging.info(f"Loaded last_purged_epoch: {self.last_purged_epoch}") - self.monitor_lookup = state_dict.get( - "monitor_lookup", {uid: 0 for uid in self.metagraph.uids.tolist()} - ) - if list(self.monitor_lookup.keys()) != self.metagraph.uids.tolist(): - bt.logging.warning( - "Monitor lookup keys do not match metagraph uids. Populating new monitor_lookup with zeros" - ) - self.monitor_lookup = { - uid: self.monitor_lookup.get(uid, 0) - for uid in self.metagraph.uids.tolist() - } - bt.logging.info(f"Loaded monitor_lookup: {self.monitor_lookup}") + # self.monitor_lookup = state_dict.get( + # "monitor_lookup", {uid: 0 for uid in self.metagraph.uids.tolist()} + # ) + # if list(self.monitor_lookup.keys()) != self.metagraph.uids.tolist(): + # bt.logging.warning( + # "Monitor lookup keys do not match metagraph uids. Populating new monitor_lookup with zeros" + # ) + # self.monitor_lookup = { + # uid: self.monitor_lookup.get(uid, 0) + # for uid in self.metagraph.uids.tolist() + # } + # bt.logging.info(f"Loaded monitor_lookup: {self.monitor_lookup}") # Check to ensure that the size of the neruon weights matches the metagraph size. if neuron_weights.shape != (self.metagraph.n,): bt.logging.warning( @@ -139,9 +136,3 @@ def load_state(self): ) except Exception as e: bt.logging.warning(f"Failed to load model with error: {e}") - - -def log_event(self, event): - # Log event - if not self.config.neuron.dont_save_events: - logger.log("EVENTS", "events", **event.__dict__) \ No newline at end of file diff --git a/subnet/validator/subtensor copy.py b/subnet/validator/subtensor copy.py new file mode 100644 index 00000000..bde6f048 --- /dev/null +++ b/subnet/validator/subtensor copy.py @@ -0,0 +1,104 @@ +import time +import typing +import asyncio +import torch +import bittensor as bt + +from subnet import protocol +from subnet.constants import SUBTENSOR_FAILURE_REWARD +from subnet.validator.utils import get_available_query_miners +from subnet.validator.bonding import get_tier_factor, update_statistics +from subnet.validator.reward import apply_reward_scores + + +CHALLENGE_NAME = "Subtensor" +CHALLENGE_TIMEOUT = 5 + + +async def handle_synapse(self, uid: int) -> typing.Tuple[bool, protocol.Subtensor]: + response = await self.dendrite( + axons=[self.metagraph.axons[uid]], + synapse=protocol.Subtensor(task=1), + deserialize=True, + timeout=CHALLENGE_TIMEOUT, + ) + + try: + # Create a subtensor with the ip return by the synapse + config = bt.subtensor.config() + config.subtensor.network = "local" + config.subtensor.chain_endpoint = f"ws://{response[0].subtensor_ip}:9944" + miner_subtensor = bt.subtensor(config) + + # Get the current block + current_block = miner_subtensor.get_current_block() + verified = current_block is not None + except Exception: + verified = False + + return verified, response + + +async def subtensor_data(self): + start_time = time.time() + bt.logging.debug(f"[{CHALLENGE_NAME}] Step starting") + + # Select the miners + uids = await get_available_query_miners(self, k=10) + bt.logging.debug(f"[{CHALLENGE_NAME}] Available uids {uids}") + + # Send synapse to miners to get their ip + tasks = [] + responses = [] + for uid in uids: + tasks.append(asyncio.create_task(handle_synapse(self, uid))) + responses = await asyncio.gather(*tasks) + + rewards: torch.FloatTensor = torch.zeros(len(responses), dtype=torch.float32).to( + self.device + ) + process_times: torch.FloatTensor = torch.zeros(len(responses), dtype=torch.float32).to( + self.device + ) + + for idx, (uid, (verified, response)) in enumerate(zip(uids, responses)): + if not verified: + message = f"[{CHALLENGE_NAME}][{uid}] The subtensor is verified" + bt.logging.success(message) + else: + message = f"[{CHALLENGE_NAME}][{uid}] The subtensor could not be verified" + bt.logging.warning(message) + + # Get the miner hotkey + hotkey = self.metagraph.hotkeys[uid] + + # Update statistics + await update_statistics( + ss58_address=hotkey, + success=verified, + task_type="subtensor", + database=self.database, + ) + + # Apply reward the challenge + tier_factor = await get_tier_factor(hotkey, self.database) + rewards[idx] = 1.0 * tier_factor if verified else SUBTENSOR_FAILURE_REWARD + + # Get the process time for each uid to apply the rewards accordingly + process_time = float(response.process_time) if verified else CHALLENGE_TIMEOUT + process_times.append(process_time) + bt.logging.debug(f"[{CHALLENGE_NAME}][{uid}] Process time {process_time}") + + # Apply rewards to the miners + bt.logging.trace(f"[{CHALLENGE_NAME}] Applying rewards") + apply_reward_scores( + self, + uids=uids, + rewards=rewards, + process_times=process_times, + timeout=CHALLENGE_TIMEOUT, + ) + + # Display step time + forward_time = time.time() - start_time + bt.logging.debug(f"[{CHALLENGE_NAME}] Step finished in {forward_time:.2f}s") diff --git a/subnet/validator/subtensor-2.py b/subnet/validator/subtensor-2.py new file mode 100644 index 00000000..32274ac0 --- /dev/null +++ b/subnet/validator/subtensor-2.py @@ -0,0 +1,116 @@ +import time +import typing +import asyncio +import torch +import bittensor as bt + +from subnet import protocol +from subnet.constants import SUBTENSOR_FAILURE_REWARD +from subnet.validator.utils import get_available_query_miners +from subnet.validator.bonding import get_tier_factor, update_statistics +from subnet.validator.reward import apply_reward_scores + + +CHALLENGE_NAME = "Subtensor" +CHALLENGE_TIMEOUT = 5 + + +async def handle_synapse(self, uid: int) -> typing.Tuple[bool, protocol.Subtensor]: + response = await self.dendrite( + axons=[self.metagraph.axons[uid]], + synapse=protocol.Subtensor(task=1), + deserialize=True, + timeout=CHALLENGE_TIMEOUT, + ) + + try: + # Create a subtensor with the ip return by the synapse + config = bt.subtensor.config() + config.subtensor.network = "local" + config.subtensor.chain_endpoint = f"ws://{response[0].subtensor_ip}:9944" + miner_subtensor = bt.subtensor(config) + + # Get the current block + current_block = miner_subtensor.get_current_block() + verified = current_block is not None + except Exception: + verified = False + + return verified, response + + +async def subtensor_data(self): + start_time = time.time() + bt.logging.debug(f"[{CHALLENGE_NAME}] Step starting") + + # Select the miners + uids = await get_available_query_miners(self, k=10) + bt.logging.debug(f"[{CHALLENGE_NAME}] Available uids {uids}") + + # Send synapse to miners to get their ip + tasks = [] + responses = [] + for uid in uids: + tasks.append(asyncio.create_task(handle_synapse(self, uid))) + responses = await asyncio.gather(*tasks) + + rewards: torch.FloatTensor = torch.zeros(len(responses), dtype=torch.float32).to( + self.device + ) + process_times: torch.FloatTensor = torch.zeros(len(responses), dtype=torch.float32).to( + self.device + ) + + for idx, (uid, (verified, response)) in enumerate(zip(uids, responses)): + if not verified: + message = f"[{CHALLENGE_NAME}][{uid}] The subtensor is verified" + bt.logging.success(message) + else: + message = f"[{CHALLENGE_NAME}][{uid}] The subtensor could not be verified" + bt.logging.warning(message) + + # Get the coldkey + axon = self.metagraph.axons[idx] + coldkey = axon.coldkey + + # Get the hotkey + hotkey = self.metagraph.hotkeys[uid] + + # Update subtensor + subs_key = f"subs:{coldkey}:{hotkey}" + await self.database.hset(subs_key, "ip", response[0].subtensor_ip) + + # # Update statistics + # stats_key = f"stats:{hotkey}" + # await self.database.hset(stats_key, "available", verified) + # await self.database.hset(stats_key, "latency", response[0].dendrite.process_time) + + # Update statistics + await update_statistics( + ss58_address=hotkey, + success=verified, + task_type="subtensor", + database=self.database, + ) + + # Apply reward the challenge + rewards[idx] = 1.0 if verified else SUBTENSOR_FAILURE_REWARD + + # Get the process time for each uid to apply the rewards accordingly + process_time = float(response.process_time) if verified else CHALLENGE_TIMEOUT + process_times.append(process_time) + bt.logging.debug(f"[{CHALLENGE_NAME}][{uid}] Process time {process_time}") + + # Apply rewards to the miners + bt.logging.trace(f"[{CHALLENGE_NAME}] Applying rewards") + apply_reward_scores( + self, + uids=uids, + rewards=rewards, + process_times=process_times, + timeout=CHALLENGE_TIMEOUT, + ) + + # Display step time + forward_time = time.time() - start_time + bt.logging.debug(f"[{CHALLENGE_NAME}] Step finished in {forward_time:.2f}s") diff --git a/subnet/validator/subtensor.py b/subnet/validator/subtensor.py index 7a8137a6..32274ac0 100644 --- a/subnet/validator/subtensor.py +++ b/subnet/validator/subtensor.py @@ -1,10 +1,18 @@ import time import typing import asyncio +import torch import bittensor as bt from subnet import protocol +from subnet.constants import SUBTENSOR_FAILURE_REWARD from subnet.validator.utils import get_available_query_miners +from subnet.validator.bonding import get_tier_factor, update_statistics +from subnet.validator.reward import apply_reward_scores + + +CHALLENGE_NAME = "Subtensor" +CHALLENGE_TIMEOUT = 5 async def handle_synapse(self, uid: int) -> typing.Tuple[bool, protocol.Subtensor]: @@ -12,6 +20,7 @@ async def handle_synapse(self, uid: int) -> typing.Tuple[bool, protocol.Subtenso axons=[self.metagraph.axons[uid]], synapse=protocol.Subtensor(task=1), deserialize=True, + timeout=CHALLENGE_TIMEOUT, ) try: @@ -32,11 +41,11 @@ async def handle_synapse(self, uid: int) -> typing.Tuple[bool, protocol.Subtenso async def subtensor_data(self): start_time = time.time() - bt.logging.debug(f"[Subtensor] Starting") + bt.logging.debug(f"[{CHALLENGE_NAME}] Step starting") # Select the miners uids = await get_available_query_miners(self, k=10) - bt.logging.debug(f"[Subtensor] Available uids {uids}") + bt.logging.debug(f"[{CHALLENGE_NAME}] Available uids {uids}") # Send synapse to miners to get their ip tasks = [] @@ -45,12 +54,20 @@ async def subtensor_data(self): tasks.append(asyncio.create_task(handle_synapse(self, uid))) responses = await asyncio.gather(*tasks) + rewards: torch.FloatTensor = torch.zeros(len(responses), dtype=torch.float32).to( + self.device + ) + process_times: torch.FloatTensor = torch.zeros(len(responses), dtype=torch.float32).to( + self.device + ) + for idx, (uid, (verified, response)) in enumerate(zip(uids, responses)): if not verified: - # TODO: do we punished miner now, later or never? - continue - - subtensor_ip = response[0].subtensor_ip + message = f"[{CHALLENGE_NAME}][{uid}] The subtensor is verified" + bt.logging.success(message) + else: + message = f"[{CHALLENGE_NAME}][{uid}] The subtensor could not be verified" + bt.logging.warning(message) # Get the coldkey axon = self.metagraph.axons[idx] @@ -59,11 +76,41 @@ async def subtensor_data(self): # Get the hotkey hotkey = self.metagraph.hotkeys[uid] - # Get the subs hash + # Update subtensor subs_key = f"subs:{coldkey}:{hotkey}" - - await self.database.hset(subs_key, "ip", subtensor_ip) + await self.database.hset(subs_key, "ip", response[0].subtensor_ip) + + # # Update statistics + # stats_key = f"stats:{hotkey}" + # await self.database.hset(stats_key, "available", verified) + # await self.database.hset(stats_key, "latency", response[0].dendrite.process_time) + + # Update statistics + await update_statistics( + ss58_address=hotkey, + success=verified, + task_type="subtensor", + database=self.database, + ) + + # Apply reward the challenge + rewards[idx] = 1.0 if verified else SUBTENSOR_FAILURE_REWARD + + # Get the process time for each uid to apply the rewards accordingly + process_time = float(response.process_time) if verified else CHALLENGE_TIMEOUT + process_times.append(process_time) + bt.logging.debug(f"[{CHALLENGE_NAME}][{uid}] Process time {process_time}") + + # Apply rewards to the miners + bt.logging.trace(f"[{CHALLENGE_NAME}] Applying rewards") + apply_reward_scores( + self, + uids=uids, + rewards=rewards, + process_times=process_times, + timeout=CHALLENGE_TIMEOUT, + ) # Display step time forward_time = time.time() - start_time - bt.logging.debug(f"[Subtensor] Step time {forward_time:.2f}s") \ No newline at end of file + bt.logging.debug(f"[{CHALLENGE_NAME}] Step finished in {forward_time:.2f}s") diff --git a/subnet/validator/utils.py b/subnet/validator/utils.py index 4b9709e3..53b55bd8 100644 --- a/subnet/validator/utils.py +++ b/subnet/validator/utils.py @@ -16,109 +16,44 @@ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. -import os -import math -import time import torch -import functools -import numpy as np +import bittensor as bt import random as pyrandom - +from typing import List from Crypto.Random import random -from itertools import combinations, cycle -from typing import List, Union -from subnet.shared.ecc import hash_data from subnet.validator.database import hotkey_at_capacity -import bittensor as bt - - -MIN_CHUNK_SIZE = 64 * 1024 * 1024 # 128 MB -MAX_CHUNK_SIZE = 512 * 1024 * 1024 # 512 MB - - -def chunk_data_generator(data, chunk_size): - """ - Generator that yields chunks of data. - - Args: - data (bytes): The data to be chunked. - chunk_size (int): The size of each chunk in bytes. - - Yields: - bytes: The next chunk of data. - """ - for i in range(0, len(data), chunk_size): - yield data[i : i + chunk_size] - - -def generate_file_size_with_lognormal( - mu: float = np.log(3 * 1024**2), sigma: float = 1.5 -) -> float: - """ - Generate a single file size using a lognormal distribution. - Default parameters are set to model a typical file size distribution, - but can be overridden for custom distributions. - - :param mu: Mean of the log values, default is set based on medium file size (20 MB). - :param sigma: Standard deviation of the log values, default is set to 1.5. - :return: File size in bytes. - """ - - # Generate a file size using the lognormal distribution - file_size = np.random.lognormal(mean=mu, sigma=sigma) - - # Scale the file size to a realistic range (e.g., bytes) - scaled_file_size = int(file_size) - return scaled_file_size +def remove_indices_from_tensor(tensor, indices_to_remove): + # Sort indices in descending order to avoid index out of range error + sorted_indices = sorted(indices_to_remove, reverse=True) + for index in sorted_indices: + tensor = torch.cat([tensor[:index], tensor[index + 1 :]]) + return tensor -def make_random_file(name: str = None, maxsize: int = None) -> Union[bytes, str]: - """ - Creates a file with random binary data or returns a bytes object with random data if no name is provided. - - Args: - name (str, optional): The name of the file to create. If None, the function returns the random data instead. - maxsize (int): The maximum size of the file or bytes object to be created, in bytes. Defaults to 1024. - - Returns: - bytes: If 'name' is not provided, returns a bytes object containing random data. - None: If 'name' is provided, a file is created and returns the filepath stored. - - Raises: - OSError: If the function encounters an error while writing to the file. - """ - size = ( - random.randint(random.randint(24, 128), maxsize) - if maxsize is not None - else generate_file_size_with_lognormal() - ) - data = os.urandom(size) - if isinstance(name, str): - with open(name, "wb") as fout: - fout.write(data) - return name # Return filepath of saved data - else: - return data # Return the data itself - - -# Determine a random chunksize between 512kb (random sample from this range) store as chunksize_E -def get_random_chunksize(minsize: int = 128, maxsize: int = 1024) -> int: +def current_block_hash(self): """ - Determines a random chunk size within a specified range for data chunking. + Get the current block hash with caching. Args: - maxsize (int): The maximum size limit for the random chunk size. Defaults to 128. + subtensor (bittensor.subtensor.Subtensor): The subtensor instance to use for getting the current block hash. Returns: - int: A random chunk size between 2kb and 'maxsize' kilobytes. - - Raises: - ValueError: If maxsize is set to a value less than 2. + str: The current block hash. """ - return random.randint(minsize, maxsize) + try: + block_hash: str = self.subtensor.get_block_hash( + self.subtensor.get_current_block() + ) + if block_hash is not None: + return block_hash + except Exception as e: + bt.logging.warning( + f"Failed to get block hash: {e}. Returning a random hash value." + ) + return int(str(random.randint(2 << 32, 2 << 64))) def check_uid_availability( @@ -143,45 +78,6 @@ def check_uid_availability( return True -def ttl_cache(maxsize=128, ttl=10): - """A simple TTL cache decorator for functions with a single argument.""" - - def wrapper_cache(func): - cache = functools.lru_cache(maxsize=maxsize)(func) - last_time = time.time() - - @functools.wraps(func) - def wrapped_func(*args, **kwargs): - nonlocal last_time - current_time = time.time() - if current_time - last_time > ttl: - cache.cache_clear() - last_time = current_time - return cache(*args, **kwargs) - - return wrapped_func - - return wrapper_cache - - -def current_block_hash(self): - """ - Get the current block hash with caching. - - Args: - subtensor (bittensor.subtensor.Subtensor): The subtensor instance to use for getting the current block hash. - - Returns: - str: The current block hash. - """ - try: - block_hash: str = self.subtensor.get_block_hash(self.subtensor.get_current_block()) - if block_hash is not None: - return block_hash - except Exception as e: - bt.logging.warning(f"Failed to get block hash: {e}. Returning a random hash value.") - return int(str(random.randint(2 << 32, 2 << 64))) - def get_block_seed(self): """ Get the block seed for the current block. @@ -197,28 +93,6 @@ def get_block_seed(self): return int(block_hash, 16) -def get_pseudorandom_uids(self, uids, k): - """ - Get a list of pseudorandom uids from the given list of uids. - - Args: - subtensor (bittensor.subtensor.Subtensor): The subtensor instance to use for getting the block_seed. - uids (list): The list of uids to generate pseudorandom uids from. - - Returns: - list: A list of pseudorandom uids. - """ - block_seed = get_block_seed(self) - pyrandom.seed(block_seed) - - # Ensure k is not larger than the number of uids - k = min(k, len(uids)) - - sampled = pyrandom.sample(uids, k=k) - bt.logging.debug(f"get_pseudorandom_uids() sampled: {k} | {sampled}") - return sampled - - def get_available_uids(self, exclude: list = None): """Returns all available uids from the metagraph. @@ -237,148 +111,26 @@ def get_available_uids(self, exclude: list = None): return avail_uids -# TODO: update this to use the block hash seed paradigm so that we don't get uids that are unavailable -def get_random_uids( - self, k: int, exclude: List[int] = None, seed: int = None -) -> torch.LongTensor: - """Returns k available random uids from the metagraph. - Args: - k (int): Number of uids to return. - exclude (List[int]): List of uids to exclude from the random sampling. - Returns: - uids (torch.LongTensor): Randomly sampled available uids. - Notes: - If `k` is larger than the number of available `uids`, set `k` to the number of available `uids`. - """ - candidate_uids = [] - avail_uids = [] - - for uid in range(self.metagraph.n.item()): - uid_is_available = check_uid_availability( - self.metagraph, uid, self.config.neuron.vpermit_tao_limit - ) - uid_is_not_excluded = exclude is None or uid not in exclude - - if uid_is_available and uid_is_not_excluded: - candidate_uids.append(uid) - elif uid_is_available: - avail_uids.append(uid) - - # If not enough candidate_uids, supplement from avail_uids, ensuring they're not in exclude list - if len(candidate_uids) < k: - additional_uids_needed = k - len(candidate_uids) - filtered_avail_uids = [uid for uid in avail_uids if uid not in exclude] - additional_uids = random.sample( - filtered_avail_uids, min(additional_uids_needed, len(filtered_avail_uids)) - ) - candidate_uids.extend(additional_uids) - - # Safeguard against trying to sample more than what is available - num_to_sample = min(k, len(candidate_uids)) - if seed: # use block hash seed if provided - random.seed(seed) - uids = random.sample(candidate_uids, num_to_sample) - bt.logging.debug(f"returning available uids: {uids}") - return uids - - -def get_all_validators_vtrust( - self, - vpermit_tao_limit: int, - vtrust_threshold: float = 0.0, - return_hotkeys: bool = False, -): - """ - Retrieves the hotkeys of all validators in the network. This method is used to - identify the validators and their corresponding hotkeys, which are essential - for various network operations, including blacklisting and peer validation. - Qualifications for validator peers: - - stake > threshold (e.g. 500, may vary per subnet) - - validator permit (implied with vtrust score) - - validator trust score > threshold (e.g. 0.5) - Returns: - List[str]: A list of hotkeys corresponding to all the validators in the network. - """ - vtrusted_uids = [ - uid for uid in torch.where(self.metagraph.validator_trust > vtrust_threshold)[0] - ] - stake_uids = [ - uid for uid in vtrusted_uids if self.metagraph.S[uid] > vpermit_tao_limit - ] - return ( - [self.metagraph.hotkeys[uid] for uid in stake_uids] - if return_hotkeys - else stake_uids - ) - - -def get_all_validators(self, return_hotkeys=False): - """ - Retrieve all validator UIDs from the metagraph. Optionally, return their hotkeys instead. - - Args: - return_hotkeys (bool): If True, returns the hotkeys of the validators; otherwise, returns the UIDs. - - Returns: - list: A list of validator UIDs or hotkeys, depending on the value of return_hotkeys. - """ - # Determine validator axons to query from metagraph - vpermits = self.metagraph.validator_permit - vpermit_uids = [uid for uid, permit in enumerate(vpermits) if permit] - vpermit_uids = torch.where(vpermits)[0] - query_idxs = torch.where( - self.metagraph.S[vpermit_uids] > self.config.neuron.vpermit_tao_limit - )[0] - query_uids = vpermit_uids[query_idxs].tolist() - - return ( - [self.metagraph.hotkeys[uid] for uid in query_uids] - if return_hotkeys - else query_uids - ) - - -def get_all_miners(self): - """ - Retrieve all miner UIDs from the metagraph, excluding those that are validators. - - Returns: - list: A list of UIDs of miners. - """ - # Determine miner axons to query from metagraph - vuids = get_all_validators(self) - return [uid for uid in self.metagraph.uids.tolist() if uid not in vuids] - - -def get_query_miners(self, k=20, exlucde=None): +def get_pseudorandom_uids(self, uids, k): """ - Obtain a list of miner UIDs selected pseudorandomly based on the current block hash. + Get a list of pseudorandom uids from the given list of uids. Args: - k (int): The number of miner UIDs to retrieve. + subtensor (bittensor.subtensor.Subtensor): The subtensor instance to use for getting the block_seed. + uids (list): The list of uids to generate pseudorandom uids from. Returns: - list: A list of pseudorandomly selected miner UIDs. - """ - # Determine miner axons to query from metagraph with pseudorandom block_hash seed - muids = get_all_miners(self) - if exlucde is not None: - muids = [muid for muid in muids if muid not in exlucde] - return get_pseudorandom_uids(self, muids, k=k) - - -def get_query_validators(self, k=3): + list: A list of pseudorandom uids. """ - Obtain a list of available validator UIDs selected pseudorandomly based on the current block hash. + block_seed = get_block_seed(self) + pyrandom.seed(block_seed) - Args: - k (int): The number of available miner UIDs to retreive. + # Ensure k is not larger than the number of uids + k = min(k, len(uids)) - Returns: - list: A list of pseudorandomly selected available validator UIDs - """ - vuids = get_all_validators(self) - return get_pseudorandom_uids(self, uids=vuids, k=k) + sampled = pyrandom.sample(uids, k=k) + bt.logging.debug(f"get_pseudorandom_uids() sampled: {k} | {sampled}") + return sampled async def get_available_query_miners( @@ -406,435 +158,73 @@ async def get_available_query_miners( return get_pseudorandom_uids(self, muids, k=k) -def get_current_validator_uid_pseudorandom(self): +async def ping_uids(self, uids): """ - Retrieve a single validator UID selected pseudorandomly based on the current block hash. - - Returns: - int: A pseudorandomly selected validator UID. + Ping a list of UIDs to check their availability. + Returns a tuple with a list of successful UIDs and a list of failed UIDs. """ - block_seed = get_block_seed(self) - pyrandom.seed(block_seed) - vuids = get_query_validators(self) - return pyrandom.choice(vuids) - - -def get_current_validtor_uid_round_robin(self): - """ - Retrieve a validator UID using a round-robin selection based on the current block and epoch length. - - Returns: - int: The UID of the validator selected via round-robin. - """ - vuids = get_all_validators(self) - vidx = self.subtensor.get_current_block() // 100 % len(vuids) - return vuids[vidx] - - -def generate_efficient_combinations(available_uids, R): - """ - Generates all possible combinations of UIDs for a given redundancy factor. - - Args: - available_uids (list): A list of UIDs that are available for storing data. - R (int): The redundancy factor specifying the number of UIDs to be used for each chunk of data. - - Returns: - list: A list of tuples, where each tuple contains a combination of UIDs. - - Raises: - ValueError: If the redundancy factor is greater than the number of available UIDs. - """ - - if R > len(available_uids): - raise ValueError( - "Redundancy factor cannot be greater than the number of available UIDs." - ) - - # Generate all combinations of available UIDs for the redundancy factor - uid_combinations = list(combinations(available_uids, R)) - - return uid_combinations - - -def assign_combinations_to_hashes_by_block_hash(self, hashes, combinations): - """ - Assigns combinations of UIDs to each data chunk hash based on a pseudorandom seed derived from the blockchain's current block hash. - - Args: - subtensor: The subtensor instance used to obtain the current block hash for pseudorandom seed generation. - hashes (list): A list of hashes, where each hash represents a unique data chunk. - combinations (list): A list of UID combinations, where each combination is a tuple of UIDs. - - Returns: - dict: A dictionary mapping each chunk hash to a pseudorandomly selected combination of UIDs. - - Raises: - ValueError: If there are not enough unique UID combinations for the number of data chunk hashes. - """ - - if len(hashes) > len(combinations): - raise ValueError( - "Not enough unique UID combinations for the given redundancy factor and number of hashes." - ) - block_seed = get_block_seed(self) - pyrandom.seed(block_seed) - - # Shuffle once and then iterate in order for assignment - pyrandom.shuffle(combinations) - return {hash_val: combinations[i] for i, hash_val in enumerate(hashes)} - - -def assign_combinations_to_hashes(hashes, combinations): - """ - Assigns combinations of UIDs to each data chunk hash in a pseudorandom manner. - - Args: - hashes (list): A list of hashes, where each hash represents a unique data chunk. - combinations (list): A list of UID combinations, where each combination is a tuple of UIDs. - - Returns: - dict: A dictionary mapping each chunk hash to a pseudorandomly selected combination of UIDs. - - Raises: - ValueError: If there are not enough unique UID combinations for the number of data chunk hashes. - """ - - if len(hashes) > len(combinations): - raise ValueError( - "Not enough unique UID combinations for the given redundancy factor and number of hashes." + axons = [self.metagraph.axons[uid] for uid in uids] + try: + responses = await self.dendrite( + axons, + bt.Synapse(), + deserialize=False, + timeout=5, ) - - # Shuffle once and then iterate in order for assignment - pyrandom.shuffle(combinations) - return {hash_val: combinations[i] for i, hash_val in enumerate(hashes)} - - -def optimal_chunk_size( - data_size, - num_available_uids, - R, - min_chunk_size=MIN_CHUNK_SIZE, - max_chunk_size=MAX_CHUNK_SIZE, -): - """ - Determines the optimal chunk size for data distribution, taking into account the total data size, - the number of available UIDs, and the desired redundancy factor. The function aims to balance - the chunk size between specified minimum and maximum limits, considering the efficient utilization - of UIDs and the number of chunks that can be created. - - Args: - data_size (int): The total size of the data to be distributed, in bytes. - num_available_uids (int): The number of available UIDs that can be assigned to data chunks. - R (int): The redundancy factor, defining how many UIDs each data chunk should be associated with. - min_chunk_size (int, optional): The minimum permissible size for each data chunk, in bytes. - Defaults to a predefined MIN_CHUNK_SIZE. - max_chunk_size (int, optional): The maximum permissible size for each data chunk, in bytes. - Defaults to a predefined MAX_CHUNK_SIZE. - - Returns: - int: The calculated optimal size for each data chunk, in bytes. The chunk size is optimized to - ensure efficient distribution across the available UIDs while respecting the minimum - and maximum chunk size constraints. - - Note: - The optimal chunk size is crucial for balancing data distribution and storage efficiency in - distributed systems or parallel processing scenarios. This function ensures that each chunk - is large enough to be meaningful yet small enough to allow for diverse distribution across - different UIDs, adhering to the specified redundancy factor. - """ - # Estimate the number of chunks based on redundancy and available UIDs - # Ensuring that we do not exceed the number of available UIDs - max_chunks = num_available_uids // R - - # Calculate the ideal chunk size based on the estimated number of chunks - if max_chunks > 0: - ideal_chunk_size = data_size / max_chunks - else: - ideal_chunk_size = max_chunk_size - - # Ensure the chunk size is within the specified bounds - chunk_size = max(min_chunk_size, min(ideal_chunk_size, max_chunk_size)) - - # Return data size if smaller than chunk size - if chunk_size > data_size: - return data_size - - return int(chunk_size) - - -def compute_chunk_distribution( - self, data, R, k, min_chunk_size=MIN_CHUNK_SIZE, max_chunk_size=MAX_CHUNK_SIZE -): - """ - Computes the distribution of data chunks to UIDs for data distribution. - - Args: - subtensor: The subtensor instance used to obtain the current block hash for pseudorandom seed generation. - data (bytes): The data to be distributed. - R (int): The redundancy factor for each data chunk. - k (int): The number of UIDs to be used for each data chunk. - min_chunk_size (int): The minimum size for each data chunk, in bytes. - max_chunk_size (int): The maximum size for each data chunk, in bytes. - - Returns: - dict: A dictionary mapping each chunk hash to a pseudorandomly selected combination of UIDs. - """ - available_uids = get_random_uids(self, k=k) - - data_size = len(data) - chunk_size = optimal_chunk_size( - data_size, len(available_uids), R, min_chunk_size, max_chunk_size - ) - - # Ensure chunk size is not larger than data size - if chunk_size > data_size: - chunk_size = data_size - uid_combinations = generate_efficient_combinations(available_uids, R) - - # Create a generator for chunking the data - data_chunks = chunk_data_generator(data, chunk_size) - - # Use multiprocessing to process chunks in parallel - block_seed = get_block_seed(self) - - # Pre-shuffle the UID combinations - pyrandom.seed(block_seed) - pyrandom.shuffle(uid_combinations) - - # Process each chunk and yield it's distribution of UIDs - for i, chunk in enumerate(data_chunks): - yield {hash_data(chunk): {"chunk": chunk, "uids": uid_combinations[i]}} - - -def partition_uids(available_uids, R): - """ - Partitions the available UIDs into non-overlapping groups of size R. - - Args: - available_uids (list): List of available UIDs. - R (int): Size of each group (redundancy factor). - - Returns: - list of tuples: A list where each tuple contains a unique group of UIDs. - """ - return [tuple(available_uids[i : i + R]) for i in range(0, len(available_uids), R)] - - -def adjust_uids_to_multiple(available_uids, R): - """ - Adjusts the list of available UIDs to ensure its length is a multiple of R. - - Args: - available_uids (list): The original list of available UIDs. - R (int): The redundancy factor. - - Returns: - list: A modified list of UIDs with a length that is a multiple of R. - """ - # Calculate the maximum number of complete groups of R that can be formed - max_complete_groups = len(available_uids) // R - - # Adjust the list length to be a multiple of R - adjusted_length = max_complete_groups * R - return available_uids[:adjusted_length] + successful_uids = [ + uid + for uid, response in zip(uids, responses) + if response.dendrite.status_code == 200 + ] + failed_uids = [ + uid + for uid, response in zip(uids, responses) + if response.dendrite.status_code != 200 + ] + except Exception as e: + bt.logging.error(f"Dendrite ping failed: {e}") + successful_uids = [] + failed_uids = uids + bt.logging.debug("ping() successful uids:", successful_uids) + bt.logging.debug("ping() failed uids :", failed_uids) + return successful_uids, failed_uids -async def compute_chunk_distribution_mut_exclusive_numpy_reuse_uids_yield( - self, data, R, k +async def ping_and_retry_uids( + self, k: int = None, max_retries: int = 3, exclude_uids: List[int] = [] ): """ - Asynchronously computes and yields chunk distributions for given data, considering the redundancy - factor and the number of query miners. This function splits the data into chunks, assigns a group - of miners to each chunk, and handles redundancy by reusing UIDs when necessary. - - Parameters: - data (bytes): The data to be distributed in chunks across the network. - R (int): Redundancy factor indicating the number of times each chunk should be replicated. - k (int): The number of unique identifiers (UIDs) or miners to be involved in the distribution. - - Yields: - dict: A dictionary for each chunk containing its hash, the chunk data itself, and the UIDs of - the miners assigned to it. - - Raises: - ValueError: If the redundancy factor R is greater than the number of available UIDs. - - Note: - - This function is essential for distributed storage systems where data needs to be stored - redundantly across multiple nodes. - - It ensures that each data chunk is stored by R different miners for redundancy. - - The distribution of chunks takes into account the availability of miners and may reuse UIDs - to meet the required redundancy factor. - - The yielded chunks can be used to parallelize storage operations across the network. - """ - available_uids = await get_available_query_miners(self, k=k) - data_size = len(data) - chunk_size = optimal_chunk_size(data_size, len(available_uids), R) - available_uids = adjust_uids_to_multiple(available_uids, R) - - if R > len(available_uids): - raise ValueError( - "Redundancy factor cannot be greater than the number of available UIDs." + Fetch available uids to minimize waiting for timeouts if they're going to fail anyways... + """ + # Select initial subset of miners to query + uids = await get_available_query_miners(self, k=k or 4, exclude=exclude_uids) + bt.logging.debug("initial ping_and_retry() uids:", uids) + + retries = 0 + successful_uids = set() + failed_uids = set() + while len(successful_uids) < k and retries < max_retries: + # Ping all UIDs + current_successful_uids, current_failed_uids = await ping_uids(self, uids) + successful_uids.update(current_successful_uids) + failed_uids.update(current_failed_uids) + + # If enough UIDs are successful, select the first k items + if len(successful_uids) >= k: + uids = list(successful_uids)[:k] + break + + # Reroll for k UIDs excluding the successful ones + new_uids = await get_available_query_miners( + self, k=k, exclude=list(successful_uids.union(failed_uids)) ) + bt.logging.debug(f"ping_and_retry() new uids: {new_uids}") + retries += 1 - # Create initial UID groups - initial_uid_groups = partition_uids(available_uids, R) - uid_groups = list(initial_uid_groups) - - # If more groups are needed, start reusing UIDs - total_chunks_needed = data_size // chunk_size - while len(uid_groups) < total_chunks_needed: - for group in cycle(initial_uid_groups): - if len(uid_groups) >= total_chunks_needed: - break - uid_groups.append(group) - - data_chunks = chunk_data_generator(data, chunk_size) - for chunk, uid_group in zip(data_chunks, uid_groups): - chunk_hash = hash_data(chunk) - yield {"chunk_hash": chunk_hash, "chunk": chunk, "uids": uid_group.tolist()} - - -def calculate_chunk_indices(data_size, chunk_size): - """ - Calculate the start and end indices for each chunk. - - :param data_size: The total size of the data to be chunked. - :param chunk_size: The chunk size. - :return: A list of tuples, each tuple containing the start and end index of a chunk. - """ - indices = [] - num_chunks = math.ceil(data_size / chunk_size) - - for i in range(num_chunks): - start_idx = i * chunk_size - end_idx = min(start_idx + chunk_size, data_size) - indices.append((start_idx, end_idx)) - - # Adjust the end index for the last chunk if necessary - if i == num_chunks - 1 and end_idx < data_size: - indices[-1] = (start_idx, data_size) - - return indices - - -def calculate_chunk_indices_from_num_chunks(data_size, num_chunks): - """ - Calculate the start and end indices for each chunk. - - :param data_size: The total size of the data to be chunked. - :param num_chunks: The desired number of chunks. - :return: A list of tuples, each tuple containing the start and end index of a chunk. - """ - chunk_size = max(1, data_size // num_chunks) # Determine the size of each chunk - indices = [] - - for i in range(num_chunks): - start_idx = i * chunk_size - end_idx = min(start_idx + chunk_size, data_size) - indices.append((start_idx, end_idx)) - - # Adjust the end index for the last chunk if necessary - if i == num_chunks - 1 and end_idx < data_size: - indices[-1] = (start_idx, data_size) - - return indices - - -async def compute_chunk_distribution_mut_exclusive_numpy_reuse_uids( - self, data_size, R, k, chunk_size=None, exclude=None -): - """ - Asynchronously computes a distribution of data chunks across a set of unique identifiers (UIDs), - taking into account redundancy and chunk size optimization. This function is useful for distributing - data across a network of nodes or miners in a way that ensures redundancy and optimal utilization. - - Parameters: - self: Reference to the class instance from which this method is called. - data_size (int): The total size of the data to be distributed, in bytes. - R (int): Redundancy factor, denoting the number of times each chunk should be replicated. - k (int): The number of unique identifiers (UIDs) to be involved in the distribution. - chunk_size (int, optional): The size of each data chunk. If not provided, an optimal chunk size - is calculated based on the data size and the number of UIDs. - - Yields: - dict: A dictionary representing a chunk's metadata, including its size, start index, end index, - the UIDs assigned to it, and its index in the chunk sequence. - - Raises: - ValueError: If the redundancy factor R is greater than the number of available UIDs. - - Note: - - This function is designed to be used in distributed storage or processing systems where - data needs to be split and stored across multiple nodes with redundancy. - - It evenly divides the data into chunks and assigns UIDs to each chunk while ensuring that - the redundancy requirements are met. - """ - - available_uids = await get_available_query_miners(self, k=k, exclude=exclude) - chunk_size = chunk_size or optimal_chunk_size(data_size, len(available_uids), R) - available_uids = adjust_uids_to_multiple(available_uids, R) - chunk_indices = calculate_chunk_indices(data_size, chunk_size) - - if R > len(available_uids): - raise ValueError( - "Redundancy factor cannot be greater than the number of available UIDs." + # Log if the maximum retries are reached without enough successful UIDs + if len(successful_uids) < k: + bt.logging.warning( + f"Insufficient successful UIDs for k: {k} Success UIDs {successful_uids} Failed UIDs: {failed_uids}" ) - # Create initial UID groups - initial_uid_groups = partition_uids(available_uids, R) - uid_groups = list(initial_uid_groups) - - # If more groups are needed, start reusing UIDs - total_chunks_needed = data_size // chunk_size - while len(uid_groups) < total_chunks_needed: - for group in cycle(initial_uid_groups): - if len(uid_groups) >= total_chunks_needed: - break - uid_groups.append(group) - - for i, ((start, end), uid_group) in enumerate(zip(chunk_indices, uid_groups)): - yield { - "chunk_size": chunk_size, - "start_idx": start, - "end_idx": end, - "uids": uid_group, - "chunk_index": i, - } - - -def get_rebalance_script_path(current_dir: str): - """ - Constructs and returns the path to the 'rebalance_deregistration.sh' script within a project directory. - - This function takes the root path of a project and appends the relative path to the 'rebalance_deregistration.sh' script. - It assumes that the script is located within the 'scripts' subdirectory of the given project root. - - Parameters: - project_root (str): The root path of the project directory. - - Returns: - str: The full path to the 'rebalance_deregistration.sh' script. - """ - project_root = os.path.join(current_dir, "..") - project_root = os.path.normpath(project_root) - script_path = os.path.join(project_root, "scripts", "rebalance_deregistration.sh") - return script_path - - -def get_current_epoch(subtensor, netuid: int = 21) -> int: - """ - Calculates the current epoch number from genesis of the network. - - Parameters: - subtensor: An object representing the Subtensor network, which provides methods to get the current block and the network's tempo. - netuid (int, optional): The network ID for which the epoch is to be calculated. Default: 21. - - Returns: - int: The current epoch calculated based on the elapsed blocks and the network's tempo. - """ - registered_at = 2009702 - blocks_since_registration = subtensor.get_current_block() - registered_at - current_epoch = blocks_since_registration // subtensor.tempo(netuid) - return current_epoch + return list(successful_uids)[:k], failed_uids \ No newline at end of file diff --git a/subnet/validator/weights.py b/subnet/validator/weights.py index ef3ec0a2..a212ce4e 100644 --- a/subnet/validator/weights.py +++ b/subnet/validator/weights.py @@ -48,6 +48,7 @@ def set_weights_for_validator( uids (torch.Tensor): miners UIDs on the network. metagraph (bt.metagraph): Bittensor metagraph. moving_averaged_scores (torch.Tensor): . + wandb_on (bool, optional): Flag to determine if logging to Weights & Biases is enabled. Defaults to False. tempo (int): Tempo for 'netuid' subnet. wait_for_inclusion (bool, optional): Wether to wait for the extrinsic to enter a block wait_for_finalization (bool, optional): Wether to wait for the extrinsic to be finalized on the chain @@ -62,10 +63,49 @@ def set_weights_for_validator( """ # Calculate the average reward for each uid across non-zero values. # Replace any NaN values with 0. - raw_weights = torch.nn.functional.normalize(moving_averaged_scores, p=1, dim=0) + nan_idxs = torch.where(torch.isnan(moving_averaged_scores))[0] + moving_averaged_scores_no_nan = torch.where( + torch.isnan(moving_averaged_scores), + torch.zeros_like(moving_averaged_scores), + moving_averaged_scores + ) + # Gather negative indices + neg_idxs = torch.where(moving_averaged_scores_no_nan < 0)[0] + + # Ensure positive + minimum = min(moving_averaged_scores_no_nan) + + # Replace nan with min + moving_averaged_scores_no_nan[nan_idxs] = minimum.clone() + + # Make all values positive + if minimum < 0: + positive_moving_averaged_scores = moving_averaged_scores_no_nan - minimum + else: + positive_moving_averaged_scores = moving_averaged_scores_no_nan + bt.logging.debug(f"Positive scores", positive_moving_averaged_scores) + + # Push all orinally negative indices to zero + positive_moving_averaged_scores[neg_idxs] = 0 + + # Normalize, ensuring no division by zero or NaNs occur + sum_scores = positive_moving_averaged_scores.sum() + bt.logging.info(f"Score sum: {sum_scores}") + if sum_scores > 0: + raw_weights = torch.nn.functional.normalize(positive_moving_averaged_scores, p=1, dim=0) + else: + raw_weights = torch.zeros_like(positive_moving_averaged_scores) + + # Doubly ensure raw_weights does not contain NaNs (this should not happen after normalization, but as an extra precaution) + raw_weights = torch.where( + torch.isnan(raw_weights), + torch.zeros_like(raw_weights), + raw_weights, + ) bt.logging.debug("raw_weights", raw_weights) bt.logging.debug("raw_weight_uids", metagraph.uids.to("cpu")) + # Process the raw weights to final_weights via subtensor limitations. ( processed_weight_uids, @@ -88,7 +128,7 @@ def set_weights_for_validator( bt.logging.debug("uint_uids", uint_uids) # Set the weights on chain via our subtensor connection. - success = set_weights( + success, message = set_weights( subtensor=subtensor, wallet=wallet, netuid=netuid, @@ -100,6 +140,6 @@ def set_weights_for_validator( ) if success is True: - bt.logging.info("set_weights on chain successfully!") + bt.logging.info("Set weights on chain successfully!") else: - bt.logging.error(f"set_weights failed.") + bt.logging.error(f"Set weights failed {message}.")