From ed33eb473f47dbd80558b318b82bdd5b4c6daa1e Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Fri, 18 Feb 2022 10:16:56 +0700 Subject: [PATCH 001/152] (refactor) Adapts Avellaneda config map for pydantic. The config map introduced in this PR is a proof of concept for the new approach to configs using pydantic. It adopts @aarmoa's suggestion to use nested models as a method of solving the interdependencies issues present with some config variables. --- hummingbot/client/config/config_data_types.py | 47 ++ hummingbot/client/config/config_helpers.py | 54 +- ...aneda_market_making_config_map_pydantic.py | 523 ++++++++++++++++++ setup.py | 1 + setup/environment-linux-aarch64.yml | 1 + setup/environment-linux.yml | 1 + setup/environment-win64.yml | 1 + setup/environment.yml | 1 + ...aneda_market_making_config_map_pydantic.py | 246 ++++++++ .../avellaneda_market_making/test_config.yml | 10 + 10 files changed, 859 insertions(+), 26 deletions(-) create mode 100644 hummingbot/client/config/config_data_types.py create mode 100644 hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py create mode 100644 test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py create mode 100644 test/hummingbot/strategy/avellaneda_market_making/test_config.yml diff --git a/hummingbot/client/config/config_data_types.py b/hummingbot/client/config/config_data_types.py new file mode 100644 index 0000000000..593c09b9a1 --- /dev/null +++ b/hummingbot/client/config/config_data_types.py @@ -0,0 +1,47 @@ +from dataclasses import dataclass +from enum import Enum +from typing import Any, Optional, Callable + +from pydantic import BaseModel +from pydantic.schema import default_ref_template + +from hummingbot.client.config.config_helpers import strategy_config_schema_encoder + + +class ClientConfigEnum(Enum): + def __str__(self): + return self.value + + +@dataclass() +class ClientFieldData: + prompt: Optional[Callable[['BaseClientModel'], str]] = None + prompt_on_new: bool = False + + +class BaseClientModel(BaseModel): + """ + Notes on configs: + - In nested models, be weary that pydantic will take the first model that fits + (see https://pydantic-docs.helpmanual.io/usage/model_config/#smart-union). + """ + @classmethod + def schema_json( + cls, *, by_alias: bool = True, ref_template: str = default_ref_template, **dumps_kwargs: Any + ) -> str: + # todo: make it ignore `client_data` all together + return cls.__config__.json_dumps( + cls.schema(by_alias=by_alias, ref_template=ref_template), + default=strategy_config_schema_encoder, + **dumps_kwargs + ) + + def get_client_prompt(self, attr_name: str) -> Optional[str]: + prompt = None + client_data = self.get_client_data(attr_name) + if client_data is not None: + prompt = client_data.prompt(self) + return prompt + + def get_client_data(self, attr_name: str) -> Optional[ClientFieldData]: + return self.__fields__[attr_name].field_info.extra["client_data"] diff --git a/hummingbot/client/config/config_helpers.py b/hummingbot/client/config/config_helpers.py index 971ecf9517..cdb98e6e00 100644 --- a/hummingbot/client/config/config_helpers.py +++ b/hummingbot/client/config/config_helpers.py @@ -1,40 +1,31 @@ +import json import logging +import shutil +from collections import OrderedDict from decimal import Decimal +from os import listdir, unlink +from os.path import isfile, join +from typing import Any, Callable, Dict, List, Optional + import ruamel.yaml -from os import ( - unlink -) -from os.path import ( - join, - isfile -) -from collections import OrderedDict -import json -from typing import ( - Any, - Callable, - Dict, - List, - Optional, -) -from os import listdir -import shutil +from eth_account import Account +from pydantic import ValidationError +from pydantic.json import pydantic_encoder +from hummingbot import get_strategy_list from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.fee_overrides_config_map import fee_overrides_config_map +from hummingbot.client.config.global_config_map import global_config_map +from hummingbot.client.config.security import Security from hummingbot.client.settings import ( - GLOBAL_CONFIG_PATH, - TRADE_FEES_CONFIG_PATH, - TEMPLATE_PATH, CONF_FILE_PATH, CONF_POSTFIX, CONF_PREFIX, - AllConnectorSettings, + GLOBAL_CONFIG_PATH, + TEMPLATE_PATH, + TRADE_FEES_CONFIG_PATH, + AllConnectorSettings ) -from hummingbot.client.config.security import Security -from hummingbot import get_strategy_list -from eth_account import Account # Use ruamel.yaml to preserve order and comments in .yml file yaml_parser = ruamel.yaml.YAML() @@ -454,3 +445,14 @@ def secondary_market_conversion_rate(strategy) -> Decimal: else: return Decimal("1") return quote_rate / base_rate + + +def retrieve_validation_error_msg(e: ValidationError) -> str: + return e.errors().pop()["msg"] + + +def strategy_config_schema_encoder(o): + if callable(o): + return None + else: + return pydantic_encoder(o) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py new file mode 100644 index 0000000000..513d9d450d --- /dev/null +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -0,0 +1,523 @@ +from datetime import datetime, time +from decimal import Decimal +from typing import Dict, Optional, Union + +from pydantic import Field, validator, root_validator + +from hummingbot.client.config.config_data_types import BaseClientModel, ClientConfigEnum, ClientFieldData +from hummingbot.client.config.config_validators import ( + validate_bool, + validate_datetime_iso_string, + validate_decimal, + validate_exchange, + validate_int, + validate_market_trading_pair, + validate_time_iso_string, +) +from hummingbot.client.settings import AllConnectorSettings, required_exchanges +from hummingbot.connector.utils import split_hb_trading_pair + + +class ExecutionTimeframe(str, ClientConfigEnum): + infinite = "infinite" + from_date_to_date = "from_date_to_date" + daily_between_times = "daily_between_times" + + +class InfiniteModel(BaseClientModel): + class Config: + title = ExecutionTimeframe.infinite + validate_assignment = True + + +class FromDateToDateModel(BaseClientModel): + start_datetime: datetime = Field( + default=..., + description="The start date and time for date-to-date execution timeframe.", + client_data=ClientFieldData( + prompt=lambda mi: "Please enter the start date and time (YYYY-MM-DD HH:MM:SS)", + prompt_on_new=True, + ), + ) + end_datetime: datetime = Field( + default=..., + description="The end date and time for date-to-date execution timeframe.", + client_data=ClientFieldData( + prompt=lambda mi: "Please enter the end date and time (YYYY-MM-DD HH:MM:SS)", + prompt_on_new=True, + ), + ) + + class Config: + title = ExecutionTimeframe.from_date_to_date + validate_assignment = True + + @validator("start_datetime", "end_datetime", pre=True) + def validate_execution_time(cls, v: str) -> Optional[str]: + ret = validate_datetime_iso_string(v) + if ret is not None: + raise ValueError(ret) + return v + + +class DailyBetweenTimesModel(BaseClientModel): + start_time: time = Field( + default=..., + description="The start time for daily-between-times execution timeframe.", + client_data=ClientFieldData( + prompt=lambda mi: "Please enter the start time (HH:MM:SS)", + prompt_on_new=True, + ), + ) + end_time: time = Field( + default=..., + description="The end time for daily-between-times execution timeframe.", + client_data=ClientFieldData( + prompt=lambda mi: "Please enter the end time (HH:MM:SS)", + prompt_on_new=True, + ), + ) + + class Config: + title = ExecutionTimeframe.daily_between_times + validate_assignment = True + + @validator("start_time", "end_time", pre=True) + def validate_execution_time(cls, v: str) -> Optional[str]: + ret = validate_time_iso_string(v) + if ret is not None: + raise ValueError(ret) + return v + + +class OrderLevelsMode(str, ClientConfigEnum): + single_order_level = "single_order_level" + multi_order_level = "multi_order_level" + + +class SingleOrderLevelModel(BaseClientModel): + class Config: + title = OrderLevelsMode.single_order_level + validate_assignment = True + + +class MultiOrderLevelModel(BaseClientModel): + order_levels: int = Field( + default=2, + description="The number of orders placed on either side of the order book.", + ge=2, + client_data=ClientFieldData( + prompt=lambda mi: "How many orders do you want to place on both sides?" + ), + ) + level_distances: Decimal = Field( + default=Decimal("0"), + description="The spread between order levels, expressed in % of optimal spread.", + ge=0, + client_data=ClientFieldData( + prompt=lambda mi: "How far apart in % of optimal spread should orders on one side be?" + ), + ) + + class Config: + title = OrderLevelsMode.multi_order_level + validate_assignment = True + + @validator("order_levels", pre=True) + def validate_int_zero_or_above(cls, v: str): + ret = validate_int(v, min_value=2) + if ret is not None: + raise ValueError(ret) + return v + + @validator("level_distances", pre=True) + def validate_decimal_zero_or_above(cls, v: str): + ret = validate_decimal(v, min_value=Decimal("0"), inclusive=True) + if ret is not None: + raise ValueError(ret) + return v + + +class HangingOrdersMode(str, ClientConfigEnum): + track_hanging_orders = "track_hanging_orders" + ignore_hanging_orders = "ignore_hanging_orders" + + +class TrackHangingOrdersModel(BaseClientModel): + hanging_orders_cancel_pct: Decimal = Field( + default=Decimal("10"), + description="The spread percentage at which hanging orders will be cancelled.", + gt=0, + lt=100, + client_data=ClientFieldData( + prompt=lambda mi: ( + "At what spread percentage (from mid price) will hanging orders be canceled?" + " (Enter 1 to indicate 1%)" + ), + ) + ) + + class Config: + title = HangingOrdersMode.track_hanging_orders + validate_assignment = True + + @validator("hanging_orders_cancel_pct", pre=True) + def validate_pct_exclusive(cls, v: str): + ret = validate_decimal(v, min_value=Decimal("0"), max_value=Decimal("100"), inclusive=False) + if ret is not None: + raise ValueError(ret) + return v + + +class IgnoreHangingOrdersModel(BaseClientModel): + class Config: + title = HangingOrdersMode.ignore_hanging_orders + validate_assignment = True + + +def maker_trading_pair_prompt(model_instance: 'AvellanedaMarketMakingConfigMap') -> str: + exchange = model_instance.exchange + example = AllConnectorSettings.get_example_pairs().get(exchange) + return ( + f"Enter the token trading pair you would like to trade on {exchange}{f' (e.g. {example})' if example else ''}" + ) + + +def order_amount_prompt(model_instance: 'AvellanedaMarketMakingConfigMap') -> str: + trading_pair = model_instance.market + base_asset, quote_asset = split_hb_trading_pair(trading_pair) + return f"What is the amount of {base_asset} per order?" + + +class AvellanedaMarketMakingConfigMap(BaseClientModel): + strategy: str = Field(default="avellaneda_market_making", client_data=None) + exchange: ClientConfigEnum( + value="Exchanges", # noqa: F821 + names={e: e for e in AllConnectorSettings.get_connector_settings().keys()}, + type=str, + ) = Field( + default=..., + description="The name of the exchange connector.", + client_data=ClientFieldData( + prompt=lambda mi: "Input your maker spot connector", + prompt_on_new=True, + ), + ) + market: str = Field( + default=..., + description="The trading pair.", + client_data=ClientFieldData( + prompt=maker_trading_pair_prompt, + prompt_on_new=True, + ), + ) + execution_timeframe_model: Union[FromDateToDateModel, DailyBetweenTimesModel, InfiniteModel] = Field( + default=..., + description="The execution timeframe.", + client_data=ClientFieldData( + prompt=lambda mi: f"Select the execution timeframe ({'/'.join(ExecutionTimeframe)})", + prompt_on_new=True, + ), + ) + order_amount: Decimal = Field( + default=..., + description="The strategy order amount.", + gt=0, + client_data=ClientFieldData(prompt=order_amount_prompt) + ) + order_optimization_enabled: bool = Field( + default=True, + description=( + "Allows the bid and ask order prices to be adjusted based on" + " the current top bid and ask prices in the market." + ), + client_data=ClientFieldData( + prompt=lambda mi: "Do you want to enable best bid ask jumping? (Yes/No)" + ), + ) + risk_factor: Decimal = Field( + default=Decimal("1"), + description="The risk factor (\u03B3).", + gt=0, + client_data=ClientFieldData( + prompt=lambda mi: "Enter risk factor (\u03B3)", + prompt_on_new=True, + ), + ) + order_amount_shape_factor: Decimal = Field( + default=Decimal("0"), + description="The amount shape factor (\u03b7)", + ge=0, + le=1, + client_data=ClientFieldData( + prompt=lambda mi: "Enter order amount shape factor (\u03B7)", + ), + ) + min_spread: Decimal = Field( + default=Decimal("0"), + description="The minimum spread limit as percentage of the mid price.", + ge=0, + client_data=ClientFieldData( + prompt=lambda mi: "Enter minimum spread limit (as % of mid price)", + ), + ) + order_refresh_time: float = Field( + default=..., + description="The frequency at which the orders' spreads will be re-evaluated.", + gt=0., + client_data=ClientFieldData( + prompt=lambda mi: "How often do you want to cancel and replace bids and asks (in seconds)?", + prompt_on_new=True, + ), + ) + max_order_age: float = Field( + default=1800., + description="A given order's maximum lifetime irrespective of spread.", + gt=0., + client_data=ClientFieldData( + prompt=lambda mi: ( + "How long do you want to cancel and replace bids and asks with the same price (in seconds)?" + ), + ), + ) + order_refresh_tolerance_pct: Decimal = Field( + default=Decimal("0"), + description=( + "The range of spreads tolerated on refresh cycles." + " Orders over that range are cancelled and re-submitted." + ), + ge=-10, + le=10, + client_data=ClientFieldData( + prompt=lambda mi: ( + "Enter the percent change in price needed to refresh orders at each cycle" + " (Enter 1 to indicate 1%)" + ) + ), + ) + filled_order_delay: float = Field( + default=60., + description="The delay before placing a new order after an order fill.", + gt=0., + client_data=ClientFieldData( + prompt=lambda mi: ( + "How long do you want to wait before placing the next order" + " if your order gets filled (in seconds)?" + ) + ), + ) + inventory_target_base_pct: Decimal = Field( + default=Decimal("50"), + description="Defines the inventory target for the base asset.", + ge=0, + le=100, + client_data=ClientFieldData( + prompt=lambda mi: "What is the inventory target for the base asset? Enter 50 for 50%", + prompt_on_new=True, + ), + ) + add_transaction_costs: bool = Field( + default=False, + description="If activated, transaction costs will be added to order prices.", + client_data=ClientFieldData( + prompt=lambda mi: "Do you want to add transaction costs automatically to order prices? (Yes/No)", + ), + ) + volatility_buffer_size: int = Field( + default=200, + description="The number of ticks that will be stored to calculate volatility.", + ge=1, + le=10_000, + client_data=ClientFieldData( + prompt=lambda mi: "Enter amount of ticks that will be stored to estimate order book liquidity", + ), + ) + trading_intensity_buffer_size: int = Field( + default=200, + description="The number of ticks that will be stored to calculate order book liquidity.", + ge=1, + le=10_000, + client_data=ClientFieldData( + prompt=lambda mi: "Enter amount of ticks that will be stored to estimate order book liquidity", + ), + ) + order_levels_mode: Union[MultiOrderLevelModel, SingleOrderLevelModel] = Field( + default=SingleOrderLevelModel.construct(), + description="Allows activating multi-order levels.", + client_data=ClientFieldData( + prompt=lambda mi: f"Select the order levels mode ({'/'.join(OrderLevelsMode)}", + ), + ) + order_override: Optional[Dict] = Field( + default=None, + description="Allows custom specification of the order levels and their spreads and amounts.", + client_data=None, + ) + hanging_orders_mode: Union[TrackHangingOrdersModel, IgnoreHangingOrdersModel] = Field( + default=IgnoreHangingOrdersModel.construct(), + description="When tracking hanging orders, the orders on the side opposite to the filled orders remain active.", + client_data=ClientFieldData( + prompt=lambda mi: f"How do you want to handle hanging orders? ({'/'.join(HangingOrdersMode)})", + ), + ) + should_wait_order_cancel_confirmation: bool = Field( + default=True, + description=( + "If activated, the strategy will await cancellation confirmation from the exchange" + " before placing a new order." + ), + client_data=ClientFieldData( + prompt=lambda mi: ( + "Should the strategy wait to receive a confirmation for orders cancellation" + " before creating a new set of orders?" + " (Not waiting requires enough available balance) (Yes/No)" + ), + ) + ) + + def __init__(self, **data): + super().__init__(**data) + required_exchanges.append(self.exchange) + + class Config: + validate_assignment = True + + @validator("exchange", pre=True) + def validate_exchange(cls, v: str): + ret = validate_exchange(v) + if ret is not None: + raise ValueError(ret) + return v + + @validator("market", pre=True) + def validate_exchange_trading_pair(cls, v: str, values: Dict): + exchange = values.get("exchange") + ret = validate_market_trading_pair(exchange, v) + if ret is not None: + raise ValueError(ret) + return v + + @validator("execution_timeframe_model", pre=True) + def validate_execution_timeframe( + cls, v: Union[str, InfiniteModel, FromDateToDateModel, DailyBetweenTimesModel] + ): + if not isinstance(v, Dict) and not hasattr(ExecutionTimeframe, v): + raise ValueError( + f"Invalid timeframe, please choose value from {[e.value for e in list(ExecutionTimeframe)]}" + ) + elif isinstance(v, (InfiniteModel, FromDateToDateModel, DailyBetweenTimesModel)): + sub_model = v + elif v == ExecutionTimeframe.infinite: + sub_model = InfiniteModel.construct() + elif v == ExecutionTimeframe.daily_between_times: + sub_model = DailyBetweenTimesModel.construct() + else: # v == ExecutionTimeframe.from_date_to_date + sub_model = FromDateToDateModel.construct() + return sub_model + + @validator("order_refresh_tolerance_pct", pre=True) + def validate_order_refresh_tolerance_pct(cls, v: str): + ret = validate_decimal(v, min_value=Decimal("-10"), max_value=Decimal("10"), inclusive=True) + if ret is not None: + raise ValueError(ret) + return v + + @validator("volatility_buffer_size", "trading_intensity_buffer_size", pre=True) + def validate_buffer_size(cls, v: str): + ret = validate_int(v, 1, 10_000) + if ret is not None: + raise ValueError(ret) + return v + + @validator("order_levels_mode", pre=True) + def validate_order_levels_mode(cls, v: Union[str, SingleOrderLevelModel, MultiOrderLevelModel]): + if not isinstance(v, Dict) and not hasattr(OrderLevelsMode, v): + raise ValueError( + f"Invalid order levels mode, please choose value from {[e.value for e in list(OrderLevelsMode)]}." + ) + elif isinstance(v, (SingleOrderLevelModel, MultiOrderLevelModel)): + sub_model = v + elif v == OrderLevelsMode.single_order_level: + sub_model = SingleOrderLevelModel.construct() + else: # v == OrderLevelsMode.multi_order_level + sub_model = MultiOrderLevelModel.construct() + return sub_model + + @validator("hanging_orders_mode", pre=True) + def validate_hanging_orders_mode(cls, v: Union[str, TrackHangingOrdersModel, IgnoreHangingOrdersModel]): + if not isinstance(v, Dict) and not hasattr(HangingOrdersMode, v): + raise ValueError( + f"Invalid hanging order mode, please choose value from {[e.value for e in list(HangingOrdersMode)]}." + ) + elif isinstance(v, (TrackHangingOrdersModel, IgnoreHangingOrdersModel)): + sub_model = v + elif v == HangingOrdersMode.track_hanging_orders: + sub_model = TrackHangingOrdersModel.construct() + else: # v == HangingOrdersMode.ignore_hanging_orders + sub_model = IgnoreHangingOrdersModel.construct() + return sub_model + + # === generic validations === + + @validator( + "order_optimization_enabled", + "add_transaction_costs", + "should_wait_order_cancel_confirmation", + pre=True, + ) + def validate_bool(cls, v: str): + if isinstance(v, str): + ret = validate_bool(v) + if ret is not None: + raise ValueError(ret) + return v + + @validator("order_amount_shape_factor", pre=True) + def validate_decimal_from_zero_to_one(cls, v: str): + ret = validate_decimal(v, min_value=Decimal("0"), max_value=Decimal("1"), inclusive=True) + if ret is not None: + raise ValueError(ret) + return v + + @validator( + "order_amount", + "risk_factor", + "order_refresh_time", + "max_order_age", + "filled_order_delay", + pre=True, + ) + def validate_decimal_above_zero(cls, v: str): + ret = validate_decimal(v, min_value=Decimal("0"), inclusive=False) + if ret is not None: + raise ValueError(ret) + return v + + @validator("min_spread", pre=True) + def validate_decimal_zero_or_above(cls, v: str): + ret = validate_decimal(v, min_value=Decimal("0"), inclusive=True) + if ret is not None: + raise ValueError(ret) + return v + + @validator("inventory_target_base_pct", pre=True) + def validate_pct_inclusive(cls, v: str): + ret = validate_decimal(v, min_value=Decimal("0"), max_value=Decimal("100"), inclusive=True) + if ret is not None: + raise ValueError(ret) + return v + + # === post-validations === + + @root_validator() + def post_validations(cls, values: Dict): + cls.execution_timeframe_post_validation(values) + return values + + @classmethod + def execution_timeframe_post_validation(cls, values: Dict): + execution_timeframe = values.get("execution_timeframe") + if execution_timeframe is not None and execution_timeframe == ExecutionTimeframe.infinite: + values["start_time"] = None + values["end_time"] = None + return values diff --git a/setup.py b/setup.py index 5c6273dda5..3b85ef9931 100755 --- a/setup.py +++ b/setup.py @@ -77,6 +77,7 @@ def main(): "pre-commit", "prompt-toolkit", "psutil", + "pydantic", "pyjwt", "pyperclip", "python-dateutil", diff --git a/setup/environment-linux-aarch64.yml b/setup/environment-linux-aarch64.yml index 0e0b45b744..1a91dc7341 100644 --- a/setup/environment-linux-aarch64.yml +++ b/setup/environment-linux-aarch64.yml @@ -14,6 +14,7 @@ dependencies: - pandas=1.2.1 - pip=21.1.2 - prompt_toolkit=3.0.20 + - pydantic=1.9.0 - pytables=3.6.1 - python=3.8.2 - python-dateutil=2.8.1 diff --git a/setup/environment-linux.yml b/setup/environment-linux.yml index c079cbcf72..a0957cf9ae 100644 --- a/setup/environment-linux.yml +++ b/setup/environment-linux.yml @@ -14,6 +14,7 @@ dependencies: - pandas=1.2.1 - pip=21.1.2 - prompt_toolkit=3.0.20 + - pydantic=1.9.0 - pytables=3.6.1 - python=3.8.2 - python-dateutil=2.8.1 diff --git a/setup/environment-win64.yml b/setup/environment-win64.yml index 26829e1e79..6c09f7eb3c 100644 --- a/setup/environment-win64.yml +++ b/setup/environment-win64.yml @@ -12,6 +12,7 @@ dependencies: - pandas=1.2.1 - pip=21.1.2 - prompt_toolkit=3.0.20 + - pydantic=1.9.0 - pytables=3.6.1 - python=3.8.2 - python-dateutil=2.8.1 diff --git a/setup/environment.yml b/setup/environment.yml index acd504f4f5..601b36d14c 100644 --- a/setup/environment.yml +++ b/setup/environment.yml @@ -15,6 +15,7 @@ dependencies: - pandas=1.2.1 - pip=21.1.2 - prompt_toolkit=3.0.20 + - pydantic=1.9.0 - pytables=3.6.1 - python=3.8.2 - python-dateutil=2.8.1 diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py new file mode 100644 index 0000000000..46dc98051b --- /dev/null +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py @@ -0,0 +1,246 @@ +import json +import unittest +from datetime import datetime, time +from pathlib import Path +from typing import Dict +from unittest.mock import patch + +import yaml +from pydantic import ValidationError, validate_model + +from hummingbot.client.config.config_data_types import BaseClientModel +from hummingbot.client.config.config_helpers import retrieve_validation_error_msg +from hummingbot.client.settings import AllConnectorSettings +from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( + AvellanedaMarketMakingConfigMap, + DailyBetweenTimesModel, + ExecutionTimeframe, + FromDateToDateModel, + InfiniteModel, +) + + +class AvellanedaMarketMakingConfigMapPydanticTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + cls.exchange = "binance" + cls.base_asset = "COINALPHA" + cls.quote_asset = "HBOT" + cls.trading_pair = f"{cls.base_asset}-{cls.quote_asset}" + + def setUp(self) -> None: + super().setUp() + config_settings = self.get_default_map() + self.config_map = AvellanedaMarketMakingConfigMap(**config_settings) + + def get_default_map(self) -> Dict[str, str]: + config_settings = { + "exchange": self.exchange, + "market": self.trading_pair, + "execution_timeframe_model": { + "start_time": "09:30:00", + "end_time": "16:00:00", + }, + "order_amount": "10", + "order_optimization_enabled": "yes", + "risk_factor": "0.5", + "order_refresh_time": "60", + "inventory_target_base_pct": "50", + } + return config_settings + + def test_schema_encoding_removes_client_data_functions(self): + s = AvellanedaMarketMakingConfigMap.schema_json() + j = json.loads(s) + expected = { + "prompt": None, + "prompt_on_new": True, + } + self.assertEqual(expected, j["properties"]["market"]["client_data"]) + + def test_initial_sequential_build(self): + config_map: AvellanedaMarketMakingConfigMap = AvellanedaMarketMakingConfigMap.construct() + config_settings = self.get_default_map() + + def build_config_map(cm: BaseClientModel, cs: Dict): + """ + This routine can be used as is for the create command, + except for the sectioned-off portion. + """ + for key, field in cm.__fields__.items(): + client_data = cm.get_client_data(key) + if client_data is not None and client_data.prompt_on_new: + self.assertIsInstance(client_data.prompt(cm), str) + # ===================================================== + if key == "execution_timeframe_model": + cm.__setattr__(key, "daily_between_times") # simulate user input + else: + cm.__setattr__(key, cs[key]) + # cm.__setattr__(key, cs[key]) # use this in the create command routine + # ===================================================== + new_value = cm.__getattribute__(key) + if isinstance(new_value, BaseClientModel): + build_config_map(new_value, cs[key]) + + build_config_map(config_map, config_settings) + validate_model(config_map.__class__, config_map.__dict__) + + def test_order_amount_prompt(self): + prompt = self.config_map.get_client_prompt("order_amount") + expected = f"What is the amount of {self.base_asset} per order?" + + self.assertEqual(expected, prompt) + + def test_maker_trading_pair_prompt(self): + exchange = self.config_map.exchange + example = AllConnectorSettings.get_example_pairs().get(exchange) + + prompt = self.config_map.get_client_prompt("market") + expected = f"Enter the token trading pair you would like to trade on {exchange} (e.g. {example})" + + self.assertEqual(expected, prompt) + + def test_execution_time_prompts(self): + self.config_map.execution_timeframe_model = ExecutionTimeframe.from_date_to_date + model = self.config_map.execution_timeframe_model + prompt = model.get_client_prompt("start_datetime") + expected = "Please enter the start date and time (YYYY-MM-DD HH:MM:SS)" + self.assertEqual(expected, prompt) + prompt = model.get_client_prompt("end_datetime") + expected = "Please enter the end date and time (YYYY-MM-DD HH:MM:SS)" + self.assertEqual(expected, prompt) + + self.config_map.execution_timeframe_model = ExecutionTimeframe.daily_between_times + model = self.config_map.execution_timeframe_model + prompt = model.get_client_prompt("start_time") + expected = "Please enter the start time (HH:MM:SS)" + self.assertEqual(expected, prompt) + prompt = model.get_client_prompt("end_time") + expected = "Please enter the end time (HH:MM:SS)" + self.assertEqual(expected, prompt) + + @patch( + "hummingbot.strategy.avellaneda_market_making" + ".avellaneda_market_making_config_map_pydantic.validate_market_trading_pair" + ) + def test_validators(self, validate_market_trading_pair_mock): + + with self.assertRaises(ValidationError) as e: + self.config_map.exchange = "test-exchange" + + error_msg = "Invalid exchange, please choose value from " + actual_msg = retrieve_validation_error_msg(e.exception) + self.assertTrue(actual_msg.startswith(error_msg)) + + alt_pair = "ETH-USDT" + error_msg = "Failed" + validate_market_trading_pair_mock.side_effect = ( + lambda m, v: None if v in [self.trading_pair, alt_pair] else error_msg + ) + + self.config_map.market = alt_pair + self.assertEqual(alt_pair, self.config_map.market) + + with self.assertRaises(ValidationError) as e: + self.config_map.market = "XXX-USDT" + + actual_msg = retrieve_validation_error_msg(e.exception) + self.assertTrue(actual_msg.startswith(error_msg)) + + self.config_map.execution_timeframe_model = "infinite" + self.assertIsInstance(self.config_map.execution_timeframe_model, InfiniteModel) + + self.config_map.execution_timeframe_model = "from_date_to_date" + self.assertIsInstance(self.config_map.execution_timeframe_model, FromDateToDateModel) + + self.config_map.execution_timeframe_model = "daily_between_times" + self.assertIsInstance(self.config_map.execution_timeframe_model, DailyBetweenTimesModel) + + with self.assertRaises(ValidationError) as e: + self.config_map.execution_timeframe_model = "XXX" + + error_msg = ( + "Invalid timeframe, please choose value from ['infinite', 'from_date_to_date', 'daily_between_times']" + ) + actual_msg = retrieve_validation_error_msg(e.exception) + self.assertEqual(error_msg, actual_msg) + + self.config_map.execution_timeframe_model = "from_date_to_date" + model = self.config_map.execution_timeframe_model + model.start_datetime = "2021-01-01 12:00:00" + model.end_datetime = "2021-01-01 15:00:00" + + self.assertEqual(datetime(2021, 1, 1, 12, 0, 0), model.start_datetime) + self.assertEqual(datetime(2021, 1, 1, 15, 0, 0), model.end_datetime) + + with self.assertRaises(ValidationError) as e: + model.start_datetime = "2021-01-01 30:00:00" + + error_msg = "Incorrect date time format (expected is YYYY-MM-DD HH:MM:SS)" + actual_msg = retrieve_validation_error_msg(e.exception) + self.assertEqual(error_msg, actual_msg) + + with self.assertRaises(ValidationError) as e: + model.start_datetime = "12:00:00" + + error_msg = "Incorrect date time format (expected is YYYY-MM-DD HH:MM:SS)" + actual_msg = retrieve_validation_error_msg(e.exception) + self.assertEqual(error_msg, actual_msg) + + self.config_map.execution_timeframe_model = "daily_between_times" + model = self.config_map.execution_timeframe_model + model.start_time = "12:00:00" + + self.assertEqual(time(12, 0, 0), model.start_time) + + with self.assertRaises(ValidationError) as e: + model.start_time = "30:00:00" + + error_msg = "Incorrect time format (expected is HH:MM:SS)" + actual_msg = retrieve_validation_error_msg(e.exception) + self.assertEqual(error_msg, actual_msg) + + with self.assertRaises(ValidationError) as e: + model.start_time = "2021-01-01 12:00:00" + + error_msg = "Incorrect time format (expected is HH:MM:SS)" + actual_msg = retrieve_validation_error_msg(e.exception) + self.assertEqual(error_msg, actual_msg) + + self.config_map.order_levels_mode = "multi_order_level" + model = self.config_map.order_levels_mode + + with self.assertRaises(ValidationError) as e: + model.order_levels = 1 + + error_msg = "Value cannot be less than 2." + actual_msg = retrieve_validation_error_msg(e.exception) + self.assertEqual(error_msg, actual_msg) + + model.order_levels = 3 + self.assertEqual(3, model.order_levels) + + self.config_map.hanging_orders_mode = "track_hanging_orders" + model = self.config_map.hanging_orders_mode + + with self.assertRaises(ValidationError) as e: + model.hanging_orders_cancel_pct = "-1" + + error_msg = "Value must be between 0 and 100 (exclusive)." + actual_msg = retrieve_validation_error_msg(e.exception) + self.assertEqual(error_msg, actual_msg) + + model.hanging_orders_cancel_pct = "3" + self.assertEqual(3, model.hanging_orders_cancel_pct) + + def test_load_configs_from_yaml(self): + cur_dir = Path(__file__).parent + f_path = cur_dir / "test_config.yml" + + with open(f_path, "r") as file: + data = yaml.safe_load(file) + + loaded_config_map = AvellanedaMarketMakingConfigMap(**data) + + self.assertEqual(self.config_map, loaded_config_map) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_config.yml b/test/hummingbot/strategy/avellaneda_market_making/test_config.yml new file mode 100644 index 0000000000..cbf4e2b487 --- /dev/null +++ b/test/hummingbot/strategy/avellaneda_market_making/test_config.yml @@ -0,0 +1,10 @@ +exchange: binance +market: COINALPHA-HBOT +execution_timeframe_model: + start_time: "09:30:00" + end_time: "16:00:00" +order_amount: 10 +order_optimization_enabled: true +risk_factor: 0.5 +order_refresh_time: 60 +inventory_target_base_pct: 50 \ No newline at end of file From d39995a13facec6b0188aab0b02dc963f0bf2e23 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Fri, 18 Feb 2022 10:27:54 +0700 Subject: [PATCH 002/152] (fix) addresses a potential problem in parsing sub-models. --- .../avellaneda_market_making_config_map_pydantic.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py index 513d9d450d..b346e9c19e 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -411,8 +411,10 @@ def validate_execution_timeframe( sub_model = InfiniteModel.construct() elif v == ExecutionTimeframe.daily_between_times: sub_model = DailyBetweenTimesModel.construct() - else: # v == ExecutionTimeframe.from_date_to_date + elif v == ExecutionTimeframe.from_date_to_date: sub_model = FromDateToDateModel.construct() + else: # isinstance(v, Dict) + sub_model = v return sub_model @validator("order_refresh_tolerance_pct", pre=True) @@ -439,8 +441,10 @@ def validate_order_levels_mode(cls, v: Union[str, SingleOrderLevelModel, MultiOr sub_model = v elif v == OrderLevelsMode.single_order_level: sub_model = SingleOrderLevelModel.construct() - else: # v == OrderLevelsMode.multi_order_level + elif v == OrderLevelsMode.multi_order_level: sub_model = MultiOrderLevelModel.construct() + else: # isinstance(v, Dict) + sub_model = v return sub_model @validator("hanging_orders_mode", pre=True) @@ -453,8 +457,10 @@ def validate_hanging_orders_mode(cls, v: Union[str, TrackHangingOrdersModel, Ign sub_model = v elif v == HangingOrdersMode.track_hanging_orders: sub_model = TrackHangingOrdersModel.construct() - else: # v == HangingOrdersMode.ignore_hanging_orders + elif v == HangingOrdersMode.ignore_hanging_orders: sub_model = IgnoreHangingOrdersModel.construct() + else: # isinstance(v, Dict) + sub_model = v return sub_model # === generic validations === From dca2f69881a6c98c550b32d86f127fb790c31557 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Fri, 18 Feb 2022 10:32:27 +0700 Subject: [PATCH 003/152] (cleanup) Edited misleading comment --- .../test_avellaneda_market_making_config_map_pydantic.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py index 46dc98051b..cc6a034d0d 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py @@ -64,21 +64,15 @@ def test_initial_sequential_build(self): config_settings = self.get_default_map() def build_config_map(cm: BaseClientModel, cs: Dict): - """ - This routine can be used as is for the create command, - except for the sectioned-off portion. - """ + """This routine can be used in the create command, with slight modifications.""" for key, field in cm.__fields__.items(): client_data = cm.get_client_data(key) if client_data is not None and client_data.prompt_on_new: self.assertIsInstance(client_data.prompt(cm), str) - # ===================================================== if key == "execution_timeframe_model": cm.__setattr__(key, "daily_between_times") # simulate user input else: cm.__setattr__(key, cs[key]) - # cm.__setattr__(key, cs[key]) # use this in the create command routine - # ===================================================== new_value = cm.__getattribute__(key) if isinstance(new_value, BaseClientModel): build_config_map(new_value, cs[key]) From e795c3016febc010d16d5df49d6d57a4752b8906 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Tue, 22 Feb 2022 11:32:53 +0700 Subject: [PATCH 004/152] Update hummingbot/client/config/config_data_types.py Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- hummingbot/client/config/config_data_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/client/config/config_data_types.py b/hummingbot/client/config/config_data_types.py index 593c09b9a1..c12d484696 100644 --- a/hummingbot/client/config/config_data_types.py +++ b/hummingbot/client/config/config_data_types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass from enum import Enum -from typing import Any, Optional, Callable +from typing import Any, Callable, Optional from pydantic import BaseModel from pydantic.schema import default_ref_template From 2913ded11d3790aef9248398680ed24e147185b2 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Tue, 22 Feb 2022 13:02:28 +0700 Subject: [PATCH 005/152] (fix) Addresses @aarmoa's PR comments --- hummingbot/client/config/config_helpers.py | 2 +- ...aneda_market_making_config_map_pydantic.py | 112 +++++++++--------- setup/environment-linux-aarch64.yml | 2 +- setup/environment-linux.yml | 2 +- setup/environment-win64.yml | 2 +- setup/environment.yml | 2 +- ...aneda_market_making_config_map_pydantic.py | 5 +- 7 files changed, 60 insertions(+), 67 deletions(-) diff --git a/hummingbot/client/config/config_helpers.py b/hummingbot/client/config/config_helpers.py index e7f8cef925..ecc6d8a0aa 100644 --- a/hummingbot/client/config/config_helpers.py +++ b/hummingbot/client/config/config_helpers.py @@ -18,13 +18,13 @@ from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.security import Security from hummingbot.client.settings import ( + AllConnectorSettings, CONF_FILE_PATH, CONF_POSTFIX, CONF_PREFIX, GLOBAL_CONFIG_PATH, TEMPLATE_PATH, TRADE_FEES_CONFIG_PATH, - AllConnectorSettings ) # Use ruamel.yaml to preserve order and comments in .yml file diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py index b346e9c19e..0d64d66d3f 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -18,15 +18,9 @@ from hummingbot.connector.utils import split_hb_trading_pair -class ExecutionTimeframe(str, ClientConfigEnum): - infinite = "infinite" - from_date_to_date = "from_date_to_date" - daily_between_times = "daily_between_times" - - class InfiniteModel(BaseClientModel): class Config: - title = ExecutionTimeframe.infinite + title = "infinite" validate_assignment = True @@ -49,7 +43,7 @@ class FromDateToDateModel(BaseClientModel): ) class Config: - title = ExecutionTimeframe.from_date_to_date + title = "from_date_to_date" validate_assignment = True @validator("start_datetime", "end_datetime", pre=True) @@ -79,7 +73,7 @@ class DailyBetweenTimesModel(BaseClientModel): ) class Config: - title = ExecutionTimeframe.daily_between_times + title = "daily_between_times" validate_assignment = True @validator("start_time", "end_time", pre=True) @@ -90,14 +84,16 @@ def validate_execution_time(cls, v: str) -> Optional[str]: return v -class OrderLevelsMode(str, ClientConfigEnum): - single_order_level = "single_order_level" - multi_order_level = "multi_order_level" +EXECUTION_TIMEFRAME_MODELS = { + InfiniteModel.Config.title: InfiniteModel, + FromDateToDateModel.Config.title: FromDateToDateModel, + DailyBetweenTimesModel.Config.title: DailyBetweenTimesModel, +} class SingleOrderLevelModel(BaseClientModel): class Config: - title = OrderLevelsMode.single_order_level + title = "single_order_level" validate_assignment = True @@ -120,7 +116,7 @@ class MultiOrderLevelModel(BaseClientModel): ) class Config: - title = OrderLevelsMode.multi_order_level + title = "multi_order_level" validate_assignment = True @validator("order_levels", pre=True) @@ -138,9 +134,10 @@ def validate_decimal_zero_or_above(cls, v: str): return v -class HangingOrdersMode(str, ClientConfigEnum): - track_hanging_orders = "track_hanging_orders" - ignore_hanging_orders = "ignore_hanging_orders" +ORDER_LEVEL_MODELS = { + SingleOrderLevelModel.Config.title: SingleOrderLevelModel, + MultiOrderLevelModel.Config.title: MultiOrderLevelModel, +} class TrackHangingOrdersModel(BaseClientModel): @@ -158,7 +155,7 @@ class TrackHangingOrdersModel(BaseClientModel): ) class Config: - title = HangingOrdersMode.track_hanging_orders + title = "track_hanging_orders" validate_assignment = True @validator("hanging_orders_cancel_pct", pre=True) @@ -171,10 +168,16 @@ def validate_pct_exclusive(cls, v: str): class IgnoreHangingOrdersModel(BaseClientModel): class Config: - title = HangingOrdersMode.ignore_hanging_orders + title = "ignore_hanging_orders" validate_assignment = True +HANGING_ORDER_MODELS = { + TrackHangingOrdersModel.Config.title: TrackHangingOrdersModel, + IgnoreHangingOrdersModel.Config.title: IgnoreHangingOrdersModel, +} + + def maker_trading_pair_prompt(model_instance: 'AvellanedaMarketMakingConfigMap') -> str: exchange = model_instance.exchange example = AllConnectorSettings.get_example_pairs().get(exchange) @@ -215,7 +218,7 @@ class AvellanedaMarketMakingConfigMap(BaseClientModel): default=..., description="The execution timeframe.", client_data=ClientFieldData( - prompt=lambda mi: f"Select the execution timeframe ({'/'.join(ExecutionTimeframe)})", + prompt=lambda mi: f"Select the execution timeframe ({'/'.join(EXECUTION_TIMEFRAME_MODELS.keys())})", prompt_on_new=True, ), ) @@ -345,7 +348,7 @@ class AvellanedaMarketMakingConfigMap(BaseClientModel): default=SingleOrderLevelModel.construct(), description="Allows activating multi-order levels.", client_data=ClientFieldData( - prompt=lambda mi: f"Select the order levels mode ({'/'.join(OrderLevelsMode)}", + prompt=lambda mi: f"Select the order levels mode ({'/'.join(list(ORDER_LEVEL_MODELS.keys()))}", ), ) order_override: Optional[Dict] = Field( @@ -357,7 +360,9 @@ class AvellanedaMarketMakingConfigMap(BaseClientModel): default=IgnoreHangingOrdersModel.construct(), description="When tracking hanging orders, the orders on the side opposite to the filled orders remain active.", client_data=ClientFieldData( - prompt=lambda mi: f"How do you want to handle hanging orders? ({'/'.join(HangingOrdersMode)})", + prompt=( + lambda mi: f"How do you want to handle hanging orders? ({'/'.join(list(HANGING_ORDER_MODELS.keys()))})" + ), ), ) should_wait_order_cancel_confirmation: bool = Field( @@ -375,10 +380,6 @@ class AvellanedaMarketMakingConfigMap(BaseClientModel): ) ) - def __init__(self, **data): - super().__init__(**data) - required_exchanges.append(self.exchange) - class Config: validate_assignment = True @@ -401,20 +402,14 @@ def validate_exchange_trading_pair(cls, v: str, values: Dict): def validate_execution_timeframe( cls, v: Union[str, InfiniteModel, FromDateToDateModel, DailyBetweenTimesModel] ): - if not isinstance(v, Dict) and not hasattr(ExecutionTimeframe, v): + if isinstance(v, (InfiniteModel, FromDateToDateModel, DailyBetweenTimesModel, Dict)): + sub_model = v + elif not isinstance(v, Dict) and v not in EXECUTION_TIMEFRAME_MODELS: raise ValueError( - f"Invalid timeframe, please choose value from {[e.value for e in list(ExecutionTimeframe)]}" + f"Invalid timeframe, please choose value from {list(EXECUTION_TIMEFRAME_MODELS.keys())}" ) - elif isinstance(v, (InfiniteModel, FromDateToDateModel, DailyBetweenTimesModel)): - sub_model = v - elif v == ExecutionTimeframe.infinite: - sub_model = InfiniteModel.construct() - elif v == ExecutionTimeframe.daily_between_times: - sub_model = DailyBetweenTimesModel.construct() - elif v == ExecutionTimeframe.from_date_to_date: - sub_model = FromDateToDateModel.construct() - else: # isinstance(v, Dict) - sub_model = v + else: + sub_model = EXECUTION_TIMEFRAME_MODELS[v].construct() return sub_model @validator("order_refresh_tolerance_pct", pre=True) @@ -433,34 +428,28 @@ def validate_buffer_size(cls, v: str): @validator("order_levels_mode", pre=True) def validate_order_levels_mode(cls, v: Union[str, SingleOrderLevelModel, MultiOrderLevelModel]): - if not isinstance(v, Dict) and not hasattr(OrderLevelsMode, v): + if isinstance(v, (SingleOrderLevelModel, MultiOrderLevelModel, Dict)): + sub_model = v + elif not isinstance(v, Dict) and v not in ORDER_LEVEL_MODELS: raise ValueError( - f"Invalid order levels mode, please choose value from {[e.value for e in list(OrderLevelsMode)]}." + f"Invalid order levels mode, please choose value from" + f" {[e.value for e in list(ORDER_LEVEL_MODELS.keys())]}." ) - elif isinstance(v, (SingleOrderLevelModel, MultiOrderLevelModel)): - sub_model = v - elif v == OrderLevelsMode.single_order_level: - sub_model = SingleOrderLevelModel.construct() - elif v == OrderLevelsMode.multi_order_level: - sub_model = MultiOrderLevelModel.construct() - else: # isinstance(v, Dict) - sub_model = v + else: + sub_model = ORDER_LEVEL_MODELS[v].construct() return sub_model @validator("hanging_orders_mode", pre=True) def validate_hanging_orders_mode(cls, v: Union[str, TrackHangingOrdersModel, IgnoreHangingOrdersModel]): - if not isinstance(v, Dict) and not hasattr(HangingOrdersMode, v): + if isinstance(v, (TrackHangingOrdersModel, IgnoreHangingOrdersModel, Dict)): + sub_model = v + elif not isinstance(v, Dict) and v not in HANGING_ORDER_MODELS: raise ValueError( - f"Invalid hanging order mode, please choose value from {[e.value for e in list(HangingOrdersMode)]}." + f"Invalid hanging order mode, please choose value from" + f" {[e.value for e in list(HANGING_ORDER_MODELS.keys())]}." ) - elif isinstance(v, (TrackHangingOrdersModel, IgnoreHangingOrdersModel)): - sub_model = v - elif v == HangingOrdersMode.track_hanging_orders: - sub_model = TrackHangingOrdersModel.construct() - elif v == HangingOrdersMode.ignore_hanging_orders: - sub_model = IgnoreHangingOrdersModel.construct() - else: # isinstance(v, Dict) - sub_model = v + else: + sub_model = HANGING_ORDER_MODELS[v].construct() return sub_model # === generic validations === @@ -518,12 +507,17 @@ def validate_pct_inclusive(cls, v: str): @root_validator() def post_validations(cls, values: Dict): cls.execution_timeframe_post_validation(values) + cls.exchange_post_validation(values) return values @classmethod def execution_timeframe_post_validation(cls, values: Dict): execution_timeframe = values.get("execution_timeframe") - if execution_timeframe is not None and execution_timeframe == ExecutionTimeframe.infinite: + if execution_timeframe is not None and execution_timeframe == InfiniteModel.Config.title: values["start_time"] = None values["end_time"] = None return values + + @classmethod + def exchange_post_validation(cls, values: Dict): + required_exchanges.append(values["exchange"]) diff --git a/setup/environment-linux-aarch64.yml b/setup/environment-linux-aarch64.yml index 2eae603363..c528d735b9 100644 --- a/setup/environment-linux-aarch64.yml +++ b/setup/environment-linux-aarch64.yml @@ -14,7 +14,7 @@ dependencies: - pandas=1.2.1 - pip=21.1.2 - prompt_toolkit=3.0.20 - - pydantic=1.9.0 + - pydantic=1.9 - pytables=3.6.1 - python=3.8.2 - python-dateutil=2.8.1 diff --git a/setup/environment-linux.yml b/setup/environment-linux.yml index 7673732228..878024dde4 100644 --- a/setup/environment-linux.yml +++ b/setup/environment-linux.yml @@ -14,7 +14,7 @@ dependencies: - pandas=1.2.1 - pip=21.1.2 - prompt_toolkit=3.0.20 - - pydantic=1.9.0 + - pydantic=1.9 - pytables=3.6.1 - python=3.8.2 - python-dateutil=2.8.1 diff --git a/setup/environment-win64.yml b/setup/environment-win64.yml index 2243ab1704..d1c5b22fb4 100644 --- a/setup/environment-win64.yml +++ b/setup/environment-win64.yml @@ -12,7 +12,7 @@ dependencies: - pandas=1.2.1 - pip=21.1.2 - prompt_toolkit=3.0.20 - - pydantic=1.9.0 + - pydantic=1.9 - pytables=3.6.1 - python=3.8.2 - python-dateutil=2.8.1 diff --git a/setup/environment.yml b/setup/environment.yml index 8065195169..194289c007 100644 --- a/setup/environment.yml +++ b/setup/environment.yml @@ -15,7 +15,7 @@ dependencies: - pandas=1.2.1 - pip=21.1.2 - prompt_toolkit=3.0.20 - - pydantic=1.9.0 + - pydantic=1.9 - pytables=3.6.1 - python=3.8.2 - python-dateutil=2.8.1 diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py index cc6a034d0d..64fec3dadd 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py @@ -14,7 +14,6 @@ from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( AvellanedaMarketMakingConfigMap, DailyBetweenTimesModel, - ExecutionTimeframe, FromDateToDateModel, InfiniteModel, ) @@ -96,7 +95,7 @@ def test_maker_trading_pair_prompt(self): self.assertEqual(expected, prompt) def test_execution_time_prompts(self): - self.config_map.execution_timeframe_model = ExecutionTimeframe.from_date_to_date + self.config_map.execution_timeframe_model = FromDateToDateModel.Config.title model = self.config_map.execution_timeframe_model prompt = model.get_client_prompt("start_datetime") expected = "Please enter the start date and time (YYYY-MM-DD HH:MM:SS)" @@ -105,7 +104,7 @@ def test_execution_time_prompts(self): expected = "Please enter the end date and time (YYYY-MM-DD HH:MM:SS)" self.assertEqual(expected, prompt) - self.config_map.execution_timeframe_model = ExecutionTimeframe.daily_between_times + self.config_map.execution_timeframe_model = DailyBetweenTimesModel.Config.title model = self.config_map.execution_timeframe_model prompt = model.get_client_prompt("start_time") expected = "Please enter the start time (HH:MM:SS)" From ad111a7e32d6c638d359b3b0a1c39467028b6925 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Tue, 22 Feb 2022 13:10:12 +0700 Subject: [PATCH 006/152] (cleanup) Adds clarification comments to client-specific validations --- .../avellaneda_market_making_config_map_pydantic.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py index 0d64d66d3f..2ed36983de 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -385,6 +385,7 @@ class Config: @validator("exchange", pre=True) def validate_exchange(cls, v: str): + """Used for client-friendly error output.""" ret = validate_exchange(v) if ret is not None: raise ValueError(ret) @@ -414,6 +415,7 @@ def validate_execution_timeframe( @validator("order_refresh_tolerance_pct", pre=True) def validate_order_refresh_tolerance_pct(cls, v: str): + """Used for client-friendly error output.""" ret = validate_decimal(v, min_value=Decimal("-10"), max_value=Decimal("10"), inclusive=True) if ret is not None: raise ValueError(ret) @@ -421,6 +423,7 @@ def validate_order_refresh_tolerance_pct(cls, v: str): @validator("volatility_buffer_size", "trading_intensity_buffer_size", pre=True) def validate_buffer_size(cls, v: str): + """Used for client-friendly error output.""" ret = validate_int(v, 1, 10_000) if ret is not None: raise ValueError(ret) @@ -461,6 +464,7 @@ def validate_hanging_orders_mode(cls, v: Union[str, TrackHangingOrdersModel, Ign pre=True, ) def validate_bool(cls, v: str): + """Used for client-friendly error output.""" if isinstance(v, str): ret = validate_bool(v) if ret is not None: @@ -469,6 +473,7 @@ def validate_bool(cls, v: str): @validator("order_amount_shape_factor", pre=True) def validate_decimal_from_zero_to_one(cls, v: str): + """Used for client-friendly error output.""" ret = validate_decimal(v, min_value=Decimal("0"), max_value=Decimal("1"), inclusive=True) if ret is not None: raise ValueError(ret) @@ -483,6 +488,7 @@ def validate_decimal_from_zero_to_one(cls, v: str): pre=True, ) def validate_decimal_above_zero(cls, v: str): + """Used for client-friendly error output.""" ret = validate_decimal(v, min_value=Decimal("0"), inclusive=False) if ret is not None: raise ValueError(ret) @@ -490,6 +496,7 @@ def validate_decimal_above_zero(cls, v: str): @validator("min_spread", pre=True) def validate_decimal_zero_or_above(cls, v: str): + """Used for client-friendly error output.""" ret = validate_decimal(v, min_value=Decimal("0"), inclusive=True) if ret is not None: raise ValueError(ret) @@ -497,6 +504,7 @@ def validate_decimal_zero_or_above(cls, v: str): @validator("inventory_target_base_pct", pre=True) def validate_pct_inclusive(cls, v: str): + """Used for client-friendly error output.""" ret = validate_decimal(v, min_value=Decimal("0"), max_value=Decimal("100"), inclusive=True) if ret is not None: raise ValueError(ret) From f85e949c95ed239be4843a0afa1e252c462650d9 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 23 Feb 2022 10:02:09 +0700 Subject: [PATCH 007/152] Update test/hummingbot/strategy/avellaneda_market_making/test_config.yml Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- .../strategy/avellaneda_market_making/test_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_config.yml b/test/hummingbot/strategy/avellaneda_market_making/test_config.yml index cbf4e2b487..c7c7cf1d39 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_config.yml +++ b/test/hummingbot/strategy/avellaneda_market_making/test_config.yml @@ -7,4 +7,4 @@ order_amount: 10 order_optimization_enabled: true risk_factor: 0.5 order_refresh_time: 60 -inventory_target_base_pct: 50 \ No newline at end of file +inventory_target_base_pct: 50 From 65b0bd75a0adb096bed3065fc844796155a1cdc2 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 23 Feb 2022 10:10:30 +0700 Subject: [PATCH 008/152] (cleanup) Makes sub-model validators a bit more precise. --- .../avellaneda_market_making_config_map_pydantic.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py index 2ed36983de..c20014d8f4 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -405,7 +405,7 @@ def validate_execution_timeframe( ): if isinstance(v, (InfiniteModel, FromDateToDateModel, DailyBetweenTimesModel, Dict)): sub_model = v - elif not isinstance(v, Dict) and v not in EXECUTION_TIMEFRAME_MODELS: + elif v not in EXECUTION_TIMEFRAME_MODELS: raise ValueError( f"Invalid timeframe, please choose value from {list(EXECUTION_TIMEFRAME_MODELS.keys())}" ) @@ -433,7 +433,7 @@ def validate_buffer_size(cls, v: str): def validate_order_levels_mode(cls, v: Union[str, SingleOrderLevelModel, MultiOrderLevelModel]): if isinstance(v, (SingleOrderLevelModel, MultiOrderLevelModel, Dict)): sub_model = v - elif not isinstance(v, Dict) and v not in ORDER_LEVEL_MODELS: + elif v not in ORDER_LEVEL_MODELS: raise ValueError( f"Invalid order levels mode, please choose value from" f" {[e.value for e in list(ORDER_LEVEL_MODELS.keys())]}." @@ -446,7 +446,7 @@ def validate_order_levels_mode(cls, v: Union[str, SingleOrderLevelModel, MultiOr def validate_hanging_orders_mode(cls, v: Union[str, TrackHangingOrdersModel, IgnoreHangingOrdersModel]): if isinstance(v, (TrackHangingOrdersModel, IgnoreHangingOrdersModel, Dict)): sub_model = v - elif not isinstance(v, Dict) and v not in HANGING_ORDER_MODELS: + elif v not in HANGING_ORDER_MODELS: raise ValueError( f"Invalid hanging order mode, please choose value from" f" {[e.value for e in list(HANGING_ORDER_MODELS.keys())]}." From e652242e1a9dea6758beca08c4e00eb4be6c5be0 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Thu, 24 Feb 2022 08:21:26 +0700 Subject: [PATCH 009/152] (cleanup) Adds the last prompt functions as class methods to the config class. --- ...aneda_market_making_config_map_pydantic.py | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py index c20014d8f4..ef04712821 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -178,20 +178,6 @@ class Config: } -def maker_trading_pair_prompt(model_instance: 'AvellanedaMarketMakingConfigMap') -> str: - exchange = model_instance.exchange - example = AllConnectorSettings.get_example_pairs().get(exchange) - return ( - f"Enter the token trading pair you would like to trade on {exchange}{f' (e.g. {example})' if example else ''}" - ) - - -def order_amount_prompt(model_instance: 'AvellanedaMarketMakingConfigMap') -> str: - trading_pair = model_instance.market - base_asset, quote_asset = split_hb_trading_pair(trading_pair) - return f"What is the amount of {base_asset} per order?" - - class AvellanedaMarketMakingConfigMap(BaseClientModel): strategy: str = Field(default="avellaneda_market_making", client_data=None) exchange: ClientConfigEnum( @@ -210,7 +196,7 @@ class AvellanedaMarketMakingConfigMap(BaseClientModel): default=..., description="The trading pair.", client_data=ClientFieldData( - prompt=maker_trading_pair_prompt, + prompt=lambda mi: AvellanedaMarketMakingConfigMap.maker_trading_pair_prompt(mi), prompt_on_new=True, ), ) @@ -226,7 +212,7 @@ class AvellanedaMarketMakingConfigMap(BaseClientModel): default=..., description="The strategy order amount.", gt=0, - client_data=ClientFieldData(prompt=order_amount_prompt) + client_data=ClientFieldData(prompt=lambda mi: AvellanedaMarketMakingConfigMap.order_amount_prompt(mi)) ) order_optimization_enabled: bool = Field( default=True, @@ -383,6 +369,21 @@ class AvellanedaMarketMakingConfigMap(BaseClientModel): class Config: validate_assignment = True + @classmethod + def maker_trading_pair_prompt(cls, model_instance: 'AvellanedaMarketMakingConfigMap') -> str: + exchange = model_instance.exchange + example = AllConnectorSettings.get_example_pairs().get(exchange) + return ( + f"Enter the token trading pair you would like to trade on" + f" {exchange}{f' (e.g. {example})' if example else ''}" + ) + + @classmethod + def order_amount_prompt(cls, model_instance: 'AvellanedaMarketMakingConfigMap') -> str: + trading_pair = model_instance.market + base_asset, quote_asset = split_hb_trading_pair(trading_pair) + return f"What is the amount of {base_asset} per order?" + @validator("exchange", pre=True) def validate_exchange(cls, v: str): """Used for client-friendly error output.""" From 832153cae420db72840629177eef47af99aa5759 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Fri, 18 Feb 2022 10:16:56 +0700 Subject: [PATCH 010/152] (refactor) Adapts Avellaneda config map for pydantic. The config map introduced in this PR is a proof of concept for the new approach to configs using pydantic. It adopts @aarmoa's suggestion to use nested models as a method of solving the interdependencies issues present with some config variables. --- hummingbot/client/config/config_data_types.py | 47 ++ hummingbot/client/config/config_helpers.py | 54 +- ...aneda_market_making_config_map_pydantic.py | 523 ++++++++++++++++++ setup.py | 1 + setup/environment-linux-aarch64.yml | 1 + setup/environment-linux.yml | 1 + setup/environment-win64.yml | 1 + setup/environment.yml | 1 + ...aneda_market_making_config_map_pydantic.py | 246 ++++++++ .../avellaneda_market_making/test_config.yml | 10 + 10 files changed, 859 insertions(+), 26 deletions(-) create mode 100644 hummingbot/client/config/config_data_types.py create mode 100644 hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py create mode 100644 test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py create mode 100644 test/hummingbot/strategy/avellaneda_market_making/test_config.yml diff --git a/hummingbot/client/config/config_data_types.py b/hummingbot/client/config/config_data_types.py new file mode 100644 index 0000000000..593c09b9a1 --- /dev/null +++ b/hummingbot/client/config/config_data_types.py @@ -0,0 +1,47 @@ +from dataclasses import dataclass +from enum import Enum +from typing import Any, Optional, Callable + +from pydantic import BaseModel +from pydantic.schema import default_ref_template + +from hummingbot.client.config.config_helpers import strategy_config_schema_encoder + + +class ClientConfigEnum(Enum): + def __str__(self): + return self.value + + +@dataclass() +class ClientFieldData: + prompt: Optional[Callable[['BaseClientModel'], str]] = None + prompt_on_new: bool = False + + +class BaseClientModel(BaseModel): + """ + Notes on configs: + - In nested models, be weary that pydantic will take the first model that fits + (see https://pydantic-docs.helpmanual.io/usage/model_config/#smart-union). + """ + @classmethod + def schema_json( + cls, *, by_alias: bool = True, ref_template: str = default_ref_template, **dumps_kwargs: Any + ) -> str: + # todo: make it ignore `client_data` all together + return cls.__config__.json_dumps( + cls.schema(by_alias=by_alias, ref_template=ref_template), + default=strategy_config_schema_encoder, + **dumps_kwargs + ) + + def get_client_prompt(self, attr_name: str) -> Optional[str]: + prompt = None + client_data = self.get_client_data(attr_name) + if client_data is not None: + prompt = client_data.prompt(self) + return prompt + + def get_client_data(self, attr_name: str) -> Optional[ClientFieldData]: + return self.__fields__[attr_name].field_info.extra["client_data"] diff --git a/hummingbot/client/config/config_helpers.py b/hummingbot/client/config/config_helpers.py index 113f105cc0..e7f8cef925 100644 --- a/hummingbot/client/config/config_helpers.py +++ b/hummingbot/client/config/config_helpers.py @@ -1,40 +1,31 @@ +import json import logging +import shutil +from collections import OrderedDict from decimal import Decimal +from os import listdir, unlink +from os.path import isfile, join +from typing import Any, Callable, Dict, List, Optional + import ruamel.yaml -from os import ( - unlink -) -from os.path import ( - join, - isfile -) -from collections import OrderedDict -import json -from typing import ( - Any, - Callable, - Dict, - List, - Optional, -) -from os import listdir -import shutil +from eth_account import Account +from pydantic import ValidationError +from pydantic.json import pydantic_encoder +from hummingbot import get_strategy_list from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.fee_overrides_config_map import fee_overrides_config_map +from hummingbot.client.config.global_config_map import global_config_map +from hummingbot.client.config.security import Security from hummingbot.client.settings import ( - GLOBAL_CONFIG_PATH, - TRADE_FEES_CONFIG_PATH, - TEMPLATE_PATH, CONF_FILE_PATH, CONF_POSTFIX, CONF_PREFIX, - AllConnectorSettings, + GLOBAL_CONFIG_PATH, + TEMPLATE_PATH, + TRADE_FEES_CONFIG_PATH, + AllConnectorSettings ) -from hummingbot.client.config.security import Security -from hummingbot import get_strategy_list -from eth_account import Account # Use ruamel.yaml to preserve order and comments in .yml file yaml_parser = ruamel.yaml.YAML() @@ -456,3 +447,14 @@ def secondary_market_conversion_rate(strategy) -> Decimal: else: return Decimal("1") return quote_rate / base_rate + + +def retrieve_validation_error_msg(e: ValidationError) -> str: + return e.errors().pop()["msg"] + + +def strategy_config_schema_encoder(o): + if callable(o): + return None + else: + return pydantic_encoder(o) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py new file mode 100644 index 0000000000..513d9d450d --- /dev/null +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -0,0 +1,523 @@ +from datetime import datetime, time +from decimal import Decimal +from typing import Dict, Optional, Union + +from pydantic import Field, validator, root_validator + +from hummingbot.client.config.config_data_types import BaseClientModel, ClientConfigEnum, ClientFieldData +from hummingbot.client.config.config_validators import ( + validate_bool, + validate_datetime_iso_string, + validate_decimal, + validate_exchange, + validate_int, + validate_market_trading_pair, + validate_time_iso_string, +) +from hummingbot.client.settings import AllConnectorSettings, required_exchanges +from hummingbot.connector.utils import split_hb_trading_pair + + +class ExecutionTimeframe(str, ClientConfigEnum): + infinite = "infinite" + from_date_to_date = "from_date_to_date" + daily_between_times = "daily_between_times" + + +class InfiniteModel(BaseClientModel): + class Config: + title = ExecutionTimeframe.infinite + validate_assignment = True + + +class FromDateToDateModel(BaseClientModel): + start_datetime: datetime = Field( + default=..., + description="The start date and time for date-to-date execution timeframe.", + client_data=ClientFieldData( + prompt=lambda mi: "Please enter the start date and time (YYYY-MM-DD HH:MM:SS)", + prompt_on_new=True, + ), + ) + end_datetime: datetime = Field( + default=..., + description="The end date and time for date-to-date execution timeframe.", + client_data=ClientFieldData( + prompt=lambda mi: "Please enter the end date and time (YYYY-MM-DD HH:MM:SS)", + prompt_on_new=True, + ), + ) + + class Config: + title = ExecutionTimeframe.from_date_to_date + validate_assignment = True + + @validator("start_datetime", "end_datetime", pre=True) + def validate_execution_time(cls, v: str) -> Optional[str]: + ret = validate_datetime_iso_string(v) + if ret is not None: + raise ValueError(ret) + return v + + +class DailyBetweenTimesModel(BaseClientModel): + start_time: time = Field( + default=..., + description="The start time for daily-between-times execution timeframe.", + client_data=ClientFieldData( + prompt=lambda mi: "Please enter the start time (HH:MM:SS)", + prompt_on_new=True, + ), + ) + end_time: time = Field( + default=..., + description="The end time for daily-between-times execution timeframe.", + client_data=ClientFieldData( + prompt=lambda mi: "Please enter the end time (HH:MM:SS)", + prompt_on_new=True, + ), + ) + + class Config: + title = ExecutionTimeframe.daily_between_times + validate_assignment = True + + @validator("start_time", "end_time", pre=True) + def validate_execution_time(cls, v: str) -> Optional[str]: + ret = validate_time_iso_string(v) + if ret is not None: + raise ValueError(ret) + return v + + +class OrderLevelsMode(str, ClientConfigEnum): + single_order_level = "single_order_level" + multi_order_level = "multi_order_level" + + +class SingleOrderLevelModel(BaseClientModel): + class Config: + title = OrderLevelsMode.single_order_level + validate_assignment = True + + +class MultiOrderLevelModel(BaseClientModel): + order_levels: int = Field( + default=2, + description="The number of orders placed on either side of the order book.", + ge=2, + client_data=ClientFieldData( + prompt=lambda mi: "How many orders do you want to place on both sides?" + ), + ) + level_distances: Decimal = Field( + default=Decimal("0"), + description="The spread between order levels, expressed in % of optimal spread.", + ge=0, + client_data=ClientFieldData( + prompt=lambda mi: "How far apart in % of optimal spread should orders on one side be?" + ), + ) + + class Config: + title = OrderLevelsMode.multi_order_level + validate_assignment = True + + @validator("order_levels", pre=True) + def validate_int_zero_or_above(cls, v: str): + ret = validate_int(v, min_value=2) + if ret is not None: + raise ValueError(ret) + return v + + @validator("level_distances", pre=True) + def validate_decimal_zero_or_above(cls, v: str): + ret = validate_decimal(v, min_value=Decimal("0"), inclusive=True) + if ret is not None: + raise ValueError(ret) + return v + + +class HangingOrdersMode(str, ClientConfigEnum): + track_hanging_orders = "track_hanging_orders" + ignore_hanging_orders = "ignore_hanging_orders" + + +class TrackHangingOrdersModel(BaseClientModel): + hanging_orders_cancel_pct: Decimal = Field( + default=Decimal("10"), + description="The spread percentage at which hanging orders will be cancelled.", + gt=0, + lt=100, + client_data=ClientFieldData( + prompt=lambda mi: ( + "At what spread percentage (from mid price) will hanging orders be canceled?" + " (Enter 1 to indicate 1%)" + ), + ) + ) + + class Config: + title = HangingOrdersMode.track_hanging_orders + validate_assignment = True + + @validator("hanging_orders_cancel_pct", pre=True) + def validate_pct_exclusive(cls, v: str): + ret = validate_decimal(v, min_value=Decimal("0"), max_value=Decimal("100"), inclusive=False) + if ret is not None: + raise ValueError(ret) + return v + + +class IgnoreHangingOrdersModel(BaseClientModel): + class Config: + title = HangingOrdersMode.ignore_hanging_orders + validate_assignment = True + + +def maker_trading_pair_prompt(model_instance: 'AvellanedaMarketMakingConfigMap') -> str: + exchange = model_instance.exchange + example = AllConnectorSettings.get_example_pairs().get(exchange) + return ( + f"Enter the token trading pair you would like to trade on {exchange}{f' (e.g. {example})' if example else ''}" + ) + + +def order_amount_prompt(model_instance: 'AvellanedaMarketMakingConfigMap') -> str: + trading_pair = model_instance.market + base_asset, quote_asset = split_hb_trading_pair(trading_pair) + return f"What is the amount of {base_asset} per order?" + + +class AvellanedaMarketMakingConfigMap(BaseClientModel): + strategy: str = Field(default="avellaneda_market_making", client_data=None) + exchange: ClientConfigEnum( + value="Exchanges", # noqa: F821 + names={e: e for e in AllConnectorSettings.get_connector_settings().keys()}, + type=str, + ) = Field( + default=..., + description="The name of the exchange connector.", + client_data=ClientFieldData( + prompt=lambda mi: "Input your maker spot connector", + prompt_on_new=True, + ), + ) + market: str = Field( + default=..., + description="The trading pair.", + client_data=ClientFieldData( + prompt=maker_trading_pair_prompt, + prompt_on_new=True, + ), + ) + execution_timeframe_model: Union[FromDateToDateModel, DailyBetweenTimesModel, InfiniteModel] = Field( + default=..., + description="The execution timeframe.", + client_data=ClientFieldData( + prompt=lambda mi: f"Select the execution timeframe ({'/'.join(ExecutionTimeframe)})", + prompt_on_new=True, + ), + ) + order_amount: Decimal = Field( + default=..., + description="The strategy order amount.", + gt=0, + client_data=ClientFieldData(prompt=order_amount_prompt) + ) + order_optimization_enabled: bool = Field( + default=True, + description=( + "Allows the bid and ask order prices to be adjusted based on" + " the current top bid and ask prices in the market." + ), + client_data=ClientFieldData( + prompt=lambda mi: "Do you want to enable best bid ask jumping? (Yes/No)" + ), + ) + risk_factor: Decimal = Field( + default=Decimal("1"), + description="The risk factor (\u03B3).", + gt=0, + client_data=ClientFieldData( + prompt=lambda mi: "Enter risk factor (\u03B3)", + prompt_on_new=True, + ), + ) + order_amount_shape_factor: Decimal = Field( + default=Decimal("0"), + description="The amount shape factor (\u03b7)", + ge=0, + le=1, + client_data=ClientFieldData( + prompt=lambda mi: "Enter order amount shape factor (\u03B7)", + ), + ) + min_spread: Decimal = Field( + default=Decimal("0"), + description="The minimum spread limit as percentage of the mid price.", + ge=0, + client_data=ClientFieldData( + prompt=lambda mi: "Enter minimum spread limit (as % of mid price)", + ), + ) + order_refresh_time: float = Field( + default=..., + description="The frequency at which the orders' spreads will be re-evaluated.", + gt=0., + client_data=ClientFieldData( + prompt=lambda mi: "How often do you want to cancel and replace bids and asks (in seconds)?", + prompt_on_new=True, + ), + ) + max_order_age: float = Field( + default=1800., + description="A given order's maximum lifetime irrespective of spread.", + gt=0., + client_data=ClientFieldData( + prompt=lambda mi: ( + "How long do you want to cancel and replace bids and asks with the same price (in seconds)?" + ), + ), + ) + order_refresh_tolerance_pct: Decimal = Field( + default=Decimal("0"), + description=( + "The range of spreads tolerated on refresh cycles." + " Orders over that range are cancelled and re-submitted." + ), + ge=-10, + le=10, + client_data=ClientFieldData( + prompt=lambda mi: ( + "Enter the percent change in price needed to refresh orders at each cycle" + " (Enter 1 to indicate 1%)" + ) + ), + ) + filled_order_delay: float = Field( + default=60., + description="The delay before placing a new order after an order fill.", + gt=0., + client_data=ClientFieldData( + prompt=lambda mi: ( + "How long do you want to wait before placing the next order" + " if your order gets filled (in seconds)?" + ) + ), + ) + inventory_target_base_pct: Decimal = Field( + default=Decimal("50"), + description="Defines the inventory target for the base asset.", + ge=0, + le=100, + client_data=ClientFieldData( + prompt=lambda mi: "What is the inventory target for the base asset? Enter 50 for 50%", + prompt_on_new=True, + ), + ) + add_transaction_costs: bool = Field( + default=False, + description="If activated, transaction costs will be added to order prices.", + client_data=ClientFieldData( + prompt=lambda mi: "Do you want to add transaction costs automatically to order prices? (Yes/No)", + ), + ) + volatility_buffer_size: int = Field( + default=200, + description="The number of ticks that will be stored to calculate volatility.", + ge=1, + le=10_000, + client_data=ClientFieldData( + prompt=lambda mi: "Enter amount of ticks that will be stored to estimate order book liquidity", + ), + ) + trading_intensity_buffer_size: int = Field( + default=200, + description="The number of ticks that will be stored to calculate order book liquidity.", + ge=1, + le=10_000, + client_data=ClientFieldData( + prompt=lambda mi: "Enter amount of ticks that will be stored to estimate order book liquidity", + ), + ) + order_levels_mode: Union[MultiOrderLevelModel, SingleOrderLevelModel] = Field( + default=SingleOrderLevelModel.construct(), + description="Allows activating multi-order levels.", + client_data=ClientFieldData( + prompt=lambda mi: f"Select the order levels mode ({'/'.join(OrderLevelsMode)}", + ), + ) + order_override: Optional[Dict] = Field( + default=None, + description="Allows custom specification of the order levels and their spreads and amounts.", + client_data=None, + ) + hanging_orders_mode: Union[TrackHangingOrdersModel, IgnoreHangingOrdersModel] = Field( + default=IgnoreHangingOrdersModel.construct(), + description="When tracking hanging orders, the orders on the side opposite to the filled orders remain active.", + client_data=ClientFieldData( + prompt=lambda mi: f"How do you want to handle hanging orders? ({'/'.join(HangingOrdersMode)})", + ), + ) + should_wait_order_cancel_confirmation: bool = Field( + default=True, + description=( + "If activated, the strategy will await cancellation confirmation from the exchange" + " before placing a new order." + ), + client_data=ClientFieldData( + prompt=lambda mi: ( + "Should the strategy wait to receive a confirmation for orders cancellation" + " before creating a new set of orders?" + " (Not waiting requires enough available balance) (Yes/No)" + ), + ) + ) + + def __init__(self, **data): + super().__init__(**data) + required_exchanges.append(self.exchange) + + class Config: + validate_assignment = True + + @validator("exchange", pre=True) + def validate_exchange(cls, v: str): + ret = validate_exchange(v) + if ret is not None: + raise ValueError(ret) + return v + + @validator("market", pre=True) + def validate_exchange_trading_pair(cls, v: str, values: Dict): + exchange = values.get("exchange") + ret = validate_market_trading_pair(exchange, v) + if ret is not None: + raise ValueError(ret) + return v + + @validator("execution_timeframe_model", pre=True) + def validate_execution_timeframe( + cls, v: Union[str, InfiniteModel, FromDateToDateModel, DailyBetweenTimesModel] + ): + if not isinstance(v, Dict) and not hasattr(ExecutionTimeframe, v): + raise ValueError( + f"Invalid timeframe, please choose value from {[e.value for e in list(ExecutionTimeframe)]}" + ) + elif isinstance(v, (InfiniteModel, FromDateToDateModel, DailyBetweenTimesModel)): + sub_model = v + elif v == ExecutionTimeframe.infinite: + sub_model = InfiniteModel.construct() + elif v == ExecutionTimeframe.daily_between_times: + sub_model = DailyBetweenTimesModel.construct() + else: # v == ExecutionTimeframe.from_date_to_date + sub_model = FromDateToDateModel.construct() + return sub_model + + @validator("order_refresh_tolerance_pct", pre=True) + def validate_order_refresh_tolerance_pct(cls, v: str): + ret = validate_decimal(v, min_value=Decimal("-10"), max_value=Decimal("10"), inclusive=True) + if ret is not None: + raise ValueError(ret) + return v + + @validator("volatility_buffer_size", "trading_intensity_buffer_size", pre=True) + def validate_buffer_size(cls, v: str): + ret = validate_int(v, 1, 10_000) + if ret is not None: + raise ValueError(ret) + return v + + @validator("order_levels_mode", pre=True) + def validate_order_levels_mode(cls, v: Union[str, SingleOrderLevelModel, MultiOrderLevelModel]): + if not isinstance(v, Dict) and not hasattr(OrderLevelsMode, v): + raise ValueError( + f"Invalid order levels mode, please choose value from {[e.value for e in list(OrderLevelsMode)]}." + ) + elif isinstance(v, (SingleOrderLevelModel, MultiOrderLevelModel)): + sub_model = v + elif v == OrderLevelsMode.single_order_level: + sub_model = SingleOrderLevelModel.construct() + else: # v == OrderLevelsMode.multi_order_level + sub_model = MultiOrderLevelModel.construct() + return sub_model + + @validator("hanging_orders_mode", pre=True) + def validate_hanging_orders_mode(cls, v: Union[str, TrackHangingOrdersModel, IgnoreHangingOrdersModel]): + if not isinstance(v, Dict) and not hasattr(HangingOrdersMode, v): + raise ValueError( + f"Invalid hanging order mode, please choose value from {[e.value for e in list(HangingOrdersMode)]}." + ) + elif isinstance(v, (TrackHangingOrdersModel, IgnoreHangingOrdersModel)): + sub_model = v + elif v == HangingOrdersMode.track_hanging_orders: + sub_model = TrackHangingOrdersModel.construct() + else: # v == HangingOrdersMode.ignore_hanging_orders + sub_model = IgnoreHangingOrdersModel.construct() + return sub_model + + # === generic validations === + + @validator( + "order_optimization_enabled", + "add_transaction_costs", + "should_wait_order_cancel_confirmation", + pre=True, + ) + def validate_bool(cls, v: str): + if isinstance(v, str): + ret = validate_bool(v) + if ret is not None: + raise ValueError(ret) + return v + + @validator("order_amount_shape_factor", pre=True) + def validate_decimal_from_zero_to_one(cls, v: str): + ret = validate_decimal(v, min_value=Decimal("0"), max_value=Decimal("1"), inclusive=True) + if ret is not None: + raise ValueError(ret) + return v + + @validator( + "order_amount", + "risk_factor", + "order_refresh_time", + "max_order_age", + "filled_order_delay", + pre=True, + ) + def validate_decimal_above_zero(cls, v: str): + ret = validate_decimal(v, min_value=Decimal("0"), inclusive=False) + if ret is not None: + raise ValueError(ret) + return v + + @validator("min_spread", pre=True) + def validate_decimal_zero_or_above(cls, v: str): + ret = validate_decimal(v, min_value=Decimal("0"), inclusive=True) + if ret is not None: + raise ValueError(ret) + return v + + @validator("inventory_target_base_pct", pre=True) + def validate_pct_inclusive(cls, v: str): + ret = validate_decimal(v, min_value=Decimal("0"), max_value=Decimal("100"), inclusive=True) + if ret is not None: + raise ValueError(ret) + return v + + # === post-validations === + + @root_validator() + def post_validations(cls, values: Dict): + cls.execution_timeframe_post_validation(values) + return values + + @classmethod + def execution_timeframe_post_validation(cls, values: Dict): + execution_timeframe = values.get("execution_timeframe") + if execution_timeframe is not None and execution_timeframe == ExecutionTimeframe.infinite: + values["start_time"] = None + values["end_time"] = None + return values diff --git a/setup.py b/setup.py index 4aa0713fc6..9d2196d5f2 100755 --- a/setup.py +++ b/setup.py @@ -78,6 +78,7 @@ def main(): "pre-commit", "prompt-toolkit", "psutil", + "pydantic", "pyjwt", "pyperclip", "python-dateutil", diff --git a/setup/environment-linux-aarch64.yml b/setup/environment-linux-aarch64.yml index c4145f4d15..2eae603363 100644 --- a/setup/environment-linux-aarch64.yml +++ b/setup/environment-linux-aarch64.yml @@ -14,6 +14,7 @@ dependencies: - pandas=1.2.1 - pip=21.1.2 - prompt_toolkit=3.0.20 + - pydantic=1.9.0 - pytables=3.6.1 - python=3.8.2 - python-dateutil=2.8.1 diff --git a/setup/environment-linux.yml b/setup/environment-linux.yml index 2bff574f81..7673732228 100644 --- a/setup/environment-linux.yml +++ b/setup/environment-linux.yml @@ -14,6 +14,7 @@ dependencies: - pandas=1.2.1 - pip=21.1.2 - prompt_toolkit=3.0.20 + - pydantic=1.9.0 - pytables=3.6.1 - python=3.8.2 - python-dateutil=2.8.1 diff --git a/setup/environment-win64.yml b/setup/environment-win64.yml index c07ab1e9c6..2243ab1704 100644 --- a/setup/environment-win64.yml +++ b/setup/environment-win64.yml @@ -12,6 +12,7 @@ dependencies: - pandas=1.2.1 - pip=21.1.2 - prompt_toolkit=3.0.20 + - pydantic=1.9.0 - pytables=3.6.1 - python=3.8.2 - python-dateutil=2.8.1 diff --git a/setup/environment.yml b/setup/environment.yml index 92a7a7dce7..8065195169 100644 --- a/setup/environment.yml +++ b/setup/environment.yml @@ -15,6 +15,7 @@ dependencies: - pandas=1.2.1 - pip=21.1.2 - prompt_toolkit=3.0.20 + - pydantic=1.9.0 - pytables=3.6.1 - python=3.8.2 - python-dateutil=2.8.1 diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py new file mode 100644 index 0000000000..46dc98051b --- /dev/null +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py @@ -0,0 +1,246 @@ +import json +import unittest +from datetime import datetime, time +from pathlib import Path +from typing import Dict +from unittest.mock import patch + +import yaml +from pydantic import ValidationError, validate_model + +from hummingbot.client.config.config_data_types import BaseClientModel +from hummingbot.client.config.config_helpers import retrieve_validation_error_msg +from hummingbot.client.settings import AllConnectorSettings +from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( + AvellanedaMarketMakingConfigMap, + DailyBetweenTimesModel, + ExecutionTimeframe, + FromDateToDateModel, + InfiniteModel, +) + + +class AvellanedaMarketMakingConfigMapPydanticTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + cls.exchange = "binance" + cls.base_asset = "COINALPHA" + cls.quote_asset = "HBOT" + cls.trading_pair = f"{cls.base_asset}-{cls.quote_asset}" + + def setUp(self) -> None: + super().setUp() + config_settings = self.get_default_map() + self.config_map = AvellanedaMarketMakingConfigMap(**config_settings) + + def get_default_map(self) -> Dict[str, str]: + config_settings = { + "exchange": self.exchange, + "market": self.trading_pair, + "execution_timeframe_model": { + "start_time": "09:30:00", + "end_time": "16:00:00", + }, + "order_amount": "10", + "order_optimization_enabled": "yes", + "risk_factor": "0.5", + "order_refresh_time": "60", + "inventory_target_base_pct": "50", + } + return config_settings + + def test_schema_encoding_removes_client_data_functions(self): + s = AvellanedaMarketMakingConfigMap.schema_json() + j = json.loads(s) + expected = { + "prompt": None, + "prompt_on_new": True, + } + self.assertEqual(expected, j["properties"]["market"]["client_data"]) + + def test_initial_sequential_build(self): + config_map: AvellanedaMarketMakingConfigMap = AvellanedaMarketMakingConfigMap.construct() + config_settings = self.get_default_map() + + def build_config_map(cm: BaseClientModel, cs: Dict): + """ + This routine can be used as is for the create command, + except for the sectioned-off portion. + """ + for key, field in cm.__fields__.items(): + client_data = cm.get_client_data(key) + if client_data is not None and client_data.prompt_on_new: + self.assertIsInstance(client_data.prompt(cm), str) + # ===================================================== + if key == "execution_timeframe_model": + cm.__setattr__(key, "daily_between_times") # simulate user input + else: + cm.__setattr__(key, cs[key]) + # cm.__setattr__(key, cs[key]) # use this in the create command routine + # ===================================================== + new_value = cm.__getattribute__(key) + if isinstance(new_value, BaseClientModel): + build_config_map(new_value, cs[key]) + + build_config_map(config_map, config_settings) + validate_model(config_map.__class__, config_map.__dict__) + + def test_order_amount_prompt(self): + prompt = self.config_map.get_client_prompt("order_amount") + expected = f"What is the amount of {self.base_asset} per order?" + + self.assertEqual(expected, prompt) + + def test_maker_trading_pair_prompt(self): + exchange = self.config_map.exchange + example = AllConnectorSettings.get_example_pairs().get(exchange) + + prompt = self.config_map.get_client_prompt("market") + expected = f"Enter the token trading pair you would like to trade on {exchange} (e.g. {example})" + + self.assertEqual(expected, prompt) + + def test_execution_time_prompts(self): + self.config_map.execution_timeframe_model = ExecutionTimeframe.from_date_to_date + model = self.config_map.execution_timeframe_model + prompt = model.get_client_prompt("start_datetime") + expected = "Please enter the start date and time (YYYY-MM-DD HH:MM:SS)" + self.assertEqual(expected, prompt) + prompt = model.get_client_prompt("end_datetime") + expected = "Please enter the end date and time (YYYY-MM-DD HH:MM:SS)" + self.assertEqual(expected, prompt) + + self.config_map.execution_timeframe_model = ExecutionTimeframe.daily_between_times + model = self.config_map.execution_timeframe_model + prompt = model.get_client_prompt("start_time") + expected = "Please enter the start time (HH:MM:SS)" + self.assertEqual(expected, prompt) + prompt = model.get_client_prompt("end_time") + expected = "Please enter the end time (HH:MM:SS)" + self.assertEqual(expected, prompt) + + @patch( + "hummingbot.strategy.avellaneda_market_making" + ".avellaneda_market_making_config_map_pydantic.validate_market_trading_pair" + ) + def test_validators(self, validate_market_trading_pair_mock): + + with self.assertRaises(ValidationError) as e: + self.config_map.exchange = "test-exchange" + + error_msg = "Invalid exchange, please choose value from " + actual_msg = retrieve_validation_error_msg(e.exception) + self.assertTrue(actual_msg.startswith(error_msg)) + + alt_pair = "ETH-USDT" + error_msg = "Failed" + validate_market_trading_pair_mock.side_effect = ( + lambda m, v: None if v in [self.trading_pair, alt_pair] else error_msg + ) + + self.config_map.market = alt_pair + self.assertEqual(alt_pair, self.config_map.market) + + with self.assertRaises(ValidationError) as e: + self.config_map.market = "XXX-USDT" + + actual_msg = retrieve_validation_error_msg(e.exception) + self.assertTrue(actual_msg.startswith(error_msg)) + + self.config_map.execution_timeframe_model = "infinite" + self.assertIsInstance(self.config_map.execution_timeframe_model, InfiniteModel) + + self.config_map.execution_timeframe_model = "from_date_to_date" + self.assertIsInstance(self.config_map.execution_timeframe_model, FromDateToDateModel) + + self.config_map.execution_timeframe_model = "daily_between_times" + self.assertIsInstance(self.config_map.execution_timeframe_model, DailyBetweenTimesModel) + + with self.assertRaises(ValidationError) as e: + self.config_map.execution_timeframe_model = "XXX" + + error_msg = ( + "Invalid timeframe, please choose value from ['infinite', 'from_date_to_date', 'daily_between_times']" + ) + actual_msg = retrieve_validation_error_msg(e.exception) + self.assertEqual(error_msg, actual_msg) + + self.config_map.execution_timeframe_model = "from_date_to_date" + model = self.config_map.execution_timeframe_model + model.start_datetime = "2021-01-01 12:00:00" + model.end_datetime = "2021-01-01 15:00:00" + + self.assertEqual(datetime(2021, 1, 1, 12, 0, 0), model.start_datetime) + self.assertEqual(datetime(2021, 1, 1, 15, 0, 0), model.end_datetime) + + with self.assertRaises(ValidationError) as e: + model.start_datetime = "2021-01-01 30:00:00" + + error_msg = "Incorrect date time format (expected is YYYY-MM-DD HH:MM:SS)" + actual_msg = retrieve_validation_error_msg(e.exception) + self.assertEqual(error_msg, actual_msg) + + with self.assertRaises(ValidationError) as e: + model.start_datetime = "12:00:00" + + error_msg = "Incorrect date time format (expected is YYYY-MM-DD HH:MM:SS)" + actual_msg = retrieve_validation_error_msg(e.exception) + self.assertEqual(error_msg, actual_msg) + + self.config_map.execution_timeframe_model = "daily_between_times" + model = self.config_map.execution_timeframe_model + model.start_time = "12:00:00" + + self.assertEqual(time(12, 0, 0), model.start_time) + + with self.assertRaises(ValidationError) as e: + model.start_time = "30:00:00" + + error_msg = "Incorrect time format (expected is HH:MM:SS)" + actual_msg = retrieve_validation_error_msg(e.exception) + self.assertEqual(error_msg, actual_msg) + + with self.assertRaises(ValidationError) as e: + model.start_time = "2021-01-01 12:00:00" + + error_msg = "Incorrect time format (expected is HH:MM:SS)" + actual_msg = retrieve_validation_error_msg(e.exception) + self.assertEqual(error_msg, actual_msg) + + self.config_map.order_levels_mode = "multi_order_level" + model = self.config_map.order_levels_mode + + with self.assertRaises(ValidationError) as e: + model.order_levels = 1 + + error_msg = "Value cannot be less than 2." + actual_msg = retrieve_validation_error_msg(e.exception) + self.assertEqual(error_msg, actual_msg) + + model.order_levels = 3 + self.assertEqual(3, model.order_levels) + + self.config_map.hanging_orders_mode = "track_hanging_orders" + model = self.config_map.hanging_orders_mode + + with self.assertRaises(ValidationError) as e: + model.hanging_orders_cancel_pct = "-1" + + error_msg = "Value must be between 0 and 100 (exclusive)." + actual_msg = retrieve_validation_error_msg(e.exception) + self.assertEqual(error_msg, actual_msg) + + model.hanging_orders_cancel_pct = "3" + self.assertEqual(3, model.hanging_orders_cancel_pct) + + def test_load_configs_from_yaml(self): + cur_dir = Path(__file__).parent + f_path = cur_dir / "test_config.yml" + + with open(f_path, "r") as file: + data = yaml.safe_load(file) + + loaded_config_map = AvellanedaMarketMakingConfigMap(**data) + + self.assertEqual(self.config_map, loaded_config_map) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_config.yml b/test/hummingbot/strategy/avellaneda_market_making/test_config.yml new file mode 100644 index 0000000000..cbf4e2b487 --- /dev/null +++ b/test/hummingbot/strategy/avellaneda_market_making/test_config.yml @@ -0,0 +1,10 @@ +exchange: binance +market: COINALPHA-HBOT +execution_timeframe_model: + start_time: "09:30:00" + end_time: "16:00:00" +order_amount: 10 +order_optimization_enabled: true +risk_factor: 0.5 +order_refresh_time: 60 +inventory_target_base_pct: 50 \ No newline at end of file From c2c42e2659345137c733ddca28e0f5742da390eb Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Fri, 18 Feb 2022 10:27:54 +0700 Subject: [PATCH 011/152] (fix) addresses a potential problem in parsing sub-models. --- .../avellaneda_market_making_config_map_pydantic.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py index 513d9d450d..b346e9c19e 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -411,8 +411,10 @@ def validate_execution_timeframe( sub_model = InfiniteModel.construct() elif v == ExecutionTimeframe.daily_between_times: sub_model = DailyBetweenTimesModel.construct() - else: # v == ExecutionTimeframe.from_date_to_date + elif v == ExecutionTimeframe.from_date_to_date: sub_model = FromDateToDateModel.construct() + else: # isinstance(v, Dict) + sub_model = v return sub_model @validator("order_refresh_tolerance_pct", pre=True) @@ -439,8 +441,10 @@ def validate_order_levels_mode(cls, v: Union[str, SingleOrderLevelModel, MultiOr sub_model = v elif v == OrderLevelsMode.single_order_level: sub_model = SingleOrderLevelModel.construct() - else: # v == OrderLevelsMode.multi_order_level + elif v == OrderLevelsMode.multi_order_level: sub_model = MultiOrderLevelModel.construct() + else: # isinstance(v, Dict) + sub_model = v return sub_model @validator("hanging_orders_mode", pre=True) @@ -453,8 +457,10 @@ def validate_hanging_orders_mode(cls, v: Union[str, TrackHangingOrdersModel, Ign sub_model = v elif v == HangingOrdersMode.track_hanging_orders: sub_model = TrackHangingOrdersModel.construct() - else: # v == HangingOrdersMode.ignore_hanging_orders + elif v == HangingOrdersMode.ignore_hanging_orders: sub_model = IgnoreHangingOrdersModel.construct() + else: # isinstance(v, Dict) + sub_model = v return sub_model # === generic validations === From 71cf8db6fed2b9463d12d0e5ab8bb2d43ba8e9d0 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Fri, 18 Feb 2022 10:32:27 +0700 Subject: [PATCH 012/152] (cleanup) Edited misleading comment --- .../test_avellaneda_market_making_config_map_pydantic.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py index 46dc98051b..cc6a034d0d 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py @@ -64,21 +64,15 @@ def test_initial_sequential_build(self): config_settings = self.get_default_map() def build_config_map(cm: BaseClientModel, cs: Dict): - """ - This routine can be used as is for the create command, - except for the sectioned-off portion. - """ + """This routine can be used in the create command, with slight modifications.""" for key, field in cm.__fields__.items(): client_data = cm.get_client_data(key) if client_data is not None and client_data.prompt_on_new: self.assertIsInstance(client_data.prompt(cm), str) - # ===================================================== if key == "execution_timeframe_model": cm.__setattr__(key, "daily_between_times") # simulate user input else: cm.__setattr__(key, cs[key]) - # cm.__setattr__(key, cs[key]) # use this in the create command routine - # ===================================================== new_value = cm.__getattribute__(key) if isinstance(new_value, BaseClientModel): build_config_map(new_value, cs[key]) From d1314cbc280d16025d96c79be1e2349eab71edca Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Tue, 22 Feb 2022 13:02:28 +0700 Subject: [PATCH 013/152] (fix) Addresses @aarmoa's PR comments --- hummingbot/client/config/config_helpers.py | 2 +- ...aneda_market_making_config_map_pydantic.py | 112 +++++++++--------- setup/environment-linux-aarch64.yml | 2 +- setup/environment-linux.yml | 2 +- setup/environment-win64.yml | 2 +- setup/environment.yml | 2 +- ...aneda_market_making_config_map_pydantic.py | 5 +- 7 files changed, 60 insertions(+), 67 deletions(-) diff --git a/hummingbot/client/config/config_helpers.py b/hummingbot/client/config/config_helpers.py index e7f8cef925..ecc6d8a0aa 100644 --- a/hummingbot/client/config/config_helpers.py +++ b/hummingbot/client/config/config_helpers.py @@ -18,13 +18,13 @@ from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.security import Security from hummingbot.client.settings import ( + AllConnectorSettings, CONF_FILE_PATH, CONF_POSTFIX, CONF_PREFIX, GLOBAL_CONFIG_PATH, TEMPLATE_PATH, TRADE_FEES_CONFIG_PATH, - AllConnectorSettings ) # Use ruamel.yaml to preserve order and comments in .yml file diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py index b346e9c19e..0d64d66d3f 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -18,15 +18,9 @@ from hummingbot.connector.utils import split_hb_trading_pair -class ExecutionTimeframe(str, ClientConfigEnum): - infinite = "infinite" - from_date_to_date = "from_date_to_date" - daily_between_times = "daily_between_times" - - class InfiniteModel(BaseClientModel): class Config: - title = ExecutionTimeframe.infinite + title = "infinite" validate_assignment = True @@ -49,7 +43,7 @@ class FromDateToDateModel(BaseClientModel): ) class Config: - title = ExecutionTimeframe.from_date_to_date + title = "from_date_to_date" validate_assignment = True @validator("start_datetime", "end_datetime", pre=True) @@ -79,7 +73,7 @@ class DailyBetweenTimesModel(BaseClientModel): ) class Config: - title = ExecutionTimeframe.daily_between_times + title = "daily_between_times" validate_assignment = True @validator("start_time", "end_time", pre=True) @@ -90,14 +84,16 @@ def validate_execution_time(cls, v: str) -> Optional[str]: return v -class OrderLevelsMode(str, ClientConfigEnum): - single_order_level = "single_order_level" - multi_order_level = "multi_order_level" +EXECUTION_TIMEFRAME_MODELS = { + InfiniteModel.Config.title: InfiniteModel, + FromDateToDateModel.Config.title: FromDateToDateModel, + DailyBetweenTimesModel.Config.title: DailyBetweenTimesModel, +} class SingleOrderLevelModel(BaseClientModel): class Config: - title = OrderLevelsMode.single_order_level + title = "single_order_level" validate_assignment = True @@ -120,7 +116,7 @@ class MultiOrderLevelModel(BaseClientModel): ) class Config: - title = OrderLevelsMode.multi_order_level + title = "multi_order_level" validate_assignment = True @validator("order_levels", pre=True) @@ -138,9 +134,10 @@ def validate_decimal_zero_or_above(cls, v: str): return v -class HangingOrdersMode(str, ClientConfigEnum): - track_hanging_orders = "track_hanging_orders" - ignore_hanging_orders = "ignore_hanging_orders" +ORDER_LEVEL_MODELS = { + SingleOrderLevelModel.Config.title: SingleOrderLevelModel, + MultiOrderLevelModel.Config.title: MultiOrderLevelModel, +} class TrackHangingOrdersModel(BaseClientModel): @@ -158,7 +155,7 @@ class TrackHangingOrdersModel(BaseClientModel): ) class Config: - title = HangingOrdersMode.track_hanging_orders + title = "track_hanging_orders" validate_assignment = True @validator("hanging_orders_cancel_pct", pre=True) @@ -171,10 +168,16 @@ def validate_pct_exclusive(cls, v: str): class IgnoreHangingOrdersModel(BaseClientModel): class Config: - title = HangingOrdersMode.ignore_hanging_orders + title = "ignore_hanging_orders" validate_assignment = True +HANGING_ORDER_MODELS = { + TrackHangingOrdersModel.Config.title: TrackHangingOrdersModel, + IgnoreHangingOrdersModel.Config.title: IgnoreHangingOrdersModel, +} + + def maker_trading_pair_prompt(model_instance: 'AvellanedaMarketMakingConfigMap') -> str: exchange = model_instance.exchange example = AllConnectorSettings.get_example_pairs().get(exchange) @@ -215,7 +218,7 @@ class AvellanedaMarketMakingConfigMap(BaseClientModel): default=..., description="The execution timeframe.", client_data=ClientFieldData( - prompt=lambda mi: f"Select the execution timeframe ({'/'.join(ExecutionTimeframe)})", + prompt=lambda mi: f"Select the execution timeframe ({'/'.join(EXECUTION_TIMEFRAME_MODELS.keys())})", prompt_on_new=True, ), ) @@ -345,7 +348,7 @@ class AvellanedaMarketMakingConfigMap(BaseClientModel): default=SingleOrderLevelModel.construct(), description="Allows activating multi-order levels.", client_data=ClientFieldData( - prompt=lambda mi: f"Select the order levels mode ({'/'.join(OrderLevelsMode)}", + prompt=lambda mi: f"Select the order levels mode ({'/'.join(list(ORDER_LEVEL_MODELS.keys()))}", ), ) order_override: Optional[Dict] = Field( @@ -357,7 +360,9 @@ class AvellanedaMarketMakingConfigMap(BaseClientModel): default=IgnoreHangingOrdersModel.construct(), description="When tracking hanging orders, the orders on the side opposite to the filled orders remain active.", client_data=ClientFieldData( - prompt=lambda mi: f"How do you want to handle hanging orders? ({'/'.join(HangingOrdersMode)})", + prompt=( + lambda mi: f"How do you want to handle hanging orders? ({'/'.join(list(HANGING_ORDER_MODELS.keys()))})" + ), ), ) should_wait_order_cancel_confirmation: bool = Field( @@ -375,10 +380,6 @@ class AvellanedaMarketMakingConfigMap(BaseClientModel): ) ) - def __init__(self, **data): - super().__init__(**data) - required_exchanges.append(self.exchange) - class Config: validate_assignment = True @@ -401,20 +402,14 @@ def validate_exchange_trading_pair(cls, v: str, values: Dict): def validate_execution_timeframe( cls, v: Union[str, InfiniteModel, FromDateToDateModel, DailyBetweenTimesModel] ): - if not isinstance(v, Dict) and not hasattr(ExecutionTimeframe, v): + if isinstance(v, (InfiniteModel, FromDateToDateModel, DailyBetweenTimesModel, Dict)): + sub_model = v + elif not isinstance(v, Dict) and v not in EXECUTION_TIMEFRAME_MODELS: raise ValueError( - f"Invalid timeframe, please choose value from {[e.value for e in list(ExecutionTimeframe)]}" + f"Invalid timeframe, please choose value from {list(EXECUTION_TIMEFRAME_MODELS.keys())}" ) - elif isinstance(v, (InfiniteModel, FromDateToDateModel, DailyBetweenTimesModel)): - sub_model = v - elif v == ExecutionTimeframe.infinite: - sub_model = InfiniteModel.construct() - elif v == ExecutionTimeframe.daily_between_times: - sub_model = DailyBetweenTimesModel.construct() - elif v == ExecutionTimeframe.from_date_to_date: - sub_model = FromDateToDateModel.construct() - else: # isinstance(v, Dict) - sub_model = v + else: + sub_model = EXECUTION_TIMEFRAME_MODELS[v].construct() return sub_model @validator("order_refresh_tolerance_pct", pre=True) @@ -433,34 +428,28 @@ def validate_buffer_size(cls, v: str): @validator("order_levels_mode", pre=True) def validate_order_levels_mode(cls, v: Union[str, SingleOrderLevelModel, MultiOrderLevelModel]): - if not isinstance(v, Dict) and not hasattr(OrderLevelsMode, v): + if isinstance(v, (SingleOrderLevelModel, MultiOrderLevelModel, Dict)): + sub_model = v + elif not isinstance(v, Dict) and v not in ORDER_LEVEL_MODELS: raise ValueError( - f"Invalid order levels mode, please choose value from {[e.value for e in list(OrderLevelsMode)]}." + f"Invalid order levels mode, please choose value from" + f" {[e.value for e in list(ORDER_LEVEL_MODELS.keys())]}." ) - elif isinstance(v, (SingleOrderLevelModel, MultiOrderLevelModel)): - sub_model = v - elif v == OrderLevelsMode.single_order_level: - sub_model = SingleOrderLevelModel.construct() - elif v == OrderLevelsMode.multi_order_level: - sub_model = MultiOrderLevelModel.construct() - else: # isinstance(v, Dict) - sub_model = v + else: + sub_model = ORDER_LEVEL_MODELS[v].construct() return sub_model @validator("hanging_orders_mode", pre=True) def validate_hanging_orders_mode(cls, v: Union[str, TrackHangingOrdersModel, IgnoreHangingOrdersModel]): - if not isinstance(v, Dict) and not hasattr(HangingOrdersMode, v): + if isinstance(v, (TrackHangingOrdersModel, IgnoreHangingOrdersModel, Dict)): + sub_model = v + elif not isinstance(v, Dict) and v not in HANGING_ORDER_MODELS: raise ValueError( - f"Invalid hanging order mode, please choose value from {[e.value for e in list(HangingOrdersMode)]}." + f"Invalid hanging order mode, please choose value from" + f" {[e.value for e in list(HANGING_ORDER_MODELS.keys())]}." ) - elif isinstance(v, (TrackHangingOrdersModel, IgnoreHangingOrdersModel)): - sub_model = v - elif v == HangingOrdersMode.track_hanging_orders: - sub_model = TrackHangingOrdersModel.construct() - elif v == HangingOrdersMode.ignore_hanging_orders: - sub_model = IgnoreHangingOrdersModel.construct() - else: # isinstance(v, Dict) - sub_model = v + else: + sub_model = HANGING_ORDER_MODELS[v].construct() return sub_model # === generic validations === @@ -518,12 +507,17 @@ def validate_pct_inclusive(cls, v: str): @root_validator() def post_validations(cls, values: Dict): cls.execution_timeframe_post_validation(values) + cls.exchange_post_validation(values) return values @classmethod def execution_timeframe_post_validation(cls, values: Dict): execution_timeframe = values.get("execution_timeframe") - if execution_timeframe is not None and execution_timeframe == ExecutionTimeframe.infinite: + if execution_timeframe is not None and execution_timeframe == InfiniteModel.Config.title: values["start_time"] = None values["end_time"] = None return values + + @classmethod + def exchange_post_validation(cls, values: Dict): + required_exchanges.append(values["exchange"]) diff --git a/setup/environment-linux-aarch64.yml b/setup/environment-linux-aarch64.yml index 2eae603363..c528d735b9 100644 --- a/setup/environment-linux-aarch64.yml +++ b/setup/environment-linux-aarch64.yml @@ -14,7 +14,7 @@ dependencies: - pandas=1.2.1 - pip=21.1.2 - prompt_toolkit=3.0.20 - - pydantic=1.9.0 + - pydantic=1.9 - pytables=3.6.1 - python=3.8.2 - python-dateutil=2.8.1 diff --git a/setup/environment-linux.yml b/setup/environment-linux.yml index 7673732228..878024dde4 100644 --- a/setup/environment-linux.yml +++ b/setup/environment-linux.yml @@ -14,7 +14,7 @@ dependencies: - pandas=1.2.1 - pip=21.1.2 - prompt_toolkit=3.0.20 - - pydantic=1.9.0 + - pydantic=1.9 - pytables=3.6.1 - python=3.8.2 - python-dateutil=2.8.1 diff --git a/setup/environment-win64.yml b/setup/environment-win64.yml index 2243ab1704..d1c5b22fb4 100644 --- a/setup/environment-win64.yml +++ b/setup/environment-win64.yml @@ -12,7 +12,7 @@ dependencies: - pandas=1.2.1 - pip=21.1.2 - prompt_toolkit=3.0.20 - - pydantic=1.9.0 + - pydantic=1.9 - pytables=3.6.1 - python=3.8.2 - python-dateutil=2.8.1 diff --git a/setup/environment.yml b/setup/environment.yml index 8065195169..194289c007 100644 --- a/setup/environment.yml +++ b/setup/environment.yml @@ -15,7 +15,7 @@ dependencies: - pandas=1.2.1 - pip=21.1.2 - prompt_toolkit=3.0.20 - - pydantic=1.9.0 + - pydantic=1.9 - pytables=3.6.1 - python=3.8.2 - python-dateutil=2.8.1 diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py index cc6a034d0d..64fec3dadd 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py @@ -14,7 +14,6 @@ from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( AvellanedaMarketMakingConfigMap, DailyBetweenTimesModel, - ExecutionTimeframe, FromDateToDateModel, InfiniteModel, ) @@ -96,7 +95,7 @@ def test_maker_trading_pair_prompt(self): self.assertEqual(expected, prompt) def test_execution_time_prompts(self): - self.config_map.execution_timeframe_model = ExecutionTimeframe.from_date_to_date + self.config_map.execution_timeframe_model = FromDateToDateModel.Config.title model = self.config_map.execution_timeframe_model prompt = model.get_client_prompt("start_datetime") expected = "Please enter the start date and time (YYYY-MM-DD HH:MM:SS)" @@ -105,7 +104,7 @@ def test_execution_time_prompts(self): expected = "Please enter the end date and time (YYYY-MM-DD HH:MM:SS)" self.assertEqual(expected, prompt) - self.config_map.execution_timeframe_model = ExecutionTimeframe.daily_between_times + self.config_map.execution_timeframe_model = DailyBetweenTimesModel.Config.title model = self.config_map.execution_timeframe_model prompt = model.get_client_prompt("start_time") expected = "Please enter the start time (HH:MM:SS)" From 8b71afe4ebc841f15448a9db8a64bb4856f158d7 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Tue, 22 Feb 2022 11:32:53 +0700 Subject: [PATCH 014/152] Update hummingbot/client/config/config_data_types.py Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- hummingbot/client/config/config_data_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/client/config/config_data_types.py b/hummingbot/client/config/config_data_types.py index 593c09b9a1..c12d484696 100644 --- a/hummingbot/client/config/config_data_types.py +++ b/hummingbot/client/config/config_data_types.py @@ -1,6 +1,6 @@ from dataclasses import dataclass from enum import Enum -from typing import Any, Optional, Callable +from typing import Any, Callable, Optional from pydantic import BaseModel from pydantic.schema import default_ref_template From c5301d1b4cb7c37c35d817be3ea4df3b4c467ade Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Tue, 22 Feb 2022 13:10:12 +0700 Subject: [PATCH 015/152] (cleanup) Adds clarification comments to client-specific validations --- .../avellaneda_market_making_config_map_pydantic.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py index 0d64d66d3f..2ed36983de 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -385,6 +385,7 @@ class Config: @validator("exchange", pre=True) def validate_exchange(cls, v: str): + """Used for client-friendly error output.""" ret = validate_exchange(v) if ret is not None: raise ValueError(ret) @@ -414,6 +415,7 @@ def validate_execution_timeframe( @validator("order_refresh_tolerance_pct", pre=True) def validate_order_refresh_tolerance_pct(cls, v: str): + """Used for client-friendly error output.""" ret = validate_decimal(v, min_value=Decimal("-10"), max_value=Decimal("10"), inclusive=True) if ret is not None: raise ValueError(ret) @@ -421,6 +423,7 @@ def validate_order_refresh_tolerance_pct(cls, v: str): @validator("volatility_buffer_size", "trading_intensity_buffer_size", pre=True) def validate_buffer_size(cls, v: str): + """Used for client-friendly error output.""" ret = validate_int(v, 1, 10_000) if ret is not None: raise ValueError(ret) @@ -461,6 +464,7 @@ def validate_hanging_orders_mode(cls, v: Union[str, TrackHangingOrdersModel, Ign pre=True, ) def validate_bool(cls, v: str): + """Used for client-friendly error output.""" if isinstance(v, str): ret = validate_bool(v) if ret is not None: @@ -469,6 +473,7 @@ def validate_bool(cls, v: str): @validator("order_amount_shape_factor", pre=True) def validate_decimal_from_zero_to_one(cls, v: str): + """Used for client-friendly error output.""" ret = validate_decimal(v, min_value=Decimal("0"), max_value=Decimal("1"), inclusive=True) if ret is not None: raise ValueError(ret) @@ -483,6 +488,7 @@ def validate_decimal_from_zero_to_one(cls, v: str): pre=True, ) def validate_decimal_above_zero(cls, v: str): + """Used for client-friendly error output.""" ret = validate_decimal(v, min_value=Decimal("0"), inclusive=False) if ret is not None: raise ValueError(ret) @@ -490,6 +496,7 @@ def validate_decimal_above_zero(cls, v: str): @validator("min_spread", pre=True) def validate_decimal_zero_or_above(cls, v: str): + """Used for client-friendly error output.""" ret = validate_decimal(v, min_value=Decimal("0"), inclusive=True) if ret is not None: raise ValueError(ret) @@ -497,6 +504,7 @@ def validate_decimal_zero_or_above(cls, v: str): @validator("inventory_target_base_pct", pre=True) def validate_pct_inclusive(cls, v: str): + """Used for client-friendly error output.""" ret = validate_decimal(v, min_value=Decimal("0"), max_value=Decimal("100"), inclusive=True) if ret is not None: raise ValueError(ret) From 9c25a74e2b57f79eed26606def640e54d4955561 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 23 Feb 2022 10:10:30 +0700 Subject: [PATCH 016/152] (cleanup) Makes sub-model validators a bit more precise. --- .../avellaneda_market_making_config_map_pydantic.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py index 2ed36983de..c20014d8f4 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -405,7 +405,7 @@ def validate_execution_timeframe( ): if isinstance(v, (InfiniteModel, FromDateToDateModel, DailyBetweenTimesModel, Dict)): sub_model = v - elif not isinstance(v, Dict) and v not in EXECUTION_TIMEFRAME_MODELS: + elif v not in EXECUTION_TIMEFRAME_MODELS: raise ValueError( f"Invalid timeframe, please choose value from {list(EXECUTION_TIMEFRAME_MODELS.keys())}" ) @@ -433,7 +433,7 @@ def validate_buffer_size(cls, v: str): def validate_order_levels_mode(cls, v: Union[str, SingleOrderLevelModel, MultiOrderLevelModel]): if isinstance(v, (SingleOrderLevelModel, MultiOrderLevelModel, Dict)): sub_model = v - elif not isinstance(v, Dict) and v not in ORDER_LEVEL_MODELS: + elif v not in ORDER_LEVEL_MODELS: raise ValueError( f"Invalid order levels mode, please choose value from" f" {[e.value for e in list(ORDER_LEVEL_MODELS.keys())]}." @@ -446,7 +446,7 @@ def validate_order_levels_mode(cls, v: Union[str, SingleOrderLevelModel, MultiOr def validate_hanging_orders_mode(cls, v: Union[str, TrackHangingOrdersModel, IgnoreHangingOrdersModel]): if isinstance(v, (TrackHangingOrdersModel, IgnoreHangingOrdersModel, Dict)): sub_model = v - elif not isinstance(v, Dict) and v not in HANGING_ORDER_MODELS: + elif v not in HANGING_ORDER_MODELS: raise ValueError( f"Invalid hanging order mode, please choose value from" f" {[e.value for e in list(HANGING_ORDER_MODELS.keys())]}." From f71994a00c9f148decb7aefee60e9eee79a20e73 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 23 Feb 2022 10:02:09 +0700 Subject: [PATCH 017/152] Update test/hummingbot/strategy/avellaneda_market_making/test_config.yml Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- .../strategy/avellaneda_market_making/test_config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_config.yml b/test/hummingbot/strategy/avellaneda_market_making/test_config.yml index cbf4e2b487..c7c7cf1d39 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_config.yml +++ b/test/hummingbot/strategy/avellaneda_market_making/test_config.yml @@ -7,4 +7,4 @@ order_amount: 10 order_optimization_enabled: true risk_factor: 0.5 order_refresh_time: 60 -inventory_target_base_pct: 50 \ No newline at end of file +inventory_target_base_pct: 50 From 0e83f3394a0da8f6139a35bc35bfc4395c8393f2 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Thu, 24 Feb 2022 08:21:26 +0700 Subject: [PATCH 018/152] (cleanup) Adds the last prompt functions as class methods to the config class. --- ...aneda_market_making_config_map_pydantic.py | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py index c20014d8f4..ef04712821 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -178,20 +178,6 @@ class Config: } -def maker_trading_pair_prompt(model_instance: 'AvellanedaMarketMakingConfigMap') -> str: - exchange = model_instance.exchange - example = AllConnectorSettings.get_example_pairs().get(exchange) - return ( - f"Enter the token trading pair you would like to trade on {exchange}{f' (e.g. {example})' if example else ''}" - ) - - -def order_amount_prompt(model_instance: 'AvellanedaMarketMakingConfigMap') -> str: - trading_pair = model_instance.market - base_asset, quote_asset = split_hb_trading_pair(trading_pair) - return f"What is the amount of {base_asset} per order?" - - class AvellanedaMarketMakingConfigMap(BaseClientModel): strategy: str = Field(default="avellaneda_market_making", client_data=None) exchange: ClientConfigEnum( @@ -210,7 +196,7 @@ class AvellanedaMarketMakingConfigMap(BaseClientModel): default=..., description="The trading pair.", client_data=ClientFieldData( - prompt=maker_trading_pair_prompt, + prompt=lambda mi: AvellanedaMarketMakingConfigMap.maker_trading_pair_prompt(mi), prompt_on_new=True, ), ) @@ -226,7 +212,7 @@ class AvellanedaMarketMakingConfigMap(BaseClientModel): default=..., description="The strategy order amount.", gt=0, - client_data=ClientFieldData(prompt=order_amount_prompt) + client_data=ClientFieldData(prompt=lambda mi: AvellanedaMarketMakingConfigMap.order_amount_prompt(mi)) ) order_optimization_enabled: bool = Field( default=True, @@ -383,6 +369,21 @@ class AvellanedaMarketMakingConfigMap(BaseClientModel): class Config: validate_assignment = True + @classmethod + def maker_trading_pair_prompt(cls, model_instance: 'AvellanedaMarketMakingConfigMap') -> str: + exchange = model_instance.exchange + example = AllConnectorSettings.get_example_pairs().get(exchange) + return ( + f"Enter the token trading pair you would like to trade on" + f" {exchange}{f' (e.g. {example})' if example else ''}" + ) + + @classmethod + def order_amount_prompt(cls, model_instance: 'AvellanedaMarketMakingConfigMap') -> str: + trading_pair = model_instance.market + base_asset, quote_asset = split_hb_trading_pair(trading_pair) + return f"What is the amount of {base_asset} per order?" + @validator("exchange", pre=True) def validate_exchange(cls, v: str): """Used for client-friendly error output.""" From 92ff8c877e22fef59251124cf67b802f4c5cce0f Mon Sep 17 00:00:00 2001 From: Daniel Tan Date: Mon, 7 Mar 2022 19:15:38 +0800 Subject: [PATCH 019/152] (feat) add functions to load pydantic configs in config_helpers.py --- hummingbot/client/config/config_helpers.py | 52 +++++++++++++++++++--- 1 file changed, 45 insertions(+), 7 deletions(-) diff --git a/hummingbot/client/config/config_helpers.py b/hummingbot/client/config/config_helpers.py index ecc6d8a0aa..2d8c222833 100644 --- a/hummingbot/client/config/config_helpers.py +++ b/hummingbot/client/config/config_helpers.py @@ -5,14 +5,14 @@ from decimal import Decimal from os import listdir, unlink from os.path import isfile, join -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, TYPE_CHECKING import ruamel.yaml from eth_account import Account from pydantic import ValidationError from pydantic.json import pydantic_encoder -from hummingbot import get_strategy_list +from hummingbot import get_strategy_list, root_path from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.fee_overrides_config_map import fee_overrides_config_map from hummingbot.client.config.global_config_map import global_config_map @@ -27,6 +27,9 @@ TRADE_FEES_CONFIG_PATH, ) +if TYPE_CHECKING: # avoid circular import problems + from hummingbot.client.config.config_data_types import BaseClientModel + # Use ruamel.yaml to preserve order and comments in .yml file yaml_parser = ruamel.yaml.YAML() @@ -222,12 +225,47 @@ def validate_strategy_file(file_path: str) -> Optional[str]: return None +def read_yml_file(yml_path: str) -> Dict[str, Any]: + with open(yml_path, "r") as file: + data = yaml_parser.load(file) or {} + return dict(data) + + +def load_pydantic_config(strategy_name: str, yml_path: str) -> Optional["BaseClientModel"]: + """ + Resolves the pydantic model with the given strategy filepath. Subsequenty load and return the config as a `Model` + + :param strategy_name: Strategy name. + :type strategy_name: str + :param yml_path: Strategy file path. + :type yml_path: str + :return: The strategy configurations. + :rtype: Optional[BaseClientModel] + """ + try: + pydantic_cm_pkg = f"{strategy_name}_config_map_pydantic" + if not isfile(f"{root_path()}/hummingbot/strategy/{strategy_name}/{pydantic_cm_pkg}.py"): + return None + + pydantic_cm_class_name = f"{''.join([s.capitalize() for s in strategy_name.split('_')])}ConfigMap" + pydantic_cm_mod = __import__(f"hummingbot.strategy.{strategy_name}.{pydantic_cm_pkg}", + fromlist=[f"{pydantic_cm_class_name}"]) + pydantic_cm_class = getattr(pydantic_cm_mod, pydantic_cm_class_name) + return pydantic_cm_class(**read_yml_file(yml_path)) + except Exception as e: + logging.getLogger().error(f"Error loading pydantic configs. Your config file may be corrupt. {e}", + exc_info=True) + return None + + async def update_strategy_config_map_from_file(yml_path: str) -> str: - strategy = strategy_name_from_file(yml_path) - config_map = get_strategy_config_map(strategy) - template_path = get_strategy_template_path(strategy) - await load_yml_into_cm(yml_path, template_path, config_map) - return strategy + strategy_name = strategy_name_from_file(yml_path) + pydantic_conf: "BaseClientModel" = load_pydantic_config(strategy_name, yml_path) + if pydantic_conf is None: + config_map = get_strategy_config_map(strategy_name) + template_path = get_strategy_template_path(strategy_name) + await load_yml_into_cm(yml_path, template_path, config_map) + return strategy_name async def load_yml_into_cm(yml_path: str, template_file_path: str, cm: Dict[str, ConfigVar]): From 36df3069fd839e0ad78cf90891e4bc98878dfc9c Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Fri, 11 Mar 2022 13:16:44 +0700 Subject: [PATCH 020/152] (refactor) Adapts the create command to work with the new pydantic configuation schemas. --- hummingbot/client/command/balance_command.py | 6 +- hummingbot/client/command/config_command.py | 24 +-- hummingbot/client/command/connect_command.py | 16 +- hummingbot/client/command/create_command.py | 178 ++++++++++++++---- hummingbot/client/command/start_command.py | 2 +- hummingbot/client/command/status_command.py | 39 +++- hummingbot/client/config/config_data_types.py | 109 ++++++++++- hummingbot/client/config/config_helpers.py | 63 +++++-- hummingbot/client/hummingbot_application.py | 12 +- hummingbot/client/ui/style.py | 4 +- ...aneda_market_making_config_map_pydantic.py | 15 +- .../avellaneda_market_making/start.py | 86 +++++---- .../client/command/test_create_command.py | 6 +- .../client/config/test_config_data_types.py | 114 +++++++++++ .../client/config/test_config_helpers.py | 30 +++ .../client/config/test_config_templates.py | 27 ++- ...aneda_market_making_config_map_pydantic.py | 30 ++- .../test_avellaneda_market_making_start.py | 68 ++++--- 18 files changed, 650 insertions(+), 179 deletions(-) create mode 100644 test/hummingbot/client/config/test_config_data_types.py diff --git a/hummingbot/client/command/balance_command.py b/hummingbot/client/command/balance_command.py index 8e1d77cb2b..635f6333a4 100644 --- a/hummingbot/client/command/balance_command.py +++ b/hummingbot/client/command/balance_command.py @@ -8,7 +8,7 @@ from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.config_helpers import ( - save_to_yml + save_to_yml_legacy ) from hummingbot.client.config.config_validators import validate_decimal, validate_exchange from hummingbot.connector.other.celo.celo_cli import CeloCLI @@ -64,7 +64,7 @@ def balance(self, elif amount >= 0: config_var.value[exchange][asset] = amount self._notify(f"Limit for {asset} on {exchange} exchange set to {amount}") - save_to_yml(file_path, config_map) + save_to_yml_legacy(file_path, config_map) elif option == "paper": config_var = config_map["paper_trade_account_balance"] @@ -81,7 +81,7 @@ def balance(self, paper_balances[asset] = amount config_var.value = paper_balances self._notify(f"Paper balance for {asset} token set to {amount}") - save_to_yml(file_path, config_map) + save_to_yml_legacy(file_path, config_map) async def show_balances(self): total_col_name = f'Total ({RateOracle.global_token_symbol})' diff --git a/hummingbot/client/command/config_command.py b/hummingbot/client/command/config_command.py index ec80041629..ffcb93a0e9 100644 --- a/hummingbot/client/command/config_command.py +++ b/hummingbot/client/command/config_command.py @@ -10,8 +10,8 @@ import pandas as pd from hummingbot.client.config.config_helpers import ( - missing_required_configs, - save_to_yml, + missing_required_configs_legacy, + save_to_yml_legacy, ) from hummingbot.client.config.config_validators import validate_bool, validate_decimal from hummingbot.client.config.config_var import ConfigVar @@ -164,16 +164,16 @@ async def _config_single_key(self, # type: HummingbotApplication elif config_var.key == "inventory_price": await self.inventory_price_prompt(config_map, input_value) else: - await self.prompt_a_config(config_var, input_value=input_value, assign_default=False) + await self.prompt_a_config_legacy(config_var, input_value=input_value, assign_default=False) if self.app.to_stop_config: self.app.to_stop_config = False return - await self.update_all_secure_configs() - missings = missing_required_configs(config_map) + await self.update_all_secure_configs_legacy() + missings = missing_required_configs_legacy(config_map) if missings: self._notify("\nThere are other configuration required, please follow the prompt to complete them.") missings = await self._prompt_missing_configs(config_map) - save_to_yml(file_path, config_map) + save_to_yml_legacy(file_path, config_map) self._notify("\nNew configuration saved:") self._notify(f"{key}: {str(config_var.value)}") self.app.app.style = load_style() @@ -196,13 +196,13 @@ async def _config_single_key(self, # type: HummingbotApplication async def _prompt_missing_configs(self, # type: HummingbotApplication config_map): - missings = missing_required_configs(config_map) + missings = missing_required_configs_legacy(config_map) for config in missings: - await self.prompt_a_config(config) + await self.prompt_a_config_legacy(config) if self.app.to_stop_config: self.app.to_stop_config = False return - if missing_required_configs(config_map): + if missing_required_configs_legacy(config_map): return missings + (await self._prompt_missing_configs(config_map)) return missings @@ -234,14 +234,14 @@ async def asset_ratio_maintenance_prompt(self, # type: HummingbotApplication required_if=lambda: True, type_str="bool", validator=validate_bool) - await self.prompt_a_config(cvar) + await self.prompt_a_config_legacy(cvar) if cvar.value: config_map['inventory_target_base_pct'].value = round(base_ratio * Decimal('100'), 1) else: if self.app.to_stop_config: self.app.to_stop_config = False return - await self.prompt_a_config(config_map["inventory_target_base_pct"]) + await self.prompt_a_config_legacy(config_map["inventory_target_base_pct"]) async def inventory_price_prompt( self, # type: HummingbotApplication @@ -275,7 +275,7 @@ async def inventory_price_prompt( v, min_value=Decimal("0"), inclusive=True ), ) - await self.prompt_a_config(cvar) + await self.prompt_a_config_legacy(cvar) config_map[key].value = cvar.value try: diff --git a/hummingbot/client/command/connect_command.py b/hummingbot/client/command/connect_command.py index b54ade8b3c..ef8b60309c 100644 --- a/hummingbot/client/command/connect_command.py +++ b/hummingbot/client/command/connect_command.py @@ -10,7 +10,7 @@ from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.client.config.global_config_map import global_config_map from hummingbot.user.user_balances import UserBalances -from hummingbot.client.config.config_helpers import save_to_yml +from hummingbot.client.config.config_helpers import save_to_yml_legacy from hummingbot.connector.other.celo.celo_cli import CeloCLI from hummingbot.connector.connector_status import get_connector_status if TYPE_CHECKING: @@ -59,7 +59,7 @@ async def connect_exchange(self, # type: HummingbotApplication to_connect = False if to_connect: for config in exchange_configs: - await self.prompt_a_config(config) + await self.prompt_a_config_legacy(config) if self.app.to_stop_config: self.app.to_stop_config = False return @@ -161,13 +161,13 @@ async def connect_ethereum(self, # type: HummingbotApplication public_address = Security.add_private_key(private_key) global_config_map["ethereum_wallet"].value = public_address if global_config_map["ethereum_rpc_url"].value is None: - await self.prompt_a_config(global_config_map["ethereum_rpc_url"]) + await self.prompt_a_config_legacy(global_config_map["ethereum_rpc_url"]) if global_config_map["ethereum_rpc_ws_url"].value is None: - await self.prompt_a_config(global_config_map["ethereum_rpc_ws_url"]) + await self.prompt_a_config_legacy(global_config_map["ethereum_rpc_ws_url"]) if self.app.to_stop_config: self.app.to_stop_config = False return - save_to_yml(GLOBAL_CONFIG_PATH, global_config_map) + save_to_yml_legacy(GLOBAL_CONFIG_PATH, global_config_map) err_msg = UserBalances.validate_ethereum_wallet() if err_msg is None: self._notify(f"Wallet {public_address} connected to hummingbot.") @@ -189,9 +189,9 @@ async def connect_celo(self, # type: HummingbotApplication if answer.lower() not in ("yes", "y"): to_connect = False if to_connect: - await self.prompt_a_config(global_config_map["celo_address"]) - await self.prompt_a_config(global_config_map["celo_password"]) - save_to_yml(GLOBAL_CONFIG_PATH, global_config_map) + await self.prompt_a_config_legacy(global_config_map["celo_address"]) + await self.prompt_a_config_legacy(global_config_map["celo_password"]) + save_to_yml_legacy(GLOBAL_CONFIG_PATH, global_config_map) err_msg = await self.validate_n_connect_celo(True, global_config_map["celo_address"].value, diff --git a/hummingbot/client/command/create_command.py b/hummingbot/client/command/create_command.py index f54ae60ea8..488782c7e7 100644 --- a/hummingbot/client/command/create_command.py +++ b/hummingbot/client/command/create_command.py @@ -3,23 +3,27 @@ import os import shutil +from pydantic import ValidationError + +from hummingbot.client.config.config_data_types import BaseClientModel, BaseStrategyConfigMap from hummingbot.client.config.config_var import ConfigVar from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.client.config.config_helpers import ( get_strategy_config_map, parse_cvar_value, default_strategy_file_path, - save_to_yml, + save_to_yml_legacy, get_strategy_template_path, format_config_file_name, - parse_config_default_to_text + parse_config_default_to_text, + retrieve_validation_error_msg, + save_to_yml, ) from hummingbot.client.settings import CONF_FILE_PATH, required_exchanges from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.security import Security -from hummingbot.client.config.config_validators import validate_strategy from hummingbot.client.ui.completer import load_completer -from typing import TYPE_CHECKING, Dict, Optional +from typing import TYPE_CHECKING, Dict, Optional, Any if TYPE_CHECKING: from hummingbot.client.hummingbot_application import HummingbotApplication @@ -36,25 +40,83 @@ def create(self, # type: HummingbotApplication safe_ensure_future(self.prompt_for_configuration(file_name)) - async def prompt_for_configuration(self, # type: HummingbotApplication - file_name): + async def prompt_for_configuration( + self, # type: HummingbotApplication + file_name, + ): self.app.clear_input() self.placeholder_mode = True self.app.hide_input = True required_exchanges.clear() - strategy_config = ConfigVar(key="strategy", - prompt="What is your market making strategy? >>> ", - validator=validate_strategy) - await self.prompt_a_config(strategy_config) + strategy = await self.get_strategy_name() + if self.app.to_stop_config: self.stop_config() return - strategy = strategy_config.value + config_map = get_strategy_config_map(strategy) - config_map_backup = copy.deepcopy(config_map) self._notify(f"Please see https://docs.hummingbot.io/strategies/{strategy.replace('_', '-')}/ " f"while setting up these below configuration.") + + if isinstance(config_map, BaseStrategyConfigMap): + await self.prompt_for_model_config(config_map) + elif config_map is not None: + await self.prompt_for_configuration_legacy(file_name, strategy, config_map) + self.app.to_stop_config = True + else: + self.app.to_stop_config = True + + if self.app.to_stop_config: + self.stop_config() + return + + file_name = await self.save_config_to_file(file_name, config_map) + self.strategy_file_name = file_name + self.strategy_name = strategy + self.strategy_config_map = config_map + # Reload completer here otherwise the new file will not appear + self.app.input_field.completer = load_completer(self) + self.placeholder_mode = False + self.app.hide_input = False + + await self.verify_status() + + async def get_strategy_name( + self, # type: HummingbotApplication + ) -> Optional[str]: + strategy = None + strategy_config = BaseStrategyConfigMap.construct() + await self.prompt_for_model_config(strategy_config) + if self.app.to_stop_config: + self.stop_config() + else: + strategy = strategy_config.strategy + return strategy + + async def prompt_for_model_config( + self, # type: HummingbotApplication + config_map: BaseClientModel, + ): + for key, field in config_map.__fields__.items(): + client_data = config_map.get_client_data(key) + if ( + client_data is not None + and (client_data.prompt_on_new and field.required) + ): + new_config_value = await self.prompt_a_config(config_map, key) + if self.app.to_stop_config: + break + elif isinstance(new_config_value, BaseClientModel): + await self.prompt_for_model_config(new_config_value) + + async def prompt_for_configuration_legacy( + self, # type: HummingbotApplication + file_name, + strategy: str, + config_map: Dict, + ): + config_map_backup = copy.deepcopy(config_map) # assign default values and reset those not required for config in config_map.values(): if config.required: @@ -64,7 +126,7 @@ async def prompt_for_configuration(self, # type: HummingbotApplication for config in config_map.values(): if config.prompt_on_new and config.required: if not self.app.to_stop_config: - await self.prompt_a_config(config) + await self.prompt_a_config_legacy(config) else: break else: @@ -84,29 +146,47 @@ async def prompt_for_configuration(self, # type: HummingbotApplication strategy_path = os.path.join(CONF_FILE_PATH, file_name) template = get_strategy_template_path(strategy) shutil.copy(template, strategy_path) - save_to_yml(strategy_path, config_map) + save_to_yml_legacy(strategy_path, config_map) self.strategy_file_name = file_name self.strategy_name = strategy + self.strategy_config = None # Reload completer here otherwise the new file will not appear self.app.input_field.completer = load_completer(self) self._notify(f"A new config file {self.strategy_file_name} created.") self.placeholder_mode = False self.app.hide_input = False + + await self.verify_status() + + async def prompt_a_config( + self, # type: HummingbotApplication + model: BaseClientModel, + config: str, + input_value=None, + ) -> Any: + if input_value is None: + prompt = await model.get_client_prompt(config) + prompt = f"{prompt} >>> " + client_data = model.get_client_data(config) + input_value = await self.app.prompt(prompt=prompt, is_password=client_data.is_secure) + + if self.app.to_stop_config: + return try: - timeout = float(global_config_map["create_command_timeout"].value) - all_status_go = await asyncio.wait_for(self.status_check_all(), timeout) - except asyncio.TimeoutError: - self._notify("\nA network error prevented the connection check to complete. See logs for more details.") - self.strategy_file_name = None - self.strategy_name = None - raise - if all_status_go: - self._notify("\nEnter \"start\" to start market making.") + model.__setattr__(config, input_value) + new_config_value = model.__getattribute__(config) + except ValidationError as e: + err_msg = retrieve_validation_error_msg(e) + self._notify(err_msg) + new_config_value = await self.prompt_a_config(model, config) + return new_config_value - async def prompt_a_config(self, # type: HummingbotApplication - config: ConfigVar, - input_value=None, - assign_default=True): + async def prompt_a_config_legacy( + self, # type: HummingbotApplication + config: ConfigVar, + input_value=None, + assign_default=True, + ): if config.key == "inventory_price": await self.inventory_price_prompt(self.strategy_config_map, input_value) return @@ -122,10 +202,26 @@ async def prompt_a_config(self, # type: HummingbotApplication err_msg = await config.validate(input_value) if err_msg is not None: self._notify(err_msg) - await self.prompt_a_config(config) + await self.prompt_a_config_legacy(config) else: config.value = value + async def save_config_to_file( + self, # type: HummingbotApplication + file_name: Optional[str], + config_map: BaseStrategyConfigMap, + ) -> str: + if file_name is None: + file_name = await self.prompt_new_file_name(config_map.strategy) + if self.app.to_stop_config: + self.stop_config() + self.app.set_text("") + return + self.app.change_prompt(prompt=">>> ") + strategy_path = os.path.join(CONF_FILE_PATH, file_name) + save_to_yml(strategy_path, config_map) + return file_name + async def prompt_new_file_name(self, # type: HummingbotApplication strategy): file_name = default_strategy_file_path(strategy) @@ -142,23 +238,39 @@ async def prompt_new_file_name(self, # type: HummingbotApplication else: return input - async def update_all_secure_configs(self # type: HummingbotApplication - ): + async def update_all_secure_configs_legacy( + self # type: HummingbotApplication + ): await Security.wait_til_decryption_done() Security.update_config_map(global_config_map) - if self.strategy_config_map is not None: + if self.strategy_config_map is not None and not isinstance(self.strategy_config_map, BaseStrategyConfigMap): Security.update_config_map(self.strategy_config_map) + async def verify_status( + self # type: HummingbotApplication + ): + try: + timeout = float(global_config_map["create_command_timeout"].value) + all_status_go = await asyncio.wait_for(self.status_check_all(), timeout) + except asyncio.TimeoutError: + self._notify("\nA network error prevented the connection check to complete. See logs for more details.") + self.strategy_file_name = None + self.strategy_name = None + self.strategy_config = None + raise + if all_status_go: + self._notify("\nEnter \"start\" to start market making.") + def stop_config( self, config_map: Optional[Dict[str, ConfigVar]] = None, config_map_backup: Optional[Dict[str, ConfigVar]] = None, ): if config_map is not None and config_map_backup is not None: - self.restore_config(config_map, config_map_backup) + self.restore_config_legacy(config_map, config_map_backup) self.app.to_stop_config = False @staticmethod - def restore_config(config_map: Dict[str, ConfigVar], config_map_backup: Dict[str, ConfigVar]): + def restore_config_legacy(config_map: Dict[str, ConfigVar], config_map_backup: Dict[str, ConfigVar]): for key in config_map: config_map[key] = config_map_backup[key] diff --git a/hummingbot/client/command/start_command.py b/hummingbot/client/command/start_command.py index 39f89854b7..2448b1a1e0 100644 --- a/hummingbot/client/command/start_command.py +++ b/hummingbot/client/command/start_command.py @@ -175,7 +175,7 @@ async def confirm_oracle_conversion_rate(self, # type: HummingbotApplication "this strategy (Yes/No) >>> ", required_if=lambda: True, validator=lambda v: validate_bool(v)) - await self.prompt_a_config(config) + await self.prompt_a_config_legacy(config) if config.value: result = True except OracleRateUnavailable: diff --git a/hummingbot/client/command/status_command.py b/hummingbot/client/command/status_command.py index 49011a1dad..7d8952e2ac 100644 --- a/hummingbot/client/command/status_command.py +++ b/hummingbot/client/command/status_command.py @@ -8,13 +8,17 @@ ) import inspect from typing import List, Dict + +from pydantic import ValidationError, validate_model + from hummingbot import check_dev_mode +from hummingbot.client.config.config_data_types import BaseStrategyConfigMap from hummingbot.logger.application_warning import ApplicationWarning from hummingbot.connector.connector_base import ConnectorBase from hummingbot.core.network_iterator import NetworkStatus from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.config_helpers import ( - missing_required_configs, + missing_required_configs_legacy, get_strategy_config_map ) from hummingbot.client.config.security import Security @@ -100,7 +104,7 @@ async def validate_required_connections(self) -> Dict[str, str]: if err_msg is not None: invalid_conns["celo"] = err_msg if not any([str(exchange).endswith("paper_trade") for exchange in required_exchanges]): - await self.update_all_secure_configs() + await self.update_all_secure_configs_legacy() connections = await UserBalances.instance().update_exchanges(exchanges=required_exchanges) invalid_conns.update({ex: err_msg for ex, err_msg in connections.items() if ex in required_exchanges and err_msg is not None}) @@ -110,11 +114,27 @@ async def validate_required_connections(self) -> Dict[str, str]: invalid_conns["ethereum"] = err_msg return invalid_conns - def missing_configurations(self) -> List[str]: - missing_globals = missing_required_configs(global_config_map) - missing_configs = missing_required_configs(get_strategy_config_map(self.strategy_name)) + def missing_configurations_legacy(self) -> List[str]: + missing_globals = missing_required_configs_legacy(global_config_map) + config_map = self.strategy_config_map + missing_configs = [] + if not isinstance(config_map, BaseStrategyConfigMap): + missing_configs = missing_required_configs_legacy(get_strategy_config_map(self.strategy_name)) return missing_globals + missing_configs + def validate_configs(self) -> List[ValidationError]: + config_map = self.strategy_config_map + validation_errors = [] + if isinstance(config_map, BaseStrategyConfigMap): + validation_results = validate_model(type(config_map), config_map.dict()) + if len(validation_results) == 3 and validation_results[2] is not None: + validation_errors = validation_results[2].errors() + validation_errors = [ + f"{'.'.join(e['loc'])} - {e['msg']}" + for e in validation_errors + ] + return validation_errors + def status(self, # type: HummingbotApplication live: bool = False): safe_ensure_future(self.status_check_all(live=live), loop=self.ev_loop) @@ -162,14 +182,19 @@ async def status_check_all(self, # type: HummingbotApplication elif notify_success: self._notify(' - Exchange check: All connections confirmed.') - missing_configs = self.missing_configurations() + missing_configs = self.missing_configurations_legacy() if missing_configs: self._notify(" - Strategy check: Incomplete strategy configuration. The following values are missing.") for config in missing_configs: self._notify(f" {config.key}") elif notify_success: self._notify(' - Strategy check: All required parameters confirmed.') - if invalid_conns or missing_configs: + validation_errors = self.validate_configs() + if len(validation_errors) != 0: + self._notify(" - Strategy check: Validation of the config maps failed. The following errors were flagged.") + for error in validation_errors: + self._notify(f" {error}") + if invalid_conns or missing_configs or len(validation_errors) != 0: return False loading_markets: List[ConnectorBase] = [] diff --git a/hummingbot/client/config/config_data_types.py b/hummingbot/client/config/config_data_types.py index c12d484696..b96c1f858a 100644 --- a/hummingbot/client/config/config_data_types.py +++ b/hummingbot/client/config/config_data_types.py @@ -1,11 +1,16 @@ +import inspect from dataclasses import dataclass +from decimal import Decimal from enum import Enum -from typing import Any, Callable, Optional +from typing import Any, Callable, Optional, List -from pydantic import BaseModel +import yaml +from pydantic import BaseModel, Field, validator from pydantic.schema import default_ref_template +from yaml import SafeDumper from hummingbot.client.config.config_helpers import strategy_config_schema_encoder +from hummingbot.client.config.config_validators import validate_strategy class ClientConfigEnum(Enum): @@ -13,10 +18,27 @@ def __str__(self): return self.value +def decimal_representer(dumper: SafeDumper, data: Decimal): + return dumper.represent_float(float(data)) + + +def enum_representer(dumper: SafeDumper, data: ClientConfigEnum): + return dumper.represent_str(str(data)) + + +yaml.add_representer( + data_type=Decimal, representer=decimal_representer, Dumper=SafeDumper +) +yaml.add_multi_representer( + data_type=ClientConfigEnum, multi_representer=enum_representer, Dumper=SafeDumper +) + + @dataclass() class ClientFieldData: prompt: Optional[Callable[['BaseClientModel'], str]] = None prompt_on_new: bool = False + is_secure: bool = False class BaseClientModel(BaseModel): @@ -36,12 +58,91 @@ def schema_json( **dumps_kwargs ) - def get_client_prompt(self, attr_name: str) -> Optional[str]: + async def get_client_prompt(self, attr_name: str) -> Optional[str]: prompt = None client_data = self.get_client_data(attr_name) if client_data is not None: - prompt = client_data.prompt(self) + prompt = client_data.prompt + if inspect.iscoroutinefunction(prompt): + prompt = await prompt(self) + else: + prompt = prompt(self) return prompt def get_client_data(self, attr_name: str) -> Optional[ClientFieldData]: return self.__fields__[attr_name].field_info.extra["client_data"] + + def get_description(self, attr_name: str) -> str: + return self.__fields__[attr_name].field_info.description + + def generate_yml_output_str_with_comments(self) -> str: + original_fragments = yaml.safe_dump(self.dict(), sort_keys=False).split("\n") + fragments_with_comments = [self._generate_title()] + self._add_model_fragments(self, fragments_with_comments, original_fragments) + fragments_with_comments.append("\n") # EOF empty line + yml_str = "".join(fragments_with_comments) + return yml_str + + def _generate_title(self) -> str: + title = f"{self.Config.title}" + title = self._adorn_title(title) + return title + + @staticmethod + def _adorn_title(title: str) -> str: + if title: + title = f"### {title} config ###" + title_len = len(title) + title = f"{'#' * title_len}\n{title}\n{'#' * title_len}" + return title + + def _add_model_fragments( + self, + model: 'BaseClientModel', + fragments_with_comments: List[str], + original_fragments: List[str], + original_fragments_idx: int = 0, + model_depth: int = 0, + ) -> int: + comment_prefix = f"\n{' ' * 2 * model_depth}# " + for attr in model.__fields__.keys(): + attr_comment = model.get_description(attr) + if attr_comment is not None: + attr_comment = "".join(f"{comment_prefix}{c}" for c in attr_comment.split("\n")) + if model_depth == 0: + attr_comment = f"\n{attr_comment}" + fragments_with_comments.extend([attr_comment, f"\n{original_fragments[original_fragments_idx]}"]) + elif model_depth == 0: + fragments_with_comments.append(f"\n\n{original_fragments[original_fragments_idx]}") + else: + fragments_with_comments.append(f"\n{original_fragments[original_fragments_idx]}") + original_fragments_idx += 1 + value = model.__getattribute__(attr) + if isinstance(value, BaseClientModel): + original_fragments_idx = self._add_model_fragments( + value, fragments_with_comments, original_fragments, original_fragments_idx, model_depth + 1 + ) + return original_fragments_idx + + +class BaseStrategyConfigMap(BaseClientModel): + strategy: str = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda mi: "What is your market making strategy?", + prompt_on_new=True, + ), + ) + + @validator("strategy", pre=True) + def validate_strategy(cls, v: str): + ret = validate_strategy(v) + if ret is not None: + raise ValueError(ret) + return v + + def _generate_title(self) -> str: + title = " ".join([w.capitalize() for w in f"{self.strategy}".split("_")]) + title = f"{title} Strategy" + title = self._adorn_title(title) + return title diff --git a/hummingbot/client/config/config_helpers.py b/hummingbot/client/config/config_helpers.py index ecc6d8a0aa..c9f17226b9 100644 --- a/hummingbot/client/config/config_helpers.py +++ b/hummingbot/client/config/config_helpers.py @@ -5,14 +5,15 @@ from decimal import Decimal from os import listdir, unlink from os.path import isfile, join -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Dict, List, Optional, TYPE_CHECKING, Union import ruamel.yaml from eth_account import Account from pydantic import ValidationError from pydantic.json import pydantic_encoder +from pydantic.main import ModelMetaclass -from hummingbot import get_strategy_list +from hummingbot import get_strategy_list, root_path from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.fee_overrides_config_map import fee_overrides_config_map from hummingbot.client.config.global_config_map import global_config_map @@ -27,6 +28,9 @@ TRADE_FEES_CONFIG_PATH, ) +if TYPE_CHECKING: # avoid circular import problems + from hummingbot.client.config.config_data_types import BaseStrategyConfigMap + # Use ruamel.yaml to preserve order and comments in .yml file yaml_parser = ruamel.yaml.YAML() @@ -169,17 +173,25 @@ def get_connector_class(connector_name: str) -> Callable: return getattr(mod, conn_setting.class_name()) -def get_strategy_config_map(strategy: str) -> Optional[Dict[str, ConfigVar]]: +def get_strategy_config_map( + strategy: str +) -> Optional[Union["BaseStrategyConfigMap", Dict[str, ConfigVar]]]: """ Given the name of a strategy, find and load strategy-specific config map. """ + config_map = None try: - cm_key = f"{strategy}_config_map" - strategy_module = __import__(f"hummingbot.strategy.{strategy}.{cm_key}", - fromlist=[f"hummingbot.strategy.{strategy}"]) - return getattr(strategy_module, cm_key) + config_map = get_strategy_pydantic_config(strategy) + if config_map is None: # legacy + cm_key = f"{strategy}_config_map" + strategy_module = __import__(f"hummingbot.strategy.{strategy}.{cm_key}", + fromlist=[f"hummingbot.strategy.{strategy}"]) + config_map = getattr(strategy_module, cm_key) + else: + config_map = config_map.construct() except Exception as e: logging.getLogger().error(e, exc_info=True) + return config_map def get_strategy_starter_file(strategy: str) -> Callable: @@ -222,6 +234,20 @@ def validate_strategy_file(file_path: str) -> Optional[str]: return None +def get_strategy_pydantic_config(strategy_name: str) -> Optional[ModelMetaclass]: + pydantic_cm_class = None + try: + pydantic_cm_pkg = f"{strategy_name}_config_map_pydantic" + if isfile(f"{root_path()}/hummingbot/strategy/{strategy_name}/{pydantic_cm_pkg}.py"): + pydantic_cm_class_name = f"{''.join([s.capitalize() for s in strategy_name.split('_')])}ConfigMap" + pydantic_cm_mod = __import__(f"hummingbot.strategy.{strategy_name}.{pydantic_cm_pkg}", + fromlist=[f"{pydantic_cm_class_name}"]) + pydantic_cm_class = getattr(pydantic_cm_mod, pydantic_cm_class_name) + except ImportError: + logging.getLogger().exception(f"Could not import Pydantic configs for {strategy_name}.") + return pydantic_cm_class + + async def update_strategy_config_map_from_file(yml_path: str) -> str: strategy = strategy_name_from_file(yml_path) config_map = get_strategy_config_map(strategy) @@ -280,7 +306,7 @@ async def load_yml_into_cm(yml_path: str, template_file_path: str, cm: Dict[str, # copy the new file template shutil.copy(template_file_path, yml_path) # save the old variables into the new config file - save_to_yml(yml_path, cm) + save_to_yml_legacy(yml_path, cm) except Exception as e: logging.getLogger().error("Error loading configs. Your config file may be corrupt. %s" % (e,), exc_info=True) @@ -299,11 +325,11 @@ async def read_system_configs_from_yml(): def save_system_configs_to_yml(): - save_to_yml(GLOBAL_CONFIG_PATH, global_config_map) - save_to_yml(TRADE_FEES_CONFIG_PATH, fee_overrides_config_map) + save_to_yml_legacy(GLOBAL_CONFIG_PATH, global_config_map) + save_to_yml_legacy(TRADE_FEES_CONFIG_PATH, fee_overrides_config_map) -def save_to_yml(yml_path: str, cm: Dict[str, ConfigVar]): +def save_to_yml_legacy(yml_path: str, cm: Dict[str, ConfigVar]): """ Write current config saved a single config map into each a single yml file """ @@ -326,11 +352,20 @@ def save_to_yml(yml_path: str, cm: Dict[str, ConfigVar]): logging.getLogger().error("Error writing configs: %s" % (str(e),), exc_info=True) +def save_to_yml(yml_path: str, cm: "BaseStrategyConfigMap"): + try: + cm_yml_str = cm.generate_yml_output_str_with_comments() + with open(yml_path, "w+") as outfile: + outfile.write(cm_yml_str) + except Exception as e: + logging.getLogger().error("Error writing configs: %s" % (str(e),), exc_info=True) + + async def write_config_to_yml(strategy_name, strategy_file_name): strategy_config_map = get_strategy_config_map(strategy_name) strategy_file_path = join(CONF_FILE_PATH, strategy_file_name) - save_to_yml(strategy_file_path, strategy_config_map) - save_to_yml(GLOBAL_CONFIG_PATH, global_config_map) + save_to_yml_legacy(strategy_file_path, strategy_config_map) + save_to_yml_legacy(GLOBAL_CONFIG_PATH, global_config_map) async def create_yml_files(): @@ -396,7 +431,7 @@ def config_map_complete(config_map): return not any(c.required and c.value is None for c in config_map.values()) -def missing_required_configs(config_map): +def missing_required_configs_legacy(config_map): return [c for c in config_map.values() if c.required and c.value is None and not c.is_connect_key] diff --git a/hummingbot/client/hummingbot_application.py b/hummingbot/client/hummingbot_application.py index ceb190a3ed..fd76f56656 100644 --- a/hummingbot/client/hummingbot_application.py +++ b/hummingbot/client/hummingbot_application.py @@ -7,6 +7,7 @@ from typing import List, Dict, Optional, Tuple, Deque from hummingbot.client.command import __all__ as commands +from hummingbot.client.config.config_data_types import BaseStrategyConfigMap from hummingbot.client.tab import __all__ as tab_classes from hummingbot.core.clock import Clock from hummingbot.exceptions import ArgumentParserError @@ -77,8 +78,9 @@ def __init__(self): self.markets: Dict[str, ExchangeBase] = {} # strategy file name and name get assigned value after import or create command - self._strategy_file_name: str = None - self.strategy_name: str = None + self._strategy_file_name: Optional[str] = None + self.strategy_name: Optional[str] = None + self._strategy_config_map: Optional[BaseStrategyConfigMap] = None self.strategy_task: Optional[asyncio.Task] = None self.strategy: Optional[StrategyBase] = None self.market_pair: Optional[CrossExchangeMarketPair] = None @@ -121,10 +123,16 @@ def strategy_file_name(self, value: Optional[str]): @property def strategy_config_map(self): + if self._strategy_config_map is not None: + return self._strategy_config_map if self.strategy_name is not None: return get_strategy_config_map(self.strategy_name) return None + @strategy_config_map.setter + def strategy_config_map(self, config_map: BaseStrategyConfigMap): + self._strategy_config_map = config_map + def _notify(self, msg: str): self.app.log(msg) for notifier in self.notifiers: diff --git a/hummingbot/client/ui/style.py b/hummingbot/client/ui/style.py index c83e7b91fc..08814fa2f9 100644 --- a/hummingbot/client/ui/style.py +++ b/hummingbot/client/ui/style.py @@ -1,7 +1,7 @@ from prompt_toolkit.styles import Style from prompt_toolkit.utils import is_windows from hummingbot.client.config.global_config_map import global_config_map -from hummingbot.client.config.config_helpers import save_to_yml +from hummingbot.client.config.config_helpers import save_to_yml_legacy from hummingbot.client.settings import GLOBAL_CONFIG_PATH @@ -111,7 +111,7 @@ def reset_style(config_map=global_config_map, save=True): # Save configuration if save: file_path = GLOBAL_CONFIG_PATH - save_to_yml(file_path, config_map) + save_to_yml_legacy(file_path, config_map) # Apply & return style return load_style(config_map) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py index ef04712821..78f185f699 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -4,7 +4,12 @@ from pydantic import Field, validator, root_validator -from hummingbot.client.config.config_data_types import BaseClientModel, ClientConfigEnum, ClientFieldData +from hummingbot.client.config.config_data_types import ( + BaseClientModel, + BaseStrategyConfigMap, + ClientConfigEnum, + ClientFieldData, +) from hummingbot.client.config.config_validators import ( validate_bool, validate_datetime_iso_string, @@ -178,7 +183,7 @@ class Config: } -class AvellanedaMarketMakingConfigMap(BaseClientModel): +class AvellanedaMarketMakingConfigMap(BaseStrategyConfigMap): strategy: str = Field(default="avellaneda_market_making", client_data=None) exchange: ClientConfigEnum( value="Exchanges", # noqa: F821 @@ -212,7 +217,10 @@ class AvellanedaMarketMakingConfigMap(BaseClientModel): default=..., description="The strategy order amount.", gt=0, - client_data=ClientFieldData(prompt=lambda mi: AvellanedaMarketMakingConfigMap.order_amount_prompt(mi)) + client_data=ClientFieldData( + prompt=lambda mi: AvellanedaMarketMakingConfigMap.order_amount_prompt(mi), + prompt_on_new=True, + ) ) order_optimization_enabled: bool = Field( default=True, @@ -367,6 +375,7 @@ class AvellanedaMarketMakingConfigMap(BaseClientModel): ) class Config: + title = "avellaneda_market_making" validate_assignment = True @classmethod diff --git a/hummingbot/strategy/avellaneda_market_making/start.py b/hummingbot/strategy/avellaneda_market_making/start.py index ad34bb79d8..d0ead627eb 100644 --- a/hummingbot/strategy/avellaneda_market_making/start.py +++ b/hummingbot/strategy/avellaneda_market_making/start.py @@ -1,4 +1,3 @@ -import datetime import pandas as pd from decimal import Decimal from typing import ( @@ -9,6 +8,13 @@ from hummingbot import data_path import os.path from hummingbot.client.hummingbot_application import HummingbotApplication +from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( + AvellanedaMarketMakingConfigMap, + MultiOrderLevelModel, + TrackHangingOrdersModel, + FromDateToDateModel, + DailyBetweenTimesModel, +) from hummingbot.strategy.conditional_execution_state import ( RunAlwaysExecutionState, RunInTimeConditionalExecutionState @@ -17,28 +23,35 @@ from hummingbot.strategy.avellaneda_market_making import ( AvellanedaMarketMakingStrategy, ) -from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map import avellaneda_market_making_config_map as c_map def start(self): try: - order_amount = c_map.get("order_amount").value - order_optimization_enabled = c_map.get("order_optimization_enabled").value - order_refresh_time = c_map.get("order_refresh_time").value - exchange = c_map.get("exchange").value.lower() - raw_trading_pair = c_map.get("market").value - max_order_age = c_map.get("max_order_age").value - inventory_target_base_pct = 0 if c_map.get("inventory_target_base_pct").value is None else \ - c_map.get("inventory_target_base_pct").value / Decimal('100') - filled_order_delay = c_map.get("filled_order_delay").value - order_refresh_tolerance_pct = c_map.get("order_refresh_tolerance_pct").value / Decimal('100') - order_levels = c_map.get("order_levels").value - level_distances = c_map.get("level_distances").value - order_override = c_map.get("order_override").value - hanging_orders_enabled = c_map.get("hanging_orders_enabled").value - - hanging_orders_cancel_pct = c_map.get("hanging_orders_cancel_pct").value / Decimal('100') - add_transaction_costs_to_orders = c_map.get("add_transaction_costs").value + c_map: AvellanedaMarketMakingConfigMap = self.strategy_config_map + order_amount = c_map.order_amount + order_optimization_enabled = c_map.order_optimization_enabled + order_refresh_time = c_map.order_refresh_time + exchange = c_map.exchange + raw_trading_pair = c_map.market + max_order_age = c_map.max_order_age + inventory_target_base_pct = 0 if c_map.inventory_target_base_pct is None else \ + c_map.inventory_target_base_pct / Decimal('100') + filled_order_delay = c_map.filled_order_delay + order_refresh_tolerance_pct = c_map.order_refresh_tolerance_pct / Decimal('100') + if isinstance(c_map.order_levels_mode, MultiOrderLevelModel): + order_levels = c_map.order_levels_mode.order_levels + level_distances = c_map.order_levels_mode.level_distances + else: + order_levels = 1 + level_distances = 0 + order_override = c_map.order_override + if isinstance(c_map.hanging_orders_mode, TrackHangingOrdersModel): + hanging_orders_enabled = True + hanging_orders_cancel_pct = c_map.hanging_orders_mode.hanging_orders_cancel_pct / Decimal('100') + else: + hanging_orders_enabled = False + hanging_orders_cancel_pct = Decimal("0") + add_transaction_costs_to_orders = c_map.add_transaction_costs trading_pair: str = raw_trading_pair maker_assets: Tuple[str, str] = self._initialize_market_assets(exchange, [trading_pair])[0] @@ -48,34 +61,33 @@ def start(self): self.market_trading_pair_tuples = [MarketTradingPairTuple(*maker_data)] strategy_logging_options = AvellanedaMarketMakingStrategy.OPTION_LOG_ALL - risk_factor = c_map.get("risk_factor").value - order_amount_shape_factor = c_map.get("order_amount_shape_factor").value - - execution_timeframe = c_map.get("execution_timeframe").value - - start_time = c_map.get("start_time").value - end_time = c_map.get("end_time").value + risk_factor = c_map.risk_factor + order_amount_shape_factor = c_map.order_amount_shape_factor - if execution_timeframe == "from_date_to_date": - start_time = datetime.datetime.fromisoformat(start_time) - end_time = datetime.datetime.fromisoformat(end_time) + execution_timeframe = c_map.execution_timeframe_model.Config.title + if isinstance(c_map.execution_timeframe_model, FromDateToDateModel): + start_time = c_map.execution_timeframe_model.start_datetime + end_time = c_map.execution_timeframe_model.end_datetime execution_state = RunInTimeConditionalExecutionState(start_timestamp=start_time, end_timestamp=end_time) - if execution_timeframe == "daily_between_times": - start_time = datetime.datetime.strptime(start_time, '%H:%M:%S').time() - end_time = datetime.datetime.strptime(end_time, '%H:%M:%S').time() + elif isinstance(c_map.execution_timeframe_model, DailyBetweenTimesModel): + start_time = c_map.execution_timeframe_model.start_time + end_time = c_map.execution_timeframe_model.end_time execution_state = RunInTimeConditionalExecutionState(start_timestamp=start_time, end_timestamp=end_time) - if execution_timeframe == "infinite": + else: + start_time = None + end_time = None execution_state = RunAlwaysExecutionState() - min_spread = c_map.get("min_spread").value - volatility_buffer_size = c_map.get("volatility_buffer_size").value - trading_intensity_buffer_size = c_map.get("trading_intensity_buffer_size").value - should_wait_order_cancel_confirmation = c_map.get("should_wait_order_cancel_confirmation") + min_spread = c_map.min_spread + volatility_buffer_size = c_map.volatility_buffer_size + trading_intensity_buffer_size = c_map.trading_intensity_buffer_size + should_wait_order_cancel_confirmation = c_map.should_wait_order_cancel_confirmation debug_csv_path = os.path.join(data_path(), HummingbotApplication.main_application().strategy_file_name.rsplit('.', 1)[0] + f"_{pd.Timestamp.now().strftime('%Y-%m-%d_%H-%M-%S')}.csv") self.strategy = AvellanedaMarketMakingStrategy() + self.logger().debug("Starting !!!!!!!!!!!!") self.strategy.init_params( market_info=MarketTradingPairTuple(*maker_data), order_amount=order_amount, diff --git a/test/hummingbot/client/command/test_create_command.py b/test/hummingbot/client/command/test_create_command.py index 5cbd5da196..c98fa8b756 100644 --- a/test/hummingbot/client/command/test_create_command.py +++ b/test/hummingbot/client/command/test_create_command.py @@ -62,7 +62,7 @@ async def run_coro_that_raises(coro: Awaitable): raise RuntimeError @patch("shutil.copy") - @patch("hummingbot.client.command.create_command.save_to_yml") + @patch("hummingbot.client.command.create_command.save_to_yml_legacy") @patch("hummingbot.client.config.security.Security.is_decryption_done") @patch("hummingbot.client.command.status_command.StatusCommand.validate_required_connections") @patch("hummingbot.core.utils.market_price.get_last_price") @@ -101,7 +101,7 @@ def test_prompt_for_configuration_re_prompts_on_lower_than_minimum_amount( self.assertTrue(self.cli_mock_assistant.check_log_called_with(msg="Value must be more than 0.")) @patch("shutil.copy") - @patch("hummingbot.client.command.create_command.save_to_yml") + @patch("hummingbot.client.command.create_command.save_to_yml_legacy") @patch("hummingbot.client.config.security.Security.is_decryption_done") @patch("hummingbot.client.command.status_command.StatusCommand.validate_required_connections") @patch("hummingbot.core.utils.market_price.get_last_price") @@ -174,7 +174,7 @@ def test_create_command_restores_config_map_after_config_stop_on_new_file_prompt self.assertEqual(original_exchange, strategy_config["exchange"].value) @patch("shutil.copy") - @patch("hummingbot.client.command.create_command.save_to_yml") + @patch("hummingbot.client.command.create_command.save_to_yml_legacy") @patch("hummingbot.client.config.security.Security.is_decryption_done") @patch("hummingbot.client.command.status_command.StatusCommand.validate_required_connections") @patch("hummingbot.core.utils.market_price.get_last_price") diff --git a/test/hummingbot/client/config/test_config_data_types.py b/test/hummingbot/client/config/test_config_data_types.py new file mode 100644 index 0000000000..47f095b19d --- /dev/null +++ b/test/hummingbot/client/config/test_config_data_types.py @@ -0,0 +1,114 @@ +import json +import unittest +from decimal import Decimal + +from pydantic import Field + +from hummingbot.client.config.config_data_types import ( + BaseClientModel, ClientFieldData, ClientConfigEnum, BaseStrategyConfigMap +) + + +class BaseClientModelTest(unittest.TestCase): + def test_schema_encoding_removes_client_data_functions(self): + class DummyModel(BaseClientModel): + some_attr: str = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda mi: "Some prompt?", + prompt_on_new=True, + ), + ) + + schema = DummyModel.schema_json() + j = json.loads(schema) + expected = { + "is_secure": False, + "prompt": None, + "prompt_on_new": True, + } + self.assertEqual(expected, j["properties"]["some_attr"]["client_data"]) + + def test_generate_yml_output_dict_with_comments(self): + class SomeEnum(ClientConfigEnum): + ONE = "one" + + class DoubleNestedModel(BaseClientModel): + double_nested_attr: float = Field( + default=3.0, + description="Double nested attr description" + ) + + class NestedModel(BaseClientModel): + nested_attr: str = Field( + default="some value", + description="Nested attr\nmultiline description", + ) + double_nested_model: DoubleNestedModel = Field( + default=DoubleNestedModel(), + ) + + class DummyModel(BaseClientModel): + some_attr: SomeEnum = Field( + default=SomeEnum.ONE, + description="Some description", + ) + nested_model: NestedModel = Field( + default=NestedModel(), + description="Nested model description", + ) + another_attr: Decimal = Field( + default=Decimal("1.0"), + description="Some other\nmultiline description", + ) + non_nested_no_description: int = Field( + default=2, + ) + + class Config: + title = "dummy_model" + + instance = DummyModel() + res_str = instance.generate_yml_output_str_with_comments() + + expected_str = """\ +############################## +### dummy_model config ### +############################## + +# Some description +some_attr: one + +# Nested model description +nested_model: + # Nested attr + # multiline description + nested_attr: some value + double_nested_model: + # Double nested attr description + double_nested_attr: 3.0 + +# Some other +# multiline description +another_attr: 1.0 + +non_nested_no_description: 2 +""" + + self.assertEqual(expected_str, res_str) + + +class BaseStrategyConfigMapTest(unittest.TestCase): + def test_generate_yml_output_dict_title(self): + instance = BaseStrategyConfigMap(strategy="pure_market_making") + res_str = instance.generate_yml_output_str_with_comments() + + expected_str = """\ +############################################## +### Pure Market Making Strategy config ### +############################################## + +strategy: pure_market_making +""" + + self.assertEqual(expected_str, res_str) diff --git a/test/hummingbot/client/config/test_config_helpers.py b/test/hummingbot/client/config/test_config_helpers.py index 18d2562605..553799d9a8 100644 --- a/test/hummingbot/client/config/test_config_helpers.py +++ b/test/hummingbot/client/config/test_config_helpers.py @@ -1,7 +1,15 @@ import asyncio import unittest +from pathlib import Path +from tempfile import TemporaryDirectory from typing import Awaitable +from hummingbot.client.config.config_data_types import BaseStrategyConfigMap +from hummingbot.client.config.config_helpers import get_strategy_config_map, save_to_yml +from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( + AvellanedaMarketMakingConfigMap +) + class ConfigHelpersTest(unittest.TestCase): def setUp(self) -> None: @@ -17,3 +25,25 @@ def get_async_sleep_fn(delay: float): async def async_sleep(*_, **__): await asyncio.sleep(delay) return async_sleep + + def test_get_strategy_config_map(self): + cm = get_strategy_config_map(strategy="avellaneda_market_making") + self.assertIsInstance(cm, AvellanedaMarketMakingConfigMap) + self.assertFalse(hasattr(cm, "market")) # uninitialized instance + + def test_save_to_yml(self): + cm = BaseStrategyConfigMap(strategy="pure_market_making") + expected_str = """\ +############################################## +### Pure Market Making Strategy config ### +############################################## + +strategy: pure_market_making +""" + with TemporaryDirectory() as d: + d = Path(d) + temp_file_name = d / "cm.yml" + save_to_yml(str(temp_file_name), cm) + with open(temp_file_name) as f: + actual_str = f.read() + self.assertEqual(expected_str, actual_str) diff --git a/test/hummingbot/client/config/test_config_templates.py b/test/hummingbot/client/config/test_config_templates.py index 389bdbfac9..f9d34c6693 100644 --- a/test/hummingbot/client/config/test_config_templates.py +++ b/test/hummingbot/client/config/test_config_templates.py @@ -1,11 +1,9 @@ #!/usr/bin/env python from os.path import ( - isdir, join, realpath, ) -from os import listdir import logging; logging.basicConfig(level=logging.INFO) import unittest import ruamel.yaml @@ -37,11 +35,26 @@ def test_global_config_template_complete(self): for key in global_config_map: self.assertTrue(key in template_data, f"{key} not in {global_config_template_path}") - def test_strategy_config_template_complete(self): - folder = realpath(join(__file__, "../../../../../hummingbot/strategy")) - # Only include valid directories - strategies = [d for d in listdir(folder) if isdir(join(folder, d)) and not d.startswith("__")] - strategies.sort() + def test_strategy_config_template_complete_legacy(self): + strategies = [ # templates is a legacy approach — new strategies won't use it + "amm_arb", + "arbitrage", + "aroon_oscillator", + "celo_arb", + "cross_exchange_market_making", + "dev_0_hello_world", + "dev_1_get_order_book", + "dev_2_perform_trade", + "dev_5_vwap", + "dev_simple_trade", + "hedge", + "liquidity_mining", + "perpetual_market_making", + "pure_market_making", + "spot_perpetual_arbitrage", + "twap", + "uniswap_v3_lp", + ] for strategy in strategies: strategy_template_path: str = get_strategy_template_path(strategy) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py index 64fec3dadd..14addcd407 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py @@ -1,8 +1,8 @@ -import json +import asyncio import unittest from datetime import datetime, time from pathlib import Path -from typing import Dict +from typing import Dict, Awaitable from unittest.mock import patch import yaml @@ -23,6 +23,7 @@ class AvellanedaMarketMakingConfigMapPydanticTest(unittest.TestCase): @classmethod def setUpClass(cls) -> None: super().setUpClass() + cls.ev_loop = asyncio.get_event_loop() cls.exchange = "binance" cls.base_asset = "COINALPHA" cls.quote_asset = "HBOT" @@ -33,6 +34,10 @@ def setUp(self) -> None: config_settings = self.get_default_map() self.config_map = AvellanedaMarketMakingConfigMap(**config_settings) + def async_run_with_timeout(self, coroutine: Awaitable, timeout: int = 1): + ret = self.ev_loop.run_until_complete(asyncio.wait_for(coroutine, timeout)) + return ret + def get_default_map(self) -> Dict[str, str]: config_settings = { "exchange": self.exchange, @@ -49,15 +54,6 @@ def get_default_map(self) -> Dict[str, str]: } return config_settings - def test_schema_encoding_removes_client_data_functions(self): - s = AvellanedaMarketMakingConfigMap.schema_json() - j = json.loads(s) - expected = { - "prompt": None, - "prompt_on_new": True, - } - self.assertEqual(expected, j["properties"]["market"]["client_data"]) - def test_initial_sequential_build(self): config_map: AvellanedaMarketMakingConfigMap = AvellanedaMarketMakingConfigMap.construct() config_settings = self.get_default_map() @@ -80,7 +76,7 @@ def build_config_map(cm: BaseClientModel, cs: Dict): validate_model(config_map.__class__, config_map.__dict__) def test_order_amount_prompt(self): - prompt = self.config_map.get_client_prompt("order_amount") + prompt = self.async_run_with_timeout(self.config_map.get_client_prompt("order_amount")) expected = f"What is the amount of {self.base_asset} per order?" self.assertEqual(expected, prompt) @@ -89,7 +85,7 @@ def test_maker_trading_pair_prompt(self): exchange = self.config_map.exchange example = AllConnectorSettings.get_example_pairs().get(exchange) - prompt = self.config_map.get_client_prompt("market") + prompt = self.async_run_with_timeout(self.config_map.get_client_prompt("market")) expected = f"Enter the token trading pair you would like to trade on {exchange} (e.g. {example})" self.assertEqual(expected, prompt) @@ -97,19 +93,19 @@ def test_maker_trading_pair_prompt(self): def test_execution_time_prompts(self): self.config_map.execution_timeframe_model = FromDateToDateModel.Config.title model = self.config_map.execution_timeframe_model - prompt = model.get_client_prompt("start_datetime") + prompt = self.async_run_with_timeout(model.get_client_prompt("start_datetime")) expected = "Please enter the start date and time (YYYY-MM-DD HH:MM:SS)" self.assertEqual(expected, prompt) - prompt = model.get_client_prompt("end_datetime") + prompt = self.async_run_with_timeout(model.get_client_prompt("end_datetime")) expected = "Please enter the end date and time (YYYY-MM-DD HH:MM:SS)" self.assertEqual(expected, prompt) self.config_map.execution_timeframe_model = DailyBetweenTimesModel.Config.title model = self.config_map.execution_timeframe_model - prompt = model.get_client_prompt("start_time") + prompt = self.async_run_with_timeout(model.get_client_prompt("start_time")) expected = "Please enter the start time (HH:MM:SS)" self.assertEqual(expected, prompt) - prompt = model.get_client_prompt("end_time") + prompt = self.async_run_with_timeout(model.get_client_prompt("end_time")) expected = "Please enter the end time (HH:MM:SS)" self.assertEqual(expected, prompt) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py index 82a284cad8..3cb4fd1256 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py @@ -1,40 +1,53 @@ import datetime +import logging from decimal import Decimal import unittest.mock import hummingbot.strategy.avellaneda_market_making.start as strategy_start from hummingbot.connector.exchange_base import ExchangeBase -from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map import ( - avellaneda_market_making_config_map as strategy_cmap +from hummingbot.connector.utils import combine_to_hb_trading_pair +from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( + AvellanedaMarketMakingConfigMap, FromDateToDateModel, + TrackHangingOrdersModel, MultiOrderLevelModel ) -from test.hummingbot.strategy import assign_config_default class AvellanedaStartTest(unittest.TestCase): + # the level is required to receive logs from the data source logger + level = 0 def setUp(self) -> None: super().setUp() self.strategy = None self.markets = {"binance": ExchangeBase()} self.notifications = [] - self.log_errors = [] - assign_config_default(strategy_cmap) - strategy_cmap.get("exchange").value = "binance" - strategy_cmap.get("market").value = "balancer" - strategy_cmap.get("execution_timeframe").value = "from_date_to_date" - strategy_cmap.get("start_time").value = "2021-11-18 15:00:00" - strategy_cmap.get("end_time").value = "2021-11-18 16:00:00" - strategy_cmap.get("order_amount").value = Decimal("1") - strategy_cmap.get("order_refresh_time").value = 60. - strategy_cmap.get("hanging_orders_enabled").value = True - strategy_cmap.get("hanging_orders_cancel_pct").value = Decimal("1") - # strategy_cmap.get("hanging_orders_aggregation_type").value = "VOLUME_WEIGHTED" - strategy_cmap.get("min_spread").value = Decimal("2") - strategy_cmap.get("risk_factor").value = Decimal("1.11") - strategy_cmap.get("order_levels").value = Decimal("4") - strategy_cmap.get("level_distances").value = Decimal("1") - strategy_cmap.get("order_amount_shape_factor").value = Decimal("3.33") + self.log_records = [] + self.base = "ETH" + self.quote = "BTC" + self.strategy_config_map = AvellanedaMarketMakingConfigMap( + exchange="binance", + market=combine_to_hb_trading_pair(self.base, self.quote), + execution_timeframe_model=FromDateToDateModel( + start_datetime="2021-11-18 15:00:00", + end_datetime="2021-11-18 16:00:00", + ), + order_amount=60, + order_refresh_time=60, + hanging_orders_model=TrackHangingOrdersModel( + hanging_orders_cancel_pct=1, + ), + order_levels_mode=MultiOrderLevelModel( + order_levels=4, + level_distances=1, + ), + min_spread=2, + risk_factor=1.11, + order_levels=4, + level_distances=1, + order_amount_shape_factor=0.33, + ) self.raise_exception_for_market_initialization = False + self._logger = None def _initialize_market_assets(self, market, trading_pairs): return [("ETH", "USDT")] @@ -47,10 +60,13 @@ def _notify(self, message): self.notifications.append(message) def logger(self): - return self + if self._logger is None: + self._logger = logging.getLogger(self.__class__.__name__) + self._logger.addHandler(self) + return self._logger - def error(self, message, exc_info): - self.log_errors.append(message) + def handle(self, record): + self.log_records.append(record) @unittest.mock.patch('hummingbot.strategy.avellaneda_market_making.start.HummingbotApplication') def test_parameters_strategy_creation(self, mock_hbot): @@ -61,7 +77,7 @@ def test_parameters_strategy_creation(self, mock_hbot): self.assertEqual(self.strategy.end_time, datetime.datetime(2021, 11, 18, 16, 0)) self.assertEqual(self.strategy.min_spread, Decimal("2")) self.assertEqual(self.strategy.gamma, Decimal("1.11")) - self.assertEqual(self.strategy.eta, Decimal("3.33")) + self.assertEqual(self.strategy.eta, Decimal("0.33")) self.assertEqual(self.strategy.order_levels, Decimal("4")) self.assertEqual(self.strategy.level_distances, Decimal("1")) self.assertTrue(all(c is not None for c in (self.strategy.gamma, self.strategy.eta))) @@ -73,5 +89,5 @@ def test_strategy_creation_when_something_fails(self): strategy_start.start(self) self.assertEqual(len(self.notifications), 1) self.assertEqual(self.notifications[0], "Exception for testing") - self.assertEqual(len(self.log_errors), 1) - self.assertEqual(self.log_errors[0], "Unknown error during initialization.") + self.assertEqual(len(self.log_records), 1) + self.assertEqual(self.log_records[0].message, "Unknown error during initialization.") From 572ea2b635fd1d258193cc438dd124f7a9b60ec9 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Mon, 21 Mar 2022 16:29:09 +0700 Subject: [PATCH 021/152] (refactor) Adapts the config command to work with the new pydantic configuration schemas. --- hummingbot/client/command/config_command.py | 236 ++++++++++++++---- hummingbot/client/command/create_command.py | 58 +++-- hummingbot/client/config/config_data_types.py | 160 ++++++++++-- hummingbot/client/ui/completer.py | 2 +- hummingbot/client/ui/interface_utils.py | 19 +- ...aneda_market_making_config_map_pydantic.py | 59 +---- .../avellaneda_market_making/start.py | 14 +- .../client/command/test_config_command.py | 175 +++++++++++-- .../client/config/test_config_data_types.py | 134 +++++++++- ...aneda_market_making_config_map_pydantic.py | 58 ++--- .../avellaneda_market_making/test_config.yml | 2 +- 11 files changed, 676 insertions(+), 241 deletions(-) diff --git a/hummingbot/client/command/config_command.py b/hummingbot/client/command/config_command.py index ffcb93a0e9..33e57e32cb 100644 --- a/hummingbot/client/command/config_command.py +++ b/hummingbot/client/command/config_command.py @@ -1,28 +1,25 @@ import asyncio from decimal import Decimal from os.path import join -from typing import ( - Any, - List, - TYPE_CHECKING, -) +from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union import pandas as pd +from prompt_toolkit.utils import is_windows -from hummingbot.client.config.config_helpers import ( - missing_required_configs_legacy, - save_to_yml_legacy, +from hummingbot.client.config.config_data_types import ( + BaseClientModel, + BaseStrategyConfigMap, + BaseTradingStrategyConfigMap ) +from hummingbot.client.config.config_helpers import missing_required_configs_legacy, save_to_yml, save_to_yml_legacy from hummingbot.client.config.config_validators import validate_bool, validate_decimal from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.security import Security -from hummingbot.client.settings import ( - CONF_FILE_PATH, - GLOBAL_CONFIG_PATH, -) +from hummingbot.client.settings import CONF_FILE_PATH, GLOBAL_CONFIG_PATH from hummingbot.client.ui.interface_utils import format_df_for_printout from hummingbot.client.ui.style import load_style +from hummingbot.connector.utils import split_hb_trading_pair from hummingbot.core.utils import map_df_to_str from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.model.inventory_cost import InventoryCost @@ -66,6 +63,7 @@ "input-pane", "logs-pane", "terminal-primary"] +columns = ["Key", "Value"] class ConfigCommand: @@ -77,14 +75,19 @@ def config(self, # type: HummingbotApplication self.list_configs() return else: - if key not in self.config_able_keys(): + if key not in self.configurable_keys(): self._notify("Invalid key, please choose from the list.") return safe_ensure_future(self._config_single_key(key, value), loop=self.ev_loop) def list_configs(self, # type: HummingbotApplication ): - columns = ["Key", " Value"] + self.list_global_configs() + self.list_strategy_configs() + + def list_global_configs( + self # type: HummingbotApplication + ): data = [[cv.key, cv.value] for cv in global_config_map.values() if cv.key in global_configs_to_display and not cv.is_secure] df = map_df_to_str(pd.DataFrame(data=data, columns=columns)) @@ -99,22 +102,58 @@ def list_configs(self, # type: HummingbotApplication lines = [" " + line for line in format_df_for_printout(df, max_col_width=50).split("\n")] self._notify("\n".join(lines)) + def list_strategy_configs( + self # type: HummingbotApplication + ): if self.strategy_name is not None: - data = [[cv.printable_key or cv.key, cv.value] for cv in self.strategy_config_map.values() if not cv.is_secure] + config_map = self.strategy_config_map + data = self.build_df_data_from_config_map(config_map) df = map_df_to_str(pd.DataFrame(data=data, columns=columns)) self._notify("\nStrategy Configurations:") lines = [" " + line for line in format_df_for_printout(df, max_col_width=50).split("\n")] self._notify("\n".join(lines)) - def config_able_keys(self # type: HummingbotApplication - ) -> List[str]: + def build_df_data_from_config_map( + self, # type: HummingbotApplication + config_map: Union[BaseClientModel, Dict[str, ConfigVar]] + ) -> List[Tuple[str, Any]]: + if isinstance(config_map, BaseClientModel): + data = self.build_model_df_data(config_map) + else: # legacy + data = [[cv.printable_key or cv.key, cv.value] for cv in self.strategy_config_map.values() if not cv.is_secure] + return data + + @staticmethod + def build_model_df_data(config_map: BaseClientModel) -> List[Tuple[str, Any]]: + model_data = [] + for traversal_item in config_map.traverse(): + attr_printout = ( + " " * (traversal_item.depth - 1) + + (u"\u221F " if not is_windows() else " ") + + traversal_item.attr + ) if traversal_item.depth else traversal_item.attr + model_data.append((attr_printout, traversal_item.printable_value)) + return model_data + + def configurable_keys(self # type: HummingbotApplication + ) -> List[str]: """ Returns a list of configurable keys - using config command, excluding exchanges api keys as they are set from connect command. """ keys = [c.key for c in global_config_map.values() if c.prompt is not None and not c.is_connect_key] if self.strategy_config_map is not None: - keys += [c.key for c in self.strategy_config_map.values() if c.prompt is not None] + if isinstance(self.strategy_config_map, BaseStrategyConfigMap): + keys.extend([ + traversal_item.config_path + for traversal_item in self.strategy_config_map.traverse() + if ( + traversal_item.client_field_data is not None + and traversal_item.client_field_data.prompt is not None + ) + ]) + else: # legacy + keys.extend([c.key for c in self.strategy_config_map.values() if c.prompt is not None]) return keys async def check_password(self, # type: HummingbotApplication @@ -149,42 +188,32 @@ async def _config_single_key(self, # type: HummingbotApplication self.app.hide_input = True try: - config_var, config_map, file_path = None, None, None - if key in global_config_map: - config_map = global_config_map - file_path = GLOBAL_CONFIG_PATH - elif self.strategy_config_map is not None and key in self.strategy_config_map: + if ( + key in global_config_map + or ( + not isinstance(self.strategy_config_map, (type(None), BaseClientModel)) + and key in self.strategy_config_map + ) + ): + await self._config_single_key_legacy(key, input_value) + else: config_map = self.strategy_config_map file_path = join(CONF_FILE_PATH, self.strategy_file_name) - config_var = config_map[key] - if input_value is None: - self._notify("Please follow the prompt to complete configurations: ") - if config_var.key == "inventory_target_base_pct": - await self.asset_ratio_maintenance_prompt(config_map, input_value) - elif config_var.key == "inventory_price": - await self.inventory_price_prompt(config_map, input_value) - else: - await self.prompt_a_config_legacy(config_var, input_value=input_value, assign_default=False) - if self.app.to_stop_config: - self.app.to_stop_config = False - return - await self.update_all_secure_configs_legacy() - missings = missing_required_configs_legacy(config_map) - if missings: - self._notify("\nThere are other configuration required, please follow the prompt to complete them.") - missings = await self._prompt_missing_configs(config_map) - save_to_yml_legacy(file_path, config_map) - self._notify("\nNew configuration saved:") - self._notify(f"{key}: {str(config_var.value)}") - self.app.app.style = load_style() - for config in missings: - self._notify(f"{config.key}: {str(config.value)}") - if isinstance(self.strategy, PureMarketMakingStrategy) or \ - isinstance(self.strategy, PerpetualMarketMakingStrategy): - updated = ConfigCommand.update_running_mm(self.strategy, key, config_var.value) - if updated: - self._notify(f"\nThe current {self.strategy_name} strategy has been updated " - f"to reflect the new configuration.") + if input_value is None: + self._notify("Please follow the prompt to complete configurations: ") + if key == "inventory_target_base_pct": + await self.asset_ratio_maintenance_prompt(config_map, input_value) + elif key == "inventory_price": + await self.inventory_price_prompt(config_map, input_value) + else: + await self.prompt_a_config(config_map, key, input_value) + if self.app.to_stop_config: + self.app.to_stop_config = False + return + save_to_yml(file_path, config_map) + self._notify("\nNew configuration saved.") + self.list_strategy_configs() + self.app.app.style = load_style() except asyncio.TimeoutError: self.logger().error("Prompt timeout") except Exception as err: @@ -194,6 +223,50 @@ async def _config_single_key(self, # type: HummingbotApplication self.placeholder_mode = False self.app.change_prompt(prompt=">>> ") + async def _config_single_key_legacy( + self, # type: HummingbotApplication + key: str, + input_value: Any, + ): # pragma: no cover + config_var, config_map, file_path = None, None, None + if key in global_config_map: + config_map = global_config_map + file_path = GLOBAL_CONFIG_PATH + elif self.strategy_config_map is not None and key in self.strategy_config_map: + config_map = self.strategy_config_map + file_path = join(CONF_FILE_PATH, self.strategy_file_name) + config_var = config_map[key] + if input_value is None: + self._notify("Please follow the prompt to complete configurations: ") + if config_var.key == "inventory_target_base_pct": + await self.asset_ratio_maintenance_prompt_legacy(config_map, input_value) + elif config_var.key == "inventory_price": + await self.inventory_price_prompt_legacy(config_map, input_value) + else: + await self.prompt_a_config_legacy(config_var, input_value=input_value, assign_default=False) + if self.app.to_stop_config: + self.app.to_stop_config = False + return + await self.update_all_secure_configs_legacy() + missings = missing_required_configs_legacy(config_map) + if missings: + self._notify("\nThere are other configuration required, please follow the prompt to complete them.") + missings = await self._prompt_missing_configs(config_map) + save_to_yml_legacy(file_path, config_map) + self._notify("\nNew configuration saved:") + self._notify(f"{key}: {str(config_var.value)}") + self.app.app.style = load_style() + for config in missings: + self._notify(f"{config.key}: {str(config.value)}") + if ( + isinstance(self.strategy, PureMarketMakingStrategy) or + isinstance(self.strategy, PerpetualMarketMakingStrategy) + ): + updated = ConfigCommand.update_running_mm(self.strategy, key, config_var.value) + if updated: + self._notify(f"\nThe current {self.strategy_name} strategy has been updated " + f"to reflect the new configuration.") + async def _prompt_missing_configs(self, # type: HummingbotApplication config_map): missings = missing_required_configs_legacy(config_map) @@ -206,9 +279,51 @@ async def _prompt_missing_configs(self, # type: HummingbotApplication return missings + (await self._prompt_missing_configs(config_map)) return missings - async def asset_ratio_maintenance_prompt(self, # type: HummingbotApplication - config_map, - input_value = None): + async def asset_ratio_maintenance_prompt( + self, # type: HummingbotApplication + config_map: BaseTradingStrategyConfigMap, + input_value: Any = None, + ): # pragma: no cover + """ + Will be removed: https://app.shortcut.com/coinalpha/story/24923/collect-special-case-prompts-via-config-models + """ + if input_value: + config_map.inventory_target_base_pct = input_value + else: + exchange = config_map.exchange + market = config_map.market + base, quote = split_hb_trading_pair(market) + balances = await UserBalances.instance().balances(exchange, base, quote) + if balances is None: + return + base_ratio = await UserBalances.base_amount_ratio(exchange, market, balances) + if base_ratio is None: + return + base_ratio = round(base_ratio, 3) + quote_ratio = 1 - base_ratio + + cvar = ConfigVar(key="temp_config", + prompt=f"On {exchange}, you have {balances.get(base, 0):.4f} {base} and " + f"{balances.get(quote, 0):.4f} {quote}. By market value, " + f"your current inventory split is {base_ratio:.1%} {base} " + f"and {quote_ratio:.1%} {quote}." + f" Would you like to keep this ratio? (Yes/No) >>> ", + required_if=lambda: True, + type_str="bool", + validator=validate_bool) + await self.prompt_a_config_legacy(cvar) + if cvar.value: + config_map.inventory_target_base_pct = round(base_ratio * Decimal('100'), 1) + elif self.app.to_stop_config: + self.app.to_stop_config = False + else: + await self.prompt_a_config(config_map, config="inventory_target_base_pct") + + async def asset_ratio_maintenance_prompt_legacy( + self, # type: HummingbotApplication + config_map, + input_value=None, + ): if input_value: config_map['inventory_target_base_pct'].value = Decimal(input_value) else: @@ -244,6 +359,17 @@ async def asset_ratio_maintenance_prompt(self, # type: HummingbotApplication await self.prompt_a_config_legacy(config_map["inventory_target_base_pct"]) async def inventory_price_prompt( + self, # type: HummingbotApplication + model: BaseTradingStrategyConfigMap, + input_value=None, + ): + """ + Not currently used. + Will be removed: https://app.shortcut.com/coinalpha/story/24923/collect-special-case-prompts-via-config-models + """ + raise NotImplementedError + + async def inventory_price_prompt_legacy( self, # type: HummingbotApplication config_map, input_value=None, diff --git a/hummingbot/client/command/create_command.py b/hummingbot/client/command/create_command.py index 488782c7e7..7db6a58365 100644 --- a/hummingbot/client/command/create_command.py +++ b/hummingbot/client/command/create_command.py @@ -2,28 +2,28 @@ import copy import os import shutil +from typing import TYPE_CHECKING, Dict, Optional from pydantic import ValidationError from hummingbot.client.config.config_data_types import BaseClientModel, BaseStrategyConfigMap -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.client.config.config_helpers import ( - get_strategy_config_map, - parse_cvar_value, default_strategy_file_path, - save_to_yml_legacy, - get_strategy_template_path, format_config_file_name, + get_strategy_config_map, + get_strategy_template_path, parse_config_default_to_text, + parse_cvar_value, retrieve_validation_error_msg, save_to_yml, + save_to_yml_legacy ) -from hummingbot.client.settings import CONF_FILE_PATH, required_exchanges +from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.security import Security +from hummingbot.client.settings import CONF_FILE_PATH, required_exchanges from hummingbot.client.ui.completer import load_completer -from typing import TYPE_CHECKING, Dict, Optional, Any +from hummingbot.core.utils.async_utils import safe_ensure_future if TYPE_CHECKING: from hummingbot.client.hummingbot_application import HummingbotApplication @@ -104,11 +104,9 @@ async def prompt_for_model_config( client_data is not None and (client_data.prompt_on_new and field.required) ): - new_config_value = await self.prompt_a_config(config_map, key) + await self.prompt_a_config(config_map, key) if self.app.to_stop_config: break - elif isinstance(new_config_value, BaseClientModel): - await self.prompt_for_model_config(new_config_value) async def prompt_for_configuration_legacy( self, # type: HummingbotApplication @@ -163,23 +161,31 @@ async def prompt_a_config( model: BaseClientModel, config: str, input_value=None, - ) -> Any: + ): + config_path = config.split(".") + while len(config_path) != 1: + sub_model_attr = config_path.pop(0) + model = model.__getattribute__(sub_model_attr) + config = config_path[0] if input_value is None: prompt = await model.get_client_prompt(config) - prompt = f"{prompt} >>> " - client_data = model.get_client_data(config) - input_value = await self.app.prompt(prompt=prompt, is_password=client_data.is_secure) + if prompt is not None: + prompt = f"{prompt} >>> " + client_data = model.get_client_data(config) + input_value = await self.app.prompt(prompt=prompt, is_password=client_data.is_secure) - if self.app.to_stop_config: - return - try: - model.__setattr__(config, input_value) - new_config_value = model.__getattribute__(config) - except ValidationError as e: - err_msg = retrieve_validation_error_msg(e) - self._notify(err_msg) - new_config_value = await self.prompt_a_config(model, config) - return new_config_value + new_config_value = None + if not self.app.to_stop_config and input_value is not None: + try: + model.__setattr__(config, input_value) + new_config_value = model.__getattribute__(config) + except ValidationError as e: + err_msg = retrieve_validation_error_msg(e) + self._notify(err_msg) + new_config_value = await self.prompt_a_config(model, config) + + if not self.app.to_stop_config and isinstance(new_config_value, BaseClientModel): + await self.prompt_for_model_config(new_config_value) async def prompt_a_config_legacy( self, # type: HummingbotApplication @@ -188,7 +194,7 @@ async def prompt_a_config_legacy( assign_default=True, ): if config.key == "inventory_price": - await self.inventory_price_prompt(self.strategy_config_map, input_value) + await self.inventory_price_prompt_legacy(self.strategy_config_map, input_value) return if input_value is None: if assign_default: diff --git a/hummingbot/client/config/config_data_types.py b/hummingbot/client/config/config_data_types.py index b96c1f858a..b08ce6d12c 100644 --- a/hummingbot/client/config/config_data_types.py +++ b/hummingbot/client/config/config_data_types.py @@ -1,16 +1,23 @@ import inspect from dataclasses import dataclass +from datetime import date, datetime, time from decimal import Decimal from enum import Enum -from typing import Any, Callable, Optional, List +from typing import Any, Callable, Dict, Generator, List, Optional import yaml from pydantic import BaseModel, Field, validator +from pydantic.fields import FieldInfo from pydantic.schema import default_ref_template from yaml import SafeDumper from hummingbot.client.config.config_helpers import strategy_config_schema_encoder -from hummingbot.client.config.config_validators import validate_strategy +from hummingbot.client.config.config_validators import ( + validate_exchange, + validate_market_trading_pair, + validate_strategy, +) +from hummingbot.client.settings import AllConnectorSettings class ClientConfigEnum(Enum): @@ -26,12 +33,33 @@ def enum_representer(dumper: SafeDumper, data: ClientConfigEnum): return dumper.represent_str(str(data)) +def date_representer(dumper: SafeDumper, data: date): + return dumper.represent_date(data) + + +def time_representer(dumper: SafeDumper, data: time): + return dumper.represent_str(data.strftime("%H:%M:%S")) + + +def datetime_representer(dumper: SafeDumper, data: datetime): + return dumper.represent_datetime(data) + + yaml.add_representer( data_type=Decimal, representer=decimal_representer, Dumper=SafeDumper ) yaml.add_multi_representer( data_type=ClientConfigEnum, multi_representer=enum_representer, Dumper=SafeDumper ) +yaml.add_representer( + data_type=date, representer=date_representer, Dumper=SafeDumper +) +yaml.add_representer( + data_type=time, representer=time_representer, Dumper=SafeDumper +) +yaml.add_representer( + data_type=datetime, representer=datetime_representer, Dumper=SafeDumper +) @dataclass() @@ -41,7 +69,22 @@ class ClientFieldData: is_secure: bool = False +@dataclass() +class TraversalItem: + depth: int + config_path: str + attr: str + value: Any + printable_value: str + client_field_data: Optional[ClientFieldData] + field_info: FieldInfo + + class BaseClientModel(BaseModel): + class Config: + validate_assignment = True + title = None + """ Notes on configs: - In nested models, be weary that pydantic will take the first model that fits @@ -58,6 +101,33 @@ def schema_json( **dumps_kwargs ) + def traverse(self) -> Generator[TraversalItem, None, None]: + """The intended use for this function is to simplify (validated) config map traversals in the client code.""" + depth = 0 + for attr, field in self.__fields__.items(): + value = self.__getattribute__(attr) + printable_value = str(value) if not isinstance(value, BaseClientModel) else value.Config.title + field_info = field.field_info + client_field_data = field_info.extra.get("client_data") + yield TraversalItem( + depth, attr, attr, value, printable_value, client_field_data, field_info + ) + if isinstance(value, BaseClientModel): + for traversal_item in value.traverse(): + traversal_item.depth += 1 + config_path = f"{attr}.{traversal_item.config_path}" + traversal_item.config_path = config_path + yield traversal_item + + def dict_in_conf_order(self) -> Dict[str, Any]: + d = {} + for attr in self.__fields__.keys(): + value = self.__getattribute__(attr) + if isinstance(value, BaseClientModel): + value = value.dict_in_conf_order() + d[attr] = value + return d + async def get_client_prompt(self, attr_name: str) -> Optional[str]: prompt = None client_data = self.get_client_data(attr_name) @@ -69,14 +139,19 @@ async def get_client_prompt(self, attr_name: str) -> Optional[str]: prompt = prompt(self) return prompt + def is_secure(self, attr_name: str) -> bool: + client_data = self.get_client_data(attr_name) + secure = client_data is not None and client_data.is_secure + return secure + def get_client_data(self, attr_name: str) -> Optional[ClientFieldData]: - return self.__fields__[attr_name].field_info.extra["client_data"] + return self.__fields__[attr_name].field_info.extra.get("client_data") def get_description(self, attr_name: str) -> str: return self.__fields__[attr_name].field_info.description def generate_yml_output_str_with_comments(self) -> str: - original_fragments = yaml.safe_dump(self.dict(), sort_keys=False).split("\n") + original_fragments = yaml.safe_dump(self.dict_in_conf_order(), sort_keys=False).split("\n") fragments_with_comments = [self._generate_title()] self._add_model_fragments(self, fragments_with_comments, original_fragments) fragments_with_comments.append("\n") # EOF empty line @@ -96,33 +171,24 @@ def _adorn_title(title: str) -> str: title = f"{'#' * title_len}\n{title}\n{'#' * title_len}" return title + @staticmethod def _add_model_fragments( - self, model: 'BaseClientModel', fragments_with_comments: List[str], original_fragments: List[str], - original_fragments_idx: int = 0, - model_depth: int = 0, - ) -> int: - comment_prefix = f"\n{' ' * 2 * model_depth}# " - for attr in model.__fields__.keys(): - attr_comment = model.get_description(attr) + ): + for i, traversal_item in enumerate(model.traverse()): + attr_comment = traversal_item.field_info.description if attr_comment is not None: + comment_prefix = f"\n{' ' * 2 * traversal_item.depth}# " attr_comment = "".join(f"{comment_prefix}{c}" for c in attr_comment.split("\n")) - if model_depth == 0: + if traversal_item.depth == 0: attr_comment = f"\n{attr_comment}" - fragments_with_comments.extend([attr_comment, f"\n{original_fragments[original_fragments_idx]}"]) - elif model_depth == 0: - fragments_with_comments.append(f"\n\n{original_fragments[original_fragments_idx]}") + fragments_with_comments.extend([attr_comment, f"\n{original_fragments[i]}"]) + elif traversal_item.depth == 0: + fragments_with_comments.append(f"\n\n{original_fragments[i]}") else: - fragments_with_comments.append(f"\n{original_fragments[original_fragments_idx]}") - original_fragments_idx += 1 - value = model.__getattribute__(attr) - if isinstance(value, BaseClientModel): - original_fragments_idx = self._add_model_fragments( - value, fragments_with_comments, original_fragments, original_fragments_idx, model_depth + 1 - ) - return original_fragments_idx + fragments_with_comments.append(f"\n{original_fragments[i]}") class BaseStrategyConfigMap(BaseClientModel): @@ -146,3 +212,51 @@ def _generate_title(self) -> str: title = f"{title} Strategy" title = self._adorn_title(title) return title + + +class BaseTradingStrategyConfigMap(BaseStrategyConfigMap): + exchange: ClientConfigEnum( + value="Exchanges", # noqa: F821 + names={e: e for e in AllConnectorSettings.get_connector_settings().keys()}, + type=str, + ) = Field( + default=..., + description="The name of the exchange connector.", + client_data=ClientFieldData( + prompt=lambda mi: "Input your maker spot connector", + prompt_on_new=True, + ), + ) + market: str = Field( + default=..., + description="The trading pair.", + client_data=ClientFieldData( + prompt=lambda mi: BaseTradingStrategyConfigMap.maker_trading_pair_prompt(mi), + prompt_on_new=True, + ), + ) + + @classmethod + def maker_trading_pair_prompt(cls, model_instance: 'BaseTradingStrategyConfigMap') -> str: + exchange = model_instance.exchange + example = AllConnectorSettings.get_example_pairs().get(exchange) + return ( + f"Enter the token trading pair you would like to trade on" + f" {exchange}{f' (e.g. {example})' if example else ''}" + ) + + @validator("exchange", pre=True) + def validate_exchange(cls, v: str): + """Used for client-friendly error output.""" + ret = validate_exchange(v) + if ret is not None: + raise ValueError(ret) + return v + + @validator("market", pre=True) + def validate_exchange_trading_pair(cls, v: str, values: Dict): + exchange = values.get("exchange") + ret = validate_market_trading_pair(exchange, v) + if ret is not None: + raise ValueError(ret) + return v diff --git a/hummingbot/client/ui/completer.py b/hummingbot/client/ui/completer.py index 2b68eda4ea..9a099cc187 100644 --- a/hummingbot/client/ui/completer.py +++ b/hummingbot/client/ui/completer.py @@ -89,7 +89,7 @@ def _option_completer(self): @property def _config_completer(self): - config_keys = self.hummingbot_application.config_able_keys() + config_keys = self.hummingbot_application.configurable_keys() return WordCompleter(config_keys, ignore_case=True) def _complete_strategies(self, document: Document) -> bool: diff --git a/hummingbot/client/ui/interface_utils.py b/hummingbot/client/ui/interface_utils.py index 6cbb954708..e846d49631 100644 --- a/hummingbot/client/ui/interface_utils.py +++ b/hummingbot/client/ui/interface_utils.py @@ -1,17 +1,17 @@ import asyncio import datetime - -import pandas as pd -import psutil -from decimal import Decimal from typing import ( List, + Optional, Set, Tuple, - Optional, ) -from tabulate import tabulate +import pandas as pd +import psutil +import tabulate +from decimal import Decimal + from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.performance import PerformanceMetrics @@ -111,5 +111,10 @@ def format_df_for_printout( ) df.columns = [c if len(c) < max_col_width else f"{c[:max_col_width - 3]}..." for c in df.columns] table_format = table_format or global_config_map.get("tables_format").value - formatted_df = tabulate(df, tablefmt=table_format, showindex=index, headers="keys") + + tabulate.PRESERVE_WHITESPACE = True + try: + formatted_df = tabulate.tabulate(df, tablefmt=table_format, showindex=index, headers="keys") + finally: + tabulate.PRESERVE_WHITESPACE = False return formatted_df diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py index 78f185f699..77941eabf5 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -6,20 +6,17 @@ from hummingbot.client.config.config_data_types import ( BaseClientModel, - BaseStrategyConfigMap, - ClientConfigEnum, + BaseTradingStrategyConfigMap, ClientFieldData, ) from hummingbot.client.config.config_validators import ( validate_bool, validate_datetime_iso_string, validate_decimal, - validate_exchange, validate_int, - validate_market_trading_pair, validate_time_iso_string, ) -from hummingbot.client.settings import AllConnectorSettings, required_exchanges +from hummingbot.client.settings import required_exchanges from hummingbot.connector.utils import split_hb_trading_pair @@ -183,29 +180,9 @@ class Config: } -class AvellanedaMarketMakingConfigMap(BaseStrategyConfigMap): +class AvellanedaMarketMakingConfigMap(BaseTradingStrategyConfigMap): strategy: str = Field(default="avellaneda_market_making", client_data=None) - exchange: ClientConfigEnum( - value="Exchanges", # noqa: F821 - names={e: e for e in AllConnectorSettings.get_connector_settings().keys()}, - type=str, - ) = Field( - default=..., - description="The name of the exchange connector.", - client_data=ClientFieldData( - prompt=lambda mi: "Input your maker spot connector", - prompt_on_new=True, - ), - ) - market: str = Field( - default=..., - description="The trading pair.", - client_data=ClientFieldData( - prompt=lambda mi: AvellanedaMarketMakingConfigMap.maker_trading_pair_prompt(mi), - prompt_on_new=True, - ), - ) - execution_timeframe_model: Union[FromDateToDateModel, DailyBetweenTimesModel, InfiniteModel] = Field( + execution_timeframe_mode: Union[FromDateToDateModel, DailyBetweenTimesModel, InfiniteModel] = Field( default=..., description="The execution timeframe.", client_data=ClientFieldData( @@ -376,16 +353,8 @@ class AvellanedaMarketMakingConfigMap(BaseStrategyConfigMap): class Config: title = "avellaneda_market_making" - validate_assignment = True - @classmethod - def maker_trading_pair_prompt(cls, model_instance: 'AvellanedaMarketMakingConfigMap') -> str: - exchange = model_instance.exchange - example = AllConnectorSettings.get_example_pairs().get(exchange) - return ( - f"Enter the token trading pair you would like to trade on" - f" {exchange}{f' (e.g. {example})' if example else ''}" - ) + # === prompts === @classmethod def order_amount_prompt(cls, model_instance: 'AvellanedaMarketMakingConfigMap') -> str: @@ -393,23 +362,9 @@ def order_amount_prompt(cls, model_instance: 'AvellanedaMarketMakingConfigMap') base_asset, quote_asset = split_hb_trading_pair(trading_pair) return f"What is the amount of {base_asset} per order?" - @validator("exchange", pre=True) - def validate_exchange(cls, v: str): - """Used for client-friendly error output.""" - ret = validate_exchange(v) - if ret is not None: - raise ValueError(ret) - return v - - @validator("market", pre=True) - def validate_exchange_trading_pair(cls, v: str, values: Dict): - exchange = values.get("exchange") - ret = validate_market_trading_pair(exchange, v) - if ret is not None: - raise ValueError(ret) - return v + # === specific validations === - @validator("execution_timeframe_model", pre=True) + @validator("execution_timeframe_mode", pre=True) def validate_execution_timeframe( cls, v: Union[str, InfiniteModel, FromDateToDateModel, DailyBetweenTimesModel] ): diff --git a/hummingbot/strategy/avellaneda_market_making/start.py b/hummingbot/strategy/avellaneda_market_making/start.py index d0ead627eb..4096680b0a 100644 --- a/hummingbot/strategy/avellaneda_market_making/start.py +++ b/hummingbot/strategy/avellaneda_market_making/start.py @@ -64,14 +64,14 @@ def start(self): risk_factor = c_map.risk_factor order_amount_shape_factor = c_map.order_amount_shape_factor - execution_timeframe = c_map.execution_timeframe_model.Config.title - if isinstance(c_map.execution_timeframe_model, FromDateToDateModel): - start_time = c_map.execution_timeframe_model.start_datetime - end_time = c_map.execution_timeframe_model.end_datetime + execution_timeframe = c_map.execution_timeframe_mode.Config.title + if isinstance(c_map.execution_timeframe_mode, FromDateToDateModel): + start_time = c_map.execution_timeframe_mode.start_datetime + end_time = c_map.execution_timeframe_mode.end_datetime execution_state = RunInTimeConditionalExecutionState(start_timestamp=start_time, end_timestamp=end_time) - elif isinstance(c_map.execution_timeframe_model, DailyBetweenTimesModel): - start_time = c_map.execution_timeframe_model.start_time - end_time = c_map.execution_timeframe_model.end_time + elif isinstance(c_map.execution_timeframe_mode, DailyBetweenTimesModel): + start_time = c_map.execution_timeframe_mode.start_time + end_time = c_map.execution_timeframe_mode.end_time execution_state = RunInTimeConditionalExecutionState(start_timestamp=start_time, end_timestamp=end_time) else: start_time = None diff --git a/test/hummingbot/client/command/test_config_command.py b/test/hummingbot/client/command/test_config_command.py index 330a2c119e..dae17bc48f 100644 --- a/test/hummingbot/client/command/test_config_command.py +++ b/test/hummingbot/client/command/test_config_command.py @@ -2,13 +2,18 @@ import unittest from collections import Awaitable from copy import deepcopy +from decimal import Decimal from unittest.mock import patch, MagicMock +from pydantic import Field + from hummingbot.client.command.config_command import color_settings_to_display, global_configs_to_display +from hummingbot.client.config.config_data_types import BaseClientModel, BaseStrategyConfigMap, ClientFieldData from hummingbot.client.config.config_helpers import read_system_configs_from_yml from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.hummingbot_application import HummingbotApplication +from test.mock.mock_cli import CLIMockingAssistant class ConfigCommandTest(unittest.TestCase): @@ -20,9 +25,12 @@ def setUp(self, _: MagicMock) -> None: self.async_run_with_timeout(read_system_configs_from_yml()) self.app = HummingbotApplication() + self.cli_mock_assistant = CLIMockingAssistant(self.app.app) + self.cli_mock_assistant.start() self.global_config_backup = deepcopy(global_config_map) def tearDown(self) -> None: + self.cli_mock_assistant.stop() self.reset_global_config() super().tearDown() @@ -68,37 +76,166 @@ def test_list_configs(self, notify_mock, get_strategy_config_map_mock): self.assertEqual("\nGlobal Configurations:", captures[0]) df_str_expected = ( - " +---------------------+-----------+" - "\n | Key | Value |" - "\n |---------------------+-----------|" - "\n | tables_format | psql |" - "\n | autofill_import | first |" - "\n | kill_switch_enabled | second |" - "\n +---------------------+-----------+" + " +---------------------+---------+" + "\n | Key | Value |" + "\n |---------------------+---------|" + "\n | tables_format | psql |" + "\n | autofill_import | first |" + "\n | kill_switch_enabled | second |" + "\n +---------------------+---------+" ) self.assertEqual(df_str_expected, captures[1]) self.assertEqual("\nColor Settings:", captures[2]) df_str_expected = ( - " +-------------+-----------+" - "\n | Key | Value |" - "\n |-------------+-----------|" - "\n | top-pane | third |" - "\n | bottom-pane | fourth |" - "\n +-------------+-----------+" + " +-------------+---------+" + "\n | Key | Value |" + "\n |-------------+---------|" + "\n | top-pane | third |" + "\n | bottom-pane | fourth |" + "\n +-------------+---------+" ) self.assertEqual(df_str_expected, captures[3]) self.assertEqual("\nStrategy Configurations:", captures[4]) df_str_expected = ( - " +-------+-----------+" - "\n | Key | Value |" - "\n |-------+-----------|" - "\n | five | fifth |" - "\n | six | sixth |" - "\n +-------+-----------+" + " +-------+---------+" + "\n | Key | Value |" + "\n |-------+---------|" + "\n | five | fifth |" + "\n | six | sixth |" + "\n +-------+---------+" + ) + + self.assertEqual(df_str_expected, captures[5]) + + @patch("hummingbot.client.hummingbot_application.get_strategy_config_map") + @patch("hummingbot.client.hummingbot_application.HummingbotApplication._notify") + def test_list_configs_pydantic_model(self, notify_mock, get_strategy_config_map_mock): + captures = [] + notify_mock.side_effect = lambda s: captures.append(s) + strategy_name = "some-strategy" + self.app.strategy_name = strategy_name + + tables_format_config_var = global_config_map["tables_format"] + global_config_map.clear() + global_config_map[tables_format_config_var.key] = tables_format_config_var + tables_format_config_var.value = "psql" + + class DoubleNestedModel(BaseClientModel): + double_nested_attr: float = Field(default=3.0) + + class Config: + title = "double_nested_model" + + class NestedModel(BaseClientModel): + nested_attr: str = Field(default="some value") + double_nested_model: DoubleNestedModel = Field(default=DoubleNestedModel()) + + class Config: + title = "nested_model" + + class DummyModel(BaseClientModel): + some_attr: int = Field(default=1) + nested_model: NestedModel = Field(default=NestedModel()) + another_attr: Decimal = Field(default=Decimal("1.0")) + + class Config: + title = "dummy_model" + + get_strategy_config_map_mock.return_value = DummyModel() + + self.app.list_configs() + + self.assertEqual(6, len(captures)) + + self.assertEqual("\nStrategy Configurations:", captures[4]) + + df_str_expected = ( + " +------------------------+---------------------+" + "\n | Key | Value |" + "\n |------------------------+---------------------|" + "\n | some_attr | 1 |" + "\n | nested_model | nested_model |" + "\n | ∟ nested_attr | some value |" + "\n | ∟ double_nested_model | double_nested_model |" + "\n | ∟ double_nested_attr | 3.0 |" + "\n | another_attr | 1.0 |" + "\n +------------------------+---------------------+" ) self.assertEqual(df_str_expected, captures[5]) + + @patch("hummingbot.client.hummingbot_application.get_strategy_config_map") + @patch("hummingbot.client.hummingbot_application.HummingbotApplication._notify") + def test_config_non_configurable_key_fails(self, notify_mock, get_strategy_config_map_mock): + class DummyModel(BaseStrategyConfigMap): + strategy: str = Field(default="pure_market_making", client_data=None) + some_attr: int = Field(default=1, client_data=ClientFieldData(prompt=lambda mi: "some prompt")) + another_attr: Decimal = Field(default=Decimal("1.0")) + + class Config: + title = "dummy_model" + + strategy_name = "some-strategy" + self.app.strategy_name = strategy_name + get_strategy_config_map_mock.return_value = DummyModel.construct() + self.app.config(key="some_attr") + + notify_mock.assert_not_called() + + self.app.config(key="another_attr") + + notify_mock.assert_called_once_with("Invalid key, please choose from the list.") + + notify_mock.reset_mock() + self.app.config(key="some_key") + + notify_mock.assert_called_once_with("Invalid key, please choose from the list.") + + @patch("hummingbot.client.command.config_command.save_to_yml") + @patch("hummingbot.client.hummingbot_application.get_strategy_config_map") + @patch("hummingbot.client.hummingbot_application.HummingbotApplication._notify") + def test_config_single_keys(self, _, get_strategy_config_map_mock, save_to_yml_mock): + class NestedModel(BaseClientModel): + nested_attr: str = Field( + default="some value", client_data=ClientFieldData(prompt=lambda mi: "some prompt") + ) + + class Config: + title = "nested_model" + + class DummyModel(BaseStrategyConfigMap): + strategy: str = Field(default="pure_market_making", client_data=None) + some_attr: int = Field(default=1, client_data=ClientFieldData(prompt=lambda mi: "some prompt")) + nested_model: NestedModel = Field(default=NestedModel()) + + class Config: + title = "dummy_model" + + strategy_name = "some-strategy" + self.app.strategy_name = strategy_name + self.app.strategy_file_name = f"{strategy_name}.yml" + config_map = DummyModel.construct() + get_strategy_config_map_mock.return_value = config_map + + self.async_run_with_timeout(self.app._config_single_key(key="some_attr", input_value=2)) + + self.assertEqual(2, config_map.some_attr) + save_to_yml_mock.assert_called_once() + + save_to_yml_mock.reset_mock() + self.cli_mock_assistant.queue_prompt_reply("3") + self.async_run_with_timeout(self.app._config_single_key(key="some_attr", input_value=None)) + + self.assertEqual(3, config_map.some_attr) + save_to_yml_mock.assert_called_once() + + save_to_yml_mock.reset_mock() + self.cli_mock_assistant.queue_prompt_reply("another value") + self.async_run_with_timeout(self.app._config_single_key(key="nested_model.nested_attr", input_value=None), 10000) + + self.assertEqual("another value", config_map.nested_model.nested_attr) + save_to_yml_mock.assert_called_once() diff --git a/test/hummingbot/client/config/test_config_data_types.py b/test/hummingbot/client/config/test_config_data_types.py index 47f095b19d..5d80a3449c 100644 --- a/test/hummingbot/client/config/test_config_data_types.py +++ b/test/hummingbot/client/config/test_config_data_types.py @@ -1,12 +1,23 @@ +import asyncio import json import unittest +from datetime import date, datetime, time from decimal import Decimal +from typing import Awaitable, Dict +from unittest.mock import patch -from pydantic import Field +from pydantic import Field, ValidationError +from pydantic.fields import FieldInfo from hummingbot.client.config.config_data_types import ( - BaseClientModel, ClientFieldData, ClientConfigEnum, BaseStrategyConfigMap + BaseClientModel, + BaseStrategyConfigMap, + BaseTradingStrategyConfigMap, + ClientConfigEnum, + ClientFieldData, + TraversalItem, ) +from hummingbot.client.config.config_helpers import retrieve_validation_error_msg class BaseClientModelTest(unittest.TestCase): @@ -29,13 +40,63 @@ class DummyModel(BaseClientModel): } self.assertEqual(expected, j["properties"]["some_attr"]["client_data"]) + def test_traverse(self): + class DoubleNestedModel(BaseClientModel): + double_nested_attr: float = Field(default=3.0) + + class Config: + title = "double_nested_model" + + class NestedModel(BaseClientModel): + nested_attr: str = Field(default="some value") + double_nested_model: DoubleNestedModel = Field(default=DoubleNestedModel()) + + class Config: + title = "nested_model" + + class DummyModel(BaseClientModel): + some_attr: int = Field(default=1, client_data=ClientFieldData()) + nested_model: NestedModel = Field(default=NestedModel()) + another_attr: Decimal = Field(default=Decimal("1.0")) + + class Config: + title = "dummy_model" + + expected = [ + TraversalItem(0, "some_attr", "some_attr", 1, "1", ClientFieldData(), None), + TraversalItem(0, "nested_model", "nested_model", NestedModel(), "nested_model", None, None), + TraversalItem(1, "nested_model.nested_attr", "nested_attr", "some value", "some value", None, None), + TraversalItem( + 1, + "nested_model.double_nested_model", + "double_nested_model", + DoubleNestedModel(), + "double_nested_model", + None, + None, + ), + TraversalItem( + 2, "nested_model.double_nested_model.double_nested_attr", "double_nested_attr", 3.0, "3.0", None, None + ), + ] + cm = DummyModel() + + for expected, actual in zip(expected, cm.traverse()): + self.assertEqual(expected.depth, actual.depth) + self.assertEqual(expected.config_path, actual.config_path) + self.assertEqual(expected.attr, actual.attr) + self.assertEqual(expected.value, actual.value) + self.assertEqual(expected.printable_value, actual.printable_value) + self.assertEqual(expected.client_field_data, actual.client_field_data) + self.assertIsInstance(actual.field_info, FieldInfo) + def test_generate_yml_output_dict_with_comments(self): class SomeEnum(ClientConfigEnum): ONE = "one" class DoubleNestedModel(BaseClientModel): - double_nested_attr: float = Field( - default=3.0, + double_nested_attr: datetime = Field( + default=datetime(2022, 1, 1, 10, 30), description="Double nested attr description" ) @@ -61,9 +122,8 @@ class DummyModel(BaseClientModel): default=Decimal("1.0"), description="Some other\nmultiline description", ) - non_nested_no_description: int = Field( - default=2, - ) + non_nested_no_description: time = Field(default=time(10, 30),) + date_attr: date = Field(default=date(2022, 1, 2)) class Config: title = "dummy_model" @@ -86,13 +146,15 @@ class Config: nested_attr: some value double_nested_model: # Double nested attr description - double_nested_attr: 3.0 + double_nested_attr: 2022-01-01 10:30:00 # Some other # multiline description another_attr: 1.0 -non_nested_no_description: 2 +non_nested_no_description: '10:30:00' + +date_attr: 2022-01-02 """ self.assertEqual(expected_str, res_str) @@ -112,3 +174,57 @@ def test_generate_yml_output_dict_title(self): """ self.assertEqual(expected_str, res_str) + + +class BaseTradingStrategyConfigMapTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + cls.ev_loop = asyncio.get_event_loop() + cls.exchange = "binance" + cls.base_asset = "COINALPHA" + cls.quote_asset = "HBOT" + cls.trading_pair = f"{cls.base_asset}-{cls.quote_asset}" + + def setUp(self) -> None: + super().setUp() + config_settings = self.get_default_map() + self.config_map = BaseTradingStrategyConfigMap(**config_settings) + + def async_run_with_timeout(self, coroutine: Awaitable, timeout: int = 1): + ret = self.ev_loop.run_until_complete(asyncio.wait_for(coroutine, timeout)) + return ret + + def get_default_map(self) -> Dict[str, str]: + config_settings = { + "strategy": "pure_market_making", + "exchange": self.exchange, + "market": self.trading_pair, + } + return config_settings + + @patch( + "hummingbot.client.config.config_data_types.validate_market_trading_pair" + ) + def test_validators(self, validate_market_trading_pair_mock): + with self.assertRaises(ValidationError) as e: + self.config_map.exchange = "test-exchange" + + error_msg = "Invalid exchange, please choose value from " + actual_msg = retrieve_validation_error_msg(e.exception) + self.assertTrue(actual_msg.startswith(error_msg)) + + alt_pair = "ETH-USDT" + error_msg = "Failed" + validate_market_trading_pair_mock.side_effect = ( + lambda m, v: None if v in [self.trading_pair, alt_pair] else error_msg + ) + + self.config_map.market = alt_pair + self.assertEqual(alt_pair, self.config_map.market) + + with self.assertRaises(ValidationError) as e: + self.config_map.market = "XXX-USDT" + + actual_msg = retrieve_validation_error_msg(e.exception) + self.assertTrue(actual_msg.startswith(error_msg)) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py index 14addcd407..d508716c37 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py @@ -42,7 +42,7 @@ def get_default_map(self) -> Dict[str, str]: config_settings = { "exchange": self.exchange, "market": self.trading_pair, - "execution_timeframe_model": { + "execution_timeframe_mode": { "start_time": "09:30:00", "end_time": "16:00:00", }, @@ -91,8 +91,8 @@ def test_maker_trading_pair_prompt(self): self.assertEqual(expected, prompt) def test_execution_time_prompts(self): - self.config_map.execution_timeframe_model = FromDateToDateModel.Config.title - model = self.config_map.execution_timeframe_model + self.config_map.execution_timeframe_mode = FromDateToDateModel.Config.title + model = self.config_map.execution_timeframe_mode prompt = self.async_run_with_timeout(model.get_client_prompt("start_datetime")) expected = "Please enter the start date and time (YYYY-MM-DD HH:MM:SS)" self.assertEqual(expected, prompt) @@ -100,8 +100,8 @@ def test_execution_time_prompts(self): expected = "Please enter the end date and time (YYYY-MM-DD HH:MM:SS)" self.assertEqual(expected, prompt) - self.config_map.execution_timeframe_model = DailyBetweenTimesModel.Config.title - model = self.config_map.execution_timeframe_model + self.config_map.execution_timeframe_mode = DailyBetweenTimesModel.Config.title + model = self.config_map.execution_timeframe_mode prompt = self.async_run_with_timeout(model.get_client_prompt("start_time")) expected = "Please enter the start time (HH:MM:SS)" self.assertEqual(expected, prompt) @@ -110,44 +110,20 @@ def test_execution_time_prompts(self): self.assertEqual(expected, prompt) @patch( - "hummingbot.strategy.avellaneda_market_making" - ".avellaneda_market_making_config_map_pydantic.validate_market_trading_pair" + "hummingbot.client.config.config_data_types.validate_market_trading_pair" ) def test_validators(self, validate_market_trading_pair_mock): + self.config_map.execution_timeframe_mode = "infinite" + self.assertIsInstance(self.config_map.execution_timeframe_mode, InfiniteModel) - with self.assertRaises(ValidationError) as e: - self.config_map.exchange = "test-exchange" - - error_msg = "Invalid exchange, please choose value from " - actual_msg = retrieve_validation_error_msg(e.exception) - self.assertTrue(actual_msg.startswith(error_msg)) - - alt_pair = "ETH-USDT" - error_msg = "Failed" - validate_market_trading_pair_mock.side_effect = ( - lambda m, v: None if v in [self.trading_pair, alt_pair] else error_msg - ) - - self.config_map.market = alt_pair - self.assertEqual(alt_pair, self.config_map.market) - - with self.assertRaises(ValidationError) as e: - self.config_map.market = "XXX-USDT" - - actual_msg = retrieve_validation_error_msg(e.exception) - self.assertTrue(actual_msg.startswith(error_msg)) - - self.config_map.execution_timeframe_model = "infinite" - self.assertIsInstance(self.config_map.execution_timeframe_model, InfiniteModel) - - self.config_map.execution_timeframe_model = "from_date_to_date" - self.assertIsInstance(self.config_map.execution_timeframe_model, FromDateToDateModel) + self.config_map.execution_timeframe_mode = "from_date_to_date" + self.assertIsInstance(self.config_map.execution_timeframe_mode, FromDateToDateModel) - self.config_map.execution_timeframe_model = "daily_between_times" - self.assertIsInstance(self.config_map.execution_timeframe_model, DailyBetweenTimesModel) + self.config_map.execution_timeframe_mode = "daily_between_times" + self.assertIsInstance(self.config_map.execution_timeframe_mode, DailyBetweenTimesModel) with self.assertRaises(ValidationError) as e: - self.config_map.execution_timeframe_model = "XXX" + self.config_map.execution_timeframe_mode = "XXX" error_msg = ( "Invalid timeframe, please choose value from ['infinite', 'from_date_to_date', 'daily_between_times']" @@ -155,8 +131,8 @@ def test_validators(self, validate_market_trading_pair_mock): actual_msg = retrieve_validation_error_msg(e.exception) self.assertEqual(error_msg, actual_msg) - self.config_map.execution_timeframe_model = "from_date_to_date" - model = self.config_map.execution_timeframe_model + self.config_map.execution_timeframe_mode = "from_date_to_date" + model = self.config_map.execution_timeframe_mode model.start_datetime = "2021-01-01 12:00:00" model.end_datetime = "2021-01-01 15:00:00" @@ -177,8 +153,8 @@ def test_validators(self, validate_market_trading_pair_mock): actual_msg = retrieve_validation_error_msg(e.exception) self.assertEqual(error_msg, actual_msg) - self.config_map.execution_timeframe_model = "daily_between_times" - model = self.config_map.execution_timeframe_model + self.config_map.execution_timeframe_mode = "daily_between_times" + model = self.config_map.execution_timeframe_mode model.start_time = "12:00:00" self.assertEqual(time(12, 0, 0), model.start_time) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_config.yml b/test/hummingbot/strategy/avellaneda_market_making/test_config.yml index c7c7cf1d39..051b140d92 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_config.yml +++ b/test/hummingbot/strategy/avellaneda_market_making/test_config.yml @@ -1,6 +1,6 @@ exchange: binance market: COINALPHA-HBOT -execution_timeframe_model: +execution_timeframe_mode: start_time: "09:30:00" end_time: "16:00:00" order_amount: 10 From 2feed79b92db98ffa1ffba7b433f4a957e755b31 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Tue, 22 Mar 2022 11:56:45 +0700 Subject: [PATCH 022/152] (fix) Fixes the paper-trade and multiple-exchange-status-printout issues. --- hummingbot/client/command/start_command.py | 2 +- hummingbot/client/config/config_data_types.py | 5 + hummingbot/client/settings.py | 2 +- .../strategy/amm_arb/amm_arb_config_map.py | 2 +- .../arbitrage/arbitrage_config_map.py | 4 +- .../aroon_oscillator_config_map.py | 2 +- .../avellaneda_market_making_config_map.py | 245 ------------------ ...aneda_market_making_config_map_pydantic.py | 2 +- .../strategy/celo_arb/celo_arb_config_map.py | 2 +- ...cross_exchange_market_making_config_map.py | 4 +- .../dev_0_hello_world_config_map.py | 2 +- .../dev_1_get_order_book_config_map.py | 2 +- .../dev_2_perform_trade_config_map.py | 2 +- .../dev_5_vwap/dev_5_vwap_config_map.py | 2 +- .../dev_simple_trade_config_map.py | 2 +- .../liquidity_mining_config_map.py | 2 +- .../perpetual_market_making_config_map.py | 2 +- .../pure_market_making_config_map.py | 2 +- .../spot_perpetual_arbitrage_config_map.py | 2 +- hummingbot/strategy/twap/twap_config_map.py | 2 +- .../uniswap_v3_lp/uniswap_v3_lp_config_map.py | 2 +- 21 files changed, 26 insertions(+), 266 deletions(-) delete mode 100644 hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map.py diff --git a/hummingbot/client/command/start_command.py b/hummingbot/client/command/start_command.py index 2448b1a1e0..ac806c4375 100644 --- a/hummingbot/client/command/start_command.py +++ b/hummingbot/client/command/start_command.py @@ -101,7 +101,7 @@ async def start_check(self, # type: HummingbotApplication f"{warning_msg}") # Display warning message if the exchange connector has outstanding issues or not working - elif status != "GREEN": + elif "GREEN" not in status: self._notify(f"\nConnector status: {status}. This connector has one or more issues.\n" "Refer to our Github page for more info: https://github.com/coinalpha/hummingbot") diff --git a/hummingbot/client/config/config_data_types.py b/hummingbot/client/config/config_data_types.py index b08ce6d12c..cc351d5371 100644 --- a/hummingbot/client/config/config_data_types.py +++ b/hummingbot/client/config/config_data_types.py @@ -251,6 +251,11 @@ def validate_exchange(cls, v: str): ret = validate_exchange(v) if ret is not None: raise ValueError(ret) + cls.__fields__["exchange"].type_ = ClientConfigEnum( # rebuild the exchanges enum + value="Exchanges", # noqa: F821 + names={e: e for e in AllConnectorSettings.get_connector_settings().keys()}, + type=str, + ) return v @validator("market", pre=True) diff --git a/hummingbot/client/settings.py b/hummingbot/client/settings.py index cff7a8873d..53fcdc5fc6 100644 --- a/hummingbot/client/settings.py +++ b/hummingbot/client/settings.py @@ -16,7 +16,7 @@ from hummingbot.core.data_type.trade_fee import TradeFeeSchema # Global variables -required_exchanges: List[str] = [] +required_exchanges: Set[str] = set() requried_connector_trading_pairs: Dict[str, List[str]] = {} # Set these two variables if a strategy uses oracle for rate conversion required_rate_oracle: bool = False diff --git a/hummingbot/strategy/amm_arb/amm_arb_config_map.py b/hummingbot/strategy/amm_arb/amm_arb_config_map.py index 807991ae8c..0750a07601 100644 --- a/hummingbot/strategy/amm_arb/amm_arb_config_map.py +++ b/hummingbot/strategy/amm_arb/amm_arb_config_map.py @@ -16,7 +16,7 @@ def exchange_on_validated(value: str) -> None: - required_exchanges.append(value) + required_exchanges.add(value) def market_1_validator(value: str) -> None: diff --git a/hummingbot/strategy/arbitrage/arbitrage_config_map.py b/hummingbot/strategy/arbitrage/arbitrage_config_map.py index 392b1de242..543f0ab900 100644 --- a/hummingbot/strategy/arbitrage/arbitrage_config_map.py +++ b/hummingbot/strategy/arbitrage/arbitrage_config_map.py @@ -38,7 +38,7 @@ def secondary_trading_pair_prompt(): def secondary_market_on_validated(value: str): - required_exchanges.append(value) + required_exchanges.add(value) def update_oracle_settings(value: str): @@ -73,7 +73,7 @@ def update_oracle_settings(value: str): prompt="Enter your primary spot connector >>> ", prompt_on_new=True, validator=validate_exchange, - on_validated=lambda value: required_exchanges.append(value), + on_validated=lambda value: required_exchanges.add(value), ), "secondary_market": ConfigVar( key="secondary_market", diff --git a/hummingbot/strategy/aroon_oscillator/aroon_oscillator_config_map.py b/hummingbot/strategy/aroon_oscillator/aroon_oscillator_config_map.py index 1ebd87aeb6..68fdb2cf3f 100644 --- a/hummingbot/strategy/aroon_oscillator/aroon_oscillator_config_map.py +++ b/hummingbot/strategy/aroon_oscillator/aroon_oscillator_config_map.py @@ -64,7 +64,7 @@ def on_validated_price_type(value: str): def exchange_on_validated(value: str): - required_exchanges.append(value) + required_exchanges.add(value) aroon_oscillator_config_map = { diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map.py deleted file mode 100644 index 2df007770f..0000000000 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map.py +++ /dev/null @@ -1,245 +0,0 @@ -from decimal import Decimal - -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_validators import ( - validate_exchange, - validate_market_trading_pair, - validate_int, - validate_bool, - validate_decimal, - validate_datetime_iso_string, - validate_time_iso_string, -) -from hummingbot.client.settings import ( - required_exchanges, - AllConnectorSettings, -) -from typing import Optional - - -def maker_trading_pair_prompt(): - exchange = avellaneda_market_making_config_map.get("exchange").value - example = AllConnectorSettings.get_example_pairs().get(exchange) - return "Enter the token trading pair you would like to trade on %s%s >>> " \ - % (exchange, f" (e.g. {example})" if example else "") - - -# strategy specific validators -def validate_exchange_trading_pair(value: str) -> Optional[str]: - exchange = avellaneda_market_making_config_map.get("exchange").value - return validate_market_trading_pair(exchange, value) - - -def validate_execution_timeframe(value: str) -> Optional[str]: - timeframes = ["infinite", "from_date_to_date", "daily_between_times"] - if value not in timeframes: - return f"Invalid timeframe, please choose value from {timeframes}" - - -def validate_execution_time(value: str) -> Optional[str]: - ret = None - if avellaneda_market_making_config_map.get("execution_timeframe").value == "from_date_to_date": - ret = validate_datetime_iso_string(value) - if avellaneda_market_making_config_map.get("execution_timeframe").value == "daily_between_times": - ret = validate_time_iso_string(value) - if ret is not None: - return ret - - -def execution_time_start_prompt() -> str: - if avellaneda_market_making_config_map.get("execution_timeframe").value == "from_date_to_date": - return "Please enter the start date and time (YYYY-MM-DD HH:MM:SS) >>> " - if avellaneda_market_making_config_map.get("execution_timeframe").value == "daily_between_times": - return "Please enter the start time (HH:MM:SS) >>> " - - -def execution_time_end_prompt() -> str: - if avellaneda_market_making_config_map.get("execution_timeframe").value == "from_date_to_date": - return "Please enter the end date and time (YYYY-MM-DD HH:MM:SS) >>> " - if avellaneda_market_making_config_map.get("execution_timeframe").value == "daily_between_times": - return "Please enter the end time (HH:MM:SS) >>> " - - -def on_validated_execution_timeframe(value: str): - avellaneda_market_making_config_map["start_time"].value = None - avellaneda_market_making_config_map["end_time"].value = None - - -def order_amount_prompt() -> str: - trading_pair = avellaneda_market_making_config_map["market"].value - base_asset, quote_asset = trading_pair.split("-") - return f"What is the amount of {base_asset} per order? >>> " - - -def on_validated_price_source_exchange(value: str): - if value is None: - avellaneda_market_making_config_map["price_source_market"].value = None - - -def exchange_on_validated(value: str): - required_exchanges.append(value) - - -avellaneda_market_making_config_map = { - "strategy": - ConfigVar(key="strategy", - prompt=None, - default="avellaneda_market_making"), - "exchange": - ConfigVar(key="exchange", - prompt="Enter your maker spot connector >>> ", - validator=validate_exchange, - on_validated=exchange_on_validated, - prompt_on_new=True), - "market": - ConfigVar(key="market", - prompt=maker_trading_pair_prompt, - validator=validate_exchange_trading_pair, - prompt_on_new=True), - "execution_timeframe": - ConfigVar(key="execution_timeframe", - prompt="Choose execution timeframe ( infinite / from_date_to_date / daily_between_times ) >>> ", - validator=validate_execution_timeframe, - on_validated=on_validated_execution_timeframe, - prompt_on_new=True), - "start_time": - ConfigVar(key="start_time", - prompt=execution_time_start_prompt, - type_str="str", - validator=validate_execution_time, - required_if=lambda: avellaneda_market_making_config_map.get("execution_timeframe").value != "infinite", - prompt_on_new=True), - "end_time": - ConfigVar(key="end_time", - prompt=execution_time_end_prompt, - type_str="str", - validator=validate_execution_time, - required_if=lambda: avellaneda_market_making_config_map.get("execution_timeframe").value != "infinite", - prompt_on_new=True), - "order_amount": - ConfigVar(key="order_amount", - prompt=order_amount_prompt, - type_str="decimal", - validator=lambda v: validate_decimal(v, min_value=Decimal("0"), inclusive=False), - prompt_on_new=True), - "order_optimization_enabled": - ConfigVar(key="order_optimization_enabled", - prompt="Do you want to enable best bid ask jumping? (Yes/No) >>> ", - type_str="bool", - default=True, - validator=validate_bool), - "risk_factor": - ConfigVar(key="risk_factor", - printable_key="risk_factor(\u03B3)", - prompt="Enter risk factor (\u03B3) >>> ", - type_str="decimal", - default=Decimal("1"), - validator=lambda v: validate_decimal(v, 0, inclusive=False), - prompt_on_new=True), - "order_amount_shape_factor": - ConfigVar(key="order_amount_shape_factor", - printable_key="order_amount_shape_factor(\u03B7)", - prompt="Enter order amount shape factor (\u03B7) >>> ", - type_str="decimal", - default=Decimal("0"), - validator=lambda v: validate_decimal(v, 0, 1, inclusive=True)), - "min_spread": - ConfigVar(key="min_spread", - prompt="Enter minimum spread limit (as % of mid price) >>> ", - type_str="decimal", - validator=lambda v: validate_decimal(v, 0, inclusive=True), - default=Decimal("0")), - "order_refresh_time": - ConfigVar(key="order_refresh_time", - prompt="How often do you want to cancel and replace bids and asks " - "(in seconds)? >>> ", - type_str="float", - validator=lambda v: validate_decimal(v, 0, inclusive=False), - prompt_on_new=True), - "max_order_age": - ConfigVar(key="max_order_age", - prompt="How long do you want to cancel and replace bids and asks " - "with the same price (in seconds)? >>> ", - type_str="float", - default=1800, - validator=lambda v: validate_decimal(v, 0, inclusive=False)), - "order_refresh_tolerance_pct": - ConfigVar(key="order_refresh_tolerance_pct", - prompt="Enter the percent change in price needed to refresh orders at each cycle " - "(Enter 1 to indicate 1%) >>> ", - type_str="decimal", - default=Decimal("0"), - validator=lambda v: validate_decimal(v, -10, 10, inclusive=True)), - "filled_order_delay": - ConfigVar(key="filled_order_delay", - prompt="How long do you want to wait before placing the next order " - "if your order gets filled (in seconds)? >>> ", - type_str="float", - validator=lambda v: validate_decimal(v, min_value=0, inclusive=False), - default=60), - "inventory_target_base_pct": - ConfigVar(key="inventory_target_base_pct", - prompt="What is the inventory target for the base asset? Enter 50 for 50% >>> ", - type_str="decimal", - validator=lambda v: validate_decimal(v, 0, 100), - prompt_on_new=True, - default=Decimal("50")), - "add_transaction_costs": - ConfigVar(key="add_transaction_costs", - prompt="Do you want to add transaction costs automatically to order prices? (Yes/No) >>> ", - type_str="bool", - default=False, - validator=validate_bool), - "volatility_buffer_size": - ConfigVar(key="volatility_buffer_size", - prompt="Enter amount of ticks that will be stored to calculate volatility >>> ", - type_str="int", - validator=lambda v: validate_decimal(v, 1, 10000), - default=200), - "trading_intensity_buffer_size": - ConfigVar(key="trading_intensity_buffer_size", - prompt="Enter amount of ticks that will be stored to estimate order book liquidity >>> ", - type_str="int", - validator=lambda v: validate_int(v, 1, 10000), - default=200), - "order_levels": - ConfigVar(key="order_levels", - prompt="How many orders do you want to place on both sides? >>> ", - type_str="int", - validator=lambda v: validate_int(v, min_value=-1, inclusive=False), - default=1), - "level_distances": - ConfigVar(key="level_distances", - prompt="How far apart in % of optimal spread should orders on one side be? >>> ", - type_str="decimal", - validator=lambda v: validate_decimal(v, min_value=0, inclusive=True), - default=0), - "order_override": - ConfigVar(key="order_override", - prompt=None, - required_if=lambda: False, - default=None, - type_str="json"), - "hanging_orders_enabled": - ConfigVar(key="hanging_orders_enabled", - prompt="Do you want to enable hanging orders? (Yes/No) >>> ", - type_str="bool", - default=False, - validator=validate_bool), - "hanging_orders_cancel_pct": - ConfigVar(key="hanging_orders_cancel_pct", - prompt="At what spread percentage (from mid price) will hanging orders be canceled? " - "(Enter 1 to indicate 1%) >>> ", - required_if=lambda: avellaneda_market_making_config_map.get("hanging_orders_enabled").value, - type_str="decimal", - default=Decimal("10"), - validator=lambda v: validate_decimal(v, 0, 100, inclusive=False)), - "should_wait_order_cancel_confirmation": - ConfigVar(key="should_wait_order_cancel_confirmation", - prompt="Should the strategy wait to receive a confirmation for orders cancellation " - "before creating a new set of orders? " - "(Not waiting requires enough available balance) (Yes/No) >>> ", - type_str="bool", - default=True, - validator=validate_bool), -} diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py index 77941eabf5..03cd05c80f 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -493,4 +493,4 @@ def execution_timeframe_post_validation(cls, values: Dict): @classmethod def exchange_post_validation(cls, values: Dict): - required_exchanges.append(values["exchange"]) + required_exchanges.add(values["exchange"]) diff --git a/hummingbot/strategy/celo_arb/celo_arb_config_map.py b/hummingbot/strategy/celo_arb/celo_arb_config_map.py index 179fa3112e..4641bfbe8d 100644 --- a/hummingbot/strategy/celo_arb/celo_arb_config_map.py +++ b/hummingbot/strategy/celo_arb/celo_arb_config_map.py @@ -12,7 +12,7 @@ def exchange_on_validated(value: str) -> None: - required_exchanges.append(value) + required_exchanges.add(value) def market_trading_pair_prompt() -> str: diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py index 14d6a2e2bc..fbe57e903c 100644 --- a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py +++ b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py @@ -53,7 +53,7 @@ def order_amount_prompt() -> str: def taker_market_on_validated(value: str): - settings.required_exchanges.append(value) + settings.required_exchanges.add(value) def update_oracle_settings(value: str): @@ -87,7 +87,7 @@ def update_oracle_settings(value: str): prompt="Enter your maker spot connector >>> ", prompt_on_new=True, validator=validate_exchange, - on_validated=lambda value: settings.required_exchanges.append(value), + on_validated=lambda value: settings.required_exchanges.add(value), ), "taker_market": ConfigVar( key="taker_market", diff --git a/hummingbot/strategy/dev_0_hello_world/dev_0_hello_world_config_map.py b/hummingbot/strategy/dev_0_hello_world/dev_0_hello_world_config_map.py index d3f01d452c..fbf913f125 100644 --- a/hummingbot/strategy/dev_0_hello_world/dev_0_hello_world_config_map.py +++ b/hummingbot/strategy/dev_0_hello_world/dev_0_hello_world_config_map.py @@ -14,7 +14,7 @@ def exchange_on_validated(value: str) -> None: - required_exchanges.append(value) + required_exchanges.add(value) def trading_pair_prompt(): diff --git a/hummingbot/strategy/dev_1_get_order_book/dev_1_get_order_book_config_map.py b/hummingbot/strategy/dev_1_get_order_book/dev_1_get_order_book_config_map.py index 52a6e4ed1f..60b28b2d2e 100644 --- a/hummingbot/strategy/dev_1_get_order_book/dev_1_get_order_book_config_map.py +++ b/hummingbot/strategy/dev_1_get_order_book/dev_1_get_order_book_config_map.py @@ -37,7 +37,7 @@ def validate_trading_pair(value: str) -> Optional[str]: ConfigVar(key="exchange", prompt="Enter the name of the exchange >>> ", validator=validate_exchange, - on_validated=lambda value: required_exchanges.append(value), + on_validated=lambda value: required_exchanges.add(value), prompt_on_new=True, ), "trading_pair": diff --git a/hummingbot/strategy/dev_2_perform_trade/dev_2_perform_trade_config_map.py b/hummingbot/strategy/dev_2_perform_trade/dev_2_perform_trade_config_map.py index c29b8516d4..c25a68a80a 100644 --- a/hummingbot/strategy/dev_2_perform_trade/dev_2_perform_trade_config_map.py +++ b/hummingbot/strategy/dev_2_perform_trade/dev_2_perform_trade_config_map.py @@ -64,7 +64,7 @@ def order_amount_prompt() -> str: ConfigVar(key="exchange", prompt="Enter the name of the exchange >>> ", validator=validate_exchange, - on_validated=lambda value: required_exchanges.append(value), + on_validated=lambda value: required_exchanges.add(value), prompt_on_new=True, ), "trading_pair": diff --git a/hummingbot/strategy/dev_5_vwap/dev_5_vwap_config_map.py b/hummingbot/strategy/dev_5_vwap/dev_5_vwap_config_map.py index ea28225a2f..41186bfdd0 100644 --- a/hummingbot/strategy/dev_5_vwap/dev_5_vwap_config_map.py +++ b/hummingbot/strategy/dev_5_vwap/dev_5_vwap_config_map.py @@ -42,7 +42,7 @@ def order_percent_of_volume_prompt(): ConfigVar(key="exchange", prompt="Enter the name of the exchange >>> ", validator=validate_exchange, - on_validated=lambda value: required_exchanges.append(value), + on_validated=lambda value: required_exchanges.add(value), prompt_on_new=True), "market": ConfigVar(key="market", diff --git a/hummingbot/strategy/dev_simple_trade/dev_simple_trade_config_map.py b/hummingbot/strategy/dev_simple_trade/dev_simple_trade_config_map.py index fde8306e95..44a122485a 100644 --- a/hummingbot/strategy/dev_simple_trade/dev_simple_trade_config_map.py +++ b/hummingbot/strategy/dev_simple_trade/dev_simple_trade_config_map.py @@ -36,7 +36,7 @@ def validate_market_trading_pair_tuple(value: str) -> Optional[str]: ConfigVar(key="market", prompt="Enter the name of the exchange >>> ", validator=validate_exchange, - on_validated=lambda value: required_exchanges.append(value)), + on_validated=lambda value: required_exchanges.add(value)), "market_trading_pair_tuple": ConfigVar(key="market_trading_pair_tuple", prompt=trading_pair_prompt, diff --git a/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py b/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py index 40a002800e..a6c0608f62 100644 --- a/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py +++ b/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py @@ -18,7 +18,7 @@ def exchange_on_validated(value: str) -> None: - required_exchanges.append(value) + required_exchanges.add(value) def market_validate(value: str) -> Optional[str]: diff --git a/hummingbot/strategy/perpetual_market_making/perpetual_market_making_config_map.py b/hummingbot/strategy/perpetual_market_making/perpetual_market_making_config_map.py index 615ec7a5bd..9801579bb7 100644 --- a/hummingbot/strategy/perpetual_market_making/perpetual_market_making_config_map.py +++ b/hummingbot/strategy/perpetual_market_making/perpetual_market_making_config_map.py @@ -103,7 +103,7 @@ def validate_price_floor_ceiling(value: str) -> Optional[str]: def derivative_on_validated(value: str): - required_exchanges.append(value) + required_exchanges.add(value) perpetual_market_making_config_map = { diff --git a/hummingbot/strategy/pure_market_making/pure_market_making_config_map.py b/hummingbot/strategy/pure_market_making/pure_market_making_config_map.py index f64e648066..54dfa4ab49 100644 --- a/hummingbot/strategy/pure_market_making/pure_market_making_config_map.py +++ b/hummingbot/strategy/pure_market_making/pure_market_making_config_map.py @@ -105,7 +105,7 @@ def on_validated_price_type(value: str): def exchange_on_validated(value: str): - required_exchanges.append(value) + required_exchanges.add(value) pure_market_making_config_map = { diff --git a/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py index 23a4a133ec..cf3fc7ca0b 100644 --- a/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py +++ b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py @@ -15,7 +15,7 @@ def exchange_on_validated(value: str) -> None: - required_exchanges.append(value) + required_exchanges.add(value) def spot_market_validator(value: str) -> None: diff --git a/hummingbot/strategy/twap/twap_config_map.py b/hummingbot/strategy/twap/twap_config_map.py index de7b7e6d3f..df612598f2 100644 --- a/hummingbot/strategy/twap/twap_config_map.py +++ b/hummingbot/strategy/twap/twap_config_map.py @@ -75,7 +75,7 @@ def validate_order_step_size(value: str = None): ConfigVar(key="connector", prompt="Enter the name of spot connector >>> ", validator=validate_exchange, - on_validated=lambda value: required_exchanges.append(value), + on_validated=lambda value: required_exchanges.add(value), prompt_on_new=True), "trading_pair": ConfigVar(key="trading_pair", diff --git a/hummingbot/strategy/uniswap_v3_lp/uniswap_v3_lp_config_map.py b/hummingbot/strategy/uniswap_v3_lp/uniswap_v3_lp_config_map.py index 168073fc3d..199efa6965 100644 --- a/hummingbot/strategy/uniswap_v3_lp/uniswap_v3_lp_config_map.py +++ b/hummingbot/strategy/uniswap_v3_lp/uniswap_v3_lp_config_map.py @@ -19,7 +19,7 @@ def market_validator(value: str) -> None: def market_on_validated(value: str) -> None: - required_exchanges.append(value) + required_exchanges.add(value) requried_connector_trading_pairs["uniswap_v3"] = [value] From c7be7428710e85368cd6bb8cc5a41e3c5261f13c Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 23 Mar 2022 10:48:43 +0700 Subject: [PATCH 023/152] Update hummingbot/client/command/config_command.py Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- hummingbot/client/command/config_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/client/command/config_command.py b/hummingbot/client/command/config_command.py index 33e57e32cb..56c7547c13 100644 --- a/hummingbot/client/command/config_command.py +++ b/hummingbot/client/command/config_command.py @@ -1,7 +1,7 @@ import asyncio from decimal import Decimal from os.path import join -from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union +from typing import Any, Dict, List, Tuple, TYPE_CHECKING, Union import pandas as pd from prompt_toolkit.utils import is_windows From 2d0cb71982e1c1882da59ad4de1d4d908ecaef29 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 23 Mar 2022 10:52:34 +0700 Subject: [PATCH 024/152] Update hummingbot/client/command/create_command.py Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- hummingbot/client/command/create_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/client/command/create_command.py b/hummingbot/client/command/create_command.py index 7db6a58365..bf5d42627f 100644 --- a/hummingbot/client/command/create_command.py +++ b/hummingbot/client/command/create_command.py @@ -2,7 +2,7 @@ import copy import os import shutil -from typing import TYPE_CHECKING, Dict, Optional +from typing import Dict, Optional, TYPE_CHECKING from pydantic import ValidationError From e138933bbdf15a9d2cd406c7bde0f94d6ccad8fd Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 23 Mar 2022 10:52:46 +0700 Subject: [PATCH 025/152] Update hummingbot/client/command/status_command.py Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- hummingbot/client/command/status_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/client/command/status_command.py b/hummingbot/client/command/status_command.py index 7d8952e2ac..e304ed3832 100644 --- a/hummingbot/client/command/status_command.py +++ b/hummingbot/client/command/status_command.py @@ -7,7 +7,7 @@ OrderedDict ) import inspect -from typing import List, Dict +from typing import Dict, List from pydantic import ValidationError, validate_model From b417433dcac658bc885714dc398c5efe115b4be6 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 23 Mar 2022 10:52:57 +0700 Subject: [PATCH 026/152] Update hummingbot/client/command/status_command.py Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- hummingbot/client/command/status_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/client/command/status_command.py b/hummingbot/client/command/status_command.py index e304ed3832..1d03628f41 100644 --- a/hummingbot/client/command/status_command.py +++ b/hummingbot/client/command/status_command.py @@ -18,8 +18,8 @@ from hummingbot.core.network_iterator import NetworkStatus from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.config_helpers import ( + get_strategy_config_map, missing_required_configs_legacy, - get_strategy_config_map ) from hummingbot.client.config.security import Security from hummingbot.user.user_balances import UserBalances From b5683393d350e66ed7dd667f46f5d0d15b4dfc03 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 23 Mar 2022 10:54:56 +0700 Subject: [PATCH 027/152] Update hummingbot/strategy/avellaneda_market_making/start.py Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- hummingbot/strategy/avellaneda_market_making/start.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/hummingbot/strategy/avellaneda_market_making/start.py b/hummingbot/strategy/avellaneda_market_making/start.py index 4096680b0a..9e6f712f32 100644 --- a/hummingbot/strategy/avellaneda_market_making/start.py +++ b/hummingbot/strategy/avellaneda_market_making/start.py @@ -10,10 +10,10 @@ from hummingbot.client.hummingbot_application import HummingbotApplication from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( AvellanedaMarketMakingConfigMap, - MultiOrderLevelModel, - TrackHangingOrdersModel, - FromDateToDateModel, DailyBetweenTimesModel, + FromDateToDateModel, + MultiOrderLevelModel, + TrackHangingOrdersModel, ) from hummingbot.strategy.conditional_execution_state import ( RunAlwaysExecutionState, From bffd78bc41f6c8fca3485160ca8f9aedcf76199b Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 23 Mar 2022 11:56:42 +0700 Subject: [PATCH 028/152] Update test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- .../test_avellaneda_market_making_config_map_pydantic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py index d508716c37..4460b7959e 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py @@ -2,7 +2,7 @@ import unittest from datetime import datetime, time from pathlib import Path -from typing import Dict, Awaitable +from typing import Awaitable, Dict from unittest.mock import patch import yaml From a3c5639d9a15781255ee72bd59f5966213da0549 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 23 Mar 2022 11:56:53 +0700 Subject: [PATCH 029/152] Update test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- .../test_avellaneda_market_making_start.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py index 3cb4fd1256..c4bb68dc80 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py @@ -6,8 +6,10 @@ from hummingbot.connector.exchange_base import ExchangeBase from hummingbot.connector.utils import combine_to_hb_trading_pair from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( - AvellanedaMarketMakingConfigMap, FromDateToDateModel, - TrackHangingOrdersModel, MultiOrderLevelModel + AvellanedaMarketMakingConfigMap, + FromDateToDateModel, + MultiOrderLevelModel, + TrackHangingOrdersModel, ) From f1fcf3be3a5b9df91114eba316a4b500cb115088 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 23 Mar 2022 11:57:53 +0700 Subject: [PATCH 030/152] (fix) Addresses @aarmoa's PR comments. --- hummingbot/client/command/config_command.py | 4 ---- hummingbot/client/ui/interface_utils.py | 5 +++-- hummingbot/strategy/avellaneda_market_making/start.py | 1 - 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/hummingbot/client/command/config_command.py b/hummingbot/client/command/config_command.py index 56c7547c13..adf3266b55 100644 --- a/hummingbot/client/command/config_command.py +++ b/hummingbot/client/command/config_command.py @@ -284,9 +284,6 @@ async def asset_ratio_maintenance_prompt( config_map: BaseTradingStrategyConfigMap, input_value: Any = None, ): # pragma: no cover - """ - Will be removed: https://app.shortcut.com/coinalpha/story/24923/collect-special-case-prompts-via-config-models - """ if input_value: config_map.inventory_target_base_pct = input_value else: @@ -365,7 +362,6 @@ async def inventory_price_prompt( ): """ Not currently used. - Will be removed: https://app.shortcut.com/coinalpha/story/24923/collect-special-case-prompts-via-config-models """ raise NotImplementedError diff --git a/hummingbot/client/ui/interface_utils.py b/hummingbot/client/ui/interface_utils.py index e846d49631..1d2da690a9 100644 --- a/hummingbot/client/ui/interface_utils.py +++ b/hummingbot/client/ui/interface_utils.py @@ -1,5 +1,6 @@ import asyncio import datetime +from decimal import Decimal from typing import ( List, Optional, @@ -10,7 +11,6 @@ import pandas as pd import psutil import tabulate -from decimal import Decimal from hummingbot.client.config.global_config_map import global_config_map @@ -112,9 +112,10 @@ def format_df_for_printout( df.columns = [c if len(c) < max_col_width else f"{c[:max_col_width - 3]}..." for c in df.columns] table_format = table_format or global_config_map.get("tables_format").value + original_preserve_whitespace = tabulate.PRESERVE_WHITESPACE tabulate.PRESERVE_WHITESPACE = True try: formatted_df = tabulate.tabulate(df, tablefmt=table_format, showindex=index, headers="keys") finally: - tabulate.PRESERVE_WHITESPACE = False + tabulate.PRESERVE_WHITESPACE = original_preserve_whitespace return formatted_df diff --git a/hummingbot/strategy/avellaneda_market_making/start.py b/hummingbot/strategy/avellaneda_market_making/start.py index 4096680b0a..4847a92de3 100644 --- a/hummingbot/strategy/avellaneda_market_making/start.py +++ b/hummingbot/strategy/avellaneda_market_making/start.py @@ -87,7 +87,6 @@ def start(self): f"_{pd.Timestamp.now().strftime('%Y-%m-%d_%H-%M-%S')}.csv") self.strategy = AvellanedaMarketMakingStrategy() - self.logger().debug("Starting !!!!!!!!!!!!") self.strategy.init_params( market_info=MarketTradingPairTuple(*maker_data), order_amount=order_amount, From e9f489d5a672902cdbea8b13a5c08560cfa848cc Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Thu, 31 Mar 2022 10:57:27 +0700 Subject: [PATCH 031/152] Merge branch 'feat/config_management_refactoring' into feat/import_command_works_with_pydantic_configs # Conflicts: # hummingbot/client/config/config_helpers.py --- hummingbot/client/command/balance_command.py | 6 +- hummingbot/client/command/config_command.py | 244 ++++++++++++----- hummingbot/client/command/connect_command.py | 16 +- hummingbot/client/command/create_command.py | 198 +++++++++++--- hummingbot/client/command/start_command.py | 4 +- hummingbot/client/command/status_command.py | 43 ++- hummingbot/client/config/config_data_types.py | 230 +++++++++++++++- hummingbot/client/hummingbot_application.py | 12 +- hummingbot/client/settings.py | 2 +- hummingbot/client/ui/completer.py | 2 +- hummingbot/client/ui/interface_utils.py | 18 +- hummingbot/client/ui/style.py | 4 +- .../strategy/amm_arb/amm_arb_config_map.py | 2 +- .../arbitrage/arbitrage_config_map.py | 4 +- .../aroon_oscillator_config_map.py | 2 +- .../avellaneda_market_making_config_map.py | 245 ------------------ ...aneda_market_making_config_map_pydantic.py | 70 ++--- .../avellaneda_market_making/start.py | 85 +++--- .../strategy/celo_arb/celo_arb_config_map.py | 2 +- ...cross_exchange_market_making_config_map.py | 4 +- .../dev_0_hello_world_config_map.py | 2 +- .../dev_1_get_order_book_config_map.py | 2 +- .../dev_2_perform_trade_config_map.py | 2 +- .../dev_5_vwap/dev_5_vwap_config_map.py | 2 +- .../dev_simple_trade_config_map.py | 2 +- .../liquidity_mining_config_map.py | 2 +- .../perpetual_market_making_config_map.py | 2 +- .../pure_market_making_config_map.py | 2 +- .../spot_perpetual_arbitrage_config_map.py | 2 +- hummingbot/strategy/twap/twap_config_map.py | 2 +- .../uniswap_v3_lp/uniswap_v3_lp_config_map.py | 2 +- .../client/command/test_config_command.py | 175 +++++++++++-- .../client/command/test_create_command.py | 6 +- .../client/config/test_config_data_types.py | 230 ++++++++++++++++ .../client/config/test_config_helpers.py | 30 +++ .../client/config/test_config_templates.py | 27 +- ...aneda_market_making_config_map_pydantic.py | 88 +++---- .../test_avellaneda_market_making_start.py | 70 +++-- .../avellaneda_market_making/test_config.yml | 2 +- 39 files changed, 1236 insertions(+), 607 deletions(-) delete mode 100644 hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map.py create mode 100644 test/hummingbot/client/config/test_config_data_types.py diff --git a/hummingbot/client/command/balance_command.py b/hummingbot/client/command/balance_command.py index 8e1d77cb2b..635f6333a4 100644 --- a/hummingbot/client/command/balance_command.py +++ b/hummingbot/client/command/balance_command.py @@ -8,7 +8,7 @@ from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.config_helpers import ( - save_to_yml + save_to_yml_legacy ) from hummingbot.client.config.config_validators import validate_decimal, validate_exchange from hummingbot.connector.other.celo.celo_cli import CeloCLI @@ -64,7 +64,7 @@ def balance(self, elif amount >= 0: config_var.value[exchange][asset] = amount self._notify(f"Limit for {asset} on {exchange} exchange set to {amount}") - save_to_yml(file_path, config_map) + save_to_yml_legacy(file_path, config_map) elif option == "paper": config_var = config_map["paper_trade_account_balance"] @@ -81,7 +81,7 @@ def balance(self, paper_balances[asset] = amount config_var.value = paper_balances self._notify(f"Paper balance for {asset} token set to {amount}") - save_to_yml(file_path, config_map) + save_to_yml_legacy(file_path, config_map) async def show_balances(self): total_col_name = f'Total ({RateOracle.global_token_symbol})' diff --git a/hummingbot/client/command/config_command.py b/hummingbot/client/command/config_command.py index ec80041629..adf3266b55 100644 --- a/hummingbot/client/command/config_command.py +++ b/hummingbot/client/command/config_command.py @@ -1,28 +1,25 @@ import asyncio from decimal import Decimal from os.path import join -from typing import ( - Any, - List, - TYPE_CHECKING, -) +from typing import Any, Dict, List, Tuple, TYPE_CHECKING, Union import pandas as pd +from prompt_toolkit.utils import is_windows -from hummingbot.client.config.config_helpers import ( - missing_required_configs, - save_to_yml, +from hummingbot.client.config.config_data_types import ( + BaseClientModel, + BaseStrategyConfigMap, + BaseTradingStrategyConfigMap ) +from hummingbot.client.config.config_helpers import missing_required_configs_legacy, save_to_yml, save_to_yml_legacy from hummingbot.client.config.config_validators import validate_bool, validate_decimal from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.security import Security -from hummingbot.client.settings import ( - CONF_FILE_PATH, - GLOBAL_CONFIG_PATH, -) +from hummingbot.client.settings import CONF_FILE_PATH, GLOBAL_CONFIG_PATH from hummingbot.client.ui.interface_utils import format_df_for_printout from hummingbot.client.ui.style import load_style +from hummingbot.connector.utils import split_hb_trading_pair from hummingbot.core.utils import map_df_to_str from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.model.inventory_cost import InventoryCost @@ -66,6 +63,7 @@ "input-pane", "logs-pane", "terminal-primary"] +columns = ["Key", "Value"] class ConfigCommand: @@ -77,14 +75,19 @@ def config(self, # type: HummingbotApplication self.list_configs() return else: - if key not in self.config_able_keys(): + if key not in self.configurable_keys(): self._notify("Invalid key, please choose from the list.") return safe_ensure_future(self._config_single_key(key, value), loop=self.ev_loop) def list_configs(self, # type: HummingbotApplication ): - columns = ["Key", " Value"] + self.list_global_configs() + self.list_strategy_configs() + + def list_global_configs( + self # type: HummingbotApplication + ): data = [[cv.key, cv.value] for cv in global_config_map.values() if cv.key in global_configs_to_display and not cv.is_secure] df = map_df_to_str(pd.DataFrame(data=data, columns=columns)) @@ -99,22 +102,58 @@ def list_configs(self, # type: HummingbotApplication lines = [" " + line for line in format_df_for_printout(df, max_col_width=50).split("\n")] self._notify("\n".join(lines)) + def list_strategy_configs( + self # type: HummingbotApplication + ): if self.strategy_name is not None: - data = [[cv.printable_key or cv.key, cv.value] for cv in self.strategy_config_map.values() if not cv.is_secure] + config_map = self.strategy_config_map + data = self.build_df_data_from_config_map(config_map) df = map_df_to_str(pd.DataFrame(data=data, columns=columns)) self._notify("\nStrategy Configurations:") lines = [" " + line for line in format_df_for_printout(df, max_col_width=50).split("\n")] self._notify("\n".join(lines)) - def config_able_keys(self # type: HummingbotApplication - ) -> List[str]: + def build_df_data_from_config_map( + self, # type: HummingbotApplication + config_map: Union[BaseClientModel, Dict[str, ConfigVar]] + ) -> List[Tuple[str, Any]]: + if isinstance(config_map, BaseClientModel): + data = self.build_model_df_data(config_map) + else: # legacy + data = [[cv.printable_key or cv.key, cv.value] for cv in self.strategy_config_map.values() if not cv.is_secure] + return data + + @staticmethod + def build_model_df_data(config_map: BaseClientModel) -> List[Tuple[str, Any]]: + model_data = [] + for traversal_item in config_map.traverse(): + attr_printout = ( + " " * (traversal_item.depth - 1) + + (u"\u221F " if not is_windows() else " ") + + traversal_item.attr + ) if traversal_item.depth else traversal_item.attr + model_data.append((attr_printout, traversal_item.printable_value)) + return model_data + + def configurable_keys(self # type: HummingbotApplication + ) -> List[str]: """ Returns a list of configurable keys - using config command, excluding exchanges api keys as they are set from connect command. """ keys = [c.key for c in global_config_map.values() if c.prompt is not None and not c.is_connect_key] if self.strategy_config_map is not None: - keys += [c.key for c in self.strategy_config_map.values() if c.prompt is not None] + if isinstance(self.strategy_config_map, BaseStrategyConfigMap): + keys.extend([ + traversal_item.config_path + for traversal_item in self.strategy_config_map.traverse() + if ( + traversal_item.client_field_data is not None + and traversal_item.client_field_data.prompt is not None + ) + ]) + else: # legacy + keys.extend([c.key for c in self.strategy_config_map.values() if c.prompt is not None]) return keys async def check_password(self, # type: HummingbotApplication @@ -149,42 +188,32 @@ async def _config_single_key(self, # type: HummingbotApplication self.app.hide_input = True try: - config_var, config_map, file_path = None, None, None - if key in global_config_map: - config_map = global_config_map - file_path = GLOBAL_CONFIG_PATH - elif self.strategy_config_map is not None and key in self.strategy_config_map: + if ( + key in global_config_map + or ( + not isinstance(self.strategy_config_map, (type(None), BaseClientModel)) + and key in self.strategy_config_map + ) + ): + await self._config_single_key_legacy(key, input_value) + else: config_map = self.strategy_config_map file_path = join(CONF_FILE_PATH, self.strategy_file_name) - config_var = config_map[key] - if input_value is None: - self._notify("Please follow the prompt to complete configurations: ") - if config_var.key == "inventory_target_base_pct": - await self.asset_ratio_maintenance_prompt(config_map, input_value) - elif config_var.key == "inventory_price": - await self.inventory_price_prompt(config_map, input_value) - else: - await self.prompt_a_config(config_var, input_value=input_value, assign_default=False) - if self.app.to_stop_config: - self.app.to_stop_config = False - return - await self.update_all_secure_configs() - missings = missing_required_configs(config_map) - if missings: - self._notify("\nThere are other configuration required, please follow the prompt to complete them.") - missings = await self._prompt_missing_configs(config_map) - save_to_yml(file_path, config_map) - self._notify("\nNew configuration saved:") - self._notify(f"{key}: {str(config_var.value)}") - self.app.app.style = load_style() - for config in missings: - self._notify(f"{config.key}: {str(config.value)}") - if isinstance(self.strategy, PureMarketMakingStrategy) or \ - isinstance(self.strategy, PerpetualMarketMakingStrategy): - updated = ConfigCommand.update_running_mm(self.strategy, key, config_var.value) - if updated: - self._notify(f"\nThe current {self.strategy_name} strategy has been updated " - f"to reflect the new configuration.") + if input_value is None: + self._notify("Please follow the prompt to complete configurations: ") + if key == "inventory_target_base_pct": + await self.asset_ratio_maintenance_prompt(config_map, input_value) + elif key == "inventory_price": + await self.inventory_price_prompt(config_map, input_value) + else: + await self.prompt_a_config(config_map, key, input_value) + if self.app.to_stop_config: + self.app.to_stop_config = False + return + save_to_yml(file_path, config_map) + self._notify("\nNew configuration saved.") + self.list_strategy_configs() + self.app.app.style = load_style() except asyncio.TimeoutError: self.logger().error("Prompt timeout") except Exception as err: @@ -194,21 +223,104 @@ async def _config_single_key(self, # type: HummingbotApplication self.placeholder_mode = False self.app.change_prompt(prompt=">>> ") + async def _config_single_key_legacy( + self, # type: HummingbotApplication + key: str, + input_value: Any, + ): # pragma: no cover + config_var, config_map, file_path = None, None, None + if key in global_config_map: + config_map = global_config_map + file_path = GLOBAL_CONFIG_PATH + elif self.strategy_config_map is not None and key in self.strategy_config_map: + config_map = self.strategy_config_map + file_path = join(CONF_FILE_PATH, self.strategy_file_name) + config_var = config_map[key] + if input_value is None: + self._notify("Please follow the prompt to complete configurations: ") + if config_var.key == "inventory_target_base_pct": + await self.asset_ratio_maintenance_prompt_legacy(config_map, input_value) + elif config_var.key == "inventory_price": + await self.inventory_price_prompt_legacy(config_map, input_value) + else: + await self.prompt_a_config_legacy(config_var, input_value=input_value, assign_default=False) + if self.app.to_stop_config: + self.app.to_stop_config = False + return + await self.update_all_secure_configs_legacy() + missings = missing_required_configs_legacy(config_map) + if missings: + self._notify("\nThere are other configuration required, please follow the prompt to complete them.") + missings = await self._prompt_missing_configs(config_map) + save_to_yml_legacy(file_path, config_map) + self._notify("\nNew configuration saved:") + self._notify(f"{key}: {str(config_var.value)}") + self.app.app.style = load_style() + for config in missings: + self._notify(f"{config.key}: {str(config.value)}") + if ( + isinstance(self.strategy, PureMarketMakingStrategy) or + isinstance(self.strategy, PerpetualMarketMakingStrategy) + ): + updated = ConfigCommand.update_running_mm(self.strategy, key, config_var.value) + if updated: + self._notify(f"\nThe current {self.strategy_name} strategy has been updated " + f"to reflect the new configuration.") + async def _prompt_missing_configs(self, # type: HummingbotApplication config_map): - missings = missing_required_configs(config_map) + missings = missing_required_configs_legacy(config_map) for config in missings: - await self.prompt_a_config(config) + await self.prompt_a_config_legacy(config) if self.app.to_stop_config: self.app.to_stop_config = False return - if missing_required_configs(config_map): + if missing_required_configs_legacy(config_map): return missings + (await self._prompt_missing_configs(config_map)) return missings - async def asset_ratio_maintenance_prompt(self, # type: HummingbotApplication - config_map, - input_value = None): + async def asset_ratio_maintenance_prompt( + self, # type: HummingbotApplication + config_map: BaseTradingStrategyConfigMap, + input_value: Any = None, + ): # pragma: no cover + if input_value: + config_map.inventory_target_base_pct = input_value + else: + exchange = config_map.exchange + market = config_map.market + base, quote = split_hb_trading_pair(market) + balances = await UserBalances.instance().balances(exchange, base, quote) + if balances is None: + return + base_ratio = await UserBalances.base_amount_ratio(exchange, market, balances) + if base_ratio is None: + return + base_ratio = round(base_ratio, 3) + quote_ratio = 1 - base_ratio + + cvar = ConfigVar(key="temp_config", + prompt=f"On {exchange}, you have {balances.get(base, 0):.4f} {base} and " + f"{balances.get(quote, 0):.4f} {quote}. By market value, " + f"your current inventory split is {base_ratio:.1%} {base} " + f"and {quote_ratio:.1%} {quote}." + f" Would you like to keep this ratio? (Yes/No) >>> ", + required_if=lambda: True, + type_str="bool", + validator=validate_bool) + await self.prompt_a_config_legacy(cvar) + if cvar.value: + config_map.inventory_target_base_pct = round(base_ratio * Decimal('100'), 1) + elif self.app.to_stop_config: + self.app.to_stop_config = False + else: + await self.prompt_a_config(config_map, config="inventory_target_base_pct") + + async def asset_ratio_maintenance_prompt_legacy( + self, # type: HummingbotApplication + config_map, + input_value=None, + ): if input_value: config_map['inventory_target_base_pct'].value = Decimal(input_value) else: @@ -234,16 +346,26 @@ async def asset_ratio_maintenance_prompt(self, # type: HummingbotApplication required_if=lambda: True, type_str="bool", validator=validate_bool) - await self.prompt_a_config(cvar) + await self.prompt_a_config_legacy(cvar) if cvar.value: config_map['inventory_target_base_pct'].value = round(base_ratio * Decimal('100'), 1) else: if self.app.to_stop_config: self.app.to_stop_config = False return - await self.prompt_a_config(config_map["inventory_target_base_pct"]) + await self.prompt_a_config_legacy(config_map["inventory_target_base_pct"]) async def inventory_price_prompt( + self, # type: HummingbotApplication + model: BaseTradingStrategyConfigMap, + input_value=None, + ): + """ + Not currently used. + """ + raise NotImplementedError + + async def inventory_price_prompt_legacy( self, # type: HummingbotApplication config_map, input_value=None, @@ -275,7 +397,7 @@ async def inventory_price_prompt( v, min_value=Decimal("0"), inclusive=True ), ) - await self.prompt_a_config(cvar) + await self.prompt_a_config_legacy(cvar) config_map[key].value = cvar.value try: diff --git a/hummingbot/client/command/connect_command.py b/hummingbot/client/command/connect_command.py index b54ade8b3c..ef8b60309c 100644 --- a/hummingbot/client/command/connect_command.py +++ b/hummingbot/client/command/connect_command.py @@ -10,7 +10,7 @@ from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.client.config.global_config_map import global_config_map from hummingbot.user.user_balances import UserBalances -from hummingbot.client.config.config_helpers import save_to_yml +from hummingbot.client.config.config_helpers import save_to_yml_legacy from hummingbot.connector.other.celo.celo_cli import CeloCLI from hummingbot.connector.connector_status import get_connector_status if TYPE_CHECKING: @@ -59,7 +59,7 @@ async def connect_exchange(self, # type: HummingbotApplication to_connect = False if to_connect: for config in exchange_configs: - await self.prompt_a_config(config) + await self.prompt_a_config_legacy(config) if self.app.to_stop_config: self.app.to_stop_config = False return @@ -161,13 +161,13 @@ async def connect_ethereum(self, # type: HummingbotApplication public_address = Security.add_private_key(private_key) global_config_map["ethereum_wallet"].value = public_address if global_config_map["ethereum_rpc_url"].value is None: - await self.prompt_a_config(global_config_map["ethereum_rpc_url"]) + await self.prompt_a_config_legacy(global_config_map["ethereum_rpc_url"]) if global_config_map["ethereum_rpc_ws_url"].value is None: - await self.prompt_a_config(global_config_map["ethereum_rpc_ws_url"]) + await self.prompt_a_config_legacy(global_config_map["ethereum_rpc_ws_url"]) if self.app.to_stop_config: self.app.to_stop_config = False return - save_to_yml(GLOBAL_CONFIG_PATH, global_config_map) + save_to_yml_legacy(GLOBAL_CONFIG_PATH, global_config_map) err_msg = UserBalances.validate_ethereum_wallet() if err_msg is None: self._notify(f"Wallet {public_address} connected to hummingbot.") @@ -189,9 +189,9 @@ async def connect_celo(self, # type: HummingbotApplication if answer.lower() not in ("yes", "y"): to_connect = False if to_connect: - await self.prompt_a_config(global_config_map["celo_address"]) - await self.prompt_a_config(global_config_map["celo_password"]) - save_to_yml(GLOBAL_CONFIG_PATH, global_config_map) + await self.prompt_a_config_legacy(global_config_map["celo_address"]) + await self.prompt_a_config_legacy(global_config_map["celo_password"]) + save_to_yml_legacy(GLOBAL_CONFIG_PATH, global_config_map) err_msg = await self.validate_n_connect_celo(True, global_config_map["celo_address"].value, diff --git a/hummingbot/client/command/create_command.py b/hummingbot/client/command/create_command.py index f54ae60ea8..bf5d42627f 100644 --- a/hummingbot/client/command/create_command.py +++ b/hummingbot/client/command/create_command.py @@ -2,24 +2,28 @@ import copy import os import shutil +from typing import Dict, Optional, TYPE_CHECKING -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.core.utils.async_utils import safe_ensure_future +from pydantic import ValidationError + +from hummingbot.client.config.config_data_types import BaseClientModel, BaseStrategyConfigMap from hummingbot.client.config.config_helpers import ( + default_strategy_file_path, + format_config_file_name, get_strategy_config_map, + get_strategy_template_path, + parse_config_default_to_text, parse_cvar_value, - default_strategy_file_path, + retrieve_validation_error_msg, save_to_yml, - get_strategy_template_path, - format_config_file_name, - parse_config_default_to_text + save_to_yml_legacy ) -from hummingbot.client.settings import CONF_FILE_PATH, required_exchanges +from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.security import Security -from hummingbot.client.config.config_validators import validate_strategy +from hummingbot.client.settings import CONF_FILE_PATH, required_exchanges from hummingbot.client.ui.completer import load_completer -from typing import TYPE_CHECKING, Dict, Optional +from hummingbot.core.utils.async_utils import safe_ensure_future if TYPE_CHECKING: from hummingbot.client.hummingbot_application import HummingbotApplication @@ -36,25 +40,81 @@ def create(self, # type: HummingbotApplication safe_ensure_future(self.prompt_for_configuration(file_name)) - async def prompt_for_configuration(self, # type: HummingbotApplication - file_name): + async def prompt_for_configuration( + self, # type: HummingbotApplication + file_name, + ): self.app.clear_input() self.placeholder_mode = True self.app.hide_input = True required_exchanges.clear() - strategy_config = ConfigVar(key="strategy", - prompt="What is your market making strategy? >>> ", - validator=validate_strategy) - await self.prompt_a_config(strategy_config) + strategy = await self.get_strategy_name() + if self.app.to_stop_config: self.stop_config() return - strategy = strategy_config.value + config_map = get_strategy_config_map(strategy) - config_map_backup = copy.deepcopy(config_map) self._notify(f"Please see https://docs.hummingbot.io/strategies/{strategy.replace('_', '-')}/ " f"while setting up these below configuration.") + + if isinstance(config_map, BaseStrategyConfigMap): + await self.prompt_for_model_config(config_map) + elif config_map is not None: + await self.prompt_for_configuration_legacy(file_name, strategy, config_map) + self.app.to_stop_config = True + else: + self.app.to_stop_config = True + + if self.app.to_stop_config: + self.stop_config() + return + + file_name = await self.save_config_to_file(file_name, config_map) + self.strategy_file_name = file_name + self.strategy_name = strategy + self.strategy_config_map = config_map + # Reload completer here otherwise the new file will not appear + self.app.input_field.completer = load_completer(self) + self.placeholder_mode = False + self.app.hide_input = False + + await self.verify_status() + + async def get_strategy_name( + self, # type: HummingbotApplication + ) -> Optional[str]: + strategy = None + strategy_config = BaseStrategyConfigMap.construct() + await self.prompt_for_model_config(strategy_config) + if self.app.to_stop_config: + self.stop_config() + else: + strategy = strategy_config.strategy + return strategy + + async def prompt_for_model_config( + self, # type: HummingbotApplication + config_map: BaseClientModel, + ): + for key, field in config_map.__fields__.items(): + client_data = config_map.get_client_data(key) + if ( + client_data is not None + and (client_data.prompt_on_new and field.required) + ): + await self.prompt_a_config(config_map, key) + if self.app.to_stop_config: + break + + async def prompt_for_configuration_legacy( + self, # type: HummingbotApplication + file_name, + strategy: str, + config_map: Dict, + ): + config_map_backup = copy.deepcopy(config_map) # assign default values and reset those not required for config in config_map.values(): if config.required: @@ -64,7 +124,7 @@ async def prompt_for_configuration(self, # type: HummingbotApplication for config in config_map.values(): if config.prompt_on_new and config.required: if not self.app.to_stop_config: - await self.prompt_a_config(config) + await self.prompt_a_config_legacy(config) else: break else: @@ -84,31 +144,57 @@ async def prompt_for_configuration(self, # type: HummingbotApplication strategy_path = os.path.join(CONF_FILE_PATH, file_name) template = get_strategy_template_path(strategy) shutil.copy(template, strategy_path) - save_to_yml(strategy_path, config_map) + save_to_yml_legacy(strategy_path, config_map) self.strategy_file_name = file_name self.strategy_name = strategy + self.strategy_config = None # Reload completer here otherwise the new file will not appear self.app.input_field.completer = load_completer(self) self._notify(f"A new config file {self.strategy_file_name} created.") self.placeholder_mode = False self.app.hide_input = False - try: - timeout = float(global_config_map["create_command_timeout"].value) - all_status_go = await asyncio.wait_for(self.status_check_all(), timeout) - except asyncio.TimeoutError: - self._notify("\nA network error prevented the connection check to complete. See logs for more details.") - self.strategy_file_name = None - self.strategy_name = None - raise - if all_status_go: - self._notify("\nEnter \"start\" to start market making.") - async def prompt_a_config(self, # type: HummingbotApplication - config: ConfigVar, - input_value=None, - assign_default=True): + await self.verify_status() + + async def prompt_a_config( + self, # type: HummingbotApplication + model: BaseClientModel, + config: str, + input_value=None, + ): + config_path = config.split(".") + while len(config_path) != 1: + sub_model_attr = config_path.pop(0) + model = model.__getattribute__(sub_model_attr) + config = config_path[0] + if input_value is None: + prompt = await model.get_client_prompt(config) + if prompt is not None: + prompt = f"{prompt} >>> " + client_data = model.get_client_data(config) + input_value = await self.app.prompt(prompt=prompt, is_password=client_data.is_secure) + + new_config_value = None + if not self.app.to_stop_config and input_value is not None: + try: + model.__setattr__(config, input_value) + new_config_value = model.__getattribute__(config) + except ValidationError as e: + err_msg = retrieve_validation_error_msg(e) + self._notify(err_msg) + new_config_value = await self.prompt_a_config(model, config) + + if not self.app.to_stop_config and isinstance(new_config_value, BaseClientModel): + await self.prompt_for_model_config(new_config_value) + + async def prompt_a_config_legacy( + self, # type: HummingbotApplication + config: ConfigVar, + input_value=None, + assign_default=True, + ): if config.key == "inventory_price": - await self.inventory_price_prompt(self.strategy_config_map, input_value) + await self.inventory_price_prompt_legacy(self.strategy_config_map, input_value) return if input_value is None: if assign_default: @@ -122,10 +208,26 @@ async def prompt_a_config(self, # type: HummingbotApplication err_msg = await config.validate(input_value) if err_msg is not None: self._notify(err_msg) - await self.prompt_a_config(config) + await self.prompt_a_config_legacy(config) else: config.value = value + async def save_config_to_file( + self, # type: HummingbotApplication + file_name: Optional[str], + config_map: BaseStrategyConfigMap, + ) -> str: + if file_name is None: + file_name = await self.prompt_new_file_name(config_map.strategy) + if self.app.to_stop_config: + self.stop_config() + self.app.set_text("") + return + self.app.change_prompt(prompt=">>> ") + strategy_path = os.path.join(CONF_FILE_PATH, file_name) + save_to_yml(strategy_path, config_map) + return file_name + async def prompt_new_file_name(self, # type: HummingbotApplication strategy): file_name = default_strategy_file_path(strategy) @@ -142,23 +244,39 @@ async def prompt_new_file_name(self, # type: HummingbotApplication else: return input - async def update_all_secure_configs(self # type: HummingbotApplication - ): + async def update_all_secure_configs_legacy( + self # type: HummingbotApplication + ): await Security.wait_til_decryption_done() Security.update_config_map(global_config_map) - if self.strategy_config_map is not None: + if self.strategy_config_map is not None and not isinstance(self.strategy_config_map, BaseStrategyConfigMap): Security.update_config_map(self.strategy_config_map) + async def verify_status( + self # type: HummingbotApplication + ): + try: + timeout = float(global_config_map["create_command_timeout"].value) + all_status_go = await asyncio.wait_for(self.status_check_all(), timeout) + except asyncio.TimeoutError: + self._notify("\nA network error prevented the connection check to complete. See logs for more details.") + self.strategy_file_name = None + self.strategy_name = None + self.strategy_config = None + raise + if all_status_go: + self._notify("\nEnter \"start\" to start market making.") + def stop_config( self, config_map: Optional[Dict[str, ConfigVar]] = None, config_map_backup: Optional[Dict[str, ConfigVar]] = None, ): if config_map is not None and config_map_backup is not None: - self.restore_config(config_map, config_map_backup) + self.restore_config_legacy(config_map, config_map_backup) self.app.to_stop_config = False @staticmethod - def restore_config(config_map: Dict[str, ConfigVar], config_map_backup: Dict[str, ConfigVar]): + def restore_config_legacy(config_map: Dict[str, ConfigVar], config_map_backup: Dict[str, ConfigVar]): for key in config_map: config_map[key] = config_map_backup[key] diff --git a/hummingbot/client/command/start_command.py b/hummingbot/client/command/start_command.py index 39f89854b7..ac806c4375 100644 --- a/hummingbot/client/command/start_command.py +++ b/hummingbot/client/command/start_command.py @@ -101,7 +101,7 @@ async def start_check(self, # type: HummingbotApplication f"{warning_msg}") # Display warning message if the exchange connector has outstanding issues or not working - elif status != "GREEN": + elif "GREEN" not in status: self._notify(f"\nConnector status: {status}. This connector has one or more issues.\n" "Refer to our Github page for more info: https://github.com/coinalpha/hummingbot") @@ -175,7 +175,7 @@ async def confirm_oracle_conversion_rate(self, # type: HummingbotApplication "this strategy (Yes/No) >>> ", required_if=lambda: True, validator=lambda v: validate_bool(v)) - await self.prompt_a_config(config) + await self.prompt_a_config_legacy(config) if config.value: result = True except OracleRateUnavailable: diff --git a/hummingbot/client/command/status_command.py b/hummingbot/client/command/status_command.py index 49011a1dad..1d03628f41 100644 --- a/hummingbot/client/command/status_command.py +++ b/hummingbot/client/command/status_command.py @@ -7,15 +7,19 @@ OrderedDict ) import inspect -from typing import List, Dict +from typing import Dict, List + +from pydantic import ValidationError, validate_model + from hummingbot import check_dev_mode +from hummingbot.client.config.config_data_types import BaseStrategyConfigMap from hummingbot.logger.application_warning import ApplicationWarning from hummingbot.connector.connector_base import ConnectorBase from hummingbot.core.network_iterator import NetworkStatus from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.config_helpers import ( - missing_required_configs, - get_strategy_config_map + get_strategy_config_map, + missing_required_configs_legacy, ) from hummingbot.client.config.security import Security from hummingbot.user.user_balances import UserBalances @@ -100,7 +104,7 @@ async def validate_required_connections(self) -> Dict[str, str]: if err_msg is not None: invalid_conns["celo"] = err_msg if not any([str(exchange).endswith("paper_trade") for exchange in required_exchanges]): - await self.update_all_secure_configs() + await self.update_all_secure_configs_legacy() connections = await UserBalances.instance().update_exchanges(exchanges=required_exchanges) invalid_conns.update({ex: err_msg for ex, err_msg in connections.items() if ex in required_exchanges and err_msg is not None}) @@ -110,11 +114,27 @@ async def validate_required_connections(self) -> Dict[str, str]: invalid_conns["ethereum"] = err_msg return invalid_conns - def missing_configurations(self) -> List[str]: - missing_globals = missing_required_configs(global_config_map) - missing_configs = missing_required_configs(get_strategy_config_map(self.strategy_name)) + def missing_configurations_legacy(self) -> List[str]: + missing_globals = missing_required_configs_legacy(global_config_map) + config_map = self.strategy_config_map + missing_configs = [] + if not isinstance(config_map, BaseStrategyConfigMap): + missing_configs = missing_required_configs_legacy(get_strategy_config_map(self.strategy_name)) return missing_globals + missing_configs + def validate_configs(self) -> List[ValidationError]: + config_map = self.strategy_config_map + validation_errors = [] + if isinstance(config_map, BaseStrategyConfigMap): + validation_results = validate_model(type(config_map), config_map.dict()) + if len(validation_results) == 3 and validation_results[2] is not None: + validation_errors = validation_results[2].errors() + validation_errors = [ + f"{'.'.join(e['loc'])} - {e['msg']}" + for e in validation_errors + ] + return validation_errors + def status(self, # type: HummingbotApplication live: bool = False): safe_ensure_future(self.status_check_all(live=live), loop=self.ev_loop) @@ -162,14 +182,19 @@ async def status_check_all(self, # type: HummingbotApplication elif notify_success: self._notify(' - Exchange check: All connections confirmed.') - missing_configs = self.missing_configurations() + missing_configs = self.missing_configurations_legacy() if missing_configs: self._notify(" - Strategy check: Incomplete strategy configuration. The following values are missing.") for config in missing_configs: self._notify(f" {config.key}") elif notify_success: self._notify(' - Strategy check: All required parameters confirmed.') - if invalid_conns or missing_configs: + validation_errors = self.validate_configs() + if len(validation_errors) != 0: + self._notify(" - Strategy check: Validation of the config maps failed. The following errors were flagged.") + for error in validation_errors: + self._notify(f" {error}") + if invalid_conns or missing_configs or len(validation_errors) != 0: return False loading_markets: List[ConnectorBase] = [] diff --git a/hummingbot/client/config/config_data_types.py b/hummingbot/client/config/config_data_types.py index c12d484696..cc351d5371 100644 --- a/hummingbot/client/config/config_data_types.py +++ b/hummingbot/client/config/config_data_types.py @@ -1,11 +1,23 @@ +import inspect from dataclasses import dataclass +from datetime import date, datetime, time +from decimal import Decimal from enum import Enum -from typing import Any, Callable, Optional +from typing import Any, Callable, Dict, Generator, List, Optional -from pydantic import BaseModel +import yaml +from pydantic import BaseModel, Field, validator +from pydantic.fields import FieldInfo from pydantic.schema import default_ref_template +from yaml import SafeDumper from hummingbot.client.config.config_helpers import strategy_config_schema_encoder +from hummingbot.client.config.config_validators import ( + validate_exchange, + validate_market_trading_pair, + validate_strategy, +) +from hummingbot.client.settings import AllConnectorSettings class ClientConfigEnum(Enum): @@ -13,13 +25,66 @@ def __str__(self): return self.value +def decimal_representer(dumper: SafeDumper, data: Decimal): + return dumper.represent_float(float(data)) + + +def enum_representer(dumper: SafeDumper, data: ClientConfigEnum): + return dumper.represent_str(str(data)) + + +def date_representer(dumper: SafeDumper, data: date): + return dumper.represent_date(data) + + +def time_representer(dumper: SafeDumper, data: time): + return dumper.represent_str(data.strftime("%H:%M:%S")) + + +def datetime_representer(dumper: SafeDumper, data: datetime): + return dumper.represent_datetime(data) + + +yaml.add_representer( + data_type=Decimal, representer=decimal_representer, Dumper=SafeDumper +) +yaml.add_multi_representer( + data_type=ClientConfigEnum, multi_representer=enum_representer, Dumper=SafeDumper +) +yaml.add_representer( + data_type=date, representer=date_representer, Dumper=SafeDumper +) +yaml.add_representer( + data_type=time, representer=time_representer, Dumper=SafeDumper +) +yaml.add_representer( + data_type=datetime, representer=datetime_representer, Dumper=SafeDumper +) + + @dataclass() class ClientFieldData: prompt: Optional[Callable[['BaseClientModel'], str]] = None prompt_on_new: bool = False + is_secure: bool = False + + +@dataclass() +class TraversalItem: + depth: int + config_path: str + attr: str + value: Any + printable_value: str + client_field_data: Optional[ClientFieldData] + field_info: FieldInfo class BaseClientModel(BaseModel): + class Config: + validate_assignment = True + title = None + """ Notes on configs: - In nested models, be weary that pydantic will take the first model that fits @@ -36,12 +101,167 @@ def schema_json( **dumps_kwargs ) - def get_client_prompt(self, attr_name: str) -> Optional[str]: + def traverse(self) -> Generator[TraversalItem, None, None]: + """The intended use for this function is to simplify (validated) config map traversals in the client code.""" + depth = 0 + for attr, field in self.__fields__.items(): + value = self.__getattribute__(attr) + printable_value = str(value) if not isinstance(value, BaseClientModel) else value.Config.title + field_info = field.field_info + client_field_data = field_info.extra.get("client_data") + yield TraversalItem( + depth, attr, attr, value, printable_value, client_field_data, field_info + ) + if isinstance(value, BaseClientModel): + for traversal_item in value.traverse(): + traversal_item.depth += 1 + config_path = f"{attr}.{traversal_item.config_path}" + traversal_item.config_path = config_path + yield traversal_item + + def dict_in_conf_order(self) -> Dict[str, Any]: + d = {} + for attr in self.__fields__.keys(): + value = self.__getattribute__(attr) + if isinstance(value, BaseClientModel): + value = value.dict_in_conf_order() + d[attr] = value + return d + + async def get_client_prompt(self, attr_name: str) -> Optional[str]: prompt = None client_data = self.get_client_data(attr_name) if client_data is not None: - prompt = client_data.prompt(self) + prompt = client_data.prompt + if inspect.iscoroutinefunction(prompt): + prompt = await prompt(self) + else: + prompt = prompt(self) return prompt + def is_secure(self, attr_name: str) -> bool: + client_data = self.get_client_data(attr_name) + secure = client_data is not None and client_data.is_secure + return secure + def get_client_data(self, attr_name: str) -> Optional[ClientFieldData]: - return self.__fields__[attr_name].field_info.extra["client_data"] + return self.__fields__[attr_name].field_info.extra.get("client_data") + + def get_description(self, attr_name: str) -> str: + return self.__fields__[attr_name].field_info.description + + def generate_yml_output_str_with_comments(self) -> str: + original_fragments = yaml.safe_dump(self.dict_in_conf_order(), sort_keys=False).split("\n") + fragments_with_comments = [self._generate_title()] + self._add_model_fragments(self, fragments_with_comments, original_fragments) + fragments_with_comments.append("\n") # EOF empty line + yml_str = "".join(fragments_with_comments) + return yml_str + + def _generate_title(self) -> str: + title = f"{self.Config.title}" + title = self._adorn_title(title) + return title + + @staticmethod + def _adorn_title(title: str) -> str: + if title: + title = f"### {title} config ###" + title_len = len(title) + title = f"{'#' * title_len}\n{title}\n{'#' * title_len}" + return title + + @staticmethod + def _add_model_fragments( + model: 'BaseClientModel', + fragments_with_comments: List[str], + original_fragments: List[str], + ): + for i, traversal_item in enumerate(model.traverse()): + attr_comment = traversal_item.field_info.description + if attr_comment is not None: + comment_prefix = f"\n{' ' * 2 * traversal_item.depth}# " + attr_comment = "".join(f"{comment_prefix}{c}" for c in attr_comment.split("\n")) + if traversal_item.depth == 0: + attr_comment = f"\n{attr_comment}" + fragments_with_comments.extend([attr_comment, f"\n{original_fragments[i]}"]) + elif traversal_item.depth == 0: + fragments_with_comments.append(f"\n\n{original_fragments[i]}") + else: + fragments_with_comments.append(f"\n{original_fragments[i]}") + + +class BaseStrategyConfigMap(BaseClientModel): + strategy: str = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda mi: "What is your market making strategy?", + prompt_on_new=True, + ), + ) + + @validator("strategy", pre=True) + def validate_strategy(cls, v: str): + ret = validate_strategy(v) + if ret is not None: + raise ValueError(ret) + return v + + def _generate_title(self) -> str: + title = " ".join([w.capitalize() for w in f"{self.strategy}".split("_")]) + title = f"{title} Strategy" + title = self._adorn_title(title) + return title + + +class BaseTradingStrategyConfigMap(BaseStrategyConfigMap): + exchange: ClientConfigEnum( + value="Exchanges", # noqa: F821 + names={e: e for e in AllConnectorSettings.get_connector_settings().keys()}, + type=str, + ) = Field( + default=..., + description="The name of the exchange connector.", + client_data=ClientFieldData( + prompt=lambda mi: "Input your maker spot connector", + prompt_on_new=True, + ), + ) + market: str = Field( + default=..., + description="The trading pair.", + client_data=ClientFieldData( + prompt=lambda mi: BaseTradingStrategyConfigMap.maker_trading_pair_prompt(mi), + prompt_on_new=True, + ), + ) + + @classmethod + def maker_trading_pair_prompt(cls, model_instance: 'BaseTradingStrategyConfigMap') -> str: + exchange = model_instance.exchange + example = AllConnectorSettings.get_example_pairs().get(exchange) + return ( + f"Enter the token trading pair you would like to trade on" + f" {exchange}{f' (e.g. {example})' if example else ''}" + ) + + @validator("exchange", pre=True) + def validate_exchange(cls, v: str): + """Used for client-friendly error output.""" + ret = validate_exchange(v) + if ret is not None: + raise ValueError(ret) + cls.__fields__["exchange"].type_ = ClientConfigEnum( # rebuild the exchanges enum + value="Exchanges", # noqa: F821 + names={e: e for e in AllConnectorSettings.get_connector_settings().keys()}, + type=str, + ) + return v + + @validator("market", pre=True) + def validate_exchange_trading_pair(cls, v: str, values: Dict): + exchange = values.get("exchange") + ret = validate_market_trading_pair(exchange, v) + if ret is not None: + raise ValueError(ret) + return v diff --git a/hummingbot/client/hummingbot_application.py b/hummingbot/client/hummingbot_application.py index ceb190a3ed..fd76f56656 100644 --- a/hummingbot/client/hummingbot_application.py +++ b/hummingbot/client/hummingbot_application.py @@ -7,6 +7,7 @@ from typing import List, Dict, Optional, Tuple, Deque from hummingbot.client.command import __all__ as commands +from hummingbot.client.config.config_data_types import BaseStrategyConfigMap from hummingbot.client.tab import __all__ as tab_classes from hummingbot.core.clock import Clock from hummingbot.exceptions import ArgumentParserError @@ -77,8 +78,9 @@ def __init__(self): self.markets: Dict[str, ExchangeBase] = {} # strategy file name and name get assigned value after import or create command - self._strategy_file_name: str = None - self.strategy_name: str = None + self._strategy_file_name: Optional[str] = None + self.strategy_name: Optional[str] = None + self._strategy_config_map: Optional[BaseStrategyConfigMap] = None self.strategy_task: Optional[asyncio.Task] = None self.strategy: Optional[StrategyBase] = None self.market_pair: Optional[CrossExchangeMarketPair] = None @@ -121,10 +123,16 @@ def strategy_file_name(self, value: Optional[str]): @property def strategy_config_map(self): + if self._strategy_config_map is not None: + return self._strategy_config_map if self.strategy_name is not None: return get_strategy_config_map(self.strategy_name) return None + @strategy_config_map.setter + def strategy_config_map(self, config_map: BaseStrategyConfigMap): + self._strategy_config_map = config_map + def _notify(self, msg: str): self.app.log(msg) for notifier in self.notifiers: diff --git a/hummingbot/client/settings.py b/hummingbot/client/settings.py index cff7a8873d..53fcdc5fc6 100644 --- a/hummingbot/client/settings.py +++ b/hummingbot/client/settings.py @@ -16,7 +16,7 @@ from hummingbot.core.data_type.trade_fee import TradeFeeSchema # Global variables -required_exchanges: List[str] = [] +required_exchanges: Set[str] = set() requried_connector_trading_pairs: Dict[str, List[str]] = {} # Set these two variables if a strategy uses oracle for rate conversion required_rate_oracle: bool = False diff --git a/hummingbot/client/ui/completer.py b/hummingbot/client/ui/completer.py index 2b68eda4ea..9a099cc187 100644 --- a/hummingbot/client/ui/completer.py +++ b/hummingbot/client/ui/completer.py @@ -89,7 +89,7 @@ def _option_completer(self): @property def _config_completer(self): - config_keys = self.hummingbot_application.config_able_keys() + config_keys = self.hummingbot_application.configurable_keys() return WordCompleter(config_keys, ignore_case=True) def _complete_strategies(self, document: Document) -> bool: diff --git a/hummingbot/client/ui/interface_utils.py b/hummingbot/client/ui/interface_utils.py index 6cbb954708..1d2da690a9 100644 --- a/hummingbot/client/ui/interface_utils.py +++ b/hummingbot/client/ui/interface_utils.py @@ -1,17 +1,17 @@ import asyncio import datetime - -import pandas as pd -import psutil from decimal import Decimal from typing import ( List, + Optional, Set, Tuple, - Optional, ) -from tabulate import tabulate +import pandas as pd +import psutil +import tabulate + from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.performance import PerformanceMetrics @@ -111,5 +111,11 @@ def format_df_for_printout( ) df.columns = [c if len(c) < max_col_width else f"{c[:max_col_width - 3]}..." for c in df.columns] table_format = table_format or global_config_map.get("tables_format").value - formatted_df = tabulate(df, tablefmt=table_format, showindex=index, headers="keys") + + original_preserve_whitespace = tabulate.PRESERVE_WHITESPACE + tabulate.PRESERVE_WHITESPACE = True + try: + formatted_df = tabulate.tabulate(df, tablefmt=table_format, showindex=index, headers="keys") + finally: + tabulate.PRESERVE_WHITESPACE = original_preserve_whitespace return formatted_df diff --git a/hummingbot/client/ui/style.py b/hummingbot/client/ui/style.py index c83e7b91fc..08814fa2f9 100644 --- a/hummingbot/client/ui/style.py +++ b/hummingbot/client/ui/style.py @@ -1,7 +1,7 @@ from prompt_toolkit.styles import Style from prompt_toolkit.utils import is_windows from hummingbot.client.config.global_config_map import global_config_map -from hummingbot.client.config.config_helpers import save_to_yml +from hummingbot.client.config.config_helpers import save_to_yml_legacy from hummingbot.client.settings import GLOBAL_CONFIG_PATH @@ -111,7 +111,7 @@ def reset_style(config_map=global_config_map, save=True): # Save configuration if save: file_path = GLOBAL_CONFIG_PATH - save_to_yml(file_path, config_map) + save_to_yml_legacy(file_path, config_map) # Apply & return style return load_style(config_map) diff --git a/hummingbot/strategy/amm_arb/amm_arb_config_map.py b/hummingbot/strategy/amm_arb/amm_arb_config_map.py index 807991ae8c..0750a07601 100644 --- a/hummingbot/strategy/amm_arb/amm_arb_config_map.py +++ b/hummingbot/strategy/amm_arb/amm_arb_config_map.py @@ -16,7 +16,7 @@ def exchange_on_validated(value: str) -> None: - required_exchanges.append(value) + required_exchanges.add(value) def market_1_validator(value: str) -> None: diff --git a/hummingbot/strategy/arbitrage/arbitrage_config_map.py b/hummingbot/strategy/arbitrage/arbitrage_config_map.py index 392b1de242..543f0ab900 100644 --- a/hummingbot/strategy/arbitrage/arbitrage_config_map.py +++ b/hummingbot/strategy/arbitrage/arbitrage_config_map.py @@ -38,7 +38,7 @@ def secondary_trading_pair_prompt(): def secondary_market_on_validated(value: str): - required_exchanges.append(value) + required_exchanges.add(value) def update_oracle_settings(value: str): @@ -73,7 +73,7 @@ def update_oracle_settings(value: str): prompt="Enter your primary spot connector >>> ", prompt_on_new=True, validator=validate_exchange, - on_validated=lambda value: required_exchanges.append(value), + on_validated=lambda value: required_exchanges.add(value), ), "secondary_market": ConfigVar( key="secondary_market", diff --git a/hummingbot/strategy/aroon_oscillator/aroon_oscillator_config_map.py b/hummingbot/strategy/aroon_oscillator/aroon_oscillator_config_map.py index 1ebd87aeb6..68fdb2cf3f 100644 --- a/hummingbot/strategy/aroon_oscillator/aroon_oscillator_config_map.py +++ b/hummingbot/strategy/aroon_oscillator/aroon_oscillator_config_map.py @@ -64,7 +64,7 @@ def on_validated_price_type(value: str): def exchange_on_validated(value: str): - required_exchanges.append(value) + required_exchanges.add(value) aroon_oscillator_config_map = { diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map.py deleted file mode 100644 index 2df007770f..0000000000 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map.py +++ /dev/null @@ -1,245 +0,0 @@ -from decimal import Decimal - -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_validators import ( - validate_exchange, - validate_market_trading_pair, - validate_int, - validate_bool, - validate_decimal, - validate_datetime_iso_string, - validate_time_iso_string, -) -from hummingbot.client.settings import ( - required_exchanges, - AllConnectorSettings, -) -from typing import Optional - - -def maker_trading_pair_prompt(): - exchange = avellaneda_market_making_config_map.get("exchange").value - example = AllConnectorSettings.get_example_pairs().get(exchange) - return "Enter the token trading pair you would like to trade on %s%s >>> " \ - % (exchange, f" (e.g. {example})" if example else "") - - -# strategy specific validators -def validate_exchange_trading_pair(value: str) -> Optional[str]: - exchange = avellaneda_market_making_config_map.get("exchange").value - return validate_market_trading_pair(exchange, value) - - -def validate_execution_timeframe(value: str) -> Optional[str]: - timeframes = ["infinite", "from_date_to_date", "daily_between_times"] - if value not in timeframes: - return f"Invalid timeframe, please choose value from {timeframes}" - - -def validate_execution_time(value: str) -> Optional[str]: - ret = None - if avellaneda_market_making_config_map.get("execution_timeframe").value == "from_date_to_date": - ret = validate_datetime_iso_string(value) - if avellaneda_market_making_config_map.get("execution_timeframe").value == "daily_between_times": - ret = validate_time_iso_string(value) - if ret is not None: - return ret - - -def execution_time_start_prompt() -> str: - if avellaneda_market_making_config_map.get("execution_timeframe").value == "from_date_to_date": - return "Please enter the start date and time (YYYY-MM-DD HH:MM:SS) >>> " - if avellaneda_market_making_config_map.get("execution_timeframe").value == "daily_between_times": - return "Please enter the start time (HH:MM:SS) >>> " - - -def execution_time_end_prompt() -> str: - if avellaneda_market_making_config_map.get("execution_timeframe").value == "from_date_to_date": - return "Please enter the end date and time (YYYY-MM-DD HH:MM:SS) >>> " - if avellaneda_market_making_config_map.get("execution_timeframe").value == "daily_between_times": - return "Please enter the end time (HH:MM:SS) >>> " - - -def on_validated_execution_timeframe(value: str): - avellaneda_market_making_config_map["start_time"].value = None - avellaneda_market_making_config_map["end_time"].value = None - - -def order_amount_prompt() -> str: - trading_pair = avellaneda_market_making_config_map["market"].value - base_asset, quote_asset = trading_pair.split("-") - return f"What is the amount of {base_asset} per order? >>> " - - -def on_validated_price_source_exchange(value: str): - if value is None: - avellaneda_market_making_config_map["price_source_market"].value = None - - -def exchange_on_validated(value: str): - required_exchanges.append(value) - - -avellaneda_market_making_config_map = { - "strategy": - ConfigVar(key="strategy", - prompt=None, - default="avellaneda_market_making"), - "exchange": - ConfigVar(key="exchange", - prompt="Enter your maker spot connector >>> ", - validator=validate_exchange, - on_validated=exchange_on_validated, - prompt_on_new=True), - "market": - ConfigVar(key="market", - prompt=maker_trading_pair_prompt, - validator=validate_exchange_trading_pair, - prompt_on_new=True), - "execution_timeframe": - ConfigVar(key="execution_timeframe", - prompt="Choose execution timeframe ( infinite / from_date_to_date / daily_between_times ) >>> ", - validator=validate_execution_timeframe, - on_validated=on_validated_execution_timeframe, - prompt_on_new=True), - "start_time": - ConfigVar(key="start_time", - prompt=execution_time_start_prompt, - type_str="str", - validator=validate_execution_time, - required_if=lambda: avellaneda_market_making_config_map.get("execution_timeframe").value != "infinite", - prompt_on_new=True), - "end_time": - ConfigVar(key="end_time", - prompt=execution_time_end_prompt, - type_str="str", - validator=validate_execution_time, - required_if=lambda: avellaneda_market_making_config_map.get("execution_timeframe").value != "infinite", - prompt_on_new=True), - "order_amount": - ConfigVar(key="order_amount", - prompt=order_amount_prompt, - type_str="decimal", - validator=lambda v: validate_decimal(v, min_value=Decimal("0"), inclusive=False), - prompt_on_new=True), - "order_optimization_enabled": - ConfigVar(key="order_optimization_enabled", - prompt="Do you want to enable best bid ask jumping? (Yes/No) >>> ", - type_str="bool", - default=True, - validator=validate_bool), - "risk_factor": - ConfigVar(key="risk_factor", - printable_key="risk_factor(\u03B3)", - prompt="Enter risk factor (\u03B3) >>> ", - type_str="decimal", - default=Decimal("1"), - validator=lambda v: validate_decimal(v, 0, inclusive=False), - prompt_on_new=True), - "order_amount_shape_factor": - ConfigVar(key="order_amount_shape_factor", - printable_key="order_amount_shape_factor(\u03B7)", - prompt="Enter order amount shape factor (\u03B7) >>> ", - type_str="decimal", - default=Decimal("0"), - validator=lambda v: validate_decimal(v, 0, 1, inclusive=True)), - "min_spread": - ConfigVar(key="min_spread", - prompt="Enter minimum spread limit (as % of mid price) >>> ", - type_str="decimal", - validator=lambda v: validate_decimal(v, 0, inclusive=True), - default=Decimal("0")), - "order_refresh_time": - ConfigVar(key="order_refresh_time", - prompt="How often do you want to cancel and replace bids and asks " - "(in seconds)? >>> ", - type_str="float", - validator=lambda v: validate_decimal(v, 0, inclusive=False), - prompt_on_new=True), - "max_order_age": - ConfigVar(key="max_order_age", - prompt="How long do you want to cancel and replace bids and asks " - "with the same price (in seconds)? >>> ", - type_str="float", - default=1800, - validator=lambda v: validate_decimal(v, 0, inclusive=False)), - "order_refresh_tolerance_pct": - ConfigVar(key="order_refresh_tolerance_pct", - prompt="Enter the percent change in price needed to refresh orders at each cycle " - "(Enter 1 to indicate 1%) >>> ", - type_str="decimal", - default=Decimal("0"), - validator=lambda v: validate_decimal(v, -10, 10, inclusive=True)), - "filled_order_delay": - ConfigVar(key="filled_order_delay", - prompt="How long do you want to wait before placing the next order " - "if your order gets filled (in seconds)? >>> ", - type_str="float", - validator=lambda v: validate_decimal(v, min_value=0, inclusive=False), - default=60), - "inventory_target_base_pct": - ConfigVar(key="inventory_target_base_pct", - prompt="What is the inventory target for the base asset? Enter 50 for 50% >>> ", - type_str="decimal", - validator=lambda v: validate_decimal(v, 0, 100), - prompt_on_new=True, - default=Decimal("50")), - "add_transaction_costs": - ConfigVar(key="add_transaction_costs", - prompt="Do you want to add transaction costs automatically to order prices? (Yes/No) >>> ", - type_str="bool", - default=False, - validator=validate_bool), - "volatility_buffer_size": - ConfigVar(key="volatility_buffer_size", - prompt="Enter amount of ticks that will be stored to calculate volatility >>> ", - type_str="int", - validator=lambda v: validate_decimal(v, 1, 10000), - default=200), - "trading_intensity_buffer_size": - ConfigVar(key="trading_intensity_buffer_size", - prompt="Enter amount of ticks that will be stored to estimate order book liquidity >>> ", - type_str="int", - validator=lambda v: validate_int(v, 1, 10000), - default=200), - "order_levels": - ConfigVar(key="order_levels", - prompt="How many orders do you want to place on both sides? >>> ", - type_str="int", - validator=lambda v: validate_int(v, min_value=-1, inclusive=False), - default=1), - "level_distances": - ConfigVar(key="level_distances", - prompt="How far apart in % of optimal spread should orders on one side be? >>> ", - type_str="decimal", - validator=lambda v: validate_decimal(v, min_value=0, inclusive=True), - default=0), - "order_override": - ConfigVar(key="order_override", - prompt=None, - required_if=lambda: False, - default=None, - type_str="json"), - "hanging_orders_enabled": - ConfigVar(key="hanging_orders_enabled", - prompt="Do you want to enable hanging orders? (Yes/No) >>> ", - type_str="bool", - default=False, - validator=validate_bool), - "hanging_orders_cancel_pct": - ConfigVar(key="hanging_orders_cancel_pct", - prompt="At what spread percentage (from mid price) will hanging orders be canceled? " - "(Enter 1 to indicate 1%) >>> ", - required_if=lambda: avellaneda_market_making_config_map.get("hanging_orders_enabled").value, - type_str="decimal", - default=Decimal("10"), - validator=lambda v: validate_decimal(v, 0, 100, inclusive=False)), - "should_wait_order_cancel_confirmation": - ConfigVar(key="should_wait_order_cancel_confirmation", - prompt="Should the strategy wait to receive a confirmation for orders cancellation " - "before creating a new set of orders? " - "(Not waiting requires enough available balance) (Yes/No) >>> ", - type_str="bool", - default=True, - validator=validate_bool), -} diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py index ef04712821..03cd05c80f 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -4,17 +4,19 @@ from pydantic import Field, validator, root_validator -from hummingbot.client.config.config_data_types import BaseClientModel, ClientConfigEnum, ClientFieldData +from hummingbot.client.config.config_data_types import ( + BaseClientModel, + BaseTradingStrategyConfigMap, + ClientFieldData, +) from hummingbot.client.config.config_validators import ( validate_bool, validate_datetime_iso_string, validate_decimal, - validate_exchange, validate_int, - validate_market_trading_pair, validate_time_iso_string, ) -from hummingbot.client.settings import AllConnectorSettings, required_exchanges +from hummingbot.client.settings import required_exchanges from hummingbot.connector.utils import split_hb_trading_pair @@ -178,29 +180,9 @@ class Config: } -class AvellanedaMarketMakingConfigMap(BaseClientModel): +class AvellanedaMarketMakingConfigMap(BaseTradingStrategyConfigMap): strategy: str = Field(default="avellaneda_market_making", client_data=None) - exchange: ClientConfigEnum( - value="Exchanges", # noqa: F821 - names={e: e for e in AllConnectorSettings.get_connector_settings().keys()}, - type=str, - ) = Field( - default=..., - description="The name of the exchange connector.", - client_data=ClientFieldData( - prompt=lambda mi: "Input your maker spot connector", - prompt_on_new=True, - ), - ) - market: str = Field( - default=..., - description="The trading pair.", - client_data=ClientFieldData( - prompt=lambda mi: AvellanedaMarketMakingConfigMap.maker_trading_pair_prompt(mi), - prompt_on_new=True, - ), - ) - execution_timeframe_model: Union[FromDateToDateModel, DailyBetweenTimesModel, InfiniteModel] = Field( + execution_timeframe_mode: Union[FromDateToDateModel, DailyBetweenTimesModel, InfiniteModel] = Field( default=..., description="The execution timeframe.", client_data=ClientFieldData( @@ -212,7 +194,10 @@ class AvellanedaMarketMakingConfigMap(BaseClientModel): default=..., description="The strategy order amount.", gt=0, - client_data=ClientFieldData(prompt=lambda mi: AvellanedaMarketMakingConfigMap.order_amount_prompt(mi)) + client_data=ClientFieldData( + prompt=lambda mi: AvellanedaMarketMakingConfigMap.order_amount_prompt(mi), + prompt_on_new=True, + ) ) order_optimization_enabled: bool = Field( default=True, @@ -367,16 +352,9 @@ class AvellanedaMarketMakingConfigMap(BaseClientModel): ) class Config: - validate_assignment = True + title = "avellaneda_market_making" - @classmethod - def maker_trading_pair_prompt(cls, model_instance: 'AvellanedaMarketMakingConfigMap') -> str: - exchange = model_instance.exchange - example = AllConnectorSettings.get_example_pairs().get(exchange) - return ( - f"Enter the token trading pair you would like to trade on" - f" {exchange}{f' (e.g. {example})' if example else ''}" - ) + # === prompts === @classmethod def order_amount_prompt(cls, model_instance: 'AvellanedaMarketMakingConfigMap') -> str: @@ -384,23 +362,9 @@ def order_amount_prompt(cls, model_instance: 'AvellanedaMarketMakingConfigMap') base_asset, quote_asset = split_hb_trading_pair(trading_pair) return f"What is the amount of {base_asset} per order?" - @validator("exchange", pre=True) - def validate_exchange(cls, v: str): - """Used for client-friendly error output.""" - ret = validate_exchange(v) - if ret is not None: - raise ValueError(ret) - return v - - @validator("market", pre=True) - def validate_exchange_trading_pair(cls, v: str, values: Dict): - exchange = values.get("exchange") - ret = validate_market_trading_pair(exchange, v) - if ret is not None: - raise ValueError(ret) - return v + # === specific validations === - @validator("execution_timeframe_model", pre=True) + @validator("execution_timeframe_mode", pre=True) def validate_execution_timeframe( cls, v: Union[str, InfiniteModel, FromDateToDateModel, DailyBetweenTimesModel] ): @@ -529,4 +493,4 @@ def execution_timeframe_post_validation(cls, values: Dict): @classmethod def exchange_post_validation(cls, values: Dict): - required_exchanges.append(values["exchange"]) + required_exchanges.add(values["exchange"]) diff --git a/hummingbot/strategy/avellaneda_market_making/start.py b/hummingbot/strategy/avellaneda_market_making/start.py index ad34bb79d8..6d3b1ec98a 100644 --- a/hummingbot/strategy/avellaneda_market_making/start.py +++ b/hummingbot/strategy/avellaneda_market_making/start.py @@ -1,4 +1,3 @@ -import datetime import pandas as pd from decimal import Decimal from typing import ( @@ -9,6 +8,13 @@ from hummingbot import data_path import os.path from hummingbot.client.hummingbot_application import HummingbotApplication +from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( + AvellanedaMarketMakingConfigMap, + DailyBetweenTimesModel, + FromDateToDateModel, + MultiOrderLevelModel, + TrackHangingOrdersModel, +) from hummingbot.strategy.conditional_execution_state import ( RunAlwaysExecutionState, RunInTimeConditionalExecutionState @@ -17,28 +23,35 @@ from hummingbot.strategy.avellaneda_market_making import ( AvellanedaMarketMakingStrategy, ) -from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map import avellaneda_market_making_config_map as c_map def start(self): try: - order_amount = c_map.get("order_amount").value - order_optimization_enabled = c_map.get("order_optimization_enabled").value - order_refresh_time = c_map.get("order_refresh_time").value - exchange = c_map.get("exchange").value.lower() - raw_trading_pair = c_map.get("market").value - max_order_age = c_map.get("max_order_age").value - inventory_target_base_pct = 0 if c_map.get("inventory_target_base_pct").value is None else \ - c_map.get("inventory_target_base_pct").value / Decimal('100') - filled_order_delay = c_map.get("filled_order_delay").value - order_refresh_tolerance_pct = c_map.get("order_refresh_tolerance_pct").value / Decimal('100') - order_levels = c_map.get("order_levels").value - level_distances = c_map.get("level_distances").value - order_override = c_map.get("order_override").value - hanging_orders_enabled = c_map.get("hanging_orders_enabled").value - - hanging_orders_cancel_pct = c_map.get("hanging_orders_cancel_pct").value / Decimal('100') - add_transaction_costs_to_orders = c_map.get("add_transaction_costs").value + c_map: AvellanedaMarketMakingConfigMap = self.strategy_config_map + order_amount = c_map.order_amount + order_optimization_enabled = c_map.order_optimization_enabled + order_refresh_time = c_map.order_refresh_time + exchange = c_map.exchange + raw_trading_pair = c_map.market + max_order_age = c_map.max_order_age + inventory_target_base_pct = 0 if c_map.inventory_target_base_pct is None else \ + c_map.inventory_target_base_pct / Decimal('100') + filled_order_delay = c_map.filled_order_delay + order_refresh_tolerance_pct = c_map.order_refresh_tolerance_pct / Decimal('100') + if isinstance(c_map.order_levels_mode, MultiOrderLevelModel): + order_levels = c_map.order_levels_mode.order_levels + level_distances = c_map.order_levels_mode.level_distances + else: + order_levels = 1 + level_distances = 0 + order_override = c_map.order_override + if isinstance(c_map.hanging_orders_mode, TrackHangingOrdersModel): + hanging_orders_enabled = True + hanging_orders_cancel_pct = c_map.hanging_orders_mode.hanging_orders_cancel_pct / Decimal('100') + else: + hanging_orders_enabled = False + hanging_orders_cancel_pct = Decimal("0") + add_transaction_costs_to_orders = c_map.add_transaction_costs trading_pair: str = raw_trading_pair maker_assets: Tuple[str, str] = self._initialize_market_assets(exchange, [trading_pair])[0] @@ -48,29 +61,27 @@ def start(self): self.market_trading_pair_tuples = [MarketTradingPairTuple(*maker_data)] strategy_logging_options = AvellanedaMarketMakingStrategy.OPTION_LOG_ALL - risk_factor = c_map.get("risk_factor").value - order_amount_shape_factor = c_map.get("order_amount_shape_factor").value - - execution_timeframe = c_map.get("execution_timeframe").value - - start_time = c_map.get("start_time").value - end_time = c_map.get("end_time").value + risk_factor = c_map.risk_factor + order_amount_shape_factor = c_map.order_amount_shape_factor - if execution_timeframe == "from_date_to_date": - start_time = datetime.datetime.fromisoformat(start_time) - end_time = datetime.datetime.fromisoformat(end_time) + execution_timeframe = c_map.execution_timeframe_mode.Config.title + if isinstance(c_map.execution_timeframe_mode, FromDateToDateModel): + start_time = c_map.execution_timeframe_mode.start_datetime + end_time = c_map.execution_timeframe_mode.end_datetime execution_state = RunInTimeConditionalExecutionState(start_timestamp=start_time, end_timestamp=end_time) - if execution_timeframe == "daily_between_times": - start_time = datetime.datetime.strptime(start_time, '%H:%M:%S').time() - end_time = datetime.datetime.strptime(end_time, '%H:%M:%S').time() + elif isinstance(c_map.execution_timeframe_mode, DailyBetweenTimesModel): + start_time = c_map.execution_timeframe_mode.start_time + end_time = c_map.execution_timeframe_mode.end_time execution_state = RunInTimeConditionalExecutionState(start_timestamp=start_time, end_timestamp=end_time) - if execution_timeframe == "infinite": + else: + start_time = None + end_time = None execution_state = RunAlwaysExecutionState() - min_spread = c_map.get("min_spread").value - volatility_buffer_size = c_map.get("volatility_buffer_size").value - trading_intensity_buffer_size = c_map.get("trading_intensity_buffer_size").value - should_wait_order_cancel_confirmation = c_map.get("should_wait_order_cancel_confirmation") + min_spread = c_map.min_spread + volatility_buffer_size = c_map.volatility_buffer_size + trading_intensity_buffer_size = c_map.trading_intensity_buffer_size + should_wait_order_cancel_confirmation = c_map.should_wait_order_cancel_confirmation debug_csv_path = os.path.join(data_path(), HummingbotApplication.main_application().strategy_file_name.rsplit('.', 1)[0] + f"_{pd.Timestamp.now().strftime('%Y-%m-%d_%H-%M-%S')}.csv") diff --git a/hummingbot/strategy/celo_arb/celo_arb_config_map.py b/hummingbot/strategy/celo_arb/celo_arb_config_map.py index 179fa3112e..4641bfbe8d 100644 --- a/hummingbot/strategy/celo_arb/celo_arb_config_map.py +++ b/hummingbot/strategy/celo_arb/celo_arb_config_map.py @@ -12,7 +12,7 @@ def exchange_on_validated(value: str) -> None: - required_exchanges.append(value) + required_exchanges.add(value) def market_trading_pair_prompt() -> str: diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py index 14d6a2e2bc..fbe57e903c 100644 --- a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py +++ b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py @@ -53,7 +53,7 @@ def order_amount_prompt() -> str: def taker_market_on_validated(value: str): - settings.required_exchanges.append(value) + settings.required_exchanges.add(value) def update_oracle_settings(value: str): @@ -87,7 +87,7 @@ def update_oracle_settings(value: str): prompt="Enter your maker spot connector >>> ", prompt_on_new=True, validator=validate_exchange, - on_validated=lambda value: settings.required_exchanges.append(value), + on_validated=lambda value: settings.required_exchanges.add(value), ), "taker_market": ConfigVar( key="taker_market", diff --git a/hummingbot/strategy/dev_0_hello_world/dev_0_hello_world_config_map.py b/hummingbot/strategy/dev_0_hello_world/dev_0_hello_world_config_map.py index d3f01d452c..fbf913f125 100644 --- a/hummingbot/strategy/dev_0_hello_world/dev_0_hello_world_config_map.py +++ b/hummingbot/strategy/dev_0_hello_world/dev_0_hello_world_config_map.py @@ -14,7 +14,7 @@ def exchange_on_validated(value: str) -> None: - required_exchanges.append(value) + required_exchanges.add(value) def trading_pair_prompt(): diff --git a/hummingbot/strategy/dev_1_get_order_book/dev_1_get_order_book_config_map.py b/hummingbot/strategy/dev_1_get_order_book/dev_1_get_order_book_config_map.py index 52a6e4ed1f..60b28b2d2e 100644 --- a/hummingbot/strategy/dev_1_get_order_book/dev_1_get_order_book_config_map.py +++ b/hummingbot/strategy/dev_1_get_order_book/dev_1_get_order_book_config_map.py @@ -37,7 +37,7 @@ def validate_trading_pair(value: str) -> Optional[str]: ConfigVar(key="exchange", prompt="Enter the name of the exchange >>> ", validator=validate_exchange, - on_validated=lambda value: required_exchanges.append(value), + on_validated=lambda value: required_exchanges.add(value), prompt_on_new=True, ), "trading_pair": diff --git a/hummingbot/strategy/dev_2_perform_trade/dev_2_perform_trade_config_map.py b/hummingbot/strategy/dev_2_perform_trade/dev_2_perform_trade_config_map.py index c29b8516d4..c25a68a80a 100644 --- a/hummingbot/strategy/dev_2_perform_trade/dev_2_perform_trade_config_map.py +++ b/hummingbot/strategy/dev_2_perform_trade/dev_2_perform_trade_config_map.py @@ -64,7 +64,7 @@ def order_amount_prompt() -> str: ConfigVar(key="exchange", prompt="Enter the name of the exchange >>> ", validator=validate_exchange, - on_validated=lambda value: required_exchanges.append(value), + on_validated=lambda value: required_exchanges.add(value), prompt_on_new=True, ), "trading_pair": diff --git a/hummingbot/strategy/dev_5_vwap/dev_5_vwap_config_map.py b/hummingbot/strategy/dev_5_vwap/dev_5_vwap_config_map.py index ea28225a2f..41186bfdd0 100644 --- a/hummingbot/strategy/dev_5_vwap/dev_5_vwap_config_map.py +++ b/hummingbot/strategy/dev_5_vwap/dev_5_vwap_config_map.py @@ -42,7 +42,7 @@ def order_percent_of_volume_prompt(): ConfigVar(key="exchange", prompt="Enter the name of the exchange >>> ", validator=validate_exchange, - on_validated=lambda value: required_exchanges.append(value), + on_validated=lambda value: required_exchanges.add(value), prompt_on_new=True), "market": ConfigVar(key="market", diff --git a/hummingbot/strategy/dev_simple_trade/dev_simple_trade_config_map.py b/hummingbot/strategy/dev_simple_trade/dev_simple_trade_config_map.py index fde8306e95..44a122485a 100644 --- a/hummingbot/strategy/dev_simple_trade/dev_simple_trade_config_map.py +++ b/hummingbot/strategy/dev_simple_trade/dev_simple_trade_config_map.py @@ -36,7 +36,7 @@ def validate_market_trading_pair_tuple(value: str) -> Optional[str]: ConfigVar(key="market", prompt="Enter the name of the exchange >>> ", validator=validate_exchange, - on_validated=lambda value: required_exchanges.append(value)), + on_validated=lambda value: required_exchanges.add(value)), "market_trading_pair_tuple": ConfigVar(key="market_trading_pair_tuple", prompt=trading_pair_prompt, diff --git a/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py b/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py index 40a002800e..a6c0608f62 100644 --- a/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py +++ b/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py @@ -18,7 +18,7 @@ def exchange_on_validated(value: str) -> None: - required_exchanges.append(value) + required_exchanges.add(value) def market_validate(value: str) -> Optional[str]: diff --git a/hummingbot/strategy/perpetual_market_making/perpetual_market_making_config_map.py b/hummingbot/strategy/perpetual_market_making/perpetual_market_making_config_map.py index 615ec7a5bd..9801579bb7 100644 --- a/hummingbot/strategy/perpetual_market_making/perpetual_market_making_config_map.py +++ b/hummingbot/strategy/perpetual_market_making/perpetual_market_making_config_map.py @@ -103,7 +103,7 @@ def validate_price_floor_ceiling(value: str) -> Optional[str]: def derivative_on_validated(value: str): - required_exchanges.append(value) + required_exchanges.add(value) perpetual_market_making_config_map = { diff --git a/hummingbot/strategy/pure_market_making/pure_market_making_config_map.py b/hummingbot/strategy/pure_market_making/pure_market_making_config_map.py index f64e648066..54dfa4ab49 100644 --- a/hummingbot/strategy/pure_market_making/pure_market_making_config_map.py +++ b/hummingbot/strategy/pure_market_making/pure_market_making_config_map.py @@ -105,7 +105,7 @@ def on_validated_price_type(value: str): def exchange_on_validated(value: str): - required_exchanges.append(value) + required_exchanges.add(value) pure_market_making_config_map = { diff --git a/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py index 23a4a133ec..cf3fc7ca0b 100644 --- a/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py +++ b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py @@ -15,7 +15,7 @@ def exchange_on_validated(value: str) -> None: - required_exchanges.append(value) + required_exchanges.add(value) def spot_market_validator(value: str) -> None: diff --git a/hummingbot/strategy/twap/twap_config_map.py b/hummingbot/strategy/twap/twap_config_map.py index de7b7e6d3f..df612598f2 100644 --- a/hummingbot/strategy/twap/twap_config_map.py +++ b/hummingbot/strategy/twap/twap_config_map.py @@ -75,7 +75,7 @@ def validate_order_step_size(value: str = None): ConfigVar(key="connector", prompt="Enter the name of spot connector >>> ", validator=validate_exchange, - on_validated=lambda value: required_exchanges.append(value), + on_validated=lambda value: required_exchanges.add(value), prompt_on_new=True), "trading_pair": ConfigVar(key="trading_pair", diff --git a/hummingbot/strategy/uniswap_v3_lp/uniswap_v3_lp_config_map.py b/hummingbot/strategy/uniswap_v3_lp/uniswap_v3_lp_config_map.py index 168073fc3d..199efa6965 100644 --- a/hummingbot/strategy/uniswap_v3_lp/uniswap_v3_lp_config_map.py +++ b/hummingbot/strategy/uniswap_v3_lp/uniswap_v3_lp_config_map.py @@ -19,7 +19,7 @@ def market_validator(value: str) -> None: def market_on_validated(value: str) -> None: - required_exchanges.append(value) + required_exchanges.add(value) requried_connector_trading_pairs["uniswap_v3"] = [value] diff --git a/test/hummingbot/client/command/test_config_command.py b/test/hummingbot/client/command/test_config_command.py index 330a2c119e..dae17bc48f 100644 --- a/test/hummingbot/client/command/test_config_command.py +++ b/test/hummingbot/client/command/test_config_command.py @@ -2,13 +2,18 @@ import unittest from collections import Awaitable from copy import deepcopy +from decimal import Decimal from unittest.mock import patch, MagicMock +from pydantic import Field + from hummingbot.client.command.config_command import color_settings_to_display, global_configs_to_display +from hummingbot.client.config.config_data_types import BaseClientModel, BaseStrategyConfigMap, ClientFieldData from hummingbot.client.config.config_helpers import read_system_configs_from_yml from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.hummingbot_application import HummingbotApplication +from test.mock.mock_cli import CLIMockingAssistant class ConfigCommandTest(unittest.TestCase): @@ -20,9 +25,12 @@ def setUp(self, _: MagicMock) -> None: self.async_run_with_timeout(read_system_configs_from_yml()) self.app = HummingbotApplication() + self.cli_mock_assistant = CLIMockingAssistant(self.app.app) + self.cli_mock_assistant.start() self.global_config_backup = deepcopy(global_config_map) def tearDown(self) -> None: + self.cli_mock_assistant.stop() self.reset_global_config() super().tearDown() @@ -68,37 +76,166 @@ def test_list_configs(self, notify_mock, get_strategy_config_map_mock): self.assertEqual("\nGlobal Configurations:", captures[0]) df_str_expected = ( - " +---------------------+-----------+" - "\n | Key | Value |" - "\n |---------------------+-----------|" - "\n | tables_format | psql |" - "\n | autofill_import | first |" - "\n | kill_switch_enabled | second |" - "\n +---------------------+-----------+" + " +---------------------+---------+" + "\n | Key | Value |" + "\n |---------------------+---------|" + "\n | tables_format | psql |" + "\n | autofill_import | first |" + "\n | kill_switch_enabled | second |" + "\n +---------------------+---------+" ) self.assertEqual(df_str_expected, captures[1]) self.assertEqual("\nColor Settings:", captures[2]) df_str_expected = ( - " +-------------+-----------+" - "\n | Key | Value |" - "\n |-------------+-----------|" - "\n | top-pane | third |" - "\n | bottom-pane | fourth |" - "\n +-------------+-----------+" + " +-------------+---------+" + "\n | Key | Value |" + "\n |-------------+---------|" + "\n | top-pane | third |" + "\n | bottom-pane | fourth |" + "\n +-------------+---------+" ) self.assertEqual(df_str_expected, captures[3]) self.assertEqual("\nStrategy Configurations:", captures[4]) df_str_expected = ( - " +-------+-----------+" - "\n | Key | Value |" - "\n |-------+-----------|" - "\n | five | fifth |" - "\n | six | sixth |" - "\n +-------+-----------+" + " +-------+---------+" + "\n | Key | Value |" + "\n |-------+---------|" + "\n | five | fifth |" + "\n | six | sixth |" + "\n +-------+---------+" + ) + + self.assertEqual(df_str_expected, captures[5]) + + @patch("hummingbot.client.hummingbot_application.get_strategy_config_map") + @patch("hummingbot.client.hummingbot_application.HummingbotApplication._notify") + def test_list_configs_pydantic_model(self, notify_mock, get_strategy_config_map_mock): + captures = [] + notify_mock.side_effect = lambda s: captures.append(s) + strategy_name = "some-strategy" + self.app.strategy_name = strategy_name + + tables_format_config_var = global_config_map["tables_format"] + global_config_map.clear() + global_config_map[tables_format_config_var.key] = tables_format_config_var + tables_format_config_var.value = "psql" + + class DoubleNestedModel(BaseClientModel): + double_nested_attr: float = Field(default=3.0) + + class Config: + title = "double_nested_model" + + class NestedModel(BaseClientModel): + nested_attr: str = Field(default="some value") + double_nested_model: DoubleNestedModel = Field(default=DoubleNestedModel()) + + class Config: + title = "nested_model" + + class DummyModel(BaseClientModel): + some_attr: int = Field(default=1) + nested_model: NestedModel = Field(default=NestedModel()) + another_attr: Decimal = Field(default=Decimal("1.0")) + + class Config: + title = "dummy_model" + + get_strategy_config_map_mock.return_value = DummyModel() + + self.app.list_configs() + + self.assertEqual(6, len(captures)) + + self.assertEqual("\nStrategy Configurations:", captures[4]) + + df_str_expected = ( + " +------------------------+---------------------+" + "\n | Key | Value |" + "\n |------------------------+---------------------|" + "\n | some_attr | 1 |" + "\n | nested_model | nested_model |" + "\n | ∟ nested_attr | some value |" + "\n | ∟ double_nested_model | double_nested_model |" + "\n | ∟ double_nested_attr | 3.0 |" + "\n | another_attr | 1.0 |" + "\n +------------------------+---------------------+" ) self.assertEqual(df_str_expected, captures[5]) + + @patch("hummingbot.client.hummingbot_application.get_strategy_config_map") + @patch("hummingbot.client.hummingbot_application.HummingbotApplication._notify") + def test_config_non_configurable_key_fails(self, notify_mock, get_strategy_config_map_mock): + class DummyModel(BaseStrategyConfigMap): + strategy: str = Field(default="pure_market_making", client_data=None) + some_attr: int = Field(default=1, client_data=ClientFieldData(prompt=lambda mi: "some prompt")) + another_attr: Decimal = Field(default=Decimal("1.0")) + + class Config: + title = "dummy_model" + + strategy_name = "some-strategy" + self.app.strategy_name = strategy_name + get_strategy_config_map_mock.return_value = DummyModel.construct() + self.app.config(key="some_attr") + + notify_mock.assert_not_called() + + self.app.config(key="another_attr") + + notify_mock.assert_called_once_with("Invalid key, please choose from the list.") + + notify_mock.reset_mock() + self.app.config(key="some_key") + + notify_mock.assert_called_once_with("Invalid key, please choose from the list.") + + @patch("hummingbot.client.command.config_command.save_to_yml") + @patch("hummingbot.client.hummingbot_application.get_strategy_config_map") + @patch("hummingbot.client.hummingbot_application.HummingbotApplication._notify") + def test_config_single_keys(self, _, get_strategy_config_map_mock, save_to_yml_mock): + class NestedModel(BaseClientModel): + nested_attr: str = Field( + default="some value", client_data=ClientFieldData(prompt=lambda mi: "some prompt") + ) + + class Config: + title = "nested_model" + + class DummyModel(BaseStrategyConfigMap): + strategy: str = Field(default="pure_market_making", client_data=None) + some_attr: int = Field(default=1, client_data=ClientFieldData(prompt=lambda mi: "some prompt")) + nested_model: NestedModel = Field(default=NestedModel()) + + class Config: + title = "dummy_model" + + strategy_name = "some-strategy" + self.app.strategy_name = strategy_name + self.app.strategy_file_name = f"{strategy_name}.yml" + config_map = DummyModel.construct() + get_strategy_config_map_mock.return_value = config_map + + self.async_run_with_timeout(self.app._config_single_key(key="some_attr", input_value=2)) + + self.assertEqual(2, config_map.some_attr) + save_to_yml_mock.assert_called_once() + + save_to_yml_mock.reset_mock() + self.cli_mock_assistant.queue_prompt_reply("3") + self.async_run_with_timeout(self.app._config_single_key(key="some_attr", input_value=None)) + + self.assertEqual(3, config_map.some_attr) + save_to_yml_mock.assert_called_once() + + save_to_yml_mock.reset_mock() + self.cli_mock_assistant.queue_prompt_reply("another value") + self.async_run_with_timeout(self.app._config_single_key(key="nested_model.nested_attr", input_value=None), 10000) + + self.assertEqual("another value", config_map.nested_model.nested_attr) + save_to_yml_mock.assert_called_once() diff --git a/test/hummingbot/client/command/test_create_command.py b/test/hummingbot/client/command/test_create_command.py index 5cbd5da196..c98fa8b756 100644 --- a/test/hummingbot/client/command/test_create_command.py +++ b/test/hummingbot/client/command/test_create_command.py @@ -62,7 +62,7 @@ async def run_coro_that_raises(coro: Awaitable): raise RuntimeError @patch("shutil.copy") - @patch("hummingbot.client.command.create_command.save_to_yml") + @patch("hummingbot.client.command.create_command.save_to_yml_legacy") @patch("hummingbot.client.config.security.Security.is_decryption_done") @patch("hummingbot.client.command.status_command.StatusCommand.validate_required_connections") @patch("hummingbot.core.utils.market_price.get_last_price") @@ -101,7 +101,7 @@ def test_prompt_for_configuration_re_prompts_on_lower_than_minimum_amount( self.assertTrue(self.cli_mock_assistant.check_log_called_with(msg="Value must be more than 0.")) @patch("shutil.copy") - @patch("hummingbot.client.command.create_command.save_to_yml") + @patch("hummingbot.client.command.create_command.save_to_yml_legacy") @patch("hummingbot.client.config.security.Security.is_decryption_done") @patch("hummingbot.client.command.status_command.StatusCommand.validate_required_connections") @patch("hummingbot.core.utils.market_price.get_last_price") @@ -174,7 +174,7 @@ def test_create_command_restores_config_map_after_config_stop_on_new_file_prompt self.assertEqual(original_exchange, strategy_config["exchange"].value) @patch("shutil.copy") - @patch("hummingbot.client.command.create_command.save_to_yml") + @patch("hummingbot.client.command.create_command.save_to_yml_legacy") @patch("hummingbot.client.config.security.Security.is_decryption_done") @patch("hummingbot.client.command.status_command.StatusCommand.validate_required_connections") @patch("hummingbot.core.utils.market_price.get_last_price") diff --git a/test/hummingbot/client/config/test_config_data_types.py b/test/hummingbot/client/config/test_config_data_types.py new file mode 100644 index 0000000000..5d80a3449c --- /dev/null +++ b/test/hummingbot/client/config/test_config_data_types.py @@ -0,0 +1,230 @@ +import asyncio +import json +import unittest +from datetime import date, datetime, time +from decimal import Decimal +from typing import Awaitable, Dict +from unittest.mock import patch + +from pydantic import Field, ValidationError +from pydantic.fields import FieldInfo + +from hummingbot.client.config.config_data_types import ( + BaseClientModel, + BaseStrategyConfigMap, + BaseTradingStrategyConfigMap, + ClientConfigEnum, + ClientFieldData, + TraversalItem, +) +from hummingbot.client.config.config_helpers import retrieve_validation_error_msg + + +class BaseClientModelTest(unittest.TestCase): + def test_schema_encoding_removes_client_data_functions(self): + class DummyModel(BaseClientModel): + some_attr: str = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda mi: "Some prompt?", + prompt_on_new=True, + ), + ) + + schema = DummyModel.schema_json() + j = json.loads(schema) + expected = { + "is_secure": False, + "prompt": None, + "prompt_on_new": True, + } + self.assertEqual(expected, j["properties"]["some_attr"]["client_data"]) + + def test_traverse(self): + class DoubleNestedModel(BaseClientModel): + double_nested_attr: float = Field(default=3.0) + + class Config: + title = "double_nested_model" + + class NestedModel(BaseClientModel): + nested_attr: str = Field(default="some value") + double_nested_model: DoubleNestedModel = Field(default=DoubleNestedModel()) + + class Config: + title = "nested_model" + + class DummyModel(BaseClientModel): + some_attr: int = Field(default=1, client_data=ClientFieldData()) + nested_model: NestedModel = Field(default=NestedModel()) + another_attr: Decimal = Field(default=Decimal("1.0")) + + class Config: + title = "dummy_model" + + expected = [ + TraversalItem(0, "some_attr", "some_attr", 1, "1", ClientFieldData(), None), + TraversalItem(0, "nested_model", "nested_model", NestedModel(), "nested_model", None, None), + TraversalItem(1, "nested_model.nested_attr", "nested_attr", "some value", "some value", None, None), + TraversalItem( + 1, + "nested_model.double_nested_model", + "double_nested_model", + DoubleNestedModel(), + "double_nested_model", + None, + None, + ), + TraversalItem( + 2, "nested_model.double_nested_model.double_nested_attr", "double_nested_attr", 3.0, "3.0", None, None + ), + ] + cm = DummyModel() + + for expected, actual in zip(expected, cm.traverse()): + self.assertEqual(expected.depth, actual.depth) + self.assertEqual(expected.config_path, actual.config_path) + self.assertEqual(expected.attr, actual.attr) + self.assertEqual(expected.value, actual.value) + self.assertEqual(expected.printable_value, actual.printable_value) + self.assertEqual(expected.client_field_data, actual.client_field_data) + self.assertIsInstance(actual.field_info, FieldInfo) + + def test_generate_yml_output_dict_with_comments(self): + class SomeEnum(ClientConfigEnum): + ONE = "one" + + class DoubleNestedModel(BaseClientModel): + double_nested_attr: datetime = Field( + default=datetime(2022, 1, 1, 10, 30), + description="Double nested attr description" + ) + + class NestedModel(BaseClientModel): + nested_attr: str = Field( + default="some value", + description="Nested attr\nmultiline description", + ) + double_nested_model: DoubleNestedModel = Field( + default=DoubleNestedModel(), + ) + + class DummyModel(BaseClientModel): + some_attr: SomeEnum = Field( + default=SomeEnum.ONE, + description="Some description", + ) + nested_model: NestedModel = Field( + default=NestedModel(), + description="Nested model description", + ) + another_attr: Decimal = Field( + default=Decimal("1.0"), + description="Some other\nmultiline description", + ) + non_nested_no_description: time = Field(default=time(10, 30),) + date_attr: date = Field(default=date(2022, 1, 2)) + + class Config: + title = "dummy_model" + + instance = DummyModel() + res_str = instance.generate_yml_output_str_with_comments() + + expected_str = """\ +############################## +### dummy_model config ### +############################## + +# Some description +some_attr: one + +# Nested model description +nested_model: + # Nested attr + # multiline description + nested_attr: some value + double_nested_model: + # Double nested attr description + double_nested_attr: 2022-01-01 10:30:00 + +# Some other +# multiline description +another_attr: 1.0 + +non_nested_no_description: '10:30:00' + +date_attr: 2022-01-02 +""" + + self.assertEqual(expected_str, res_str) + + +class BaseStrategyConfigMapTest(unittest.TestCase): + def test_generate_yml_output_dict_title(self): + instance = BaseStrategyConfigMap(strategy="pure_market_making") + res_str = instance.generate_yml_output_str_with_comments() + + expected_str = """\ +############################################## +### Pure Market Making Strategy config ### +############################################## + +strategy: pure_market_making +""" + + self.assertEqual(expected_str, res_str) + + +class BaseTradingStrategyConfigMapTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + cls.ev_loop = asyncio.get_event_loop() + cls.exchange = "binance" + cls.base_asset = "COINALPHA" + cls.quote_asset = "HBOT" + cls.trading_pair = f"{cls.base_asset}-{cls.quote_asset}" + + def setUp(self) -> None: + super().setUp() + config_settings = self.get_default_map() + self.config_map = BaseTradingStrategyConfigMap(**config_settings) + + def async_run_with_timeout(self, coroutine: Awaitable, timeout: int = 1): + ret = self.ev_loop.run_until_complete(asyncio.wait_for(coroutine, timeout)) + return ret + + def get_default_map(self) -> Dict[str, str]: + config_settings = { + "strategy": "pure_market_making", + "exchange": self.exchange, + "market": self.trading_pair, + } + return config_settings + + @patch( + "hummingbot.client.config.config_data_types.validate_market_trading_pair" + ) + def test_validators(self, validate_market_trading_pair_mock): + with self.assertRaises(ValidationError) as e: + self.config_map.exchange = "test-exchange" + + error_msg = "Invalid exchange, please choose value from " + actual_msg = retrieve_validation_error_msg(e.exception) + self.assertTrue(actual_msg.startswith(error_msg)) + + alt_pair = "ETH-USDT" + error_msg = "Failed" + validate_market_trading_pair_mock.side_effect = ( + lambda m, v: None if v in [self.trading_pair, alt_pair] else error_msg + ) + + self.config_map.market = alt_pair + self.assertEqual(alt_pair, self.config_map.market) + + with self.assertRaises(ValidationError) as e: + self.config_map.market = "XXX-USDT" + + actual_msg = retrieve_validation_error_msg(e.exception) + self.assertTrue(actual_msg.startswith(error_msg)) diff --git a/test/hummingbot/client/config/test_config_helpers.py b/test/hummingbot/client/config/test_config_helpers.py index 18d2562605..553799d9a8 100644 --- a/test/hummingbot/client/config/test_config_helpers.py +++ b/test/hummingbot/client/config/test_config_helpers.py @@ -1,7 +1,15 @@ import asyncio import unittest +from pathlib import Path +from tempfile import TemporaryDirectory from typing import Awaitable +from hummingbot.client.config.config_data_types import BaseStrategyConfigMap +from hummingbot.client.config.config_helpers import get_strategy_config_map, save_to_yml +from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( + AvellanedaMarketMakingConfigMap +) + class ConfigHelpersTest(unittest.TestCase): def setUp(self) -> None: @@ -17,3 +25,25 @@ def get_async_sleep_fn(delay: float): async def async_sleep(*_, **__): await asyncio.sleep(delay) return async_sleep + + def test_get_strategy_config_map(self): + cm = get_strategy_config_map(strategy="avellaneda_market_making") + self.assertIsInstance(cm, AvellanedaMarketMakingConfigMap) + self.assertFalse(hasattr(cm, "market")) # uninitialized instance + + def test_save_to_yml(self): + cm = BaseStrategyConfigMap(strategy="pure_market_making") + expected_str = """\ +############################################## +### Pure Market Making Strategy config ### +############################################## + +strategy: pure_market_making +""" + with TemporaryDirectory() as d: + d = Path(d) + temp_file_name = d / "cm.yml" + save_to_yml(str(temp_file_name), cm) + with open(temp_file_name) as f: + actual_str = f.read() + self.assertEqual(expected_str, actual_str) diff --git a/test/hummingbot/client/config/test_config_templates.py b/test/hummingbot/client/config/test_config_templates.py index 389bdbfac9..f9d34c6693 100644 --- a/test/hummingbot/client/config/test_config_templates.py +++ b/test/hummingbot/client/config/test_config_templates.py @@ -1,11 +1,9 @@ #!/usr/bin/env python from os.path import ( - isdir, join, realpath, ) -from os import listdir import logging; logging.basicConfig(level=logging.INFO) import unittest import ruamel.yaml @@ -37,11 +35,26 @@ def test_global_config_template_complete(self): for key in global_config_map: self.assertTrue(key in template_data, f"{key} not in {global_config_template_path}") - def test_strategy_config_template_complete(self): - folder = realpath(join(__file__, "../../../../../hummingbot/strategy")) - # Only include valid directories - strategies = [d for d in listdir(folder) if isdir(join(folder, d)) and not d.startswith("__")] - strategies.sort() + def test_strategy_config_template_complete_legacy(self): + strategies = [ # templates is a legacy approach — new strategies won't use it + "amm_arb", + "arbitrage", + "aroon_oscillator", + "celo_arb", + "cross_exchange_market_making", + "dev_0_hello_world", + "dev_1_get_order_book", + "dev_2_perform_trade", + "dev_5_vwap", + "dev_simple_trade", + "hedge", + "liquidity_mining", + "perpetual_market_making", + "pure_market_making", + "spot_perpetual_arbitrage", + "twap", + "uniswap_v3_lp", + ] for strategy in strategies: strategy_template_path: str = get_strategy_template_path(strategy) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py index 64fec3dadd..4460b7959e 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py @@ -1,8 +1,8 @@ -import json +import asyncio import unittest from datetime import datetime, time from pathlib import Path -from typing import Dict +from typing import Awaitable, Dict from unittest.mock import patch import yaml @@ -23,6 +23,7 @@ class AvellanedaMarketMakingConfigMapPydanticTest(unittest.TestCase): @classmethod def setUpClass(cls) -> None: super().setUpClass() + cls.ev_loop = asyncio.get_event_loop() cls.exchange = "binance" cls.base_asset = "COINALPHA" cls.quote_asset = "HBOT" @@ -33,11 +34,15 @@ def setUp(self) -> None: config_settings = self.get_default_map() self.config_map = AvellanedaMarketMakingConfigMap(**config_settings) + def async_run_with_timeout(self, coroutine: Awaitable, timeout: int = 1): + ret = self.ev_loop.run_until_complete(asyncio.wait_for(coroutine, timeout)) + return ret + def get_default_map(self) -> Dict[str, str]: config_settings = { "exchange": self.exchange, "market": self.trading_pair, - "execution_timeframe_model": { + "execution_timeframe_mode": { "start_time": "09:30:00", "end_time": "16:00:00", }, @@ -49,15 +54,6 @@ def get_default_map(self) -> Dict[str, str]: } return config_settings - def test_schema_encoding_removes_client_data_functions(self): - s = AvellanedaMarketMakingConfigMap.schema_json() - j = json.loads(s) - expected = { - "prompt": None, - "prompt_on_new": True, - } - self.assertEqual(expected, j["properties"]["market"]["client_data"]) - def test_initial_sequential_build(self): config_map: AvellanedaMarketMakingConfigMap = AvellanedaMarketMakingConfigMap.construct() config_settings = self.get_default_map() @@ -80,7 +76,7 @@ def build_config_map(cm: BaseClientModel, cs: Dict): validate_model(config_map.__class__, config_map.__dict__) def test_order_amount_prompt(self): - prompt = self.config_map.get_client_prompt("order_amount") + prompt = self.async_run_with_timeout(self.config_map.get_client_prompt("order_amount")) expected = f"What is the amount of {self.base_asset} per order?" self.assertEqual(expected, prompt) @@ -89,69 +85,45 @@ def test_maker_trading_pair_prompt(self): exchange = self.config_map.exchange example = AllConnectorSettings.get_example_pairs().get(exchange) - prompt = self.config_map.get_client_prompt("market") + prompt = self.async_run_with_timeout(self.config_map.get_client_prompt("market")) expected = f"Enter the token trading pair you would like to trade on {exchange} (e.g. {example})" self.assertEqual(expected, prompt) def test_execution_time_prompts(self): - self.config_map.execution_timeframe_model = FromDateToDateModel.Config.title - model = self.config_map.execution_timeframe_model - prompt = model.get_client_prompt("start_datetime") + self.config_map.execution_timeframe_mode = FromDateToDateModel.Config.title + model = self.config_map.execution_timeframe_mode + prompt = self.async_run_with_timeout(model.get_client_prompt("start_datetime")) expected = "Please enter the start date and time (YYYY-MM-DD HH:MM:SS)" self.assertEqual(expected, prompt) - prompt = model.get_client_prompt("end_datetime") + prompt = self.async_run_with_timeout(model.get_client_prompt("end_datetime")) expected = "Please enter the end date and time (YYYY-MM-DD HH:MM:SS)" self.assertEqual(expected, prompt) - self.config_map.execution_timeframe_model = DailyBetweenTimesModel.Config.title - model = self.config_map.execution_timeframe_model - prompt = model.get_client_prompt("start_time") + self.config_map.execution_timeframe_mode = DailyBetweenTimesModel.Config.title + model = self.config_map.execution_timeframe_mode + prompt = self.async_run_with_timeout(model.get_client_prompt("start_time")) expected = "Please enter the start time (HH:MM:SS)" self.assertEqual(expected, prompt) - prompt = model.get_client_prompt("end_time") + prompt = self.async_run_with_timeout(model.get_client_prompt("end_time")) expected = "Please enter the end time (HH:MM:SS)" self.assertEqual(expected, prompt) @patch( - "hummingbot.strategy.avellaneda_market_making" - ".avellaneda_market_making_config_map_pydantic.validate_market_trading_pair" + "hummingbot.client.config.config_data_types.validate_market_trading_pair" ) def test_validators(self, validate_market_trading_pair_mock): + self.config_map.execution_timeframe_mode = "infinite" + self.assertIsInstance(self.config_map.execution_timeframe_mode, InfiniteModel) - with self.assertRaises(ValidationError) as e: - self.config_map.exchange = "test-exchange" - - error_msg = "Invalid exchange, please choose value from " - actual_msg = retrieve_validation_error_msg(e.exception) - self.assertTrue(actual_msg.startswith(error_msg)) - - alt_pair = "ETH-USDT" - error_msg = "Failed" - validate_market_trading_pair_mock.side_effect = ( - lambda m, v: None if v in [self.trading_pair, alt_pair] else error_msg - ) - - self.config_map.market = alt_pair - self.assertEqual(alt_pair, self.config_map.market) - - with self.assertRaises(ValidationError) as e: - self.config_map.market = "XXX-USDT" - - actual_msg = retrieve_validation_error_msg(e.exception) - self.assertTrue(actual_msg.startswith(error_msg)) - - self.config_map.execution_timeframe_model = "infinite" - self.assertIsInstance(self.config_map.execution_timeframe_model, InfiniteModel) - - self.config_map.execution_timeframe_model = "from_date_to_date" - self.assertIsInstance(self.config_map.execution_timeframe_model, FromDateToDateModel) + self.config_map.execution_timeframe_mode = "from_date_to_date" + self.assertIsInstance(self.config_map.execution_timeframe_mode, FromDateToDateModel) - self.config_map.execution_timeframe_model = "daily_between_times" - self.assertIsInstance(self.config_map.execution_timeframe_model, DailyBetweenTimesModel) + self.config_map.execution_timeframe_mode = "daily_between_times" + self.assertIsInstance(self.config_map.execution_timeframe_mode, DailyBetweenTimesModel) with self.assertRaises(ValidationError) as e: - self.config_map.execution_timeframe_model = "XXX" + self.config_map.execution_timeframe_mode = "XXX" error_msg = ( "Invalid timeframe, please choose value from ['infinite', 'from_date_to_date', 'daily_between_times']" @@ -159,8 +131,8 @@ def test_validators(self, validate_market_trading_pair_mock): actual_msg = retrieve_validation_error_msg(e.exception) self.assertEqual(error_msg, actual_msg) - self.config_map.execution_timeframe_model = "from_date_to_date" - model = self.config_map.execution_timeframe_model + self.config_map.execution_timeframe_mode = "from_date_to_date" + model = self.config_map.execution_timeframe_mode model.start_datetime = "2021-01-01 12:00:00" model.end_datetime = "2021-01-01 15:00:00" @@ -181,8 +153,8 @@ def test_validators(self, validate_market_trading_pair_mock): actual_msg = retrieve_validation_error_msg(e.exception) self.assertEqual(error_msg, actual_msg) - self.config_map.execution_timeframe_model = "daily_between_times" - model = self.config_map.execution_timeframe_model + self.config_map.execution_timeframe_mode = "daily_between_times" + model = self.config_map.execution_timeframe_mode model.start_time = "12:00:00" self.assertEqual(time(12, 0, 0), model.start_time) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py index 82a284cad8..96e6bc1f8c 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py @@ -1,40 +1,55 @@ import datetime +import logging from decimal import Decimal import unittest.mock import hummingbot.strategy.avellaneda_market_making.start as strategy_start from hummingbot.connector.exchange_base import ExchangeBase -from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map import ( - avellaneda_market_making_config_map as strategy_cmap +from hummingbot.connector.utils import combine_to_hb_trading_pair +from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( + AvellanedaMarketMakingConfigMap, + FromDateToDateModel, + MultiOrderLevelModel, + TrackHangingOrdersModel, ) -from test.hummingbot.strategy import assign_config_default class AvellanedaStartTest(unittest.TestCase): + # the level is required to receive logs from the data source logger + level = 0 def setUp(self) -> None: super().setUp() self.strategy = None self.markets = {"binance": ExchangeBase()} self.notifications = [] - self.log_errors = [] - assign_config_default(strategy_cmap) - strategy_cmap.get("exchange").value = "binance" - strategy_cmap.get("market").value = "balancer" - strategy_cmap.get("execution_timeframe").value = "from_date_to_date" - strategy_cmap.get("start_time").value = "2021-11-18 15:00:00" - strategy_cmap.get("end_time").value = "2021-11-18 16:00:00" - strategy_cmap.get("order_amount").value = Decimal("1") - strategy_cmap.get("order_refresh_time").value = 60. - strategy_cmap.get("hanging_orders_enabled").value = True - strategy_cmap.get("hanging_orders_cancel_pct").value = Decimal("1") - # strategy_cmap.get("hanging_orders_aggregation_type").value = "VOLUME_WEIGHTED" - strategy_cmap.get("min_spread").value = Decimal("2") - strategy_cmap.get("risk_factor").value = Decimal("1.11") - strategy_cmap.get("order_levels").value = Decimal("4") - strategy_cmap.get("level_distances").value = Decimal("1") - strategy_cmap.get("order_amount_shape_factor").value = Decimal("3.33") + self.log_records = [] + self.base = "ETH" + self.quote = "BTC" + self.strategy_config_map = AvellanedaMarketMakingConfigMap( + exchange="binance", + market=combine_to_hb_trading_pair(self.base, self.quote), + execution_timeframe_model=FromDateToDateModel( + start_datetime="2021-11-18 15:00:00", + end_datetime="2021-11-18 16:00:00", + ), + order_amount=60, + order_refresh_time=60, + hanging_orders_model=TrackHangingOrdersModel( + hanging_orders_cancel_pct=1, + ), + order_levels_mode=MultiOrderLevelModel( + order_levels=4, + level_distances=1, + ), + min_spread=2, + risk_factor=1.11, + order_levels=4, + level_distances=1, + order_amount_shape_factor=0.33, + ) self.raise_exception_for_market_initialization = False + self._logger = None def _initialize_market_assets(self, market, trading_pairs): return [("ETH", "USDT")] @@ -47,10 +62,13 @@ def _notify(self, message): self.notifications.append(message) def logger(self): - return self + if self._logger is None: + self._logger = logging.getLogger(self.__class__.__name__) + self._logger.addHandler(self) + return self._logger - def error(self, message, exc_info): - self.log_errors.append(message) + def handle(self, record): + self.log_records.append(record) @unittest.mock.patch('hummingbot.strategy.avellaneda_market_making.start.HummingbotApplication') def test_parameters_strategy_creation(self, mock_hbot): @@ -61,7 +79,7 @@ def test_parameters_strategy_creation(self, mock_hbot): self.assertEqual(self.strategy.end_time, datetime.datetime(2021, 11, 18, 16, 0)) self.assertEqual(self.strategy.min_spread, Decimal("2")) self.assertEqual(self.strategy.gamma, Decimal("1.11")) - self.assertEqual(self.strategy.eta, Decimal("3.33")) + self.assertEqual(self.strategy.eta, Decimal("0.33")) self.assertEqual(self.strategy.order_levels, Decimal("4")) self.assertEqual(self.strategy.level_distances, Decimal("1")) self.assertTrue(all(c is not None for c in (self.strategy.gamma, self.strategy.eta))) @@ -73,5 +91,5 @@ def test_strategy_creation_when_something_fails(self): strategy_start.start(self) self.assertEqual(len(self.notifications), 1) self.assertEqual(self.notifications[0], "Exception for testing") - self.assertEqual(len(self.log_errors), 1) - self.assertEqual(self.log_errors[0], "Unknown error during initialization.") + self.assertEqual(len(self.log_records), 1) + self.assertEqual(self.log_records[0].message, "Unknown error during initialization.") diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_config.yml b/test/hummingbot/strategy/avellaneda_market_making/test_config.yml index c7c7cf1d39..051b140d92 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_config.yml +++ b/test/hummingbot/strategy/avellaneda_market_making/test_config.yml @@ -1,6 +1,6 @@ exchange: binance market: COINALPHA-HBOT -execution_timeframe_model: +execution_timeframe_mode: start_time: "09:30:00" end_time: "16:00:00" order_amount: 10 From ee5fb8bf5df9ccbeb2989af4036ba87e9deea15d Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Fri, 1 Apr 2022 17:34:18 +0700 Subject: [PATCH 032/152] (feat) Allows the import command to handle pydantic-based configs. - Adds an extra abstraction layer `ClientConfigAdapter` - All things Pydantic-related are hidden by it from the client - Allows loading incomplete configs and to complete them after the fact - The same behaviour as we currently have for ConfigVars - This required some changes in already existing code --- bin/hummingbot.py | 4 +- bin/hummingbot_quickstart.py | 8 +- hummingbot/client/command/config_command.py | 21 +- hummingbot/client/command/create_command.py | 55 ++- hummingbot/client/command/import_command.py | 13 +- hummingbot/client/command/status_command.py | 50 ++- hummingbot/client/config/config_data_types.py | 160 +------- hummingbot/client/config/config_helpers.py | 370 +++++++++++++----- hummingbot/client/config/config_methods.py | 9 + hummingbot/client/ui/style.py | 1 + ...aneda_market_making_config_map_pydantic.py | 8 +- .../avellaneda_market_making/start.py | 11 +- .../client/command/test_config_command.py | 31 +- .../client/command/test_import_command.py | 142 ++++++- .../client/command/test_status_command.py | 6 +- .../client/config/test_config_data_types.py | 51 +-- .../client/config/test_config_helpers.py | 18 +- ...est_avellaneda_market_making_config_map.py | 136 ------- ...aneda_market_making_config_map_pydantic.py | 70 ++-- .../test_avellaneda_market_making_start.py | 43 +- 20 files changed, 623 insertions(+), 584 deletions(-) delete mode 100644 test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map.py diff --git a/bin/hummingbot.py b/bin/hummingbot.py index c738eeeb4a..7da4736196 100755 --- a/bin/hummingbot.py +++ b/bin/hummingbot.py @@ -16,7 +16,7 @@ from hummingbot.client.hummingbot_application import HummingbotApplication from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.config_helpers import ( - create_yml_files, + create_yml_files_legacy, read_system_configs_from_yml, write_config_to_yml, ) @@ -71,7 +71,7 @@ async def ui_start_handler(self): async def main(): - await create_yml_files() + await create_yml_files_legacy() # This init_logging() call is important, to skip over the missing config warnings. init_logging("hummingbot_logs.yml") diff --git a/bin/hummingbot_quickstart.py b/bin/hummingbot_quickstart.py index 7f75a46725..79056157fe 100755 --- a/bin/hummingbot_quickstart.py +++ b/bin/hummingbot_quickstart.py @@ -19,9 +19,9 @@ from hummingbot.client.hummingbot_application import HummingbotApplication from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.config_helpers import ( - create_yml_files, + create_yml_files_legacy, read_system_configs_from_yml, - update_strategy_config_map_from_file, + load_strategy_config_map_from_file, all_configs_complete, ) from hummingbot.client.ui import login_prompt @@ -73,7 +73,7 @@ async def quick_start(args): return await Security.wait_til_decryption_done() - await create_yml_files() + await create_yml_files_legacy() init_logging("hummingbot_logs.yml") await read_system_configs_from_yml() @@ -84,7 +84,7 @@ async def quick_start(args): if config_file_name is not None: hb.strategy_file_name = config_file_name - hb.strategy_name = await update_strategy_config_map_from_file(os.path.join(CONF_FILE_PATH, config_file_name)) + hb.strategy_name = await load_strategy_config_map_from_file(os.path.join(CONF_FILE_PATH, config_file_name)) # To ensure quickstart runs with the default value of False for kill_switch_enabled if not present if not global_config_map.get("kill_switch_enabled"): diff --git a/hummingbot/client/command/config_command.py b/hummingbot/client/command/config_command.py index adf3266b55..8889a274fe 100644 --- a/hummingbot/client/command/config_command.py +++ b/hummingbot/client/command/config_command.py @@ -6,12 +6,13 @@ import pandas as pd from prompt_toolkit.utils import is_windows -from hummingbot.client.config.config_data_types import ( - BaseClientModel, - BaseStrategyConfigMap, - BaseTradingStrategyConfigMap +from hummingbot.client.config.config_data_types import BaseTradingStrategyConfigMap +from hummingbot.client.config.config_helpers import ( + missing_required_configs_legacy, + save_to_yml, + save_to_yml_legacy, + ClientConfigAdapter, ) -from hummingbot.client.config.config_helpers import missing_required_configs_legacy, save_to_yml, save_to_yml_legacy from hummingbot.client.config.config_validators import validate_bool, validate_decimal from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.global_config_map import global_config_map @@ -115,16 +116,16 @@ def list_strategy_configs( def build_df_data_from_config_map( self, # type: HummingbotApplication - config_map: Union[BaseClientModel, Dict[str, ConfigVar]] + config_map: Union[ClientConfigAdapter, Dict[str, ConfigVar]] ) -> List[Tuple[str, Any]]: - if isinstance(config_map, BaseClientModel): + if isinstance(config_map, ClientConfigAdapter): data = self.build_model_df_data(config_map) else: # legacy data = [[cv.printable_key or cv.key, cv.value] for cv in self.strategy_config_map.values() if not cv.is_secure] return data @staticmethod - def build_model_df_data(config_map: BaseClientModel) -> List[Tuple[str, Any]]: + def build_model_df_data(config_map: ClientConfigAdapter) -> List[Tuple[str, Any]]: model_data = [] for traversal_item in config_map.traverse(): attr_printout = ( @@ -143,7 +144,7 @@ def configurable_keys(self # type: HummingbotApplication """ keys = [c.key for c in global_config_map.values() if c.prompt is not None and not c.is_connect_key] if self.strategy_config_map is not None: - if isinstance(self.strategy_config_map, BaseStrategyConfigMap): + if isinstance(self.strategy_config_map, ClientConfigAdapter): keys.extend([ traversal_item.config_path for traversal_item in self.strategy_config_map.traverse() @@ -191,7 +192,7 @@ async def _config_single_key(self, # type: HummingbotApplication if ( key in global_config_map or ( - not isinstance(self.strategy_config_map, (type(None), BaseClientModel)) + not isinstance(self.strategy_config_map, (type(None), ClientConfigAdapter)) and key in self.strategy_config_map ) ): diff --git a/hummingbot/client/command/create_command.py b/hummingbot/client/command/create_command.py index bf5d42627f..b73034003f 100644 --- a/hummingbot/client/command/create_command.py +++ b/hummingbot/client/command/create_command.py @@ -4,9 +4,7 @@ import shutil from typing import Dict, Optional, TYPE_CHECKING -from pydantic import ValidationError - -from hummingbot.client.config.config_data_types import BaseClientModel, BaseStrategyConfigMap +from hummingbot.client.config.config_data_types import BaseStrategyConfigMap from hummingbot.client.config.config_helpers import ( default_strategy_file_path, format_config_file_name, @@ -14,9 +12,10 @@ get_strategy_template_path, parse_config_default_to_text, parse_cvar_value, - retrieve_validation_error_msg, save_to_yml, - save_to_yml_legacy + save_to_yml_legacy, + ClientConfigAdapter, + ConfigValidationError, ) from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.global_config_map import global_config_map @@ -59,11 +58,11 @@ async def prompt_for_configuration( self._notify(f"Please see https://docs.hummingbot.io/strategies/{strategy.replace('_', '-')}/ " f"while setting up these below configuration.") - if isinstance(config_map, BaseStrategyConfigMap): + if isinstance(config_map, ClientConfigAdapter): await self.prompt_for_model_config(config_map) + file_name = await self.save_config_to_file(file_name, config_map) elif config_map is not None: - await self.prompt_for_configuration_legacy(file_name, strategy, config_map) - self.app.to_stop_config = True + file_name = await self.prompt_for_configuration_legacy(file_name, strategy, config_map) else: self.app.to_stop_config = True @@ -71,12 +70,12 @@ async def prompt_for_configuration( self.stop_config() return - file_name = await self.save_config_to_file(file_name, config_map) self.strategy_file_name = file_name self.strategy_name = strategy self.strategy_config_map = config_map # Reload completer here otherwise the new file will not appear self.app.input_field.completer = load_completer(self) + self._notify(f"A new config file {self.strategy_file_name} created.") self.placeholder_mode = False self.app.hide_input = False @@ -86,7 +85,7 @@ async def get_strategy_name( self, # type: HummingbotApplication ) -> Optional[str]: strategy = None - strategy_config = BaseStrategyConfigMap.construct() + strategy_config = ClientConfigAdapter(BaseStrategyConfigMap.construct()) await self.prompt_for_model_config(strategy_config) if self.app.to_stop_config: self.stop_config() @@ -96,13 +95,13 @@ async def get_strategy_name( async def prompt_for_model_config( self, # type: HummingbotApplication - config_map: BaseClientModel, + config_map: ClientConfigAdapter, ): - for key, field in config_map.__fields__.items(): + for key in config_map.keys(): client_data = config_map.get_client_data(key) if ( client_data is not None - and (client_data.prompt_on_new and field.required) + and (client_data.prompt_on_new and config_map.is_required(key)) ): await self.prompt_a_config(config_map, key) if self.app.to_stop_config: @@ -145,27 +144,18 @@ async def prompt_for_configuration_legacy( template = get_strategy_template_path(strategy) shutil.copy(template, strategy_path) save_to_yml_legacy(strategy_path, config_map) - self.strategy_file_name = file_name - self.strategy_name = strategy - self.strategy_config = None - # Reload completer here otherwise the new file will not appear - self.app.input_field.completer = load_completer(self) - self._notify(f"A new config file {self.strategy_file_name} created.") - self.placeholder_mode = False - self.app.hide_input = False - - await self.verify_status() + return file_name async def prompt_a_config( self, # type: HummingbotApplication - model: BaseClientModel, + model: ClientConfigAdapter, config: str, input_value=None, ): config_path = config.split(".") while len(config_path) != 1: sub_model_attr = config_path.pop(0) - model = model.__getattribute__(sub_model_attr) + model = getattr(model, sub_model_attr) config = config_path[0] if input_value is None: prompt = await model.get_client_prompt(config) @@ -177,14 +167,13 @@ async def prompt_a_config( new_config_value = None if not self.app.to_stop_config and input_value is not None: try: - model.__setattr__(config, input_value) - new_config_value = model.__getattribute__(config) - except ValidationError as e: - err_msg = retrieve_validation_error_msg(e) - self._notify(err_msg) + setattr(model, config, input_value) + new_config_value = getattr(model, config) + except ConfigValidationError as e: + self._notify(str(e)) new_config_value = await self.prompt_a_config(model, config) - if not self.app.to_stop_config and isinstance(new_config_value, BaseClientModel): + if not self.app.to_stop_config and isinstance(new_config_value, ClientConfigAdapter): await self.prompt_for_model_config(new_config_value) async def prompt_a_config_legacy( @@ -215,7 +204,7 @@ async def prompt_a_config_legacy( async def save_config_to_file( self, # type: HummingbotApplication file_name: Optional[str], - config_map: BaseStrategyConfigMap, + config_map: ClientConfigAdapter, ) -> str: if file_name is None: file_name = await self.prompt_new_file_name(config_map.strategy) @@ -249,7 +238,7 @@ async def update_all_secure_configs_legacy( ): await Security.wait_til_decryption_done() Security.update_config_map(global_config_map) - if self.strategy_config_map is not None and not isinstance(self.strategy_config_map, BaseStrategyConfigMap): + if self.strategy_config_map is not None and not isinstance(self.strategy_config_map, ClientConfigAdapter): Security.update_config_map(self.strategy_config_map) async def verify_status( diff --git a/hummingbot/client/command/import_command.py b/hummingbot/client/command/import_command.py index 2ac6b9cde6..766e98e175 100644 --- a/hummingbot/client/command/import_command.py +++ b/hummingbot/client/command/import_command.py @@ -4,13 +4,14 @@ from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.config_helpers import ( - update_strategy_config_map_from_file, + load_strategy_config_map_from_file, short_strategy_name, format_config_file_name, validate_strategy_file ) from hummingbot.client.settings import CONF_FILE_PATH, CONF_PREFIX, required_exchanges from typing import TYPE_CHECKING + if TYPE_CHECKING: from hummingbot.client.hummingbot_application import HummingbotApplication @@ -36,9 +37,14 @@ async def import_config_file(self, # type: HummingbotApplication self.app.to_stop_config = False return strategy_path = os.path.join(CONF_FILE_PATH, file_name) - strategy = await update_strategy_config_map_from_file(strategy_path) + config_map = await load_strategy_config_map_from_file(strategy_path) self.strategy_file_name = file_name - self.strategy_name = strategy + self.strategy_name = ( + config_map.strategy + if not isinstance(config_map, dict) + else config_map.get("strategy").value # legacy + ) + self.strategy_config_map = config_map self._notify(f"Configuration from {self.strategy_file_name} file is imported.") self.placeholder_mode = False self.app.hide_input = False @@ -48,6 +54,7 @@ async def import_config_file(self, # type: HummingbotApplication except asyncio.TimeoutError: self.strategy_file_name = None self.strategy_name = None + self.strategy_config_map = None raise if all_status_go: self._notify("\nEnter \"start\" to start market making.") diff --git a/hummingbot/client/command/status_command.py b/hummingbot/client/command/status_command.py index 1d03628f41..4870d60a65 100644 --- a/hummingbot/client/command/status_command.py +++ b/hummingbot/client/command/status_command.py @@ -9,10 +9,7 @@ import inspect from typing import Dict, List -from pydantic import ValidationError, validate_model - from hummingbot import check_dev_mode -from hummingbot.client.config.config_data_types import BaseStrategyConfigMap from hummingbot.logger.application_warning import ApplicationWarning from hummingbot.connector.connector_base import ConnectorBase from hummingbot.core.network_iterator import NetworkStatus @@ -20,6 +17,7 @@ from hummingbot.client.config.config_helpers import ( get_strategy_config_map, missing_required_configs_legacy, + ClientConfigAdapter, ) from hummingbot.client.config.security import Security from hummingbot.user.user_balances import UserBalances @@ -114,25 +112,21 @@ async def validate_required_connections(self) -> Dict[str, str]: invalid_conns["ethereum"] = err_msg return invalid_conns - def missing_configurations_legacy(self) -> List[str]: + def missing_configurations_legacy( + self, # type: HummingbotApplication + ) -> List[str]: missing_globals = missing_required_configs_legacy(global_config_map) config_map = self.strategy_config_map missing_configs = [] - if not isinstance(config_map, BaseStrategyConfigMap): + if not isinstance(config_map, ClientConfigAdapter): missing_configs = missing_required_configs_legacy(get_strategy_config_map(self.strategy_name)) return missing_globals + missing_configs - def validate_configs(self) -> List[ValidationError]: + def validate_configs( + self, # type: HummingbotApplication + ) -> List[str]: config_map = self.strategy_config_map - validation_errors = [] - if isinstance(config_map, BaseStrategyConfigMap): - validation_results = validate_model(type(config_map), config_map.dict()) - if len(validation_results) == 3 and validation_results[2] is not None: - validation_errors = validation_results[2].errors() - validation_errors = [ - f"{'.'.join(e['loc'])} - {e['msg']}" - for e in validation_errors - ] + validation_errors = config_map.validate_model() if isinstance(config_map, ClientConfigAdapter) else [] return validation_errors def status(self, # type: HummingbotApplication @@ -169,6 +163,20 @@ async def status_check_all(self, # type: HummingbotApplication self._notify(' - Security check: Encrypted files are being processed. Please wait and try again later.') return False + missing_configs = self.missing_configurations_legacy() + if missing_configs: + self._notify(" - Strategy check: Incomplete strategy configuration. The following values are missing.") + for config in missing_configs: + self._notify(f" {config.key}") + elif notify_success: + self._notify(' - Strategy check: All required parameters confirmed.') + validation_errors = self.validate_configs() + if len(validation_errors) != 0: + self._notify(" - Strategy check: Validation of the config maps failed. The following errors were flagged.") + for error in validation_errors: + self._notify(f" {error}") + return False + network_timeout = float(global_config_map["other_commands_timeout"].value) try: invalid_conns = await asyncio.wait_for(self.validate_required_connections(), network_timeout) @@ -182,18 +190,6 @@ async def status_check_all(self, # type: HummingbotApplication elif notify_success: self._notify(' - Exchange check: All connections confirmed.') - missing_configs = self.missing_configurations_legacy() - if missing_configs: - self._notify(" - Strategy check: Incomplete strategy configuration. The following values are missing.") - for config in missing_configs: - self._notify(f" {config.key}") - elif notify_success: - self._notify(' - Strategy check: All required parameters confirmed.') - validation_errors = self.validate_configs() - if len(validation_errors) != 0: - self._notify(" - Strategy check: Validation of the config maps failed. The following errors were flagged.") - for error in validation_errors: - self._notify(f" {error}") if invalid_conns or missing_configs or len(validation_errors) != 0: return False diff --git a/hummingbot/client/config/config_data_types.py b/hummingbot/client/config/config_data_types.py index cc351d5371..2c44f727b8 100644 --- a/hummingbot/client/config/config_data_types.py +++ b/hummingbot/client/config/config_data_types.py @@ -1,17 +1,11 @@ -import inspect from dataclasses import dataclass -from datetime import date, datetime, time -from decimal import Decimal from enum import Enum -from typing import Any, Callable, Dict, Generator, List, Optional +from typing import Any, Callable, Dict, Optional -import yaml from pydantic import BaseModel, Field, validator -from pydantic.fields import FieldInfo from pydantic.schema import default_ref_template -from yaml import SafeDumper -from hummingbot.client.config.config_helpers import strategy_config_schema_encoder +from hummingbot.client.config.config_methods import strategy_config_schema_encoder from hummingbot.client.config.config_validators import ( validate_exchange, validate_market_trading_pair, @@ -25,43 +19,6 @@ def __str__(self): return self.value -def decimal_representer(dumper: SafeDumper, data: Decimal): - return dumper.represent_float(float(data)) - - -def enum_representer(dumper: SafeDumper, data: ClientConfigEnum): - return dumper.represent_str(str(data)) - - -def date_representer(dumper: SafeDumper, data: date): - return dumper.represent_date(data) - - -def time_representer(dumper: SafeDumper, data: time): - return dumper.represent_str(data.strftime("%H:%M:%S")) - - -def datetime_representer(dumper: SafeDumper, data: datetime): - return dumper.represent_datetime(data) - - -yaml.add_representer( - data_type=Decimal, representer=decimal_representer, Dumper=SafeDumper -) -yaml.add_multi_representer( - data_type=ClientConfigEnum, multi_representer=enum_representer, Dumper=SafeDumper -) -yaml.add_representer( - data_type=date, representer=date_representer, Dumper=SafeDumper -) -yaml.add_representer( - data_type=time, representer=time_representer, Dumper=SafeDumper -) -yaml.add_representer( - data_type=datetime, representer=datetime_representer, Dumper=SafeDumper -) - - @dataclass() class ClientFieldData: prompt: Optional[Callable[['BaseClientModel'], str]] = None @@ -69,27 +26,12 @@ class ClientFieldData: is_secure: bool = False -@dataclass() -class TraversalItem: - depth: int - config_path: str - attr: str - value: Any - printable_value: str - client_field_data: Optional[ClientFieldData] - field_info: FieldInfo - - class BaseClientModel(BaseModel): class Config: validate_assignment = True title = None + smart_union = True - """ - Notes on configs: - - In nested models, be weary that pydantic will take the first model that fits - (see https://pydantic-docs.helpmanual.io/usage/model_config/#smart-union). - """ @classmethod def schema_json( cls, *, by_alias: bool = True, ref_template: str = default_ref_template, **dumps_kwargs: Any @@ -101,94 +43,8 @@ def schema_json( **dumps_kwargs ) - def traverse(self) -> Generator[TraversalItem, None, None]: - """The intended use for this function is to simplify (validated) config map traversals in the client code.""" - depth = 0 - for attr, field in self.__fields__.items(): - value = self.__getattribute__(attr) - printable_value = str(value) if not isinstance(value, BaseClientModel) else value.Config.title - field_info = field.field_info - client_field_data = field_info.extra.get("client_data") - yield TraversalItem( - depth, attr, attr, value, printable_value, client_field_data, field_info - ) - if isinstance(value, BaseClientModel): - for traversal_item in value.traverse(): - traversal_item.depth += 1 - config_path = f"{attr}.{traversal_item.config_path}" - traversal_item.config_path = config_path - yield traversal_item - - def dict_in_conf_order(self) -> Dict[str, Any]: - d = {} - for attr in self.__fields__.keys(): - value = self.__getattribute__(attr) - if isinstance(value, BaseClientModel): - value = value.dict_in_conf_order() - d[attr] = value - return d - - async def get_client_prompt(self, attr_name: str) -> Optional[str]: - prompt = None - client_data = self.get_client_data(attr_name) - if client_data is not None: - prompt = client_data.prompt - if inspect.iscoroutinefunction(prompt): - prompt = await prompt(self) - else: - prompt = prompt(self) - return prompt - - def is_secure(self, attr_name: str) -> bool: - client_data = self.get_client_data(attr_name) - secure = client_data is not None and client_data.is_secure - return secure - - def get_client_data(self, attr_name: str) -> Optional[ClientFieldData]: - return self.__fields__[attr_name].field_info.extra.get("client_data") - - def get_description(self, attr_name: str) -> str: - return self.__fields__[attr_name].field_info.description - - def generate_yml_output_str_with_comments(self) -> str: - original_fragments = yaml.safe_dump(self.dict_in_conf_order(), sort_keys=False).split("\n") - fragments_with_comments = [self._generate_title()] - self._add_model_fragments(self, fragments_with_comments, original_fragments) - fragments_with_comments.append("\n") # EOF empty line - yml_str = "".join(fragments_with_comments) - return yml_str - - def _generate_title(self) -> str: - title = f"{self.Config.title}" - title = self._adorn_title(title) - return title - - @staticmethod - def _adorn_title(title: str) -> str: - if title: - title = f"### {title} config ###" - title_len = len(title) - title = f"{'#' * title_len}\n{title}\n{'#' * title_len}" - return title - - @staticmethod - def _add_model_fragments( - model: 'BaseClientModel', - fragments_with_comments: List[str], - original_fragments: List[str], - ): - for i, traversal_item in enumerate(model.traverse()): - attr_comment = traversal_item.field_info.description - if attr_comment is not None: - comment_prefix = f"\n{' ' * 2 * traversal_item.depth}# " - attr_comment = "".join(f"{comment_prefix}{c}" for c in attr_comment.split("\n")) - if traversal_item.depth == 0: - attr_comment = f"\n{attr_comment}" - fragments_with_comments.extend([attr_comment, f"\n{original_fragments[i]}"]) - elif traversal_item.depth == 0: - fragments_with_comments.append(f"\n\n{original_fragments[i]}") - else: - fragments_with_comments.append(f"\n{original_fragments[i]}") + def is_required(self, attr: str) -> bool: + return self.__fields__[attr].required class BaseStrategyConfigMap(BaseClientModel): @@ -207,12 +63,6 @@ def validate_strategy(cls, v: str): raise ValueError(ret) return v - def _generate_title(self) -> str: - title = " ".join([w.capitalize() for w in f"{self.strategy}".split("_")]) - title = f"{title} Strategy" - title = self._adorn_title(title) - return title - class BaseTradingStrategyConfigMap(BaseStrategyConfigMap): exchange: ClientConfigEnum( diff --git a/hummingbot/client/config/config_helpers.py b/hummingbot/client/config/config_helpers.py index 5b9ade6674..f24899eef3 100644 --- a/hummingbot/client/config/config_helpers.py +++ b/hummingbot/client/config/config_helpers.py @@ -1,19 +1,36 @@ +import contextlib +import inspect import json import logging import shutil from collections import OrderedDict +from dataclasses import dataclass +from datetime import date, time, datetime from decimal import Decimal from os import listdir, unlink from os.path import isfile, join -from typing import Any, Callable, Dict, List, Optional, TYPE_CHECKING, Union +from typing import ( + Any, + Callable, + Dict, + Generator, + List, + Optional, + Union, +) import ruamel.yaml +import yaml from eth_account import Account from pydantic import ValidationError -from pydantic.json import pydantic_encoder -from pydantic.main import ModelMetaclass +from pydantic.fields import FieldInfo +from pydantic.main import ModelMetaclass, validate_model +from yaml import SafeDumper from hummingbot import get_strategy_list, root_path +from hummingbot.client.config.config_data_types import ( + BaseClientModel, ClientConfigEnum, ClientFieldData +) from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.fee_overrides_config_map import fee_overrides_config_map from hummingbot.client.config.global_config_map import global_config_map @@ -28,11 +45,232 @@ TRADE_FEES_CONFIG_PATH, ) -if TYPE_CHECKING: # avoid circular import problems - from hummingbot.client.config.config_data_types import BaseClientModel, BaseStrategyConfigMap - # Use ruamel.yaml to preserve order and comments in .yml file -yaml_parser = ruamel.yaml.YAML() +yaml_parser = ruamel.yaml.YAML() # legacy + + +def decimal_representer(dumper: SafeDumper, data: Decimal): + return dumper.represent_float(float(data)) + + +def enum_representer(dumper: SafeDumper, data: ClientConfigEnum): + return dumper.represent_str(str(data)) + + +def date_representer(dumper: SafeDumper, data: date): + return dumper.represent_date(data) + + +def time_representer(dumper: SafeDumper, data: time): + return dumper.represent_str(data.strftime("%H:%M:%S")) + + +def datetime_representer(dumper: SafeDumper, data: datetime): + return dumper.represent_datetime(data) + + +yaml.add_representer( + data_type=Decimal, representer=decimal_representer, Dumper=SafeDumper +) +yaml.add_multi_representer( + data_type=ClientConfigEnum, multi_representer=enum_representer, Dumper=SafeDumper +) +yaml.add_representer( + data_type=date, representer=date_representer, Dumper=SafeDumper +) +yaml.add_representer( + data_type=time, representer=time_representer, Dumper=SafeDumper +) +yaml.add_representer( + data_type=datetime, representer=datetime_representer, Dumper=SafeDumper +) + + +class ConfigValidationError(Exception): + pass + + +@dataclass() +class ConfigTraversalItem: + depth: int + config_path: str + attr: str + value: Any + printable_value: str + client_field_data: Optional[ClientFieldData] + field_info: FieldInfo + + +class ClientConfigAdapter: + def __init__(self, hb_config: BaseClientModel): + self._hb_config = hb_config + + def __getattr__(self, item): + if item == "_hb_config": + value = super().__getattribute__(item) + else: + value = getattr(self._hb_config, item) + if isinstance(value, BaseClientModel): + value = ClientConfigAdapter(value) + return value + + def __setattr__(self, key, value): + if key == "_hb_config": + super().__setattr__(key, value) + else: + try: + self._hb_config.__setattr__(key, value) + except ValidationError as e: + raise ConfigValidationError(retrieve_validation_error_msg(e)) + + def __repr__(self): + return self._hb_config.__repr__() + + def __eq__(self, other): + if isinstance(other, ClientConfigAdapter): + eq = self._hb_config.__eq__(other._hb_config) + else: + eq = super().__eq__(other) + return eq + + @property + def hb_config(self) -> BaseClientModel: + return self._hb_config + + @property + def title(self) -> str: + return self._hb_config.Config.title + + def is_required(self, attr: str) -> bool: + return self._hb_config.is_required(attr) + + def keys(self) -> Generator[str, None, None]: + return self._hb_config.__fields__.keys() + + def setattr_no_validation(self, attr: str, value: Any): + with self._disable_validation(): + setattr(self, attr, value) + + def traverse(self) -> Generator[ConfigTraversalItem, None, None]: + """The intended use for this function is to simplify config map traversals in the client code. + + If the field is missing, its value will be set to `None` and its printable value will be set to + 'MISSING_AND_REQUIRED'. + """ + depth = 0 + for attr, field in self._hb_config.__fields__.items(): + if hasattr(self, attr): + value = getattr(self, attr) + printable_value = ( + str(value) if not isinstance(value, ClientConfigAdapter) else value.hb_config.Config.title + ) + field_info = field.field_info + client_field_data = field_info.extra.get("client_data") + else: + value = None + printable_value = "&cMISSING_AND_REQUIRED" + client_field_data = self.get_client_data(attr) + field_info = self._hb_config.__fields__[attr].field_info + yield ConfigTraversalItem( + depth, attr, attr, value, printable_value, client_field_data, field_info + ) + if isinstance(value, ClientConfigAdapter): + for traversal_item in value.traverse(): + traversal_item.depth += 1 + config_path = f"{attr}.{traversal_item.config_path}" + traversal_item.config_path = config_path + yield traversal_item + + async def get_client_prompt(self, attr_name: str) -> Optional[str]: + prompt = None + client_data = self.get_client_data(attr_name) + if client_data is not None: + prompt_fn = client_data.prompt + if inspect.iscoroutinefunction(prompt_fn): + prompt = await prompt_fn(self._hb_config) + else: + prompt = prompt_fn(self._hb_config) + return prompt + + def is_secure(self, attr_name: str) -> bool: + client_data = self.get_client_data(attr_name) + secure = client_data is not None and client_data.is_secure + return secure + + def get_client_data(self, attr_name: str) -> Optional[ClientFieldData]: + return self._hb_config.__fields__[attr_name].field_info.extra.get("client_data") + + def get_description(self, attr_name: str) -> str: + return self._hb_config.__fields__[attr_name].field_info.description + + def generate_yml_output_str_with_comments(self) -> str: + original_fragments = yaml.safe_dump(self._dict_in_conf_order(), sort_keys=False).split("\n") + fragments_with_comments = [self._generate_title()] + self._add_model_fragments(fragments_with_comments, original_fragments) + fragments_with_comments.append("\n") # EOF empty line + yml_str = "".join(fragments_with_comments) + return yml_str + + def validate_model(self) -> List[str]: + results = validate_model(type(self._hb_config), self._hb_config.dict()) + self._hb_config = self._hb_config.__class__.construct() + for key, value in results[0].items(): + self.setattr_no_validation(key, value) + errors = results[2] + validation_errors = [] + if errors is not None: + errors = errors.errors() + validation_errors = [ + f"{'.'.join(e['loc'])} - {e['msg']}" + for e in errors + ] + return validation_errors + + @contextlib.contextmanager + def _disable_validation(self): + self._hb_config.Config.validate_assignment = False + yield + self._hb_config.Config.validate_assignment = True + + def _dict_in_conf_order(self) -> Dict[str, Any]: + d = {} + for attr in self._hb_config.__fields__.keys(): + value = getattr(self, attr) + if isinstance(value, ClientConfigAdapter): + value = value._dict_in_conf_order() + d[attr] = value + return d + + def _generate_title(self) -> str: + title = f"{self._hb_config.Config.title}" + title = self._adorn_title(title) + return title + + @staticmethod + def _adorn_title(title: str) -> str: + if title: + title = f"### {title} config ###" + title_len = len(title) + title = f"{'#' * title_len}\n{title}\n{'#' * title_len}" + return title + + def _add_model_fragments( + self, + fragments_with_comments: List[str], + original_fragments: List[str], + ): + for i, traversal_item in enumerate(self.traverse()): + attr_comment = traversal_item.field_info.description + if attr_comment is not None: + comment_prefix = f"\n{' ' * 2 * traversal_item.depth}# " + attr_comment = "".join(f"{comment_prefix}{c}" for c in attr_comment.split("\n")) + if traversal_item.depth == 0: + attr_comment = f"\n{attr_comment}" + fragments_with_comments.extend([attr_comment, f"\n{original_fragments[i]}"]) + elif traversal_item.depth == 0: + fragments_with_comments.append(f"\n\n{original_fragments[i]}") + else: + fragments_with_comments.append(f"\n{original_fragments[i]}") def parse_cvar_value(cvar: ConfigVar, value: Any) -> Any: @@ -175,20 +413,21 @@ def get_connector_class(connector_name: str) -> Callable: def get_strategy_config_map( strategy: str -) -> Optional[Union["BaseStrategyConfigMap", Dict[str, ConfigVar]]]: +) -> Optional[Union[ClientConfigAdapter, Dict[str, ConfigVar]]]: """ Given the name of a strategy, find and load strategy-specific config map. """ config_map = None try: - config_map = get_strategy_pydantic_config(strategy) - if config_map is None: # legacy + config_cls = get_strategy_pydantic_config_cls(strategy) + if config_cls is None: # legacy cm_key = f"{strategy}_config_map" strategy_module = __import__(f"hummingbot.strategy.{strategy}.{cm_key}", fromlist=[f"hummingbot.strategy.{strategy}"]) config_map = getattr(strategy_module, cm_key) else: - config_map = config_map.construct() + hb_config = config_cls.construct() + config_map = ClientConfigAdapter(hb_config) except Exception as e: logging.getLogger().error(e, exc_info=True) return config_map @@ -209,17 +448,9 @@ def get_strategy_starter_file(strategy: str) -> Callable: logging.getLogger().error(e, exc_info=True) -def load_required_configs(strategy_name) -> OrderedDict: - strategy_config_map = get_strategy_config_map(strategy_name) - # create an ordered dict where `strategy` is inserted first - # so that strategy-specific configs are prompted first and populate required_exchanges - return _merge_dicts(strategy_config_map, global_config_map) - - def strategy_name_from_file(file_path: str) -> str: - with open(file_path) as stream: - data = yaml_parser.load(stream) or {} - strategy = data.get("strategy") + data = read_yml_file(file_path) + strategy = data.get("strategy") return strategy @@ -234,40 +465,13 @@ def validate_strategy_file(file_path: str) -> Optional[str]: return None -def read_yml_file(yml_path: str) -> Dict[str, Any]: # todo: revise +def read_yml_file(yml_path: str) -> Dict[str, Any]: with open(yml_path, "r") as file: - data = yaml_parser.load(file) or {} + data = yaml.safe_load(file) or {} return dict(data) -def load_pydantic_config(strategy_name: str, yml_path: str) -> Optional["BaseClientModel"]: - """ todo: revise - Resolves the pydantic model with the given strategy filepath. Subsequenty load and return the config as a `Model` - - :param strategy_name: Strategy name. - :type strategy_name: str - :param yml_path: Strategy file path. - :type yml_path: str - :return: The strategy configurations. - :rtype: Optional[BaseClientModel] - """ - try: - pydantic_cm_pkg = f"{strategy_name}_config_map_pydantic" - if not isfile(f"{root_path()}/hummingbot/strategy/{strategy_name}/{pydantic_cm_pkg}.py"): - return None - - pydantic_cm_class_name = f"{''.join([s.capitalize() for s in strategy_name.split('_')])}ConfigMap" - pydantic_cm_mod = __import__(f"hummingbot.strategy.{strategy_name}.{pydantic_cm_pkg}", - fromlist=[f"{pydantic_cm_class_name}"]) - pydantic_cm_class = getattr(pydantic_cm_mod, pydantic_cm_class_name) - return pydantic_cm_class(**read_yml_file(yml_path)) - except Exception as e: - logging.getLogger().error(f"Error loading pydantic configs. Your config file may be corrupt. {e}", - exc_info=True) - return None - - -def get_strategy_pydantic_config(strategy_name: str) -> Optional[ModelMetaclass]: +def get_strategy_pydantic_config_cls(strategy_name: str) -> Optional[ModelMetaclass]: pydantic_cm_class = None try: pydantic_cm_pkg = f"{strategy_name}_config_map_pydantic" @@ -281,17 +485,28 @@ def get_strategy_pydantic_config(strategy_name: str) -> Optional[ModelMetaclass] return pydantic_cm_class -async def update_strategy_config_map_from_file(yml_path: str) -> str: # todo: revise +async def load_strategy_config_map_from_file(yml_path: str) -> Union[ClientConfigAdapter, Dict[str, ConfigVar]]: strategy_name = strategy_name_from_file(yml_path) - pydantic_conf: "BaseClientModel" = load_pydantic_config(strategy_name, yml_path) - if pydantic_conf is None: + config_cls = get_strategy_pydantic_config_cls(strategy_name) + if config_cls is None: # legacy config_map = get_strategy_config_map(strategy_name) template_path = get_strategy_template_path(strategy_name) - await load_yml_into_cm(yml_path, template_path, config_map) - return strategy_name + await load_yml_into_cm_legacy(yml_path, template_path, config_map) + else: + config_data = read_yml_file(yml_path) + hb_config = config_cls.construct() + config_map = ClientConfigAdapter(hb_config) + for key in config_map.keys(): + if key in config_data: + config_map.setattr_no_validation(key, config_data[key]) + try: + config_map.validate_model() # try to coerce the values to the appropriate type + except Exception: + pass # but don't raise if it fails + return config_map -async def load_yml_into_cm(yml_path: str, template_file_path: str, cm: Dict[str, ConfigVar]): +async def load_yml_into_cm_legacy(yml_path: str, template_file_path: str, cm: Dict[str, ConfigVar]): try: data = {} conf_version = -1 @@ -352,9 +567,9 @@ async def read_system_configs_from_yml(): Read global config and selected strategy yml files and save the values to corresponding config map If a yml file is outdated, it gets reformatted with the new template """ - await load_yml_into_cm(GLOBAL_CONFIG_PATH, join(TEMPLATE_PATH, "conf_global_TEMPLATE.yml"), global_config_map) - await load_yml_into_cm(TRADE_FEES_CONFIG_PATH, join(TEMPLATE_PATH, "conf_fee_overrides_TEMPLATE.yml"), - fee_overrides_config_map) + await load_yml_into_cm_legacy(GLOBAL_CONFIG_PATH, join(TEMPLATE_PATH, "conf_global_TEMPLATE.yml"), global_config_map) + await load_yml_into_cm_legacy(TRADE_FEES_CONFIG_PATH, join(TEMPLATE_PATH, "conf_fee_overrides_TEMPLATE.yml"), + fee_overrides_config_map) # In case config maps get updated (due to default values) save_system_configs_to_yml() @@ -387,7 +602,7 @@ def save_to_yml_legacy(yml_path: str, cm: Dict[str, ConfigVar]): logging.getLogger().error("Error writing configs: %s" % (str(e),), exc_info=True) -def save_to_yml(yml_path: str, cm: "BaseStrategyConfigMap"): +def save_to_yml(yml_path: str, cm: ClientConfigAdapter): try: cm_yml_str = cm.generate_yml_output_str_with_comments() with open(yml_path, "w+") as outfile: @@ -399,11 +614,14 @@ def save_to_yml(yml_path: str, cm: "BaseStrategyConfigMap"): async def write_config_to_yml(strategy_name, strategy_file_name): strategy_config_map = get_strategy_config_map(strategy_name) strategy_file_path = join(CONF_FILE_PATH, strategy_file_name) - save_to_yml_legacy(strategy_file_path, strategy_config_map) + if isinstance(strategy_config_map, ClientConfigAdapter): + save_to_yml(strategy_file_path, strategy_config_map) + else: + save_to_yml_legacy(strategy_file_path, strategy_config_map) save_to_yml_legacy(GLOBAL_CONFIG_PATH, global_config_map) -async def create_yml_files(): +async def create_yml_files_legacy(): """ Copy `hummingbot_logs.yml` and `conf_global.yml` templates to the `conf` directory on start up """ @@ -470,12 +688,6 @@ def missing_required_configs_legacy(config_map): return [c for c in config_map.values() if c.required and c.value is None and not c.is_connect_key] -def load_all_secure_values(strategy): - strategy_map = get_strategy_config_map(strategy) - load_secure_values(global_config_map) - load_secure_values(strategy_map) - - def load_secure_values(config_map): for key, config in config_map.items(): if config.is_secure: @@ -506,25 +718,5 @@ def parse_config_default_to_text(config: ConfigVar) -> str: return default -def secondary_market_conversion_rate(strategy) -> Decimal: - config_map = get_strategy_config_map(strategy) - if "secondary_to_primary_quote_conversion_rate" in config_map: - base_rate = config_map["secondary_to_primary_base_conversion_rate"].value - quote_rate = config_map["secondary_to_primary_quote_conversion_rate"].value - elif "taker_to_maker_quote_conversion_rate" in config_map: - base_rate = config_map["taker_to_maker_base_conversion_rate"].value - quote_rate = config_map["taker_to_maker_quote_conversion_rate"].value - else: - return Decimal("1") - return quote_rate / base_rate - - def retrieve_validation_error_msg(e: ValidationError) -> str: return e.errors().pop()["msg"] - - -def strategy_config_schema_encoder(o): - if callable(o): - return None - else: - return pydantic_encoder(o) diff --git a/hummingbot/client/config/config_methods.py b/hummingbot/client/config/config_methods.py index d6b879b788..9c7dbc182e 100644 --- a/hummingbot/client/config/config_methods.py +++ b/hummingbot/client/config/config_methods.py @@ -1,3 +1,5 @@ +from pydantic.json import pydantic_encoder + from hummingbot.client.config.config_var import ConfigVar from typing import Callable @@ -12,3 +14,10 @@ def new_fee_config_var(key: str, type_str: str = "decimal"): def using_exchange(exchange: str) -> Callable: from hummingbot.client.settings import required_exchanges return lambda: exchange in required_exchanges + + +def strategy_config_schema_encoder(o): + if callable(o): + return None + else: + return pydantic_encoder(o) diff --git a/hummingbot/client/ui/style.py b/hummingbot/client/ui/style.py index 08814fa2f9..0e84d445ca 100644 --- a/hummingbot/client/ui/style.py +++ b/hummingbot/client/ui/style.py @@ -156,6 +156,7 @@ def hex_to_ansi(color_hex): "&cGREEN": "success-label", "&cYELLOW": "warning-label", "&cRED": "error-label", + "&cMISSING_AND_REQUIRED": "error-label", } default_ui_style = { diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py index 03cd05c80f..69687bc369 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -319,7 +319,7 @@ class AvellanedaMarketMakingConfigMap(BaseTradingStrategyConfigMap): default=SingleOrderLevelModel.construct(), description="Allows activating multi-order levels.", client_data=ClientFieldData( - prompt=lambda mi: f"Select the order levels mode ({'/'.join(list(ORDER_LEVEL_MODELS.keys()))}", + prompt=lambda mi: f"Select the order levels mode ({'/'.join(list(ORDER_LEVEL_MODELS.keys()))})", ), ) order_override: Optional[Dict] = Field( @@ -400,8 +400,7 @@ def validate_order_levels_mode(cls, v: Union[str, SingleOrderLevelModel, MultiOr sub_model = v elif v not in ORDER_LEVEL_MODELS: raise ValueError( - f"Invalid order levels mode, please choose value from" - f" {[e.value for e in list(ORDER_LEVEL_MODELS.keys())]}." + f"Invalid order levels mode, please choose value from {list(ORDER_LEVEL_MODELS.keys())}." ) else: sub_model = ORDER_LEVEL_MODELS[v].construct() @@ -413,8 +412,7 @@ def validate_hanging_orders_mode(cls, v: Union[str, TrackHangingOrdersModel, Ign sub_model = v elif v not in HANGING_ORDER_MODELS: raise ValueError( - f"Invalid hanging order mode, please choose value from" - f" {[e.value for e in list(HANGING_ORDER_MODELS.keys())]}." + f"Invalid hanging order mode, please choose value from {list(HANGING_ORDER_MODELS.keys())}." ) else: sub_model = HANGING_ORDER_MODELS[v].construct() diff --git a/hummingbot/strategy/avellaneda_market_making/start.py b/hummingbot/strategy/avellaneda_market_making/start.py index 6d3b1ec98a..d42fc8ea22 100644 --- a/hummingbot/strategy/avellaneda_market_making/start.py +++ b/hummingbot/strategy/avellaneda_market_making/start.py @@ -9,7 +9,6 @@ import os.path from hummingbot.client.hummingbot_application import HummingbotApplication from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( - AvellanedaMarketMakingConfigMap, DailyBetweenTimesModel, FromDateToDateModel, MultiOrderLevelModel, @@ -27,7 +26,7 @@ def start(self): try: - c_map: AvellanedaMarketMakingConfigMap = self.strategy_config_map + c_map = self.strategy_config_map order_amount = c_map.order_amount order_optimization_enabled = c_map.order_optimization_enabled order_refresh_time = c_map.order_refresh_time @@ -38,14 +37,14 @@ def start(self): c_map.inventory_target_base_pct / Decimal('100') filled_order_delay = c_map.filled_order_delay order_refresh_tolerance_pct = c_map.order_refresh_tolerance_pct / Decimal('100') - if isinstance(c_map.order_levels_mode, MultiOrderLevelModel): + if c_map.order_levels_mode.title == MultiOrderLevelModel.Config.title: order_levels = c_map.order_levels_mode.order_levels level_distances = c_map.order_levels_mode.level_distances else: order_levels = 1 level_distances = 0 order_override = c_map.order_override - if isinstance(c_map.hanging_orders_mode, TrackHangingOrdersModel): + if c_map.hanging_orders_mode.title == TrackHangingOrdersModel.Config.title: hanging_orders_enabled = True hanging_orders_cancel_pct = c_map.hanging_orders_mode.hanging_orders_cancel_pct / Decimal('100') else: @@ -65,11 +64,11 @@ def start(self): order_amount_shape_factor = c_map.order_amount_shape_factor execution_timeframe = c_map.execution_timeframe_mode.Config.title - if isinstance(c_map.execution_timeframe_mode, FromDateToDateModel): + if c_map.execution_timeframe_mode.title == FromDateToDateModel.Config.title: start_time = c_map.execution_timeframe_mode.start_datetime end_time = c_map.execution_timeframe_mode.end_datetime execution_state = RunInTimeConditionalExecutionState(start_timestamp=start_time, end_timestamp=end_time) - elif isinstance(c_map.execution_timeframe_mode, DailyBetweenTimesModel): + elif c_map.execution_timeframe_mode.title == DailyBetweenTimesModel.Config.title: start_time = c_map.execution_timeframe_mode.start_time end_time = c_map.execution_timeframe_mode.end_time execution_state = RunInTimeConditionalExecutionState(start_timestamp=start_time, end_timestamp=end_time) diff --git a/test/hummingbot/client/command/test_config_command.py b/test/hummingbot/client/command/test_config_command.py index dae17bc48f..038091dd00 100644 --- a/test/hummingbot/client/command/test_config_command.py +++ b/test/hummingbot/client/command/test_config_command.py @@ -9,7 +9,8 @@ from hummingbot.client.command.config_command import color_settings_to_display, global_configs_to_display from hummingbot.client.config.config_data_types import BaseClientModel, BaseStrategyConfigMap, ClientFieldData -from hummingbot.client.config.config_helpers import read_system_configs_from_yml +from hummingbot.client.config.config_helpers import \ + read_system_configs_from_yml, ClientConfigAdapter from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.hummingbot_application import HummingbotApplication @@ -141,11 +142,12 @@ class DummyModel(BaseClientModel): some_attr: int = Field(default=1) nested_model: NestedModel = Field(default=NestedModel()) another_attr: Decimal = Field(default=Decimal("1.0")) + missing_no_default: int = Field(default=...) class Config: title = "dummy_model" - get_strategy_config_map_mock.return_value = DummyModel() + get_strategy_config_map_mock.return_value = ClientConfigAdapter(DummyModel.construct()) self.app.list_configs() @@ -154,16 +156,17 @@ class Config: self.assertEqual("\nStrategy Configurations:", captures[4]) df_str_expected = ( - " +------------------------+---------------------+" - "\n | Key | Value |" - "\n |------------------------+---------------------|" - "\n | some_attr | 1 |" - "\n | nested_model | nested_model |" - "\n | ∟ nested_attr | some value |" - "\n | ∟ double_nested_model | double_nested_model |" - "\n | ∟ double_nested_attr | 3.0 |" - "\n | another_attr | 1.0 |" - "\n +------------------------+---------------------+" + " +------------------------+------------------------+" + "\n | Key | Value |" + "\n |------------------------+------------------------|" + "\n | some_attr | 1 |" + "\n | nested_model | nested_model |" + "\n | ∟ nested_attr | some value |" + "\n | ∟ double_nested_model | double_nested_model |" + "\n | ∟ double_nested_attr | 3.0 |" + "\n | another_attr | 1.0 |" + "\n | missing_no_default | &cMISSING_AND_REQUIRED |" + "\n +------------------------+------------------------+" ) self.assertEqual(df_str_expected, captures[5]) @@ -181,7 +184,7 @@ class Config: strategy_name = "some-strategy" self.app.strategy_name = strategy_name - get_strategy_config_map_mock.return_value = DummyModel.construct() + get_strategy_config_map_mock.return_value = ClientConfigAdapter(DummyModel.construct()) self.app.config(key="some_attr") notify_mock.assert_not_called() @@ -218,7 +221,7 @@ class Config: strategy_name = "some-strategy" self.app.strategy_name = strategy_name self.app.strategy_file_name = f"{strategy_name}.yml" - config_map = DummyModel.construct() + config_map = ClientConfigAdapter(DummyModel.construct()) get_strategy_config_map_mock.return_value = config_map self.async_run_with_timeout(self.app._config_single_key(key="some_attr", input_value=2)) diff --git a/test/hummingbot/client/command/test_import_command.py b/test/hummingbot/client/command/test_import_command.py index 6d371ee099..04b8c737af 100644 --- a/test/hummingbot/client/command/test_import_command.py +++ b/test/hummingbot/client/command/test_import_command.py @@ -1,9 +1,18 @@ import asyncio import unittest -from typing import Awaitable +from datetime import datetime, time, date +from decimal import Decimal +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Awaitable, Type from unittest.mock import patch, MagicMock, AsyncMock -from hummingbot.client.config.config_helpers import read_system_configs_from_yml +from pydantic import Field + +from hummingbot.client.command import import_command +from hummingbot.client.config.config_data_types import BaseClientModel, ClientConfigEnum, BaseTradingStrategyConfigMap +from hummingbot.client.config.config_helpers import read_system_configs_from_yml, save_to_yml, ClientConfigAdapter +from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.hummingbot_application import HummingbotApplication from test.mock.mock_cli import CLIMockingAssistant @@ -49,15 +58,62 @@ async def run_coro_that_raises(coro: Awaitable): except asyncio.TimeoutError: # the coroutine did not finish on time raise RuntimeError - @patch("hummingbot.client.command.import_command.update_strategy_config_map_from_file") + @staticmethod + def build_dummy_strategy_config_cls(strategy_name: str) -> Type[BaseClientModel]: + class SomeEnum(ClientConfigEnum): + ONE = "one" + + class DoubleNestedModel(BaseClientModel): + double_nested_attr: datetime = Field( + default=datetime(2022, 1, 1, 10, 30), + description="Double nested attr description" + ) + + class NestedModel(BaseClientModel): + nested_attr: str = Field( + default="some value", + description="Nested attr\nmultiline description", + ) + double_nested_model: DoubleNestedModel = Field( + default=DoubleNestedModel(), + ) + + class DummyModel(BaseTradingStrategyConfigMap): + strategy: str = strategy_name + exchange: str = "binance" + market: str = "BTC-USDT" + some_attr: SomeEnum = Field( + default=SomeEnum.ONE, + description="Some description", + ) + nested_model: NestedModel = Field( + default=NestedModel(), + description="Nested model description", + ) + another_attr: Decimal = Field( + default=Decimal("1.0"), + description="Some other\nmultiline description", + ) + non_nested_no_description: time = Field(default=time(10, 30),) + date_attr: date = Field(default=date(2022, 1, 2)) + no_default: str = Field(default=...) + + class Config: + title = "dummy_model" + + return DummyModel + + @patch("hummingbot.client.command.import_command.load_strategy_config_map_from_file") @patch("hummingbot.client.command.status_command.StatusCommand.status_check_all") - def test_import_config_file_success( - self, status_check_all_mock: AsyncMock, update_strategy_config_map_from_file: AsyncMock + def test_import_config_file_success_legacy( + self, status_check_all_mock: AsyncMock, load_strategy_config_map_from_file: AsyncMock ): - strategy_name = "some-strategy" + strategy_name = "some_strategy" strategy_file_name = f"{strategy_name}.yml" status_check_all_mock.return_value = True - update_strategy_config_map_from_file.return_value = strategy_name + strategy_conf_var = ConfigVar("strategy", None) + strategy_conf_var.value = strategy_name + load_strategy_config_map_from_file.return_value = {"strategy": strategy_conf_var} self.async_run_with_timeout(self.app.import_config_file(strategy_file_name)) self.assertEqual(strategy_file_name, self.app.strategy_file_name) @@ -66,15 +122,17 @@ def test_import_config_file_success( self.cli_mock_assistant.check_log_called_with("\nEnter \"start\" to start market making.") ) - @patch("hummingbot.client.command.import_command.update_strategy_config_map_from_file") + @patch("hummingbot.client.command.import_command.load_strategy_config_map_from_file") @patch("hummingbot.client.command.status_command.StatusCommand.status_check_all") - def test_import_config_file_handles_network_timeouts( - self, status_check_all_mock: AsyncMock, update_strategy_config_map_from_file: AsyncMock + def test_import_config_file_handles_network_timeouts_legacy( + self, status_check_all_mock: AsyncMock, load_strategy_config_map_from_file: AsyncMock ): - strategy_name = "some-strategy" + strategy_name = "some_strategy" strategy_file_name = f"{strategy_name}.yml" status_check_all_mock.side_effect = self.raise_timeout - update_strategy_config_map_from_file.return_value = strategy_name + strategy_conf_var = ConfigVar("strategy", None) + strategy_conf_var.value = strategy_name + load_strategy_config_map_from_file.return_value = {"strategy": strategy_conf_var} with self.assertRaises(asyncio.TimeoutError): self.async_run_with_timeout_coroutine_must_raise_timeout( @@ -82,3 +140,63 @@ def test_import_config_file_handles_network_timeouts( ) self.assertEqual(None, self.app.strategy_file_name) self.assertEqual(None, self.app.strategy_name) + + @patch("hummingbot.client.config.config_helpers.get_strategy_pydantic_config_cls") + @patch("hummingbot.client.command.status_command.StatusCommand.status_check_all") + def test_import_config_file_success( + self, status_check_all_mock: AsyncMock, get_strategy_pydantic_config_cls: MagicMock + ): + strategy_name = "perpetual_market_making" + strategy_file_name = f"{strategy_name}.yml" + status_check_all_mock.return_value = True + dummy_strategy_config_cls = self.build_dummy_strategy_config_cls(strategy_name) + get_strategy_pydantic_config_cls.return_value = dummy_strategy_config_cls + cm = ClientConfigAdapter(dummy_strategy_config_cls(no_default="some value")) + + with TemporaryDirectory() as d: + import_command.CONF_FILE_PATH = str(d) + d = Path(d) + temp_file_name = d / strategy_file_name + save_to_yml(str(temp_file_name), cm) + self.async_run_with_timeout(self.app.import_config_file(strategy_file_name)) + + self.assertEqual(strategy_file_name, self.app.strategy_file_name) + self.assertEqual(strategy_name, self.app.strategy_name) + self.assertTrue( + self.cli_mock_assistant.check_log_called_with("\nEnter \"start\" to start market making.") + ) + self.assertEqual(cm, self.app.strategy_config_map) + + @patch("hummingbot.client.config.config_helpers.get_strategy_pydantic_config_cls") + @patch("hummingbot.client.command.status_command.StatusCommand.status_check_all") + def test_import_incomplete_config_file_success( + self, status_check_all_mock: AsyncMock, get_strategy_pydantic_config_cls: MagicMock + ): + strategy_name = "perpetual_market_making" + strategy_file_name = f"{strategy_name}.yml" + status_check_all_mock.return_value = True + dummy_strategy_config_cls = self.build_dummy_strategy_config_cls(strategy_name) + get_strategy_pydantic_config_cls.return_value = dummy_strategy_config_cls + cm = ClientConfigAdapter(dummy_strategy_config_cls(no_default="some value")) + + with TemporaryDirectory() as d: + import_command.CONF_FILE_PATH = str(d) + d = Path(d) + temp_file_name = d / strategy_file_name + cm_yml_str = cm.generate_yml_output_str_with_comments() + cm_yml_str = cm_yml_str.replace("\nno_default: some value\n", "") + with open(temp_file_name, "w+") as outfile: + outfile.write(cm_yml_str) + self.async_run_with_timeout(self.app.import_config_file(strategy_file_name)) + + self.assertEqual(strategy_file_name, self.app.strategy_file_name) + self.assertEqual(strategy_name, self.app.strategy_name) + self.assertTrue( + self.cli_mock_assistant.check_log_called_with("\nEnter \"start\" to start market making.") + ) + self.assertNotEqual(cm, self.app.strategy_config_map) + + validation_errors = self.app.strategy_config_map.validate_model() + + self.assertEqual(1, len(validation_errors)) + self.assertEqual("no_default - field required", validation_errors[0]) diff --git a/test/hummingbot/client/command/test_status_command.py b/test/hummingbot/client/command/test_status_command.py index 2dc41f38fe..105db95916 100644 --- a/test/hummingbot/client/command/test_status_command.py +++ b/test/hummingbot/client/command/test_status_command.py @@ -59,15 +59,17 @@ async def run_coro_that_raises(coro: Awaitable): except asyncio.TimeoutError: # the coroutine did not finish on time raise RuntimeError + @patch("hummingbot.client.command.status_command.StatusCommand.validate_configs") @patch("hummingbot.client.command.status_command.StatusCommand.validate_required_connections") @patch("hummingbot.client.config.security.Security.is_decryption_done") def test_status_check_all_handles_network_timeouts( - self, is_decryption_done_mock, validate_required_connections_mock + self, is_decryption_done_mock, validate_required_connections_mock, validate_configs_mock ): validate_required_connections_mock.side_effect = self.get_async_sleep_fn(delay=0.02) + validate_configs_mock.return_value = [] global_config_map["other_commands_timeout"].value = 0.01 is_decryption_done_mock.return_value = True - strategy_name = "some-strategy" + strategy_name = "avellaneda_market_making" self.app.strategy_name = strategy_name self.app.strategy_file_name = f"{strategy_name}.yml" diff --git a/test/hummingbot/client/config/test_config_data_types.py b/test/hummingbot/client/config/test_config_data_types.py index 5d80a3449c..21c96d558f 100644 --- a/test/hummingbot/client/config/test_config_data_types.py +++ b/test/hummingbot/client/config/test_config_data_types.py @@ -6,7 +6,7 @@ from typing import Awaitable, Dict from unittest.mock import patch -from pydantic import Field, ValidationError +from pydantic import Field from pydantic.fields import FieldInfo from hummingbot.client.config.config_data_types import ( @@ -15,9 +15,10 @@ BaseTradingStrategyConfigMap, ClientConfigEnum, ClientFieldData, - TraversalItem, ) -from hummingbot.client.config.config_helpers import retrieve_validation_error_msg +from hummingbot.client.config.config_helpers import ( + ConfigTraversalItem, ClientConfigAdapter, ConfigValidationError +) class BaseClientModelTest(unittest.TestCase): @@ -63,23 +64,25 @@ class Config: title = "dummy_model" expected = [ - TraversalItem(0, "some_attr", "some_attr", 1, "1", ClientFieldData(), None), - TraversalItem(0, "nested_model", "nested_model", NestedModel(), "nested_model", None, None), - TraversalItem(1, "nested_model.nested_attr", "nested_attr", "some value", "some value", None, None), - TraversalItem( + ConfigTraversalItem(0, "some_attr", "some_attr", 1, "1", ClientFieldData(), None), + ConfigTraversalItem( + 0, "nested_model", "nested_model", ClientConfigAdapter(NestedModel()), "nested_model", None, None + ), + ConfigTraversalItem(1, "nested_model.nested_attr", "nested_attr", "some value", "some value", None, None), + ConfigTraversalItem( 1, "nested_model.double_nested_model", "double_nested_model", - DoubleNestedModel(), + ClientConfigAdapter(DoubleNestedModel()), "double_nested_model", None, None, ), - TraversalItem( + ConfigTraversalItem( 2, "nested_model.double_nested_model.double_nested_attr", "double_nested_attr", 3.0, "3.0", None, None ), ] - cm = DummyModel() + cm = ClientConfigAdapter(DummyModel()) for expected, actual in zip(expected, cm.traverse()): self.assertEqual(expected.depth, actual.depth) @@ -128,7 +131,7 @@ class DummyModel(BaseClientModel): class Config: title = "dummy_model" - instance = DummyModel() + instance = ClientConfigAdapter(DummyModel()) res_str = instance.generate_yml_output_str_with_comments() expected_str = """\ @@ -162,13 +165,19 @@ class Config: class BaseStrategyConfigMapTest(unittest.TestCase): def test_generate_yml_output_dict_title(self): - instance = BaseStrategyConfigMap(strategy="pure_market_making") + class DummyStrategy(BaseStrategyConfigMap): + class Config: + title = "pure_market_making" + + strategy: str = "pure_market_making" + + instance = ClientConfigAdapter(DummyStrategy()) res_str = instance.generate_yml_output_str_with_comments() expected_str = """\ -############################################## -### Pure Market Making Strategy config ### -############################################## +##################################### +### pure_market_making config ### +##################################### strategy: pure_market_making """ @@ -189,7 +198,7 @@ def setUpClass(cls) -> None: def setUp(self) -> None: super().setUp() config_settings = self.get_default_map() - self.config_map = BaseTradingStrategyConfigMap(**config_settings) + self.config_map = ClientConfigAdapter(BaseTradingStrategyConfigMap(**config_settings)) def async_run_with_timeout(self, coroutine: Awaitable, timeout: int = 1): ret = self.ev_loop.run_until_complete(asyncio.wait_for(coroutine, timeout)) @@ -207,12 +216,11 @@ def get_default_map(self) -> Dict[str, str]: "hummingbot.client.config.config_data_types.validate_market_trading_pair" ) def test_validators(self, validate_market_trading_pair_mock): - with self.assertRaises(ValidationError) as e: + with self.assertRaises(ConfigValidationError) as e: self.config_map.exchange = "test-exchange" error_msg = "Invalid exchange, please choose value from " - actual_msg = retrieve_validation_error_msg(e.exception) - self.assertTrue(actual_msg.startswith(error_msg)) + self.assertTrue(str(e.exception).startswith(error_msg)) alt_pair = "ETH-USDT" error_msg = "Failed" @@ -223,8 +231,7 @@ def test_validators(self, validate_market_trading_pair_mock): self.config_map.market = alt_pair self.assertEqual(alt_pair, self.config_map.market) - with self.assertRaises(ValidationError) as e: + with self.assertRaises(ConfigValidationError) as e: self.config_map.market = "XXX-USDT" - actual_msg = retrieve_validation_error_msg(e.exception) - self.assertTrue(actual_msg.startswith(error_msg)) + self.assertTrue(str(e.exception).startswith(error_msg)) diff --git a/test/hummingbot/client/config/test_config_helpers.py b/test/hummingbot/client/config/test_config_helpers.py index 553799d9a8..beef381242 100644 --- a/test/hummingbot/client/config/test_config_helpers.py +++ b/test/hummingbot/client/config/test_config_helpers.py @@ -5,7 +5,7 @@ from typing import Awaitable from hummingbot.client.config.config_data_types import BaseStrategyConfigMap -from hummingbot.client.config.config_helpers import get_strategy_config_map, save_to_yml +from hummingbot.client.config.config_helpers import get_strategy_config_map, save_to_yml, ClientConfigAdapter from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( AvellanedaMarketMakingConfigMap ) @@ -28,15 +28,21 @@ async def async_sleep(*_, **__): def test_get_strategy_config_map(self): cm = get_strategy_config_map(strategy="avellaneda_market_making") - self.assertIsInstance(cm, AvellanedaMarketMakingConfigMap) + self.assertIsInstance(cm.hb_config, AvellanedaMarketMakingConfigMap) self.assertFalse(hasattr(cm, "market")) # uninitialized instance def test_save_to_yml(self): - cm = BaseStrategyConfigMap(strategy="pure_market_making") + class DummyStrategy(BaseStrategyConfigMap): + class Config: + title = "pure_market_making" + + strategy: str = "pure_market_making" + + cm = ClientConfigAdapter(DummyStrategy()) expected_str = """\ -############################################## -### Pure Market Making Strategy config ### -############################################## +##################################### +### pure_market_making config ### +##################################### strategy: pure_market_making """ diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map.py deleted file mode 100644 index 905f1d4e04..0000000000 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map.py +++ /dev/null @@ -1,136 +0,0 @@ -import unittest -from copy import deepcopy - -from hummingbot.client.settings import AllConnectorSettings -from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map import ( - avellaneda_market_making_config_map, - maker_trading_pair_prompt, - order_amount_prompt, - execution_time_start_prompt, - execution_time_end_prompt, - validate_exchange_trading_pair, - validate_execution_timeframe, - validate_execution_time, - on_validated_execution_timeframe -) - - -class AvellanedaMarketMakingConfigMapTest(unittest.TestCase): - @classmethod - def setUpClass(cls) -> None: - super().setUpClass() - cls.base_asset = "COINALPHA" - cls.quote_asset = "HBOT" - cls.trading_pair = f"{cls.base_asset}-{cls.quote_asset}" - - def setUp(self) -> None: - super().setUp() - self.config_backup = deepcopy(avellaneda_market_making_config_map) - - def tearDown(self) -> None: - self.reset_config_map() - super().tearDown() - - def reset_config_map(self): - for key, value in self.config_backup.items(): - avellaneda_market_making_config_map[key] = value - - def test_order_amount_prompt(self): - avellaneda_market_making_config_map["market"].value = self.trading_pair - prompt = order_amount_prompt() - expected = f"What is the amount of {self.base_asset} per order? >>> " - - self.assertEqual(expected, prompt) - - def test_maker_trading_pair_prompt(self): - exchange = avellaneda_market_making_config_map["exchange"].value = "binance" - example = AllConnectorSettings.get_example_pairs().get(exchange) - - prompt = maker_trading_pair_prompt() - expected = f"Enter the token trading pair you would like to trade on {exchange} (e.g. {example}) >>> " - - self.assertEqual(expected, prompt) - - def test_execution_time_prompts(self): - avellaneda_market_making_config_map["execution_timeframe"].value = "from_date_to_date" - prompt = execution_time_start_prompt() - expected = "Please enter the start date and time (YYYY-MM-DD HH:MM:SS) >>> " - self.assertEqual(expected, prompt) - - avellaneda_market_making_config_map["execution_timeframe"].value = "daily_between_times" - prompt = execution_time_start_prompt() - expected = "Please enter the start time (HH:MM:SS) >>> " - self.assertEqual(expected, prompt) - - avellaneda_market_making_config_map["execution_timeframe"].value = "from_date_to_date" - prompt = execution_time_end_prompt() - expected = "Please enter the end date and time (YYYY-MM-DD HH:MM:SS) >>> " - self.assertEqual(expected, prompt) - - avellaneda_market_making_config_map["execution_timeframe"].value = "daily_between_times" - prompt = execution_time_end_prompt() - expected = "Please enter the end time (HH:MM:SS) >>> " - self.assertEqual(expected, prompt) - - def test_validators(self): - avellaneda_market_making_config_map["exchange"].value = "binance" - value = validate_exchange_trading_pair("ETH-USDT") - self.assertIsNone(value) - - value = validate_exchange_trading_pair("XXX-USDT") - self.assertFalse(value) - - value = validate_execution_timeframe("infinite") - self.assertIsNone(value) - - value = validate_execution_timeframe("from_date_to_date") - self.assertIsNone(value) - - value = validate_execution_timeframe("daily_between_times") - self.assertIsNone(value) - - value = validate_execution_timeframe("XXX") - expected = "Invalid timeframe, please choose value from ['infinite', 'from_date_to_date', 'daily_between_times']" - self.assertEqual(expected, value) - - avellaneda_market_making_config_map["execution_timeframe"].value = "from_date_to_date" - - value = validate_execution_time("2021-01-01 12:00:00") - self.assertIsNone(value) - - value = validate_execution_time("2021-01-01 30:00:00") - expected = "Incorrect date time format (expected is YYYY-MM-DD HH:MM:SS)" - self.assertEqual(expected, value) - - value = validate_execution_time("12:00:00") - expected = "Incorrect date time format (expected is YYYY-MM-DD HH:MM:SS)" - self.assertEqual(expected, value) - - avellaneda_market_making_config_map["execution_timeframe"].value = "daily_between_times" - - value = validate_execution_time("12:00:00") - self.assertIsNone(value) - - value = validate_execution_time("30:00:00") - expected = "Incorrect time format (expected is HH:MM:SS)" - self.assertEqual(expected, value) - - value = validate_execution_time("2021-01-01 12:00:00") - expected = "Incorrect time format (expected is HH:MM:SS)" - self.assertEqual(expected, value) - - avellaneda_market_making_config_map["execution_timeframe"].value = "infinite" - - value = validate_execution_time("12:00:00") - self.assertIsNone(value) - - avellaneda_market_making_config_map["start_time"].value = "12:00:00" - avellaneda_market_making_config_map["end_time"].value = "13:00:00" - - on_validated_execution_timeframe("") - - value = avellaneda_market_making_config_map["start_time"].value - self.assertIsNone(value) - - value = avellaneda_market_making_config_map["end_time"].value - self.assertIsNone(value) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py index 4460b7959e..a98370435d 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py @@ -6,10 +6,9 @@ from unittest.mock import patch import yaml -from pydantic import ValidationError, validate_model +from pydantic import validate_model -from hummingbot.client.config.config_data_types import BaseClientModel -from hummingbot.client.config.config_helpers import retrieve_validation_error_msg +from hummingbot.client.config.config_helpers import ClientConfigAdapter, ConfigValidationError from hummingbot.client.settings import AllConnectorSettings from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( AvellanedaMarketMakingConfigMap, @@ -32,7 +31,7 @@ def setUpClass(cls) -> None: def setUp(self) -> None: super().setUp() config_settings = self.get_default_map() - self.config_map = AvellanedaMarketMakingConfigMap(**config_settings) + self.config_map = ClientConfigAdapter(AvellanedaMarketMakingConfigMap(**config_settings)) def async_run_with_timeout(self, coroutine: Awaitable, timeout: int = 1): ret = self.ev_loop.run_until_complete(asyncio.wait_for(coroutine, timeout)) @@ -55,25 +54,27 @@ def get_default_map(self) -> Dict[str, str]: return config_settings def test_initial_sequential_build(self): - config_map: AvellanedaMarketMakingConfigMap = AvellanedaMarketMakingConfigMap.construct() + config_map = ClientConfigAdapter(AvellanedaMarketMakingConfigMap.construct()) config_settings = self.get_default_map() - def build_config_map(cm: BaseClientModel, cs: Dict): + def build_config_map(cm: ClientConfigAdapter, cs: Dict): """This routine can be used in the create command, with slight modifications.""" - for key, field in cm.__fields__.items(): + for key in cm.keys(): client_data = cm.get_client_data(key) if client_data is not None and client_data.prompt_on_new: self.assertIsInstance(client_data.prompt(cm), str) if key == "execution_timeframe_model": - cm.__setattr__(key, "daily_between_times") # simulate user input + setattr(cm, key, "daily_between_times") # simulate user input else: - cm.__setattr__(key, cs[key]) - new_value = cm.__getattribute__(key) - if isinstance(new_value, BaseClientModel): + setattr(cm, key, cs[key]) + new_value = getattr(cm, key) + if isinstance(new_value, ClientConfigAdapter): build_config_map(new_value, cs[key]) build_config_map(config_map, config_settings) - validate_model(config_map.__class__, config_map.__dict__) + hb_config = config_map.hb_config + validate_model(hb_config.__class__, hb_config.__dict__) + self.assertEqual(0, len(config_map.validate_model())) def test_order_amount_prompt(self): prompt = self.async_run_with_timeout(self.config_map.get_client_prompt("order_amount")) @@ -112,24 +113,23 @@ def test_execution_time_prompts(self): @patch( "hummingbot.client.config.config_data_types.validate_market_trading_pair" ) - def test_validators(self, validate_market_trading_pair_mock): + def test_validators(self, _): self.config_map.execution_timeframe_mode = "infinite" - self.assertIsInstance(self.config_map.execution_timeframe_mode, InfiniteModel) + self.assertIsInstance(self.config_map.execution_timeframe_mode.hb_config, InfiniteModel) self.config_map.execution_timeframe_mode = "from_date_to_date" - self.assertIsInstance(self.config_map.execution_timeframe_mode, FromDateToDateModel) + self.assertIsInstance(self.config_map.execution_timeframe_mode.hb_config, FromDateToDateModel) self.config_map.execution_timeframe_mode = "daily_between_times" - self.assertIsInstance(self.config_map.execution_timeframe_mode, DailyBetweenTimesModel) + self.assertIsInstance(self.config_map.execution_timeframe_mode.hb_config, DailyBetweenTimesModel) - with self.assertRaises(ValidationError) as e: + with self.assertRaises(ConfigValidationError) as e: self.config_map.execution_timeframe_mode = "XXX" error_msg = ( "Invalid timeframe, please choose value from ['infinite', 'from_date_to_date', 'daily_between_times']" ) - actual_msg = retrieve_validation_error_msg(e.exception) - self.assertEqual(error_msg, actual_msg) + self.assertEqual(error_msg, str(e.exception)) self.config_map.execution_timeframe_mode = "from_date_to_date" model = self.config_map.execution_timeframe_mode @@ -139,19 +139,17 @@ def test_validators(self, validate_market_trading_pair_mock): self.assertEqual(datetime(2021, 1, 1, 12, 0, 0), model.start_datetime) self.assertEqual(datetime(2021, 1, 1, 15, 0, 0), model.end_datetime) - with self.assertRaises(ValidationError) as e: + with self.assertRaises(ConfigValidationError) as e: model.start_datetime = "2021-01-01 30:00:00" error_msg = "Incorrect date time format (expected is YYYY-MM-DD HH:MM:SS)" - actual_msg = retrieve_validation_error_msg(e.exception) - self.assertEqual(error_msg, actual_msg) + self.assertEqual(error_msg, str(e.exception)) - with self.assertRaises(ValidationError) as e: + with self.assertRaises(ConfigValidationError) as e: model.start_datetime = "12:00:00" error_msg = "Incorrect date time format (expected is YYYY-MM-DD HH:MM:SS)" - actual_msg = retrieve_validation_error_msg(e.exception) - self.assertEqual(error_msg, actual_msg) + self.assertEqual(error_msg, str(e.exception)) self.config_map.execution_timeframe_mode = "daily_between_times" model = self.config_map.execution_timeframe_mode @@ -159,29 +157,26 @@ def test_validators(self, validate_market_trading_pair_mock): self.assertEqual(time(12, 0, 0), model.start_time) - with self.assertRaises(ValidationError) as e: + with self.assertRaises(ConfigValidationError) as e: model.start_time = "30:00:00" error_msg = "Incorrect time format (expected is HH:MM:SS)" - actual_msg = retrieve_validation_error_msg(e.exception) - self.assertEqual(error_msg, actual_msg) + self.assertEqual(error_msg, str(e.exception)) - with self.assertRaises(ValidationError) as e: + with self.assertRaises(ConfigValidationError) as e: model.start_time = "2021-01-01 12:00:00" error_msg = "Incorrect time format (expected is HH:MM:SS)" - actual_msg = retrieve_validation_error_msg(e.exception) - self.assertEqual(error_msg, actual_msg) + self.assertEqual(error_msg, str(e.exception)) self.config_map.order_levels_mode = "multi_order_level" model = self.config_map.order_levels_mode - with self.assertRaises(ValidationError) as e: + with self.assertRaises(ConfigValidationError) as e: model.order_levels = 1 error_msg = "Value cannot be less than 2." - actual_msg = retrieve_validation_error_msg(e.exception) - self.assertEqual(error_msg, actual_msg) + self.assertEqual(error_msg, str(e.exception)) model.order_levels = 3 self.assertEqual(3, model.order_levels) @@ -189,12 +184,11 @@ def test_validators(self, validate_market_trading_pair_mock): self.config_map.hanging_orders_mode = "track_hanging_orders" model = self.config_map.hanging_orders_mode - with self.assertRaises(ValidationError) as e: + with self.assertRaises(ConfigValidationError) as e: model.hanging_orders_cancel_pct = "-1" error_msg = "Value must be between 0 and 100 (exclusive)." - actual_msg = retrieve_validation_error_msg(e.exception) - self.assertEqual(error_msg, actual_msg) + self.assertEqual(error_msg, str(e.exception)) model.hanging_orders_cancel_pct = "3" self.assertEqual(3, model.hanging_orders_cancel_pct) @@ -206,6 +200,6 @@ def test_load_configs_from_yaml(self): with open(f_path, "r") as file: data = yaml.safe_load(file) - loaded_config_map = AvellanedaMarketMakingConfigMap(**data) + loaded_config_map = ClientConfigAdapter(AvellanedaMarketMakingConfigMap(**data)) self.assertEqual(self.config_map, loaded_config_map) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py index 96e6bc1f8c..b7c97fa9b2 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py @@ -3,6 +3,7 @@ from decimal import Decimal import unittest.mock import hummingbot.strategy.avellaneda_market_making.start as strategy_start +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange_base import ExchangeBase from hummingbot.connector.utils import combine_to_hb_trading_pair from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( @@ -25,27 +26,29 @@ def setUp(self) -> None: self.log_records = [] self.base = "ETH" self.quote = "BTC" - self.strategy_config_map = AvellanedaMarketMakingConfigMap( - exchange="binance", - market=combine_to_hb_trading_pair(self.base, self.quote), - execution_timeframe_model=FromDateToDateModel( - start_datetime="2021-11-18 15:00:00", - end_datetime="2021-11-18 16:00:00", - ), - order_amount=60, - order_refresh_time=60, - hanging_orders_model=TrackHangingOrdersModel( - hanging_orders_cancel_pct=1, - ), - order_levels_mode=MultiOrderLevelModel( + self.strategy_config_map = ClientConfigAdapter( + AvellanedaMarketMakingConfigMap( + exchange="binance", + market=combine_to_hb_trading_pair(self.base, self.quote), + execution_timeframe_mode=FromDateToDateModel( + start_datetime="2021-11-18 15:00:00", + end_datetime="2021-11-18 16:00:00", + ), + order_amount=60, + order_refresh_time=60, + hanging_orders_model=TrackHangingOrdersModel( + hanging_orders_cancel_pct=1, + ), + order_levels_mode=MultiOrderLevelModel( + order_levels=4, + level_distances=1, + ), + min_spread=2, + risk_factor=1.11, order_levels=4, level_distances=1, - ), - min_spread=2, - risk_factor=1.11, - order_levels=4, - level_distances=1, - order_amount_shape_factor=0.33, + order_amount_shape_factor=0.33, + ) ) self.raise_exception_for_market_initialization = False @@ -72,7 +75,7 @@ def handle(self, record): @unittest.mock.patch('hummingbot.strategy.avellaneda_market_making.start.HummingbotApplication') def test_parameters_strategy_creation(self, mock_hbot): - mock_hbot.main_application().strategy_file_name = "test.csv" + mock_hbot.main_application().strategy_file_name = "test.yml" strategy_start.start(self) self.assertEqual(self.strategy.execution_timeframe, "from_date_to_date") self.assertEqual(self.strategy.start_time, datetime.datetime(2021, 11, 18, 15, 0)) From 254379ec2ec09d4a42a658ba3a49f3911a97a993 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Mon, 4 Apr 2022 15:22:18 +0300 Subject: [PATCH 033/152] (fix) Ensures that AMM config can convert between modes successfully. --- hummingbot/client/config/config_data_types.py | 7 +- hummingbot/client/config/config_helpers.py | 4 +- ...aneda_market_making_config_map_pydantic.py | 23 ++----- .../avellaneda_market_making/start.py | 2 +- ...aneda_market_making_config_map_pydantic.py | 64 ++++++++++++++++++- .../test_avellaneda_market_making_start.py | 4 +- 6 files changed, 77 insertions(+), 27 deletions(-) diff --git a/hummingbot/client/config/config_data_types.py b/hummingbot/client/config/config_data_types.py index 2c44f727b8..ead513a1e5 100644 --- a/hummingbot/client/config/config_data_types.py +++ b/hummingbot/client/config/config_data_types.py @@ -1,8 +1,9 @@ from dataclasses import dataclass +from datetime import datetime from enum import Enum from typing import Any, Callable, Dict, Optional -from pydantic import BaseModel, Field, validator +from pydantic import BaseModel, Field, validator, Extra from pydantic.schema import default_ref_template from hummingbot.client.config.config_methods import strategy_config_schema_encoder @@ -31,6 +32,10 @@ class Config: validate_assignment = True title = None smart_union = True + extra = Extra.forbid + json_encoders = { + datetime: lambda dt: dt.strftime("%Y-%m-%d %H:%M:%S"), + } @classmethod def schema_json( diff --git a/hummingbot/client/config/config_helpers.py b/hummingbot/client/config/config_helpers.py index f24899eef3..b75fe8687d 100644 --- a/hummingbot/client/config/config_helpers.py +++ b/hummingbot/client/config/config_helpers.py @@ -124,7 +124,7 @@ def __setattr__(self, key, value): raise ConfigValidationError(retrieve_validation_error_msg(e)) def __repr__(self): - return self._hb_config.__repr__() + return f"{self.__class__.__name__}.{self._hb_config.__repr__()}" def __eq__(self, other): if isinstance(other, ClientConfigAdapter): @@ -212,7 +212,7 @@ def generate_yml_output_str_with_comments(self) -> str: return yml_str def validate_model(self) -> List[str]: - results = validate_model(type(self._hb_config), self._hb_config.dict()) + results = validate_model(type(self._hb_config), json.loads(self._hb_config.json())) self._hb_config = self._hb_config.__class__.construct() for key, value in results[0].items(): self.setattr_no_validation(key, value) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py index 69687bc369..122c825dd0 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -23,7 +23,6 @@ class InfiniteModel(BaseClientModel): class Config: title = "infinite" - validate_assignment = True class FromDateToDateModel(BaseClientModel): @@ -46,7 +45,6 @@ class FromDateToDateModel(BaseClientModel): class Config: title = "from_date_to_date" - validate_assignment = True @validator("start_datetime", "end_datetime", pre=True) def validate_execution_time(cls, v: str) -> Optional[str]: @@ -76,7 +74,6 @@ class DailyBetweenTimesModel(BaseClientModel): class Config: title = "daily_between_times" - validate_assignment = True @validator("start_time", "end_time", pre=True) def validate_execution_time(cls, v: str) -> Optional[str]: @@ -96,7 +93,6 @@ def validate_execution_time(cls, v: str) -> Optional[str]: class SingleOrderLevelModel(BaseClientModel): class Config: title = "single_order_level" - validate_assignment = True class MultiOrderLevelModel(BaseClientModel): @@ -119,7 +115,6 @@ class MultiOrderLevelModel(BaseClientModel): class Config: title = "multi_order_level" - validate_assignment = True @validator("order_levels", pre=True) def validate_int_zero_or_above(cls, v: str): @@ -158,7 +153,6 @@ class TrackHangingOrdersModel(BaseClientModel): class Config: title = "track_hanging_orders" - validate_assignment = True @validator("hanging_orders_cancel_pct", pre=True) def validate_pct_exclusive(cls, v: str): @@ -182,7 +176,7 @@ class Config: class AvellanedaMarketMakingConfigMap(BaseTradingStrategyConfigMap): strategy: str = Field(default="avellaneda_market_making", client_data=None) - execution_timeframe_mode: Union[FromDateToDateModel, DailyBetweenTimesModel, InfiniteModel] = Field( + execution_timeframe_mode: Union[InfiniteModel, FromDateToDateModel, DailyBetweenTimesModel] = Field( default=..., description="The execution timeframe.", client_data=ClientFieldData( @@ -315,7 +309,7 @@ class AvellanedaMarketMakingConfigMap(BaseTradingStrategyConfigMap): prompt=lambda mi: "Enter amount of ticks that will be stored to estimate order book liquidity", ), ) - order_levels_mode: Union[MultiOrderLevelModel, SingleOrderLevelModel] = Field( + order_levels_mode: Union[SingleOrderLevelModel, MultiOrderLevelModel] = Field( default=SingleOrderLevelModel.construct(), description="Allows activating multi-order levels.", client_data=ClientFieldData( @@ -327,7 +321,7 @@ class AvellanedaMarketMakingConfigMap(BaseTradingStrategyConfigMap): description="Allows custom specification of the order levels and their spreads and amounts.", client_data=None, ) - hanging_orders_mode: Union[TrackHangingOrdersModel, IgnoreHangingOrdersModel] = Field( + hanging_orders_mode: Union[IgnoreHangingOrdersModel, TrackHangingOrdersModel] = Field( default=IgnoreHangingOrdersModel.construct(), description="When tracking hanging orders, the orders on the side opposite to the filled orders remain active.", client_data=ClientFieldData( @@ -407,7 +401,7 @@ def validate_order_levels_mode(cls, v: Union[str, SingleOrderLevelModel, MultiOr return sub_model @validator("hanging_orders_mode", pre=True) - def validate_hanging_orders_mode(cls, v: Union[str, TrackHangingOrdersModel, IgnoreHangingOrdersModel]): + def validate_hanging_orders_mode(cls, v: Union[str, IgnoreHangingOrdersModel, TrackHangingOrdersModel]): if isinstance(v, (TrackHangingOrdersModel, IgnoreHangingOrdersModel, Dict)): sub_model = v elif v not in HANGING_ORDER_MODELS: @@ -477,18 +471,9 @@ def validate_pct_inclusive(cls, v: str): @root_validator() def post_validations(cls, values: Dict): - cls.execution_timeframe_post_validation(values) cls.exchange_post_validation(values) return values - @classmethod - def execution_timeframe_post_validation(cls, values: Dict): - execution_timeframe = values.get("execution_timeframe") - if execution_timeframe is not None and execution_timeframe == InfiniteModel.Config.title: - values["start_time"] = None - values["end_time"] = None - return values - @classmethod def exchange_post_validation(cls, values: Dict): required_exchanges.add(values["exchange"]) diff --git a/hummingbot/strategy/avellaneda_market_making/start.py b/hummingbot/strategy/avellaneda_market_making/start.py index d42fc8ea22..f3f98a1757 100644 --- a/hummingbot/strategy/avellaneda_market_making/start.py +++ b/hummingbot/strategy/avellaneda_market_making/start.py @@ -16,7 +16,7 @@ ) from hummingbot.strategy.conditional_execution_state import ( RunAlwaysExecutionState, - RunInTimeConditionalExecutionState + RunInTimeConditionalExecutionState, ) from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple from hummingbot.strategy.avellaneda_market_making import ( diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py index a98370435d..a6a2dbb0d2 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py @@ -1,6 +1,7 @@ import asyncio import unittest from datetime import datetime, time +from decimal import Decimal from pathlib import Path from typing import Awaitable, Dict from unittest.mock import patch @@ -14,7 +15,8 @@ AvellanedaMarketMakingConfigMap, DailyBetweenTimesModel, FromDateToDateModel, - InfiniteModel, + InfiniteModel, SingleOrderLevelModel, MultiOrderLevelModel, + IgnoreHangingOrdersModel, TrackHangingOrdersModel, ) @@ -203,3 +205,63 @@ def test_load_configs_from_yaml(self): loaded_config_map = ClientConfigAdapter(AvellanedaMarketMakingConfigMap(**data)) self.assertEqual(self.config_map, loaded_config_map) + + def test_configuring_execution_timeframe_mode(self): + self.config_map.execution_timeframe_mode = InfiniteModel() + + self.config_map.execution_timeframe_mode = { + "start_datetime": "2022-01-01 10:00:00", + "end_datetime": "2022-01-02 10:00:00", + } + self.config_map.validate_model() + + self.assertIsInstance(self.config_map.execution_timeframe_mode.hb_config, FromDateToDateModel) + self.assertEqual(self.config_map.execution_timeframe_mode.start_datetime, datetime(2022, 1, 1, 10)) + self.assertEqual(self.config_map.execution_timeframe_mode.end_datetime, datetime(2022, 1, 2, 10)) + + self.config_map.execution_timeframe_mode = { + "start_time": "10:00:00", + "end_time": "11:00:00", + } + self.config_map.validate_model() + + self.assertIsInstance(self.config_map.execution_timeframe_mode.hb_config, DailyBetweenTimesModel) + self.assertEqual(self.config_map.execution_timeframe_mode.start_time, time(10)) + self.assertEqual(self.config_map.execution_timeframe_mode.end_time, time(11)) + + self.config_map.execution_timeframe_mode = {} + self.config_map.validate_model() + + self.assertIsInstance(self.config_map.execution_timeframe_mode.hb_config, InfiniteModel) + + def test_configuring_order_levels_mode(self): + self.config_map.order_levels_mode = SingleOrderLevelModel() + + self.config_map.order_levels_mode = { + "order_levels": 2, + "level_distances": 1, + } + self.config_map.validate_model() + + self.assertIsInstance(self.config_map.order_levels_mode.hb_config, MultiOrderLevelModel) + self.assertEqual(self.config_map.order_levels_mode.order_levels, 2) + self.assertEqual(self.config_map.order_levels_mode.level_distances, 1) + + self.config_map.order_levels_mode = {} + self.config_map.validate_model() + + self.assertIsInstance(self.config_map.order_levels_mode.hb_config, SingleOrderLevelModel) + + def test_configuring_hanging_orders_mode(self): + self.config_map.hanging_orders_mode = IgnoreHangingOrdersModel() + + self.config_map.hanging_orders_mode = {"hanging_orders_cancel_pct": 1} + self.config_map.validate_model() + + self.assertIsInstance(self.config_map.hanging_orders_mode.hb_config, TrackHangingOrdersModel) + self.assertEqual(self.config_map.hanging_orders_mode.hanging_orders_cancel_pct, Decimal("1")) + + self.config_map.hanging_orders_mode = {} + self.config_map.validate_model() + + self.assertIsInstance(self.config_map.hanging_orders_mode.hb_config, IgnoreHangingOrdersModel) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py index b7c97fa9b2..91bd6cd66f 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py @@ -36,7 +36,7 @@ def setUp(self) -> None: ), order_amount=60, order_refresh_time=60, - hanging_orders_model=TrackHangingOrdersModel( + hanging_orders_mode=TrackHangingOrdersModel( hanging_orders_cancel_pct=1, ), order_levels_mode=MultiOrderLevelModel( @@ -45,8 +45,6 @@ def setUp(self) -> None: ), min_spread=2, risk_factor=1.11, - order_levels=4, - level_distances=1, order_amount_shape_factor=0.33, ) ) From 6d337f16cd23f6c5e4f9c90dcc05e429457f347e Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Tue, 5 Apr 2022 14:33:59 +0300 Subject: [PATCH 034/152] (fix) Fixes failure to cancel the strategy creation process. --- hummingbot/client/command/create_command.py | 22 ++++----------------- 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/hummingbot/client/command/create_command.py b/hummingbot/client/command/create_command.py index b73034003f..065c499d93 100644 --- a/hummingbot/client/command/create_command.py +++ b/hummingbot/client/command/create_command.py @@ -51,7 +51,6 @@ async def prompt_for_configuration( strategy = await self.get_strategy_name() if self.app.to_stop_config: - self.stop_config() return config_map = get_strategy_config_map(strategy) @@ -60,14 +59,14 @@ async def prompt_for_configuration( if isinstance(config_map, ClientConfigAdapter): await self.prompt_for_model_config(config_map) - file_name = await self.save_config_to_file(file_name, config_map) + if not self.app.to_stop_config: + file_name = await self.save_config_to_file(file_name, config_map) elif config_map is not None: file_name = await self.prompt_for_configuration_legacy(file_name, strategy, config_map) else: self.app.to_stop_config = True if self.app.to_stop_config: - self.stop_config() return self.strategy_file_name = file_name @@ -87,9 +86,7 @@ async def get_strategy_name( strategy = None strategy_config = ClientConfigAdapter(BaseStrategyConfigMap.construct()) await self.prompt_for_model_config(strategy_config) - if self.app.to_stop_config: - self.stop_config() - else: + if not self.app.to_stop_config: strategy = strategy_config.strategy return strategy @@ -130,13 +127,12 @@ async def prompt_for_configuration_legacy( config.value = config.default if self.app.to_stop_config: - self.stop_config(config_map, config_map_backup) return if file_name is None: file_name = await self.prompt_new_file_name(strategy) if self.app.to_stop_config: - self.stop_config(config_map, config_map_backup) + self.restore_config_legacy(config_map, config_map_backup) self.app.set_text("") return self.app.change_prompt(prompt=">>> ") @@ -209,7 +205,6 @@ async def save_config_to_file( if file_name is None: file_name = await self.prompt_new_file_name(config_map.strategy) if self.app.to_stop_config: - self.stop_config() self.app.set_text("") return self.app.change_prompt(prompt=">>> ") @@ -256,15 +251,6 @@ async def verify_status( if all_status_go: self._notify("\nEnter \"start\" to start market making.") - def stop_config( - self, - config_map: Optional[Dict[str, ConfigVar]] = None, - config_map_backup: Optional[Dict[str, ConfigVar]] = None, - ): - if config_map is not None and config_map_backup is not None: - self.restore_config_legacy(config_map, config_map_backup) - self.app.to_stop_config = False - @staticmethod def restore_config_legacy(config_map: Dict[str, ConfigVar], config_map_backup: Dict[str, ConfigVar]): for key in config_map: From f5c405d863991bc719c50a01b3451e5840fe1640 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 6 Apr 2022 12:49:10 +0700 Subject: [PATCH 035/152] Update hummingbot/client/command/config_command.py Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- hummingbot/client/command/config_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/client/command/config_command.py b/hummingbot/client/command/config_command.py index 8889a274fe..bf3be284d1 100644 --- a/hummingbot/client/command/config_command.py +++ b/hummingbot/client/command/config_command.py @@ -8,10 +8,10 @@ from hummingbot.client.config.config_data_types import BaseTradingStrategyConfigMap from hummingbot.client.config.config_helpers import ( + ClientConfigAdapter, missing_required_configs_legacy, save_to_yml, save_to_yml_legacy, - ClientConfigAdapter, ) from hummingbot.client.config.config_validators import validate_bool, validate_decimal from hummingbot.client.config.config_var import ConfigVar From 5531684044639cd42506296f8c28ca3daa67dac6 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 6 Apr 2022 12:54:02 +0700 Subject: [PATCH 036/152] Update hummingbot/client/command/create_command.py Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- hummingbot/client/command/create_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/client/command/create_command.py b/hummingbot/client/command/create_command.py index 065c499d93..22b5e02cc5 100644 --- a/hummingbot/client/command/create_command.py +++ b/hummingbot/client/command/create_command.py @@ -74,7 +74,7 @@ async def prompt_for_configuration( self.strategy_config_map = config_map # Reload completer here otherwise the new file will not appear self.app.input_field.completer = load_completer(self) - self._notify(f"A new config file {self.strategy_file_name} created.") + self._notify(f"A new config file has been created: {self.strategy_file_name}") self.placeholder_mode = False self.app.hide_input = False From 53fde5c9378b7b78953ca39bee34430af2990388 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 6 Apr 2022 12:56:17 +0700 Subject: [PATCH 037/152] Update hummingbot/client/config/config_data_types.py Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- hummingbot/client/config/config_data_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/client/config/config_data_types.py b/hummingbot/client/config/config_data_types.py index ead513a1e5..8242260d36 100644 --- a/hummingbot/client/config/config_data_types.py +++ b/hummingbot/client/config/config_data_types.py @@ -3,7 +3,7 @@ from enum import Enum from typing import Any, Callable, Dict, Optional -from pydantic import BaseModel, Field, validator, Extra +from pydantic import BaseModel, Field, Extra, validator from pydantic.schema import default_ref_template from hummingbot.client.config.config_methods import strategy_config_schema_encoder From 7db80eef63b279860905b4d03ca79408c4dca5ac Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 6 Apr 2022 12:56:33 +0700 Subject: [PATCH 038/152] Update hummingbot/client/config/config_helpers.py Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- hummingbot/client/config/config_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/client/config/config_helpers.py b/hummingbot/client/config/config_helpers.py index b75fe8687d..ab90e3fbcc 100644 --- a/hummingbot/client/config/config_helpers.py +++ b/hummingbot/client/config/config_helpers.py @@ -5,7 +5,7 @@ import shutil from collections import OrderedDict from dataclasses import dataclass -from datetime import date, time, datetime +from datetime import date, datetime, time from decimal import Decimal from os import listdir, unlink from os.path import isfile, join From 89f9063aa66bd536a01af67ed78b603b68da9071 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 6 Apr 2022 12:56:59 +0700 Subject: [PATCH 039/152] Update hummingbot/client/config/config_helpers.py Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- hummingbot/client/config/config_helpers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/hummingbot/client/config/config_helpers.py b/hummingbot/client/config/config_helpers.py index ab90e3fbcc..cfb548099e 100644 --- a/hummingbot/client/config/config_helpers.py +++ b/hummingbot/client/config/config_helpers.py @@ -28,9 +28,7 @@ from yaml import SafeDumper from hummingbot import get_strategy_list, root_path -from hummingbot.client.config.config_data_types import ( - BaseClientModel, ClientConfigEnum, ClientFieldData -) +from hummingbot.client.config.config_data_types import BaseClientModel, ClientConfigEnum, ClientFieldData from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.fee_overrides_config_map import fee_overrides_config_map from hummingbot.client.config.global_config_map import global_config_map From 7eaea15d721405621ce9bbce6f4c4fa79cf1d0c9 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 6 Apr 2022 14:53:14 +0700 Subject: [PATCH 040/152] Update test/hummingbot/client/command/test_import_command.py Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- test/hummingbot/client/command/test_import_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/hummingbot/client/command/test_import_command.py b/test/hummingbot/client/command/test_import_command.py index 04b8c737af..4577c83b3c 100644 --- a/test/hummingbot/client/command/test_import_command.py +++ b/test/hummingbot/client/command/test_import_command.py @@ -10,7 +10,7 @@ from pydantic import Field from hummingbot.client.command import import_command -from hummingbot.client.config.config_data_types import BaseClientModel, ClientConfigEnum, BaseTradingStrategyConfigMap +from hummingbot.client.config.config_data_types import BaseClientModel, BaseTradingStrategyConfigMap, ClientConfigEnum from hummingbot.client.config.config_helpers import read_system_configs_from_yml, save_to_yml, ClientConfigAdapter from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.hummingbot_application import HummingbotApplication From 58ec95efd9c292ccee9d053ede42f21f339ada41 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 6 Apr 2022 14:53:52 +0700 Subject: [PATCH 041/152] Update test/hummingbot/client/command/test_import_command.py Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- test/hummingbot/client/command/test_import_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/hummingbot/client/command/test_import_command.py b/test/hummingbot/client/command/test_import_command.py index 4577c83b3c..55adaf7fa9 100644 --- a/test/hummingbot/client/command/test_import_command.py +++ b/test/hummingbot/client/command/test_import_command.py @@ -1,6 +1,6 @@ import asyncio import unittest -from datetime import datetime, time, date +from datetime import date, datetime, time from decimal import Decimal from pathlib import Path from tempfile import TemporaryDirectory From 9f6794cb53056461a35aba05f7af981c4bf7c7fb Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 6 Apr 2022 14:54:12 +0700 Subject: [PATCH 042/152] Update test/hummingbot/client/command/test_import_command.py Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- test/hummingbot/client/command/test_import_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/hummingbot/client/command/test_import_command.py b/test/hummingbot/client/command/test_import_command.py index 55adaf7fa9..593c3657ce 100644 --- a/test/hummingbot/client/command/test_import_command.py +++ b/test/hummingbot/client/command/test_import_command.py @@ -11,7 +11,7 @@ from hummingbot.client.command import import_command from hummingbot.client.config.config_data_types import BaseClientModel, BaseTradingStrategyConfigMap, ClientConfigEnum -from hummingbot.client.config.config_helpers import read_system_configs_from_yml, save_to_yml, ClientConfigAdapter +from hummingbot.client.config.config_helpers import ClientConfigAdapter, read_system_configs_from_yml, save_to_yml from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.hummingbot_application import HummingbotApplication from test.mock.mock_cli import CLIMockingAssistant From a8c0545cda3fe9c32103b6b77d2b5f713d91a262 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 6 Apr 2022 14:54:23 +0700 Subject: [PATCH 043/152] Update test/hummingbot/client/config/test_config_data_types.py Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- test/hummingbot/client/config/test_config_data_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/hummingbot/client/config/test_config_data_types.py b/test/hummingbot/client/config/test_config_data_types.py index 21c96d558f..9d8dfedeaa 100644 --- a/test/hummingbot/client/config/test_config_data_types.py +++ b/test/hummingbot/client/config/test_config_data_types.py @@ -17,7 +17,7 @@ ClientFieldData, ) from hummingbot.client.config.config_helpers import ( - ConfigTraversalItem, ClientConfigAdapter, ConfigValidationError + ClientConfigAdapter, ConfigTraversalItem, ConfigValidationError ) From 279e5c567b253d37c67f1508cb0a6d81fc4ae13e Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 6 Apr 2022 14:54:35 +0700 Subject: [PATCH 044/152] Update test/hummingbot/client/config/test_config_helpers.py Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- test/hummingbot/client/config/test_config_helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/hummingbot/client/config/test_config_helpers.py b/test/hummingbot/client/config/test_config_helpers.py index beef381242..fe00a2b5cb 100644 --- a/test/hummingbot/client/config/test_config_helpers.py +++ b/test/hummingbot/client/config/test_config_helpers.py @@ -5,7 +5,7 @@ from typing import Awaitable from hummingbot.client.config.config_data_types import BaseStrategyConfigMap -from hummingbot.client.config.config_helpers import get_strategy_config_map, save_to_yml, ClientConfigAdapter +from hummingbot.client.config.config_helpers import ClientConfigAdapter, get_strategy_config_map, save_to_yml from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( AvellanedaMarketMakingConfigMap ) From 8cecbbf470a1726eab226d4613cf7df8c8d612c2 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 6 Apr 2022 10:57:58 +0300 Subject: [PATCH 045/152] (cleanup) Addresses @aarmoa's PR comments. --- hummingbot/client/command/create_command.py | 6 ++-- hummingbot/client/command/import_command.py | 8 +++--- hummingbot/client/command/status_command.py | 28 ++++++++----------- hummingbot/client/config/config_helpers.py | 9 ++---- ...aneda_market_making_config_map_pydantic.py | 7 +++-- 5 files changed, 27 insertions(+), 31 deletions(-) diff --git a/hummingbot/client/command/create_command.py b/hummingbot/client/command/create_command.py index 065c499d93..6b0546c034 100644 --- a/hummingbot/client/command/create_command.py +++ b/hummingbot/client/command/create_command.py @@ -6,6 +6,8 @@ from hummingbot.client.config.config_data_types import BaseStrategyConfigMap from hummingbot.client.config.config_helpers import ( + ClientConfigAdapter, + ConfigValidationError, default_strategy_file_path, format_config_file_name, get_strategy_config_map, @@ -13,9 +15,7 @@ parse_config_default_to_text, parse_cvar_value, save_to_yml, - save_to_yml_legacy, - ClientConfigAdapter, - ConfigValidationError, + save_to_yml_legacy ) from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.global_config_map import global_config_map diff --git a/hummingbot/client/command/import_command.py b/hummingbot/client/command/import_command.py index 766e98e175..33728d1544 100644 --- a/hummingbot/client/command/import_command.py +++ b/hummingbot/client/command/import_command.py @@ -1,16 +1,16 @@ import asyncio import os +from typing import TYPE_CHECKING -from hummingbot.core.utils.async_utils import safe_ensure_future -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.config_helpers import ( + format_config_file_name, load_strategy_config_map_from_file, short_strategy_name, - format_config_file_name, validate_strategy_file ) +from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.settings import CONF_FILE_PATH, CONF_PREFIX, required_exchanges -from typing import TYPE_CHECKING +from hummingbot.core.utils.async_utils import safe_ensure_future if TYPE_CHECKING: from hummingbot.client.hummingbot_application import HummingbotApplication diff --git a/hummingbot/client/command/status_command.py b/hummingbot/client/command/status_command.py index 4870d60a65..1acbdf3142 100644 --- a/hummingbot/client/command/status_command.py +++ b/hummingbot/client/command/status_command.py @@ -1,30 +1,26 @@ import asyncio +import inspect +import time +from collections import OrderedDict, deque +from typing import Dict, List, TYPE_CHECKING import pandas as pd -import time -from collections import ( - deque, - OrderedDict -) -import inspect -from typing import Dict, List from hummingbot import check_dev_mode -from hummingbot.logger.application_warning import ApplicationWarning -from hummingbot.connector.connector_base import ConnectorBase -from hummingbot.core.network_iterator import NetworkStatus -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.config_helpers import ( - get_strategy_config_map, - missing_required_configs_legacy, ClientConfigAdapter, + get_strategy_config_map, + missing_required_configs_legacy ) +from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.security import Security -from hummingbot.user.user_balances import UserBalances -from hummingbot.client.settings import required_exchanges, ethereum_wallet_required +from hummingbot.client.settings import ethereum_wallet_required, required_exchanges +from hummingbot.connector.connector_base import ConnectorBase +from hummingbot.core.network_iterator import NetworkStatus from hummingbot.core.utils.async_utils import safe_ensure_future +from hummingbot.logger.application_warning import ApplicationWarning +from hummingbot.user.user_balances import UserBalances -from typing import TYPE_CHECKING if TYPE_CHECKING: from hummingbot.client.hummingbot_application import HummingbotApplication diff --git a/hummingbot/client/config/config_helpers.py b/hummingbot/client/config/config_helpers.py index b75fe8687d..c24296fed2 100644 --- a/hummingbot/client/config/config_helpers.py +++ b/hummingbot/client/config/config_helpers.py @@ -106,12 +106,9 @@ def __init__(self, hb_config: BaseClientModel): self._hb_config = hb_config def __getattr__(self, item): - if item == "_hb_config": - value = super().__getattribute__(item) - else: - value = getattr(self._hb_config, item) - if isinstance(value, BaseClientModel): - value = ClientConfigAdapter(value) + value = getattr(self._hb_config, item) + if isinstance(value, BaseClientModel): + value = ClientConfigAdapter(value) return value def __setattr__(self, key, value): diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py index a6a2dbb0d2..f528001fa7 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py @@ -15,8 +15,11 @@ AvellanedaMarketMakingConfigMap, DailyBetweenTimesModel, FromDateToDateModel, - InfiniteModel, SingleOrderLevelModel, MultiOrderLevelModel, - IgnoreHangingOrdersModel, TrackHangingOrdersModel, + IgnoreHangingOrdersModel, + InfiniteModel, + MultiOrderLevelModel, + SingleOrderLevelModel, + TrackHangingOrdersModel ) From 7159e222df49f15ba0b176e7e0c0b52487654a4c Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 6 Apr 2022 11:50:12 +0300 Subject: [PATCH 046/152] Attempting to add isort hook --- .pre-commit-config.yaml | 6 ++++++ pyproject.toml | 7 +++++++ .../test_avellaneda_market_making_config_map_pydantic.py | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 976d7e0152..fced5d40e7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,3 +5,9 @@ repos: - id: flake8 types: ['file'] files: \.(py|pyx|pxd)$ +- repo: https://github.com/pycqa/isort + rev: 5.10.1 + hooks: + - id: isort + files: "\\.(py)$" + args: [--settings-path=pyproject.toml] diff --git a/pyproject.toml b/pyproject.toml index dc3aef1755..3fa91c0016 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,3 +17,10 @@ exclude = ''' [build-system] requires = ["setuptools", "wheel", "numpy", "cython==0.29.15"] + +[tool.isort] +line_length = 120 +multi_line_output = 3 +include_trailing_comma = true +use_parentheses = true +ensure_newline_before_comments = true \ No newline at end of file diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py index f528001fa7..8bb531897c 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py @@ -19,7 +19,7 @@ InfiniteModel, MultiOrderLevelModel, SingleOrderLevelModel, - TrackHangingOrdersModel + TrackHangingOrdersModel, ) From 2b0e132f6da7d19e9c15aec8287653330accd228 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 6 Apr 2022 11:51:14 +0300 Subject: [PATCH 047/152] Revert "Attempting to add isort hook" This reverts commit 7159e222df49f15ba0b176e7e0c0b52487654a4c. --- .pre-commit-config.yaml | 6 ------ pyproject.toml | 7 ------- .../test_avellaneda_market_making_config_map_pydantic.py | 2 +- 3 files changed, 1 insertion(+), 14 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fced5d40e7..976d7e0152 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,9 +5,3 @@ repos: - id: flake8 types: ['file'] files: \.(py|pyx|pxd)$ -- repo: https://github.com/pycqa/isort - rev: 5.10.1 - hooks: - - id: isort - files: "\\.(py)$" - args: [--settings-path=pyproject.toml] diff --git a/pyproject.toml b/pyproject.toml index 3fa91c0016..dc3aef1755 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,10 +17,3 @@ exclude = ''' [build-system] requires = ["setuptools", "wheel", "numpy", "cython==0.29.15"] - -[tool.isort] -line_length = 120 -multi_line_output = 3 -include_trailing_comma = true -use_parentheses = true -ensure_newline_before_comments = true \ No newline at end of file diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py index 8bb531897c..f528001fa7 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py @@ -19,7 +19,7 @@ InfiniteModel, MultiOrderLevelModel, SingleOrderLevelModel, - TrackHangingOrdersModel, + TrackHangingOrdersModel ) From b03598e69d3b9942764f4a80e1e3f8a0a9c3f456 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Thu, 7 Apr 2022 13:07:04 +0700 Subject: [PATCH 048/152] Update hummingbot/client/config/config_data_types.py Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- hummingbot/client/config/config_data_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/client/config/config_data_types.py b/hummingbot/client/config/config_data_types.py index 8242260d36..dbd1c0721e 100644 --- a/hummingbot/client/config/config_data_types.py +++ b/hummingbot/client/config/config_data_types.py @@ -3,7 +3,7 @@ from enum import Enum from typing import Any, Callable, Dict, Optional -from pydantic import BaseModel, Field, Extra, validator +from pydantic import BaseModel, Extra, Field, validator from pydantic.schema import default_ref_template from hummingbot.client.config.config_methods import strategy_config_schema_encoder From e5bf76e3b59aef782c0ba1104df2acac193c0e00 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Fri, 15 Apr 2022 13:25:42 +0300 Subject: [PATCH 049/152] (fix) Fixes failing test case --- hummingbot/client/command/create_command.py | 2 ++ hummingbot/client/ui/interface_utils.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/hummingbot/client/command/create_command.py b/hummingbot/client/command/create_command.py index b5bcac06bf..aab81b4da7 100644 --- a/hummingbot/client/command/create_command.py +++ b/hummingbot/client/command/create_command.py @@ -127,6 +127,8 @@ async def prompt_for_configuration_legacy( config.value = config.default if self.app.to_stop_config: + self.restore_config_legacy(config_map, config_map_backup) + self.app.set_text("") return if file_name is None: diff --git a/hummingbot/client/ui/interface_utils.py b/hummingbot/client/ui/interface_utils.py index 0be1c78a90..72474985a5 100644 --- a/hummingbot/client/ui/interface_utils.py +++ b/hummingbot/client/ui/interface_utils.py @@ -10,7 +10,7 @@ import pandas as pd import psutil -from tabulate import tabulate +import tabulate from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.performance import PerformanceMetrics From e2d6af60c63c142a5fbfadbcdcaf7099ac3c8c02 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Thu, 28 Apr 2022 09:51:51 +0300 Subject: [PATCH 050/152] (cleanup) Reverts the env name --- setup/environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup/environment.yml b/setup/environment.yml index 8098cca299..bcb2c127b6 100644 --- a/setup/environment.yml +++ b/setup/environment.yml @@ -1,4 +1,4 @@ -name: hummingbot-hs +name: hummingbot channels: - conda-forge - defaults From b1148ee17edc7532b16f9eb32ce2e46f49bafb57 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 4 May 2022 11:06:47 +0300 Subject: [PATCH 051/152] (feat) Refactor secure configs stroing This PR refactors the way we store secure configs. For strategies, the secure configs can be stored alongside the rest of the strategy configs. They will be loaded with the strategy configs and have the same lifetime. For connectors, the login values / connector configs are loaded on startup and are stored in the `Security` object as previously. This allows for the export keys command and several other parts of the code to continue working as expected. Connector configs are now stored in the same format as strategy configs, and the secure values are no longer separated in individual files. All connectors have been refactored to this new approach to configs storing. The secure configs are no longer stored in the global config map. --- .gitignore | 3 + bin/hummingbot.py | 4 +- bin/hummingbot_quickstart.py | 20 +- conf/__init__.py | 7 +- hummingbot/__init__.py | 32 ++- hummingbot/client/command/balance_command.py | 3 +- hummingbot/client/command/config_command.py | 12 +- hummingbot/client/command/connect_command.py | 142 ++++++------- hummingbot/client/command/create_command.py | 24 +-- hummingbot/client/command/export_command.py | 28 +-- hummingbot/client/command/gateway_command.py | 47 ++-- hummingbot/client/command/import_command.py | 7 +- hummingbot/client/command/start_command.py | 8 +- hummingbot/client/command/status_command.py | 5 +- hummingbot/client/config/config_crypt.py | 130 ++++++----- hummingbot/client/config/config_data_types.py | 25 ++- hummingbot/client/config/config_helpers.py | 201 ++++++++++++------ hummingbot/client/config/global_config_map.py | 38 +--- hummingbot/client/config/security.py | 144 ++++--------- hummingbot/client/settings.py | 73 +++++-- hummingbot/client/ui/__init__.py | 35 +-- hummingbot/client/ui/completer.py | 20 +- .../binance_perpetual_utils.py | 80 ++++--- .../bybit_perpetual/bybit_perpetual_utils.py | 81 ++++--- .../dydx_perpetual/dydx_perpetual_utils.py | 104 +++++---- .../exchange/altmarkets/altmarkets_utils.py | 51 +++-- .../exchange/ascend_ex/ascend_ex_utils.py | 42 ++-- .../connector/exchange/beaxy/beaxy_utils.py | 43 ++-- .../exchange/binance/binance_utils.py | 81 ++++--- .../exchange/bitfinex/bitfinex_utils.py | 45 ++-- .../exchange/bitmart/bitmart_utils.py | 57 +++-- .../exchange/bittrex/bittrex_utils.py | 43 ++-- .../exchange/blocktane/blocktane_utils.py | 46 ++-- .../coinbase_pro/coinbase_pro_utils.py | 58 +++-- .../exchange/coinflex/coinflex_utils.py | 81 ++++--- .../exchange/coinzoom/coinzoom_utils.py | 65 +++--- .../exchange/crypto_com/crypto_com_utils.py | 45 ++-- .../exchange/digifinex/digifinex_utils.py | 45 ++-- .../connector/exchange/ftx/ftx_utils.py | 60 +++--- .../exchange/gate_io/gate_io_utils.py | 50 +++-- .../connector/exchange/hitbtc/hitbtc_utils.py | 53 ++--- .../connector/exchange/huobi/huobi_utils.py | 43 ++-- hummingbot/connector/exchange/k2/k2_utils.py | 54 ++--- .../connector/exchange/kraken/kraken_utils.py | 69 +++--- .../connector/exchange/kucoin/kucoin_utils.py | 113 ++++++---- .../connector/exchange/liquid/liquid_utils.py | 42 ++-- .../exchange/loopring/loopring_utils.py | 75 ++++--- .../connector/exchange/mexc/mexc_utils.py | 44 ++-- .../connector/exchange/ndax/ndax_utils.py | 143 ++++++++----- .../connector/exchange/okex/okex_utils.py | 58 +++-- .../connector/exchange/probit/probit_utils.py | 82 ++++--- .../connector/exchange/wazirx/wazirx_utils.py | 48 +++-- .../connector/other/celo/celo_data_types.py | 34 ++- hummingbot/core/utils/wallet_setup.py | 72 ------- hummingbot/pmm_script/pmm_script_iterator.pyx | 6 +- hummingbot/templates/conf_global_TEMPLATE.yml | 136 +----------- hummingbot/user/user_balances.py | 20 +- scripts/conf_migration_script.py | 127 +++++++++++ setup/environment.yml | 1 + test/debug/test_config_process.py | 31 ++- .../client/command/test_connect_command.py | 52 +++-- .../client/command/test_import_command.py | 10 +- .../client/config/test_config_data_types.py | 61 ++++-- .../client/config/test_config_helpers.py | 41 +++- .../client/config/test_config_security.py | 72 ------- .../client/config/test_config_templates.py | 5 +- .../hummingbot/client/config/test_security.py | 124 +++++++++++ test/hummingbot/connector/test_utils.py | 40 ++++ .../core/utils/test_wallet_setup.py | 97 --------- 69 files changed, 2133 insertions(+), 1705 deletions(-) delete mode 100644 hummingbot/core/utils/wallet_setup.py create mode 100644 scripts/conf_migration_script.py delete mode 100644 test/hummingbot/client/config/test_config_security.py create mode 100644 test/hummingbot/client/config/test_security.py delete mode 100644 test/hummingbot/core/utils/test_wallet_setup.py diff --git a/.gitignore b/.gitignore index b5f1613211..debf2c1055 100644 --- a/.gitignore +++ b/.gitignore @@ -110,3 +110,6 @@ coverage.xml # Debug console .debug_console_ssh_host_key + +# password file +.password_verification diff --git a/bin/hummingbot.py b/bin/hummingbot.py index 24f102e204..4af7f9d7a1 100755 --- a/bin/hummingbot.py +++ b/bin/hummingbot.py @@ -8,6 +8,7 @@ from bin.docker_connection import fork_and_start from hummingbot import chdir_to_data_directory, init_logging +from hummingbot.client.config.config_crypt import ETHKeyFileSecretManger from hummingbot.client.config.config_helpers import ( create_yml_files_legacy, read_system_configs_from_yml, @@ -75,7 +76,8 @@ async def main_async(): def main(): chdir_to_data_directory() - if login_prompt(): + secrets_manager_cls = ETHKeyFileSecretManger + if login_prompt(secrets_manager_cls): ev_loop: asyncio.AbstractEventLoop = asyncio.get_event_loop() ev_loop.run_until_complete(main_async()) diff --git a/bin/hummingbot_quickstart.py b/bin/hummingbot_quickstart.py index 011ab7325a..f916e203c4 100755 --- a/bin/hummingbot_quickstart.py +++ b/bin/hummingbot_quickstart.py @@ -15,6 +15,7 @@ from bin.docker_connection import fork_and_start from bin.hummingbot import UIStartListener, detect_available_port from hummingbot import init_logging +from hummingbot.client.config.config_crypt import BaseSecretsManager, ETHKeyFileSecretManger from hummingbot.client.config.config_helpers import ( all_configs_complete, create_yml_files_legacy, @@ -24,7 +25,7 @@ from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.security import Security from hummingbot.client.hummingbot_application import HummingbotApplication -from hummingbot.client.settings import CONF_FILE_PATH, AllConnectorSettings +from hummingbot.client.settings import STRATEGIES_CONF_DIR_PATH, AllConnectorSettings from hummingbot.client.ui import login_prompt from hummingbot.core.event.events import HummingbotUIEvent from hummingbot.core.gateway import start_existing_gateway_container @@ -70,14 +71,13 @@ def autofix_permissions(user_group_spec: str): os.setuid(uid) -async def quick_start(args: argparse.Namespace): +async def quick_start(args: argparse.Namespace, secrets_manager: BaseSecretsManager): config_file_name = args.config_file_name - password = args.config_password if args.auto_set_permissions is not None: autofix_permissions(args.auto_set_permissions) - if password is not None and not Security.login(password): + if Security.login(secrets_manager): logging.getLogger().error("Invalid password.") return @@ -93,7 +93,9 @@ async def quick_start(args: argparse.Namespace): if config_file_name is not None: hb.strategy_file_name = config_file_name - hb.strategy_name = await load_strategy_config_map_from_file(os.path.join(CONF_FILE_PATH, config_file_name)) + hb.strategy_name = await load_strategy_config_map_from_file( + STRATEGIES_CONF_DIR_PATH / config_file_name + ) # To ensure quickstart runs with the default value of False for kill_switch_enabled if not present if not global_config_map.get("kill_switch_enabled"): @@ -128,11 +130,15 @@ def main(): args.config_password = os.environ["CONFIG_PASSWORD"] # If no password is given from the command line, prompt for one. + secrets_manager_cls = ETHKeyFileSecretManger if args.config_password is None: - if not login_prompt(): + secrets_manager = login_prompt(secrets_manager_cls) + if not secrets_manager: return + else: + secrets_manager = secrets_manager_cls(args.config_password) - asyncio.get_event_loop().run_until_complete(quick_start(args)) + asyncio.get_event_loop().run_until_complete(quick_start(args, secrets_manager)) if __name__ == "__main__": diff --git a/conf/__init__.py b/conf/__init__.py index 1a45edd695..960d4522d9 100644 --- a/conf/__init__.py +++ b/conf/__init__.py @@ -1,9 +1,8 @@ #!/usr/bin/env python +import logging as _logging import os -from hummingbot.client.config.global_config_map import connector_keys -import logging as _logging _logger = _logging.getLogger(__name__) master_host = "***REMOVED***" @@ -38,10 +37,6 @@ # whether to enable api mocking in unit test cases mock_api_enabled = os.getenv("MOCK_API_ENABLED") -# ALL TEST KEYS -for key in connector_keys().keys(): - locals()[key] = os.getenv(key.upper()) - """ # AscendEX Tests ascend_ex_api_key = os.getenv("ASCEND_EX_KEY") diff --git a/hummingbot/__init__.py b/hummingbot/__init__.py index 09188b1d29..ca2060fd42 100644 --- a/hummingbot/__init__.py +++ b/hummingbot/__init__.py @@ -3,6 +3,7 @@ import sys from concurrent.futures import ThreadPoolExecutor from os import listdir, path +from pathlib import Path from typing import List, Optional from hummingbot.logger.struct_logger import StructLogger, StructLogRecord @@ -20,9 +21,9 @@ _cert_path = None -def root_path() -> str: - from os.path import realpath, join - return realpath(join(__file__, "../../")) +def root_path() -> Path: + from os.path import join, realpath + return Path(realpath(join(__file__, "../../"))) def get_executor() -> ThreadPoolExecutor: @@ -35,26 +36,20 @@ def get_executor() -> ThreadPoolExecutor: def prefix_path() -> str: global _prefix_path if _prefix_path is None: - from os.path import ( - realpath, - join - ) + from os.path import join, realpath _prefix_path = realpath(join(__file__, "../../")) return _prefix_path -def set_prefix_path(path: str): +def set_prefix_path(p: str): global _prefix_path - _prefix_path = path + _prefix_path = p def data_path() -> str: global _data_path if _data_path is None: - from os.path import ( - realpath, - join - ) + from os.path import join, realpath _data_path = realpath(join(prefix_path(), "data")) import os @@ -97,8 +92,9 @@ def chdir_to_data_directory(): # Do nothing. return - import appdirs import os + + import appdirs app_data_dir: str = appdirs.user_data_dir("Hummingbot", "hummingbot.io") os.makedirs(os.path.join(app_data_dir, "logs"), 0o711, exist_ok=True) os.makedirs(os.path.join(app_data_dir, "conf"), 0o711, exist_ok=True) @@ -115,15 +111,13 @@ def init_logging(conf_filename: str, import io import logging.config from os.path import join - import pandas as pd from typing import Dict + + import pandas as pd from ruamel.yaml import YAML from hummingbot.client.config.global_config_map import global_config_map - from hummingbot.logger.struct_logger import ( - StructLogRecord, - StructLogger - ) + from hummingbot.logger.struct_logger import StructLogger, StructLogRecord global STRUCT_LOGGER_SET if not STRUCT_LOGGER_SET: logging.setLogRecordFactory(StructLogRecord) diff --git a/hummingbot/client/command/balance_command.py b/hummingbot/client/command/balance_command.py index b301d1c106..e6845c95fb 100644 --- a/hummingbot/client/command/balance_command.py +++ b/hummingbot/client/command/balance_command.py @@ -11,6 +11,7 @@ from hummingbot.client.performance import PerformanceMetrics from hummingbot.client.settings import GLOBAL_CONFIG_PATH from hummingbot.connector.other.celo.celo_cli import CeloCLI +from hummingbot.connector.other.celo.celo_data_types import KEYS as CELO_KEYS from hummingbot.core.rate_oracle.rate_oracle import RateOracle from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.user.user_balances import UserBalances @@ -117,7 +118,7 @@ async def show_balances(self): self.notify(f"\n\nExchanges Total: {RateOracle.global_token_symbol} {exchanges_total:.0f} ") - celo_address = global_config_map["celo_address"].value + celo_address = CELO_KEYS.celo_address if hasattr("celo_address", CELO_KEYS) else None if celo_address is not None: try: if not CeloCLI.unlocked: diff --git a/hummingbot/client/command/config_command.py b/hummingbot/client/command/config_command.py index 38fa098b1d..c5a93a6679 100644 --- a/hummingbot/client/command/config_command.py +++ b/hummingbot/client/command/config_command.py @@ -1,6 +1,5 @@ import asyncio from decimal import Decimal -from os.path import join from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union import pandas as pd @@ -17,7 +16,7 @@ from hummingbot.client.config.config_validators import validate_bool, validate_decimal from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.security import Security -from hummingbot.client.settings import CONF_FILE_PATH, GLOBAL_CONFIG_PATH +from hummingbot.client.settings import GLOBAL_CONFIG_PATH, STRATEGIES_CONF_DIR_PATH from hummingbot.client.ui.interface_utils import format_df_for_printout from hummingbot.client.ui.style import load_style from hummingbot.connector.utils import split_hb_trading_pair @@ -167,7 +166,7 @@ def configurable_keys(self # type: HummingbotApplication async def check_password(self, # type: HummingbotApplication ): password = await self.app.prompt(prompt="Enter your password >>> ", is_password=True) - if password != Security.password: + if password != Security.secrets_manager.password.get_secret_value(): self.notify("Invalid password, please try again.") return False else: @@ -206,7 +205,7 @@ async def _config_single_key(self, # type: HummingbotApplication await self._config_single_key_legacy(key, input_value) else: config_map = self.strategy_config_map - file_path = join(CONF_FILE_PATH, self.strategy_file_name) + file_path = STRATEGIES_CONF_DIR_PATH / self.strategy_file_name if input_value is None: self.notify("Please follow the prompt to complete configurations: ") if key == "inventory_target_base_pct": @@ -242,7 +241,7 @@ async def _config_single_key_legacy( file_path = GLOBAL_CONFIG_PATH elif self.strategy_config_map is not None and key in self.strategy_config_map: config_map = self.strategy_config_map - file_path = join(CONF_FILE_PATH, self.strategy_file_name) + file_path = STRATEGIES_CONF_DIR_PATH / self.strategy_file_name config_var = config_map[key] if input_value is None: self.notify("Please follow the prompt to complete configurations: ") @@ -255,12 +254,11 @@ async def _config_single_key_legacy( if self.app.to_stop_config: self.app.to_stop_config = False return - await self.update_all_secure_configs_legacy() missings = missing_required_configs_legacy(config_map) if missings: self.notify("\nThere are other configuration required, please follow the prompt to complete them.") missings = await self._prompt_missing_configs(config_map) - save_to_yml_legacy(file_path, config_map) + save_to_yml_legacy(str(file_path), config_map) self.notify("\nNew configuration saved:") self.notify(f"{key}: {str(config_var.value)}") self.app.app.style = load_style() diff --git a/hummingbot/client/command/connect_command.py b/hummingbot/client/command/connect_command.py index 6724183536..85cf9a381e 100644 --- a/hummingbot/client/command/connect_command.py +++ b/hummingbot/client/command/connect_command.py @@ -3,13 +3,14 @@ import pandas as pd -from hummingbot.client.config.config_helpers import save_to_yml_legacy +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.security import Security -from hummingbot.client.settings import GLOBAL_CONFIG_PATH, AllConnectorSettings +from hummingbot.client.settings import AllConnectorSettings from hummingbot.client.ui.interface_utils import format_df_for_printout from hummingbot.connector.connector_status import get_connector_status from hummingbot.connector.other.celo.celo_cli import CeloCLI +from hummingbot.connector.other.celo.celo_data_types import KEYS as CELO_KEYS from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.user.user_balances import UserBalances @@ -27,36 +28,39 @@ def connect(self, # type: HummingbotApplication safe_ensure_future(self.show_connections()) elif option == "ethereum": safe_ensure_future(self.connect_ethereum()) - elif option == "celo": - safe_ensure_future(self.connect_celo()) else: safe_ensure_future(self.connect_exchange(option)) async def connect_exchange(self, # type: HummingbotApplication - exchange): + connector_name): # instruct users to use gateway connect if connector is a gateway connector - if AllConnectorSettings.get_connector_settings()[exchange].uses_gateway_generic_connector(): + if ( + connector_name != "celo" + and AllConnectorSettings.get_connector_settings()[connector_name].uses_gateway_generic_connector() + ): self.notify("This is a gateway connector. Use `gateway connect` command instead.") return self.app.clear_input() self.placeholder_mode = True self.app.hide_input = True - if exchange == "kraken": + if connector_name == "kraken": self.notify("Reminder: Please ensure your Kraken API Key Nonce Window is at least 10.") - exchange_configs = [c for c in global_config_map.values() - if c.key in AllConnectorSettings.get_connector_settings()[exchange].config_keys and c.is_connect_key] - + if connector_name == "celo": + connector_config = ClientConfigAdapter(CELO_KEYS) + else: + connector_config = ClientConfigAdapter(AllConnectorSettings.get_connector_config_keys(connector_name)) to_connect = True - if Security.encrypted_file_exists(exchange_configs[0].key): + if Security.connector_config_file_exists(connector_name): await Security.wait_til_decryption_done() - api_key_config = [c for c in exchange_configs if "api_key" in c.key] + api_key_config = [c for c in connector_config.traverse() if "api_key" in c.attr] if api_key_config: - api_key_config = api_key_config[0] - api_key = Security.decrypted_value(api_key_config.key) - prompt = f"Would you like to replace your existing {exchange} API key {api_key} (Yes/No)? >>> " + api_key = api_key_config[0] + prompt = ( + f"Would you like to replace your existing {connector_name} API key {api_key} (Yes/No)? >>> " + ) else: - prompt = f"Would you like to replace your existing {exchange_configs[0].key} (Yes/No)? >>> " + prompt = f"Would you like to replace your existing {connector_name} key (Yes/No)? >>> " answer = await self.app.prompt(prompt=prompt) if self.app.to_stop_config: self.app.to_stop_config = False @@ -64,26 +68,17 @@ async def connect_exchange(self, # type: HummingbotApplication if answer.lower() not in ("yes", "y"): to_connect = False if to_connect: - for config in exchange_configs: - await self.prompt_a_config_legacy(config) - if self.app.to_stop_config: - self.app.to_stop_config = False - return - Security.update_secure_config(config.key, config.value) - api_keys = await Security.api_keys(exchange) - network_timeout = float(global_config_map["other_commands_timeout"].value) - try: - err_msg = await asyncio.wait_for( - UserBalances.instance().add_exchange(exchange, **api_keys), network_timeout - ) - except asyncio.TimeoutError: - self.notify("\nA network error prevented the connection to complete. See logs for more details.") - self.placeholder_mode = False - self.app.hide_input = False - self.app.change_prompt(prompt=">>> ") - raise + await self.prompt_for_model_config(connector_config) + if self.app.to_stop_config: + self.app.to_stop_config = False + return + Security.update_secure_config(connector_config) + if connector_name == "celo": + err_msg = await self.validate_n_connect_celo(to_reconnect=True) + else: + err_msg = await self.validate_n_connect_connector(connector_name) if err_msg is None: - self.notify(f"\nYou are now connected to {exchange}.") + self.notify(f"\nYou are now connected to {connector_name}.") else: self.notify(f"\nError: {err_msg}") self.placeholder_mode = False @@ -93,7 +88,6 @@ async def connect_exchange(self, # type: HummingbotApplication async def show_connections(self # type: HummingbotApplication ): self.notify("\nTesting connections, please wait...") - await Security.wait_til_decryption_done() df, failed_msgs = await self.connection_df() lines = [" " + line for line in format_df_for_printout(df).split("\n")] if failed_msgs: @@ -103,6 +97,7 @@ async def show_connections(self # type: HummingbotApplication async def connection_df(self # type: HummingbotApplication ): + await Security.wait_til_decryption_done() columns = ["Exchange", " Keys Added", " Keys Confirmed", " Status"] data = [] failed_msgs = {} @@ -116,26 +111,30 @@ async def connection_df(self # type: HummingbotApplication raise for option in sorted(OPTIONS): keys_added = "No" - keys_confirmed = 'No' + keys_confirmed = "No" status = get_connector_status(option) if option == "celo": - celo_address = global_config_map["celo_address"].value - if celo_address is not None and Security.encrypted_file_exists("celo_password"): + celo_config = Security.decrypted_value(option) + if celo_config is not None: keys_added = "Yes" err_msg = await self.validate_n_connect_celo(True) if err_msg is not None: failed_msgs[option] = err_msg else: - keys_confirmed = 'Yes' + keys_confirmed = "Yes" else: - api_keys = (await Security.api_keys(option)).values() + api_keys = ( + (await Security.api_keys(option)).values() + if not UserBalances.instance().is_gateway_market(option) + else {} + ) if len(api_keys) > 0: keys_added = "Yes" err_msg = err_msgs.get(option) if err_msg is not None: failed_msgs[option] = err_msg else: - keys_confirmed = 'Yes' + keys_confirmed = "Yes" data.append([option, keys_added, keys_confirmed, status]) return pd.DataFrame(data=data, columns=columns), failed_msgs @@ -143,46 +142,31 @@ async def connect_ethereum(self, # type: HummingbotApplication ): self.notify("\nError: Feature deprecated. Use 'gateway connect' instead.") - async def connect_celo(self, # type: HummingbotApplication - ): - self.placeholder_mode = True - self.app.hide_input = True - celo_address = global_config_map["celo_address"].value - to_connect = True - if celo_address is not None: - answer = await self.app.prompt(prompt=f"Would you like to replace your existing Celo account address " - f"{celo_address} (Yes/No)? >>> ") - if answer.lower() not in ("yes", "y"): - to_connect = False - if to_connect: - await self.prompt_a_config_legacy(global_config_map["celo_address"]) - await self.prompt_a_config_legacy(global_config_map["celo_password"]) - save_to_yml_legacy(GLOBAL_CONFIG_PATH, global_config_map) - - err_msg = await self.validate_n_connect_celo(True, - global_config_map["celo_address"].value, - global_config_map["celo_password"].value) - if err_msg is None: - self.notify("You are now connected to Celo network.") - else: - self.notify(err_msg) - self.placeholder_mode = False - self.app.hide_input = False - self.app.change_prompt(prompt=">>> ") - - async def validate_n_connect_celo(self, to_reconnect: bool = False, celo_address: str = None, - celo_password: str = None) -> Optional[str]: - if celo_address is None: - celo_address = global_config_map["celo_address"].value - if celo_password is None: - await Security.wait_til_decryption_done() - celo_password = Security.decrypted_value("celo_password") - if celo_address is None or celo_password is None: - return "Celo address and/or password have not been added." + async def validate_n_connect_celo(self, to_reconnect: bool = False) -> Optional[str]: + await Security.wait_til_decryption_done() + celo_config = Security.decrypted_value(key="celo") + if celo_config is None: + return "No Celo connection has been configured." if CeloCLI.unlocked and not to_reconnect: return None err_msg = CeloCLI.validate_node_synced() if err_msg is not None: return err_msg - err_msg = CeloCLI.unlock_account(celo_address, celo_password) + err_msg = CeloCLI.unlock_account(celo_config.celo_address, celo_config.celo_password.get_secret_value()) + return err_msg + + async def validate_n_connect_connector(self, connector_name: str) -> Optional[str]: + api_keys = await Security.api_keys(connector_name) + network_timeout = float(global_config_map["other_commands_timeout"].value) + try: + err_msg = await asyncio.wait_for( + UserBalances.instance().add_exchange(connector_name, **api_keys), network_timeout + ) + except asyncio.TimeoutError: + self.notify( + "\nA network error prevented the connection to complete. See logs for more details.") + self.placeholder_mode = False + self.app.hide_input = False + self.app.change_prompt(prompt=">>> ") + raise return err_msg diff --git a/hummingbot/client/command/create_command.py b/hummingbot/client/command/create_command.py index 1e3bd60ffe..8860c365de 100644 --- a/hummingbot/client/command/create_command.py +++ b/hummingbot/client/command/create_command.py @@ -2,6 +2,7 @@ import copy import os import shutil +from pathlib import Path from typing import TYPE_CHECKING, Dict, Optional from hummingbot.client.config.config_data_types import BaseStrategyConfigMap @@ -19,8 +20,7 @@ ) from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.global_config_map import global_config_map -from hummingbot.client.config.security import Security -from hummingbot.client.settings import CONF_FILE_PATH, required_exchanges +from hummingbot.client.settings import STRATEGIES_CONF_DIR_PATH, required_exchanges from hummingbot.client.ui.completer import load_completer from hummingbot.core.utils.async_utils import safe_ensure_future @@ -33,7 +33,7 @@ def create(self, # type: HummingbotApplication file_name): if file_name is not None: file_name = format_config_file_name(file_name) - if os.path.exists(os.path.join(CONF_FILE_PATH, file_name)): + if (STRATEGIES_CONF_DIR_PATH / file_name).exists(): self.notify(f"{file_name} already exists.") return @@ -98,7 +98,7 @@ async def prompt_for_model_config( client_data = config_map.get_client_data(key) if ( client_data is not None - and (client_data.prompt_on_new and config_map.is_required(key)) + and (client_data.prompt_on_new or config_map.is_required(key)) ): await self.prompt_a_config(config_map, key) if self.app.to_stop_config: @@ -138,10 +138,10 @@ async def prompt_for_configuration_legacy( self.app.set_text("") return self.app.change_prompt(prompt=">>> ") - strategy_path = os.path.join(CONF_FILE_PATH, file_name) + strategy_path = STRATEGIES_CONF_DIR_PATH / file_name template = get_strategy_template_path(strategy) shutil.copy(template, strategy_path) - save_to_yml_legacy(strategy_path, config_map) + save_to_yml_legacy(str(strategy_path), config_map) return file_name async def prompt_a_config( @@ -211,7 +211,7 @@ async def save_config_to_file( self.app.set_text("") return self.app.change_prompt(prompt=">>> ") - strategy_path = os.path.join(CONF_FILE_PATH, file_name) + strategy_path = Path(STRATEGIES_CONF_DIR_PATH) / file_name save_to_yml(strategy_path, config_map) return file_name @@ -221,7 +221,7 @@ async def prompt_new_file_name(self, # type: HummingbotApplication self.app.set_text(file_name) input = await self.app.prompt(prompt="Enter a new file name for your configuration >>> ") input = format_config_file_name(input) - file_path = os.path.join(CONF_FILE_PATH, input) + file_path = os.path.join(STRATEGIES_CONF_DIR_PATH, input) if input is None or input == "": self.notify("Value is required.") return await self.prompt_new_file_name(strategy) @@ -231,14 +231,6 @@ async def prompt_new_file_name(self, # type: HummingbotApplication else: return input - async def update_all_secure_configs_legacy( - self # type: HummingbotApplication - ): - await Security.wait_til_decryption_done() - Security.update_config_map(global_config_map) - if self.strategy_config_map is not None and not isinstance(self.strategy_config_map, ClientConfigAdapter): - Security.update_config_map(self.strategy_config_map) - async def verify_status( self # type: HummingbotApplication ): diff --git a/hummingbot/client/command/export_command.py b/hummingbot/client/command/export_command.py index 2a22c6e954..a3248f4f6c 100644 --- a/hummingbot/client/command/export_command.py +++ b/hummingbot/client/command/export_command.py @@ -1,17 +1,15 @@ import os -from typing import List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, List, Optional import pandas as pd -from sqlalchemy.orm import ( - Session, - Query -) +from sqlalchemy.orm import Query, Session from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.security import Security from hummingbot.client.settings import DEFAULT_LOG_FILE_PATH from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.model.trade_fill import TradeFill + if TYPE_CHECKING: from hummingbot.client.hummingbot_application import HummingbotApplication @@ -29,24 +27,20 @@ def export(self, # type: HummingbotApplication async def export_keys(self, # type: HummingbotApplication ): - if not Security.any_encryped_files() and not Security.any_wallets(): + await Security.wait_til_decryption_done() + if not Security.any_secure_configs(): self.notify("There are no keys to export.") return - await Security.wait_til_decryption_done() self.placeholder_mode = True self.app.hide_input = True if await self.check_password(): - await Security.wait_til_decryption_done() self.notify("\nWarning: Never disclose API keys or private keys. Anyone with your keys can steal any " "assets held in your account.") - if Security.all_decrypted_values(): - self.notify("\nAPI keys:") - for key, value in Security.all_decrypted_values().items(): - self.notify(f"{key}: {value}") - if Security.private_keys(): - self.notify("\nEthereum wallets:") - for key, value in Security.private_keys().items(): - self.notify(f"Public address: {key}\nPrivate Key: {value.hex()}") + self.notify("\nAPI keys:") + for key, cm in Security.all_decrypted_values().items(): + for el in cm.traverse(secure=False): + if el.client_field_data is not None and el.client_field_data.is_secure: + self.notify(f"{el.attr}: {el.printable_value}") self.app.change_prompt(prompt=">>> ") self.app.hide_input = False self.placeholder_mode = False @@ -79,7 +73,7 @@ async def export_trades(self, # type: HummingbotApplication self.app.hide_input = True path = global_config_map["log_file_path"].value if path is None: - path = DEFAULT_LOG_FILE_PATH + path = str(DEFAULT_LOG_FILE_PATH) file_name = await self.prompt_new_export_file_name(path) file_path = os.path.join(path, file_name) try: diff --git a/hummingbot/client/command/gateway_command.py b/hummingbot/client/command/gateway_command.py index f691f1e3db..6542e0b4d0 100644 --- a/hummingbot/client/command/gateway_command.py +++ b/hummingbot/client/command/gateway_command.py @@ -1,55 +1,48 @@ #!/usr/bin/env python -import aiohttp import asyncio -from contextlib import contextmanager -import docker import itertools import json +from contextlib import contextmanager +from pathlib import Path +from typing import TYPE_CHECKING, Any, Dict, Generator, List + +import aiohttp import pandas as pd -from typing import ( - Dict, - Any, - TYPE_CHECKING, - List, - Generator, -) +import docker +from hummingbot.client.config.config_helpers import refresh_trade_fees_config, save_to_yml +from hummingbot.client.config.global_config_map import global_config_map +from hummingbot.client.config.security import Security from hummingbot.client.settings import ( GATEWAY_CONNECTORS, GLOBAL_CONFIG_PATH, - GatewayConnectionSetting + AllConnectorSettings, + GatewayConnectionSetting, ) +from hummingbot.client.ui.completer import load_completer from hummingbot.core.gateway import ( - docker_ipc, - docker_ipc_with_generator, - get_gateway_container_name, - get_gateway_paths, GATEWAY_DOCKER_REPO, GATEWAY_DOCKER_TAG, GatewayPaths, + docker_ipc, + docker_ipc_with_generator, get_default_gateway_port, + get_gateway_container_name, + get_gateway_paths, start_gateway, - stop_gateway + stop_gateway, ) +from hummingbot.core.gateway.gateway_http_client import GatewayHttpClient from hummingbot.core.gateway.status_monitor import Status from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.core.utils.gateway_config_utils import ( - search_configs, build_config_dict_display, build_connector_display, build_wallet_display, native_tokens, + search_configs, ) -from hummingbot.core.gateway.gateway_http_client import GatewayHttpClient from hummingbot.core.utils.ssl_cert import certs_files_exist, create_self_sign_certs -from hummingbot.client.config.config_helpers import ( - save_to_yml, - refresh_trade_fees_config, -) -from hummingbot.client.config.global_config_map import global_config_map -from hummingbot.client.config.security import Security -from hummingbot.client.settings import AllConnectorSettings -from hummingbot.client.ui.completer import load_completer if TYPE_CHECKING: from hummingbot.client.hummingbot_application import HummingbotApplication @@ -262,7 +255,7 @@ async def _create_gateway(self): if global_config_map.get("gateway_api_port").value != gateway_port: global_config_map["gateway_api_port"].value = gateway_port global_config_map["gateway_api_host"].value = "localhost" - save_to_yml(GLOBAL_CONFIG_PATH, global_config_map) + save_to_yml(Path(GLOBAL_CONFIG_PATH), global_config_map) GatewayHttpClient.get_instance().base_url = f"https://{global_config_map['gateway_api_host'].value}:" \ f"{global_config_map['gateway_api_port'].value}" diff --git a/hummingbot/client/command/import_command.py b/hummingbot/client/command/import_command.py index 42cfc7af6a..7a8c1ab8e6 100644 --- a/hummingbot/client/command/import_command.py +++ b/hummingbot/client/command/import_command.py @@ -1,5 +1,4 @@ import asyncio -import os from typing import TYPE_CHECKING from hummingbot.client.config.config_helpers import ( @@ -9,7 +8,7 @@ validate_strategy_file, ) from hummingbot.client.config.global_config_map import global_config_map -from hummingbot.client.settings import CONF_FILE_PATH, CONF_PREFIX, required_exchanges +from hummingbot.client.settings import CONF_PREFIX, STRATEGIES_CONF_DIR_PATH, required_exchanges from hummingbot.core.utils.async_utils import safe_ensure_future if TYPE_CHECKING: @@ -36,7 +35,7 @@ async def import_config_file(self, # type: HummingbotApplication if self.app.to_stop_config: self.app.to_stop_config = False return - strategy_path = os.path.join(CONF_FILE_PATH, file_name) + strategy_path = STRATEGIES_CONF_DIR_PATH / file_name config_map = await load_strategy_config_map_from_file(strategy_path) self.strategy_file_name = file_name self.strategy_name = ( @@ -68,7 +67,7 @@ async def prompt_a_file_name(self # type: HummingbotApplication file_name = await self.app.prompt(prompt=f'Enter path to your strategy file (e.g. "{example}") >>> ') if self.app.to_stop_config: return - file_path = os.path.join(CONF_FILE_PATH, file_name) + file_path = STRATEGIES_CONF_DIR_PATH / file_name err_msg = validate_strategy_file(file_path) if err_msg is not None: self.notify(f"Error: {err_msg}") diff --git a/hummingbot/client/command/start_command.py b/hummingbot/client/command/start_command.py index 9698ac22d9..96860df271 100644 --- a/hummingbot/client/command/start_command.py +++ b/hummingbot/client/command/start_command.py @@ -2,7 +2,7 @@ import platform import threading import time -from os.path import dirname, exists, join +from os.path import dirname from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional import pandas as pd @@ -160,8 +160,8 @@ def start_script_strategy(self): self.strategy = script_strategy(self.markets) def is_current_strategy_script_strategy(self) -> bool: - script_file_name = join(settings.SCRIPT_STRATEGIES_PATH, f"{self.strategy_file_name}.py") - return exists(script_file_name) + script_file_name = settings.SCRIPT_STRATEGIES_PATH / f"{self.strategy_file_name}.py" + return script_file_name.exists() async def start_market_making(self, # type: HummingbotApplication restore: Optional[bool] = False): @@ -184,7 +184,7 @@ async def start_market_making(self, # type: HummingbotApplication pmm_script_file = global_config.global_config_map[global_config.PMM_SCRIPT_FILE_PATH_KEY].value folder = dirname(pmm_script_file) if folder == "": - pmm_script_file = join(settings.PMM_SCRIPTS_PATH, pmm_script_file) + pmm_script_file = settings.PMM_SCRIPTS_PATH / pmm_script_file if self.strategy_name != "pure_market_making": self.notify("Error: PMM script feature is only available for pure_market_making strategy.") else: diff --git a/hummingbot/client/command/status_command.py b/hummingbot/client/command/status_command.py index a8210a598d..606a79b562 100644 --- a/hummingbot/client/command/status_command.py +++ b/hummingbot/client/command/status_command.py @@ -98,7 +98,6 @@ async def validate_required_connections(self) -> Dict[str, str]: if err_msg is not None: invalid_conns["celo"] = err_msg if not any([str(exchange).endswith("paper_trade") for exchange in required_exchanges]): - await self.update_all_secure_configs_legacy() connections = await UserBalances.instance().update_exchanges(exchanges=required_exchanges) invalid_conns.update({ex: err_msg for ex, err_msg in connections.items() if ex in required_exchanges and err_msg is not None}) @@ -115,7 +114,9 @@ def missing_configurations_legacy( config_map = self.strategy_config_map missing_configs = [] if not isinstance(config_map, ClientConfigAdapter): - missing_configs = missing_required_configs_legacy(get_strategy_config_map(self.strategy_name)) + missing_configs = missing_required_configs_legacy( + get_strategy_config_map(self.strategy_name) + ) return missing_globals + missing_configs def validate_configs( diff --git a/hummingbot/client/config/config_crypt.py b/hummingbot/client/config/config_crypt.py index c68b00cb25..18eae9d641 100644 --- a/hummingbot/client/config/config_crypt.py +++ b/hummingbot/client/config/config_crypt.py @@ -1,71 +1,83 @@ -from eth_account import Account -from hummingbot.core.utils.wallet_setup import get_key_file_path +import binascii import json -import os +from abc import ABC, abstractmethod + +from eth_account import Account from eth_keyfile.keyfile import ( + DKLEN, + SCRYPT_P, + SCRYPT_R, Random, - get_default_work_factor_for_kdf, _pbkdf2_hash, - DKLEN, - encode_hex_no_prefix, _scrypt_hash, - SCRYPT_R, - SCRYPT_P, big_endian_to_int, + encode_hex_no_prefix, encrypt_aes_ctr, + get_default_work_factor_for_kdf, + int_to_big_endian, keccak, - int_to_big_endian ) -from hummingbot.client.settings import ENCYPTED_CONF_PREFIX, ENCYPTED_CONF_POSTFIX - - -def list_encrypted_file_paths(): - file_paths = [] - for f in sorted(os.listdir(get_key_file_path())): - f_path = os.path.join(get_key_file_path(), f) - if os.path.isfile(f_path) and f.startswith(ENCYPTED_CONF_PREFIX) and f.endswith(ENCYPTED_CONF_POSTFIX): - file_paths.append(f_path) - return file_paths - - -def encrypted_file_path(config_key: str): - return os.path.join(get_key_file_path(), f"{ENCYPTED_CONF_PREFIX}{config_key}{ENCYPTED_CONF_POSTFIX}") - - -def secure_config_key(encrypted_file_path: str): - _, file_name = os.path.split(encrypted_file_path) - return file_name[file_name.find(ENCYPTED_CONF_PREFIX) + len(ENCYPTED_CONF_PREFIX): - file_name.find(ENCYPTED_CONF_POSTFIX)] - - -def encrypted_file_exists(config_key: str): - return os.path.exists(encrypted_file_path(config_key)) - - -def encrypt_n_save_config_value(config_key, config_value, password): - """ - encrypt configuration value and store in a file, file name is derived from config_var key (in conf folder) - """ - password_bytes = password.encode() - message = config_value.encode() - encrypted = _create_v3_keyfile_json(message, password_bytes) - file_path = encrypted_file_path(config_key) - with open(file_path, 'w+') as f: - f.write(json.dumps(encrypted)) - - -def decrypt_config_value(config_key, password): - if not encrypted_file_exists(config_key): - return None - file_path = encrypted_file_path(config_key) - return decrypt_file(file_path, password) - - -def decrypt_file(file_path, password): - with open(file_path, 'r') as f: - encrypted = f.read() - secured_value = Account.decrypt(encrypted, password) - return secured_value.decode() +from pydantic import SecretStr + +from hummingbot import root_path + +PASSWORD_VERIFICATION_WORD = "HummingBot" +PASSWORD_VERIFICATION_PATH = root_path() / ".password_verification" + + +class BaseSecretsManager(ABC): + def __init__(self, password: str): + self._password = password + + @property + def password(self) -> SecretStr: + return SecretStr(self._password) + + @abstractmethod + def encrypt_secret_value(self, attr: str, value: str): + pass + + @abstractmethod + def decrypt_secret_value(self, attr: str, value: str) -> str: + pass + + +class ETHKeyFileSecretManger(BaseSecretsManager): + def encrypt_secret_value(self, attr: str, value: str): + if self._password is None: + raise ValueError(f"Could not encrypt secret attribute {attr} because no password was provided.") + password_bytes = self._password.encode() + value_bytes = value.encode() + keyfile_json = _create_v3_keyfile_json(value_bytes, password_bytes) + json_str = json.dumps(keyfile_json) + encrypted_value = binascii.hexlify(json_str.encode()).decode() + return encrypted_value + + def decrypt_secret_value(self, attr: str, value: str) -> str: + if self._password is None: + raise ValueError(f"Could not decrypt secret attribute {attr} because no password was provided.") + value = binascii.unhexlify(value) + decrypted_value = Account.decrypt(value.decode(), self._password).decode() + return decrypted_value + + +def store_password_verification(secrets_manager: BaseSecretsManager): + encrypted_word = secrets_manager.encrypt_secret_value(PASSWORD_VERIFICATION_WORD, PASSWORD_VERIFICATION_WORD) + with open(PASSWORD_VERIFICATION_PATH, "w") as f: + f.write(encrypted_word) + + +def validate_password(secrets_manager: BaseSecretsManager) -> bool: + valid = False + with open(PASSWORD_VERIFICATION_PATH, "r") as f: + encrypted_word = f.read() + try: + decrypted_word = secrets_manager.decrypt_secret_value(PASSWORD_VERIFICATION_WORD, encrypted_word) + valid = decrypted_word == PASSWORD_VERIFICATION_WORD + except ValueError as e: + if str(e) != "MAC mismatch": + raise e + return valid def _create_v3_keyfile_json(message_to_encrypt, password, kdf="pbkdf2", work_factor=None): diff --git a/hummingbot/client/config/config_data_types.py b/hummingbot/client/config/config_data_types.py index dbd1c0721e..4deea70e6c 100644 --- a/hummingbot/client/config/config_data_types.py +++ b/hummingbot/client/config/config_data_types.py @@ -8,6 +8,7 @@ from hummingbot.client.config.config_methods import strategy_config_schema_encoder from hummingbot.client.config.config_validators import ( + validate_connector, validate_exchange, validate_market_trading_pair, validate_strategy, @@ -25,6 +26,7 @@ class ClientFieldData: prompt: Optional[Callable[['BaseClientModel'], str]] = None prompt_on_new: bool = False is_secure: bool = False + is_connect_key: bool = False class BaseClientModel(BaseModel): @@ -70,9 +72,9 @@ def validate_strategy(cls, v: str): class BaseTradingStrategyConfigMap(BaseStrategyConfigMap): - exchange: ClientConfigEnum( + exchange: ClientConfigEnum( # rebuild the exchanges enum value="Exchanges", # noqa: F821 - names={e: e for e in AllConnectorSettings.get_connector_settings().keys()}, + names={e: e for e in AllConnectorSettings.get_all_connectors()}, type=str, ) = Field( default=..., @@ -108,7 +110,7 @@ def validate_exchange(cls, v: str): raise ValueError(ret) cls.__fields__["exchange"].type_ = ClientConfigEnum( # rebuild the exchanges enum value="Exchanges", # noqa: F821 - names={e: e for e in AllConnectorSettings.get_connector_settings().keys()}, + names={e: e for e in AllConnectorSettings.get_all_connectors()}, type=str, ) return v @@ -120,3 +122,20 @@ def validate_exchange_trading_pair(cls, v: str, values: Dict): if ret is not None: raise ValueError(ret) return v + + +class BaseConnectorConfigMap(BaseClientModel): + connector: str = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda mi: "What is your connector?", + prompt_on_new=True, + ), + ) + + @validator("connector", pre=True) + def validate_connector(cls, v: str): + ret = validate_connector(v) + if ret is not None: + raise ValueError(ret) + return v diff --git a/hummingbot/client/config/config_helpers.py b/hummingbot/client/config/config_helpers.py index dc36c73a82..f0534d69e8 100644 --- a/hummingbot/client/config/config_helpers.py +++ b/hummingbot/client/config/config_helpers.py @@ -7,13 +7,14 @@ from dataclasses import dataclass from datetime import date, datetime, time from decimal import Decimal -from os import listdir, unlink +from os import listdir, scandir, unlink from os.path import isfile, join -from typing import Any, Callable, Dict, Generator, List, Optional, Union +from pathlib import Path +from typing import Any, Callable, Dict, Generator, List, Optional, Type, Union import ruamel.yaml import yaml -from pydantic import ValidationError +from pydantic import SecretStr, ValidationError from pydantic.fields import FieldInfo from pydantic.main import ModelMetaclass, validate_model from yaml import SafeDumper @@ -23,16 +24,18 @@ from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.fee_overrides_config_map import fee_overrides_config_map, init_fee_overrides_config from hummingbot.client.config.global_config_map import global_config_map -from hummingbot.client.config.security import Security from hummingbot.client.settings import ( - CONF_FILE_PATH, + CONF_DIR_PATH, CONF_POSTFIX, CONF_PREFIX, + CONNECTORS_CONF_DIR_PATH, GLOBAL_CONFIG_PATH, + STRATEGIES_CONF_DIR_PATH, TEMPLATE_PATH, TRADE_FEES_CONFIG_PATH, AllConnectorSettings, ) +from hummingbot.connector.other.celo.celo_data_types import KEYS as CELO_KEYS # Use ruamel.yaml to preserve order and comments in .yml file yaml_parser = ruamel.yaml.YAML() # legacy @@ -88,6 +91,7 @@ class ConfigTraversalItem: printable_value: str client_field_data: Optional[ClientFieldData] field_info: FieldInfo + type_: Type class ClientConfigAdapter: @@ -133,11 +137,7 @@ def is_required(self, attr: str) -> bool: def keys(self) -> Generator[str, None, None]: return self._hb_config.__fields__.keys() - def setattr_no_validation(self, attr: str, value: Any): - with self._disable_validation(): - setattr(self, attr, value) - - def traverse(self) -> Generator[ConfigTraversalItem, None, None]: + def traverse(self, secure: bool = True) -> Generator[ConfigTraversalItem, None, None]: """The intended use for this function is to simplify config map traversals in the client code. If the field is missing, its value will be set to `None` and its printable value will be set to @@ -145,20 +145,18 @@ def traverse(self) -> Generator[ConfigTraversalItem, None, None]: """ depth = 0 for attr, field in self._hb_config.__fields__.items(): + field_info = field.field_info + type_ = field.type_ if hasattr(self, attr): value = getattr(self, attr) - printable_value = ( - str(value) if not isinstance(value, ClientConfigAdapter) else value.hb_config.Config.title - ) - field_info = field.field_info + printable_value = self._get_printable_value(value, secure) client_field_data = field_info.extra.get("client_data") else: value = None printable_value = "&cMISSING_AND_REQUIRED" client_field_data = self.get_client_data(attr) - field_info = self._hb_config.__fields__[attr].field_info yield ConfigTraversalItem( - depth, attr, attr, value, printable_value, client_field_data, field_info + depth, attr, attr, value, printable_value, client_field_data, field_info, type_ ) if isinstance(value, ClientConfigAdapter): for traversal_item in value.traverse(): @@ -190,7 +188,9 @@ def get_description(self, attr_name: str) -> str: return self._hb_config.__fields__[attr_name].field_info.description def generate_yml_output_str_with_comments(self) -> str: - original_fragments = yaml.safe_dump(self._dict_in_conf_order(), sort_keys=False).split("\n") + conf_dict = self._dict_in_conf_order() + self._encrypt_secrets(conf_dict) + original_fragments = yaml.safe_dump(conf_dict, sort_keys=False).split("\n") fragments_with_comments = [self._generate_title()] self._add_model_fragments(fragments_with_comments, original_fragments) fragments_with_comments.append("\n") # EOF empty line @@ -199,8 +199,9 @@ def generate_yml_output_str_with_comments(self) -> str: def validate_model(self) -> List[str]: results = validate_model(type(self._hb_config), json.loads(self._hb_config.json())) - self._hb_config = self._hb_config.__class__.construct() - for key, value in results[0].items(): + conf_dict = results[0] + self._decrypt_secrets(conf_dict) + for key, value in conf_dict.items(): self.setattr_no_validation(key, value) errors = results[2] validation_errors = [] @@ -212,12 +213,26 @@ def validate_model(self) -> List[str]: ] return validation_errors + def setattr_no_validation(self, attr: str, value: Any): + with self._disable_validation(): + setattr(self, attr, value) + @contextlib.contextmanager def _disable_validation(self): self._hb_config.Config.validate_assignment = False yield self._hb_config.Config.validate_assignment = True + @staticmethod + def _get_printable_value(value: Any, secure: bool) -> str: + if isinstance(value, ClientConfigAdapter): + printable_value = value.hb_config.Config.title + elif isinstance(value, SecretStr) and not secure: + printable_value = value.get_secret_value() + else: + printable_value = str(value) + return printable_value + def _dict_in_conf_order(self) -> Dict[str, Any]: d = {} for attr in self._hb_config.__fields__.keys(): @@ -227,6 +242,21 @@ def _dict_in_conf_order(self) -> Dict[str, Any]: d[attr] = value return d + def _encrypt_secrets(self, conf_dict: Dict[str, Any]): + from hummingbot.client.config.security import Security # avoids circular import + for attr, value in conf_dict.items(): + attr_type = self._hb_config.__fields__[attr].type_ + if attr_type == SecretStr: + conf_dict[attr] = Security.secrets_manager.encrypt_secret_value(attr, value.get_secret_value()) + + def _decrypt_secrets(self, conf_dict: Dict[str, Any]): + from hummingbot.client.config.security import Security # avoids circular import + for attr, value in conf_dict.items(): + attr_type = self._hb_config.__fields__[attr].type_ + if attr_type == SecretStr: + decrypted_value = Security.secrets_manager.decrypt_secret_value(attr, value.get_secret_value()) + conf_dict[attr] = SecretStr(decrypted_value) + def _generate_title(self) -> str: title = f"{self._hb_config.Config.title}" title = self._adorn_title(title) @@ -355,20 +385,20 @@ async def copy_strategy_template(strategy: str) -> str: old_path = get_strategy_template_path(strategy) i = 0 new_fname = f"{CONF_PREFIX}{strategy}{CONF_POSTFIX}_{i}.yml" - new_path = join(CONF_FILE_PATH, new_fname) + new_path = STRATEGIES_CONF_DIR_PATH / new_fname while isfile(new_path): new_fname = f"{CONF_PREFIX}{strategy}{CONF_POSTFIX}_{i}.yml" - new_path = join(CONF_FILE_PATH, new_fname) + new_path = STRATEGIES_CONF_DIR_PATH / new_fname i += 1 shutil.copy(old_path, new_path) return new_fname -def get_strategy_template_path(strategy: str) -> str: +def get_strategy_template_path(strategy: str) -> Path: """ Given the strategy name, return its template config `yml` file name. """ - return join(TEMPLATE_PATH, f"{CONF_PREFIX}{strategy}{CONF_POSTFIX}_TEMPLATE.yml") + return TEMPLATE_PATH / f"{CONF_PREFIX}{strategy}{CONF_POSTFIX}_TEMPLATE.yml" def _merge_dicts(*args: Dict[str, ConfigVar]) -> OrderedDict: @@ -425,13 +455,19 @@ def get_strategy_starter_file(strategy: str) -> Callable: logging.getLogger().error(e, exc_info=True) -def strategy_name_from_file(file_path: str) -> str: +def strategy_name_from_file(file_path: Path) -> str: data = read_yml_file(file_path) strategy = data.get("strategy") return strategy -def validate_strategy_file(file_path: str) -> Optional[str]: +def connector_name_from_file(file_path: Path) -> str: + data = read_yml_file(file_path) + connector = data["connector"] + return connector + + +def validate_strategy_file(file_path: Path) -> Optional[str]: if not isfile(file_path): return f"{file_path} file does not exist." strategy = strategy_name_from_file(file_path) @@ -442,7 +478,7 @@ def validate_strategy_file(file_path: str) -> Optional[str]: return None -def read_yml_file(yml_path: str) -> Dict[str, Any]: +def read_yml_file(yml_path: Path) -> Dict[str, Any]: with open(yml_path, "r") as file: data = yaml.safe_load(file) or {} return dict(data) @@ -452,7 +488,8 @@ def get_strategy_pydantic_config_cls(strategy_name: str) -> Optional[ModelMetacl pydantic_cm_class = None try: pydantic_cm_pkg = f"{strategy_name}_config_map_pydantic" - if isfile(f"{root_path()}/hummingbot/strategy/{strategy_name}/{pydantic_cm_pkg}.py"): + pydantic_cm_path = root_path() / "hummingbot" / "strategy" / strategy_name / f"{pydantic_cm_pkg}.py" + if pydantic_cm_path.exists(): pydantic_cm_class_name = f"{''.join([s.capitalize() for s in strategy_name.split('_')])}ConfigMap" pydantic_cm_mod = __import__(f"hummingbot.strategy.{strategy_name}.{pydantic_cm_pkg}", fromlist=[f"{pydantic_cm_class_name}"]) @@ -462,27 +499,70 @@ def get_strategy_pydantic_config_cls(strategy_name: str) -> Optional[ModelMetacl return pydantic_cm_class -async def load_strategy_config_map_from_file(yml_path: str) -> Union[ClientConfigAdapter, Dict[str, ConfigVar]]: +async def load_strategy_config_map_from_file(yml_path: Path) -> Union[ClientConfigAdapter, Dict[str, ConfigVar]]: strategy_name = strategy_name_from_file(yml_path) config_cls = get_strategy_pydantic_config_cls(strategy_name) if config_cls is None: # legacy config_map = get_strategy_config_map(strategy_name) template_path = get_strategy_template_path(strategy_name) - await load_yml_into_cm_legacy(yml_path, template_path, config_map) + await load_yml_into_cm_legacy(str(yml_path), str(template_path), config_map) else: config_data = read_yml_file(yml_path) hb_config = config_cls.construct() config_map = ClientConfigAdapter(hb_config) - for key in config_map.keys(): - if key in config_data: - config_map.setattr_no_validation(key, config_data[key]) - try: - config_map.validate_model() # try to coerce the values to the appropriate type - except Exception: - pass # but don't raise if it fails + _load_yml_data_into_map(config_data, config_map) return config_map +def load_connector_config_map_from_file(yml_path: Path) -> ClientConfigAdapter: + config_data = read_yml_file(yml_path) + connector_name = connector_name_from_file(yml_path) + hb_config = get_connector_hb_config(connector_name) + config_map = ClientConfigAdapter(hb_config) + _load_yml_data_into_map(config_data, config_map) + return config_map + + +def get_connector_hb_config(connector_name: str) -> BaseClientModel: + if connector_name == "celo": + hb_config = CELO_KEYS + else: + hb_config = AllConnectorSettings.get_connector_config_keys(connector_name) + return hb_config + + +def api_keys_from_connector_config_map(cm: ClientConfigAdapter) -> Dict[str, str]: + api_keys = {} + for c in cm.traverse(): + if c.value is not None and c.client_field_data is not None and c.client_field_data.is_connect_key: + value = c.value.get_secret_value() if isinstance(c.value, SecretStr) else c.value + api_keys[c.attr] = value + return api_keys + + +def get_connector_config_yml_path(connector_name: str) -> Path: + connector_path = Path(CONNECTORS_CONF_DIR_PATH) / f"{connector_name}.yml" + return connector_path + + +def list_connector_configs() -> List[Path]: + connector_configs = [ + Path(f.path) for f in scandir(str(CONNECTORS_CONF_DIR_PATH)) + if f.is_file() and not f.name.startswith("_") and not f.name.startswith(".") + ] + return connector_configs + + +def _load_yml_data_into_map(yml_data: Dict[str, Any], cm: ClientConfigAdapter): + for key in cm.keys(): + if key in yml_data: + cm.setattr_no_validation(key, yml_data[key]) + try: + cm.validate_model() # try coercing values to appropriate type + except Exception: + pass # but don't raise if it fails + + async def load_yml_into_dict(yml_path: str) -> Dict[str, Any]: data = {} if isfile(yml_path): @@ -526,10 +606,8 @@ async def load_yml_into_cm_legacy(yml_path: str, template_file_path: str, cm: Di logging.getLogger().error(f"Cannot find corresponding config to key {key} in template.") continue - # Skip this step since the values are not saved in the yml file if cvar.is_secure: - cvar.value = Security.decrypted_value(key) - continue + raise DeprecationWarning("Secure values are no longer supported in legacy configs.") val_in_file = data.get(key, None) if (val_in_file is None or val_in_file == "") and cvar.default is not None: @@ -565,16 +643,19 @@ async def read_system_configs_from_yml(): Read global config and selected strategy yml files and save the values to corresponding config map If a yml file is outdated, it gets reformatted with the new template """ - await load_yml_into_cm_legacy(GLOBAL_CONFIG_PATH, join(TEMPLATE_PATH, "conf_global_TEMPLATE.yml"), global_config_map) - await load_yml_into_cm_legacy(TRADE_FEES_CONFIG_PATH, join(TEMPLATE_PATH, "conf_fee_overrides_TEMPLATE.yml"), - fee_overrides_config_map) + await load_yml_into_cm_legacy( + GLOBAL_CONFIG_PATH, str(TEMPLATE_PATH / "conf_global_TEMPLATE.yml"), global_config_map + ) + await load_yml_into_cm_legacy( + str(TRADE_FEES_CONFIG_PATH), str(TEMPLATE_PATH / "conf_fee_overrides_TEMPLATE.yml"), fee_overrides_config_map + ) # In case config maps get updated (due to default values) save_system_configs_to_yml() def save_system_configs_to_yml(): save_to_yml_legacy(GLOBAL_CONFIG_PATH, global_config_map) - save_to_yml_legacy(TRADE_FEES_CONFIG_PATH, fee_overrides_config_map) + save_to_yml_legacy(str(TRADE_FEES_CONFIG_PATH), fee_overrides_config_map) async def refresh_trade_fees_config(): @@ -582,8 +663,10 @@ async def refresh_trade_fees_config(): Refresh the trade fees config, after new connectors have been added (e.g. gateway connectors). """ init_fee_overrides_config() - await load_yml_into_cm_legacy(GLOBAL_CONFIG_PATH, join(TEMPLATE_PATH, "conf_global_TEMPLATE.yml"), global_config_map) - save_to_yml_legacy(TRADE_FEES_CONFIG_PATH, fee_overrides_config_map) + await load_yml_into_cm_legacy( + GLOBAL_CONFIG_PATH, str(TEMPLATE_PATH / "conf_global_TEMPLATE.yml"), global_config_map + ) + save_to_yml_legacy(str(TRADE_FEES_CONFIG_PATH), fee_overrides_config_map) def save_to_yml_legacy(yml_path: str, cm: Dict[str, ConfigVar]): @@ -595,11 +678,7 @@ def save_to_yml_legacy(yml_path: str, cm: Dict[str, ConfigVar]): data = yaml_parser.load(stream) or {} for key in cm: cvar = cm.get(key) - if cvar.is_secure: - Security.update_secure_config(key, cvar.value) - if key in data: - data.pop(key) - elif type(cvar.value) == Decimal: + if type(cvar.value) == Decimal: data[key] = float(cvar.value) else: data[key] = cvar.value @@ -609,10 +688,10 @@ def save_to_yml_legacy(yml_path: str, cm: Dict[str, ConfigVar]): logging.getLogger().error("Error writing configs: %s" % (str(e),), exc_info=True) -def save_to_yml(yml_path: str, cm: ClientConfigAdapter): +def save_to_yml(yml_path: Path, cm: ClientConfigAdapter): try: cm_yml_str = cm.generate_yml_output_str_with_comments() - with open(yml_path, "w+") as outfile: + with open(yml_path, "w") as outfile: outfile.write(cm_yml_str) except Exception as e: logging.getLogger().error("Error writing configs: %s" % (str(e),), exc_info=True) @@ -620,7 +699,7 @@ def save_to_yml(yml_path: str, cm: ClientConfigAdapter): async def write_config_to_yml(strategy_name, strategy_file_name): strategy_config_map = get_strategy_config_map(strategy_name) - strategy_file_path = join(CONF_FILE_PATH, strategy_file_name) + strategy_file_path = Path(STRATEGIES_CONF_DIR_PATH) / strategy_file_name if isinstance(strategy_config_map, ClientConfigAdapter): save_to_yml(strategy_file_path, strategy_config_map) else: @@ -635,8 +714,8 @@ async def create_yml_files_legacy(): for fname in listdir(TEMPLATE_PATH): if "_TEMPLATE" in fname and CONF_POSTFIX not in fname: stripped_fname = fname.replace("_TEMPLATE", "") - template_path = join(TEMPLATE_PATH, fname) - conf_path = join(CONF_FILE_PATH, stripped_fname) + template_path = str(TEMPLATE_PATH / fname) + conf_path = join(CONF_DIR_PATH, stripped_fname) if not isfile(conf_path): shutil.copy(template_path, conf_path) @@ -663,10 +742,10 @@ def default_strategy_file_path(strategy: str) -> str: """ i = 1 new_fname = f"{CONF_PREFIX}{short_strategy_name(strategy)}_{i}.yml" - new_path = join(CONF_FILE_PATH, new_fname) - while isfile(new_path): + new_path = STRATEGIES_CONF_DIR_PATH / new_fname + while new_path.is_file(): new_fname = f"{CONF_PREFIX}{short_strategy_name(strategy)}_{i}.yml" - new_path = join(CONF_FILE_PATH, new_fname) + new_path = STRATEGIES_CONF_DIR_PATH / new_fname i += 1 return new_fname @@ -695,12 +774,6 @@ def missing_required_configs_legacy(config_map): return [c for c in config_map.values() if c.required and c.value is None and not c.is_connect_key] -def load_secure_values(config_map): - for key, config in config_map.items(): - if config.is_secure: - config.value = Security.decrypted_value(key) - - def format_config_file_name(file_name): if "." not in file_name: return file_name + ".yml" diff --git a/hummingbot/client/config/global_config_map.py b/hummingbot/client/config/global_config_map.py index ff1d12dcfc..df7bb62c05 100644 --- a/hummingbot/client/config/global_config_map.py +++ b/hummingbot/client/config/global_config_map.py @@ -2,14 +2,14 @@ import random import re from decimal import Decimal -from typing import Callable, Optional, Dict +from typing import Callable, Optional from tabulate import tabulate_formats from hummingbot.client.config.config_methods import using_exchange as using_exchange_pointer from hummingbot.client.config.config_validators import validate_bool, validate_decimal from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.settings import AllConnectorSettings, DEFAULT_KEY_FILE_PATH, DEFAULT_LOG_FILE_PATH +from hummingbot.client.settings import DEFAULT_LOG_FILE_PATH, AllConnectorSettings from hummingbot.core.rate_oracle.rate_oracle import RateOracle, RateOracleSource PMM_SCRIPT_ENABLED_KEY = "pmm_script_enabled" @@ -34,13 +34,6 @@ def validate_pmm_script_file_path(file_path: str) -> Optional[bool]: return f"{file_path} file does not exist." -def connector_keys() -> Dict[str, ConfigVar]: - all_keys = {} - for connector_setting in AllConnectorSettings.get_connector_settings().values(): - all_keys.update(connector_setting.config_keys) - return all_keys - - def validate_rate_oracle_source(value: str) -> Optional[str]: if value not in (r.name for r in RateOracleSource): return f"Invalid source, please choose value from {','.join(r.name for r in RateOracleSource)}" @@ -96,32 +89,11 @@ def global_token_symbol_on_validated(value: str): "conf" ], type_str="list"), - "key_file_path": - ConfigVar(key="key_file_path", - prompt=f"Where would you like to save your private key file? " - f"(default '{DEFAULT_KEY_FILE_PATH}') >>> ", - required_if=lambda: False, - default=DEFAULT_KEY_FILE_PATH), "log_file_path": ConfigVar(key="log_file_path", prompt=f"Where would you like to save your logs? (default '{DEFAULT_LOG_FILE_PATH}') >>> ", required_if=lambda: False, - default=DEFAULT_LOG_FILE_PATH), - - # Required by chosen CEXes or DEXes - "celo_address": - ConfigVar(key="celo_address", - prompt="Enter your Celo account address >>> ", - type_str="str", - required_if=lambda: False, - is_connect_key=True), - "celo_password": - ConfigVar(key="celo_password", - prompt="Enter your Celo account password >>> ", - type_str="str", - required_if=lambda: global_config_map["celo_address"].value is not None, - is_secure=True, - is_connect_key=True), + default=str(DEFAULT_LOG_FILE_PATH)), "kill_switch_enabled": ConfigVar(key="kill_switch_enabled", prompt="Would you like to enable the kill switch? (Yes/No) >>> ", @@ -315,8 +287,6 @@ def global_token_symbol_on_validated(value: str): default="psql"), } -key_config_map = connector_keys() - color_config_map = { # The variables below are usually not prompted during setup process "top-pane": @@ -425,4 +395,4 @@ def global_token_symbol_on_validated(value: str): ), } -global_config_map = {**key_config_map, **main_config_map, **color_config_map, **paper_trade_config_map} +global_config_map = {**main_config_map, **color_config_map, **paper_trade_config_map} diff --git a/hummingbot/client/config/security.py b/hummingbot/client/config/security.py index daae578d5f..a4429404ce 100644 --- a/hummingbot/client/config/security.py +++ b/hummingbot/client/config/security.py @@ -1,143 +1,93 @@ -from hummingbot.client.config.config_crypt import ( - list_encrypted_file_paths, - decrypt_file, - secure_config_key, - encrypted_file_exists, - encrypt_n_save_config_value, - encrypted_file_path -) -from hummingbot.core.utils.wallet_setup import ( - list_wallets, - unlock_wallet, - import_and_save_wallet +import asyncio +from pathlib import Path +from typing import Dict, Optional + +from hummingbot.client.config.config_crypt import PASSWORD_VERIFICATION_PATH, BaseSecretsManager, validate_password +from hummingbot.client.config.config_helpers import ( + ClientConfigAdapter, + api_keys_from_connector_config_map, + connector_name_from_file, + get_connector_config_yml_path, + list_connector_configs, + load_connector_config_map_from_file, + save_to_yml, ) -from hummingbot.client.config.global_config_map import global_config_map -from hummingbot.client.settings import AllConnectorSettings -from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.core.utils.async_call_scheduler import AsyncCallScheduler -import asyncio -from os import unlink +from hummingbot.core.utils.async_utils import safe_ensure_future class Security: __instance = None - password = None + secrets_manager: Optional[BaseSecretsManager] = None _secure_configs = {} - _private_keys = {} _decryption_done = asyncio.Event() @staticmethod - def new_password_required(): - encrypted_files = list_encrypted_file_paths() - wallets = list_wallets() - return len(encrypted_files) == 0 and len(wallets) == 0 + def new_password_required() -> bool: + return not PASSWORD_VERIFICATION_PATH.exists() - @staticmethod - def any_encryped_files(): - encrypted_files = list_encrypted_file_paths() - return len(encrypted_files) > 0 - - @staticmethod - def any_wallets(): - return len(list_wallets()) > 0 + @classmethod + def any_secure_configs(cls): + return len(cls._secure_configs) > 0 @staticmethod - def encrypted_file_exists(config_key): - return encrypted_file_exists(config_key) + def connector_config_file_exists(connector_name: str) -> bool: + connector_configs_path = get_connector_config_yml_path(connector_name) + return connector_configs_path.exists() @classmethod - def login(cls, password): - encrypted_files = list_encrypted_file_paths() - wallets = list_wallets() - if encrypted_files: - try: - decrypt_file(encrypted_files[0], password) - except ValueError as err: - if str(err) == "MAC mismatch": - return False - raise err - elif wallets: - try: - unlock_wallet(wallets[0], password) - except ValueError as err: - if str(err) == "MAC mismatch": - return False - raise err - Security.password = password + def login(cls, secrets_manager: BaseSecretsManager) -> bool: + if not validate_password(secrets_manager): + return False + cls.secrets_manager = secrets_manager coro = AsyncCallScheduler.shared_instance().call_async(cls.decrypt_all, timeout_seconds=30) safe_ensure_future(coro) return True - @classmethod - def decrypt_file(cls, file_path): - key_name = secure_config_key(file_path) - cls._secure_configs[key_name] = decrypt_file(file_path, Security.password) - - @classmethod - def unlock_wallet(cls, public_key): - if public_key not in cls._private_keys: - cls._private_keys[public_key] = unlock_wallet(wallet_address=public_key, password=Security.password) - return cls._private_keys[public_key] - @classmethod def decrypt_all(cls): cls._secure_configs.clear() - cls._private_keys.clear() cls._decryption_done.clear() - encrypted_files = list_encrypted_file_paths() + encrypted_files = list_connector_configs() for file in encrypted_files: - cls.decrypt_file(file) - wallets = list_wallets() - for wallet in wallets: - cls.unlock_wallet(wallet) + cls.decrypt_connector_config(file) cls._decryption_done.set() @classmethod - def update_secure_config(cls, key, new_value): - if new_value is None: - return - if encrypted_file_exists(key): - unlink(encrypted_file_path(key)) - encrypt_n_save_config_value(key, new_value, cls.password) - cls._secure_configs[key] = new_value + def decrypt_connector_config(cls, file_path: Path): + connector_name = connector_name_from_file(file_path) + cls._secure_configs[connector_name] = load_connector_config_map_from_file(file_path) @classmethod - def add_private_key(cls, private_key) -> str: - # Add private key and return the account address - account = import_and_save_wallet(cls.password, private_key) - cls._private_keys[account.address] = account.privateKey - return account.address - - @classmethod - def update_config_map(cls, config_map): - for config in config_map.values(): - if config.is_secure and config.value is None: - config.value = cls.decrypted_value(config.key) + def update_secure_config(cls, connector_config: ClientConfigAdapter): + connector_name = connector_config.connector + file_path = get_connector_config_yml_path(connector_name) + save_to_yml(file_path, connector_config) + cls._secure_configs[connector_name] = connector_config @classmethod def is_decryption_done(cls): return cls._decryption_done.is_set() @classmethod - def decrypted_value(cls, key): + def decrypted_value(cls, key: str) -> Optional[ClientConfigAdapter]: return cls._secure_configs.get(key, None) @classmethod - def all_decrypted_values(cls): + def all_decrypted_values(cls) -> Dict[str, ClientConfigAdapter]: return cls._secure_configs.copy() - @classmethod - def private_keys(cls): - return cls._private_keys.copy() - @classmethod async def wait_til_decryption_done(cls): await cls._decryption_done.wait() @classmethod - async def api_keys(cls, exchange): + async def api_keys(cls, connector_name: str) -> Dict[str, Optional[str]]: await cls.wait_til_decryption_done() - exchange_configs = [c for c in global_config_map.values() - if c.key in AllConnectorSettings.get_connector_settings()[exchange].config_keys - and c.key in cls._secure_configs] - return {c.key: cls.decrypted_value(c.key) for c in exchange_configs} + connector_config = cls.decrypted_value(connector_name) + keys = ( + api_keys_from_connector_config_map(connector_config) + if connector_config is not None + else {} + ) + return keys diff --git a/hummingbot/client/settings.py b/hummingbot/client/settings.py index ca5189ef5d..22fb112e41 100644 --- a/hummingbot/client/settings.py +++ b/hummingbot/client/settings.py @@ -9,12 +9,14 @@ from enum import Enum from os import DirEntry, scandir from os.path import exists, join, realpath -from typing import Any, Dict, List, NamedTuple, Optional, Set, Union, cast +from typing import TYPE_CHECKING, Any, Dict, List, NamedTuple, Optional, Set, Union, cast from hummingbot import get_strategy_list, root_path -from hummingbot.client.config.config_var import ConfigVar from hummingbot.core.data_type.trade_fee import TradeFeeSchema +if TYPE_CHECKING: + from hummingbot.client.config.config_data_types import BaseClientModel + # Global variables required_exchanges: Set[str] = set() requried_connector_trading_pairs: Dict[str, List[str]] = {} @@ -24,27 +26,27 @@ # Global static values KEYFILE_PREFIX = "key_file_" -KEYFILE_POSTFIX = ".json" -ENCYPTED_CONF_PREFIX = "encrypted_" +KEYFILE_POSTFIX = ".yml" ENCYPTED_CONF_POSTFIX = ".json" -GLOBAL_CONFIG_PATH = "conf/conf_global.yml" -TRADE_FEES_CONFIG_PATH = "conf/conf_fee_overrides.yml" -DEFAULT_KEY_FILE_PATH = "conf/" -DEFAULT_LOG_FILE_PATH = "logs/" +GLOBAL_CONFIG_PATH = str(root_path() / "conf" / "conf_global.yml") +TRADE_FEES_CONFIG_PATH = root_path() / "conf" / "conf_fee_overrides.yml" +DEFAULT_LOG_FILE_PATH = root_path() / "logs" DEFAULT_ETHEREUM_RPC_URL = "https://mainnet.coinalpha.com/hummingbot-test-node" -TEMPLATE_PATH = realpath(join(__file__, "../../templates/")) -CONF_FILE_PATH = "conf/" +TEMPLATE_PATH = root_path() / "hummingbot" / "templates" +CONF_DIR_PATH = root_path() / "conf" +STRATEGIES_CONF_DIR_PATH = CONF_DIR_PATH / "strategies" +CONNECTORS_CONF_DIR_PATH = CONF_DIR_PATH / "connectors" CONF_PREFIX = "conf_" CONF_POSTFIX = "_strategy" -PMM_SCRIPTS_PATH = realpath(join(__file__, "../../../pmm_scripts/")) +PMM_SCRIPTS_PATH = root_path() / "pmm_scripts" SCRIPT_STRATEGIES_MODULE = "scripts" -SCRIPT_STRATEGIES_PATH = realpath(join(__file__, f"../../../{SCRIPT_STRATEGIES_MODULE}/")) -CERTS_PATH = "certs/" +SCRIPT_STRATEGIES_PATH = root_path() / SCRIPT_STRATEGIES_MODULE +CERTS_PATH = root_path() / "certs" # Certificates for securely communicating with the gateway api -GATEAWAY_CA_CERT_PATH = realpath(join(__file__, join(f"../../../{CERTS_PATH}/ca_cert.pem"))) -GATEAWAY_CLIENT_CERT_PATH = realpath(join(__file__, join(f"../../../{CERTS_PATH}/client_cert.pem"))) -GATEAWAY_CLIENT_KEY_PATH = realpath(join(__file__, join(f"../../../{CERTS_PATH}/client_key.pem"))) +GATEAWAY_CA_CERT_PATH = CERTS_PATH / "ca_cert.pem" +GATEAWAY_CLIENT_CERT_PATH = CERTS_PATH / "client_cert.pem" +GATEAWAY_CLIENT_KEY_PATH = CERTS_PATH / "client_key.pem" class ConnectorType(Enum): @@ -61,7 +63,7 @@ class ConnectorType(Enum): class GatewayConnectionSetting: @staticmethod def conf_path() -> str: - return realpath(join(CONF_FILE_PATH, "gateway_connections.json")) + return realpath(join(CONF_DIR_PATH, "gateway_connections.json")) @staticmethod def load() -> List[Dict[str, str]]: @@ -120,7 +122,7 @@ class ConnectorSetting(NamedTuple): centralised: bool use_ethereum_wallet: bool trade_fee_schema: TradeFeeSchema - config_keys: Dict[str, ConfigVar] + config_keys: Optional["BaseClientModel"] is_sub_domain: bool parent_name: Optional[str] domain_parameter: Optional[str] @@ -154,9 +156,12 @@ def class_name(self) -> str: return "".join(splited_name) return "".join([o.capitalize() for o in self.module_name().split("_")]) - def conn_init_parameters(self, api_keys: Dict[str, Any] = {}) -> Dict[str, Any]: + def conn_init_parameters(self, api_keys: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + api_keys = api_keys or {} if self.uses_gateway_generic_connector(): # init parameters for gateway connectors - params: Dict[str, Any] = {k: v.value for k, v in self.config_keys.items()} + params = {} + if self.config_keys is not None: + params: Dict[str, Any] = {k: v.value for k, v in self.config_keys.items()} connector_spec: Dict[str, str] = GatewayConnectionSetting.get_connector_spec_from_market_name(self.name) params.update( connector_name=connector_spec["connector"], @@ -199,7 +204,7 @@ def create_connector_settings(cls): connector_exceptions = ["paper_trade"] type_dirs: List[DirEntry] = [ - cast(DirEntry, f) for f in scandir(f"{root_path()}/hummingbot/connector") + cast(DirEntry, f) for f in scandir(f"{root_path() / 'hummingbot' / 'connector'}") if f.is_dir() ] for type_dir in type_dirs: @@ -229,7 +234,7 @@ def create_connector_settings(cls): example_pair=getattr(util_module, "EXAMPLE_PAIR", ""), use_ethereum_wallet=getattr(util_module, "USE_ETHEREUM_WALLET", False), trade_fee_schema=trade_fee_schema, - config_keys=getattr(util_module, "KEYS", {}), + config_keys=getattr(util_module, "KEYS", None), is_sub_domain=False, parent_name=None, domain_parameter=None, @@ -269,7 +274,7 @@ def create_connector_settings(cls): example_pair="WETH-USDC", use_ethereum_wallet=False, trade_fee_schema=trade_fee_schema, - config_keys={}, + config_keys=None, is_sub_domain=False, parent_name=None, domain_parameter=None, @@ -298,12 +303,34 @@ def initialize_paper_trade_settings(cls, paper_trade_exchanges: List[str]): ) cls.all_connector_settings.update({f"{e}_paper_trade": paper_trade_settings}) + @classmethod + def get_all_connectors(cls) -> List[str]: + """Avoids circular import problems introduced by `create_connector_settings`.""" + connector_names = [] + type_dirs: List[DirEntry] = [ + cast(DirEntry, f) for f in + scandir(f"{root_path() / 'hummingbot' / 'connector'}") + if f.is_dir() + ] + for type_dir in type_dirs: + connector_dirs: List[DirEntry] = [ + cast(DirEntry, f) for f in scandir(type_dir.path) + if f.is_dir() and exists(join(f.path, "__init__.py")) + ] + connector_names.extend( + [connector_dir.name for connector_dir in connector_dirs]) + return connector_names + @classmethod def get_connector_settings(cls) -> Dict[str, ConnectorSetting]: if len(cls.all_connector_settings) == 0: cls.all_connector_settings = cls.create_connector_settings() return cls.all_connector_settings + @classmethod + def get_connector_config_keys(cls, connector: str) -> Optional["BaseClientModel"]: + return cls.get_connector_settings()[connector].config_keys + @classmethod def get_exchange_names(cls) -> Set[str]: return {cs.name for cs in cls.all_connector_settings.values() if cs.type is ConnectorType.Exchange} diff --git a/hummingbot/client/ui/__init__.py b/hummingbot/client/ui/__init__.py index bad9f1b5aa..382db3d3ff 100644 --- a/hummingbot/client/ui/__init__.py +++ b/hummingbot/client/ui/__init__.py @@ -1,11 +1,16 @@ +from os.path import dirname, join, realpath +from typing import Optional, Type + from prompt_toolkit.shortcuts import input_dialog, message_dialog from prompt_toolkit.styles import Style -from os.path import join, realpath, dirname +from hummingbot.client.config.config_crypt import BaseSecretsManager, store_password_verification from hummingbot.client.config.global_config_map import color_config_map +from hummingbot.client.config.security import Security import sys; sys.path.insert(0, realpath(join(__file__, "../../../"))) + with open(realpath(join(dirname(__file__), '../../VERSION'))) as version_file: version = version_file.read().strip() @@ -80,36 +85,35 @@ def show_welcome(): style=dialog_style).run() -def login_prompt(): - from hummingbot.client.config.security import Security - import time - +def login_prompt(secrets_manager_cls: Type[BaseSecretsManager]) -> Optional[BaseSecretsManager]: err_msg = None + secrets_manager = None if Security.new_password_required(): show_welcome() password = input_dialog( title="Set Password", text="Create a password to protect your sensitive data. " "This password is not shared with us nor with anyone else, so please store it securely." + "\n\nIf you have used hummingbot before and already have secure configs stored," + " input your previous password in this prompt, then run the scripts/conf_migration_script.py script" + " to migrate your existing secure configs to the new management system." "\n\nEnter your new password:", password=True, style=dialog_style).run() if password is None: - return False + return None re_password = input_dialog( title="Set Password", text="Please re-enter your password:", password=True, style=dialog_style).run() if re_password is None: - return False + return None if password != re_password: err_msg = "Passwords entered do not match, please try again." else: - Security.login(password) - # encrypt current timestamp as a dummy to prevent promping for password if bot exits without connecting an exchange - dummy = f"{time.time()}" - Security.update_secure_config("default", dummy) + secrets_manager = secrets_manager_cls(password) + store_password_verification(secrets_manager) else: password = input_dialog( title="Welcome back to Hummingbot", @@ -117,13 +121,14 @@ def login_prompt(): password=True, style=dialog_style).run() if password is None: - return False - if not Security.login(password): + return None + secrets_manager = secrets_manager_cls(password) + if not Security.login(secrets_manager): err_msg = "Invalid password - please try again." if err_msg is not None: message_dialog( title='Error', text=err_msg, style=dialog_style).run() - return login_prompt() - return True + return login_prompt(secrets_manager_cls) + return secrets_manager diff --git a/hummingbot/client/ui/completer.py b/hummingbot/client/ui/completer.py index 09104f2480..7a75d5eb89 100644 --- a/hummingbot/client/ui/completer.py +++ b/hummingbot/client/ui/completer.py @@ -8,18 +8,17 @@ from hummingbot.client.command.connect_command import OPTIONS as CONNECT_OPTIONS from hummingbot.client.settings import ( - CONF_FILE_PATH, GATEWAY_CONNECTORS, PMM_SCRIPTS_PATH, SCRIPT_STRATEGIES_PATH, STRATEGIES, + STRATEGIES_CONF_DIR_PATH, AllConnectorSettings, ) from hummingbot.client.ui.parser import ThrowingArgumentParser from hummingbot.core.rate_oracle.rate_oracle import RateOracleSource from hummingbot.core.utils.gateway_config_utils import list_gateway_wallets from hummingbot.core.utils.trading_pair_fetcher import TradingPairFetcher -from hummingbot.core.utils.wallet_setup import list_wallets def file_name_list(path, file_extension): @@ -32,7 +31,7 @@ class HummingbotCompleter(Completer): def __init__(self, hummingbot_application): super(HummingbotCompleter, self).__init__() self.hummingbot_application = hummingbot_application - self._path_completer = WordCompleter(file_name_list(CONF_FILE_PATH, "yml")) + self._path_completer = WordCompleter(file_name_list(str(STRATEGIES_CONF_DIR_PATH), "yml")) self._command_completer = WordCompleter(self.parser.commands, ignore_case=True) self._exchange_completer = WordCompleter(sorted(AllConnectorSettings.get_connector_settings().keys()), ignore_case=True) self._spot_exchange_completer = WordCompleter(sorted(AllConnectorSettings.get_exchange_names()), ignore_case=True) @@ -48,8 +47,8 @@ def __init__(self, hummingbot_application): self._gateway_connect_completer = WordCompleter(GATEWAY_CONNECTORS, ignore_case=True) self._gateway_config_completer = WordCompleter(hummingbot_application.gateway_config_keys, ignore_case=True) self._strategy_completer = WordCompleter(STRATEGIES, ignore_case=True) - self._py_file_completer = WordCompleter(file_name_list(PMM_SCRIPTS_PATH, "py")) - self._script_strategy_completer = WordCompleter(file_name_list(SCRIPT_STRATEGIES_PATH, "py")) + self._py_file_completer = WordCompleter(file_name_list(str(PMM_SCRIPTS_PATH), "py")) + self._script_strategy_completer = WordCompleter(file_name_list(str(SCRIPT_STRATEGIES_PATH), "py")) self._rate_oracle_completer = WordCompleter([r.name for r in RateOracleSource], ignore_case=True) self._gateway_networks = [] self._list_gateway_wallets_parameters = {"wallets": [], "chain": ""} @@ -83,10 +82,6 @@ def _trading_pair_completer(self) -> Completer: trading_pairs = trading_pair_fetcher.trading_pairs.get(market, []) if trading_pair_fetcher.ready and market else [] return WordCompleter(trading_pairs, ignore_case=True, sentence=True) - @property - def _wallet_address_completer(self): - return WordCompleter(list_wallets(), ignore_case=True) - @property def _gateway_network_completer(self): return WordCompleter(self._gateway_networks, ignore_case=True) @@ -180,9 +175,6 @@ def _complete_paths(self, document: Document) -> bool: return (("path" in self.prompt_text and "file" in self.prompt_text) or "import" in text_before_cursor) - def _complete_wallet_addresses(self, document: Document) -> bool: - return "Which wallet" in self.prompt_text - def _complete_gateway_network(self, document: Document) -> bool: return "Which network do you want" in self.prompt_text @@ -228,10 +220,6 @@ def get_completions(self, document: Document, complete_event: CompleteEvent): for c in self._strategy_completer.get_completions(document, complete_event): yield c - elif self._complete_wallet_addresses(document): - for c in self._wallet_address_completer.get_completions(document, complete_event): - yield c - elif self._complete_gateway_network(document): for c in self._gateway_network_completer.get_completions(document, complete_event): yield c diff --git a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_utils.py b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_utils.py index 58ad071ad0..939dbcfdf6 100644 --- a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_utils.py +++ b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_utils.py @@ -2,8 +2,9 @@ import socket from typing import Any, Dict -from hummingbot.client.config.config_methods import using_exchange -from hummingbot.client.config.config_var import ConfigVar +from pydantic import Field, SecretStr + +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData from hummingbot.core.utils.tracking_nonce import get_tracking_nonce CENTRALIZED = True @@ -38,43 +39,56 @@ def is_exchange_information_valid(exchange_info: Dict[str, Any]) -> bool: return exchange_info.get("status", None) == "TRADING" -KEYS = { - "binance_perpetual_api_key": ConfigVar( - key="binance_perpetual_api_key", - prompt="Enter your Binance Perpetual API key >>> ", - required_if=using_exchange("binance_perpetual"), - is_secure=True, - is_connect_key=True, - ), - "binance_perpetual_api_secret": ConfigVar( - key="binance_perpetual_api_secret", - prompt="Enter your Binance Perpetual API secret >>> ", - required_if=using_exchange("binance_perpetual"), - is_secure=True, - is_connect_key=True, - ), -} +class BinancePerpetualConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="binance_perpetual", client_data=None) + binance_perpetual_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Binance Perpetual API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + binance_perpetual_api_secret: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Binance Perpetual API secret", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = BinancePerpetualConfigMap.construct() OTHER_DOMAINS = ["binance_perpetual_testnet"] OTHER_DOMAINS_PARAMETER = {"binance_perpetual_testnet": "binance_perpetual_testnet"} OTHER_DOMAINS_EXAMPLE_PAIR = {"binance_perpetual_testnet": "BTC-USDT"} OTHER_DOMAINS_DEFAULT_FEES = {"binance_perpetual_testnet": [0.02, 0.04]} -OTHER_DOMAINS_KEYS = { - "binance_perpetual_testnet": { - # add keys for testnet - "binance_perpetual_testnet_api_key": ConfigVar( - key="binance_perpetual_testnet_api_key", - prompt="Enter your Binance Perpetual testnet API key >>> ", - required_if=using_exchange("binance_perpetual_testnet"), + + +class BinancePerpetualTestnetConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="binance_perpetual_testnet", client_data=None) + binance_perpetual_testnet_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Binance Perpetual testnet API key", is_secure=True, is_connect_key=True, - ), - "binance_perpetual_testnet_api_secret": ConfigVar( - key="binance_perpetual_testnet_api_secret", - prompt="Enter your Binance Perpetual testnet API secret >>> ", - required_if=using_exchange("binance_perpetual_testnet"), + prompt_on_new=True, + ) + ) + binance_perpetual_testnet_api_secret: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Binance Perpetual testnet API secret", is_secure=True, is_connect_key=True, - ), - } -} + prompt_on_new=True, + ) + ) + + +OTHER_DOMAINS_KEYS = {"binance_perpetual_testnet": BinancePerpetualTestnetConfigMap.construct()} diff --git a/hummingbot/connector/derivative/bybit_perpetual/bybit_perpetual_utils.py b/hummingbot/connector/derivative/bybit_perpetual/bybit_perpetual_utils.py index 80d5cf1f2b..f791ed1016 100644 --- a/hummingbot/connector/derivative/bybit_perpetual/bybit_perpetual_utils.py +++ b/hummingbot/connector/derivative/bybit_perpetual/bybit_perpetual_utils.py @@ -1,7 +1,8 @@ from typing import Dict, List, Optional -from hummingbot.client.config.config_methods import using_exchange -from hummingbot.client.config.config_var import ConfigVar +from pydantic import Field, SecretStr + +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData from hummingbot.connector.derivative.bybit_perpetual import bybit_perpetual_constants as CONSTANTS from hummingbot.connector.utils import split_hb_trading_pair from hummingbot.core.api_throttler.data_types import LinkedLimitWeightPair, RateLimit @@ -96,40 +97,60 @@ def get_next_funding_timestamp(current_timestamp: float) -> float: return float(int_ts - mod + eight_hours) -KEYS = { - "bybit_perpetual_api_key": - ConfigVar(key="bybit_perpetual_api_key", - prompt="Enter your Bybit Perpetual API key >>> ", - required_if=using_exchange("bybit_perpetual"), - is_secure=True, - is_connect_key=True), - "bybit_perpetual_secret_key": - ConfigVar(key="bybit_perpetual_secret_key", - prompt="Enter your Bybit Perpetual secret key >>> ", - required_if=using_exchange("bybit_perpetual"), - is_secure=True, - is_connect_key=True), -} +class BybitPerpetualConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="bybit_perpetual", client_data=None) + bybit_perpetual_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Bybit Perpetual API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + bybit_perpetual_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Bybit Perpetual secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = BybitPerpetualConfigMap.construct() OTHER_DOMAINS = ["bybit_perpetual_testnet"] OTHER_DOMAINS_PARAMETER = {"bybit_perpetual_testnet": "bybit_perpetual_testnet"} OTHER_DOMAINS_EXAMPLE_PAIR = {"bybit_perpetual_testnet": "BTC-USDT"} OTHER_DOMAINS_DEFAULT_FEES = {"bybit_perpetual_testnet": [-0.025, 0.075]} + + +class BybitPerpetualTestnetConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="bybit_perpetual_testnet", client_data=None) + bybit_perpetual_testnet_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Bybit Perpetual Testnet API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + bybit_perpetual_testnet_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Bybit Perpetual Testnet secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + OTHER_DOMAINS_KEYS = { - "bybit_perpetual_testnet": { - "bybit_perpetual_testnet_api_key": - ConfigVar(key="bybit_perpetual_testnet_api_key", - prompt="Enter your Bybit Perpetual Testnet API key >>> ", - required_if=using_exchange("bybit_perpetual_testnet"), - is_secure=True, - is_connect_key=True), - "bybit_perpetual_testnet_secret_key": - ConfigVar(key="bybit_perpetual_testnet_secret_key", - prompt="Enter your Bybit Perpetual Testnet secret key >>> ", - required_if=using_exchange("bybit_perpetual_testnet"), - is_secure=True, - is_connect_key=True), - } + "bybit_perpetual_testnet": BybitPerpetualTestnetConfigMap.construct() } diff --git a/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_utils.py b/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_utils.py index f3aa9271a4..0997015c91 100644 --- a/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_utils.py +++ b/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_utils.py @@ -1,7 +1,8 @@ -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_methods import using_exchange -from hummingbot.core.web_assistant.web_assistants_factory import WebAssistantsFactory +from pydantic import Field, SecretStr + +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData +from hummingbot.core.web_assistant.web_assistants_factory import WebAssistantsFactory CENTRALIZED = True @@ -17,41 +18,62 @@ def build_api_factory() -> WebAssistantsFactory: return api_factory -KEYS = { - "dydx_perpetual_api_key": - ConfigVar(key="dydx_perpetual_api_key", - prompt="Enter your dydx Perpetual API key >>> ", - required_if=using_exchange("dydx_perpetual"), - is_secure=True, - is_connect_key=True), - "dydx_perpetual_api_secret": - ConfigVar(key="dydx_perpetual_api_secret", - prompt="Enter your dydx Perpetual API secret >>> ", - required_if=using_exchange("dydx_perpetual"), - is_secure=True, - is_connect_key=True), - "dydx_perpetual_passphrase": - ConfigVar(key="dydx_perpetual_passphrase", - prompt="Enter your dydx Perpetual API passphrase >>> ", - required_if=using_exchange("dydx_perpetual"), - is_secure=True, - is_connect_key=True), - "dydx_perpetual_account_number": - ConfigVar(key="dydx_perpetual_account_number", - prompt="Enter your dydx Perpetual API account_number >>> ", - required_if=using_exchange("dydx_perpetual"), - is_secure=True, - is_connect_key=True), - "dydx_perpetual_stark_private_key": - ConfigVar(key="dydx_perpetual_stark_private_key", - prompt="Enter your stark private key >>> ", - required_if=using_exchange("dydx_perpetual"), - is_secure=True, - is_connect_key=True), - "dydx_perpetual_ethereum_address": - ConfigVar(key="dydx_perpetual_ethereum_address", - prompt="Enter your ethereum wallet address >>> ", - required_if=using_exchange("dydx_perpetual"), - is_secure=True, - is_connect_key=True), -} +class DydxPerpetualConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="dydx_perpetual", client_data=None) + dydx_perpetual_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your dydx Perpetual API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + dydx_perpetual_api_secret: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your dydx Perpetual API secret", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + dydx_perpetual_passphrase: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your dydx Perpetual API passphrase", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + dydx_perpetual_account_number: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your dydx Perpetual API account_number", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + dydx_perpetual_stark_private_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your stark private key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + dydx_perpetual_ethereum_address: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your ethereum wallet address", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = DydxPerpetualConfigMap.construct() diff --git a/hummingbot/connector/exchange/altmarkets/altmarkets_utils.py b/hummingbot/connector/exchange/altmarkets/altmarkets_utils.py index 8b478576c6..d28acaf913 100644 --- a/hummingbot/connector/exchange/altmarkets/altmarkets_utils.py +++ b/hummingbot/connector/exchange/altmarkets/altmarkets_utils.py @@ -1,17 +1,13 @@ import re +from typing import Any, Dict, Optional, Tuple + from dateutil.parser import parse as dateparse -from typing import ( - Any, - Dict, - Optional, - Tuple, -) +from pydantic import Field, SecretStr +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData from hummingbot.core.utils.tracking_nonce import get_tracking_nonce -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_methods import using_exchange -from .altmarkets_constants import Constants +from .altmarkets_constants import Constants TRADING_PAIR_SPLITTER = re.compile(Constants.TRADING_PAIR_SPLITTER) @@ -78,17 +74,26 @@ def get_new_client_order_id(is_buy: bool, trading_pair: str) -> str: return f"{Constants.HBOT_BROKER_ID}-{side}{base_str}{quote_str}{get_tracking_nonce()}" -KEYS = { - "altmarkets_api_key": - ConfigVar(key="altmarkets_api_key", - prompt=f"Enter your {Constants.EXCHANGE_NAME} API key >>> ", - required_if=using_exchange("altmarkets"), - is_secure=True, - is_connect_key=True), - "altmarkets_secret_key": - ConfigVar(key="altmarkets_secret_key", - prompt=f"Enter your {Constants.EXCHANGE_NAME} secret key >>> ", - required_if=using_exchange("altmarkets"), - is_secure=True, - is_connect_key=True), -} +class AltmarketsConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="altmarkets", client_data=None) + altmarkets_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: f"Enter your {Constants.EXCHANGE_NAME} API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + altmarkets_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: f"Enter your {Constants.EXCHANGE_NAME} secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = AltmarketsConfigMap.construct() diff --git a/hummingbot/connector/exchange/ascend_ex/ascend_ex_utils.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_utils.py index f7aefaa739..6f7333ce03 100644 --- a/hummingbot/connector/exchange/ascend_ex/ascend_ex_utils.py +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_utils.py @@ -3,8 +3,9 @@ import time from typing import Any, Dict, Optional, Tuple -from hummingbot.client.config.config_methods import using_exchange -from hummingbot.client.config.config_var import ConfigVar +from pydantic import Field, SecretStr + +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData from hummingbot.core.web_assistant.auth import AuthBase from hummingbot.core.web_assistant.connections.data_types import RESTRequest from hummingbot.core.web_assistant.rest_pre_processors import RESTPreProcessorBase @@ -127,20 +128,29 @@ def gen_exchange_order_id(userUid: str, client_order_id: str, timestamp: Optiona ] -KEYS = { - "ascend_ex_api_key": - ConfigVar(key="ascend_ex_api_key", - prompt="Enter your AscendEx API key >>> ", - required_if=using_exchange("ascend_ex"), - is_secure=True, - is_connect_key=True), - "ascend_ex_secret_key": - ConfigVar(key="ascend_ex_secret_key", - prompt="Enter your AscendEx secret key >>> ", - required_if=using_exchange("ascend_ex"), - is_secure=True, - is_connect_key=True), -} +class AscendExConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="ascend_ex", client_data=None) + ascend_ex_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your AscendEx API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + ascend_ex_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your AscendEx secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = AscendExConfigMap.construct() def _time(): diff --git a/hummingbot/connector/exchange/beaxy/beaxy_utils.py b/hummingbot/connector/exchange/beaxy/beaxy_utils.py index c7f6aace79..8734141c0e 100644 --- a/hummingbot/connector/exchange/beaxy/beaxy_utils.py +++ b/hummingbot/connector/exchange/beaxy/beaxy_utils.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- +from pydantic import Field, SecretStr -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_methods import using_exchange - +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData CENTRALIZED = True @@ -10,17 +9,27 @@ DEFAULT_FEES = [0.15, 0.25] -KEYS = { - 'beaxy_api_key': - ConfigVar(key='beaxy_api_key', - prompt='Enter your Beaxy API key >>> ', - required_if=using_exchange('beaxy'), - is_secure=True, - is_connect_key=True), - 'beaxy_secret_key': - ConfigVar(key='beaxy_secret_key', - prompt='Enter your Beaxy secret key >>> ', - required_if=using_exchange('beaxy'), - is_secure=True, - is_connect_key=True), -} + +class BeaxyConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="beaxy", client_data=None) + beaxy_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Beaxy API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + beaxy_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Beaxy secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = BeaxyConfigMap.construct() diff --git a/hummingbot/connector/exchange/binance/binance_utils.py b/hummingbot/connector/exchange/binance/binance_utils.py index 5625b42735..b0846268d9 100644 --- a/hummingbot/connector/exchange/binance/binance_utils.py +++ b/hummingbot/connector/exchange/binance/binance_utils.py @@ -1,7 +1,8 @@ from typing import Any, Dict -from hummingbot.client.config.config_methods import using_exchange -from hummingbot.client.config.config_var import ConfigVar +from pydantic import Field, SecretStr + +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData CENTRALIZED = True EXAMPLE_PAIR = "ZRX-ETH" @@ -17,36 +18,56 @@ def is_exchange_information_valid(exchange_info: Dict[str, Any]) -> bool: return exchange_info.get("status", None) == "TRADING" and "SPOT" in exchange_info.get("permissions", list()) -KEYS = { - "binance_api_key": - ConfigVar(key="binance_api_key", - prompt="Enter your Binance API key >>> ", - required_if=using_exchange("binance"), - is_secure=True, - is_connect_key=True), - "binance_api_secret": - ConfigVar(key="binance_api_secret", - prompt="Enter your Binance API secret >>> ", - required_if=using_exchange("binance"), - is_secure=True, - is_connect_key=True), -} +class BinanceConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="binance", client_data=None) + binance_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Binance API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + binance_api_secret: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Binance API secret", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = BinanceConfigMap.construct() OTHER_DOMAINS = ["binance_us"] OTHER_DOMAINS_PARAMETER = {"binance_us": "us"} OTHER_DOMAINS_EXAMPLE_PAIR = {"binance_us": "BTC-USDT"} OTHER_DOMAINS_DEFAULT_FEES = {"binance_us": [0.1, 0.1]} -OTHER_DOMAINS_KEYS = {"binance_us": { - "binance_us_api_key": - ConfigVar(key="binance_us_api_key", - prompt="Enter your Binance US API key >>> ", - required_if=using_exchange("binance_us"), - is_secure=True, - is_connect_key=True), - "binance_us_api_secret": - ConfigVar(key="binance_us_api_secret", - prompt="Enter your Binance US API secret >>> ", - required_if=using_exchange("binance_us"), - is_secure=True, - is_connect_key=True), -}} + + +class BinanceUSConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="binance_us", client_data=None) + binance_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Binance US API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + binance_api_secret: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Binance US API secret", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +OTHER_DOMAINS_KEYS = {"binance_us": BinanceUSConfigMap.construct()} diff --git a/hummingbot/connector/exchange/bitfinex/bitfinex_utils.py b/hummingbot/connector/exchange/bitfinex/bitfinex_utils.py index dee0706b6f..bfcf308d21 100644 --- a/hummingbot/connector/exchange/bitfinex/bitfinex_utils.py +++ b/hummingbot/connector/exchange/bitfinex/bitfinex_utils.py @@ -1,12 +1,10 @@ import math - -from typing import Dict, List, Tuple, Optional from decimal import Decimal +from typing import Dict, List, Optional, Tuple +from pydantic import Field, SecretStr -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_methods import using_exchange - +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData CENTRALIZED = True @@ -20,20 +18,29 @@ DEFAULT_FEES = [0.1, 0.2] -KEYS = { - "bitfinex_api_key": - ConfigVar(key="bitfinex_api_key", - prompt="Enter your Bitfinex API key >>> ", - required_if=using_exchange("bitfinex"), - is_secure=True, - is_connect_key=True), - "bitfinex_secret_key": - ConfigVar(key="bitfinex_secret_key", - prompt="Enter your Bitfinex secret key >>> ", - required_if=using_exchange("bitfinex"), - is_secure=True, - is_connect_key=True), -} +class BitfinexConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="bitfinex", client_data=None) + bitfinex_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Bitfinex API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + bitfinex_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Bitfinex secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = BitfinexConfigMap.construct() # deeply merge two dictionaries diff --git a/hummingbot/connector/exchange/bitmart/bitmart_utils.py b/hummingbot/connector/exchange/bitmart/bitmart_utils.py index 356a070fe9..7902429c47 100644 --- a/hummingbot/connector/exchange/bitmart/bitmart_utils.py +++ b/hummingbot/connector/exchange/bitmart/bitmart_utils.py @@ -2,8 +2,9 @@ import zlib from typing import Dict, List, Tuple -from hummingbot.client.config.config_methods import using_exchange -from hummingbot.client.config.config_var import ConfigVar +from pydantic import Field, SecretStr + +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData from hummingbot.core.data_type.order_book_message import OrderBookMessage from hummingbot.core.data_type.order_book_row import OrderBookRow from hummingbot.core.utils.tracking_nonce import get_tracking_nonce, get_tracking_nonce_low_res @@ -149,26 +150,38 @@ def compress_ws_message(message): return message -KEYS = { - "bitmart_api_key": - ConfigVar(key="bitmart_api_key", - prompt="Enter your BitMart API key >>> ", - required_if=using_exchange("bitmart"), - is_secure=True, - is_connect_key=True), - "bitmart_secret_key": - ConfigVar(key="bitmart_secret_key", - prompt="Enter your BitMart secret key >>> ", - required_if=using_exchange("bitmart"), - is_secure=True, - is_connect_key=True), - "bitmart_memo": - ConfigVar(key="bitmart_memo", - prompt="Enter your BitMart API Memo >>> ", - required_if=using_exchange("bitmart"), - is_secure=True, - is_connect_key=True), -} +class BitmartConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="bitmart", client_data=None) + bitmart_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your BitMart API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + bitmart_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your BitMart secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + bitmart_memo: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your BitMart API Memo", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = BitmartConfigMap.construct() def build_api_factory() -> WebAssistantsFactory: diff --git a/hummingbot/connector/exchange/bittrex/bittrex_utils.py b/hummingbot/connector/exchange/bittrex/bittrex_utils.py index acde0217fd..0807e13f8a 100644 --- a/hummingbot/connector/exchange/bittrex/bittrex_utils.py +++ b/hummingbot/connector/exchange/bittrex/bittrex_utils.py @@ -1,7 +1,8 @@ from decimal import Decimal -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_methods import using_exchange +from pydantic import Field, SecretStr + +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData from hummingbot.core.data_type.trade_fee import TradeFeeSchema CENTRALIZED = True @@ -13,17 +14,27 @@ taker_percent_fee_decimal=Decimal("0.0035"), ) -KEYS = { - "bittrex_api_key": - ConfigVar(key="bittrex_api_key", - prompt="Enter your Bittrex API key >>> ", - required_if=using_exchange("bittrex"), - is_secure=True, - is_connect_key=True), - "bittrex_secret_key": - ConfigVar(key="bittrex_secret_key", - prompt="Enter your Bittrex secret key >>> ", - required_if=using_exchange("bittrex"), - is_secure=True, - is_connect_key=True), -} + +class BittrexConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="bittrex", client_data=None) + bittrex_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Bittrex API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + bittrex_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Bittrex secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = BittrexConfigMap.construct() diff --git a/hummingbot/connector/exchange/blocktane/blocktane_utils.py b/hummingbot/connector/exchange/blocktane/blocktane_utils.py index 23b99700f4..fc872708a9 100644 --- a/hummingbot/connector/exchange/blocktane/blocktane_utils.py +++ b/hummingbot/connector/exchange/blocktane/blocktane_utils.py @@ -1,10 +1,10 @@ import re -import requests - from typing import Optional, Tuple -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_methods import using_exchange +import requests +from pydantic import Field, SecretStr + +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData CENTRALIZED = True @@ -12,20 +12,30 @@ DEFAULT_FEES = [0.35, 0.45] # The actual fees -KEYS = { - "blocktane_api_key": - ConfigVar(key="blocktane_api_key", - prompt="Enter your Blocktane API key >>> ", - required_if=using_exchange("blocktane"), - is_secure=True, - is_connect_key=True), - "blocktane_api_secret": - ConfigVar(key="blocktane_api_secret", - prompt="Enter your Blocktane API secret >>> ", - required_if=using_exchange("blocktane"), - is_secure=True, - is_connect_key=True) -} + +class BlocktaneConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="blocktane", client_data=None) + blocktane_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Blocktane API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + blocktane_api_secret: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Blocktane API secret", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = BlocktaneConfigMap.construct() TRADING_PAIR_SPLITTER = re.compile(r"^(\w+)(BTC|btc|ETH|eth|BRL|brl|PAX|pax|USDT|usdt|PAXG|paxg|LETH|leth|EURS|eurs|LRC|lrc|BKT|bkt)$") MARKET_DATA = None diff --git a/hummingbot/connector/exchange/coinbase_pro/coinbase_pro_utils.py b/hummingbot/connector/exchange/coinbase_pro/coinbase_pro_utils.py index 70b2d67ea3..00f5b8423a 100644 --- a/hummingbot/connector/exchange/coinbase_pro/coinbase_pro_utils.py +++ b/hummingbot/connector/exchange/coinbase_pro/coinbase_pro_utils.py @@ -2,8 +2,9 @@ from dataclasses import dataclass from typing import Optional -from hummingbot.client.config.config_methods import using_exchange -from hummingbot.client.config.config_var import ConfigVar +from pydantic import Field, SecretStr + +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData from hummingbot.connector.exchange.coinbase_pro import coinbase_pro_constants as CONSTANTS from hummingbot.core.web_assistant.connections.data_types import EndpointRESTRequest from hummingbot.core.web_assistant.web_assistants_factory import WebAssistantsFactory @@ -17,26 +18,39 @@ DEFAULT_FEES = [0.5, 0.5] -KEYS = { - "coinbase_pro_api_key": - ConfigVar(key="coinbase_pro_api_key", - prompt="Enter your Coinbase API key >>> ", - required_if=using_exchange("coinbase_pro"), - is_secure=True, - is_connect_key=True), - "coinbase_pro_secret_key": - ConfigVar(key="coinbase_pro_secret_key", - prompt="Enter your Coinbase secret key >>> ", - required_if=using_exchange("coinbase_pro"), - is_secure=True, - is_connect_key=True), - "coinbase_pro_passphrase": - ConfigVar(key="coinbase_pro_passphrase", - prompt="Enter your Coinbase passphrase >>> ", - required_if=using_exchange("coinbase_pro"), - is_secure=True, - is_connect_key=True), -} + +class CoinbaseProConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="coinbase_pro", client_data=None) + coinbase_pro_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Coinbase API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + coinbase_pro_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Coinbase secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + coinbase_pro_passphrase: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Coinbase passphrase", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = CoinbaseProConfigMap.construct() @dataclass diff --git a/hummingbot/connector/exchange/coinflex/coinflex_utils.py b/hummingbot/connector/exchange/coinflex/coinflex_utils.py index e509ba4cf4..37b933ae80 100644 --- a/hummingbot/connector/exchange/coinflex/coinflex_utils.py +++ b/hummingbot/connector/exchange/coinflex/coinflex_utils.py @@ -1,9 +1,10 @@ from decimal import Decimal from typing import Any, Dict +from pydantic import Field, SecretStr + import hummingbot.connector.exchange.coinflex.coinflex_constants as CONSTANTS -from hummingbot.client.config.config_methods import using_exchange -from hummingbot.client.config.config_var import ConfigVar +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData from hummingbot.core.utils.tracking_nonce import get_tracking_nonce CENTRALIZED = True @@ -35,36 +36,56 @@ def decimal_val_or_none(string_value: str): return Decimal(string_value) if string_value else None -KEYS = { - "coinflex_api_key": - ConfigVar(key="coinflex_api_key", - prompt="Enter your CoinFLEX API key >>> ", - required_if=using_exchange("coinflex"), - is_secure=True, - is_connect_key=True), - "coinflex_api_secret": - ConfigVar(key="coinflex_api_secret", - prompt="Enter your CoinFLEX API secret >>> ", - required_if=using_exchange("coinflex"), - is_secure=True, - is_connect_key=True), -} +class CoinflexConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="coinflex", client_data=None) + coinflex_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your CoinFLEX API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + coinflex_api_secret: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your CoinFLEX API secret", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = CoinflexConfigMap.construct() OTHER_DOMAINS = ["coinflex_test"] OTHER_DOMAINS_PARAMETER = {"coinflex_test": "test"} OTHER_DOMAINS_EXAMPLE_PAIR = {"coinflex_test": "BTC-USDT"} OTHER_DOMAINS_DEFAULT_FEES = {"coinflex_test": [0.1, 0.1]} -OTHER_DOMAINS_KEYS = {"coinflex_test": { - "coinflex_test_api_key": - ConfigVar(key="coinflex_test_api_key", - prompt="Enter your CoinFLEX Staging API key >>> ", - required_if=using_exchange("coinflex_test"), - is_secure=True, - is_connect_key=True), - "coinflex_test_api_secret": - ConfigVar(key="coinflex_test_api_secret", - prompt="Enter your CoinFLEX Staging API secret >>> ", - required_if=using_exchange("coinflex_test"), - is_secure=True, - is_connect_key=True), -}} + + +class CoinflexTestConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="coinflex_test", client_data=None) + coinflex_test_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your CoinFLEX Staging API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + coinflex_test_api_secret: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your CoinFLEX Staging API secret", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +OTHER_DOMAINS_KEYS = {"coinflex_test": CoinflexTestConfigMap.construct()} diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py b/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py index c2970510d0..0039aaf989 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py @@ -1,15 +1,12 @@ +from typing import Any, Dict, Optional + from dateutil.parser import parse as dateparse -from typing import ( - Any, - Dict, - Optional, -) +from pydantic import Field, SecretStr +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData from hummingbot.core.utils.tracking_nonce import get_tracking_nonce -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_methods import using_exchange -from .coinzoom_constants import Constants +from .coinzoom_constants import Constants CENTRALIZED = True @@ -64,23 +61,35 @@ def get_new_client_order_id(is_buy: bool, trading_pair: str) -> str: return f"{Constants.HBOT_BROKER_ID}{side}{base_str}{quote_str}{get_tracking_nonce()}" -KEYS = { - "coinzoom_api_key": - ConfigVar(key="coinzoom_api_key", - prompt=f"Enter your {Constants.EXCHANGE_NAME} API key >>> ", - required_if=using_exchange("coinzoom"), - is_secure=True, - is_connect_key=True), - "coinzoom_secret_key": - ConfigVar(key="coinzoom_secret_key", - prompt=f"Enter your {Constants.EXCHANGE_NAME} secret key >>> ", - required_if=using_exchange("coinzoom"), - is_secure=True, - is_connect_key=True), - "coinzoom_username": - ConfigVar(key="coinzoom_username", - prompt=f"Enter your {Constants.EXCHANGE_NAME} ZoomMe username >>> ", - required_if=using_exchange("coinzoom"), - is_secure=True, - is_connect_key=True), -} +class CoinzoomConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="coinzoom", client_data=None) + coinzoom_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: f"Enter your {Constants.EXCHANGE_NAME} API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + coinzoom_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: f"Enter your {Constants.EXCHANGE_NAME} secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + coinzoom_username: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: f"Enter your {Constants.EXCHANGE_NAME} ZoomMe username", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = CoinzoomConfigMap.construct() diff --git a/hummingbot/connector/exchange/crypto_com/crypto_com_utils.py b/hummingbot/connector/exchange/crypto_com/crypto_com_utils.py index aaf17d8f27..cc73e947d1 100644 --- a/hummingbot/connector/exchange/crypto_com/crypto_com_utils.py +++ b/hummingbot/connector/exchange/crypto_com/crypto_com_utils.py @@ -1,12 +1,12 @@ import math from typing import Dict, List -from hummingbot.core.utils.tracking_nonce import get_tracking_nonce, get_tracking_nonce_low_res -from . import crypto_com_constants as CONSTANTS +from pydantic import Field, SecretStr -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_methods import using_exchange +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData +from hummingbot.core.utils.tracking_nonce import get_tracking_nonce, get_tracking_nonce_low_res +from . import crypto_com_constants as CONSTANTS CENTRALIZED = True @@ -78,17 +78,26 @@ def get_rest_url(path_url: str, api_version: str = CONSTANTS.API_VERSION) -> str return f"{CONSTANTS.REST_URL}{api_version}{path_url}" -KEYS = { - "crypto_com_api_key": - ConfigVar(key="crypto_com_api_key", - prompt="Enter your Crypto.com API key >>> ", - required_if=using_exchange("crypto_com"), - is_secure=True, - is_connect_key=True), - "crypto_com_secret_key": - ConfigVar(key="crypto_com_secret_key", - prompt="Enter your Crypto.com secret key >>> ", - required_if=using_exchange("crypto_com"), - is_secure=True, - is_connect_key=True), -} +class CryptoComConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="crypto_com", client_data=None) + crypto_com_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Crypto.com API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + crypto_com_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Crypto.com secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = CryptoComConfigMap.construct() diff --git a/hummingbot/connector/exchange/digifinex/digifinex_utils.py b/hummingbot/connector/exchange/digifinex/digifinex_utils.py index a48ac5d3b1..8e9675776a 100644 --- a/hummingbot/connector/exchange/digifinex/digifinex_utils.py +++ b/hummingbot/connector/exchange/digifinex/digifinex_utils.py @@ -1,12 +1,12 @@ import math from typing import Dict, List -from hummingbot.core.utils.tracking_nonce import get_tracking_nonce, get_tracking_nonce_low_res -from . import digifinex_constants as Constants +from pydantic import Field, SecretStr -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_methods import using_exchange +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData +from hummingbot.core.utils.tracking_nonce import get_tracking_nonce, get_tracking_nonce_low_res +from . import digifinex_constants as Constants CENTRALIZED = True @@ -82,17 +82,26 @@ def get_api_reason(code: str) -> str: return Constants.API_REASONS.get(int(code), code) -KEYS = { - "digifinex_api_key": - ConfigVar(key="digifinex_api_key", - prompt="Enter your Digifinex API key >>> ", - required_if=using_exchange("digifinex"), - is_secure=True, - is_connect_key=True), - "digifinex_secret_key": - ConfigVar(key="digifinex_secret_key", - prompt="Enter your Digifinex secret key >>> ", - required_if=using_exchange("digifinex"), - is_secure=True, - is_connect_key=True), -} +class DigifinexConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="digifinex", client_data=None) + digifinex_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Digifinex API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + digifinex_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Digifinex secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = DigifinexConfigMap.construct() diff --git a/hummingbot/connector/exchange/ftx/ftx_utils.py b/hummingbot/connector/exchange/ftx/ftx_utils.py index cbd82f2bdd..479e1aa42b 100644 --- a/hummingbot/connector/exchange/ftx/ftx_utils.py +++ b/hummingbot/connector/exchange/ftx/ftx_utils.py @@ -1,10 +1,8 @@ -from typing import ( - Optional, - Tuple) +from typing import Optional, Tuple -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_methods import using_exchange +from pydantic import Field, SecretStr +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData CENTRALIZED = True @@ -36,23 +34,35 @@ def convert_to_exchange_trading_pair(hb_trading_pair: str) -> str: return hb_trading_pair.replace("-", "/") -KEYS = { - "ftx_api_key": - ConfigVar(key="ftx_api_key", - prompt="Enter your FTX API key >>> ", - required_if=using_exchange("ftx"), - is_secure=True, - is_connect_key=True), - "ftx_secret_key": - ConfigVar(key="ftx_secret_key", - prompt="Enter your FTX API secret >>> ", - required_if=using_exchange("ftx"), - is_secure=True, - is_connect_key=True), - "ftx_subaccount_name": - ConfigVar(key="ftx_subaccount_name", - prompt="Enter your FTX subaccount name (if this is not a subaccount, leave blank) >>> ", - required_if=using_exchange("ftx"), - is_secure=True, - is_connect_key=True), -} +class FtxConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="ftx", client_data=None) + ftx_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your FTX API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + ftx_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your FTX API secret", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + ftx_subaccount_name: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your FTX subaccount name (if this is not a subaccount, leave blank)", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = FtxConfigMap.construct() diff --git a/hummingbot/connector/exchange/gate_io/gate_io_utils.py b/hummingbot/connector/exchange/gate_io/gate_io_utils.py index 7607518b42..4ce091312f 100644 --- a/hummingbot/connector/exchange/gate_io/gate_io_utils.py +++ b/hummingbot/connector/exchange/gate_io/gate_io_utils.py @@ -4,16 +4,15 @@ from dataclasses import dataclass from typing import Any, Dict, Optional, Tuple -from hummingbot.client.config.config_methods import using_exchange -from hummingbot.client.config.config_var import ConfigVar +from pydantic import Field, SecretStr + +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData from hummingbot.connector.exchange.gate_io import gate_io_constants as CONSTANTS from hummingbot.connector.exchange.gate_io.gate_io_auth import GateIoAuth -from hummingbot.core.web_assistant.web_assistants_factory import WebAssistantsFactory -from hummingbot.core.web_assistant.connections.data_types import ( - RESTMethod, RESTResponse, EndpointRESTRequest -) -from hummingbot.core.web_assistant.rest_assistant import RESTAssistant from hummingbot.core.api_throttler.async_throttler import AsyncThrottler +from hummingbot.core.web_assistant.connections.data_types import EndpointRESTRequest, RESTMethod, RESTResponse +from hummingbot.core.web_assistant.rest_assistant import RESTAssistant +from hummingbot.core.web_assistant.web_assistants_factory import WebAssistantsFactory CENTRALIZED = True @@ -160,17 +159,26 @@ async def api_call_with_retries(request: GateIORESTRequest, return parsed_response -KEYS = { - "gate_io_api_key": - ConfigVar(key="gate_io_api_key", - prompt=f"Enter your {CONSTANTS.EXCHANGE_NAME} API key >>> ", - required_if=using_exchange("gate_io"), - is_secure=True, - is_connect_key=True), - "gate_io_secret_key": - ConfigVar(key="gate_io_secret_key", - prompt=f"Enter your {CONSTANTS.EXCHANGE_NAME} secret key >>> ", - required_if=using_exchange("gate_io"), - is_secure=True, - is_connect_key=True), -} +class GateIOConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="gate_io", client_data=None) + gate_io_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: f"Enter your {CONSTANTS.EXCHANGE_NAME} API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + gate_io_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: f"Enter your {CONSTANTS.EXCHANGE_NAME} secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = GateIOConfigMap.construct() diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py index 3ce8d79c48..977b9dc7f2 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py @@ -1,20 +1,16 @@ -import aiohttp import asyncio import random import re +from typing import Any, Dict, Optional, Tuple + +import aiohttp from dateutil.parser import parse as dateparse -from typing import ( - Any, - Dict, - Optional, - Tuple, -) +from pydantic import Field, SecretStr +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData from hummingbot.core.utils.tracking_nonce import get_tracking_nonce -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_methods import using_exchange -from .hitbtc_constants import Constants +from .hitbtc_constants import Constants TRADING_PAIR_SPLITTER = re.compile(Constants.TRADING_PAIR_SPLITTER) @@ -141,17 +137,26 @@ async def api_call_with_retries(method, return parsed_response -KEYS = { - "hitbtc_api_key": - ConfigVar(key="hitbtc_api_key", - prompt=f"Enter your {Constants.EXCHANGE_NAME} API key >>> ", - required_if=using_exchange("hitbtc"), - is_secure=True, - is_connect_key=True), - "hitbtc_secret_key": - ConfigVar(key="hitbtc_secret_key", - prompt=f"Enter your {Constants.EXCHANGE_NAME} secret key >>> ", - required_if=using_exchange("hitbtc"), - is_secure=True, - is_connect_key=True), -} +class HitbtcConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="hitbtc", client_data=None) + hitbtc_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: f"Enter your {Constants.EXCHANGE_NAME} API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + hitbtc_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: f"Enter your {Constants.EXCHANGE_NAME} secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = HitbtcConfigMap.construct() diff --git a/hummingbot/connector/exchange/huobi/huobi_utils.py b/hummingbot/connector/exchange/huobi/huobi_utils.py index b2b5a8e94b..4207ecb5f2 100644 --- a/hummingbot/connector/exchange/huobi/huobi_utils.py +++ b/hummingbot/connector/exchange/huobi/huobi_utils.py @@ -1,12 +1,12 @@ import re from typing import Optional, Tuple -from hummingbot.client.config.config_methods import using_exchange -from hummingbot.client.config.config_var import ConfigVar +from pydantic import Field, SecretStr + +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData from hummingbot.connector.exchange.huobi.huobi_ws_post_processor import HuobiWSPostProcessor from hummingbot.core.web_assistant.web_assistants_factory import WebAssistantsFactory - RE_4_LETTERS_QUOTE = re.compile(r"^(\w+)(usdt|husd|usdc)$") RE_3_LETTERS_QUOTE = re.compile(r"^(\w+)(btc|eth|trx)$") RE_2_LETTERS_QUOTE = re.compile(r"^(\w+)(ht)$") @@ -51,17 +51,26 @@ def build_api_factory() -> WebAssistantsFactory: return api_factory -KEYS = { - "huobi_api_key": - ConfigVar(key="huobi_api_key", - prompt="Enter your Huobi API key >>> ", - required_if=using_exchange("huobi"), - is_secure=True, - is_connect_key=True), - "huobi_secret_key": - ConfigVar(key="huobi_secret_key", - prompt="Enter your Huobi secret key >>> ", - required_if=using_exchange("huobi"), - is_secure=True, - is_connect_key=True), -} +class HuobiConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="huobi", client_data=None) + huobi_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Huobi API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + huobi_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Huobi secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = HuobiConfigMap.construct() diff --git a/hummingbot/connector/exchange/k2/k2_utils.py b/hummingbot/connector/exchange/k2/k2_utils.py index f63ebc69e1..bbe1c91509 100644 --- a/hummingbot/connector/exchange/k2/k2_utils.py +++ b/hummingbot/connector/exchange/k2/k2_utils.py @@ -1,17 +1,12 @@ +from typing import Any, Dict, List, Tuple + import dateutil.parser -from typing import ( - Any, - Dict, - List, - Tuple -) +from pydantic import Field, SecretStr -from hummingbot.core.utils.tracking_nonce import get_tracking_nonce -from hummingbot.core.data_type.order_book_row import OrderBookRow +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData from hummingbot.core.data_type.order_book_message import OrderBookMessage - -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_methods import using_exchange +from hummingbot.core.data_type.order_book_row import OrderBookRow +from hummingbot.core.utils.tracking_nonce import get_tracking_nonce CENTRALIZED = True @@ -88,17 +83,26 @@ def convert_to_epoch_timestamp(timestamp: str) -> int: return int(dateutil.parser.parse(timestamp).timestamp() * 1e3) -KEYS = { - "k2_api_key": - ConfigVar(key="k2_api_key", - prompt="Enter your K2 API key >>> ", - required_if=using_exchange("k2"), - is_secure=True, - is_connect_key=True), - "k2_secret_key": - ConfigVar(key="k2_secret_key", - prompt="Enter your K2 secret key >>> ", - required_if=using_exchange("k2"), - is_secure=True, - is_connect_key=True), -} +class K2ConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="k2", client_data=None) + k2_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your K2 API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + k2_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your K2 secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = K2ConfigMap.construct() diff --git a/hummingbot/connector/exchange/kraken/kraken_utils.py b/hummingbot/connector/exchange/kraken/kraken_utils.py index 666bcbfb93..34bf68d8d7 100644 --- a/hummingbot/connector/exchange/kraken/kraken_utils.py +++ b/hummingbot/connector/exchange/kraken/kraken_utils.py @@ -1,20 +1,13 @@ -import hummingbot.connector.exchange.kraken.kraken_constants as CONSTANTS +from typing import Any, Dict, List, Optional, Tuple -from typing import ( - Any, - Dict, - List, - Optional, - Tuple, -) +from pydantic import Field, SecretStr -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_methods import using_exchange +import hummingbot.connector.exchange.kraken.kraken_constants as CONSTANTS +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData from hummingbot.connector.exchange.kraken.kraken_constants import KrakenAPITier from hummingbot.core.api_throttler.data_types import LinkedLimitWeightPair, RateLimit from hummingbot.core.web_assistant.web_assistants_factory import WebAssistantsFactory - CENTRALIZED = True EXAMPLE_PAIR = "ETH-USDC" @@ -182,29 +175,37 @@ def _api_tier_validator(value: str) -> Optional[str]: return "No such Kraken API Tier." -KEYS = { - "kraken_api_key": - ConfigVar(key="kraken_api_key", - prompt="Enter your Kraken API key >>> ", - required_if=using_exchange("kraken"), - is_secure=True, - is_connect_key=True), - "kraken_secret_key": - ConfigVar(key="kraken_secret_key", - prompt="Enter your Kraken secret key >>> ", - required_if=using_exchange("kraken"), - is_secure=True, - is_connect_key=True), - "kraken_api_tier": - ConfigVar(key="kraken_api_tier", - prompt="Enter your Kraken API Tier (Starter/Intermediate/Pro) >>> ", - required_if=using_exchange("kraken"), - default="Starter", - is_secure=False, - is_connect_key=True, - validator=lambda v: _api_tier_validator(v), - ), -} +class KrakenConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="kraken", client_data=None) + kraken_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Kraken API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + kraken_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Kraken secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + kraken_api_tier: str = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Kraken API Tier (Starter/Intermediate/Pro)", + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = KrakenConfigMap.construct() def build_api_factory() -> WebAssistantsFactory: diff --git a/hummingbot/connector/exchange/kucoin/kucoin_utils.py b/hummingbot/connector/exchange/kucoin/kucoin_utils.py index 76915a6ba3..8a2c78f678 100644 --- a/hummingbot/connector/exchange/kucoin/kucoin_utils.py +++ b/hummingbot/connector/exchange/kucoin/kucoin_utils.py @@ -1,8 +1,9 @@ from decimal import Decimal from typing import Any, Dict -from hummingbot.client.config.config_methods import using_exchange -from hummingbot.client.config.config_var import ConfigVar +from pydantic import Field, SecretStr + +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData from hummingbot.core.data_type.trade_fee import TradeFeeSchema CENTRALIZED = True @@ -26,50 +27,74 @@ def is_pair_information_valid(pair_info: Dict[str, Any]) -> bool: return pair_info.get("enableTrading", False) -KEYS = { - "kucoin_api_key": - ConfigVar(key="kucoin_api_key", - prompt="Enter your KuCoin API key >>> ", - required_if=using_exchange("kucoin"), - is_secure=True, - is_connect_key=True), - "kucoin_secret_key": - ConfigVar(key="kucoin_secret_key", - prompt="Enter your KuCoin secret key >>> ", - required_if=using_exchange("kucoin"), - is_secure=True, - is_connect_key=True), - "kucoin_passphrase": - ConfigVar(key="kucoin_passphrase", - prompt="Enter your KuCoin passphrase >>> ", - required_if=using_exchange("kucoin"), - is_secure=True, - is_connect_key=True), -} +class KuCoinConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="kucoin", client_data=None) + kucoin_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your KuCoin API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + kucoin_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your KuCoin secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + kucoin_passphrase: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your KuCoin passphrase", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = KuCoinConfigMap.construct() OTHER_DOMAINS = ["kucoin_testnet"] OTHER_DOMAINS_PARAMETER = {"kucoin_testnet": "testnet"} OTHER_DOMAINS_EXAMPLE_PAIR = {"kucoin_testnet": "ETH-USDT"} OTHER_DOMAINS_DEFAULT_FEES = {"kucoin_testnet": [0.1, 0.1]} -OTHER_DOMAINS_KEYS = { - "kucoin_testnet": { - "kucoin_testnet_api_key": - ConfigVar(key="kucoin_testnet_api_key", - prompt="Enter your KuCoin API key >>> ", - required_if=using_exchange("kucoin_testnet"), - is_secure=True, - is_connect_key=True), - "kucoin_testnet_secret_key": - ConfigVar(key="kucoin_testnet_secret_key", - prompt="Enter your KuCoin secret key >>> ", - required_if=using_exchange("kucoin_testnet"), - is_secure=True, - is_connect_key=True), - "kucoin_testnet_passphrase": - ConfigVar(key="kucoin_testnet_passphrase", - prompt="Enter your KuCoin passphrase >>> ", - required_if=using_exchange("kucoin_testnet"), - is_secure=True, - is_connect_key=True), - } -} + + +class KuCoinTestnetConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="kucoin_testnet", client_data=None) + kucoin_testnet_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your KuCoin Testnet API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + kucoin_testnet_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your KuCoin Testnet secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + kucoin_testnet_passphrase: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your KuCoin Testnet passphrase", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +OTHER_DOMAINS_KEYS = {"kucoin_testnet": KuCoinTestnetConfigMap.construct()} diff --git a/hummingbot/connector/exchange/liquid/liquid_utils.py b/hummingbot/connector/exchange/liquid/liquid_utils.py index 7134427195..3ba827453f 100644 --- a/hummingbot/connector/exchange/liquid/liquid_utils.py +++ b/hummingbot/connector/exchange/liquid/liquid_utils.py @@ -1,5 +1,6 @@ -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_methods import using_exchange +from pydantic import Field, SecretStr + +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData CENTRALIZED = True @@ -8,17 +9,26 @@ DEFAULT_FEES = [0.1, 0.1] -KEYS = { - "liquid_api_key": - ConfigVar(key="liquid_api_key", - prompt="Enter your Liquid API key >>> ", - required_if=using_exchange("liquid"), - is_secure=True, - is_connect_key=True), - "liquid_secret_key": - ConfigVar(key="liquid_secret_key", - prompt="Enter your Liquid secret key >>> ", - required_if=using_exchange("liquid"), - is_secure=True, - is_connect_key=True), -} +class LiquidConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="liquid", client_data=None) + liquid_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Liquid API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + liquid_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Liquid secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = LiquidConfigMap.construct() diff --git a/hummingbot/connector/exchange/loopring/loopring_utils.py b/hummingbot/connector/exchange/loopring/loopring_utils.py index 2a8ec79d40..cfee94b783 100644 --- a/hummingbot/connector/exchange/loopring/loopring_utils.py +++ b/hummingbot/connector/exchange/loopring/loopring_utils.py @@ -1,8 +1,9 @@ +from typing import Any, Dict + import aiohttp -from typing import Dict, Any +from pydantic import Field, SecretStr -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_methods import using_exchange +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData CENTRALIZED = True @@ -13,32 +14,48 @@ LOOPRING_ROOT_API = "https://api3.loopring.io" LOOPRING_WS_KEY_PATH = "/v2/ws/key" -KEYS = { - "loopring_accountid": - ConfigVar(key="loopring_accountid", - prompt="Enter your Loopring account id >>> ", - required_if=using_exchange("loopring"), - is_secure=True, - is_connect_key=True), - "loopring_exchangeaddress": - ConfigVar(key="loopring_exchangeaddress", - prompt="Enter the Loopring exchange address >>> ", - required_if=using_exchange("loopring"), - is_secure=True, - is_connect_key=True), - "loopring_private_key": - ConfigVar(key="loopring_private_key", - prompt="Enter your Loopring private key >>> ", - required_if=using_exchange("loopring"), - is_secure=True, - is_connect_key=True), - "loopring_api_key": - ConfigVar(key="loopring_api_key", - prompt="Enter your loopring api key >>> ", - required_if=using_exchange("loopring"), - is_secure=True, - is_connect_key=True) -} + +class LoopringConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="loopring", client_data=None) + loopring_accountid: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Loopring account id", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + loopring_exchangeaddress: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter the Loopring exchange address", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + loopring_private_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Loopring private key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + loopring_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your loopring api key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = LoopringConfigMap.construct() def convert_from_exchange_trading_pair(exchange_trading_pair: str) -> str: diff --git a/hummingbot/connector/exchange/mexc/mexc_utils.py b/hummingbot/connector/exchange/mexc/mexc_utils.py index 7628e2b559..990b7e8cac 100644 --- a/hummingbot/connector/exchange/mexc/mexc_utils.py +++ b/hummingbot/connector/exchange/mexc/mexc_utils.py @@ -1,11 +1,11 @@ #!/usr/bin/env python # -*- coding: UTF-8 -*- +import time from decimal import Decimal -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_methods import using_exchange +from pydantic import Field, SecretStr -import time +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData def num_to_increment(num): @@ -18,20 +18,30 @@ def num_to_increment(num): DEFAULT_FEES = [0.2, 0.2] -KEYS = { - "mexc_api_key": - ConfigVar(key="mexc_api_key", - prompt="Enter your MEXC API key >>> ", - required_if=using_exchange("mexc"), - is_secure=True, - is_connect_key=True), - "mexc_secret_key": - ConfigVar(key="mexc_secret_key", - prompt="Enter your MEXC secret key >>> ", - required_if=using_exchange("mexc"), - is_secure=True, - is_connect_key=True), -} + +class MexcConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="mexc", client_data=None) + mexc_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your MEXC API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + mexc_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your MEXC secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = MexcConfigMap.construct() ws_status = { 1: 'NEW', diff --git a/hummingbot/connector/exchange/ndax/ndax_utils.py b/hummingbot/connector/exchange/ndax/ndax_utils.py index 345229863f..bbaedd653f 100644 --- a/hummingbot/connector/exchange/ndax/ndax_utils.py +++ b/hummingbot/connector/exchange/ndax/ndax_utils.py @@ -1,7 +1,8 @@ from typing import Optional -from hummingbot.client.config.config_methods import using_exchange -from hummingbot.client.config.config_var import ConfigVar +from pydantic import Field, SecretStr + +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData from hummingbot.connector.exchange.ndax import ndax_constants as CONSTANTS from hummingbot.core.utils.tracking_nonce import get_tracking_nonce @@ -38,62 +39,92 @@ def wss_url(connector_variant_label: Optional[str]) -> str: return CONSTANTS.WSS_URLS.get(variant) -KEYS = { - "ndax_uid": - ConfigVar(key="ndax_uid", - prompt="Enter your NDAX user ID (uid) >>> ", - required_if=using_exchange("ndax"), - is_secure=True, - is_connect_key=True), - "ndax_account_name": - ConfigVar(key="ndax_account_name", - prompt="Enter the name of the account you want to use >>> ", - required_if=using_exchange("ndax"), - is_secure=True, - is_connect_key=True), - "ndax_api_key": - ConfigVar(key="ndax_api_key", - prompt="Enter your NDAX API key >>> ", - required_if=using_exchange("ndax"), - is_secure=True, - is_connect_key=True), - "ndax_secret_key": - ConfigVar(key="ndax_secret_key", - prompt="Enter your NDAX secret key >>> ", - required_if=using_exchange("ndax"), - is_secure=True, - is_connect_key=True), -} +class NdaxConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="ndax", client_data=None) + ndax_uid: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your NDAX user ID (uid)", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + ndax_account_name: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter the name of the account you want to use", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + ndax_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your NDAX API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + ndax_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your NDAX secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = NdaxConfigMap.construct() OTHER_DOMAINS = ["ndax_testnet"] OTHER_DOMAINS_PARAMETER = {"ndax_testnet": "ndax_testnet"} OTHER_DOMAINS_EXAMPLE_PAIR = {"ndax_testnet": "BTC-CAD"} OTHER_DOMAINS_DEFAULT_FEES = {"ndax_testnet": [0.2, 0.2]} -OTHER_DOMAINS_KEYS = { - "ndax_testnet": { - "ndax_testnet_uid": - ConfigVar(key="ndax_testnet_uid", - prompt="Enter your NDAX user ID (uid) >>> ", - required_if=using_exchange("ndax_testnet"), - is_secure=True, - is_connect_key=True), - "ndax_testnet_account_name": - ConfigVar(key="ndax_testnet_account_name", - prompt="Enter the name of the account you want to use >>> ", - required_if=using_exchange("ndax_testnet"), - is_secure=True, - is_connect_key=True), - "ndax_testnet_api_key": - ConfigVar(key="ndax_testnet_api_key", - prompt="Enter your NDAX API key >>> ", - required_if=using_exchange("ndax_testnet"), - is_secure=True, - is_connect_key=True), - "ndax_testnet_secret_key": - ConfigVar(key="ndax_testnet_secret_key", - prompt="Enter your NDAX secret key >>> ", - required_if=using_exchange("ndax_testnet"), - is_secure=True, - is_connect_key=True), - } -} + + +class NdaxTestnetConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="ndax_testnet", client_data=None) + ndax_testnet_uid: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your NDAX Testnet user ID (uid)", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + ndax_testnet_account_name: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter the name of the account you want to use", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + ndax_testnet_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your NDAX Testnet API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + ndax_testnet_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your NDAX Testnet secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +OTHER_DOMAINS_KEYS = {"ndax_testnet": NdaxTestnetConfigMap.construct()} diff --git a/hummingbot/connector/exchange/okex/okex_utils.py b/hummingbot/connector/exchange/okex/okex_utils.py index 4b579ccef5..d5e77d37a7 100644 --- a/hummingbot/connector/exchange/okex/okex_utils.py +++ b/hummingbot/connector/exchange/okex/okex_utils.py @@ -1,7 +1,9 @@ -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_methods import using_exchange import zlib +from pydantic import Field, SecretStr + +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData + CENTRALIZED = True @@ -11,26 +13,38 @@ DEFAULT_FEES = [0.1, 0.15] -KEYS = { - "okex_api_key": - ConfigVar(key="okex_api_key", - prompt="Enter your OKEx API key >>> ", - required_if=using_exchange("okex"), - is_secure=True, - is_connect_key=True), - "okex_secret_key": - ConfigVar(key="okex_secret_key", - prompt="Enter your OKEx secret key >>> ", - required_if=using_exchange("okex"), - is_secure=True, - is_connect_key=True), - "okex_passphrase": - ConfigVar(key="okex_passphrase", - prompt="Enter your OKEx passphrase key >>> ", - required_if=using_exchange("okex"), - is_secure=True, - is_connect_key=True), -} +class OkexConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="okex", client_data=None) + okex_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your OKEx API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + okex_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your OKEx secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + okex_passphrase: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your OKEx passphrase key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = OkexConfigMap.construct() def inflate(data): diff --git a/hummingbot/connector/exchange/probit/probit_utils.py b/hummingbot/connector/exchange/probit/probit_utils.py index 60b6bcad7c..6ebf8b8228 100644 --- a/hummingbot/connector/exchange/probit/probit_utils.py +++ b/hummingbot/connector/exchange/probit/probit_utils.py @@ -4,9 +4,9 @@ from typing import Any, Dict, List, Tuple import dateutil.parser as dp +from pydantic import Field, SecretStr -from hummingbot.client.config.config_methods import using_exchange -from hummingbot.client.config.config_var import ConfigVar +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData from hummingbot.core.data_type.order_book_message import OrderBookMessage from hummingbot.core.data_type.order_book_row import OrderBookRow @@ -60,38 +60,56 @@ def convert_diff_message_to_order_book_row(message: OrderBookMessage) -> Tuple[L return bids, asks -KEYS = { - "probit_api_key": - ConfigVar(key="probit_api_key", - prompt="Enter your ProBit Client ID >>> ", - required_if=using_exchange("probit"), - is_secure=True, - is_connect_key=True), - "probit_secret_key": - ConfigVar(key="probit_secret_key", - prompt="Enter your ProBit secret key >>> ", - required_if=using_exchange("probit"), - is_secure=True, - is_connect_key=True), -} +class ProbitConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="probit", client_data=None) + probit_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your ProBit Client ID", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + probit_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your ProBit secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = ProbitConfigMap.construct() OTHER_DOMAINS = ["probit_kr"] OTHER_DOMAINS_PARAMETER = {"probit_kr": "kr"} OTHER_DOMAINS_EXAMPLE_PAIR = {"probit_kr": "BTC-USDT"} OTHER_DOMAINS_DEFAULT_FEES = {"probit_kr": [0.2, 0.2]} -OTHER_DOMAINS_KEYS = { - "probit_kr": { - "probit_kr_api_key": - ConfigVar(key="probit_kr_api_key", - prompt="Enter your ProBit KR Client ID >>> ", - required_if=using_exchange("probit_kr"), - is_secure=True, - is_connect_key=True), - "probit_kr_secret_key": - ConfigVar(key="probit_kr_secret_key", - prompt="Enter your ProBit KR secret key >>> ", - required_if=using_exchange("probit_kr"), - is_secure=True, - is_connect_key=True), - } -} + + +class ProbitKrConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="probit_kr", client_data=None) + probit_kr_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your ProBit KR Client ID", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + probit_kr_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your ProBit KR secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +OTHER_DOMAINS_KEYS = {"probit_kr": ProbitKrConfigMap.construct()} diff --git a/hummingbot/connector/exchange/wazirx/wazirx_utils.py b/hummingbot/connector/exchange/wazirx/wazirx_utils.py index ce0e5812b7..b14288b47f 100644 --- a/hummingbot/connector/exchange/wazirx/wazirx_utils.py +++ b/hummingbot/connector/exchange/wazirx/wazirx_utils.py @@ -1,11 +1,12 @@ -import re import math +import re from typing import Dict, List -from hummingbot.core.utils.tracking_nonce import get_tracking_nonce, get_tracking_nonce_low_res -from hummingbot.connector.exchange.wazirx import wazirx_constants as CONSTANTS -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_methods import using_exchange +from pydantic import Field, SecretStr + +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData +from hummingbot.connector.exchange.wazirx import wazirx_constants as CONSTANTS +from hummingbot.core.utils.tracking_nonce import get_tracking_nonce, get_tracking_nonce_low_res TRADING_PAIR_SPLITTER = re.compile(r"^(\w+)(btc|usdt|inr|wrx)$") @@ -96,17 +97,26 @@ def get_api_reason(code: str) -> str: return CONSTANTS.API_REASONS.get(int(code), code) -KEYS = { - "wazirx_api_key": - ConfigVar(key="wazirx_api_key", - prompt="Enter your WazirX API key >>> ", - required_if=using_exchange("wazirx"), - is_secure=True, - is_connect_key=True), - "wazirx_secret_key": - ConfigVar(key="wazirx_secret_key", - prompt="Enter your WazirX secret key >>> ", - required_if=using_exchange("wazirx"), - is_secure=True, - is_connect_key=True), -} +class WazirxConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="wazirx", client_data=None) + wazirx_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your WazirX API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + wazirx_secret_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your WazirX secret key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = WazirxConfigMap.construct() diff --git a/hummingbot/connector/other/celo/celo_data_types.py b/hummingbot/connector/other/celo/celo_data_types.py index cc5aa736cb..63537eece1 100644 --- a/hummingbot/connector/other/celo/celo_data_types.py +++ b/hummingbot/connector/other/celo/celo_data_types.py @@ -1,5 +1,9 @@ -from typing import NamedTuple from decimal import Decimal +from typing import NamedTuple + +from pydantic import Field, SecretStr + +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData class CeloExchangeRate(NamedTuple): @@ -44,3 +48,31 @@ def __repr__(self) -> str: f"Celo Order - tx_hash: {self.tx_hash}, side: {'buy' if self.is_buy else 'sell'}, " f"price: {self.price}, amount: {self.amount}." ) + + +class CeloConfigMap(BaseConnectorConfigMap): + """ + As Celo is treated as a special case everywhere else in the code, + its canfigs cannot be stored and handled the conventional way either (i.e. using celo_utils.py). + """ + connector: str = Field(default="celo", client_data=None) + celo_address: str = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Celo account address", + is_connect_key=True, + prompt_on_new=True, + ) + ) + celo_password: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Celo account password", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + +KEYS = CeloConfigMap.construct() diff --git a/hummingbot/core/utils/wallet_setup.py b/hummingbot/core/utils/wallet_setup.py deleted file mode 100644 index 516bc627a8..0000000000 --- a/hummingbot/core/utils/wallet_setup.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python - -""" -Functions for storing encrypted wallets and decrypting stored wallets. -""" - -from eth_account import Account -from hummingbot.client.settings import ( - KEYFILE_PREFIX, - KEYFILE_POSTFIX, - DEFAULT_KEY_FILE_PATH, -) -from hummingbot.client.config.global_config_map import global_config_map -import json -from os import listdir -from os.path import ( - join, - isfile -) -from typing import Dict, List - - -def get_key_file_path() -> str: - """ - The key file path is where encrypted wallet files are stored. - Get the key file path from the global config map. - If it is not defined, then return DEFAULT_KEY_FILE_PATH - """ - path = global_config_map["key_file_path"].value - return path if path is not None else DEFAULT_KEY_FILE_PATH - - -def import_and_save_wallet(password: str, private_key: str) -> Account: - """ - Create an account for a private key, then encryt the private key and store it in the path from get_key_file_path() - """ - acct: Account = Account.privateKeyToAccount(private_key) - return save_wallet(acct, password) - - -def save_wallet(acct: Account, password: str) -> Account: - """ - For a given account and password, encrypt the account address and store it in the path from get_key_file_path() - """ - encrypted: Dict = Account.encrypt(acct.privateKey, password) - file_path: str = "%s%s%s%s" % (get_key_file_path(), KEYFILE_PREFIX, acct.address, KEYFILE_POSTFIX) - with open(file_path, 'w+') as f: - f.write(json.dumps(encrypted)) - return acct - - -def unlock_wallet(wallet_address: str, password: str) -> str: - """ - Search get_key_file_path() by a public key for an account file, then decrypt the private key from the file with the - provided password - """ - file_path: str = "%s%s%s%s" % (get_key_file_path(), KEYFILE_PREFIX, wallet_address, KEYFILE_POSTFIX) - with open(file_path, 'r') as f: - encrypted = f.read() - private_key: str = Account.decrypt(encrypted, password) - return private_key - - -def list_wallets() -> List[str]: - """ - Return a list of wallets in get_key_file_path() - """ - wallets = [] - for f in listdir(get_key_file_path()): - if isfile(join(get_key_file_path(), f)) and f.startswith(KEYFILE_PREFIX) and f.endswith(KEYFILE_POSTFIX): - wallets.append(f[len(KEYFILE_PREFIX):-len(KEYFILE_POSTFIX)]) - return wallets diff --git a/hummingbot/pmm_script/pmm_script_iterator.pyx b/hummingbot/pmm_script/pmm_script_iterator.pyx index 277662265e..59b57439a1 100644 --- a/hummingbot/pmm_script/pmm_script_iterator.pyx +++ b/hummingbot/pmm_script/pmm_script_iterator.pyx @@ -3,6 +3,7 @@ import asyncio import logging from multiprocessing import Process, Queue +from pathlib import Path from typing import List from hummingbot.connector.exchange_base import ExchangeBase @@ -41,13 +42,12 @@ cdef class PMMScriptIterator(TimeIterator): return sir_logger def __init__(self, - script_file_path: str, + script_file_path: Path, markets: List[ExchangeBase], strategy: PureMarketMakingStrategy, queue_check_interval: float = 0.01, is_unit_testing_mode: bool = False): super().__init__() - self._script_file_path = script_file_path self._markets = markets self._strategy = strategy self._is_unit_testing_mode = is_unit_testing_mode @@ -65,7 +65,7 @@ cdef class PMMScriptIterator(TimeIterator): self._script_process = Process( target=run_pmm_script, - args=(script_file_path, self._parent_queue, self._child_queue, queue_check_interval,) + args=(str(script_file_path), self._parent_queue, self._child_queue, queue_check_interval,) ) self.logger().info(f"starting PMM script in {script_file_path}") self._script_process.start() diff --git a/hummingbot/templates/conf_global_TEMPLATE.yml b/hummingbot/templates/conf_global_TEMPLATE.yml index d7c6a43c8c..290dc73893 100644 --- a/hummingbot/templates/conf_global_TEMPLATE.yml +++ b/hummingbot/templates/conf_global_TEMPLATE.yml @@ -3,140 +3,7 @@ ################################# # For more detailed information: https://docs.hummingbot.io -template_version: 35 - -# Exchange configs - -beaxy_api_key: null -beaxy_secret_key: null - -binance_api_key: null -binance_api_secret: null - -binance_us_api_key: null -binance_us_api_secret: null - -binance_perpetual_api_key: null -binance_perpetual_api_secret: null - -binance_perpetual_testnet_api_key: null -binance_perpetual_testnet_api_secret: null - -bittrex_api_key: null -bittrex_secret_key: null - -blocktane_api_key: null -blocktane_api_secret: null - -coinbase_pro_api_key: null -coinbase_pro_secret_key: null -coinbase_pro_passphrase: null - -coinflex_api_key: null -coinflex_api_secret: null -coinflex_test_api_key: null -coinflex_test_api_secret: null - -coinzoom_api_key: null -coinzoom_secret_key: null -coinzoom_username: null - -dydx_perpetual_api_key: null -dydx_perpetual_api_secret: null -dydx_perpetual_passphrase: null -dydx_perpetual_account_number: null -dydx_perpetual_stark_private_key: null -dydx_perpetual_ethereum_address: null - -ftx_api_key: null -ftx_secret_key: null -ftx_subaccount_name: null - -huobi_api_key: null -huobi_secret_key: null - -liquid_api_key: null -liquid_secret_key: null - -loopring_accountid: null -loopring_exchangeaddress: null -loopring_api_key: null -loopring_private_key: null - -kucoin_api_key: null -kucoin_secret_key: null -kucoin_passphrase: null - -kucoin_testnet_api_key: null -kucoin_testnet_secret_key: null -kucoin_testnet_passphrase: null - -altmarkets_api_key: null -altmarkets_secret_key: null - -kraken_api_key: null -kraken_secret_key: null -kraken_api_tier: null - -bitmart_api_key: null -bitmart_secret_key: null -bitmart_memo: null - -crypto_com_api_key: null -crypto_com_secret_key: null - - -hitbtc_api_key: null -hitbtc_secret_key: null - -gate_io_api_key: null -gate_io_secret_key: null - -bitfinex_api_key: null -bitfinex_secret_key: null - -okex_api_key: null -okex_secret_key: null -okex_passphrase: null - -ascend_ex_api_key: null -ascend_ex_secret_key: null - -celo_address: null -celo_password: null - -digifinex_api_key: null -digifinex_secret_key: null - -k2_api_key: null -k2_secret_key: null - -probit_api_key: null -probit_secret_key: null - -probit_kr_api_key: null -probit_kr_secret_key: null - -ndax_uid: null -ndax_account_name: null -ndax_api_key: null -ndax_secret_key: null - -ndax_testnet_uid: null -ndax_testnet_account_name: null -ndax_testnet_api_key: null -ndax_testnet_secret_key: null - -bybit_perpetual_api_key: null -bybit_perpetual_secret_key: null - -bybit_perpetual_testnet_api_key: null -bybit_perpetual_testnet_secret_key: null - -wazirx_api_key: null -wazirx_secret_key: null -mexc_api_key: null -mexc_secret_key: null +template_version: 36 # Kill switch kill_switch_enabled: null @@ -179,7 +46,6 @@ logger_override_whitelist: - hummingbot.strategy.arbitrage - hummingbot.strategy.cross_exchange_market_making - conf -key_file_path: conf/ log_file_path: logs/ # Advanced database options, currently supports SQLAlchemy's included dialects diff --git a/hummingbot/user/user_balances.py b/hummingbot/user/user_balances.py index dbbd6e7a26..ed7ae452f6 100644 --- a/hummingbot/user/user_balances.py +++ b/hummingbot/user/user_balances.py @@ -1,12 +1,12 @@ from decimal import Decimal from functools import lru_cache -from typing import Optional, Dict, List +from typing import Dict, List, Optional -from hummingbot.core.utils.market_price import get_last_price -from hummingbot.client.settings import AllConnectorSettings, gateway_connector_trading_pairs -from hummingbot.client.config.security import Security from hummingbot.client.config.config_helpers import get_connector_class +from hummingbot.client.config.security import Security +from hummingbot.client.settings import AllConnectorSettings, gateway_connector_trading_pairs from hummingbot.core.utils.async_utils import safe_gather +from hummingbot.core.utils.market_price import get_last_price class UserBalances: @@ -66,22 +66,24 @@ def all_balances(self, exchange) -> Dict[str, Decimal]: return self._markets[exchange].get_all_balances() async def update_exchange_balance(self, exchange_name: str) -> Optional[str]: - if self.is_gateway_market(exchange_name) and exchange_name in self._markets: + is_gateway_market = self.is_gateway_market(exchange_name) + if is_gateway_market and exchange_name in self._markets: # we want to refresh gateway connectors always, since the applicable tokens change over time. # doing this will reinitialize and fetch balances for active trading pair del self._markets[exchange_name] if exchange_name in self._markets: return await self._update_balances(self._markets[exchange_name]) else: - api_keys = await Security.api_keys(exchange_name) + api_keys = await Security.api_keys(exchange_name) if not is_gateway_market else {} return await self.add_exchange(exchange_name, **api_keys) # returns error message for each exchange async def update_exchanges( - self, - reconnect: bool = False, - exchanges: List[str] = [] + self, + reconnect: bool = False, + exchanges: Optional[List[str]] = None ) -> Dict[str, Optional[str]]: + exchanges = exchanges or [] tasks = [] # Update user balances if len(exchanges) == 0: diff --git a/scripts/conf_migration_script.py b/scripts/conf_migration_script.py new file mode 100644 index 0000000000..03658eee5e --- /dev/null +++ b/scripts/conf_migration_script.py @@ -0,0 +1,127 @@ +import argparse +import binascii +import importlib +import shutil +from os import DirEntry, scandir +from os.path import exists, join +from typing import List, cast + +import yaml + +from hummingbot import root_path +from hummingbot.client.config.config_crypt import ETHKeyFileSecretManger +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter +from hummingbot.client.config.security import Security +from hummingbot.client.settings import CONF_DIR_PATH, STRATEGIES_CONF_DIR_PATH + +encrypted_conf_prefix = "encrypted_" +encrypted_conf_postfix = ".json" +conf_dir_path = CONF_DIR_PATH +strategies_conf_dir_path = STRATEGIES_CONF_DIR_PATH + + +def migrate(password: str): + print("Starting conf migration.") + backup_existing_dir() + migrate_strategy_confs() + migrate_connector_confs(password) + + +def backup_existing_dir(): + if conf_dir_path.exists(): + backup_path = conf_dir_path.parent / "conf_backup" + if backup_path.exists(): + raise RuntimeError( + f"\nBackup path {backup_path} already exists. The migration script cannot backup you" + f" exiting conf files without overwriting that directory. Please remove it and" + f" run the script again." + ) + shutil.copytree(conf_dir_path, backup_path) + print(f"\nCreated a backup of your existing conf directory to {backup_path}") + + +def migrate_strategy_confs(): + print("\nMigrating strategies...") + for child in conf_dir_path.iterdir(): + if child.is_file() and child.name.endswith(".yml"): + with open(str(child), "r") as f: + conf = yaml.safe_load(f) + if "strategy" in conf and "exchange" in conf: + new_path = strategies_conf_dir_path / child.name + child.rename(new_path) + print(f"Migrated conf for {conf['strategy']}") + + +def migrate_connector_confs(password: str): + print("\nMigrating connector secure keys...") + secrets_manager = ETHKeyFileSecretManger(password) + Security.secrets_manager = secrets_manager + connector_exceptions = ["paper_trade"] + type_dirs: List[DirEntry] = [ + cast(DirEntry, f) for f in + scandir(f"{root_path() / 'hummingbot' / 'connector'}") + if f.is_dir() + ] + for type_dir in type_dirs: + connector_dirs: List[DirEntry] = [ + cast(DirEntry, f) for f in scandir(type_dir.path) + if f.is_dir() and exists(join(f.path, "__init__.py")) + ] + for connector_dir in connector_dirs: + if connector_dir.name.startswith("_") or connector_dir.name in connector_exceptions: + continue + try: + util_module_path: str = ( + f"hummingbot.connector.{type_dir.name}.{connector_dir.name}.{connector_dir.name}_utils" + ) + util_module = importlib.import_module(util_module_path) + config_keys = getattr(util_module, "KEYS", None) + if config_keys is not None: + _maybe_migrate_encrypted_confs(config_keys) + other_domains = getattr(util_module, "OTHER_DOMAINS", []) + for domain in other_domains: + config_keys = getattr(util_module, "OTHER_DOMAINS_KEYS")[domain] + if config_keys is not None: + _maybe_migrate_encrypted_confs(config_keys) + except ModuleNotFoundError: + continue + + +def _maybe_migrate_encrypted_confs(config_keys: BaseConnectorConfigMap): + cm = ClientConfigAdapter(config_keys) + found_one = False + files_to_remove = [] + missing_fields = [] + for el in cm.traverse(): + if el.client_field_data is not None and el.client_field_data.is_secure: + key_path = conf_dir_path / f"{encrypted_conf_prefix}{el.attr}{encrypted_conf_postfix}" + if key_path.exists(): + with open(key_path, 'r') as f: + json_str = f.read() + encrypted = binascii.hexlify(json_str.encode()).decode() + cm.setattr_no_validation(el.attr, encrypted) + files_to_remove.append(key_path) + found_one = True + else: + missing_fields.append(el.attr) + if found_one: + if len(missing_fields) != 0: + raise RuntimeError( + f"The migration of {config_keys.connector} failed because of missing fields: {missing_fields}" + ) + errors = cm.validate_model() + if errors: + raise RuntimeError(f"The migration of {config_keys.connector} failed with errors: {errors}") + Security.update_secure_config(cm.connector, cm) + for f in files_to_remove: + f.unlink() + print(f"Migrated secure keys for {config_keys.connector}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Migrate the HummingBot confs") + parser.add_argument("password", type=str, help="Required to migrate all encrypted configs.") + args = parser.parse_args() + migrate(args.password) + print("\nConf migration done.") diff --git a/setup/environment.yml b/setup/environment.yml index bcb2c127b6..6dc3a6e03d 100644 --- a/setup/environment.yml +++ b/setup/environment.yml @@ -59,6 +59,7 @@ dependencies: - hexbytes==0.2.0 - importlib-metadata==0.23 - mypy-extensions==0.4.3 + - path_util==0.1.3, - pre-commit==2.18.1 - psutil==5.7.2 - ptpython==3.0.20 diff --git a/test/debug/test_config_process.py b/test/debug/test_config_process.py index edd4f4b024..21d9cb8f39 100644 --- a/test/debug/test_config_process.py +++ b/test/debug/test_config_process.py @@ -1,22 +1,21 @@ #!/usr/bin/env python -from os.path import ( - join, - realpath, -) -import sys; sys.path.insert(0, realpath(join(__file__, "../../"))) -import sys; sys.path.append(realpath(join(__file__, "../../bin"))) -from bin.hummingbot import main as hb_main -from hummingbot.client.hummingbot_application import HummingbotApplication -import unittest import asyncio -import time import inspect import os +import time +import unittest +from os.path import join, realpath +from test.debug.fixture_configs import FixtureConfigs + +from bin.hummingbot import main as hb_main from hummingbot.client import settings +from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.security import Security +from hummingbot.client.hummingbot_application import HummingbotApplication from hummingbot.strategy.pure_market_making.pure_market_making_config_map import pure_market_making_config_map -from hummingbot.client.config.global_config_map import global_config_map -from test.debug.fixture_configs import FixtureConfigs + +import sys; sys.path.insert(0, realpath(join(__file__, "../../"))) +import sys; sys.path.append(realpath(join(__file__, "../../bin"))) async def wait_til(condition_func, timeout=10): @@ -77,14 +76,14 @@ def setUpClass(cls): @classmethod def tearDownClass(cls) -> None: - remove_files(settings.CONF_FILE_PATH, [".yml", ".json"]) - remove_files_extension(settings.CONF_FILE_PATH, ".temp") + remove_files(settings.STRATEGIES_CONF_DIR_PATH, [".yml", ".json"]) + remove_files_extension(settings.STRATEGIES_CONF_DIR_PATH, ".temp") user_response("stop") cls.ev_loop.run_until_complete(wait_til(lambda: cls.hb.markets_recorder is None)) @classmethod async def set_up_class(cls): - add_files_extension(settings.CONF_FILE_PATH, [".yml", ".json"], ".temp") + add_files_extension(settings.STRATEGIES_CONF_DIR_PATH, [".yml", ".json"], ".temp") asyncio.ensure_future(hb_main()) cls.hb = HummingbotApplication.main_application() await wait_til(lambda: 'Enter "config" to create a bot' in cls.hb.app.output_field.document.text) @@ -143,7 +142,7 @@ def test_pure_mm_basic_til_start(self): async def _test_pure_mm_basic_import_config_file(self): config_file_name = f"{settings.CONF_PREFIX}pure_market_making{settings.CONF_POSTFIX}_0.yml" # update the config file to put in some blank and invalid values. - with open(os.path.join(settings.CONF_FILE_PATH, config_file_name), "r+") as f: + with open(os.path.join(settings.STRATEGIES_CONF_DIR_PATH, config_file_name), "r+") as f: content = f.read() # read everything in the file f.seek(0) # rewind content = content.replace("bid_place_threshold: 0.01", "bid_place_threshold: ") diff --git a/test/hummingbot/client/command/test_connect_command.py b/test/hummingbot/client/command/test_connect_command.py index 01ec97a2f5..a10fbaeb70 100644 --- a/test/hummingbot/client/command/test_connect_command.py +++ b/test/hummingbot/client/command/test_connect_command.py @@ -1,8 +1,9 @@ import asyncio import unittest from copy import deepcopy +from test.mock.mock_cli import CLIMockingAssistant from typing import Awaitable -from unittest.mock import patch, MagicMock, AsyncMock +from unittest.mock import AsyncMock, MagicMock, patch import pandas as pd @@ -10,7 +11,6 @@ from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.security import Security from hummingbot.client.hummingbot_application import HummingbotApplication -from test.mock.mock_cli import CLIMockingAssistant class ConnectCommandTest(unittest.TestCase): @@ -64,22 +64,22 @@ async def run_coro_that_raises(coro: Awaitable): raise RuntimeError @patch("hummingbot.client.config.security.Security.update_secure_config") - @patch("hummingbot.client.config.security.Security.encrypted_file_exists") + @patch("hummingbot.client.config.security.Security.connector_config_file_exists") @patch("hummingbot.client.config.security.Security.api_keys") @patch("hummingbot.user.user_balances.UserBalances.add_exchange") def test_connect_exchange_success( self, add_exchange_mock: AsyncMock, api_keys_mock: AsyncMock, - encrypted_file_exists_mock: MagicMock, - _: MagicMock, + connector_config_file_exists_mock: MagicMock, + update_secure_config_mock: MagicMock, ): add_exchange_mock.return_value = None exchange = "binance" api_key = "someKey" api_secret = "someSecret" api_keys_mock.return_value = {"binance_api_key": api_key, "binance_api_secret": api_secret} - encrypted_file_exists_mock.return_value = False + connector_config_file_exists_mock.return_value = False global_config_map["other_commands_timeout"].value = 30 self.cli_mock_assistant.queue_prompt_reply(api_key) # binance API key self.cli_mock_assistant.queue_prompt_reply(api_secret) # binance API secret @@ -88,16 +88,42 @@ def test_connect_exchange_success( self.assertTrue(self.cli_mock_assistant.check_log_called_with(msg=f"\nYou are now connected to {exchange}.")) self.assertFalse(self.app.placeholder_mode) self.assertFalse(self.app.app.hide_input) + self.assertEqual(update_secure_config_mock.call_count, 1) + + @patch("hummingbot.client.config.security.Security.wait_til_decryption_done") + @patch("hummingbot.client.config.security.save_to_yml") + @patch("hummingbot.client.command.connect_command.CeloCLI") + @patch("hummingbot.client.config.security.Security.connector_config_file_exists") + def test_connect_celo_success( + self, + connector_config_file_exists_mock: MagicMock, + celo_cli_mock: MagicMock, + _: MagicMock, + __: AsyncMock, + ): + connector_config_file_exists_mock.return_value = False + exchange = "celo" + celo_address = "someAddress" + celo_password = "somePassword" + self.cli_mock_assistant.queue_prompt_reply(celo_address) + self.cli_mock_assistant.queue_prompt_reply(celo_password) + celo_cli_mock.validate_node_synced.return_value = None + celo_cli_mock.unlock_account.return_value = None + + self.async_run_with_timeout(self.app.connect_exchange(exchange), 1000) + self.assertTrue(self.cli_mock_assistant.check_log_called_with(msg="\nYou are now connected to celo.")) + self.assertFalse(self.app.placeholder_mode) + self.assertFalse(self.app.app.hide_input) @patch("hummingbot.client.config.security.Security.update_secure_config") - @patch("hummingbot.client.config.security.Security.encrypted_file_exists") + @patch("hummingbot.client.config.security.Security.connector_config_file_exists") @patch("hummingbot.client.config.security.Security.api_keys") @patch("hummingbot.user.user_balances.UserBalances.add_exchange") def test_connect_exchange_handles_network_timeouts( self, add_exchange_mock: AsyncMock, api_keys_mock: AsyncMock, - encrypted_file_exists_mock: MagicMock, + connector_config_file_exists_mock: MagicMock, _: MagicMock, ): add_exchange_mock.side_effect = self.get_async_sleep_fn(delay=0.02) @@ -105,7 +131,7 @@ def test_connect_exchange_handles_network_timeouts( api_key = "someKey" api_secret = "someSecret" api_keys_mock.return_value = {"binance_api_key": api_key, "binance_api_secret": api_secret} - encrypted_file_exists_mock.return_value = False + connector_config_file_exists_mock.return_value = False self.cli_mock_assistant.queue_prompt_reply(api_key) # binance API key self.cli_mock_assistant.queue_prompt_reply(api_secret) # binance API secret @@ -120,12 +146,13 @@ def test_connect_exchange_handles_network_timeouts( self.assertFalse(self.app.app.hide_input) @patch("hummingbot.user.user_balances.UserBalances.update_exchanges") - def test_connection_df_handles_network_timeouts(self, update_exchanges_mock: AsyncMock): + @patch("hummingbot.client.config.security.Security.wait_til_decryption_done") + def test_connection_df_handles_network_timeouts(self, _: AsyncMock, update_exchanges_mock: AsyncMock): update_exchanges_mock.side_effect = self.get_async_sleep_fn(delay=0.02) global_config_map["other_commands_timeout"].value = 0.01 with self.assertRaises(asyncio.TimeoutError): - self.async_run_with_timeout_coroutine_must_raise_timeout(self.app.connection_df()) + self.async_run_with_timeout_coroutine_must_raise_timeout(self.app.connection_df(), 10000) self.assertTrue( self.cli_mock_assistant.check_log_called_with( msg="\nA network error prevented the connection table to populate. See logs for more details." @@ -133,7 +160,8 @@ def test_connection_df_handles_network_timeouts(self, update_exchanges_mock: Asy ) @patch("hummingbot.user.user_balances.UserBalances.update_exchanges") - def test_connection_df_handles_network_timeouts_logs_hidden(self, update_exchanges_mock: AsyncMock): + @patch("hummingbot.client.config.security.Security.wait_til_decryption_done") + def test_connection_df_handles_network_timeouts_logs_hidden(self, _: AsyncMock, update_exchanges_mock: AsyncMock): self.cli_mock_assistant.toggle_logs() update_exchanges_mock.side_effect = self.get_async_sleep_fn(delay=0.02) diff --git a/test/hummingbot/client/command/test_import_command.py b/test/hummingbot/client/command/test_import_command.py index 593c3657ce..d7acd0862f 100644 --- a/test/hummingbot/client/command/test_import_command.py +++ b/test/hummingbot/client/command/test_import_command.py @@ -4,8 +4,9 @@ from decimal import Decimal from pathlib import Path from tempfile import TemporaryDirectory +from test.mock.mock_cli import CLIMockingAssistant from typing import Awaitable, Type -from unittest.mock import patch, MagicMock, AsyncMock +from unittest.mock import AsyncMock, MagicMock, patch from pydantic import Field @@ -14,7 +15,6 @@ from hummingbot.client.config.config_helpers import ClientConfigAdapter, read_system_configs_from_yml, save_to_yml from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.hummingbot_application import HummingbotApplication -from test.mock.mock_cli import CLIMockingAssistant class ImportCommandTest(unittest.TestCase): @@ -154,10 +154,10 @@ def test_import_config_file_success( cm = ClientConfigAdapter(dummy_strategy_config_cls(no_default="some value")) with TemporaryDirectory() as d: - import_command.CONF_FILE_PATH = str(d) d = Path(d) + import_command.STRATEGIES_CONF_DIR_PATH = d temp_file_name = d / strategy_file_name - save_to_yml(str(temp_file_name), cm) + save_to_yml(temp_file_name, cm) self.async_run_with_timeout(self.app.import_config_file(strategy_file_name)) self.assertEqual(strategy_file_name, self.app.strategy_file_name) @@ -180,8 +180,8 @@ def test_import_incomplete_config_file_success( cm = ClientConfigAdapter(dummy_strategy_config_cls(no_default="some value")) with TemporaryDirectory() as d: - import_command.CONF_FILE_PATH = str(d) d = Path(d) + import_command.STRATEGIES_CONF_DIR_PATH = d temp_file_name = d / strategy_file_name cm_yml_str = cm.generate_yml_output_str_with_comments() cm_yml_str = cm_yml_str.replace("\nno_default: some value\n", "") diff --git a/test/hummingbot/client/config/test_config_data_types.py b/test/hummingbot/client/config/test_config_data_types.py index 9d8dfedeaa..8c2f4dc31a 100644 --- a/test/hummingbot/client/config/test_config_data_types.py +++ b/test/hummingbot/client/config/test_config_data_types.py @@ -6,19 +6,18 @@ from typing import Awaitable, Dict from unittest.mock import patch -from pydantic import Field +from pydantic import Field, SecretStr from pydantic.fields import FieldInfo +from hummingbot.client.config.config_crypt import ETHKeyFileSecretManger from hummingbot.client.config.config_data_types import ( BaseClientModel, - BaseStrategyConfigMap, BaseTradingStrategyConfigMap, ClientConfigEnum, ClientFieldData, ) -from hummingbot.client.config.config_helpers import ( - ClientConfigAdapter, ConfigTraversalItem, ConfigValidationError -) +from hummingbot.client.config.config_helpers import ClientConfigAdapter, ConfigTraversalItem, ConfigValidationError +from hummingbot.client.config.security import Security class BaseClientModelTest(unittest.TestCase): @@ -35,9 +34,10 @@ class DummyModel(BaseClientModel): schema = DummyModel.schema_json() j = json.loads(schema) expected = { - "is_secure": False, "prompt": None, "prompt_on_new": True, + "is_secure": False, + "is_connect_key": False, } self.assertEqual(expected, j["properties"]["some_attr"]["client_data"]) @@ -64,11 +64,20 @@ class Config: title = "dummy_model" expected = [ - ConfigTraversalItem(0, "some_attr", "some_attr", 1, "1", ClientFieldData(), None), + ConfigTraversalItem(0, "some_attr", "some_attr", 1, "1", ClientFieldData(), None, int), + ConfigTraversalItem( + 0, + "nested_model", + "nested_model", + ClientConfigAdapter(NestedModel()), + "nested_model", + None, + None, + NestedModel, + ), ConfigTraversalItem( - 0, "nested_model", "nested_model", ClientConfigAdapter(NestedModel()), "nested_model", None, None + 1, "nested_model.nested_attr", "nested_attr", "some value", "some value", None, None, str ), - ConfigTraversalItem(1, "nested_model.nested_attr", "nested_attr", "some value", "some value", None, None), ConfigTraversalItem( 1, "nested_model.double_nested_model", @@ -77,9 +86,17 @@ class Config: "double_nested_model", None, None, + DoubleNestedModel, ), ConfigTraversalItem( - 2, "nested_model.double_nested_model.double_nested_attr", "double_nested_attr", 3.0, "3.0", None, None + 2, + "nested_model.double_nested_model.double_nested_attr", + "double_nested_attr", + 3.0, + "3.0", + None, + None, + float, ), ] cm = ClientConfigAdapter(DummyModel()) @@ -133,7 +150,6 @@ class Config: instance = ClientConfigAdapter(DummyModel()) res_str = instance.generate_yml_output_str_with_comments() - expected_str = """\ ############################## ### dummy_model config ### @@ -162,10 +178,31 @@ class Config: self.assertEqual(expected_str, res_str) + def test_generate_yml_output_dict_with_secret(self): + class DummyModel(BaseClientModel): + secret_attr: SecretStr + + class Config: + title = "dummy_model" + + Security.secrets_manager = ETHKeyFileSecretManger(password="some-password") + secret_value = "some_secret" + instance = ClientConfigAdapter(DummyModel(secret_attr=secret_value)) + res_str = instance.generate_yml_output_str_with_comments() + expected_str = """\ +############################## +### dummy_model config ### +############################## + +secret_attr: """ + + self.assertTrue(res_str.startswith(expected_str)) + self.assertNotIn(secret_value, res_str) + class BaseStrategyConfigMapTest(unittest.TestCase): def test_generate_yml_output_dict_title(self): - class DummyStrategy(BaseStrategyConfigMap): + class DummyStrategy(BaseClientModel): class Config: title = "pure_market_making" diff --git a/test/hummingbot/client/config/test_config_helpers.py b/test/hummingbot/client/config/test_config_helpers.py index fe00a2b5cb..27c81f4cc8 100644 --- a/test/hummingbot/client/config/test_config_helpers.py +++ b/test/hummingbot/client/config/test_config_helpers.py @@ -2,12 +2,24 @@ import unittest from pathlib import Path from tempfile import TemporaryDirectory -from typing import Awaitable +from typing import Awaitable, Optional +from unittest.mock import MagicMock, patch -from hummingbot.client.config.config_data_types import BaseStrategyConfigMap -from hummingbot.client.config.config_helpers import ClientConfigAdapter, get_strategy_config_map, save_to_yml +from pydantic import Field, SecretStr + +from hummingbot.client.config import config_helpers +from hummingbot.client.config.config_crypt import ETHKeyFileSecretManger +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, BaseStrategyConfigMap +from hummingbot.client.config.config_helpers import ( + ClientConfigAdapter, + get_connector_config_yml_path, + get_strategy_config_map, + load_connector_config_map_from_file, + save_to_yml, +) +from hummingbot.client.config.security import Security from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( - AvellanedaMarketMakingConfigMap + AvellanedaMarketMakingConfigMap, ) @@ -49,7 +61,26 @@ class Config: with TemporaryDirectory() as d: d = Path(d) temp_file_name = d / "cm.yml" - save_to_yml(str(temp_file_name), cm) + save_to_yml(temp_file_name, cm) with open(temp_file_name) as f: actual_str = f.read() self.assertEqual(expected_str, actual_str) + + @patch("hummingbot.client.config.config_helpers.AllConnectorSettings.get_connector_config_keys") + def test_load_connector_config_map_from_file_with_secrets(self, get_connector_config_keys_mock: MagicMock): + class DummyConnectorModel(BaseConnectorConfigMap): + connector = "some-connector" + secret_attr: Optional[SecretStr] = Field(default=None) + + password = "some-pass" + Security.secrets_manager = ETHKeyFileSecretManger(password) + cm = ClientConfigAdapter(DummyConnectorModel(secret_attr="some_secret")) + get_connector_config_keys_mock.return_value = DummyConnectorModel() + with TemporaryDirectory() as d: + d = Path(d) + config_helpers.CONNECTORS_CONF_DIR_PATH = d + temp_file_name = get_connector_config_yml_path(cm.connector) + save_to_yml(temp_file_name, cm) + cm_loaded = load_connector_config_map_from_file(temp_file_name) + + self.assertEqual(cm, cm_loaded) diff --git a/test/hummingbot/client/config/test_config_security.py b/test/hummingbot/client/config/test_config_security.py deleted file mode 100644 index b49487541c..0000000000 --- a/test/hummingbot/client/config/test_config_security.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python - -import asyncio -from contextlib import ExitStack -import os -import tempfile -import unittest - -from hummingbot.client.config.security import Security -from hummingbot.client import settings -from hummingbot.client.config.global_config_map import global_config_map -from hummingbot.client.config.config_crypt import encrypt_n_save_config_value - - -class ConfigSecurityNewPasswordUnitTest(unittest.TestCase): - def setUp(self): - self._exit_stack: ExitStack = ExitStack() - self._temp_folder: str = self._exit_stack.enter_context(tempfile.TemporaryDirectory()) - settings.CONF_FILE_PATH = self._temp_folder - global_config_map["key_file_path"].value = self._temp_folder - - def tearDown(self): - self._exit_stack.close() - - def test_new_password_process(self): - # empty folder, new password is required - self.assertFalse(Security.any_encryped_files()) - self.assertTrue(Security.new_password_required()) - # login will pass with any password - result = Security.login("a") - self.assertTrue(result) - Security.update_secure_config("new_key", "new_value") - self.assertTrue(os.path.exists(os.path.join(self._temp_folder, "encrypted_new_key.json"))) - self.assertTrue(Security.encrypted_file_exists("new_key")) - - -class ConfigSecurityExistingPasswordUnitTest(unittest.TestCase): - def setUp(self): - self._exit_stack: ExitStack = ExitStack() - self._temp_folder: str = self._exit_stack.enter_context(tempfile.TemporaryDirectory()) - settings.CONF_FILE_PATH = self._temp_folder - global_config_map["key_file_path"].value = self._temp_folder - encrypt_n_save_config_value("test_key_1", "test_value_1", "a") - encrypt_n_save_config_value("test_key_2", "test_value_2", "a") - - def tearDown(self): - self._exit_stack.close() - - async def _test_existing_password(self): - # check the 2 encrypted files exist - self.assertTrue(os.path.exists(os.path.join(self._temp_folder, "encrypted_test_key_1.json"))) - self.assertTrue(os.path.exists(os.path.join(self._temp_folder, "encrypted_test_key_2.json"))) - self.assertTrue(Security.any_encryped_files()) - self.assertFalse(Security.new_password_required()) - # login fails with incorrect password - result = Security.login("b") - self.assertFalse(result) - # login passes with correct password - result = Security.login("a") - self.assertTrue(result) - # right after logging in, the decryption shouldn't finished yet - self.assertFalse(Security.is_decryption_done()) - await Security.wait_til_decryption_done() - self.assertEqual(len(Security.all_decrypted_values()), 2) - config_value = Security.decrypted_value("test_key_1") - self.assertEqual("test_value_1", config_value) - Security.update_secure_config("test_key_1", "new_value") - self.assertEqual("new_value", Security.decrypted_value("test_key_1")) - - def test_existing_password(self): - loop = asyncio.get_event_loop() - loop.run_until_complete(self._test_existing_password()) diff --git a/test/hummingbot/client/config/test_config_templates.py b/test/hummingbot/client/config/test_config_templates.py index ba39115eaf..22d996a40f 100644 --- a/test/hummingbot/client/config/test_config_templates.py +++ b/test/hummingbot/client/config/test_config_templates.py @@ -1,5 +1,4 @@ import unittest -from os.path import join import ruamel.yaml @@ -16,7 +15,7 @@ class ConfigTemplatesUnitTest(unittest.TestCase): def test_global_config_template_complete(self): - global_config_template_path: str = join(root_path(), "hummingbot/templates/conf_global_TEMPLATE.yml") + global_config_template_path = root_path() / "hummingbot" / "templates" / "conf_global_TEMPLATE.yml" with open(global_config_template_path, "r") as template_fd: template_data = yaml_parser.load(template_fd) @@ -51,7 +50,7 @@ def test_strategy_config_template_complete_legacy(self): ] for strategy in strategies: - strategy_template_path: str = get_strategy_template_path(strategy) + strategy_template_path = get_strategy_template_path(strategy) strategy_config_map = get_strategy_config_map(strategy) with open(strategy_template_path, "r") as template_fd: diff --git a/test/hummingbot/client/config/test_security.py b/test/hummingbot/client/config/test_security.py new file mode 100644 index 0000000000..89ce666105 --- /dev/null +++ b/test/hummingbot/client/config/test_security.py @@ -0,0 +1,124 @@ +import asyncio +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Awaitable + +from hummingbot.client.config import config_crypt, config_helpers, security +from hummingbot.client.config.config_crypt import ETHKeyFileSecretManger, store_password_verification, validate_password +from hummingbot.client.config.config_helpers import ( + ClientConfigAdapter, + api_keys_from_connector_config_map, + get_connector_config_yml_path, + save_to_yml, +) +from hummingbot.client.config.security import Security +from hummingbot.connector.exchange.binance.binance_utils import BinanceConfigMap + + +class SecurityTest(unittest.TestCase): + def setUp(self) -> None: + super().setUp() + self.ev_loop = asyncio.get_event_loop() + self.new_conf_dir_path = TemporaryDirectory() + self.default_pswrd_verification_path = security.PASSWORD_VERIFICATION_PATH + self.default_connectors_conf_dir_path = config_helpers.CONNECTORS_CONF_DIR_PATH + config_crypt.PASSWORD_VERIFICATION_PATH = ( + Path(self.new_conf_dir_path.name) / ".password_verification" + ) + security.PASSWORD_VERIFICATION_PATH = config_crypt.PASSWORD_VERIFICATION_PATH + config_helpers.CONNECTORS_CONF_DIR_PATH = ( + Path(self.new_conf_dir_path.name) / "connectors" + ) + config_helpers.CONNECTORS_CONF_DIR_PATH.mkdir(parents=True, exist_ok=True) + self.connector = "binance" + self.api_key = "someApiKey" + self.api_secret = "someSecret" + + def tearDown(self) -> None: + config_crypt.PASSWORD_VERIFICATION_PATH = self.default_pswrd_verification_path + security.PASSWORD_VERIFICATION_PATH = config_crypt.PASSWORD_VERIFICATION_PATH + config_helpers.CONNECTORS_CONF_DIR_PATH = self.default_connectors_conf_dir_path + self.new_conf_dir_path.cleanup() + self.reset_security() + super().tearDown() + + def async_run_with_timeout(self, coroutine: Awaitable, timeout: float = 1): + ret = self.ev_loop.run_until_complete(asyncio.wait_for(coroutine, timeout)) + return ret + + def store_binance_config(self) -> ClientConfigAdapter: + config_map = ClientConfigAdapter( + BinanceConfigMap(binance_api_key=self.api_key, binance_api_secret=self.api_secret) + ) + file_path = get_connector_config_yml_path(self.connector) + save_to_yml(file_path, config_map) + return config_map + + @staticmethod + def reset_security(): + Security.__instance = None + Security.secrets_manager = None + Security._secure_configs = {} + Security._decryption_done = asyncio.Event() + + def test_password_process(self): + self.assertTrue(Security.new_password_required()) + + password = "som-password" + secrets_manager = ETHKeyFileSecretManger(password) + store_password_verification(secrets_manager) + + self.assertFalse(Security.new_password_required()) + self.assertTrue(validate_password(secrets_manager)) + + another_secrets_manager = ETHKeyFileSecretManger("another-password") + + self.assertFalse(validate_password(another_secrets_manager)) + + def test_login(self): + password = "som-password" + secrets_manager = ETHKeyFileSecretManger(password) + store_password_verification(secrets_manager) + + Security.login(secrets_manager) + config_map = self.store_binance_config() + self.async_run_with_timeout(Security.wait_til_decryption_done(), timeout=2) + + self.assertTrue(Security.is_decryption_done()) + self.assertTrue(Security.any_secure_configs()) + self.assertTrue(Security.connector_config_file_exists(self.connector)) + + api_keys = self.async_run_with_timeout(Security.api_keys(self.connector)) + expected_keys = api_keys_from_connector_config_map(config_map) + + self.assertEqual(expected_keys, api_keys) + + def test_update_secure_config(self): + password = "som-password" + secrets_manager = ETHKeyFileSecretManger(password) + store_password_verification(secrets_manager) + Security.login(secrets_manager) + binance_config = ClientConfigAdapter( + BinanceConfigMap(binance_api_key=self.api_key, binance_api_secret=self.api_secret) + ) + self.async_run_with_timeout(Security.wait_til_decryption_done()) + + Security.update_secure_config(binance_config) + self.reset_security() + + Security.login(secrets_manager) + self.async_run_with_timeout(Security.wait_til_decryption_done(), timeout=2) + binance_loaded_config = Security.decrypted_value(binance_config.connector) + + self.assertEqual(binance_config, binance_loaded_config) + + binance_config.binance_api_key = "someOtherApiKey" + Security.update_secure_config(binance_config) + self.reset_security() + + Security.login(secrets_manager) + self.async_run_with_timeout(Security.wait_til_decryption_done(), timeout=2) + binance_loaded_config = Security.decrypted_value(binance_config.connector) + + self.assertEqual(binance_config, binance_loaded_config) diff --git a/test/hummingbot/connector/test_utils.py b/test/hummingbot/connector/test_utils.py index 8a5805d754..97f98182f9 100644 --- a/test/hummingbot/connector/test_utils.py +++ b/test/hummingbot/connector/test_utils.py @@ -1,5 +1,14 @@ +import importlib import unittest +from os import DirEntry, scandir +from os.path import exists, join +from typing import cast +from pydantic import SecretStr + +from hummingbot import root_path +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.utils import get_new_client_order_id @@ -22,3 +31,34 @@ def test_get_new_client_order_id(self): id2 = get_new_client_order_id(is_buy=True, trading_pair=self.trading_pair, max_id_len=len(id0) - 2) self.assertEqual(len(id0) - 2, len(id2)) + + def test_connector_config_maps(self): + connector_exceptions = ["paper_trade", "celo"] + + type_dirs = [ + cast(DirEntry, f) for f in + scandir(f"{root_path() / 'hummingbot' / 'connector'}") + if f.is_dir() + ] + for type_dir in type_dirs: + connector_dirs = [ + cast(DirEntry, f) for f in scandir(type_dir.path) + if f.is_dir() and exists(join(f.path, "__init__.py")) + ] + for connector_dir in connector_dirs: + if connector_dir.name.startswith("_") or connector_dir.name in connector_exceptions: + continue + util_module_path: str = ( + f"hummingbot.connector.{type_dir.name}.{connector_dir.name}.{connector_dir.name}_utils" + ) + util_module = importlib.import_module(util_module_path) + connector_config = getattr(util_module, "KEYS") + + self.assertIsInstance(connector_config, BaseConnectorConfigMap) + for el in ClientConfigAdapter(connector_config).traverse(): + if el.attr == "connector": + self.assertEqual(el.value, connector_dir.name) + elif el.client_field_data.is_secure: + self.assertEqual(el.type_, SecretStr) + else: + self.assertEqual(el.type_, str) diff --git a/test/hummingbot/core/utils/test_wallet_setup.py b/test/hummingbot/core/utils/test_wallet_setup.py deleted file mode 100644 index c3ee82dc15..0000000000 --- a/test/hummingbot/core/utils/test_wallet_setup.py +++ /dev/null @@ -1,97 +0,0 @@ -""" -Unit tests for hummingbot.core.utils.wallet_setup -""" - -from eth_account import Account -from hummingbot.client.settings import DEFAULT_KEY_FILE_PATH, KEYFILE_PREFIX, KEYFILE_POSTFIX -from hummingbot.client.config.global_config_map import global_config_map -from hummingbot.core.utils.wallet_setup import get_key_file_path, list_wallets, import_and_save_wallet, save_wallet -import os -import tempfile -import unittest.mock - - -class WalletSetupTest(unittest.TestCase): - def test_get_key_file_path(self): - """ - test get_key_file_path - """ - global_config_map["key_file_path"].value = "/my_wallets" - self.assertEqual(get_key_file_path(), global_config_map["key_file_path"].value) - - global_config_map["key_file_path"].value = None - self.assertEqual(get_key_file_path(), DEFAULT_KEY_FILE_PATH) - - def test_save_wallet(self): - """ - test save_wallet - """ - # set the temp_dir as the file that will get returned from get_key_file_path - temp_dir = tempfile.gettempdir() - global_config_map["key_file_path"].value = temp_dir + "/" - - # this private key must be in the correct format or it will fail - # account isn't our code but it gets tested indirectly in test_import_and_save_wallet - private_key = "0x8da4ef21b864d2cc526dbdb2a120bd2874c36c9d0a1fb7f8c63d7f7a8b41de8f" # noqa: mock - acct = Account.privateKeyToAccount(private_key) - - # there is no check on the format of the password in save_wallet - save_wallet(acct, "topsecret") - file_path = "%s%s%s%s" % (get_key_file_path(), KEYFILE_PREFIX, acct.address, KEYFILE_POSTFIX) - self.assertEqual(os.path.exists(file_path), True) - - def test_import_and_save_wallet(self): - """ - test import_and_save_wallet - this is almost the same as test_save_wallet, but good to have in case the functions diverge and we want to be - notified if the behavior changes unexpectedly - """ - - temp_dir = tempfile.gettempdir() - global_config_map["key_file_path"].value = temp_dir + "/" - password = "topsecret" - - ill_formed_private_key1 = "not_hex" - self.assertRaisesRegex(ValueError, "^when sending a str, it must be a hex string", import_and_save_wallet, password, ill_formed_private_key1) - - ill_formed_private_key2 = "0x123123123" # not the expected length - self.assertRaisesRegex(ValueError, "^The private key must be exactly 32 bytes long", import_and_save_wallet, password, ill_formed_private_key2) - - # this private key must be in the correct format or it will fail - private_key = "0x8da4ef21b864d2cc526dbdb2a120bd2874c36c9d0a1fb7f8c63d7f7a8b41de8f" # noqa: mock - password = "topsecret" - acct = import_and_save_wallet(password, private_key) - file_path = "%s%s%s%s" % (get_key_file_path(), KEYFILE_PREFIX, acct.address, KEYFILE_POSTFIX) - self.assertEqual(os.path.exists(file_path), True) - - def test_list_wallets(self): - """ - test list_wallets - """ - # remove any wallets we might have created in other tests - temp_dir = tempfile.gettempdir() - for f in os.listdir(temp_dir): - if f.startswith(KEYFILE_PREFIX) and f.endswith(KEYFILE_POSTFIX): - os.remove(os.path.join(temp_dir, f)) - - # there should be no wallets - self.assertEqual(list_wallets(), []) - - # make one wallet - private_key = "0x8da4ef21b864d2cc526dbdb2a120bd2874c36c9d0a1fb7f8c63d7f7a8b41de8f" # noqa: mock - password = "topsecret" - import_and_save_wallet(password, private_key) - - self.assertEqual(len(list_wallets()), 1) - - # reimporting an existing wallet should not change the count - import_and_save_wallet(password, private_key) - - self.assertEqual(len(list_wallets()), 1) - - # make a second wallet - private_key2 = "0xaaaaaf21b864d2cc526dbdb2a120bd2874c36c9d0a1fb7f8c63d7f7a8b41eeee" # noqa: mock - password2 = "topsecrettopsecret" - import_and_save_wallet(password2, private_key2) - - self.assertEqual(len(list_wallets()), 2) From 5b61de781bfcc8c6867851cef1a375a4f79e66a3 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 4 May 2022 15:37:25 +0300 Subject: [PATCH 052/152] (feat) Better migration process This commit improves the configs migration process for existing hummingbot users. --- conf/connectors/.gitignore | 1 + conf/strategies/.gitignore | 1 + hummingbot/client/ui/__init__.py | 149 +++++++++++++++++++------------ scripts/conf_migration_script.py | 40 +++++---- 4 files changed, 118 insertions(+), 73 deletions(-) create mode 100644 conf/connectors/.gitignore create mode 100644 conf/strategies/.gitignore diff --git a/conf/connectors/.gitignore b/conf/connectors/.gitignore new file mode 100644 index 0000000000..1cda54be93 --- /dev/null +++ b/conf/connectors/.gitignore @@ -0,0 +1 @@ +*.yml diff --git a/conf/strategies/.gitignore b/conf/strategies/.gitignore new file mode 100644 index 0000000000..1cda54be93 --- /dev/null +++ b/conf/strategies/.gitignore @@ -0,0 +1 @@ +*.yml diff --git a/hummingbot/client/ui/__init__.py b/hummingbot/client/ui/__init__.py index 382db3d3ff..4b9932cf14 100644 --- a/hummingbot/client/ui/__init__.py +++ b/hummingbot/client/ui/__init__.py @@ -1,3 +1,5 @@ +import os +import sys from os.path import dirname, join, realpath from typing import Optional, Type @@ -7,8 +9,10 @@ from hummingbot.client.config.config_crypt import BaseSecretsManager, store_password_verification from hummingbot.client.config.global_config_map import color_config_map from hummingbot.client.config.security import Security +from hummingbot.client.settings import CONF_DIR_PATH +from scripts.conf_migration_script import migrate -import sys; sys.path.insert(0, realpath(join(__file__, "../../../"))) +sys.path.insert(0, realpath(join(__file__, "../../../"))) with open(realpath(join(dirname(__file__), '../../VERSION'))) as version_file: @@ -24,6 +28,94 @@ }) +def login_prompt(secrets_manager_cls: Type[BaseSecretsManager]) -> Optional[BaseSecretsManager]: + err_msg = None + secrets_manager = None + if Security.new_password_required() and legacy_confs_exist(): + migrate_configs(secrets_manager_cls) + if Security.new_password_required(): + show_welcome() + password = input_dialog( + title="Set Password", + text="Create a password to protect your sensitive data. " + "This password is not shared with us nor with anyone else, so please store it securely." + "\n\nIf you have used hummingbot before and already have secure configs stored," + " input your previous password in this prompt, then run the scripts/conf_migration_script.py script" + " to migrate your existing secure configs to the new management system." + "\n\nEnter your new password:", + password=True, + style=dialog_style).run() + if password is None: + return None + re_password = input_dialog( + title="Set Password", + text="Please re-enter your password:", + password=True, + style=dialog_style).run() + if re_password is None: + return None + if password != re_password: + err_msg = "Passwords entered do not match, please try again." + else: + secrets_manager = secrets_manager_cls(password) + store_password_verification(secrets_manager) + else: + password = input_dialog( + title="Welcome back to Hummingbot", + text="Enter your password:", + password=True, + style=dialog_style).run() + if password is None: + return None + secrets_manager = secrets_manager_cls(password) + if not Security.login(secrets_manager): + err_msg = "Invalid password - please try again." + if err_msg is not None: + message_dialog( + title='Error', + text=err_msg, + style=dialog_style).run() + return login_prompt(secrets_manager_cls) + return secrets_manager + + +def legacy_confs_exist() -> bool: + encrypted_conf_prefix = "encrypted_" + encrypted_conf_postfix = ".json" + exist = False + for f in sorted(os.listdir(CONF_DIR_PATH)): + f_path = CONF_DIR_PATH / f + if os.path.isfile(f_path) and f.startswith(encrypted_conf_prefix) and f.endswith(encrypted_conf_postfix): + exist = True + break + return exist + + +def migrate_configs(secrets_manager_cls: Type[BaseSecretsManager]): + message_dialog( + title='Configs Migration', + text=""" + + + CONFIGS MIGRATION: + + We have recently refactored the way hummingbot handles configurations. + To migrate your legacy configuration files to the new format, + please enter your password on the following screen. + + """, + style=dialog_style).run() + password = input_dialog( + title="Input Password", + text="\n\nEnter your password:", + password=True, + style=dialog_style).run() + if password is None: + raise ValueError("Wrong password.") + secrets_manager = secrets_manager_cls(password) + migrate(secrets_manager) + + def show_welcome(): message_dialog( title='Welcome to Hummingbot', @@ -59,12 +151,6 @@ def show_welcome(): the time to understand how each strategy works before you risk real capital with it. You are solely responsible for the trades that you perform using Hummingbot. - To use Hummingbot, you first need to give it access to your crypto assets by entering - API keys and/or private keys. These keys are not shared with anyone, including us. - - On the next screen, you will set a password to protect your use of Hummingbot. Please - store this password safely, since only you have access to it and we cannot reset it. - """, style=dialog_style).run() message_dialog( @@ -83,52 +169,3 @@ def show_welcome(): """, style=dialog_style).run() - - -def login_prompt(secrets_manager_cls: Type[BaseSecretsManager]) -> Optional[BaseSecretsManager]: - err_msg = None - secrets_manager = None - if Security.new_password_required(): - show_welcome() - password = input_dialog( - title="Set Password", - text="Create a password to protect your sensitive data. " - "This password is not shared with us nor with anyone else, so please store it securely." - "\n\nIf you have used hummingbot before and already have secure configs stored," - " input your previous password in this prompt, then run the scripts/conf_migration_script.py script" - " to migrate your existing secure configs to the new management system." - "\n\nEnter your new password:", - password=True, - style=dialog_style).run() - if password is None: - return None - re_password = input_dialog( - title="Set Password", - text="Please re-enter your password:", - password=True, - style=dialog_style).run() - if re_password is None: - return None - if password != re_password: - err_msg = "Passwords entered do not match, please try again." - else: - secrets_manager = secrets_manager_cls(password) - store_password_verification(secrets_manager) - else: - password = input_dialog( - title="Welcome back to Hummingbot", - text="Enter your password:", - password=True, - style=dialog_style).run() - if password is None: - return None - secrets_manager = secrets_manager_cls(password) - if not Security.login(secrets_manager): - err_msg = "Invalid password - please try again." - if err_msg is not None: - message_dialog( - title='Error', - text=err_msg, - style=dialog_style).run() - return login_prompt(secrets_manager_cls) - return secrets_manager diff --git a/scripts/conf_migration_script.py b/scripts/conf_migration_script.py index 03658eee5e..7004896dc2 100644 --- a/scripts/conf_migration_script.py +++ b/scripts/conf_migration_script.py @@ -9,7 +9,11 @@ import yaml from hummingbot import root_path -from hummingbot.client.config.config_crypt import ETHKeyFileSecretManger +from hummingbot.client.config.config_crypt import ( + BaseSecretsManager, + ETHKeyFileSecretManger, + store_password_verification, +) from hummingbot.client.config.config_data_types import BaseConnectorConfigMap from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.client.config.security import Security @@ -21,11 +25,12 @@ strategies_conf_dir_path = STRATEGIES_CONF_DIR_PATH -def migrate(password: str): +def migrate(secrets_manager: BaseSecretsManager): print("Starting conf migration.") backup_existing_dir() - migrate_strategy_confs() - migrate_connector_confs(password) + migrate_strategy_confs_paths() + migrate_connector_confs(secrets_manager) + store_password_verification(secrets_manager) def backup_existing_dir(): @@ -41,7 +46,7 @@ def backup_existing_dir(): print(f"\nCreated a backup of your existing conf directory to {backup_path}") -def migrate_strategy_confs(): +def migrate_strategy_confs_paths(): print("\nMigrating strategies...") for child in conf_dir_path.iterdir(): if child.is_file() and child.name.endswith(".yml"): @@ -53,9 +58,8 @@ def migrate_strategy_confs(): print(f"Migrated conf for {conf['strategy']}") -def migrate_connector_confs(password: str): +def migrate_connector_confs(secrets_manager: BaseSecretsManager): print("\nMigrating connector secure keys...") - secrets_manager = ETHKeyFileSecretManger(password) Security.secrets_manager = secrets_manager connector_exceptions = ["paper_trade"] type_dirs: List[DirEntry] = [ @@ -106,22 +110,24 @@ def _maybe_migrate_encrypted_confs(config_keys: BaseConnectorConfigMap): else: missing_fields.append(el.attr) if found_one: + errors = [] if len(missing_fields) != 0: - raise RuntimeError( - f"The migration of {config_keys.connector} failed because of missing fields: {missing_fields}" - ) - errors = cm.validate_model() + errors = [f"missing fields: {missing_fields}"] + if len(errors) == 0: + errors = cm.validate_model() if errors: - raise RuntimeError(f"The migration of {config_keys.connector} failed with errors: {errors}") - Security.update_secure_config(cm.connector, cm) - for f in files_to_remove: - f.unlink() - print(f"Migrated secure keys for {config_keys.connector}") + print(f"The migration of {config_keys.connector} failed with errors: {errors}") + else: + Security.update_secure_config(cm) + for f in files_to_remove: + f.unlink() + print(f"Migrated secure keys for {config_keys.connector}") if __name__ == "__main__": parser = argparse.ArgumentParser(description="Migrate the HummingBot confs") parser.add_argument("password", type=str, help="Required to migrate all encrypted configs.") args = parser.parse_args() - migrate(args.password) + secrets_manager_ = ETHKeyFileSecretManger(args.password) + migrate(secrets_manager_) print("\nConf migration done.") From 68828bc992528e3ea7e53e4e5fa933ba550d3cf2 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Thu, 5 May 2022 17:06:13 +0300 Subject: [PATCH 053/152] (fix) Fixing QA-flagged issues --- bin/conf_migration_script.py | 12 +++++++ hummingbot/client/command/connect_command.py | 11 +++++-- hummingbot/client/command/gateway_command.py | 9 +++-- hummingbot/client/config/config_helpers.py | 11 +++++-- hummingbot/client/config/security.py | 12 +++++-- hummingbot/client/hummingbot_application.py | 4 +-- hummingbot/client/settings.py | 13 ++++++++ hummingbot/client/ui/__init__.py | 33 ++++++++++++++++--- .../exchange/gate_io/gate_io_exchange.py | 10 +++--- .../core/gateway/gateway_http_client.py | 14 +++----- hummingbot/user/user_balances.py | 3 +- setup/environment.yml | 1 - .../client/command/test_connect_command.py | 6 +++- .../hummingbot/client/config/test_security.py | 2 +- .../exchange/gate_io/test_gate_io_exchange.py | 15 +++++++-- 15 files changed, 117 insertions(+), 39 deletions(-) create mode 100644 bin/conf_migration_script.py diff --git a/bin/conf_migration_script.py b/bin/conf_migration_script.py new file mode 100644 index 0000000000..6188103f90 --- /dev/null +++ b/bin/conf_migration_script.py @@ -0,0 +1,12 @@ +import argparse + +from hummingbot.client.config.conf_migration import migrate_configs +from hummingbot.client.config.config_crypt import ETHKeyFileSecretManger + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Migrate the HummingBot confs") + parser.add_argument("password", type=str, help="Required to migrate all encrypted configs.") + args = parser.parse_args() + secrets_manager_ = ETHKeyFileSecretManger(args.password) + migrate_configs(secrets_manager_) + print("\nConf migration done.") diff --git a/hummingbot/client/command/connect_command.py b/hummingbot/client/command/connect_command.py index 85cf9a381e..83ebb3cb68 100644 --- a/hummingbot/client/command/connect_command.py +++ b/hummingbot/client/command/connect_command.py @@ -53,7 +53,9 @@ async def connect_exchange(self, # type: HummingbotApplication to_connect = True if Security.connector_config_file_exists(connector_name): await Security.wait_til_decryption_done() - api_key_config = [c for c in connector_config.traverse() if "api_key" in c.attr] + api_key_config = [ + c.printable_value for c in connector_config.traverse(secure=False) if "api_key" in c.attr + ] if api_key_config: api_key = api_key_config[0] prompt = ( @@ -69,6 +71,7 @@ async def connect_exchange(self, # type: HummingbotApplication to_connect = False if to_connect: await self.prompt_for_model_config(connector_config) + self.app.change_prompt(prompt=">>> ") if self.app.to_stop_config: self.app.to_stop_config = False return @@ -81,6 +84,7 @@ async def connect_exchange(self, # type: HummingbotApplication self.notify(f"\nYou are now connected to {connector_name}.") else: self.notify(f"\nError: {err_msg}") + Security.remove_secure_config(connector_config) self.placeholder_mode = False self.app.hide_input = False self.app.change_prompt(prompt=">>> ") @@ -124,7 +128,7 @@ async def connection_df(self # type: HummingbotApplication keys_confirmed = "Yes" else: api_keys = ( - (await Security.api_keys(option)).values() + Security.api_keys(option).values() if not UserBalances.instance().is_gateway_market(option) else {} ) @@ -156,7 +160,8 @@ async def validate_n_connect_celo(self, to_reconnect: bool = False) -> Optional[ return err_msg async def validate_n_connect_connector(self, connector_name: str) -> Optional[str]: - api_keys = await Security.api_keys(connector_name) + await Security.wait_til_decryption_done() + api_keys = Security.api_keys(connector_name) network_timeout = float(global_config_map["other_commands_timeout"].value) try: err_msg = await asyncio.wait_for( diff --git a/hummingbot/client/command/gateway_command.py b/hummingbot/client/command/gateway_command.py index 6542e0b4d0..ce5579f2b3 100644 --- a/hummingbot/client/command/gateway_command.py +++ b/hummingbot/client/command/gateway_command.py @@ -3,14 +3,13 @@ import itertools import json from contextlib import contextmanager -from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, Generator, List import aiohttp import pandas as pd import docker -from hummingbot.client.config.config_helpers import refresh_trade_fees_config, save_to_yml +from hummingbot.client.config.config_helpers import refresh_trade_fees_config, save_to_yml_legacy from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.security import Security from hummingbot.client.settings import ( @@ -125,7 +124,7 @@ async def _generate_certs( break self.notify("Error: Invalid pass phase") else: - pass_phase = Security.password + pass_phase = Security.secrets_manager.password.get_secret_value() create_self_sign_certs(pass_phase) self.notify(f"Gateway SSL certification files are created in {cert_path}.") GatewayHttpClient.get_instance().reload_certs() @@ -246,7 +245,7 @@ async def _create_gateway(self): logs_mount_path ], host_config=host_config, - environment=[f"GATEWAY_PASSPHRASE={Security.password}"] + environment=[f"GATEWAY_PASSPHRASE={Security.secrets_manager.password.get_secret_value()}"] ) self.notify(f"New Gateway docker container id is {container_info['Id']}.") @@ -255,7 +254,7 @@ async def _create_gateway(self): if global_config_map.get("gateway_api_port").value != gateway_port: global_config_map["gateway_api_port"].value = gateway_port global_config_map["gateway_api_host"].value = "localhost" - save_to_yml(Path(GLOBAL_CONFIG_PATH), global_config_map) + save_to_yml_legacy(GLOBAL_CONFIG_PATH, global_config_map) GatewayHttpClient.get_instance().base_url = f"https://{global_config_map['gateway_api_host'].value}:" \ f"{global_config_map['gateway_api_port'].value}" diff --git a/hummingbot/client/config/config_helpers.py b/hummingbot/client/config/config_helpers.py index f0534d69e8..ec78dd3953 100644 --- a/hummingbot/client/config/config_helpers.py +++ b/hummingbot/client/config/config_helpers.py @@ -35,7 +35,7 @@ TRADE_FEES_CONFIG_PATH, AllConnectorSettings, ) -from hummingbot.connector.other.celo.celo_data_types import KEYS as CELO_KEYS +from hummingbot.connector.other.celo import celo_data_types # Use ruamel.yaml to preserve order and comments in .yml file yaml_parser = ruamel.yaml.YAML() # legacy @@ -525,12 +525,19 @@ def load_connector_config_map_from_file(yml_path: Path) -> ClientConfigAdapter: def get_connector_hb_config(connector_name: str) -> BaseClientModel: if connector_name == "celo": - hb_config = CELO_KEYS + hb_config = celo_data_types.KEYS else: hb_config = AllConnectorSettings.get_connector_config_keys(connector_name) return hb_config +def reset_connector_hb_config(connector_name: str): + if connector_name == "celo": + celo_data_types.KEYS = celo_data_types.KEYS.__class__.construct() + else: + AllConnectorSettings.reset_connector_config_keys(connector_name) + + def api_keys_from_connector_config_map(cm: ClientConfigAdapter) -> Dict[str, str]: api_keys = {} for c in cm.traverse(): diff --git a/hummingbot/client/config/security.py b/hummingbot/client/config/security.py index a4429404ce..e7b5a2c4d7 100644 --- a/hummingbot/client/config/security.py +++ b/hummingbot/client/config/security.py @@ -10,6 +10,7 @@ get_connector_config_yml_path, list_connector_configs, load_connector_config_map_from_file, + reset_connector_hb_config, save_to_yml, ) from hummingbot.core.utils.async_call_scheduler import AsyncCallScheduler @@ -65,6 +66,14 @@ def update_secure_config(cls, connector_config: ClientConfigAdapter): save_to_yml(file_path, connector_config) cls._secure_configs[connector_name] = connector_config + @classmethod + def remove_secure_config(cls, connector_config: ClientConfigAdapter): + connector_name = connector_config.connector + file_path = get_connector_config_yml_path(connector_name) + file_path.unlink(missing_ok=True) + reset_connector_hb_config(connector_name) + cls._secure_configs.pop(connector_name) + @classmethod def is_decryption_done(cls): return cls._decryption_done.is_set() @@ -82,8 +91,7 @@ async def wait_til_decryption_done(cls): await cls._decryption_done.wait() @classmethod - async def api_keys(cls, connector_name: str) -> Dict[str, Optional[str]]: - await cls.wait_til_decryption_done() + def api_keys(cls, connector_name: str) -> Dict[str, Optional[str]]: connector_config = cls.decrypted_value(connector_name) keys = ( api_keys_from_connector_config_map(connector_config) diff --git a/hummingbot/client/hummingbot_application.py b/hummingbot/client/hummingbot_application.py index 3abb66c725..f263449aef 100644 --- a/hummingbot/client/hummingbot_application.py +++ b/hummingbot/client/hummingbot_application.py @@ -274,9 +274,7 @@ def _initialize_markets(self, market_names: List[Tuple[str, List[str]]]): for asset, balance in paper_trade_account_balance.items(): connector.set_balance(asset, balance) else: - Security.update_config_map(global_config_map) - keys = {key: config.value for key, config in global_config_map.items() - if key in conn_setting.config_keys} + keys = Security.api_keys(connector_name) init_params = conn_setting.conn_init_parameters(keys) init_params.update(trading_pairs=trading_pairs, trading_required=self._trading_required) connector_class = get_connector_class(connector_name) diff --git a/hummingbot/client/settings.py b/hummingbot/client/settings.py index 22fb112e41..4c4afc29c6 100644 --- a/hummingbot/client/settings.py +++ b/hummingbot/client/settings.py @@ -331,6 +331,19 @@ def get_connector_settings(cls) -> Dict[str, ConnectorSetting]: def get_connector_config_keys(cls, connector: str) -> Optional["BaseClientModel"]: return cls.get_connector_settings()[connector].config_keys + @classmethod + def reset_connector_config_keys(cls, connector: str): + current_settings = cls.get_connector_settings()[connector] + current_keys = current_settings.config_keys + new_keys = ( + current_keys if current_keys is None else current_keys.__class__.construct() + ) + new_keys_settings_dict = current_settings._asdict() + new_keys_settings_dict.update({"config_keys": new_keys}) + cls.get_connector_settings()[connector] = ConnectorSetting( + **new_keys_settings_dict + ) + @classmethod def get_exchange_names(cls) -> Set[str]: return {cs.name for cs in cls.all_connector_settings.values() if cs.type is ConnectorType.Exchange} diff --git a/hummingbot/client/ui/__init__.py b/hummingbot/client/ui/__init__.py index 4b9932cf14..87f38549d1 100644 --- a/hummingbot/client/ui/__init__.py +++ b/hummingbot/client/ui/__init__.py @@ -6,11 +6,11 @@ from prompt_toolkit.shortcuts import input_dialog, message_dialog from prompt_toolkit.styles import Style +from hummingbot.client.config.conf_migration import migrate_configs from hummingbot.client.config.config_crypt import BaseSecretsManager, store_password_verification from hummingbot.client.config.global_config_map import color_config_map from hummingbot.client.config.security import Security from hummingbot.client.settings import CONF_DIR_PATH -from scripts.conf_migration_script import migrate sys.path.insert(0, realpath(join(__file__, "../../../"))) @@ -32,7 +32,7 @@ def login_prompt(secrets_manager_cls: Type[BaseSecretsManager]) -> Optional[Base err_msg = None secrets_manager = None if Security.new_password_required() and legacy_confs_exist(): - migrate_configs(secrets_manager_cls) + migrate_configs_prompt(secrets_manager_cls) if Security.new_password_required(): show_welcome() password = input_dialog( @@ -91,7 +91,7 @@ def legacy_confs_exist() -> bool: return exist -def migrate_configs(secrets_manager_cls: Type[BaseSecretsManager]): +def migrate_configs_prompt(secrets_manager_cls: Type[BaseSecretsManager]): message_dialog( title='Configs Migration', text=""" @@ -113,7 +113,32 @@ def migrate_configs(secrets_manager_cls: Type[BaseSecretsManager]): if password is None: raise ValueError("Wrong password.") secrets_manager = secrets_manager_cls(password) - migrate(secrets_manager) + errors = migrate_configs(secrets_manager) + if len(errors) != 0: + errors_str = "\n ".join(errors) + message_dialog( + title='Configs Migration Errors', + text=f""" + + + CONFIGS MIGRATION ERRORS: + + {errors_str} + + """, + style=dialog_style).run() + else: + message_dialog( + title='Configs Migration Success', + text=""" + + + CONFIGS MIGRATION SUCCESS: + + The migration process was completed successfully. + + """, + style=dialog_style).run() def show_welcome(): diff --git a/hummingbot/connector/exchange/gate_io/gate_io_exchange.py b/hummingbot/connector/exchange/gate_io/gate_io_exchange.py index d6281428da..6fb7bed4bf 100644 --- a/hummingbot/connector/exchange/gate_io/gate_io_exchange.py +++ b/hummingbot/connector/exchange/gate_io/gate_io_exchange.py @@ -14,12 +14,12 @@ from hummingbot.connector.exchange.gate_io.gate_io_order_book_tracker import GateIoOrderBookTracker from hummingbot.connector.exchange.gate_io.gate_io_user_stream_tracker import GateIoUserStreamTracker from hummingbot.connector.exchange.gate_io.gate_io_utils import ( + GateIoAPIError, + GateIORESTRequest, api_call_with_retries, build_gate_io_api_factory, convert_from_exchange_trading_pair, convert_to_exchange_trading_pair, - GateIoAPIError, - GateIORESTRequest, ) from hummingbot.connector.exchange_base import ExchangeBase from hummingbot.connector.trading_rule import TradingRule @@ -27,9 +27,10 @@ from hummingbot.core.api_throttler.async_throttler import AsyncThrottler from hummingbot.core.clock import Clock from hummingbot.core.data_type.cancellation_result import CancellationResult -from hummingbot.core.data_type.common import OpenOrder +from hummingbot.core.data_type.common import OpenOrder, OrderType, TradeType from hummingbot.core.data_type.limit_order import LimitOrder from hummingbot.core.data_type.order_book import OrderBook +from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee, TokenAmount from hummingbot.core.event.events import ( BuyOrderCompletedEvent, BuyOrderCreatedEvent, @@ -40,8 +41,6 @@ SellOrderCompletedEvent, SellOrderCreatedEvent, ) -from hummingbot.core.data_type.common import OrderType, TradeType -from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee, TokenAmount from hummingbot.core.network_iterator import NetworkStatus from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather from hummingbot.core.web_assistant.connections.data_types import RESTMethod @@ -643,6 +642,7 @@ async def _update_balances(self): warn_msg = (f"Could not fetch balance update from {CONSTANTS.EXCHANGE_NAME}") self.logger().network(f"Unexpected error while fetching balance update - {str(e)}", exc_info=True, app_warning_msg=warn_msg) + raise def stop_tracking_order_exceed_not_found_limit(self, tracked_order: GateIoInFlightOrder): """ diff --git a/hummingbot/core/gateway/gateway_http_client.py b/hummingbot/core/gateway/gateway_http_client.py index dd1e920e7c..8077f9c40c 100644 --- a/hummingbot/core/gateway/gateway_http_client.py +++ b/hummingbot/core/gateway/gateway_http_client.py @@ -1,19 +1,15 @@ -import aiohttp import logging import ssl - from decimal import Decimal from enum import Enum -from typing import Optional, Any, Dict, List, Union +from typing import Any, Dict, List, Optional, Union + +import aiohttp from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.security import Security from hummingbot.core.event.events import TradeType -from hummingbot.core.gateway import ( - detect_existing_gateway_container, - get_gateway_paths, - restart_gateway -) +from hummingbot.core.gateway import detect_existing_gateway_container, get_gateway_paths, restart_gateway from hummingbot.logger import HummingbotLogger @@ -75,7 +71,7 @@ def _http_client(cls, re_init: bool = False) -> aiohttp.ClientSession: ssl_ctx = ssl.create_default_context(cafile=f"{cert_path}/ca_cert.pem") ssl_ctx.load_cert_chain(certfile=f"{cert_path}/client_cert.pem", keyfile=f"{cert_path}/client_key.pem", - password=Security.password) + password=Security.secrets_manager.password.get_secret_value()) conn = aiohttp.TCPConnector(ssl_context=ssl_ctx) cls._shared_client = aiohttp.ClientSession(connector=conn) return cls._shared_client diff --git a/hummingbot/user/user_balances.py b/hummingbot/user/user_balances.py index ed7ae452f6..c36f44ec43 100644 --- a/hummingbot/user/user_balances.py +++ b/hummingbot/user/user_balances.py @@ -74,7 +74,8 @@ async def update_exchange_balance(self, exchange_name: str) -> Optional[str]: if exchange_name in self._markets: return await self._update_balances(self._markets[exchange_name]) else: - api_keys = await Security.api_keys(exchange_name) if not is_gateway_market else {} + await Security.wait_til_decryption_done() + api_keys = Security.api_keys(exchange_name) if not is_gateway_market else {} return await self.add_exchange(exchange_name, **api_keys) # returns error message for each exchange diff --git a/setup/environment.yml b/setup/environment.yml index 6dc3a6e03d..bcb2c127b6 100644 --- a/setup/environment.yml +++ b/setup/environment.yml @@ -59,7 +59,6 @@ dependencies: - hexbytes==0.2.0 - importlib-metadata==0.23 - mypy-extensions==0.4.3 - - path_util==0.1.3, - pre-commit==2.18.1 - psutil==5.7.2 - ptpython==3.0.20 diff --git a/test/hummingbot/client/command/test_connect_command.py b/test/hummingbot/client/command/test_connect_command.py index a10fbaeb70..4dd5f25b59 100644 --- a/test/hummingbot/client/command/test_connect_command.py +++ b/test/hummingbot/client/command/test_connect_command.py @@ -63,6 +63,7 @@ async def run_coro_that_raises(coro: Awaitable): except asyncio.TimeoutError: # the coroutine did not finish on time raise RuntimeError + @patch("hummingbot.client.config.security.Security.wait_til_decryption_done") @patch("hummingbot.client.config.security.Security.update_secure_config") @patch("hummingbot.client.config.security.Security.connector_config_file_exists") @patch("hummingbot.client.config.security.Security.api_keys") @@ -73,6 +74,7 @@ def test_connect_exchange_success( api_keys_mock: AsyncMock, connector_config_file_exists_mock: MagicMock, update_secure_config_mock: MagicMock, + _: MagicMock, ): add_exchange_mock.return_value = None exchange = "binance" @@ -115,6 +117,7 @@ def test_connect_celo_success( self.assertFalse(self.app.placeholder_mode) self.assertFalse(self.app.app.hide_input) + @patch("hummingbot.client.config.security.Security.wait_til_decryption_done") @patch("hummingbot.client.config.security.Security.update_secure_config") @patch("hummingbot.client.config.security.Security.connector_config_file_exists") @patch("hummingbot.client.config.security.Security.api_keys") @@ -125,6 +128,7 @@ def test_connect_exchange_handles_network_timeouts( api_keys_mock: AsyncMock, connector_config_file_exists_mock: MagicMock, _: MagicMock, + __: MagicMock, ): add_exchange_mock.side_effect = self.get_async_sleep_fn(delay=0.02) global_config_map["other_commands_timeout"].value = 0.01 @@ -136,7 +140,7 @@ def test_connect_exchange_handles_network_timeouts( self.cli_mock_assistant.queue_prompt_reply(api_secret) # binance API secret with self.assertRaises(asyncio.TimeoutError): - self.async_run_with_timeout_coroutine_must_raise_timeout(self.app.connect_exchange("binance")) + self.async_run_with_timeout_coroutine_must_raise_timeout(self.app.connect_exchange("binance"), timeout=1000000) self.assertTrue( self.cli_mock_assistant.check_log_called_with( msg="\nA network error prevented the connection to complete. See logs for more details." diff --git a/test/hummingbot/client/config/test_security.py b/test/hummingbot/client/config/test_security.py index 89ce666105..84cde5cb28 100644 --- a/test/hummingbot/client/config/test_security.py +++ b/test/hummingbot/client/config/test_security.py @@ -89,7 +89,7 @@ def test_login(self): self.assertTrue(Security.any_secure_configs()) self.assertTrue(Security.connector_config_file_exists(self.connector)) - api_keys = self.async_run_with_timeout(Security.api_keys(self.connector)) + api_keys = Security.api_keys(self.connector) expected_keys = api_keys_from_connector_config_map(config_map) self.assertEqual(expected_keys, api_keys) diff --git a/test/hummingbot/connector/exchange/gate_io/test_gate_io_exchange.py b/test/hummingbot/connector/exchange/gate_io/test_gate_io_exchange.py index b8c06ce616..bc8a49f213 100644 --- a/test/hummingbot/connector/exchange/gate_io/test_gate_io_exchange.py +++ b/test/hummingbot/connector/exchange/gate_io/test_gate_io_exchange.py @@ -4,7 +4,8 @@ import time import unittest from decimal import Decimal -from typing import Any, Awaitable, List, Dict +from test.hummingbot.connector.network_mocking_assistant import NetworkMockingAssistant +from typing import Any, Awaitable, Dict, List from unittest.mock import patch from aioresponses import aioresponses @@ -12,12 +13,12 @@ from hummingbot.connector.exchange.gate_io import gate_io_constants as CONSTANTS from hummingbot.connector.exchange.gate_io.gate_io_exchange import GateIoExchange from hummingbot.connector.exchange.gate_io.gate_io_in_flight_order import GateIoInFlightOrder +from hummingbot.connector.exchange.gate_io.gate_io_utils import GateIoAPIError from hummingbot.connector.utils import get_new_client_order_id from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.event.event_logger import EventLogger from hummingbot.core.event.events import MarketEvent from hummingbot.core.network_iterator import NetworkStatus -from test.hummingbot.connector.network_mocking_assistant import NetworkMockingAssistant class TestGateIoExchange(unittest.TestCase): @@ -620,3 +621,13 @@ def test_client_order_id_on_order(self, mocked_nonce): ) self.assertEqual(result, expected_client_order_id) + + @aioresponses() + def test_update_balances_raises_on_error(self, mock_api): + url = f"{CONSTANTS.REST_URL}/{CONSTANTS.USER_BALANCES_PATH_URL}" + regex_url = re.compile(f"^{url}".replace(".", r"\.").replace("?", r"\?")) + resp = {"label": "INVALID_KEY", "message": "Invalid key provided"} + mock_api.get(regex_url, body=json.dumps(resp)) + + with self.assertRaises(GateIoAPIError): + self.async_run_with_timeout(coroutine=self.exchange._update_balances()) From 6f8acfff517c2f5590204769dc51a8d610f623a7 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Fri, 6 May 2022 12:53:44 +0300 Subject: [PATCH 054/152] (fix) Additional fixes --- bin/conf_migration_script.py | 1 - hummingbot/client/command/connect_command.py | 9 ++- .../client/config/conf_migration.py | 71 ++++++++++--------- hummingbot/client/config/config_helpers.py | 8 +++ hummingbot/client/config/security.py | 5 +- hummingbot/client/settings.py | 15 ++-- 6 files changed, 65 insertions(+), 44 deletions(-) rename scripts/conf_migration_script.py => hummingbot/client/config/conf_migration.py (71%) diff --git a/bin/conf_migration_script.py b/bin/conf_migration_script.py index 6188103f90..f833618ef3 100644 --- a/bin/conf_migration_script.py +++ b/bin/conf_migration_script.py @@ -9,4 +9,3 @@ args = parser.parse_args() secrets_manager_ = ETHKeyFileSecretManger(args.password) migrate_configs(secrets_manager_) - print("\nConf migration done.") diff --git a/hummingbot/client/command/connect_command.py b/hummingbot/client/command/connect_command.py index 83ebb3cb68..f75a9a6d91 100644 --- a/hummingbot/client/command/connect_command.py +++ b/hummingbot/client/command/connect_command.py @@ -51,6 +51,7 @@ async def connect_exchange(self, # type: HummingbotApplication else: connector_config = ClientConfigAdapter(AllConnectorSettings.get_connector_config_keys(connector_name)) to_connect = True + previous_keys = None if Security.connector_config_file_exists(connector_name): await Security.wait_til_decryption_done() api_key_config = [ @@ -69,6 +70,8 @@ async def connect_exchange(self, # type: HummingbotApplication return if answer.lower() not in ("yes", "y"): to_connect = False + else: + previous_keys = Security.api_keys(connector_name) if to_connect: await self.prompt_for_model_config(connector_config) self.app.change_prompt(prompt=">>> ") @@ -84,7 +87,11 @@ async def connect_exchange(self, # type: HummingbotApplication self.notify(f"\nYou are now connected to {connector_name}.") else: self.notify(f"\nError: {err_msg}") - Security.remove_secure_config(connector_config) + if previous_keys is not None: + previous_config = ClientConfigAdapter(connector_config.hb_config.__class__(**previous_keys)) + Security.update_secure_config(previous_config) + else: + Security.remove_secure_config(connector_name) self.placeholder_mode = False self.app.hide_input = False self.app.change_prompt(prompt=">>> ") diff --git a/scripts/conf_migration_script.py b/hummingbot/client/config/conf_migration.py similarity index 71% rename from scripts/conf_migration_script.py rename to hummingbot/client/config/conf_migration.py index 7004896dc2..0f5953d1e3 100644 --- a/scripts/conf_migration_script.py +++ b/hummingbot/client/config/conf_migration.py @@ -1,4 +1,3 @@ -import argparse import binascii import importlib import shutil @@ -9,11 +8,7 @@ import yaml from hummingbot import root_path -from hummingbot.client.config.config_crypt import ( - BaseSecretsManager, - ETHKeyFileSecretManger, - store_password_verification, -) +from hummingbot.client.config.config_crypt import BaseSecretsManager, store_password_verification from hummingbot.client.config.config_data_types import BaseConnectorConfigMap from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.client.config.security import Security @@ -25,25 +20,36 @@ strategies_conf_dir_path = STRATEGIES_CONF_DIR_PATH -def migrate(secrets_manager: BaseSecretsManager): +def migrate_configs(secrets_manager: BaseSecretsManager) -> List[str]: print("Starting conf migration.") - backup_existing_dir() - migrate_strategy_confs_paths() - migrate_connector_confs(secrets_manager) - store_password_verification(secrets_manager) + errors = backup_existing_dir() + if len(errors) == 0: + migrate_strategy_confs_paths() + errors.extend(migrate_connector_confs(secrets_manager)) + store_password_verification(secrets_manager) + print("\nConf migration done.") + else: + print("\nConf migration failed.") + return errors -def backup_existing_dir(): +def backup_existing_dir() -> List[str]: + errors = [] if conf_dir_path.exists(): backup_path = conf_dir_path.parent / "conf_backup" if backup_path.exists(): - raise RuntimeError( - f"\nBackup path {backup_path} already exists. The migration script cannot backup you" - f" exiting conf files without overwriting that directory. Please remove it and" - f" run the script again." - ) - shutil.copytree(conf_dir_path, backup_path) - print(f"\nCreated a backup of your existing conf directory to {backup_path}") + errors = [ + ( + f"\nBackup path {backup_path} already exists." + f"\nThe migration script cannot backup you exiting" + f"\nconf files without overwriting that directory." + f"\nPlease remove it and run the script again." + ) + ] + else: + shutil.copytree(conf_dir_path, backup_path) + print(f"\nCreated a backup of your existing conf directory to {backup_path}") + return errors def migrate_strategy_confs_paths(): @@ -60,6 +66,7 @@ def migrate_strategy_confs_paths(): def migrate_connector_confs(secrets_manager: BaseSecretsManager): print("\nMigrating connector secure keys...") + errors = [] Security.secrets_manager = secrets_manager connector_exceptions = ["paper_trade"] type_dirs: List[DirEntry] = [ @@ -82,17 +89,18 @@ def migrate_connector_confs(secrets_manager: BaseSecretsManager): util_module = importlib.import_module(util_module_path) config_keys = getattr(util_module, "KEYS", None) if config_keys is not None: - _maybe_migrate_encrypted_confs(config_keys) + errors.extend(_maybe_migrate_encrypted_confs(config_keys)) other_domains = getattr(util_module, "OTHER_DOMAINS", []) for domain in other_domains: config_keys = getattr(util_module, "OTHER_DOMAINS_KEYS")[domain] if config_keys is not None: - _maybe_migrate_encrypted_confs(config_keys) + errors.extend(_maybe_migrate_encrypted_confs(config_keys)) except ModuleNotFoundError: continue + return errors -def _maybe_migrate_encrypted_confs(config_keys: BaseConnectorConfigMap): +def _maybe_migrate_encrypted_confs(config_keys: BaseConnectorConfigMap) -> List[str]: cm = ClientConfigAdapter(config_keys) found_one = False files_to_remove = [] @@ -109,25 +117,18 @@ def _maybe_migrate_encrypted_confs(config_keys: BaseConnectorConfigMap): found_one = True else: missing_fields.append(el.attr) + errors = [] if found_one: - errors = [] if len(missing_fields) != 0: - errors = [f"missing fields: {missing_fields}"] + errors = [f"{config_keys.connector} - missing fields: {missing_fields}"] if len(errors) == 0: errors = cm.validate_model() if errors: + errors = [f"{config_keys.connector} - {e}" for e in errors] print(f"The migration of {config_keys.connector} failed with errors: {errors}") else: Security.update_secure_config(cm) - for f in files_to_remove: - f.unlink() print(f"Migrated secure keys for {config_keys.connector}") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Migrate the HummingBot confs") - parser.add_argument("password", type=str, help="Required to migrate all encrypted configs.") - args = parser.parse_args() - secrets_manager_ = ETHKeyFileSecretManger(args.password) - migrate(secrets_manager_) - print("\nConf migration done.") + for f in files_to_remove: + f.unlink() + return errors diff --git a/hummingbot/client/config/config_helpers.py b/hummingbot/client/config/config_helpers.py index ec78dd3953..ae51600561 100644 --- a/hummingbot/client/config/config_helpers.py +++ b/hummingbot/client/config/config_helpers.py @@ -538,6 +538,14 @@ def reset_connector_hb_config(connector_name: str): AllConnectorSettings.reset_connector_config_keys(connector_name) +def update_connector_hb_config(connector_config: ClientConfigAdapter): + connector_name = connector_config.connector + if connector_name == "celo": + celo_data_types.KEYS = connector_config.hb_config + else: + AllConnectorSettings.update_connector_config_keys(connector_config.hb_config) + + def api_keys_from_connector_config_map(cm: ClientConfigAdapter) -> Dict[str, str]: api_keys = {} for c in cm.traverse(): diff --git a/hummingbot/client/config/security.py b/hummingbot/client/config/security.py index e7b5a2c4d7..736bfaf1f4 100644 --- a/hummingbot/client/config/security.py +++ b/hummingbot/client/config/security.py @@ -12,6 +12,7 @@ load_connector_config_map_from_file, reset_connector_hb_config, save_to_yml, + update_connector_hb_config, ) from hummingbot.core.utils.async_call_scheduler import AsyncCallScheduler from hummingbot.core.utils.async_utils import safe_ensure_future @@ -64,11 +65,11 @@ def update_secure_config(cls, connector_config: ClientConfigAdapter): connector_name = connector_config.connector file_path = get_connector_config_yml_path(connector_name) save_to_yml(file_path, connector_config) + update_connector_hb_config(connector_config) cls._secure_configs[connector_name] = connector_config @classmethod - def remove_secure_config(cls, connector_config: ClientConfigAdapter): - connector_name = connector_config.connector + def remove_secure_config(cls, connector_name: str): file_path = get_connector_config_yml_path(connector_name) file_path.unlink(missing_ok=True) reset_connector_hb_config(connector_name) diff --git a/hummingbot/client/settings.py b/hummingbot/client/settings.py index 4c4afc29c6..fd19cc5a9a 100644 --- a/hummingbot/client/settings.py +++ b/hummingbot/client/settings.py @@ -15,7 +15,7 @@ from hummingbot.core.data_type.trade_fee import TradeFeeSchema if TYPE_CHECKING: - from hummingbot.client.config.config_data_types import BaseClientModel + from hummingbot.client.config.config_data_types import BaseConnectorConfigMap # Global variables required_exchanges: Set[str] = set() @@ -122,7 +122,7 @@ class ConnectorSetting(NamedTuple): centralised: bool use_ethereum_wallet: bool trade_fee_schema: TradeFeeSchema - config_keys: Optional["BaseClientModel"] + config_keys: Optional["BaseConnectorConfigMap"] is_sub_domain: bool parent_name: Optional[str] domain_parameter: Optional[str] @@ -328,7 +328,7 @@ def get_connector_settings(cls) -> Dict[str, ConnectorSetting]: return cls.all_connector_settings @classmethod - def get_connector_config_keys(cls, connector: str) -> Optional["BaseClientModel"]: + def get_connector_config_keys(cls, connector: str) -> Optional["BaseConnectorConfigMap"]: return cls.get_connector_settings()[connector].config_keys @classmethod @@ -338,9 +338,14 @@ def reset_connector_config_keys(cls, connector: str): new_keys = ( current_keys if current_keys is None else current_keys.__class__.construct() ) + cls.update_connector_config_keys(new_keys) + + @classmethod + def update_connector_config_keys(cls, new_config_keys: "BaseConnectorConfigMap"): + current_settings = cls.get_connector_settings()[new_config_keys.connector] new_keys_settings_dict = current_settings._asdict() - new_keys_settings_dict.update({"config_keys": new_keys}) - cls.get_connector_settings()[connector] = ConnectorSetting( + new_keys_settings_dict.update({"config_keys": new_config_keys}) + cls.get_connector_settings()[new_config_keys.connector] = ConnectorSetting( **new_keys_settings_dict ) From 77cf7e4f269c29fc9a0805edf182d594d500a06d Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Fri, 6 May 2022 13:19:30 +0300 Subject: [PATCH 055/152] (fix) Fixes password check --- bin/hummingbot_quickstart.py | 2 +- hummingbot/client/config/conf_migration.py | 19 ++++++++++--------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/bin/hummingbot_quickstart.py b/bin/hummingbot_quickstart.py index f916e203c4..89bd56681d 100755 --- a/bin/hummingbot_quickstart.py +++ b/bin/hummingbot_quickstart.py @@ -77,7 +77,7 @@ async def quick_start(args: argparse.Namespace, secrets_manager: BaseSecretsMana if args.auto_set_permissions is not None: autofix_permissions(args.auto_set_permissions) - if Security.login(secrets_manager): + if not Security.login(secrets_manager): logging.getLogger().error("Invalid password.") return diff --git a/hummingbot/client/config/conf_migration.py b/hummingbot/client/config/conf_migration.py index 0f5953d1e3..7b0b92f974 100644 --- a/hummingbot/client/config/conf_migration.py +++ b/hummingbot/client/config/conf_migration.py @@ -1,5 +1,6 @@ import binascii import importlib +import logging import shutil from os import DirEntry, scandir from os.path import exists, join @@ -21,15 +22,15 @@ def migrate_configs(secrets_manager: BaseSecretsManager) -> List[str]: - print("Starting conf migration.") + logging.getLogger().info("Starting conf migration.") errors = backup_existing_dir() if len(errors) == 0: migrate_strategy_confs_paths() errors.extend(migrate_connector_confs(secrets_manager)) store_password_verification(secrets_manager) - print("\nConf migration done.") + logging.getLogger().info("\nConf migration done.") else: - print("\nConf migration failed.") + logging.getLogger().error("\nConf migration failed.") return errors @@ -48,12 +49,12 @@ def backup_existing_dir() -> List[str]: ] else: shutil.copytree(conf_dir_path, backup_path) - print(f"\nCreated a backup of your existing conf directory to {backup_path}") + logging.getLogger().info(f"\nCreated a backup of your existing conf directory to {backup_path}") return errors def migrate_strategy_confs_paths(): - print("\nMigrating strategies...") + logging.getLogger().info("\nMigrating strategies...") for child in conf_dir_path.iterdir(): if child.is_file() and child.name.endswith(".yml"): with open(str(child), "r") as f: @@ -61,11 +62,11 @@ def migrate_strategy_confs_paths(): if "strategy" in conf and "exchange" in conf: new_path = strategies_conf_dir_path / child.name child.rename(new_path) - print(f"Migrated conf for {conf['strategy']}") + logging.getLogger().info(f"Migrated conf for {conf['strategy']}") def migrate_connector_confs(secrets_manager: BaseSecretsManager): - print("\nMigrating connector secure keys...") + logging.getLogger().info("\nMigrating connector secure keys...") errors = [] Security.secrets_manager = secrets_manager connector_exceptions = ["paper_trade"] @@ -125,10 +126,10 @@ def _maybe_migrate_encrypted_confs(config_keys: BaseConnectorConfigMap) -> List[ errors = cm.validate_model() if errors: errors = [f"{config_keys.connector} - {e}" for e in errors] - print(f"The migration of {config_keys.connector} failed with errors: {errors}") + logging.getLogger().error(f"The migration of {config_keys.connector} failed with errors: {errors}") else: Security.update_secure_config(cm) - print(f"Migrated secure keys for {config_keys.connector}") + logging.getLogger().info(f"Migrated secure keys for {config_keys.connector}") for f in files_to_remove: f.unlink() return errors From 357e131c68a881112757c9aa0102441c871f0bdb Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Fri, 6 May 2022 14:11:41 +0300 Subject: [PATCH 056/152] (fix) Fixes folder creation in docker container --- hummingbot/client/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/hummingbot/client/settings.py b/hummingbot/client/settings.py index fd19cc5a9a..d099fc4115 100644 --- a/hummingbot/client/settings.py +++ b/hummingbot/client/settings.py @@ -35,7 +35,9 @@ TEMPLATE_PATH = root_path() / "hummingbot" / "templates" CONF_DIR_PATH = root_path() / "conf" STRATEGIES_CONF_DIR_PATH = CONF_DIR_PATH / "strategies" +STRATEGIES_CONF_DIR_PATH.mkdir(parents=True, exist_ok=True) CONNECTORS_CONF_DIR_PATH = CONF_DIR_PATH / "connectors" +CONNECTORS_CONF_DIR_PATH.mkdir(parents=True, exist_ok=True) CONF_PREFIX = "conf_" CONF_POSTFIX = "_strategy" PMM_SCRIPTS_PATH = root_path() / "pmm_scripts" From 0ee9d6402bf4bd2da39e77fbbb5d3da610fe83ac Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Tue, 10 May 2022 11:31:08 +0300 Subject: [PATCH 057/152] (fix) Fixes QA-raised issues - Adds the `connectors` and `strategies` folders to the docker image. - Bug fixes. --- .dockerignore | 1 - Dockerfile | 2 ++ bin/hummingbot.py | 5 ++--- bin/hummingbot_quickstart.py | 9 ++++++--- conf/.dockerignore | 10 ++++++++++ conf/connectors/.dockerignore | 1 + conf/connectors/__init__.py | 0 conf/strategies/.dockerignore | 1 + conf/strategies/__init__.py | 0 hummingbot/client/command/balance_command.py | 2 +- hummingbot/client/config/config_helpers.py | 18 +++++++++++------- hummingbot/client/settings.py | 11 ++++++++--- hummingbot/client/ui/__init__.py | 9 +++++---- 13 files changed, 47 insertions(+), 22 deletions(-) create mode 100644 conf/.dockerignore create mode 100644 conf/connectors/.dockerignore create mode 100644 conf/connectors/__init__.py create mode 100644 conf/strategies/.dockerignore create mode 100644 conf/strategies/__init__.py diff --git a/.dockerignore b/.dockerignore index 32b80173aa..6cee1089f7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,3 @@ -/conf /data /logs /build diff --git a/Dockerfile b/Dockerfile index 3f91fcd59e..a3c54dded4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -114,6 +114,8 @@ VOLUME /conf /logs /data /pmm_scripts /scripts \ COPY --chown=hummingbot:hummingbot pmm_scripts/ pmm_scripts/ # Pre-populate scripts/ volume with default scripts COPY --chown=hummingbot:hummingbot scripts/ scripts/ +# Copy the conf folder structure +COPY --chown=hummingbot:hummingbot conf/ conf/ # Install packages required in runtime RUN apt-get update && \ diff --git a/bin/hummingbot.py b/bin/hummingbot.py index 4af7f9d7a1..aba2b95648 100755 --- a/bin/hummingbot.py +++ b/bin/hummingbot.py @@ -39,9 +39,8 @@ def hummingbot_app(self) -> HummingbotApplication: async def ui_start_handler(self): hb: HummingbotApplication = self.hummingbot_app - - if hb.strategy_file_name is not None and hb.strategy_name is not None: - await write_config_to_yml(hb.strategy_name, hb.strategy_file_name) + if hb.strategy_config_map is not None: + write_config_to_yml(hb.strategy_config_map, hb.strategy_file_name) hb.start(global_config_map.get("log_level").value) diff --git a/bin/hummingbot_quickstart.py b/bin/hummingbot_quickstart.py index 89bd56681d..a811a34c4a 100755 --- a/bin/hummingbot_quickstart.py +++ b/bin/hummingbot_quickstart.py @@ -91,18 +91,21 @@ async def quick_start(args: argparse.Namespace, secrets_manager: BaseSecretsMana hb = HummingbotApplication.main_application() # Todo: validate strategy and config_file_name before assinging + strategy_config = None if config_file_name is not None: hb.strategy_file_name = config_file_name - hb.strategy_name = await load_strategy_config_map_from_file( + strategy_config = await load_strategy_config_map_from_file( STRATEGIES_CONF_DIR_PATH / config_file_name ) + hb.strategy_name = strategy_config.strategy + hb.strategy_config_map = strategy_config # To ensure quickstart runs with the default value of False for kill_switch_enabled if not present if not global_config_map.get("kill_switch_enabled"): global_config_map.get("kill_switch_enabled").value = False - if hb.strategy_name and hb.strategy_file_name: - if not all_configs_complete(hb.strategy_name): + if strategy_config is not None: + if not all_configs_complete(strategy_config): hb.status() # The listener needs to have a named variable for keeping reference, since the event listener system diff --git a/conf/.dockerignore b/conf/.dockerignore new file mode 100644 index 0000000000..1ac225396f --- /dev/null +++ b/conf/.dockerignore @@ -0,0 +1,10 @@ +/config_local.py +/quote_api_secret.py +/web3_wallet_secret.py +/binance_secret.py +/api_secrets.py +/*secret* +*encrypted* +*key* +*.yml +/gateway_connections.json diff --git a/conf/connectors/.dockerignore b/conf/connectors/.dockerignore new file mode 100644 index 0000000000..1cda54be93 --- /dev/null +++ b/conf/connectors/.dockerignore @@ -0,0 +1 @@ +*.yml diff --git a/conf/connectors/__init__.py b/conf/connectors/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/conf/strategies/.dockerignore b/conf/strategies/.dockerignore new file mode 100644 index 0000000000..1cda54be93 --- /dev/null +++ b/conf/strategies/.dockerignore @@ -0,0 +1 @@ +*.yml diff --git a/conf/strategies/__init__.py b/conf/strategies/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hummingbot/client/command/balance_command.py b/hummingbot/client/command/balance_command.py index e6845c95fb..858956de19 100644 --- a/hummingbot/client/command/balance_command.py +++ b/hummingbot/client/command/balance_command.py @@ -118,7 +118,7 @@ async def show_balances(self): self.notify(f"\n\nExchanges Total: {RateOracle.global_token_symbol} {exchanges_total:.0f} ") - celo_address = CELO_KEYS.celo_address if hasattr("celo_address", CELO_KEYS) else None + celo_address = CELO_KEYS.celo_address if hasattr(CELO_KEYS, "celo_address") else None if celo_address is not None: try: if not CeloCLI.unlocked: diff --git a/hummingbot/client/config/config_helpers.py b/hummingbot/client/config/config_helpers.py index ae51600561..b7bc88b518 100644 --- a/hummingbot/client/config/config_helpers.py +++ b/hummingbot/client/config/config_helpers.py @@ -424,7 +424,6 @@ def get_strategy_config_map( """ Given the name of a strategy, find and load strategy-specific config map. """ - config_map = None try: config_cls = get_strategy_pydantic_config_cls(strategy) if config_cls is None: # legacy @@ -712,8 +711,9 @@ def save_to_yml(yml_path: Path, cm: ClientConfigAdapter): logging.getLogger().error("Error writing configs: %s" % (str(e),), exc_info=True) -async def write_config_to_yml(strategy_name, strategy_file_name): - strategy_config_map = get_strategy_config_map(strategy_name) +def write_config_to_yml( + strategy_config_map: Union[ClientConfigAdapter, Dict], strategy_file_name: str +): strategy_file_path = Path(STRATEGIES_CONF_DIR_PATH) / strategy_file_name if isinstance(strategy_config_map, ClientConfigAdapter): save_to_yml(strategy_file_path, strategy_config_map) @@ -776,12 +776,16 @@ def short_strategy_name(strategy: str) -> str: return strategy -def all_configs_complete(strategy): - strategy_map = get_strategy_config_map(strategy) - return config_map_complete(global_config_map) and config_map_complete(strategy_map) +def all_configs_complete(strategy_config): + strategy_valid = ( + config_map_complete_legacy(strategy_config) + if isinstance(strategy_config, Dict) + else len(strategy_config.validate_model()) == 0 + ) + return config_map_complete_legacy(global_config_map) and strategy_valid -def config_map_complete(config_map): +def config_map_complete_legacy(config_map): return not any(c.required and c.value is None for c in config_map.values()) diff --git a/hummingbot/client/settings.py b/hummingbot/client/settings.py index d099fc4115..739864ef4e 100644 --- a/hummingbot/client/settings.py +++ b/hummingbot/client/settings.py @@ -35,9 +35,7 @@ TEMPLATE_PATH = root_path() / "hummingbot" / "templates" CONF_DIR_PATH = root_path() / "conf" STRATEGIES_CONF_DIR_PATH = CONF_DIR_PATH / "strategies" -STRATEGIES_CONF_DIR_PATH.mkdir(parents=True, exist_ok=True) CONNECTORS_CONF_DIR_PATH = CONF_DIR_PATH / "connectors" -CONNECTORS_CONF_DIR_PATH.mkdir(parents=True, exist_ok=True) CONF_PREFIX = "conf_" CONF_POSTFIX = "_strategy" PMM_SCRIPTS_PATH = root_path() / "pmm_scripts" @@ -50,6 +48,13 @@ GATEAWAY_CLIENT_CERT_PATH = CERTS_PATH / "client_cert.pem" GATEAWAY_CLIENT_KEY_PATH = CERTS_PATH / "client_key.pem" +PAPER_TRADE_EXCHANGES = [ # todo: fix after global config map refactor + "binance_paper_trade", + "kucoin_paper_trade", + "ascend_ex_paper_trade", + "gate_io_paper_trade", +] + class ConnectorType(Enum): """ @@ -308,7 +313,7 @@ def initialize_paper_trade_settings(cls, paper_trade_exchanges: List[str]): @classmethod def get_all_connectors(cls) -> List[str]: """Avoids circular import problems introduced by `create_connector_settings`.""" - connector_names = [] + connector_names = PAPER_TRADE_EXCHANGES type_dirs: List[DirEntry] = [ cast(DirEntry, f) for f in scandir(f"{root_path() / 'hummingbot' / 'connector'}") diff --git a/hummingbot/client/ui/__init__.py b/hummingbot/client/ui/__init__.py index 87f38549d1..f5d231adfe 100644 --- a/hummingbot/client/ui/__init__.py +++ b/hummingbot/client/ui/__init__.py @@ -32,7 +32,7 @@ def login_prompt(secrets_manager_cls: Type[BaseSecretsManager]) -> Optional[Base err_msg = None secrets_manager = None if Security.new_password_required() and legacy_confs_exist(): - migrate_configs_prompt(secrets_manager_cls) + secrets_manager = migrate_configs_prompt(secrets_manager_cls) if Security.new_password_required(): show_welcome() password = input_dialog( @@ -68,8 +68,8 @@ def login_prompt(secrets_manager_cls: Type[BaseSecretsManager]) -> Optional[Base if password is None: return None secrets_manager = secrets_manager_cls(password) - if not Security.login(secrets_manager): - err_msg = "Invalid password - please try again." + if err_msg is None and not Security.login(secrets_manager): + err_msg = "Invalid password - please try again." if err_msg is not None: message_dialog( title='Error', @@ -91,7 +91,7 @@ def legacy_confs_exist() -> bool: return exist -def migrate_configs_prompt(secrets_manager_cls: Type[BaseSecretsManager]): +def migrate_configs_prompt(secrets_manager_cls: Type[BaseSecretsManager]) -> BaseSecretsManager: message_dialog( title='Configs Migration', text=""" @@ -139,6 +139,7 @@ def migrate_configs_prompt(secrets_manager_cls: Type[BaseSecretsManager]): """, style=dialog_style).run() + return secrets_manager def show_welcome(): From 877b186663d0be83697b9b278a011cdcc5b4a608 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Tue, 10 May 2022 14:16:48 +0200 Subject: [PATCH 058/152] (refactor) base trailing indicator - sampling & pr buffer length handling --- .../base_trailing_indicator.py | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/hummingbot/strategy/__utils__/trailing_indicators/base_trailing_indicator.py b/hummingbot/strategy/__utils__/trailing_indicators/base_trailing_indicator.py index 8df46b58b0..ddd1694e6d 100644 --- a/hummingbot/strategy/__utils__/trailing_indicators/base_trailing_indicator.py +++ b/hummingbot/strategy/__utils__/trailing_indicators/base_trailing_indicator.py @@ -19,6 +19,7 @@ def __init__(self, sampling_length: int = 30, processing_length: int = 15): self._sampling_buffer = RingBuffer(sampling_length) self._processing_length = processing_length self._processing_buffer = RingBuffer(processing_length) + self._samples_length = 0 def add_sample(self, value: float): self._sampling_buffer.add_value(value) @@ -47,3 +48,28 @@ def is_sampling_buffer_full(self) -> bool: @property def is_processing_buffer_full(self) -> bool: return self._processing_buffer.is_full + + @property + def is_sampling_buffer_changed(self) -> bool: + buffer_len = len(self._sampling_buffer.get_as_numpy_array()) + is_changed = self._samples_length != buffer_len + self._samples_length = buffer_len + return is_changed + + @property + def sampling_length(self) -> int: + return self._sampling_length + + @sampling_length.setter + def sampling_length(self, value): + self._sampling_length = value + self._sampling_buffer.length = value + + @property + def processing_length(self) -> int: + return self._processing_length + + @processing_length.setter + def processing_length(self, value): + self._processing_length = value + self._processing_buffer.length = value From 3284e78b3ad2b820e494c2db73b68964b56a9a95 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Tue, 10 May 2022 14:17:02 +0200 Subject: [PATCH 059/152] (refactor) ring buffer length setting --- hummingbot/strategy/__utils__/ring_buffer.pyx | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/hummingbot/strategy/__utils__/ring_buffer.pyx b/hummingbot/strategy/__utils__/ring_buffer.pyx index 3c572bb580..4f480d4c3e 100644 --- a/hummingbot/strategy/__utils__/ring_buffer.pyx +++ b/hummingbot/strategy/__utils__/ring_buffer.pyx @@ -100,3 +100,19 @@ cdef class RingBuffer: @property def variance(self): return self.c_variance() + + @property + def length(self) -> int: + return self._length + + @length.setter + def length(self, value): + data = self.get_as_numpy_array() + + self._length = value + self._buffer = np.zeros(value, dtype=np.float64) + self._delimiter = 0 + self._is_full = False + + for val in data[-value:]: + self.add_value(val) From 3a3d0c8e5541a4735e9f2f4f39dda2260b051b17 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Tue, 10 May 2022 14:17:33 +0200 Subject: [PATCH 060/152] (refactor) trading intensity sampling length setting --- .../__utils__/trailing_indicators/trading_intensity.pyx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/hummingbot/strategy/__utils__/trailing_indicators/trading_intensity.pyx b/hummingbot/strategy/__utils__/trailing_indicators/trading_intensity.pyx index b1a350da8f..7b8cb79a1e 100644 --- a/hummingbot/strategy/__utils__/trailing_indicators/trading_intensity.pyx +++ b/hummingbot/strategy/__utils__/trailing_indicators/trading_intensity.pyx @@ -172,3 +172,9 @@ cdef class TradingIntensityIndicator(): @property def sampling_length(self) -> int: return self._sampling_length + + @sampling_length.setter + def sampling_length(self, value): + self._sampling_length = value + if self._sampling_length < len(self._trades): + self._trades = self._trades[-self._sampling_length:] From e8e713082a8668eacb812f78ae75156a41c8ca1c Mon Sep 17 00:00:00 2001 From: mhrvth Date: Tue, 10 May 2022 14:17:54 +0200 Subject: [PATCH 061/152] (refactor) avellaneda - settings from config map --- .../avellaneda_market_making.pxd | 15 +- .../avellaneda_market_making.pyx | 335 +++++++++++------- .../avellaneda_market_making/start.py | 74 +--- 3 files changed, 220 insertions(+), 204 deletions(-) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pxd b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pxd index b0759ae5e8..6dc03db4e9 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pxd +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pxd @@ -6,20 +6,12 @@ from hummingbot.strategy.strategy_base cimport StrategyBase cdef class AvellanedaMarketMakingStrategy(StrategyBase): cdef: + object _config_map object _market_info object _minimum_spread - object _order_amount - double _order_refresh_time - double _max_order_age - object _order_refresh_tolerance_pct - double _filled_order_delay - int _order_levels - object _level_distances - object _order_override bint _hanging_orders_enabled + object _hanging_orders_cancel_pct object _hanging_orders_tracker - object _inventory_target_base_pct - bint _order_optimization_enabled bint _add_transaction_costs_to_orders bint _hb_app_notification bint _is_debug @@ -37,11 +29,14 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): int _volatility_sampling_period double _last_sampling_timestamp bint _parameters_based_on_spread + int _volatility_buffer_size + int _trading_intensity_buffer_size int _ticks_to_be_ready object _alpha object _kappa object _gamma object _eta + str _execution_mode str _execution_timeframe object _execution_state object _start_time diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx index 25902050fe..26988aaad9 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx @@ -19,7 +19,19 @@ from hummingbot.core.network_iterator import NetworkStatus from hummingbot.core.utils import map_df_to_str from hummingbot.strategy.__utils__.trailing_indicators.instant_volatility import InstantVolatilityIndicator from hummingbot.strategy.__utils__.trailing_indicators.trading_intensity import TradingIntensityIndicator -from hummingbot.strategy.conditional_execution_state import ConditionalExecutionState, RunAlwaysExecutionState +from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( + AvellanedaMarketMakingConfigMap, + DailyBetweenTimesModel, + FromDateToDateModel, + MultiOrderLevelModel, + SingleOrderLevelModel, + TrackHangingOrdersModel, +) +from hummingbot.strategy.conditional_execution_state import ( + ConditionalExecutionState, + RunAlwaysExecutionState, + RunInTimeConditionalExecutionState +) from hummingbot.strategy.data_types import ( PriceSize, Proposal, @@ -54,54 +66,22 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): return pmm_logger def init_params(self, + config_map: AvellanedaMarketMakingConfigMap, market_info: MarketTradingPairTuple, - order_amount: Decimal, - order_refresh_time: float = 30.0, - max_order_age: float = 1800, - order_refresh_tolerance_pct: Decimal = s_decimal_neg_one, - order_optimization_enabled = True, - filled_order_delay: float = 60.0, - order_levels: int = 0, - level_distances: Decimal = Decimal("0.0"), - order_override: Dict[str, List[str]] = {}, - hanging_orders_enabled: bool = False, - hanging_orders_cancel_pct: Decimal = Decimal("0.1"), - inventory_target_base_pct: Decimal = s_decimal_zero, - add_transaction_costs_to_orders: bool = True, logging_options: int = OPTION_LOG_ALL, status_report_interval: float = 900, hb_app_notification: bool = False, - risk_factor: Decimal = Decimal("0.5"), - order_amount_shape_factor: Decimal = Decimal("0.0"), - execution_timeframe: str = "infinite", - execution_state: ConditionalExecutionState = RunAlwaysExecutionState(), - start_time: datetime.datetime = None, - end_time: datetime.datetime = None, - min_spread: Decimal = Decimal("0"), debug_csv_path: str = '', - volatility_buffer_size: int = 200, - trading_intensity_buffer_size: int = 200, - trading_intensity_price_levels: Tuple[float] = tuple(np.geomspace(1, 2, 10) - 1), - should_wait_order_cancel_confirmation = True, is_debug: bool = False, ): self._sb_order_tracker = OrderTracker() + self._config_map = config_map self._market_info = market_info - self._order_amount = order_amount - self._order_optimization_enabled = order_optimization_enabled - self._order_refresh_time = order_refresh_time - self._max_order_age = max_order_age - self._order_refresh_tolerance_pct = order_refresh_tolerance_pct - self._filled_order_delay = filled_order_delay - self._order_levels = order_levels - self._level_distances = level_distances - self._order_override = order_override - self._inventory_target_base_pct = inventory_target_base_pct - self._add_transaction_costs_to_orders = add_transaction_costs_to_orders self._hb_app_notification = hb_app_notification - self._hanging_orders_enabled = hanging_orders_enabled + self._hanging_orders_enabled = False + self._hanging_orders_cancel_pct = Decimal("10") self._hanging_orders_tracker = HangingOrdersTracker(self, - hanging_orders_cancel_pct) + self._hanging_orders_cancel_pct / Decimal('100')) self._cancel_timestamp = 0 self._create_timestamp = 0 @@ -115,24 +95,23 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): self._last_own_trade_price = Decimal('nan') self.c_add_markets([market_info.market]) - self._ticks_to_be_ready = max(volatility_buffer_size, trading_intensity_buffer_size) - self._avg_vol = InstantVolatilityIndicator(sampling_length=volatility_buffer_size) - self._trading_intensity = TradingIntensityIndicator(trading_intensity_buffer_size) + self._volatility_buffer_size = 0 + self._trading_intensity_buffer_size = 0 + self._ticks_to_be_ready = -1 + self._avg_vol = None + self._trading_intensity = None self._last_sampling_timestamp = 0 self._alpha = None self._kappa = None - self._gamma = risk_factor - self._eta = order_amount_shape_factor - self._execution_timeframe = execution_timeframe - self._execution_state = execution_state - self._start_time = start_time - self._end_time = end_time - self._min_spread = min_spread + self._execution_mode = None + self._execution_timeframe = None + self._execution_state = None + self._start_time = None + self._end_time = None self._reservation_price = s_decimal_zero self._optimal_spread = s_decimal_zero self._optimal_ask = s_decimal_zero self._optimal_bid = s_decimal_zero - self._should_wait_order_cancel_confirmation = should_wait_order_cancel_confirmation self._debug_csv_path = debug_csv_path self._is_debug = is_debug try: @@ -141,16 +120,18 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): except FileNotFoundError: pass + self.update_from_config_map() + def all_markets_ready(self): return all([market.ready for market in self._sb_markets]) @property def min_spread(self): - return self._min_spread + return self._config_map.min_spread @min_spread.setter def min_spread(self, value): - self._min_spread = value + self._config_map.min_spread = value @property def avg_vol(self): @@ -174,91 +155,125 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): @property def order_refresh_tolerance_pct(self) -> Decimal: - return self._order_refresh_tolerance_pct + if self._config_map.order_refresh_tolerance_pct is not None: + return self._config_map.order_refresh_tolerance_pct + else: + return Decimal("0") @order_refresh_tolerance_pct.setter def order_refresh_tolerance_pct(self, value: Decimal): - self._order_refresh_tolerance_pct = value + if self._config_map.order_refresh_tolerance_pct is not None: + self._config_map.order_refresh_tolerance_pct = value + + @property + def order_refresh_tolerance(self) -> Decimal: + return self._config_map.order_refresh_tolerance_pct / Decimal('100') + + @order_refresh_tolerance.setter + def order_refresh_tolerance(self, value: Decimal): + self._config_map.order_refresh_tolerance_pct = value * Decimal('100') @property def order_amount(self) -> Decimal: - return self._order_amount + return self._config_map.order_amount @order_amount.setter def order_amount(self, value: Decimal): - self._order_amount = value + self._config_map.order_amount = value @property def inventory_target_base_pct(self) -> Decimal: - return self._inventory_target_base_pct + if self._config_map.inventory_target_base_pct is not None: + return self._config_map.inventory_target_base_pct + else: + return 0 @inventory_target_base_pct.setter def inventory_target_base_pct(self, value: Decimal): - self._inventory_target_base_pct = value + self._config_map.inventory_target_base_pct = value + + @property + def inventory_target_base(self) -> Decimal: + return self.inventory_target_base_pct / Decimal('100') + + @inventory_target_base.setter + def inventory_target_base(self, value: Decimal): + self.inventory_target_base_pct = value * Decimal('100') @property def order_optimization_enabled(self) -> bool: - return self._order_optimization_enabled + return self._config_map.order_optimization_enabled @order_optimization_enabled.setter def order_optimization_enabled(self, value: bool): - self._order_optimization_enabled = value + self._config_map.order_optimization_enabled = value @property def order_refresh_time(self) -> float: - return self._order_refresh_time + return self._config_map.order_refresh_time @order_refresh_time.setter def order_refresh_time(self, value: float): - self._order_refresh_time = value + self._config_map.order_refresh_time = value @property def filled_order_delay(self) -> float: - return self._filled_order_delay + return self._config_map.filled_order_delay @filled_order_delay.setter def filled_order_delay(self, value: float): - self._filled_order_delay = value + self._config_map.filled_order_delay = value @property def order_override(self) -> Dict[str, any]: - return self._order_override + return self._config_map.order_override @order_override.setter def order_override(self, value): - self._order_override = value + self._config_map.order_override = value @property def order_levels(self) -> int: - return self._order_levels + if self._config_map.order_levels_mode.title == MultiOrderLevelModel.Config.title: + return self._config_map.order_levels_mode.order_levels + else: + return 0 @order_levels.setter def order_levels(self, value): - self._order_levels = value + if value == 0: + self._config_map.order_levels_mode = SingleOrderLevelModel() + else: + self._config_map.order_levels_mode = MultiOrderLevelModel() + self._config_map.order_levels_mode.order_levels = value @property def level_distances(self) -> int: - return self._level_distances + if self._config_map.order_levels_mode.title == MultiOrderLevelModel.Config.title: + return self._config_map.order_levels_mode.level_distances + else: + return 0 @level_distances.setter def level_distances(self, value): - self._level_distances = value + if self._config_map.order_levels_mode.title == MultiOrderLevelModel.Config.title: + self._config_map.order_levels_mode.level_distances = value @property def max_order_age(self): - return self._max_order_age + return self._config_map.max_order_age @max_order_age.setter def max_order_age(self, value): - self._max_order_age = value + self._config_map.max_order_age = value @property def add_transaction_costs_to_orders(self) -> bool: - return self._add_transaction_costs_to_orders + return self._config_map.add_transaction_costs @add_transaction_costs_to_orders.setter def add_transaction_costs_to_orders(self, value: bool): - self._add_transaction_costs_to_orders = value + self._config_map.add_transaction_costs = value @property def base_asset(self): @@ -274,11 +289,11 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): @property def gamma(self): - return self._gamma + return self._config_map.risk_factor @gamma.setter def gamma(self, value): - self._gamma = value + self._config_map.risk_factor = value @property def alpha(self): @@ -298,11 +313,11 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): @property def eta(self): - return self._eta + return self._config_map.order_amount_shape_factor @eta.setter def eta(self, value): - self._eta = value + self._config_map.order_amount_shape_factor = value @property def reservation_price(self): @@ -409,6 +424,79 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): def hanging_orders_tracker(self): return self._hanging_orders_tracker + def update_from_config_map(self): + self.get_config_map_execution_mode() + self.get_config_map_hanging_orders() + self.get_config_map_indicators() + + def get_config_map_execution_mode(self): + execution_mode = self._config_map.execution_timeframe_mode.title + execution_timeframe = self._config_map.execution_timeframe_mode.Config.title + if execution_mode == FromDateToDateModel.Config.title: + start_time = self._config_map.execution_timeframe_mode.start_datetime + end_time = self._config_map.execution_timeframe_mode.end_datetime + execution_state = RunInTimeConditionalExecutionState(start_timestamp=start_time, end_timestamp=end_time) + elif execution_mode == DailyBetweenTimesModel.Config.title: + start_time = self._config_map.execution_timeframe_mode.start_time + end_time = self._config_map.execution_timeframe_mode.end_time + execution_state = RunInTimeConditionalExecutionState(start_timestamp=start_time, end_timestamp=end_time) + else: + start_time = None + end_time = None + execution_state = RunAlwaysExecutionState() + + # Something has changed? + if self._execution_state is None or self._execution_state != execution_state: + self._execution_state = execution_state + self._execution_mode = execution_mode + self._execution_timeframe = execution_timeframe + self._start_time = start_time + self._end_time = end_time + + def get_config_map_hanging_orders(self): + if self._config_map.hanging_orders_mode.title == TrackHangingOrdersModel.Config.title: + hanging_orders_enabled = True + hanging_orders_cancel_pct = self._config_map.hanging_orders_mode.hanging_orders_cancel_pct + else: + hanging_orders_enabled = False + hanging_orders_cancel_pct = Decimal("0") + + if self._hanging_orders_enabled != hanging_orders_enabled: + self._hanging_orders_enabled = hanging_orders_enabled + self._hanging_orders_cancel_pct = hanging_orders_cancel_pct + self._hanging_orders_tracker = HangingOrdersTracker(self, + hanging_orders_cancel_pct / Decimal('100')) + elif self._hanging_orders_cancel_pct != hanging_orders_cancel_pct: + self._hanging_orders_cancel_pct = hanging_orders_cancel_pct + self._hanging_orders_tracker = HangingOrdersTracker(self, + hanging_orders_cancel_pct / Decimal('100')) + + def get_config_map_indicators(self): + volatility_buffer_size = self._config_map.volatility_buffer_size + trading_intensity_buffer_size = self._config_map.trading_intensity_buffer_size + ticks_to_be_ready_after = max(volatility_buffer_size, trading_intensity_buffer_size) + ticks_to_be_ready_before = max(self._volatility_buffer_size, self._trading_intensity_buffer_size) + + if self._volatility_buffer_size == 0 or self._volatility_buffer_size != volatility_buffer_size: + self._volatility_buffer_size = volatility_buffer_size + + if self._avg_vol is None: + self._avg_vol = InstantVolatilityIndicator(sampling_length=volatility_buffer_size) + else: + self._avg_vol.sampling_length = volatility_buffer_size + + if self._trading_intensity_buffer_size == 0 or self._trading_intensity_buffer_size != trading_intensity_buffer_size: + self._trading_intensity_buffer_size = trading_intensity_buffer_size + + if self._trading_intensity is None: + self._trading_intensity = TradingIntensityIndicator(trading_intensity_buffer_size) + else: + self._trading_intensity.sampling_length = trading_intensity_buffer_size + + self._ticks_to_be_ready += (ticks_to_be_ready_after - ticks_to_be_ready_before) + if self._ticks_to_be_ready < 0: + self._ticks_to_be_ready = 0 + def pure_mm_assets_df(self, to_show_current_pct: bool) -> pd.DataFrame: market, trading_pair, base_asset, quote_asset = self._market_info price = self._market_info.get_mid_price() @@ -454,7 +542,7 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): spread = 0 if price == 0 else abs(order.price - price) / price age = pd.Timestamp(order_age(order, self._current_timestamp), unit='s').strftime('%H:%M:%S') - amount_orig = self._order_amount + amount_orig = self._config_map.order_amount if is_hanging_order: amount_orig = float(order.quantity) level = "hang" @@ -516,9 +604,9 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): lines.extend(["", " No active maker orders."]) volatility_pct = self._avg_vol.current_value / float(self.get_price()) * 100.0 - if all((self._gamma, self._alpha, self._kappa, not isnan(volatility_pct))): + if all((self.gamma, self._alpha, self._kappa, not isnan(volatility_pct))): lines.extend(["", f" Strategy parameters:", - f" risk_factor(\u03B3)= {self._gamma:.5E}", + f" risk_factor(\u03B3)= {self.gamma:.5E}", f" order_book_intensity_factor(\u0391)= {self._alpha:.5E}", f" order_book_depth_factor(\u03BA)= {self._kappa:.5E}", f" volatility= {volatility_pct:.3f}%"]) @@ -586,7 +674,11 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): self.logger().warning(f"WARNING: Some markets are not connected or are down at the moment. Market " f"making may be dangerous when markets or networks are unstable.") + # Updates settings from config map if changed + self.update_from_config_map() + self.c_collect_market_variables(timestamp) + if self.c_is_algorithm_ready(): if self._create_timestamp <= self._current_timestamp: # Measure order book liquidity @@ -690,7 +782,6 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): # The amount of stocks owned - q - has to be in relative units, not absolute, because changing the portfolio size shouldn't change the reservation price # The reservation price should concern itself only with the strategy performance, i.e. amount of stocks relative to the target inventory = Decimal(str(self.c_calculate_inventory())) - if inventory == 0: return @@ -702,7 +793,7 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): # order book liquidity - kappa and alpha have to represent absolute values because the second member of the optimal spread equation has to be an absolute price # and from the reservation price calculation we know that gamma's unit is not absolute price - if all((self._gamma, self._kappa)) and self._alpha != 0 and self._kappa > 0 and vol != 0: + if all((self.gamma, self._kappa)) and self._alpha != 0 and self._kappa > 0 and vol != 0: if self._execution_state.time_left is not None and self._execution_state.closing_time is not None: # Avellaneda-Stoikov for a fixed timespan time_left_fraction = Decimal(str(self._execution_state.time_left / self._execution_state.closing_time)) @@ -723,12 +814,12 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): # current mid price # This leads to normalization of the risk_factor and will guaranetee consistent behavior on all price ranges of the asset, and across assets - self._reservation_price = price - (q * self._gamma * vol * time_left_fraction) + self._reservation_price = price - (q * self.gamma * vol * time_left_fraction) - self._optimal_spread = self._gamma * vol * time_left_fraction - self._optimal_spread += 2 * Decimal(1 + self._gamma / self._kappa).ln() / self._gamma + self._optimal_spread = self.gamma * vol * time_left_fraction + self._optimal_spread += 2 * Decimal(1 + self.gamma / self._kappa).ln() / self.gamma - min_spread = price / 100 * Decimal(str(self._min_spread)) + min_spread = price / 100 * Decimal(str(self._config_map.min_spread)) max_limit_bid = price - min_spread / 2 min_limit_ask = price + min_spread / 2 @@ -769,7 +860,7 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): # Total inventory value in quote asset prices inventory_value = base_value + quote_asset_amount # Target base asset value in quote asset prices - target_inventory_value = inventory_value * self._inventory_target_base_pct + target_inventory_value = inventory_value * self.inventory_target_base # Target base asset amount target_inventory_amount = target_inventory_value / price return market.c_quantize_order_amount(trading_pair, Decimal(str(target_inventory_amount))) @@ -806,7 +897,7 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): return self._avg_vol.is_sampling_buffer_full and self._trading_intensity.is_sampling_buffer_full cdef bint c_is_algorithm_changed(self): - return self._trading_intensity.is_sampling_buffer_changed + return self._trading_intensity.is_sampling_buffer_changed or self._avg_vol.is_sampling_buffer_changed def is_algorithm_ready(self) -> bool: return self.c_is_algorithm_ready() @@ -815,10 +906,10 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): return self.c_is_algorithm_changed() def _get_level_spreads(self): - level_step = ((self._optimal_spread / 2) / 100) * self._level_distances + level_step = ((self._optimal_spread / 2) / 100) * self.level_distances - bid_level_spreads = [i * level_step for i in range(self._order_levels)] - ask_level_spreads = [i * level_step for i in range(self._order_levels)] + bid_level_spreads = [i * level_step for i in range(self.order_levels)] + ask_level_spreads = [i * level_step for i in range(self.order_levels)] return bid_level_spreads, ask_level_spreads @@ -828,18 +919,19 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): list buys = [] list sells = [] reference_price = self.get_price() - for key, value in self._order_override.items(): - if str(value[0]) in ["buy", "sell"]: - list_to_be_appended = buys if str(value[0]) == "buy" else sells - size = Decimal(str(value[2])) - size = market.c_quantize_order_amount(self.trading_pair, size) - if str(value[0]) == "buy": - price = reference_price * (Decimal("1") - Decimal(str(value[1])) / Decimal("100")) - elif str(value[0]) == "sell": - price = reference_price * (Decimal("1") + Decimal(str(value[1])) / Decimal("100")) - price = market.c_quantize_order_price(self.trading_pair, price) - if size > 0 and price > 0: - list_to_be_appended.append(PriceSize(price, size)) + if self.order_override is not None: + for key, value in self.order_override.items(): + if str(value[0]) in ["buy", "sell"]: + list_to_be_appended = buys if str(value[0]) == "buy" else sells + size = Decimal(str(value[2])) + size = market.c_quantize_order_amount(self.trading_pair, size) + if str(value[0]) == "buy": + price = reference_price * (Decimal("1") - Decimal(str(value[1])) / Decimal("100")) + elif str(value[0]) == "sell": + price = reference_price * (Decimal("1") + Decimal(str(value[1])) / Decimal("100")) + price = market.c_quantize_order_price(self.trading_pair, price) + if size > 0 and price > 0: + list_to_be_appended.append(PriceSize(price, size)) return buys, sells def create_proposal_based_on_order_override(self) -> Tuple[List[Proposal], List[Proposal]]: @@ -851,9 +943,9 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): list buys = [] list sells = [] bid_level_spreads, ask_level_spreads = self._get_level_spreads() - size = market.c_quantize_order_amount(self.trading_pair, self._order_amount) + size = market.c_quantize_order_amount(self.trading_pair, self._config_map.order_amount) if size > 0: - for level in range(self._order_levels): + for level in range(self.order_levels): bid_price = market.c_quantize_order_price(self.trading_pair, self._optimal_bid - Decimal(str(bid_level_spreads[level]))) ask_price = market.c_quantize_order_price(self.trading_pair, @@ -872,12 +964,12 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): list buys = [] list sells = [] price = market.c_quantize_order_price(self.trading_pair, Decimal(str(self._optimal_bid))) - size = market.c_quantize_order_amount(self.trading_pair, self._order_amount) + size = market.c_quantize_order_amount(self.trading_pair, self._config_map.order_amount) if size > 0: buys.append(PriceSize(price, size)) price = market.c_quantize_order_price(self.trading_pair, Decimal(str(self._optimal_ask))) - size = market.c_quantize_order_amount(self.trading_pair, self._order_amount) + size = market.c_quantize_order_amount(self.trading_pair, self._config_map.order_amount) if size > 0: sells.append(PriceSize(price, size)) return buys, sells @@ -891,10 +983,10 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): list buys = [] list sells = [] - if self._order_override is not None and len(self._order_override) > 0: + if self.order_override is not None and len(self.order_override) > 0: # If order_override is set, it will override order_levels buys, sells = self._create_proposal_based_on_order_override() - elif self._order_levels > 0: + elif self.order_levels > 0: # Simple order levels buys, sells = self._create_proposal_based_on_order_levels() else: @@ -928,10 +1020,10 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): return self.c_get_adjusted_available_balance(orders) cdef c_apply_order_price_modifiers(self, object proposal): - if self._order_optimization_enabled: + if self._config_map.order_optimization_enabled: self.c_apply_order_optimization(proposal) - if self._add_transaction_costs_to_orders: + if self._config_map.add_transaction_costs: self.c_apply_add_transaction_costs(proposal) def apply_order_price_modifiers(self, proposal: Proposal): @@ -1054,7 +1146,7 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): str trading_pair = self._market_info.trading_pair # Order amounts should be changed only if order_override is not active - if (self._order_override is None) or (len(self._order_override) == 0): + if (self.order_override is None) or (len(self.order_override) == 0): # eta parameter is described in the paper as the shape parameter for having exponentially decreasing order amount # for orders that go against inventory target (i.e. Want to buy when excess inventory or sell when deficit inventory) @@ -1072,13 +1164,13 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): if q > 0: for i, proposed in enumerate(proposal.buys): - proposal.buys[i].size = market.c_quantize_order_amount(trading_pair, proposal.buys[i].size * Decimal.exp(-self._eta * q)) + proposal.buys[i].size = market.c_quantize_order_amount(trading_pair, proposal.buys[i].size * Decimal.exp(-self.eta * q)) proposal.buys = [o for o in proposal.buys if o.size > 0] if len(proposal.sells) > 0: if q < 0: for i, proposed in enumerate(proposal.sells): - proposal.sells[i].size = market.c_quantize_order_amount(trading_pair, proposal.sells[i].size * Decimal.exp(self._eta * q)) + proposal.sells[i].size = market.c_quantize_order_amount(trading_pair, proposal.sells[i].size * Decimal.exp(self.eta * q)) proposal.sells = [o for o in proposal.sells if o.size > 0] def apply_order_amount_eta_transformation(self, proposal: Proposal): @@ -1144,7 +1236,7 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): if (not self._hanging_orders_tracker.is_order_id_in_hanging_orders(order_id) and not self.hanging_orders_tracker.is_order_id_in_completed_hanging_orders(order_id)): # delay order creation by filled_order_delay (in seconds) - self._create_timestamp = self._current_timestamp + self._filled_order_delay + self._create_timestamp = self._current_timestamp + self.filled_order_delay self._cancel_timestamp = min(self._cancel_timestamp, self._create_timestamp) if limit_order_record.is_buy: @@ -1175,7 +1267,7 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): proposal_prices = sorted(proposal_prices) for current, proposal in zip(current_prices, proposal_prices): # if spread diff is more than the tolerance or order quantities are different, return false. - if abs(proposal - current) / current > self._order_refresh_tolerance_pct: + if abs(proposal - current) / current > self.order_refresh_tolerance: return False return True @@ -1189,7 +1281,7 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): cdef: list active_orders = self.active_non_hanging_orders for order in active_orders: - if order_age(order, self._current_timestamp) > self._max_order_age: + if order_age(order, self._current_timestamp) > self._config_map.max_order_age: self.c_cancel_order(self._market_info, order.client_order_id) cdef c_cancel_active_orders(self, object proposal): @@ -1200,6 +1292,7 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): list active_buy_prices = [] list active_sells = [] bint to_defer_canceling = False + if len(self.active_non_hanging_orders) == 0: return if proposal is not None: @@ -1229,7 +1322,7 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): self._hanging_orders_tracker.is_potential_hanging_order(o)] return (self._create_timestamp < self._current_timestamp - and (not self._should_wait_order_cancel_confirmation or + and (not self._config_map.should_wait_order_cancel_confirmation or len(self._sb_order_tracker.in_flight_cancels) == 0) and proposal is not None and len(non_hanging_orders_non_cancelled) == 0) @@ -1297,7 +1390,7 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): self.c_execute_orders_proposal(proposal) cdef c_set_timers(self): - cdef double next_cycle = self._current_timestamp + self._order_refresh_time + cdef double next_cycle = self._current_timestamp + self.order_refresh_time if self._create_timestamp <= self._current_timestamp: self._create_timestamp = next_cycle if self._cancel_timestamp <= self._current_timestamp: @@ -1365,10 +1458,10 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): self.c_calculate_target_inventory(), time_left_fraction, self._avg_vol.current_value, - self._gamma, + self.gamma, self._alpha, self._kappa, - self._eta, + self.eta, vol, mid_price_variance, self.inventory_target_base_pct)]) diff --git a/hummingbot/strategy/avellaneda_market_making/start.py b/hummingbot/strategy/avellaneda_market_making/start.py index a71c11045c..6b228b986b 100644 --- a/hummingbot/strategy/avellaneda_market_making/start.py +++ b/hummingbot/strategy/avellaneda_market_making/start.py @@ -1,5 +1,4 @@ import os.path -from decimal import Decimal from typing import List, Tuple import pandas as pd @@ -7,43 +6,14 @@ from hummingbot import data_path from hummingbot.client.hummingbot_application import HummingbotApplication from hummingbot.strategy.avellaneda_market_making import AvellanedaMarketMakingStrategy -from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( - DailyBetweenTimesModel, - FromDateToDateModel, - MultiOrderLevelModel, - TrackHangingOrdersModel, -) -from hummingbot.strategy.conditional_execution_state import RunAlwaysExecutionState, RunInTimeConditionalExecutionState from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple def start(self): try: c_map = self.strategy_config_map - order_amount = c_map.order_amount - order_optimization_enabled = c_map.order_optimization_enabled - order_refresh_time = c_map.order_refresh_time exchange = c_map.exchange raw_trading_pair = c_map.market - max_order_age = c_map.max_order_age - inventory_target_base_pct = 0 if c_map.inventory_target_base_pct is None else \ - c_map.inventory_target_base_pct / Decimal('100') - filled_order_delay = c_map.filled_order_delay - order_refresh_tolerance_pct = c_map.order_refresh_tolerance_pct / Decimal('100') - if c_map.order_levels_mode.title == MultiOrderLevelModel.Config.title: - order_levels = c_map.order_levels_mode.order_levels - level_distances = c_map.order_levels_mode.level_distances - else: - order_levels = 1 - level_distances = 0 - order_override = c_map.order_override - if c_map.hanging_orders_mode.title == TrackHangingOrdersModel.Config.title: - hanging_orders_enabled = True - hanging_orders_cancel_pct = c_map.hanging_orders_mode.hanging_orders_cancel_pct / Decimal('100') - else: - hanging_orders_enabled = False - hanging_orders_cancel_pct = Decimal("0") - add_transaction_costs_to_orders = c_map.add_transaction_costs trading_pair: str = raw_trading_pair maker_assets: Tuple[str, str] = self._initialize_market_assets(exchange, [trading_pair])[0] @@ -53,60 +23,18 @@ def start(self): self.market_trading_pair_tuples = [MarketTradingPairTuple(*maker_data)] strategy_logging_options = AvellanedaMarketMakingStrategy.OPTION_LOG_ALL - risk_factor = c_map.risk_factor - order_amount_shape_factor = c_map.order_amount_shape_factor - execution_timeframe = c_map.execution_timeframe_mode.Config.title - if c_map.execution_timeframe_mode.title == FromDateToDateModel.Config.title: - start_time = c_map.execution_timeframe_mode.start_datetime - end_time = c_map.execution_timeframe_mode.end_datetime - execution_state = RunInTimeConditionalExecutionState(start_timestamp=start_time, end_timestamp=end_time) - elif c_map.execution_timeframe_mode.title == DailyBetweenTimesModel.Config.title: - start_time = c_map.execution_timeframe_mode.start_time - end_time = c_map.execution_timeframe_mode.end_time - execution_state = RunInTimeConditionalExecutionState(start_timestamp=start_time, end_timestamp=end_time) - else: - start_time = None - end_time = None - execution_state = RunAlwaysExecutionState() - - min_spread = c_map.min_spread - volatility_buffer_size = c_map.volatility_buffer_size - trading_intensity_buffer_size = c_map.trading_intensity_buffer_size - should_wait_order_cancel_confirmation = c_map.should_wait_order_cancel_confirmation debug_csv_path = os.path.join(data_path(), HummingbotApplication.main_application().strategy_file_name.rsplit('.', 1)[0] + f"_{pd.Timestamp.now().strftime('%Y-%m-%d_%H-%M-%S')}.csv") self.strategy = AvellanedaMarketMakingStrategy() self.strategy.init_params( + config_map=c_map, market_info=MarketTradingPairTuple(*maker_data), - order_amount=order_amount, - order_optimization_enabled=order_optimization_enabled, - inventory_target_base_pct=inventory_target_base_pct, - order_refresh_time=order_refresh_time, - max_order_age=max_order_age, - order_refresh_tolerance_pct=order_refresh_tolerance_pct, - filled_order_delay=filled_order_delay, - order_levels=order_levels, - level_distances=level_distances, - order_override=order_override, - hanging_orders_enabled=hanging_orders_enabled, - hanging_orders_cancel_pct=hanging_orders_cancel_pct, - add_transaction_costs_to_orders=add_transaction_costs_to_orders, logging_options=strategy_logging_options, hb_app_notification=True, - risk_factor=risk_factor, - order_amount_shape_factor=order_amount_shape_factor, - execution_timeframe=execution_timeframe, - execution_state=execution_state, - start_time=start_time, - end_time=end_time, - min_spread=min_spread, debug_csv_path=debug_csv_path, - volatility_buffer_size=volatility_buffer_size, - trading_intensity_buffer_size=trading_intensity_buffer_size, - should_wait_order_cancel_confirmation=should_wait_order_cancel_confirmation, is_debug=False ) except Exception as e: From 5ea01c7a2be8764bcb769bf33ea435d352cb2609 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Tue, 10 May 2022 14:18:12 +0200 Subject: [PATCH 062/152] (refactor) tests --- .../test_avellaneda_market_making.py | 137 +++++++++++++----- .../test_avellaneda_market_making_start.py | 2 +- 2 files changed, 103 insertions(+), 36 deletions(-) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py index d516213571..a0636dd10d 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py @@ -3,11 +3,16 @@ import unittest from copy import deepcopy from decimal import Decimal -from typing import List, Tuple +from typing import ( + Dict, + List, + Tuple +) import numpy as np import pandas as pd +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode @@ -25,9 +30,15 @@ from hummingbot.strategy.__utils__.trailing_indicators.instant_volatility import InstantVolatilityIndicator from hummingbot.strategy.__utils__.trailing_indicators.trading_intensity import TradingIntensityIndicator from hummingbot.strategy.avellaneda_market_making import AvellanedaMarketMakingStrategy +from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( + AvellanedaMarketMakingConfigMap, + MultiOrderLevelModel, + TrackHangingOrdersModel +) from hummingbot.strategy.data_types import PriceSize, Proposal from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple + s_decimal_zero = Decimal(0) s_decimal_one = Decimal(1) s_decimal_nan = Decimal("NaN") @@ -64,7 +75,7 @@ def setUpClass(cls): # Strategy Initial Configuration Parameters cls.order_amount: Decimal = Decimal("10") - cls.inventory_target_base_pct: Decimal = Decimal("0.5") # Indicates 50% + cls.inventory_target_base_pct: Decimal = Decimal("50") # 50% cls.min_spread: Decimal = Decimal("0.0") # Default strategy value cls.risk_factor_finite: Decimal = Decimal("0.8") cls.risk_factor_infinite: Decimal = Decimal("1") @@ -98,13 +109,13 @@ def setUp(self): ) ) + config_settings = self.get_default_map() + self.config_map = ClientConfigAdapter(AvellanedaMarketMakingConfigMap(**config_settings)) + self.strategy: AvellanedaMarketMakingStrategy = AvellanedaMarketMakingStrategy() self.strategy.init_params( + config_map=self.config_map, market_info=self.market_info, - order_amount=self.order_amount, - min_spread=self.min_spread, - inventory_target_base_pct=self.inventory_target_base_pct, - risk_factor=self.risk_factor_finite ) self.avg_vol_indicator: InstantVolatilityIndicator = InstantVolatilityIndicator(sampling_length=100, @@ -126,6 +137,21 @@ def tearDown(self) -> None: self.strategy.stop(self.clock) super().tearDown() + def get_default_map(self) -> Dict[str, str]: + config_settings = { + "exchange": self.market.name, + "market": self.trading_pair, + "execution_timeframe_mode": "infinite", + "order_amount": self.order_amount, + "order_optimization_enabled": "yes", + "min_spread": self.min_spread, + "risk_factor": self.risk_factor_finite, + "order_refresh_time": "30", + "inventory_target_base_pct": self.inventory_target_base_pct, + "add_transaction_costs": "yes", + } + return config_settings + def simulate_low_volatility(self, strategy: AvellanedaMarketMakingStrategy): if self.volatility_indicator_low_vol is None: N_SAMPLES = 350 @@ -369,7 +395,7 @@ def test_market_info(self): def test_order_refresh_tolerance_pct(self): # Default value for order_refresh_tolerance_pct - self.assertEqual(Decimal(-1), self.strategy.order_refresh_tolerance_pct) + self.assertEqual(Decimal(0), self.strategy.order_refresh_tolerance_pct) # Test setter method self.strategy.order_refresh_tolerance_pct = Decimal("1") @@ -593,7 +619,7 @@ def test_calculate_target_inventory(self): quote_asset_amount = self.market.get_balance(self.trading_pair.split("-")[1]) base_value = base_asset_amount * current_price inventory_value = base_value + quote_asset_amount - target_inventory_value = Decimal((inventory_value * self.inventory_target_base_pct) / current_price) + target_inventory_value = Decimal((inventory_value * self.inventory_target_base_pct / Decimal('100')) / current_price) expected_quantize_order_amount = self.market.quantize_order_amount(self.trading_pair, target_inventory_value) @@ -689,16 +715,29 @@ def test_create_proposal_based_on_order_override(self): self.assertEqual(str(expected_proposal), str(self.strategy.create_proposal_based_on_order_override())) def test_get_level_spreads(self): + order_levels_mode = MultiOrderLevelModel() + order_levels_mode.order_levels = 4 + order_levels_mode.level_distances = 1 + + config_settings = { + "exchange": self.market.name, + "market": self.trading_pair, + "execution_timeframe_mode": "infinite", + "order_amount": self.order_amount, + "order_optimization_enabled": "yes", + "order_levels_mode": order_levels_mode, + "min_spread": self.min_spread, + "risk_factor": self.risk_factor_infinite, + "order_refresh_time": "60", + "inventory_target_base_pct": self.inventory_target_base_pct, + } + config_map = ClientConfigAdapter(AvellanedaMarketMakingConfigMap(**config_settings)) + # Re-initialize strategy with order_level configurations self.strategy = AvellanedaMarketMakingStrategy() self.strategy.init_params( + config_map=config_map, market_info=self.market_info, - order_amount=self.order_amount, - order_levels=4, - level_distances=1, - risk_factor=self.risk_factor_infinite, - execution_timeframe="infinite", - inventory_target_base_pct=self.inventory_target_base_pct, ) self.strategy.start(self.clock, self.start_timestamp) @@ -976,7 +1015,8 @@ def test_apply_order_price_modifiers(self): initial_proposal: Proposal = Proposal([PriceSize(bid_price, order_amount)], [PriceSize(ask_price, order_amount)]) # <<<<< Test Preparation End - # (1) Default Both Enabled: order_optimization = True, add_transaction_costs_to_orders = True + # (1) Default: order_optimization = True, add_transaction_costs_to_orders = False + #self.strategy.add_transaction_costs_to_orders = True # Intentionally make top_bid/ask_price lower/higher respectively & set TradeFees ob_bids: List[OrderBookRow] = [OrderBookRow(bid_price * Decimal("0.5"), self.order_amount, 2)] @@ -1147,6 +1187,9 @@ def test_is_within_tolerance(self): buy_prices: List[Decimal] = [bid_price] sell_prices: List[Decimal] = [ask_price] + bid_price: Decimal = Decimal("98.5") + ask_price: Decimal = Decimal("100.5") + proposal: Proposal = Proposal( [PriceSize(bid_price, self.order_amount)], # Bids [PriceSize(ask_price, self.order_amount)] # Sells @@ -1154,11 +1197,11 @@ def test_is_within_tolerance(self): proposal_buys = [buy.price for buy in proposal.buys] proposal_sells = [sell.price for sell in proposal.sells] - # Default order_refresh_tolerance_pct is -1. So it will always NOT be within tolerance + # Default order_refresh_tolerance_pct is 0. So it will always NOT be within tolerance self.assertFalse(self.strategy.is_within_tolerance(buy_prices, proposal_buys)) self.assertFalse(self.strategy.is_within_tolerance(sell_prices, proposal_sells)) - self.strategy.order_refresh_tolerance_pct = Decimal("1.0") + self.strategy.order_refresh_tolerance_pct = Decimal("10.0") self.assertTrue(self.strategy.is_within_tolerance(buy_prices, proposal_buys)) self.assertTrue(self.strategy.is_within_tolerance(sell_prices, proposal_sells)) @@ -1193,14 +1236,20 @@ def test_cancel_active_orders(self): # Case (2): Has active orders and within _order_refresh_tolerance_pct. # Note: Order will NOT be cancelled - self.strategy.order_refresh_tolerance_pct = Decimal("100") + self.strategy.order_refresh_tolerance_pct = Decimal("10") self.simulate_place_limit_order(self.strategy, self.market_info, limit_buy_order) self.simulate_place_limit_order(self.strategy, self.market_info, limit_sell_order) self.assertEqual(2, len(self.strategy.active_orders)) # Case (3a): Has active orders and EXCEED _order_refresh_tolerance_pct BUT cancel_timestamp > current_timestamp # Note: Orders will NOT be cancelled - self.strategy.order_refresh_tolerance_pct = Decimal("-1") + self.strategy.order_refresh_tolerance_pct = Decimal("0") + bid_price: Decimal = Decimal("98.5") + ask_price: Decimal = Decimal("100.5") + proposal: Proposal = Proposal( + [PriceSize(bid_price, self.order_amount)], # Bids + [PriceSize(ask_price, self.order_amount)] # Sells + ) self.assertEqual(2, len(self.strategy.active_orders)) self.strategy.cancel_active_orders(proposal) @@ -1217,7 +1266,7 @@ def test_cancel_active_orders(self): # Case (4): Has active orders and within _order_refresh_tolerance_pct BUT cancel_timestamp > current_timestamp # Note: Order not cancelled - self.strategy.order_refresh_tolerance_pct = Decimal("100") + self.strategy.order_refresh_tolerance_pct = Decimal("10") self.simulate_place_limit_order(self.strategy, self.market_info, limit_buy_order) self.simulate_place_limit_order(self.strategy, self.market_info, limit_sell_order) self.assertEqual(2, len(self.strategy.active_orders)) @@ -1272,6 +1321,19 @@ def test_to_create_orders(self): self.assertTrue(self.strategy.to_create_orders(proposal)) def test_existing_hanging_orders_are_included_in_budget_constraint(self): + config_settings = { + "exchange": self.market.name, + "market": self.trading_pair, + "execution_timeframe_mode": "infinite", + "order_amount": self.order_amount, + "min_spread": self.min_spread, + "risk_factor": self.risk_factor_finite, + "hanging_orders_mode": {"hanging_orders_cancel_pct": 99}, + "order_refresh_time": "60", + "inventory_target_base_pct": self.inventory_target_base_pct, + "filled_order_delay": 30, + } + config_map = ClientConfigAdapter(AvellanedaMarketMakingConfigMap(**config_settings)) self.market.set_balance("COINALPHA", 100) self.market.set_balance("HBOT", 50000) @@ -1279,14 +1341,8 @@ def test_existing_hanging_orders_are_included_in_budget_constraint(self): # Create a new strategy, with hanging orders enabled self.strategy = AvellanedaMarketMakingStrategy() self.strategy.init_params( + config_map=config_map, market_info=self.market_info, - order_amount=self.order_amount, - min_spread=self.min_spread, - inventory_target_base_pct=self.inventory_target_base_pct, - risk_factor=self.risk_factor_finite, - hanging_orders_enabled=True, - hanging_orders_cancel_pct=Decimal(1), - filled_order_delay=30 ) # Create a new clock to start the strategy from scratch @@ -1335,26 +1391,36 @@ def test_existing_hanging_orders_are_included_in_budget_constraint(self): self.assertEqual(expected_quote_balance, current_quote_balance) def test_not_filled_order_changed_to_hanging_order_after_refresh_time(self): + hanging_orders_model = TrackHangingOrdersModel() + hanging_orders_model.hanging_orders_cancel_pct = "99" # Refresh has to happend after filled_order_delay refresh_time = 80 filled_extension_time = 60 + config_settings = { + "exchange": self.market.name, + "market": self.trading_pair, + "execution_timeframe_mode": "infinite", + "order_amount": self.order_amount, + "order_optimization_enabled": "yes", + "min_spread": self.min_spread, + "risk_factor": self.risk_factor_finite, + "order_refresh_time": refresh_time, + "inventory_target_base_pct": self.inventory_target_base_pct, + "hanging_orders_mode": hanging_orders_model, + "filled_order_delay": filled_extension_time + } + config_map = ClientConfigAdapter(AvellanedaMarketMakingConfigMap(**config_settings)) + self.market.set_balance("COINALPHA", 100) self.market.set_balance("HBOT", 50000) # Create a new strategy, with hanging orders enabled self.strategy = AvellanedaMarketMakingStrategy() self.strategy.init_params( + config_map=config_map, market_info=self.market_info, - order_amount=self.order_amount, - min_spread=self.min_spread, - inventory_target_base_pct=self.inventory_target_base_pct, - risk_factor=self.risk_factor_finite, - hanging_orders_enabled=True, - hanging_orders_cancel_pct=Decimal(1), - order_refresh_time=refresh_time, - filled_order_delay=filled_extension_time ) # Create a new clock to start the strategy from scratch @@ -1415,6 +1481,7 @@ def test_no_new_orders_created_until_previous_orders_cancellation_confirmed(self refresh_time = self.strategy.order_refresh_time self.strategy.avg_vol = self.avg_vol_indicator + self.strategy.order_refresh_tolerance_pct = -1 # Simulate low volatility self.simulate_low_volatility(self.strategy) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py index 45956d5f4a..dce3f1dd33 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py @@ -94,4 +94,4 @@ def test_strategy_creation_when_something_fails(self): self.assertEqual(len(self.notifications), 1) self.assertEqual(self.notifications[0], "Exception for testing") self.assertEqual(len(self.log_records), 1) - self.assertEqual(self.log_records[0].message, "Unknown error during initialization.") + self.assertEqual(self.log_records[0].getMessage(), "Unknown error during initialization.") From 5f6face44551ac4dfce94912597c46c77d13fa95 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Tue, 10 May 2022 17:19:56 +0300 Subject: [PATCH 063/152] (fix) Fixes further issues with quick_start.py --- bin/hummingbot_quickstart.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/hummingbot_quickstart.py b/bin/hummingbot_quickstart.py index a811a34c4a..8604528e36 100755 --- a/bin/hummingbot_quickstart.py +++ b/bin/hummingbot_quickstart.py @@ -16,6 +16,7 @@ from bin.hummingbot import UIStartListener, detect_available_port from hummingbot import init_logging from hummingbot.client.config.config_crypt import BaseSecretsManager, ETHKeyFileSecretManger +from hummingbot.client.config.config_data_types import BaseStrategyConfigMap from hummingbot.client.config.config_helpers import ( all_configs_complete, create_yml_files_legacy, @@ -97,7 +98,11 @@ async def quick_start(args: argparse.Namespace, secrets_manager: BaseSecretsMana strategy_config = await load_strategy_config_map_from_file( STRATEGIES_CONF_DIR_PATH / config_file_name ) - hb.strategy_name = strategy_config.strategy + hb.strategy_name = ( + strategy_config.strategy + if isinstance(strategy_config, BaseStrategyConfigMap) + else strategy_config.get("strategy").value + ) hb.strategy_config_map = strategy_config # To ensure quickstart runs with the default value of False for kill_switch_enabled if not present From 85d433c930d56816d8197a25808f1c0200b840e0 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Wed, 11 May 2022 13:23:29 +0200 Subject: [PATCH 064/152] (fix) execution timeframe mode parameter exception handling --- .../avellaneda_market_making.pyx | 47 ++++++++++--------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx index 26988aaad9..eb230dfca8 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx @@ -430,28 +430,31 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): self.get_config_map_indicators() def get_config_map_execution_mode(self): - execution_mode = self._config_map.execution_timeframe_mode.title - execution_timeframe = self._config_map.execution_timeframe_mode.Config.title - if execution_mode == FromDateToDateModel.Config.title: - start_time = self._config_map.execution_timeframe_mode.start_datetime - end_time = self._config_map.execution_timeframe_mode.end_datetime - execution_state = RunInTimeConditionalExecutionState(start_timestamp=start_time, end_timestamp=end_time) - elif execution_mode == DailyBetweenTimesModel.Config.title: - start_time = self._config_map.execution_timeframe_mode.start_time - end_time = self._config_map.execution_timeframe_mode.end_time - execution_state = RunInTimeConditionalExecutionState(start_timestamp=start_time, end_timestamp=end_time) - else: - start_time = None - end_time = None - execution_state = RunAlwaysExecutionState() - - # Something has changed? - if self._execution_state is None or self._execution_state != execution_state: - self._execution_state = execution_state - self._execution_mode = execution_mode - self._execution_timeframe = execution_timeframe - self._start_time = start_time - self._end_time = end_time + try: + execution_mode = self._config_map.execution_timeframe_mode.title + execution_timeframe = self._config_map.execution_timeframe_mode.Config.title + if execution_mode == FromDateToDateModel.Config.title: + start_time = self._config_map.execution_timeframe_mode.start_datetime + end_time = self._config_map.execution_timeframe_mode.end_datetime + execution_state = RunInTimeConditionalExecutionState(start_timestamp=start_time, end_timestamp=end_time) + elif execution_mode == DailyBetweenTimesModel.Config.title: + start_time = self._config_map.execution_timeframe_mode.start_time + end_time = self._config_map.execution_timeframe_mode.end_time + execution_state = RunInTimeConditionalExecutionState(start_timestamp=start_time, end_timestamp=end_time) + else: + start_time = None + end_time = None + execution_state = RunAlwaysExecutionState() + + # Something has changed? + if self._execution_state is None or self._execution_state != execution_state: + self._execution_state = execution_state + self._execution_mode = execution_mode + self._execution_timeframe = execution_timeframe + self._start_time = start_time + self._end_time = end_time + except AttributeError: + self.logger().info("A parameter is missing in the execution timeframe mode configuration.") def get_config_map_hanging_orders(self): if self._config_map.hanging_orders_mode.title == TrackHangingOrdersModel.Config.title: From 5d3a064a5ed9d4008c2cced056baa935fdf94911 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Wed, 11 May 2022 13:46:00 +0200 Subject: [PATCH 065/152] (fix) handing orders config re-instantiation --- .../avellaneda_market_making/avellaneda_market_making.pyx | 6 ++++-- hummingbot/strategy/hanging_orders_tracker.py | 8 ++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx index eb230dfca8..aba65939ec 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx @@ -465,14 +465,16 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): hanging_orders_cancel_pct = Decimal("0") if self._hanging_orders_enabled != hanging_orders_enabled: + # Hanging order tracker instance doesn't exist - create from scratch self._hanging_orders_enabled = hanging_orders_enabled self._hanging_orders_cancel_pct = hanging_orders_cancel_pct self._hanging_orders_tracker = HangingOrdersTracker(self, hanging_orders_cancel_pct / Decimal('100')) + self._hanging_orders_tracker.register_events(self.active_markets) elif self._hanging_orders_cancel_pct != hanging_orders_cancel_pct: + # Hanging order tracker instance existst - only update variable self._hanging_orders_cancel_pct = hanging_orders_cancel_pct - self._hanging_orders_tracker = HangingOrdersTracker(self, - hanging_orders_cancel_pct / Decimal('100')) + self._hanging_orders_tracker.hanging_orders_cancel_pct = hanging_orders_cancel_pct / Decimal('100') def get_config_map_indicators(self): volatility_buffer_size = self._config_map.volatility_buffer_size diff --git a/hummingbot/strategy/hanging_orders_tracker.py b/hummingbot/strategy/hanging_orders_tracker.py index 2006b201d7..4e9486fee5 100644 --- a/hummingbot/strategy/hanging_orders_tracker.py +++ b/hummingbot/strategy/hanging_orders_tracker.py @@ -75,6 +75,14 @@ def __init__(self, (MarketEvent.BuyOrderCompleted, self._complete_buy_order_forwarder), (MarketEvent.SellOrderCompleted, self._complete_sell_order_forwarder)] + @property + def hanging_orders_cancel_pct(self): + return self._hanging_orders_cancel_pct + + @hanging_orders_cancel_pct.setter + def hanging_orders_cancel_pct(self, value): + self._hanging_orders_cancel_pct = value + def register_events(self, markets: List[ConnectorBase]): """Start listening to events from the given markets.""" for market in markets: From 761deb20ee6287ec31d9beb90033414d8b8a86b2 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 11 May 2022 15:13:57 +0300 Subject: [PATCH 066/152] (fix) Fixing missing sub-folder on container create --- installation/docker-commands/create.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/installation/docker-commands/create.sh b/installation/docker-commands/create.sh index 79e79582db..79207ce538 100755 --- a/installation/docker-commands/create.sh +++ b/installation/docker-commands/create.sh @@ -77,6 +77,8 @@ create_instance () { mkdir $FOLDER # 2) Create subfolders for hummingbot files mkdir $CONF_FOLDER + mkdir $CONF_FOLDER/connectors + mkdir $CONF_FOLDER/strategies mkdir $LOGS_FOLDER mkdir $DATA_FOLDER mkdir $PMM_SCRIPTS_FOLDER From 6cc4808ff5a53e35ae2f5e5cc1e9cea53705a4d0 Mon Sep 17 00:00:00 2001 From: mhrvth <7762229+mhrvth@users.noreply.github.com> Date: Thu, 12 May 2022 21:41:01 +0200 Subject: [PATCH 067/152] Update hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx Co-authored-by: Petio Petrov --- .../avellaneda_market_making/avellaneda_market_making.pyx | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx index aba65939ec..e8b3440fe8 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx @@ -155,10 +155,7 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): @property def order_refresh_tolerance_pct(self) -> Decimal: - if self._config_map.order_refresh_tolerance_pct is not None: - return self._config_map.order_refresh_tolerance_pct - else: - return Decimal("0") + return self._config_map.order_refresh_tolerance_pct @order_refresh_tolerance_pct.setter def order_refresh_tolerance_pct(self, value: Decimal): From e3d90ddd04dc9a61d0d3ff233df9d6a95f3e895b Mon Sep 17 00:00:00 2001 From: mhrvth <7762229+mhrvth@users.noreply.github.com> Date: Thu, 12 May 2022 21:41:33 +0200 Subject: [PATCH 068/152] Update hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx Co-authored-by: Petio Petrov --- .../avellaneda_market_making/avellaneda_market_making.pyx | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx index e8b3440fe8..755b68dfd4 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx @@ -180,10 +180,7 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): @property def inventory_target_base_pct(self) -> Decimal: - if self._config_map.inventory_target_base_pct is not None: - return self._config_map.inventory_target_base_pct - else: - return 0 + return self._config_map.inventory_target_base_pct @inventory_target_base_pct.setter def inventory_target_base_pct(self, value: Decimal): From b2b78eba50a8fe160f0543bec435c43a683c8a5c Mon Sep 17 00:00:00 2001 From: mhrvth <7762229+mhrvth@users.noreply.github.com> Date: Thu, 12 May 2022 21:45:36 +0200 Subject: [PATCH 069/152] Update hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx Co-authored-by: Petio Petrov --- .../avellaneda_market_making/avellaneda_market_making.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx index 755b68dfd4..f8e9ecbd92 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx @@ -942,7 +942,7 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): list buys = [] list sells = [] bid_level_spreads, ask_level_spreads = self._get_level_spreads() - size = market.c_quantize_order_amount(self.trading_pair, self._config_map.order_amount) + size = market.c_quantize_order_amount(self.trading_pair, self._config_map.order_amount) if size > 0: for level in range(self.order_levels): bid_price = market.c_quantize_order_price(self.trading_pair, From 09142fa9716c783136fa8542dc8c9dbee630ae33 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Thu, 12 May 2022 21:47:44 +0200 Subject: [PATCH 070/152] (fix) ring buffer - sampling length, processing length variables removed --- .../trailing_indicators/base_trailing_indicator.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/hummingbot/strategy/__utils__/trailing_indicators/base_trailing_indicator.py b/hummingbot/strategy/__utils__/trailing_indicators/base_trailing_indicator.py index ddd1694e6d..2afb47dd5d 100644 --- a/hummingbot/strategy/__utils__/trailing_indicators/base_trailing_indicator.py +++ b/hummingbot/strategy/__utils__/trailing_indicators/base_trailing_indicator.py @@ -15,9 +15,7 @@ def logger(cls): return pmm_logger def __init__(self, sampling_length: int = 30, processing_length: int = 15): - self._sampling_length = sampling_length self._sampling_buffer = RingBuffer(sampling_length) - self._processing_length = processing_length self._processing_buffer = RingBuffer(processing_length) self._samples_length = 0 @@ -58,18 +56,16 @@ def is_sampling_buffer_changed(self) -> bool: @property def sampling_length(self) -> int: - return self._sampling_length + return self._sampling_buffer.length @sampling_length.setter def sampling_length(self, value): - self._sampling_length = value self._sampling_buffer.length = value @property def processing_length(self) -> int: - return self._processing_length + return self._processing_buffer.length @processing_length.setter def processing_length(self, value): - self._processing_length = value self._processing_buffer.length = value From 6671c3d8b02c66f8acb0175bf146bec2bdb8ea48 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Thu, 12 May 2022 21:47:57 +0200 Subject: [PATCH 071/152] (refactor) warning silent --- .../avellaneda_market_making/avellaneda_market_making.pyx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx index f8e9ecbd92..21dab3a1af 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx @@ -448,7 +448,8 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): self._start_time = start_time self._end_time = end_time except AttributeError: - self.logger().info("A parameter is missing in the execution timeframe mode configuration.") + # A parameter is missing in the execution timeframe mode configuration + pass def get_config_map_hanging_orders(self): if self._config_map.hanging_orders_mode.title == TrackHangingOrdersModel.Config.title: From c3d3d94cd361920c914fde2fccb7b1af58dad759 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Thu, 12 May 2022 23:20:26 +0200 Subject: [PATCH 072/152] (fix) conditional execution state __eq__ added --- hummingbot/strategy/conditional_execution_state.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/hummingbot/strategy/conditional_execution_state.py b/hummingbot/strategy/conditional_execution_state.py index 870dd1e781..12bd471be8 100644 --- a/hummingbot/strategy/conditional_execution_state.py +++ b/hummingbot/strategy/conditional_execution_state.py @@ -16,6 +16,9 @@ class ConditionalExecutionState(ABC): _closing_time: int = None _time_left: int = None + def __eq__(self, other): + return type(self) == type(other) + @property def time_left(self): return self._time_left @@ -75,6 +78,11 @@ def __str__(self): if self._end_timestamp is not None: return f"run daily between {self._start_timestamp} and {self._end_timestamp}" + def __eq__(self, other): + return type(self) == type(other) and \ + self._start_timestamp == other._start_timestamp and \ + self._end_timestamp == other._end_timestamp + def process_tick(self, timestamp: float, strategy: StrategyBase): if isinstance(self._start_timestamp, datetime): # From datetime From 8d35f3944c446c19ccc6eb34897a2ae6b3e6f3a1 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Fri, 13 May 2022 10:28:20 +0300 Subject: [PATCH 073/152] (fix) Fixes Kraken configs migration --- hummingbot/connector/exchange/kraken/kraken_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/connector/exchange/kraken/kraken_utils.py b/hummingbot/connector/exchange/kraken/kraken_utils.py index 34bf68d8d7..449bcddba7 100644 --- a/hummingbot/connector/exchange/kraken/kraken_utils.py +++ b/hummingbot/connector/exchange/kraken/kraken_utils.py @@ -196,7 +196,7 @@ class KrakenConfigMap(BaseConnectorConfigMap): ) ) kraken_api_tier: str = Field( - default=..., + default="Starter", client_data=ClientFieldData( prompt=lambda cm: "Enter your Kraken API Tier (Starter/Intermediate/Pro)", is_connect_key=True, From f06d07d1726554bfab3071396d668841096f8081 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Fri, 13 May 2022 10:43:36 +0300 Subject: [PATCH 074/152] (fix) Removes check for secure param on migration --- hummingbot/client/config/conf_migration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/client/config/conf_migration.py b/hummingbot/client/config/conf_migration.py index 7b0b92f974..b8c48ab1ed 100644 --- a/hummingbot/client/config/conf_migration.py +++ b/hummingbot/client/config/conf_migration.py @@ -107,7 +107,7 @@ def _maybe_migrate_encrypted_confs(config_keys: BaseConnectorConfigMap) -> List[ files_to_remove = [] missing_fields = [] for el in cm.traverse(): - if el.client_field_data is not None and el.client_field_data.is_secure: + if el.client_field_data is not None: key_path = conf_dir_path / f"{encrypted_conf_prefix}{el.attr}{encrypted_conf_postfix}" if key_path.exists(): with open(key_path, 'r') as f: From 4fc73eef66005ec73e7ca9754e87065027b2fcda Mon Sep 17 00:00:00 2001 From: mhrvth Date: Fri, 13 May 2022 12:41:09 +0200 Subject: [PATCH 075/152] (refactor) setters removed --- .../avellaneda_market_making.pyx | 70 ------------ .../test_avellaneda_market_making.py | 104 +++++------------- 2 files changed, 28 insertions(+), 146 deletions(-) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx index 21dab3a1af..e5967dff43 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx @@ -129,10 +129,6 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): def min_spread(self): return self._config_map.min_spread - @min_spread.setter - def min_spread(self, value): - self._config_map.min_spread = value - @property def avg_vol(self): return self._avg_vol @@ -157,35 +153,18 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): def order_refresh_tolerance_pct(self) -> Decimal: return self._config_map.order_refresh_tolerance_pct - @order_refresh_tolerance_pct.setter - def order_refresh_tolerance_pct(self, value: Decimal): - if self._config_map.order_refresh_tolerance_pct is not None: - self._config_map.order_refresh_tolerance_pct = value - @property def order_refresh_tolerance(self) -> Decimal: return self._config_map.order_refresh_tolerance_pct / Decimal('100') - @order_refresh_tolerance.setter - def order_refresh_tolerance(self, value: Decimal): - self._config_map.order_refresh_tolerance_pct = value * Decimal('100') - @property def order_amount(self) -> Decimal: return self._config_map.order_amount - @order_amount.setter - def order_amount(self, value: Decimal): - self._config_map.order_amount = value - @property def inventory_target_base_pct(self) -> Decimal: return self._config_map.inventory_target_base_pct - @inventory_target_base_pct.setter - def inventory_target_base_pct(self, value: Decimal): - self._config_map.inventory_target_base_pct = value - @property def inventory_target_base(self) -> Decimal: return self.inventory_target_base_pct / Decimal('100') @@ -198,34 +177,18 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): def order_optimization_enabled(self) -> bool: return self._config_map.order_optimization_enabled - @order_optimization_enabled.setter - def order_optimization_enabled(self, value: bool): - self._config_map.order_optimization_enabled = value - @property def order_refresh_time(self) -> float: return self._config_map.order_refresh_time - @order_refresh_time.setter - def order_refresh_time(self, value: float): - self._config_map.order_refresh_time = value - @property def filled_order_delay(self) -> float: return self._config_map.filled_order_delay - @filled_order_delay.setter - def filled_order_delay(self, value: float): - self._config_map.filled_order_delay = value - @property def order_override(self) -> Dict[str, any]: return self._config_map.order_override - @order_override.setter - def order_override(self, value): - self._config_map.order_override = value - @property def order_levels(self) -> int: if self._config_map.order_levels_mode.title == MultiOrderLevelModel.Config.title: @@ -233,14 +196,6 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): else: return 0 - @order_levels.setter - def order_levels(self, value): - if value == 0: - self._config_map.order_levels_mode = SingleOrderLevelModel() - else: - self._config_map.order_levels_mode = MultiOrderLevelModel() - self._config_map.order_levels_mode.order_levels = value - @property def level_distances(self) -> int: if self._config_map.order_levels_mode.title == MultiOrderLevelModel.Config.title: @@ -248,27 +203,14 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): else: return 0 - @level_distances.setter - def level_distances(self, value): - if self._config_map.order_levels_mode.title == MultiOrderLevelModel.Config.title: - self._config_map.order_levels_mode.level_distances = value - @property def max_order_age(self): return self._config_map.max_order_age - @max_order_age.setter - def max_order_age(self, value): - self._config_map.max_order_age = value - @property def add_transaction_costs_to_orders(self) -> bool: return self._config_map.add_transaction_costs - @add_transaction_costs_to_orders.setter - def add_transaction_costs_to_orders(self, value: bool): - self._config_map.add_transaction_costs = value - @property def base_asset(self): return self._market_info.base_asset @@ -285,10 +227,6 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): def gamma(self): return self._config_map.risk_factor - @gamma.setter - def gamma(self, value): - self._config_map.risk_factor = value - @property def alpha(self): return self._alpha @@ -309,10 +247,6 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): def eta(self): return self._config_map.order_amount_shape_factor - @eta.setter - def eta(self, value): - self._config_map.order_amount_shape_factor = value - @property def reservation_price(self): return self._reservation_price @@ -345,10 +279,6 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): def execution_timeframe(self): return self._execution_timeframe - @execution_timeframe.setter - def execution_timeframe(self, value): - self._execution_timeframe = value - @property def start_time(self) -> time: return self._start_time diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py index a0636dd10d..5823e041c5 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py @@ -32,6 +32,9 @@ from hummingbot.strategy.avellaneda_market_making import AvellanedaMarketMakingStrategy from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( AvellanedaMarketMakingConfigMap, + DailyBetweenTimesModel, + FromDateToDateModel, + InfiniteModel, MultiOrderLevelModel, TrackHangingOrdersModel ) @@ -393,63 +396,6 @@ def test_all_markets_ready(self): def test_market_info(self): self.assertEqual(self.market_info, self.strategy.market_info) - def test_order_refresh_tolerance_pct(self): - # Default value for order_refresh_tolerance_pct - self.assertEqual(Decimal(0), self.strategy.order_refresh_tolerance_pct) - - # Test setter method - self.strategy.order_refresh_tolerance_pct = Decimal("1") - - self.assertEqual(Decimal("1"), self.strategy.order_refresh_tolerance_pct) - - def test_order_amount(self): - self.assertEqual(self.order_amount, self.strategy.order_amount) - - # Test setter method - self.strategy.order_amount = Decimal("1") - - self.assertEqual(Decimal("1"), self.strategy.order_amount) - - def test_inventory_target_base_pct(self): - self.assertEqual(self.inventory_target_base_pct, self.strategy.inventory_target_base_pct) - - # Test setter method - self.strategy.inventory_target_base_pct = Decimal("1") - - self.assertEqual(Decimal("1"), self.strategy.inventory_target_base_pct) - - def test_order_optimization_enabled(self): - self.assertFalse(s_decimal_zero, self.strategy.order_optimization_enabled) - - # Test setter method - self.strategy.order_optimization_enabled = True - - self.assertTrue(self.strategy.order_optimization_enabled) - - def test_order_refresh_time(self): - self.assertEqual(float(30.0), self.strategy.order_refresh_time) - - # Test setter method - self.strategy.order_refresh_time = float(1.0) - - self.assertEqual(float(1.0), self.strategy.order_refresh_time) - - def test_filled_order_delay(self): - self.assertEqual(float(60.0), self.strategy.filled_order_delay) - - # Test setter method - self.strategy.filled_order_delay = float(1.0) - - self.assertEqual(float(1.0), self.strategy.filled_order_delay) - - def test_add_transaction_costs_to_orders(self): - self.assertTrue(self.strategy.order_optimization_enabled) - - # Test setter method - self.strategy.order_optimization_enabled = False - - self.assertFalse(self.strategy.order_optimization_enabled) - def test_base_asset(self): self.assertEqual(self.trading_pair.split("-")[0], self.strategy.base_asset) @@ -645,9 +591,9 @@ def test_liquidity_estimation(self): def test_calculate_reservation_price_and_optimal_spread_timeframe_constrained(self): # Init params - self.strategy.execution_timeframe = "daily_between_times" - self.strategy.start_time = (datetime.datetime.fromtimestamp(self.strategy.current_timestamp) - datetime.timedelta(minutes=30)).time() - self.strategy.end_time = (datetime.datetime.fromtimestamp(self.strategy.current_timestamp) + datetime.timedelta(minutes=30)).time() + start_time = (datetime.datetime.fromtimestamp(self.strategy.current_timestamp) - datetime.timedelta(minutes=30)).time().strftime("%H:%M:%S") + end_time = (datetime.datetime.fromtimestamp(self.strategy.current_timestamp) + datetime.timedelta(minutes=30)).time().strftime("%H:%M:%S") + self.config_map.execution_timeframe_mode = DailyBetweenTimesModel(start_time=start_time, end_time=end_time) # Simulate low volatility self.simulate_low_volatility(self.strategy) @@ -667,8 +613,8 @@ def test_calculate_reservation_price_and_optimal_spread_timeframe_constrained(se def test_calculate_reservation_price_and_optimal_spread_timeframe_infinite(self): # Init params - self.strategy.execution_timeframe = "infinite" - self.strategy.gamma = self.risk_factor_infinite + self.config_map.execution_timeframe_mode = InfiniteModel() + self.config_map.risk_factor = self.risk_factor_infinite # Simulate low volatility self.simulate_low_volatility(self.strategy) @@ -697,7 +643,7 @@ def test_create_proposal_based_on_order_override(self): } # Re-configure strategy with order_ride configurations - self.strategy.order_override = order_override + self.config_map.order_override = order_override expected_proposal = (list(), list()) for order in order_override.values(): @@ -801,7 +747,10 @@ def test_create_proposal_based_on_order_levels(self): self.assertEqual(empty_proposal, self.strategy.create_proposal_based_on_order_levels()) # Re-initialize strategy with order_level configurations - self.strategy.order_levels = 2 + order_levels_mode = MultiOrderLevelModel() + order_levels_mode.order_levels = 2 + order_levels_mode.level_distances = 1 + self.config_map.order_levels_mode = order_levels_mode # Calculate order levels bid_level_spreads, ask_level_spreads = self.strategy._get_level_spreads() @@ -878,7 +827,7 @@ def test_create_base_proposal(self): } # Re-configure strategy with order_ride configurations - self.strategy.order_override = order_override + self.config_map.order_override = order_override expected_buys = [] expected_sells = [] @@ -899,10 +848,13 @@ def test_create_base_proposal(self): self.assertEqual(str(expected_proposal), str(self.strategy.create_base_proposal())) # Reset order_override configuration - self.strategy.order_override = {} + self.config_map.order_override = {} # (3) With order_levels - self.strategy.order_levels = 2 + order_levels_mode = MultiOrderLevelModel() + order_levels_mode.order_levels = 2 + order_levels_mode.level_distances = 1 + self.config_map.order_levels_mode = order_levels_mode # Calculate order levels bid_level_spreads, ask_level_spreads = self.strategy._get_level_spreads() @@ -1043,7 +995,7 @@ def test_apply_order_price_modifiers(self): self.assertAlmostEqual(expected_ask_price, new_ask_price, 6) # (2) With none enabled - self.strategy.order_optimization_enabled = self.strategy.add_transaction_costs_to_orders = False + self.config_map.order_optimization_enabled = self.config_map.add_transaction_costs = False new_proposal: Proposal = deepcopy(initial_proposal) self.strategy.apply_order_price_modifiers(new_proposal) @@ -1135,7 +1087,7 @@ def test_apply_order_amount_eta_transformation(self): } # Re-configure strategy with order_ride configurations - self.strategy.order_override = order_override + self.config_map.order_override = order_override self.strategy.apply_order_amount_eta_transformation(proposal) @@ -1144,7 +1096,7 @@ def test_apply_order_amount_eta_transformation(self): # Test (2) Check proposal when order_override is None # Re-configure strategy with order_ride configurations - self.strategy.order_override = None + self.config_map.order_override = None proposal: Proposal = deepcopy(initial_proposal) @@ -1201,7 +1153,7 @@ def test_is_within_tolerance(self): self.assertFalse(self.strategy.is_within_tolerance(buy_prices, proposal_buys)) self.assertFalse(self.strategy.is_within_tolerance(sell_prices, proposal_sells)) - self.strategy.order_refresh_tolerance_pct = Decimal("10.0") + self.config_map.order_refresh_tolerance_pct = Decimal("10.0") self.assertTrue(self.strategy.is_within_tolerance(buy_prices, proposal_buys)) self.assertTrue(self.strategy.is_within_tolerance(sell_prices, proposal_sells)) @@ -1236,14 +1188,14 @@ def test_cancel_active_orders(self): # Case (2): Has active orders and within _order_refresh_tolerance_pct. # Note: Order will NOT be cancelled - self.strategy.order_refresh_tolerance_pct = Decimal("10") + self.config_map.order_refresh_tolerance_pct = Decimal("10") self.simulate_place_limit_order(self.strategy, self.market_info, limit_buy_order) self.simulate_place_limit_order(self.strategy, self.market_info, limit_sell_order) self.assertEqual(2, len(self.strategy.active_orders)) # Case (3a): Has active orders and EXCEED _order_refresh_tolerance_pct BUT cancel_timestamp > current_timestamp # Note: Orders will NOT be cancelled - self.strategy.order_refresh_tolerance_pct = Decimal("0") + self.config_map.order_refresh_tolerance_pct = Decimal("0") bid_price: Decimal = Decimal("98.5") ask_price: Decimal = Decimal("100.5") proposal: Proposal = Proposal( @@ -1266,7 +1218,7 @@ def test_cancel_active_orders(self): # Case (4): Has active orders and within _order_refresh_tolerance_pct BUT cancel_timestamp > current_timestamp # Note: Order not cancelled - self.strategy.order_refresh_tolerance_pct = Decimal("10") + self.config_map.order_refresh_tolerance_pct = Decimal("10") self.simulate_place_limit_order(self.strategy, self.market_info, limit_buy_order) self.simulate_place_limit_order(self.strategy, self.market_info, limit_sell_order) self.assertEqual(2, len(self.strategy.active_orders)) @@ -1276,7 +1228,7 @@ def test_cancel_active_orders(self): self.assertEqual(2, len(self.strategy.active_orders)) # Case (5): Has active orders and within _order_refresh_tolerance_pct AND cancel_timestamp <= current_timestamp - self.strategy.order_refresh_tolerance_pct = s_decimal_neg_one + self.config_map.order_refresh_tolerance_pct = s_decimal_neg_one self.clock.backtest_til(self.strategy.current_timestamp + self.strategy.order_refresh_time + 1) @@ -1481,7 +1433,7 @@ def test_no_new_orders_created_until_previous_orders_cancellation_confirmed(self refresh_time = self.strategy.order_refresh_time self.strategy.avg_vol = self.avg_vol_indicator - self.strategy.order_refresh_tolerance_pct = -1 + self.config_map.order_refresh_tolerance_pct = -1 # Simulate low volatility self.simulate_low_volatility(self.strategy) From 9cc40b2ddf5bd802a623f5f74717879c509b393e Mon Sep 17 00:00:00 2001 From: mhrvth <7762229+mhrvth@users.noreply.github.com> Date: Fri, 13 May 2022 13:47:05 +0200 Subject: [PATCH 076/152] Update test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py Co-authored-by: Petio Petrov --- .../test_avellaneda_market_making.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py index 5823e041c5..5ec68794c0 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py @@ -591,8 +591,12 @@ def test_liquidity_estimation(self): def test_calculate_reservation_price_and_optimal_spread_timeframe_constrained(self): # Init params - start_time = (datetime.datetime.fromtimestamp(self.strategy.current_timestamp) - datetime.timedelta(minutes=30)).time().strftime("%H:%M:%S") - end_time = (datetime.datetime.fromtimestamp(self.strategy.current_timestamp) + datetime.timedelta(minutes=30)).time().strftime("%H:%M:%S") + start_time = ( + datetime.datetime.fromtimestamp(self.strategy.current_timestamp) - datetime.timedelta(minutes=30) + ).time().strftime("%H:%M:%S") + end_time = ( + datetime.datetime.fromtimestamp(self.strategy.current_timestamp) + datetime.timedelta(minutes=30) + ).time().strftime("%H:%M:%S") self.config_map.execution_timeframe_mode = DailyBetweenTimesModel(start_time=start_time, end_time=end_time) # Simulate low volatility From 911fe2cff954302dc56c0504fbac247bb519e3b9 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Fri, 13 May 2022 15:04:53 +0300 Subject: [PATCH 077/152] (fix) Fixing a typo in the install script --- install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install b/install index 625b01b51d..bd005f7598 100755 --- a/install +++ b/install @@ -39,7 +39,7 @@ else ${CONDA_EXE} env create -f $ENV_FILE fi -source "${CONDA_BIN}/activate" hummingbot-hs +source "${CONDA_BIN}/activate" hummingbot # Add the project directory to module search paths. conda develop . From b966c88192a9432c4fd757b1f59a765373ea987a Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Mon, 16 May 2022 10:35:17 +0300 Subject: [PATCH 078/152] (fix) Fixing loading encrypted configs that are not supposed to secret --- hummingbot/client/config/conf_migration.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hummingbot/client/config/conf_migration.py b/hummingbot/client/config/conf_migration.py index b8c48ab1ed..46b33fb37f 100644 --- a/hummingbot/client/config/conf_migration.py +++ b/hummingbot/client/config/conf_migration.py @@ -112,8 +112,10 @@ def _maybe_migrate_encrypted_confs(config_keys: BaseConnectorConfigMap) -> List[ if key_path.exists(): with open(key_path, 'r') as f: json_str = f.read() - encrypted = binascii.hexlify(json_str.encode()).decode() - cm.setattr_no_validation(el.attr, encrypted) + value = binascii.hexlify(json_str.encode()).decode() + if not el.client_field_data.is_secure: + value = Security.secrets_manager.decrypt_secret_value(el.attr, value) + cm.setattr_no_validation(el.attr, value) files_to_remove.append(key_path) found_one = True else: From 0feb9e2684795c250675bb77350545ff816a5f7c Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Mon, 16 May 2022 10:49:35 +0300 Subject: [PATCH 079/152] (cleanup) Adding config title to connector configs This will allow proper titles of yml config files. --- .../derivative/binance_perpetual/binance_perpetual_utils.py | 3 +++ .../derivative/bybit_perpetual/bybit_perpetual_utils.py | 6 ++++++ .../derivative/dydx_perpetual/dydx_perpetual_utils.py | 3 +++ .../connector/exchange/altmarkets/altmarkets_utils.py | 3 +++ hummingbot/connector/exchange/ascend_ex/ascend_ex_utils.py | 3 +++ hummingbot/connector/exchange/beaxy/beaxy_utils.py | 3 +++ hummingbot/connector/exchange/binance/binance_utils.py | 6 ++++++ hummingbot/connector/exchange/bitfinex/bitfinex_utils.py | 3 +++ hummingbot/connector/exchange/bitmart/bitmart_utils.py | 3 +++ hummingbot/connector/exchange/bittrex/bittrex_utils.py | 3 +++ hummingbot/connector/exchange/blocktane/blocktane_utils.py | 3 +++ .../connector/exchange/coinbase_pro/coinbase_pro_utils.py | 3 +++ hummingbot/connector/exchange/coinflex/coinflex_utils.py | 6 ++++++ hummingbot/connector/exchange/coinzoom/coinzoom_utils.py | 3 +++ .../connector/exchange/crypto_com/crypto_com_utils.py | 3 +++ hummingbot/connector/exchange/digifinex/digifinex_utils.py | 3 +++ hummingbot/connector/exchange/ftx/ftx_utils.py | 3 +++ hummingbot/connector/exchange/gate_io/gate_io_utils.py | 3 +++ hummingbot/connector/exchange/hitbtc/hitbtc_utils.py | 3 +++ hummingbot/connector/exchange/huobi/huobi_utils.py | 3 +++ hummingbot/connector/exchange/k2/k2_utils.py | 3 +++ hummingbot/connector/exchange/kraken/kraken_utils.py | 3 +++ hummingbot/connector/exchange/kucoin/kucoin_utils.py | 6 ++++++ hummingbot/connector/exchange/liquid/liquid_utils.py | 3 +++ hummingbot/connector/exchange/loopring/loopring_utils.py | 3 +++ hummingbot/connector/exchange/mexc/mexc_utils.py | 3 +++ hummingbot/connector/exchange/ndax/ndax_utils.py | 6 ++++++ hummingbot/connector/exchange/okex/okex_utils.py | 3 +++ hummingbot/connector/exchange/probit/probit_utils.py | 6 ++++++ hummingbot/connector/exchange/wazirx/wazirx_utils.py | 3 +++ hummingbot/connector/other/celo/celo_data_types.py | 3 +++ 31 files changed, 111 insertions(+) diff --git a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_utils.py b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_utils.py index b22512ca62..71221b354b 100644 --- a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_utils.py +++ b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_utils.py @@ -96,5 +96,8 @@ class BinancePerpetualTestnetConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "binance_perpetual" + OTHER_DOMAINS_KEYS = {"binance_perpetual_testnet": BinancePerpetualTestnetConfigMap.construct()} diff --git a/hummingbot/connector/derivative/bybit_perpetual/bybit_perpetual_utils.py b/hummingbot/connector/derivative/bybit_perpetual/bybit_perpetual_utils.py index 9bcb4f2184..dd222a8b3d 100644 --- a/hummingbot/connector/derivative/bybit_perpetual/bybit_perpetual_utils.py +++ b/hummingbot/connector/derivative/bybit_perpetual/bybit_perpetual_utils.py @@ -123,6 +123,9 @@ class BybitPerpetualConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "bybit_perpetual" + KEYS = BybitPerpetualConfigMap.construct() @@ -153,6 +156,9 @@ class BybitPerpetualTestnetConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "bybit_perpetual_testnet" + OTHER_DOMAINS_KEYS = { "bybit_perpetual_testnet": BybitPerpetualTestnetConfigMap.construct() diff --git a/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_utils.py b/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_utils.py index 0997015c91..1f3af42e63 100644 --- a/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_utils.py +++ b/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_utils.py @@ -75,5 +75,8 @@ class DydxPerpetualConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "dydx_perpetual" + KEYS = DydxPerpetualConfigMap.construct() diff --git a/hummingbot/connector/exchange/altmarkets/altmarkets_utils.py b/hummingbot/connector/exchange/altmarkets/altmarkets_utils.py index d28acaf913..494fc16657 100644 --- a/hummingbot/connector/exchange/altmarkets/altmarkets_utils.py +++ b/hummingbot/connector/exchange/altmarkets/altmarkets_utils.py @@ -95,5 +95,8 @@ class AltmarketsConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "altmarkets" + KEYS = AltmarketsConfigMap.construct() diff --git a/hummingbot/connector/exchange/ascend_ex/ascend_ex_utils.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_utils.py index 831c49aa22..e10181489d 100644 --- a/hummingbot/connector/exchange/ascend_ex/ascend_ex_utils.py +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_utils.py @@ -154,6 +154,9 @@ class AscendExConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "ascend_ex" + KEYS = AscendExConfigMap.construct() diff --git a/hummingbot/connector/exchange/beaxy/beaxy_utils.py b/hummingbot/connector/exchange/beaxy/beaxy_utils.py index 8734141c0e..9452226503 100644 --- a/hummingbot/connector/exchange/beaxy/beaxy_utils.py +++ b/hummingbot/connector/exchange/beaxy/beaxy_utils.py @@ -31,5 +31,8 @@ class BeaxyConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "beaxy" + KEYS = BeaxyConfigMap.construct() diff --git a/hummingbot/connector/exchange/binance/binance_utils.py b/hummingbot/connector/exchange/binance/binance_utils.py index d4443ab3b2..43024ce5be 100644 --- a/hummingbot/connector/exchange/binance/binance_utils.py +++ b/hummingbot/connector/exchange/binance/binance_utils.py @@ -46,6 +46,9 @@ class BinanceConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "binance" + KEYS = BinanceConfigMap.construct() @@ -76,5 +79,8 @@ class BinanceUSConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "binance_us" + OTHER_DOMAINS_KEYS = {"binance_us": BinanceUSConfigMap.construct()} diff --git a/hummingbot/connector/exchange/bitfinex/bitfinex_utils.py b/hummingbot/connector/exchange/bitfinex/bitfinex_utils.py index bfcf308d21..95dedd530c 100644 --- a/hummingbot/connector/exchange/bitfinex/bitfinex_utils.py +++ b/hummingbot/connector/exchange/bitfinex/bitfinex_utils.py @@ -39,6 +39,9 @@ class BitfinexConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "bitfinex" + KEYS = BitfinexConfigMap.construct() diff --git a/hummingbot/connector/exchange/bitmart/bitmart_utils.py b/hummingbot/connector/exchange/bitmart/bitmart_utils.py index 7902429c47..42cca62027 100644 --- a/hummingbot/connector/exchange/bitmart/bitmart_utils.py +++ b/hummingbot/connector/exchange/bitmart/bitmart_utils.py @@ -180,6 +180,9 @@ class BitmartConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "bitmart" + KEYS = BitmartConfigMap.construct() diff --git a/hummingbot/connector/exchange/bittrex/bittrex_utils.py b/hummingbot/connector/exchange/bittrex/bittrex_utils.py index 0807e13f8a..876d639811 100644 --- a/hummingbot/connector/exchange/bittrex/bittrex_utils.py +++ b/hummingbot/connector/exchange/bittrex/bittrex_utils.py @@ -36,5 +36,8 @@ class BittrexConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "bitrex" + KEYS = BittrexConfigMap.construct() diff --git a/hummingbot/connector/exchange/blocktane/blocktane_utils.py b/hummingbot/connector/exchange/blocktane/blocktane_utils.py index fc872708a9..fa03b6dc44 100644 --- a/hummingbot/connector/exchange/blocktane/blocktane_utils.py +++ b/hummingbot/connector/exchange/blocktane/blocktane_utils.py @@ -34,6 +34,9 @@ class BlocktaneConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "blocktane" + KEYS = BlocktaneConfigMap.construct() diff --git a/hummingbot/connector/exchange/coinbase_pro/coinbase_pro_utils.py b/hummingbot/connector/exchange/coinbase_pro/coinbase_pro_utils.py index 00f5b8423a..5600f2b101 100644 --- a/hummingbot/connector/exchange/coinbase_pro/coinbase_pro_utils.py +++ b/hummingbot/connector/exchange/coinbase_pro/coinbase_pro_utils.py @@ -49,6 +49,9 @@ class CoinbaseProConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "coinbase_pro" + KEYS = CoinbaseProConfigMap.construct() diff --git a/hummingbot/connector/exchange/coinflex/coinflex_utils.py b/hummingbot/connector/exchange/coinflex/coinflex_utils.py index 37b933ae80..61e82c8a3e 100644 --- a/hummingbot/connector/exchange/coinflex/coinflex_utils.py +++ b/hummingbot/connector/exchange/coinflex/coinflex_utils.py @@ -57,6 +57,9 @@ class CoinflexConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "coinflex" + KEYS = CoinflexConfigMap.construct() @@ -87,5 +90,8 @@ class CoinflexTestConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "coinflex_test" + OTHER_DOMAINS_KEYS = {"coinflex_test": CoinflexTestConfigMap.construct()} diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py b/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py index 0039aaf989..9d50fa1370 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_utils.py @@ -91,5 +91,8 @@ class CoinzoomConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "coinzoom" + KEYS = CoinzoomConfigMap.construct() diff --git a/hummingbot/connector/exchange/crypto_com/crypto_com_utils.py b/hummingbot/connector/exchange/crypto_com/crypto_com_utils.py index cc73e947d1..e21c6bf14b 100644 --- a/hummingbot/connector/exchange/crypto_com/crypto_com_utils.py +++ b/hummingbot/connector/exchange/crypto_com/crypto_com_utils.py @@ -99,5 +99,8 @@ class CryptoComConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "crypto_com" + KEYS = CryptoComConfigMap.construct() diff --git a/hummingbot/connector/exchange/digifinex/digifinex_utils.py b/hummingbot/connector/exchange/digifinex/digifinex_utils.py index 8e9675776a..a6905cbab8 100644 --- a/hummingbot/connector/exchange/digifinex/digifinex_utils.py +++ b/hummingbot/connector/exchange/digifinex/digifinex_utils.py @@ -103,5 +103,8 @@ class DigifinexConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "digifinex" + KEYS = DigifinexConfigMap.construct() diff --git a/hummingbot/connector/exchange/ftx/ftx_utils.py b/hummingbot/connector/exchange/ftx/ftx_utils.py index 479e1aa42b..fd9331d387 100644 --- a/hummingbot/connector/exchange/ftx/ftx_utils.py +++ b/hummingbot/connector/exchange/ftx/ftx_utils.py @@ -64,5 +64,8 @@ class FtxConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "ftx" + KEYS = FtxConfigMap.construct() diff --git a/hummingbot/connector/exchange/gate_io/gate_io_utils.py b/hummingbot/connector/exchange/gate_io/gate_io_utils.py index 48f0c546d0..90738bcf28 100644 --- a/hummingbot/connector/exchange/gate_io/gate_io_utils.py +++ b/hummingbot/connector/exchange/gate_io/gate_io_utils.py @@ -185,5 +185,8 @@ class GateIOConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "gate_io" + KEYS = GateIOConfigMap.construct() diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py index 977b9dc7f2..0e00fc4277 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_utils.py @@ -158,5 +158,8 @@ class HitbtcConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "hitbtc" + KEYS = HitbtcConfigMap.construct() diff --git a/hummingbot/connector/exchange/huobi/huobi_utils.py b/hummingbot/connector/exchange/huobi/huobi_utils.py index 0600057749..6a39a95c04 100644 --- a/hummingbot/connector/exchange/huobi/huobi_utils.py +++ b/hummingbot/connector/exchange/huobi/huobi_utils.py @@ -78,5 +78,8 @@ class HuobiConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "huobi" + KEYS = HuobiConfigMap.construct() diff --git a/hummingbot/connector/exchange/k2/k2_utils.py b/hummingbot/connector/exchange/k2/k2_utils.py index bbe1c91509..bb2f1492b3 100644 --- a/hummingbot/connector/exchange/k2/k2_utils.py +++ b/hummingbot/connector/exchange/k2/k2_utils.py @@ -104,5 +104,8 @@ class K2ConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "k2" + KEYS = K2ConfigMap.construct() diff --git a/hummingbot/connector/exchange/kraken/kraken_utils.py b/hummingbot/connector/exchange/kraken/kraken_utils.py index 449bcddba7..f623d619ce 100644 --- a/hummingbot/connector/exchange/kraken/kraken_utils.py +++ b/hummingbot/connector/exchange/kraken/kraken_utils.py @@ -204,6 +204,9 @@ class KrakenConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "kraken" + KEYS = KrakenConfigMap.construct() diff --git a/hummingbot/connector/exchange/kucoin/kucoin_utils.py b/hummingbot/connector/exchange/kucoin/kucoin_utils.py index 8a2c78f678..aaa835caa3 100644 --- a/hummingbot/connector/exchange/kucoin/kucoin_utils.py +++ b/hummingbot/connector/exchange/kucoin/kucoin_utils.py @@ -57,6 +57,9 @@ class KuCoinConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "kucoin" + KEYS = KuCoinConfigMap.construct() @@ -96,5 +99,8 @@ class KuCoinTestnetConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "kucoin_testnet" + OTHER_DOMAINS_KEYS = {"kucoin_testnet": KuCoinTestnetConfigMap.construct()} diff --git a/hummingbot/connector/exchange/liquid/liquid_utils.py b/hummingbot/connector/exchange/liquid/liquid_utils.py index 3ba827453f..11ead72cff 100644 --- a/hummingbot/connector/exchange/liquid/liquid_utils.py +++ b/hummingbot/connector/exchange/liquid/liquid_utils.py @@ -30,5 +30,8 @@ class LiquidConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "liquid" + KEYS = LiquidConfigMap.construct() diff --git a/hummingbot/connector/exchange/loopring/loopring_utils.py b/hummingbot/connector/exchange/loopring/loopring_utils.py index cfee94b783..f2113e8067 100644 --- a/hummingbot/connector/exchange/loopring/loopring_utils.py +++ b/hummingbot/connector/exchange/loopring/loopring_utils.py @@ -54,6 +54,9 @@ class LoopringConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "loopring" + KEYS = LoopringConfigMap.construct() diff --git a/hummingbot/connector/exchange/mexc/mexc_utils.py b/hummingbot/connector/exchange/mexc/mexc_utils.py index 990b7e8cac..7f25cdd9c1 100644 --- a/hummingbot/connector/exchange/mexc/mexc_utils.py +++ b/hummingbot/connector/exchange/mexc/mexc_utils.py @@ -40,6 +40,9 @@ class MexcConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "mexc" + KEYS = MexcConfigMap.construct() diff --git a/hummingbot/connector/exchange/ndax/ndax_utils.py b/hummingbot/connector/exchange/ndax/ndax_utils.py index bbaedd653f..8ea07625e1 100644 --- a/hummingbot/connector/exchange/ndax/ndax_utils.py +++ b/hummingbot/connector/exchange/ndax/ndax_utils.py @@ -78,6 +78,9 @@ class NdaxConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "ndax" + KEYS = NdaxConfigMap.construct() @@ -126,5 +129,8 @@ class NdaxTestnetConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "ndax_testnet" + OTHER_DOMAINS_KEYS = {"ndax_testnet": NdaxTestnetConfigMap.construct()} diff --git a/hummingbot/connector/exchange/okex/okex_utils.py b/hummingbot/connector/exchange/okex/okex_utils.py index 4a13e5a15a..d873ca7f4c 100644 --- a/hummingbot/connector/exchange/okex/okex_utils.py +++ b/hummingbot/connector/exchange/okex/okex_utils.py @@ -46,6 +46,9 @@ class OkexConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "okex" + KEYS = OkexConfigMap.construct() diff --git a/hummingbot/connector/exchange/probit/probit_utils.py b/hummingbot/connector/exchange/probit/probit_utils.py index d034da1cbb..8dbbb70980 100644 --- a/hummingbot/connector/exchange/probit/probit_utils.py +++ b/hummingbot/connector/exchange/probit/probit_utils.py @@ -87,6 +87,9 @@ class ProbitConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "probit" + KEYS = ProbitConfigMap.construct() @@ -117,5 +120,8 @@ class ProbitKrConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "probit_kr" + OTHER_DOMAINS_KEYS = {"probit_kr": ProbitKrConfigMap.construct()} diff --git a/hummingbot/connector/exchange/wazirx/wazirx_utils.py b/hummingbot/connector/exchange/wazirx/wazirx_utils.py index b14288b47f..ed06bb8db0 100644 --- a/hummingbot/connector/exchange/wazirx/wazirx_utils.py +++ b/hummingbot/connector/exchange/wazirx/wazirx_utils.py @@ -118,5 +118,8 @@ class WazirxConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "wazirx" + KEYS = WazirxConfigMap.construct() diff --git a/hummingbot/connector/other/celo/celo_data_types.py b/hummingbot/connector/other/celo/celo_data_types.py index 63537eece1..2d9142eef2 100644 --- a/hummingbot/connector/other/celo/celo_data_types.py +++ b/hummingbot/connector/other/celo/celo_data_types.py @@ -74,5 +74,8 @@ class CeloConfigMap(BaseConnectorConfigMap): ) ) + class Config: + title = "celo" + KEYS = CeloConfigMap.construct() From 4991a300cef3ce9c11239068fc2b9f11b5ddfd9f Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Mon, 16 May 2022 13:21:08 +0300 Subject: [PATCH 080/152] (fix) Removing the migration functions from coverage reports --- .coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index 10e98d7a93..3b208cef0f 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,6 +4,7 @@ omit = hummingbot/core/gateway/* hummingbot/core/management/* hummingbot/client/config/config_helpers.py + hummingbot/client/config/conf_migration.py hummingbot/client/config/security.py hummingbot/client/hummingbot_application.py hummingbot/client/command/* From e4ba35f74f256afba981e414844513c829625524 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Mon, 16 May 2022 14:24:45 +0200 Subject: [PATCH 081/152] (refactor) formatting --- .../strategy/conditional_execution_state.py | 7 ++-- .../test_avellaneda_market_making.py | 34 +++++++------------ 2 files changed, 16 insertions(+), 25 deletions(-) diff --git a/hummingbot/strategy/conditional_execution_state.py b/hummingbot/strategy/conditional_execution_state.py index 12bd471be8..e409250ff2 100644 --- a/hummingbot/strategy/conditional_execution_state.py +++ b/hummingbot/strategy/conditional_execution_state.py @@ -1,6 +1,5 @@ from abc import ABC, abstractmethod -from datetime import datetime -from datetime import time +from datetime import datetime, time from typing import Union from hummingbot.strategy.strategy_base import StrategyBase @@ -80,8 +79,8 @@ def __str__(self): def __eq__(self, other): return type(self) == type(other) and \ - self._start_timestamp == other._start_timestamp and \ - self._end_timestamp == other._end_timestamp + self._start_timestamp == other._start_timestamp and \ + self._end_timestamp == other._end_timestamp def process_tick(self, timestamp: float, strategy: StrategyBase): if isinstance(self._start_timestamp, datetime): diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py index 5ec68794c0..7cc5fdc9de 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py @@ -3,11 +3,7 @@ import unittest from copy import deepcopy from decimal import Decimal -from typing import ( - Dict, - List, - Tuple -) +from typing import Dict, List, Tuple import numpy as np import pandas as pd @@ -21,27 +17,20 @@ from hummingbot.core.data_type.order_book import OrderBook from hummingbot.core.data_type.order_book_row import OrderBookRow from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee, TradeFeeSchema -from hummingbot.core.event.events import ( - BuyOrderCompletedEvent, - MarketEvent, - OrderFilledEvent, - SellOrderCompletedEvent -) +from hummingbot.core.event.events import BuyOrderCompletedEvent, MarketEvent, OrderFilledEvent, SellOrderCompletedEvent from hummingbot.strategy.__utils__.trailing_indicators.instant_volatility import InstantVolatilityIndicator from hummingbot.strategy.__utils__.trailing_indicators.trading_intensity import TradingIntensityIndicator from hummingbot.strategy.avellaneda_market_making import AvellanedaMarketMakingStrategy from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( AvellanedaMarketMakingConfigMap, DailyBetweenTimesModel, - FromDateToDateModel, InfiniteModel, MultiOrderLevelModel, - TrackHangingOrdersModel + TrackHangingOrdersModel, ) from hummingbot.strategy.data_types import PriceSize, Proposal from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple - s_decimal_zero = Decimal(0) s_decimal_one = Decimal(1) s_decimal_nan = Decimal("NaN") @@ -647,7 +636,7 @@ def test_create_proposal_based_on_order_override(self): } # Re-configure strategy with order_ride configurations - self.config_map.order_override = order_override + self.config_map.order_override = order_override expected_proposal = (list(), list()) for order in order_override.values(): @@ -831,7 +820,7 @@ def test_create_base_proposal(self): } # Re-configure strategy with order_ride configurations - self.config_map.order_override = order_override + self.config_map.order_override = order_override expected_buys = [] expected_sells = [] @@ -852,7 +841,7 @@ def test_create_base_proposal(self): self.assertEqual(str(expected_proposal), str(self.strategy.create_base_proposal())) # Reset order_override configuration - self.config_map.order_override = {} + self.config_map.order_override = {} # (3) With order_levels order_levels_mode = MultiOrderLevelModel() @@ -972,7 +961,7 @@ def test_apply_order_price_modifiers(self): # <<<<< Test Preparation End # (1) Default: order_optimization = True, add_transaction_costs_to_orders = False - #self.strategy.add_transaction_costs_to_orders = True + # self.strategy.add_transaction_costs_to_orders = True # Intentionally make top_bid/ask_price lower/higher respectively & set TradeFees ob_bids: List[OrderBookRow] = [OrderBookRow(bid_price * Decimal("0.5"), self.order_amount, 2)] @@ -1080,7 +1069,10 @@ def test_apply_order_amount_eta_transformation(self): bid_price: Decimal = self.market.quantize_order_price(self.trading_pair, self.strategy.optimal_bid) ask_price: Decimal = self.market.quantize_order_price(self.trading_pair, self.strategy.optimal_ask) - initial_proposal: Proposal = Proposal([PriceSize(bid_price, order_amount)], [PriceSize(ask_price, order_amount)]) + initial_proposal: Proposal = Proposal( + [PriceSize(bid_price, order_amount)], + [PriceSize(ask_price, order_amount)] + ) # Test (1) Check proposal when order_override is NOT None proposal: Proposal = deepcopy(initial_proposal) @@ -1091,7 +1083,7 @@ def test_apply_order_amount_eta_transformation(self): } # Re-configure strategy with order_ride configurations - self.config_map.order_override = order_override + self.config_map.order_override = order_override self.strategy.apply_order_amount_eta_transformation(proposal) @@ -1100,7 +1092,7 @@ def test_apply_order_amount_eta_transformation(self): # Test (2) Check proposal when order_override is None # Re-configure strategy with order_ride configurations - self.config_map.order_override = None + self.config_map.order_override = None proposal: Proposal = deepcopy(initial_proposal) From 55bee8c7126403940d3b0f82c5f0f5e9f474a415 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Mon, 16 May 2022 14:51:53 +0200 Subject: [PATCH 082/152] (refactor) formatting --- .../avellaneda_market_making/avellaneda_market_making.pyx | 4 ++-- .../test_avellaneda_market_making_config_map_pydantic.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx index 8e0227a140..814d51df85 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx @@ -894,12 +894,12 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): list buys = [] list sells = [] price = market.c_quantize_order_price(self.trading_pair, Decimal(str(self._optimal_bid))) - size = market.c_quantize_order_amount(self.trading_pair, self._config_map.order_amount) + size = market.c_quantize_order_amount(self.trading_pair, self._config_map.order_amount) if size > 0: buys.append(PriceSize(price, size)) price = market.c_quantize_order_price(self.trading_pair, Decimal(str(self._optimal_ask))) - size = market.c_quantize_order_amount(self.trading_pair, self._config_map.order_amount) + size = market.c_quantize_order_amount(self.trading_pair, self._config_map.order_amount) if size > 0: sells.append(PriceSize(price, size)) return buys, sells diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py index f528001fa7..8bb531897c 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py @@ -19,7 +19,7 @@ InfiniteModel, MultiOrderLevelModel, SingleOrderLevelModel, - TrackHangingOrdersModel + TrackHangingOrdersModel, ) From e787e5c7d43dc22b5ac91eede07bf276275d44db Mon Sep 17 00:00:00 2001 From: mhrvth Date: Mon, 16 May 2022 14:52:24 +0200 Subject: [PATCH 083/152] (refactor) formatting --- .../avellaneda_market_making_config_map_pydantic.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py index 122c825dd0..5cb6115bc0 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -2,13 +2,9 @@ from decimal import Decimal from typing import Dict, Optional, Union -from pydantic import Field, validator, root_validator +from pydantic import Field, root_validator, validator -from hummingbot.client.config.config_data_types import ( - BaseClientModel, - BaseTradingStrategyConfigMap, - ClientFieldData, -) +from hummingbot.client.config.config_data_types import BaseClientModel, BaseTradingStrategyConfigMap, ClientFieldData from hummingbot.client.config.config_validators import ( validate_bool, validate_datetime_iso_string, From d2da015192d387baf19f4dff006b14f7ea197a9f Mon Sep 17 00:00:00 2001 From: mhrvth Date: Mon, 16 May 2022 14:59:28 +0200 Subject: [PATCH 084/152] (refactor) formatting --- hummingbot/client/config/config_methods.py | 3 ++- hummingbot/client/ui/interface_utils.py | 7 +------ hummingbot/client/ui/style.py | 3 ++- .../base_trailing_indicator.py | 4 +++- .../arbitrage/arbitrage_config_map.py | 14 +++++++------- .../aroon_oscillator_config_map.py | 19 +++++++------------ .../strategy/celo_arb/celo_arb_config_map.py | 14 ++++---------- ...cross_exchange_market_making_config_map.py | 15 ++++++++------- .../dev_0_hello_world_config_map.py | 14 +++----------- .../dev_1_get_order_book_config_map.py | 13 ++++--------- .../dev_2_perform_trade_config_map.py | 10 ++-------- .../liquidity_mining_config_map.py | 12 +++--------- .../perpetual_market_making_config_map.py | 15 ++++++--------- .../spot_perpetual_arbitrage.py | 7 ++----- .../spot_perpetual_arbitrage_config_map.py | 17 +++++++---------- .../client/command/test_create_command.py | 4 ++-- .../client/command/test_import_command.py | 4 ++-- .../client/command/test_status_command.py | 4 ++-- .../client/config/test_config_data_types.py | 4 +--- .../client/config/test_config_helpers.py | 2 +- .../test_spot_perpetual_arbitrage.py | 8 ++------ 21 files changed, 71 insertions(+), 122 deletions(-) diff --git a/hummingbot/client/config/config_methods.py b/hummingbot/client/config/config_methods.py index 9c7dbc182e..4c2f671fbd 100644 --- a/hummingbot/client/config/config_methods.py +++ b/hummingbot/client/config/config_methods.py @@ -1,7 +1,8 @@ +from typing import Callable + from pydantic.json import pydantic_encoder from hummingbot.client.config.config_var import ConfigVar -from typing import Callable def new_fee_config_var(key: str, type_str: str = "decimal"): diff --git a/hummingbot/client/ui/interface_utils.py b/hummingbot/client/ui/interface_utils.py index 72474985a5..177550382a 100644 --- a/hummingbot/client/ui/interface_utils.py +++ b/hummingbot/client/ui/interface_utils.py @@ -1,12 +1,7 @@ import asyncio import datetime from decimal import Decimal -from typing import ( - List, - Optional, - Set, - Tuple, -) +from typing import List, Optional, Set, Tuple import pandas as pd import psutil diff --git a/hummingbot/client/ui/style.py b/hummingbot/client/ui/style.py index 0e84d445ca..da06ad5409 100644 --- a/hummingbot/client/ui/style.py +++ b/hummingbot/client/ui/style.py @@ -1,7 +1,8 @@ from prompt_toolkit.styles import Style from prompt_toolkit.utils import is_windows -from hummingbot.client.config.global_config_map import global_config_map + from hummingbot.client.config.config_helpers import save_to_yml_legacy +from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.settings import GLOBAL_CONFIG_PATH diff --git a/hummingbot/strategy/__utils__/trailing_indicators/base_trailing_indicator.py b/hummingbot/strategy/__utils__/trailing_indicators/base_trailing_indicator.py index 2afb47dd5d..0718eaf763 100644 --- a/hummingbot/strategy/__utils__/trailing_indicators/base_trailing_indicator.py +++ b/hummingbot/strategy/__utils__/trailing_indicators/base_trailing_indicator.py @@ -1,6 +1,8 @@ +import logging from abc import ABC, abstractmethod + import numpy as np -import logging + from ..ring_buffer import RingBuffer pmm_logger = None diff --git a/hummingbot/strategy/arbitrage/arbitrage_config_map.py b/hummingbot/strategy/arbitrage/arbitrage_config_map.py index 543f0ab900..729cded10d 100644 --- a/hummingbot/strategy/arbitrage/arbitrage_config_map.py +++ b/hummingbot/strategy/arbitrage/arbitrage_config_map.py @@ -1,16 +1,16 @@ -import hummingbot.client.settings as settings +from decimal import Decimal +from typing import Optional -from hummingbot.client.config.config_var import ConfigVar +import hummingbot.client.settings as settings +from hummingbot.client.config.config_helpers import parse_cvar_value from hummingbot.client.config.config_validators import ( + validate_bool, + validate_decimal, validate_exchange, validate_market_trading_pair, - validate_decimal, - validate_bool ) -from hummingbot.client.config.config_helpers import parse_cvar_value +from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.settings import AllConnectorSettings, required_exchanges -from decimal import Decimal -from typing import Optional def validate_primary_market_trading_pair(value: str) -> Optional[str]: diff --git a/hummingbot/strategy/aroon_oscillator/aroon_oscillator_config_map.py b/hummingbot/strategy/aroon_oscillator/aroon_oscillator_config_map.py index 68fdb2cf3f..c0d99c28b9 100644 --- a/hummingbot/strategy/aroon_oscillator/aroon_oscillator_config_map.py +++ b/hummingbot/strategy/aroon_oscillator/aroon_oscillator_config_map.py @@ -1,21 +1,16 @@ from decimal import Decimal +from typing import Optional -from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.config_validators import ( - validate_exchange, - validate_market_trading_pair, validate_bool, validate_decimal, - validate_int -) -from hummingbot.client.settings import ( - required_exchanges, - AllConnectorSettings, -) -from hummingbot.client.config.global_config_map import ( - using_exchange + validate_exchange, + validate_int, + validate_market_trading_pair, ) -from typing import Optional +from hummingbot.client.config.config_var import ConfigVar +from hummingbot.client.config.global_config_map import using_exchange +from hummingbot.client.settings import AllConnectorSettings, required_exchanges def maker_trading_pair_prompt(): diff --git a/hummingbot/strategy/celo_arb/celo_arb_config_map.py b/hummingbot/strategy/celo_arb/celo_arb_config_map.py index 4641bfbe8d..11a66cd2dc 100644 --- a/hummingbot/strategy/celo_arb/celo_arb_config_map.py +++ b/hummingbot/strategy/celo_arb/celo_arb_config_map.py @@ -1,15 +1,9 @@ -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_validators import ( - validate_exchange, - validate_market_trading_pair, - validate_decimal -) -from hummingbot.client.settings import ( - required_exchanges, - AllConnectorSettings, -) from decimal import Decimal +from hummingbot.client.config.config_validators import validate_decimal, validate_exchange, validate_market_trading_pair +from hummingbot.client.config.config_var import ConfigVar +from hummingbot.client.settings import AllConnectorSettings, required_exchanges + def exchange_on_validated(value: str) -> None: required_exchanges.add(value) diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py index fbe57e903c..8a471bd872 100644 --- a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py +++ b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py @@ -1,14 +1,15 @@ -from hummingbot.client.config.config_var import ConfigVar +from decimal import Decimal +from typing import Optional + +import hummingbot.client.settings as settings +from hummingbot.client.config.config_helpers import parse_cvar_value from hummingbot.client.config.config_validators import ( + validate_bool, + validate_decimal, validate_exchange, validate_market_trading_pair, - validate_decimal, - validate_bool ) -from hummingbot.client.config.config_helpers import parse_cvar_value -import hummingbot.client.settings as settings -from decimal import Decimal -from typing import Optional +from hummingbot.client.config.config_var import ConfigVar def maker_trading_pair_prompt(): diff --git a/hummingbot/strategy/dev_0_hello_world/dev_0_hello_world_config_map.py b/hummingbot/strategy/dev_0_hello_world/dev_0_hello_world_config_map.py index fbf913f125..3bdf852186 100644 --- a/hummingbot/strategy/dev_0_hello_world/dev_0_hello_world_config_map.py +++ b/hummingbot/strategy/dev_0_hello_world/dev_0_hello_world_config_map.py @@ -1,16 +1,8 @@ -from typing import ( - Optional, -) +from typing import Optional +from hummingbot.client.config.config_validators import validate_exchange, validate_market_trading_pair from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_validators import ( - validate_exchange, - validate_market_trading_pair -) -from hummingbot.client.settings import ( - required_exchanges, - AllConnectorSettings, -) +from hummingbot.client.settings import AllConnectorSettings, required_exchanges def exchange_on_validated(value: str) -> None: diff --git a/hummingbot/strategy/dev_1_get_order_book/dev_1_get_order_book_config_map.py b/hummingbot/strategy/dev_1_get_order_book/dev_1_get_order_book_config_map.py index 60b28b2d2e..acded003db 100644 --- a/hummingbot/strategy/dev_1_get_order_book/dev_1_get_order_book_config_map.py +++ b/hummingbot/strategy/dev_1_get_order_book/dev_1_get_order_book_config_map.py @@ -1,14 +1,9 @@ -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_validators import ( - validate_exchange, - validate_market_trading_pair, -) -from hummingbot.client.settings import ( - required_exchanges, - AllConnectorSettings, -) from typing import Optional +from hummingbot.client.config.config_validators import validate_exchange, validate_market_trading_pair +from hummingbot.client.config.config_var import ConfigVar +from hummingbot.client.settings import AllConnectorSettings, required_exchanges + def trading_pair_prompt(): market = dev_1_get_order_book_config_map.get("exchange").value diff --git a/hummingbot/strategy/dev_2_perform_trade/dev_2_perform_trade_config_map.py b/hummingbot/strategy/dev_2_perform_trade/dev_2_perform_trade_config_map.py index c25a68a80a..df304c01e6 100644 --- a/hummingbot/strategy/dev_2_perform_trade/dev_2_perform_trade_config_map.py +++ b/hummingbot/strategy/dev_2_perform_trade/dev_2_perform_trade_config_map.py @@ -2,15 +2,9 @@ from decimal import Decimal from typing import Optional -from hummingbot.client.config.config_validators import ( - validate_exchange, - validate_market_trading_pair, -) +from hummingbot.client.config.config_validators import validate_exchange, validate_market_trading_pair from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.settings import ( - required_exchanges, - AllConnectorSettings, -) +from hummingbot.client.settings import AllConnectorSettings, required_exchanges def trading_pair_prompt(): diff --git a/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py b/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py index a6c0608f62..95dfb8540f 100644 --- a/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py +++ b/hummingbot/strategy/liquidity_mining/liquidity_mining_config_map.py @@ -5,16 +5,10 @@ import re from decimal import Decimal from typing import Optional + +from hummingbot.client.config.config_validators import validate_bool, validate_decimal, validate_exchange, validate_int from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.config_validators import ( - validate_exchange, - validate_decimal, - validate_int, - validate_bool -) -from hummingbot.client.settings import ( - required_exchanges, -) +from hummingbot.client.settings import required_exchanges def exchange_on_validated(value: str) -> None: diff --git a/hummingbot/strategy/perpetual_market_making/perpetual_market_making_config_map.py b/hummingbot/strategy/perpetual_market_making/perpetual_market_making_config_map.py index 9801579bb7..31f9a53e85 100644 --- a/hummingbot/strategy/perpetual_market_making/perpetual_market_making_config_map.py +++ b/hummingbot/strategy/perpetual_market_making/perpetual_market_making_config_map.py @@ -1,19 +1,16 @@ from decimal import Decimal from typing import Optional -from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.config_validators import ( - validate_exchange, - validate_derivative, - validate_market_trading_pair, validate_bool, validate_decimal, - validate_int -) -from hummingbot.client.settings import ( - AllConnectorSettings, - required_exchanges, + validate_derivative, + validate_exchange, + validate_int, + validate_market_trading_pair, ) +from hummingbot.client.config.config_var import ConfigVar +from hummingbot.client.settings import AllConnectorSettings, required_exchanges def maker_trading_pair_prompt(): diff --git a/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py index 9b88b6d848..99db15b79e 100644 --- a/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py +++ b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage.py @@ -9,14 +9,11 @@ from hummingbot.connector.connector_base import ConnectorBase from hummingbot.connector.derivative.position import Position from hummingbot.core.clock import Clock +from hummingbot.core.data_type.common import OrderType, PositionAction, PositionMode, TradeType from hummingbot.core.data_type.limit_order import LimitOrder from hummingbot.core.data_type.market_order import MarketOrder from hummingbot.core.data_type.order_candidate import OrderCandidate, PerpetualOrderCandidate -from hummingbot.core.event.events import ( - BuyOrderCompletedEvent, - SellOrderCompletedEvent, -) -from hummingbot.core.data_type.common import OrderType, PositionAction, PositionMode, TradeType +from hummingbot.core.event.events import BuyOrderCompletedEvent, SellOrderCompletedEvent from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather from hummingbot.logger import HummingbotLogger from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple diff --git a/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py index cf3fc7ca0b..d23942a40a 100644 --- a/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py +++ b/hummingbot/strategy/spot_perpetual_arbitrage/spot_perpetual_arbitrage_config_map.py @@ -1,17 +1,14 @@ -from hummingbot.client.config.config_var import ConfigVar +from decimal import Decimal + from hummingbot.client.config.config_validators import ( - validate_market_trading_pair, validate_connector, - validate_derivative, validate_decimal, - validate_int -) -from hummingbot.client.settings import ( - required_exchanges, - requried_connector_trading_pairs, - AllConnectorSettings, + validate_derivative, + validate_int, + validate_market_trading_pair, ) -from decimal import Decimal +from hummingbot.client.config.config_var import ConfigVar +from hummingbot.client.settings import AllConnectorSettings, required_exchanges, requried_connector_trading_pairs def exchange_on_validated(value: str) -> None: diff --git a/test/hummingbot/client/command/test_create_command.py b/test/hummingbot/client/command/test_create_command.py index c98fa8b756..b2640aa8ef 100644 --- a/test/hummingbot/client/command/test_create_command.py +++ b/test/hummingbot/client/command/test_create_command.py @@ -2,13 +2,13 @@ import unittest from copy import deepcopy from decimal import Decimal +from test.mock.mock_cli import CLIMockingAssistant from typing import Awaitable -from unittest.mock import patch, MagicMock, AsyncMock +from unittest.mock import AsyncMock, MagicMock, patch from hummingbot.client.config.config_helpers import get_strategy_config_map, read_system_configs_from_yml from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.hummingbot_application import HummingbotApplication -from test.mock.mock_cli import CLIMockingAssistant class CreateCommandTest(unittest.TestCase): diff --git a/test/hummingbot/client/command/test_import_command.py b/test/hummingbot/client/command/test_import_command.py index 593c3657ce..2e4aa48001 100644 --- a/test/hummingbot/client/command/test_import_command.py +++ b/test/hummingbot/client/command/test_import_command.py @@ -4,8 +4,9 @@ from decimal import Decimal from pathlib import Path from tempfile import TemporaryDirectory +from test.mock.mock_cli import CLIMockingAssistant from typing import Awaitable, Type -from unittest.mock import patch, MagicMock, AsyncMock +from unittest.mock import AsyncMock, MagicMock, patch from pydantic import Field @@ -14,7 +15,6 @@ from hummingbot.client.config.config_helpers import ClientConfigAdapter, read_system_configs_from_yml, save_to_yml from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.hummingbot_application import HummingbotApplication -from test.mock.mock_cli import CLIMockingAssistant class ImportCommandTest(unittest.TestCase): diff --git a/test/hummingbot/client/command/test_status_command.py b/test/hummingbot/client/command/test_status_command.py index 105db95916..163e7c399f 100644 --- a/test/hummingbot/client/command/test_status_command.py +++ b/test/hummingbot/client/command/test_status_command.py @@ -1,13 +1,13 @@ import asyncio import unittest from copy import deepcopy +from test.mock.mock_cli import CLIMockingAssistant from typing import Awaitable -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock, patch from hummingbot.client.config.config_helpers import read_system_configs_from_yml from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.hummingbot_application import HummingbotApplication -from test.mock.mock_cli import CLIMockingAssistant class StatusCommandTest(unittest.TestCase): diff --git a/test/hummingbot/client/config/test_config_data_types.py b/test/hummingbot/client/config/test_config_data_types.py index 9d8dfedeaa..933858908e 100644 --- a/test/hummingbot/client/config/test_config_data_types.py +++ b/test/hummingbot/client/config/test_config_data_types.py @@ -16,9 +16,7 @@ ClientConfigEnum, ClientFieldData, ) -from hummingbot.client.config.config_helpers import ( - ClientConfigAdapter, ConfigTraversalItem, ConfigValidationError -) +from hummingbot.client.config.config_helpers import ClientConfigAdapter, ConfigTraversalItem, ConfigValidationError class BaseClientModelTest(unittest.TestCase): diff --git a/test/hummingbot/client/config/test_config_helpers.py b/test/hummingbot/client/config/test_config_helpers.py index fe00a2b5cb..79673838ab 100644 --- a/test/hummingbot/client/config/test_config_helpers.py +++ b/test/hummingbot/client/config/test_config_helpers.py @@ -7,7 +7,7 @@ from hummingbot.client.config.config_data_types import BaseStrategyConfigMap from hummingbot.client.config.config_helpers import ClientConfigAdapter, get_strategy_config_map, save_to_yml from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( - AvellanedaMarketMakingConfigMap + AvellanedaMarketMakingConfigMap, ) diff --git a/test/hummingbot/strategy/spot_perpetual_arbitrage/test_spot_perpetual_arbitrage.py b/test/hummingbot/strategy/spot_perpetual_arbitrage/test_spot_perpetual_arbitrage.py index 08e985c0ca..4a17a828c6 100644 --- a/test/hummingbot/strategy/spot_perpetual_arbitrage/test_spot_perpetual_arbitrage.py +++ b/test/hummingbot/strategy/spot_perpetual_arbitrage/test_spot_perpetual_arbitrage.py @@ -1,6 +1,7 @@ import asyncio import unittest from decimal import Decimal +from test.mock.mock_perp_connector import MockPerpConnector import pandas as pd @@ -11,18 +12,13 @@ from hummingbot.core.clock import Clock, ClockMode from hummingbot.core.data_type.common import OrderType, PositionMode, PositionSide from hummingbot.core.event.event_logger import EventLogger -from hummingbot.core.event.events import ( - BuyOrderCompletedEvent, - MarketEvent, - SellOrderCompletedEvent, -) +from hummingbot.core.event.events import BuyOrderCompletedEvent, MarketEvent, SellOrderCompletedEvent from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple from hummingbot.strategy.spot_perpetual_arbitrage.arb_proposal import ArbProposal, ArbProposalSide from hummingbot.strategy.spot_perpetual_arbitrage.spot_perpetual_arbitrage import ( SpotPerpetualArbitrageStrategy, StrategyState, ) -from test.mock.mock_perp_connector import MockPerpConnector trading_pair = "HBOT-USDT" base_asset = trading_pair.split("-")[0] From 7461aa6a7ac7b450f9fdfcac16de37e7fb0c3070 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Tue, 17 May 2022 10:54:17 +0300 Subject: [PATCH 085/152] (fix) Moving mock paper trade exchange in a module This is done to allow `AllConnectorSettings` to properly identify the mock exchange as such. --- hummingbot/client/settings.py | 2 +- .../mock/mock_paper_exchange/__init__.py | 0 .../mock_order_tracker.py | 2 +- .../mock_paper_exchange.pxd | 0 .../mock_paper_exchange.pyx | 4 +-- .../mock/mock_pure_python_paper_exchange.py | 8 ----- .../__init__.py | 0 .../mock_pure_python_paper_exchange.py | 8 +++++ ...aneda_market_making_config_map_pydantic.py | 2 +- .../client/command/test_order_book_command.py | 4 +-- .../client/command/test_ticker_command.py | 6 ++-- .../connector/test_budget_checker.py | 2 +- test/hummingbot/connector/test_utils.py | 2 +- .../strategy/arbitrage/test_arbitrage.py | 2 +- .../test_avellaneda_market_making.py | 2 +- .../strategy/celo_arb/test_celo_arb.py | 4 +-- .../test_cross_exchange_market_making.py | 4 +-- .../test_dev_0_hello_world.py | 2 +- .../test_dev_1_get_order_book.py | 2 +- .../test_dev_2_perform_trade.py | 2 +- .../strategy/dev_5_vwap/test_vwap.py | 9 ++---- .../dev_simple_trade/test_simple_trade.py | 4 +-- .../liquidity_mining/test_liquidity_mining.py | 2 +- .../test_perpetual_market_making.py | 14 ++++----- .../strategy/pure_market_making/test_pmm.py | 4 +-- .../pure_market_making/test_pmm_ping_pong.py | 12 ++------ .../test_pmm_refresh_tolerance.py | 12 ++------ .../test_pmm_take_if_cross.py | 8 ++--- .../test_spot_perpetual_arbitrage.py | 30 +++++++++---------- .../test_market_trading_pair_tuple.py | 2 +- .../hummingbot/strategy/test_order_tracker.py | 7 ++--- .../strategy/test_script_strategy_base.py | 2 +- .../hummingbot/strategy/test_strategy_base.py | 15 ++-------- .../strategy/test_strategy_py_base.py | 2 +- test/hummingbot/strategy/twap/test_twap.py | 22 +++++++------- test/mock/mock_perp_connector.py | 4 +-- 36 files changed, 87 insertions(+), 120 deletions(-) create mode 100644 hummingbot/connector/mock/mock_paper_exchange/__init__.py rename hummingbot/connector/mock/{ => mock_paper_exchange}/mock_order_tracker.py (97%) rename hummingbot/connector/mock/{ => mock_paper_exchange}/mock_paper_exchange.pxd (100%) rename hummingbot/connector/mock/{ => mock_paper_exchange}/mock_paper_exchange.pyx (98%) delete mode 100644 hummingbot/connector/mock/mock_pure_python_paper_exchange.py create mode 100644 hummingbot/connector/mock/mock_pure_python_paper_exchange/__init__.py create mode 100644 hummingbot/connector/mock/mock_pure_python_paper_exchange/mock_pure_python_paper_exchange.py diff --git a/hummingbot/client/settings.py b/hummingbot/client/settings.py index 739864ef4e..68173d124a 100644 --- a/hummingbot/client/settings.py +++ b/hummingbot/client/settings.py @@ -208,7 +208,7 @@ def create_connector_settings(cls): Iterate over files in specific Python directories to create a dictionary of exchange names to ConnectorSetting. """ cls.all_connector_settings = {} # reset - connector_exceptions = ["paper_trade"] + connector_exceptions = ["mock_paper_exchange", "mock_pure_python_paper_exchange", "paper_trade"] type_dirs: List[DirEntry] = [ cast(DirEntry, f) for f in scandir(f"{root_path() / 'hummingbot' / 'connector'}") diff --git a/hummingbot/connector/mock/mock_paper_exchange/__init__.py b/hummingbot/connector/mock/mock_paper_exchange/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hummingbot/connector/mock/mock_order_tracker.py b/hummingbot/connector/mock/mock_paper_exchange/mock_order_tracker.py similarity index 97% rename from hummingbot/connector/mock/mock_order_tracker.py rename to hummingbot/connector/mock/mock_paper_exchange/mock_order_tracker.py index 24f1fd3049..8ddedf3270 100644 --- a/hummingbot/connector/mock/mock_order_tracker.py +++ b/hummingbot/connector/mock/mock_paper_exchange/mock_order_tracker.py @@ -1,5 +1,5 @@ import asyncio -from typing import List, Dict +from typing import Dict, List from hummingbot.core.data_type.order_book import OrderBook from hummingbot.core.data_type.order_book_tracker import OrderBookTracker, OrderBookTrackerDataSource diff --git a/hummingbot/connector/mock/mock_paper_exchange.pxd b/hummingbot/connector/mock/mock_paper_exchange/mock_paper_exchange.pxd similarity index 100% rename from hummingbot/connector/mock/mock_paper_exchange.pxd rename to hummingbot/connector/mock/mock_paper_exchange/mock_paper_exchange.pxd diff --git a/hummingbot/connector/mock/mock_paper_exchange.pyx b/hummingbot/connector/mock/mock_paper_exchange/mock_paper_exchange.pyx similarity index 98% rename from hummingbot/connector/mock/mock_paper_exchange.pyx rename to hummingbot/connector/mock/mock_paper_exchange/mock_paper_exchange.pyx index f9011f04bf..2780e4d4c4 100644 --- a/hummingbot/connector/mock/mock_paper_exchange.pyx +++ b/hummingbot/connector/mock/mock_paper_exchange/mock_paper_exchange.pyx @@ -9,7 +9,7 @@ from hummingbot.connector.connector_base cimport ConnectorBase from hummingbot.connector.exchange.paper_trade.paper_trade_exchange cimport PaperTradeExchange, QuantizationParams from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.exchange.paper_trade.trading_pair import TradingPair -from hummingbot.connector.mock.mock_order_tracker import MockOrderTracker +from hummingbot.connector.mock.mock_paper_exchange.mock_order_tracker import MockOrderTracker from hummingbot.core.clock cimport Clock from hummingbot.core.data_type.common import OrderType from hummingbot.core.data_type.composite_order_book cimport CompositeOrderBook @@ -45,7 +45,7 @@ cdef class MockPaperExchange(PaperTradeExchange): @property def name(self) -> str: - return "MockPaperExchange" + return "mock_paper_exchange" @property def display_name(self) -> str: diff --git a/hummingbot/connector/mock/mock_pure_python_paper_exchange.py b/hummingbot/connector/mock/mock_pure_python_paper_exchange.py deleted file mode 100644 index 6445415f6f..0000000000 --- a/hummingbot/connector/mock/mock_pure_python_paper_exchange.py +++ /dev/null @@ -1,8 +0,0 @@ -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange - - -class MockPurePythonPaperExchange(MockPaperExchange): - - @property - def name(self) -> str: - return "MockPurePythonPaperExchange" diff --git a/hummingbot/connector/mock/mock_pure_python_paper_exchange/__init__.py b/hummingbot/connector/mock/mock_pure_python_paper_exchange/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hummingbot/connector/mock/mock_pure_python_paper_exchange/mock_pure_python_paper_exchange.py b/hummingbot/connector/mock/mock_pure_python_paper_exchange/mock_pure_python_paper_exchange.py new file mode 100644 index 0000000000..921256253e --- /dev/null +++ b/hummingbot/connector/mock/mock_pure_python_paper_exchange/mock_pure_python_paper_exchange.py @@ -0,0 +1,8 @@ +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange + + +class MockPurePythonPaperExchange(MockPaperExchange): + + @property + def name(self) -> str: + return "mock_pure_python_paper_exchange" diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py index 5cb6115bc0..0596180065 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -465,7 +465,7 @@ def validate_pct_inclusive(cls, v: str): # === post-validations === - @root_validator() + @root_validator(skip_on_failure=True) def post_validations(cls, values: Dict): cls.exchange_post_validation(values) return values diff --git a/test/hummingbot/client/command/test_order_book_command.py b/test/hummingbot/client/command/test_order_book_command.py index 446b95ea58..fe6128975d 100644 --- a/test/hummingbot/client/command/test_order_book_command.py +++ b/test/hummingbot/client/command/test_order_book_command.py @@ -7,7 +7,7 @@ from hummingbot.client.config.config_helpers import read_system_configs_from_yml from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.hummingbot_application import HummingbotApplication -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange class OrderBookCommandTest(unittest.TestCase): @@ -58,7 +58,7 @@ def test_show_order_book(self, notify_mock): self.assertEqual(1, len(captures)) df_str_expected = ( - " market: MockPaperExchange BTC-USDT" + " market: mock_paper_exchange BTC-USDT" "\n +-------------+--------------+-------------+--------------+" "\n | bid_price | bid_volume | ask_price | ask_volume |" "\n |-------------+--------------+-------------+--------------|" diff --git a/test/hummingbot/client/command/test_ticker_command.py b/test/hummingbot/client/command/test_ticker_command.py index 615983f8d7..77f0ec92c2 100644 --- a/test/hummingbot/client/command/test_ticker_command.py +++ b/test/hummingbot/client/command/test_ticker_command.py @@ -2,12 +2,12 @@ import unittest from collections import Awaitable from copy import deepcopy -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock, patch from hummingbot.client.config.config_helpers import read_system_configs_from_yml from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.hummingbot_application import HummingbotApplication -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange class TickerCommandTest(unittest.TestCase): @@ -58,7 +58,7 @@ def test_show_ticker(self, notify_mock): self.assertEqual(1, len(captures)) df_str_expected = ( - " Market: MockPaperExchange" + " Market: mock_paper_exchange" "\n+------------+------------+-------------+--------------+" "\n| Best Bid | Best Ask | Mid Price | Last Trade |" "\n|------------+------------+-------------+--------------|" diff --git a/test/hummingbot/connector/test_budget_checker.py b/test/hummingbot/connector/test_budget_checker.py index 1ebc44f646..ea2e5731fc 100644 --- a/test/hummingbot/connector/test_budget_checker.py +++ b/test/hummingbot/connector/test_budget_checker.py @@ -3,7 +3,7 @@ from hummingbot.connector.budget_checker import BudgetChecker from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.connector.utils import combine_to_hb_trading_pair from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.data_type.order_candidate import OrderCandidate diff --git a/test/hummingbot/connector/test_utils.py b/test/hummingbot/connector/test_utils.py index 97f98182f9..5eed751434 100644 --- a/test/hummingbot/connector/test_utils.py +++ b/test/hummingbot/connector/test_utils.py @@ -33,7 +33,7 @@ def test_get_new_client_order_id(self): self.assertEqual(len(id0) - 2, len(id2)) def test_connector_config_maps(self): - connector_exceptions = ["paper_trade", "celo"] + connector_exceptions = ["mock_paper_exchange", "mock_pure_python_paper_exchange", "paper_trade", "celo"] type_dirs = [ cast(DirEntry, f) for f in diff --git a/test/hummingbot/strategy/arbitrage/test_arbitrage.py b/test/hummingbot/strategy/arbitrage/test_arbitrage.py index 0738daea90..07c180c7f0 100644 --- a/test/hummingbot/strategy/arbitrage/test_arbitrage.py +++ b/test/hummingbot/strategy/arbitrage/test_arbitrage.py @@ -6,7 +6,7 @@ from nose.plugins.attrib import attr from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode from hummingbot.core.data_type.order_book_row import OrderBookRow from hummingbot.core.event.event_logger import EventLogger diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py index 7cc5fdc9de..957afac2ed 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py @@ -10,7 +10,7 @@ from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.data_type.limit_order import LimitOrder diff --git a/test/hummingbot/strategy/celo_arb/test_celo_arb.py b/test/hummingbot/strategy/celo_arb/test_celo_arb.py index d98565da7d..d0fe524bf3 100644 --- a/test/hummingbot/strategy/celo_arb/test_celo_arb.py +++ b/test/hummingbot/strategy/celo_arb/test_celo_arb.py @@ -1,20 +1,20 @@ import logging import unittest from decimal import Decimal +from test.connector.fixture_celo import TEST_ADDRESS, TEST_PASSWORD, outputs as celo_outputs import mock import pandas as pd from nose.plugins.attrib import attr from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.connector.other.celo.celo_cli import CeloCLI from hummingbot.core.clock import Clock, ClockMode from hummingbot.core.event.event_logger import EventLogger from hummingbot.core.event.events import MarketEvent from hummingbot.strategy.celo_arb.celo_arb import CeloArbStrategy, get_trade_profits from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple -from test.connector.fixture_celo import outputs as celo_outputs, TEST_ADDRESS, TEST_PASSWORD logging.basicConfig(level=logging.ERROR) diff --git a/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making.py b/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making.py index 2a96c68a88..1fcada0963 100644 --- a/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making.py +++ b/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making.py @@ -7,7 +7,7 @@ from nose.plugins.attrib import attr from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.data_type.limit_order import LimitOrder @@ -21,8 +21,8 @@ MarketEvent, OrderBookTradeEvent, OrderFilledEvent, - SellOrderCreatedEvent, SellOrderCompletedEvent, + SellOrderCreatedEvent, ) from hummingbot.strategy.cross_exchange_market_making import CrossExchangeMarketMakingStrategy from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_pair import CrossExchangeMarketPair diff --git a/test/hummingbot/strategy/dev_0_hello_world/test_dev_0_hello_world.py b/test/hummingbot/strategy/dev_0_hello_world/test_dev_0_hello_world.py index a2d22517ba..f1352a74e9 100644 --- a/test/hummingbot/strategy/dev_0_hello_world/test_dev_0_hello_world.py +++ b/test/hummingbot/strategy/dev_0_hello_world/test_dev_0_hello_world.py @@ -4,7 +4,7 @@ import pandas as pd from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode from hummingbot.strategy.dev_0_hello_world import HelloWorldStrategy diff --git a/test/hummingbot/strategy/dev_1_get_order_book/test_dev_1_get_order_book.py b/test/hummingbot/strategy/dev_1_get_order_book/test_dev_1_get_order_book.py index fa717337a8..e68313bdee 100644 --- a/test/hummingbot/strategy/dev_1_get_order_book/test_dev_1_get_order_book.py +++ b/test/hummingbot/strategy/dev_1_get_order_book/test_dev_1_get_order_book.py @@ -5,7 +5,7 @@ import pandas as pd from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode from hummingbot.strategy.dev_1_get_order_book import GetOrderBookStrategy diff --git a/test/hummingbot/strategy/dev_2_perform_trade/test_dev_2_perform_trade.py b/test/hummingbot/strategy/dev_2_perform_trade/test_dev_2_perform_trade.py index e18112e66d..1c015759a0 100644 --- a/test/hummingbot/strategy/dev_2_perform_trade/test_dev_2_perform_trade.py +++ b/test/hummingbot/strategy/dev_2_perform_trade/test_dev_2_perform_trade.py @@ -5,7 +5,7 @@ import pandas as pd from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode from hummingbot.core.data_type.common import OrderType, PriceType, TradeType from hummingbot.core.data_type.limit_order import LimitOrder diff --git a/test/hummingbot/strategy/dev_5_vwap/test_vwap.py b/test/hummingbot/strategy/dev_5_vwap/test_vwap.py index a2bfdd6218..7ae0673a27 100644 --- a/test/hummingbot/strategy/dev_5_vwap/test_vwap.py +++ b/test/hummingbot/strategy/dev_5_vwap/test_vwap.py @@ -6,18 +6,13 @@ import pandas as pd from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.data_type.limit_order import LimitOrder from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee from hummingbot.core.event.event_logger import EventLogger -from hummingbot.core.event.events import ( - BuyOrderCompletedEvent, - MarketEvent, - OrderFilledEvent, - SellOrderCompletedEvent -) +from hummingbot.core.event.events import BuyOrderCompletedEvent, MarketEvent, OrderFilledEvent, SellOrderCompletedEvent from hummingbot.strategy.dev_5_vwap import Dev5TwapTradeStrategy from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple diff --git a/test/hummingbot/strategy/dev_simple_trade/test_simple_trade.py b/test/hummingbot/strategy/dev_simple_trade/test_simple_trade.py index 0407ea5edd..817cd8bd4e 100644 --- a/test/hummingbot/strategy/dev_simple_trade/test_simple_trade.py +++ b/test/hummingbot/strategy/dev_simple_trade/test_simple_trade.py @@ -5,7 +5,7 @@ import pandas as pd from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.data_type.limit_order import LimitOrder @@ -16,7 +16,7 @@ MarketEvent, OrderCancelledEvent, OrderFilledEvent, - SellOrderCompletedEvent + SellOrderCompletedEvent, ) from hummingbot.strategy.dev_simple_trade.dev_simple_trade import SimpleTradeStrategy from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple diff --git a/test/hummingbot/strategy/liquidity_mining/test_liquidity_mining.py b/test/hummingbot/strategy/liquidity_mining/test_liquidity_mining.py index b522707865..5f3d17fff3 100644 --- a/test/hummingbot/strategy/liquidity_mining/test_liquidity_mining.py +++ b/test/hummingbot/strategy/liquidity_mining/test_liquidity_mining.py @@ -6,7 +6,7 @@ from hummingbot.client.hummingbot_application import HummingbotApplication from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode from hummingbot.core.data_type.common import TradeType from hummingbot.core.data_type.limit_order import LimitOrder diff --git a/test/hummingbot/strategy/perpetual_market_making/test_perpetual_market_making.py b/test/hummingbot/strategy/perpetual_market_making/test_perpetual_market_making.py index 5bd4441a2d..6c0865d71c 100644 --- a/test/hummingbot/strategy/perpetual_market_making/test_perpetual_market_making.py +++ b/test/hummingbot/strategy/perpetual_market_making/test_perpetual_market_making.py @@ -1,4 +1,5 @@ from decimal import Decimal +from test.mock.mock_perp_connector import MockPerpConnector from unittest import TestCase from unittest.mock import patch @@ -6,7 +7,7 @@ from hummingbot.connector.derivative.position import Position from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock from hummingbot.core.clock_mode import ClockMode from hummingbot.core.data_type.common import OrderType, PositionMode, PositionSide, PriceType, TradeType @@ -20,11 +21,10 @@ PositionModeChangeEvent, SellOrderCompletedEvent, ) -from hummingbot.strategy.data_types import Proposal, PriceSize +from hummingbot.strategy.data_types import PriceSize, Proposal from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple from hummingbot.strategy.perpetual_market_making import PerpetualMarketMakingStrategy from hummingbot.strategy.strategy_base import StrategyBase -from test.mock.mock_perp_connector import MockPerpConnector class PerpetualMarketMakingTests(TestCase): @@ -702,8 +702,8 @@ def test_status_text_when_no_open_positions_and_two_orders(self): self.assertEqual(1, len(self.strategy.active_sells)) expected_status = ("\n Markets:" - "\n Exchange Market Best Bid Best Ask Ref Price (MidPrice)" - "\n MockPerpConnector COINALPHA-HBOT 99.5 100.5 100" + "\n Exchange Market Best Bid Best Ask Ref Price (MidPrice)" + "\n mock_perp_connector COINALPHA-HBOT 99.5 100.5 100" "\n\n Assets:" "\n HBOT" "\n Total Balance 50000" @@ -730,8 +730,8 @@ def test_status_text_with_one_open_position_and_no_orders_alive(self): self.clock.backtest_til(self.start_timestamp + 1) expected_status = ("\n Markets:" - "\n Exchange Market Best Bid Best Ask Ref Price (MidPrice)" - "\n MockPerpConnector COINALPHA-HBOT 99.5 100.5 100" + "\n Exchange Market Best Bid Best Ask Ref Price (MidPrice)" + "\n mock_perp_connector COINALPHA-HBOT 99.5 100.5 100" "\n\n Assets:" "\n HBOT" "\n Total Balance 50000" diff --git a/test/hummingbot/strategy/pure_market_making/test_pmm.py b/test/hummingbot/strategy/pure_market_making/test_pmm.py index 2f1e7c7dac..7b51b6bcf9 100644 --- a/test/hummingbot/strategy/pure_market_making/test_pmm.py +++ b/test/hummingbot/strategy/pure_market_making/test_pmm.py @@ -1,12 +1,13 @@ import unittest from decimal import Decimal +from test.mock.mock_asset_price_delegate import MockAssetPriceDelegate from typing import List, Optional import pandas as pd from hummingbot.client.command.config_command import ConfigCommand from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode from hummingbot.core.data_type.common import PriceType, TradeType from hummingbot.core.data_type.limit_order import LimitOrder @@ -19,7 +20,6 @@ from hummingbot.strategy.order_book_asset_price_delegate import OrderBookAssetPriceDelegate from hummingbot.strategy.pure_market_making.inventory_cost_price_delegate import InventoryCostPriceDelegate from hummingbot.strategy.pure_market_making.pure_market_making import PureMarketMakingStrategy -from test.mock.mock_asset_price_delegate import MockAssetPriceDelegate # Update the orderbook so that the top bids and asks are lower than actual for a wider bid ask spread diff --git a/test/hummingbot/strategy/pure_market_making/test_pmm_ping_pong.py b/test/hummingbot/strategy/pure_market_making/test_pmm_ping_pong.py index e5ad60137f..9c273b10a2 100644 --- a/test/hummingbot/strategy/pure_market_making/test_pmm_ping_pong.py +++ b/test/hummingbot/strategy/pure_market_making/test_pmm_ping_pong.py @@ -5,19 +5,13 @@ import pandas as pd from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams -from hummingbot.core.clock import ( - Clock, - ClockMode -) +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange +from hummingbot.core.clock import Clock, ClockMode from hummingbot.core.data_type.common import TradeType from hummingbot.core.event.event_logger import EventLogger -from hummingbot.core.event.events import ( - MarketEvent, - OrderBookTradeEvent, -) +from hummingbot.core.event.events import MarketEvent, OrderBookTradeEvent from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple from hummingbot.strategy.pure_market_making.pure_market_making import PureMarketMakingStrategy -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange logging.basicConfig(level=logging.ERROR) diff --git a/test/hummingbot/strategy/pure_market_making/test_pmm_refresh_tolerance.py b/test/hummingbot/strategy/pure_market_making/test_pmm_refresh_tolerance.py index 7d6faf631f..dedbd75540 100644 --- a/test/hummingbot/strategy/pure_market_making/test_pmm_refresh_tolerance.py +++ b/test/hummingbot/strategy/pure_market_making/test_pmm_refresh_tolerance.py @@ -6,20 +6,14 @@ import pandas as pd from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams -from hummingbot.core.clock import ( - Clock, - ClockMode -) +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange +from hummingbot.core.clock import Clock, ClockMode from hummingbot.core.data_type.common import TradeType from hummingbot.core.data_type.order_book_row import OrderBookRow from hummingbot.core.event.event_logger import EventLogger -from hummingbot.core.event.events import ( - MarketEvent, - OrderBookTradeEvent, -) +from hummingbot.core.event.events import MarketEvent, OrderBookTradeEvent from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple from hummingbot.strategy.pure_market_making.pure_market_making import PureMarketMakingStrategy -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange logging.basicConfig(level=logging.ERROR) diff --git a/test/hummingbot/strategy/pure_market_making/test_pmm_take_if_cross.py b/test/hummingbot/strategy/pure_market_making/test_pmm_take_if_cross.py index 5a2a1d381b..193de936de 100644 --- a/test/hummingbot/strategy/pure_market_making/test_pmm_take_if_cross.py +++ b/test/hummingbot/strategy/pure_market_making/test_pmm_take_if_cross.py @@ -6,20 +6,16 @@ import pandas as pd from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode from hummingbot.core.data_type.common import TradeType from hummingbot.core.data_type.order_book import OrderBook from hummingbot.core.data_type.order_book_row import OrderBookRow from hummingbot.core.event.event_logger import EventLogger -from hummingbot.core.event.events import ( - MarketEvent, - OrderBookTradeEvent, -) +from hummingbot.core.event.events import MarketEvent, OrderBookTradeEvent from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple from hummingbot.strategy.order_book_asset_price_delegate import OrderBookAssetPriceDelegate from hummingbot.strategy.pure_market_making.pure_market_making import PureMarketMakingStrategy -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange - logging.basicConfig(level=logging.ERROR) diff --git a/test/hummingbot/strategy/spot_perpetual_arbitrage/test_spot_perpetual_arbitrage.py b/test/hummingbot/strategy/spot_perpetual_arbitrage/test_spot_perpetual_arbitrage.py index 4a17a828c6..800ec8a584 100644 --- a/test/hummingbot/strategy/spot_perpetual_arbitrage/test_spot_perpetual_arbitrage.py +++ b/test/hummingbot/strategy/spot_perpetual_arbitrage/test_spot_perpetual_arbitrage.py @@ -8,7 +8,7 @@ from hummingbot.connector.connector_base import ConnectorBase from hummingbot.connector.derivative.position import Position from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode from hummingbot.core.data_type.common import OrderType, PositionMode, PositionSide from hummingbot.core.event.event_logger import EventLogger @@ -289,8 +289,8 @@ def test_arbitrage_buy_spot_sell_perp(self): # asyncio.get_event_loop().run_until_complete(asyncio.sleep(0.01)) self.assertTrue(self._is_logged("INFO", "Arbitrage position opening opportunity found.")) self.assertTrue(self._is_logged("INFO", "Profitability (8.96%) is now above min_opening_arbitrage_pct.")) - self.assertTrue(self._is_logged("INFO", "Placing BUY order for 1 HBOT at MockPaperExchange at 100.500 price")) - self.assertTrue(self._is_logged("INFO", "Placing SELL order for 1 HBOT at MockPerpConnector at 109.500 price " + self.assertTrue(self._is_logged("INFO", "Placing BUY order for 1 HBOT at mock_paper_exchange at 100.500 price")) + self.assertTrue(self._is_logged("INFO", "Placing SELL order for 1 HBOT at mock_perp_connector at 109.500 price " "to OPEN position.")) placed_orders = self.strategy.tracked_market_orders self.assertEqual(2, len(placed_orders)) @@ -316,24 +316,24 @@ def test_arbitrage_buy_spot_sell_perp(self): status = asyncio.get_event_loop().run_until_complete(self.strategy.format_status()) expected_status = (""" Markets: - Exchange Market Sell Price Buy Price Mid Price - MockPaperExchange HBOT-USDT 99.5 100.5 100 - MockPerpConnector HBOT-USDT 109.5 110.5 110 + Exchange Market Sell Price Buy Price Mid Price + mock_paper_exchange HBOT-USDT 99.5 100.5 100 + mock_perp_connector HBOT-USDT 109.5 110.5 110 Positions: Symbol Type Entry Price Amount Leverage Unrealized PnL HBOT-USDT SHORT 109.5 -1 5 0 Assets: - Exchange Asset Total Balance Available Balance - 0 MockPaperExchange HBOT 5 5 - 1 MockPaperExchange USDT 500 500 - 2 MockPerpConnector HBOT 5 5 - 3 MockPerpConnector USDT 500 500 + Exchange Asset Total Balance Available Balance + 0 mock_paper_exchange HBOT 5 5 + 1 mock_paper_exchange USDT 500 500 + 2 mock_perp_connector HBOT 5 5 + 3 mock_perp_connector USDT 500 500 Opportunity: - buy at MockPaperExchange, sell at MockPerpConnector: 8.96% - sell at MockPaperExchange, buy at MockPerpConnector: -9.95%""") + buy at mock_paper_exchange, sell at mock_perp_connector: 8.96% + sell at mock_paper_exchange, buy at mock_perp_connector: -9.95%""") self.assertEqual(expected_status, status) @@ -387,8 +387,8 @@ def test_arbitrage_sell_spot_buy_perp_opening(self): # asyncio.get_event_loop().run_until_complete(asyncio.sleep(0.01)) self.assertTrue(self._is_logged("INFO", "Arbitrage position opening opportunity found.")) self.assertTrue(self._is_logged("INFO", "Profitability (9.94%) is now above min_opening_arbitrage_pct.")) - self.assertTrue(self._is_logged("INFO", "Placing SELL order for 1 HBOT at MockPaperExchange at 99.5000 price")) - self.assertTrue(self._is_logged("INFO", "Placing BUY order for 1 HBOT at MockPerpConnector at 90.5000 price to " + self.assertTrue(self._is_logged("INFO", "Placing SELL order for 1 HBOT at mock_paper_exchange at 99.5000 price")) + self.assertTrue(self._is_logged("INFO", "Placing BUY order for 1 HBOT at mock_perp_connector at 90.5000 price to " "OPEN position.")) placed_orders = self.strategy.tracked_market_orders self.assertEqual(2, len(placed_orders)) diff --git a/test/hummingbot/strategy/test_market_trading_pair_tuple.py b/test/hummingbot/strategy/test_market_trading_pair_tuple.py index b550081889..470a9f28e1 100644 --- a/test/hummingbot/strategy/test_market_trading_pair_tuple.py +++ b/test/hummingbot/strategy/test_market_trading_pair_tuple.py @@ -7,7 +7,7 @@ import pandas as pd from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode from hummingbot.core.data_type.common import OrderType, PriceType, TradeType from hummingbot.core.data_type.limit_order import LimitOrder diff --git a/test/hummingbot/strategy/test_order_tracker.py b/test/hummingbot/strategy/test_order_tracker.py index 1e55a9be46..c497e90c83 100644 --- a/test/hummingbot/strategy/test_order_tracker.py +++ b/test/hummingbot/strategy/test_order_tracker.py @@ -2,19 +2,16 @@ import time import unittest from decimal import Decimal -from typing import ( - List, - Union, -) +from typing import List, Union import pandas as pd +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode from hummingbot.core.data_type.limit_order import LimitOrder from hummingbot.core.data_type.market_order import MarketOrder from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple from hummingbot.strategy.order_tracker import OrderTracker -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange class OrderTrackerUnitTests(unittest.TestCase): diff --git a/test/hummingbot/strategy/test_script_strategy_base.py b/test/hummingbot/strategy/test_script_strategy_base.py index 9a3e2f8661..1564d5283f 100644 --- a/test/hummingbot/strategy/test_script_strategy_base.py +++ b/test/hummingbot/strategy/test_script_strategy_base.py @@ -5,7 +5,7 @@ import pandas as pd from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock from hummingbot.core.clock_mode import ClockMode from hummingbot.core.event.events import OrderType diff --git a/test/hummingbot/strategy/test_strategy_base.py b/test/hummingbot/strategy/test_strategy_base.py index 1c1378bfa4..e4da7a67d3 100644 --- a/test/hummingbot/strategy/test_strategy_base.py +++ b/test/hummingbot/strategy/test_strategy_base.py @@ -5,28 +5,19 @@ import unittest.mock from datetime import datetime from decimal import Decimal -from typing import ( - Any, - Dict, - List, - Union, - Tuple, -) +from typing import Any, Dict, List, Tuple, Union from hummingbot.client.hummingbot_application import HummingbotApplication from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.in_flight_order_base import InFlightOrderBase +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.data_type.limit_order import LimitOrder from hummingbot.core.data_type.market_order import MarketOrder -from hummingbot.core.event.events import ( - MarketEvent, - OrderFilledEvent, -) +from hummingbot.core.event.events import MarketEvent, OrderFilledEvent from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple from hummingbot.strategy.order_tracker import OrderTracker from hummingbot.strategy.strategy_base import StrategyBase -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange ms_logger = None diff --git a/test/hummingbot/strategy/test_strategy_py_base.py b/test/hummingbot/strategy/test_strategy_py_base.py index d7f51daf85..06e8475e71 100644 --- a/test/hummingbot/strategy/test_strategy_py_base.py +++ b/test/hummingbot/strategy/test_strategy_py_base.py @@ -5,6 +5,7 @@ from decimal import Decimal from typing import Union +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.data_type.limit_order import LimitOrder from hummingbot.core.data_type.market_order import MarketOrder @@ -22,7 +23,6 @@ ) from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple from hummingbot.strategy.strategy_py_base import StrategyPyBase -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange class MockPyStrategy(StrategyPyBase): diff --git a/test/hummingbot/strategy/twap/test_twap.py b/test/hummingbot/strategy/twap/test_twap.py index 0c7fb3d41e..c8d2228c9b 100644 --- a/test/hummingbot/strategy/twap/test_twap.py +++ b/test/hummingbot/strategy/twap/test_twap.py @@ -7,7 +7,7 @@ import pandas as pd from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.data_type.limit_order import LimitOrder @@ -387,14 +387,14 @@ def test_status_after_first_order_filled(self): " Order size: 1 COINALPHA\n" " Execution type: run continuously\n\n" " Markets:\n" - " Exchange Market Best Bid Price Best Ask Price Mid Price\n" - " 0 MockPaperExchange COINALPHA-WETH 99.5 100.5 100\n\n" + " Exchange Market Best Bid Price Best Ask Price Mid Price\n" + " 0 mock_paper_exchange COINALPHA-WETH 99.5 100.5 100\n\n" " Assets:\n" - " Exchange Asset Total Balance Available Balance\n" - " 0 MockPaperExchange COINALPHA " + " Exchange Asset Total Balance Available Balance\n" + " 0 mock_paper_exchange COINALPHA " f"{base_balance:.2f} " f"{available_base_balance:.2f}\n" - " 1 MockPaperExchange WETH " + " 1 mock_paper_exchange WETH " f"{quote_balance:.2f} " f"{available_quote_balance:.2f}\n\n" " No active maker orders.\n\n" @@ -408,14 +408,14 @@ def test_status_after_first_order_filled(self): " Order size: 1.67 COINALPHA\n" " Execution type: run continuously\n\n" " Markets:\n" - " Exchange Market Best Bid Price Best Ask Price Mid Price\n" - " 0 MockPaperExchange COINALPHA-WETH 99.5 100.5 100\n\n" + " Exchange Market Best Bid Price Best Ask Price Mid Price\n" + " 0 mock_paper_exchange COINALPHA-WETH 99.5 100.5 100\n\n" " Assets:\n" - " Exchange Asset Total Balance Available Balance\n" - " 0 MockPaperExchange COINALPHA " + " Exchange Asset Total Balance Available Balance\n" + " 0 mock_paper_exchange COINALPHA " f"{base_balance:.2f} " f"{available_base_balance:.2f}\n" - " 1 MockPaperExchange WETH " + " 1 mock_paper_exchange WETH " f"{quote_balance:.2f} " f"{available_quote_balance:.2f}\n\n" " Active orders:\n" diff --git a/test/mock/mock_perp_connector.py b/test/mock/mock_perp_connector.py index 45913072b0..a22f866663 100644 --- a/test/mock/mock_perp_connector.py +++ b/test/mock/mock_perp_connector.py @@ -2,7 +2,7 @@ from typing import Optional from hummingbot.connector.derivative.perpetual_budget_checker import PerpetualBudgetChecker -from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange +from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.connector.perpetual_trading import PerpetualTrading from hummingbot.core.data_type.common import OrderType, PositionAction, PositionMode, TradeType from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee, TradeFeeSchema @@ -28,7 +28,7 @@ def supported_position_modes(self): @property def name(self): - return "MockPerpConnector" + return "mock_perp_connector" @property def budget_checker(self) -> PerpetualBudgetChecker: From b4e613e1365d55674df95232434c5c9f86337259 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 18 May 2022 12:45:20 +0300 Subject: [PATCH 086/152] (fix) Addressing the issue of strategies not migrating --- hummingbot/client/config/conf_migration.py | 29 ++++++++++- hummingbot/client/ui/__init__.py | 56 ++++++++++++++++++---- 2 files changed, 73 insertions(+), 12 deletions(-) diff --git a/hummingbot/client/config/conf_migration.py b/hummingbot/client/config/conf_migration.py index 46b33fb37f..487cc91d42 100644 --- a/hummingbot/client/config/conf_migration.py +++ b/hummingbot/client/config/conf_migration.py @@ -4,7 +4,7 @@ import shutil from os import DirEntry, scandir from os.path import exists, join -from typing import List, cast +from typing import Dict, List, cast import yaml @@ -34,6 +34,17 @@ def migrate_configs(secrets_manager: BaseSecretsManager) -> List[str]: return errors +def migrate_strategies_only() -> List[str]: + logging.getLogger().info("Starting strategies conf migration.") + errors = backup_existing_dir() + if len(errors) == 0: + migrate_strategy_confs_paths() + logging.getLogger().info("\nConf migration done.") + else: + logging.getLogger().error("\nConf migration failed.") + return errors + + def backup_existing_dir() -> List[str]: errors = [] if conf_dir_path.exists(): @@ -59,12 +70,26 @@ def migrate_strategy_confs_paths(): if child.is_file() and child.name.endswith(".yml"): with open(str(child), "r") as f: conf = yaml.safe_load(f) - if "strategy" in conf and "exchange" in conf: + if "strategy" in conf and _has_connector_field(conf): new_path = strategies_conf_dir_path / child.name child.rename(new_path) logging.getLogger().info(f"Migrated conf for {conf['strategy']}") +def _has_connector_field(conf: Dict) -> bool: + return ( + "exchange" in conf + or "connector_1" in conf # amm arb + or "primary_market" in conf # arbitrage + or "secondary_exchange" in conf # celo arb + or "maker_market" in conf # XEMM + or "market" in conf # dev simple trade + or "maker_exchange" in conf # hedge + or "spot_connector" in conf # spot-perp arb + or "connector" in conf # twap + ) + + def migrate_connector_confs(secrets_manager: BaseSecretsManager): logging.getLogger().info("\nMigrating connector secure keys...") errors = [] diff --git a/hummingbot/client/ui/__init__.py b/hummingbot/client/ui/__init__.py index f5d231adfe..f631e0e260 100644 --- a/hummingbot/client/ui/__init__.py +++ b/hummingbot/client/ui/__init__.py @@ -6,7 +6,7 @@ from prompt_toolkit.shortcuts import input_dialog, message_dialog from prompt_toolkit.styles import Style -from hummingbot.client.config.conf_migration import migrate_configs +from hummingbot.client.config.conf_migration import migrate_configs, migrate_strategies_only from hummingbot.client.config.config_crypt import BaseSecretsManager, store_password_verification from hummingbot.client.config.global_config_map import color_config_map from hummingbot.client.config.security import Security @@ -59,6 +59,7 @@ def login_prompt(secrets_manager_cls: Type[BaseSecretsManager]) -> Optional[Base else: secrets_manager = secrets_manager_cls(password) store_password_verification(secrets_manager) + migrate_strategies_only_prompt() else: password = input_dialog( title="Welcome back to Hummingbot", @@ -115,31 +116,66 @@ def migrate_configs_prompt(secrets_manager_cls: Type[BaseSecretsManager]) -> Bas secrets_manager = secrets_manager_cls(password) errors = migrate_configs(secrets_manager) if len(errors) != 0: - errors_str = "\n ".join(errors) + _migration_errors_dialog(errors) + else: message_dialog( - title='Configs Migration Errors', - text=f""" + title='Configs Migration Success', + text=""" - CONFIGS MIGRATION ERRORS: + CONFIGS MIGRATION SUCCESS: - {errors_str} + The migration process was completed successfully. - """, + """, style=dialog_style).run() + return secrets_manager + + +def migrate_strategies_only_prompt(): + message_dialog( + title='Strategies Configs Migration', + text=""" + + + STRATEGIES CONFIGS MIGRATION: + + We have recently refactored the way hummingbot handles configurations. + We will now attempt to migrate any legacy strategy config files + to the new format. + + """, + style=dialog_style).run() + errors = migrate_strategies_only() + if len(errors) != 0: + _migration_errors_dialog(errors) else: message_dialog( - title='Configs Migration Success', + title='Strategies Configs Migration Success', text=""" - CONFIGS MIGRATION SUCCESS: + STRATEGIES CONFIGS MIGRATION SUCCESS: The migration process was completed successfully. """, style=dialog_style).run() - return secrets_manager + + +def _migration_errors_dialog(errors): + errors_str = "\n ".join(errors) + message_dialog( + title='Configs Migration Errors', + text=f""" + + + CONFIGS MIGRATION ERRORS: + + {errors_str} + + """, + style=dialog_style).run() def show_welcome(): From 1619fba991561ba37bcb502ec1cad5bb6d3edfe1 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 18 May 2022 17:01:48 +0700 Subject: [PATCH 087/152] Update hummingbot/connector/derivative/binance_perpetual/binance_perpetual_utils.py Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- .../derivative/binance_perpetual/binance_perpetual_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_utils.py b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_utils.py index 71221b354b..a44f0dda56 100644 --- a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_utils.py +++ b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_utils.py @@ -89,7 +89,7 @@ class BinancePerpetualTestnetConfigMap(BaseConnectorConfigMap): binance_perpetual_testnet_api_secret: SecretStr = Field( default=..., client_data=ClientFieldData( - prompt=lambda cm: "Enter your Binance Perpetual testnet API secret", + prompt=lambda cm: "Enter your Binance Perpetual testnet API secret", is_secure=True, is_connect_key=True, prompt_on_new=True, From 017d1e8099cf37d2867185c5bc0a1d37718748ce Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 18 May 2022 17:02:39 +0700 Subject: [PATCH 088/152] Update hummingbot/client/config/conf_migration.py Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- hummingbot/client/config/conf_migration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/client/config/conf_migration.py b/hummingbot/client/config/conf_migration.py index 487cc91d42..11f69c4886 100644 --- a/hummingbot/client/config/conf_migration.py +++ b/hummingbot/client/config/conf_migration.py @@ -53,7 +53,7 @@ def backup_existing_dir() -> List[str]: errors = [ ( f"\nBackup path {backup_path} already exists." - f"\nThe migration script cannot backup you exiting" + f"\nThe migration script cannot backup you existing" f"\nconf files without overwriting that directory." f"\nPlease remove it and run the script again." ) From 9f286e3f5f875e4ae908a4cf89980e13b97c0a4b Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 18 May 2022 13:03:50 +0300 Subject: [PATCH 089/152] (fix) Addresses @aarmoa's PR comments --- hummingbot/client/command/connect_command.py | 43 +++++++++----------- hummingbot/client/ui/__init__.py | 3 +- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/hummingbot/client/command/connect_command.py b/hummingbot/client/command/connect_command.py index f75a9a6d91..26b63f50b2 100644 --- a/hummingbot/client/command/connect_command.py +++ b/hummingbot/client/command/connect_command.py @@ -50,8 +50,6 @@ async def connect_exchange(self, # type: HummingbotApplication connector_config = ClientConfigAdapter(CELO_KEYS) else: connector_config = ClientConfigAdapter(AllConnectorSettings.get_connector_config_keys(connector_name)) - to_connect = True - previous_keys = None if Security.connector_config_file_exists(connector_name): await Security.wait_til_decryption_done() api_key_config = [ @@ -68,30 +66,27 @@ async def connect_exchange(self, # type: HummingbotApplication if self.app.to_stop_config: self.app.to_stop_config = False return - if answer.lower() not in ("yes", "y"): - to_connect = False - else: + if answer.lower() in ("yes", "y"): previous_keys = Security.api_keys(connector_name) - if to_connect: - await self.prompt_for_model_config(connector_config) - self.app.change_prompt(prompt=">>> ") - if self.app.to_stop_config: - self.app.to_stop_config = False - return - Security.update_secure_config(connector_config) - if connector_name == "celo": - err_msg = await self.validate_n_connect_celo(to_reconnect=True) - else: - err_msg = await self.validate_n_connect_connector(connector_name) - if err_msg is None: - self.notify(f"\nYou are now connected to {connector_name}.") - else: - self.notify(f"\nError: {err_msg}") - if previous_keys is not None: - previous_config = ClientConfigAdapter(connector_config.hb_config.__class__(**previous_keys)) - Security.update_secure_config(previous_config) + await self.prompt_for_model_config(connector_config) + self.app.change_prompt(prompt=">>> ") + if self.app.to_stop_config: + self.app.to_stop_config = False + return + Security.update_secure_config(connector_config) + if connector_name == "celo": + err_msg = await self.validate_n_connect_celo(to_reconnect=True) else: - Security.remove_secure_config(connector_name) + err_msg = await self.validate_n_connect_connector(connector_name) + if err_msg is None: + self.notify(f"\nYou are now connected to {connector_name}.") + else: + self.notify(f"\nError: {err_msg}") + if previous_keys is not None: + previous_config = ClientConfigAdapter(connector_config.hb_config.__class__(**previous_keys)) + Security.update_secure_config(previous_config) + else: + Security.remove_secure_config(connector_name) self.placeholder_mode = False self.app.hide_input = False self.app.change_prompt(prompt=">>> ") diff --git a/hummingbot/client/ui/__init__.py b/hummingbot/client/ui/__init__.py index f631e0e260..86f8096f73 100644 --- a/hummingbot/client/ui/__init__.py +++ b/hummingbot/client/ui/__init__.py @@ -6,13 +6,14 @@ from prompt_toolkit.shortcuts import input_dialog, message_dialog from prompt_toolkit.styles import Style +from hummingbot import root_path from hummingbot.client.config.conf_migration import migrate_configs, migrate_strategies_only from hummingbot.client.config.config_crypt import BaseSecretsManager, store_password_verification from hummingbot.client.config.global_config_map import color_config_map from hummingbot.client.config.security import Security from hummingbot.client.settings import CONF_DIR_PATH -sys.path.insert(0, realpath(join(__file__, "../../../"))) +sys.path.insert(0, str(root_path())) with open(realpath(join(dirname(__file__), '../../VERSION'))) as version_file: From 5f7a91b067fe2fc5dbe996ad01eebf1358ce78da Mon Sep 17 00:00:00 2001 From: mhrvth Date: Wed, 18 May 2022 21:47:12 +0200 Subject: [PATCH 090/152] (refactor) config map --- hummingbot/client/config/config_data_types.py | 104 ++++++- ...aneda_market_making_config_map_pydantic.py | 8 +- .../cross_exchange_market_making.pxd | 17 +- .../cross_exchange_market_making.pyx | 186 ++++++------ ...cross_exchange_market_making_config_map.py | 239 --------------- ...hange_market_making_config_map_pydantic.py | 278 ++++++++++++++++++ .../cross_exchange_market_making/start.py | 64 +--- ...aneda_market_making_config_map_pydantic.py | 14 +- .../test_avellaneda_market_making_start.py | 8 +- .../test_cross_exchange_market_making.py | 256 ++++++++++------ ...cross_exchange_market_making_config_map.py | 61 ---- ...test_cross_exchange_market_making_start.py | 32 +- 12 files changed, 698 insertions(+), 569 deletions(-) delete mode 100644 hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py create mode 100644 hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py delete mode 100644 test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_config_map.py diff --git a/hummingbot/client/config/config_data_types.py b/hummingbot/client/config/config_data_types.py index dbd1c0721e..a8bf9c605d 100644 --- a/hummingbot/client/config/config_data_types.py +++ b/hummingbot/client/config/config_data_types.py @@ -69,8 +69,8 @@ def validate_strategy(cls, v: str): return v -class BaseTradingStrategyConfigMap(BaseStrategyConfigMap): - exchange: ClientConfigEnum( +class BaseTradingStrategyMakerConfigMap(BaseStrategyConfigMap): + market: ClientConfigEnum( value="Exchanges", # noqa: F821 names={e: e for e in AllConnectorSettings.get_connector_settings().keys()}, type=str, @@ -82,17 +82,17 @@ class BaseTradingStrategyConfigMap(BaseStrategyConfigMap): prompt_on_new=True, ), ) - market: str = Field( + trading_pair: str = Field( default=..., description="The trading pair.", client_data=ClientFieldData( - prompt=lambda mi: BaseTradingStrategyConfigMap.maker_trading_pair_prompt(mi), + prompt=lambda mi: BaseTradingStrategyMakerConfigMap.trading_pair_prompt(mi), prompt_on_new=True, ), ) @classmethod - def maker_trading_pair_prompt(cls, model_instance: 'BaseTradingStrategyConfigMap') -> str: + def trading_pair_prompt(cls, model_instance: 'BaseTradingStrategyMakerConfigMap') -> str: exchange = model_instance.exchange example = AllConnectorSettings.get_example_pairs().get(exchange) return ( @@ -100,7 +100,7 @@ def maker_trading_pair_prompt(cls, model_instance: 'BaseTradingStrategyConfigMap f" {exchange}{f' (e.g. {example})' if example else ''}" ) - @validator("exchange", pre=True) + @validator("market", pre=True) def validate_exchange(cls, v: str): """Used for client-friendly error output.""" ret = validate_exchange(v) @@ -113,10 +113,100 @@ def validate_exchange(cls, v: str): ) return v - @validator("market", pre=True) + @validator("trading_pair", pre=True) def validate_exchange_trading_pair(cls, v: str, values: Dict): exchange = values.get("exchange") ret = validate_market_trading_pair(exchange, v) if ret is not None: raise ValueError(ret) return v + + +class BaseTradingStrategyMakerTakerConfigMap(BaseStrategyConfigMap): + market_maker: str = Field( + default=..., + description="", + client_data=ClientFieldData( + prompt=lambda mi: "Enter your maker spot connector", + prompt_on_new=True, + ), + ) + market_taker: str = Field( + default=..., + description="", + client_data=ClientFieldData( + prompt=lambda mi: "Enter your taker spot connector", + prompt_on_new=True, + ), + ) + trading_pair_maker: str = Field( + default=..., + description="", + client_data=ClientFieldData( + prompt=lambda mi: BaseTradingStrategyMakerTakerConfigMap.trading_pair_prompt(mi, True), + prompt_on_new=True, + ), + ) + trading_pair_taker: str = Field( + default=..., + description="", + client_data=ClientFieldData( + prompt=lambda mi: BaseTradingStrategyMakerTakerConfigMap.trading_pair_prompt(mi, False), + prompt_on_new=True, + ), + ) + + @classmethod + def trading_pair_prompt(cls, model_instance: 'BaseTradingStrategyMakerTakerConfigMap', is_maker: bool) -> str: + if is_maker: + exchange = model_instance.market_maker + example = AllConnectorSettings.get_example_pairs().get(exchange) + market_type = "maker" + else: + exchange = model_instance.market_taker + example = AllConnectorSettings.get_example_pairs().get(exchange) + market_type = "taker" + return ( + f"Enter the token trading pair you would like to trade on {market_type} market:" + f" {exchange}{f' (e.g. {example})' if example else ''}" + ) + + @validator( + "market_maker", + "market_taker", + pre=True + ) + def validate_exchange(cls, v: str, values: Dict, config: BaseModel.Config, field: Field): + """Used for client-friendly error output.""" + ret = validate_exchange(v) + if ret is not None: + raise ValueError(ret) + if field.name == "trading_pair_maker": + cls.__fields__["market_maker"].type_ = ClientConfigEnum( # rebuild the exchanges enum + value="Exchanges", # noqa: F821 + names={e: e for e in AllConnectorSettings.get_connector_settings().keys()}, + type=str, + ) + if field.name == "trading_pair_taker": + cls.__fields__["market_taker"].type_ = ClientConfigEnum( # rebuild the exchanges enum + value="Exchanges", # noqa: F821 + names={e: e for e in AllConnectorSettings.get_connector_settings().keys()}, + type=str, + ) + return v + + @validator( + "trading_pair_maker", + "trading_pair_taker", + pre=True, + ) + def validate_exchange_trading_pair(cls, v: str, values: Dict, config: BaseModel.Config, field: Field): + if field.name == "trading_pair_maker": + exchange = values.get("market_maker") + ret = validate_market_trading_pair(exchange, v) + if field.name == "trading_pair_taker": + exchange = values.get("market_taker") + ret = validate_market_trading_pair(exchange, v) + if ret is not None: + raise ValueError(ret) + return v diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py index 5cb6115bc0..ab11a0015d 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -4,7 +4,11 @@ from pydantic import Field, root_validator, validator -from hummingbot.client.config.config_data_types import BaseClientModel, BaseTradingStrategyConfigMap, ClientFieldData +from hummingbot.client.config.config_data_types import ( + BaseClientModel, + BaseTradingStrategyMakerConfigMap, + ClientFieldData, +) from hummingbot.client.config.config_validators import ( validate_bool, validate_datetime_iso_string, @@ -170,7 +174,7 @@ class Config: } -class AvellanedaMarketMakingConfigMap(BaseTradingStrategyConfigMap): +class AvellanedaMarketMakingConfigMap(BaseTradingStrategyMakerConfigMap): strategy: str = Field(default="avellaneda_market_making", client_data=None) execution_timeframe_mode: Union[InfiniteModel, FromDateToDateModel, DailyBetweenTimesModel] = Field( default=..., diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pxd b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pxd index 6d5ff130b5..e3174a7a59 100755 --- a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pxd +++ b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pxd @@ -8,33 +8,20 @@ from .order_id_market_pair_tracker cimport OrderIDMarketPairTracker cdef class CrossExchangeMarketMakingStrategy(StrategyBase): cdef: + object _config_map set _maker_markets set _taker_markets bint _all_markets_ready - bint _active_order_canceling bint _adjust_orders_enabled dict _anti_hysteresis_timers - object _min_profitability - object _order_size_taker_volume_factor - object _order_size_taker_balance_factor - object _order_size_portfolio_ratio_limit - object _order_amount - object _cancel_order_threshold - object _top_depth_tolerance - double _anti_hysteresis_duration - double _status_report_interval double _last_timestamp - double _limit_order_min_expiration + double _status_report_interval dict _order_fill_buy_events dict _order_fill_sell_events dict _suggested_price_samples dict _market_pairs int64_t _logging_options OrderIDMarketPairTracker _market_pair_tracker - bint _use_oracle_conversion_rate - object _taker_to_maker_base_conversion_rate - object _taker_to_maker_quote_conversion_rate - object _slippage_buffer bint _hb_app_notification list _maker_order_ids double _last_conv_rates_logged diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pyx b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pyx index 1a7d86366b..752f7e63d4 100755 --- a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pyx +++ b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pyx @@ -28,6 +28,10 @@ from hummingbot.strategy.strategy_base import StrategyBase from .cross_exchange_market_pair import CrossExchangeMarketPair from .order_id_market_pair_tracker import OrderIDMarketPairTracker +from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_making_config_map_pydantic import ( + CrossExchangeMarketMakingConfigMap, +) + NaN = float("nan") s_decimal_zero = Decimal(0) s_decimal_nan = Decimal("nan") @@ -58,55 +62,21 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): return s_logger def init_params(self, + config_map: CrossExchangeMarketMakingConfigMap, market_pairs: List[CrossExchangeMarketPair], - min_profitability: Decimal, - order_amount: Optional[Decimal] = Decimal("0.0"), - order_size_taker_volume_factor: Decimal = Decimal("0.25"), - order_size_taker_balance_factor: Decimal = Decimal("0.995"), - order_size_portfolio_ratio_limit: Decimal = Decimal("0.1667"), - limit_order_min_expiration: float = 130.0, - adjust_order_enabled: bool = True, - anti_hysteresis_duration: float = 60.0, - active_order_canceling: bint = True, - cancel_order_threshold: Decimal = Decimal("0.05"), - top_depth_tolerance: Decimal = Decimal(0), - logging_options: int = OPTION_LOG_ALL, status_report_interval: float = 900, - use_oracle_conversion_rate: bool = False, - taker_to_maker_base_conversion_rate: Decimal = Decimal("1"), - taker_to_maker_quote_conversion_rate: Decimal = Decimal("1"), - slippage_buffer: Decimal = Decimal("0.05"), + logging_options: int = OPTION_LOG_ALL, hb_app_notification: bool = False ): """ Initializes a cross exchange market making strategy object. + :param config_map: Strategy configuration map :param market_pairs: list of cross exchange market pairs - :param min_profitability: minimum profitability ratio threshold, for actively canceling unprofitable orders - :param order_amount: override the limit order trade size, in base asset unit - :param order_size_taker_volume_factor: maximum size limit of new limit orders, in terms of ratio of hedge-able - volume on taker side - :param order_size_taker_balance_factor: maximum size limit of new limit orders, in terms of ratio of asset - balance available for hedging trade on taker side - :param order_size_portfolio_ratio_limit: maximum size limit of new limit orders, in terms of ratio of total - portfolio value on both maker and taker markets - :param limit_order_min_expiration: amount of time after which limit order will expire to be used alongside - cancel_order_threshold - :param cancel_order_threshold: if active order cancelation is disabled, the hedging loss ratio required for the - strategy to force an order cancelation - :param active_order_canceling: True if active order cancelation is enabled, False if disabled - :param anti_hysteresis_duration: the minimum amount of time interval between adjusting limit order prices :param logging_options: bit field for what types of logging to enable in this strategy object - :param status_report_interval: what is the time interval between outputting new network warnings - :param slippage_buffer: Buffer added to the price to account for slippage for taker orders + :param hb_app_notification: """ - if len(market_pairs) < 0: - raise ValueError(f"market_pairs must not be empty.") - if not 0 <= order_size_taker_volume_factor <= 1: - raise ValueError(f"order_size_taker_volume_factor must be between 0 and 1.") - if not 0 <= order_size_taker_balance_factor <= 1: - raise ValueError(f"order_size_taker_balance_factor must be between 0 and 1.") - + self._config_map = config_map self._market_pairs = { (market_pair.maker.market, market_pair.maker.trading_pair): market_pair for market_pair in market_pairs @@ -114,29 +84,15 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): self._maker_markets = set([market_pair.maker.market for market_pair in market_pairs]) self._taker_markets = set([market_pair.taker.market for market_pair in market_pairs]) self._all_markets_ready = False - self._min_profitability = min_profitability - self._order_size_taker_volume_factor = order_size_taker_volume_factor - self._order_size_taker_balance_factor = order_size_taker_balance_factor - self._order_amount = order_amount - self._order_size_portfolio_ratio_limit = order_size_portfolio_ratio_limit - self._top_depth_tolerance = top_depth_tolerance - self._cancel_order_threshold = cancel_order_threshold + self._anti_hysteresis_timers = {} self._order_fill_buy_events = {} self._order_fill_sell_events = {} self._suggested_price_samples = {} - self._active_order_canceling = active_order_canceling - self._anti_hysteresis_duration = anti_hysteresis_duration self._logging_options = logging_options self._last_timestamp = 0 - self._limit_order_min_expiration = limit_order_min_expiration self._status_report_interval = status_report_interval self._market_pair_tracker = OrderIDMarketPairTracker() - self._adjust_orders_enabled = adjust_order_enabled - self._use_oracle_conversion_rate = use_oracle_conversion_rate - self._taker_to_maker_base_conversion_rate = taker_to_maker_base_conversion_rate - self._taker_to_maker_quote_conversion_rate = taker_to_maker_quote_conversion_rate - self._slippage_buffer = slippage_buffer self._last_conv_rates_logged = 0 self._hb_app_notification = hb_app_notification @@ -148,11 +104,67 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): @property def order_amount(self): - return self._order_amount + return self._config_map.order_amount @property def min_profitability(self): - return self._min_profitability + return self._config_map.min_profitability / Decimal("100") + + @property + def order_size_taker_volume_factor(self): + return self._config_map.order_size_taker_volume_factor / Decimal("100") + + @property + def order_size_taker_balance_factor(self): + return self._config_map.order_size_taker_balance_factor / Decimal("100") + + @property + def order_size_portfolio_ratio_limit(self): + return self._config_map.order_size_portfolio_ratio_limit / Decimal("100") + + @property + def top_depth_tolerance(self): + return self._config_map.top_depth_tolerance + + @property + def cancel_order_threshold(self): + return self._config_map.cancel_order_threshold / Decimal("100") + + @property + def active_order_canceling(self): + return self._config_map.active_order_canceling + + @property + def anti_hysteresis_duration(self): + return self._config_map.anti_hysteresis_duration + + @property + def limit_order_min_expiration(self): + return self._config_map.limit_order_min_expiration + + @property + def status_report_interval(self): + return self._status_report_interval + + @property + def adjust_order_enabled(self): + return self._config_map.adjust_order_enabled + + @property + def use_oracle_conversion_rate(self): + return self._config_map.use_oracle_conversion_rate + + @property + def taker_to_maker_base_conversion_rate(self): + return self._config_map.taker_to_maker_base_conversion_rate + + @property + def taker_to_maker_quote_conversion_rate(self): + return self._config_map.taker_to_maker_quote_conversion_rate + + @property + def slippage_buffer(self): + return self._config_map.slippage_buffer / Decimal("100") @property def active_limit_orders(self) -> List[Tuple[ExchangeBase, LimitOrder]]: @@ -189,21 +201,21 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): market_pairs = list(self._market_pairs.values())[0] quote_pair = f"{market_pairs.taker.quote_asset}-{market_pairs.maker.quote_asset}" quote_rate_source = "fixed" - if self._use_oracle_conversion_rate: + if self.use_oracle_conversion_rate: if market_pairs.taker.quote_asset != market_pairs.maker.quote_asset: quote_rate_source = RateOracle.source.name quote_rate = RateOracle.get_instance().rate(quote_pair) else: - quote_rate = self._taker_to_maker_quote_conversion_rate + quote_rate = self.taker_to_maker_quote_conversion_rate base_rate = Decimal("1") base_pair = f"{market_pairs.taker.base_asset}-{market_pairs.maker.base_asset}" base_rate_source = "fixed" - if self._use_oracle_conversion_rate: + if self.use_oracle_conversion_rate: if market_pairs.taker.base_asset != market_pairs.maker.base_asset: base_rate_source = RateOracle.source.name base_rate = RateOracle.get_instance().rate(base_pair) else: - base_rate = self._taker_to_maker_base_conversion_rate + base_rate = self.taker_to_maker_base_conversion_rate return quote_pair, quote_rate_source, quote_rate, base_pair, base_rate_source, base_rate def log_conversion_rates(self): @@ -325,8 +337,8 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): StrategyBase.c_tick(self, timestamp) cdef: - int64_t current_tick = (timestamp // self._status_report_interval) - int64_t last_tick = (self._last_timestamp // self._status_report_interval) + int64_t current_tick = (timestamp // self.status_report_interval) + int64_t last_tick = (self._last_timestamp // self.status_report_interval) bint should_report_warnings = ((current_tick > last_tick) and (self._logging_options & self.OPTION_LOG_STATUS_REPORT)) list active_limit_orders = self.active_limit_orders @@ -443,7 +455,7 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): if not self.c_check_if_still_profitable(market_pair, active_order, current_hedging_price): continue - if not self._active_order_canceling: + if not self.active_order_canceling: continue # See if I still have enough balance on my wallet to fill the order on maker market, and to hedge the @@ -461,7 +473,7 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): # If order adjustment is needed in the next tick, set the anti-hysteresis timer s.t. the next order adjustment # for the same pair wouldn't happen within the time limit. if need_adjust_order: - self._anti_hysteresis_timers[market_pair] = self._current_timestamp + self._anti_hysteresis_duration + self._anti_hysteresis_timers[market_pair] = self._current_timestamp + self.anti_hysteresis_duration # If there's both an active bid and ask, then there's no need to think about making new limit orders. if has_active_bid and has_active_ask: @@ -672,7 +684,7 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): hedged_order_quantity = min( buy_fill_quantity, taker_market.c_get_available_balance(market_pair.taker.base_asset) * - self._order_size_taker_balance_factor + self.order_size_taker_balance_factor ) quantized_hedge_amount = taker_market.c_quantize_order_amount(taker_trading_pair, Decimal(hedged_order_quantity)) taker_top = taker_market.c_get_price(taker_trading_pair, False) @@ -683,7 +695,7 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): ).result_price self.log_with_clock(logging.INFO, f"Calculated by HB order_price: {order_price}") - order_price *= Decimal("1") - self._slippage_buffer + order_price *= Decimal("1") - self.slippage_buffer order_price = taker_market.c_quantize_order_price(taker_trading_pair, order_price) self.log_with_clock(logging.INFO, f"Slippage buffer adjusted order_price: {order_price}") @@ -711,7 +723,7 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): sell_fill_quantity, taker_market.c_get_available_balance(market_pair.taker.quote_asset) / market_pair.taker.get_price_for_volume(True, sell_fill_quantity).result_price * - self._order_size_taker_balance_factor + self.order_size_taker_balance_factor ) quantized_hedge_amount = taker_market.c_quantize_order_amount(taker_trading_pair, Decimal(hedged_order_quantity)) taker_top = taker_market.c_get_price(taker_trading_pair, True) @@ -721,7 +733,7 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): quantized_hedge_amount).result_price self.log_with_clock(logging.INFO, f"Calculated by HB order_price: {order_price}") - order_price *= Decimal("1") + self._slippage_buffer + order_price *= Decimal("1") + self.slippage_buffer order_price = taker_market.quantize_order_price(taker_trading_pair, order_price) self.log_with_clock(logging.INFO, f"Slippage buffer adjusted order_price: {order_price}") @@ -759,8 +771,8 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): cdef: ExchangeBase maker_market = market_pair.maker.market str trading_pair = market_pair.maker.trading_pair - if self._order_amount and self._order_amount > 0: - base_order_size = self._order_amount + if self.order_amount and self.order_amount > 0: + base_order_size = self.order_amount return maker_market.c_quantize_order_amount(trading_pair, Decimal(base_order_size)) else: return self.c_get_order_size_after_portfolio_ratio_limit(market_pair) @@ -784,7 +796,7 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): object current_price = (maker_market.c_get_price(trading_pair, True) + maker_market.c_get_price(trading_pair, False)) * Decimal(0.5) object maker_portfolio_value = base_balance + quote_balance / current_price - object adjusted_order_size = maker_portfolio_value * self._order_size_portfolio_ratio_limit + object adjusted_order_size = maker_portfolio_value * self.order_size_portfolio_ratio_limit return maker_market.c_quantize_order_amount(trading_pair, Decimal(adjusted_order_size)) @@ -813,7 +825,7 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): maker_balance_in_quote = maker_market.c_get_available_balance(market_pair.maker.quote_asset) taker_balance = taker_market.c_get_available_balance(market_pair.taker.base_asset) * \ - self._order_size_taker_balance_factor + self.order_size_taker_balance_factor user_order = self.c_get_adjusted_limit_order_size(market_pair) @@ -833,7 +845,7 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): maker_balance = maker_market.c_get_available_balance(market_pair.maker.base_asset) taker_balance_in_quote = taker_market.c_get_available_balance(market_pair.taker.quote_asset) * \ - self._order_size_taker_balance_factor + self.order_size_taker_balance_factor user_order = self.c_get_adjusted_limit_order_size(market_pair) @@ -845,7 +857,7 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): assert user_order == s_decimal_zero return s_decimal_zero - taker_slippage_adjustment_factor = Decimal("1") + self._slippage_buffer + taker_slippage_adjustment_factor = Decimal("1") + self.slippage_buffer taker_balance = taker_balance_in_quote / (taker_price * taker_slippage_adjustment_factor) order_amount = min(maker_balance, taker_balance, user_order) @@ -897,10 +909,10 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): taker_price *= self.market_conversion_rate() # you are buying on the maker market and selling on the taker market - maker_price = taker_price / (1 + self._min_profitability) + maker_price = taker_price / (1 + self.min_profitability) # # If your bid is higher than highest bid price, reduce it to one tick above the top bid price - if self._adjust_orders_enabled: + if self.adjust_order_enabled: # If maker bid order book is not empty if not Decimal.is_nan(price_above_bid): maker_price = min(maker_price, price_above_bid) @@ -932,10 +944,10 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): taker_price *= self.market_conversion_rate() # You are selling on the maker market and buying on the taker market - maker_price = taker_price * (1 + self._min_profitability) + maker_price = taker_price * (1 + self.min_profitability) # If your ask is lower than the the top ask, increase it to just one tick below top ask - if self._adjust_orders_enabled: + if self.adjust_order_enabled: # If maker ask order book is not empty if not Decimal.is_nan(next_price_below_top_ask): maker_price = max(maker_price, next_price_below_top_ask) @@ -1008,7 +1020,7 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): str trading_pair = market_pair.maker.trading_pair ExchangeBase maker_market = market_pair.maker.market - if self._top_depth_tolerance == 0: + if self.top_depth_tolerance == 0: top_bid_price = maker_market.c_get_price(trading_pair, False) top_ask_price = maker_market.c_get_price(trading_pair, True) @@ -1017,12 +1029,12 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): # Use bid entries in maker order book top_bid_price = maker_market.c_get_price_for_volume(trading_pair, False, - self._top_depth_tolerance).result_price + self.top_depth_tolerance).result_price # Use ask entries in maker order book top_ask_price = maker_market.c_get_price_for_volume(trading_pair, True, - self._top_depth_tolerance).result_price + self.top_depth_tolerance).result_price return top_bid_price, top_ask_price @@ -1097,10 +1109,10 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): object order_price = active_order.price object cancel_order_threshold - if not self._active_order_canceling: - cancel_order_threshold = self._cancel_order_threshold + if not self.active_order_canceling: + cancel_order_threshold = self.cancel_order_threshold else: - cancel_order_threshold = self._min_profitability + cancel_order_threshold = self.min_profitability if current_hedging_price is None: if self._logging_options & self.OPTION_LOG_REMOVING_ORDER: @@ -1158,7 +1170,7 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): else: base_asset_amount = maker_market.c_get_balance(market_pair.maker.base_asset) quote_asset_amount = taker_market.c_get_balance(market_pair.taker.quote_asset) * quote_rate - taker_slippage_adjustment_factor = Decimal("1") + self._slippage_buffer + taker_slippage_adjustment_factor = Decimal("1") + self.slippage_buffer taker_price = taker_market.c_get_price_for_quote_volume( taker_trading_pair, True, quote_asset_amount ).result_price @@ -1299,8 +1311,8 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): market_info.market.get_taker_order_type() if order_type is OrderType.MARKET: price = s_decimal_nan - if not self._active_order_canceling: - expiration_seconds = self._limit_order_min_expiration + if not self.active_order_canceling: + expiration_seconds = self.limit_order_min_expiration if is_buy: order_id = StrategyBase.c_buy_with_specific_market(self, market_info, amount, order_type=order_type, price=price, diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py deleted file mode 100644 index 8a471bd872..0000000000 --- a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map.py +++ /dev/null @@ -1,239 +0,0 @@ -from decimal import Decimal -from typing import Optional - -import hummingbot.client.settings as settings -from hummingbot.client.config.config_helpers import parse_cvar_value -from hummingbot.client.config.config_validators import ( - validate_bool, - validate_decimal, - validate_exchange, - validate_market_trading_pair, -) -from hummingbot.client.config.config_var import ConfigVar - - -def maker_trading_pair_prompt(): - maker_market = cross_exchange_market_making_config_map.get("maker_market").value - example = settings.AllConnectorSettings.get_example_pairs().get(maker_market) - return "Enter the token trading pair you would like to trade on maker market: %s%s >>> " % ( - maker_market, - f" (e.g. {example})" if example else "", - ) - - -def taker_trading_pair_prompt(): - taker_market = cross_exchange_market_making_config_map.get("taker_market").value - example = settings.AllConnectorSettings.get_example_pairs().get(taker_market) - return "Enter the token trading pair you would like to trade on taker market: %s%s >>> " % ( - taker_market, - f" (e.g. {example})" if example else "", - ) - - -def top_depth_tolerance_prompt() -> str: - maker_market = cross_exchange_market_making_config_map["maker_market_trading_pair"].value - base_asset, quote_asset = maker_market.split("-") - return f"What is your top depth tolerance? (in {base_asset}) >>> " - - -# strategy specific validators -def validate_maker_market_trading_pair(value: str) -> Optional[str]: - maker_market = cross_exchange_market_making_config_map.get("maker_market").value - return validate_market_trading_pair(maker_market, value) - - -def validate_taker_market_trading_pair(value: str) -> Optional[str]: - taker_market = cross_exchange_market_making_config_map.get("taker_market").value - return validate_market_trading_pair(taker_market, value) - - -def order_amount_prompt() -> str: - trading_pair = cross_exchange_market_making_config_map["maker_market_trading_pair"].value - base_asset, quote_asset = trading_pair.split("-") - return f"What is the amount of {base_asset} per order? >>> " - - -def taker_market_on_validated(value: str): - settings.required_exchanges.add(value) - - -def update_oracle_settings(value: str): - c_map = cross_exchange_market_making_config_map - if not (c_map["use_oracle_conversion_rate"].value is not None and - c_map["maker_market_trading_pair"].value is not None and - c_map["taker_market_trading_pair"].value is not None): - return - use_oracle = parse_cvar_value(c_map["use_oracle_conversion_rate"], c_map["use_oracle_conversion_rate"].value) - first_base, first_quote = c_map["maker_market_trading_pair"].value.split("-") - second_base, second_quote = c_map["taker_market_trading_pair"].value.split("-") - if use_oracle and (first_base != second_base or first_quote != second_quote): - settings.required_rate_oracle = True - settings.rate_oracle_pairs = [] - if first_base != second_base: - settings.rate_oracle_pairs.append(f"{second_base}-{first_base}") - if first_quote != second_quote: - settings.rate_oracle_pairs.append(f"{second_quote}-{first_quote}") - else: - settings.required_rate_oracle = False - settings.rate_oracle_pairs = [] - - -cross_exchange_market_making_config_map = { - "strategy": ConfigVar(key="strategy", - prompt="", - default="cross_exchange_market_making" - ), - "maker_market": ConfigVar( - key="maker_market", - prompt="Enter your maker spot connector >>> ", - prompt_on_new=True, - validator=validate_exchange, - on_validated=lambda value: settings.required_exchanges.add(value), - ), - "taker_market": ConfigVar( - key="taker_market", - prompt="Enter your taker spot connector >>> ", - prompt_on_new=True, - validator=validate_exchange, - on_validated=taker_market_on_validated, - ), - "maker_market_trading_pair": ConfigVar( - key="maker_market_trading_pair", - prompt=maker_trading_pair_prompt, - prompt_on_new=True, - validator=validate_maker_market_trading_pair, - on_validated=update_oracle_settings - ), - "taker_market_trading_pair": ConfigVar( - key="taker_market_trading_pair", - prompt=taker_trading_pair_prompt, - prompt_on_new=True, - validator=validate_taker_market_trading_pair, - on_validated=update_oracle_settings - ), - "min_profitability": ConfigVar( - key="min_profitability", - prompt="What is the minimum profitability for you to make a trade? (Enter 1 to indicate 1%) >>> ", - prompt_on_new=True, - validator=lambda v: validate_decimal(v, Decimal(-100), Decimal("100"), inclusive=True), - type_str="decimal", - ), - "order_amount": ConfigVar( - key="order_amount", - prompt=order_amount_prompt, - prompt_on_new=True, - type_str="decimal", - validator=lambda v: validate_decimal(v, min_value=Decimal("0"), inclusive=True), - ), - "adjust_order_enabled": ConfigVar( - key="adjust_order_enabled", - prompt="Do you want to enable adjust order? (Yes/No) >>> ", - default=True, - type_str="bool", - validator=validate_bool, - required_if=lambda: False, - ), - "active_order_canceling": ConfigVar( - key="active_order_canceling", - prompt="Do you want to enable active order canceling? (Yes/No) >>> ", - type_str="bool", - default=True, - required_if=lambda: False, - validator=validate_bool, - ), - # Setting the default threshold to 0.05 when to active_order_canceling is disabled - # prevent canceling orders after it has expired - "cancel_order_threshold": ConfigVar( - key="cancel_order_threshold", - prompt="What is the threshold of profitability to cancel a trade? (Enter 1 to indicate 1%) >>> ", - default=5, - type_str="decimal", - required_if=lambda: False, - validator=lambda v: validate_decimal(v, min_value=Decimal(-100), max_value=Decimal(100), inclusive=False), - ), - "limit_order_min_expiration": ConfigVar( - key="limit_order_min_expiration", - prompt="How often do you want limit orders to expire (in seconds)? >>> ", - default=130.0, - type_str="float", - required_if=lambda: False, - validator=lambda v: validate_decimal(v, min_value=0, inclusive=False) - ), - "top_depth_tolerance": ConfigVar( - key="top_depth_tolerance", - prompt=top_depth_tolerance_prompt, - default=0, - type_str="decimal", - required_if=lambda: False, - validator=lambda v: validate_decimal(v, min_value=0, inclusive=True) - ), - "anti_hysteresis_duration": ConfigVar( - key="anti_hysteresis_duration", - prompt="What is the minimum time interval you want limit orders to be adjusted? (in seconds) >>> ", - default=60, - type_str="float", - required_if=lambda: False, - validator=lambda v: validate_decimal(v, min_value=0, inclusive=False) - ), - "order_size_taker_volume_factor": ConfigVar( - key="order_size_taker_volume_factor", - prompt="What percentage of hedge-able volume would you like to be traded on the taker market? " - "(Enter 1 to indicate 1%) >>> ", - default=25, - type_str="decimal", - required_if=lambda: False, - validator=lambda v: validate_decimal(v, Decimal(0), Decimal(100), inclusive=False) - ), - "order_size_taker_balance_factor": ConfigVar( - key="order_size_taker_balance_factor", - prompt="What percentage of asset balance would you like to use for hedging trades on the taker market? " - "(Enter 1 to indicate 1%) >>> ", - default=Decimal("99.5"), - type_str="decimal", - required_if=lambda: False, - validator=lambda v: validate_decimal(v, Decimal(0), Decimal(100), inclusive=False) - ), - "order_size_portfolio_ratio_limit": ConfigVar( - key="order_size_portfolio_ratio_limit", - prompt="What ratio of your total portfolio value would you like to trade on the maker and taker markets? " - "Enter 50 for 50% >>> ", - default=Decimal("16.67"), - type_str="decimal", - required_if=lambda: False, - validator=lambda v: validate_decimal(v, Decimal(0), Decimal(100), inclusive=False) - ), - "use_oracle_conversion_rate": ConfigVar( - key="use_oracle_conversion_rate", - type_str="bool", - prompt="Do you want to use rate oracle on unmatched trading pairs? (Yes/No) >>> ", - prompt_on_new=True, - validator=lambda v: validate_bool(v), - on_validated=update_oracle_settings), - "taker_to_maker_base_conversion_rate": ConfigVar( - key="taker_to_maker_base_conversion_rate", - prompt="Enter conversion rate for taker base asset value to maker base asset value, e.g. " - "if maker base asset is USD and the taker is DAI, 1 DAI is valued at 1.25 USD, " - "the conversion rate is 1.25 >>> ", - default=Decimal("1"), - validator=lambda v: validate_decimal(v, Decimal(0), inclusive=False), - type_str="decimal" - ), - "taker_to_maker_quote_conversion_rate": ConfigVar( - key="taker_to_maker_quote_conversion_rate", - prompt="Enter conversion rate for taker quote asset value to maker quote asset value, e.g. " - "if maker quote asset is USD and the taker is DAI, 1 DAI is valued at 1.25 USD, " - "the conversion rate is 1.25 >>> ", - default=Decimal("1"), - validator=lambda v: validate_decimal(v, Decimal(0), inclusive=False), - type_str="decimal" - ), - "slippage_buffer": ConfigVar( - key="slippage_buffer", - prompt="How much buffer do you want to add to the price to account for slippage for taker orders " - "Enter 1 to indicate 1% >>> ", - prompt_on_new=True, - default=Decimal("5"), - type_str="decimal", - validator=lambda v: validate_decimal(v, Decimal(0), Decimal(100), inclusive=True) - ) -} diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py new file mode 100644 index 0000000000..c478158e7a --- /dev/null +++ b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py @@ -0,0 +1,278 @@ +from decimal import Decimal +from typing import Dict + +from pydantic import BaseModel, Field, root_validator, validator + +import hummingbot.client.settings as settings +from hummingbot.client.config.config_data_types import BaseTradingStrategyMakerTakerConfigMap, ClientFieldData +from hummingbot.client.config.config_validators import validate_bool, validate_decimal + + +class CrossExchangeMarketMakingConfigMap(BaseTradingStrategyMakerTakerConfigMap): + strategy: str = Field(default="cross_exchange_market_making", client_data=None) + + min_profitability: Decimal = Field( + default=..., + description="", + ge=-100.0, + le=100.0, + client_data=ClientFieldData( + prompt=lambda mi: "What is the minimum profitability for you to make a trade? (Enter 1 to indicate 1%)", + prompt_on_new=True, + ), + ) + order_amount: Decimal = Field( + default=..., + description="The strategy order amount.", + ge=0.0, + client_data=ClientFieldData( + prompt=lambda mi: CrossExchangeMarketMakingConfigMap.order_amount_prompt(mi), + prompt_on_new=True, + ) + ) + adjust_order_enabled: bool = Field( + default=True, + description="", + client_data=ClientFieldData( + prompt=lambda mi: "Do you want to enable adjust order? (Yes/No)" + ), + ) + active_order_canceling: bool = Field( + default=True, + description="", + client_data=ClientFieldData( + prompt=lambda mi: "Do you want to enable active order canceling? (Yes/No)" + ), + ) + cancel_order_threshold: Decimal = Field( + default=Decimal("5.0"), + description="", + gt=-100.0, + lt=100.0, + client_data=ClientFieldData( + prompt=lambda mi: "What is the threshold of profitability to cancel a trade? (Enter 1 to indicate 1%)", + ), + ) + limit_order_min_expiration: float = Field( + default=130.0, + description="", + gt=0.0, + client_data=ClientFieldData( + prompt=lambda mi: "How often do you want limit orders to expire (in seconds)?", + ), + ) + top_depth_tolerance: Decimal = Field( + default=Decimal("0.0"), + description="", + ge=0.0, + client_data=ClientFieldData( + prompt=lambda mi: CrossExchangeMarketMakingConfigMap.top_depth_tolerance_prompt(mi), + ), + ) + anti_hysteresis_duration: float = Field( + default=60.0, + description="", + gt=0.0, + client_data=ClientFieldData( + prompt=lambda mi: "What is the minimum time interval you want limit orders to be adjusted? (in seconds)", + ), + ) + order_size_taker_volume_factor: Decimal = Field( + default=Decimal("25.0"), + description="", + ge=0.0, + le=100.0, + client_data=ClientFieldData( + prompt=lambda mi: ( + "What percentage of hedge-able volume would you like to be traded on the taker market? " + "(Enter 1 to indicate 1%)" + ), + ), + ) + order_size_taker_balance_factor: Decimal = Field( + default=Decimal("99.5"), + description="", + ge=0.0, + le=100.0, + client_data=ClientFieldData( + prompt=lambda mi: ( + "What percentage of asset balance would you like to use for hedging trades on the taker market? " + "(Enter 1 to indicate 1%)" + ), + ), + ) + order_size_portfolio_ratio_limit: Decimal = Field( + default=Decimal("16.67"), + description="", + ge=0.0, + le=100.0, + client_data=ClientFieldData( + prompt=lambda mi: ( + "What ratio of your total portfolio value would you like to trade on the maker and taker markets? " + "Enter 50 for 50%" + ), + ), + ) + use_oracle_conversion_rate: bool = Field( + default=True, + description="", + client_data=ClientFieldData( + prompt=lambda mi: "Do you want to use rate oracle on unmatched trading pairs? (Yes/No)", + prompt_on_new=True, + ), + ) + taker_to_maker_base_conversion_rate: Decimal = Field( + default=Decimal("1.0"), + description="", + gt=0.0, + client_data=ClientFieldData( + prompt=lambda mi: ( + "Enter conversion rate for taker base asset value to maker base asset value, e.g. " + "if maker base asset is USD and the taker is DAI, 1 DAI is valued at 1.25 USD, " + "the conversion rate is 1.25" + ), + ), + ) + taker_to_maker_quote_conversion_rate: Decimal = Field( + default=Decimal("1.0"), + description="", + gt=0.0, + client_data=ClientFieldData( + prompt=lambda mi: ( + "Enter conversion rate for taker quote asset value to maker quote asset value, e.g. " + "if maker quote asset is USD and the taker is DAI, 1 DAI is valued at 1.25 USD, " + "the conversion rate is 1.25" + ), + ), + ) + slippage_buffer: Decimal = Field( + default=Decimal("5.0"), + description="", + ge=0.0, + le=100.0, + client_data=ClientFieldData( + prompt=lambda mi: ( + "How much buffer do you want to add to the price to account for slippage for taker orders " + "Enter 1 to indicate 1%" + ), + prompt_on_new=True, + ), + ) + + # === prompts === + + @classmethod + def top_depth_tolerance_prompt(cls, model_instance: 'CrossExchangeMarketMakingConfigMap') -> str: + market_maker = model_instance.trading_pair_maker + base_asset, quote_asset = market_maker.split("-") + return f"What is your top depth tolerance? (in {base_asset})" + + @classmethod + def order_amount_prompt(cls, model_instance: 'CrossExchangeMarketMakingConfigMap') -> str: + trading_pair = model_instance.trading_pair_maker + base_asset, quote_asset = trading_pair.split("-") + return f"What is the amount of {base_asset} per order?" + + # === generic validations === + + @validator( + "adjust_order_enabled", + "active_order_canceling", + "use_oracle_conversion_rate", + pre=True, + ) + def validate_bool(cls, v: str): + """Used for client-friendly error output.""" + if isinstance(v, str): + ret = validate_bool(v) + if ret is not None: + raise ValueError(ret) + return v + + @validator( + "min_profitability", + "order_amount", + "cancel_order_threshold", + "limit_order_min_expiration", + "top_depth_tolerance", + "anti_hysteresis_duration", + "order_size_taker_volume_factor", + "order_size_taker_balance_factor", + "order_size_portfolio_ratio_limit", + "taker_to_maker_base_conversion_rate", + "taker_to_maker_quote_conversion_rate", + "slippage_buffer", + pre=True, + ) + def validate_decimal(cls, v: str, values: Dict, config: BaseModel.Config, field: Field): + """Used for client-friendly error output.""" + range_min = None + range_max = None + range_inclusive = None + + field = field.field_info + + if field.gt is not None: + range_min = field.gt + range_inclusive = False + elif field.ge is not None: + range_min = field.ge + range_inclusive = True + if field.lt is not None: + range_max = field.lt + range_inclusive = False + elif field.le is not None: + range_max = field.le + range_inclusive = True + + if range_min is not None and range_max is not None: + ret = validate_decimal(v, + min_value=Decimal(str(range_min)), + max_value=Decimal(str(range_max)), + inclusive=str(range_inclusive)) + elif range_min is not None: + ret = validate_decimal(v, + min_value=Decimal(str(range_min)), + inclusive=str(range_inclusive)) + elif range_max is not None: + ret = validate_decimal(v, + max_value=Decimal(str(range_max)), + inclusive=str(range_inclusive)) + if ret is not None: + raise ValueError(ret) + return v + + # === post-validations === + + @root_validator() + def post_validations(cls, values: Dict): + cls.exchange_post_validation(values) + cls.update_oracle_settings(values) + return values + + @classmethod + def exchange_post_validation(cls, values: Dict): + if "market_maker" in values.keys(): + settings.required_exchanges.add(values["market_maker"]) + if "market_taker" in values.keys(): + settings.required_exchanges.add(values["market_taker"]) + + @classmethod + def update_oracle_settings(cls, values: str): + if not ("use_oracle_conversion_rate" in values.keys() and + "trading_pair_maker" in values.keys() and + "trading_pair_taker" in values.keys()): + return + use_oracle = values["use_oracle_conversion_rate"] + first_base, first_quote = values["trading_pair_maker"].split("-") + second_base, second_quote = values["trading_pair_taker"].split("-") + if use_oracle and (first_base != second_base or first_quote != second_quote): + settings.required_rate_oracle = True + settings.rate_oracle_pairs = [] + if first_base != second_base: + settings.rate_oracle_pairs.append(f"{second_base}-{first_base}") + if first_quote != second_quote: + settings.rate_oracle_pairs.append(f"{second_quote}-{first_quote}") + else: + settings.required_rate_oracle = False + settings.rate_oracle_pairs = [] diff --git a/hummingbot/strategy/cross_exchange_market_making/start.py b/hummingbot/strategy/cross_exchange_market_making/start.py index f08d5c7383..f33d821520 100644 --- a/hummingbot/strategy/cross_exchange_market_making/start.py +++ b/hummingbot/strategy/cross_exchange_market_making/start.py @@ -1,42 +1,20 @@ -from typing import ( - List, - Tuple -) -from decimal import Decimal +from typing import List, Tuple + from hummingbot.client.config.global_config_map import global_config_map -from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple +from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_making import ( + CrossExchangeMarketMakingStrategy, +) from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_pair import CrossExchangeMarketPair -from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_making import CrossExchangeMarketMakingStrategy -from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_making_config_map import \ - cross_exchange_market_making_config_map as xemm_map +from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple def start(self): - maker_market = xemm_map.get("maker_market").value.lower() - taker_market = xemm_map.get("taker_market").value.lower() - raw_maker_trading_pair = xemm_map.get("maker_market_trading_pair").value - raw_taker_trading_pair = xemm_map.get("taker_market_trading_pair").value - min_profitability = xemm_map.get("min_profitability").value / Decimal("100") - order_amount = xemm_map.get("order_amount").value - strategy_report_interval = global_config_map.get("strategy_report_interval").value - limit_order_min_expiration = xemm_map.get("limit_order_min_expiration").value - cancel_order_threshold = xemm_map.get("cancel_order_threshold").value / Decimal("100") - active_order_canceling = xemm_map.get("active_order_canceling").value - adjust_order_enabled = xemm_map.get("adjust_order_enabled").value - top_depth_tolerance = xemm_map.get("top_depth_tolerance").value - order_size_taker_volume_factor = xemm_map.get("order_size_taker_volume_factor").value / Decimal("100") - order_size_taker_balance_factor = xemm_map.get("order_size_taker_balance_factor").value / Decimal("100") - order_size_portfolio_ratio_limit = xemm_map.get("order_size_portfolio_ratio_limit").value / Decimal("100") - anti_hysteresis_duration = xemm_map.get("anti_hysteresis_duration").value - use_oracle_conversion_rate = xemm_map.get("use_oracle_conversion_rate").value - taker_to_maker_base_conversion_rate = xemm_map.get("taker_to_maker_base_conversion_rate").value - taker_to_maker_quote_conversion_rate = xemm_map.get("taker_to_maker_quote_conversion_rate").value - slippage_buffer = xemm_map.get("slippage_buffer").value / Decimal("100") - - # check if top depth tolerance is a list or if trade size override exists - if isinstance(top_depth_tolerance, list) or "trade_size_override" in xemm_map: - self.notify("Current config is not compatible with cross exchange market making strategy. Please reconfigure") - return + c_map = self.strategy_config_map + maker_market = c_map.market_maker.lower() + taker_market = c_map.market_taker.lower() + raw_maker_trading_pair = c_map.trading_pair_maker + raw_taker_trading_pair = c_map.trading_pair_taker + status_report_interval = global_config_map.get("strategy_report_interval").value try: maker_trading_pair: str = raw_maker_trading_pair @@ -70,23 +48,9 @@ def start(self): ) self.strategy = CrossExchangeMarketMakingStrategy() self.strategy.init_params( + config_map=c_map, market_pairs=[self.market_pair], - min_profitability=min_profitability, - status_report_interval=strategy_report_interval, + status_report_interval=status_report_interval, logging_options=strategy_logging_options, - order_amount=order_amount, - limit_order_min_expiration=limit_order_min_expiration, - cancel_order_threshold=cancel_order_threshold, - active_order_canceling=active_order_canceling, - adjust_order_enabled=adjust_order_enabled, - top_depth_tolerance=top_depth_tolerance, - order_size_taker_volume_factor=order_size_taker_volume_factor, - order_size_taker_balance_factor=order_size_taker_balance_factor, - order_size_portfolio_ratio_limit=order_size_portfolio_ratio_limit, - anti_hysteresis_duration=anti_hysteresis_duration, - use_oracle_conversion_rate=use_oracle_conversion_rate, - taker_to_maker_base_conversion_rate=taker_to_maker_base_conversion_rate, - taker_to_maker_quote_conversion_rate=taker_to_maker_quote_conversion_rate, - slippage_buffer=slippage_buffer, hb_app_notification=True, ) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py index 8bb531897c..6418ab6652 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py @@ -12,7 +12,7 @@ from hummingbot.client.config.config_helpers import ClientConfigAdapter, ConfigValidationError from hummingbot.client.settings import AllConnectorSettings from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( - AvellanedaMarketMakingConfigMap, + AvellanedaMarketMakingMakerConfigMap, DailyBetweenTimesModel, FromDateToDateModel, IgnoreHangingOrdersModel, @@ -28,7 +28,7 @@ class AvellanedaMarketMakingConfigMapPydanticTest(unittest.TestCase): def setUpClass(cls) -> None: super().setUpClass() cls.ev_loop = asyncio.get_event_loop() - cls.exchange = "binance" + cls.market = "binance" cls.base_asset = "COINALPHA" cls.quote_asset = "HBOT" cls.trading_pair = f"{cls.base_asset}-{cls.quote_asset}" @@ -36,7 +36,7 @@ def setUpClass(cls) -> None: def setUp(self) -> None: super().setUp() config_settings = self.get_default_map() - self.config_map = ClientConfigAdapter(AvellanedaMarketMakingConfigMap(**config_settings)) + self.config_map = ClientConfigAdapter(AvellanedaMarketMakingMakerConfigMap(**config_settings)) def async_run_with_timeout(self, coroutine: Awaitable, timeout: int = 1): ret = self.ev_loop.run_until_complete(asyncio.wait_for(coroutine, timeout)) @@ -44,8 +44,8 @@ def async_run_with_timeout(self, coroutine: Awaitable, timeout: int = 1): def get_default_map(self) -> Dict[str, str]: config_settings = { - "exchange": self.exchange, - "market": self.trading_pair, + "market": self.market, + "trading_pair": self.trading_pair, "execution_timeframe_mode": { "start_time": "09:30:00", "end_time": "16:00:00", @@ -59,7 +59,7 @@ def get_default_map(self) -> Dict[str, str]: return config_settings def test_initial_sequential_build(self): - config_map = ClientConfigAdapter(AvellanedaMarketMakingConfigMap.construct()) + config_map = ClientConfigAdapter(AvellanedaMarketMakingMakerConfigMap.construct()) config_settings = self.get_default_map() def build_config_map(cm: ClientConfigAdapter, cs: Dict): @@ -205,7 +205,7 @@ def test_load_configs_from_yaml(self): with open(f_path, "r") as file: data = yaml.safe_load(file) - loaded_config_map = ClientConfigAdapter(AvellanedaMarketMakingConfigMap(**data)) + loaded_config_map = ClientConfigAdapter(AvellanedaMarketMakingMakerConfigMap(**data)) self.assertEqual(self.config_map, loaded_config_map) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py index dce3f1dd33..8dcf482ad6 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py @@ -8,7 +8,7 @@ from hummingbot.connector.exchange_base import ExchangeBase from hummingbot.connector.utils import combine_to_hb_trading_pair from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( - AvellanedaMarketMakingConfigMap, + AvellanedaMarketMakingMakerConfigMap, FromDateToDateModel, MultiOrderLevelModel, TrackHangingOrdersModel, @@ -28,9 +28,9 @@ def setUp(self) -> None: self.base = "ETH" self.quote = "BTC" self.strategy_config_map = ClientConfigAdapter( - AvellanedaMarketMakingConfigMap( - exchange="binance", - market=combine_to_hb_trading_pair(self.base, self.quote), + AvellanedaMarketMakingMakerConfigMap( + market="binance", + trading_pair=combine_to_hb_trading_pair(self.base, self.quote), execution_timeframe_mode=FromDateToDateModel( start_datetime="2021-11-18 15:00:00", end_datetime="2021-11-18 16:00:00", diff --git a/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making.py b/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making.py index 2a96c68a88..71a1e35a2f 100644 --- a/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making.py +++ b/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making.py @@ -1,4 +1,5 @@ import unittest +from copy import deepcopy from decimal import Decimal from math import ceil, floor from typing import List @@ -6,6 +7,7 @@ import pandas as pd from nose.plugins.attrib import attr +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.mock.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode @@ -21,10 +23,13 @@ MarketEvent, OrderBookTradeEvent, OrderFilledEvent, - SellOrderCreatedEvent, SellOrderCompletedEvent, + SellOrderCreatedEvent, ) from hummingbot.strategy.cross_exchange_market_making import CrossExchangeMarketMakingStrategy +from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_making_config_map_pydantic import ( + CrossExchangeMarketMakingConfigMap, +) from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_pair import CrossExchangeMarketPair from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple @@ -35,27 +40,59 @@ class HedgedMarketMakingUnitTest(unittest.TestCase): end: pd.Timestamp = pd.Timestamp("2019-01-01 01:00:00", tz="UTC") start_timestamp: float = start.timestamp() end_timestamp: float = end.timestamp() - maker_trading_pairs: List[str] = ["COINALPHA-WETH", "COINALPHA", "WETH"] - taker_trading_pairs: List[str] = ["COINALPHA-ETH", "COINALPHA", "ETH"] + exchange_name_maker = "binance" + exchange_name_taker = "kucoin" + trading_pairs_maker: List[str] = ["COINALPHA-WETH", "COINALPHA", "WETH"] + trading_pairs_taker: List[str] = ["COINALPHA-ETH", "COINALPHA", "ETH"] def setUp(self): self.clock: Clock = Clock(ClockMode.BACKTEST, 1.0, self.start_timestamp, self.end_timestamp) - self.min_profitbality = Decimal("0.005") + self.min_profitability = Decimal("0.5") self.maker_market: MockPaperExchange = MockPaperExchange() self.taker_market: MockPaperExchange = MockPaperExchange() - self.maker_market.set_balanced_order_book(self.maker_trading_pairs[0], 1.0, 0.5, 1.5, 0.01, 10) - self.taker_market.set_balanced_order_book(self.taker_trading_pairs[0], 1.0, 0.5, 1.5, 0.001, 4) + self.maker_market.set_balanced_order_book(self.trading_pairs_maker[0], 1.0, 0.5, 1.5, 0.01, 10) + self.taker_market.set_balanced_order_book(self.trading_pairs_taker[0], 1.0, 0.5, 1.5, 0.001, 4) self.maker_market.set_balance("COINALPHA", 5) self.maker_market.set_balance("WETH", 5) self.maker_market.set_balance("QETH", 5) self.taker_market.set_balance("COINALPHA", 5) self.taker_market.set_balance("ETH", 5) - self.maker_market.set_quantization_param(QuantizationParams(self.maker_trading_pairs[0], 5, 5, 5, 5)) - self.taker_market.set_quantization_param(QuantizationParams(self.taker_trading_pairs[0], 5, 5, 5, 5)) + self.maker_market.set_quantization_param(QuantizationParams(self.trading_pairs_maker[0], 5, 5, 5, 5)) + self.taker_market.set_quantization_param(QuantizationParams(self.trading_pairs_taker[0], 5, 5, 5, 5)) self.market_pair: CrossExchangeMarketPair = CrossExchangeMarketPair( - MarketTradingPairTuple(self.maker_market, *self.maker_trading_pairs), - MarketTradingPairTuple(self.taker_market, *self.taker_trading_pairs), + MarketTradingPairTuple(self.maker_market, *self.trading_pairs_maker), + MarketTradingPairTuple(self.taker_market, *self.trading_pairs_taker), + ) + + self.config_map_raw = CrossExchangeMarketMakingConfigMap( + market_maker=self.exchange_name_maker, + market_taker=self.exchange_name_taker, + trading_pair_maker=self.trading_pairs_maker[0], + trading_pair_taker=self.trading_pairs_taker[0], + min_profitability=Decimal(self.min_profitability), + slippage_buffer=Decimal("0"), + order_amount=Decimal("0"), + # Default values folllow + order_size_taker_volume_factor=Decimal("25"), + order_size_taker_balance_factor=Decimal("99.5"), + order_size_portfolio_ratio_limit=Decimal("30"), + limit_order_min_expiration=130.0, + adjust_order_enabled=True, + anti_hysteresis_duration=60.0, + active_order_canceling=True, + cancel_order_threshold=Decimal("5"), + top_depth_tolerance=Decimal(0), + use_oracle_conversion_rate=False, + taker_to_maker_base_conversion_rate=Decimal("1"), + taker_to_maker_quote_conversion_rate=Decimal("1") + ) + + self.config_map = ClientConfigAdapter(self.config_map_raw) + config_map_with_top_depth_tolerance_raw = deepcopy(self.config_map_raw) + config_map_with_top_depth_tolerance_raw.top_depth_tolerance = Decimal("1") + config_map_with_top_depth_tolerance = ClientConfigAdapter( + config_map_with_top_depth_tolerance_raw ) logging_options: int = ( @@ -64,20 +101,15 @@ def setUp(self): ) self.strategy: CrossExchangeMarketMakingStrategy = CrossExchangeMarketMakingStrategy() self.strategy.init_params( - [self.market_pair], - order_size_portfolio_ratio_limit=Decimal("0.3"), - min_profitability=Decimal(self.min_profitbality), + config_map=self.config_map, + market_pairs=[self.market_pair], logging_options=logging_options, - slippage_buffer=Decimal("0"), ) self.strategy_with_top_depth_tolerance: CrossExchangeMarketMakingStrategy = CrossExchangeMarketMakingStrategy() self.strategy_with_top_depth_tolerance.init_params( - [self.market_pair], - order_size_portfolio_ratio_limit=Decimal("0.3"), - min_profitability=Decimal(self.min_profitbality), + config_map=config_map_with_top_depth_tolerance, + market_pairs=[self.market_pair], logging_options=logging_options, - top_depth_tolerance=1, - slippage_buffer=Decimal("0"), ) self.logging_options = logging_options self.clock.add_iterator(self.maker_market) @@ -97,8 +129,8 @@ def setUp(self): self.taker_market.add_listener(MarketEvent.BuyOrderCreated, self.taker_order_created_logger) self.taker_market.add_listener(MarketEvent.SellOrderCreated, self.taker_order_created_logger) - def simulate_maker_market_trade(self, is_buy: bool, quantity: Decimal, price: Decimal): - maker_trading_pair: str = self.maker_trading_pairs[0] + def simulate_market_maker_trade(self, is_buy: bool, quantity: Decimal, price: Decimal): + maker_trading_pair: str = self.trading_pairs_maker[0] order_book: OrderBook = self.maker_market.get_order_book(maker_trading_pair) trade_event: OrderBookTradeEvent = OrderBookTradeEvent( maker_trading_pair, self.clock.current_timestamp, TradeType.BUY if is_buy else TradeType.SELL, price, quantity @@ -239,7 +271,7 @@ def test_both_sides_profitable(self): self.assertEqual(Decimal("3.0"), bid_order.quantity) self.assertEqual(Decimal("3.0"), ask_order.quantity) - self.simulate_maker_market_trade(False, Decimal("10.0"), bid_order.price * Decimal("0.99")) + self.simulate_market_maker_trade(False, Decimal("10.0"), bid_order.price * Decimal("0.99")) self.clock.backtest_til(self.start_timestamp + 10) self.assertEqual(1, len(self.maker_order_fill_logger.event_log)) @@ -292,7 +324,7 @@ def test_top_depth_tolerance(self): # TODO self.assertEqual(Decimal("3.0"), bid_order.quantity) self.assertEqual(Decimal("3.0"), ask_order.quantity) - self.simulate_order_book_widening(self.taker_market.order_books[self.taker_trading_pairs[0]], 0.99, 1.01) + self.simulate_order_book_widening(self.taker_market.order_books[self.trading_pairs_taker[0]], 0.99, 1.01) self.clock.backtest_til(self.start_timestamp + 100) @@ -341,7 +373,7 @@ def test_market_became_wider(self): ) ) - self.simulate_order_book_widening(self.taker_market.order_books[self.taker_trading_pairs[0]], 0.99, 1.01) + self.simulate_order_book_widening(self.taker_market.order_books[self.trading_pairs_taker[0]], 0.99, 1.01) self.clock.backtest_til(self.start_timestamp + 100) @@ -363,7 +395,7 @@ def test_market_became_narrower(self): self.assertEqual(Decimal("3.0"), bid_order.quantity) self.assertEqual(Decimal("3.0"), ask_order.quantity) - self.maker_market.order_books[self.maker_trading_pairs[0]].apply_diffs( + self.maker_market.order_books[self.trading_pairs_maker[0]].apply_diffs( [OrderBookRow(0.996, 30, 2)], [OrderBookRow(1.004, 30, 2)], 2) self.clock.backtest_til(self.start_timestamp + 10) @@ -411,7 +443,7 @@ def test_order_fills_after_cancellation(self): # TODO ) ) - self.simulate_order_book_widening(self.taker_market.order_books[self.taker_trading_pairs[0]], 0.99, 1.01) + self.simulate_order_book_widening(self.taker_market.order_books[self.trading_pairs_taker[0]], 0.99, 1.01) self.clock.backtest_til(self.start_timestamp + 10) @@ -435,7 +467,7 @@ def test_order_fills_after_cancellation(self): # TODO self.assertEqual(1, len(bid_hedges)) self.assertEqual(1, len(ask_hedges)) self.assertGreater( - self.maker_market.get_balance(self.maker_trading_pairs[2]) + self.taker_market.get_balance(self.taker_trading_pairs[2]), + self.maker_market.get_balance(self.trading_pairs_maker[2]) + self.taker_market.get_balance(self.trading_pairs_taker[2]), Decimal("10"), ) self.assertEqual(2, len(self.taker_order_fill_logger.event_log)) @@ -452,15 +484,23 @@ def test_with_conversion(self): self.clock.remove_iterator(self.strategy) self.market_pair: CrossExchangeMarketPair = CrossExchangeMarketPair( MarketTradingPairTuple(self.maker_market, *["COINALPHA-QETH", "COINALPHA", "QETH"]), - MarketTradingPairTuple(self.taker_market, *self.taker_trading_pairs), + MarketTradingPairTuple(self.taker_market, *self.trading_pairs_taker), ) self.maker_market.set_balanced_order_book("COINALPHA-QETH", 1.05, 0.55, 1.55, 0.01, 10) + + config_map_raw = deepcopy(self.config_map_raw) + config_map_raw.min_profitability = Decimal("1") + config_map_raw.order_size_portfolio_ratio_limit = Decimal("30") + config_map_raw.taker_to_maker_base_conversion_rate = Decimal("0.95") + config_map = ClientConfigAdapter( + config_map_raw + ) + self.strategy: CrossExchangeMarketMakingStrategy = CrossExchangeMarketMakingStrategy() self.strategy.init_params( - [self.market_pair], Decimal("0.01"), - order_size_portfolio_ratio_limit=Decimal("0.3"), + config_map=config_map, + market_pairs=[self.market_pair], logging_options=self.logging_options, - taker_to_maker_base_conversion_rate=Decimal("0.95") ) self.clock.add_iterator(self.strategy) self.clock.backtest_til(self.start_timestamp + 5) @@ -482,9 +522,9 @@ def test_maker_price(self): self.clock.backtest_til(self.start_timestamp + 5) bid_order: LimitOrder = self.strategy.active_bids[0][1] ask_order: LimitOrder = self.strategy.active_asks[0][1] - bid_maker_price = sell_taker_price * (1 - self.min_profitbality) + bid_maker_price = sell_taker_price * (1 - self.min_profitability / Decimal(100)) bid_maker_price = (floor(bid_maker_price / price_quantum)) * price_quantum - ask_maker_price = buy_taker_price * (1 + self.min_profitbality) + ask_maker_price = buy_taker_price * (1 + self.min_profitability / Decimal(100)) ask_maker_price = (ceil(ask_maker_price / price_quantum) * price_quantum) self.assertEqual(bid_maker_price, round(bid_order.price, 4)) self.assertEqual(ask_maker_price, round(ask_order.price, 4)) @@ -495,22 +535,30 @@ def test_with_adjust_orders_enabled(self): self.clock.remove_iterator(self.strategy) self.clock.remove_iterator(self.maker_market) self.maker_market: MockPaperExchange = MockPaperExchange() - self.maker_market.set_balanced_order_book(self.maker_trading_pairs[0], 1.0, 0.5, 1.5, 0.1, 10) + self.maker_market.set_balanced_order_book(self.trading_pairs_maker[0], 1.0, 0.5, 1.5, 0.1, 10) self.market_pair: CrossExchangeMarketPair = CrossExchangeMarketPair( - MarketTradingPairTuple(self.maker_market, *self.maker_trading_pairs), - MarketTradingPairTuple(self.taker_market, *self.taker_trading_pairs), + MarketTradingPairTuple(self.maker_market, *self.trading_pairs_maker), + MarketTradingPairTuple(self.taker_market, *self.trading_pairs_taker), ) + + config_map_raw = deepcopy(self.config_map_raw) + config_map_raw.order_size_portfolio_ratio_limit = Decimal("30") + config_map_raw.min_profitability = Decimal("0.5") + config_map_raw.adjust_order_enabled = True + config_map = ClientConfigAdapter( + config_map_raw + ) + self.strategy: CrossExchangeMarketMakingStrategy = CrossExchangeMarketMakingStrategy() self.strategy.init_params( - [self.market_pair], - order_size_portfolio_ratio_limit=Decimal("0.3"), - min_profitability=Decimal("0.005"), + config_map=config_map, + market_pairs=[self.market_pair], logging_options=self.logging_options, ) self.maker_market.set_balance("COINALPHA", 5) self.maker_market.set_balance("WETH", 5) self.maker_market.set_balance("QETH", 5) - self.maker_market.set_quantization_param(QuantizationParams(self.maker_trading_pairs[0], 4, 4, 4, 4)) + self.maker_market.set_quantization_param(QuantizationParams(self.trading_pairs_maker[0], 4, 4, 4, 4)) self.clock.add_iterator(self.strategy) self.clock.add_iterator(self.maker_market) self.clock.backtest_til(self.start_timestamp + 5) @@ -530,24 +578,31 @@ def test_with_adjust_orders_disabled(self): self.clock.remove_iterator(self.maker_market) self.maker_market: MockPaperExchange = MockPaperExchange() - self.maker_market.set_balanced_order_book(self.maker_trading_pairs[0], 1.0, 0.5, 1.5, 0.1, 10) - self.taker_market.set_balanced_order_book(self.taker_trading_pairs[0], 1.0, 0.5, 1.5, 0.001, 20) + self.maker_market.set_balanced_order_book(self.trading_pairs_maker[0], 1.0, 0.5, 1.5, 0.1, 10) + self.taker_market.set_balanced_order_book(self.trading_pairs_taker[0], 1.0, 0.5, 1.5, 0.001, 20) self.market_pair: CrossExchangeMarketPair = CrossExchangeMarketPair( - MarketTradingPairTuple(self.maker_market, *self.maker_trading_pairs), - MarketTradingPairTuple(self.taker_market, *self.taker_trading_pairs), + MarketTradingPairTuple(self.maker_market, *self.trading_pairs_maker), + MarketTradingPairTuple(self.taker_market, *self.trading_pairs_taker), + ) + + config_map_raw = deepcopy(self.config_map_raw) + config_map_raw.order_size_portfolio_ratio_limit = Decimal("30") + config_map_raw.min_profitability = Decimal("0.5") + config_map_raw.adjust_order_enabled = False + config_map = ClientConfigAdapter( + config_map_raw ) + self.strategy: CrossExchangeMarketMakingStrategy = CrossExchangeMarketMakingStrategy() self.strategy.init_params( - [self.market_pair], - order_size_portfolio_ratio_limit=Decimal("0.3"), - min_profitability=Decimal("0.005"), + config_map=config_map, + market_pairs=[self.market_pair], logging_options=self.logging_options, - adjust_order_enabled=False ) self.maker_market.set_balance("COINALPHA", 5) self.maker_market.set_balance("WETH", 5) self.maker_market.set_balance("QETH", 5) - self.maker_market.set_quantization_param(QuantizationParams(self.maker_trading_pairs[0], 4, 4, 4, 4)) + self.maker_market.set_quantization_param(QuantizationParams(self.trading_pairs_maker[0], 4, 4, 4, 4)) self.clock.add_iterator(self.strategy) self.clock.add_iterator(self.maker_market) self.clock.backtest_til(self.start_timestamp + 5) @@ -561,7 +616,7 @@ def test_with_adjust_orders_disabled(self): self.assertAlmostEqual(Decimal("3"), round(ask_order.quantity, 4)) def test_price_and_size_limit_calculation(self): - self.taker_market.set_balanced_order_book(self.taker_trading_pairs[0], 1.0, 0.5, 1.5, 0.001, 20) + self.taker_market.set_balanced_order_book(self.trading_pairs_taker[0], 1.0, 0.5, 1.5, 0.001, 20) bid_size = self.strategy.get_market_making_size(self.market_pair, True) bid_price = self.strategy.get_market_making_price(self.market_pair, True, bid_size) ask_size = self.strategy.get_market_making_size(self.market_pair, False) @@ -572,34 +627,51 @@ def test_price_and_size_limit_calculation(self): def test_price_and_size_limit_calculation_with_slippage_buffer(self): self.taker_market.set_balance("ETH", 3) self.taker_market.set_balanced_order_book( - self.taker_trading_pairs[0], + self.trading_pairs_taker[0], mid_price=Decimal("1.0"), min_price=Decimal("0.5"), max_price=Decimal("1.5"), price_step_size=Decimal("0.1"), volume_step_size=Decimal("100"), ) + + config_map_raw = deepcopy(self.config_map_raw) + config_map_raw.order_size_taker_volume_factor = Decimal("100") + config_map_raw.order_size_taker_balance_factor = Decimal("100") + config_map_raw.order_size_portfolio_ratio_limit = Decimal("100") + config_map_raw.min_profitability = Decimal("25") + config_map_raw.slippage_buffer = Decimal("0") + config_map_raw.order_amount = Decimal("4") + config_map = ClientConfigAdapter( + config_map_raw + ) + self.strategy: CrossExchangeMarketMakingStrategy = CrossExchangeMarketMakingStrategy() self.strategy.init_params( - [self.market_pair], - order_size_taker_volume_factor=Decimal("1"), - order_size_taker_balance_factor=Decimal("1"), - order_size_portfolio_ratio_limit=Decimal("1"), - min_profitability=Decimal("0.25"), + config_map=config_map, + market_pairs=[self.market_pair], logging_options=self.logging_options, - slippage_buffer=Decimal("0"), - order_amount=Decimal("4"), + ) + + config_map_with_slippage_buffer = ClientConfigAdapter( + CrossExchangeMarketMakingConfigMap( + market_maker=self.exchange_name_maker, + market_taker=self.exchange_name_taker, + trading_pair_maker=self.trading_pairs_maker[0], + trading_pair_taker=self.trading_pairs_taker[0], + order_amount=Decimal("4"), + min_profitability=Decimal("25"), + order_size_taker_volume_factor=Decimal("100"), + order_size_taker_balance_factor=Decimal("100"), + order_size_portfolio_ratio_limit=Decimal("100"), + slippage_buffer=Decimal("25"), + ) ) strategy_with_slippage_buffer: CrossExchangeMarketMakingStrategy = CrossExchangeMarketMakingStrategy() strategy_with_slippage_buffer.init_params( - [self.market_pair], - order_size_taker_volume_factor=Decimal("1"), - order_size_taker_balance_factor=Decimal("1"), - order_size_portfolio_ratio_limit=Decimal("1"), - min_profitability=Decimal("0.25"), + config_map=config_map_with_slippage_buffer, + market_pairs=[self.market_pair], logging_options=self.logging_options, - slippage_buffer=Decimal("0.25"), - order_amount=Decimal("4"), ) bid_size = self.strategy.get_market_making_size(self.market_pair, True) @@ -628,23 +700,31 @@ def test_check_if_sufficient_balance_adjusts_including_slippage(self): self.taker_market.set_balance("COINALPHA", 4) self.taker_market.set_balance("ETH", 3) self.taker_market.set_balanced_order_book( - self.taker_trading_pairs[0], + self.trading_pairs_taker[0], mid_price=Decimal("1.0"), min_price=Decimal("0.5"), max_price=Decimal("1.5"), price_step_size=Decimal("0.1"), volume_step_size=Decimal("1"), ) + + config_map_raw = deepcopy(self.config_map_raw) + config_map_raw.order_size_taker_volume_factor = Decimal("100") + config_map_raw.order_size_taker_balance_factor = Decimal("100") + config_map_raw.order_size_portfolio_ratio_limit = Decimal("100") + config_map_raw.min_profitability = Decimal("25") + config_map_raw.slippage_buffer = Decimal("25") + config_map_raw.order_amount = Decimal("4") + + config_map = ClientConfigAdapter( + config_map_raw + ) + strategy_with_slippage_buffer: CrossExchangeMarketMakingStrategy = CrossExchangeMarketMakingStrategy() strategy_with_slippage_buffer.init_params( - [self.market_pair], - order_size_taker_volume_factor=Decimal("1"), - order_size_taker_balance_factor=Decimal("1"), - order_size_portfolio_ratio_limit=Decimal("1"), - min_profitability=Decimal("0.25"), + config_map=config_map, + market_pairs=[self.market_pair], logging_options=self.logging_options, - slippage_buffer=Decimal("0.25"), - order_amount=Decimal("4"), ) self.clock.remove_iterator(self.strategy) self.clock.add_iterator(strategy_with_slippage_buffer) @@ -673,10 +753,10 @@ def test_check_if_sufficient_balance_adjusts_including_slippage(self): active_bid = active_bids[0][1] active_ask = active_asks[0][1] bids_quantum = self.taker_market.get_order_size_quantum( - self.taker_trading_pairs[0], active_bid.quantity + self.trading_pairs_taker[0], active_bid.quantity ) asks_quantum = self.taker_market.get_order_size_quantum( - self.taker_trading_pairs[0], active_ask.quantity + self.trading_pairs_taker[0], active_ask.quantity ) self.taker_market.set_balance("COINALPHA", Decimal("4") - bids_quantum) @@ -709,26 +789,34 @@ def test_empty_maker_orderbook(self): self.maker_market: MockPaperExchange = MockPaperExchange() # Orderbook is empty - self.maker_market.new_empty_order_book(self.maker_trading_pairs[0]) + self.maker_market.new_empty_order_book(self.trading_pairs_maker[0]) self.market_pair: CrossExchangeMarketPair = CrossExchangeMarketPair( - MarketTradingPairTuple(self.maker_market, *self.maker_trading_pairs), - MarketTradingPairTuple(self.taker_market, *self.taker_trading_pairs), + MarketTradingPairTuple(self.maker_market, *self.trading_pairs_maker), + MarketTradingPairTuple(self.taker_market, *self.trading_pairs_taker), ) + + config_map_raw = deepcopy(self.config_map_raw) + config_map_raw.min_profitability = Decimal("0.5") + config_map_raw.adjust_order_enabled = False + config_map_raw.order_amount = Decimal("1") + + config_map = ClientConfigAdapter( + config_map_raw + ) + self.strategy: CrossExchangeMarketMakingStrategy = CrossExchangeMarketMakingStrategy() self.strategy.init_params( - [self.market_pair], - order_amount=1, - min_profitability=Decimal("0.005"), + config_map=config_map, + market_pairs=[self.market_pair], logging_options=self.logging_options, - adjust_order_enabled=False ) self.maker_market.set_balance("COINALPHA", 5) self.maker_market.set_balance("WETH", 5) self.maker_market.set_balance("QETH", 5) - self.maker_market.set_quantization_param(QuantizationParams(self.maker_trading_pairs[0], 4, 4, 4, 4)) + self.maker_market.set_quantization_param(QuantizationParams(self.trading_pairs_maker[0], 4, 4, 4, 4)) self.clock.add_iterator(self.strategy) self.clock.add_iterator(self.maker_market) - self.clock.backtest_til(self.start_timestamp + 5) + self.clock.backtest_til(self.start_timestamp + 4) self.assertEqual(1, len(self.strategy.active_bids)) self.assertEqual(1, len(self.strategy.active_asks)) bid_order: LimitOrder = self.strategy.active_bids[0][1] diff --git a/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_config_map.py b/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_config_map.py deleted file mode 100644 index e988644011..0000000000 --- a/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_config_map.py +++ /dev/null @@ -1,61 +0,0 @@ -import unittest -from copy import deepcopy - -from hummingbot.client.settings import AllConnectorSettings -from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_making_config_map import ( - cross_exchange_market_making_config_map, - order_amount_prompt, - maker_trading_pair_prompt, - taker_trading_pair_prompt -) - - -class CrossExchangeMarketMakingConfigMapTest(unittest.TestCase): - @classmethod - def setUpClass(cls) -> None: - super().setUpClass() - cls.base_asset = "COINALPHA" - cls.quote_asset = "HBOT" - cls.trading_pair = f"{cls.base_asset}-{cls.quote_asset}" - - cls.maker_exchange = "binance" - cls.taker_exchange = "kucoin" - - def setUp(self) -> None: - super().setUp() - self.config_backup = deepcopy(cross_exchange_market_making_config_map) - - def tearDown(self) -> None: - self.reset_config_map() - super().tearDown() - - def reset_config_map(self): - for key, value in self.config_backup.items(): - cross_exchange_market_making_config_map[key] = value - - def test_order_amount_prompt(self): - cross_exchange_market_making_config_map["maker_market_trading_pair"].value = self.trading_pair - prompt = order_amount_prompt() - expected = f"What is the amount of {self.base_asset} per order? >>> " - - self.assertEqual(expected, prompt) - - def test_maker_trading_pair_prompt(self): - exchange = cross_exchange_market_making_config_map["maker_market"].value = self.maker_exchange - example = AllConnectorSettings.get_example_pairs().get(exchange) - - prompt = maker_trading_pair_prompt() - expected = f"Enter the token trading pair you would like to trade on maker market: {exchange} " \ - f"(e.g. {example}) >>> " - - self.assertEqual(expected, prompt) - - def test_taker_trading_pair_prompt(self): - exchange = cross_exchange_market_making_config_map["taker_market"].value = self.taker_exchange - example = AllConnectorSettings.get_example_pairs().get(exchange) - - prompt = taker_trading_pair_prompt() - expected = f"Enter the token trading pair you would like to trade on taker market: {exchange} " \ - f"(e.g. {example}) >>> " - - self.assertEqual(expected, prompt) diff --git a/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_start.py b/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_start.py index 2930df6925..1762bc8422 100644 --- a/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_start.py +++ b/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_start.py @@ -1,12 +1,13 @@ -from decimal import Decimal import unittest.mock +from decimal import Decimal + import hummingbot.strategy.cross_exchange_market_making.start as strategy_start +from hummingbot.client.config.config_helpers import ClientConfigAdapter +from hummingbot.client.config.global_config_map import global_config_map from hummingbot.connector.exchange_base import ExchangeBase -from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_making_config_map import ( - cross_exchange_market_making_config_map as strategy_cmap +from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_making_config_map_pydantic import ( + CrossExchangeMarketMakingConfigMap, ) -from hummingbot.client.config.global_config_map import global_config_map -from test.hummingbot.strategy import assign_config_default class XEMMStartTest(unittest.TestCase): @@ -17,15 +18,20 @@ def setUp(self) -> None: self.markets = {"binance": ExchangeBase(), "kucoin": ExchangeBase()} self.notifications = [] self.log_errors = [] - assign_config_default(strategy_cmap) - strategy_cmap.get("maker_market").value = "binance" - strategy_cmap.get("taker_market").value = "kucoin" - strategy_cmap.get("maker_market_trading_pair").value = "ETH-USDT" - strategy_cmap.get("taker_market_trading_pair").value = "ETH-USDT" - strategy_cmap.get("order_amount").value = Decimal("1") - strategy_cmap.get("min_profitability").value = Decimal("2") + + self.strategy_config_map = ClientConfigAdapter( + CrossExchangeMarketMakingConfigMap( + market_maker="binance", + market_taker="kucoin", + trading_pair_maker="ETH-USDT", + trading_pair_taker="ETH-USDT", + order_amount=1.0, + min_profitability=2.0, + use_oracle_conversion_rate=False, + ) + ) + global_config_map.get("strategy_report_interval").value = 60. - strategy_cmap.get("use_oracle_conversion_rate").value = False def _initialize_market_assets(self, market, trading_pairs): return [("ETH", "USDT")] From ad4456d551348f3de27d977c68aa3fa51c618ac4 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Wed, 18 May 2022 21:53:43 +0200 Subject: [PATCH 091/152] (feat) refactoring --- hummingbot/client/config/config_data_types.py | 14 +++++++------- .../bybit_perpetual/bybit_perpetual_derivative.py | 11 ++++++----- ...avellaneda_market_making_config_map_pydantic.py | 8 ++------ .../test_bybit_perpetual_derivative.py | 7 ++++--- 4 files changed, 19 insertions(+), 21 deletions(-) diff --git a/hummingbot/client/config/config_data_types.py b/hummingbot/client/config/config_data_types.py index a8bf9c605d..36855e3845 100644 --- a/hummingbot/client/config/config_data_types.py +++ b/hummingbot/client/config/config_data_types.py @@ -69,8 +69,8 @@ def validate_strategy(cls, v: str): return v -class BaseTradingStrategyMakerConfigMap(BaseStrategyConfigMap): - market: ClientConfigEnum( +class BaseTradingStrategyConfigMap(BaseStrategyConfigMap): + exchange: ClientConfigEnum( value="Exchanges", # noqa: F821 names={e: e for e in AllConnectorSettings.get_connector_settings().keys()}, type=str, @@ -82,17 +82,17 @@ class BaseTradingStrategyMakerConfigMap(BaseStrategyConfigMap): prompt_on_new=True, ), ) - trading_pair: str = Field( + market: str = Field( default=..., description="The trading pair.", client_data=ClientFieldData( - prompt=lambda mi: BaseTradingStrategyMakerConfigMap.trading_pair_prompt(mi), + prompt=lambda mi: BaseTradingStrategyConfigMap.trading_pair_prompt(mi), prompt_on_new=True, ), ) @classmethod - def trading_pair_prompt(cls, model_instance: 'BaseTradingStrategyMakerConfigMap') -> str: + def trading_pair_prompt(cls, model_instance: 'BaseTradingStrategyConfigMap') -> str: exchange = model_instance.exchange example = AllConnectorSettings.get_example_pairs().get(exchange) return ( @@ -100,7 +100,7 @@ def trading_pair_prompt(cls, model_instance: 'BaseTradingStrategyMakerConfigMap' f" {exchange}{f' (e.g. {example})' if example else ''}" ) - @validator("market", pre=True) + @validator("exchange", pre=True) def validate_exchange(cls, v: str): """Used for client-friendly error output.""" ret = validate_exchange(v) @@ -113,7 +113,7 @@ def validate_exchange(cls, v: str): ) return v - @validator("trading_pair", pre=True) + @validator("market", pre=True) def validate_exchange_trading_pair(cls, v: str, values: Dict): exchange = values.get("exchange") ret = validate_market_trading_pair(exchange, v) diff --git a/hummingbot/connector/derivative/bybit_perpetual/bybit_perpetual_derivative.py b/hummingbot/connector/derivative/bybit_perpetual/bybit_perpetual_derivative.py index 10420921b0..f46b6de109 100644 --- a/hummingbot/connector/derivative/bybit_perpetual/bybit_perpetual_derivative.py +++ b/hummingbot/connector/derivative/bybit_perpetual/bybit_perpetual_derivative.py @@ -14,17 +14,18 @@ import hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_constants as CONSTANTS import hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_utils as bybit_utils from hummingbot.connector.client_order_tracker import ClientOrderTracker -from hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_api_order_book_data_source import \ - BybitPerpetualAPIOrderBookDataSource as OrderBookDataSource +from hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_api_order_book_data_source import ( + BybitPerpetualAPIOrderBookDataSource as OrderBookDataSource, +) from hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_auth import BybitPerpetualAuth from hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_order_book_tracker import ( - BybitPerpetualOrderBookTracker + BybitPerpetualOrderBookTracker, ) from hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_user_stream_tracker import ( - BybitPerpetualUserStreamTracker + BybitPerpetualUserStreamTracker, ) from hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_websocket_adaptor import ( - BybitPerpetualWebSocketAdaptor + BybitPerpetualWebSocketAdaptor, ) from hummingbot.connector.derivative.perpetual_budget_checker import PerpetualBudgetChecker from hummingbot.connector.derivative.position import Position diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py index ab11a0015d..5cb6115bc0 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -4,11 +4,7 @@ from pydantic import Field, root_validator, validator -from hummingbot.client.config.config_data_types import ( - BaseClientModel, - BaseTradingStrategyMakerConfigMap, - ClientFieldData, -) +from hummingbot.client.config.config_data_types import BaseClientModel, BaseTradingStrategyConfigMap, ClientFieldData from hummingbot.client.config.config_validators import ( validate_bool, validate_datetime_iso_string, @@ -174,7 +170,7 @@ class Config: } -class AvellanedaMarketMakingConfigMap(BaseTradingStrategyMakerConfigMap): +class AvellanedaMarketMakingConfigMap(BaseTradingStrategyConfigMap): strategy: str = Field(default="avellaneda_market_making", client_data=None) execution_timeframe_mode: Union[InfiniteModel, FromDateToDateModel, DailyBetweenTimesModel] = Field( default=..., diff --git a/test/hummingbot/connector/derivative/bybit_perpetual/test_bybit_perpetual_derivative.py b/test/hummingbot/connector/derivative/bybit_perpetual/test_bybit_perpetual_derivative.py index 99d839ed3d..f46001975a 100644 --- a/test/hummingbot/connector/derivative/bybit_perpetual/test_bybit_perpetual_derivative.py +++ b/test/hummingbot/connector/derivative/bybit_perpetual/test_bybit_perpetual_derivative.py @@ -3,6 +3,7 @@ import re import time from decimal import Decimal +from test.hummingbot.connector.network_mocking_assistant import NetworkMockingAssistant from typing import Dict from unittest import TestCase from unittest.mock import AsyncMock, patch @@ -12,8 +13,9 @@ import hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_constants as CONSTANTS import hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_utils as bybit_utils -from hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_api_order_book_data_source import \ - BybitPerpetualAPIOrderBookDataSource +from hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_api_order_book_data_source import ( + BybitPerpetualAPIOrderBookDataSource, +) from hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_derivative import BybitPerpetualDerivative from hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_order_book import BybitPerpetualOrderBook from hummingbot.connector.trading_rule import TradingRule @@ -23,7 +25,6 @@ from hummingbot.core.event.event_logger import EventLogger from hummingbot.core.event.events import FundingInfo, MarketEvent from hummingbot.core.network_iterator import NetworkStatus -from test.hummingbot.connector.network_mocking_assistant import NetworkMockingAssistant class BybitPerpetualDerivativeTests(TestCase): From 287eb28fbdf1f4f3fcdd9904c5ca17d6d5b07c62 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Wed, 18 May 2022 22:11:36 +0200 Subject: [PATCH 092/152] (feat) fix --- .../test_avellaneda_market_making_config_map_pydantic.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py index 6418ab6652..f2e1ab315a 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py @@ -12,7 +12,7 @@ from hummingbot.client.config.config_helpers import ClientConfigAdapter, ConfigValidationError from hummingbot.client.settings import AllConnectorSettings from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( - AvellanedaMarketMakingMakerConfigMap, + AvellanedaMarketMakingConfigMap, DailyBetweenTimesModel, FromDateToDateModel, IgnoreHangingOrdersModel, @@ -36,7 +36,7 @@ def setUpClass(cls) -> None: def setUp(self) -> None: super().setUp() config_settings = self.get_default_map() - self.config_map = ClientConfigAdapter(AvellanedaMarketMakingMakerConfigMap(**config_settings)) + self.config_map = ClientConfigAdapter(AvellanedaMarketMakingConfigMap(**config_settings)) def async_run_with_timeout(self, coroutine: Awaitable, timeout: int = 1): ret = self.ev_loop.run_until_complete(asyncio.wait_for(coroutine, timeout)) @@ -59,7 +59,7 @@ def get_default_map(self) -> Dict[str, str]: return config_settings def test_initial_sequential_build(self): - config_map = ClientConfigAdapter(AvellanedaMarketMakingMakerConfigMap.construct()) + config_map = ClientConfigAdapter(AvellanedaMarketMakingConfigMap.construct()) config_settings = self.get_default_map() def build_config_map(cm: ClientConfigAdapter, cs: Dict): @@ -205,7 +205,7 @@ def test_load_configs_from_yaml(self): with open(f_path, "r") as file: data = yaml.safe_load(file) - loaded_config_map = ClientConfigAdapter(AvellanedaMarketMakingMakerConfigMap(**data)) + loaded_config_map = ClientConfigAdapter(AvellanedaMarketMakingConfigMap(**data)) self.assertEqual(self.config_map, loaded_config_map) From 686ac6b06266f9f6b65b39dcffd14e6924e4fa51 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Wed, 18 May 2022 22:33:50 +0200 Subject: [PATCH 093/152] (feat) fix --- .../test_avellaneda_market_making_config_map_pydantic.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py index f2e1ab315a..7a8f963cf5 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py @@ -28,7 +28,7 @@ class AvellanedaMarketMakingConfigMapPydanticTest(unittest.TestCase): def setUpClass(cls) -> None: super().setUpClass() cls.ev_loop = asyncio.get_event_loop() - cls.market = "binance" + cls.exchange = "binance" cls.base_asset = "COINALPHA" cls.quote_asset = "HBOT" cls.trading_pair = f"{cls.base_asset}-{cls.quote_asset}" @@ -44,8 +44,8 @@ def async_run_with_timeout(self, coroutine: Awaitable, timeout: int = 1): def get_default_map(self) -> Dict[str, str]: config_settings = { - "market": self.market, - "trading_pair": self.trading_pair, + "exchange": self.exchange, + "market": self.trading_pair, "execution_timeframe_mode": { "start_time": "09:30:00", "end_time": "16:00:00", @@ -91,7 +91,7 @@ def test_maker_trading_pair_prompt(self): exchange = self.config_map.exchange example = AllConnectorSettings.get_example_pairs().get(exchange) - prompt = self.async_run_with_timeout(self.config_map.get_client_prompt("market")) + prompt = self.async_run_with_timeout(self.config_map.get_client_prompt("exchange")) expected = f"Enter the token trading pair you would like to trade on {exchange} (e.g. {example})" self.assertEqual(expected, prompt) From d8f05b92573b83365cf0b2b857480b26d659c38f Mon Sep 17 00:00:00 2001 From: mhrvth Date: Wed, 18 May 2022 22:45:46 +0200 Subject: [PATCH 094/152] (feat) refactor --- .../test_avellaneda_market_making_start.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py index 8dcf482ad6..431ca41c08 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py @@ -8,7 +8,7 @@ from hummingbot.connector.exchange_base import ExchangeBase from hummingbot.connector.utils import combine_to_hb_trading_pair from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( - AvellanedaMarketMakingMakerConfigMap, + AvellanedaMarketMakingConfigMap, FromDateToDateModel, MultiOrderLevelModel, TrackHangingOrdersModel, @@ -28,7 +28,7 @@ def setUp(self) -> None: self.base = "ETH" self.quote = "BTC" self.strategy_config_map = ClientConfigAdapter( - AvellanedaMarketMakingMakerConfigMap( + AvellanedaMarketMakingConfigMap( market="binance", trading_pair=combine_to_hb_trading_pair(self.base, self.quote), execution_timeframe_mode=FromDateToDateModel( From f2a2fbb7823d21d3039f315b770afed2ada80cd1 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Thu, 19 May 2022 07:58:20 +0200 Subject: [PATCH 095/152] (feat) tests --- .../test_avellaneda_market_making_config_map_pydantic.py | 2 +- .../test_avellaneda_market_making_start.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py index 7a8f963cf5..8bb531897c 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_config_map_pydantic.py @@ -91,7 +91,7 @@ def test_maker_trading_pair_prompt(self): exchange = self.config_map.exchange example = AllConnectorSettings.get_example_pairs().get(exchange) - prompt = self.async_run_with_timeout(self.config_map.get_client_prompt("exchange")) + prompt = self.async_run_with_timeout(self.config_map.get_client_prompt("market")) expected = f"Enter the token trading pair you would like to trade on {exchange} (e.g. {example})" self.assertEqual(expected, prompt) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py index 431ca41c08..dce3f1dd33 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py @@ -29,8 +29,8 @@ def setUp(self) -> None: self.quote = "BTC" self.strategy_config_map = ClientConfigAdapter( AvellanedaMarketMakingConfigMap( - market="binance", - trading_pair=combine_to_hb_trading_pair(self.base, self.quote), + exchange="binance", + market=combine_to_hb_trading_pair(self.base, self.quote), execution_timeframe_mode=FromDateToDateModel( start_datetime="2021-11-18 15:00:00", end_datetime="2021-11-18 16:00:00", From e8200ad718eb465e2a8bf6fc12d0f7965aaa2811 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Thu, 19 May 2022 08:14:06 +0200 Subject: [PATCH 096/152] (feat) refactor --- hummingbot/client/config/config_data_types.py | 36 +++++++++---------- ...hange_market_making_config_map_pydantic.py | 22 ++++++------ .../cross_exchange_market_making/start.py | 8 ++--- .../test_cross_exchange_market_making.py | 16 ++++----- ...test_cross_exchange_market_making_start.py | 8 ++--- 5 files changed, 45 insertions(+), 45 deletions(-) diff --git a/hummingbot/client/config/config_data_types.py b/hummingbot/client/config/config_data_types.py index 36855e3845..a03bccd45b 100644 --- a/hummingbot/client/config/config_data_types.py +++ b/hummingbot/client/config/config_data_types.py @@ -123,7 +123,7 @@ def validate_exchange_trading_pair(cls, v: str, values: Dict): class BaseTradingStrategyMakerTakerConfigMap(BaseStrategyConfigMap): - market_maker: str = Field( + maker_market: str = Field( default=..., description="", client_data=ClientFieldData( @@ -131,7 +131,7 @@ class BaseTradingStrategyMakerTakerConfigMap(BaseStrategyConfigMap): prompt_on_new=True, ), ) - market_taker: str = Field( + taker_market: str = Field( default=..., description="", client_data=ClientFieldData( @@ -139,7 +139,7 @@ class BaseTradingStrategyMakerTakerConfigMap(BaseStrategyConfigMap): prompt_on_new=True, ), ) - trading_pair_maker: str = Field( + maker_market_trading_pair: str = Field( default=..., description="", client_data=ClientFieldData( @@ -147,7 +147,7 @@ class BaseTradingStrategyMakerTakerConfigMap(BaseStrategyConfigMap): prompt_on_new=True, ), ) - trading_pair_taker: str = Field( + taker_market_trading_pair: str = Field( default=..., description="", client_data=ClientFieldData( @@ -159,11 +159,11 @@ class BaseTradingStrategyMakerTakerConfigMap(BaseStrategyConfigMap): @classmethod def trading_pair_prompt(cls, model_instance: 'BaseTradingStrategyMakerTakerConfigMap', is_maker: bool) -> str: if is_maker: - exchange = model_instance.market_maker + exchange = model_instance.maker_market example = AllConnectorSettings.get_example_pairs().get(exchange) market_type = "maker" else: - exchange = model_instance.market_taker + exchange = model_instance.taker_market example = AllConnectorSettings.get_example_pairs().get(exchange) market_type = "taker" return ( @@ -172,8 +172,8 @@ def trading_pair_prompt(cls, model_instance: 'BaseTradingStrategyMakerTakerConfi ) @validator( - "market_maker", - "market_taker", + "maker_market", + "taker_market", pre=True ) def validate_exchange(cls, v: str, values: Dict, config: BaseModel.Config, field: Field): @@ -181,14 +181,14 @@ def validate_exchange(cls, v: str, values: Dict, config: BaseModel.Config, field ret = validate_exchange(v) if ret is not None: raise ValueError(ret) - if field.name == "trading_pair_maker": - cls.__fields__["market_maker"].type_ = ClientConfigEnum( # rebuild the exchanges enum + if field.name == "maker_market_trading_pair": + cls.__fields__["maker_market"].type_ = ClientConfigEnum( # rebuild the exchanges enum value="Exchanges", # noqa: F821 names={e: e for e in AllConnectorSettings.get_connector_settings().keys()}, type=str, ) - if field.name == "trading_pair_taker": - cls.__fields__["market_taker"].type_ = ClientConfigEnum( # rebuild the exchanges enum + if field.name == "taker_market_trading_pair": + cls.__fields__["taker_market"].type_ = ClientConfigEnum( # rebuild the exchanges enum value="Exchanges", # noqa: F821 names={e: e for e in AllConnectorSettings.get_connector_settings().keys()}, type=str, @@ -196,16 +196,16 @@ def validate_exchange(cls, v: str, values: Dict, config: BaseModel.Config, field return v @validator( - "trading_pair_maker", - "trading_pair_taker", + "maker_market_trading_pair", + "taker_market_trading_pair", pre=True, ) def validate_exchange_trading_pair(cls, v: str, values: Dict, config: BaseModel.Config, field: Field): - if field.name == "trading_pair_maker": - exchange = values.get("market_maker") + if field.name == "maker_market_trading_pair": + exchange = values.get("maker_market") ret = validate_market_trading_pair(exchange, v) - if field.name == "trading_pair_taker": - exchange = values.get("market_taker") + if field.name == "taker_market_trading_pair": + exchange = values.get("taker_market") ret = validate_market_trading_pair(exchange, v) if ret is not None: raise ValueError(ret) diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py index c478158e7a..3bebc58d36 100644 --- a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py @@ -163,13 +163,13 @@ class CrossExchangeMarketMakingConfigMap(BaseTradingStrategyMakerTakerConfigMap) @classmethod def top_depth_tolerance_prompt(cls, model_instance: 'CrossExchangeMarketMakingConfigMap') -> str: - market_maker = model_instance.trading_pair_maker - base_asset, quote_asset = market_maker.split("-") + maker_market = model_instance.maker_market_trading_pair + base_asset, quote_asset = maker_market.split("-") return f"What is your top depth tolerance? (in {base_asset})" @classmethod def order_amount_prompt(cls, model_instance: 'CrossExchangeMarketMakingConfigMap') -> str: - trading_pair = model_instance.trading_pair_maker + trading_pair = model_instance.maker_market_trading_pair base_asset, quote_asset = trading_pair.split("-") return f"What is the amount of {base_asset} per order?" @@ -252,20 +252,20 @@ def post_validations(cls, values: Dict): @classmethod def exchange_post_validation(cls, values: Dict): - if "market_maker" in values.keys(): - settings.required_exchanges.add(values["market_maker"]) - if "market_taker" in values.keys(): - settings.required_exchanges.add(values["market_taker"]) + if "maker_market" in values.keys(): + settings.required_exchanges.add(values["maker_market"]) + if "taker_market" in values.keys(): + settings.required_exchanges.add(values["taker_market"]) @classmethod def update_oracle_settings(cls, values: str): if not ("use_oracle_conversion_rate" in values.keys() and - "trading_pair_maker" in values.keys() and - "trading_pair_taker" in values.keys()): + "maker_market_trading_pair" in values.keys() and + "taker_market_trading_pair" in values.keys()): return use_oracle = values["use_oracle_conversion_rate"] - first_base, first_quote = values["trading_pair_maker"].split("-") - second_base, second_quote = values["trading_pair_taker"].split("-") + first_base, first_quote = values["maker_market_trading_pair"].split("-") + second_base, second_quote = values["taker_market_trading_pair"].split("-") if use_oracle and (first_base != second_base or first_quote != second_quote): settings.required_rate_oracle = True settings.rate_oracle_pairs = [] diff --git a/hummingbot/strategy/cross_exchange_market_making/start.py b/hummingbot/strategy/cross_exchange_market_making/start.py index f33d821520..3624efcaec 100644 --- a/hummingbot/strategy/cross_exchange_market_making/start.py +++ b/hummingbot/strategy/cross_exchange_market_making/start.py @@ -10,10 +10,10 @@ def start(self): c_map = self.strategy_config_map - maker_market = c_map.market_maker.lower() - taker_market = c_map.market_taker.lower() - raw_maker_trading_pair = c_map.trading_pair_maker - raw_taker_trading_pair = c_map.trading_pair_taker + maker_market = c_map.maker_market.lower() + taker_market = c_map.taker_market.lower() + raw_maker_trading_pair = c_map.maker_market_trading_pair + raw_taker_trading_pair = c_map.taker_market_trading_pair status_report_interval = global_config_map.get("strategy_report_interval").value try: diff --git a/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making.py b/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making.py index 71a1e35a2f..5566137905 100644 --- a/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making.py +++ b/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making.py @@ -66,10 +66,10 @@ def setUp(self): ) self.config_map_raw = CrossExchangeMarketMakingConfigMap( - market_maker=self.exchange_name_maker, - market_taker=self.exchange_name_taker, - trading_pair_maker=self.trading_pairs_maker[0], - trading_pair_taker=self.trading_pairs_taker[0], + maker_market=self.exchange_name_maker, + taker_market=self.exchange_name_taker, + maker_market_trading_pair=self.trading_pairs_maker[0], + taker_market_trading_pair=self.trading_pairs_taker[0], min_profitability=Decimal(self.min_profitability), slippage_buffer=Decimal("0"), order_amount=Decimal("0"), @@ -655,10 +655,10 @@ def test_price_and_size_limit_calculation_with_slippage_buffer(self): config_map_with_slippage_buffer = ClientConfigAdapter( CrossExchangeMarketMakingConfigMap( - market_maker=self.exchange_name_maker, - market_taker=self.exchange_name_taker, - trading_pair_maker=self.trading_pairs_maker[0], - trading_pair_taker=self.trading_pairs_taker[0], + maker_market=self.exchange_name_maker, + taker_market=self.exchange_name_taker, + maker_market_trading_pair=self.trading_pairs_maker[0], + taker_market_trading_pair=self.trading_pairs_taker[0], order_amount=Decimal("4"), min_profitability=Decimal("25"), order_size_taker_volume_factor=Decimal("100"), diff --git a/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_start.py b/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_start.py index 1762bc8422..9d653ac1b6 100644 --- a/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_start.py +++ b/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_start.py @@ -21,10 +21,10 @@ def setUp(self) -> None: self.strategy_config_map = ClientConfigAdapter( CrossExchangeMarketMakingConfigMap( - market_maker="binance", - market_taker="kucoin", - trading_pair_maker="ETH-USDT", - trading_pair_taker="ETH-USDT", + maker_market="binance", + taker_market="kucoin", + maker_market_trading_pair="ETH-USDT", + taker_market_trading_pair="ETH-USDT", order_amount=1.0, min_profitability=2.0, use_oracle_conversion_rate=False, From f572aa5ae24b135cdee1fa7de0890c42435e3c54 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Thu, 19 May 2022 08:43:14 +0200 Subject: [PATCH 097/152] (feat) removed XEMM from test --- test/hummingbot/client/config/test_config_templates.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/hummingbot/client/config/test_config_templates.py b/test/hummingbot/client/config/test_config_templates.py index ba39115eaf..d137cec30c 100644 --- a/test/hummingbot/client/config/test_config_templates.py +++ b/test/hummingbot/client/config/test_config_templates.py @@ -36,7 +36,6 @@ def test_strategy_config_template_complete_legacy(self): "arbitrage", "aroon_oscillator", "celo_arb", - "cross_exchange_market_making", "dev_0_hello_world", "dev_1_get_order_book", "dev_2_perform_trade", From 1b067c5925fefdc663920af651250449451cbd80 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Thu, 19 May 2022 09:59:18 +0300 Subject: [PATCH 098/152] (fix) Reverting previous changes to connect command Also, removing debugging timeout values when testing asyncio functions in unit-tests. --- hummingbot/client/command/connect_command.py | 43 +++++++++++-------- .../client/command/test_config_command.py | 2 +- .../client/command/test_connect_command.py | 6 +-- .../client/command/test_history_command.py | 6 +-- 4 files changed, 31 insertions(+), 26 deletions(-) diff --git a/hummingbot/client/command/connect_command.py b/hummingbot/client/command/connect_command.py index 26b63f50b2..f75a9a6d91 100644 --- a/hummingbot/client/command/connect_command.py +++ b/hummingbot/client/command/connect_command.py @@ -50,6 +50,8 @@ async def connect_exchange(self, # type: HummingbotApplication connector_config = ClientConfigAdapter(CELO_KEYS) else: connector_config = ClientConfigAdapter(AllConnectorSettings.get_connector_config_keys(connector_name)) + to_connect = True + previous_keys = None if Security.connector_config_file_exists(connector_name): await Security.wait_til_decryption_done() api_key_config = [ @@ -66,27 +68,30 @@ async def connect_exchange(self, # type: HummingbotApplication if self.app.to_stop_config: self.app.to_stop_config = False return - if answer.lower() in ("yes", "y"): + if answer.lower() not in ("yes", "y"): + to_connect = False + else: previous_keys = Security.api_keys(connector_name) - await self.prompt_for_model_config(connector_config) - self.app.change_prompt(prompt=">>> ") - if self.app.to_stop_config: - self.app.to_stop_config = False - return - Security.update_secure_config(connector_config) - if connector_name == "celo": - err_msg = await self.validate_n_connect_celo(to_reconnect=True) - else: - err_msg = await self.validate_n_connect_connector(connector_name) - if err_msg is None: - self.notify(f"\nYou are now connected to {connector_name}.") + if to_connect: + await self.prompt_for_model_config(connector_config) + self.app.change_prompt(prompt=">>> ") + if self.app.to_stop_config: + self.app.to_stop_config = False + return + Security.update_secure_config(connector_config) + if connector_name == "celo": + err_msg = await self.validate_n_connect_celo(to_reconnect=True) + else: + err_msg = await self.validate_n_connect_connector(connector_name) + if err_msg is None: + self.notify(f"\nYou are now connected to {connector_name}.") + else: + self.notify(f"\nError: {err_msg}") + if previous_keys is not None: + previous_config = ClientConfigAdapter(connector_config.hb_config.__class__(**previous_keys)) + Security.update_secure_config(previous_config) else: - self.notify(f"\nError: {err_msg}") - if previous_keys is not None: - previous_config = ClientConfigAdapter(connector_config.hb_config.__class__(**previous_keys)) - Security.update_secure_config(previous_config) - else: - Security.remove_secure_config(connector_name) + Security.remove_secure_config(connector_name) self.placeholder_mode = False self.app.hide_input = False self.app.change_prompt(prompt=">>> ") diff --git a/test/hummingbot/client/command/test_config_command.py b/test/hummingbot/client/command/test_config_command.py index 1529a3ad70..1e751235f9 100644 --- a/test/hummingbot/client/command/test_config_command.py +++ b/test/hummingbot/client/command/test_config_command.py @@ -237,7 +237,7 @@ class Config: save_to_yml_mock.reset_mock() self.cli_mock_assistant.queue_prompt_reply("another value") - self.async_run_with_timeout(self.app._config_single_key(key="nested_model.nested_attr", input_value=None), 10000) + self.async_run_with_timeout(self.app._config_single_key(key="nested_model.nested_attr", input_value=None)) self.assertEqual("another value", config_map.nested_model.nested_attr) save_to_yml_mock.assert_called_once() diff --git a/test/hummingbot/client/command/test_connect_command.py b/test/hummingbot/client/command/test_connect_command.py index 4dd5f25b59..ffef970e49 100644 --- a/test/hummingbot/client/command/test_connect_command.py +++ b/test/hummingbot/client/command/test_connect_command.py @@ -112,7 +112,7 @@ def test_connect_celo_success( celo_cli_mock.validate_node_synced.return_value = None celo_cli_mock.unlock_account.return_value = None - self.async_run_with_timeout(self.app.connect_exchange(exchange), 1000) + self.async_run_with_timeout(self.app.connect_exchange(exchange)) self.assertTrue(self.cli_mock_assistant.check_log_called_with(msg="\nYou are now connected to celo.")) self.assertFalse(self.app.placeholder_mode) self.assertFalse(self.app.app.hide_input) @@ -140,7 +140,7 @@ def test_connect_exchange_handles_network_timeouts( self.cli_mock_assistant.queue_prompt_reply(api_secret) # binance API secret with self.assertRaises(asyncio.TimeoutError): - self.async_run_with_timeout_coroutine_must_raise_timeout(self.app.connect_exchange("binance"), timeout=1000000) + self.async_run_with_timeout_coroutine_must_raise_timeout(self.app.connect_exchange("binance")) self.assertTrue( self.cli_mock_assistant.check_log_called_with( msg="\nA network error prevented the connection to complete. See logs for more details." @@ -156,7 +156,7 @@ def test_connection_df_handles_network_timeouts(self, _: AsyncMock, update_excha global_config_map["other_commands_timeout"].value = 0.01 with self.assertRaises(asyncio.TimeoutError): - self.async_run_with_timeout_coroutine_must_raise_timeout(self.app.connection_df(), 10000) + self.async_run_with_timeout_coroutine_must_raise_timeout(self.app.connection_df()) self.assertTrue( self.cli_mock_assistant.check_log_called_with( msg="\nA network error prevented the connection table to populate. See logs for more details." diff --git a/test/hummingbot/client/command/test_history_command.py b/test/hummingbot/client/command/test_history_command.py index de3bfc7b29..4d1bc39a8c 100644 --- a/test/hummingbot/client/command/test_history_command.py +++ b/test/hummingbot/client/command/test_history_command.py @@ -5,8 +5,9 @@ from copy import deepcopy from decimal import Decimal from pathlib import Path +from test.mock.mock_cli import CLIMockingAssistant from typing import Awaitable, List -from unittest.mock import patch, MagicMock, AsyncMock +from unittest.mock import AsyncMock, MagicMock, patch from hummingbot.client.config.config_helpers import read_system_configs_from_yml from hummingbot.client.config.global_config_map import global_config_map @@ -16,7 +17,6 @@ from hummingbot.model.order import Order from hummingbot.model.sql_connection_manager import SQLConnectionManager from hummingbot.model.trade_fill import TradeFill -from test.mock.mock_cli import CLIMockingAssistant class HistoryCommandTest(unittest.TestCase): @@ -104,7 +104,7 @@ def test_history_report_raises_on_get_current_balances_network_timeout(self, get with self.assertRaises(asyncio.TimeoutError): self.async_run_with_timeout_coroutine_must_raise_timeout( - self.app.history_report(start_time=time.time(), trades=trades), 10000 + self.app.history_report(start_time=time.time(), trades=trades) ) self.assertTrue( self.cli_mock_assistant.check_log_called_with( From 8e0d4ba82d2ef26f15a5be2abcbaf18a9d60d763 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Thu, 19 May 2022 10:04:39 +0200 Subject: [PATCH 099/152] (feat) descriptions --- hummingbot/client/config/config_data_types.py | 8 +++--- ...hange_market_making_config_map_pydantic.py | 28 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/hummingbot/client/config/config_data_types.py b/hummingbot/client/config/config_data_types.py index a03bccd45b..7f9c92b046 100644 --- a/hummingbot/client/config/config_data_types.py +++ b/hummingbot/client/config/config_data_types.py @@ -125,7 +125,7 @@ def validate_exchange_trading_pair(cls, v: str, values: Dict): class BaseTradingStrategyMakerTakerConfigMap(BaseStrategyConfigMap): maker_market: str = Field( default=..., - description="", + description="The name of the maker exchange connector.", client_data=ClientFieldData( prompt=lambda mi: "Enter your maker spot connector", prompt_on_new=True, @@ -133,7 +133,7 @@ class BaseTradingStrategyMakerTakerConfigMap(BaseStrategyConfigMap): ) taker_market: str = Field( default=..., - description="", + description="The name of the taker exchange connector.", client_data=ClientFieldData( prompt=lambda mi: "Enter your taker spot connector", prompt_on_new=True, @@ -141,7 +141,7 @@ class BaseTradingStrategyMakerTakerConfigMap(BaseStrategyConfigMap): ) maker_market_trading_pair: str = Field( default=..., - description="", + description="The name of the maker trading pair.", client_data=ClientFieldData( prompt=lambda mi: BaseTradingStrategyMakerTakerConfigMap.trading_pair_prompt(mi, True), prompt_on_new=True, @@ -149,7 +149,7 @@ class BaseTradingStrategyMakerTakerConfigMap(BaseStrategyConfigMap): ) taker_market_trading_pair: str = Field( default=..., - description="", + description="The name of the taker trading pair.", client_data=ClientFieldData( prompt=lambda mi: BaseTradingStrategyMakerTakerConfigMap.trading_pair_prompt(mi, False), prompt_on_new=True, diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py index 3bebc58d36..44d4295322 100644 --- a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py @@ -13,7 +13,7 @@ class CrossExchangeMarketMakingConfigMap(BaseTradingStrategyMakerTakerConfigMap) min_profitability: Decimal = Field( default=..., - description="", + description="The minimum estimated profitability required to open a position.", ge=-100.0, le=100.0, client_data=ClientFieldData( @@ -32,21 +32,21 @@ class CrossExchangeMarketMakingConfigMap(BaseTradingStrategyMakerTakerConfigMap) ) adjust_order_enabled: bool = Field( default=True, - description="", + description="Adjust order price to be one tick above the top bid or below the top ask.", client_data=ClientFieldData( prompt=lambda mi: "Do you want to enable adjust order? (Yes/No)" ), ) active_order_canceling: bool = Field( default=True, - description="", + description="An option to refresh orders by cancellation instead of letting them expire.", client_data=ClientFieldData( prompt=lambda mi: "Do you want to enable active order canceling? (Yes/No)" ), ) cancel_order_threshold: Decimal = Field( default=Decimal("5.0"), - description="", + description="Profitability threshold to cancel a trade.", gt=-100.0, lt=100.0, client_data=ClientFieldData( @@ -55,7 +55,7 @@ class CrossExchangeMarketMakingConfigMap(BaseTradingStrategyMakerTakerConfigMap) ) limit_order_min_expiration: float = Field( default=130.0, - description="", + description="Limit order expiration time limit.", gt=0.0, client_data=ClientFieldData( prompt=lambda mi: "How often do you want limit orders to expire (in seconds)?", @@ -63,7 +63,7 @@ class CrossExchangeMarketMakingConfigMap(BaseTradingStrategyMakerTakerConfigMap) ) top_depth_tolerance: Decimal = Field( default=Decimal("0.0"), - description="", + description="Volume requirement for determning a possible top bid or ask price from the order book.", ge=0.0, client_data=ClientFieldData( prompt=lambda mi: CrossExchangeMarketMakingConfigMap.top_depth_tolerance_prompt(mi), @@ -71,7 +71,7 @@ class CrossExchangeMarketMakingConfigMap(BaseTradingStrategyMakerTakerConfigMap) ) anti_hysteresis_duration: float = Field( default=60.0, - description="", + description="Minimum time limit between two subsequent order adjustments.", gt=0.0, client_data=ClientFieldData( prompt=lambda mi: "What is the minimum time interval you want limit orders to be adjusted? (in seconds)", @@ -79,7 +79,7 @@ class CrossExchangeMarketMakingConfigMap(BaseTradingStrategyMakerTakerConfigMap) ) order_size_taker_volume_factor: Decimal = Field( default=Decimal("25.0"), - description="", + description="Taker order size as a percentage of volume.", ge=0.0, le=100.0, client_data=ClientFieldData( @@ -91,7 +91,7 @@ class CrossExchangeMarketMakingConfigMap(BaseTradingStrategyMakerTakerConfigMap) ) order_size_taker_balance_factor: Decimal = Field( default=Decimal("99.5"), - description="", + description="Taker order size as a percentage of the available balance.", ge=0.0, le=100.0, client_data=ClientFieldData( @@ -103,7 +103,7 @@ class CrossExchangeMarketMakingConfigMap(BaseTradingStrategyMakerTakerConfigMap) ) order_size_portfolio_ratio_limit: Decimal = Field( default=Decimal("16.67"), - description="", + description="Order size as a maker and taker account balance ratio.", ge=0.0, le=100.0, client_data=ClientFieldData( @@ -115,7 +115,7 @@ class CrossExchangeMarketMakingConfigMap(BaseTradingStrategyMakerTakerConfigMap) ) use_oracle_conversion_rate: bool = Field( default=True, - description="", + description="Use rate oracle to determine a conversion rate between two different trading pairs.", client_data=ClientFieldData( prompt=lambda mi: "Do you want to use rate oracle on unmatched trading pairs? (Yes/No)", prompt_on_new=True, @@ -123,7 +123,7 @@ class CrossExchangeMarketMakingConfigMap(BaseTradingStrategyMakerTakerConfigMap) ) taker_to_maker_base_conversion_rate: Decimal = Field( default=Decimal("1.0"), - description="", + description="A fixed conversion rate between the maker and taker tradin pairs based on the maker base asset.", gt=0.0, client_data=ClientFieldData( prompt=lambda mi: ( @@ -135,7 +135,7 @@ class CrossExchangeMarketMakingConfigMap(BaseTradingStrategyMakerTakerConfigMap) ) taker_to_maker_quote_conversion_rate: Decimal = Field( default=Decimal("1.0"), - description="", + description="A fixed conversion rate between the maker and taker tradin pairs based on the maker quote asset.", gt=0.0, client_data=ClientFieldData( prompt=lambda mi: ( @@ -147,7 +147,7 @@ class CrossExchangeMarketMakingConfigMap(BaseTradingStrategyMakerTakerConfigMap) ) slippage_buffer: Decimal = Field( default=Decimal("5.0"), - description="", + description="Allowed slippage to fill ensure taker orders are filled.", ge=0.0, le=100.0, client_data=ClientFieldData( From 39915090264270bb5562c42c8d46d53089a8278f Mon Sep 17 00:00:00 2001 From: mhrvth <7762229+mhrvth@users.noreply.github.com> Date: Thu, 19 May 2022 14:06:52 +0200 Subject: [PATCH 100/152] Update hummingbot/client/config/config_data_types.py Co-authored-by: Petio Petrov --- hummingbot/client/config/config_data_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/client/config/config_data_types.py b/hummingbot/client/config/config_data_types.py index a03bccd45b..0ca1877b5b 100644 --- a/hummingbot/client/config/config_data_types.py +++ b/hummingbot/client/config/config_data_types.py @@ -176,7 +176,7 @@ def trading_pair_prompt(cls, model_instance: 'BaseTradingStrategyMakerTakerConfi "taker_market", pre=True ) - def validate_exchange(cls, v: str, values: Dict, config: BaseModel.Config, field: Field): + def validate_exchange(cls, v: str, field: Field): """Used for client-friendly error output.""" ret = validate_exchange(v) if ret is not None: From 1b68a12bb7b4e67637f4607787d929e3fbe335ae Mon Sep 17 00:00:00 2001 From: mhrvth <7762229+mhrvth@users.noreply.github.com> Date: Thu, 19 May 2022 14:07:06 +0200 Subject: [PATCH 101/152] Update hummingbot/client/config/config_data_types.py Co-authored-by: Petio Petrov --- hummingbot/client/config/config_data_types.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hummingbot/client/config/config_data_types.py b/hummingbot/client/config/config_data_types.py index 0ca1877b5b..0e611b81d6 100644 --- a/hummingbot/client/config/config_data_types.py +++ b/hummingbot/client/config/config_data_types.py @@ -200,7 +200,8 @@ def validate_exchange(cls, v: str, field: Field): "taker_market_trading_pair", pre=True, ) - def validate_exchange_trading_pair(cls, v: str, values: Dict, config: BaseModel.Config, field: Field): + def validate_exchange_trading_pair(cls, v: str, values: Dict, field: Field): + ret = None if field.name == "maker_market_trading_pair": exchange = values.get("maker_market") ret = validate_market_trading_pair(exchange, v) From c824fd86d859a69d7b9fd0252d0f8f988a38633b Mon Sep 17 00:00:00 2001 From: mhrvth <7762229+mhrvth@users.noreply.github.com> Date: Thu, 19 May 2022 14:07:34 +0200 Subject: [PATCH 102/152] Update hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pyx Co-authored-by: Petio Petrov --- .../cross_exchange_market_making.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pyx b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pyx index 752f7e63d4..1a5fa37318 100755 --- a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pyx +++ b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pyx @@ -771,7 +771,7 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): cdef: ExchangeBase maker_market = market_pair.maker.market str trading_pair = market_pair.maker.trading_pair - if self.order_amount and self.order_amount > 0: + if self.order_amount > 0: base_order_size = self.order_amount return maker_market.c_quantize_order_amount(trading_pair, Decimal(base_order_size)) else: From c230e6df7b8bfc28b3847d51c7e8ee6b55c3752d Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Fri, 20 May 2022 14:34:17 +0300 Subject: [PATCH 103/152] (refactor) Refactors connect command to simplify the code --- hummingbot/client/command/connect_command.py | 53 ++++++++++---------- 1 file changed, 27 insertions(+), 26 deletions(-) diff --git a/hummingbot/client/command/connect_command.py b/hummingbot/client/command/connect_command.py index f75a9a6d91..592f6b7cc6 100644 --- a/hummingbot/client/command/connect_command.py +++ b/hummingbot/client/command/connect_command.py @@ -1,5 +1,5 @@ import asyncio -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING, Dict, Optional import pandas as pd @@ -50,8 +50,6 @@ async def connect_exchange(self, # type: HummingbotApplication connector_config = ClientConfigAdapter(CELO_KEYS) else: connector_config = ClientConfigAdapter(AllConnectorSettings.get_connector_config_keys(connector_name)) - to_connect = True - previous_keys = None if Security.connector_config_file_exists(connector_name): await Security.wait_til_decryption_done() api_key_config = [ @@ -68,30 +66,11 @@ async def connect_exchange(self, # type: HummingbotApplication if self.app.to_stop_config: self.app.to_stop_config = False return - if answer.lower() not in ("yes", "y"): - to_connect = False - else: + if answer.lower() in ("yes", "y"): previous_keys = Security.api_keys(connector_name) - if to_connect: - await self.prompt_for_model_config(connector_config) - self.app.change_prompt(prompt=">>> ") - if self.app.to_stop_config: - self.app.to_stop_config = False - return - Security.update_secure_config(connector_config) - if connector_name == "celo": - err_msg = await self.validate_n_connect_celo(to_reconnect=True) - else: - err_msg = await self.validate_n_connect_connector(connector_name) - if err_msg is None: - self.notify(f"\nYou are now connected to {connector_name}.") - else: - self.notify(f"\nError: {err_msg}") - if previous_keys is not None: - previous_config = ClientConfigAdapter(connector_config.hb_config.__class__(**previous_keys)) - Security.update_secure_config(previous_config) - else: - Security.remove_secure_config(connector_name) + await self._perform_connect(connector_config, previous_keys) + else: + await self._perform_connect(connector_config) self.placeholder_mode = False self.app.hide_input = False self.app.change_prompt(prompt=">>> ") @@ -182,3 +161,25 @@ async def validate_n_connect_connector(self, connector_name: str) -> Optional[st self.app.change_prompt(prompt=">>> ") raise return err_msg + + async def _perform_connect(self, connector_config: ClientConfigAdapter, previous_keys: Optional[Dict] = None): + connector_name = connector_config.connector + await self.prompt_for_model_config(connector_config) + self.app.change_prompt(prompt=">>> ") + if self.app.to_stop_config: + self.app.to_stop_config = False + return + Security.update_secure_config(connector_config) + if connector_name == "celo": + err_msg = await self.validate_n_connect_celo(to_reconnect=True) + else: + err_msg = await self.validate_n_connect_connector(connector_name) + if err_msg is None: + self.notify(f"\nYou are now connected to {connector_name}.") + else: + self.notify(f"\nError: {err_msg}") + if previous_keys is not None: + previous_config = ClientConfigAdapter(connector_config.hb_config.__class__(**previous_keys)) + Security.update_secure_config(previous_config) + else: + Security.remove_secure_config(connector_name) From a5dc6d19e71cea1d30700ad4e8abbce15498d5a3 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Fri, 20 May 2022 18:35:25 +0200 Subject: [PATCH 104/152] (fix) create command --- hummingbot/client/command/create_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/client/command/create_command.py b/hummingbot/client/command/create_command.py index 1e3bd60ffe..efb38b14e1 100644 --- a/hummingbot/client/command/create_command.py +++ b/hummingbot/client/command/create_command.py @@ -98,7 +98,7 @@ async def prompt_for_model_config( client_data = config_map.get_client_data(key) if ( client_data is not None - and (client_data.prompt_on_new and config_map.is_required(key)) + and (client_data.prompt_on_new or config_map.is_required(key)) ): await self.prompt_a_config(config_map, key) if self.app.to_stop_config: From 9096f07f63dd358d02d7e4f0ac257bb5ed68440e Mon Sep 17 00:00:00 2001 From: mhrvth Date: Fri, 20 May 2022 18:35:39 +0200 Subject: [PATCH 105/152] (fix) avellaneda --- ...aneda_market_making_config_map_pydantic.py | 6 +- ...laneda_market_making_strategy_TEMPLATE.yml | 86 ------------------- 2 files changed, 4 insertions(+), 88 deletions(-) delete mode 100644 hummingbot/templates/conf_avellaneda_market_making_strategy_TEMPLATE.yml diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py index 5cb6115bc0..1763cdad2e 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -97,7 +97,8 @@ class MultiOrderLevelModel(BaseClientModel): description="The number of orders placed on either side of the order book.", ge=2, client_data=ClientFieldData( - prompt=lambda mi: "How many orders do you want to place on both sides?" + prompt=lambda mi: "How many orders do you want to place on both sides?", + prompt_on_new=True, ), ) level_distances: Decimal = Field( @@ -105,7 +106,8 @@ class MultiOrderLevelModel(BaseClientModel): description="The spread between order levels, expressed in % of optimal spread.", ge=0, client_data=ClientFieldData( - prompt=lambda mi: "How far apart in % of optimal spread should orders on one side be?" + prompt=lambda mi: "How far apart in % of optimal spread should orders on one side be?", + prompt_on_new=True, ), ) diff --git a/hummingbot/templates/conf_avellaneda_market_making_strategy_TEMPLATE.yml b/hummingbot/templates/conf_avellaneda_market_making_strategy_TEMPLATE.yml deleted file mode 100644 index 34e2a1d770..0000000000 --- a/hummingbot/templates/conf_avellaneda_market_making_strategy_TEMPLATE.yml +++ /dev/null @@ -1,86 +0,0 @@ -######################################################## -### Avellaneda market making strategy config ### -######################################################## - -template_version: 8 -strategy: null - -# Exchange and token parameters. -exchange: null - -# Token trading pair for the exchange, e.g. BTC-USDT -market: null - -# Type of execution timeframe -execution_timeframe: null - -# Start time for running between specified times of a day or between specified dates -start_time: null - -# End time for running between specified times of a day or between specified dates -end_time: null - -# Time in seconds before cancelling and placing new orders. -# If the value is 60, the bot cancels active orders and placing new ones after a minute. -order_refresh_time: null - -# Whether to enable order optimization mode (true/false). -order_optimization_enabled: true - -# Time in seconds before replacing existing order with new orders at the same price. -max_order_age: null - -# The spread (from mid price) to defer order refresh process to the next cycle. -# (Enter 1 to indicate 1%), value below 0, e.g. -1, is to disable this feature - not recommended. -order_refresh_tolerance_pct: null - -# Size of your bid and ask order. -order_amount: null - -# How long to wait before placing the next order in case your order gets filled. -filled_order_delay: null - -# Target base asset inventory percentage target to be maintained (for Inventory skew feature). -inventory_target_base_pct: null - -# Number of levels of orders to place on each side of the order book. -order_levels: null - -# Distance between levels in % of optimal spread -level_distances: null - -# Whether to enable adding transaction costs to order price calculation (true/false). -add_transaction_costs: null - -# Whether to stop cancellations of orders on the other side (of the order book), -# when one side is filled (hanging orders feature) (true/false). -hanging_orders_enabled: null - -# Spread (from mid price, in percentage) hanging orders will be canceled (Enter 1 to indicate 1%) -hanging_orders_cancel_pct: null - -# Use user provided orders to directly override the orders placed by order_amount and order_level_parameter -# This is an advanced feature and user is expected to directly edit this field in config file -# Below is an sample input, the format is a dictionary, the key is user-defined order name, the value is a list which includes buy/sell, order spread, and order amount -# order_override: -# order_1: [buy, 0.5, 100] -# order_2: [buy, 0.75, 200] -# order_3: [sell, 0.1, 500] -# Please make sure there is a space between : and [ -order_override: null - - - -# Avellaneda - Stoikov algorithm parameters -risk_factor: 1 -order_amount_shape_factor: 0 -min_spread: 0 - -# Buffer size used to store historic samples and calculate volatility -volatility_buffer_size: 60 - -# Buffer size used to store historic trades and calculate order book liquidity -trading_intensity_buffer_size: 200 - -# If the strategy should wait to receive cancellations confirmation before creating new orders during refresh time -should_wait_order_cancel_confirmation: True From ef516518e9d3836fa2c95e41b3b4c80ccf5c9f26 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Fri, 20 May 2022 18:37:20 +0200 Subject: [PATCH 106/152] (fix) OOP submodels & tests --- .../cross_exchange_market_making.pyx | 58 +--- ...hange_market_making_config_map_pydantic.py | 277 ++++++++++++++---- .../test_cross_exchange_market_making.py | 15 +- ...test_cross_exchange_market_making_start.py | 24 +- 4 files changed, 254 insertions(+), 120 deletions(-) diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pyx b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pyx index 1a5fa37318..d80f517e17 100755 --- a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pyx +++ b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pyx @@ -30,6 +30,7 @@ from .order_id_market_pair_tracker import OrderIDMarketPairTracker from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_making_config_map_pydantic import ( CrossExchangeMarketMakingConfigMap, + PassiveOrderRefreshMode ) NaN = float("nan") @@ -126,14 +127,6 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): def top_depth_tolerance(self): return self._config_map.top_depth_tolerance - @property - def cancel_order_threshold(self): - return self._config_map.cancel_order_threshold / Decimal("100") - - @property - def active_order_canceling(self): - return self._config_map.active_order_canceling - @property def anti_hysteresis_duration(self): return self._config_map.anti_hysteresis_duration @@ -191,36 +184,10 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): def logging_options(self, int64_t logging_options): self._logging_options = logging_options - def get_taker_to_maker_conversion_rate(self) -> Tuple[str, Decimal, str, Decimal]: - """ - Find conversion rates from taker market to maker market - :return: A tuple of quote pair symbol, quote conversion rate source, quote conversion rate, - base pair symbol, base conversion rate source, base conversion rate - """ - quote_rate = Decimal("1") - market_pairs = list(self._market_pairs.values())[0] - quote_pair = f"{market_pairs.taker.quote_asset}-{market_pairs.maker.quote_asset}" - quote_rate_source = "fixed" - if self.use_oracle_conversion_rate: - if market_pairs.taker.quote_asset != market_pairs.maker.quote_asset: - quote_rate_source = RateOracle.source.name - quote_rate = RateOracle.get_instance().rate(quote_pair) - else: - quote_rate = self.taker_to_maker_quote_conversion_rate - base_rate = Decimal("1") - base_pair = f"{market_pairs.taker.base_asset}-{market_pairs.maker.base_asset}" - base_rate_source = "fixed" - if self.use_oracle_conversion_rate: - if market_pairs.taker.base_asset != market_pairs.maker.base_asset: - base_rate_source = RateOracle.source.name - base_rate = RateOracle.get_instance().rate(base_pair) - else: - base_rate = self.taker_to_maker_base_conversion_rate - return quote_pair, quote_rate_source, quote_rate, base_pair, base_rate_source, base_rate - def log_conversion_rates(self): + market_pair = list(self._market_pairs.values())[0] quote_pair, quote_rate_source, quote_rate, base_pair, base_rate_source, base_rate = \ - self.get_taker_to_maker_conversion_rate() + self._config_map.conversion_rate_mode.get_taker_to_maker_conversion_rate(market_pair) if quote_pair.split("-")[0] != quote_pair.split("-")[1]: self.logger().info(f"{quote_pair} ({quote_rate_source}) conversion rate: {PerformanceMetrics.smart_round(quote_rate)}") if base_pair.split("-")[0] != base_pair.split("-")[1]: @@ -229,8 +196,9 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): def oracle_status_df(self): columns = ["Source", "Pair", "Rate"] data = [] + market_pair = list(self._market_pairs.values())[0] quote_pair, quote_rate_source, quote_rate, base_pair, base_rate_source, base_rate = \ - self.get_taker_to_maker_conversion_rate() + self._config_map.conversion_rate_mode.get_taker_to_maker_conversion_rate(market_pair) if quote_pair.split("-")[0] != quote_pair.split("-")[1]: data.extend([ [quote_rate_source, quote_pair, PerformanceMetrics.smart_round(quote_rate)], @@ -455,7 +423,7 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): if not self.c_check_if_still_profitable(market_pair, active_order, current_hedging_price): continue - if not self.active_order_canceling: + if isinstance(self._config_map.order_refresh_mode, PassiveOrderRefreshMode): continue # See if I still have enough balance on my wallet to fill the order on maker market, and to hedge the @@ -1109,9 +1077,8 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): object order_price = active_order.price object cancel_order_threshold - if not self.active_order_canceling: - cancel_order_threshold = self.cancel_order_threshold - else: + cancel_order_threshold = self._config_map.order_refresh_mode.get_cancel_order_threshold() + if cancel_order_threshold.is_nan(): cancel_order_threshold = self.min_profitability if current_hedging_price is None: @@ -1160,8 +1127,9 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): object order_size_limit str taker_trading_pair = market_pair.taker.trading_pair + market_pair = list(self._market_pairs.values())[0] quote_pair, quote_rate_source, quote_rate, base_pair, base_rate_source, base_rate = \ - self.get_taker_to_maker_conversion_rate() + self._config_map.conversion_rate_mode.get_taker_to_maker_conversion_rate(market_pair) if is_buy: quote_asset_amount = maker_market.c_get_balance(market_pair.maker.quote_asset) @@ -1195,7 +1163,8 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): """ Return price conversion rate for a taker market (to convert it into maker base asset value) """ - _, _, quote_rate, _, _, base_rate = self.get_taker_to_maker_conversion_rate() + market_pair = list(self._market_pairs.values())[0] + _, _, quote_rate, _, _, base_rate = self._config_map.conversion_rate_mode.get_taker_to_maker_conversion_rate(market_pair) return quote_rate / base_rate # else: # market_pairs = list(self._market_pairs.values())[0] @@ -1311,8 +1280,7 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): market_info.market.get_taker_order_type() if order_type is OrderType.MARKET: price = s_decimal_nan - if not self.active_order_canceling: - expiration_seconds = self.limit_order_min_expiration + expiration_seconds = self._config_map.order_refresh_mode.get_expiration_seconds() if is_buy: order_id = StrategyBase.c_buy_with_specific_market(self, market_info, amount, order_type=order_type, price=price, diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py index 44d4295322..34591845c6 100644 --- a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py @@ -1,11 +1,193 @@ +from abc import ABC, abstractmethod from decimal import Decimal -from typing import Dict +from typing import Dict, Tuple, Union from pydantic import BaseModel, Field, root_validator, validator import hummingbot.client.settings as settings -from hummingbot.client.config.config_data_types import BaseTradingStrategyMakerTakerConfigMap, ClientFieldData +from hummingbot.client.config.config_data_types import ( + BaseClientModel, + BaseTradingStrategyMakerTakerConfigMap, + ClientFieldData, +) from hummingbot.client.config.config_validators import validate_bool, validate_decimal +from hummingbot.core.rate_oracle.rate_oracle import RateOracle + +from .cross_exchange_market_pair import CrossExchangeMarketPair + + +class ConversionRateModel(BaseClientModel, ABC): + @abstractmethod + def get_taker_to_maker_conversion_rate( + self, market_pair: CrossExchangeMarketPair + ) -> Tuple[str, str, Decimal, str, str, Decimal]: + pass + + +class OracleConversionRateMode(ConversionRateModel): + class Config: + title = "rate_oracle_conversion_rate" + + def get_taker_to_maker_conversion_rate( + self, market_pair: CrossExchangeMarketPair + ) -> Tuple[str, str, Decimal, str, str, Decimal]: + """ + Find conversion rates from taker market to maker market + :param market_pair: maker and taker trading pairs for which to do conversion + :return: A tuple of quote pair symbol, quote conversion rate source, quote conversion rate, + base pair symbol, base conversion rate source, base conversion rate + """ + quote_pair = f"{market_pair.taker.quote_asset}-{market_pair.maker.quote_asset}" + if market_pair.taker.quote_asset != market_pair.maker.quote_asset: + quote_rate_source = RateOracle.source.name + quote_rate = RateOracle.get_instance().rate(quote_pair) + else: + quote_rate_source = "fixed" + quote_rate = Decimal("1") + + base_pair = f"{market_pair.taker.base_asset}-{market_pair.maker.base_asset}" + if market_pair.taker.base_asset != market_pair.maker.base_asset: + base_rate_source = RateOracle.source.name + base_rate = RateOracle.get_instance().rate(base_pair) + else: + base_rate_source = "fixed" + base_rate = Decimal("1") + + return quote_pair, quote_rate_source, quote_rate, base_pair, base_rate_source, base_rate + + +class TakerToMakerConversionRateMode(ConversionRateModel): + taker_to_maker_base_conversion_rate: Decimal = Field( + default=Decimal("1.0"), + description="A fixed conversion rate between the maker and taker tradin pairs based on the maker base asset.", + gt=0.0, + client_data=ClientFieldData( + prompt=lambda mi: ( + "Enter conversion rate for taker base asset value to maker base asset value, e.g. " + "if maker base asset is USD and the taker is DAI, 1 DAI is valued at 1.25 USD, " + "the conversion rate is 1.25" + ), + prompt_on_new=True, + ), + ) + taker_to_maker_quote_conversion_rate: Decimal = Field( + default=Decimal("1.0"), + description="A fixed conversion rate between the maker and taker tradin pairs based on the maker quote asset.", + gt=0.0, + client_data=ClientFieldData( + prompt=lambda mi: ( + "Enter conversion rate for taker quote asset value to maker quote asset value, e.g. " + "if maker quote asset is USD and the taker is DAI, 1 DAI is valued at 1.25 USD, " + "the conversion rate is 1.25" + ), + prompt_on_new=True, + ), + ) + + class Config: + title = "fixed_conversion_rate" + + def get_taker_to_maker_conversion_rate( + self, market_pair: CrossExchangeMarketPair + ) -> Tuple[str, str, Decimal, str, str, Decimal]: + """ + Find conversion rates from taker market to maker market + :param market_pair: maker and taker trading pairs for which to do conversion + :return: A tuple of quote pair symbol, quote conversion rate source, quote conversion rate, + base pair symbol, base conversion rate source, base conversion rate + """ + quote_rate = Decimal("1") + quote_pair = f"{market_pair.taker.quote_asset}-{market_pair.maker.quote_asset}" + quote_rate_source = "fixed" + quote_rate = self.taker_to_maker_quote_conversion_rate + + base_rate = Decimal("1") + base_pair = f"{market_pair.taker.base_asset}-{market_pair.maker.base_asset}" + base_rate_source = "fixed" + base_rate = self.taker_to_maker_base_conversion_rate + + return quote_pair, quote_rate_source, quote_rate, base_pair, base_rate_source, base_rate + + @validator( + "taker_to_maker_base_conversion_rate", + "taker_to_maker_quote_conversion_rate", + pre=True, + ) + def validate_decimal(cls, v: str, values: Dict, config: BaseModel.Config, field: Field): + return CrossExchangeMarketMakingConfigMap.validate_decimal(v=v, field=field) + + +CONVERSION_RATE_MODELS = { + OracleConversionRateMode.Config.title: OracleConversionRateMode, + TakerToMakerConversionRateMode.Config.title: TakerToMakerConversionRateMode, +} + + +class OrderRefreshMode(BaseClientModel, ABC): + @abstractmethod + def get_cancel_order_threshold(self) -> Decimal: + pass + + @abstractmethod + def get_expiration_seconds(self) -> Decimal: + pass + + +class PassiveOrderRefreshMode(OrderRefreshMode): + cancel_order_threshold: Decimal = Field( + default=Decimal("5.0"), + description="Profitability threshold to cancel a trade.", + gt=-100.0, + lt=100.0, + client_data=ClientFieldData( + prompt=lambda mi: "What is the threshold of profitability to cancel a trade? (Enter 1 to indicate 1%)", + prompt_on_new=True, + ), + ) + + limit_order_min_expiration: float = Field( + default=130.0, + description="Limit order expiration time limit.", + gt=0.0, + client_data=ClientFieldData( + prompt=lambda mi: "How often do you want limit orders to expire (in seconds)?", + prompt_on_new=True, + ), + ) + + class Config: + title = "passive_order_refresh" + + def get_cancel_order_threshold(self) -> Decimal: + return self.cancel_order_threshold / Decimal("100") + + def get_expiration_seconds(self) -> Decimal: + return self.limit_order_min_expiration + + @validator( + "cancel_order_threshold", + "limit_order_min_expiration", + pre=True, + ) + def validate_decimal(cls, v: str, values: Dict, config: BaseModel.Config, field: Field): + return CrossExchangeMarketMakingConfigMap.validate_decimal(v=v, field=field) + + +class ActiveOrderRefreshMode(OrderRefreshMode): + class Config: + title = "active_order_refresh" + + def get_cancel_order_threshold(self) -> Decimal: + return Decimal('nan') + + def get_expiration_seconds(self) -> Decimal: + return Decimal('nan') + + +ORDER_REFRESH_MODELS = { + PassiveOrderRefreshMode.Config.title: PassiveOrderRefreshMode, + ActiveOrderRefreshMode.Config.title: ActiveOrderRefreshMode, +} class CrossExchangeMarketMakingConfigMap(BaseTradingStrategyMakerTakerConfigMap): @@ -37,28 +219,12 @@ class CrossExchangeMarketMakingConfigMap(BaseTradingStrategyMakerTakerConfigMap) prompt=lambda mi: "Do you want to enable adjust order? (Yes/No)" ), ) - active_order_canceling: bool = Field( - default=True, - description="An option to refresh orders by cancellation instead of letting them expire.", - client_data=ClientFieldData( - prompt=lambda mi: "Do you want to enable active order canceling? (Yes/No)" - ), - ) - cancel_order_threshold: Decimal = Field( - default=Decimal("5.0"), - description="Profitability threshold to cancel a trade.", - gt=-100.0, - lt=100.0, - client_data=ClientFieldData( - prompt=lambda mi: "What is the threshold of profitability to cancel a trade? (Enter 1 to indicate 1%)", - ), - ) - limit_order_min_expiration: float = Field( - default=130.0, - description="Limit order expiration time limit.", - gt=0.0, + order_refresh_mode: Union[PassiveOrderRefreshMode, ActiveOrderRefreshMode] = Field( + default=ActiveOrderRefreshMode.construct(), + description="Refresh orders by cancellation or by letting them expire.", client_data=ClientFieldData( - prompt=lambda mi: "How often do you want limit orders to expire (in seconds)?", + prompt=lambda mi: f"Select the order refresh mode ({'/'.join(list(ORDER_REFRESH_MODELS.keys()))})", + prompt_on_new=True, ), ) top_depth_tolerance: Decimal = Field( @@ -113,38 +279,14 @@ class CrossExchangeMarketMakingConfigMap(BaseTradingStrategyMakerTakerConfigMap) ), ), ) - use_oracle_conversion_rate: bool = Field( - default=True, - description="Use rate oracle to determine a conversion rate between two different trading pairs.", + conversion_rate_mode: Union[OracleConversionRateMode, TakerToMakerConversionRateMode] = Field( + default=OracleConversionRateMode.construct(), + description="Convert between different trading pairs using fixed conversion rates or using the rate oracle.", client_data=ClientFieldData( - prompt=lambda mi: "Do you want to use rate oracle on unmatched trading pairs? (Yes/No)", + prompt=lambda mi: f"Select the conversion rate mode ({'/'.join(list(CONVERSION_RATE_MODELS.keys()))})", prompt_on_new=True, ), ) - taker_to_maker_base_conversion_rate: Decimal = Field( - default=Decimal("1.0"), - description="A fixed conversion rate between the maker and taker tradin pairs based on the maker base asset.", - gt=0.0, - client_data=ClientFieldData( - prompt=lambda mi: ( - "Enter conversion rate for taker base asset value to maker base asset value, e.g. " - "if maker base asset is USD and the taker is DAI, 1 DAI is valued at 1.25 USD, " - "the conversion rate is 1.25" - ), - ), - ) - taker_to_maker_quote_conversion_rate: Decimal = Field( - default=Decimal("1.0"), - description="A fixed conversion rate between the maker and taker tradin pairs based on the maker quote asset.", - gt=0.0, - client_data=ClientFieldData( - prompt=lambda mi: ( - "Enter conversion rate for taker quote asset value to maker quote asset value, e.g. " - "if maker quote asset is USD and the taker is DAI, 1 DAI is valued at 1.25 USD, " - "the conversion rate is 1.25" - ), - ), - ) slippage_buffer: Decimal = Field( default=Decimal("5.0"), description="Allowed slippage to fill ensure taker orders are filled.", @@ -173,12 +315,35 @@ def order_amount_prompt(cls, model_instance: 'CrossExchangeMarketMakingConfigMap base_asset, quote_asset = trading_pair.split("-") return f"What is the amount of {base_asset} per order?" + # === specific validations === + @validator("order_refresh_mode", pre=True) + def validate_order_refresh_mode(cls, v: Union[str, ActiveOrderRefreshMode, PassiveOrderRefreshMode]): + if isinstance(v, (ActiveOrderRefreshMode, PassiveOrderRefreshMode, Dict)): + sub_model = v + elif v not in ORDER_REFRESH_MODELS: + raise ValueError( + f"Invalid order refresh mode, please choose value from {list(ORDER_REFRESH_MODELS.keys())}." + ) + else: + sub_model = ORDER_REFRESH_MODELS[v].construct() + return sub_model + + @validator("conversion_rate_mode", pre=True) + def validate_conversion_rate_mode(cls, v: Union[str, OracleConversionRateMode, TakerToMakerConversionRateMode]): + if isinstance(v, (OracleConversionRateMode, TakerToMakerConversionRateMode, Dict)): + sub_model = v + elif v not in CONVERSION_RATE_MODELS: + raise ValueError( + f"Invalid conversion rate mode, please choose value from {list(CONVERSION_RATE_MODELS.keys())}." + ) + else: + sub_model = CONVERSION_RATE_MODELS[v].construct() + return sub_model + # === generic validations === @validator( "adjust_order_enabled", - "active_order_canceling", - "use_oracle_conversion_rate", pre=True, ) def validate_bool(cls, v: str): @@ -192,19 +357,15 @@ def validate_bool(cls, v: str): @validator( "min_profitability", "order_amount", - "cancel_order_threshold", - "limit_order_min_expiration", "top_depth_tolerance", "anti_hysteresis_duration", "order_size_taker_volume_factor", "order_size_taker_balance_factor", "order_size_portfolio_ratio_limit", - "taker_to_maker_base_conversion_rate", - "taker_to_maker_quote_conversion_rate", "slippage_buffer", pre=True, ) - def validate_decimal(cls, v: str, values: Dict, config: BaseModel.Config, field: Field): + def validate_decimal(cls, v: str, field: Field): """Used for client-friendly error output.""" range_min = None range_max = None diff --git a/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making.py b/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making.py index 5566137905..7b2e9c98f9 100644 --- a/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making.py +++ b/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making.py @@ -28,7 +28,9 @@ ) from hummingbot.strategy.cross_exchange_market_making import CrossExchangeMarketMakingStrategy from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_making_config_map_pydantic import ( + ActiveOrderRefreshMode, CrossExchangeMarketMakingConfigMap, + TakerToMakerConversionRateMode, ) from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_pair import CrossExchangeMarketPair from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple @@ -77,17 +79,16 @@ def setUp(self): order_size_taker_volume_factor=Decimal("25"), order_size_taker_balance_factor=Decimal("99.5"), order_size_portfolio_ratio_limit=Decimal("30"), - limit_order_min_expiration=130.0, adjust_order_enabled=True, anti_hysteresis_duration=60.0, - active_order_canceling=True, - cancel_order_threshold=Decimal("5"), + order_refresh_mode=ActiveOrderRefreshMode(), top_depth_tolerance=Decimal(0), - use_oracle_conversion_rate=False, - taker_to_maker_base_conversion_rate=Decimal("1"), - taker_to_maker_quote_conversion_rate=Decimal("1") + conversion_rate_mode=TakerToMakerConversionRateMode(), ) + self.config_map_raw.conversion_rate_mode.taker_to_maker_base_conversion_rate = Decimal("1.0") + self.config_map_raw.conversion_rate_mode.taker_to_maker_quote_conversion_rate = Decimal("1.0") + self.config_map = ClientConfigAdapter(self.config_map_raw) config_map_with_top_depth_tolerance_raw = deepcopy(self.config_map_raw) config_map_with_top_depth_tolerance_raw.top_depth_tolerance = Decimal("1") @@ -491,7 +492,7 @@ def test_with_conversion(self): config_map_raw = deepcopy(self.config_map_raw) config_map_raw.min_profitability = Decimal("1") config_map_raw.order_size_portfolio_ratio_limit = Decimal("30") - config_map_raw.taker_to_maker_base_conversion_rate = Decimal("0.95") + config_map_raw.conversion_rate_mode.taker_to_maker_base_conversion_rate = Decimal("0.95") config_map = ClientConfigAdapter( config_map_raw ) diff --git a/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_start.py b/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_start.py index 9d653ac1b6..8d411c080a 100644 --- a/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_start.py +++ b/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_start.py @@ -7,6 +7,7 @@ from hummingbot.connector.exchange_base import ExchangeBase from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_making_config_map_pydantic import ( CrossExchangeMarketMakingConfigMap, + TakerToMakerConversionRateMode, ) @@ -19,18 +20,21 @@ def setUp(self) -> None: self.notifications = [] self.log_errors = [] - self.strategy_config_map = ClientConfigAdapter( - CrossExchangeMarketMakingConfigMap( - maker_market="binance", - taker_market="kucoin", - maker_market_trading_pair="ETH-USDT", - taker_market_trading_pair="ETH-USDT", - order_amount=1.0, - min_profitability=2.0, - use_oracle_conversion_rate=False, - ) + config_map_raw = CrossExchangeMarketMakingConfigMap( + maker_market="binance", + taker_market="kucoin", + maker_market_trading_pair="ETH-USDT", + taker_market_trading_pair="ETH-USDT", + order_amount=1.0, + min_profitability=2.0, + conversion_rate_mode=TakerToMakerConversionRateMode(), ) + config_map_raw.conversion_rate_mode.taker_to_maker_base_conversion_rate = Decimal("1.0") + config_map_raw.conversion_rate_mode.taker_to_maker_quote_conversion_rate = Decimal("1.0") + + self.strategy_config_map = ClientConfigAdapter(config_map_raw) + global_config_map.get("strategy_report_interval").value = 60. def _initialize_market_assets(self, market, trading_pairs): From 88fc62e25c2e0cbd96f9f44762ad90afef12f9c4 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Fri, 20 May 2022 18:37:37 +0200 Subject: [PATCH 107/152] (feat) config template yml deleted --- ...change_market_making_strategy_TEMPLATE.yml | 84 ------------------- 1 file changed, 84 deletions(-) delete mode 100644 hummingbot/templates/conf_cross_exchange_market_making_strategy_TEMPLATE.yml diff --git a/hummingbot/templates/conf_cross_exchange_market_making_strategy_TEMPLATE.yml b/hummingbot/templates/conf_cross_exchange_market_making_strategy_TEMPLATE.yml deleted file mode 100644 index e2b9cf58de..0000000000 --- a/hummingbot/templates/conf_cross_exchange_market_making_strategy_TEMPLATE.yml +++ /dev/null @@ -1,84 +0,0 @@ -######################################################## -### Cross exchange market making strategy config ### -######################################################## - -template_version: 6 -strategy: null - -# The following configuations are only required for the -# cross exchange market making strategy - -# Exchange and token parameters -maker_market: null -taker_market: null -maker_market_trading_pair: null -taker_market_trading_pair: null - -# Minimum profitability target required to place an order -# Expressed in percentage value, e.g. 1 = 1% target profit -min_profitability: null - -# Maximum order size in terms of base currency -order_amount: null - -# Have Hummingbot actively adjust/cancel orders if necessary. -# If set to true, outstanding orders are adjusted if -# profitability falls below min_profitability. -# If set to false, outstanding orders are adjusted if -# profitability falls below cancel_order_threshold. -active_order_canceling: null - -# If active_order_canceling = false, this is the profitability/ -# loss threshold at which to cancel the order. -# Expressed in decimals: 1 = 1% target profit -cancel_order_threshold: null - -# An amount in seconds, which is the minimum duration for any -# placed limit orders. Default value = 130 seconds. -limit_order_min_expiration: null - -# If enabled (parameter set to `True`), the strategy will place the order on top of the top bid and ask, -# if its more profitable to place it there -adjust_order_enabled: null - -# An amount expressed in base currency which is used for getting the top bid and ask, -# ignoring dust orders on top of the order book -top_depth_tolerance: null - -# An amount in seconds, which is the minimum amount of time interval between adjusting limit order prices -anti_hysteresis_duration: null - -# An amount expressed in decimals (i.e. input of `1` corresponds to 1%), which is the maximum size limit of new limit orders, -# in terms of ratio of hedge-able volume on taker side. -order_size_taker_volume_factor: null - -# An amount expressed in decimals (i.e. input of `1` corresponds to 1%), which is the maximum size limit of new limit orders, -# in terms of ratio of asset balance available for hedging trade on taker side -order_size_taker_balance_factor: null - -# An amount expressed in decimals (i.e. input of `1` corresponds to 1%), which is the maximum size limit of new limit orders, -# in terms of ratio of total portfolio value on both maker and taker markets -order_size_portfolio_ratio_limit: null - -# Whether to use rate oracle on unmatched trading pairs -# Set this to either True or False -use_oracle_conversion_rate: null - -# The conversion rate for taker base asset value to maker base asset value. -# e.g. if maker base asset is USD, taker is DAI and 1 USD is worth 1.25 DAI, " -# the conversion rate is 0.8 (1 / 1.25) -taker_to_maker_base_conversion_rate: null - -# The conversion rate for taker quote asset value to maker quote asset value. -# e.g. if maker quote asset is USD, taker is DAI and 1 USD is worth 1.25 DAI, " -# the conversion rate is 0.8 (1 / 1.25) -taker_to_maker_quote_conversion_rate: null - -# A buffer for which to adjust order price for higher chance of the order getting filled. -# Since we hedge on markets a slippage is acceptable rather than having the transaction get rejected. -# The submitted order price will be adjusted higher (by percentage value) for buy orders -# and lower for sell orders. (Enter 1 for 1%) -slippage_buffer: null - -# For more detailed information, see: -# https://docs.hummingbot.io/strategies/cross-exchange-market-making/#configuration-parameters \ No newline at end of file From 696b67f2fbb313b5f3c26c821df6a6e5e134fbdb Mon Sep 17 00:00:00 2001 From: mhrvth Date: Fri, 20 May 2022 18:51:11 +0200 Subject: [PATCH 108/152] (feat) validate_decimal() --- hummingbot/client/config/config_data_types.py | 15 ++++++++ ...hange_market_making_config_map_pydantic.py | 38 +------------------ 2 files changed, 17 insertions(+), 36 deletions(-) diff --git a/hummingbot/client/config/config_data_types.py b/hummingbot/client/config/config_data_types.py index 5bb78a30ed..974b1d1b28 100644 --- a/hummingbot/client/config/config_data_types.py +++ b/hummingbot/client/config/config_data_types.py @@ -1,5 +1,6 @@ from dataclasses import dataclass from datetime import datetime +from decimal import Decimal from enum import Enum from typing import Any, Callable, Dict, Optional @@ -9,6 +10,7 @@ from hummingbot.client.config.config_methods import strategy_config_schema_encoder from hummingbot.client.config.config_validators import ( validate_connector, + validate_decimal, validate_exchange, validate_market_trading_pair, validate_strategy, @@ -53,6 +55,19 @@ def schema_json( def is_required(self, attr: str) -> bool: return self.__fields__[attr].required + def validate_decimal(v: str, field: Field): + """Used for client-friendly error output.""" + field_info = field.field_info + inclusive = field_info.ge is not None or field_info.le is not None + min_value = field_info.gt if field_info.gt is not None else field_info.ge + min_value = Decimal(min_value) if min_value is not None else min_value + max_value = field_info.lt if field_info.lt is not None else field_info.le + max_value = Decimal(max_value) if max_value is not None else max_value + ret = validate_decimal(v, min_value, max_value, inclusive) + if ret is not None: + raise ValueError(ret) + return v + class BaseStrategyConfigMap(BaseClientModel): strategy: str = Field( diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py index 34591845c6..d705740f54 100644 --- a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py @@ -10,7 +10,7 @@ BaseTradingStrategyMakerTakerConfigMap, ClientFieldData, ) -from hummingbot.client.config.config_validators import validate_bool, validate_decimal +from hummingbot.client.config.config_validators import validate_bool from hummingbot.core.rate_oracle.rate_oracle import RateOracle from .cross_exchange_market_pair import CrossExchangeMarketPair @@ -367,41 +367,7 @@ def validate_bool(cls, v: str): ) def validate_decimal(cls, v: str, field: Field): """Used for client-friendly error output.""" - range_min = None - range_max = None - range_inclusive = None - - field = field.field_info - - if field.gt is not None: - range_min = field.gt - range_inclusive = False - elif field.ge is not None: - range_min = field.ge - range_inclusive = True - if field.lt is not None: - range_max = field.lt - range_inclusive = False - elif field.le is not None: - range_max = field.le - range_inclusive = True - - if range_min is not None and range_max is not None: - ret = validate_decimal(v, - min_value=Decimal(str(range_min)), - max_value=Decimal(str(range_max)), - inclusive=str(range_inclusive)) - elif range_min is not None: - ret = validate_decimal(v, - min_value=Decimal(str(range_min)), - inclusive=str(range_inclusive)) - elif range_max is not None: - ret = validate_decimal(v, - max_value=Decimal(str(range_max)), - inclusive=str(range_inclusive)) - if ret is not None: - raise ValueError(ret) - return v + return super().validate_decimal(v, field) # === post-validations === From 31b76f9a91fd1f7e06c27318e9ce08cf1319d278 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Fri, 20 May 2022 18:53:52 +0200 Subject: [PATCH 109/152] (feat) config map pydantic tests --- ...hange_market_making_config_map_pydantic.py | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_config_map_pydantic.py diff --git a/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_config_map_pydantic.py b/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_config_map_pydantic.py new file mode 100644 index 0000000000..174fb3a041 --- /dev/null +++ b/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_config_map_pydantic.py @@ -0,0 +1,63 @@ +import unittest +from typing import Dict + +from hummingbot.client.config.config_helpers import ClientConfigAdapter +from hummingbot.client.settings import AllConnectorSettings +from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_making_config_map_pydantic import ( + CrossExchangeMarketMakingConfigMap, +) + + +class CrossExchangeMarketMakingConfigMapPydanticTest(unittest.TestCase): + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + cls.base_asset = "COINALPHA" + cls.quote_asset = "HBOT" + cls.trading_pair = f"{cls.base_asset}-{cls.quote_asset}" + + cls.maker_exchange = "binance" + cls.taker_exchange = "kucoin" + + def setUp(self) -> None: + super().setUp() + config_settings = self.get_default_map() + self.config_map = ClientConfigAdapter(CrossExchangeMarketMakingConfigMap(**config_settings)) + + def get_default_map(self) -> Dict[str, str]: + config_settings = { + "maker_market": self.maker_exchange, + "taker_market": self.taker_exchange, + "maker_market_trading_pair": self.trading_pair, + "taker_market_trading_pair": self.trading_pair, + "order_amount": "10", + "min_profitability": "0", + } + return config_settings + + def test_order_amount_prompt(self): + self.config_map.maker_market_trading_pair = self.trading_pair + prompt = self.config_map.order_amount_prompt(self.config_map) + expected = f"What is the amount of {self.base_asset} per order?" + + self.assertEqual(expected, prompt) + + def test_maker_trading_pair_prompt(self): + exchange = self.config_map.maker_market = self.maker_exchange + example = AllConnectorSettings.get_example_pairs().get(exchange) + + prompt = self.config_map.trading_pair_prompt(self.config_map, True) + expected = f"Enter the token trading pair you would like to trade on maker market: {exchange} " \ + f"(e.g. {example})" + + self.assertEqual(expected, prompt) + + def test_taker_trading_pair_prompt(self): + exchange = self.config_map.maker_market = self.taker_exchange + example = AllConnectorSettings.get_example_pairs().get(exchange) + + prompt = self.config_map.trading_pair_prompt(self.config_map, False) + expected = f"Enter the token trading pair you would like to trade on taker market: {exchange} " \ + f"(e.g. {example})" + + self.assertEqual(expected, prompt) From 7ec1c008564d4d455503622d65665f618172d1a4 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Fri, 20 May 2022 19:28:37 +0200 Subject: [PATCH 110/152] (feat) config map pydantic tests --- ...hange_market_making_config_map_pydantic.py | 74 +++++++++++++++---- 1 file changed, 58 insertions(+), 16 deletions(-) diff --git a/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_config_map_pydantic.py b/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_config_map_pydantic.py index 174fb3a041..5fa565cf69 100644 --- a/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_config_map_pydantic.py +++ b/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_config_map_pydantic.py @@ -1,10 +1,18 @@ import unittest +from decimal import Decimal +from pathlib import Path from typing import Dict +from unittest.mock import patch -from hummingbot.client.config.config_helpers import ClientConfigAdapter -from hummingbot.client.settings import AllConnectorSettings +import yaml + +from hummingbot.client.config.config_helpers import ClientConfigAdapter, ConfigValidationError from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_making_config_map_pydantic import ( + ActiveOrderRefreshMode, CrossExchangeMarketMakingConfigMap, + OracleConversionRateMode, + PassiveOrderRefreshMode, + TakerToMakerConversionRateMode, ) @@ -35,6 +43,13 @@ def get_default_map(self) -> Dict[str, str]: } return config_settings + def test_top_depth_tolerance_prompt(self): + self.config_map.maker_market_trading_pair = self.trading_pair + prompt = self.config_map.top_depth_tolerance_prompt(self.config_map) + expected = f"What is your top depth tolerance? (in {self.base_asset})" + + self.assertEqual(expected, prompt) + def test_order_amount_prompt(self): self.config_map.maker_market_trading_pair = self.trading_pair prompt = self.config_map.order_amount_prompt(self.config_map) @@ -42,22 +57,49 @@ def test_order_amount_prompt(self): self.assertEqual(expected, prompt) - def test_maker_trading_pair_prompt(self): - exchange = self.config_map.maker_market = self.maker_exchange - example = AllConnectorSettings.get_example_pairs().get(exchange) + @patch( + "hummingbot.client.config.config_data_types.validate_market_trading_pair" + ) + def test_validators(self, _): + self.config_map.order_refresh_mode = "active_order_refresh" + self.assertIsInstance(self.config_map.order_refresh_mode.hb_config, ActiveOrderRefreshMode) - prompt = self.config_map.trading_pair_prompt(self.config_map, True) - expected = f"Enter the token trading pair you would like to trade on maker market: {exchange} " \ - f"(e.g. {example})" + self.config_map.order_refresh_mode = "passive_order_refresh" + self.config_map.order_refresh_mode.cancel_order_threshold = Decimal("1.0") + self.config_map.order_refresh_mode.cancel_order_threshold = Decimal("2.0") + self.assertIsInstance(self.config_map.order_refresh_mode.hb_config, PassiveOrderRefreshMode) - self.assertEqual(expected, prompt) + with self.assertRaises(ConfigValidationError) as e: + self.config_map.order_refresh_mode = "XXX" - def test_taker_trading_pair_prompt(self): - exchange = self.config_map.maker_market = self.taker_exchange - example = AllConnectorSettings.get_example_pairs().get(exchange) + error_msg = ( + "Invalid order refresh mode, please choose value from ['passive_order_refresh', 'active_order_refresh']." + ) + self.assertEqual(error_msg, str(e.exception)) - prompt = self.config_map.trading_pair_prompt(self.config_map, False) - expected = f"Enter the token trading pair you would like to trade on taker market: {exchange} " \ - f"(e.g. {example})" + self.config_map.conversion_rate_mode = "rate_oracle_conversion_rate" + self.assertIsInstance(self.config_map.conversion_rate_mode.hb_config, OracleConversionRateMode) - self.assertEqual(expected, prompt) + self.config_map.conversion_rate_mode = "fixed_conversion_rate" + self.config_map.conversion_rate_mode.taker_to_maker_base_conversion_rate = Decimal("1.0") + self.config_map.conversion_rate_mode.taker_to_maker_quote_conversion_rate = Decimal("2.0") + self.assertIsInstance(self.config_map.conversion_rate_mode.hb_config, TakerToMakerConversionRateMode) + + with self.assertRaises(ConfigValidationError) as e: + self.config_map.conversion_rate_mode = "XXX" + + error_msg = ( + "Invalid conversion rate mode, please choose value from ['rate_oracle_conversion_rate', 'fixed_conversion_rate']." + ) + self.assertEqual(error_msg, str(e.exception)) + + def test_load_configs_from_yaml(self): + cur_dir = Path(__file__).parent + f_path = cur_dir / "test_config.yml" + + with open(f_path, "r") as file: + data = yaml.safe_load(file) + + loaded_config_map = ClientConfigAdapter(CrossExchangeMarketMakingConfigMap(**data)) + + self.assertEqual(self.config_map, loaded_config_map) From ee8e5f3313e5eace21ec514aa5f5033b55003b8d Mon Sep 17 00:00:00 2001 From: mhrvth Date: Fri, 20 May 2022 19:31:55 +0200 Subject: [PATCH 111/152] (feat) test yml --- .../cross_exchange_market_making/test_config.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 test/hummingbot/strategy/cross_exchange_market_making/test_config.yml diff --git a/test/hummingbot/strategy/cross_exchange_market_making/test_config.yml b/test/hummingbot/strategy/cross_exchange_market_making/test_config.yml new file mode 100644 index 0000000000..be1fcf409a --- /dev/null +++ b/test/hummingbot/strategy/cross_exchange_market_making/test_config.yml @@ -0,0 +1,16 @@ +strategy: cross_exchange_market_making +maker_market: binance +taker_market: kucoin +maker_market_trading_pair: COINALPHA-HBOT +taker_market_trading_pair: COINALPHA-HBOT +min_profitability: 0 +order_amount: 10 +adjust_order_enabled: true +order_refresh_mode: active_order_refresh +top_depth_tolerance: 0.0 +anti_hysteresis_duration: 60.0 +order_size_taker_volume_factor: 25.0 +order_size_taker_balance_factor: 99.5 +order_size_portfolio_ratio_limit: 16.67 +conversion_rate_mode: {} +slippage_buffer: 5.0 From 35945012e02083221fe1a17228fe300a67eec747 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Mon, 23 May 2022 12:06:40 +0300 Subject: [PATCH 112/152] (refactor) Completed implementation - no debugging, no testing --- bin/hummingbot.py | 21 +- bin/hummingbot_quickstart.py | 17 +- hummingbot/__init__.py | 10 +- hummingbot/client/command/balance_command.py | 54 +- hummingbot/client/command/config_command.py | 72 +- hummingbot/client/command/connect_command.py | 17 +- hummingbot/client/command/create_command.py | 3 +- hummingbot/client/command/export_command.py | 3 +- hummingbot/client/command/gateway_command.py | 90 +- hummingbot/client/command/history_command.py | 22 +- hummingbot/client/command/import_command.py | 6 +- .../client/command/order_book_command.py | 13 +- hummingbot/client/command/start_command.py | 32 +- hummingbot/client/command/status_command.py | 11 +- hummingbot/client/command/ticker_command.py | 2 +- hummingbot/client/config/client_config_map.py | 785 ++++++++++++++++++ hummingbot/client/config/conf_migration.py | 136 ++- hummingbot/client/config/config_data_types.py | 37 +- hummingbot/client/config/config_helpers.py | 49 +- hummingbot/client/config/global_config_map.py | 398 --------- hummingbot/client/hummingbot_application.py | 57 +- hummingbot/client/settings.py | 2 +- hummingbot/client/ui/__init__.py | 26 +- hummingbot/client/ui/custom_widgets.py | 22 +- hummingbot/client/ui/hummingbot_cli.py | 25 +- hummingbot/client/ui/interface_utils.py | 5 +- hummingbot/client/ui/keybindings.py | 23 +- hummingbot/client/ui/layout.py | 8 +- hummingbot/client/ui/parser.py | 25 +- hummingbot/client/ui/style.py | 73 +- hummingbot/connector/connector_base.pxd | 1 + hummingbot/connector/connector_base.pyx | 22 +- .../connector/connector_metrics_collector.py | 48 +- .../binance_perpetual_derivative.py | 11 +- .../binance_perpetual_web_utils.py | 2 +- hummingbot/connector/derivative_base.py | 8 +- .../altmarkets/altmarkets_exchange.py | 16 +- .../exchange/beaxy/beaxy_exchange.pyx | 8 +- .../exchange/binance/binance_exchange.py | 10 +- .../exchange/bitfinex/bitfinex_exchange.pyx | 8 +- .../exchange/bitmart/bitmart_exchange.py | 8 +- .../exchange/bittrex/bittrex_exchange.pyx | 8 +- .../exchange/blocktane/blocktane_exchange.pyx | 8 +- .../coinbase_pro/coinbase_pro_exchange.pyx | 8 +- .../exchange/coinflex/coinflex_exchange.py | 23 +- .../exchange/coinzoom/coinzoom_exchange.py | 14 +- .../crypto_com/crypto_com_exchange.py | 11 +- .../exchange/digifinex/digifinex_exchange.py | 11 +- .../connector/exchange/ftx/ftx_exchange.pyx | 8 +- .../exchange/gate_io/gate_io_exchange.py | 8 +- .../exchange/hitbtc/hitbtc_exchange.py | 19 +- .../exchange/huobi/huobi_exchange.pyx | 8 +- .../connector/exchange/k2/k2_exchange.py | 11 +- .../exchange/kraken/kraken_exchange.pyx | 12 +- .../exchange/liquid/liquid_exchange.pyx | 8 +- .../exchange/loopring/loopring_exchange.pyx | 8 +- .../connector/exchange/ndax/ndax_exchange.py | 23 +- .../connector/exchange/okex/okex_exchange.pyx | 8 +- .../exchange/paper_trade/__init__.py | 6 +- .../paper_trade/paper_trade_exchange.pyx | 15 +- .../exchange/probit/probit_exchange.py | 17 +- .../exchange/wazirx/wazirx_exchange.py | 11 +- hummingbot/connector/exchange_base.pyx | 9 +- hummingbot/connector/gateway_EVM_AMM.py | 85 +- .../api_throttler/async_throttler_base.py | 19 +- hummingbot/core/gateway/__init__.py | 53 +- .../core/gateway/gateway_http_client.py | 27 +- hummingbot/core/gateway/status_monitor.py | 14 +- hummingbot/core/utils/kill_switch.py | 30 +- hummingbot/core/utils/ssl_cert.py | 22 +- hummingbot/model/db_migration/migrator.py | 9 +- hummingbot/model/sql_connection_manager.py | 74 +- hummingbot/notifier/telegram_notifier.py | 43 +- .../aroon_oscillator_config_map.py | 2 +- ...aneda_market_making_config_map_pydantic.py | 3 +- .../cross_exchange_market_making/start.py | 20 +- hummingbot/user/user_balances.py | 27 +- .../client/command/test_config_command.py | 10 +- .../client/ui/test_hummingbot_cli.py | 8 +- .../connector/test_markets_recorder.py | 6 +- .../test_inventory_cost_price_delegate.py | 11 +- .../strategy/pure_market_making/test_pmm.py | 4 +- 82 files changed, 1749 insertions(+), 1158 deletions(-) create mode 100644 hummingbot/client/config/client_config_map.py delete mode 100644 hummingbot/client/config/global_config_map.py diff --git a/bin/hummingbot.py b/bin/hummingbot.py index aba2b95648..23511b08db 100755 --- a/bin/hummingbot.py +++ b/bin/hummingbot.py @@ -11,10 +11,9 @@ from hummingbot.client.config.config_crypt import ETHKeyFileSecretManger from hummingbot.client.config.config_helpers import ( create_yml_files_legacy, - read_system_configs_from_yml, + load_client_config_map_from_file, write_config_to_yml, ) -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.hummingbot_application import HummingbotApplication from hummingbot.client.settings import AllConnectorSettings from hummingbot.client.ui import login_prompt @@ -40,29 +39,29 @@ def hummingbot_app(self) -> HummingbotApplication: async def ui_start_handler(self): hb: HummingbotApplication = self.hummingbot_app if hb.strategy_config_map is not None: - write_config_to_yml(hb.strategy_config_map, hb.strategy_file_name) - hb.start(global_config_map.get("log_level").value) + write_config_to_yml(hb.strategy_config_map, hb.strategy_file_name, hb.client_config_map) + hb.start(self._hb_ref.client_config_map.log_level) async def main_async(): await create_yml_files_legacy() - # This init_logging() call is important, to skip over the missing config warnings. - init_logging("hummingbot_logs.yml") + client_config_map = load_client_config_map_from_file() - await read_system_configs_from_yml() + # This init_logging() call is important, to skip over the missing config warnings. + init_logging("hummingbot_logs.yml", client_config_map) - AllConnectorSettings.initialize_paper_trade_settings(global_config_map.get("paper_trade_exchanges").value) + AllConnectorSettings.initialize_paper_trade_settings(client_config_map.paper_trade.paper_trade_exchanges) - hb = HummingbotApplication.main_application() + hb = HummingbotApplication.main_application(client_config_map) # The listener needs to have a named variable for keeping reference, since the event listener system # uses weak references to remove unneeded listeners. start_listener: UIStartListener = UIStartListener(hb) hb.app.add_listener(HummingbotUIEvent.Start, start_listener) - tasks: List[Coroutine] = [hb.run(), start_existing_gateway_container()] - if global_config_map.get("debug_console").value: + tasks: List[Coroutine] = [hb.run(), start_existing_gateway_container(client_config_map)] + if client_config_map.debug_console: if not hasattr(__builtins__, "help"): import _sitebuiltins __builtins__.help = _sitebuiltins._Helper() diff --git a/bin/hummingbot_quickstart.py b/bin/hummingbot_quickstart.py index 8604528e36..6867c0f486 100755 --- a/bin/hummingbot_quickstart.py +++ b/bin/hummingbot_quickstart.py @@ -20,10 +20,10 @@ from hummingbot.client.config.config_helpers import ( all_configs_complete, create_yml_files_legacy, + load_client_config_map_from_file, load_strategy_config_map_from_file, read_system_configs_from_yml, ) -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.security import Security from hummingbot.client.hummingbot_application import HummingbotApplication from hummingbot.client.settings import STRATEGIES_CONF_DIR_PATH, AllConnectorSettings @@ -74,6 +74,7 @@ def autofix_permissions(user_group_spec: str): async def quick_start(args: argparse.Namespace, secrets_manager: BaseSecretsManager): config_file_name = args.config_file_name + client_config = load_client_config_map_from_file() if args.auto_set_permissions is not None: autofix_permissions(args.auto_set_permissions) @@ -84,10 +85,10 @@ async def quick_start(args: argparse.Namespace, secrets_manager: BaseSecretsMana await Security.wait_til_decryption_done() await create_yml_files_legacy() - init_logging("hummingbot_logs.yml") + init_logging("hummingbot_logs.yml", client_config) await read_system_configs_from_yml() - AllConnectorSettings.initialize_paper_trade_settings(global_config_map.get("paper_trade_exchanges").value) + AllConnectorSettings.initialize_paper_trade_settings(client_config.paper_trade.paper_trade_exchanges) hb = HummingbotApplication.main_application() # Todo: validate strategy and config_file_name before assinging @@ -105,12 +106,8 @@ async def quick_start(args: argparse.Namespace, secrets_manager: BaseSecretsMana ) hb.strategy_config_map = strategy_config - # To ensure quickstart runs with the default value of False for kill_switch_enabled if not present - if not global_config_map.get("kill_switch_enabled"): - global_config_map.get("kill_switch_enabled").value = False - if strategy_config is not None: - if not all_configs_complete(strategy_config): + if not all_configs_complete(strategy_config, hb.client_config_map): hb.status() # The listener needs to have a named variable for keeping reference, since the event listener system @@ -118,8 +115,8 @@ async def quick_start(args: argparse.Namespace, secrets_manager: BaseSecretsMana start_listener: UIStartListener = UIStartListener(hb) hb.app.add_listener(HummingbotUIEvent.Start, start_listener) - tasks: List[Coroutine] = [hb.run(), start_existing_gateway_container()] - if global_config_map.get("debug_console").value: + tasks: List[Coroutine] = [hb.run(), start_existing_gateway_container(client_config)] + if client_config.debug_console: management_port: int = detect_available_port(8211) tasks.append(start_management_console(locals(), host="localhost", port=management_port)) diff --git a/hummingbot/__init__.py b/hummingbot/__init__.py index ca2060fd42..a71e55dd36 100644 --- a/hummingbot/__init__.py +++ b/hummingbot/__init__.py @@ -4,10 +4,13 @@ from concurrent.futures import ThreadPoolExecutor from os import listdir, path from pathlib import Path -from typing import List, Optional +from typing import TYPE_CHECKING, List, Optional from hummingbot.logger.struct_logger import StructLogger, StructLogRecord +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + STRUCT_LOGGER_SET = False DEV_STRATEGY_PREFIX = "dev" _prefix_path = None @@ -106,6 +109,7 @@ def chdir_to_data_directory(): def init_logging(conf_filename: str, + client_config_map: "ClientConfigAdapter", override_log_level: Optional[str] = None, strategy_file_path: str = "hummingbot"): import io @@ -116,7 +120,6 @@ def init_logging(conf_filename: str, import pandas as pd from ruamel.yaml import YAML - from hummingbot.client.config.global_config_map import global_config_map from hummingbot.logger.struct_logger import StructLogger, StructLogRecord global STRUCT_LOGGER_SET if not STRUCT_LOGGER_SET: @@ -138,8 +141,7 @@ def init_logging(conf_filename: str, config_dict: Dict = yaml_parser.load(io_stream) if override_log_level is not None and "loggers" in config_dict: for logger in config_dict["loggers"]: - if global_config_map["logger_override_whitelist"].value and \ - logger in global_config_map["logger_override_whitelist"].value: + if logger in client_config_map.logger_override_whitelist: config_dict["loggers"][logger]["level"] = override_log_level logging.config.dictConfig(config_dict) diff --git a/hummingbot/client/command/balance_command.py b/hummingbot/client/command/balance_command.py index 858956de19..c57fc43372 100644 --- a/hummingbot/client/command/balance_command.py +++ b/hummingbot/client/command/balance_command.py @@ -1,15 +1,12 @@ import asyncio import threading from decimal import Decimal -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING, Dict, List import pandas as pd -from hummingbot.client.config.config_helpers import save_to_yml_legacy from hummingbot.client.config.config_validators import validate_decimal, validate_exchange -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.performance import PerformanceMetrics -from hummingbot.client.settings import GLOBAL_CONFIG_PATH from hummingbot.connector.other.celo.celo_cli import CeloCLI from hummingbot.connector.other.celo.celo_data_types import KEYS as CELO_KEYS from hummingbot.core.rate_oracle.rate_oracle import RateOracle @@ -26,7 +23,7 @@ class BalanceCommand: - def balance(self, + def balance(self, # type: HummingbotApplication option: str = None, args: List[str] = None ): @@ -39,10 +36,8 @@ def balance(self, safe_ensure_future(self.show_balances()) elif option in OPTIONS: - config_map = global_config_map - file_path = GLOBAL_CONFIG_PATH if option == "limit": - config_var = config_map["balance_asset_limit"] + balance_asset_limit = self.client_config_map.balance_asset_limit if args is None or len(args) == 0: safe_ensure_future(self.show_asset_limits()) return @@ -53,18 +48,18 @@ def balance(self, exchange = args[0] asset = args[1].upper() amount = float(args[2]) - if exchange not in config_var.value or config_var.value[exchange] is None: - config_var.value[exchange] = {} - if amount < 0 and asset in config_var.value[exchange].keys(): - config_var.value[exchange].pop(asset) + if exchange not in balance_asset_limit or balance_asset_limit[exchange] is None: + balance_asset_limit[exchange] = {} + if amount < 0 and asset in balance_asset_limit[exchange].keys(): + balance_asset_limit[exchange].pop(asset) self.notify(f"Limit for {asset} on {exchange} exchange removed.") elif amount >= 0: - config_var.value[exchange][asset] = amount + balance_asset_limit[exchange][asset] = amount self.notify(f"Limit for {asset} on {exchange} exchange set to {amount}") - save_to_yml_legacy(file_path, config_map) + self.save_client_config() elif option == "paper": - config_var = config_map["paper_trade_account_balance"] + paper_balances = self.client_config_map.paper_trade.paper_trade_account_balance if args is None or len(args) == 0: safe_ensure_future(self.show_paper_account_balance()) return @@ -74,29 +69,25 @@ def balance(self, return asset = args[0].upper() amount = float(args[1]) - paper_balances = dict(config_var.value) if config_var.value else {} paper_balances[asset] = amount - config_var.value = paper_balances self.notify(f"Paper balance for {asset} token set to {amount}") - save_to_yml_legacy(file_path, config_map) + self.save_client_config() - async def show_balances(self): + async def show_balances( + self # type: HummingbotApplication + ): total_col_name = f'Total ({RateOracle.global_token_symbol})' sum_not_for_show_name = "sum_not_for_show" self.notify("Updating balances, please wait...") - network_timeout = float(global_config_map["other_commands_timeout"].value) + network_timeout = self.client_config_map.commands_timeout.other_commands_timeout try: all_ex_bals = await asyncio.wait_for( - UserBalances.instance().all_balances_all_exchanges(), network_timeout + UserBalances.instance().all_balances_all_exchanges(self.client_config_map), network_timeout ) except asyncio.TimeoutError: self.notify("\nA network error prevented the balances to update. See logs for more details.") raise all_ex_avai_bals = UserBalances.instance().all_available_balances_all_exchanges() - all_ex_limits: Optional[Dict[str, Dict[str, str]]] = global_config_map["balance_asset_limit"].value - - if all_ex_limits is None: - all_ex_limits = {} exchanges_total = 0 @@ -175,9 +166,10 @@ async def asset_limits_df(self, df.sort_values(by=["Asset"], inplace=True) return df - async def show_asset_limits(self): - config_var = global_config_map["balance_asset_limit"] - exchange_limit_conf: Dict[str, Dict[str, str]] = config_var.value + async def show_asset_limits( + self # type: HummingbotApplication + ): + exchange_limit_conf = self.client_config_map.balance_asset_limit if not any(list(exchange_limit_conf.values())): self.notify("You have not set any limits.") @@ -218,8 +210,10 @@ def notify_balance_paper_set(self): " balance paper [ASSET] [AMOUNT]\n" "e.g. balance paper BTC 0.1") - async def show_paper_account_balance(self): - paper_balances = global_config_map["paper_trade_account_balance"].value + async def show_paper_account_balance( + self # type: HummingbotApplication + ): + paper_balances = self.client_config_map.paper_trade.paper_trade_account_balance if not paper_balances: self.notify("You have not set any paper account balance.") self.notify_balance_paper_set() diff --git a/hummingbot/client/command/config_command.py b/hummingbot/client/command/config_command.py index c5a93a6679..c82c1f3e5b 100644 --- a/hummingbot/client/command/config_command.py +++ b/hummingbot/client/command/config_command.py @@ -1,11 +1,10 @@ import asyncio from decimal import Decimal -from typing import TYPE_CHECKING, Any, Dict, List, Tuple, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union import pandas as pd from prompt_toolkit.utils import is_windows -import hummingbot.client.config.global_config_map as global_config from hummingbot.client.config.config_data_types import BaseTradingStrategyConfigMap from hummingbot.client.config.config_helpers import ( ClientConfigAdapter, @@ -16,7 +15,7 @@ from hummingbot.client.config.config_validators import validate_bool, validate_decimal from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.security import Security -from hummingbot.client.settings import GLOBAL_CONFIG_PATH, STRATEGIES_CONF_DIR_PATH +from hummingbot.client.settings import CLIENT_CONFIG_PATH, STRATEGIES_CONF_DIR_PATH from hummingbot.client.ui.interface_utils import format_df_for_printout from hummingbot.client.ui.style import load_style from hummingbot.connector.utils import split_hb_trading_pair @@ -43,15 +42,15 @@ "price_floor_pct", "price_band_refresh_time" ] -global_configs_to_display = ["autofill_import", - "kill_switch_enabled", +client_configs_to_display = ["autofill_import", + "kill_switch_mode", "kill_switch_rate", - "telegram_enabled", + "telegram_mode", "telegram_token", "telegram_chat_id", "send_error_logs", - global_config.PMM_SCRIPT_ENABLED_KEY, - global_config.PMM_SCRIPT_FILE_PATH_KEY, + "pmm_script_mode", + "pmm_script_file_path", "ethereum_chain_name", "gateway_enabled", "gateway_cert_passphrase", @@ -89,21 +88,19 @@ def config(self, # type: HummingbotApplication def list_configs(self, # type: HummingbotApplication ): - self.list_global_configs() + self.list_client_configs() self.list_strategy_configs() - def list_global_configs( + def list_client_configs( self # type: HummingbotApplication ): - data = [[cv.key, cv.value] for cv in global_config.global_config_map.values() - if cv.key in global_configs_to_display and not cv.is_secure] + data = self.build_model_df_data(self.client_config_map, to_print=client_configs_to_display) df = map_df_to_str(pd.DataFrame(data=data, columns=columns)) self.notify("\nGlobal Configurations:") lines = [" " + line for line in format_df_for_printout(df, max_col_width=50).split("\n")] self.notify("\n".join(lines)) - data = [[cv.key, cv.value] for cv in global_config.global_config_map.values() - if cv.key in color_settings_to_display and not cv.is_secure] + data = self.build_model_df_data(self.client_config_map, to_print=color_settings_to_display) df = map_df_to_str(pd.DataFrame(data=data, columns=columns)) self.notify("\nColor Settings:") lines = [" " + line for line in format_df_for_printout(df, max_col_width=50).split("\n")] @@ -131,9 +128,13 @@ def build_df_data_from_config_map( return data @staticmethod - def build_model_df_data(config_map: ClientConfigAdapter) -> List[Tuple[str, Any]]: + def build_model_df_data( + config_map: ClientConfigAdapter, to_print: Optional[List[str]] = None + ) -> List[Tuple[str, Any]]: model_data = [] for traversal_item in config_map.traverse(): + if to_print is not None and traversal_item.attr not in to_print: + continue attr_printout = ( " " * (traversal_item.depth - 1) + (u"\u221F " if not is_windows() else " ") @@ -148,7 +149,14 @@ def configurable_keys(self # type: HummingbotApplication Returns a list of configurable keys - using config command, excluding exchanges api keys as they are set from connect command. """ - keys = [c.key for c in global_config.global_config_map.values() if c.prompt is not None and not c.is_connect_key] + keys = [ + traversal_item.config_path + for traversal_item in self.client_config_map.traverse() + if ( + traversal_item.client_field_data is not None + and traversal_item.client_field_data.prompt is not None + ) + ] if self.strategy_config_map is not None: if isinstance(self.strategy_config_map, ClientConfigAdapter): keys.extend([ @@ -196,18 +204,20 @@ async def _config_single_key(self, # type: HummingbotApplication try: if ( - key in global_config.global_config_map - or ( - not isinstance(self.strategy_config_map, (type(None), ClientConfigAdapter)) - and key in self.strategy_config_map - ) + not isinstance(self.strategy_config_map, (type(None), ClientConfigAdapter)) + and key in self.strategy_config_map ): await self._config_single_key_legacy(key, input_value) else: - config_map = self.strategy_config_map - file_path = STRATEGIES_CONF_DIR_PATH / self.strategy_file_name if input_value is None: self.notify("Please follow the prompt to complete configurations: ") + client_config_key = key in self.client_config_map.keys() + if client_config_key: + config_map = self.strategy_config_map + file_path = CLIENT_CONFIG_PATH + else: + config_map = self.strategy_config_map + file_path = STRATEGIES_CONF_DIR_PATH / self.strategy_file_name if key == "inventory_target_base_pct": await self.asset_ratio_maintenance_prompt(config_map, input_value) elif key == "inventory_price": @@ -219,8 +229,11 @@ async def _config_single_key(self, # type: HummingbotApplication return save_to_yml(file_path, config_map) self.notify("\nNew configuration saved.") - self.list_strategy_configs() - self.app.app.style = load_style() + if client_config_key: + self.list_client_configs() + else: + self.list_strategy_configs() + self.app.app.style = load_style(self.client_config_map) except asyncio.TimeoutError: self.logger().error("Prompt timeout") except Exception as err: @@ -236,10 +249,7 @@ async def _config_single_key_legacy( input_value: Any, ): # pragma: no cover config_var, config_map, file_path = None, None, None - if key in global_config.global_config_map: - config_map = global_config.global_config_map - file_path = GLOBAL_CONFIG_PATH - elif self.strategy_config_map is not None and key in self.strategy_config_map: + if self.strategy_config_map is not None and key in self.strategy_config_map: config_map = self.strategy_config_map file_path = STRATEGIES_CONF_DIR_PATH / self.strategy_file_name config_var = config_map[key] @@ -261,7 +271,7 @@ async def _config_single_key_legacy( save_to_yml_legacy(str(file_path), config_map) self.notify("\nNew configuration saved:") self.notify(f"{key}: {str(config_var.value)}") - self.app.app.style = load_style() + self.app.app.style = load_style(self.client_config_map) for config in missings: self.notify(f"{config.key}: {str(config.value)}") if ( @@ -385,7 +395,7 @@ async def inventory_price_prompt_legacy( base_asset, quote_asset = market.split("-") if exchange.endswith("paper_trade"): - balances = global_config.global_config_map["paper_trade_account_balance"].value + balances = self.client_config_map.paper_trade.paper_trade_account_balance else: balances = await UserBalances.instance().balances( exchange, base_asset, quote_asset diff --git a/hummingbot/client/command/connect_command.py b/hummingbot/client/command/connect_command.py index 592f6b7cc6..c622731847 100644 --- a/hummingbot/client/command/connect_command.py +++ b/hummingbot/client/command/connect_command.py @@ -4,7 +4,6 @@ import pandas as pd from hummingbot.client.config.config_helpers import ClientConfigAdapter -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.security import Security from hummingbot.client.settings import AllConnectorSettings from hummingbot.client.ui.interface_utils import format_df_for_printout @@ -79,7 +78,7 @@ async def show_connections(self # type: HummingbotApplication ): self.notify("\nTesting connections, please wait...") df, failed_msgs = await self.connection_df() - lines = [" " + line for line in format_df_for_printout(df).split("\n")] + lines = [" " + line for line in format_df_for_printout(df, self.client_config_map.tables_format).split("\n")] if failed_msgs: lines.append("\nFailed connections:") lines.extend([" " + k + ": " + v for k, v in failed_msgs.items()]) @@ -91,10 +90,10 @@ async def connection_df(self # type: HummingbotApplication columns = ["Exchange", " Keys Added", " Keys Confirmed", " Status"] data = [] failed_msgs = {} - network_timeout = float(global_config_map["other_commands_timeout"].value) + network_timeout = self.client_config_map.commands_timeout.other_commands_timeout try: err_msgs = await asyncio.wait_for( - UserBalances.instance().update_exchanges(reconnect=True), network_timeout + UserBalances.instance().update_exchanges(self.client_config_map, reconnect=True), network_timeout ) except asyncio.TimeoutError: self.notify("\nA network error prevented the connection table to populate. See logs for more details.") @@ -145,13 +144,17 @@ async def validate_n_connect_celo(self, to_reconnect: bool = False) -> Optional[ err_msg = CeloCLI.unlock_account(celo_config.celo_address, celo_config.celo_password.get_secret_value()) return err_msg - async def validate_n_connect_connector(self, connector_name: str) -> Optional[str]: + async def validate_n_connect_connector( + self, # type: HummingbotApplication + connector_name: str, + ) -> Optional[str]: await Security.wait_til_decryption_done() api_keys = Security.api_keys(connector_name) - network_timeout = float(global_config_map["other_commands_timeout"].value) + network_timeout = self.client_config_map.commands_timeout.other_commands_timeout try: err_msg = await asyncio.wait_for( - UserBalances.instance().add_exchange(connector_name, **api_keys), network_timeout + UserBalances.instance().add_exchange(connector_name, self.client_config_map, **api_keys), + network_timeout, ) except asyncio.TimeoutError: self.notify( diff --git a/hummingbot/client/command/create_command.py b/hummingbot/client/command/create_command.py index 8860c365de..f6de78e760 100644 --- a/hummingbot/client/command/create_command.py +++ b/hummingbot/client/command/create_command.py @@ -19,7 +19,6 @@ save_to_yml_legacy, ) from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.settings import STRATEGIES_CONF_DIR_PATH, required_exchanges from hummingbot.client.ui.completer import load_completer from hummingbot.core.utils.async_utils import safe_ensure_future @@ -235,7 +234,7 @@ async def verify_status( self # type: HummingbotApplication ): try: - timeout = float(global_config_map["create_command_timeout"].value) + timeout = self.client_config_map.commands_timeout.create_command_timeout all_status_go = await asyncio.wait_for(self.status_check_all(), timeout) except asyncio.TimeoutError: self.notify("\nA network error prevented the connection check to complete. See logs for more details.") diff --git a/hummingbot/client/command/export_command.py b/hummingbot/client/command/export_command.py index a3248f4f6c..74756b709d 100644 --- a/hummingbot/client/command/export_command.py +++ b/hummingbot/client/command/export_command.py @@ -4,7 +4,6 @@ import pandas as pd from sqlalchemy.orm import Query, Session -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.security import Security from hummingbot.client.settings import DEFAULT_LOG_FILE_PATH from hummingbot.core.utils.async_utils import safe_ensure_future @@ -71,7 +70,7 @@ async def export_trades(self, # type: HummingbotApplication return self.placeholder_mode = True self.app.hide_input = True - path = global_config_map["log_file_path"].value + path = self.client_config_map.log_file_path if path is None: path = str(DEFAULT_LOG_FILE_PATH) file_name = await self.prompt_new_export_file_name(path) diff --git a/hummingbot/client/command/gateway_command.py b/hummingbot/client/command/gateway_command.py index ce5579f2b3..0bbaba9c66 100644 --- a/hummingbot/client/command/gateway_command.py +++ b/hummingbot/client/command/gateway_command.py @@ -9,12 +9,11 @@ import pandas as pd import docker -from hummingbot.client.config.config_helpers import refresh_trade_fees_config, save_to_yml_legacy -from hummingbot.client.config.global_config_map import global_config_map +from hummingbot.client.config.config_helpers import refresh_trade_fees_config, save_to_yml from hummingbot.client.config.security import Security from hummingbot.client.settings import ( + CLIENT_CONFIG_PATH, GATEWAY_CONNECTORS, - GLOBAL_CONFIG_PATH, AllConnectorSettings, GatewayConnectionSetting, ) @@ -68,14 +67,18 @@ def create_gateway(self): def gateway_connect(self, connector: str = None): safe_ensure_future(self._gateway_connect(connector), loop=self.ev_loop) - def gateway_start(self): - safe_ensure_future(start_gateway(), loop=self.ev_loop) + def gateway_start( + self # type: HummingbotApplication + ): + safe_ensure_future(start_gateway(self.client_config_map), loop=self.ev_loop) def gateway_status(self): safe_ensure_future(self._gateway_status(), loop=self.ev_loop) - def gateway_stop(self): - safe_ensure_future(stop_gateway(), loop=self.ev_loop) + def gateway_stop( + self # type: HummingbotApplication + ): + safe_ensure_future(stop_gateway(self.client_config_map), loop=self.ev_loop) def generate_certs(self): safe_ensure_future(self._generate_certs(), loop=self.ev_loop) @@ -98,7 +101,7 @@ async def check_gateway_image(docker_repo: str, docker_tag: str) -> bool: async def _test_connection(self): # test that the gateway is running - if await GatewayHttpClient.get_instance().ping_gateway(): + if await self._get_gateway_instance().ping_gateway(): self.notify("\nSuccesfully pinged gateway.") else: self.notify("\nUnable to ping gateway.") @@ -107,9 +110,9 @@ async def _generate_certs( self, # type: HummingbotApplication from_client_password: bool = False ): - cert_path: str = get_gateway_paths().local_certs_path.as_posix() + cert_path: str = get_gateway_paths(self.client_config_map).local_certs_path.as_posix() if not from_client_password: - if certs_files_exist(): + if certs_files_exist(self.client_config_map): self.notify(f"Gateway SSL certification files exist in {cert_path}.") self.notify("To create new certification files, please first manually delete those files.") return @@ -125,9 +128,9 @@ async def _generate_certs( self.notify("Error: Invalid pass phase") else: pass_phase = Security.secrets_manager.password.get_secret_value() - create_self_sign_certs(pass_phase) + create_self_sign_certs(pass_phase, self.client_config_map) self.notify(f"Gateway SSL certification files are created in {cert_path}.") - GatewayHttpClient.get_instance().reload_certs() + self._get_gateway_instance().reload_certs() async def _generate_gateway_confs( self, # type: HummingbotApplication @@ -175,13 +178,15 @@ async def _generate_gateway_confs( except Exception: raise - async def _create_gateway(self): - gateway_paths: GatewayPaths = get_gateway_paths() - gateway_container_name: str = get_gateway_container_name() + async def _create_gateway( + self # type: HummingbotApplication + ): + gateway_paths: GatewayPaths = get_gateway_paths(self.client_config_map) + gateway_container_name: str = get_gateway_container_name(self.client_config_map) gateway_conf_mount_path: str = gateway_paths.mount_conf_path.as_posix() certificate_mount_path: str = gateway_paths.mount_certs_path.as_posix() logs_mount_path: str = gateway_paths.mount_logs_path.as_posix() - gateway_port: int = get_default_gateway_port() + gateway_port: int = get_default_gateway_port(self.client_config_map) # remove existing container(s) try: @@ -251,14 +256,16 @@ async def _create_gateway(self): self.notify(f"New Gateway docker container id is {container_info['Id']}.") # Save the gateway port number, if it's not already there. - if global_config_map.get("gateway_api_port").value != gateway_port: - global_config_map["gateway_api_port"].value = gateway_port - global_config_map["gateway_api_host"].value = "localhost" - save_to_yml_legacy(GLOBAL_CONFIG_PATH, global_config_map) - - GatewayHttpClient.get_instance().base_url = f"https://{global_config_map['gateway_api_host'].value}:" \ - f"{global_config_map['gateway_api_port'].value}" - await start_gateway() + gateway_config_map = self.client_config_map.gateway + if gateway_config_map.gateway_api_port != gateway_port: + gateway_config_map.gateway_api_port = gateway_port + gateway_config_map.gateway_api_host = "localhost" + save_to_yml(CLIENT_CONFIG_PATH, self.client_config_map) + + self._get_gateway_instance().base_url = ( + f"https://{gateway_config_map.gateway_api_host}:{gateway_config_map.gateway_api_port}" + ) + await start_gateway(self.client_config_map) # create Gateway configs await self._generate_gateway_confs(container_id=container_info["Id"]) @@ -296,7 +303,7 @@ async def _gateway_status(self): if self._gateway_monitor.current_status == Status.ONLINE: try: - status = await GatewayHttpClient.get_instance().get_gateway_status() + status = await self._get_gateway_instance().get_gateway_status() self.notify(pd.DataFrame(status)) except Exception: self.notify("\nError: Unable to fetch status of connected Gateway server.") @@ -305,15 +312,18 @@ async def _gateway_status(self): async def _update_gateway_configuration(self, key: str, value: Any): try: - response = await GatewayHttpClient.get_instance().update_config(key, value) + response = await self._get_gateway_instance().update_config(key, value) self.notify(response["message"]) await self._gateway_monitor.update_gateway_config_key_list() except Exception: self.notify("\nError: Gateway configuration update failed. See log file for more details.") - async def _show_gateway_configuration(self, key: str): - host = global_config_map['gateway_api_host'].value - port = global_config_map['gateway_api_port'].value + async def _show_gateway_configuration( + self, # type: HummingbotApplication + key: str, + ): + host = self.client_config_map.gateway.gateway_api_host + port = self.client_config_map.gateway.gateway_api_port try: config_dict: Dict[str, Any] = await self._gateway_monitor._fetch_gateway_configs() if key is not None: @@ -343,7 +353,7 @@ async def _gateway_connect( self.notify(connector_df.to_string(index=False)) else: # get available networks - connector_configs: Dict[str, Any] = await GatewayHttpClient.get_instance().get_connectors() + connector_configs: Dict[str, Any] = await self._get_gateway_instance().get_connectors() connector_config: List[Dict[str, Any]] = [ d for d in connector_configs["connectors"] if d["name"] == connector ] @@ -393,7 +403,7 @@ async def _gateway_connect( self.notify("Error: Invalid network") # get wallets for the selected chain - wallets_response: List[Dict[str, Any]] = await GatewayHttpClient.get_instance().get_wallets() + wallets_response: List[Dict[str, Any]] = await self._get_gateway_instance().get_wallets() matching_wallets: List[Dict[str, Any]] = [w for w in wallets_response if w["chain"] == chain] wallets: List[str] if len(matching_wallets) < 1: @@ -412,7 +422,7 @@ async def _gateway_connect( self.app.clear_input() if self.app.to_stop_config: return - response: Dict[str, Any] = await GatewayHttpClient.get_instance().add_wallet( + response: Dict[str, Any] = await self._get_gateway_instance().add_wallet( chain, network, wallet_private_key ) wallet_address: str = response["address"] @@ -437,7 +447,7 @@ async def _gateway_connect( native_token: str = native_tokens[chain] wallet_table: List[Dict[str, Any]] = [] for w in wallets: - balances: Dict[str, Any] = await GatewayHttpClient.get_instance().get_balances( + balances: Dict[str, Any] = await self._get_gateway_instance().get_balances( chain, network, w, [native_token] ) wallet_table.append({"balance": balances['balances'][native_token], "address": w}) @@ -467,7 +477,7 @@ async def _gateway_connect( if self.app.to_stop_config: return - response: Dict[str, Any] = await GatewayHttpClient.get_instance().add_wallet( + response: Dict[str, Any] = await self._get_gateway_instance().add_wallet( chain, network, wallet_private_key ) wallet_address = response["address"] @@ -483,8 +493,16 @@ async def _gateway_connect( # update AllConnectorSettings and fee overrides. AllConnectorSettings.create_connector_settings() - AllConnectorSettings.initialize_paper_trade_settings(global_config_map.get("paper_trade_exchanges").value) - await refresh_trade_fees_config() + AllConnectorSettings.initialize_paper_trade_settings( + self.client_config_map.paper_trade.paper_trade_exchanges + ) + await refresh_trade_fees_config(self.client_config_map) # Reload completer here to include newly added gateway connectors self.app.input_field.completer = load_completer(self) + + def _get_gateway_instance( + self # type: HummingbotApplication + ) -> GatewayHttpClient: + gateway_instance = GatewayHttpClient.get_instance(self.client_config_map) + return gateway_instance diff --git a/hummingbot/client/command/history_command.py b/hummingbot/client/command/history_command.py index e65e47d0a0..4b0801e22c 100644 --- a/hummingbot/client/command/history_command.py +++ b/hummingbot/client/command/history_command.py @@ -3,22 +3,12 @@ import time from datetime import datetime from decimal import Decimal -from typing import ( - List, - Optional, - Set, - Tuple, - TYPE_CHECKING, -) +from typing import TYPE_CHECKING, List, Optional, Set, Tuple import pandas as pd -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.performance import PerformanceMetrics -from hummingbot.client.settings import ( - AllConnectorSettings, - MAXIMUM_TRADE_FILLS_DISPLAY_OUTPUT, -) +from hummingbot.client.settings import MAXIMUM_TRADE_FILLS_DISPLAY_OUTPUT, AllConnectorSettings from hummingbot.client.ui.interface_utils import format_df_for_printout from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.model.trade_fill import TradeFill @@ -74,7 +64,7 @@ async def history_report(self, # type: HummingbotApplication return_pcts = [] for market, symbol in market_info: cur_trades = [t for t in trades if t.market == market and t.symbol == symbol] - network_timeout = float(global_config_map["other_commands_timeout"].value) + network_timeout = float(self.client_config_map.commands_timeout.other_commands_timeout) try: cur_balances = await asyncio.wait_for(self.get_current_balances(market), network_timeout) except asyncio.TimeoutError: @@ -96,12 +86,12 @@ async def get_current_balances(self, # type: HummingbotApplication if market in self.markets and self.markets[market].ready: return self.markets[market].get_all_balances() elif "Paper" in market: - paper_balances = global_config_map["paper_trade_account_balance"].value + paper_balances = self.client_config_map.paper_trade.paper_trade_account_balance if paper_balances is None: return {} return {token: Decimal(str(bal)) for token, bal in paper_balances.items()} else: - await UserBalances.instance().update_exchange_balance(market) + await UserBalances.instance().update_exchange_balance(market, self.client_config_map) return UserBalances.instance().all_balances(market) def report_header(self, # type: HummingbotApplication @@ -235,7 +225,7 @@ def list_trades(self, # type: HummingbotApplication df = df[:MAXIMUM_TRADE_FILLS_DISPLAY_OUTPUT] self.notify( f"\n Showing last {MAXIMUM_TRADE_FILLS_DISPLAY_OUTPUT} trades in the current session.") - df_lines = format_df_for_printout(df).split("\n") + df_lines = format_df_for_printout(df, self.client_config_map.tables_format).split("\n") lines.extend(["", " Recent trades:"] + [" " + line for line in df_lines]) else: diff --git a/hummingbot/client/command/import_command.py b/hummingbot/client/command/import_command.py index 7a8c1ab8e6..a1169faf80 100644 --- a/hummingbot/client/command/import_command.py +++ b/hummingbot/client/command/import_command.py @@ -1,13 +1,13 @@ import asyncio from typing import TYPE_CHECKING +from hummingbot.client.config.client_config_map import AutofillImportEnum from hummingbot.client.config.config_helpers import ( format_config_file_name, load_strategy_config_map_from_file, short_strategy_name, validate_strategy_file, ) -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.settings import CONF_PREFIX, STRATEGIES_CONF_DIR_PATH, required_exchanges from hummingbot.core.utils.async_utils import safe_ensure_future @@ -57,8 +57,8 @@ async def import_config_file(self, # type: HummingbotApplication raise if all_status_go: self.notify("\nEnter \"start\" to start market making.") - autofill_import = global_config_map.get("autofill_import").value - if autofill_import is not None: + autofill_import = self.client_config_map.autofill_import + if autofill_import != AutofillImportEnum.disabled: self.app.set_text(autofill_import) async def prompt_a_file_name(self # type: HummingbotApplication diff --git a/hummingbot/client/command/order_book_command.py b/hummingbot/client/command/order_book_command.py index bee2284706..f97dcaa271 100644 --- a/hummingbot/client/command/order_book_command.py +++ b/hummingbot/client/command/order_book_command.py @@ -1,9 +1,13 @@ +from typing import TYPE_CHECKING + +import pandas as pd + from hummingbot.client.ui.interface_utils import format_df_for_printout from hummingbot.core.utils.async_utils import safe_ensure_future -import pandas as pd -from typing import TYPE_CHECKING + if TYPE_CHECKING: from hummingbot.client.hummingbot_application import HummingbotApplication + import threading @@ -48,7 +52,10 @@ def get_order_book(lines): asks = order_book.snapshot[1][['price', 'amount']].head(lines) asks.rename(columns={'price': 'ask_price', 'amount': 'ask_volume'}, inplace=True) joined_df = pd.concat([bids, asks], axis=1) - text_lines = [" " + line for line in format_df_for_printout(joined_df).split("\n")] + text_lines = [ + " " + line + for line in format_df_for_printout(joined_df, self.client_config_map.tables_format).split("\n") + ] header = f" market: {market_connector.name} {trading_pair}\n" return header + "\n".join(text_lines) diff --git a/hummingbot/client/command/start_command.py b/hummingbot/client/command/start_command.py index 22e114afe4..b66da8d931 100644 --- a/hummingbot/client/command/start_command.py +++ b/hummingbot/client/command/start_command.py @@ -2,12 +2,10 @@ import platform import threading import time -from os.path import dirname from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional import pandas as pd -import hummingbot.client.config.global_config_map as global_config import hummingbot.client.settings as settings from hummingbot import init_logging from hummingbot.client.command.rate_command import RateCommand @@ -19,9 +17,7 @@ from hummingbot.core.clock import Clock, ClockMode from hummingbot.core.rate_oracle.rate_oracle import RateOracle from hummingbot.core.utils.async_utils import safe_ensure_future -from hummingbot.core.utils.kill_switch import KillSwitch from hummingbot.exceptions import OracleRateUnavailable -from hummingbot.pmm_script.pmm_script_iterator import PMMScriptIterator from hummingbot.strategy.script_strategy_base import ScriptStrategyBase from hummingbot.user.user_balances import UserBalances @@ -76,6 +72,7 @@ async def start_check(self, # type: HummingbotApplication return if self._last_started_strategy_file != self.strategy_file_name: init_logging("hummingbot_logs.yml", + self.client_config_map, override_log_level=log_level.upper() if log_level else None, strategy_file_path=self.strategy_file_name) self._last_started_strategy_file = self.strategy_file_name @@ -112,7 +109,7 @@ async def start_check(self, # type: HummingbotApplication ["network", connector_details['network']], ["wallet_address", connector_details['wallet_address']] ] - await UserBalances.instance().update_exchange_balance(connector) + await UserBalances.instance().update_exchange_balance(connector, self.client_config_map) balances: List[str] = [ f"{str(PerformanceMetrics.smart_round(v, 8))} {k}" for k, v in UserBalances.instance().all_balances(connector).items() @@ -180,27 +177,22 @@ async def start_market_making(self, # type: HummingbotApplication self.notify(f"Restored {len(market.limit_orders)} limit orders on {market.name}...") if self.strategy: self.clock.add_iterator(self.strategy) - if global_config.global_config_map[global_config.PMM_SCRIPT_ENABLED_KEY].value: - pmm_script_file = global_config.global_config_map[global_config.PMM_SCRIPT_FILE_PATH_KEY].value - folder = dirname(pmm_script_file) - if folder == "": - pmm_script_file = settings.PMM_SCRIPTS_PATH / pmm_script_file - if self.strategy_name != "pure_market_making": - self.notify("Error: PMM script feature is only available for pure_market_making strategy.") - else: - self._pmm_script_iterator = PMMScriptIterator(pmm_script_file, - list(self.markets.values()), - self.strategy, 0.1) - self.clock.add_iterator(self._pmm_script_iterator) - self.notify(f"PMM script ({pmm_script_file}) started.") - + try: + self._pmm_script_iterator = self.client_config_map.pmm_script_mode.get_iterator( + self.strategy_name, list(self.markets.values()), self.strategy + ) + except ValueError as e: + self.notify(f"Error: {e}") + else: + self.clock.add_iterator(self._pmm_script_iterator) + self.notify(f"PMM script ({self.client_config_map.pmm_script_mode.pmm_script_file_path}) started.") self.strategy_task: asyncio.Task = safe_ensure_future(self._run_clock(), loop=self.ev_loop) self.notify(f"\n'{self.strategy_name}' strategy started.\n" f"Run `status` command to query the progress.") self.logger().info("start command initiated.") if self._trading_required: - self.kill_switch = KillSwitch(self) + self.kill_switch = self.client_config_map.kill_switch_mode.get_kill_switch(self) await self.wait_till_ready(self.kill_switch.start) except Exception as e: self.logger().error(str(e), exc_info=True) diff --git a/hummingbot/client/command/status_command.py b/hummingbot/client/command/status_command.py index 606a79b562..67083c1211 100644 --- a/hummingbot/client/command/status_command.py +++ b/hummingbot/client/command/status_command.py @@ -12,7 +12,6 @@ get_strategy_config_map, missing_required_configs_legacy, ) -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.security import Security from hummingbot.client.settings import ethereum_wallet_required, required_exchanges from hummingbot.connector.connector_base import ConnectorBase @@ -91,14 +90,16 @@ def application_warning(self): self.notify(app_warning) return app_warning - async def validate_required_connections(self) -> Dict[str, str]: + async def validate_required_connections( + self # type: HummingbotApplication + ) -> Dict[str, str]: invalid_conns = {} if self.strategy_name == "celo_arb": err_msg = await self.validate_n_connect_celo(True) if err_msg is not None: invalid_conns["celo"] = err_msg if not any([str(exchange).endswith("paper_trade") for exchange in required_exchanges]): - connections = await UserBalances.instance().update_exchanges(exchanges=required_exchanges) + connections = await UserBalances.instance().update_exchanges(self.client_config_map, exchanges=required_exchanges) invalid_conns.update({ex: err_msg for ex, err_msg in connections.items() if ex in required_exchanges and err_msg is not None}) if ethereum_wallet_required(): @@ -110,7 +111,7 @@ async def validate_required_connections(self) -> Dict[str, str]: def missing_configurations_legacy( self, # type: HummingbotApplication ) -> List[str]: - missing_globals = missing_required_configs_legacy(global_config_map) + missing_globals = missing_required_configs_legacy(self.client_config_map) config_map = self.strategy_config_map missing_configs = [] if not isinstance(config_map, ClientConfigAdapter): @@ -174,7 +175,7 @@ async def status_check_all(self, # type: HummingbotApplication self.notify(f" {error}") return False - network_timeout = float(global_config_map["other_commands_timeout"].value) + network_timeout = self.client_config_map.commands_timeout.other_commands_timeout try: invalid_conns = await asyncio.wait_for(self.validate_required_connections(), network_timeout) except asyncio.TimeoutError: diff --git a/hummingbot/client/command/ticker_command.py b/hummingbot/client/command/ticker_command.py index 3822e6dc62..0563cc835e 100644 --- a/hummingbot/client/command/ticker_command.py +++ b/hummingbot/client/command/ticker_command.py @@ -53,7 +53,7 @@ def get_ticker(): float(market_connector.get_price_by_type(trading_pair, PriceType.LastTrade)) ]] ticker_df = pd.DataFrame(data=data, columns=columns) - ticker_df_str = format_df_for_printout(ticker_df) + ticker_df_str = format_df_for_printout(ticker_df, self.client_config_map.tables_format) return f" Market: {market_connector.name}\n{ticker_df_str}" if live: diff --git a/hummingbot/client/config/client_config_map.py b/hummingbot/client/config/client_config_map.py new file mode 100644 index 0000000000..6d9a1ce7e8 --- /dev/null +++ b/hummingbot/client/config/client_config_map.py @@ -0,0 +1,785 @@ +import json +import os.path +import random +import re +from abc import ABC, abstractmethod +from decimal import Decimal +from os.path import dirname +from pathlib import Path +from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Union + +from pydantic import BaseModel, Field, Json, root_validator, validator +from tabulate import tabulate_formats + +from hummingbot.client.config.config_data_types import BaseClientModel, ClientConfigEnum, ClientFieldData +from hummingbot.client.config.config_methods import using_exchange as using_exchange_pointer +from hummingbot.client.config.config_validators import validate_bool +from hummingbot.client.settings import DEFAULT_LOG_FILE_PATH, PMM_SCRIPTS_PATH, AllConnectorSettings +from hummingbot.connector.connector_base import ConnectorBase +from hummingbot.connector.connector_metrics_collector import ( + DummyMetricsCollector, + MetricsCollector, + TradeVolumeMetricCollector, +) +from hummingbot.connector.exchange.ascend_ex.ascend_ex_utils import AscendExConfigMap +from hummingbot.connector.exchange.binance.binance_utils import BinanceConfigMap +from hummingbot.connector.exchange.gate_io.gate_io_utils import GateIOConfigMap +from hummingbot.connector.exchange.kucoin.kucoin_utils import KuCoinConfigMap +from hummingbot.connector.exchange_base import ExchangeBase +from hummingbot.core.rate_oracle.rate_oracle import RateOracle, RateOracleSource +from hummingbot.core.utils.kill_switch import ActiveKillSwitch, KillSwitch, PassThroughKillSwitch +from hummingbot.notifier.telegram_notifier import TelegramNotifier +from hummingbot.pmm_script.pmm_script_iterator import PMMScriptIterator +from hummingbot.strategy.strategy_base import StrategyBase + +if TYPE_CHECKING: + from hummingbot.client.hummingbot_application import HummingbotApplication + +PMM_SCRIPT_ENABLED_KEY = "pmm_script_enabled" +PMM_SCRIPT_FILE_PATH_KEY = "pmm_script_file_path" + + +def generate_client_id() -> str: + vals = [random.choice(range(0, 256)) for i in range(0, 20)] + return "".join([f"{val:02x}" for val in vals]) + + +def using_exchange(exchange: str) -> Callable: + return using_exchange_pointer(exchange) + + +class ColorConfigMap(BaseClientModel): + top_pane: str = Field( + default="#000000", + client_data=ClientFieldData( + prompt=lambda cm: "What is the background color of the top pane?", + ), + ) + bottom_pane: str = Field( + default="#000000", + client_data=ClientFieldData( + prompt=lambda cm: "What is the background color of the bottom pane?", + ), + ) + output_pane: str = Field( + default="#262626", + client_data=ClientFieldData( + prompt=lambda cm: "What is the background color of the output pane?", + ), + ) + input_pane: str = Field( + default="#1C1C1C", + client_data=ClientFieldData( + prompt=lambda cm: "What is the background color of the input pane?", + ), + ) + logs_pane: str = Field( + default="#121212", + client_data=ClientFieldData( + prompt=lambda cm: "What is the background color of the logs pane?", + ), + ) + terminal_primary: str = Field( + default="#5FFFD7", + client_data=ClientFieldData( + prompt=lambda cm: "What is the terminal primary color?", + ), + ) + primary_label: str = Field( + default="#5FFFD7", + client_data=ClientFieldData( + prompt=lambda cm: "What is the background color for secondary label?", + ), + ) + secondary_label: str = Field( + default="#FFFFFF", + client_data=ClientFieldData( + prompt=lambda cm: "What is the background color for secondary label?", + ), + ) + success_label: str = Field( + default="#5FFFD7", + client_data=ClientFieldData( + prompt=lambda cm: "What is the background color for success label?", + ), + ) + warning_label: str = Field( + default="#FFFF00", + client_data=ClientFieldData( + prompt=lambda cm: "What is the background color for warning label?", + ), + ) + info_label: str = Field( + default="#5FD7FF", + client_data=ClientFieldData( + prompt=lambda cm: "What is the background color for info label?", + ), + ) + error_label: str = Field( + default="#FF0000", + client_data=ClientFieldData( + prompt=lambda cm: "What is the background color for error label?", + ), + ) + + @validator( + "top_pane", + "bottom_pane", + "output_pane", + "input_pane", + "logs_pane", + "terminal_primary", + "primary_label", + "secondary_label", + "success_label", + "warning_label", + "info_label", + "error_label", + pre=True, + ) + def validate_color(cls, v: str): + if not re.search(r'^#(?:[0-9a-fA-F]{2}){3}$', v): + raise ValueError("Invalid color code") + return v + + +class PaperTradeConfigMap(BaseClientModel): + paper_trade_exchanges: List = Field( + default=[ + BinanceConfigMap.Config.title, + KuCoinConfigMap.Config.title, + AscendExConfigMap.Config.title, + GateIOConfigMap.Config.title, + ], + ) + paper_trade_account_balance: Json = Field( + default={ + "BTC": 1, + "USDT": 1000, + "ONE": 1000, + "USDQ": 1000, + "TUSD": 1000, + "ETH": 10, + "WETH": 10, + "USDC": 1000, + "DAI": 1000, + }, + client_data=ClientFieldData( + prompt=lambda cm: ( + "Enter paper trade balance settings (Input must be valid json — " + "e.g. {\"ETH\": 10, \"USDC\": 50000})" + ), + ), + ) + + +class KillSwitchMode(BaseClientModel, ABC): + @abstractmethod + def get_kill_switch(self, hb: "HummingbotApplication") -> KillSwitch: + ... + + +class KillSwitchEnabledMode(KillSwitchMode): + kill_switch_rate: Decimal = Field( + default=..., + ge=Decimal(-100), + le=Decimal(100), + client_data=ClientFieldData( + prompt=lambda cm: ( + "At what profit/loss rate would you like the bot to stop?" + " (e.g. -5 equals 5 percent loss)" + ), + ), + ) + + class Config: + title = "kill_switch_enabled" + + def get_kill_switch(self, hb: "HummingbotApplication") -> ActiveKillSwitch: + kill_switch = ActiveKillSwitch(kill_switch_rate=self.kill_switch_rate, hummingbot_application=hb) + return kill_switch + + @validator("kill_switch_rate", pre=True) + def validate_decimal(cls, v: str, field: Field): + """Used for client-friendly error output.""" + super().validate_decimal(v, field) + + +class KillSwitchDisabledMode(KillSwitchMode): + class Config: + title = "kill_switch_disabled" + + def get_kill_switch(self, hb: "HummingbotApplication") -> PassThroughKillSwitch: + kill_switch = PassThroughKillSwitch() + return kill_switch + + +KILL_SWITCH_MODES = { + KillSwitchEnabledMode.Config.title: KillSwitchEnabledMode, + KillSwitchDisabledMode.Config.title: KillSwitchDisabledMode, +} + + +class AutofillImportEnum(str, ClientConfigEnum): + start = "start" + config = "config" + disabled = "disabled" + + +class TelegramMode(BaseClientModel, ABC): + @abstractmethod + def get_notifiers(self, hb: "HummingbotApplication") -> List[TelegramNotifier]: + ... + + +class TelegramEnabledMode(TelegramMode): + telegram_token: str = Field( + default=..., + client_data=ClientFieldData(prompt=lambda cm: "What is your telegram token?"), + ) + telegram_chat_id: str = Field( + default=..., + client_data=ClientFieldData(prompt=lambda cm: "What is your telegram chat id?"), + ) + + class Config: + title = "telegram_enabled" + + def get_notifiers(self, hb: "HummingbotApplication") -> List[TelegramNotifier]: + notifiers = [ + TelegramNotifier(token=self.telegram_token, chat_id=self.telegram_chat_id, hb=hb) + ] + return notifiers + + +class TelegramDisabledMode(TelegramMode): + class Config: + title = "telegram_disabled" + + def get_notifiers(self, hb: "HummingbotApplication") -> List[TelegramNotifier]: + return [] + + +TELEGRAM_MODES = { + TelegramEnabledMode.Config.title: TelegramEnabledMode, + TelegramDisabledMode.Config.title: TelegramDisabledMode, +} + + +class DBMode(BaseClientModel, ABC): + @abstractmethod + def get_url(self, db_path: str) -> str: + ... + + +class DBSqliteMode(DBMode): + db_engine: str = Field( + default="sqlite", + const=True, + client_data=ClientFieldData( + prompt=lambda cm: ( + "Please enter database engine you want to use (reference: https://docs.sqlalchemy.org/en/13/dialects/)" + ), + ), + ) + + class Config: + title = "sqlite_db_engine" + + def get_url(self, db_path: str) -> str: + return f"{self.db_engine}:///{db_path}" + + +class DBOtherMode(DBMode): + db_engine: str = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: ( + "Please enter database engine you want to use (reference: https://docs.sqlalchemy.org/en/13/dialects/)" + ), + ), + ) + db_host: str = Field( + default="127.0.0.1", + client_data=ClientFieldData( + prompt=lambda cm: "Please enter your DB host address", + ), + ) + db_port: int = Field( + default=3306, + client_data=ClientFieldData( + prompt=lambda cm: "Please enter your DB port", + ), + ) + db_username: str = Field( + defaul="username", + client_data=ClientFieldData( + prompt=lambda cm: "Please enter your DB username", + ), + ) + db_password: str = Field( + default="password", + client_data=ClientFieldData( + prompt=lambda cm: "Please enter your DB password", + ), + ) + db_name: str = Field( + default="dbname", + client_data=ClientFieldData( + prompt=lambda cm: "Please enter your the name of your DB", + ), + ) + + class Config: + title = "other_db_engine" + + def get_url(self, db_path: str) -> str: + return f"{self.db_engine}://{self.db_username}:{self.db_password}@{self.db_host}:{self.db_port}/{self.db_name}" + + @validator("db_engine") + def validate_db_engine(cls, v: str): + assert v != "sqlite" + return v + + +DB_MODES = { + DBSqliteMode.Config.title: DBSqliteMode, + DBOtherMode.Config.title: DBOtherMode, +} + + +class PMMScriptMode(BaseClientModel, ABC): + @abstractmethod + def get_iterator(self, strategy_name: str, markets: List[ExchangeBase], strategy: StrategyBase) -> Optional[PMMScriptIterator]: + ... + + +class PMMScriptDisabledMode(PMMScriptMode): + class Config: + title = "pmm_script_disabled" + + def get_iterator(self, strategy_name: str, markets: List[ExchangeBase], strategy: StrategyBase) -> Optional[PMMScriptIterator]: + return None + + +class PMMScriptEnabledMode(PMMScriptMode): + pmm_script_file_path: str = Field( + default=..., + client_data=ClientFieldData(prompt=lambda cm: "Enter path to your PMM script file"), + ) + + class Config: + title = "pmm_script_enabled" + + def get_iterator(self, strategy_name: str, markets: List[ExchangeBase], strategy: StrategyBase) -> Optional[PMMScriptIterator]: + if strategy_name != "pure_market_making": + raise ValueError("PMM script feature is only available for pure_market_making strategy.") + folder = dirname(self.pmm_script_file_path) + pmm_script_file = ( + PMM_SCRIPTS_PATH / self.pmm_script_file_path + if folder == "" + else self.pmm_script_file_path + ) + pmm_script_iterator = PMMScriptIterator( + pmm_script_file, + markets, + self.strategy, + queue_check_interval=0.1 + ) + return pmm_script_iterator + + @validator("pmm_script_file_path", pre=True) + def validate_pmm_script_file_path(cls, v: str): + import hummingbot.client.settings as settings + file_path = v + path, name = os.path.split(file_path) + if path == "": + file_path = os.path.join(settings.PMM_SCRIPTS_PATH, file_path) + if not os.path.isfile(file_path): + raise ValueError(f"{file_path} file does not exist.") + return file_path + + +PMM_SCRIPT_MODES = { + PMMScriptDisabledMode.Config.title: PMMScriptDisabledMode, + PMMScriptEnabledMode.Config.title: PMMScriptEnabledMode, +} + + +class GatewayConfigMap(BaseClientModel): + gateway_api_host: str = Field(default="localhost") + gateway_api_port: str = Field( + default="5000", + client_data=ClientFieldData( + prompt=lambda cm: "Please enter your Gateway API port", + ), + ) + + +class GlobalTokenConfigMap(BaseClientModel): + global_token_name: str = Field( + default="USD", + client_data=ClientFieldData( + prompt=lambda + cm: "What is your default display token? (e.g. USD,EUR,BTC)", + ), + ) + global_token_symbol: str = Field( + default="$", + client_data=ClientFieldData( + prompt=lambda + cm: "What is your default display token symbol? (e.g. $,€)", + ), + ) + + # === post-validations === + + @root_validator() + def post_validations(cls, values: Dict): + cls.global_token_on_validated(values) + cls.global_token_symbol_on_validated(values) + return values + + @classmethod + def global_token_on_validated(cls, values: Dict): + RateOracle.global_token = values["global_token_name"].upper() + + @classmethod + def global_token_symbol_on_validated(cls, values: Dict): + RateOracle.global_token_symbol = values["global_token_symbol"] + + +class CommandsTimeoutConfigMap(BaseClientModel): + create_command_timeout: Decimal = Field( + default=Decimal("10"), + gt=Decimal("0"), + client_data=ClientFieldData( + prompt=lambda cm: ( + "Network timeout when fetching the minimum order amount in the create command (in seconds)" + ), + ), + ) + other_commands_timeout: Decimal = Field( + default=Decimal("30"), + gt=Decimal("0"), + client_data=ClientFieldData( + prompt=lambda cm: ( + "Network timeout to apply to the other commands' API calls" + " (i.e. import, connect, balance, history; in seconds)" + ), + ), + ) + + @validator( + "create_command_timeout", + "other_commands_timeout", + pre=True, + ) + def validate_decimals(cls, v: str, field: Field): + """Used for client-friendly error output.""" + super().validate_decimal(v, field) + + +class AnonymizedMetricsMode(BaseClientModel, ABC): + @abstractmethod + def get_collector( + self, + connector: ConnectorBase, + rate_provider: RateOracle, + instance_id: str, + valuation_token: str = "USDT", + ) -> MetricsCollector: + ... + + +class AnonymizedMetricsDisabledMode(AnonymizedMetricsMode): + class Config: + title = "anonymized_metrics_disabled" + + def get_collector( + self, + connector: ConnectorBase, + rate_provider: RateOracle, + instance_id: str, + valuation_token: str = "USDT", + ) -> MetricsCollector: + return DummyMetricsCollector() + + +class AnonymizedMetricsEnabledMode(AnonymizedMetricsMode): + anonymized_metrics_interval_min: Decimal = Field( + default=Decimal("15"), + gt=Decimal("0"), + client_data=ClientFieldData( + prompt=lambda cm: "How often do you want to send the anonymized metrics (Enter 5 for 5 minutes)?", + ), + ) + + class Config: + title = "anonymized_metrics_enabled" + + def get_collector( + self, + connector: ConnectorBase, + rate_provider: RateOracle, + instance_id: str, + valuation_token: str = "USDT", + ) -> MetricsCollector: + instance = TradeVolumeMetricCollector( + connector=connector, + activation_interval=self.anonymized_metrics_interval_min, + rate_provider=rate_provider, + instance_id=instance_id, + valuation_token=valuation_token, + ) + return instance + + @validator("anonymized_metrics_interval_min", pre=True) + def validate_decimal(cls, v: str, field: Field): + """Used for client-friendly error output.""" + super().validate_decimal(v, field) + + +METRICS_MODES = { + AnonymizedMetricsDisabledMode.Config.title: AnonymizedMetricsDisabledMode, + AnonymizedMetricsEnabledMode.Config.title: AnonymizedMetricsEnabledMode, +} + + +class CommandShortcutModel(BaseModel): + command: str + help: str + arguments: List[str] + output: List[str] + + +class ClientConfigMap(BaseClientModel): + instance_id: str = Field(default=generate_client_id()) + log_level: str = Field(default="INFO") + debug_console: bool = Field(default=False) + strategy_report_interval: float = Field(default=900) + logger_override_whitelist: List = Field( + default=["hummingbot.strategy.arbitrage", "hummingbot.strategy.cross_exchange_market_making", "conf"] + ) + log_file_path: Path = Field( + default=DEFAULT_LOG_FILE_PATH, + client_data=ClientFieldData( + prompt=lambda cm: f"Where would you like to save your logs? (default '{DEFAULT_LOG_FILE_PATH}')", + ), + ) + kill_switch_mode: Union[tuple(KILL_SWITCH_MODES.values())] = Field( + default=KillSwitchDisabledMode(), + client_data=ClientFieldData( + prompt=lambda cm: f"Select your kill-switch mode ({'/'.join(list(KILL_SWITCH_MODES.keys()))})", + ), + ) + autofill_import: AutofillImportEnum = Field( + default=AutofillImportEnum.disabled, + client_data=ClientFieldData( + prompt=lambda cm: ( + f"What to auto-fill in the prompt after each import command? ({'/'.join(list(AutofillImportEnum))})" + ), + ) + ) + telegram_mode: Union[tuple(TELEGRAM_MODES.values())] = Field( + default=TelegramDisabledMode(), + client_data=ClientFieldData( + prompt=lambda cm: f"Select the desired telegram mode ({'/'.join(list(TELEGRAM_MODES.keys()))})" + ) + ) + send_error_logs: bool = Field( + default=True, + client_data=ClientFieldData( + prompt=lambda cm: "Would you like to send error logs to hummingbot? (Yes/No)", + ), + ) + db_mode: Union[tuple(DB_MODES.values())] = Field( + default=DBSqliteMode(), + client_data=ClientFieldData( + prompt=lambda cm: f"Select the desired db mode ({'/'.join(list(DB_MODES.keys()))})", + ), + ) + pmm_script_mode: Union[tuple(PMM_SCRIPT_MODES.values())] = Field( + default=PMMScriptDisabledMode(), + client_data=ClientFieldData( + prompt=lambda cm: f"Select the desired PMM script mode ({'/'.join(list(PMM_SCRIPT_MODES.keys()))})", + ), + ) + balance_asset_limit: Json = Field( + default=json.dumps({exchange: None for exchange in AllConnectorSettings.get_exchange_names()}), + client_data=ClientFieldData( + prompt=lambda cm: ( + "Use the `balance limit` command e.g. balance limit [EXCHANGE] [ASSET] [AMOUNT]" + ), + ), + ) + manual_gas_price: Decimal = Field( + default=Decimal("50"), + gt=Decimal("0"), + client_data=ClientFieldData( + prompt=lambda cm: "Enter fixed gas price (in Gwei) you want to use for Ethereum transactions", + ), + ) + gateway: GatewayConfigMap = Field(default=GatewayConfigMap()) + anonymized_metrics_mode: Union[tuple(METRICS_MODES.values())] = Field( + default=AnonymizedMetricsDisabledMode(), + client_data=ClientFieldData( + prompt=lambda cm: f"Select the desired metrics mode ({'/'.join(list(METRICS_MODES.keys()))})", + ), + ) + command_shortcuts: List[CommandShortcutModel] = Field( # todo: test it + default=[ + CommandShortcutModel( + command="spreads", + help="Set bid and ask spread", + arguments=["Bid Spread", "Ask Spread"], + output=["config bid_spread $1", "config ask_spread $2"] + ) + ] + ) + rate_oracle_source: str = Field( + default=RateOracleSource.binance.name, + client_data=ClientFieldData( + prompt=lambda cm: ( + f"What source do you want rate oracle to pull data from? ({','.join(r.name for r in RateOracleSource)})" + ), + ), + ) + global_token: GlobalTokenConfigMap = Field(default=GlobalTokenConfigMap()) + rate_limits_share_pct: Decimal = Field( + default=Decimal("100"), + gt=Decimal("0"), + le=Decimal("100"), + client_data=ClientFieldData( + prompt=lambda cm: ( + "What percentage of API rate limits do you want to allocate to this bot instance?" + " (Enter 50 to indicate 50%)" + ), + ), + ) + commands_timeout: CommandsTimeoutConfigMap = Field(default=CommandsTimeoutConfigMap()) + tables_format: ClientConfigEnum( + value="TabulateFormats", # noqa: F821 + names={e: e for e in tabulate_formats}, + type=str, + ) = Field( + default="psql", + client_data=ClientFieldData( + prompt=lambda cm: ( + "What tabulate formatting to apply to the tables?" + " [https://github.com/astanin/python-tabulate#table-format]" + ), + ), + ) + paper_trade: PaperTradeConfigMap = Field(default=PaperTradeConfigMap()) + color: ColorConfigMap = Field(default=ColorConfigMap()) + + @validator("kill_switch_mode", pre=True) + def validate_kill_switch_mode(cls, v: Union[(str,) + tuple(KILL_SWITCH_MODES.values())]): + if isinstance(v, tuple(KILL_SWITCH_MODES.values())): + sub_model = v + elif v not in KILL_SWITCH_MODES: + raise ValueError( + f"Invalid kill switch mode, please choose a value from {list(KILL_SWITCH_MODES.keys())}." + ) + else: + sub_model = KILL_SWITCH_MODES[v].construct() + return sub_model + + @validator("autofill_import", pre=True) + def validate_autofill_import(cls, v: Union[str, AutofillImportEnum]): + if isinstance(v, str) and v not in AutofillImportEnum: + raise ValueError(f"The value must be one of {', '.join(list(AutofillImportEnum))}.") + return v + + @validator("telegram_mode", pre=True) + def validate_telegram_mode(cls, v: Union[(str,) + tuple(TELEGRAM_MODES.values())]): + if isinstance(v, tuple(TELEGRAM_MODES.values())): + sub_model = v + elif v not in TELEGRAM_MODES: + raise ValueError( + f"Invalid telegram mode, please choose a value from {list(TELEGRAM_MODES.keys())}." + ) + else: + sub_model = TELEGRAM_MODES[v].construct() + return sub_model + + @validator("send_error_logs", pre=True) + def validate_bool(cls, v: str): + """Used for client-friendly error output.""" + if isinstance(v, str): + ret = validate_bool(v) + if ret is not None: + raise ValueError(ret) + return v + + @validator("db_mode", pre=True) + def validate_db_mode(cls, v: Union[(str,) + tuple(DB_MODES.values())]): + if isinstance(v, tuple(DB_MODES.values())): + sub_model = v + elif v not in DB_MODES: + raise ValueError( + f"Invalid DB mode, please choose a value from {list(DB_MODES.keys())}." + ) + else: + sub_model = DB_MODES[v].construct() + return sub_model + + @validator("pmm_script_mode", pre=True) + def validate_pmm_script_mode(cls, v: Union[(str,) + tuple(PMM_SCRIPT_MODES.values())]): + if isinstance(v, tuple(PMM_SCRIPT_MODES.values())): + sub_model = v + elif v not in PMM_SCRIPT_MODES: + raise ValueError( + f"Invalid PMM script mode, please choose a value from {list(PMM_SCRIPT_MODES.keys())}." + ) + else: + sub_model = PMM_SCRIPT_MODES[v].construct() + return sub_model + + @validator("anonymized_metrics_mode", pre=True) + def validate_anonymized_metrics_mode(cls, v: Union[(str,) + tuple(METRICS_MODES.values())]): + if isinstance(v, tuple(METRICS_MODES.values())): + sub_model = v + elif v not in METRICS_MODES: + raise ValueError( + f"Invalid metrics mode, please choose a value from {list(METRICS_MODES.keys())}." + ) + else: + sub_model = METRICS_MODES[v].construct() + return sub_model + + @validator("rate_oracle_source", pre=True) + def validate_rate_oracle_source(cls, v: str): + if v not in (r.name for r in RateOracleSource): + raise ValueError( + f"Invalid source, please choose value from {','.join(r.name for r in RateOracleSource)}" + ) + return v + + @validator("tables_format", pre=True) + def validate_tables_format(cls, v: str): + """Used for client-friendly error output.""" + if v not in tabulate_formats: + raise ValueError("Invalid table format.") + return v + + @validator( + "manual_gas_price", + "rate_limits_share_pct", + pre=True, + ) + def validate_decimals(cls, v: str, field: Field): + """Used for client-friendly error output.""" + super().validate_decimal(v, field) + + # === post-validations === + + @root_validator() + def post_validations(cls, values: Dict): + cls.rate_oracle_source_on_validated(values) + return values + + @classmethod + def rate_oracle_source_on_validated(cls, values: Dict): + RateOracle.source = RateOracleSource[values["rate_oracle_source"]] diff --git a/hummingbot/client/config/conf_migration.py b/hummingbot/client/config/conf_migration.py index 11f69c4886..4ae789a39d 100644 --- a/hummingbot/client/config/conf_migration.py +++ b/hummingbot/client/config/conf_migration.py @@ -4,11 +4,25 @@ import shutil from os import DirEntry, scandir from os.path import exists, join -from typing import Dict, List, cast +from typing import Dict, List, Union, cast import yaml from hummingbot import root_path +from hummingbot.client.config.client_config_map import ( + AnonymizedMetricsDisabledMode, + AnonymizedMetricsEnabledMode, + ClientConfigMap, + ColorConfigMap, + DBOtherMode, + DBSqliteMode, + KillSwitchDisabledMode, + KillSwitchEnabledMode, + PMMScriptDisabledMode, + PMMScriptEnabledMode, + TelegramDisabledMode, + TelegramEnabledMode, +) from hummingbot.client.config.config_crypt import BaseSecretsManager, store_password_verification from hummingbot.client.config.config_data_types import BaseConnectorConfigMap from hummingbot.client.config.config_helpers import ClientConfigAdapter @@ -25,21 +39,25 @@ def migrate_configs(secrets_manager: BaseSecretsManager) -> List[str]: logging.getLogger().info("Starting conf migration.") errors = backup_existing_dir() if len(errors) == 0: - migrate_strategy_confs_paths() - errors.extend(migrate_connector_confs(secrets_manager)) - store_password_verification(secrets_manager) - logging.getLogger().info("\nConf migration done.") + errors = migrate_global_config() + if len(errors) == 0: + migrate_strategy_confs_paths() + errors.extend(migrate_connector_confs(secrets_manager)) + store_password_verification(secrets_manager) + logging.getLogger().info("\nConf migration done.") else: logging.getLogger().error("\nConf migration failed.") return errors -def migrate_strategies_only() -> List[str]: +def migrate_non_secure_configs_only() -> List[str]: logging.getLogger().info("Starting strategies conf migration.") errors = backup_existing_dir() if len(errors) == 0: - migrate_strategy_confs_paths() - logging.getLogger().info("\nConf migration done.") + errors = migrate_global_config() + if len(errors) == 0: + migrate_strategy_confs_paths() + logging.getLogger().info("\nConf migration done.") else: logging.getLogger().error("\nConf migration failed.") return errors @@ -64,6 +82,108 @@ def backup_existing_dir() -> List[str]: return errors +def migrate_global_config() -> List[str]: + global_config_path = CONF_DIR_PATH / "conf_global.yml" + errors = [] + if global_config_path.exists(): + with open(str(global_config_path), "r") as f: + data = yaml.safe_load(f) + del data["template_version"] + client_config_map = ClientConfigAdapter(ClientConfigMap()) + migrate_global_config_modes(client_config_map, data) + for key, value in data.items(): + if key not in client_config_map.keys(): + errors.append( + f"Could not match the attribute {key} from the legacy config file to the new config map." + ) + continue + if value is not None: + client_config_map.setattr_no_validation(key, value) + return errors + + +def migrate_global_config_modes(client_config_map: ClientConfigAdapter, data: Dict): + client_config_map: Union[ClientConfigAdapter, ClientConfigMap] = client_config_map # for IDE autocomplete + + kill_switch_enabled = data.pop("kill_switch_enabled") + kill_switch_rate = data.pop("kill_switch_rate") + if kill_switch_enabled: + client_config_map.kill_switch_mode = KillSwitchEnabledMode(kill_switch_rate=kill_switch_rate) + else: + client_config_map.kill_switch_mode = KillSwitchDisabledMode() + + client_config_map.paper_trade.paper_trade_exchanges = data.pop("paper_trade_exchanges") + client_config_map.paper_trade.paper_trade_account_balance = data.pop("paper_trade_account_balance") + + telegram_enabled = data.pop("telegram_enabled") + telegram_token = data.pop("telegram_token") + telegram_chat_id = data.pop("telegram_chat_id") + if telegram_enabled: + client_config_map.telegram_mode = TelegramEnabledMode( + telegram_token=telegram_token, + telegram_chat_id=telegram_chat_id, + ) + else: + client_config_map.telegram_mode = TelegramDisabledMode() + + db_engine = data.pop("db_engine") + db_host = data.pop("db_host") + db_port = data.pop("db_port") + db_username = data.pop("db_username") + db_password = data.pop("db_password") + db_name = data.pop("db_name") + if db_engine == "sqlite": + client_config_map.db_mode = DBSqliteMode() + else: + client_config_map.db_mode = DBOtherMode( + db_engine=db_engine, + db_host=db_host, + db_port=db_port, + db_username=db_username, + db_password=db_password, + db_name=db_name, + ) + + pmm_script_enabled = data.pop("pmm_script_enabled") + pmm_script_file_path = data.pop("pmm_script_file_path") + if pmm_script_enabled: + client_config_map.pmm_script_mode = PMMScriptEnabledMode(pmm_script_file_path=pmm_script_file_path) + else: + client_config_map.pmm_script_mode = PMMScriptDisabledMode() + + client_config_map.gateway.gateway_api_host = data.pop("gateway_api_host") + client_config_map.gateway.gateway_api_port = data.pop("gateway_api_port") + + anonymized_metrics_enabled = data.pop("anonymized_metrics_enabled") + anonymized_metrics_interval_min = data.pop("anonymized_metrics_interval_min") + if anonymized_metrics_enabled: + client_config_map.anonymized_metrics_mode = AnonymizedMetricsEnabledMode( + anonymized_metrics_interval_min=anonymized_metrics_interval_min + ) + else: + client_config_map.anonymized_metrics_mode = AnonymizedMetricsDisabledMode() + + client_config_map.global_token.global_token_name = data.pop("global_token") + client_config_map.global_token.global_token_symbol = data.pop("global_token_symbol") + + client_config_map.commands_timeout.create_command_timeout = data.pop("create_command_timeout") + client_config_map.commands_timeout.other_commands_timeout = data.pop("other_commands_timeout") + + color_map: ColorConfigMap = client_config_map.color + color_map.top_pane = data.pop("top-pane") + color_map.bottom_pane = data.pop("bottom-pane") + color_map.output_pane = data.pop("output-pane") + color_map.input_pane = data.pop("input-pane") + color_map.logs_pane = data.pop("logs-pane") + color_map.terminal_primary = data.pop("terminal-primary") + color_map.primary_label = data.pop("primary-label") + color_map.secondary_label = data.pop("secondary-label") + color_map.success_label = data.pop("success-label") + color_map.warning_label = data.pop("warning-label") + color_map.info_label = data.pop("info-label") + color_map.error_label = data.pop("error-label") + + def migrate_strategy_confs_paths(): logging.getLogger().info("\nMigrating strategies...") for child in conf_dir_path.iterdir(): diff --git a/hummingbot/client/config/config_data_types.py b/hummingbot/client/config/config_data_types.py index 4deea70e6c..84ed9ba455 100644 --- a/hummingbot/client/config/config_data_types.py +++ b/hummingbot/client/config/config_data_types.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from datetime import datetime from enum import Enum -from typing import Any, Callable, Dict, Optional +from typing import Any, Callable, Dict, List, Optional from pydantic import BaseModel, Extra, Field, validator from pydantic.schema import default_ref_template @@ -9,6 +9,7 @@ from hummingbot.client.config.config_methods import strategy_config_schema_encoder from hummingbot.client.config.config_validators import ( validate_connector, + validate_decimal, validate_exchange, validate_market_trading_pair, validate_strategy, @@ -53,6 +54,18 @@ def schema_json( def is_required(self, attr: str) -> bool: return self.__fields__[attr].required + @staticmethod + def validate_decimal(v: str, field: Field): + """Used for client-friendly error output.""" + field_info = field.field_info + inclusive = field_info.ge is not None or field_info.le is not None + min_value = field_info.gt if field_info.gt is not None else field_info.ge + max_value = field_info.lt if field_info.lt is not None else field_info.le + ret = validate_decimal(v, min_value, max_value, inclusive) + if ret is not None: + raise ValueError(ret) + return v + class BaseStrategyConfigMap(BaseClientModel): strategy: str = Field( @@ -105,15 +118,19 @@ def maker_trading_pair_prompt(cls, model_instance: 'BaseTradingStrategyConfigMap @validator("exchange", pre=True) def validate_exchange(cls, v: str): """Used for client-friendly error output.""" - ret = validate_exchange(v) - if ret is not None: - raise ValueError(ret) - cls.__fields__["exchange"].type_ = ClientConfigEnum( # rebuild the exchanges enum - value="Exchanges", # noqa: F821 - names={e: e for e in AllConnectorSettings.get_all_connectors()}, - type=str, - ) - return v + exchanges = v.split(", ") + for e in exchanges: + ret = validate_exchange(e) + if ret is not None: + raise ValueError(ret) + cls.__fields__["exchange"].type_ = List[ + ClientConfigEnum( # rebuild the exchanges enum + value="Exchanges", # noqa: F821 + names={e: e for e in AllConnectorSettings.get_all_connectors()}, + type=str, + ) + ] + return exchanges @validator("market", pre=True) def validate_exchange_trading_pair(cls, v: str, values: Dict): diff --git a/hummingbot/client/config/config_helpers.py b/hummingbot/client/config/config_helpers.py index b7bc88b518..82d0b223fd 100644 --- a/hummingbot/client/config/config_helpers.py +++ b/hummingbot/client/config/config_helpers.py @@ -20,16 +20,16 @@ from yaml import SafeDumper from hummingbot import get_strategy_list, root_path +from hummingbot.client.config.client_config_map import ClientConfigMap from hummingbot.client.config.config_data_types import BaseClientModel, ClientConfigEnum, ClientFieldData from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.fee_overrides_config_map import fee_overrides_config_map, init_fee_overrides_config -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.settings import ( + CLIENT_CONFIG_PATH, CONF_DIR_PATH, CONF_POSTFIX, CONF_PREFIX, CONNECTORS_CONF_DIR_PATH, - GLOBAL_CONFIG_PATH, STRATEGIES_CONF_DIR_PATH, TEMPLATE_PATH, TRADE_FEES_CONFIG_PATH, @@ -187,6 +187,9 @@ def get_client_data(self, attr_name: str) -> Optional[ClientFieldData]: def get_description(self, attr_name: str) -> str: return self._hb_config.__fields__[attr_name].field_info.description + def get_default(self, attr_name: str) -> Any: + return self._hb_config.__fields__[attr_name].field_info.default + def generate_yml_output_str_with_comments(self) -> str: conf_dict = self._dict_in_conf_order() self._encrypt_secrets(conf_dict) @@ -289,6 +292,15 @@ def _add_model_fragments( fragments_with_comments.append(f"\n{original_fragments[i]}") +class ReadOnlyClientConfigAdapter(ClientConfigAdapter): + def __setattr__(self, key, value): + raise AttributeError("Cannot set an attribute on a read-only client adapter.") + + @classmethod + def lock_config(cls, config_map: ClientConfigMap): + return cls(config_map._hb_config) + + def parse_cvar_value(cvar: ConfigVar, value: Any) -> Any: """ Based on the target type specified in `ConfigVar.type_str`, parses a string value into the target type. @@ -522,6 +534,18 @@ def load_connector_config_map_from_file(yml_path: Path) -> ClientConfigAdapter: return config_map +def load_client_config_map_from_file() -> ClientConfigAdapter: + yml_path = CLIENT_CONFIG_PATH + if yml_path.exists(): + config_data = read_yml_file(yml_path) + else: + config_data = {} + client_config = ClientConfigMap() + config_map = ClientConfigAdapter(client_config) + _load_yml_data_into_map(config_data, config_map) + return config_map + + def get_connector_hb_config(connector_name: str) -> BaseClientModel: if connector_name == "celo": hb_config = celo_data_types.KEYS @@ -657,9 +681,6 @@ async def read_system_configs_from_yml(): Read global config and selected strategy yml files and save the values to corresponding config map If a yml file is outdated, it gets reformatted with the new template """ - await load_yml_into_cm_legacy( - GLOBAL_CONFIG_PATH, str(TEMPLATE_PATH / "conf_global_TEMPLATE.yml"), global_config_map - ) await load_yml_into_cm_legacy( str(TRADE_FEES_CONFIG_PATH), str(TEMPLATE_PATH / "conf_fee_overrides_TEMPLATE.yml"), fee_overrides_config_map ) @@ -668,18 +689,15 @@ async def read_system_configs_from_yml(): def save_system_configs_to_yml(): - save_to_yml_legacy(GLOBAL_CONFIG_PATH, global_config_map) save_to_yml_legacy(str(TRADE_FEES_CONFIG_PATH), fee_overrides_config_map) -async def refresh_trade_fees_config(): +async def refresh_trade_fees_config(client_config_map: ClientConfigAdapter): """ Refresh the trade fees config, after new connectors have been added (e.g. gateway connectors). """ init_fee_overrides_config() - await load_yml_into_cm_legacy( - GLOBAL_CONFIG_PATH, str(TEMPLATE_PATH / "conf_global_TEMPLATE.yml"), global_config_map - ) + save_to_yml(CLIENT_CONFIG_PATH, client_config_map) save_to_yml_legacy(str(TRADE_FEES_CONFIG_PATH), fee_overrides_config_map) @@ -712,14 +730,16 @@ def save_to_yml(yml_path: Path, cm: ClientConfigAdapter): def write_config_to_yml( - strategy_config_map: Union[ClientConfigAdapter, Dict], strategy_file_name: str + strategy_config_map: Union[ClientConfigAdapter, Dict], + strategy_file_name: str, + client_config_map: ClientConfigAdapter, ): strategy_file_path = Path(STRATEGIES_CONF_DIR_PATH) / strategy_file_name if isinstance(strategy_config_map, ClientConfigAdapter): save_to_yml(strategy_file_path, strategy_config_map) else: save_to_yml_legacy(strategy_file_path, strategy_config_map) - save_to_yml_legacy(GLOBAL_CONFIG_PATH, global_config_map) + save_to_yml(CLIENT_CONFIG_PATH, client_config_map) async def create_yml_files_legacy(): @@ -776,13 +796,14 @@ def short_strategy_name(strategy: str) -> str: return strategy -def all_configs_complete(strategy_config): +def all_configs_complete(strategy_config: Union[ClientConfigAdapter, Dict], client_config_map: ClientConfigAdapter): strategy_valid = ( config_map_complete_legacy(strategy_config) if isinstance(strategy_config, Dict) else len(strategy_config.validate_model()) == 0 ) - return config_map_complete_legacy(global_config_map) and strategy_valid + client_config_valid = len(client_config_map.validate_model()) == 0 + return client_config_valid and strategy_valid def config_map_complete_legacy(config_map): diff --git a/hummingbot/client/config/global_config_map.py b/hummingbot/client/config/global_config_map.py deleted file mode 100644 index df7bb62c05..0000000000 --- a/hummingbot/client/config/global_config_map.py +++ /dev/null @@ -1,398 +0,0 @@ -import os.path -import random -import re -from decimal import Decimal -from typing import Callable, Optional - -from tabulate import tabulate_formats - -from hummingbot.client.config.config_methods import using_exchange as using_exchange_pointer -from hummingbot.client.config.config_validators import validate_bool, validate_decimal -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.settings import DEFAULT_LOG_FILE_PATH, AllConnectorSettings -from hummingbot.core.rate_oracle.rate_oracle import RateOracle, RateOracleSource - -PMM_SCRIPT_ENABLED_KEY = "pmm_script_enabled" -PMM_SCRIPT_FILE_PATH_KEY = "pmm_script_file_path" - - -def generate_client_id() -> str: - vals = [random.choice(range(0, 256)) for i in range(0, 20)] - return "".join([f"{val:02x}" for val in vals]) - - -def using_exchange(exchange: str) -> Callable: - return using_exchange_pointer(exchange) - - -def validate_pmm_script_file_path(file_path: str) -> Optional[bool]: - import hummingbot.client.settings as settings - path, name = os.path.split(file_path) - if path == "": - file_path = os.path.join(settings.PMM_SCRIPTS_PATH, file_path) - if not os.path.isfile(file_path): - return f"{file_path} file does not exist." - - -def validate_rate_oracle_source(value: str) -> Optional[str]: - if value not in (r.name for r in RateOracleSource): - return f"Invalid source, please choose value from {','.join(r.name for r in RateOracleSource)}" - - -def rate_oracle_source_on_validated(value: str): - RateOracle.source = RateOracleSource[value] - - -def validate_color(value: str) -> Optional[str]: - if not re.search(r'^#(?:[0-9a-fA-F]{2}){3}$', value): - return "Invalid color code" - - -def global_token_on_validated(value: str): - RateOracle.global_token = value.upper() - - -def global_token_symbol_on_validated(value: str): - RateOracle.global_token_symbol = value - - -# Main global config store -main_config_map = { - # The variables below are usually not prompted during setup process - "instance_id": - ConfigVar(key="instance_id", - prompt=None, - required_if=lambda: False, - default=generate_client_id()), - "log_level": - ConfigVar(key="log_level", - prompt=None, - required_if=lambda: False, - default="INFO"), - "debug_console": - ConfigVar(key="debug_console", - prompt=None, - type_str="bool", - required_if=lambda: False, - default=False), - "strategy_report_interval": - ConfigVar(key="strategy_report_interval", - prompt=None, - type_str="float", - required_if=lambda: False, - default=900), - "logger_override_whitelist": - ConfigVar(key="logger_override_whitelist", - prompt=None, - required_if=lambda: False, - default=["hummingbot.strategy", - "conf" - ], - type_str="list"), - "log_file_path": - ConfigVar(key="log_file_path", - prompt=f"Where would you like to save your logs? (default '{DEFAULT_LOG_FILE_PATH}') >>> ", - required_if=lambda: False, - default=str(DEFAULT_LOG_FILE_PATH)), - "kill_switch_enabled": - ConfigVar(key="kill_switch_enabled", - prompt="Would you like to enable the kill switch? (Yes/No) >>> ", - required_if=lambda: False, - type_str="bool", - default=False, - validator=validate_bool), - "kill_switch_rate": - ConfigVar(key="kill_switch_rate", - prompt="At what profit/loss rate would you like the bot to stop? " - "(e.g. -5 equals 5 percent loss) >>> ", - type_str="decimal", - default=-100, - validator=lambda v: validate_decimal(v, Decimal(-100), Decimal(100)), - required_if=lambda: global_config_map["kill_switch_enabled"].value), - "autofill_import": - ConfigVar(key="autofill_import", - prompt="What to auto-fill in the prompt after each import command? (start/config) >>> ", - type_str="str", - default=None, - validator=lambda s: None if s in {"start", - "config"} else "Invalid auto-fill prompt.", - required_if=lambda: False), - "telegram_enabled": - ConfigVar(key="telegram_enabled", - prompt="Would you like to enable telegram? >>> ", - type_str="bool", - default=False, - required_if=lambda: False), - "telegram_token": - ConfigVar(key="telegram_token", - prompt="What is your telegram token? >>> ", - required_if=lambda: False), - "telegram_chat_id": - ConfigVar(key="telegram_chat_id", - prompt="What is your telegram chat id? >>> ", - required_if=lambda: False), - "send_error_logs": - ConfigVar(key="send_error_logs", - prompt="Would you like to send error logs to hummingbot? (Yes/No) >>> ", - type_str="bool", - default=True), - # Database options - "db_engine": - ConfigVar(key="db_engine", - prompt="Please enter database engine you want to use (reference: https://docs.sqlalchemy.org/en/13/dialects/) >>> ", - type_str="str", - required_if=lambda: False, - default="sqlite"), - "db_host": - ConfigVar(key="db_host", - prompt="Please enter your DB host address >>> ", - type_str="str", - required_if=lambda: global_config_map.get("db_engine").value != "sqlite", - default="127.0.0.1"), - "db_port": - ConfigVar(key="db_port", - prompt="Please enter your DB port >>> ", - type_str="str", - required_if=lambda: global_config_map.get("db_engine").value != "sqlite", - default="3306"), - "db_username": - ConfigVar(key="db_username", - prompt="Please enter your DB username >>> ", - type_str="str", - required_if=lambda: global_config_map.get("db_engine").value != "sqlite", - default="username"), - "db_password": - ConfigVar(key="db_password", - prompt="Please enter your DB password >>> ", - type_str="str", - required_if=lambda: global_config_map.get("db_engine").value != "sqlite", - default="password"), - "db_name": - ConfigVar(key="db_name", - prompt="Please enter your the name of your DB >>> ", - type_str="str", - required_if=lambda: global_config_map.get("db_engine").value != "sqlite", - default="dbname"), - PMM_SCRIPT_ENABLED_KEY: - ConfigVar(key=PMM_SCRIPT_ENABLED_KEY, - prompt="Would you like to enable PMM script feature? (Yes/No) >>> ", - type_str="bool", - default=False, - validator=validate_bool), - PMM_SCRIPT_FILE_PATH_KEY: - ConfigVar(key=PMM_SCRIPT_FILE_PATH_KEY, - prompt='Enter path to your PMM script file >>> ', - type_str="str", - required_if=lambda: global_config_map[PMM_SCRIPT_ENABLED_KEY].value, - validator=validate_pmm_script_file_path), - "balance_asset_limit": - ConfigVar(key="balance_asset_limit", - prompt="Use the `balance limit` command" - "e.g. balance limit [EXCHANGE] [ASSET] [AMOUNT]", - required_if=lambda: False, - type_str="json", - default={exchange: None for exchange in AllConnectorSettings.get_exchange_names()}), - "manual_gas_price": - ConfigVar(key="manual_gas_price", - prompt="Enter fixed gas price (in Gwei) you want to use for Ethereum transactions >>> ", - required_if=lambda: False, - type_str="decimal", - validator=lambda v: validate_decimal(v, Decimal(0), inclusive=False), - default=50), - "gateway_api_host": - ConfigVar(key="gateway_api_host", - prompt=None, - required_if=lambda: False, - default='localhost'), - "gateway_api_port": - ConfigVar(key="gateway_api_port", - prompt="Please enter your Gateway API port >>> ", - type_str="str", - required_if=lambda: False, - default="5000"), - "anonymized_metrics_enabled": - ConfigVar(key="anonymized_metrics_enabled", - prompt="Do you want to report aggregated, anonymized trade volume by exchange to " - "Hummingbot Foundation? >>> ", - required_if=lambda: False, - type_str="bool", - validator=validate_bool, - default=True), - "anonymized_metrics_interval_min": - ConfigVar(key="anonymized_metrics_interval_min", - prompt="How often do you want to send the anonymized metrics (Enter 5 for 5 minutes)? >>> ", - required_if=lambda: False, - type_str="decimal", - validator=lambda v: validate_decimal(v, Decimal(0), inclusive=False), - default=Decimal("15")), - "command_shortcuts": - ConfigVar(key="command_shortcuts", - prompt=None, - required_if=lambda: False, - type_str="list"), - "rate_oracle_source": - ConfigVar(key="rate_oracle_source", - prompt=f"What source do you want rate oracle to pull data from? " - f"({','.join(r.name for r in RateOracleSource)}) >>> ", - type_str="str", - required_if=lambda: False, - validator=validate_rate_oracle_source, - on_validated=rate_oracle_source_on_validated, - default=RateOracleSource.binance.name), - "global_token": - ConfigVar(key="global_token", - prompt="What is your default display token? (e.g. USD,EUR,BTC) >>> ", - type_str="str", - required_if=lambda: False, - on_validated=global_token_on_validated, - default="USD"), - "global_token_symbol": - ConfigVar(key="global_token_symbol", - prompt="What is your default display token symbol? (e.g. $,€) >>> ", - type_str="str", - required_if=lambda: False, - on_validated=global_token_symbol_on_validated, - default="$"), - "rate_limits_share_pct": - ConfigVar(key="rate_limits_share_pct", - prompt="What percentage of API rate limits do you want to allocate to this bot instance? " - "(Enter 50 to indicate 50%) >>> ", - type_str="decimal", - validator=lambda v: validate_decimal(v, 1, 100, inclusive=True), - required_if=lambda: False, - default=Decimal("100")), - "create_command_timeout": - ConfigVar(key="create_command_timeout", - prompt="Network timeout when fetching the minimum order amount" - " in the create command (in seconds) >>> ", - type_str="decimal", - validator=lambda v: validate_decimal(v, min_value=Decimal("0"), inclusive=False), - required_if=lambda: False, - default=Decimal("10")), - "other_commands_timeout": - ConfigVar(key="other_commands_timeout", - prompt="Network timeout to apply to the other commands' API calls" - " (i.e. import, connect, balance, history; in seconds) >>> ", - type_str="decimal", - validator=lambda v: validate_decimal(v, min_value=Decimal("0"), inclusive=False), - required_if=lambda: False, - default=Decimal("30")), - "tables_format": - ConfigVar(key="tables_format", - prompt="What tabulate formatting to apply to the tables?" - " [https://github.com/astanin/python-tabulate#table-format] >>> ", - type_str="str", - required_if=lambda: False, - validator=lambda value: "Invalid format" if value not in tabulate_formats else None, - default="psql"), -} - -color_config_map = { - # The variables below are usually not prompted during setup process - "top-pane": - ConfigVar(key="top-pane", - prompt="What is the background color of the top pane? ", - type_str="str", - required_if=lambda: False, - validator=validate_color, - default="#000000"), - "bottom-pane": - ConfigVar(key="bottom-pane", - prompt="What is the background color of the bottom pane? ", - type_str="str", - required_if=lambda: False, - validator=validate_color, - default="#000000"), - "output-pane": - ConfigVar(key="output-pane", - prompt="What is the background color of the output pane? ", - type_str="str", - required_if=lambda: False, - validator=validate_color, - default="#282C2F"), - "input-pane": - ConfigVar(key="input-pane", - prompt="What is the background color of the input pane? ", - type_str="str", - required_if=lambda: False, - validator=validate_color, - default="#151819"), - "logs-pane": - ConfigVar(key="logs-pane", - prompt="What is the background color of the logs pane? ", - type_str="str", - required_if=lambda: False, - validator=validate_color, - default="#151819"), - "terminal-primary": - ConfigVar(key="terminal-primary", - prompt="What is the terminal primary color? ", - type_str="str", - required_if=lambda: False, - validator=validate_color, - default="#00FFE5"), - "primary-label": - ConfigVar(key="primary-label", - prompt="What is the background color for primary label? ", - type_str="str", - required_if=lambda: False, - validator=validate_color, - default="#5FFFD7"), - "secondary-label": - ConfigVar(key="secondary-label", - prompt="What is the background color for secondary label? ", - type_str="str", - required_if=lambda: False, - validator=validate_color, - default="#FFFFFF"), - "success-label": - ConfigVar(key="success-label", - prompt="What is the background color for success label? ", - type_str="str", - required_if=lambda: False, - validator=validate_color, - default="#5FFFD7"), - "warning-label": - ConfigVar(key="warning-label", - prompt="What is the background color for warning label? ", - type_str="str", - required_if=lambda: False, - validator=validate_color, - default="#FFFF00"), - "info-label": - ConfigVar(key="info-label", - prompt="What is the background color for info label? ", - type_str="str", - required_if=lambda: False, - validator=validate_color, - default="#5FD7FF"), - "error-label": - ConfigVar(key="error-label", - prompt="What is the background color for error label? ", - type_str="str", - required_if=lambda: False, - validator=validate_color, - default="#FF0000"), -} - -paper_trade_config_map = { - "paper_trade_exchanges": - ConfigVar(key="paper_trade_exchanges", - prompt=None, - required_if=lambda: False, - default=["binance", - "kucoin", - "ascend_ex", - "gate_io", - ], - type_str="list"), - "paper_trade_account_balance": - ConfigVar(key="paper_trade_account_balance", - prompt="Enter paper trade balance settings (Input must be valid json: " - "e.g. {\"ETH\": 10, \"USDC\": 50000}) >>> ", - required_if=lambda: False, - type_str="json", - ), -} - -global_config_map = {**main_config_map, **color_config_map, **paper_trade_config_map} diff --git a/hummingbot/client/hummingbot_application.py b/hummingbot/client/hummingbot_application.py index 014857c08d..99e68165ab 100644 --- a/hummingbot/client/hummingbot_application.py +++ b/hummingbot/client/hummingbot_application.py @@ -4,14 +4,21 @@ import logging import time from collections import deque -from typing import Deque, Dict, List, Optional, Tuple +from typing import Deque, Dict, List, Optional, Tuple, Union from hummingbot.client.command import __all__ as commands +from hummingbot.client.config.client_config_map import ClientConfigMap from hummingbot.client.config.config_data_types import BaseStrategyConfigMap -from hummingbot.client.config.config_helpers import get_connector_class, get_strategy_config_map -from hummingbot.client.config.global_config_map import global_config_map +from hummingbot.client.config.config_helpers import ( + ClientConfigAdapter, + ReadOnlyClientConfigAdapter, + get_connector_class, + get_strategy_config_map, + load_client_config_map_from_file, + save_to_yml, +) from hummingbot.client.config.security import Security -from hummingbot.client.settings import AllConnectorSettings, ConnectorType +from hummingbot.client.settings import CLIENT_CONFIG_PATH, AllConnectorSettings, ConnectorType from hummingbot.client.tab import __all__ as tab_classes from hummingbot.client.tab.data_types import CommandTab from hummingbot.client.ui.completer import load_completer @@ -31,7 +38,6 @@ from hummingbot.logger.application_warning import ApplicationWarning from hummingbot.model.sql_connection_manager import SQLConnectionManager from hummingbot.notifier.notifier_base import NotifierBase -from hummingbot.notifier.telegram_notifier import TelegramNotifier from hummingbot.strategy.cross_exchange_market_making import CrossExchangeMarketPair from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple from hummingbot.strategy.strategy_base import StrategyBase @@ -54,12 +60,16 @@ def logger(cls) -> HummingbotLogger: return s_logger @classmethod - def main_application(cls) -> "HummingbotApplication": + def main_application(cls, client_config_map: Optional[ClientConfigAdapter] = None) -> "HummingbotApplication": if cls._main_app is None: - cls._main_app = HummingbotApplication() + cls._main_app = HummingbotApplication(client_config_map) return cls._main_app - def __init__(self): + def __init__(self, client_config_map: Optional[ClientConfigAdapter] = None): + self.client_config_map: Union[ClientConfigMap, ClientConfigAdapter] = ( # type-hint enables IDE auto-complete + client_config_map or load_client_config_map_from_file() + ) + # This is to start fetching trading pairs for auto-complete TradingPairFetcher.get_instance() self.ev_loop: asyncio.AbstractEventLoop = asyncio.get_event_loop() @@ -99,6 +109,7 @@ def __init__(self): command_tabs = self.init_command_tabs() self.parser: ThrowingArgumentParser = load_parser(self, command_tabs) self.app = HummingbotCLI( + self.client_config_map, input_handler=self._handle_command, bindings=load_key_bindings(self), completer=load_completer(self), @@ -120,7 +131,9 @@ def strategy_file_name(self, value: Optional[str]): self._strategy_file_name = value if value is not None: db_name = value.split(".")[0] - self.trade_fill_db = SQLConnectionManager.get_trade_fills_instance(db_name) + self.trade_fill_db = SQLConnectionManager.get_trade_fills_instance( + self.client_config_map, db_name + ) else: self.trade_fill_db = None @@ -172,7 +185,7 @@ def _handle_command(self, raw_command: str): self.help(raw_command) return - shortcuts = global_config_map.get("command_shortcuts").value + shortcuts = self.client_config_map.command_shortcuts shortcut = None # see if we match against shortcut command if shortcuts is not None: @@ -269,7 +282,7 @@ def _initialize_markets(self, market_names: List[Tuple[str, List[str]]]): if connector_name.endswith("paper_trade") and conn_setting.type == ConnectorType.Exchange: connector = create_paper_trade_market(conn_setting.parent_name, trading_pairs) - paper_trade_account_balance = global_config_map.get("paper_trade_account_balance").value + paper_trade_account_balance = self.client_config_map.paper_trade.paper_trade_account_balance if paper_trade_account_balance is not None: for asset, balance in paper_trade_account_balance.items(): connector.set_balance(asset, balance) @@ -278,7 +291,8 @@ def _initialize_markets(self, market_names: List[Tuple[str, List[str]]]): init_params = conn_setting.conn_init_parameters(keys) init_params.update(trading_pairs=trading_pairs, trading_required=self._trading_required) connector_class = get_connector_class(connector_name) - connector = connector_class(**init_params) + read_only_config = ReadOnlyClientConfigAdapter.lock_config(self.client_config_map) + connector = connector_class(read_only_config, **init_params) self.markets[connector_name] = connector self.markets_recorder = MarketsRecorder( @@ -290,16 +304,12 @@ def _initialize_markets(self, market_names: List[Tuple[str, List[str]]]): self.markets_recorder.start() def _initialize_notifiers(self): - if global_config_map.get("telegram_enabled").value: - # TODO: refactor to use single instance - if not any([isinstance(n, TelegramNotifier) for n in self.notifiers]): - self.notifiers.append( - TelegramNotifier( - token=global_config_map["telegram_token"].value, - chat_id=global_config_map["telegram_chat_id"].value, - hb=self, - ) - ) + self.notifiers.extend( + [ + notifier for notifier in self.client_config_map.telegram_mode.get_notifiers() + if notifier not in self.notifiers + ] + ) for notifier in self.notifiers: notifier.start() @@ -313,3 +323,6 @@ def init_command_tabs(self) -> Dict[str, CommandTab]: name = tab_class.get_command_name() command_tabs[name] = CommandTab(name, None, None, None, tab_class) return command_tabs + + def save_client_config(self): + save_to_yml(CLIENT_CONFIG_PATH, self.client_config_map) diff --git a/hummingbot/client/settings.py b/hummingbot/client/settings.py index 68173d124a..ce5e183ab0 100644 --- a/hummingbot/client/settings.py +++ b/hummingbot/client/settings.py @@ -28,7 +28,7 @@ KEYFILE_PREFIX = "key_file_" KEYFILE_POSTFIX = ".yml" ENCYPTED_CONF_POSTFIX = ".json" -GLOBAL_CONFIG_PATH = str(root_path() / "conf" / "conf_global.yml") +CLIENT_CONFIG_PATH = root_path() / "conf" / "conf_client.yml" TRADE_FEES_CONFIG_PATH = root_path() / "conf" / "conf_fee_overrides.yml" DEFAULT_LOG_FILE_PATH = root_path() / "logs" DEFAULT_ETHEREUM_RPC_URL = "https://mainnet.coinalpha.com/hummingbot-test-node" diff --git a/hummingbot/client/ui/__init__.py b/hummingbot/client/ui/__init__.py index 86f8096f73..3f21ea7f9e 100644 --- a/hummingbot/client/ui/__init__.py +++ b/hummingbot/client/ui/__init__.py @@ -7,9 +7,10 @@ from prompt_toolkit.styles import Style from hummingbot import root_path -from hummingbot.client.config.conf_migration import migrate_configs, migrate_strategies_only +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.conf_migration import migrate_configs, migrate_non_secure_configs_only from hummingbot.client.config.config_crypt import BaseSecretsManager, store_password_verification -from hummingbot.client.config.global_config_map import color_config_map +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.client.config.security import Security from hummingbot.client.settings import CONF_DIR_PATH @@ -19,10 +20,12 @@ with open(realpath(join(dirname(__file__), '../../VERSION'))) as version_file: version = version_file.read().strip() +client_config_map = ClientConfigAdapter(ClientConfigMap()) +terminal_primary = client_config_map.color.terminal_primary dialog_style = Style.from_dict({ 'dialog': 'bg:#171E2B', 'dialog frame.label': 'bg:#ffffff #000000', - 'dialog.body': 'bg:#000000 ' + color_config_map["terminal-primary"].default, + 'dialog.body': 'bg:#000000 ' + terminal_primary, 'dialog shadow': 'bg:#171E2B', 'button': 'bg:#000000', 'text-area': 'bg:#000000 #ffffff', @@ -60,7 +63,7 @@ def login_prompt(secrets_manager_cls: Type[BaseSecretsManager]) -> Optional[Base else: secrets_manager = secrets_manager_cls(password) store_password_verification(secrets_manager) - migrate_strategies_only_prompt() + migrate_non_secure_only_prompt() else: password = input_dialog( title="Welcome back to Hummingbot", @@ -133,30 +136,29 @@ def migrate_configs_prompt(secrets_manager_cls: Type[BaseSecretsManager]) -> Bas return secrets_manager -def migrate_strategies_only_prompt(): +def migrate_non_secure_only_prompt(): message_dialog( - title='Strategies Configs Migration', + title='Configs Migration', text=""" - STRATEGIES CONFIGS MIGRATION: + CONFIGS MIGRATION: We have recently refactored the way hummingbot handles configurations. - We will now attempt to migrate any legacy strategy config files - to the new format. + We will now attempt to migrate any legacy config files to the new format. """, style=dialog_style).run() - errors = migrate_strategies_only() + errors = migrate_non_secure_configs_only() if len(errors) != 0: _migration_errors_dialog(errors) else: message_dialog( - title='Strategies Configs Migration Success', + title='Configs Migration Success', text=""" - STRATEGIES CONFIGS MIGRATION SUCCESS: + CONFIGS MIGRATION SUCCESS: The migration process was completed successfully. diff --git a/hummingbot/client/ui/custom_widgets.py b/hummingbot/client/ui/custom_widgets.py index 47a65aa7ea..ae60a9e334 100644 --- a/hummingbot/client/ui/custom_widgets.py +++ b/hummingbot/client/ui/custom_widgets.py @@ -9,22 +9,18 @@ from prompt_toolkit.buffer import Buffer from prompt_toolkit.completion import DynamicCompleter from prompt_toolkit.document import Document -from prompt_toolkit.filters import (Condition, has_focus, is_done, is_true, - to_filter) +from prompt_toolkit.filters import Condition, has_focus, is_done, is_true, to_filter from prompt_toolkit.formatted_text.base import StyleAndTextTuples from prompt_toolkit.layout.containers import Window, WindowAlign from prompt_toolkit.layout.controls import BufferControl from prompt_toolkit.layout.margins import NumberedMargin, ScrollbarMargin -from prompt_toolkit.layout.processors import (AppendAutoSuggestion, - BeforeInput, - ConditionalProcessor, - PasswordProcessor) +from prompt_toolkit.layout.processors import AppendAutoSuggestion, BeforeInput, ConditionalProcessor, PasswordProcessor from prompt_toolkit.lexers import DynamicLexer from prompt_toolkit.lexers.base import Lexer from prompt_toolkit.widgets.toolbars import SearchToolbar +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.client.ui.style import load_style, text_ui_style -from hummingbot.client.config.global_config_map import color_config_map class CustomBuffer(Buffer): @@ -43,13 +39,15 @@ class FormattedTextLexer(Lexer): PROMPT_TEXT = ">>> " - def __init__(self) -> None: + def __init__(self, client_config_map: ClientConfigAdapter) -> None: super().__init__() - self.html_tag_css_style_map: Dict[str, str] = {style: css for style, css in load_style().style_rules} + self.html_tag_css_style_map: Dict[str, str] = { + style: css for style, css in load_style(client_config_map).style_rules + } self.html_tag_css_style_map.update({ - style: config.value - for style, config in color_config_map.items() - if style not in self.html_tag_css_style_map.keys() + ti.attr: ti.value + for ti in client_config_map.color.traverse() + if ti.attr not in self.html_tag_css_style_map }) # Maps specific text to its corresponding UI styles diff --git a/hummingbot/client/ui/hummingbot_cli.py b/hummingbot/client/ui/hummingbot_cli.py index c91d3ab35a..9fb1e4bf5b 100644 --- a/hummingbot/client/ui/hummingbot_cli.py +++ b/hummingbot/client/ui/hummingbot_cli.py @@ -2,7 +2,7 @@ import logging import threading from contextlib import ExitStack -from typing import Any, Callable, Dict, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Callable, Dict, Optional, Union from prompt_toolkit.application import Application from prompt_toolkit.clipboard.pyperclip import PyperclipClipboard @@ -12,7 +12,8 @@ from prompt_toolkit.layout.processors import BeforeInput, PasswordProcessor from hummingbot import init_logging -from hummingbot.client.config.global_config_map import global_config_map +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.client.tab.data_types import CommandTab from hummingbot.client.ui.interface_utils import start_process_monitor, start_timer, start_trade_monitor from hummingbot.client.ui.layout import ( @@ -50,15 +51,17 @@ def _handle_exception_patch(self, loop, context): class HummingbotCLI(PubSub): def __init__(self, + client_config_map: ClientConfigAdapter, input_handler: Callable, bindings: KeyBindings, completer: Completer, command_tabs: Dict[str, CommandTab]): super().__init__() + self.client_config_map: Union[ClientConfigAdapter, ClientConfigMap] = client_config_map self.command_tabs = command_tabs self.search_field = create_search_field() self.input_field = create_input_field(completer=completer) - self.output_field = create_output_field() + self.output_field = create_output_field(client_config_map) self.log_field = create_log_field(self.search_field) self.right_pane_toggle = create_log_toggle(self.toggle_right_pane) self.live_field = create_live_field() @@ -95,17 +98,23 @@ def __init__(self, loop.create_task(start_process_monitor(self.process_usage)) loop.create_task(start_trade_monitor(self.trade_monitor)) - def did_start_ui(self): + def did_start_ui(self, hummingbot: "HummingbotApplication"): self._stdout_redirect_context.enter_context(patch_stdout(log_field=self.log_field)) - log_level = global_config_map.get("log_level").value - init_logging("hummingbot_logs.yml", override_log_level=log_level) + log_level = hummingbot.client_config_map.log_level + init_logging("hummingbot_logs.yml", self.client_config_map, override_log_level=log_level) self.trigger_event(HummingbotUIEvent.Start, self) async def run(self): - self.app = Application(layout=self.layout, full_screen=True, key_bindings=self.bindings, style=load_style(), - mouse_support=True, clipboard=PyperclipClipboard()) + self.app = Application( + layout=self.layout, + full_screen=True, + key_bindings=self.bindings, + style=load_style(self.client_config_map), + mouse_support=True, + clipboard=PyperclipClipboard(), + ) await self.app.run_async(pre_run=self.did_start_ui) self._stdout_redirect_context.close() diff --git a/hummingbot/client/ui/interface_utils.py b/hummingbot/client/ui/interface_utils.py index 177550382a..955f16dfa1 100644 --- a/hummingbot/client/ui/interface_utils.py +++ b/hummingbot/client/ui/interface_utils.py @@ -7,7 +7,7 @@ import psutil import tabulate -from hummingbot.client.config.global_config_map import global_config_map +from hummingbot.client.config.config_data_types import ClientConfigEnum from hummingbot.client.performance import PerformanceMetrics from hummingbot.model.trade_fill import TradeFill @@ -93,7 +93,7 @@ async def start_trade_monitor(trade_monitor): def format_df_for_printout( - df: pd.DataFrame, max_col_width: Optional[int] = None, index: bool = False, table_format: Optional[str] = None + df: pd.DataFrame, table_format: ClientConfigEnum, max_col_width: Optional[int] = None, index: bool = False ) -> str: if max_col_width is not None: # in anticipation of the next release of tabulate which will include maxcolwidth max_col_width = max(max_col_width, 4) @@ -103,7 +103,6 @@ def format_df_for_printout( ) ) df.columns = [c if len(c) < max_col_width else f"{c[:max_col_width - 3]}..." for c in df.columns] - table_format = table_format or global_config_map.get("tables_format").value original_preserve_whitespace = tabulate.PRESERVE_WHITESPACE tabulate.PRESERVE_WHITESPACE = True diff --git a/hummingbot/client/ui/keybindings.py b/hummingbot/client/ui/keybindings.py index 83769a014d..2c515e6ccd 100644 --- a/hummingbot/client/ui/keybindings.py +++ b/hummingbot/client/ui/keybindings.py @@ -1,24 +1,13 @@ #!/usr/bin/env python from prompt_toolkit.application.current import get_app -from prompt_toolkit.filters import ( - is_searching, - to_filter, -) +from prompt_toolkit.filters import is_searching, to_filter from prompt_toolkit.key_binding import KeyBindings -from prompt_toolkit.search import ( - start_search, - stop_search, - do_incremental_search, - SearchDirection, -) - -from hummingbot.client.ui.scroll_handlers import ( - scroll_down, - scroll_up, -) -from hummingbot.core.utils.async_utils import safe_ensure_future +from prompt_toolkit.search import SearchDirection, do_incremental_search, start_search, stop_search + +from hummingbot.client.ui.scroll_handlers import scroll_down, scroll_up from hummingbot.client.ui.style import reset_style +from hummingbot.core.utils.async_utils import safe_ensure_future def load_key_bindings(hb) -> KeyBindings: @@ -100,7 +89,7 @@ def stop_live_update(event): @bindings.add("c-r") def do_reset_style(event): - hb.app.app.style = reset_style() + hb.app.app.style = reset_style(hb.client_config_map) @bindings.add("c-t") def toggle_logs(event): diff --git a/hummingbot/client/ui/layout.py b/hummingbot/client/ui/layout.py index 23e1d105e2..001d92a1bd 100644 --- a/hummingbot/client/ui/layout.py +++ b/hummingbot/client/ui/layout.py @@ -1,4 +1,4 @@ -from os.path import join, realpath, dirname +from os.path import dirname, join, realpath from typing import Dict from prompt_toolkit.auto_suggest import AutoSuggestFromHistory @@ -18,12 +18,12 @@ from prompt_toolkit.layout.menus import CompletionsMenu from prompt_toolkit.widgets import Box, Button, SearchToolbar +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.client.settings import MAXIMUM_LOG_PANE_LINE_COUNT, MAXIMUM_OUTPUT_PANE_LINE_COUNT from hummingbot.client.tab.data_types import CommandTab from hummingbot.client.ui.custom_widgets import CustomTextArea as TextArea, FormattedTextLexer from hummingbot.core.gateway.status_monitor import Status as GatewayStatus - HEADER = """ *,. *,,,* @@ -96,7 +96,7 @@ def create_input_field(lexer=None, completer: Completer = None): ) -def create_output_field(): +def create_output_field(client_config_map: ClientConfigAdapter): return TextArea( style='class:output-field', focus_on_click=False, @@ -104,7 +104,7 @@ def create_output_field(): scrollbar=True, max_line_count=MAXIMUM_OUTPUT_PANE_LINE_COUNT, initial_text=HEADER, - lexer=FormattedTextLexer() + lexer=FormattedTextLexer(client_config_map) ) diff --git a/hummingbot/client/ui/parser.py b/hummingbot/client/ui/parser.py index bbaf04d76c..4168a28b3c 100644 --- a/hummingbot/client/ui/parser.py +++ b/hummingbot/client/ui/parser.py @@ -1,10 +1,12 @@ import argparse -from typing import Any, List +from typing import TYPE_CHECKING, Any, List from hummingbot.client.command.connect_command import OPTIONS as CONNECT_OPTIONS -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.exceptions import ArgumentParserError +if TYPE_CHECKING: + from hummingbot.client.hummingbot_application import HummingbotApplication + class ThrowingArgumentParser(argparse.ArgumentParser): def error(self, message): @@ -35,7 +37,7 @@ def subcommands_from(self, top_level_command: str) -> List[str]: return filtered -def load_parser(hummingbot, command_tabs) -> [ThrowingArgumentParser, Any]: +def load_parser(hummingbot: "HummingbotApplication", command_tabs) -> [ThrowingArgumentParser, Any]: parser = ThrowingArgumentParser(prog="", add_help=False) subparsers = parser.add_subparsers() @@ -139,15 +141,14 @@ def load_parser(hummingbot, command_tabs) -> [ThrowingArgumentParser, Any]: pmm_script_parser.set_defaults(func=hummingbot.pmm_script_command) # add shortcuts so they appear in command help - shortcuts = global_config_map.get("command_shortcuts").value - if shortcuts is not None: - for shortcut in shortcuts: - help_str = shortcut['help'] - command = shortcut['command'] - shortcut_parser = subparsers.add_parser(command, help=help_str) - args = shortcut['arguments'] - for i in range(len(args)): - shortcut_parser.add_argument(f'${i+1}', help=args[i]) + shortcuts = hummingbot.client_config_map.command_shortcuts + for shortcut in shortcuts: + help_str = shortcut.help + command = shortcut.command + shortcut_parser = subparsers.add_parser(command, help=help_str) + args = shortcut.arguments + for i in range(len(args)): + shortcut_parser.add_argument(f'${i+1}', help=args[i]) rate_parser = subparsers.add_parser('rate', help="Show rate of a given trading pair") rate_parser.add_argument("-p", "--pair", default=None, diff --git a/hummingbot/client/ui/style.py b/hummingbot/client/ui/style.py index da06ad5409..7af252e96e 100644 --- a/hummingbot/client/ui/style.py +++ b/hummingbot/client/ui/style.py @@ -1,33 +1,32 @@ +from typing import Union + from prompt_toolkit.styles import Style from prompt_toolkit.utils import is_windows -from hummingbot.client.config.config_helpers import save_to_yml_legacy -from hummingbot.client.config.global_config_map import global_config_map -from hummingbot.client.settings import GLOBAL_CONFIG_PATH +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter, save_to_yml +from hummingbot.client.settings import CLIENT_CONFIG_PATH -def load_style(config_map=global_config_map): +def load_style(config_map: ClientConfigAdapter): """ Return a dict mapping {ui_style_name -> style_dict}. """ - + config_map: Union[ClientConfigAdapter, ClientConfigMap] = config_map # to enable IDE auto-complete # Load config - color_top_pane = config_map.get("top-pane").value - color_bottom_pane = config_map.get("bottom-pane").value - color_output_pane = config_map.get("output-pane").value - color_input_pane = config_map.get("input-pane").value - color_logs_pane = config_map.get("logs-pane").value - color_terminal_primary = config_map.get("terminal-primary").value - - color_primary_label = config_map.get("primary-label").value - color_secondary_label = config_map.get("secondary-label").value - color_success_label = config_map.get("success-label").value - color_warning_label = config_map.get("warning-label").value - color_info_label = config_map.get("info-label").value - color_error_label = config_map.get("error-label").value - - # Load default style - style = default_ui_style + color_top_pane = config_map.color.top_pane + color_bottom_pane = config_map.color.bottom_pane + color_output_pane = config_map.color.output_pane + color_input_pane = config_map.color.input_pane + color_logs_pane = config_map.color.logs_pane + color_terminal_primary = config_map.color.terminal_primary + + color_primary_label = config_map.color.primary_label + color_secondary_label = config_map.color.secondary_label + color_success_label = config_map.color.success_label + color_warning_label = config_map.color.warning_label + color_info_label = config_map.color.info_label + color_error_label = config_map.color.error_label if is_windows(): # Load default style for Windows @@ -93,26 +92,26 @@ def load_style(config_map=global_config_map): return Style.from_dict(style) -def reset_style(config_map=global_config_map, save=True): +def reset_style(config_map: ClientConfigAdapter, save=True): # Reset config - config_map.get("top-pane").value = config_map.get("top-pane").default - config_map.get("bottom-pane").value = config_map.get("bottom-pane").default - config_map.get("output-pane").value = config_map.get("output-pane").default - config_map.get("input-pane").value = config_map.get("input-pane").default - config_map.get("logs-pane").value = config_map.get("logs-pane").default - config_map.get("terminal-primary").value = config_map.get("terminal-primary").default - - config_map.get("primary-label").value = config_map.get("primary-label").default - config_map.get("secondary-label").value = config_map.get("secondary-label").default - config_map.get("success-label").value = config_map.get("success-label").default - config_map.get("warning-label").value = config_map.get("warning-label").default - config_map.get("info-label").value = config_map.get("info-label").default - config_map.get("error-label").value = config_map.get("error-label").default + + config_map.color.top_pane = config_map.color.get_default("top_pane") + config_map.color.bottom_pane = config_map.color.get_default("bottom_pane") + config_map.color.output_pane = config_map.color.get_default("output_pane") + config_map.color.input_pane = config_map.color.get_default("input_pane") + config_map.color.logs_pane = config_map.color.get_default("logs_pane") + config_map.color.terminal_primary = config_map.color.get_default("terminal_primary") + + config_map.color.primary_label = config_map.color.get_default("primary_label") + config_map.color.secondary_label = config_map.color.get_default("secondary_label") + config_map.color.success_label = config_map.color.get_default("success_label") + config_map.color.warning_label = config_map.color.get_default("warning_label") + config_map.color.info_label = config_map.color.get_default("info_label") + config_map.color.error_label = config_map.color.get_default("error_label") # Save configuration if save: - file_path = GLOBAL_CONFIG_PATH - save_to_yml_legacy(file_path, config_map) + save_to_yml(CLIENT_CONFIG_PATH, config_map) # Apply & return style return load_style(config_map) diff --git a/hummingbot/connector/connector_base.pxd b/hummingbot/connector/connector_base.pxd index a382ec26f9..4ac7497590 100644 --- a/hummingbot/connector/connector_base.pxd +++ b/hummingbot/connector/connector_base.pxd @@ -16,6 +16,7 @@ cdef class ConnectorBase(NetworkIterator): public dict _exchange_order_ids public object _trade_fee_schema public object _trade_volume_metric_collector + public object _client_config cdef str c_buy(self, str trading_pair, object amount, object order_type=*, object price=*, dict kwargs=*) cdef str c_sell(self, str trading_pair, object amount, object order_type=*, object price=*, dict kwargs=*) diff --git a/hummingbot/connector/connector_base.pyx b/hummingbot/connector/connector_base.pyx index b75d9be912..7e51f72ab0 100644 --- a/hummingbot/connector/connector_base.pyx +++ b/hummingbot/connector/connector_base.pyx @@ -1,10 +1,8 @@ import time from decimal import Decimal -from typing import Dict, List, Set, Tuple +from typing import Dict, List, Set, Tuple, TYPE_CHECKING -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.trade_fee_schema_loader import TradeFeeSchemaLoader -from hummingbot.connector.connector_metrics_collector import TradeVolumeMetricCollector from hummingbot.connector.in_flight_order_base import InFlightOrderBase from hummingbot.connector.utils import split_hb_trading_pair, TradeFillOrderDetails from hummingbot.core.clock cimport Clock @@ -16,6 +14,10 @@ from hummingbot.core.network_iterator import NetworkIterator from hummingbot.core.rate_oracle.rate_oracle import RateOracle from hummingbot.core.utils.estimate_fee import estimate_fee +if TYPE_CHECKING: + from hummingbot.client.config.client_config_map import ClientConfigMap + from hummingbot.client.config.config_helpers import ClientConfigAdapter + NaN = float("nan") s_decimal_NaN = Decimal("nan") s_decimal_0 = Decimal(0) @@ -42,7 +44,7 @@ cdef class ConnectorBase(NetworkIterator): MarketEvent.RangePositionInitiated, ] - def __init__(self): + def __init__(self, client_config_map: "ClientConfigAdapter"): super().__init__() self._event_reporter = EventReporter(event_source=self.display_name) @@ -63,9 +65,12 @@ cdef class ConnectorBase(NetworkIterator): self._current_trade_fills = set() self._exchange_order_ids = dict() self._trade_fee_schema = None - self._trade_volume_metric_collector = TradeVolumeMetricCollector.from_configuration( + self._trade_volume_metric_collector = client_config_map.anonymized_metrics_mode.get_collector( connector=self, - rate_provider=RateOracle.get_instance()) + rate_provider=RateOracle.get_instance(), + instance_id=client_config_map.instance_id, + ) + self._client_config: Union[ClientConfigAdapter, ClientConfigMap] = client_config_map # for IDE autocomplete @property def real_time_balance_update(self) -> bool: @@ -161,10 +166,7 @@ cdef class ConnectorBase(NetworkIterator): """ Retrieves the Balance Limits for the specified market. """ - all_ex_limit = global_config_map["balance_asset_limit"].value - if all_ex_limit is None: - return {} - exchange_limits = all_ex_limit.get(market, {}) + exchange_limits = self._client_config.balance_asset_limit.get(market, {}) return exchange_limits if exchange_limits is not None else {} @property diff --git a/hummingbot/connector/connector_metrics_collector.py b/hummingbot/connector/connector_metrics_collector.py index 6bca2981c6..4305f8d140 100644 --- a/hummingbot/connector/connector_metrics_collector.py +++ b/hummingbot/connector/connector_metrics_collector.py @@ -5,9 +5,8 @@ from abc import ABC, abstractmethod from decimal import Decimal from os.path import dirname, join, realpath -from typing import List, Tuple, TYPE_CHECKING +from typing import TYPE_CHECKING, List, Tuple -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.connector.utils import combine_to_hb_trading_pair, split_hb_trading_pair from hummingbot.core.event.event_forwarder import EventForwarder from hummingbot.core.event.events import MarketEvent, OrderFilledEvent @@ -26,7 +25,6 @@ class MetricsCollector(ABC): DEFAULT_METRICS_SERVER_URL = "https://api.coinalpha.com/reporting-proxy-v2" - DEFAULT_ACTIVATION_INTERVAL_MINUTES = 15 @abstractmethod def start(self): @@ -64,19 +62,17 @@ class TradeVolumeMetricCollector(MetricsCollector): def __init__(self, connector: 'ConnectorBase', - activation_interval: float, - metrics_dispatcher: LogServerClient, + activation_interval: Decimal, rate_provider: RateOracle, instance_id: str, - client_version: str, valuation_token: str = "USDT"): super().__init__() self._connector = connector self._activation_interval = activation_interval - self._dispatcher = metrics_dispatcher + self._dispatcher = LogServerClient(log_server_url=self.DEFAULT_METRICS_SERVER_URL) self._rate_provider = rate_provider self._instance_id = instance_id - self._client_version = client_version + self._client_version = CLIENT_VERSION self._valuation_token = valuation_token self._last_process_tick_timestamp = 0 self._last_executed_collection_process = None @@ -94,42 +90,6 @@ def logger(cls) -> HummingbotLogger: cls._logger = logging.getLogger(__name__) return cls._logger - @classmethod - def from_configuration(cls, connector: 'ConnectorBase', rate_provider: RateOracle, valuation_token: str = "USDT"): - instance = DummyMetricsCollector() - - anonymized_metrics_enabled = global_config_map.get("anonymized_metrics_enabled") - if anonymized_metrics_enabled is not None and anonymized_metrics_enabled.value: - dispatcher_url = global_config_map.get("log_server_url") - if dispatcher_url is None: - dispatcher_url = cls.DEFAULT_METRICS_SERVER_URL - else: - dispatcher_url = dispatcher_url.value - dispatcher = LogServerClient(log_server_url=dispatcher_url) - - activation_interval = global_config_map.get("anonymized_metrics_interval_min") - if activation_interval is None: - activation_interval = cls.DEFAULT_ACTIVATION_INTERVAL_MINUTES * 60 - else: - activation_interval = float(activation_interval.value) * 60 - - instance_id = global_config_map.get("instance_id") - if instance_id is None: - instance_id = "" - else: - instance_id = instance_id.value - - instance = cls( - connector=connector, - activation_interval=activation_interval, - metrics_dispatcher=dispatcher, - rate_provider=rate_provider, - instance_id=instance_id, - client_version=CLIENT_VERSION, - valuation_token=valuation_token) - - return instance - def start(self): self._dispatcher.start() for event_pair in self._event_pairs: diff --git a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py index 70bb23f752..d6cc5bed05 100644 --- a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py +++ b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py @@ -12,14 +12,15 @@ import hummingbot.connector.derivative.binance_perpetual.constants as CONSTANTS from hummingbot.connector.client_order_tracker import ClientOrderTracker from hummingbot.connector.derivative.binance_perpetual.binance_perpetual_api_order_book_data_source import ( - BinancePerpetualAPIOrderBookDataSource + BinancePerpetualAPIOrderBookDataSource, ) from hummingbot.connector.derivative.binance_perpetual.binance_perpetual_auth import BinancePerpetualAuth from hummingbot.connector.derivative.binance_perpetual.binance_perpetual_order_book_tracker import ( - BinancePerpetualOrderBookTracker + BinancePerpetualOrderBookTracker, +) +from hummingbot.connector.derivative.binance_perpetual.binance_perpetual_user_stream_data_source import ( + BinancePerpetualUserStreamDataSource, ) -from hummingbot.connector.derivative.binance_perpetual.binance_perpetual_user_stream_data_source import \ - BinancePerpetualUserStreamDataSource from hummingbot.connector.derivative.perpetual_budget_checker import PerpetualBudgetChecker from hummingbot.connector.derivative.position import Position from hummingbot.connector.exchange_base import ExchangeBase, s_decimal_NaN @@ -83,7 +84,7 @@ def __init__( time_provider=self._binance_time_synchronizer) self._trading_pairs = trading_pairs self._trading_required = trading_required - self._throttler = AsyncThrottler(CONSTANTS.RATE_LIMITS) + self._throttler = AsyncThrottler(CONSTANTS.RATE_LIMITS, self._client_config.rate_limits_share_pct) self._domain = domain self._api_factory = web_utils.build_api_factory( throttler=self._throttler, diff --git a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_web_utils.py b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_web_utils.py index df69acb674..8effad002f 100644 --- a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_web_utils.py +++ b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_web_utils.py @@ -5,7 +5,7 @@ from hummingbot.connector.utils import TimeSynchronizerRESTPreProcessor from hummingbot.core.api_throttler.async_throttler import AsyncThrottler from hummingbot.core.web_assistant.auth import AuthBase -from hummingbot.core.web_assistant.connections.data_types import RESTRequest, RESTMethod +from hummingbot.core.web_assistant.connections.data_types import RESTMethod, RESTRequest from hummingbot.core.web_assistant.rest_pre_processors import RESTPreProcessorBase from hummingbot.core.web_assistant.web_assistants_factory import WebAssistantsFactory diff --git a/hummingbot/connector/derivative_base.py b/hummingbot/connector/derivative_base.py index 90579eefa5..e08fafb3f1 100644 --- a/hummingbot/connector/derivative_base.py +++ b/hummingbot/connector/derivative_base.py @@ -1,8 +1,12 @@ from decimal import Decimal +from typing import TYPE_CHECKING from hummingbot.connector.exchange_base import ExchangeBase from hummingbot.core.data_type.common import PositionMode +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + NaN = float("nan") s_decimal_NaN = Decimal("nan") s_decimal_0 = Decimal(0) @@ -13,8 +17,8 @@ class DerivativeBase(ExchangeBase): DerivativeBase provide extra funtionality in addition to the ExchangeBase for derivative exchanges """ - def __init__(self): - super().__init__() + def __init__(self, client_config_map: "ClientConfigAdapter"): + super().__init__(client_config_map) self._funding_info = {} self._account_positions = {} self._position_mode = None diff --git a/hummingbot/connector/exchange/altmarkets/altmarkets_exchange.py b/hummingbot/connector/exchange/altmarkets/altmarkets_exchange.py index 0980eca2bd..ea92ccf19c 100644 --- a/hummingbot/connector/exchange/altmarkets/altmarkets_exchange.py +++ b/hummingbot/connector/exchange/altmarkets/altmarkets_exchange.py @@ -4,13 +4,7 @@ import time import traceback from decimal import Decimal -from typing import ( - Dict, - List, - Optional, - Any, - AsyncIterable, -) +from typing import TYPE_CHECKING, Any, AsyncIterable, Dict, List, Optional import aiohttp from async_timeout import timeout @@ -51,6 +45,9 @@ from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + ctce_logger = None s_decimal_NaN = Decimal("nan") s_decimal_0 = Decimal(0) @@ -73,6 +70,7 @@ def logger(cls) -> HummingbotLogger: return ctce_logger def __init__(self, + client_config_map: "ClientConfigAdapter", altmarkets_api_key: str, altmarkets_secret_key: str, trading_pairs: Optional[List[str]] = None, @@ -84,10 +82,10 @@ def __init__(self, :param trading_pairs: The market trading pairs which to track order book data. :param trading_required: Whether actual trading is needed. """ - super().__init__() + super().__init__(client_config_map) self._trading_required = trading_required self._trading_pairs = trading_pairs - self._throttler = AsyncThrottler(Constants.RATE_LIMITS) + self._throttler = AsyncThrottler(Constants.RATE_LIMITS, self._client_config.rate_limits_share_pct) self._altmarkets_auth = AltmarketsAuth(altmarkets_api_key, altmarkets_secret_key) self._order_book_tracker = AltmarketsOrderBookTracker(throttler=self._throttler, trading_pairs=trading_pairs) diff --git a/hummingbot/connector/exchange/beaxy/beaxy_exchange.pyx b/hummingbot/connector/exchange/beaxy/beaxy_exchange.pyx index a8fe9ddfab..e6798d305e 100644 --- a/hummingbot/connector/exchange/beaxy/beaxy_exchange.pyx +++ b/hummingbot/connector/exchange/beaxy/beaxy_exchange.pyx @@ -4,7 +4,7 @@ import logging from async_timeout import timeout from datetime import datetime, timedelta from decimal import Decimal -from typing import Any, AsyncIterable, Dict, List, Optional +from typing import Any, AsyncIterable, Dict, List, Optional, TYPE_CHECKING import aiohttp from aiohttp.client_exceptions import ContentTypeError @@ -43,6 +43,9 @@ from hummingbot.core.utils.estimate_fee import estimate_fee from hummingbot.core.utils.tracking_nonce import get_tracking_nonce from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + s_logger = None s_decimal_0 = Decimal('0.0') s_decimal_NaN = Decimal('NaN') @@ -83,13 +86,14 @@ cdef class BeaxyExchange(ExchangeBase): def __init__( self, + client_config_map: "ClientConfigAdapter", beaxy_api_key: str, beaxy_secret_key: str, poll_interval: float = 5.0, # interval which the class periodically pulls status from the rest API trading_pairs: Optional[List[str]] = None, trading_required: bool = True ): - super().__init__() + super().__init__(client_config_map) self._trading_required = trading_required self._beaxy_auth = BeaxyAuth(beaxy_api_key, beaxy_secret_key) self._order_book_tracker = BeaxyOrderBookTracker(trading_pairs=trading_pairs) diff --git a/hummingbot/connector/exchange/binance/binance_exchange.py b/hummingbot/connector/exchange/binance/binance_exchange.py index e37b170703..03246b1343 100755 --- a/hummingbot/connector/exchange/binance/binance_exchange.py +++ b/hummingbot/connector/exchange/binance/binance_exchange.py @@ -2,7 +2,7 @@ import logging import time from decimal import Decimal -from typing import Any, AsyncIterable, Dict, List, Optional +from typing import TYPE_CHECKING, Any, AsyncIterable, Dict, List, Optional from async_timeout import timeout @@ -20,7 +20,7 @@ from hummingbot.core.api_throttler.async_throttler import AsyncThrottler from hummingbot.core.data_type.cancellation_result import CancellationResult from hummingbot.core.data_type.common import OrderType, TradeType -from hummingbot.core.data_type.in_flight_order import InFlightOrder, OrderUpdate, OrderState, TradeUpdate +from hummingbot.core.data_type.in_flight_order import InFlightOrder, OrderState, OrderUpdate, TradeUpdate from hummingbot.core.data_type.limit_order import LimitOrder from hummingbot.core.data_type.order_book import OrderBook from hummingbot.core.data_type.order_book_tracker import OrderBookTracker @@ -33,6 +33,9 @@ from hummingbot.core.web_assistant.rest_assistant import RESTAssistant from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + s_logger = None s_decimal_0 = Decimal(0) s_decimal_NaN = Decimal("nan") @@ -44,6 +47,7 @@ class BinanceExchange(ExchangeBase): LONG_POLL_INTERVAL = 120.0 def __init__(self, + client_config_map: "ClientConfigAdapter", binance_api_key: str, binance_api_secret: str, trading_pairs: Optional[List[str]] = None, @@ -52,7 +56,7 @@ def __init__(self, ): self._domain = domain self._binance_time_synchronizer = TimeSynchronizer() - super().__init__() + super().__init__(client_config_map) self._trading_required = trading_required self._auth = BinanceAuth( api_key=binance_api_key, diff --git a/hummingbot/connector/exchange/bitfinex/bitfinex_exchange.pyx b/hummingbot/connector/exchange/bitfinex/bitfinex_exchange.pyx index e481be2cfd..9acd4409dc 100644 --- a/hummingbot/connector/exchange/bitfinex/bitfinex_exchange.pyx +++ b/hummingbot/connector/exchange/bitfinex/bitfinex_exchange.pyx @@ -5,7 +5,7 @@ import logging import time import uuid from decimal import Decimal -from typing import Any, AsyncIterable, Dict, List, Optional +from typing import Any, AsyncIterable, Dict, List, Optional, TYPE_CHECKING import aiohttp from libc.stdint cimport int64_t @@ -53,6 +53,9 @@ from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather from hummingbot.core.utils.estimate_fee import estimate_fee from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + s_logger = None s_decimal_0 = Decimal(0) s_decimal_nan = Decimal("nan") @@ -104,13 +107,14 @@ cdef class BitfinexExchange(ExchangeBase): return s_logger def __init__(self, + client_config_map: "ClientConfigAdapter", bitfinex_api_key: str, bitfinex_secret_key: str, # interval which the class periodically pulls status from the rest API poll_interval: float = 5.0, trading_pairs: Optional[List[str]] = None, trading_required: bool = True): - super().__init__() + super().__init__(client_config_map) self._ev_loop = asyncio.get_event_loop() self._poll_notifier = asyncio.Event() diff --git a/hummingbot/connector/exchange/bitmart/bitmart_exchange.py b/hummingbot/connector/exchange/bitmart/bitmart_exchange.py index 77c9ebeaef..ad1334e7d5 100644 --- a/hummingbot/connector/exchange/bitmart/bitmart_exchange.py +++ b/hummingbot/connector/exchange/bitmart/bitmart_exchange.py @@ -4,7 +4,7 @@ import logging import math from decimal import Decimal -from typing import Any, AsyncIterable, Dict, List, Optional +from typing import TYPE_CHECKING, Any, AsyncIterable, Dict, List, Optional from hummingbot.connector.exchange.bitmart import bitmart_constants as CONSTANTS, bitmart_utils from hummingbot.connector.exchange.bitmart.bitmart_auth import BitmartAuth @@ -37,6 +37,9 @@ from hummingbot.core.web_assistant.rest_assistant import RESTAssistant from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + ctce_logger = None s_decimal_NaN = Decimal("nan") s_decimal_0 = Decimal(0) @@ -60,6 +63,7 @@ def logger(cls) -> HummingbotLogger: return ctce_logger def __init__(self, + client_config_map: "ClientConfigAdapter", bitmart_api_key: str, bitmart_secret_key: str, bitmart_memo: str, @@ -72,7 +76,7 @@ def __init__(self, :param trading_pairs: The market trading pairs which to track order book data. :param trading_required: Whether actual trading is needed. """ - super().__init__() + super().__init__(client_config_map) self._api_factory = bitmart_utils.build_api_factory() self._rest_assistant = None self._trading_required = trading_required diff --git a/hummingbot/connector/exchange/bittrex/bittrex_exchange.pyx b/hummingbot/connector/exchange/bittrex/bittrex_exchange.pyx index 7e446b0ee1..8ae8b8b6c7 100644 --- a/hummingbot/connector/exchange/bittrex/bittrex_exchange.pyx +++ b/hummingbot/connector/exchange/bittrex/bittrex_exchange.pyx @@ -1,7 +1,7 @@ import asyncio import logging from decimal import Decimal -from typing import Any, AsyncIterable, Dict, List, Optional +from typing import Any, AsyncIterable, Dict, List, Optional, TYPE_CHECKING import aiohttp from async_timeout import timeout @@ -36,6 +36,9 @@ from hummingbot.core.utils.estimate_fee import estimate_fee from hummingbot.core.utils.tracking_nonce import get_tracking_nonce from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + bm_logger = None s_decimal_0 = Decimal(0) s_decimal_NaN = Decimal("NaN") @@ -79,12 +82,13 @@ cdef class BittrexExchange(ExchangeBase): return bm_logger def __init__(self, + client_config_map: "ClientConfigAdapter", bittrex_api_key: str, bittrex_secret_key: str, poll_interval: float = 5.0, trading_pairs: Optional[List[str]] = None, trading_required: bool = True): - super().__init__() + super().__init__(client_config_map) self._account_available_balances = {} self._account_balances = {} self._account_id = "" diff --git a/hummingbot/connector/exchange/blocktane/blocktane_exchange.pyx b/hummingbot/connector/exchange/blocktane/blocktane_exchange.pyx index c8da1c564a..38f2b64201 100644 --- a/hummingbot/connector/exchange/blocktane/blocktane_exchange.pyx +++ b/hummingbot/connector/exchange/blocktane/blocktane_exchange.pyx @@ -11,7 +11,7 @@ from typing import ( Dict, List, Optional, - Tuple, + Tuple, TYPE_CHECKING, ) import aiohttp @@ -53,6 +53,9 @@ from hummingbot.core.utils.estimate_fee import build_trade_fee from hummingbot.core.utils.tracking_nonce import get_tracking_nonce from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + bm_logger = None s_decimal_0 = Decimal(0) @@ -104,12 +107,13 @@ cdef class BlocktaneExchange(ExchangeBase): return bm_logger def __init__(self, + client_config_map: "ClientConfigAdapter", blocktane_api_key: str, blocktane_api_secret: str, poll_interval: float = 5.0, trading_pairs: Optional[List[str]] = None, trading_required: bool = True): - super().__init__() + super().__init__(client_config_map) self._real_time_balance_update = True self._account_id = "" self._account_available_balances = {} diff --git a/hummingbot/connector/exchange/coinbase_pro/coinbase_pro_exchange.pyx b/hummingbot/connector/exchange/coinbase_pro/coinbase_pro_exchange.pyx index 691ad60ace..f803cbd3a3 100755 --- a/hummingbot/connector/exchange/coinbase_pro/coinbase_pro_exchange.pyx +++ b/hummingbot/connector/exchange/coinbase_pro/coinbase_pro_exchange.pyx @@ -2,7 +2,7 @@ import asyncio import copy import logging from decimal import Decimal -from typing import Any, AsyncIterable, Dict, List, Optional +from typing import Any, AsyncIterable, Dict, List, Optional, TYPE_CHECKING from async_timeout import timeout from libc.stdint cimport int64_t @@ -46,6 +46,9 @@ from hummingbot.core.web_assistant.connections.data_types import RESTMethod from hummingbot.core.web_assistant.rest_assistant import RESTAssistant from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + s_logger = None s_decimal_0 = Decimal("0.0") s_decimal_nan = Decimal("nan") @@ -87,13 +90,14 @@ cdef class CoinbaseProExchange(ExchangeBase): return s_logger def __init__(self, + client_config_map: "ClientConfigAdapter", coinbase_pro_api_key: str, coinbase_pro_secret_key: str, coinbase_pro_passphrase: str, poll_interval: float = 5.0, # interval which the class periodically pulls status from the rest API trading_pairs: Optional[List[str]] = None, trading_required: bool = True): - super().__init__() + super().__init__(client_config_map) self._trading_required = trading_required auth = CoinbaseProAuth(coinbase_pro_api_key, coinbase_pro_secret_key, coinbase_pro_passphrase) self._web_assistants_factory = build_coinbase_pro_web_assistant_factory(auth) diff --git a/hummingbot/connector/exchange/coinflex/coinflex_exchange.py b/hummingbot/connector/exchange/coinflex/coinflex_exchange.py index 738ff0c60f..cc347bff4b 100755 --- a/hummingbot/connector/exchange/coinflex/coinflex_exchange.py +++ b/hummingbot/connector/exchange/coinflex/coinflex_exchange.py @@ -2,35 +2,24 @@ import logging import time from decimal import Decimal -from typing import ( - Any, - AsyncIterable, - Dict, - List, - Optional, -) +from typing import TYPE_CHECKING, Any, AsyncIterable, Dict, List, Optional from async_timeout import timeout import hummingbot.connector.exchange.coinflex.coinflex_constants as CONSTANTS import hummingbot.connector.exchange.coinflex.coinflex_web_utils as web_utils from hummingbot.connector.client_order_tracker import ClientOrderTracker -from hummingbot.connector.exchange_base import ExchangeBase from hummingbot.connector.exchange.coinflex import coinflex_utils from hummingbot.connector.exchange.coinflex.coinflex_api_order_book_data_source import CoinflexAPIOrderBookDataSource from hummingbot.connector.exchange.coinflex.coinflex_auth import CoinflexAuth from hummingbot.connector.exchange.coinflex.coinflex_order_book_tracker import CoinflexOrderBookTracker from hummingbot.connector.exchange.coinflex.coinflex_user_stream_tracker import CoinflexUserStreamTracker +from hummingbot.connector.exchange_base import ExchangeBase from hummingbot.connector.trading_rule import TradingRule from hummingbot.core.api_throttler.async_throttler import AsyncThrottler from hummingbot.core.data_type.cancellation_result import CancellationResult from hummingbot.core.data_type.common import OrderType, TradeType -from hummingbot.core.data_type.in_flight_order import ( - InFlightOrder, - OrderState, - OrderUpdate, - TradeUpdate, -) +from hummingbot.core.data_type.in_flight_order import InFlightOrder, OrderState, OrderUpdate, TradeUpdate from hummingbot.core.data_type.limit_order import LimitOrder from hummingbot.core.data_type.order_book import OrderBook from hummingbot.core.data_type.trade_fee import DeductedFromReturnsTradeFee, TokenAmount, TradeFeeBase @@ -39,6 +28,9 @@ from hummingbot.core.web_assistant.connections.data_types import RESTMethod from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + s_logger = None s_decimal_0 = Decimal(0) s_decimal_NaN = Decimal("nan") @@ -52,6 +44,7 @@ class CoinflexExchange(ExchangeBase): MAX_ORDER_UPDATE_RETRIEVAL_RETRIES_WITH_FAILURES = 3 def __init__(self, + client_config_map: "ClientConfigAdapter", coinflex_api_key: str, coinflex_api_secret: str, trading_pairs: Optional[List[str]] = None, @@ -59,7 +52,7 @@ def __init__(self, domain: str = CONSTANTS.DEFAULT_DOMAIN ): self._domain = domain - super().__init__() + super().__init__(client_config_map) self._trading_required = trading_required self._auth = CoinflexAuth( api_key=coinflex_api_key, diff --git a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py index ff6b90d15d..e18b90049e 100644 --- a/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py +++ b/hummingbot/connector/exchange/coinzoom/coinzoom_exchange.py @@ -4,13 +4,7 @@ import math import time from decimal import Decimal -from typing import ( - Any, - AsyncIterable, - Dict, - List, - Optional, -) +from typing import TYPE_CHECKING, Any, AsyncIterable, Dict, List, Optional import aiohttp from async_timeout import timeout @@ -51,6 +45,9 @@ from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + ctce_logger = None s_decimal_NaN = Decimal("nan") @@ -71,6 +68,7 @@ def logger(cls) -> HummingbotLogger: return ctce_logger def __init__(self, + client_config_map: "ClientConfigAdapter", coinzoom_api_key: str, coinzoom_secret_key: str, coinzoom_username: str, @@ -84,7 +82,7 @@ def __init__(self, :param trading_pairs: The market trading pairs which to track order book data. :param trading_required: Whether actual trading is needed. """ - super().__init__() + super().__init__(client_config_map) self._trading_required = trading_required self._trading_pairs = trading_pairs self._throttler = AsyncThrottler(Constants.RATE_LIMITS) diff --git a/hummingbot/connector/exchange/crypto_com/crypto_com_exchange.py b/hummingbot/connector/exchange/crypto_com/crypto_com_exchange.py index aa2a1d46af..60de0f0491 100644 --- a/hummingbot/connector/exchange/crypto_com/crypto_com_exchange.py +++ b/hummingbot/connector/exchange/crypto_com/crypto_com_exchange.py @@ -4,7 +4,7 @@ import math import time from decimal import Decimal -from typing import Any, AsyncIterable, Dict, List, Optional +from typing import TYPE_CHECKING, Any, AsyncIterable, Dict, List, Optional import aiohttp @@ -18,7 +18,7 @@ from hummingbot.core.api_throttler.async_throttler import AsyncThrottler from hummingbot.core.clock import Clock from hummingbot.core.data_type.cancellation_result import CancellationResult -from hummingbot.core.data_type.common import OpenOrder +from hummingbot.core.data_type.common import OpenOrder, OrderType, TradeType from hummingbot.core.data_type.limit_order import LimitOrder from hummingbot.core.data_type.order_book import OrderBook from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee, TokenAmount @@ -32,11 +32,13 @@ SellOrderCompletedEvent, SellOrderCreatedEvent, ) -from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.network_iterator import NetworkStatus from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + ctce_logger = None s_decimal_NaN = Decimal("nan") @@ -59,6 +61,7 @@ def logger(cls) -> HummingbotLogger: return ctce_logger def __init__(self, + client_config_map: "ClientConfigAdapter", crypto_com_api_key: str, crypto_com_secret_key: str, trading_pairs: Optional[List[str]] = None, @@ -70,7 +73,7 @@ def __init__(self, :param trading_pairs: The market trading pairs which to track order book data. :param trading_required: Whether actual trading is needed. """ - super().__init__() + super().__init__(client_config_map) self._trading_required = trading_required self._trading_pairs = trading_pairs self._crypto_com_auth = CryptoComAuth(crypto_com_api_key, crypto_com_secret_key) diff --git a/hummingbot/connector/exchange/digifinex/digifinex_exchange.py b/hummingbot/connector/exchange/digifinex/digifinex_exchange.py index f44aa34182..4b925b978a 100644 --- a/hummingbot/connector/exchange/digifinex/digifinex_exchange.py +++ b/hummingbot/connector/exchange/digifinex/digifinex_exchange.py @@ -3,7 +3,7 @@ import math import time from decimal import Decimal -from typing import Any, AsyncIterable, Dict, List, Optional +from typing import TYPE_CHECKING, Any, AsyncIterable, Dict, List, Optional from hummingbot.connector.exchange.digifinex import digifinex_utils from hummingbot.connector.exchange.digifinex.digifinex_global import DigifinexGlobal @@ -14,7 +14,7 @@ from hummingbot.connector.trading_rule import TradingRule from hummingbot.core.clock import Clock from hummingbot.core.data_type.cancellation_result import CancellationResult -from hummingbot.core.data_type.common import OpenOrder +from hummingbot.core.data_type.common import OpenOrder, OrderType, TradeType from hummingbot.core.data_type.limit_order import LimitOrder from hummingbot.core.data_type.order_book import OrderBook from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee @@ -28,12 +28,14 @@ SellOrderCompletedEvent, SellOrderCreatedEvent, ) -from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.network_iterator import NetworkStatus from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather from hummingbot.core.utils.estimate_fee import estimate_fee from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + ctce_logger = None s_decimal_NaN = Decimal("nan") @@ -56,6 +58,7 @@ def logger(cls) -> HummingbotLogger: return ctce_logger def __init__(self, + client_config_map: "ClientConfigAdapter", digifinex_api_key: str, digifinex_secret_key: str, trading_pairs: Optional[List[str]] = None, @@ -67,7 +70,7 @@ def __init__(self, :param trading_pairs: The market trading pairs which to track order book data. :param trading_required: Whether actual trading is needed. """ - super().__init__() + super().__init__(client_config_map) self._trading_required = trading_required self._trading_pairs = trading_pairs self._global = DigifinexGlobal(digifinex_api_key, digifinex_secret_key) diff --git a/hummingbot/connector/exchange/ftx/ftx_exchange.pyx b/hummingbot/connector/exchange/ftx/ftx_exchange.pyx index 9e182e5cb2..6af55ff66e 100644 --- a/hummingbot/connector/exchange/ftx/ftx_exchange.pyx +++ b/hummingbot/connector/exchange/ftx/ftx_exchange.pyx @@ -8,7 +8,7 @@ from typing import ( AsyncIterable, Dict, List, - Optional, + Optional, TYPE_CHECKING, ) import aiohttp @@ -50,6 +50,9 @@ from hummingbot.core.utils.estimate_fee import estimate_fee from hummingbot.core.utils.tracking_nonce import get_tracking_nonce from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + bm_logger = None s_decimal_0 = Decimal(0) UNRECOGNIZED_ORDER_DEBOUCE = 20 # seconds @@ -92,13 +95,14 @@ cdef class FtxExchange(ExchangeBase): return bm_logger def __init__(self, + client_config_map: "ClientConfigAdapter", ftx_secret_key: str, ftx_api_key: str, ftx_subaccount_name: str = None, poll_interval: float = 5.0, trading_pairs: Optional[List[str]] = None, trading_required: bool = True): - super().__init__() + super().__init__(client_config_map) self._real_time_balance_update = False self._account_available_balances = {} self._account_balances = {} diff --git a/hummingbot/connector/exchange/gate_io/gate_io_exchange.py b/hummingbot/connector/exchange/gate_io/gate_io_exchange.py index a4bb03b128..03b492906d 100644 --- a/hummingbot/connector/exchange/gate_io/gate_io_exchange.py +++ b/hummingbot/connector/exchange/gate_io/gate_io_exchange.py @@ -4,7 +4,7 @@ import math import time from decimal import Decimal -from typing import Any, AsyncIterable, Dict, List, Optional +from typing import TYPE_CHECKING, Any, AsyncIterable, Dict, List, Optional from async_timeout import timeout @@ -47,6 +47,9 @@ from hummingbot.core.web_assistant.rest_assistant import RESTAssistant from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + ctce_logger = None s_decimal_NaN = Decimal("nan") @@ -67,6 +70,7 @@ def logger(cls) -> HummingbotLogger: return ctce_logger def __init__(self, + client_config_map: "ClientConfigAdapter", gate_io_api_key: str, gate_io_secret_key: str, trading_pairs: Optional[List[str]] = None, @@ -78,7 +82,7 @@ def __init__(self, :param trading_pairs: The market trading pairs which to track order book data. :param trading_required: Whether actual trading is needed. """ - super().__init__() + super().__init__(client_config_map) self._trading_required = trading_required self._trading_pairs = trading_pairs self._gate_io_auth = GateIoAuth(gate_io_api_key, gate_io_secret_key) diff --git a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py index 6ef917ce72..3798f97e69 100644 --- a/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py +++ b/hummingbot/connector/exchange/hitbtc/hitbtc_exchange.py @@ -3,13 +3,7 @@ import math import time from decimal import Decimal -from typing import ( - Any, - AsyncIterable, - Dict, - List, - Optional, -) +from typing import TYPE_CHECKING, Any, AsyncIterable, Dict, List, Optional import aiohttp from async_timeout import timeout @@ -21,9 +15,9 @@ from hummingbot.connector.exchange.hitbtc.hitbtc_order_book_tracker import HitbtcOrderBookTracker from hummingbot.connector.exchange.hitbtc.hitbtc_user_stream_tracker import HitbtcUserStreamTracker from hummingbot.connector.exchange.hitbtc.hitbtc_utils import ( + HitbtcAPIError, aiohttp_response_with_errors, get_new_client_order_id, - HitbtcAPIError, retry_sleep_time, str_date_to_ts, translate_asset, @@ -32,8 +26,7 @@ from hummingbot.connector.trading_rule import TradingRule from hummingbot.core.clock import Clock from hummingbot.core.data_type.cancellation_result import CancellationResult -from hummingbot.core.data_type.common import OpenOrder -from hummingbot.core.data_type.common import OrderType, TradeType +from hummingbot.core.data_type.common import OpenOrder, OrderType, TradeType from hummingbot.core.data_type.limit_order import LimitOrder from hummingbot.core.data_type.order_book import OrderBook from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee, TokenAmount @@ -51,6 +44,9 @@ from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + ctce_logger = None s_decimal_NaN = Decimal("nan") @@ -71,6 +67,7 @@ def logger(cls) -> HummingbotLogger: return ctce_logger def __init__(self, + client_config_map: "ClientConfigAdapter", hitbtc_api_key: str, hitbtc_secret_key: str, trading_pairs: Optional[List[str]] = None, @@ -82,7 +79,7 @@ def __init__(self, :param trading_pairs: The market trading pairs which to track order book data. :param trading_required: Whether actual trading is needed. """ - super().__init__() + super().__init__(client_config_map) self._trading_required = trading_required self._trading_pairs = trading_pairs self._hitbtc_auth = HitbtcAuth(hitbtc_api_key, hitbtc_secret_key) diff --git a/hummingbot/connector/exchange/huobi/huobi_exchange.pyx b/hummingbot/connector/exchange/huobi/huobi_exchange.pyx index 995db10919..273b8ae421 100644 --- a/hummingbot/connector/exchange/huobi/huobi_exchange.pyx +++ b/hummingbot/connector/exchange/huobi/huobi_exchange.pyx @@ -7,7 +7,7 @@ from typing import ( AsyncIterable, Dict, List, - Optional + Optional, TYPE_CHECKING ) import ujson @@ -55,6 +55,9 @@ from hummingbot.core.web_assistant.connections.data_types import RESTMethod, RES from hummingbot.core.web_assistant.rest_assistant import RESTAssistant from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + hm_logger = None s_decimal_0 = Decimal(0) s_decimal_NaN = Decimal("NaN") @@ -102,12 +105,13 @@ cdef class HuobiExchange(ExchangeBase): return hm_logger def __init__(self, + client_config_map: "ClientConfigAdapter", huobi_api_key: str, huobi_secret_key: str, trading_pairs: Optional[List[str]] = None, trading_required: bool = True): - super().__init__() + super().__init__(client_config_map) self._account_id = "" self._async_scheduler = AsyncCallScheduler(call_interval=0.5) self._ev_loop = asyncio.get_event_loop() diff --git a/hummingbot/connector/exchange/k2/k2_exchange.py b/hummingbot/connector/exchange/k2/k2_exchange.py index bf1ee63c17..b31b825072 100644 --- a/hummingbot/connector/exchange/k2/k2_exchange.py +++ b/hummingbot/connector/exchange/k2/k2_exchange.py @@ -3,7 +3,7 @@ import math import time from decimal import Decimal -from typing import Any, AsyncIterable, Dict, List, Optional +from typing import TYPE_CHECKING, Any, AsyncIterable, Dict, List, Optional import aiohttp @@ -16,7 +16,7 @@ from hummingbot.connector.trading_rule import TradingRule from hummingbot.core.clock import Clock from hummingbot.core.data_type.cancellation_result import CancellationResult -from hummingbot.core.data_type.common import OpenOrder +from hummingbot.core.data_type.common import OpenOrder, OrderType, TradeType from hummingbot.core.data_type.limit_order import LimitOrder from hummingbot.core.data_type.order_book import OrderBook from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee, TokenAmount @@ -30,11 +30,13 @@ SellOrderCompletedEvent, SellOrderCreatedEvent, ) -from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.network_iterator import NetworkStatus from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + k2_logger = None s_decimal_NaN = Decimal("nan") @@ -57,6 +59,7 @@ def logger(cls) -> HummingbotLogger: return k2_logger def __init__(self, + client_config_map: "ClientConfigAdapter", k2_api_key: str, k2_secret_key: str, trading_pairs: Optional[List[str]] = None, @@ -68,7 +71,7 @@ def __init__(self, :param trading_pairs: The market trading pairs which to track order book data. :param trading_required: Whether actual trading is needed. """ - super().__init__() + super().__init__(client_config_map) self._trading_required = trading_required self._trading_pairs = trading_pairs self._k2_auth = K2Auth(k2_api_key, k2_secret_key) diff --git a/hummingbot/connector/exchange/kraken/kraken_exchange.pyx b/hummingbot/connector/exchange/kraken/kraken_exchange.pyx index 12c5fa38aa..d988c75f4c 100755 --- a/hummingbot/connector/exchange/kraken/kraken_exchange.pyx +++ b/hummingbot/connector/exchange/kraken/kraken_exchange.pyx @@ -9,13 +9,12 @@ from typing import ( AsyncIterable, Dict, List, - Optional, + Optional, TYPE_CHECKING, ) from async_timeout import timeout from libc.stdint cimport int32_t, int64_t -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.connector.exchange.kraken import kraken_constants as CONSTANTS from hummingbot.connector.exchange.kraken.kraken_auth import KrakenAuth from hummingbot.connector.exchange.kraken.kraken_constants import KrakenAPITier @@ -66,6 +65,9 @@ from hummingbot.core.web_assistant.connections.data_types import RESTMethod, RES from hummingbot.core.web_assistant.rest_assistant import RESTAssistant from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + s_logger = None s_decimal_0 = Decimal(0) s_decimal_NaN = Decimal("NaN") @@ -107,6 +109,7 @@ cdef class KrakenExchange(ExchangeBase): return s_logger def __init__(self, + client_config_map: "ClientConfigAdapter", kraken_api_key: str, kraken_secret_key: str, poll_interval: float = 30.0, @@ -114,7 +117,7 @@ cdef class KrakenExchange(ExchangeBase): trading_required: bool = True, kraken_api_tier: str = "starter"): - super().__init__() + super().__init__(client_config_map) self._trading_required = trading_required self._api_factory = build_api_factory() self._rest_assistant = None @@ -1128,8 +1131,7 @@ cdef class KrakenExchange(ExchangeBase): return self.c_get_order_book(trading_pair) def _build_async_throttler(self, api_tier: KrakenAPITier) -> AsyncThrottler: - limits_pct_conf: Optional[Decimal] = global_config_map["rate_limits_share_pct"].value - limits_pct = Decimal("100") if limits_pct_conf is None else limits_pct_conf + limits_pct = self._client_config.rate_limits_share_pct if limits_pct < Decimal("100"): self.logger().warning( f"The Kraken API does not allow enough bandwidth for a reduced rate-limit share percentage." diff --git a/hummingbot/connector/exchange/liquid/liquid_exchange.pyx b/hummingbot/connector/exchange/liquid/liquid_exchange.pyx index 6288851a8c..32b596ef11 100644 --- a/hummingbot/connector/exchange/liquid/liquid_exchange.pyx +++ b/hummingbot/connector/exchange/liquid/liquid_exchange.pyx @@ -10,7 +10,7 @@ from typing import ( AsyncIterable, Dict, List, - Optional, + Optional, TYPE_CHECKING, ) import aiohttp @@ -53,6 +53,9 @@ from hummingbot.core.utils.estimate_fee import estimate_fee from hummingbot.core.utils.tracking_nonce import get_tracking_nonce from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + s_logger = None s_decimal_0 = Decimal(0) s_decimal_nan = Decimal("nan") @@ -91,12 +94,13 @@ cdef class LiquidExchange(ExchangeBase): return s_logger def __init__(self, + client_config_map: "ClientConfigAdapter", liquid_api_key: str, liquid_secret_key: str, poll_interval: float = 5.0, # interval which the class periodically pulls status from the rest API trading_pairs: Optional[List[str]] = None, trading_required: bool = True): - super().__init__() + super().__init__(client_config_map) self._trading_required = trading_required self._liquid_auth = LiquidAuth(liquid_api_key, liquid_secret_key) diff --git a/hummingbot/connector/exchange/loopring/loopring_exchange.pyx b/hummingbot/connector/exchange/loopring/loopring_exchange.pyx index 927b8b1bdb..a9ae31e77c 100644 --- a/hummingbot/connector/exchange/loopring/loopring_exchange.pyx +++ b/hummingbot/connector/exchange/loopring/loopring_exchange.pyx @@ -10,7 +10,7 @@ from typing import ( AsyncIterable, Dict, List, - Optional, + Optional, TYPE_CHECKING, ) import aiohttp @@ -48,6 +48,9 @@ from hummingbot.core.utils.estimate_fee import estimate_fee from hummingbot.core.utils.tracking_nonce import get_tracking_nonce from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + s_logger = None s_decimal_0 = Decimal(0) s_decimal_NaN = Decimal("nan") @@ -136,6 +139,7 @@ cdef class LoopringExchange(ExchangeBase): return s_logger def __init__(self, + client_config_map: "ClientConfigAdapter", loopring_accountid: int, loopring_exchangeaddress: str, loopring_private_key: str, @@ -144,7 +148,7 @@ cdef class LoopringExchange(ExchangeBase): trading_pairs: Optional[List[str]] = None, trading_required: bool = True): - super().__init__() + super().__init__(client_config_map) self._real_time_balance_update = True diff --git a/hummingbot/connector/exchange/ndax/ndax_exchange.py b/hummingbot/connector/exchange/ndax/ndax_exchange.py index 504e7e03ba..eb9970b59b 100644 --- a/hummingbot/connector/exchange/ndax/ndax_exchange.py +++ b/hummingbot/connector/exchange/ndax/ndax_exchange.py @@ -3,24 +3,14 @@ import math import time from decimal import Decimal -from typing import ( - Any, - AsyncIterable, - Dict, - List, - Optional, - Union, -) +from typing import TYPE_CHECKING, Any, AsyncIterable, Dict, List, Optional, Union import aiohttp import ujson from hummingbot.connector.exchange.ndax import ndax_constants as CONSTANTS, ndax_utils from hummingbot.connector.exchange.ndax.ndax_auth import NdaxAuth -from hummingbot.connector.exchange.ndax.ndax_in_flight_order import ( - NdaxInFlightOrder, - NdaxInFlightOrderNotCreated, -) +from hummingbot.connector.exchange.ndax.ndax_in_flight_order import NdaxInFlightOrder, NdaxInFlightOrderNotCreated from hummingbot.connector.exchange.ndax.ndax_order_book_tracker import NdaxOrderBookTracker from hummingbot.connector.exchange.ndax.ndax_user_stream_tracker import NdaxUserStreamTracker from hummingbot.connector.exchange.ndax.ndax_websocket_adaptor import NdaxWebSocketAdaptor @@ -29,8 +19,7 @@ from hummingbot.core.api_throttler.async_throttler import AsyncThrottler from hummingbot.core.clock import Clock from hummingbot.core.data_type.cancellation_result import CancellationResult -from hummingbot.core.data_type.common import OpenOrder -from hummingbot.core.data_type.common import OrderType, TradeType +from hummingbot.core.data_type.common import OpenOrder, OrderType, TradeType from hummingbot.core.data_type.limit_order import LimitOrder from hummingbot.core.data_type.order_book import OrderBook from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee @@ -48,6 +37,9 @@ from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + s_decimal_NaN = Decimal("nan") s_decimal_0 = Decimal(0) @@ -74,6 +66,7 @@ def logger(cls) -> HummingbotLogger: return cls._logger def __init__(self, + client_config_map: "ClientConfigAdapter", ndax_uid: str, ndax_api_key: str, ndax_secret_key: str, @@ -90,7 +83,7 @@ def __init__(self, :param trading_pairs: The market trading pairs which to track order book data. :param trading_required: Whether actual trading is needed. """ - super().__init__() + super().__init__(client_config_map) self._trading_required = trading_required self._trading_pairs = trading_pairs self._auth = NdaxAuth(uid=ndax_uid, diff --git a/hummingbot/connector/exchange/okex/okex_exchange.pyx b/hummingbot/connector/exchange/okex/okex_exchange.pyx index b73555f55f..51f3fb1b1f 100644 --- a/hummingbot/connector/exchange/okex/okex_exchange.pyx +++ b/hummingbot/connector/exchange/okex/okex_exchange.pyx @@ -7,7 +7,7 @@ from typing import ( AsyncIterable, Dict, List, - Optional, + Optional, TYPE_CHECKING, ) import aiohttp @@ -53,6 +53,9 @@ from hummingbot.core.utils.estimate_fee import estimate_fee from hummingbot.core.utils.tracking_nonce import get_tracking_nonce from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + hm_logger = None s_decimal_0 = Decimal(0) TRADING_PAIR_SPLITTER = "-" @@ -101,6 +104,7 @@ cdef class OkexExchange(ExchangeBase): return hm_logger def __init__(self, + client_config_map: "ClientConfigAdapter", okex_api_key: str, okex_secret_key: str, okex_passphrase: str, @@ -109,7 +113,7 @@ cdef class OkexExchange(ExchangeBase): trading_pairs: Optional[List[str]] = None, trading_required: bool = True): - super().__init__() + super().__init__(client_config_map) # self._account_id = "" self._async_scheduler = AsyncCallScheduler(call_interval=0.5) self._data_source_type = order_book_tracker_data_source_type diff --git a/hummingbot/connector/exchange/paper_trade/__init__.py b/hummingbot/connector/exchange/paper_trade/__init__.py index b0953b1cf0..9069d243ee 100644 --- a/hummingbot/connector/exchange/paper_trade/__init__.py +++ b/hummingbot/connector/exchange/paper_trade/__init__.py @@ -1,6 +1,7 @@ import importlib from typing import List +from hummingbot.client.config.client_config_map import ClientConfigMap from hummingbot.client.config.config_helpers import get_connector_class from hummingbot.client.settings import AllConnectorSettings from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import PaperTradeExchange @@ -31,8 +32,9 @@ def get_order_book_tracker(connector_name: str, trading_pairs: List[str]) -> Ord raise Exception(f"Connector {connector_name} OrderBookTracker class not found ({exception})") -def create_paper_trade_market(exchange_name: str, trading_pairs: List[str]): +def create_paper_trade_market(exchange_name: str, client_config_map: ClientConfigMap, trading_pairs: List[str]): tracker = get_order_book_tracker(connector_name=exchange_name, trading_pairs=trading_pairs) - return PaperTradeExchange(tracker, + return PaperTradeExchange(client_config_map, + tracker, get_connector_class(exchange_name), exchange_name=exchange_name) diff --git a/hummingbot/connector/exchange/paper_trade/paper_trade_exchange.pyx b/hummingbot/connector/exchange/paper_trade/paper_trade_exchange.pyx index 3d3736db9b..888915205c 100644 --- a/hummingbot/connector/exchange/paper_trade/paper_trade_exchange.pyx +++ b/hummingbot/connector/exchange/paper_trade/paper_trade_exchange.pyx @@ -5,7 +5,7 @@ import math import random from collections import defaultdict, deque from decimal import Decimal -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Tuple, TYPE_CHECKING from cpython cimport PyObject from cython.operator cimport address, dereference as deref, postincrement as inc @@ -45,6 +45,9 @@ from hummingbot.core.Utils cimport getIteratorFromReverseIterator, reverse_itera from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.core.utils.estimate_fee import build_trade_fee +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + ptm_logger = None s_decimal_0 = Decimal(0) @@ -148,11 +151,17 @@ cdef class PaperTradeExchange(ExchangeBase): MARKET_SELL_ORDER_CREATED_EVENT_TAG = MarketEvent.SellOrderCreated.value MARKET_BUY_ORDER_CREATED_EVENT_TAG = MarketEvent.BuyOrderCreated.value - def __init__(self, order_book_tracker: OrderBookTracker, target_market: type, exchange_name: str): + def __init__( + self, + client_config_map: "ClientConfigAdapter", + order_book_tracker: OrderBookTracker, + target_market: type, + exchange_name: str, + ): order_book_tracker.data_source.order_book_create_function = lambda: CompositeOrderBook() self._order_book_tracker = order_book_tracker self._budget_checker = BudgetChecker(exchange=self) - super(ExchangeBase, self).__init__() + super(ExchangeBase, self).__init__(client_config_map) self._exchange_name = exchange_name self._account_balances = {} self._account_available_balances = {} diff --git a/hummingbot/connector/exchange/probit/probit_exchange.py b/hummingbot/connector/exchange/probit/probit_exchange.py index c234b40932..19d1c02950 100644 --- a/hummingbot/connector/exchange/probit/probit_exchange.py +++ b/hummingbot/connector/exchange/probit/probit_exchange.py @@ -3,13 +3,7 @@ import math import time from decimal import Decimal -from typing import ( - Any, - AsyncIterable, - Dict, - List, - Optional, -) +from typing import TYPE_CHECKING, Any, AsyncIterable, Dict, List, Optional import aiohttp import ujson @@ -24,7 +18,7 @@ from hummingbot.connector.utils import get_new_client_order_id from hummingbot.core.clock import Clock from hummingbot.core.data_type.cancellation_result import CancellationResult -from hummingbot.core.data_type.common import OpenOrder +from hummingbot.core.data_type.common import OpenOrder, OrderType, TradeType from hummingbot.core.data_type.limit_order import LimitOrder from hummingbot.core.data_type.order_book import OrderBook from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee, TokenAmount @@ -38,11 +32,13 @@ SellOrderCompletedEvent, SellOrderCreatedEvent, ) -from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.network_iterator import NetworkStatus from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + probit_logger = None s_decimal_NaN = Decimal("nan") @@ -65,6 +61,7 @@ def logger(cls) -> HummingbotLogger: return probit_logger def __init__(self, + client_config_map: "ClientConfigAdapter", probit_api_key: str, probit_secret_key: str, trading_pairs: Optional[List[str]] = None, @@ -78,7 +75,7 @@ def __init__(self, :param trading_required: Whether actual trading is needed. """ self._domain = domain - super().__init__() + super().__init__(client_config_map) self._trading_required = trading_required self._trading_pairs = trading_pairs self._shared_client = aiohttp.ClientSession() diff --git a/hummingbot/connector/exchange/wazirx/wazirx_exchange.py b/hummingbot/connector/exchange/wazirx/wazirx_exchange.py index 2e754a48ec..b6008d324d 100644 --- a/hummingbot/connector/exchange/wazirx/wazirx_exchange.py +++ b/hummingbot/connector/exchange/wazirx/wazirx_exchange.py @@ -4,13 +4,12 @@ import math import time from decimal import Decimal -from typing import Any, AsyncIterable, Dict, List, Optional +from typing import TYPE_CHECKING, Any, AsyncIterable, Dict, List, Optional import aiohttp from async_timeout import timeout -from hummingbot.connector.exchange.wazirx import wazirx_constants as CONSTANTS -from hummingbot.connector.exchange.wazirx import wazirx_utils +from hummingbot.connector.exchange.wazirx import wazirx_constants as CONSTANTS, wazirx_utils from hummingbot.connector.exchange.wazirx.wazirx_auth import WazirxAuth from hummingbot.connector.exchange.wazirx.wazirx_in_flight_order import WazirxInFlightOrder from hummingbot.connector.exchange.wazirx.wazirx_order_book_tracker import WazirxOrderBookTracker @@ -38,6 +37,9 @@ from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather, wait_til from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + ctce_logger = None s_decimal_NaN = Decimal("nan") @@ -60,6 +62,7 @@ def logger(cls) -> HummingbotLogger: return ctce_logger def __init__(self, + client_config_map: "ClientConfigAdapter", wazirx_api_key: str, wazirx_secret_key: str, trading_pairs: Optional[List[str]] = None, @@ -71,7 +74,7 @@ def __init__(self, :param trading_pairs: The market trading pairs which to track order book data. :param trading_required: Whether actual trading is needed. """ - super().__init__() + super().__init__(client_config_map) self._trading_required = trading_required self._trading_pairs = trading_pairs self._wazirx_auth = WazirxAuth(wazirx_api_key, wazirx_secret_key) diff --git a/hummingbot/connector/exchange_base.pyx b/hummingbot/connector/exchange_base.pyx index 034b24acfc..7f16075976 100644 --- a/hummingbot/connector/exchange_base.pyx +++ b/hummingbot/connector/exchange_base.pyx @@ -1,5 +1,5 @@ from decimal import Decimal -from typing import Dict, List, Optional, Iterator +from typing import Dict, List, Optional, Iterator, TYPE_CHECKING from hummingbot.connector.budget_checker import BudgetChecker from hummingbot.connector.connector_base import ConnectorBase @@ -10,6 +10,9 @@ from hummingbot.core.data_type.order_book_query_result import OrderBookQueryResu from hummingbot.core.data_type.order_book_row import ClientOrderBookRow from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + NaN = float("nan") s_decimal_NaN = Decimal("nan") s_decimal_0 = Decimal(0) @@ -21,8 +24,8 @@ cdef class ExchangeBase(ConnectorBase): interface. """ - def __init__(self): - super().__init__() + def __init__(self, client_config_map: "ClientConfigAdapter"): + super().__init__(client_config_map) self._order_book_tracker = None self._budget_checker = BudgetChecker(exchange=self) diff --git a/hummingbot/connector/gateway_EVM_AMM.py b/hummingbot/connector/gateway_EVM_AMM.py index dafd5fa126..d93c66a622 100644 --- a/hummingbot/connector/gateway_EVM_AMM.py +++ b/hummingbot/connector/gateway_EVM_AMM.py @@ -1,52 +1,48 @@ import asyncio import copy -from decimal import Decimal import itertools as it -from async_timeout import timeout import logging import re import time -from typing import ( - Dict, - List, - Set, - Optional, - Any, - Type, - Union, - cast, -) +from decimal import Decimal +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Type, Union, cast + +from async_timeout import timeout from hummingbot.connector.connector_base import ConnectorBase from hummingbot.connector.gateway_in_flight_order import GatewayInFlightOrder -from hummingbot.core.utils import async_ttl_cache -from hummingbot.core.gateway import check_transaction_exceptions -from hummingbot.core.gateway.gateway_http_client import GatewayHttpClient -from hummingbot.core.network_iterator import NetworkStatus -from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather -from hummingbot.core.utils.tracking_nonce import get_tracking_nonce -from hummingbot.core.data_type.limit_order import LimitOrder from hummingbot.core.data_type.cancellation_result import CancellationResult +from hummingbot.core.data_type.limit_order import LimitOrder from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee, TokenAmount from hummingbot.core.event.events import ( - MarketEvent, - TokenApprovalEvent, - BuyOrderCreatedEvent, - SellOrderCreatedEvent, BuyOrderCompletedEvent, - SellOrderCompletedEvent, + BuyOrderCreatedEvent, + MarketEvent, MarketOrderFailureEvent, OrderCancelledEvent, OrderFilledEvent, - TokenApprovalSuccessEvent, - TokenApprovalFailureEvent, - TokenApprovalCancelledEvent, OrderType, + SellOrderCompletedEvent, + SellOrderCreatedEvent, + TokenApprovalCancelledEvent, + TokenApprovalEvent, + TokenApprovalFailureEvent, + TokenApprovalSuccessEvent, TradeType, ) +from hummingbot.core.gateway import check_transaction_exceptions +from hummingbot.core.gateway.gateway_http_client import GatewayHttpClient +from hummingbot.core.network_iterator import NetworkStatus +from hummingbot.core.utils import async_ttl_cache +from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather +from hummingbot.core.utils.tracking_nonce import get_tracking_nonce from hummingbot.logger import HummingbotLogger + from .gateway_price_shim import GatewayPriceShim +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + s_logger = None s_decimal_0 = Decimal("0") s_decimal_NaN = Decimal("nan") @@ -85,6 +81,7 @@ class GatewayEVMAMM(ConnectorBase): _native_currency: str def __init__(self, + client_config_map: "ClientConfigAdapter", connector_name: str, chain: str, network: str, @@ -102,7 +99,7 @@ def __init__(self, """ self._connector_name = connector_name self._name = "_".join([connector_name, chain, network]) - super().__init__() + super().__init__(client_config_map) self._chain = chain self._network = network self._trading_pairs = trading_pairs @@ -247,7 +244,7 @@ async def get_chain_info(self): Calls the base endpoint of the connector on Gateway to know basic info about chain being used. """ try: - self._chain_info = await GatewayHttpClient.get_instance().get_network_status( + self._chain_info = await self._get_gateway_instance().get_network_status( chain=self.chain, network=self.network ) if type(self._chain_info) != list: @@ -266,7 +263,7 @@ async def get_gas_estimate(self): Gets the gas estimates for the connector. """ try: - response: Dict[Any] = await GatewayHttpClient.get_instance().amm_estimate_gas( + response: Dict[Any] = await self._get_gateway_instance().amm_estimate_gas( chain=self.chain, network=self.network, connector=self.connector_name ) self.network_transaction_fee = TokenAmount( @@ -298,7 +295,7 @@ async def approve_token(self, token_symbol: str, **request_args) -> Optional[Gat """ order_id: str = self.create_approval_order_id(token_symbol) await self._update_nonce() - resp: Dict[str, Any] = await GatewayHttpClient.get_instance().approve_token( + resp: Dict[str, Any] = await self._get_gateway_instance().approve_token( self.chain, self.network, self.address, @@ -332,7 +329,7 @@ async def get_allowances(self) -> Dict[str, Decimal]: :return: A dictionary of token and its allowance. """ ret_val = {} - resp: Dict[str, Any] = await GatewayHttpClient.get_instance().get_allowances( + resp: Dict[str, Any] = await self._get_gateway_instance().get_allowances( self.chain, self.network, self.address, list(self._tokens), self.connector_name ) for token, amount in resp["approvals"].items(): @@ -373,7 +370,7 @@ async def get_quote_price( if test_price is not None: # Grab the gas price for test net. try: - resp: Dict[str, Any] = await GatewayHttpClient.get_instance().get_price( + resp: Dict[str, Any] = await self._get_gateway_instance().get_price( self.chain, self.network, self.connector_name, base, quote, amount, side ) gas_price_token: str = resp["gasPriceToken"] @@ -387,7 +384,7 @@ async def get_quote_price( # Pull the price from gateway. try: - resp: Dict[str, Any] = await GatewayHttpClient.get_instance().get_price( + resp: Dict[str, Any] = await self._get_gateway_instance().get_price( self.chain, self.network, self.connector_name, base, quote, amount, side ) required_items = ["price", "gasLimit", "gasPrice", "gasCost", "gasPriceToken"] @@ -510,7 +507,7 @@ async def _create_order( amount=amount) await self._update_nonce() try: - order_result: Dict[str, Any] = await GatewayHttpClient.get_instance().amm_trade( + order_result: Dict[str, Any] = await self._get_gateway_instance().amm_trade( self.chain, self.network, self.connector_name, @@ -618,7 +615,7 @@ async def update_token_approval_status(self, tracked_approvals: List[GatewayInFl tracked_approval.get_exchange_order_id() for tracked_approval in tracked_approvals ]) transaction_states: List[Union[Dict[str, Any], Exception]] = await safe_gather(*[ - GatewayHttpClient.get_instance().get_transaction_status( + self._get_gateway_instance().get_transaction_status( self.chain, self.network, tx_hash @@ -675,7 +672,7 @@ async def update_canceling_transactions(self, canceled_tracked_orders: List[Gate len(canceled_tracked_orders) ) update_results: List[Union[Dict[str, Any], Exception]] = await safe_gather(*[ - GatewayHttpClient.get_instance().get_transaction_status( + self._get_gateway_instance().get_transaction_status( self.chain, self.network, tx_hash @@ -737,7 +734,7 @@ async def update_order_status(self, tracked_orders: List[GatewayInFlightOrder]): len(tracked_orders) ) update_results: List[Union[Dict[str, Any], Exception]] = await safe_gather(*[ - GatewayHttpClient.get_instance().get_transaction_status( + self._get_gateway_instance().get_transaction_status( self.chain, self.network, tx_hash @@ -857,7 +854,7 @@ async def stop_network(self): async def check_network(self) -> NetworkStatus: try: - if await GatewayHttpClient.get_instance().ping_gateway(): + if await self._get_gateway_instance().ping_gateway(): return NetworkStatus.CONNECTED except asyncio.CancelledError: raise @@ -878,7 +875,7 @@ async def _update_nonce(self): """ Call the gateway API to get the current nonce for self.address """ - resp_json = await GatewayHttpClient.get_instance().get_evm_nonce(self.chain, self.network, self.address) + resp_json = await self._get_gateway_instance().get_evm_nonce(self.chain, self.network, self.address) self._nonce = resp_json['nonce'] async def _status_polling_loop(self): @@ -911,7 +908,7 @@ async def update_balances(self, on_interval=False): self._last_balance_poll_timestamp = current_tick local_asset_names = set(self._account_balances.keys()) remote_asset_names = set() - resp_json: Dict[str, Any] = await GatewayHttpClient.get_instance().get_balances( + resp_json: Dict[str, Any] = await self._get_gateway_instance().get_balances( self.chain, self.network, self.address, list(self._tokens) + [self._native_currency] ) for token, bal in resp_json["balances"].items(): @@ -962,7 +959,7 @@ async def _execute_cancel(self, order_id: str, cancel_age: int) -> Optional[str] self.logger().info(f"The blockchain transaction for {order_id} with nonce {tracked_order.nonce} has " f"expired. Canceling the order...") - resp: Dict[str, Any] = await GatewayHttpClient.get_instance().cancel_evm_transaction( + resp: Dict[str, Any] = await self._get_gateway_instance().cancel_evm_transaction( self.chain, self.network, self.address, @@ -1030,3 +1027,7 @@ async def cancel_outdated_orders(self, cancel_age: int) -> List[CancellationResu @property def in_flight_orders(self) -> Dict[str, GatewayInFlightOrder]: return self._in_flight_orders + + def _get_gateway_instance(self) -> GatewayHttpClient: + gateway_instance = GatewayHttpClient.get_instance(self.client_config_map) + return gateway_instance diff --git a/hummingbot/core/api_throttler/async_throttler_base.py b/hummingbot/core/api_throttler/async_throttler_base.py index b440c5573d..8c551bec4a 100644 --- a/hummingbot/core/api_throttler/async_throttler_base.py +++ b/hummingbot/core/api_throttler/async_throttler_base.py @@ -2,21 +2,12 @@ import copy import logging import math - from abc import ABC, abstractmethod from decimal import Decimal -from typing import ( - Dict, - List, - Optional, - Tuple, -) +from typing import Dict, List, Optional, Tuple from hummingbot.core.api_throttler.async_request_context_base import AsyncRequestContextBase -from hummingbot.core.api_throttler.data_types import ( - RateLimit, - TaskLog -) +from hummingbot.core.api_throttler.data_types import RateLimit, TaskLog from hummingbot.logger.logger import HummingbotLogger @@ -44,14 +35,14 @@ def __init__(self, :param retry_interval: Time between every capacity check. :param safety_margin: Percentage of limit to be added as a safety margin when calculating capacity to ensure calls are within the limit. """ - from hummingbot.client.config.global_config_map import global_config_map # avoids chance of circular import + from hummingbot.client.hummingbot_application import HummingbotApplication # avoids circular import # Rate Limit Definitions self._rate_limits: List[RateLimit] = copy.deepcopy(rate_limits) - limits_pct_conf: Optional[Decimal] = global_config_map["rate_limits_share_pct"].value + client_config = HummingbotApplication.main_application().client_config_map # If configured, users can define the percentage of rate limits to allocate to the throttler. - self.limits_pct: Optional[Decimal] = Decimal("1") if limits_pct_conf is None else limits_pct_conf / Decimal("100") + self.limits_pct: Decimal = client_config.rate_limits_share_pct for rate_limit in self._rate_limits: rate_limit.limit = max(Decimal("1"), math.floor(Decimal(str(rate_limit.limit)) * self.limits_pct)) diff --git a/hummingbot/core/gateway/__init__.py b/hummingbot/core/gateway/__init__.py index 346509ff3f..50f42e3624 100644 --- a/hummingbot/core/gateway/__init__.py +++ b/hummingbot/core/gateway/__init__.py @@ -1,13 +1,17 @@ -import aioprocessing +import os from dataclasses import dataclass from decimal import Decimal -import os from pathlib import Path -from typing import Optional, Any, Dict, AsyncIterable, List +from typing import TYPE_CHECKING, Any, AsyncIterable, Dict, List, Optional + +import aioprocessing from hummingbot.core.event.events import TradeType from hummingbot.core.utils import detect_available_port +if TYPE_CHECKING: + from hummingbot import ClientConfigAdapter + _default_paths: Optional["GatewayPaths"] = None _hummingbot_pipe: Optional[aioprocessing.AioConnection] = None @@ -33,14 +37,13 @@ def is_inside_docker() -> bool: return False -def get_gateway_container_name() -> str: +def get_gateway_container_name(client_config_map: "ClientConfigAdapter") -> str: """ Calculates the name for the gateway container, for this Hummingbot instance. :return: Gateway container name """ - from hummingbot.client.config.global_config_map import global_config_map - instance_id_suffix: str = global_config_map["instance_id"].value[:8] + instance_id_suffix = client_config_map.instance_id[:8] return f"hummingbot-gateway-{instance_id_suffix}" @@ -74,7 +77,7 @@ def __post_init__(self): path.mkdir(mode=0o755, parents=True, exist_ok=True) -def get_gateway_paths() -> GatewayPaths: +def get_gateway_paths(client_config_map: "ClientConfigAdapter") -> GatewayPaths: """ Calculates the default paths for a gateway container. @@ -90,7 +93,7 @@ def get_gateway_paths() -> GatewayPaths: inside_docker: bool = is_inside_docker() - gateway_container_name: str = get_gateway_container_name() + gateway_container_name: str = get_gateway_container_name(client_config_map) external_certs_path: Optional[Path] = os.getenv("CERTS_FOLDER") and Path(os.getenv("CERTS_FOLDER")) external_conf_path: Optional[Path] = os.getenv("GATEWAY_CONF_FOLDER") and Path(os.getenv("GATEWAY_CONF_FOLDER")) external_logs_path: Optional[Path] = os.getenv("GATEWAY_LOGS_FOLDER") and Path(os.getenv("GATEWAY_LOGS_FOLDER")) @@ -122,9 +125,9 @@ def get_gateway_paths() -> GatewayPaths: return _default_paths -def get_default_gateway_port() -> int: - from hummingbot.client.config.global_config_map import global_config_map - return detect_available_port(16000 + int(global_config_map.get("instance_id").value[:4], 16) % 16000) +def get_default_gateway_port(client_config_map: "ClientConfigAdapter") -> int: + instance_id_portion = client_config_map.instance_id[:4] + return detect_available_port(16000 + int(instance_id_portion, 16) % 16000) def set_hummingbot_pipe(conn: aioprocessing.AioConnection): @@ -132,13 +135,13 @@ def set_hummingbot_pipe(conn: aioprocessing.AioConnection): _hummingbot_pipe = conn -async def detect_existing_gateway_container() -> Optional[Dict[str, Any]]: +async def detect_existing_gateway_container(client_config_map: "ClientConfigAdapter") -> Optional[Dict[str, Any]]: try: results: List[Dict[str, Any]] = await docker_ipc( "containers", all=True, filters={ - "name": get_gateway_container_name(), + "name": get_gateway_container_name(client_config_map), }) if len(results) > 0: return results[0] @@ -147,10 +150,10 @@ async def detect_existing_gateway_container() -> Optional[Dict[str, Any]]: return -async def start_existing_gateway_container(): - container_info: Optional[Dict[str, Any]] = await detect_existing_gateway_container() +async def start_existing_gateway_container(client_config_map: "ClientConfigAdapter"): + container_info: Optional[Dict[str, Any]] = await detect_existing_gateway_container(client_config_map) if container_info is not None and container_info["State"] != "running": - await docker_ipc("start", get_gateway_container_name()) + await docker_ipc("start", get_gateway_container_name(client_config_map)) async def docker_ipc(method_name: str, *args, **kwargs) -> Any: @@ -240,16 +243,16 @@ def check_transaction_exceptions( return exception_list -async def start_gateway(): +async def start_gateway(client_config_map: "ClientConfigAdapter"): from hummingbot.client.hummingbot_application import HummingbotApplication try: response = await docker_ipc( "containers", all=True, - filters={"name": get_gateway_container_name()} + filters={"name": get_gateway_container_name(client_config_map)} ) if len(response) == 0: - raise ValueError(f"Gateway container {get_gateway_container_name()} not found. ") + raise ValueError(f"Gateway container {get_gateway_container_name(client_config_map)} not found. ") container_info = response[0] if container_info["State"] == "running": @@ -265,16 +268,16 @@ async def start_gateway(): HummingbotApplication.main_application().notify(f"Error occurred starting Gateway container. {e}") -async def stop_gateway(): +async def stop_gateway(client_config_map: "ClientConfigAdapter"): from hummingbot.client.hummingbot_application import HummingbotApplication try: response = await docker_ipc( "containers", all=True, - filters={"name": get_gateway_container_name()} + filters={"name": get_gateway_container_name(client_config_map)} ) if len(response) == 0: - raise ValueError(f"Gateway container {get_gateway_container_name()} not found.") + raise ValueError(f"Gateway container {get_gateway_container_name(client_config_map)} not found.") container_info = response[0] if container_info["State"] != "running": @@ -290,8 +293,8 @@ async def stop_gateway(): HummingbotApplication.main_application().notify(f"Error occurred stopping Gateway container. {e}") -async def restart_gateway(): +async def restart_gateway(client_config_map: "ClientConfigAdapter"): from hummingbot.client.hummingbot_application import HummingbotApplication - await stop_gateway() - await start_gateway() + await stop_gateway(client_config_map) + await start_gateway(client_config_map) HummingbotApplication.main_application().notify("Gateway will be ready momentarily.") diff --git a/hummingbot/core/gateway/gateway_http_client.py b/hummingbot/core/gateway/gateway_http_client.py index a240626dc5..9720dc6701 100644 --- a/hummingbot/core/gateway/gateway_http_client.py +++ b/hummingbot/core/gateway/gateway_http_client.py @@ -2,16 +2,18 @@ import ssl from decimal import Decimal from enum import Enum -from typing import Any, Dict, List, Optional, Union +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union import aiohttp -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.config.security import Security from hummingbot.core.event.events import TradeType from hummingbot.core.gateway import get_gateway_paths from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + class GatewayError(Enum): """ @@ -44,15 +46,20 @@ class GatewayHttpClient: __instance = None @staticmethod - def get_instance() -> "GatewayHttpClient": + def get_instance(client_config_map: Optional["ClientConfigAdapter"] = None) -> "GatewayHttpClient": if GatewayHttpClient.__instance is None: - GatewayHttpClient() + GatewayHttpClient(client_config_map) return GatewayHttpClient.__instance - def __init__(self): + def __init__(self, client_config_map: Optional["ClientConfigAdapter"] = None): + if client_config_map is None: + from hummingbot.client.hummingbot_application import HummingbotApplication + client_config_map = HummingbotApplication.main_application().client_config_map + api_host = client_config_map.gateway.gateway_api_host + api_port = client_config_map.gateway.gateway_api_port if GatewayHttpClient.__instance is None: - self._base_url = f"https://{global_config_map['gateway_api_host'].value}:" \ - f"{global_config_map['gateway_api_port'].value}" + self._base_url = f"https://{api_host}:{api_port}" + self._client_confi_map = client_config_map GatewayHttpClient.__instance = self @classmethod @@ -62,12 +69,12 @@ def logger(cls) -> HummingbotLogger: return cls._ghc_logger @classmethod - def _http_client(cls, re_init: bool = False) -> aiohttp.ClientSession: + def _http_client(cls, client_config_map: "ClientConfigAdapter", re_init: bool = False) -> aiohttp.ClientSession: """ :returns Shared client session instance """ if cls._shared_client is None or re_init: - cert_path = get_gateway_paths().local_certs_path.as_posix() + cert_path = get_gateway_paths(client_config_map).local_certs_path.as_posix() ssl_ctx = ssl.create_default_context(cafile=f"{cert_path}/ca_cert.pem") ssl_ctx.load_cert_chain(certfile=f"{cert_path}/client_cert.pem", keyfile=f"{cert_path}/client_key.pem", @@ -140,7 +147,7 @@ async def api_request( :returns A response in json format. """ url = f"{self.base_url}/{path_url}" - client = self._http_client() + client = self._http_client(self._client_confi_map) parsed_response = {} try: diff --git a/hummingbot/core/gateway/status_monitor.py b/hummingbot/core/gateway/status_monitor.py index c592c5cbf5..0c3bb82edd 100644 --- a/hummingbot/core/gateway/status_monitor.py +++ b/hummingbot/core/gateway/status_monitor.py @@ -1,8 +1,7 @@ import asyncio import logging - from enum import Enum -from typing import Any, Dict, List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Any, Dict, List, Optional from hummingbot.client.settings import GATEWAY_CONNECTORS from hummingbot.client.ui.completer import load_completer @@ -62,9 +61,10 @@ def stop(self): async def _monitor_loop(self): while True: try: - if await asyncio.wait_for(GatewayHttpClient.get_instance().ping_gateway(), timeout=POLL_TIMEOUT): + gateway_instance = self._get_gateway_instance() + if await asyncio.wait_for(gateway_instance.ping_gateway(), timeout=POLL_TIMEOUT): if self._current_status is Status.OFFLINE: - gateway_connectors = await GatewayHttpClient.get_instance().get_connectors(fail_silently=True) + gateway_connectors = await gateway_instance.get_connectors(fail_silently=True) GATEWAY_CONNECTORS.clear() GATEWAY_CONNECTORS.extend([connector["name"] for connector in gateway_connectors.get("connectors", [])]) @@ -80,7 +80,7 @@ async def _monitor_loop(self): await asyncio.sleep(POLL_INTERVAL) async def _fetch_gateway_configs(self) -> Dict[str, Any]: - return await GatewayHttpClient.get_instance().get_configuration(fail_silently=False) + return await self._get_gateway_instance().get_configuration(fail_silently=False) async def update_gateway_config_key_list(self): try: @@ -93,3 +93,7 @@ async def update_gateway_config_key_list(self): except Exception: self.logger().error("Error fetching gateway configs. Please check that Gateway service is online. ", exc_info=True) + + def _get_gateway_instance(self) -> GatewayHttpClient: + gateway_instance = GatewayHttpClient.get_instance(self._app.client_config_map) + return gateway_instance diff --git a/hummingbot/core/utils/kill_switch.py b/hummingbot/core/utils/kill_switch.py index e27ed7d307..6a1b720021 100644 --- a/hummingbot/core/utils/kill_switch.py +++ b/hummingbot/core/utils/kill_switch.py @@ -1,13 +1,24 @@ import asyncio import logging +from abc import ABC, abstractmethod from decimal import Decimal from typing import Optional -from hummingbot.client.config.global_config_map import global_config_map -from hummingbot.logger import HummingbotLogger + from hummingbot.core.utils.async_utils import safe_ensure_future +from hummingbot.logger import HummingbotLogger + +class KillSwitch(ABC): + @abstractmethod + def start(self): + ... + + @abstractmethod + def stop(self): + ... -class KillSwitch: + +class ActiveKillSwitch(KillSwitch): ks_logger: Optional[HummingbotLogger] = None @classmethod @@ -17,12 +28,11 @@ def logger(cls) -> HummingbotLogger: return cls.ks_logger def __init__(self, + kill_switch_rate: Decimal, hummingbot_application: "HummingbotApplication"): # noqa F821 self._hummingbot_application = hummingbot_application - self._kill_switch_enabled: bool = global_config_map.get("kill_switch_enabled").value - self._kill_switch_rate: Decimal = Decimal(global_config_map.get("kill_switch_rate").value or "0.0") / \ - Decimal(100) + self._kill_switch_rate: Decimal = kill_switch_rate / Decimal(100) self._started = False self._update_interval = 10.0 self._check_profitability_task: Optional[asyncio.Task] = None @@ -63,3 +73,11 @@ def stop(self): if self._check_profitability_task and not self._check_profitability_task.done(): self._check_profitability_task.cancel() self._started = False + + +class PassThroughKillSwitch(KillSwitch): + def start(self): + pass + + def stop(self): + pass diff --git a/hummingbot/core/utils/ssl_cert.py b/hummingbot/core/utils/ssl_cert.py index 442e9909ac..eb5d58cc16 100644 --- a/hummingbot/core/utils/ssl_cert.py +++ b/hummingbot/core/utils/ssl_cert.py @@ -1,17 +1,21 @@ """ Functions for generating keys and certificates """ +from datetime import datetime, timedelta +from os import listdir +from os.path import join +from typing import TYPE_CHECKING from cryptography import x509 from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes -from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.x509.oid import NameOID -from datetime import datetime, timedelta + from hummingbot.core.gateway import get_gateway_paths -from os import listdir -from os.path import join + +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter CERT_SUBJECT = [ x509.NameAttribute(NameOID.ORGANIZATION_NAME, 'localhost'), @@ -166,7 +170,7 @@ def sign_csr(csr, ca_public_key, ca_private_key, filepath): client_csr_filename = 'client_csr.pem' -def certs_files_exist() -> bool: +def certs_files_exist(client_config_map: "ClientConfigAdapter") -> bool: """ Check if the necessary key and certificate files exist """ @@ -174,15 +178,15 @@ def certs_files_exist() -> bool: server_key_filename, server_cert_filename, client_key_filename, client_cert_filename] - file_list = listdir(get_gateway_paths().local_certs_path.as_posix()) + file_list = listdir(get_gateway_paths(client_config_map).local_certs_path.as_posix()) return all(elem in file_list for elem in required_certs) -def create_self_sign_certs(pass_phase: str): +def create_self_sign_certs(pass_phase: str, client_config_map: "ClientConfigAdapter"): """ Create self-sign CA Cert """ - cert_directory: str = get_gateway_paths().local_certs_path.as_posix() + cert_directory: str = get_gateway_paths(client_config_map).local_certs_path.as_posix() filepath_list = { 'ca_key': join(cert_directory, ca_key_filename), diff --git a/hummingbot/model/db_migration/migrator.py b/hummingbot/model/db_migration/migrator.py index f1fdff85ed..ec884c5814 100644 --- a/hummingbot/model/db_migration/migrator.py +++ b/hummingbot/model/db_migration/migrator.py @@ -1,11 +1,12 @@ import logging -from shutil import copyfile, move from inspect import getmembers, isabstract, isclass from pathlib import Path +from shutil import copyfile, move import pandas as pd from sqlalchemy.exc import SQLAlchemyError +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.model.db_migration.base_transformation import DatabaseTransformation from hummingbot.model.sql_connection_manager import SQLConnectionManager, SQLConnectionType @@ -22,7 +23,7 @@ def _get_transformations(cls): def __init__(self): self.transformations = [t(self) for t in self._get_transformations()] - def migrate_db_to_version(self, db_handle, from_version, to_version): + def migrate_db_to_version(self, client_config_map: ClientConfigAdapter, db_handle, from_version, to_version): original_db_path = db_handle.db_path original_db_name = Path(original_db_path).stem backup_db_path = original_db_path + '.backup_' + pd.Timestamp.utcnow().strftime("%Y%m%d-%H%M%S") @@ -31,7 +32,9 @@ def migrate_db_to_version(self, db_handle, from_version, to_version): copyfile(original_db_path, backup_db_path) db_handle.engine.dispose() - new_db_handle = SQLConnectionManager(SQLConnectionType.TRADE_FILLS, new_db_path, original_db_name, True) + new_db_handle = SQLConnectionManager( + client_config_map, SQLConnectionType.TRADE_FILLS, new_db_path, original_db_name, True + ) relevant_transformations = [t for t in self.transformations if t.does_apply_to_version(from_version, to_version)] diff --git a/hummingbot/model/sql_connection_manager.py b/hummingbot/model/sql_connection_manager.py index 538f6a9bd9..0e49e44f05 100644 --- a/hummingbot/model/sql_connection_manager.py +++ b/hummingbot/model/sql_connection_manager.py @@ -1,29 +1,22 @@ import logging - from enum import Enum from os.path import join -from typing import Optional +from typing import TYPE_CHECKING, Optional -from sqlalchemy import ( - create_engine, - inspect, - MetaData, -) +from sqlalchemy import MetaData, create_engine, inspect from sqlalchemy.engine.base import Engine -from sqlalchemy.orm import ( - Query, - Session, - sessionmaker, -) +from sqlalchemy.orm import Query, Session, sessionmaker from sqlalchemy.schema import DropConstraint, ForeignKeyConstraint, Table from hummingbot import data_path -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.logger.logger import HummingbotLogger from hummingbot.model import get_declarative_base from hummingbot.model.metadata import Metadata as LocalMetadata from hummingbot.model.transaction_base import TransactionBase +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + class SQLConnectionType(Enum): TRADE_FILLS = 1 @@ -47,11 +40,17 @@ def get_declarative_base(cls): return get_declarative_base() @classmethod - def get_trade_fills_instance(cls, db_name: Optional[str] = None) -> "SQLConnectionManager": + def get_trade_fills_instance( + cls, client_config_map: "ClientConfigAdapter", db_name: Optional[str] = None + ) -> "SQLConnectionManager": if cls._scm_trade_fills_instance is None: - cls._scm_trade_fills_instance = SQLConnectionManager(SQLConnectionType.TRADE_FILLS, db_name=db_name) + cls._scm_trade_fills_instance = SQLConnectionManager( + client_config_map, SQLConnectionType.TRADE_FILLS, db_name=db_name + ) elif cls.create_db_path(db_name=db_name) != cls._scm_trade_fills_instance.db_path: - cls._scm_trade_fills_instance = SQLConnectionManager(SQLConnectionType.TRADE_FILLS, db_name=db_name) + cls._scm_trade_fills_instance = SQLConnectionManager( + client_config_map, SQLConnectionType.TRADE_FILLS, db_name=db_name + ) return cls._scm_trade_fills_instance @classmethod @@ -63,28 +62,8 @@ def create_db_path(cls, db_path: Optional[str] = None, db_name: Optional[str] = else: return join(data_path(), "hummingbot_trades.sqlite") - @classmethod - def get_db_engine(cls, - dialect: str, - params: dict) -> Engine: - # Fallback to `sqlite` if dialect is None - if dialect is None: - dialect = "sqlite" - - if "sqlite" in dialect: - db_path = params.get("db_path") - - return create_engine(f"{dialect}:///{db_path}") - else: - username = params.get("db_username") - password = params.get("db_password") - host = params.get("db_host") - port = params.get("db_port") - db_name = params.get("db_name") - - return create_engine(f"{dialect}://{username}:{password}@{host}:{port}/{db_name}") - def __init__(self, + client_config_map: "ClientConfigAdapter", connection_type: SQLConnectionType, db_path: Optional[str] = None, db_name: Optional[str] = None, @@ -92,20 +71,8 @@ def __init__(self, db_path = self.create_db_path(db_path, db_name) self.db_path = db_path - engine_options = { - "db_engine": global_config_map.get("db_engine").value, - "db_host": global_config_map.get("db_host").value, - "db_port": global_config_map.get("db_port").value, - "db_username": global_config_map.get("db_username").value, - "db_password": global_config_map.get("db_password").value, - "db_name": global_config_map.get("db_name").value, - "db_path": db_path - } - if connection_type is SQLConnectionType.TRADE_FILLS: - self._engine: Engine = self.get_db_engine( - engine_options.get("db_engine"), - engine_options) + self._engine: Engine = create_engine(client_config_map.db_mode.get_url(self.db_path)) self._metadata: MetaData = self.get_declarative_base().metadata self._metadata.create_all(self._engine) @@ -127,7 +94,7 @@ def __init__(self, self._session_cls = sessionmaker(bind=self._engine) if connection_type is SQLConnectionType.TRADE_FILLS and (not called_from_migrator): - self.check_and_migrate_db() + self.check_and_migrate_db(client_config_map) @property def engine(self) -> Engine: @@ -142,7 +109,7 @@ def get_local_db_version(self, session: Session): result: Optional[LocalMetadata] = query.one_or_none() return result - def check_and_migrate_db(self): + def check_and_migrate_db(self, client_config_map: "ClientConfigAdapter"): from hummingbot.model.db_migration.migrator import Migrator with self.get_new_session() as session: with session.begin(): @@ -157,7 +124,8 @@ def check_and_migrate_db(self): # if needed. if local_db_version.value < self.LOCAL_DB_VERSION_VALUE: was_migration_successful = Migrator().migrate_db_to_version( - self, int(local_db_version.value), int(self.LOCAL_DB_VERSION_VALUE)) + client_config_map, self, int(local_db_version.value), int(self.LOCAL_DB_VERSION_VALUE) + ) if was_migration_successful: # Cannot use variable local_db_version because reference is not valid # since Migrator changed it diff --git a/hummingbot/notifier/telegram_notifier.py b/hummingbot/notifier/telegram_notifier.py index 4599cd090b..6f3df538c9 100644 --- a/hummingbot/notifier/telegram_notifier.py +++ b/hummingbot/notifier/telegram_notifier.py @@ -1,38 +1,25 @@ #!/usr/bin/env python -from os.path import join, realpath -import sys; sys.path.insert(0, realpath(join(__file__, "../../../"))) - import asyncio import logging -from typing import ( - Any, - List, - Callable, - Optional, -) +from os.path import join, realpath +from typing import Any, Callable, List, Optional + +import pandas as pd from telegram.bot import Bot +from telegram.error import NetworkError, TelegramError +from telegram.ext import Filters, MessageHandler, Updater from telegram.parsemode import ParseMode from telegram.replykeyboardmarkup import ReplyKeyboardMarkup from telegram.update import Update -from telegram.error import ( - NetworkError, - TelegramError, -) -from telegram.ext import ( - MessageHandler, - Filters, - Updater, -) import hummingbot -import pandas as pd -from hummingbot.logger import HummingbotLogger -from hummingbot.notifier.notifier_base import NotifierBase -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.core.utils.async_call_scheduler import AsyncCallScheduler from hummingbot.core.utils.async_utils import safe_ensure_future +from hummingbot.logger import HummingbotLogger +from hummingbot.notifier.notifier_base import NotifierBase +import sys; sys.path.insert(0, realpath(join(__file__, "../../../"))) DISABLED_COMMANDS = { "connect", # disabled because telegram can't display secondary prompt @@ -79,8 +66,8 @@ def __init__(self, chat_id: str, hb: "hummingbot.client.hummingbot_application.HummingbotApplication") -> None: super().__init__() - self._token = token or global_config_map.get("telegram_token").value - self._chat_id = chat_id or global_config_map.get("telegram_chat_id").value + self._token = token + self._chat_id = chat_id self._updater = Updater(token=token, workers=0) self._hb = hb self._ev_loop = asyncio.get_event_loop() @@ -93,6 +80,14 @@ def __init__(self, for handle in handles: self._updater.dispatcher.add_handler(handle) + def __eq__(self, other): + return ( + isinstance(other, self.__class__) + and self._token == other._token + and self._chat_id == other._chat_id + and id(self._hb) == id(other._hb) + ) + def start(self): if not self._started: self._started = True diff --git a/hummingbot/strategy/aroon_oscillator/aroon_oscillator_config_map.py b/hummingbot/strategy/aroon_oscillator/aroon_oscillator_config_map.py index c0d99c28b9..d6f951d920 100644 --- a/hummingbot/strategy/aroon_oscillator/aroon_oscillator_config_map.py +++ b/hummingbot/strategy/aroon_oscillator/aroon_oscillator_config_map.py @@ -1,6 +1,7 @@ from decimal import Decimal from typing import Optional +from hummingbot.client.config.client_config_map import using_exchange from hummingbot.client.config.config_validators import ( validate_bool, validate_decimal, @@ -9,7 +10,6 @@ validate_market_trading_pair, ) from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.global_config_map import using_exchange from hummingbot.client.settings import AllConnectorSettings, required_exchanges diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py index 0596180065..1dafa3cfcc 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making_config_map_pydantic.py @@ -161,7 +161,6 @@ def validate_pct_exclusive(cls, v: str): class IgnoreHangingOrdersModel(BaseClientModel): class Config: title = "ignore_hanging_orders" - validate_assignment = True HANGING_ORDER_MODELS = { @@ -318,7 +317,7 @@ class AvellanedaMarketMakingConfigMap(BaseTradingStrategyConfigMap): client_data=None, ) hanging_orders_mode: Union[IgnoreHangingOrdersModel, TrackHangingOrdersModel] = Field( - default=IgnoreHangingOrdersModel.construct(), + default=IgnoreHangingOrdersModel(), description="When tracking hanging orders, the orders on the side opposite to the filled orders remain active.", client_data=ClientFieldData( prompt=( diff --git a/hummingbot/strategy/cross_exchange_market_making/start.py b/hummingbot/strategy/cross_exchange_market_making/start.py index f08d5c7383..ee9de162c5 100644 --- a/hummingbot/strategy/cross_exchange_market_making/start.py +++ b/hummingbot/strategy/cross_exchange_market_making/start.py @@ -1,14 +1,14 @@ -from typing import ( - List, - Tuple -) from decimal import Decimal -from hummingbot.client.config.global_config_map import global_config_map -from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple +from typing import List, Tuple + +from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_making import ( + CrossExchangeMarketMakingStrategy, +) +from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_making_config_map import ( + cross_exchange_market_making_config_map as xemm_map, +) from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_pair import CrossExchangeMarketPair -from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_making import CrossExchangeMarketMakingStrategy -from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_making_config_map import \ - cross_exchange_market_making_config_map as xemm_map +from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple def start(self): @@ -18,7 +18,7 @@ def start(self): raw_taker_trading_pair = xemm_map.get("taker_market_trading_pair").value min_profitability = xemm_map.get("min_profitability").value / Decimal("100") order_amount = xemm_map.get("order_amount").value - strategy_report_interval = global_config_map.get("strategy_report_interval").value + strategy_report_interval = self.clientconfig_map.strategy_report_interval limit_order_min_expiration = xemm_map.get("limit_order_min_expiration").value cancel_order_threshold = xemm_map.get("cancel_order_threshold").value / Decimal("100") active_order_canceling = xemm_map.get("active_order_canceling").value diff --git a/hummingbot/user/user_balances.py b/hummingbot/user/user_balances.py index c36f44ec43..4eea20d68c 100644 --- a/hummingbot/user/user_balances.py +++ b/hummingbot/user/user_balances.py @@ -2,7 +2,8 @@ from functools import lru_cache from typing import Dict, List, Optional -from hummingbot.client.config.config_helpers import get_connector_class +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ReadOnlyClientConfigAdapter, get_connector_class from hummingbot.client.config.security import Security from hummingbot.client.settings import AllConnectorSettings, gateway_connector_trading_pairs from hummingbot.core.utils.async_utils import safe_gather @@ -13,14 +14,15 @@ class UserBalances: __instance = None @staticmethod - def connect_market(exchange, **api_details): + def connect_market(exchange, client_config_map: ClientConfigMap, **api_details): connector = None conn_setting = AllConnectorSettings.get_connector_settings()[exchange] if api_details or conn_setting.uses_gateway_generic_connector(): connector_class = get_connector_class(exchange) init_params = conn_setting.conn_init_parameters(api_details) init_params.update(trading_pairs=gateway_connector_trading_pairs(conn_setting.name)) - connector = connector_class(**init_params) + read_only_client_config = ReadOnlyClientConfigAdapter.lock_config(client_config_map) + connector = connector_class(read_only_client_config, **init_params) return connector # return error message if the _update_balances fails @@ -50,9 +52,9 @@ def __init__(self): UserBalances.__instance = self self._markets = {} - async def add_exchange(self, exchange, **api_details) -> Optional[str]: + async def add_exchange(self, exchange, client_config_map: ClientConfigMap, **api_details) -> Optional[str]: self._markets.pop(exchange, None) - market = UserBalances.connect_market(exchange, **api_details) + market = UserBalances.connect_market(exchange, client_config_map, **api_details) if not market: return "API keys have not been added." err_msg = await UserBalances._update_balances(market) @@ -65,7 +67,7 @@ def all_balances(self, exchange) -> Dict[str, Decimal]: return {} return self._markets[exchange].get_all_balances() - async def update_exchange_balance(self, exchange_name: str) -> Optional[str]: + async def update_exchange_balance(self, exchange_name: str, client_config_map: ClientConfigMap) -> Optional[str]: is_gateway_market = self.is_gateway_market(exchange_name) if is_gateway_market and exchange_name in self._markets: # we want to refresh gateway connectors always, since the applicable tokens change over time. @@ -76,11 +78,12 @@ async def update_exchange_balance(self, exchange_name: str) -> Optional[str]: else: await Security.wait_til_decryption_done() api_keys = Security.api_keys(exchange_name) if not is_gateway_market else {} - return await self.add_exchange(exchange_name, **api_keys) + return await self.add_exchange(exchange_name, client_config_map, **api_keys) # returns error message for each exchange async def update_exchanges( self, + client_config_map: ClientConfigMap, reconnect: bool = False, exchanges: Optional[List[str]] = None ) -> Dict[str, Optional[str]]: @@ -100,19 +103,19 @@ async def update_exchanges( if reconnect: self._markets.clear() for exchange in exchanges: - tasks.append(self.update_exchange_balance(exchange)) + tasks.append(self.update_exchange_balance(exchange, client_config_map)) results = await safe_gather(*tasks) return {ex: err_msg for ex, err_msg in zip(exchanges, results)} - async def all_balances_all_exchanges(self) -> Dict[str, Dict[str, Decimal]]: - await self.update_exchanges() + async def all_balances_all_exchanges(self, client_config_map: ClientConfigMap) -> Dict[str, Dict[str, Decimal]]: + await self.update_exchanges(client_config_map) return {k: v.get_all_balances() for k, v in sorted(self._markets.items(), key=lambda x: x[0])} def all_available_balances_all_exchanges(self) -> Dict[str, Dict[str, Decimal]]: return {k: v.available_balances for k, v in sorted(self._markets.items(), key=lambda x: x[0])} - async def balances(self, exchange, *symbols) -> Dict[str, Decimal]: - if await self.update_exchange_balance(exchange) is None: + async def balances(self, exchange, client_config_map: ClientConfigMap, *symbols) -> Dict[str, Decimal]: + if await self.update_exchange_balance(exchange, client_config_map) is None: results = {} for token, bal in self.all_balances(exchange).items(): matches = [s for s in symbols if s.lower() == token.lower()] diff --git a/test/hummingbot/client/command/test_config_command.py b/test/hummingbot/client/command/test_config_command.py index 1e751235f9..31ad01cb66 100644 --- a/test/hummingbot/client/command/test_config_command.py +++ b/test/hummingbot/client/command/test_config_command.py @@ -8,7 +8,7 @@ from pydantic import Field -from hummingbot.client.command.config_command import color_settings_to_display, global_configs_to_display +from hummingbot.client.command.config_command import client_configs_to_display, color_settings_to_display from hummingbot.client.config.config_data_types import BaseClientModel, BaseStrategyConfigMap, ClientFieldData from hummingbot.client.config.config_helpers import ClientConfigAdapter, read_system_configs_from_yml from hummingbot.client.config.config_var import ConfigVar @@ -54,10 +54,10 @@ def test_list_configs(self, notify_mock, get_strategy_config_map_mock): global_config_map.clear() global_config_map[tables_format_config_var.key] = tables_format_config_var tables_format_config_var.value = "psql" - global_config_map[global_configs_to_display[0]] = ConfigVar(key=global_configs_to_display[0], prompt="") - global_config_map[global_configs_to_display[0]].value = "first" - global_config_map[global_configs_to_display[1]] = ConfigVar(key=global_configs_to_display[1], prompt="") - global_config_map[global_configs_to_display[1]].value = "second" + global_config_map[client_configs_to_display[0]] = ConfigVar(key=client_configs_to_display[0], prompt="") + global_config_map[client_configs_to_display[0]].value = "first" + global_config_map[client_configs_to_display[1]] = ConfigVar(key=client_configs_to_display[1], prompt="") + global_config_map[client_configs_to_display[1]].value = "second" global_config_map[color_settings_to_display[0]] = ConfigVar(key=color_settings_to_display[0], prompt="") global_config_map[color_settings_to_display[0]].value = "third" global_config_map[color_settings_to_display[1]] = ConfigVar(key=color_settings_to_display[1], prompt="") diff --git a/test/hummingbot/client/ui/test_hummingbot_cli.py b/test/hummingbot/client/ui/test_hummingbot_cli.py index 10398291c0..2de321efad 100644 --- a/test/hummingbot/client/ui/test_hummingbot_cli.py +++ b/test/hummingbot/client/ui/test_hummingbot_cli.py @@ -5,6 +5,7 @@ from prompt_toolkit.widgets import Button from hummingbot.client.config.config_helpers import read_system_configs_from_yml +from hummingbot.client.hummingbot_application import HummingbotApplication from hummingbot.client.tab.data_types import CommandTab from hummingbot.client.ui.custom_widgets import CustomTextArea from hummingbot.client.ui.hummingbot_cli import HummingbotCLI @@ -28,6 +29,7 @@ def setUp(self) -> None: self.mock_hb = MagicMock() self.app = HummingbotCLI(None, None, None, tabs) self.app.app = MagicMock() + self.hb = HummingbotApplication() def test_handle_tab_command_on_close_argument(self): tab = self.app.command_tabs[self.command_name] @@ -110,9 +112,8 @@ def test_tab_navigation(self, mock_vsplit, mock_hsplit, mock_box, moc_cc, moc_fc self.assertFalse(tab1.is_selected) self.assertTrue(tab2.is_selected) - @patch("hummingbot.client.ui.hummingbot_cli.global_config_map") @patch("hummingbot.client.ui.hummingbot_cli.init_logging") - def test_did_start_ui(self, mock_init_logging: MagicMock, mock_config_map: MagicMock): + def test_did_start_ui(self, mock_init_logging: MagicMock): class UIStartHandler(EventListener): def __init__(self): super().__init__() @@ -123,8 +124,7 @@ def __call__(self, _): handler: UIStartHandler = UIStartHandler() self.app.add_listener(HummingbotUIEvent.Start, handler) - self.app.did_start_ui() + self.app.did_start_ui(self.hb) - mock_config_map.get.assert_called() mock_init_logging.assert_called() handler.mock.assert_called() diff --git a/test/hummingbot/connector/test_markets_recorder.py b/test/hummingbot/connector/test_markets_recorder.py index 34610132b4..07c409fe24 100644 --- a/test/hummingbot/connector/test_markets_recorder.py +++ b/test/hummingbot/connector/test_markets_recorder.py @@ -5,6 +5,8 @@ from sqlalchemy import create_engine +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.markets_recorder import MarketsRecorder from hummingbot.core.data_type.common import OrderType, PositionAction, TradeType from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee @@ -35,7 +37,9 @@ def setUp(self, engine_mock) -> None: self.trading_pair = f"{self.base}-{self.quote}" engine_mock.return_value = create_engine("sqlite:///:memory:") - self.manager = SQLConnectionManager(SQLConnectionType.TRADE_FILLS, db_name="test_DB") + self.manager = SQLConnectionManager( + ClientConfigAdapter(ClientConfigMap()), SQLConnectionType.TRADE_FILLS, db_name="test_DB" + ) self.tracking_states = dict() diff --git a/test/hummingbot/strategy/pure_market_making/test_inventory_cost_price_delegate.py b/test/hummingbot/strategy/pure_market_making/test_inventory_cost_price_delegate.py index f562ef6383..ed85a6154b 100644 --- a/test/hummingbot/strategy/pure_market_making/test_inventory_cost_price_delegate.py +++ b/test/hummingbot/strategy/pure_market_making/test_inventory_cost_price_delegate.py @@ -1,14 +1,13 @@ import unittest from decimal import Decimal -from hummingbot.core.event.events import OrderFilledEvent +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee +from hummingbot.core.event.events import OrderFilledEvent from hummingbot.model.inventory_cost import InventoryCost -from hummingbot.model.sql_connection_manager import ( - SQLConnectionManager, - SQLConnectionType, -) +from hummingbot.model.sql_connection_manager import SQLConnectionManager, SQLConnectionType from hummingbot.strategy.pure_market_making.inventory_cost_price_delegate import InventoryCostPriceDelegate @@ -16,7 +15,7 @@ class TestInventoryCostPriceDelegate(unittest.TestCase): @classmethod def setUpClass(cls): cls.trade_fill_sql = SQLConnectionManager( - SQLConnectionType.TRADE_FILLS, db_path="" + ClientConfigAdapter(ClientConfigMap()), SQLConnectionType.TRADE_FILLS, db_path="" ) cls.trading_pair = "BTC-USDT" cls.base_asset, cls.quote_asset = cls.trading_pair.split("-") diff --git a/test/hummingbot/strategy/pure_market_making/test_pmm.py b/test/hummingbot/strategy/pure_market_making/test_pmm.py index 7b51b6bcf9..e445fb2297 100644 --- a/test/hummingbot/strategy/pure_market_making/test_pmm.py +++ b/test/hummingbot/strategy/pure_market_making/test_pmm.py @@ -6,6 +6,8 @@ import pandas as pd from hummingbot.client.command.config_command import ConfigCommand +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode @@ -147,7 +149,7 @@ def setUp(self): volume_step_size=10) self.order_book_asset_del = OrderBookAssetPriceDelegate(self.ext_market, self.trading_pair) trade_fill_sql = SQLConnectionManager( - SQLConnectionType.TRADE_FILLS, db_path="" + ClientConfigAdapter(ClientConfigMap()), SQLConnectionType.TRADE_FILLS, db_path="" ) self.inventory_cost_price_del = InventoryCostPriceDelegate(trade_fill_sql, self.trading_pair) From 7fe3b80f2ee7d5509e4ced34b2a0433bcb2bb2ec Mon Sep 17 00:00:00 2001 From: mhrvth <7762229+mhrvth@users.noreply.github.com> Date: Mon, 23 May 2022 13:48:09 +0200 Subject: [PATCH 113/152] Update hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pyx Co-authored-by: Petio Petrov --- .../cross_exchange_market_making.pyx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pyx b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pyx index d80f517e17..c3234a6b2a 100755 --- a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pyx +++ b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making.pyx @@ -1164,7 +1164,9 @@ cdef class CrossExchangeMarketMakingStrategy(StrategyBase): Return price conversion rate for a taker market (to convert it into maker base asset value) """ market_pair = list(self._market_pairs.values())[0] - _, _, quote_rate, _, _, base_rate = self._config_map.conversion_rate_mode.get_taker_to_maker_conversion_rate(market_pair) + _, _, quote_rate, _, _, base_rate = ( + self._config_map.conversion_rate_mode.get_taker_to_maker_conversion_rate(market_pair) + ) return quote_rate / base_rate # else: # market_pairs = list(self._market_pairs.values())[0] From 0626eb1ae0489ae6d0af556ddced9eafd3e3ace4 Mon Sep 17 00:00:00 2001 From: mhrvth <7762229+mhrvth@users.noreply.github.com> Date: Mon, 23 May 2022 13:49:32 +0200 Subject: [PATCH 114/152] Update hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py Co-authored-by: Petio Petrov --- .../cross_exchange_market_making_config_map_pydantic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py index d705740f54..d6b4faa666 100644 --- a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py @@ -114,7 +114,7 @@ def get_taker_to_maker_conversion_rate( pre=True, ) def validate_decimal(cls, v: str, values: Dict, config: BaseModel.Config, field: Field): - return CrossExchangeMarketMakingConfigMap.validate_decimal(v=v, field=field) + return super().validate_decimal(v=v, field=field) CONVERSION_RATE_MODELS = { From 74dc84e90ac4a5d1a8d8260697b791b1f563862c Mon Sep 17 00:00:00 2001 From: mhrvth <7762229+mhrvth@users.noreply.github.com> Date: Mon, 23 May 2022 13:49:44 +0200 Subject: [PATCH 115/152] Update hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py Co-authored-by: Petio Petrov --- .../cross_exchange_market_making_config_map_pydantic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py index d6b4faa666..bca7b98f1d 100644 --- a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py @@ -170,7 +170,7 @@ def get_expiration_seconds(self) -> Decimal: pre=True, ) def validate_decimal(cls, v: str, values: Dict, config: BaseModel.Config, field: Field): - return CrossExchangeMarketMakingConfigMap.validate_decimal(v=v, field=field) + return super().validate_decimal(v=v, field=field) class ActiveOrderRefreshMode(OrderRefreshMode): From f846b7cf74c0bd6bd65cea9084223a2ebf66e67e Mon Sep 17 00:00:00 2001 From: mhrvth <7762229+mhrvth@users.noreply.github.com> Date: Mon, 23 May 2022 13:51:08 +0200 Subject: [PATCH 116/152] Update hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py Co-authored-by: Petio Petrov --- .../cross_exchange_market_making_config_map_pydantic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py index bca7b98f1d..e584874b3f 100644 --- a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py @@ -96,7 +96,6 @@ def get_taker_to_maker_conversion_rate( :return: A tuple of quote pair symbol, quote conversion rate source, quote conversion rate, base pair symbol, base conversion rate source, base conversion rate """ - quote_rate = Decimal("1") quote_pair = f"{market_pair.taker.quote_asset}-{market_pair.maker.quote_asset}" quote_rate_source = "fixed" quote_rate = self.taker_to_maker_quote_conversion_rate From 276a85f6c5bbf503a5d337d6a7a842c55d6f059a Mon Sep 17 00:00:00 2001 From: mhrvth <7762229+mhrvth@users.noreply.github.com> Date: Mon, 23 May 2022 13:51:22 +0200 Subject: [PATCH 117/152] Update hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py Co-authored-by: Petio Petrov --- .../cross_exchange_market_making_config_map_pydantic.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py index e584874b3f..bbbc4f813c 100644 --- a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py @@ -100,7 +100,6 @@ def get_taker_to_maker_conversion_rate( quote_rate_source = "fixed" quote_rate = self.taker_to_maker_quote_conversion_rate - base_rate = Decimal("1") base_pair = f"{market_pair.taker.base_asset}-{market_pair.maker.base_asset}" base_rate_source = "fixed" base_rate = self.taker_to_maker_base_conversion_rate From 3bd6b415e2a864e9ac9e542189822d577d02d4fb Mon Sep 17 00:00:00 2001 From: mhrvth Date: Mon, 23 May 2022 13:54:42 +0200 Subject: [PATCH 118/152] (feat) fix --- .../cross_exchange_market_making_config_map_pydantic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py index bbbc4f813c..49fdb50fe3 100644 --- a/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py +++ b/hummingbot/strategy/cross_exchange_market_making/cross_exchange_market_making_config_map_pydantic.py @@ -143,7 +143,7 @@ class PassiveOrderRefreshMode(OrderRefreshMode): ), ) - limit_order_min_expiration: float = Field( + limit_order_min_expiration: Decimal = Field( default=130.0, description="Limit order expiration time limit.", gt=0.0, From 741f029c0846db1a4b52ea4e6de3727a0a0c5c90 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Mon, 23 May 2022 14:30:12 +0200 Subject: [PATCH 119/152] (feat) migration script --- hummingbot/client/config/xemm_migration.py | 90 ++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 hummingbot/client/config/xemm_migration.py diff --git a/hummingbot/client/config/xemm_migration.py b/hummingbot/client/config/xemm_migration.py new file mode 100644 index 0000000000..bf8b779947 --- /dev/null +++ b/hummingbot/client/config/xemm_migration.py @@ -0,0 +1,90 @@ +import logging +import shutil +from typing import List + +import yaml + +from hummingbot.client.config.config_helpers import ClientConfigAdapter, save_to_yml +from hummingbot.client.settings import CONF_DIR_PATH +from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_making_config_map_pydantic import ( + CrossExchangeMarketMakingConfigMap, +) + +conf_dir_path = CONF_DIR_PATH + + +def migrate_xemm() -> List[str]: + logging.getLogger().info("Starting conf migration.") + errors = backup_existing_dir() + if len(errors) == 0: + errors.extend(migrate_xemm_confs()) + logging.getLogger().info("\nConf migration done.") + else: + logging.getLogger().error("\nConf migration failed.") + return errors + + +def backup_existing_dir() -> List[str]: + errors = [] + if conf_dir_path.exists(): + backup_path = conf_dir_path.parent / "conf_backup" + if backup_path.exists(): + errors = [ + ( + f"\nBackup path {backup_path} already exists." + f"\nThe migration script cannot backup you existing" + f"\nconf files without overwriting that directory." + f"\nPlease remove it and run the script again." + ) + ] + else: + shutil.copytree(conf_dir_path, backup_path) + logging.getLogger().info(f"\nCreated a backup of your existing conf directory to {backup_path}") + return errors + + +def migrate_xemm_confs(): + errors = [] + logging.getLogger().info("\nMigrating strategies...") + for child in conf_dir_path.iterdir(): + if child.is_file() and child.name.endswith(".yml"): + with open(str(child), "r") as f: + conf = yaml.safe_load(f) + if "strategy" in conf: + if conf["strategy"] == "cross_exchange_market_making": + if "active_order_canceling" in conf: + if conf["active_order_canceling"]: + conf["order_refresh_mode"] = {} + else: + conf["order_refresh_mode"] = { + "cancel_order_threshold": conf["cancel_order_threshold"], + "limit_order_min_expiration": conf["limit_order_min_expiration"] + } + conf.pop("active_order_canceling") + conf.pop("cancel_order_threshold") + conf.pop("limit_order_min_expiration") + + if "use_oracle_conversion_rate" in conf: + if conf["use_oracle_conversion_rate"]: + conf["conversion_rate_mode"] = {} + else: + conf["conversion_rate_mode"] = { + "taker_to_maker_base_conversion_rate": conf["taker_to_maker_base_conversion_rate"], + "taker_to_maker_quote_conversion_rate": conf["taker_to_maker_quote_conversion_rate"] + } + conf.pop("use_oracle_conversion_rate") + conf.pop("taker_to_maker_base_conversion_rate") + conf.pop("taker_to_maker_quote_conversion_rate") + + if "template_version" in conf: + conf.pop("template_version") + + try: + config_map = ClientConfigAdapter(CrossExchangeMarketMakingConfigMap(**conf)) + + save_to_yml(child.absolute(), config_map) + + logging.getLogger().info(f"Migrated conf for {conf['strategy']}") + except Exception as e: + errors.extend((str(e))) + return errors From a4b6b890b7827e324ab91c8b188c253d26ff2709 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Mon, 23 May 2022 18:53:41 +0300 Subject: [PATCH 120/152] (fix) Fixing bugs --- hummingbot/client/config/client_config_map.py | 45 +++++--- hummingbot/client/config/conf_migration.py | 106 +++++++++++++----- hummingbot/client/settings.py | 4 +- hummingbot/client/ui/__init__.py | 5 +- hummingbot/client/ui/hummingbot_cli.py | 1 + 5 files changed, 111 insertions(+), 50 deletions(-) diff --git a/hummingbot/client/config/client_config_map.py b/hummingbot/client/config/client_config_map.py index 6d9a1ce7e8..a2eb46fe11 100644 --- a/hummingbot/client/config/client_config_map.py +++ b/hummingbot/client/config/client_config_map.py @@ -8,7 +8,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Callable, Dict, List, Optional, Union -from pydantic import BaseModel, Field, Json, root_validator, validator +from pydantic import BaseModel, Field, root_validator, validator from tabulate import tabulate_formats from hummingbot.client.config.config_data_types import BaseClientModel, ClientConfigEnum, ClientFieldData @@ -152,7 +152,7 @@ class PaperTradeConfigMap(BaseClientModel): GateIOConfigMap.Config.title, ], ) - paper_trade_account_balance: Json = Field( + paper_trade_account_balance: Dict[str, float] = Field( default={ "BTC": 1, "USDT": 1000, @@ -172,6 +172,12 @@ class PaperTradeConfigMap(BaseClientModel): ), ) + @validator("paper_trade_account_balance", pre=True) + def validate_paper_trade_account_balance(cls, v: Union[str, Dict[str, float]]): + if isinstance(v, str): + v = json.loads(v) + return v + class KillSwitchMode(BaseClientModel, ABC): @abstractmethod @@ -202,7 +208,7 @@ def get_kill_switch(self, hb: "HummingbotApplication") -> ActiveKillSwitch: @validator("kill_switch_rate", pre=True) def validate_decimal(cls, v: str, field: Field): """Used for client-friendly error output.""" - super().validate_decimal(v, field) + return super().validate_decimal(v, field) class KillSwitchDisabledMode(KillSwitchMode): @@ -477,7 +483,7 @@ class CommandsTimeoutConfigMap(BaseClientModel): ) def validate_decimals(cls, v: str, field: Field): """Used for client-friendly error output.""" - super().validate_decimal(v, field) + return super().validate_decimal(v, field) class AnonymizedMetricsMode(BaseClientModel, ABC): @@ -537,7 +543,7 @@ def get_collector( @validator("anonymized_metrics_interval_min", pre=True) def validate_decimal(cls, v: str, field: Field): """Used for client-friendly error output.""" - super().validate_decimal(v, field) + return super().validate_decimal(v, field) METRICS_MODES = { @@ -605,7 +611,7 @@ class ClientConfigMap(BaseClientModel): prompt=lambda cm: f"Select the desired PMM script mode ({'/'.join(list(PMM_SCRIPT_MODES.keys()))})", ), ) - balance_asset_limit: Json = Field( + balance_asset_limit: Dict[str, Dict[str, Decimal]] = Field( default=json.dumps({exchange: None for exchange in AllConnectorSettings.get_exchange_names()}), client_data=ClientFieldData( prompt=lambda cm: ( @@ -674,9 +680,12 @@ class ClientConfigMap(BaseClientModel): paper_trade: PaperTradeConfigMap = Field(default=PaperTradeConfigMap()) color: ColorConfigMap = Field(default=ColorConfigMap()) + class Config: + title = "client_config_map" + @validator("kill_switch_mode", pre=True) - def validate_kill_switch_mode(cls, v: Union[(str,) + tuple(KILL_SWITCH_MODES.values())]): - if isinstance(v, tuple(KILL_SWITCH_MODES.values())): + def validate_kill_switch_mode(cls, v: Union[(str, Dict) + tuple(KILL_SWITCH_MODES.values())]): + if isinstance(v, tuple(KILL_SWITCH_MODES.values()) + (Dict,)): sub_model = v elif v not in KILL_SWITCH_MODES: raise ValueError( @@ -688,13 +697,13 @@ def validate_kill_switch_mode(cls, v: Union[(str,) + tuple(KILL_SWITCH_MODES.val @validator("autofill_import", pre=True) def validate_autofill_import(cls, v: Union[str, AutofillImportEnum]): - if isinstance(v, str) and v not in AutofillImportEnum: + if isinstance(v, str) and v not in AutofillImportEnum.__members__: raise ValueError(f"The value must be one of {', '.join(list(AutofillImportEnum))}.") return v @validator("telegram_mode", pre=True) - def validate_telegram_mode(cls, v: Union[(str,) + tuple(TELEGRAM_MODES.values())]): - if isinstance(v, tuple(TELEGRAM_MODES.values())): + def validate_telegram_mode(cls, v: Union[(str, Dict) + tuple(TELEGRAM_MODES.values())]): + if isinstance(v, tuple(TELEGRAM_MODES.values()) + (Dict,)): sub_model = v elif v not in TELEGRAM_MODES: raise ValueError( @@ -714,8 +723,8 @@ def validate_bool(cls, v: str): return v @validator("db_mode", pre=True) - def validate_db_mode(cls, v: Union[(str,) + tuple(DB_MODES.values())]): - if isinstance(v, tuple(DB_MODES.values())): + def validate_db_mode(cls, v: Union[(str, Dict) + tuple(DB_MODES.values())]): + if isinstance(v, tuple(DB_MODES.values()) + (Dict,)): sub_model = v elif v not in DB_MODES: raise ValueError( @@ -726,8 +735,8 @@ def validate_db_mode(cls, v: Union[(str,) + tuple(DB_MODES.values())]): return sub_model @validator("pmm_script_mode", pre=True) - def validate_pmm_script_mode(cls, v: Union[(str,) + tuple(PMM_SCRIPT_MODES.values())]): - if isinstance(v, tuple(PMM_SCRIPT_MODES.values())): + def validate_pmm_script_mode(cls, v: Union[(str, Dict) + tuple(PMM_SCRIPT_MODES.values())]): + if isinstance(v, tuple(PMM_SCRIPT_MODES.values()) + (Dict,)): sub_model = v elif v not in PMM_SCRIPT_MODES: raise ValueError( @@ -738,8 +747,8 @@ def validate_pmm_script_mode(cls, v: Union[(str,) + tuple(PMM_SCRIPT_MODES.value return sub_model @validator("anonymized_metrics_mode", pre=True) - def validate_anonymized_metrics_mode(cls, v: Union[(str,) + tuple(METRICS_MODES.values())]): - if isinstance(v, tuple(METRICS_MODES.values())): + def validate_anonymized_metrics_mode(cls, v: Union[(str, Dict) + tuple(METRICS_MODES.values())]): + if isinstance(v, tuple(METRICS_MODES.values()) + (Dict,)): sub_model = v elif v not in METRICS_MODES: raise ValueError( @@ -771,7 +780,7 @@ def validate_tables_format(cls, v: str): ) def validate_decimals(cls, v: str, field: Field): """Used for client-friendly error output.""" - super().validate_decimal(v, field) + return super().validate_decimal(v, field) # === post-validations === diff --git a/hummingbot/client/config/conf_migration.py b/hummingbot/client/config/conf_migration.py index 4ae789a39d..b40cc58a86 100644 --- a/hummingbot/client/config/conf_migration.py +++ b/hummingbot/client/config/conf_migration.py @@ -4,7 +4,7 @@ import shutil from os import DirEntry, scandir from os.path import exists, join -from typing import Dict, List, Union, cast +from typing import Any, Dict, List, Optional, Union, cast import yaml @@ -25,9 +25,9 @@ ) from hummingbot.client.config.config_crypt import BaseSecretsManager, store_password_verification from hummingbot.client.config.config_data_types import BaseConnectorConfigMap -from hummingbot.client.config.config_helpers import ClientConfigAdapter +from hummingbot.client.config.config_helpers import ClientConfigAdapter, save_to_yml from hummingbot.client.config.security import Security -from hummingbot.client.settings import CONF_DIR_PATH, STRATEGIES_CONF_DIR_PATH +from hummingbot.client.settings import CLIENT_CONFIG_PATH, CONF_DIR_PATH, STRATEGIES_CONF_DIR_PATH encrypted_conf_prefix = "encrypted_" encrypted_conf_postfix = ".json" @@ -83,6 +83,7 @@ def backup_existing_dir() -> List[str]: def migrate_global_config() -> List[str]: + logging.getLogger().info("\nMigrating the global config...") global_config_path = CONF_DIR_PATH / "conf_global.yml" errors = [] if global_config_path.exists(): @@ -91,14 +92,24 @@ def migrate_global_config() -> List[str]: del data["template_version"] client_config_map = ClientConfigAdapter(ClientConfigMap()) migrate_global_config_modes(client_config_map, data) - for key, value in data.items(): + keys = list(data.keys()) + for key in keys: if key not in client_config_map.keys(): errors.append( f"Could not match the attribute {key} from the legacy config file to the new config map." ) - continue - if value is not None: - client_config_map.setattr_no_validation(key, value) + else: + migrate_global_config_field(client_config_map, data, key) + for key in data: + errors.append(f"ConfigVar {key} was not migrated.") + errors.extend(client_config_map.validate_model()) + if len(errors) == 0: + save_to_yml(CLIENT_CONFIG_PATH, client_config_map) + global_config_path.unlink() + logging.getLogger().info("\nSuccessfully migrated the global config.") + else: + errors = [f"client_config_map - {e}" for e in errors] + logging.getLogger().error(f"The migration of the global config map failed with errors: {errors}") return errors @@ -112,8 +123,12 @@ def migrate_global_config_modes(client_config_map: ClientConfigAdapter, data: Di else: client_config_map.kill_switch_mode = KillSwitchDisabledMode() - client_config_map.paper_trade.paper_trade_exchanges = data.pop("paper_trade_exchanges") - client_config_map.paper_trade.paper_trade_account_balance = data.pop("paper_trade_account_balance") + migrate_global_config_field( + client_config_map.paper_trade, data, "paper_trade_exchanges" + ) + migrate_global_config_field( + client_config_map.paper_trade, data, "paper_trade_account_balance" + ) telegram_enabled = data.pop("telegram_enabled") telegram_token = data.pop("telegram_token") @@ -151,8 +166,12 @@ def migrate_global_config_modes(client_config_map: ClientConfigAdapter, data: Di else: client_config_map.pmm_script_mode = PMMScriptDisabledMode() - client_config_map.gateway.gateway_api_host = data.pop("gateway_api_host") - client_config_map.gateway.gateway_api_port = data.pop("gateway_api_port") + migrate_global_config_field( + client_config_map.gateway, data, "gateway_api_host" + ) + migrate_global_config_field( + client_config_map.gateway, data, "gateway_api_port" + ) anonymized_metrics_enabled = data.pop("anonymized_metrics_enabled") anonymized_metrics_interval_min = data.pop("anonymized_metrics_interval_min") @@ -163,25 +182,55 @@ def migrate_global_config_modes(client_config_map: ClientConfigAdapter, data: Di else: client_config_map.anonymized_metrics_mode = AnonymizedMetricsDisabledMode() - client_config_map.global_token.global_token_name = data.pop("global_token") - client_config_map.global_token.global_token_symbol = data.pop("global_token_symbol") + migrate_global_config_field( + client_config_map.global_token, data, "global_token", "global_token_name" + ) + migrate_global_config_field( + client_config_map.global_token, data, "global_token_symbol" + ) + + migrate_global_config_field( + client_config_map.commands_timeout, data, "create_command_timeout" + ) + migrate_global_config_field( + client_config_map.commands_timeout, data, "other_commands_timeout" + ) - client_config_map.commands_timeout.create_command_timeout = data.pop("create_command_timeout") - client_config_map.commands_timeout.other_commands_timeout = data.pop("other_commands_timeout") + color_map: Union[ClientConfigAdapter, ColorConfigMap] = client_config_map.color + migrate_global_config_field(color_map, data, "top-pane", "top_pane") + migrate_global_config_field(color_map, data, "bottom-pane", "bottom_pane") + migrate_global_config_field(color_map, data, "output-pane", "output_pane") + migrate_global_config_field(color_map, data, "input-pane", "input_pane") + migrate_global_config_field(color_map, data, "logs-pane", "logs_pane") + migrate_global_config_field(color_map, data, "terminal-primary", "terminal_primary") + migrate_global_config_field(color_map, data, "primary-label", "primary_label") + migrate_global_config_field(color_map, data, "secondary-label", "secondary_label") + migrate_global_config_field(color_map, data, "success-label", "success_label") + migrate_global_config_field(color_map, data, "warning-label", "warning_label") + migrate_global_config_field(color_map, data, "info-label", "info_label") + migrate_global_config_field(color_map, data, "error-label", "error_label") + + balance_asset_limit = data.pop("balance_asset_limit") + if balance_asset_limit is not None: + exchanges = list(balance_asset_limit.keys()) + for e in exchanges: + if balance_asset_limit[e] is None: + balance_asset_limit.pop(e) + else: + assets = balance_asset_limit[e].keys() + for a in assets: + if balance_asset_limit[e][a] is None: + balance_asset_limit[e].pop(a) + client_config_map.balance_asset_limit = balance_asset_limit - color_map: ColorConfigMap = client_config_map.color - color_map.top_pane = data.pop("top-pane") - color_map.bottom_pane = data.pop("bottom-pane") - color_map.output_pane = data.pop("output-pane") - color_map.input_pane = data.pop("input-pane") - color_map.logs_pane = data.pop("logs-pane") - color_map.terminal_primary = data.pop("terminal-primary") - color_map.primary_label = data.pop("primary-label") - color_map.secondary_label = data.pop("secondary-label") - color_map.success_label = data.pop("success-label") - color_map.warning_label = data.pop("warning-label") - color_map.info_label = data.pop("info-label") - color_map.error_label = data.pop("error-label") + +def migrate_global_config_field( + cm: ClientConfigAdapter, global_config_data: Dict[str, Any], attr: str, cm_attr: Optional[str] = None +): + value = global_config_data.pop(attr) + cm_attr = cm_attr if cm_attr is not None else attr + if value is not None: + cm.setattr_no_validation(cm_attr, value) def migrate_strategy_confs_paths(): @@ -191,6 +240,7 @@ def migrate_strategy_confs_paths(): with open(str(child), "r") as f: conf = yaml.safe_load(f) if "strategy" in conf and _has_connector_field(conf): + new_path = strategies_conf_dir_path / child.name child.rename(new_path) logging.getLogger().info(f"Migrated conf for {conf['strategy']}") diff --git a/hummingbot/client/settings.py b/hummingbot/client/settings.py index ce5e183ab0..725e9fdae4 100644 --- a/hummingbot/client/settings.py +++ b/hummingbot/client/settings.py @@ -28,12 +28,12 @@ KEYFILE_PREFIX = "key_file_" KEYFILE_POSTFIX = ".yml" ENCYPTED_CONF_POSTFIX = ".json" -CLIENT_CONFIG_PATH = root_path() / "conf" / "conf_client.yml" -TRADE_FEES_CONFIG_PATH = root_path() / "conf" / "conf_fee_overrides.yml" DEFAULT_LOG_FILE_PATH = root_path() / "logs" DEFAULT_ETHEREUM_RPC_URL = "https://mainnet.coinalpha.com/hummingbot-test-node" TEMPLATE_PATH = root_path() / "hummingbot" / "templates" CONF_DIR_PATH = root_path() / "conf" +CLIENT_CONFIG_PATH = CONF_DIR_PATH / "conf_client.yml" +TRADE_FEES_CONFIG_PATH = CONF_DIR_PATH / "conf_fee_overrides.yml" STRATEGIES_CONF_DIR_PATH = CONF_DIR_PATH / "strategies" CONNECTORS_CONF_DIR_PATH = CONF_DIR_PATH / "connectors" CONF_PREFIX = "conf_" diff --git a/hummingbot/client/ui/__init__.py b/hummingbot/client/ui/__init__.py index 3f21ea7f9e..fc480b1a99 100644 --- a/hummingbot/client/ui/__init__.py +++ b/hummingbot/client/ui/__init__.py @@ -63,7 +63,7 @@ def login_prompt(secrets_manager_cls: Type[BaseSecretsManager]) -> Optional[Base else: secrets_manager = secrets_manager_cls(password) store_password_verification(secrets_manager) - migrate_non_secure_only_prompt() + migrate_non_secure_only_prompt() else: password = input_dialog( title="Welcome back to Hummingbot", @@ -167,7 +167,8 @@ def migrate_non_secure_only_prompt(): def _migration_errors_dialog(errors): - errors_str = "\n ".join(errors) + padding = "\n " + errors_str = padding + padding.join(errors) message_dialog( title='Configs Migration Errors', text=f""" diff --git a/hummingbot/client/ui/hummingbot_cli.py b/hummingbot/client/ui/hummingbot_cli.py index 9fb1e4bf5b..29ccf47fa6 100644 --- a/hummingbot/client/ui/hummingbot_cli.py +++ b/hummingbot/client/ui/hummingbot_cli.py @@ -116,6 +116,7 @@ async def run(self): clipboard=PyperclipClipboard(), ) await self.app.run_async(pre_run=self.did_start_ui) + # await self.app.run_async(pre_run=partial(self.did_start_ui, self.app)) self._stdout_redirect_context.close() def accept(self, buff): From b8f042858681a430fd47d01735cea1bb8439a532 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Mon, 23 May 2022 19:20:14 +0300 Subject: [PATCH 121/152] (fix) Fixing more bugs --- hummingbot/client/config/client_config_map.py | 4 ++-- hummingbot/client/config/config_helpers.py | 12 +++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/hummingbot/client/config/client_config_map.py b/hummingbot/client/config/client_config_map.py index a2eb46fe11..fbf5ff48ad 100644 --- a/hummingbot/client/config/client_config_map.py +++ b/hummingbot/client/config/client_config_map.py @@ -611,7 +611,7 @@ class ClientConfigMap(BaseClientModel): prompt=lambda cm: f"Select the desired PMM script mode ({'/'.join(list(PMM_SCRIPT_MODES.keys()))})", ), ) - balance_asset_limit: Dict[str, Dict[str, Decimal]] = Field( + balance_asset_limit: Dict[str, Dict[str, Decimal]] = Field( # todo: validator that can handle inputs default=json.dumps({exchange: None for exchange in AllConnectorSettings.get_exchange_names()}), client_data=ClientFieldData( prompt=lambda cm: ( @@ -633,7 +633,7 @@ class ClientConfigMap(BaseClientModel): prompt=lambda cm: f"Select the desired metrics mode ({'/'.join(list(METRICS_MODES.keys()))})", ), ) - command_shortcuts: List[CommandShortcutModel] = Field( # todo: test it + command_shortcuts: List[CommandShortcutModel] = Field( # todo: test it — does it load properly? default=[ CommandShortcutModel( command="spreads", diff --git a/hummingbot/client/config/config_helpers.py b/hummingbot/client/config/config_helpers.py index 82d0b223fd..54c5e5f89f 100644 --- a/hummingbot/client/config/config_helpers.py +++ b/hummingbot/client/config/config_helpers.py @@ -9,7 +9,7 @@ from decimal import Decimal from os import listdir, scandir, unlink from os.path import isfile, join -from pathlib import Path +from pathlib import Path, PosixPath from typing import Any, Callable, Dict, Generator, List, Optional, Type, Union import ruamel.yaml @@ -61,6 +61,10 @@ def datetime_representer(dumper: SafeDumper, data: datetime): return dumper.represent_datetime(data) +def path_representer(dumper: SafeDumper, data: Path): + return dumper.represent_str(str(data)) + + yaml.add_representer( data_type=Decimal, representer=decimal_representer, Dumper=SafeDumper ) @@ -76,6 +80,12 @@ def datetime_representer(dumper: SafeDumper, data: datetime): yaml.add_representer( data_type=datetime, representer=datetime_representer, Dumper=SafeDumper ) +yaml.add_representer( + data_type=Path, representer=path_representer, Dumper=SafeDumper +) +yaml.add_representer( + data_type=PosixPath, representer=path_representer, Dumper=SafeDumper +) class ConfigValidationError(Exception): From 962bcc7f477ff1b4b28f75bbd20ca3cfbf1aeae7 Mon Sep 17 00:00:00 2001 From: mhrvth Date: Mon, 23 May 2022 22:58:21 +0200 Subject: [PATCH 122/152] (feat) migration script improvements --- hummingbot/client/config/conf_migration.py | 2 ++ hummingbot/client/config/xemm_migration.py | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/hummingbot/client/config/conf_migration.py b/hummingbot/client/config/conf_migration.py index 11f69c4886..057e284b05 100644 --- a/hummingbot/client/config/conf_migration.py +++ b/hummingbot/client/config/conf_migration.py @@ -13,6 +13,7 @@ from hummingbot.client.config.config_data_types import BaseConnectorConfigMap from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.client.config.security import Security +from hummingbot.client.config.xemm_migration import migrate_xemm_confs from hummingbot.client.settings import CONF_DIR_PATH, STRATEGIES_CONF_DIR_PATH encrypted_conf_prefix = "encrypted_" @@ -28,6 +29,7 @@ def migrate_configs(secrets_manager: BaseSecretsManager) -> List[str]: migrate_strategy_confs_paths() errors.extend(migrate_connector_confs(secrets_manager)) store_password_verification(secrets_manager) + migrate_xemm_confs() logging.getLogger().info("\nConf migration done.") else: logging.getLogger().error("\nConf migration failed.") diff --git a/hummingbot/client/config/xemm_migration.py b/hummingbot/client/config/xemm_migration.py index bf8b779947..988515ea08 100644 --- a/hummingbot/client/config/xemm_migration.py +++ b/hummingbot/client/config/xemm_migration.py @@ -5,12 +5,13 @@ import yaml from hummingbot.client.config.config_helpers import ClientConfigAdapter, save_to_yml -from hummingbot.client.settings import CONF_DIR_PATH +from hummingbot.client.settings import CONF_DIR_PATH, STRATEGIES_CONF_DIR_PATH from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_making_config_map_pydantic import ( CrossExchangeMarketMakingConfigMap, ) conf_dir_path = CONF_DIR_PATH +conf_dir_path_strategy = STRATEGIES_CONF_DIR_PATH def migrate_xemm() -> List[str]: @@ -46,7 +47,7 @@ def backup_existing_dir() -> List[str]: def migrate_xemm_confs(): errors = [] logging.getLogger().info("\nMigrating strategies...") - for child in conf_dir_path.iterdir(): + for child in conf_dir_path_strategy.iterdir(): if child.is_file() and child.name.endswith(".yml"): with open(str(child), "r") as f: conf = yaml.safe_load(f) From 664fea8efe8b6e9d69db9324dd2bdc39839890ad Mon Sep 17 00:00:00 2001 From: mhrvth Date: Tue, 24 May 2022 14:00:34 +0200 Subject: [PATCH 123/152] (feat) migration script refactor --- hummingbot/client/config/conf_migration.py | 54 ++++++++++++- hummingbot/client/config/xemm_migration.py | 91 ---------------------- 2 files changed, 51 insertions(+), 94 deletions(-) delete mode 100644 hummingbot/client/config/xemm_migration.py diff --git a/hummingbot/client/config/conf_migration.py b/hummingbot/client/config/conf_migration.py index 057e284b05..f89c3f3a66 100644 --- a/hummingbot/client/config/conf_migration.py +++ b/hummingbot/client/config/conf_migration.py @@ -11,10 +11,12 @@ from hummingbot import root_path from hummingbot.client.config.config_crypt import BaseSecretsManager, store_password_verification from hummingbot.client.config.config_data_types import BaseConnectorConfigMap -from hummingbot.client.config.config_helpers import ClientConfigAdapter +from hummingbot.client.config.config_helpers import ClientConfigAdapter, save_to_yml from hummingbot.client.config.security import Security -from hummingbot.client.config.xemm_migration import migrate_xemm_confs from hummingbot.client.settings import CONF_DIR_PATH, STRATEGIES_CONF_DIR_PATH +from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_making_config_map_pydantic import ( + CrossExchangeMarketMakingConfigMap, +) encrypted_conf_prefix = "encrypted_" encrypted_conf_postfix = ".json" @@ -29,7 +31,6 @@ def migrate_configs(secrets_manager: BaseSecretsManager) -> List[str]: migrate_strategy_confs_paths() errors.extend(migrate_connector_confs(secrets_manager)) store_password_verification(secrets_manager) - migrate_xemm_confs() logging.getLogger().info("\nConf migration done.") else: logging.getLogger().error("\nConf migration failed.") @@ -75,9 +76,56 @@ def migrate_strategy_confs_paths(): if "strategy" in conf and _has_connector_field(conf): new_path = strategies_conf_dir_path / child.name child.rename(new_path) + if conf["strategy"] == "cross_exchange_market_making": + migrate_xemm_confs() logging.getLogger().info(f"Migrated conf for {conf['strategy']}") +def migrate_xemm_confs(): + logging.getLogger().info("\nMigrating strategies...") + for child in strategies_conf_dir_path.iterdir(): + if child.is_file() and child.name.endswith(".yml"): + with open(str(child), "r") as f: + conf = yaml.safe_load(f) + if "strategy" in conf: + if conf["strategy"] == "cross_exchange_market_making": + if "active_order_canceling" in conf: + if conf["active_order_canceling"]: + conf["order_refresh_mode"] = {} + else: + conf["order_refresh_mode"] = { + "cancel_order_threshold": conf["cancel_order_threshold"], + "limit_order_min_expiration": conf["limit_order_min_expiration"] + } + conf.pop("active_order_canceling") + conf.pop("cancel_order_threshold") + conf.pop("limit_order_min_expiration") + + if "use_oracle_conversion_rate" in conf: + if conf["use_oracle_conversion_rate"]: + conf["conversion_rate_mode"] = {} + else: + conf["conversion_rate_mode"] = { + "taker_to_maker_base_conversion_rate": conf["taker_to_maker_base_conversion_rate"], + "taker_to_maker_quote_conversion_rate": conf["taker_to_maker_quote_conversion_rate"] + } + conf.pop("use_oracle_conversion_rate") + conf.pop("taker_to_maker_base_conversion_rate") + conf.pop("taker_to_maker_quote_conversion_rate") + + if "template_version" in conf: + conf.pop("template_version") + + try: + config_map = ClientConfigAdapter(CrossExchangeMarketMakingConfigMap(**conf)) + + save_to_yml(child.absolute(), config_map) + + logging.getLogger().info(f"Migrated conf for {conf['strategy']}") + except Exception as e: + logging.getLogger().error(str(e)) + + def _has_connector_field(conf: Dict) -> bool: return ( "exchange" in conf diff --git a/hummingbot/client/config/xemm_migration.py b/hummingbot/client/config/xemm_migration.py deleted file mode 100644 index 988515ea08..0000000000 --- a/hummingbot/client/config/xemm_migration.py +++ /dev/null @@ -1,91 +0,0 @@ -import logging -import shutil -from typing import List - -import yaml - -from hummingbot.client.config.config_helpers import ClientConfigAdapter, save_to_yml -from hummingbot.client.settings import CONF_DIR_PATH, STRATEGIES_CONF_DIR_PATH -from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_making_config_map_pydantic import ( - CrossExchangeMarketMakingConfigMap, -) - -conf_dir_path = CONF_DIR_PATH -conf_dir_path_strategy = STRATEGIES_CONF_DIR_PATH - - -def migrate_xemm() -> List[str]: - logging.getLogger().info("Starting conf migration.") - errors = backup_existing_dir() - if len(errors) == 0: - errors.extend(migrate_xemm_confs()) - logging.getLogger().info("\nConf migration done.") - else: - logging.getLogger().error("\nConf migration failed.") - return errors - - -def backup_existing_dir() -> List[str]: - errors = [] - if conf_dir_path.exists(): - backup_path = conf_dir_path.parent / "conf_backup" - if backup_path.exists(): - errors = [ - ( - f"\nBackup path {backup_path} already exists." - f"\nThe migration script cannot backup you existing" - f"\nconf files without overwriting that directory." - f"\nPlease remove it and run the script again." - ) - ] - else: - shutil.copytree(conf_dir_path, backup_path) - logging.getLogger().info(f"\nCreated a backup of your existing conf directory to {backup_path}") - return errors - - -def migrate_xemm_confs(): - errors = [] - logging.getLogger().info("\nMigrating strategies...") - for child in conf_dir_path_strategy.iterdir(): - if child.is_file() and child.name.endswith(".yml"): - with open(str(child), "r") as f: - conf = yaml.safe_load(f) - if "strategy" in conf: - if conf["strategy"] == "cross_exchange_market_making": - if "active_order_canceling" in conf: - if conf["active_order_canceling"]: - conf["order_refresh_mode"] = {} - else: - conf["order_refresh_mode"] = { - "cancel_order_threshold": conf["cancel_order_threshold"], - "limit_order_min_expiration": conf["limit_order_min_expiration"] - } - conf.pop("active_order_canceling") - conf.pop("cancel_order_threshold") - conf.pop("limit_order_min_expiration") - - if "use_oracle_conversion_rate" in conf: - if conf["use_oracle_conversion_rate"]: - conf["conversion_rate_mode"] = {} - else: - conf["conversion_rate_mode"] = { - "taker_to_maker_base_conversion_rate": conf["taker_to_maker_base_conversion_rate"], - "taker_to_maker_quote_conversion_rate": conf["taker_to_maker_quote_conversion_rate"] - } - conf.pop("use_oracle_conversion_rate") - conf.pop("taker_to_maker_base_conversion_rate") - conf.pop("taker_to_maker_quote_conversion_rate") - - if "template_version" in conf: - conf.pop("template_version") - - try: - config_map = ClientConfigAdapter(CrossExchangeMarketMakingConfigMap(**conf)) - - save_to_yml(child.absolute(), config_map) - - logging.getLogger().info(f"Migrated conf for {conf['strategy']}") - except Exception as e: - errors.extend((str(e))) - return errors From cb3568154c6642c626ff23194323f3a7c25de7bb Mon Sep 17 00:00:00 2001 From: mhrvth Date: Tue, 24 May 2022 14:01:19 +0200 Subject: [PATCH 124/152] (feat) remove double logging --- hummingbot/client/config/conf_migration.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/hummingbot/client/config/conf_migration.py b/hummingbot/client/config/conf_migration.py index f89c3f3a66..4ffc9bfdf4 100644 --- a/hummingbot/client/config/conf_migration.py +++ b/hummingbot/client/config/conf_migration.py @@ -82,7 +82,6 @@ def migrate_strategy_confs_paths(): def migrate_xemm_confs(): - logging.getLogger().info("\nMigrating strategies...") for child in strategies_conf_dir_path.iterdir(): if child.is_file() and child.name.endswith(".yml"): with open(str(child), "r") as f: @@ -120,8 +119,6 @@ def migrate_xemm_confs(): config_map = ClientConfigAdapter(CrossExchangeMarketMakingConfigMap(**conf)) save_to_yml(child.absolute(), config_map) - - logging.getLogger().info(f"Migrated conf for {conf['strategy']}") except Exception as e: logging.getLogger().error(str(e)) From eba99d47ae67cf52514a9c10c7e84b01250a073b Mon Sep 17 00:00:00 2001 From: mhrvth <7762229+mhrvth@users.noreply.github.com> Date: Tue, 24 May 2022 18:09:17 +0200 Subject: [PATCH 125/152] Update hummingbot/client/config/conf_migration.py Co-authored-by: Petio Petrov --- hummingbot/client/config/conf_migration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/client/config/conf_migration.py b/hummingbot/client/config/conf_migration.py index 4ffc9bfdf4..b8341c5487 100644 --- a/hummingbot/client/config/conf_migration.py +++ b/hummingbot/client/config/conf_migration.py @@ -28,7 +28,7 @@ def migrate_configs(secrets_manager: BaseSecretsManager) -> List[str]: logging.getLogger().info("Starting conf migration.") errors = backup_existing_dir() if len(errors) == 0: - migrate_strategy_confs_paths() + errors.extend(migrate_strategy_confs_paths()) errors.extend(migrate_connector_confs(secrets_manager)) store_password_verification(secrets_manager) logging.getLogger().info("\nConf migration done.") From e5133d20203ca74de33a2b827af95a58f42ab7bc Mon Sep 17 00:00:00 2001 From: mhrvth Date: Tue, 24 May 2022 18:18:03 +0200 Subject: [PATCH 126/152] (feat) improvements --- hummingbot/client/config/conf_migration.py | 79 ++++++++++------------ 1 file changed, 37 insertions(+), 42 deletions(-) diff --git a/hummingbot/client/config/conf_migration.py b/hummingbot/client/config/conf_migration.py index b8341c5487..eba77cb243 100644 --- a/hummingbot/client/config/conf_migration.py +++ b/hummingbot/client/config/conf_migration.py @@ -41,7 +41,7 @@ def migrate_strategies_only() -> List[str]: logging.getLogger().info("Starting strategies conf migration.") errors = backup_existing_dir() if len(errors) == 0: - migrate_strategy_confs_paths() + errors.extend(migrate_strategy_confs_paths()) logging.getLogger().info("\nConf migration done.") else: logging.getLogger().error("\nConf migration failed.") @@ -68,6 +68,7 @@ def backup_existing_dir() -> List[str]: def migrate_strategy_confs_paths(): + errors = [] logging.getLogger().info("\nMigrating strategies...") for child in conf_dir_path.iterdir(): if child.is_file() and child.name.endswith(".yml"): @@ -77,50 +78,44 @@ def migrate_strategy_confs_paths(): new_path = strategies_conf_dir_path / child.name child.rename(new_path) if conf["strategy"] == "cross_exchange_market_making": - migrate_xemm_confs() + errors.extend(migrate_xemm_confs(conf, new_path)) logging.getLogger().info(f"Migrated conf for {conf['strategy']}") + return errors -def migrate_xemm_confs(): - for child in strategies_conf_dir_path.iterdir(): - if child.is_file() and child.name.endswith(".yml"): - with open(str(child), "r") as f: - conf = yaml.safe_load(f) - if "strategy" in conf: - if conf["strategy"] == "cross_exchange_market_making": - if "active_order_canceling" in conf: - if conf["active_order_canceling"]: - conf["order_refresh_mode"] = {} - else: - conf["order_refresh_mode"] = { - "cancel_order_threshold": conf["cancel_order_threshold"], - "limit_order_min_expiration": conf["limit_order_min_expiration"] - } - conf.pop("active_order_canceling") - conf.pop("cancel_order_threshold") - conf.pop("limit_order_min_expiration") - - if "use_oracle_conversion_rate" in conf: - if conf["use_oracle_conversion_rate"]: - conf["conversion_rate_mode"] = {} - else: - conf["conversion_rate_mode"] = { - "taker_to_maker_base_conversion_rate": conf["taker_to_maker_base_conversion_rate"], - "taker_to_maker_quote_conversion_rate": conf["taker_to_maker_quote_conversion_rate"] - } - conf.pop("use_oracle_conversion_rate") - conf.pop("taker_to_maker_base_conversion_rate") - conf.pop("taker_to_maker_quote_conversion_rate") - - if "template_version" in conf: - conf.pop("template_version") - - try: - config_map = ClientConfigAdapter(CrossExchangeMarketMakingConfigMap(**conf)) - - save_to_yml(child.absolute(), config_map) - except Exception as e: - logging.getLogger().error(str(e)) +def migrate_xemm_confs(conf, new_path) -> List[str]: + if "active_order_canceling" in conf: + if conf["active_order_canceling"]: + conf["order_refresh_mode"] = {} + else: + conf["order_refresh_mode"] = { + "cancel_order_threshold": conf["cancel_order_threshold"], + "limit_order_min_expiration": conf["limit_order_min_expiration"] + } + conf.pop("active_order_canceling") + conf.pop("cancel_order_threshold") + conf.pop("limit_order_min_expiration") + if "use_oracle_conversion_rate" in conf: + if conf["use_oracle_conversion_rate"]: + conf["conversion_rate_mode"] = {} + else: + conf["conversion_rate_mode"] = { + "taker_to_maker_base_conversion_rate": conf["taker_to_maker_base_conversion_rate"], + "taker_to_maker_quote_conversion_rate": conf["taker_to_maker_quote_conversion_rate"] + } + conf.pop("use_oracle_conversion_rate") + conf.pop("taker_to_maker_base_conversion_rate") + conf.pop("taker_to_maker_quote_conversion_rate") + if "template_version" in conf: + conf.pop("template_version") + try: + config_map = ClientConfigAdapter(CrossExchangeMarketMakingConfigMap(**conf)) + save_to_yml(new_path, config_map) + errors = [] + except Exception as e: + logging.getLogger().error(str(e)) + errors = [str(e)] + return errors def _has_connector_field(conf: Dict) -> bool: From a6e4da633016e38c49b701b064220ddfdf953507 Mon Sep 17 00:00:00 2001 From: abel Date: Tue, 31 May 2022 17:54:11 -0300 Subject: [PATCH 127/152] (feat) Fix all failing unit tests after refactoring to transfor client_config_map to Pydantic configuration Since all connectors have now a new parameter (the client config), all connectors' unit tests had to be updated to instantiate them passing the new configuration object. Added a new ReadOnly configuration adapter to be used in the connectors, to ensure no connector will change the client configuration by mistake. --- bin/conf_migration_script.py | 2 + hummingbot/client/command/balance_command.py | 2 +- hummingbot/client/command/config_command.py | 33 ++-- hummingbot/client/command/connect_command.py | 8 +- hummingbot/client/command/create_command.py | 2 +- hummingbot/client/command/status_command.py | 5 +- hummingbot/client/config/client_config_map.py | 95 +++++++--- hummingbot/client/config/conf_migration.py | 8 +- hummingbot/client/config/config_data_types.py | 24 ++- hummingbot/client/config/config_helpers.py | 71 +++++--- hummingbot/client/hummingbot_application.py | 12 +- hummingbot/client/ui/__init__.py | 2 +- hummingbot/client/ui/custom_widgets.py | 11 +- hummingbot/client/ui/hummingbot_cli.py | 4 +- hummingbot/client/ui/layout.py | 22 +-- hummingbot/client/ui/style.py | 56 +++--- .../binance_perpetual_derivative.py | 10 +- .../bybit_perpetual_derivative.py | 19 +- .../dydx_perpetual_derivative.py | 26 +-- .../exchange/ascend_ex/ascend_ex_exchange.py | 11 +- .../exchange/kucoin/kucoin_exchange.py | 12 +- .../connector/exchange/mexc/mexc_exchange.py | 10 +- hummingbot/connector/gateway_EVM_AMM.py | 2 +- .../mock_paper_exchange.pyx | 14 +- .../api_throttler/async_throttler_base.py | 11 +- .../cross_exchange_market_making/start.py | 2 +- .../strategy/pure_market_making/start.py | 23 +-- hummingbot/user/user_balances.py | 6 +- .../client/command/test_balance_command.py | 15 +- .../client/command/test_config_command.py | 79 ++++---- .../client/command/test_connect_command.py | 23 +-- .../client/command/test_create_command.py | 30 ++-- .../client/command/test_history_command.py | 18 +- .../client/command/test_order_book_command.py | 21 +-- .../client/command/test_status_command.py | 16 +- .../client/command/test_ticker_command.py | 21 +-- .../client/config/test_config_data_types.py | 104 ++++++----- .../client/config/test_config_helpers.py | 74 +++++++- .../client/config/test_config_templates.py | 72 -------- .../client/ui/test_custom_widgets.py | 14 +- .../client/ui/test_hummingbot_cli.py | 13 +- .../client/ui/test_interface_utils.py | 31 ++-- test/hummingbot/client/ui/test_layout.py | 17 +- test/hummingbot/client/ui/test_style.py | 169 +++++++++--------- .../connector/gateway/test_gateway_cancel.py | 32 ++-- .../connector/gateway/test_gateway_evm_amm.py | 18 +- .../test_binance_perpetual_derivative.py | 21 +-- .../test_bybit_perpetual_derivative.py | 87 +++++---- .../test_dydx_perpetual_derivative.py | 13 +- .../test_perpetual_budget_checker.py | 12 +- .../altmarkets/test_altmarkets_exchange.py | 7 +- .../ascend_ex/test_ascend_ex_exchange.py | 18 +- .../exchange/binance/test_binance_exchange.py | 4 + .../bitfinex/test_bitfinex_exchange.py | 4 + .../exchange/bitmart/test_bitmart_exchange.py | 21 +-- .../exchange/bittrex/test_bittrex_exchange.py | 14 +- .../test_coinbase_pro_exchange.py | 15 +- .../test_coingbase_pro_exchange.py | 4 + .../coinflex/test_coinflex_exchange.py | 8 +- .../coinzoom/test_coinzoom_exchange.py | 5 + .../exchange/ftx/test_ftx_exchange.py | 9 +- .../exchange/gate_io/test_gate_io_exchange.py | 9 +- .../exchange/huobi/test_huobi_exchange.py | 11 +- .../exchange/kraken/test_kraken_exchange.py | 7 +- .../exchange/kucoin/test_kucoin_exchange.py | 14 +- .../exchange/liquid/test_liquid_exchange.py | 9 +- .../exchange/mexc/test_mexc_exchange.py | 13 +- .../exchange/ndax/test_ndax_exchange.py | 24 +-- .../exchange/okex/test_okex_exchange.py | 9 +- .../paper_trade/test_paper_trade_exchange.py | 12 +- .../exchange/probit/test_probit_exchange.py | 11 +- .../connector/test_budget_checker.py | 34 +++- .../connector/test_client_order_tracker.py | 4 +- .../connector/test_connector_base.py | 4 +- .../test_connector_metrics_collector.py | 74 ++------ .../connector/test_markets_recorder.py | 2 +- .../api_throttler/test_async_throttler.py | 24 ++- .../debug_collect_gatewy_test_samples.py | 16 +- test/hummingbot/core/utils/test_ssl_cert.py | 21 ++- .../strategy/amm_arb/test_amm_arb.py | 29 +-- .../hummingbot/strategy/amm_arb/test_utils.py | 18 +- .../strategy/arbitrage/test_arbitrage.py | 6 +- .../arbitrage/test_arbitrage_start.py | 14 +- .../test_avellaneda_market_making.py | 5 +- .../test_avellaneda_market_making_start.py | 3 +- .../strategy/celo_arb/test_celo_arb.py | 4 +- .../strategy/celo_arb/test_celo_arb_start.py | 9 +- .../test_cross_exchange_market_making.py | 20 ++- ...test_cross_exchange_market_making_start.py | 17 +- .../test_dev_0_hello_world.py | 6 +- .../test_dev_1_get_order_book.py | 6 +- .../test_dev_2_perform_trade.py | 6 +- .../strategy/dev_5_vwap/test_vwap.py | 6 +- .../dev_simple_trade/test_simple_trade.py | 6 +- .../liquidity_mining/test_liquidity_mining.py | 6 +- .../test_liquidity_mining_start.py | 11 +- .../test_perpetual_market_making.py | 6 +- .../test_perpetual_market_making_start.py | 11 +- .../strategy/pure_market_making/test_pmm.py | 12 +- .../pure_market_making/test_pmm_ping_pong.py | 6 +- .../test_pmm_refresh_tolerance.py | 6 +- .../test_pmm_take_if_cross.py | 10 +- .../test_pure_market_making_start.py | 11 +- .../test_spot_perpetual_arbitrage.py | 10 +- .../test_spot_perpetual_arbitrage_start.py | 16 +- .../test_market_trading_pair_tuple.py | 6 +- .../hummingbot/strategy/test_order_tracker.py | 6 +- .../strategy/test_script_strategy_base.py | 6 +- .../hummingbot/strategy/test_strategy_base.py | 14 +- .../strategy/test_strategy_py_base.py | 6 +- test/hummingbot/strategy/twap/test_twap.py | 6 +- .../strategy/twap/test_twap_trade_strategy.py | 26 ++- .../strategy/twap/twap_test_support.py | 11 +- .../hummingbot/test_hummingbot_application.py | 4 +- test/mock/mock_cli.py | 9 +- test/mock/mock_perp_connector.py | 11 +- 116 files changed, 1246 insertions(+), 929 deletions(-) mode change 100644 => 100755 bin/conf_migration_script.py delete mode 100644 test/hummingbot/client/config/test_config_templates.py diff --git a/bin/conf_migration_script.py b/bin/conf_migration_script.py old mode 100644 new mode 100755 index f833618ef3..931a825284 --- a/bin/conf_migration_script.py +++ b/bin/conf_migration_script.py @@ -1,5 +1,7 @@ import argparse +import path_util # noqa: F401 + from hummingbot.client.config.conf_migration import migrate_configs from hummingbot.client.config.config_crypt import ETHKeyFileSecretManger diff --git a/hummingbot/client/command/balance_command.py b/hummingbot/client/command/balance_command.py index c57fc43372..d9d1743a7f 100644 --- a/hummingbot/client/command/balance_command.py +++ b/hummingbot/client/command/balance_command.py @@ -79,7 +79,7 @@ async def show_balances( total_col_name = f'Total ({RateOracle.global_token_symbol})' sum_not_for_show_name = "sum_not_for_show" self.notify("Updating balances, please wait...") - network_timeout = self.client_config_map.commands_timeout.other_commands_timeout + network_timeout = float(self.client_config_map.commands_timeout.other_commands_timeout) try: all_ex_bals = await asyncio.wait_for( UserBalances.instance().all_balances_all_exchanges(self.client_config_map), network_timeout diff --git a/hummingbot/client/command/config_command.py b/hummingbot/client/command/config_command.py index c82c1f3e5b..8cbf46e0f6 100644 --- a/hummingbot/client/command/config_command.py +++ b/hummingbot/client/command/config_command.py @@ -52,6 +52,7 @@ "pmm_script_mode", "pmm_script_file_path", "ethereum_chain_name", + "gateway", "gateway_enabled", "gateway_cert_passphrase", "gateway_api_host", @@ -60,15 +61,16 @@ "global_token", "global_token_symbol", "rate_limits_share_pct", + "commands_timeout", "create_command_timeout", "other_commands_timeout", "tables_format"] -color_settings_to_display = ["top-pane", - "bottom-pane", - "output-pane", - "input-pane", - "logs-pane", - "terminal-primary"] +color_settings_to_display = ["top_pane", + "bottom_pane", + "output_pane", + "input_pane", + "logs_pane", + "terminal_primary"] columns = ["Key", "Value"] @@ -97,13 +99,19 @@ def list_client_configs( data = self.build_model_df_data(self.client_config_map, to_print=client_configs_to_display) df = map_df_to_str(pd.DataFrame(data=data, columns=columns)) self.notify("\nGlobal Configurations:") - lines = [" " + line for line in format_df_for_printout(df, max_col_width=50).split("\n")] + lines = [" " + line for line in format_df_for_printout( + df, + table_format=self.client_config_map.tables_format, + max_col_width=50).split("\n")] self.notify("\n".join(lines)) data = self.build_model_df_data(self.client_config_map, to_print=color_settings_to_display) df = map_df_to_str(pd.DataFrame(data=data, columns=columns)) self.notify("\nColor Settings:") - lines = [" " + line for line in format_df_for_printout(df, max_col_width=50).split("\n")] + lines = [" " + line for line in format_df_for_printout( + df, + table_format=self.client_config_map.tables_format, + max_col_width=50).split("\n")] self.notify("\n".join(lines)) def list_strategy_configs( @@ -114,7 +122,10 @@ def list_strategy_configs( data = self.build_df_data_from_config_map(config_map) df = map_df_to_str(pd.DataFrame(data=data, columns=columns)) self.notify("\nStrategy Configurations:") - lines = [" " + line for line in format_df_for_printout(df, max_col_width=50).split("\n")] + lines = [" " + line for line in format_df_for_printout( + df, + table_format=self.client_config_map.tables_format, + max_col_width=50).split("\n")] self.notify("\n".join(lines)) def build_df_data_from_config_map( @@ -211,9 +222,9 @@ async def _config_single_key(self, # type: HummingbotApplication else: if input_value is None: self.notify("Please follow the prompt to complete configurations: ") - client_config_key = key in self.client_config_map.keys() + client_config_key = key in self.client_config_map.config_paths() if client_config_key: - config_map = self.strategy_config_map + config_map = self.client_config_map file_path = CLIENT_CONFIG_PATH else: config_map = self.strategy_config_map diff --git a/hummingbot/client/command/connect_command.py b/hummingbot/client/command/connect_command.py index c622731847..6684f99ab5 100644 --- a/hummingbot/client/command/connect_command.py +++ b/hummingbot/client/command/connect_command.py @@ -78,7 +78,9 @@ async def show_connections(self # type: HummingbotApplication ): self.notify("\nTesting connections, please wait...") df, failed_msgs = await self.connection_df() - lines = [" " + line for line in format_df_for_printout(df, self.client_config_map.tables_format).split("\n")] + lines = [" " + line for line in format_df_for_printout( + df, + table_format=self.client_config_map.tables_format).split("\n")] if failed_msgs: lines.append("\nFailed connections:") lines.extend([" " + k + ": " + v for k, v in failed_msgs.items()]) @@ -90,7 +92,7 @@ async def connection_df(self # type: HummingbotApplication columns = ["Exchange", " Keys Added", " Keys Confirmed", " Status"] data = [] failed_msgs = {} - network_timeout = self.client_config_map.commands_timeout.other_commands_timeout + network_timeout = float(self.client_config_map.commands_timeout.other_commands_timeout) try: err_msgs = await asyncio.wait_for( UserBalances.instance().update_exchanges(self.client_config_map, reconnect=True), network_timeout @@ -150,7 +152,7 @@ async def validate_n_connect_connector( ) -> Optional[str]: await Security.wait_til_decryption_done() api_keys = Security.api_keys(connector_name) - network_timeout = self.client_config_map.commands_timeout.other_commands_timeout + network_timeout = float(self.client_config_map.commands_timeout.other_commands_timeout) try: err_msg = await asyncio.wait_for( UserBalances.instance().add_exchange(connector_name, self.client_config_map, **api_keys), diff --git a/hummingbot/client/command/create_command.py b/hummingbot/client/command/create_command.py index f6de78e760..afa911306a 100644 --- a/hummingbot/client/command/create_command.py +++ b/hummingbot/client/command/create_command.py @@ -234,7 +234,7 @@ async def verify_status( self # type: HummingbotApplication ): try: - timeout = self.client_config_map.commands_timeout.create_command_timeout + timeout = float(self.client_config_map.commands_timeout.create_command_timeout) all_status_go = await asyncio.wait_for(self.status_check_all(), timeout) except asyncio.TimeoutError: self.notify("\nA network error prevented the connection check to complete. See logs for more details.") diff --git a/hummingbot/client/command/status_command.py b/hummingbot/client/command/status_command.py index 67083c1211..5970c653bf 100644 --- a/hummingbot/client/command/status_command.py +++ b/hummingbot/client/command/status_command.py @@ -111,14 +111,13 @@ async def validate_required_connections( def missing_configurations_legacy( self, # type: HummingbotApplication ) -> List[str]: - missing_globals = missing_required_configs_legacy(self.client_config_map) config_map = self.strategy_config_map missing_configs = [] if not isinstance(config_map, ClientConfigAdapter): missing_configs = missing_required_configs_legacy( get_strategy_config_map(self.strategy_name) ) - return missing_globals + missing_configs + return missing_configs def validate_configs( self, # type: HummingbotApplication @@ -175,7 +174,7 @@ async def status_check_all(self, # type: HummingbotApplication self.notify(f" {error}") return False - network_timeout = self.client_config_map.commands_timeout.other_commands_timeout + network_timeout = float(self.client_config_map.commands_timeout.other_commands_timeout) try: invalid_conns = await asyncio.wait_for(self.validate_required_connections(), network_timeout) except asyncio.TimeoutError: diff --git a/hummingbot/client/config/client_config_map.py b/hummingbot/client/config/client_config_map.py index fbf5ff48ad..3992bb2018 100644 --- a/hummingbot/client/config/client_config_map.py +++ b/hummingbot/client/config/client_config_map.py @@ -356,7 +356,11 @@ def validate_db_engine(cls, v: str): class PMMScriptMode(BaseClientModel, ABC): @abstractmethod - def get_iterator(self, strategy_name: str, markets: List[ExchangeBase], strategy: StrategyBase) -> Optional[PMMScriptIterator]: + def get_iterator( + self, + strategy_name: str, + markets: List[ExchangeBase], + strategy: StrategyBase) -> Optional[PMMScriptIterator]: ... @@ -364,7 +368,11 @@ class PMMScriptDisabledMode(PMMScriptMode): class Config: title = "pmm_script_disabled" - def get_iterator(self, strategy_name: str, markets: List[ExchangeBase], strategy: StrategyBase) -> Optional[PMMScriptIterator]: + def get_iterator( + self, + strategy_name: str, + markets: List[ExchangeBase], + strategy: StrategyBase) -> Optional[PMMScriptIterator]: return None @@ -377,7 +385,11 @@ class PMMScriptEnabledMode(PMMScriptMode): class Config: title = "pmm_script_enabled" - def get_iterator(self, strategy_name: str, markets: List[ExchangeBase], strategy: StrategyBase) -> Optional[PMMScriptIterator]: + def get_iterator( + self, + strategy_name: str, + markets: List[ExchangeBase], + strategy: StrategyBase) -> Optional[PMMScriptIterator]: if strategy_name != "pure_market_making": raise ValueError("PMM script feature is only available for pure_market_making strategy.") folder = dirname(self.pmm_script_file_path) @@ -421,6 +433,9 @@ class GatewayConfigMap(BaseClientModel): ), ) + class Config: + title = "gateway" + class GlobalTokenConfigMap(BaseClientModel): global_token_name: str = Field( @@ -489,11 +504,11 @@ def validate_decimals(cls, v: str, field: Field): class AnonymizedMetricsMode(BaseClientModel, ABC): @abstractmethod def get_collector( - self, - connector: ConnectorBase, - rate_provider: RateOracle, - instance_id: str, - valuation_token: str = "USDT", + self, + connector: ConnectorBase, + rate_provider: RateOracle, + instance_id: str, + valuation_token: str = "USDT", ) -> MetricsCollector: ... @@ -503,11 +518,11 @@ class Config: title = "anonymized_metrics_disabled" def get_collector( - self, - connector: ConnectorBase, - rate_provider: RateOracle, - instance_id: str, - valuation_token: str = "USDT", + self, + connector: ConnectorBase, + rate_provider: RateOracle, + instance_id: str, + valuation_token: str = "USDT", ) -> MetricsCollector: return DummyMetricsCollector() @@ -525,11 +540,11 @@ class Config: title = "anonymized_metrics_enabled" def get_collector( - self, - connector: ConnectorBase, - rate_provider: RateOracle, - instance_id: str, - valuation_token: str = "USDT", + self, + connector: ConnectorBase, + rate_provider: RateOracle, + instance_id: str, + valuation_token: str = "USDT", ) -> MetricsCollector: instance = TradeVolumeMetricCollector( connector=connector, @@ -581,6 +596,7 @@ class ClientConfigMap(BaseClientModel): ) autofill_import: AutofillImportEnum = Field( default=AutofillImportEnum.disabled, + description="What to auto-fill in the prompt after each import command (start/config)", client_data=ClientFieldData( prompt=lambda cm: ( f"What to auto-fill in the prompt after each import command? ({'/'.join(list(AutofillImportEnum))})" @@ -595,12 +611,19 @@ class ClientConfigMap(BaseClientModel): ) send_error_logs: bool = Field( default=True, + description="Error log sharing", client_data=ClientFieldData( prompt=lambda cm: "Would you like to send error logs to hummingbot? (Yes/No)", ), ) db_mode: Union[tuple(DB_MODES.values())] = Field( default=DBSqliteMode(), + description=("Advanced database options, currently supports SQLAlchemy's included dialects" + "\nReference: https://docs.sqlalchemy.org/en/13/dialects/" + "\nTo use an instance of SQLite DB the required configuration is \n db_engine: sqlite" + "\nTo use a DBMS the required configuration is" + "\n db_host: 127.0.0.1\n db_port: 3306\n db_username: username\n db_password: password" + "\n db_name: dbname"), client_data=ClientFieldData( prompt=lambda cm: f"Select the desired db mode ({'/'.join(list(DB_MODES.keys()))})", ), @@ -611,8 +634,14 @@ class ClientConfigMap(BaseClientModel): prompt=lambda cm: f"Select the desired PMM script mode ({'/'.join(list(PMM_SCRIPT_MODES.keys()))})", ), ) - balance_asset_limit: Dict[str, Dict[str, Decimal]] = Field( # todo: validator that can handle inputs - default=json.dumps({exchange: None for exchange in AllConnectorSettings.get_exchange_names()}), + balance_asset_limit: Dict[str, Dict[str, Decimal]] = Field( + default={exchange: {} for exchange in AllConnectorSettings.get_exchange_names()}, + description=("Balance Limit Configurations" + "\ne.g. Setting USDT and BTC limits on Binance." + "\nbalance_asset_limit:" + "\n binance:" + "\n BTC: 0.1" + "\n USDT: 1000"), client_data=ClientFieldData( prompt=lambda cm: ( "Use the `balance limit` command e.g. balance limit [EXCHANGE] [ASSET] [AMOUNT]" @@ -621,19 +650,26 @@ class ClientConfigMap(BaseClientModel): ) manual_gas_price: Decimal = Field( default=Decimal("50"), + description="Fixed gas price (in Gwei) for Ethereum transactions", gt=Decimal("0"), client_data=ClientFieldData( prompt=lambda cm: "Enter fixed gas price (in Gwei) you want to use for Ethereum transactions", ), ) - gateway: GatewayConfigMap = Field(default=GatewayConfigMap()) + gateway: GatewayConfigMap = Field( + default=GatewayConfigMap(), + description=("Gateway API Configurations" + "\ndefault host to only use localhost" + "\nPort need to match the final installation port for Gateway"), + ) anonymized_metrics_mode: Union[tuple(METRICS_MODES.values())] = Field( default=AnonymizedMetricsDisabledMode(), + description="Whether to enable aggregated order and trade data collection", client_data=ClientFieldData( prompt=lambda cm: f"Select the desired metrics mode ({'/'.join(list(METRICS_MODES.keys()))})", ), ) - command_shortcuts: List[CommandShortcutModel] = Field( # todo: test it — does it load properly? + command_shortcuts: List[CommandShortcutModel] = Field( default=[ CommandShortcutModel( command="spreads", @@ -641,19 +677,29 @@ class ClientConfigMap(BaseClientModel): arguments=["Bid Spread", "Ask Spread"], output=["config bid_spread $1", "config ask_spread $2"] ) - ] + ], + description=("Command Shortcuts" + "\nDefine abbreviations for often used commands" + "\nor batch grouped commands together"), ) rate_oracle_source: str = Field( default=RateOracleSource.binance.name, + description="A source for rate oracle, currently binance, ascendex, kucoin or coingecko", client_data=ClientFieldData( prompt=lambda cm: ( f"What source do you want rate oracle to pull data from? ({','.join(r.name for r in RateOracleSource)})" ), ), ) - global_token: GlobalTokenConfigMap = Field(default=GlobalTokenConfigMap()) + global_token: GlobalTokenConfigMap = Field( + default=GlobalTokenConfigMap(), + description="A universal token which to display tokens values in, e.g. USD,EUR,BTC" + ) rate_limits_share_pct: Decimal = Field( default=Decimal("100"), + description=("Percentage of API rate limits (on any exchange and any end point) allocated to this bot instance." + "\nEnter 50 to indicate 50%. E.g. if the API rate limit is 100 calls per second, and you allocate " + "\n50% to this setting, the bot will have a maximum (limit) of 50 calls per second"), gt=Decimal("0"), le=Decimal("100"), client_data=ClientFieldData( @@ -670,6 +716,7 @@ class ClientConfigMap(BaseClientModel): type=str, ) = Field( default="psql", + description="Tabulate table format style (https://github.com/astanin/python-tabulate#table-format)", client_data=ClientFieldData( prompt=lambda cm: ( "What tabulate formatting to apply to the tables?" diff --git a/hummingbot/client/config/conf_migration.py b/hummingbot/client/config/conf_migration.py index b40cc58a86..b5e69de50a 100644 --- a/hummingbot/client/config/conf_migration.py +++ b/hummingbot/client/config/conf_migration.py @@ -94,14 +94,10 @@ def migrate_global_config() -> List[str]: migrate_global_config_modes(client_config_map, data) keys = list(data.keys()) for key in keys: - if key not in client_config_map.keys(): - errors.append( - f"Could not match the attribute {key} from the legacy config file to the new config map." - ) - else: + if key in client_config_map.keys(): migrate_global_config_field(client_config_map, data, key) for key in data: - errors.append(f"ConfigVar {key} was not migrated.") + logging.getLogger().warning(f"ConfigVar {key} was not migrated.") errors.extend(client_config_map.validate_model()) if len(errors) == 0: save_to_yml(CLIENT_CONFIG_PATH, client_config_map) diff --git a/hummingbot/client/config/config_data_types.py b/hummingbot/client/config/config_data_types.py index 84ed9ba455..e1a9c259cc 100644 --- a/hummingbot/client/config/config_data_types.py +++ b/hummingbot/client/config/config_data_types.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from datetime import datetime from enum import Enum -from typing import Any, Callable, Dict, List, Optional +from typing import Any, Callable, Dict, Optional from pydantic import BaseModel, Extra, Field, validator from pydantic.schema import default_ref_template @@ -118,19 +118,15 @@ def maker_trading_pair_prompt(cls, model_instance: 'BaseTradingStrategyConfigMap @validator("exchange", pre=True) def validate_exchange(cls, v: str): """Used for client-friendly error output.""" - exchanges = v.split(", ") - for e in exchanges: - ret = validate_exchange(e) - if ret is not None: - raise ValueError(ret) - cls.__fields__["exchange"].type_ = List[ - ClientConfigEnum( # rebuild the exchanges enum - value="Exchanges", # noqa: F821 - names={e: e for e in AllConnectorSettings.get_all_connectors()}, - type=str, - ) - ] - return exchanges + ret = validate_exchange(v) + if ret is not None: + raise ValueError(ret) + cls.__fields__["exchange"].type_ = ClientConfigEnum( # rebuild the exchanges enum + value="Exchanges", # noqa: F821 + names={e: e for e in AllConnectorSettings.get_all_connectors()}, + type=str, + ) + return v @validator("market", pre=True) def validate_exchange_trading_pair(cls, v: str, values: Dict): diff --git a/hummingbot/client/config/config_helpers.py b/hummingbot/client/config/config_helpers.py index 54c5e5f89f..1a63949904 100644 --- a/hummingbot/client/config/config_helpers.py +++ b/hummingbot/client/config/config_helpers.py @@ -20,7 +20,7 @@ from yaml import SafeDumper from hummingbot import get_strategy_list, root_path -from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.client_config_map import ClientConfigMap, CommandShortcutModel from hummingbot.client.config.config_data_types import BaseClientModel, ClientConfigEnum, ClientFieldData from hummingbot.client.config.config_var import ConfigVar from hummingbot.client.config.fee_overrides_config_map import fee_overrides_config_map, init_fee_overrides_config @@ -65,6 +65,10 @@ def path_representer(dumper: SafeDumper, data: Path): return dumper.represent_str(str(data)) +def command_shortcut_representer(dumper: SafeDumper, data: CommandShortcutModel): + return dumper.represent_dict(data.__dict__) + + yaml.add_representer( data_type=Decimal, representer=decimal_representer, Dumper=SafeDumper ) @@ -86,6 +90,9 @@ def path_representer(dumper: SafeDumper, data: Path): yaml.add_representer( data_type=PosixPath, representer=path_representer, Dumper=SafeDumper ) +yaml.add_representer( + data_type=CommandShortcutModel, representer=command_shortcut_representer, Dumper=SafeDumper +) class ConfigValidationError(Exception): @@ -147,6 +154,9 @@ def is_required(self, attr: str) -> bool: def keys(self) -> Generator[str, None, None]: return self._hb_config.__fields__.keys() + def config_paths(self) -> Generator[str, None, None]: + return (traversal_item.config_path for traversal_item in self.traverse()) + def traverse(self, secure: bool = True) -> Generator[ConfigTraversalItem, None, None]: """The intended use for this function is to simplify config map traversals in the client code. @@ -201,12 +211,8 @@ def get_default(self, attr_name: str) -> Any: return self._hb_config.__fields__[attr_name].field_info.default def generate_yml_output_str_with_comments(self) -> str: - conf_dict = self._dict_in_conf_order() - self._encrypt_secrets(conf_dict) - original_fragments = yaml.safe_dump(conf_dict, sort_keys=False).split("\n") fragments_with_comments = [self._generate_title()] - self._add_model_fragments(fragments_with_comments, original_fragments) - fragments_with_comments.append("\n") # EOF empty line + self._add_model_fragments(fragments_with_comments) yml_str = "".join(fragments_with_comments) return yml_str @@ -286,25 +292,38 @@ def _adorn_title(title: str) -> str: def _add_model_fragments( self, fragments_with_comments: List[str], - original_fragments: List[str], ): - for i, traversal_item in enumerate(self.traverse()): + + fragments_with_comments.append("\n") + first_level_conf_items_generator = (item for item in self.traverse() if item.depth == 0) + + for traversal_item in first_level_conf_items_generator: + fragments_with_comments.append("\n") + attr_comment = traversal_item.field_info.description if attr_comment is not None: - comment_prefix = f"\n{' ' * 2 * traversal_item.depth}# " - attr_comment = "".join(f"{comment_prefix}{c}" for c in attr_comment.split("\n")) - if traversal_item.depth == 0: - attr_comment = f"\n{attr_comment}" - fragments_with_comments.extend([attr_comment, f"\n{original_fragments[i]}"]) - elif traversal_item.depth == 0: - fragments_with_comments.append(f"\n\n{original_fragments[i]}") - else: - fragments_with_comments.append(f"\n{original_fragments[i]}") + comment_prefix = f"{' ' * 2 * traversal_item.depth}# " + attr_comment = "\n".join(f"{comment_prefix}{c}" for c in attr_comment.split("\n")) + fragments_with_comments.append(attr_comment) + fragments_with_comments.append("\n") + + attribute = traversal_item.attr + value = getattr(self, attribute) + if isinstance(value, ClientConfigAdapter): + value = value._dict_in_conf_order() + conf_as_dictionary = {attribute: value} + self._encrypt_secrets(conf_as_dictionary) + + yaml_config = yaml.safe_dump(conf_as_dictionary, sort_keys=False) + fragments_with_comments.append(yaml_config) class ReadOnlyClientConfigAdapter(ClientConfigAdapter): def __setattr__(self, key, value): - raise AttributeError("Cannot set an attribute on a read-only client adapter.") + if key == "_hb_config": + super().__setattr__(key, value) + else: + raise AttributeError("Cannot set an attribute on a read-only client adapter") @classmethod def lock_config(cls, config_map: ClientConfigMap): @@ -552,7 +571,12 @@ def load_client_config_map_from_file() -> ClientConfigAdapter: config_data = {} client_config = ClientConfigMap() config_map = ClientConfigAdapter(client_config) - _load_yml_data_into_map(config_data, config_map) + config_validation_errors = _load_yml_data_into_map(config_data, config_map) + + if len(config_validation_errors) > 0: + all_errors = "\n".join(config_validation_errors) + raise ConfigValidationError(f"There are errors in the client global configuration (\n{all_errors})") + return config_map @@ -601,14 +625,13 @@ def list_connector_configs() -> List[Path]: return connector_configs -def _load_yml_data_into_map(yml_data: Dict[str, Any], cm: ClientConfigAdapter): +def _load_yml_data_into_map(yml_data: Dict[str, Any], cm: ClientConfigAdapter) -> List[str]: for key in cm.keys(): if key in yml_data: cm.setattr_no_validation(key, yml_data[key]) - try: - cm.validate_model() # try coercing values to appropriate type - except Exception: - pass # but don't raise if it fails + + config_validation_errors = cm.validate_model() # try coercing values to appropriate type + return config_validation_errors async def load_yml_into_dict(yml_path: str) -> Dict[str, Any]: diff --git a/hummingbot/client/hummingbot_application.py b/hummingbot/client/hummingbot_application.py index 99e68165ab..3911a35411 100644 --- a/hummingbot/client/hummingbot_application.py +++ b/hummingbot/client/hummingbot_application.py @@ -189,20 +189,20 @@ def _handle_command(self, raw_command: str): shortcut = None # see if we match against shortcut command if shortcuts is not None: - for s in shortcuts: - if command_split[0] == s['command']: - shortcut = s + for each_shortcut in shortcuts: + if command_split[0] == each_shortcut.command: + shortcut = each_shortcut break # perform shortcut expansion if shortcut is not None: # check number of arguments - num_shortcut_args = len(shortcut['arguments']) + num_shortcut_args = len(shortcut.arguments) if len(command_split) == num_shortcut_args + 1: # notify each expansion if there's more than 1 - verbose = True if len(shortcut['output']) > 1 else False + verbose = True if len(shortcut.output) > 1 else False # do argument replace and re-enter this function with the expanded command - for output_cmd in shortcut['output']: + for output_cmd in shortcut.output: final_cmd = output_cmd for i in range(1, num_shortcut_args + 1): final_cmd = final_cmd.replace(f'${i}', command_split[i]) diff --git a/hummingbot/client/ui/__init__.py b/hummingbot/client/ui/__init__.py index fc480b1a99..cd72691f48 100644 --- a/hummingbot/client/ui/__init__.py +++ b/hummingbot/client/ui/__init__.py @@ -112,7 +112,7 @@ def migrate_configs_prompt(secrets_manager_cls: Type[BaseSecretsManager]) -> Bas style=dialog_style).run() password = input_dialog( title="Input Password", - text="\n\nEnter your password:", + text="\n\nEnter your previous password:", password=True, style=dialog_style).run() if password is None: diff --git a/hummingbot/client/ui/custom_widgets.py b/hummingbot/client/ui/custom_widgets.py index ae60a9e334..566eb1fcba 100644 --- a/hummingbot/client/ui/custom_widgets.py +++ b/hummingbot/client/ui/custom_widgets.py @@ -45,16 +45,17 @@ def __init__(self, client_config_map: ClientConfigAdapter) -> None: style: css for style, css in load_style(client_config_map).style_rules } self.html_tag_css_style_map.update({ - ti.attr: ti.value + ti.attr.replace("_", "-"): ti.value for ti in client_config_map.color.traverse() - if ti.attr not in self.html_tag_css_style_map + if ti.attr.replace("_", "-") not in self.html_tag_css_style_map }) # Maps specific text to its corresponding UI styles self.text_style_tag_map: Dict[str, str] = text_ui_style def get_css_style(self, tag: str) -> str: - return self.html_tag_css_style_map.get(tag, "") + style = self.html_tag_css_style_map.get(tag, "") + return style def lex_document(self, document: Document) -> Callable[[int], StyleAndTextTuples]: lines = document.lines @@ -66,7 +67,7 @@ def get_line(lineno: int) -> StyleAndTextTuples: # Apply styling to command prompt if current_line.startswith(self.PROMPT_TEXT): - return [(self.get_css_style("primary-label"), current_line)] + return [(self.get_css_style("primary_label"), current_line)] matched_indexes: List[Tuple[int, int, str]] = [(match.start(), match.end(), style) for special_word, style in self.text_style_tag_map.items() @@ -80,7 +81,7 @@ def get_line(lineno: int) -> StyleAndTextTuples: for start_idx, end_idx, style in matched_indexes: line_fragments.extend([ ("", current_line[previous_idx:start_idx]), - (self.get_css_style("output-pane"), current_line[start_idx:start_idx + 2]), + (self.get_css_style("output_pane"), current_line[start_idx:start_idx + 2]), (self.get_css_style(style), current_line[start_idx + 2:end_idx]) ]) previous_idx = end_idx diff --git a/hummingbot/client/ui/hummingbot_cli.py b/hummingbot/client/ui/hummingbot_cli.py index 29ccf47fa6..7209955a80 100644 --- a/hummingbot/client/ui/hummingbot_cli.py +++ b/hummingbot/client/ui/hummingbot_cli.py @@ -98,10 +98,10 @@ def __init__(self, loop.create_task(start_process_monitor(self.process_usage)) loop.create_task(start_trade_monitor(self.trade_monitor)) - def did_start_ui(self, hummingbot: "HummingbotApplication"): + def did_start_ui(self): self._stdout_redirect_context.enter_context(patch_stdout(log_field=self.log_field)) - log_level = hummingbot.client_config_map.log_level + log_level = self.client_config_map.log_level init_logging("hummingbot_logs.yml", self.client_config_map, override_log_level=log_level) self.trigger_event(HummingbotUIEvent.Start, self) diff --git a/hummingbot/client/ui/layout.py b/hummingbot/client/ui/layout.py index 001d92a1bd..de74929391 100644 --- a/hummingbot/client/ui/layout.py +++ b/hummingbot/client/ui/layout.py @@ -86,7 +86,7 @@ def create_input_field(lexer=None, completer: Completer = None): return TextArea( height=10, prompt='>>> ', - style='class:input-field', + style='class:input_field', multiline=False, focus_on_click=True, lexer=lexer, @@ -98,7 +98,7 @@ def create_input_field(lexer=None, completer: Completer = None): def create_output_field(client_config_map: ClientConfigAdapter): return TextArea( - style='class:output-field', + style='class:output_field', focus_on_click=False, read_only=False, scrollbar=True, @@ -148,7 +148,7 @@ def create_search_field() -> SearchToolbar: def create_log_field(search_field: SearchToolbar): return TextArea( - style='class:log-field', + style='class:log_field', text="Running Logs\n", focus_on_click=False, read_only=False, @@ -162,7 +162,7 @@ def create_log_field(search_field: SearchToolbar): def create_live_field(): return TextArea( - style='class:log-field', + style='class:log_field', focus_on_click=False, read_only=False, scrollbar=True, @@ -197,14 +197,14 @@ def get_version(): def get_active_strategy(): from hummingbot.client.hummingbot_application import HummingbotApplication hb = HummingbotApplication.main_application() - style = "class:log-field" + style = "class:log_field" return [(style, f"Strategy: {hb.strategy_name}")] def get_strategy_file(): from hummingbot.client.hummingbot_application import HummingbotApplication hb = HummingbotApplication.main_application() - style = "class:log-field" + style = "class:log_field" return [(style, f"Strategy File: {hb._strategy_file_name}")] @@ -212,7 +212,7 @@ def get_gateway_status(): from hummingbot.client.hummingbot_application import HummingbotApplication hb = HummingbotApplication.main_application() gateway_status = "ON" if hb._gateway_monitor.current_status is GatewayStatus.ONLINE else "OFF" - style = "class:log-field" + style = "class:log_field" return [(style, f"Gateway: {gateway_status}")] @@ -242,8 +242,8 @@ def generate_layout(input_field: TextArea, components["pane_bottom"] = VSplit([trade_monitor, process_monitor, timer], height=1) - output_pane = Box(body=output_field, padding=0, padding_left=2, style="class:output-field") - input_pane = Box(body=input_field, padding=0, padding_left=2, padding_top=1, style="class:input-field") + output_pane = Box(body=output_field, padding=0, padding_left=2, style="class:output_field") + input_pane = Box(body=input_field, padding=0, padding_left=2, padding_top=1, style="class:input_field") components["pane_left"] = HSplit([output_pane, input_pane], width=Dimension(weight=1)) if all(not t.is_selected for t in command_tabs.values()): log_field_button.window.style = "class:tab_button.focused" @@ -262,10 +262,10 @@ def generate_layout(input_field: TextArea, focused_right_field = [tab.output_field for tab in command_tabs.values() if tab.is_selected] if focused_right_field: pane_right_field = focused_right_field[0] - components["pane_right_top"] = VSplit(tab_buttons, height=1, style="class:log-field", padding_char=" ", padding=2) + components["pane_right_top"] = VSplit(tab_buttons, height=1, style="class:log_field", padding_char=" ", padding=2) components["pane_right"] = ConditionalContainer( Box(body=HSplit([components["pane_right_top"], pane_right_field, search_field], width=Dimension(weight=1)), - padding=0, padding_left=2, style="class:log-field"), + padding=0, padding_left=2, style="class:log_field"), filter=True ) components["hint_menus"] = [Float(xcursor=True, diff --git a/hummingbot/client/ui/style.py b/hummingbot/client/ui/style.py index 7af252e96e..e5572cd86c 100644 --- a/hummingbot/client/ui/style.py +++ b/hummingbot/client/ui/style.py @@ -48,9 +48,9 @@ def load_style(config_map: ClientConfigAdapter): color_error_label = hex_to_ansi(color_error_label) # Apply custom configuration - style["output-field"] = "bg:" + color_output_pane + " " + color_terminal_primary - style["input-field"] = "bg:" + color_input_pane + " " + style["input-field"].split(' ')[-1] - style["log-field"] = "bg:" + color_logs_pane + " " + style["log-field"].split(' ')[-1] + style["output_field"] = "bg:" + color_output_pane + " " + color_terminal_primary + style["input_field"] = "bg:" + color_input_pane + " " + style["input_field"].split(' ')[-1] + style["log_field"] = "bg:" + color_logs_pane + " " + style["log_field"].split(' ')[-1] style["tab_button.focused"] = "bg:" + color_terminal_primary + " " + color_logs_pane style["tab_button"] = style["tab_button"].split(' ')[0] + " " + color_logs_pane style["header"] = "bg:" + color_top_pane + " " + style["header"].split(' ')[-1] @@ -59,12 +59,12 @@ def load_style(config_map: ClientConfigAdapter): style["search"] = color_terminal_primary style["search.current"] = color_terminal_primary - style["primary-label"] = "bg:" + color_primary_label + " " + color_output_pane - style["secondary-label"] = "bg:" + color_secondary_label + " " + color_output_pane - style["success-label"] = "bg:" + color_success_label + " " + color_output_pane - style["warning-label"] = "bg:" + color_warning_label + " " + color_output_pane - style["info-label"] = "bg:" + color_info_label + " " + color_output_pane - style["error-label"] = "bg:" + color_error_label + " " + color_output_pane + style["primary_label"] = "bg:" + color_primary_label + " " + color_output_pane + style["secondary_label"] = "bg:" + color_secondary_label + " " + color_output_pane + style["success_label"] = "bg:" + color_success_label + " " + color_output_pane + style["warning_label"] = "bg:" + color_warning_label + " " + color_output_pane + style["info_label"] = "bg:" + color_info_label + " " + color_output_pane + style["error_label"] = "bg:" + color_error_label + " " + color_output_pane return Style.from_dict(style) @@ -73,21 +73,21 @@ def load_style(config_map: ClientConfigAdapter): style = default_ui_style # Apply custom configuration - style["output-field"] = "bg:" + color_output_pane + " " + color_terminal_primary - style["input-field"] = "bg:" + color_input_pane + " " + style["input-field"].split(' ')[-1] - style["log-field"] = "bg:" + color_logs_pane + " " + style["log-field"].split(' ')[-1] + style["output_field"] = "bg:" + color_output_pane + " " + color_terminal_primary + style["input_field"] = "bg:" + color_input_pane + " " + style["input_field"].split(' ')[-1] + style["log_field"] = "bg:" + color_logs_pane + " " + style["log_field"].split(' ')[-1] style["header"] = "bg:" + color_top_pane + " " + style["header"].split(' ')[-1] style["footer"] = "bg:" + color_bottom_pane + " " + style["footer"].split(' ')[-1] style["primary"] = color_terminal_primary style["tab_button.focused"] = "bg:" + color_terminal_primary + " " + color_logs_pane style["tab_button"] = style["tab_button"].split(' ')[0] + " " + color_logs_pane - style["primary-label"] = "bg:" + color_primary_label + " " + color_output_pane - style["secondary-label"] = "bg:" + color_secondary_label + " " + color_output_pane - style["success-label"] = "bg:" + color_success_label + " " + color_output_pane - style["warning-label"] = "bg:" + color_warning_label + " " + color_output_pane - style["info-label"] = "bg:" + color_info_label + " " + color_output_pane - style["error-label"] = "bg:" + color_error_label + " " + color_output_pane + style["primary_label"] = "bg:" + color_primary_label + " " + color_output_pane + style["secondary_label"] = "bg:" + color_secondary_label + " " + color_output_pane + style["success_label"] = "bg:" + color_success_label + " " + color_output_pane + style["warning_label"] = "bg:" + color_warning_label + " " + color_output_pane + style["info_label"] = "bg:" + color_info_label + " " + color_output_pane + style["error_label"] = "bg:" + color_error_label + " " + color_output_pane return Style.from_dict(style) @@ -153,16 +153,16 @@ def hex_to_ansi(color_hex): text_ui_style = { - "&cGREEN": "success-label", - "&cYELLOW": "warning-label", - "&cRED": "error-label", - "&cMISSING_AND_REQUIRED": "error-label", + "&cGREEN": "success_label", + "&cYELLOW": "warning_label", + "&cRED": "error_label", + "&cMISSING_AND_REQUIRED": "error_label", } default_ui_style = { - "output-field": "bg:#171E2B #1CD085", # noqa: E241 - "input-field": "bg:#000000 #FFFFFF", # noqa: E241 - "log-field": "bg:#171E2B #FFFFFF", # noqa: E241 + "output_field": "bg:#171E2B #1CD085", # noqa: E241 + "input_field": "bg:#000000 #FFFFFF", # noqa: E241 + "log_field": "bg:#171E2B #FFFFFF", # noqa: E241 "header": "bg:#000000 #AAAAAA", # noqa: E241 "footer": "bg:#000000 #AAAAAA", # noqa: E241 "search": "bg:#000000 #93C36D", # noqa: E241 @@ -178,9 +178,9 @@ def hex_to_ansi(color_hex): # Style for an older version of Windows consoles. They support only 16 colors, # so we choose a combination that displays nicely. win32_code_style = { - "output-field": "#ansigreen", # noqa: E241 - "input-field": "#ansiwhite", # noqa: E241 - "log-field": "#ansiwhite", # noqa: E241 + "output_field": "#ansigreen", # noqa: E241 + "input_field": "#ansiwhite", # noqa: E241 + "log_field": "#ansiwhite", # noqa: E241 "header": "#ansiwhite", # noqa: E241 "footer": "#ansiwhite", # noqa: E241 "search": "#ansigreen", # noqa: E241 diff --git a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py index d6cc5bed05..883818ab53 100644 --- a/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py +++ b/hummingbot/connector/derivative/binance_perpetual/binance_perpetual_derivative.py @@ -4,7 +4,7 @@ import warnings from collections import defaultdict from decimal import Decimal -from typing import Any, AsyncIterable, Dict, List, Optional +from typing import TYPE_CHECKING, Any, AsyncIterable, Dict, List, Optional from async_timeout import timeout @@ -50,6 +50,9 @@ from hummingbot.core.web_assistant.ws_assistant import WSAssistant from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + bpm_logger = None @@ -72,6 +75,7 @@ def logger(cls) -> HummingbotLogger: def __init__( self, + client_config_map: "ClientConfigAdapter", binance_perpetual_api_key: str = None, binance_perpetual_api_secret: str = None, trading_pairs: Optional[List[str]] = None, @@ -84,7 +88,7 @@ def __init__( time_provider=self._binance_time_synchronizer) self._trading_pairs = trading_pairs self._trading_required = trading_required - self._throttler = AsyncThrottler(CONSTANTS.RATE_LIMITS, self._client_config.rate_limits_share_pct) + self._throttler = AsyncThrottler(CONSTANTS.RATE_LIMITS, client_config_map.rate_limits_share_pct) self._domain = domain self._api_factory = web_utils.build_api_factory( throttler=self._throttler, @@ -94,7 +98,7 @@ def __init__( self._rest_assistant: Optional[RESTAssistant] = None self._ws_assistant: Optional[WSAssistant] = None - ExchangeBase.__init__(self) + ExchangeBase.__init__(self, client_config_map=client_config_map) PerpetualTrading.__init__(self) self._user_stream_tracker = UserStreamTracker( diff --git a/hummingbot/connector/derivative/bybit_perpetual/bybit_perpetual_derivative.py b/hummingbot/connector/derivative/bybit_perpetual/bybit_perpetual_derivative.py index 10420921b0..1651f72053 100644 --- a/hummingbot/connector/derivative/bybit_perpetual/bybit_perpetual_derivative.py +++ b/hummingbot/connector/derivative/bybit_perpetual/bybit_perpetual_derivative.py @@ -4,7 +4,7 @@ import time import warnings from decimal import Decimal -from typing import Any, AsyncIterable, Dict, List, Optional +from typing import TYPE_CHECKING, Any, AsyncIterable, Dict, List, Optional import aiohttp import pandas as pd @@ -14,17 +14,18 @@ import hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_constants as CONSTANTS import hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_utils as bybit_utils from hummingbot.connector.client_order_tracker import ClientOrderTracker -from hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_api_order_book_data_source import \ - BybitPerpetualAPIOrderBookDataSource as OrderBookDataSource +from hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_api_order_book_data_source import ( + BybitPerpetualAPIOrderBookDataSource as OrderBookDataSource, +) from hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_auth import BybitPerpetualAuth from hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_order_book_tracker import ( - BybitPerpetualOrderBookTracker + BybitPerpetualOrderBookTracker, ) from hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_user_stream_tracker import ( - BybitPerpetualUserStreamTracker + BybitPerpetualUserStreamTracker, ) from hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_websocket_adaptor import ( - BybitPerpetualWebSocketAdaptor + BybitPerpetualWebSocketAdaptor, ) from hummingbot.connector.derivative.perpetual_budget_checker import PerpetualBudgetChecker from hummingbot.connector.derivative.position import Position @@ -50,6 +51,9 @@ from hummingbot.core.utils.async_utils import safe_ensure_future, safe_gather from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + s_decimal_NaN = Decimal("nan") s_decimal_0 = Decimal(0) @@ -69,13 +73,14 @@ def logger(cls) -> HummingbotLogger: return cls._logger def __init__(self, + client_config_map: "ClientConfigAdapter", bybit_perpetual_api_key: str = None, bybit_perpetual_secret_key: str = None, trading_pairs: Optional[List[str]] = None, trading_required: bool = True, domain: Optional[str] = None): - ExchangeBase.__init__(self) + ExchangeBase.__init__(self, client_config_map=client_config_map) PerpetualTrading.__init__(self) self._trading_pairs = trading_pairs diff --git a/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_derivative.py b/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_derivative.py index 74f370020b..52931f8f35 100644 --- a/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_derivative.py +++ b/hummingbot/connector/derivative/dydx_perpetual/dydx_perpetual_derivative.py @@ -5,7 +5,7 @@ import warnings from collections import defaultdict from decimal import Decimal -from typing import Any, AsyncIterable, Dict, List, Optional +from typing import TYPE_CHECKING, Any, AsyncIterable, Dict, List, Optional from dateutil.parser import parse as dateparse from dydx3.errors import DydxApiError @@ -14,11 +14,13 @@ from hummingbot.connector.derivative.dydx_perpetual.dydx_perpetual_client_wrapper import DydxPerpetualClientWrapper from hummingbot.connector.derivative.dydx_perpetual.dydx_perpetual_fill_report import DydxPerpetualFillReport from hummingbot.connector.derivative.dydx_perpetual.dydx_perpetual_in_flight_order import DydxPerpetualInFlightOrder -from hummingbot.connector.derivative.dydx_perpetual.dydx_perpetual_order_book_tracker import \ - DydxPerpetualOrderBookTracker +from hummingbot.connector.derivative.dydx_perpetual.dydx_perpetual_order_book_tracker import ( + DydxPerpetualOrderBookTracker, +) from hummingbot.connector.derivative.dydx_perpetual.dydx_perpetual_position import DydxPerpetualPosition -from hummingbot.connector.derivative.dydx_perpetual.dydx_perpetual_user_stream_tracker import \ - DydxPerpetualUserStreamTracker +from hummingbot.connector.derivative.dydx_perpetual.dydx_perpetual_user_stream_tracker import ( + DydxPerpetualUserStreamTracker, +) from hummingbot.connector.derivative.dydx_perpetual.dydx_perpetual_utils import build_api_factory from hummingbot.connector.derivative.perpetual_budget_checker import PerpetualBudgetChecker from hummingbot.connector.exchange_base import ExchangeBase @@ -26,13 +28,7 @@ from hummingbot.connector.trading_rule import TradingRule from hummingbot.core.clock import Clock from hummingbot.core.data_type.cancellation_result import CancellationResult -from hummingbot.core.data_type.common import ( - OrderType, - PositionAction, - PositionMode, - PositionSide, - TradeType -) +from hummingbot.core.data_type.common import OrderType, PositionAction, PositionMode, PositionSide, TradeType from hummingbot.core.data_type.limit_order import LimitOrder from hummingbot.core.data_type.order_book import OrderBook from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee, TokenAmount @@ -58,6 +54,9 @@ from hummingbot.core.utils.tracking_nonce import get_tracking_nonce from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + s_logger = None s_decimal_0 = Decimal(0) s_decimal_NaN = Decimal("nan") @@ -132,6 +131,7 @@ def logger(cls) -> HummingbotLogger: def __init__( self, + client_config_map: "ClientConfigAdapter", dydx_perpetual_api_key: str, dydx_perpetual_api_secret: str, dydx_perpetual_passphrase: str, @@ -142,7 +142,7 @@ def __init__( trading_required: bool = True, ): - ExchangeBase.__init__(self) + ExchangeBase.__init__(self, client_config_map=client_config_map) PerpetualTrading.__init__(self) self._real_time_balance_update = True self._api_factory = build_api_factory() diff --git a/hummingbot/connector/exchange/ascend_ex/ascend_ex_exchange.py b/hummingbot/connector/exchange/ascend_ex/ascend_ex_exchange.py index d0d6478f97..06d4c95bd8 100644 --- a/hummingbot/connector/exchange/ascend_ex/ascend_ex_exchange.py +++ b/hummingbot/connector/exchange/ascend_ex/ascend_ex_exchange.py @@ -5,11 +5,10 @@ from collections import namedtuple from decimal import Decimal from enum import Enum -from typing import Any, AsyncIterable, Dict, List, Optional +from typing import TYPE_CHECKING, Any, AsyncIterable, Dict, List, Optional from hummingbot.connector.client_order_tracker import ClientOrderTracker -from hummingbot.connector.exchange.ascend_ex import ascend_ex_constants as CONSTANTS -from hummingbot.connector.exchange.ascend_ex import ascend_ex_utils +from hummingbot.connector.exchange.ascend_ex import ascend_ex_constants as CONSTANTS, ascend_ex_utils from hummingbot.connector.exchange.ascend_ex.ascend_ex_api_order_book_data_source import AscendExAPIOrderBookDataSource from hummingbot.connector.exchange.ascend_ex.ascend_ex_auth import AscendExAuth from hummingbot.connector.exchange.ascend_ex.ascend_ex_order_book_tracker import AscendExOrderBookTracker @@ -32,6 +31,9 @@ from hummingbot.core.web_assistant.rest_assistant import RESTAssistant from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + ctce_logger = None s_decimal_NaN = Decimal("nan") s_decimal_0 = Decimal("0") @@ -95,6 +97,7 @@ def logger(cls) -> HummingbotLogger: def __init__( self, + client_config_map: "ClientConfigAdapter", ascend_ex_api_key: str, ascend_ex_secret_key: str, trading_pairs: Optional[List[str]] = None, @@ -106,7 +109,7 @@ def __init__( :param trading_pairs: The market trading pairs which to track order book data. :param trading_required: Whether actual trading is needed. """ - super().__init__() + super().__init__(client_config_map=client_config_map) self._trading_required = trading_required self._trading_pairs = trading_pairs self._ascend_ex_auth = AscendExAuth(ascend_ex_api_key, ascend_ex_secret_key) diff --git a/hummingbot/connector/exchange/kucoin/kucoin_exchange.py b/hummingbot/connector/exchange/kucoin/kucoin_exchange.py index ece8b01170..e5089e2366 100644 --- a/hummingbot/connector/exchange/kucoin/kucoin_exchange.py +++ b/hummingbot/connector/exchange/kucoin/kucoin_exchange.py @@ -1,7 +1,7 @@ import asyncio import logging from decimal import Decimal -from typing import Any, AsyncIterable, Dict, List, Optional +from typing import TYPE_CHECKING, Any, AsyncIterable, Dict, List, Optional from async_timeout import timeout @@ -17,11 +17,11 @@ from hummingbot.connector.exchange_py_base import ExchangePyBase from hummingbot.connector.time_synchronizer import TimeSynchronizer from hummingbot.connector.trading_rule import TradingRule -from hummingbot.connector.utils import get_new_client_order_id, combine_to_hb_trading_pair +from hummingbot.connector.utils import combine_to_hb_trading_pair, get_new_client_order_id from hummingbot.core.api_throttler.async_throttler import AsyncThrottler from hummingbot.core.data_type.cancellation_result import CancellationResult from hummingbot.core.data_type.common import OrderType, TradeType -from hummingbot.core.data_type.in_flight_order import InFlightOrder, OrderUpdate, OrderState, TradeUpdate +from hummingbot.core.data_type.in_flight_order import InFlightOrder, OrderState, OrderUpdate, TradeUpdate from hummingbot.core.data_type.limit_order import LimitOrder from hummingbot.core.data_type.order_book import OrderBook from hummingbot.core.data_type.order_book_tracker import OrderBookTracker @@ -33,6 +33,9 @@ from hummingbot.core.web_assistant.connections.data_types import RESTMethod from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + s_decimal_0 = Decimal(0) s_decimal_NaN = Decimal("nan") @@ -54,6 +57,7 @@ def logger(cls) -> HummingbotLogger: return cls._logger def __init__(self, + client_config_map: "ClientConfigAdapter", kucoin_api_key: str, kucoin_passphrase: str, kucoin_secret_key: str, @@ -63,7 +67,7 @@ def __init__(self, self._domain = domain self._time_synchronizer = TimeSynchronizer() - super().__init__() + super().__init__(client_config_map=client_config_map) self._auth = KucoinAuth( api_key=kucoin_api_key, passphrase=kucoin_passphrase, diff --git a/hummingbot/connector/exchange/mexc/mexc_exchange.py b/hummingbot/connector/exchange/mexc/mexc_exchange.py index 1999115298..6682e09411 100644 --- a/hummingbot/connector/exchange/mexc/mexc_exchange.py +++ b/hummingbot/connector/exchange/mexc/mexc_exchange.py @@ -1,7 +1,7 @@ import asyncio import logging from decimal import Decimal -from typing import Any, AsyncIterable, Dict, List, Optional +from typing import TYPE_CHECKING, Any, AsyncIterable, Dict, List, Optional from urllib.parse import quote, urljoin import aiohttp @@ -16,7 +16,7 @@ convert_from_exchange_trading_pair, convert_to_exchange_trading_pair, num_to_increment, - ws_order_status_convert_to_str + ws_order_status_convert_to_str, ) from hummingbot.connector.exchange_base import ExchangeBase, s_decimal_NaN from hummingbot.connector.trading_rule import TradingRule @@ -44,6 +44,9 @@ from hummingbot.core.utils.tracking_nonce import get_tracking_nonce from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + hm_logger = None s_decimal_0 = Decimal(0) @@ -81,6 +84,7 @@ def logger(cls) -> HummingbotLogger: return cls._logger def __init__(self, + client_config_map: "ClientConfigAdapter", mexc_api_key: str, mexc_secret_key: str, poll_interval: float = 5.0, @@ -88,7 +92,7 @@ def __init__(self, trading_pairs: Optional[List[str]] = None, trading_required: bool = True): - super().__init__() + super().__init__(client_config_map=client_config_map) self._throttler = AsyncThrottler(CONSTANTS.RATE_LIMITS) self._shared_client = aiohttp.ClientSession() self._async_scheduler = AsyncCallScheduler(call_interval=0.5) diff --git a/hummingbot/connector/gateway_EVM_AMM.py b/hummingbot/connector/gateway_EVM_AMM.py index d93c66a622..d60393eed4 100644 --- a/hummingbot/connector/gateway_EVM_AMM.py +++ b/hummingbot/connector/gateway_EVM_AMM.py @@ -1029,5 +1029,5 @@ def in_flight_orders(self) -> Dict[str, GatewayInFlightOrder]: return self._in_flight_orders def _get_gateway_instance(self) -> GatewayHttpClient: - gateway_instance = GatewayHttpClient.get_instance(self.client_config_map) + gateway_instance = GatewayHttpClient.get_instance(self._client_config) return gateway_instance diff --git a/hummingbot/connector/mock/mock_paper_exchange/mock_paper_exchange.pyx b/hummingbot/connector/mock/mock_paper_exchange/mock_paper_exchange.pyx index 2780e4d4c4..c9b7735bd6 100644 --- a/hummingbot/connector/mock/mock_paper_exchange/mock_paper_exchange.pyx +++ b/hummingbot/connector/mock/mock_paper_exchange/mock_paper_exchange.pyx @@ -1,5 +1,5 @@ from decimal import Decimal -from typing import List, Optional, Tuple +from typing import List, Optional, Tuple, TYPE_CHECKING import numpy as np @@ -17,13 +17,21 @@ from hummingbot.core.data_type.order_book import OrderBookRow from hummingbot.core.data_type.trade_fee import TradeFeeSchema from hummingbot.core.network_iterator import NetworkStatus +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + s_decimal_0 = Decimal("0") cdef class MockPaperExchange(PaperTradeExchange): - def __init__(self, trade_fee_schema: Optional[TradeFeeSchema] = None): - PaperTradeExchange.__init__(self, MockOrderTracker(), MockPaperExchange, exchange_name="mock") + def __init__(self, client_config_map: "ClientConfigAdapter", trade_fee_schema: Optional[TradeFeeSchema] = None): + PaperTradeExchange.__init__( + self, + client_config_map, + MockOrderTracker(), + MockPaperExchange, + exchange_name="mock") trade_fee_schema = trade_fee_schema or TradeFeeSchema( maker_percent_fee_decimal=Decimal("0"), taker_percent_fee_decimal=Decimal("0") diff --git a/hummingbot/core/api_throttler/async_throttler_base.py b/hummingbot/core/api_throttler/async_throttler_base.py index 8c551bec4a..d66537510a 100644 --- a/hummingbot/core/api_throttler/async_throttler_base.py +++ b/hummingbot/core/api_throttler/async_throttler_base.py @@ -35,14 +35,12 @@ def __init__(self, :param retry_interval: Time between every capacity check. :param safety_margin: Percentage of limit to be added as a safety margin when calculating capacity to ensure calls are within the limit. """ - from hummingbot.client.hummingbot_application import HummingbotApplication # avoids circular import - # Rate Limit Definitions self._rate_limits: List[RateLimit] = copy.deepcopy(rate_limits) - client_config = HummingbotApplication.main_application().client_config_map + client_config = self._client_config_map() # If configured, users can define the percentage of rate limits to allocate to the throttler. - self.limits_pct: Decimal = client_config.rate_limits_share_pct + self.limits_pct: Decimal = client_config.rate_limits_share_pct / 100 for rate_limit in self._rate_limits: rate_limit.limit = max(Decimal("1"), math.floor(Decimal(str(rate_limit.limit)) * self.limits_pct)) @@ -63,6 +61,11 @@ def __init__(self, # Shared asyncio.Lock instance to prevent multiple async ContextManager from accessing the _task_logs variable self._lock = asyncio.Lock() + def _client_config_map(self): + from hummingbot.client.hummingbot_application import HummingbotApplication # avoids circular import + + return HummingbotApplication.main_application().client_config_map + def get_related_limits(self, limit_id: str) -> Tuple[RateLimit, List[Tuple[RateLimit, int]]]: rate_limit: Optional[RateLimit] = self._id_to_limit_map.get(limit_id, None) diff --git a/hummingbot/strategy/cross_exchange_market_making/start.py b/hummingbot/strategy/cross_exchange_market_making/start.py index ee9de162c5..9b7f797f70 100644 --- a/hummingbot/strategy/cross_exchange_market_making/start.py +++ b/hummingbot/strategy/cross_exchange_market_making/start.py @@ -18,7 +18,7 @@ def start(self): raw_taker_trading_pair = xemm_map.get("taker_market_trading_pair").value min_profitability = xemm_map.get("min_profitability").value / Decimal("100") order_amount = xemm_map.get("order_amount").value - strategy_report_interval = self.clientconfig_map.strategy_report_interval + strategy_report_interval = self.client_config_map.strategy_report_interval limit_order_min_expiration = xemm_map.get("limit_order_min_expiration").value cancel_order_threshold = xemm_map.get("cancel_order_threshold").value / Decimal("100") active_order_canceling = xemm_map.get("active_order_canceling").value diff --git a/hummingbot/strategy/pure_market_making/start.py b/hummingbot/strategy/pure_market_making/start.py index 3b7d958e12..92dcb856fd 100644 --- a/hummingbot/strategy/pure_market_making/start.py +++ b/hummingbot/strategy/pure_market_making/start.py @@ -1,22 +1,15 @@ -from typing import ( - List, - Tuple, - Optional -) +from decimal import Decimal +from typing import List, Optional, Tuple from hummingbot.client.hummingbot_application import HummingbotApplication -from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple -from hummingbot.strategy.order_book_asset_price_delegate import OrderBookAssetPriceDelegate -from hummingbot.strategy.api_asset_price_delegate import APIAssetPriceDelegate -from hummingbot.strategy.pure_market_making import ( - PureMarketMakingStrategy, - InventoryCostPriceDelegate, -) -from hummingbot.strategy.pure_market_making.pure_market_making_config_map import pure_market_making_config_map as c_map from hummingbot.connector.exchange.paper_trade import create_paper_trade_market from hummingbot.connector.exchange_base import ExchangeBase -from decimal import Decimal +from hummingbot.strategy.api_asset_price_delegate import APIAssetPriceDelegate +from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple +from hummingbot.strategy.order_book_asset_price_delegate import OrderBookAssetPriceDelegate +from hummingbot.strategy.pure_market_making import InventoryCostPriceDelegate, PureMarketMakingStrategy from hummingbot.strategy.pure_market_making.moving_price_band import MovingPriceBand +from hummingbot.strategy.pure_market_making.pure_market_making_config_map import pure_market_making_config_map as c_map def start(self): @@ -93,7 +86,7 @@ def convert_decimal_string_to_list(string: Optional[str], divisor: Decimal = Dec asset_price_delegate = None if price_source == "external_market": asset_trading_pair: str = price_source_market - ext_market = create_paper_trade_market(price_source_exchange, [asset_trading_pair]) + ext_market = create_paper_trade_market(price_source_exchange, self.client_config_map, [asset_trading_pair]) self.markets[price_source_exchange]: ExchangeBase = ext_market asset_price_delegate = OrderBookAssetPriceDelegate(ext_market, asset_trading_pair) elif price_source == "custom_api": diff --git a/hummingbot/user/user_balances.py b/hummingbot/user/user_balances.py index 4eea20d68c..27b4cae624 100644 --- a/hummingbot/user/user_balances.py +++ b/hummingbot/user/user_balances.py @@ -20,9 +20,11 @@ def connect_market(exchange, client_config_map: ClientConfigMap, **api_details): if api_details or conn_setting.uses_gateway_generic_connector(): connector_class = get_connector_class(exchange) init_params = conn_setting.conn_init_parameters(api_details) - init_params.update(trading_pairs=gateway_connector_trading_pairs(conn_setting.name)) read_only_client_config = ReadOnlyClientConfigAdapter.lock_config(client_config_map) - connector = connector_class(read_only_client_config, **init_params) + init_params.update( + trading_pairs=gateway_connector_trading_pairs(conn_setting.name), + client_config_map=read_only_client_config) + connector = connector_class(**init_params) return connector # return error message if the _update_balances fails diff --git a/test/hummingbot/client/command/test_balance_command.py b/test/hummingbot/client/command/test_balance_command.py index 6170139a20..b39873a6f3 100644 --- a/test/hummingbot/client/command/test_balance_command.py +++ b/test/hummingbot/client/command/test_balance_command.py @@ -1,13 +1,12 @@ import asyncio import unittest -from copy import deepcopy +from decimal import Decimal +from test.mock.mock_cli import CLIMockingAssistant from typing import Awaitable -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock, patch from hummingbot.client.config.config_helpers import read_system_configs_from_yml -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.hummingbot_application import HummingbotApplication -from test.mock.mock_cli import CLIMockingAssistant class BalanceCommandTest(unittest.TestCase): @@ -21,17 +20,11 @@ def setUp(self, _: MagicMock) -> None: self.app = HummingbotApplication() self.cli_mock_assistant = CLIMockingAssistant(self.app.app) self.cli_mock_assistant.start() - self.global_config_backup = deepcopy(global_config_map) def tearDown(self) -> None: self.cli_mock_assistant.stop() - self.reset_global_config() super().tearDown() - def reset_global_config(self): - for key, value in self.global_config_backup.items(): - global_config_map[key] = value - @staticmethod def get_async_sleep_fn(delay: float): async def async_sleep(*_, **__): @@ -64,7 +57,7 @@ def test_show_balances_handles_network_timeouts( self, all_balances_all_exchanges_mock ): all_balances_all_exchanges_mock.side_effect = self.get_async_sleep_fn(delay=0.02) - global_config_map["other_commands_timeout"].value = 0.01 + self.app.client_config_map.commands_timeout.other_commands_timeout = Decimal("0.01") with self.assertRaises(asyncio.TimeoutError): self.async_run_with_timeout_coroutine_must_raise_timeout(self.app.show_balances()) diff --git a/test/hummingbot/client/command/test_config_command.py b/test/hummingbot/client/command/test_config_command.py index 31ad01cb66..a8ac81c31b 100644 --- a/test/hummingbot/client/command/test_config_command.py +++ b/test/hummingbot/client/command/test_config_command.py @@ -1,18 +1,16 @@ import asyncio import unittest from collections import Awaitable -from copy import deepcopy from decimal import Decimal from test.mock.mock_cli import CLIMockingAssistant from unittest.mock import MagicMock, patch from pydantic import Field -from hummingbot.client.command.config_command import client_configs_to_display, color_settings_to_display +from hummingbot.client.config.client_config_map import ClientConfigMap from hummingbot.client.config.config_data_types import BaseClientModel, BaseStrategyConfigMap, ClientFieldData from hummingbot.client.config.config_helpers import ClientConfigAdapter, read_system_configs_from_yml from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.hummingbot_application import HummingbotApplication @@ -24,24 +22,21 @@ def setUp(self, _: MagicMock) -> None: self.async_run_with_timeout(read_system_configs_from_yml()) - self.app = HummingbotApplication() + self.client_config = ClientConfigMap() + self.config_adapter = ClientConfigAdapter(self.client_config) + + self.app = HummingbotApplication(client_config_map=self.config_adapter) self.cli_mock_assistant = CLIMockingAssistant(self.app.app) self.cli_mock_assistant.start() - self.global_config_backup = deepcopy(global_config_map) def tearDown(self) -> None: self.cli_mock_assistant.stop() - self.reset_global_config() super().tearDown() def async_run_with_timeout(self, coroutine: Awaitable, timeout: float = 1): ret = self.ev_loop.run_until_complete(asyncio.wait_for(coroutine, timeout)) return ret - def reset_global_config(self): - for key, value in self.global_config_backup.items(): - global_config_map[key] = value - @patch("hummingbot.client.hummingbot_application.get_strategy_config_map") @patch("hummingbot.client.hummingbot_application.HummingbotApplication.notify") def test_list_configs(self, notify_mock, get_strategy_config_map_mock): @@ -50,18 +45,6 @@ def test_list_configs(self, notify_mock, get_strategy_config_map_mock): strategy_name = "some-strategy" self.app.strategy_name = strategy_name - tables_format_config_var = global_config_map["tables_format"] - global_config_map.clear() - global_config_map[tables_format_config_var.key] = tables_format_config_var - tables_format_config_var.value = "psql" - global_config_map[client_configs_to_display[0]] = ConfigVar(key=client_configs_to_display[0], prompt="") - global_config_map[client_configs_to_display[0]].value = "first" - global_config_map[client_configs_to_display[1]] = ConfigVar(key=client_configs_to_display[1], prompt="") - global_config_map[client_configs_to_display[1]].value = "second" - global_config_map[color_settings_to_display[0]] = ConfigVar(key=color_settings_to_display[0], prompt="") - global_config_map[color_settings_to_display[0]].value = "third" - global_config_map[color_settings_to_display[1]] = ConfigVar(key=color_settings_to_display[1], prompt="") - global_config_map[color_settings_to_display[1]].value = "fourth" strategy_config_map_mock = { "five": ConfigVar(key="five", prompt=""), "six": ConfigVar(key="six", prompt="", default="sixth"), @@ -75,27 +58,40 @@ def test_list_configs(self, notify_mock, get_strategy_config_map_mock): self.assertEqual(6, len(captures)) self.assertEqual("\nGlobal Configurations:", captures[0]) - df_str_expected = ( - " +---------------------+---------+" - "\n | Key | Value |" - "\n |---------------------+---------|" - "\n | tables_format | psql |" - "\n | autofill_import | first |" - "\n | kill_switch_enabled | second |" - "\n +---------------------+---------+" - ) + df_str_expected = (" +--------------------------+----------------------+\n" + " | Key | Value |\n" + " |--------------------------+----------------------|\n" + " | kill_switch_mode | kill_switch_disabled |\n" + " | autofill_import | disabled |\n" + " | telegram_mode | telegram_disabled |\n" + " | send_error_logs | True |\n" + " | pmm_script_mode | pmm_script_disabled |\n" + " | gateway | gateway |\n" + " | ∟ gateway_api_host | localhost |\n" + " | ∟ gateway_api_port | 5000 |\n" + " | rate_oracle_source | binance |\n" + " | global_token | None |\n" + " | ∟ global_token_symbol | $ |\n" + " | rate_limits_share_pct | 100 |\n" + " | commands_timeout | None |\n" + " | ∟ create_command_timeout | 10 |\n" + " | ∟ other_commands_timeout | 30 |\n" + " | tables_format | psql |\n" + " +--------------------------+----------------------+") self.assertEqual(df_str_expected, captures[1]) self.assertEqual("\nColor Settings:", captures[2]) - df_str_expected = ( - " +-------------+---------+" - "\n | Key | Value |" - "\n |-------------+---------|" - "\n | top-pane | third |" - "\n | bottom-pane | fourth |" - "\n +-------------+---------+" - ) + df_str_expected = (" +--------------------+---------+\n" + " | Key | Value |\n" + " |--------------------+---------|\n" + " | ∟ top_pane | #000000 |\n" + " | ∟ bottom_pane | #000000 |\n" + " | ∟ output_pane | #262626 |\n" + " | ∟ input_pane | #1C1C1C |\n" + " | ∟ logs_pane | #121212 |\n" + " | ∟ terminal_primary | #5FFFD7 |\n" + " +--------------------+---------+") self.assertEqual(df_str_expected, captures[3]) self.assertEqual("\nStrategy Configurations:", captures[4]) @@ -119,11 +115,6 @@ def test_list_configs_pydantic_model(self, notify_mock, get_strategy_config_map_ strategy_name = "some-strategy" self.app.strategy_name = strategy_name - tables_format_config_var = global_config_map["tables_format"] - global_config_map.clear() - global_config_map[tables_format_config_var.key] = tables_format_config_var - tables_format_config_var.value = "psql" - class DoubleNestedModel(BaseClientModel): double_nested_attr: float = Field(default=3.0) diff --git a/test/hummingbot/client/command/test_connect_command.py b/test/hummingbot/client/command/test_connect_command.py index ffef970e49..f2bda3ff1f 100644 --- a/test/hummingbot/client/command/test_connect_command.py +++ b/test/hummingbot/client/command/test_connect_command.py @@ -1,14 +1,13 @@ import asyncio import unittest -from copy import deepcopy from test.mock.mock_cli import CLIMockingAssistant from typing import Awaitable from unittest.mock import AsyncMock, MagicMock, patch import pandas as pd -from hummingbot.client.config.config_helpers import read_system_configs_from_yml -from hummingbot.client.config.global_config_map import global_config_map +from hummingbot.client.config.client_config_map import ClientConfigMap, DBSqliteMode +from hummingbot.client.config.config_helpers import ClientConfigAdapter, read_system_configs_from_yml from hummingbot.client.config.security import Security from hummingbot.client.hummingbot_application import HummingbotApplication @@ -20,22 +19,17 @@ def setUp(self, _: MagicMock) -> None: self.ev_loop = asyncio.get_event_loop() self.async_run_with_timeout(read_system_configs_from_yml()) + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) - self.app = HummingbotApplication() + self.app = HummingbotApplication(client_config_map=self.client_config_map) self.cli_mock_assistant = CLIMockingAssistant(self.app.app) self.cli_mock_assistant.start() - self.global_config_backup = deepcopy(global_config_map) def tearDown(self) -> None: self.cli_mock_assistant.stop() - self.reset_global_config() Security._decryption_done.clear() super().tearDown() - def reset_global_config(self): - for key, value in self.global_config_backup.items(): - global_config_map[key] = value - @staticmethod def get_async_sleep_fn(delay: float): async def async_sleep(*_, **__): @@ -82,7 +76,6 @@ def test_connect_exchange_success( api_secret = "someSecret" api_keys_mock.return_value = {"binance_api_key": api_key, "binance_api_secret": api_secret} connector_config_file_exists_mock.return_value = False - global_config_map["other_commands_timeout"].value = 30 self.cli_mock_assistant.queue_prompt_reply(api_key) # binance API key self.cli_mock_assistant.queue_prompt_reply(api_secret) # binance API secret @@ -131,7 +124,7 @@ def test_connect_exchange_handles_network_timeouts( __: MagicMock, ): add_exchange_mock.side_effect = self.get_async_sleep_fn(delay=0.02) - global_config_map["other_commands_timeout"].value = 0.01 + self.client_config_map.commands_timeout.other_commands_timeout = 0.01 api_key = "someKey" api_secret = "someSecret" api_keys_mock.return_value = {"binance_api_key": api_key, "binance_api_secret": api_secret} @@ -153,7 +146,7 @@ def test_connect_exchange_handles_network_timeouts( @patch("hummingbot.client.config.security.Security.wait_til_decryption_done") def test_connection_df_handles_network_timeouts(self, _: AsyncMock, update_exchanges_mock: AsyncMock): update_exchanges_mock.side_effect = self.get_async_sleep_fn(delay=0.02) - global_config_map["other_commands_timeout"].value = 0.01 + self.client_config_map.commands_timeout.other_commands_timeout = 0.01 with self.assertRaises(asyncio.TimeoutError): self.async_run_with_timeout_coroutine_must_raise_timeout(self.app.connection_df()) @@ -169,7 +162,7 @@ def test_connection_df_handles_network_timeouts_logs_hidden(self, _: AsyncMock, self.cli_mock_assistant.toggle_logs() update_exchanges_mock.side_effect = self.get_async_sleep_fn(delay=0.02) - global_config_map["other_commands_timeout"].value = 0.01 + self.client_config_map.commands_timeout.other_commands_timeout = 0.01 with self.assertRaises(asyncio.TimeoutError): self.async_run_with_timeout_coroutine_must_raise_timeout(self.app.connection_df()) @@ -182,7 +175,7 @@ def test_connection_df_handles_network_timeouts_logs_hidden(self, _: AsyncMock, @patch("hummingbot.client.hummingbot_application.HummingbotApplication.notify") @patch("hummingbot.client.hummingbot_application.HummingbotApplication.connection_df") def test_show_connections(self, connection_df_mock, notify_mock): - global_config_map["tables_format"].value = "psql" + self.client_config_map.db_mode = DBSqliteMode() Security._decryption_done.set() diff --git a/test/hummingbot/client/command/test_create_command.py b/test/hummingbot/client/command/test_create_command.py index b2640aa8ef..15ad53a5d0 100644 --- a/test/hummingbot/client/command/test_create_command.py +++ b/test/hummingbot/client/command/test_create_command.py @@ -1,13 +1,16 @@ import asyncio import unittest -from copy import deepcopy from decimal import Decimal from test.mock.mock_cli import CLIMockingAssistant from typing import Awaitable from unittest.mock import AsyncMock, MagicMock, patch -from hummingbot.client.config.config_helpers import get_strategy_config_map, read_system_configs_from_yml -from hummingbot.client.config.global_config_map import global_config_map +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ( + ClientConfigAdapter, + get_strategy_config_map, + read_system_configs_from_yml, +) from hummingbot.client.hummingbot_application import HummingbotApplication @@ -18,21 +21,16 @@ def setUp(self, _: MagicMock) -> None: self.ev_loop = asyncio.get_event_loop() self.async_run_with_timeout(read_system_configs_from_yml()) + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) - self.app = HummingbotApplication() + self.app = HummingbotApplication(client_config_map=self.client_config_map) self.cli_mock_assistant = CLIMockingAssistant(self.app.app) self.cli_mock_assistant.start() - self.global_config_backup = deepcopy(global_config_map) def tearDown(self) -> None: self.cli_mock_assistant.stop() - self.reset_global_config() super().tearDown() - def reset_global_config(self): - for key, value in self.global_config_backup.items(): - global_config_map[key] = value - @staticmethod def get_async_sleep_fn(delay: float): async def async_sleep(*_, **__): @@ -80,8 +78,8 @@ def test_prompt_for_configuration_re_prompts_on_lower_than_minimum_amount( config_maps = [] save_to_yml_mock.side_effect = lambda _, cm: config_maps.append(cm) - global_config_map["create_command_timeout"].value = 10 - global_config_map["other_commands_timeout"].value = 30 + self.client_config_map.commands_timeout.create_command_timeout = 10 + self.client_config_map.commands_timeout.other_commands_timeout = 30 strategy_name = "some-strategy" strategy_file_name = f"{strategy_name}.yml" base_strategy = "pure_market_making" @@ -119,8 +117,8 @@ def test_prompt_for_configuration_accepts_zero_amount_on_get_last_price_network_ config_maps = [] save_to_yml_mock.side_effect = lambda _, cm: config_maps.append(cm) - global_config_map["create_command_timeout"].value = 0.005 - global_config_map["other_commands_timeout"].value = 0.01 + self.client_config_map.commands_timeout.create_command_timeout = 0.005 + self.client_config_map.commands_timeout.other_commands_timeout = 0.01 strategy_name = "some-strategy" strategy_file_name = f"{strategy_name}.yml" base_strategy = "pure_market_making" @@ -189,8 +187,8 @@ def test_prompt_for_configuration_handles_status_network_timeout( get_last_price_mock.return_value = None validate_required_connections_mock.side_effect = self.get_async_sleep_fn(delay=0.02) is_decryption_done_mock.return_value = True - global_config_map["create_command_timeout"].value = 0.005 - global_config_map["other_commands_timeout"].value = 0.01 + self.client_config_map.commands_timeout.create_command_timeout = 0.005 + self.client_config_map.commands_timeout.other_commands_timeout = 0.01 strategy_file_name = "some-strategy.yml" self.cli_mock_assistant.queue_prompt_reply("pure_market_making") # strategy self.cli_mock_assistant.queue_prompt_reply("binance") # spot connector diff --git a/test/hummingbot/client/command/test_history_command.py b/test/hummingbot/client/command/test_history_command.py index 4d1bc39a8c..989cbe8870 100644 --- a/test/hummingbot/client/command/test_history_command.py +++ b/test/hummingbot/client/command/test_history_command.py @@ -2,15 +2,14 @@ import datetime import time import unittest -from copy import deepcopy from decimal import Decimal from pathlib import Path from test.mock.mock_cli import CLIMockingAssistant from typing import Awaitable, List from unittest.mock import AsyncMock, MagicMock, patch -from hummingbot.client.config.config_helpers import read_system_configs_from_yml -from hummingbot.client.config.global_config_map import global_config_map +from hummingbot.client.config.client_config_map import ClientConfigMap, DBSqliteMode +from hummingbot.client.config.config_helpers import ClientConfigAdapter, read_system_configs_from_yml from hummingbot.client.hummingbot_application import HummingbotApplication from hummingbot.connector.exchange.paper_trade import PaperTradeExchange from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee @@ -26,25 +25,20 @@ def setUp(self, _: MagicMock) -> None: self.ev_loop = asyncio.get_event_loop() self.async_run_with_timeout(read_system_configs_from_yml()) + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) - self.app = HummingbotApplication() + self.app = HummingbotApplication(client_config_map=self.client_config_map) self.cli_mock_assistant = CLIMockingAssistant(self.app.app) self.cli_mock_assistant.start() - self.global_config_backup = deepcopy(global_config_map) self.mock_strategy_name = "test-strategy" def tearDown(self) -> None: self.cli_mock_assistant.stop() - self.reset_global_config() db_path = Path(SQLConnectionManager.create_db_path(db_name=self.mock_strategy_name)) db_path.unlink(missing_ok=True) super().tearDown() - def reset_global_config(self): - for key, value in self.global_config_backup.items(): - global_config_map[key] = value - @staticmethod def get_async_sleep_fn(delay: float): async def async_sleep(*_, **__): @@ -99,7 +93,7 @@ def get_trades(self) -> List[TradeFill]: @patch("hummingbot.client.command.history_command.HistoryCommand.get_current_balances") def test_history_report_raises_on_get_current_balances_network_timeout(self, get_current_balances_mock: AsyncMock): get_current_balances_mock.side_effect = self.get_async_sleep_fn(delay=0.02) - global_config_map["other_commands_timeout"].value = 0.01 + self.client_config_map.commands_timeout.other_commands_timeout = 0.01 trades = self.get_trades() with self.assertRaises(asyncio.TimeoutError): @@ -114,7 +108,7 @@ def test_history_report_raises_on_get_current_balances_network_timeout(self, get @patch("hummingbot.client.hummingbot_application.HummingbotApplication.notify") def test_list_trades(self, notify_mock): - global_config_map["tables_format"].value = "psql" + self.client_config_map.db_mode = DBSqliteMode() captures = [] notify_mock.side_effect = lambda s: captures.append(s) diff --git a/test/hummingbot/client/command/test_order_book_command.py b/test/hummingbot/client/command/test_order_book_command.py index fe6128975d..554a013d1d 100644 --- a/test/hummingbot/client/command/test_order_book_command.py +++ b/test/hummingbot/client/command/test_order_book_command.py @@ -1,11 +1,10 @@ import asyncio import unittest from collections import Awaitable -from copy import deepcopy from unittest.mock import MagicMock, patch -from hummingbot.client.config.config_helpers import read_system_configs_from_yml -from hummingbot.client.config.global_config_map import global_config_map +from hummingbot.client.config.client_config_map import ClientConfigMap, DBSqliteMode +from hummingbot.client.config.config_helpers import ClientConfigAdapter, read_system_configs_from_yml from hummingbot.client.hummingbot_application import HummingbotApplication from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange @@ -17,31 +16,23 @@ def setUp(self, _: MagicMock) -> None: self.ev_loop = asyncio.get_event_loop() self.async_run_with_timeout(read_system_configs_from_yml()) + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) - self.app = HummingbotApplication() - self.global_config_backup = deepcopy(global_config_map) - - def tearDown(self) -> None: - self.reset_global_config() - super().tearDown() + self.app = HummingbotApplication(client_config_map=self.client_config_map) def async_run_with_timeout(self, coroutine: Awaitable, timeout: float = 1): ret = self.ev_loop.run_until_complete(asyncio.wait_for(coroutine, timeout)) return ret - def reset_global_config(self): - for key, value in self.global_config_backup.items(): - global_config_map[key] = value - @patch("hummingbot.client.hummingbot_application.HummingbotApplication.notify") def test_show_order_book(self, notify_mock): - global_config_map["tables_format"].value = "psql" + self.client_config_map.db_mode = DBSqliteMode() captures = [] notify_mock.side_effect = lambda s: captures.append(s) exchange_name = "paper" - exchange = MockPaperExchange() + exchange = MockPaperExchange(client_config_map=ClientConfigAdapter(ClientConfigMap())) self.app.markets[exchange_name] = exchange trading_pair = "BTC-USDT" exchange.set_balanced_order_book( diff --git a/test/hummingbot/client/command/test_status_command.py b/test/hummingbot/client/command/test_status_command.py index 163e7c399f..8cc83d922f 100644 --- a/test/hummingbot/client/command/test_status_command.py +++ b/test/hummingbot/client/command/test_status_command.py @@ -1,12 +1,11 @@ import asyncio import unittest -from copy import deepcopy from test.mock.mock_cli import CLIMockingAssistant from typing import Awaitable from unittest.mock import MagicMock, patch -from hummingbot.client.config.config_helpers import read_system_configs_from_yml -from hummingbot.client.config.global_config_map import global_config_map +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter, read_system_configs_from_yml from hummingbot.client.hummingbot_application import HummingbotApplication @@ -17,21 +16,16 @@ def setUp(self, _: MagicMock) -> None: self.ev_loop = asyncio.get_event_loop() self.async_run_with_timeout(read_system_configs_from_yml()) + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) - self.app = HummingbotApplication() + self.app = HummingbotApplication(client_config_map=self.client_config_map) self.cli_mock_assistant = CLIMockingAssistant(self.app.app) self.cli_mock_assistant.start() - self.global_config_backup = deepcopy(global_config_map) def tearDown(self) -> None: self.cli_mock_assistant.stop() - self.reset_global_config() super().tearDown() - def reset_global_config(self): - for key, value in self.global_config_backup.items(): - global_config_map[key] = value - @staticmethod def get_async_sleep_fn(delay: float): async def async_sleep(*_, **__): @@ -67,7 +61,7 @@ def test_status_check_all_handles_network_timeouts( ): validate_required_connections_mock.side_effect = self.get_async_sleep_fn(delay=0.02) validate_configs_mock.return_value = [] - global_config_map["other_commands_timeout"].value = 0.01 + self.client_config_map.commands_timeout.other_commands_timeout = 0.01 is_decryption_done_mock.return_value = True strategy_name = "avellaneda_market_making" self.app.strategy_name = strategy_name diff --git a/test/hummingbot/client/command/test_ticker_command.py b/test/hummingbot/client/command/test_ticker_command.py index 77f0ec92c2..e59df1ef12 100644 --- a/test/hummingbot/client/command/test_ticker_command.py +++ b/test/hummingbot/client/command/test_ticker_command.py @@ -1,11 +1,10 @@ import asyncio import unittest from collections import Awaitable -from copy import deepcopy from unittest.mock import MagicMock, patch -from hummingbot.client.config.config_helpers import read_system_configs_from_yml -from hummingbot.client.config.global_config_map import global_config_map +from hummingbot.client.config.client_config_map import ClientConfigMap, DBSqliteMode +from hummingbot.client.config.config_helpers import ClientConfigAdapter, read_system_configs_from_yml from hummingbot.client.hummingbot_application import HummingbotApplication from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange @@ -17,31 +16,23 @@ def setUp(self, _: MagicMock) -> None: self.ev_loop = asyncio.get_event_loop() self.async_run_with_timeout(read_system_configs_from_yml()) + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) - self.app = HummingbotApplication() - self.global_config_backup = deepcopy(global_config_map) - - def tearDown(self) -> None: - self.reset_global_config() - super().tearDown() + self.app = HummingbotApplication(client_config_map=self.client_config_map) def async_run_with_timeout(self, coroutine: Awaitable, timeout: float = 1): ret = self.ev_loop.run_until_complete(asyncio.wait_for(coroutine, timeout)) return ret - def reset_global_config(self): - for key, value in self.global_config_backup.items(): - global_config_map[key] = value - @patch("hummingbot.client.hummingbot_application.HummingbotApplication.notify") def test_show_ticker(self, notify_mock): - global_config_map["tables_format"].value = "psql" + self.client_config_map.db_mode = DBSqliteMode() captures = [] notify_mock.side_effect = lambda s: captures.append(s) exchange_name = "paper" - exchange = MockPaperExchange() + exchange = MockPaperExchange(client_config_map=ClientConfigAdapter(ClientConfigMap())) self.app.markets[exchange_name] = exchange trading_pair = "BTC-USDT" exchange.set_balanced_order_book( diff --git a/test/hummingbot/client/config/test_config_data_types.py b/test/hummingbot/client/config/test_config_data_types.py index 8c2f4dc31a..ff64f799f4 100644 --- a/test/hummingbot/client/config/test_config_data_types.py +++ b/test/hummingbot/client/config/test_config_data_types.py @@ -20,6 +20,48 @@ from hummingbot.client.config.security import Security +class SomeEnum(ClientConfigEnum): + ONE = "one" + + +class DoubleNestedModel(BaseClientModel): + double_nested_attr: datetime = Field( + default=datetime(2022, 1, 1, 10, 30), + description="Double nested attr description" + ) + + +class NestedModel(BaseClientModel): + nested_attr: str = Field( + default="some value", + description="Nested attr\nmultiline description", + ) + double_nested_model: DoubleNestedModel = Field( + default=DoubleNestedModel(), + ) + + +class DummyModel(BaseClientModel): + some_attr: SomeEnum = Field( + default=SomeEnum.ONE, + description="Some description", + client_data=ClientFieldData(), + ) + nested_model: NestedModel = Field( + default=NestedModel(), + description="Nested model description", + ) + another_attr: Decimal = Field( + default=Decimal("1.0"), + description="Some other\nmultiline description", + ) + non_nested_no_description: time = Field(default=time(10, 30), ) + date_attr: date = Field(default=date(2022, 1, 2)) + + class Config: + title = "dummy_model" + + class BaseClientModelTest(unittest.TestCase): def test_schema_encoding_removes_client_data_functions(self): class DummyModel(BaseClientModel): @@ -111,44 +153,7 @@ class Config: self.assertIsInstance(actual.field_info, FieldInfo) def test_generate_yml_output_dict_with_comments(self): - class SomeEnum(ClientConfigEnum): - ONE = "one" - - class DoubleNestedModel(BaseClientModel): - double_nested_attr: datetime = Field( - default=datetime(2022, 1, 1, 10, 30), - description="Double nested attr description" - ) - - class NestedModel(BaseClientModel): - nested_attr: str = Field( - default="some value", - description="Nested attr\nmultiline description", - ) - double_nested_model: DoubleNestedModel = Field( - default=DoubleNestedModel(), - ) - - class DummyModel(BaseClientModel): - some_attr: SomeEnum = Field( - default=SomeEnum.ONE, - description="Some description", - ) - nested_model: NestedModel = Field( - default=NestedModel(), - description="Nested model description", - ) - another_attr: Decimal = Field( - default=Decimal("1.0"), - description="Some other\nmultiline description", - ) - non_nested_no_description: time = Field(default=time(10, 30),) - date_attr: date = Field(default=date(2022, 1, 2)) - - class Config: - title = "dummy_model" - - instance = ClientConfigAdapter(DummyModel()) + instance = self._nested_config_adapter() res_str = instance.generate_yml_output_str_with_comments() expected_str = """\ ############################## @@ -160,11 +165,8 @@ class Config: # Nested model description nested_model: - # Nested attr - # multiline description nested_attr: some value double_nested_model: - # Double nested attr description double_nested_attr: 2022-01-01 10:30:00 # Some other @@ -199,6 +201,26 @@ class Config: self.assertTrue(res_str.startswith(expected_str)) self.assertNotIn(secret_value, res_str) + def test_config_paths_includes_all_intermediate_keys(self): + adapter = self._nested_config_adapter() + + all_config_paths = list(adapter.config_paths()) + expected_config_paths = [ + 'some_attr', + 'nested_model', + 'nested_model.nested_attr', + 'nested_model.double_nested_model', + 'nested_model.double_nested_model.double_nested_attr', + 'another_attr', + 'non_nested_no_description', + 'date_attr', + ] + + self.assertEqual(expected_config_paths, all_config_paths) + + def _nested_config_adapter(self): + return ClientConfigAdapter(DummyModel()) + class BaseStrategyConfigMapTest(unittest.TestCase): def test_generate_yml_output_dict_title(self): diff --git a/test/hummingbot/client/config/test_config_helpers.py b/test/hummingbot/client/config/test_config_helpers.py index 27c81f4cc8..ed8c481e4c 100644 --- a/test/hummingbot/client/config/test_config_helpers.py +++ b/test/hummingbot/client/config/test_config_helpers.py @@ -1,17 +1,20 @@ import asyncio import unittest +from decimal import Decimal from pathlib import Path from tempfile import TemporaryDirectory -from typing import Awaitable, Optional +from typing import Awaitable, List, Optional from unittest.mock import MagicMock, patch from pydantic import Field, SecretStr from hummingbot.client.config import config_helpers +from hummingbot.client.config.client_config_map import ClientConfigMap, CommandShortcutModel from hummingbot.client.config.config_crypt import ETHKeyFileSecretManger -from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, BaseStrategyConfigMap +from hummingbot.client.config.config_data_types import BaseClientModel, BaseConnectorConfigMap, BaseStrategyConfigMap from hummingbot.client.config.config_helpers import ( ClientConfigAdapter, + ReadOnlyClientConfigAdapter, get_connector_config_yml_path, get_strategy_config_map, load_connector_config_map_from_file, @@ -36,6 +39,7 @@ def async_run_with_timeout(self, coroutine: Awaitable, timeout: float = 1): def get_async_sleep_fn(delay: float): async def async_sleep(*_, **__): await asyncio.sleep(delay) + return async_sleep def test_get_strategy_config_map(self): @@ -66,6 +70,53 @@ class Config: actual_str = f.read() self.assertEqual(expected_str, actual_str) + def test_save_command_shortcuts_to_yml(self): + class DummyStrategy(BaseClientModel): + command_shortcuts: List[CommandShortcutModel] = Field( + default=[ + CommandShortcutModel( + command="spreads", + help="Set bid and ask spread", + arguments=["Bid Spread", "Ask Spread"], + output=["config bid_spread $1", "config ask_spread $2"] + ) + ] + ) + another_attr: Decimal = Field( + default=Decimal("1.0"), + description="Some other\nmultiline description", + ) + + class Config: + title = "dummy_global_config" + + cm = ClientConfigAdapter(DummyStrategy()) + expected_str = ( + "######################################\n" + "### dummy_global_config config ###\n" + "######################################\n\n" + "command_shortcuts:\n" + "- command: spreads\n" + " help: Set bid and ask spread\n" + " arguments:\n" + " - Bid Spread\n" + " - Ask Spread\n" + " output:\n" + " - config bid_spread $1\n" + " - config ask_spread $2\n\n" + "# Some other\n" + "# multiline description\n" + "another_attr: 1.0\n" + ) + + with TemporaryDirectory() as d: + d = Path(d) + temp_file_name = d / "cm.yml" + save_to_yml(temp_file_name, cm) + with open(temp_file_name) as f: + actual_str = f.read() + self.assertEqual(expected_str, actual_str) + @patch("hummingbot.client.config.config_helpers.AllConnectorSettings.get_connector_config_keys") def test_load_connector_config_map_from_file_with_secrets(self, get_connector_config_keys_mock: MagicMock): class DummyConnectorModel(BaseConnectorConfigMap): @@ -84,3 +135,22 @@ class DummyConnectorModel(BaseConnectorConfigMap): cm_loaded = load_connector_config_map_from_file(temp_file_name) self.assertEqual(cm, cm_loaded) + + +class ReadOnlyClientAdapterTest(unittest.TestCase): + + def test_read_only_adapter_can_be_created(self): + adapter = ClientConfigAdapter(ClientConfigMap()) + read_only_adapter = ReadOnlyClientConfigAdapter(adapter.hb_config) + + self.assertEqual(adapter.hb_config, read_only_adapter.hb_config) + + def test_read_only_adapter_raises_exception_when_setting_value(self): + read_only_adapter = ReadOnlyClientConfigAdapter(ClientConfigMap()) + initial_instance_id = read_only_adapter.instance_id + + with self.assertRaises(AttributeError) as context: + read_only_adapter.instance_id = "newInstanceID" + + self.assertEqual("Cannot set an attribute on a read-only client adapter", str(context.exception)) + self.assertEqual(initial_instance_id, read_only_adapter.instance_id) diff --git a/test/hummingbot/client/config/test_config_templates.py b/test/hummingbot/client/config/test_config_templates.py deleted file mode 100644 index 22d996a40f..0000000000 --- a/test/hummingbot/client/config/test_config_templates.py +++ /dev/null @@ -1,72 +0,0 @@ -import unittest - -import ruamel.yaml - -from hummingbot import root_path -from hummingbot.client.config.config_helpers import get_strategy_config_map, get_strategy_template_path -from hummingbot.client.config.global_config_map import global_config_map - -import logging; logging.basicConfig(level=logging.INFO) - - -yaml_parser = ruamel.yaml.YAML() - - -class ConfigTemplatesUnitTest(unittest.TestCase): - - def test_global_config_template_complete(self): - global_config_template_path = root_path() / "hummingbot" / "templates" / "conf_global_TEMPLATE.yml" - - with open(global_config_template_path, "r") as template_fd: - template_data = yaml_parser.load(template_fd) - template_version = template_data.get("template_version", 0) - self.assertGreaterEqual(template_version, 1) - for key in template_data: - if key == "template_version": - continue - self.assertTrue(key in global_config_map, f"{key} not in global_config_map") - - for key in global_config_map: - self.assertTrue(key in template_data, f"{key} not in {global_config_template_path}") - - def test_strategy_config_template_complete_legacy(self): - strategies = [ # templates is a legacy approach — new strategies won't use it - "amm_arb", - "arbitrage", - "aroon_oscillator", - "celo_arb", - "cross_exchange_market_making", - "dev_0_hello_world", - "dev_1_get_order_book", - "dev_2_perform_trade", - "dev_5_vwap", - "dev_simple_trade", - "hedge", - "liquidity_mining", - "perpetual_market_making", - "pure_market_making", - "spot_perpetual_arbitrage", - "twap", - ] - - for strategy in strategies: - strategy_template_path = get_strategy_template_path(strategy) - strategy_config_map = get_strategy_config_map(strategy) - - with open(strategy_template_path, "r") as template_fd: - template_data = yaml_parser.load(template_fd) - template_version = template_data.get("template_version", 0) - self.assertGreaterEqual(template_version, 1, f"Template version too low at {strategy_template_path}") - for key in template_data: - if key == "template_version": - continue - self.assertTrue(key in strategy_config_map, f"{key} not in {strategy}_config_map") - - for key in strategy_config_map: - self.assertTrue(key in template_data, f"{key} not in {strategy_template_path}") - - def test_global_config_prompt_exists(self): - for key in global_config_map: - cvar = global_config_map[key] - if cvar.required: - self.assertTrue(cvar.prompt is not None) diff --git a/test/hummingbot/client/ui/test_custom_widgets.py b/test/hummingbot/client/ui/test_custom_widgets.py index 5f37269fe3..2c92cb0f12 100644 --- a/test/hummingbot/client/ui/test_custom_widgets.py +++ b/test/hummingbot/client/ui/test_custom_widgets.py @@ -1,10 +1,11 @@ import asyncio import unittest - from typing import Awaitable + from prompt_toolkit.document import Document -from hummingbot.client.config.config_helpers import read_system_configs_from_yml +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter, read_system_configs_from_yml from hummingbot.client.ui.custom_widgets import FormattedTextLexer @@ -20,7 +21,8 @@ def setUpClass(cls) -> None: def setUp(self) -> None: super().setUp() - self.lexer = FormattedTextLexer() + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) + self.lexer = FormattedTextLexer(client_config_map=self.client_config_map) self.lexer.text_style_tag_map.update(self.text_style_tag) self.lexer.html_tag_css_style_map.update(self.tag_css_style) @@ -43,7 +45,7 @@ def test_get_line_command_promt(self): document = Document(text=TEST_PROMPT_TEXT) get_line = self.lexer.lex_document(document) - expected_fragments = [(self.lexer.get_css_style("primary-label"), TEST_PROMPT_TEXT)] + expected_fragments = [(self.lexer.get_css_style("primary_label"), TEST_PROMPT_TEXT)] line_fragments = get_line(0) self.assertEqual(1, len(line_fragments)) @@ -56,10 +58,10 @@ def test_get_line_match_found(self): expected_fragments = [ ("", "SOME RANDOM TEXT WITH "), - (self.lexer.get_css_style("output-pane"), "&c"), + (self.lexer.get_css_style("output_pane"), "&c"), (self.lexer.get_css_style("SPECIAL_LABEL"), "SPECIAL_WORD"), ("", " AND "), - (self.lexer.get_css_style("output-pane"), "&c"), + (self.lexer.get_css_style("output_pane"), "&c"), (self.lexer.get_css_style("SPECIAL_LABEL"), "SPECIAL_WORD"), ("", ""), ] diff --git a/test/hummingbot/client/ui/test_hummingbot_cli.py b/test/hummingbot/client/ui/test_hummingbot_cli.py index 2de321efad..8ec9858c5a 100644 --- a/test/hummingbot/client/ui/test_hummingbot_cli.py +++ b/test/hummingbot/client/ui/test_hummingbot_cli.py @@ -4,7 +4,8 @@ from prompt_toolkit.widgets import Button -from hummingbot.client.config.config_helpers import read_system_configs_from_yml +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter, read_system_configs_from_yml from hummingbot.client.hummingbot_application import HummingbotApplication from hummingbot.client.tab.data_types import CommandTab from hummingbot.client.ui.custom_widgets import CustomTextArea @@ -25,9 +26,15 @@ def setUpClass(cls) -> None: def setUp(self) -> None: super().setUp() + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) tabs = {self.command_name: CommandTab(self.command_name, None, None, None, MagicMock())} self.mock_hb = MagicMock() - self.app = HummingbotCLI(None, None, None, tabs) + self.app = HummingbotCLI( + client_config_map=self.client_config_map, + input_handler=None, + bindings=None, + completer=None, + command_tabs=tabs) self.app.app = MagicMock() self.hb = HummingbotApplication() @@ -124,7 +131,7 @@ def __call__(self, _): handler: UIStartHandler = UIStartHandler() self.app.add_listener(HummingbotUIEvent.Start, handler) - self.app.did_start_ui(self.hb) + self.app.did_start_ui() mock_init_logging.assert_called() handler.mock.assert_called() diff --git a/test/hummingbot/client/ui/test_interface_utils.py b/test/hummingbot/client/ui/test_interface_utils.py index 080e71b93d..338fde1f63 100644 --- a/test/hummingbot/client/ui/test_interface_utils.py +++ b/test/hummingbot/client/ui/test_interface_utils.py @@ -1,15 +1,18 @@ +import asyncio import unittest -from copy import deepcopy from decimal import Decimal -import asyncio from typing import Awaitable -from unittest.mock import patch, MagicMock, AsyncMock, PropertyMock +from unittest.mock import AsyncMock, MagicMock, PropertyMock, patch import pandas as pd -from hummingbot.client.config.global_config_map import global_config_map -from hummingbot.client.ui.interface_utils import start_trade_monitor, \ - format_bytes, start_timer, start_process_monitor, format_df_for_printout +from hummingbot.client.ui.interface_utils import ( + format_bytes, + format_df_for_printout, + start_process_monitor, + start_timer, + start_trade_monitor, +) class ExpectedException(Exception): @@ -27,20 +30,10 @@ def setUp(self) -> None: super().setUp() self.ev_loop = asyncio.get_event_loop() - self.global_config_backup = deepcopy(global_config_map) - - def tearDown(self) -> None: - self.reset_global_config() - super().tearDown() - def async_run_with_timeout(self, coroutine: Awaitable, timeout: float = 1): ret = self.ev_loop.run_until_complete(asyncio.wait_for(coroutine, timeout)) return ret - def reset_global_config(self): - for key, value in self.global_config_backup.items(): - global_config_map[key] = value - def test_format_bytes(self): size = 1024. self.assertEqual("1.00 KB", format_bytes(size)) @@ -227,8 +220,7 @@ def test_format_df_for_printout_table_format_from_global_config(self): } ) - global_config_map.get("tables_format").value = "psql" - df_str = format_df_for_printout(df) + df_str = format_df_for_printout(df, table_format="psql") target_str = ( "+---------+----------+" "\n| first | second |" @@ -240,8 +232,7 @@ def test_format_df_for_printout_table_format_from_global_config(self): self.assertEqual(target_str, df_str) - global_config_map.get("tables_format").value = "simple" - df_str = format_df_for_printout(df) + df_str = format_df_for_printout(df, table_format="simple") target_str = ( " first second" "\n------- --------" diff --git a/test/hummingbot/client/ui/test_layout.py b/test/hummingbot/client/ui/test_layout.py index 5511493de1..ee56d63f15 100644 --- a/test/hummingbot/client/ui/test_layout.py +++ b/test/hummingbot/client/ui/test_layout.py @@ -1,23 +1,10 @@ import unittest -from copy import deepcopy -from hummingbot.client.config.global_config_map import global_config_map from hummingbot.client.hummingbot_application import HummingbotApplication from hummingbot.client.ui.layout import get_active_strategy, get_strategy_file class LayoutTest(unittest.TestCase): - def setUp(self) -> None: - super().setUp() - self.global_config_backup = deepcopy(global_config_map) - - def tearDown(self) -> None: - self.reset_global_config() - super().tearDown() - - def reset_global_config(self): - for key, value in self.global_config_backup.items(): - global_config_map[key] = value def test_get_active_strategy(self): hb = HummingbotApplication.main_application() @@ -25,7 +12,7 @@ def test_get_active_strategy(self): res = get_active_strategy() style, text = res[0] - self.assertEqual("class:log-field", style) + self.assertEqual("class:log_field", style) self.assertEqual(f"Strategy: {hb.strategy_name}", text) def test_get_strategy_file(self): @@ -34,5 +21,5 @@ def test_get_strategy_file(self): res = get_strategy_file() style, text = res[0] - self.assertEqual("class:log-field", style) + self.assertEqual("class:log_field", style) self.assertEqual(f"Strategy File: {hb._strategy_file_name}", text) diff --git a/test/hummingbot/client/ui/test_style.py b/test/hummingbot/client/ui/test_style.py index c056df47d0..74753add37 100644 --- a/test/hummingbot/client/ui/test_style.py +++ b/test/hummingbot/client/ui/test_style.py @@ -1,13 +1,14 @@ import unittest +from unittest.mock import patch from prompt_toolkit.styles import Style -from unittest.mock import patch -from hummingbot.client.ui.style import load_style, reset_style, hex_to_ansi +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter +from hummingbot.client.ui.style import hex_to_ansi, load_style, reset_style class StyleTest(unittest.TestCase): - class ConfigVar: value = None default = None @@ -20,26 +21,28 @@ def __init__(self, value, default=None): def test_load_style_unix(self, is_windows_mock): is_windows_mock.return_value = False - global_config_map = {} - global_config_map["top-pane"] = self.ConfigVar("#FAFAFA") - global_config_map["bottom-pane"] = self.ConfigVar("#FAFAFA") - global_config_map["output-pane"] = self.ConfigVar("#FAFAFA") - global_config_map["input-pane"] = self.ConfigVar("#FAFAFA") - global_config_map["logs-pane"] = self.ConfigVar("#FAFAFA") - global_config_map["terminal-primary"] = self.ConfigVar("#FCFCFC") - - global_config_map["primary-label"] = self.ConfigVar("#5FFFD7") - global_config_map["secondary-label"] = self.ConfigVar("#FFFFFF") - global_config_map["success-label"] = self.ConfigVar("#5FFFD7") - global_config_map["warning-label"] = self.ConfigVar("#FFFF00") - global_config_map["info-label"] = self.ConfigVar("#5FD7FF") - global_config_map["error-label"] = self.ConfigVar("#FF0000") + global_config_map = ClientConfigMap() + global_config_map.color.top_pane = "#FAFAFA" + global_config_map.color.bottom_pane = "#FAFAFA" + global_config_map.color.output_pane = "#FAFAFA" + global_config_map.color.input_pane = "#FAFAFA" + global_config_map.color.logs_pane = "#FAFAFA" + global_config_map.color.terminal_primary = "#FCFCFC" + + global_config_map.color.primary_label = "#5FFFD7" + global_config_map.color.secondary_label = "#FFFFFF" + global_config_map.color.success_label = "#5FFFD7" + global_config_map.color.warning_label = "#FFFF00" + global_config_map.color.info_label = "#5FD7FF" + global_config_map.color.error_label = "#FF0000" + + adapter = ClientConfigAdapter(global_config_map) style = Style.from_dict( { - "output-field": "bg:#FAFAFA #FCFCFC", - "input-field": "bg:#FAFAFA #FFFFFF", - "log-field": "bg:#FAFAFA #FFFFFF", + "output_field": "bg:#FAFAFA #FCFCFC", + "input_field": "bg:#FAFAFA #FFFFFF", + "log_field": "bg:#FAFAFA #FFFFFF", "header": "bg:#FAFAFA #AAAAAA", "footer": "bg:#FAFAFA #AAAAAA", "search": "bg:#000000 #93C36D", @@ -50,41 +53,43 @@ def test_load_style_unix(self, is_windows_mock): "tab_button.focused": "bg:#FCFCFC #FAFAFA", "tab_button": "bg:#FFFFFF #FAFAFA", # Label bg and font color - "primary-label": "bg:#5FFFD7 #FAFAFA", - "secondary-label": "bg:#FFFFFF #FAFAFA", - "success-label": "bg:#5FFFD7 #FAFAFA", - "warning-label": "bg:#FFFF00 #FAFAFA", - "info-label": "bg:#5FD7FF #FAFAFA", - "error-label": "bg:#FF0000 #FAFAFA", + "primary_label": "bg:#5FFFD7 #FAFAFA", + "secondary_label": "bg:#FFFFFF #FAFAFA", + "success_label": "bg:#5FFFD7 #FAFAFA", + "warning_label": "bg:#FFFF00 #FAFAFA", + "info_label": "bg:#5FD7FF #FAFAFA", + "error_label": "bg:#FF0000 #FAFAFA", } ) - self.assertEqual(style.class_names_and_attrs, load_style(global_config_map).class_names_and_attrs) + self.assertEqual(style.class_names_and_attrs, load_style(adapter).class_names_and_attrs) @patch("hummingbot.client.ui.style.is_windows") def test_load_style_windows(self, is_windows_mock): is_windows_mock.return_value = True - global_config_map = {} - global_config_map["top-pane"] = self.ConfigVar("#FAFAFA") - global_config_map["bottom-pane"] = self.ConfigVar("#FAFAFA") - global_config_map["output-pane"] = self.ConfigVar("#FAFAFA") - global_config_map["input-pane"] = self.ConfigVar("#FAFAFA") - global_config_map["logs-pane"] = self.ConfigVar("#FAFAFA") - global_config_map["terminal-primary"] = self.ConfigVar("#FCFCFC") - - global_config_map["primary-label"] = self.ConfigVar("#5FFFD7") - global_config_map["secondary-label"] = self.ConfigVar("#FFFFFF") - global_config_map["success-label"] = self.ConfigVar("#5FFFD7") - global_config_map["warning-label"] = self.ConfigVar("#FFFF00") - global_config_map["info-label"] = self.ConfigVar("#5FD7FF") - global_config_map["error-label"] = self.ConfigVar("#FF0000") + global_config_map = ClientConfigMap() + global_config_map.color.top_pane = "#FAFAFA" + global_config_map.color.bottom_pane = "#FAFAFA" + global_config_map.color.output_pane = "#FAFAFA" + global_config_map.color.input_pane = "#FAFAFA" + global_config_map.color.logs_pane = "#FAFAFA" + global_config_map.color.terminal_primary = "#FCFCFC" + + global_config_map.color.primary_label = "#5FFFD7" + global_config_map.color.secondary_label = "#FFFFFF" + global_config_map.color.success_label = "#5FFFD7" + global_config_map.color.warning_label = "#FFFF00" + global_config_map.color.info_label = "#5FD7FF" + global_config_map.color.error_label = "#FF0000" + + adapter = ClientConfigAdapter(global_config_map) style = Style.from_dict( { - "output-field": "bg:#ansiwhite #ansiwhite", - "input-field": "bg:#ansiwhite #ansiwhite", - "log-field": "bg:#ansiwhite #ansiwhite", + "output_field": "bg:#ansiwhite #ansiwhite", + "input_field": "bg:#ansiwhite #ansiwhite", + "log_field": "bg:#ansiwhite #ansiwhite", "header": "bg:#ansiwhite #ansiwhite", "footer": "bg:#ansiwhite #ansiwhite", "search": "#ansiwhite", @@ -95,60 +100,60 @@ def test_load_style_windows(self, is_windows_mock): "tab_button.focused": "bg:#ansiwhite #ansiwhite", "tab_button": "bg:#ansiwhite #ansiwhite", # Label bg and font color - "primary-label": "bg:#ansicyan #ansiwhite", - "secondary-label": "bg:#ansiwhite #ansiwhite", - "success-label": "bg:#ansicyan #ansiwhite", - "warning-label": "bg:#ansiyellow #ansiwhite", - "info-label": "bg:#ansicyan #ansiwhite", - "error-label": "bg:#ansired #ansiwhite", + "primary_label": "bg:#ansicyan #ansiwhite", + "secondary_label": "bg:#ansiwhite #ansiwhite", + "success_label": "bg:#ansicyan #ansiwhite", + "warning_label": "bg:#ansiyellow #ansiwhite", + "info_label": "bg:#ansicyan #ansiwhite", + "error_label": "bg:#ansired #ansiwhite", } ) - self.assertEqual(style.class_names_and_attrs, load_style(global_config_map).class_names_and_attrs) + self.assertEqual(style.class_names_and_attrs, load_style(adapter).class_names_and_attrs) def test_reset_style(self): + global_config_map = ClientConfigMap() + global_config_map.color.top_pane = "#FAFAFA" + global_config_map.color.bottom_pane = "#FAFAFA" + global_config_map.color.output_pane = "#FAFAFA" + global_config_map.color.input_pane = "#FAFAFA" + global_config_map.color.logs_pane = "#FAFAFA" + global_config_map.color.terminal_primary = "#FCFCFC" + + global_config_map.color.primary_label = "#FAFAFA" + global_config_map.color.secondary_label = "#FAFAFA" + global_config_map.color.success_label = "#FAFAFA" + global_config_map.color.warning_label = "#FAFAFA" + global_config_map.color.info_label = "#FAFAFA" + global_config_map.color.error_label = "#FAFAFA" + + adapter = ClientConfigAdapter(global_config_map) - global_config_map = {} - global_config_map["top-pane"] = self.ConfigVar("#FAFAFA", "#333333") - global_config_map["bottom-pane"] = self.ConfigVar("#FAFAFA", "#333333") - global_config_map["output-pane"] = self.ConfigVar("#FAFAFA", "#333333") - global_config_map["input-pane"] = self.ConfigVar("#FAFAFA", "#333333") - global_config_map["logs-pane"] = self.ConfigVar("#FAFAFA", "#333333") - global_config_map["terminal-primary"] = self.ConfigVar("#FCFCFC", "#010101") - - global_config_map["primary-label"] = self.ConfigVar("#FAFAFA", "#5FFFD7") - global_config_map["secondary-label"] = self.ConfigVar("#FAFAFA", "#FFFFFF") - global_config_map["success-label"] = self.ConfigVar("#FAFAFA", "#5FFFD7") - global_config_map["warning-label"] = self.ConfigVar("#FAFAFA", "#FFFF00") - global_config_map["info-label"] = self.ConfigVar("#FAFAFA", "#5FD7FF") - global_config_map["error-label"] = self.ConfigVar("#FAFAFA", "#FF0000") style = Style.from_dict( { - "output-field": "bg:#333333 #010101", - "input-field": "bg:#333333 #FFFFFF", - "log-field": "bg:#333333 #FFFFFF", - "header": "bg:#333333 #AAAAAA", - "footer": "bg:#333333 #AAAAAA", + "output_field": "bg:#262626 #5FFFD7", + "input_field": "bg:#1C1C1C #FFFFFF", + "log_field": "bg:#121212 #FFFFFF", + "header": "bg:#000000 #AAAAAA", + "footer": "bg:#000000 #AAAAAA", "search": "bg:#000000 #93C36D", "search.current": "bg:#000000 #1CD085", - "primary": "#010101", + "primary": "#5FFFD7", "warning": "#93C36D", "error": "#F5634A", - "tab_button.focused": "bg:#010101 #333333", - "tab_button": "bg:#FFFFFF #333333", - # Label bg and font color - "primary-label": "bg:#5FFFD7 #333333", - "secondary-label": "bg:#FFFFFF #333333", - "success-label": "bg:#5FFFD7 #333333", - "warning-label": "bg:#FFFF00 #333333", - "info-label": "bg:#5FD7FF #333333", - "error-label": "bg:#FF0000 #333333", - + "tab_button.focused": "bg:#5FFFD7 #121212", + "tab_button": "bg:#FFFFFF #121212", + "primary_label": "bg:#5FFFD7 #262626", + "secondary_label": "bg:#FFFFFF #262626", + "success_label": "bg:#5FFFD7 #262626", + "warning_label": "bg:#FFFF00 #262626", + "info_label": "bg:#5FD7FF #262626", + "error_label": "bg:#FF0000 #262626" } ) - self.assertEqual(style.class_names_and_attrs, reset_style(config_map=global_config_map, save=False).class_names_and_attrs) + self.assertEqual(style.class_names_and_attrs, reset_style(config_map=adapter, save=False).class_names_and_attrs) def test_hex_to_ansi(self): self.assertEqual("#ansiblack", hex_to_ansi("#000000")) diff --git a/test/hummingbot/connector/connector/gateway/test_gateway_cancel.py b/test/hummingbot/connector/connector/gateway/test_gateway_cancel.py index 434de6345f..f097ed3c38 100644 --- a/test/hummingbot/connector/connector/gateway/test_gateway_cancel.py +++ b/test/hummingbot/connector/connector/gateway/test_gateway_cancel.py @@ -1,31 +1,33 @@ -from bin import path_util # noqa: F401 - -from aiounittest import async_test -from aiohttp import ClientSession import asyncio -from async_timeout import timeout +import time +import unittest from contextlib import ExitStack, asynccontextmanager from decimal import Decimal from os.path import join, realpath -import time +from test.mock.http_recorder import HttpPlayer from typing import Generator, Optional, Set -import unittest from unittest.mock import patch +from aiohttp import ClientSession +from aiounittest import async_test +from async_timeout import timeout + +from bin import path_util # noqa: F401 +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.gateway_EVM_AMM import GatewayEVMAMM from hummingbot.connector.gateway_in_flight_order import GatewayInFlightOrder from hummingbot.core.clock import Clock, ClockMode from hummingbot.core.event.event_logger import EventLogger from hummingbot.core.event.events import ( - TradeType, MarketEvent, OrderCancelledEvent, - TokenApprovalEvent, TokenApprovalCancelledEvent, + TokenApprovalEvent, + TradeType, ) from hummingbot.core.gateway.gateway_http_client import GatewayHttpClient from hummingbot.core.utils.async_utils import safe_ensure_future -from test.mock.http_recorder import HttpPlayer WALLET_ADDRESS = "0x5821715133bB451bDE2d5BC6a4cE3430a4fdAF92" NETWORK = "ropsten" @@ -49,11 +51,13 @@ def setUpClass(cls) -> None: cls._db_path = realpath(join(__file__, "../fixtures/gateway_cancel_fixture.db")) cls._http_player = HttpPlayer(cls._db_path) cls._clock: Clock = Clock(ClockMode.REALTIME) + cls._client_config_map = ClientConfigAdapter(ClientConfigMap()) cls._connector: GatewayEVMAMM = GatewayEVMAMM( - "uniswap", - "ethereum", - NETWORK, - WALLET_ADDRESS, + client_config_map=cls._client_config_map, + connector_name="uniswap", + chain="ethereum", + network=NETWORK, + wallet_address=WALLET_ADDRESS, trading_pairs=[TRADING_PAIR], trading_required=True ) diff --git a/test/hummingbot/connector/connector/gateway/test_gateway_evm_amm.py b/test/hummingbot/connector/connector/gateway/test_gateway_evm_amm.py index 139e8035b9..07515deaa2 100644 --- a/test/hummingbot/connector/connector/gateway/test_gateway_evm_amm.py +++ b/test/hummingbot/connector/connector/gateway/test_gateway_evm_amm.py @@ -13,6 +13,8 @@ from async_timeout import timeout from bin import path_util # noqa: F401 +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.gateway_EVM_AMM import GatewayEVMAMM, GatewayInFlightOrder from hummingbot.core.clock import Clock, ClockMode from hummingbot.core.event.event_logger import EventLogger @@ -43,14 +45,17 @@ class GatewayEVMAMMConnectorUnitTest(unittest.TestCase): @classmethod def setUpClass(cls) -> None: super().setUpClass() + GatewayHttpClient.__instance = None cls._db_path = realpath(join(__file__, "../fixtures/gateway_evm_amm_fixture.db")) cls._http_player = HttpPlayer(cls._db_path) cls._clock: Clock = Clock(ClockMode.REALTIME) + cls._client_config_map = ClientConfigAdapter(ClientConfigMap()) cls._connector: GatewayEVMAMM = GatewayEVMAMM( - "uniswap", - "ethereum", - "ropsten", - "0x5821715133bB451bDE2d5BC6a4cE3430a4fdAF92", + client_config_map=cls._client_config_map, + connector_name="uniswap", + chain="ethereum", + network="ropsten", + wallet_address="0x5821715133bB451bDE2d5BC6a4cE3430a4fdAF92", trading_pairs=["DAI-WETH"], trading_required=True ) @@ -64,14 +69,17 @@ def setUpClass(cls) -> None: ) ) cls._patch_stack.enter_context(cls._clock) - GatewayHttpClient.get_instance().base_url = "https://localhost:5000" + GatewayHttpClient.get_instance(client_config_map=cls._client_config_map).base_url = "https://localhost:5000" ev_loop.run_until_complete(cls.wait_til_ready()) @classmethod def tearDownClass(cls) -> None: cls._patch_stack.close() + GatewayHttpClient.__instance = None + super().tearDownClass() def setUp(self) -> None: + super().setUp() self._http_player.replay_timestamp_ms = None @classmethod diff --git a/test/hummingbot/connector/derivative/binance_perpetual/test_binance_perpetual_derivative.py b/test/hummingbot/connector/derivative/binance_perpetual/test_binance_perpetual_derivative.py index a5fba0436a..2ea5725b62 100644 --- a/test/hummingbot/connector/derivative/binance_perpetual/test_binance_perpetual_derivative.py +++ b/test/hummingbot/connector/derivative/binance_perpetual/test_binance_perpetual_derivative.py @@ -4,6 +4,7 @@ import re import unittest from decimal import Decimal +from test.hummingbot.connector.network_mocking_assistant import NetworkMockingAssistant from typing import Any, Awaitable, Callable, Dict, List, Optional from unittest.mock import AsyncMock, patch @@ -13,20 +14,18 @@ import hummingbot.connector.derivative.binance_perpetual.binance_perpetual_web_utils as web_utils import hummingbot.connector.derivative.binance_perpetual.constants as CONSTANTS -from hummingbot.connector.derivative.binance_perpetual.binance_perpetual_api_order_book_data_source import \ - BinancePerpetualAPIOrderBookDataSource -from hummingbot.connector.derivative.binance_perpetual.binance_perpetual_derivative import \ - BinancePerpetualDerivative +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter +from hummingbot.connector.derivative.binance_perpetual.binance_perpetual_api_order_book_data_source import ( + BinancePerpetualAPIOrderBookDataSource, +) +from hummingbot.connector.derivative.binance_perpetual.binance_perpetual_derivative import BinancePerpetualDerivative from hummingbot.connector.utils import get_new_client_order_id from hummingbot.core.data_type.common import OrderType, PositionAction, PositionMode, TradeType -from hummingbot.core.data_type.in_flight_order import OrderState, InFlightOrder +from hummingbot.core.data_type.in_flight_order import InFlightOrder, OrderState from hummingbot.core.data_type.trade_fee import TokenAmount from hummingbot.core.event.event_logger import EventLogger -from hummingbot.core.event.events import ( - MarketEvent, - OrderFilledEvent, -) -from test.hummingbot.connector.network_mocking_assistant import NetworkMockingAssistant +from hummingbot.core.event.events import MarketEvent, OrderFilledEvent class BinancePerpetualDerivativeUnitTest(unittest.TestCase): @@ -55,8 +54,10 @@ def setUp(self) -> None: self.ws_sent_messages = [] self.ws_incoming_messages = asyncio.Queue() self.resume_test_event = asyncio.Event() + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) self.exchange = BinancePerpetualDerivative( + client_config_map=self.client_config_map, binance_perpetual_api_key="testAPIKey", binance_perpetual_api_secret="testSecret", trading_pairs=[self.trading_pair], diff --git a/test/hummingbot/connector/derivative/bybit_perpetual/test_bybit_perpetual_derivative.py b/test/hummingbot/connector/derivative/bybit_perpetual/test_bybit_perpetual_derivative.py index 99d839ed3d..c8752d6827 100644 --- a/test/hummingbot/connector/derivative/bybit_perpetual/test_bybit_perpetual_derivative.py +++ b/test/hummingbot/connector/derivative/bybit_perpetual/test_bybit_perpetual_derivative.py @@ -3,6 +3,7 @@ import re import time from decimal import Decimal +from test.hummingbot.connector.network_mocking_assistant import NetworkMockingAssistant from typing import Dict from unittest import TestCase from unittest.mock import AsyncMock, patch @@ -12,8 +13,11 @@ import hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_constants as CONSTANTS import hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_utils as bybit_utils -from hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_api_order_book_data_source import \ - BybitPerpetualAPIOrderBookDataSource +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter +from hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_api_order_book_data_source import ( + BybitPerpetualAPIOrderBookDataSource, +) from hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_derivative import BybitPerpetualDerivative from hummingbot.connector.derivative.bybit_perpetual.bybit_perpetual_order_book import BybitPerpetualOrderBook from hummingbot.connector.trading_rule import TradingRule @@ -23,7 +27,6 @@ from hummingbot.core.event.event_logger import EventLogger from hummingbot.core.event.events import FundingInfo, MarketEvent from hummingbot.core.network_iterator import NetworkStatus -from test.hummingbot.connector.network_mocking_assistant import NetworkMockingAssistant class BybitPerpetualDerivativeTests(TestCase): @@ -44,11 +47,14 @@ def setUp(self) -> None: self.log_records = [] self._finalMessage = 'FinalDummyMessage' self.async_task = None + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) - self.connector = BybitPerpetualDerivative(bybit_perpetual_api_key='testApiKey', - bybit_perpetual_secret_key='testSecretKey', - trading_pairs=[self.trading_pair, self.non_linear_trading_pair], - domain=self.domain) + self.connector = BybitPerpetualDerivative( + client_config_map=self.client_config_map, + bybit_perpetual_api_key='testApiKey', + bybit_perpetual_secret_key='testSecretKey', + trading_pairs=[self.trading_pair, self.non_linear_trading_pair], + domain=self.domain) self.connector.logger().setLevel(1) self.connector.logger().addHandler(self) @@ -116,12 +122,12 @@ def _authentication_response(self, authenticated: bool) -> str: return message def _get_symbols_mock_response( - self, - linear: bool = True, - min_order_size: float = 1, - max_order_size: float = 2, - min_price_increment: float = 3, - min_base_amount_increment: float = 4, + self, + linear: bool = True, + min_order_size: float = 1, + max_order_size: float = 2, + min_price_increment: float = 3, + min_base_amount_increment: float = 4, ) -> Dict: base, quote = ( (self.base_asset, self.quote_asset) @@ -1037,14 +1043,14 @@ def test_cancel_all_in_flight_orders(self, post_mock): position=PositionAction.OPEN ) - cancellation_results = asyncio.get_event_loop().run_until_complete(self.connector.cancel_all(timeout_seconds=10)) + cancellation_results = asyncio.get_event_loop().run_until_complete( + self.connector.cancel_all(timeout_seconds=10)) self.assertEqual(2, len(cancellation_results)) self.assertTrue(any(map(lambda result: result.order_id == "O1" and result.success, cancellation_results))) self.assertTrue(any(map(lambda result: result.order_id == "O2" and not result.success, cancellation_results))) def test_cancel_all_logs_warning_when_process_times_out(self): - self._simulate_trading_rules_initialized() self.connector._set_current_timestamp(1640001112.0) @@ -1096,10 +1102,12 @@ def test_connector_ready_status(self): self.assertTrue(self.connector.ready) def test_connector_ready_status_when_trading_not_required(self): - local_connector = BybitPerpetualDerivative(bybit_perpetual_api_key='testApiKey', - bybit_perpetual_secret_key='testSecretKey', - trading_pairs=[self.trading_pair], - trading_required=False) + local_connector = BybitPerpetualDerivative( + client_config_map=self.client_config_map, + bybit_perpetual_api_key='testApiKey', + bybit_perpetual_secret_key='testSecretKey', + trading_pairs=[self.trading_pair], + trading_required=False) self.assertFalse(local_connector.ready) @@ -2031,14 +2039,18 @@ def test_get_order_book_for_invalid_trading_pair_raises_error(self): "BTC-USDT") def test_supported_position_modes(self): - testnet_linear_connector = BybitPerpetualDerivative(bybit_perpetual_api_key='testApiKey', - bybit_perpetual_secret_key='testSecretKey', - trading_pairs=[self.trading_pair], - domain="bybit_perpetual_testnet") - testnet_non_linear_connector = BybitPerpetualDerivative(bybit_perpetual_api_key='testApiKey', - bybit_perpetual_secret_key='testSecretKey', - trading_pairs=[self.non_linear_trading_pair], - domain="bybit_perpetual_testnet") + testnet_linear_connector = BybitPerpetualDerivative( + client_config_map=self.client_config_map, + bybit_perpetual_api_key='testApiKey', + bybit_perpetual_secret_key='testSecretKey', + trading_pairs=[self.trading_pair], + domain="bybit_perpetual_testnet") + testnet_non_linear_connector = BybitPerpetualDerivative( + client_config_map=self.client_config_map, + bybit_perpetual_api_key='testApiKey', + bybit_perpetual_secret_key='testSecretKey', + trading_pairs=[self.non_linear_trading_pair], + domain="bybit_perpetual_testnet") # Case 1: Linear Perpetual expected_result = [PositionMode.HEDGE] @@ -2206,7 +2218,8 @@ def test_fetch_funding_fee_supported_linear_trading_pair_receive_funding(self, g @aioresponses() def test_fetch_funding_fee_supported_non_linear_trading_pair_receive_funding(self, get_mock): - path_url = bybit_utils.rest_api_path_for_endpoint(CONSTANTS.GET_LAST_FUNDING_RATE_PATH_URL, self.non_linear_trading_pair) + path_url = bybit_utils.rest_api_path_for_endpoint(CONSTANTS.GET_LAST_FUNDING_RATE_PATH_URL, + self.non_linear_trading_pair) url = bybit_utils.rest_api_url_for_endpoint(path_url, self.domain) regex_url = re.compile(f"^{url}") @@ -2299,7 +2312,8 @@ def test_user_funding_fee_polling_loop(self, get_mock): } get_mock.get(regex_url, body=json.dumps(linear_mock_response)) - path_url = bybit_utils.rest_api_path_for_endpoint(CONSTANTS.GET_LAST_FUNDING_RATE_PATH_URL, self.non_linear_trading_pair) + path_url = bybit_utils.rest_api_path_for_endpoint(CONSTANTS.GET_LAST_FUNDING_RATE_PATH_URL, + self.non_linear_trading_pair) url = bybit_utils.rest_api_url_for_endpoint(path_url, self.domain) regex_url = re.compile(f"^{url}") non_linear_mock_response = { @@ -2336,7 +2350,8 @@ def test_user_funding_fee_polling_loop(self, get_mock): self.assertFalse(self.connector._funding_fee_poll_notifier.is_set()) self.assertGreater(self.connector._next_funding_fee_timestamp, initial_funding_fee_ts) self.assertTrue(self._is_logged("INFO", f"Funding payment of 0.0001 received on {self.trading_pair} market.")) - self.assertTrue(self._is_logged("INFO", f"Funding payment of 0.0001 received on {self.non_linear_trading_pair} market.")) + self.assertTrue( + self._is_logged("INFO", f"Funding payment of 0.0001 received on {self.non_linear_trading_pair} market.")) def test_set_leverage_unsupported_trading_pair(self): self.connector_task = asyncio.get_event_loop().create_task( @@ -2535,7 +2550,8 @@ def test_listening_process_receives_updates_position(self, ws_connect_mock): self.connector._user_stream_tracker.start()) # Add the authentication response for the websocket - self.mocking_assistant.add_websocket_json_message(ws_connect_mock.return_value, self._authentication_response(True)) + self.mocking_assistant.add_websocket_json_message(ws_connect_mock.return_value, + self._authentication_response(True)) self.mocking_assistant.add_websocket_json_message( ws_connect_mock.return_value, @@ -2605,7 +2621,8 @@ def test_listening_process_receives_updates_order(self, ws_connect_mock): "Market", 0, 1, "") # Add the authentication response for the websocket - self.mocking_assistant.add_websocket_json_message(ws_connect_mock.return_value, self._authentication_response(True)) + self.mocking_assistant.add_websocket_json_message(ws_connect_mock.return_value, + self._authentication_response(True)) self.mocking_assistant.add_websocket_json_message( ws_connect_mock.return_value, @@ -2663,7 +2680,8 @@ def test_listening_process_receives_updates_execution(self, ws_connect_mock): "Market", 0, 1, "") # Add the authentication response for the websocket - self.mocking_assistant.add_websocket_json_message(ws_connect_mock.return_value, self._authentication_response(True)) + self.mocking_assistant.add_websocket_json_message(ws_connect_mock.return_value, + self._authentication_response(True)) self.mocking_assistant.add_websocket_json_message( ws_connect_mock.return_value, @@ -2726,7 +2744,8 @@ def test_get_funding_info_trading_pair_does_not_exist(self, get_mock): next_funding_utc_timestamp=int(pd.Timestamp('2021-08-23T08:00:00Z', tz="UTC").timestamp()), rate=(Decimal('-15')), ) - path_url = bybit_utils.rest_api_path_for_endpoint(CONSTANTS.LATEST_SYMBOL_INFORMATION_ENDPOINT, self.trading_pair) + path_url = bybit_utils.rest_api_path_for_endpoint(CONSTANTS.LATEST_SYMBOL_INFORMATION_ENDPOINT, + self.trading_pair) url = bybit_utils.rest_api_url_for_endpoint(path_url, self.domain) regex_url = re.compile(f"^{url}") diff --git a/test/hummingbot/connector/derivative/dydx_perpetual/test_dydx_perpetual_derivative.py b/test/hummingbot/connector/derivative/dydx_perpetual/test_dydx_perpetual_derivative.py index 33c3af3c30..af53cfaeb5 100644 --- a/test/hummingbot/connector/derivative/dydx_perpetual/test_dydx_perpetual_derivative.py +++ b/test/hummingbot/connector/derivative/dydx_perpetual/test_dydx_perpetual_derivative.py @@ -5,20 +5,20 @@ from datetime import datetime from decimal import Decimal from typing import Dict, Optional -from unittest.mock import AsyncMock, patch, PropertyMock +from unittest.mock import AsyncMock, PropertyMock, patch import pandas as pd from dydx3 import DydxApiError from requests import Response +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.derivative.dydx_perpetual.dydx_perpetual_derivative import DydxPerpetualDerivative from hummingbot.connector.derivative.dydx_perpetual.dydx_perpetual_position import DydxPerpetualPosition from hummingbot.connector.trading_rule import TradingRule from hummingbot.core.data_type.cancellation_result import CancellationResult -from hummingbot.core.data_type.common import ( - PositionSide, - TradeType -) +from hummingbot.core.data_type.common import PositionSide, TradeType +from hummingbot.core.event.event_logger import EventLogger from hummingbot.core.event.events import ( BuyOrderCreatedEvent, FundingInfo, @@ -27,7 +27,6 @@ PositionAction, SellOrderCreatedEvent, ) -from hummingbot.core.event.event_logger import EventLogger class DydxPerpetualDerivativeTest(unittest.TestCase): @@ -50,8 +49,10 @@ def setUp(self) -> None: self.return_values_queue = asyncio.Queue() self.resume_test_event = asyncio.Event() self.log_records = [] + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) self.exchange = DydxPerpetualDerivative( + client_config_map=self.client_config_map, dydx_perpetual_api_key="someAPIKey", dydx_perpetual_api_secret="someAPISecret", dydx_perpetual_passphrase="somePassPhrase", diff --git a/test/hummingbot/connector/derivative/test_perpetual_budget_checker.py b/test/hummingbot/connector/derivative/test_perpetual_budget_checker.py index 23d32947f5..854aa16ed0 100644 --- a/test/hummingbot/connector/derivative/test_perpetual_budget_checker.py +++ b/test/hummingbot/connector/derivative/test_perpetual_budget_checker.py @@ -1,13 +1,15 @@ import unittest from decimal import Decimal +from test.mock.mock_perp_connector import MockPerpConnector +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.derivative.perpetual_budget_checker import PerpetualBudgetChecker from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.utils import combine_to_hb_trading_pair from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.data_type.order_candidate import PerpetualOrderCandidate from hummingbot.core.data_type.trade_fee import TradeFeeSchema -from test.mock.mock_perp_connector import MockPerpConnector class PerpetualBudgetCheckerTest(unittest.TestCase): @@ -21,7 +23,9 @@ def setUp(self) -> None: trade_fee_schema = TradeFeeSchema( maker_percent_fee_decimal=Decimal("0.01"), taker_percent_fee_decimal=Decimal("0.02") ) - self.exchange = MockPerpConnector(trade_fee_schema) + self.exchange = MockPerpConnector( + client_config_map=ClientConfigAdapter(ClientConfigMap()), + trade_fee_schema=trade_fee_schema) self.budget_checker = self.exchange.budget_checker def test_populate_collateral_fields_buy_order(self): @@ -133,7 +137,9 @@ def test_populate_collateral_fields_percent_fees_in_third_token(self): maker_percent_fee_decimal=Decimal("0.01"), taker_percent_fee_decimal=Decimal("0.01"), ) - exchange = MockPerpConnector(trade_fee_schema) + exchange = MockPerpConnector( + client_config_map=ClientConfigAdapter(ClientConfigMap()), + trade_fee_schema=trade_fee_schema) pfc_quote_pair = combine_to_hb_trading_pair(self.quote_asset, pfc_token) exchange.set_balanced_order_book( # the quote to pfc price will be 1:2 trading_pair=pfc_quote_pair, diff --git a/test/hummingbot/connector/exchange/altmarkets/test_altmarkets_exchange.py b/test/hummingbot/connector/exchange/altmarkets/test_altmarkets_exchange.py index 4e67a6d13b..1b5117c07e 100644 --- a/test/hummingbot/connector/exchange/altmarkets/test_altmarkets_exchange.py +++ b/test/hummingbot/connector/exchange/altmarkets/test_altmarkets_exchange.py @@ -4,12 +4,14 @@ import time from decimal import Decimal from functools import partial -from typing import Awaitable, List, Dict +from typing import Awaitable, Dict, List from unittest import TestCase from unittest.mock import AsyncMock, patch from aioresponses import aioresponses +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.altmarkets.altmarkets_constants import Constants from hummingbot.connector.exchange.altmarkets.altmarkets_exchange import AltmarketsExchange from hummingbot.connector.exchange.altmarkets.altmarkets_in_flight_order import AltmarketsInFlightOrder @@ -47,7 +49,10 @@ def setUp(self) -> None: super().setUp() self.log_records = [] self.async_tasks: List[asyncio.Task] = [] + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) + self.exchange = AltmarketsExchange( + client_config_map=self.client_config_map, altmarkets_api_key=self.api_key, altmarkets_secret_key=self.api_secret_key, trading_pairs=[self.trading_pair] diff --git a/test/hummingbot/connector/exchange/ascend_ex/test_ascend_ex_exchange.py b/test/hummingbot/connector/exchange/ascend_ex/test_ascend_ex_exchange.py index cd3b068554..9d1bde198b 100644 --- a/test/hummingbot/connector/exchange/ascend_ex/test_ascend_ex_exchange.py +++ b/test/hummingbot/connector/exchange/ascend_ex/test_ascend_ex_exchange.py @@ -3,12 +3,15 @@ import re import unittest from decimal import Decimal +from test.hummingbot.connector.network_mocking_assistant import NetworkMockingAssistant from typing import Awaitable, List, Optional from unittest.mock import AsyncMock, MagicMock, patch from aioresponses import aioresponses from bidict import bidict +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.ascend_ex import ascend_ex_constants as CONSTANTS, ascend_ex_utils from hummingbot.connector.exchange.ascend_ex.ascend_ex_api_order_book_data_source import AscendExAPIOrderBookDataSource from hummingbot.connector.exchange.ascend_ex.ascend_ex_exchange import ( @@ -23,15 +26,9 @@ from hummingbot.core.data_type.in_flight_order import InFlightOrder, OrderState from hummingbot.core.data_type.trade_fee import TokenAmount from hummingbot.core.event.event_logger import EventLogger -from hummingbot.core.event.events import ( - BuyOrderCompletedEvent, - MarketEvent, - MarketOrderFailureEvent, - OrderFilledEvent, -) +from hummingbot.core.event.events import BuyOrderCompletedEvent, MarketEvent, MarketOrderFailureEvent, OrderFilledEvent from hummingbot.core.network_iterator import NetworkStatus from hummingbot.core.web_assistant.connections.data_types import RESTMethod -from test.hummingbot.connector.network_mocking_assistant import NetworkMockingAssistant class TestAscendExExchange(unittest.TestCase): @@ -53,8 +50,13 @@ def setUp(self) -> None: super().setUp() self.log_records = [] self.async_task: Optional[asyncio.Task] = None + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) - self.exchange = AscendExExchange(self.api_key, self.api_secret_key, trading_pairs=[self.trading_pair]) + self.exchange = AscendExExchange( + client_config_map=self.client_config_map, + ascend_ex_api_key=self.api_key, + ascend_ex_secret_key=self.api_secret_key, + trading_pairs=[self.trading_pair]) self.mocking_assistant = NetworkMockingAssistant() self.resume_test_event = asyncio.Event() self._initialize_event_loggers() diff --git a/test/hummingbot/connector/exchange/binance/test_binance_exchange.py b/test/hummingbot/connector/exchange/binance/test_binance_exchange.py index b78119a357..4356524eaa 100644 --- a/test/hummingbot/connector/exchange/binance/test_binance_exchange.py +++ b/test/hummingbot/connector/exchange/binance/test_binance_exchange.py @@ -9,6 +9,8 @@ from aioresponses import aioresponses from bidict import bidict +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.binance import binance_constants as CONSTANTS, binance_web_utils as web_utils from hummingbot.connector.exchange.binance.binance_api_order_book_data_source import BinanceAPIOrderBookDataSource from hummingbot.connector.exchange.binance.binance_exchange import BinanceExchange @@ -48,8 +50,10 @@ def setUp(self) -> None: self.log_records = [] self.test_task: Optional[asyncio.Task] = None + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) self.exchange = BinanceExchange( + client_config_map=self.client_config_map, binance_api_key="testAPIKey", binance_api_secret="testSecret", trading_pairs=[self.trading_pair], diff --git a/test/hummingbot/connector/exchange/bitfinex/test_bitfinex_exchange.py b/test/hummingbot/connector/exchange/bitfinex/test_bitfinex_exchange.py index 20bc0710d2..f57f4bcabf 100644 --- a/test/hummingbot/connector/exchange/bitfinex/test_bitfinex_exchange.py +++ b/test/hummingbot/connector/exchange/bitfinex/test_bitfinex_exchange.py @@ -3,6 +3,8 @@ from typing import Awaitable, Optional from unittest import TestCase +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.bitfinex.bitfinex_exchange import BitfinexExchange from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.data_type.trade_fee import TokenAmount @@ -28,8 +30,10 @@ def setUp(self) -> None: self.log_records = [] self.test_task: Optional[asyncio.Task] = None + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) self.exchange = BitfinexExchange( + client_config_map=self.client_config_map, bitfinex_api_key="testAPIKey", bitfinex_secret_key="testSecret", trading_pairs=[self.trading_pair], diff --git a/test/hummingbot/connector/exchange/bitmart/test_bitmart_exchange.py b/test/hummingbot/connector/exchange/bitmart/test_bitmart_exchange.py index e3a276b029..3ab639ac08 100644 --- a/test/hummingbot/connector/exchange/bitmart/test_bitmart_exchange.py +++ b/test/hummingbot/connector/exchange/bitmart/test_bitmart_exchange.py @@ -3,25 +3,24 @@ import re import unittest from decimal import Decimal -from typing import Dict, Awaitable, List -from unittest.mock import AsyncMock, patch, MagicMock +from test.hummingbot.connector.network_mocking_assistant import NetworkMockingAssistant +from typing import Awaitable, Dict, List +from unittest.mock import AsyncMock, MagicMock, patch from aioresponses import aioresponses -from hummingbot.connector.exchange.bitmart.bitmart_exchange import BitmartExchange +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.bitmart import bitmart_constants as CONSTANTS -from hummingbot.core.event.event_logger import EventLogger - -from hummingbot.connector.trading_rule import TradingRule - +from hummingbot.connector.exchange.bitmart.bitmart_exchange import BitmartExchange from hummingbot.connector.exchange.bitmart.bitmart_utils import HBOT_BROKER_ID +from hummingbot.connector.trading_rule import TradingRule from hummingbot.core.clock import Clock, ClockMode -from hummingbot.core.event.events import MarketEvent from hummingbot.core.data_type.common import OrderType, TradeType +from hummingbot.core.event.event_logger import EventLogger +from hummingbot.core.event.events import MarketEvent from hummingbot.core.network_iterator import NetworkStatus - from hummingbot.core.time_iterator import TimeIterator -from test.hummingbot.connector.network_mocking_assistant import NetworkMockingAssistant class BitmartExchangeTests(unittest.TestCase): @@ -44,8 +43,10 @@ def setUp(self) -> None: self.log_records = [] self.return_values_queue = asyncio.Queue() self.resume_test_event = asyncio.Event() + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) self.exchange = BitmartExchange( + client_config_map=self.client_config_map, bitmart_api_key="someKey", bitmart_secret_key="someSecret", bitmart_memo="someMemo", diff --git a/test/hummingbot/connector/exchange/bittrex/test_bittrex_exchange.py b/test/hummingbot/connector/exchange/bittrex/test_bittrex_exchange.py index 3beab90153..394906f31e 100644 --- a/test/hummingbot/connector/exchange/bittrex/test_bittrex_exchange.py +++ b/test/hummingbot/connector/exchange/bittrex/test_bittrex_exchange.py @@ -9,14 +9,13 @@ from aioresponses import aioresponses +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.bittrex.bittrex_exchange import BittrexExchange from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.data_type.trade_fee import TokenAmount from hummingbot.core.event.event_logger import EventLogger -from hummingbot.core.event.events import ( - MarketEvent, - OrderFilledEvent, -) +from hummingbot.core.event.events import MarketEvent, OrderFilledEvent class BittrexExchangeTest(unittest.TestCase): @@ -39,8 +38,13 @@ def setUp(self) -> None: self.log_records = [] self.test_task: Optional[asyncio.Task] = None self.resume_test_event = asyncio.Event() + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) - self.exchange = BittrexExchange(self.api_key, self.secret_key, trading_pairs=[self.trading_pair]) + self.exchange = BittrexExchange( + client_config_map=self.client_config_map, + bittrex_api_key=self.api_key, + bittrex_secret_key=self.secret_key, + trading_pairs=[self.trading_pair]) self.exchange.logger().setLevel(1) self.exchange.logger().addHandler(self) diff --git a/test/hummingbot/connector/exchange/coinbase_pro/test_coinbase_pro_exchange.py b/test/hummingbot/connector/exchange/coinbase_pro/test_coinbase_pro_exchange.py index 240c198556..9fe91dc4f2 100644 --- a/test/hummingbot/connector/exchange/coinbase_pro/test_coinbase_pro_exchange.py +++ b/test/hummingbot/connector/exchange/coinbase_pro/test_coinbase_pro_exchange.py @@ -3,24 +3,26 @@ import re import unittest from decimal import Decimal +from test.hummingbot.connector.network_mocking_assistant import NetworkMockingAssistant from typing import Awaitable, Dict, List from aioresponses import aioresponses +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.coinbase_pro import coinbase_pro_constants as CONSTANTS from hummingbot.connector.exchange.coinbase_pro.coinbase_pro_exchange import CoinbaseProExchange from hummingbot.connector.trading_rule import TradingRule +from hummingbot.core.data_type.common import OrderType from hummingbot.core.event.event_logger import EventLogger from hummingbot.core.event.events import ( BuyOrderCreatedEvent, MarketEvent, MarketOrderFailureEvent, OrderCancelledEvent, - SellOrderCreatedEvent + SellOrderCreatedEvent, ) -from hummingbot.core.data_type.common import OrderType from hummingbot.core.network_iterator import NetworkStatus -from test.hummingbot.connector.network_mocking_assistant import NetworkMockingAssistant class TestCoinbaseProExchange(unittest.TestCase): @@ -44,9 +46,14 @@ def setUp(self) -> None: self.log_records = [] self.mocking_assistant = NetworkMockingAssistant() self.async_tasks: List[asyncio.Task] = [] + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) self.exchange = CoinbaseProExchange( - self.api_key, self.api_secret, self.api_passphrase, trading_pairs=[self.trading_pair] + client_config_map=self.client_config_map, + coinbase_pro_api_key=self.api_key, + coinbase_pro_secret_key=self.api_secret, + coinbase_pro_passphrase=self.api_passphrase, + trading_pairs=[self.trading_pair] ) self.event_listener = EventLogger() diff --git a/test/hummingbot/connector/exchange/coinbase_pro/test_coingbase_pro_exchange.py b/test/hummingbot/connector/exchange/coinbase_pro/test_coingbase_pro_exchange.py index f49bb24414..63c52bedba 100644 --- a/test/hummingbot/connector/exchange/coinbase_pro/test_coingbase_pro_exchange.py +++ b/test/hummingbot/connector/exchange/coinbase_pro/test_coingbase_pro_exchange.py @@ -5,6 +5,8 @@ from unittest import TestCase from unittest.mock import AsyncMock +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.coinbase_pro.coinbase_pro_exchange import CoinbaseProExchange from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.event.event_logger import EventLogger @@ -30,8 +32,10 @@ def setUp(self) -> None: self.log_records = [] self.test_task: Optional[asyncio.Task] = None self.resume_test_event = asyncio.Event() + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) self.exchange = CoinbaseProExchange( + client_config_map=self.client_config_map, coinbase_pro_api_key="testAPIKey", coinbase_pro_secret_key="testSecret", coinbase_pro_passphrase="testPassphrase", diff --git a/test/hummingbot/connector/exchange/coinflex/test_coinflex_exchange.py b/test/hummingbot/connector/exchange/coinflex/test_coinflex_exchange.py index ff45de19b5..5b655deb1f 100644 --- a/test/hummingbot/connector/exchange/coinflex/test_coinflex_exchange.py +++ b/test/hummingbot/connector/exchange/coinflex/test_coinflex_exchange.py @@ -12,8 +12,10 @@ from aioresponses import aioresponses from async_timeout import timeout from bidict import bidict -from hummingbot.connector.exchange.coinflex import coinflex_constants as CONSTANTS -from hummingbot.connector.exchange.coinflex import coinflex_web_utils + +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter +from hummingbot.connector.exchange.coinflex import coinflex_constants as CONSTANTS, coinflex_web_utils from hummingbot.connector.exchange.coinflex.coinflex_api_order_book_data_source import CoinflexAPIOrderBookDataSource from hummingbot.connector.exchange.coinflex.coinflex_exchange import CoinflexExchange from hummingbot.connector.trading_rule import TradingRule @@ -57,8 +59,10 @@ def setUp(self) -> None: self.log_records = [] self.test_task: Optional[asyncio.Task] = None self.mocking_assistant = NetworkMockingAssistant() + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) self.exchange = CoinflexExchange( + client_config_map=self.client_config_map, coinflex_api_key="testAPIKey", coinflex_api_secret="testSecret", trading_pairs=[self.trading_pair], diff --git a/test/hummingbot/connector/exchange/coinzoom/test_coinzoom_exchange.py b/test/hummingbot/connector/exchange/coinzoom/test_coinzoom_exchange.py index 5337302efe..a2c6a7d543 100644 --- a/test/hummingbot/connector/exchange/coinzoom/test_coinzoom_exchange.py +++ b/test/hummingbot/connector/exchange/coinzoom/test_coinzoom_exchange.py @@ -7,6 +7,8 @@ from aioresponses import aioresponses +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.coinzoom.coinzoom_constants import Constants from hummingbot.connector.exchange.coinzoom.coinzoom_exchange import CoinzoomExchange from hummingbot.connector.trading_rule import TradingRule @@ -28,7 +30,10 @@ def setUpClass(cls) -> None: def setUp(self) -> None: super().setUp() + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) + self.exchange = CoinzoomExchange( + client_config_map=self.client_config_map, coinzoom_api_key=self.api_key, coinzoom_secret_key=self.api_secret_key, coinzoom_username=self.username, diff --git a/test/hummingbot/connector/exchange/ftx/test_ftx_exchange.py b/test/hummingbot/connector/exchange/ftx/test_ftx_exchange.py index 2574b97d6e..becf3b5ae1 100644 --- a/test/hummingbot/connector/exchange/ftx/test_ftx_exchange.py +++ b/test/hummingbot/connector/exchange/ftx/test_ftx_exchange.py @@ -5,14 +5,13 @@ from unittest import TestCase from unittest.mock import AsyncMock +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.ftx.ftx_exchange import FtxExchange from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.data_type.trade_fee import TokenAmount from hummingbot.core.event.event_logger import EventLogger -from hummingbot.core.event.events import ( - MarketEvent, - OrderFilledEvent, -) +from hummingbot.core.event.events import MarketEvent, OrderFilledEvent class FtxExchangeTests(TestCase): @@ -34,8 +33,10 @@ def setUp(self) -> None: self.log_records = [] self.test_task: Optional[asyncio.Task] = None self.resume_test_event = asyncio.Event() + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) self.exchange = FtxExchange( + client_config_map=self.client_config_map, ftx_api_key="testAPIKey", ftx_secret_key="testSecret", trading_pairs=[self.trading_pair], diff --git a/test/hummingbot/connector/exchange/gate_io/test_gate_io_exchange.py b/test/hummingbot/connector/exchange/gate_io/test_gate_io_exchange.py index bc8a49f213..aa614d6bc8 100644 --- a/test/hummingbot/connector/exchange/gate_io/test_gate_io_exchange.py +++ b/test/hummingbot/connector/exchange/gate_io/test_gate_io_exchange.py @@ -10,6 +10,8 @@ from aioresponses import aioresponses +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.gate_io import gate_io_constants as CONSTANTS from hummingbot.connector.exchange.gate_io.gate_io_exchange import GateIoExchange from hummingbot.connector.exchange.gate_io.gate_io_in_flight_order import GateIoInFlightOrder @@ -41,8 +43,13 @@ def setUp(self) -> None: self.log_records = [] self.mocking_assistant = NetworkMockingAssistant() self.async_tasks: List[asyncio.Task] = [] + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) - self.exchange = GateIoExchange(self.api_key, self.api_secret, trading_pairs=[self.trading_pair]) + self.exchange = GateIoExchange( + client_config_map=self.client_config_map, + gate_io_api_key=self.api_key, + gate_io_secret_key=self.api_secret, + trading_pairs=[self.trading_pair]) self.event_listener = EventLogger() self.exchange.logger().setLevel(1) diff --git a/test/hummingbot/connector/exchange/huobi/test_huobi_exchange.py b/test/hummingbot/connector/exchange/huobi/test_huobi_exchange.py index f38b5da758..c87e407007 100644 --- a/test/hummingbot/connector/exchange/huobi/test_huobi_exchange.py +++ b/test/hummingbot/connector/exchange/huobi/test_huobi_exchange.py @@ -4,16 +4,15 @@ from unittest import TestCase from unittest.mock import AsyncMock, patch +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.huobi import huobi_constants as CONSTANTS, huobi_utils from hummingbot.connector.exchange.huobi.huobi_exchange import HuobiExchange -from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.connector.utils import get_new_client_order_id +from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.data_type.trade_fee import TokenAmount from hummingbot.core.event.event_logger import EventLogger -from hummingbot.core.event.events import ( - MarketEvent, - OrderFilledEvent, -) +from hummingbot.core.event.events import MarketEvent, OrderFilledEvent class HuobiExchangeTests(TestCase): @@ -34,8 +33,10 @@ def setUp(self) -> None: self.log_records = [] self.test_task: Optional[asyncio.Task] = None + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) self.exchange = HuobiExchange( + client_config_map=self.client_config_map, huobi_api_key="testAPIKey", huobi_secret_key="testSecret", trading_pairs=[self.trading_pair], diff --git a/test/hummingbot/connector/exchange/kraken/test_kraken_exchange.py b/test/hummingbot/connector/exchange/kraken/test_kraken_exchange.py index 5728527ff2..f6a4d9a9ee 100644 --- a/test/hummingbot/connector/exchange/kraken/test_kraken_exchange.py +++ b/test/hummingbot/connector/exchange/kraken/test_kraken_exchange.py @@ -4,10 +4,13 @@ import unittest from decimal import Decimal from functools import partial +from test.hummingbot.connector.network_mocking_assistant import NetworkMockingAssistant from typing import Awaitable, Dict from aioresponses import aioresponses +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.kraken import kraken_constants as CONSTANTS from hummingbot.connector.exchange.kraken.kraken_exchange import KrakenExchange from hummingbot.connector.exchange.kraken.kraken_in_flight_order import KrakenInFlightOrderNotCreated @@ -22,7 +25,6 @@ SellOrderCreatedEvent, ) from hummingbot.core.network_iterator import NetworkStatus -from test.hummingbot.connector.network_mocking_assistant import NetworkMockingAssistant class KrakenExchangeTest(unittest.TestCase): @@ -39,7 +41,10 @@ def setUp(self) -> None: self.mocking_assistant = NetworkMockingAssistant() self.event_listener = EventLogger() not_a_real_secret = "kQH5HW/8p1uGOVjbgWA7FunAmGO8lsSUXNsu3eow76sz84Q18fWxnyRzBHCd3pd5nE9qa99HAZtuZuj6F1huXg==" + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) + self.exchange = KrakenExchange( + client_config_map=self.client_config_map, kraken_api_key="someKey", kraken_secret_key=not_a_real_secret, trading_pairs=[self.trading_pair], diff --git a/test/hummingbot/connector/exchange/kucoin/test_kucoin_exchange.py b/test/hummingbot/connector/exchange/kucoin/test_kucoin_exchange.py index b7d51ae8ec..bb52e3f2a1 100644 --- a/test/hummingbot/connector/exchange/kucoin/test_kucoin_exchange.py +++ b/test/hummingbot/connector/exchange/kucoin/test_kucoin_exchange.py @@ -10,11 +10,10 @@ from aioresponses import aioresponses from bidict import bidict +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.client_order_tracker import ClientOrderTracker -from hummingbot.connector.exchange.kucoin import ( - kucoin_constants as CONSTANTS, - kucoin_web_utils as web_utils, -) +from hummingbot.connector.exchange.kucoin import kucoin_constants as CONSTANTS, kucoin_web_utils as web_utils from hummingbot.connector.exchange.kucoin.kucoin_api_order_book_data_source import KucoinAPIOrderBookDataSource from hummingbot.connector.exchange.kucoin.kucoin_exchange import KucoinExchange from hummingbot.connector.trading_rule import TradingRule @@ -56,9 +55,14 @@ def setUp(self) -> None: self.log_records = [] self.test_task: Optional[asyncio.Task] = None + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) self.exchange = KucoinExchange( - self.api_key, self.api_passphrase, self.api_secret_key, trading_pairs=[self.trading_pair] + client_config_map=self.client_config_map, + kucoin_api_key=self.api_key, + kucoin_passphrase=self.api_passphrase, + kucoin_secret_key=self.api_secret_key, + trading_pairs=[self.trading_pair] ) self.exchange.logger().setLevel(1) diff --git a/test/hummingbot/connector/exchange/liquid/test_liquid_exchange.py b/test/hummingbot/connector/exchange/liquid/test_liquid_exchange.py index d5b6884420..7261b22cb1 100644 --- a/test/hummingbot/connector/exchange/liquid/test_liquid_exchange.py +++ b/test/hummingbot/connector/exchange/liquid/test_liquid_exchange.py @@ -5,14 +5,13 @@ from unittest import TestCase from unittest.mock import AsyncMock +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.liquid.liquid_exchange import LiquidExchange from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.data_type.trade_fee import TokenAmount from hummingbot.core.event.event_logger import EventLogger -from hummingbot.core.event.events import ( - MarketEvent, - OrderFilledEvent, -) +from hummingbot.core.event.events import MarketEvent, OrderFilledEvent class LiquidExchangeTests(TestCase): @@ -34,8 +33,10 @@ def setUp(self) -> None: self.log_records = [] self.test_task: Optional[asyncio.Task] = None self.resume_test_event = asyncio.Event() + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) self.exchange = LiquidExchange( + client_config_map=self.client_config_map, liquid_api_key="testAPIKey", liquid_secret_key="testSecret", trading_pairs=[self.trading_pair], diff --git a/test/hummingbot/connector/exchange/mexc/test_mexc_exchange.py b/test/hummingbot/connector/exchange/mexc/test_mexc_exchange.py index 4a771d629b..e479170627 100644 --- a/test/hummingbot/connector/exchange/mexc/test_mexc_exchange.py +++ b/test/hummingbot/connector/exchange/mexc/test_mexc_exchange.py @@ -5,6 +5,7 @@ import time from collections import Awaitable from decimal import Decimal +from test.hummingbot.connector.network_mocking_assistant import NetworkMockingAssistant from typing import Any, Callable, Dict, List from unittest import TestCase from unittest.mock import AsyncMock, PropertyMock, patch @@ -14,6 +15,8 @@ from aioresponses import aioresponses import hummingbot.connector.exchange.mexc.mexc_constants as CONSTANTS +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.mexc.mexc_exchange import MexcExchange from hummingbot.connector.exchange.mexc.mexc_in_flight_order import MexcInFlightOrder from hummingbot.connector.exchange.mexc.mexc_order_book import MexcOrderBook @@ -22,7 +25,6 @@ from hummingbot.core.event.events import OrderCancelledEvent, SellOrderCompletedEvent from hummingbot.core.network_iterator import NetworkStatus from hummingbot.core.utils.async_utils import safe_ensure_future -from test.hummingbot.connector.network_mocking_assistant import NetworkMockingAssistant class MexcExchangeTests(TestCase): @@ -47,10 +49,13 @@ def setUp(self) -> None: self.log_records = [] self.resume_test_event = asyncio.Event() self._account_name = "hbot" + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) - self.exchange = MexcExchange(mexc_api_key='testAPIKey', - mexc_secret_key='testSecret', - trading_pairs=[self.trading_pair]) + self.exchange = MexcExchange( + client_config_map=self.client_config_map, + mexc_api_key='testAPIKey', + mexc_secret_key='testSecret', + trading_pairs=[self.trading_pair]) self.exchange.logger().setLevel(1) self.exchange.logger().addHandler(self) diff --git a/test/hummingbot/connector/exchange/ndax/test_ndax_exchange.py b/test/hummingbot/connector/exchange/ndax/test_ndax_exchange.py index e8d81c9afc..e42a24bdee 100644 --- a/test/hummingbot/connector/exchange/ndax/test_ndax_exchange.py +++ b/test/hummingbot/connector/exchange/ndax/test_ndax_exchange.py @@ -4,33 +4,30 @@ import re import time from decimal import Decimal +from test.hummingbot.connector.network_mocking_assistant import NetworkMockingAssistant from typing import Any, Awaitable, Callable, Dict, List from unittest import TestCase -from unittest.mock import AsyncMock, patch, PropertyMock +from unittest.mock import AsyncMock, PropertyMock, patch import pandas as pd from aioresponses import aioresponses +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.ndax import ndax_constants as CONSTANTS, ndax_utils from hummingbot.connector.exchange.ndax.ndax_exchange import NdaxExchange from hummingbot.connector.exchange.ndax.ndax_in_flight_order import ( + WORKING_LOCAL_STATUS, NdaxInFlightOrder, NdaxInFlightOrderNotCreated, - WORKING_LOCAL_STATUS, ) from hummingbot.connector.exchange.ndax.ndax_order_book import NdaxOrderBook from hummingbot.connector.trading_rule import TradingRule -from hummingbot.core.event.event_logger import EventLogger -from hummingbot.core.event.events import ( - MarketEvent, - MarketOrderFailureEvent, - OrderCancelledEvent, - OrderFilledEvent, -) from hummingbot.core.data_type.common import OrderType, TradeType +from hummingbot.core.event.event_logger import EventLogger +from hummingbot.core.event.events import MarketEvent, MarketOrderFailureEvent, OrderCancelledEvent, OrderFilledEvent from hummingbot.core.network_iterator import NetworkStatus from hummingbot.core.utils.async_utils import safe_ensure_future -from test.hummingbot.connector.network_mocking_assistant import NetworkMockingAssistant class NdaxExchangeTests(TestCase): @@ -54,8 +51,10 @@ def setUp(self) -> None: self.log_records = [] self.resume_test_event = asyncio.Event() self._account_name = "hbot" + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) - self.exchange = NdaxExchange(ndax_uid='001', + self.exchange = NdaxExchange(client_config_map=self.client_config_map, + ndax_uid='001', ndax_api_key='testAPIKey', ndax_secret_key='testSecret', ndax_account_name=self._account_name, @@ -1655,7 +1654,8 @@ def test_execute_cancel_exception_raised_stop_tracking_order(self, mock_api): self.exchange._execute_cancel(self.trading_pair, order.client_order_id) ) - self._is_logged("WARNING", f"Order {order.client_order_id} does not seem to be active, will stop tracking order...") + self._is_logged("WARNING", + f"Order {order.client_order_id} does not seem to be active, will stop tracking order...") @patch("hummingbot.connector.exchange.ndax.ndax_exchange.NdaxExchange._execute_cancel", new_callable=AsyncMock) def test_cancel(self, mock_cancel): diff --git a/test/hummingbot/connector/exchange/okex/test_okex_exchange.py b/test/hummingbot/connector/exchange/okex/test_okex_exchange.py index 217714d9ba..c7aa68c323 100644 --- a/test/hummingbot/connector/exchange/okex/test_okex_exchange.py +++ b/test/hummingbot/connector/exchange/okex/test_okex_exchange.py @@ -3,6 +3,8 @@ from decimal import Decimal from unittest.mock import patch +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.okex.constants import CLIENT_ID_PREFIX, MAX_ID_LEN from hummingbot.connector.exchange.okex.okex_exchange import OkexExchange from hummingbot.connector.utils import get_new_client_order_id @@ -23,8 +25,13 @@ def setUpClass(cls) -> None: def setUp(self) -> None: super().setUp() + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) self.exchange = OkexExchange( - self.api_key, self.api_secret_key, self.api_passphrase, trading_pairs=[self.trading_pair] + client_config_map=self.client_config_map, + okex_api_key=self.api_key, + okex_secret_key=self.api_secret_key, + okex_passphrase=self.api_passphrase, + trading_pairs=[self.trading_pair] ) @patch("hummingbot.connector.utils.get_tracking_nonce_low_res") diff --git a/test/hummingbot/connector/exchange/paper_trade/test_paper_trade_exchange.py b/test/hummingbot/connector/exchange/paper_trade/test_paper_trade_exchange.py index 28115c6ba0..9e9c4d33cb 100644 --- a/test/hummingbot/connector/exchange/paper_trade/test_paper_trade_exchange.py +++ b/test/hummingbot/connector/exchange/paper_trade/test_paper_trade_exchange.py @@ -1,5 +1,7 @@ from unittest import TestCase +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.binance.binance_api_order_book_data_source import BinanceAPIOrderBookDataSource from hummingbot.connector.exchange.kucoin.kucoin_api_order_book_data_source import KucoinAPIOrderBookDataSource from hummingbot.connector.exchange.paper_trade import create_paper_trade_market, get_order_book_tracker @@ -16,8 +18,14 @@ def test_get_order_book_tracker_for_connector_using_generic_tracker(self): self.assertEqual(OrderBookTracker, type(tracker)) def test_create_paper_trade_market_for_connector_using_generic_tracker(self): - paper_exchange = create_paper_trade_market(exchange_name="binance", trading_pairs=["COINALPHA-HBOT"]) + paper_exchange = create_paper_trade_market( + exchange_name="binance", + client_config_map=ClientConfigAdapter(ClientConfigMap()), + trading_pairs=["COINALPHA-HBOT"]) self.assertEqual(BinanceAPIOrderBookDataSource, type(paper_exchange.order_book_tracker.data_source)) - paper_exchange = create_paper_trade_market(exchange_name="kucoin", trading_pairs=["COINALPHA-HBOT"]) + paper_exchange = create_paper_trade_market( + exchange_name="kucoin", + client_config_map=ClientConfigAdapter(ClientConfigMap()), + trading_pairs=["COINALPHA-HBOT"]) self.assertEqual(KucoinAPIOrderBookDataSource, type(paper_exchange.order_book_tracker.data_source)) diff --git a/test/hummingbot/connector/exchange/probit/test_probit_exchange.py b/test/hummingbot/connector/exchange/probit/test_probit_exchange.py index 44b775006c..29aa225dc9 100644 --- a/test/hummingbot/connector/exchange/probit/test_probit_exchange.py +++ b/test/hummingbot/connector/exchange/probit/test_probit_exchange.py @@ -3,8 +3,9 @@ from decimal import Decimal from unittest.mock import patch -from hummingbot.connector.exchange.probit.probit_constants import \ - MAX_ORDER_ID_LEN +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter +from hummingbot.connector.exchange.probit.probit_constants import MAX_ORDER_ID_LEN from hummingbot.connector.exchange.probit.probit_exchange import ProbitExchange from hummingbot.connector.utils import get_new_client_order_id from hummingbot.core.event.events import OrderType @@ -23,8 +24,12 @@ def setUpClass(cls) -> None: def setUp(self) -> None: super().setUp() + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) self.exchange = ProbitExchange( - self.api_key, self.api_secret_key, trading_pairs=[self.trading_pair] + client_config_map=self.client_config_map, + probit_api_key=self.api_key, + probit_secret_key=self.api_secret_key, + trading_pairs=[self.trading_pair] ) @patch("hummingbot.connector.utils.get_tracking_nonce_low_res") diff --git a/test/hummingbot/connector/test_budget_checker.py b/test/hummingbot/connector/test_budget_checker.py index ea2e5731fc..16bd16a24d 100644 --- a/test/hummingbot/connector/test_budget_checker.py +++ b/test/hummingbot/connector/test_budget_checker.py @@ -1,6 +1,8 @@ import unittest from decimal import Decimal +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.budget_checker import BudgetChecker from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange @@ -20,7 +22,9 @@ def setUp(self) -> None: trade_fee_schema = TradeFeeSchema( maker_percent_fee_decimal=Decimal("0.01"), taker_percent_fee_decimal=Decimal("0.02") ) - self.exchange = MockPaperExchange(trade_fee_schema) + self.exchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()), + trade_fee_schema=trade_fee_schema) self.budget_checker: BudgetChecker = self.exchange.budget_checker def test_populate_collateral_fields_buy_order(self): @@ -71,7 +75,9 @@ def test_populate_collateral_fields_buy_order_percent_fee_from_returns(self): taker_percent_fee_decimal=Decimal("0.01"), buy_percent_fee_deducted_from_returns=True, ) - exchange = MockPaperExchange(trade_fee_schema) + exchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()), + trade_fee_schema=trade_fee_schema) budget_checker: BudgetChecker = exchange.budget_checker order_candidate = OrderCandidate( trading_pair=self.trading_pair, @@ -119,7 +125,9 @@ def test_populate_collateral_fields_percent_fees_in_third_token(self): maker_percent_fee_decimal=Decimal("0.01"), taker_percent_fee_decimal=Decimal("0.01"), ) - exchange = MockPaperExchange(trade_fee_schema) + exchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()), + trade_fee_schema=trade_fee_schema) pfc_quote_pair = combine_to_hb_trading_pair(self.quote_asset, pfc_token) exchange.set_balanced_order_book( # the quote to pfc price will be 1:2 trading_pair=pfc_quote_pair, @@ -156,7 +164,9 @@ def test_populate_collateral_fields_fixed_fees_in_quote_token(self): maker_fixed_fees=[TokenAmount(self.quote_asset, Decimal("1"))], taker_fixed_fees=[TokenAmount(self.base_asset, Decimal("2"))], ) - exchange = MockPaperExchange(trade_fee_schema) + exchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()), + trade_fee_schema=trade_fee_schema) budget_checker: BudgetChecker = exchange.budget_checker order_candidate = OrderCandidate( @@ -285,7 +295,9 @@ def test_adjust_candidate_insufficient_funds_for_flat_fees_same_token(self): trade_fee_schema = TradeFeeSchema( maker_fixed_fees=[TokenAmount(self.quote_asset, Decimal("1"))], ) - exchange = MockPaperExchange(trade_fee_schema) + exchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()), + trade_fee_schema=trade_fee_schema) budget_checker: BudgetChecker = exchange.budget_checker exchange.set_balance(self.quote_asset, Decimal("11")) @@ -318,7 +330,9 @@ def test_adjust_candidate_insufficient_funds_for_flat_fees_third_token(self): trade_fee_schema = TradeFeeSchema( maker_fixed_fees=[TokenAmount(fee_asset, Decimal("11"))], ) - exchange = MockPaperExchange(trade_fee_schema) + exchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()), + trade_fee_schema=trade_fee_schema) budget_checker: BudgetChecker = exchange.budget_checker exchange.set_balance(self.quote_asset, Decimal("100")) exchange.set_balance(fee_asset, Decimal("10")) @@ -340,7 +354,9 @@ def test_adjust_candidate_insufficient_funds_for_flat_fees_and_percent_fees(self maker_percent_fee_decimal=Decimal("0.1"), maker_fixed_fees=[TokenAmount(self.quote_asset, Decimal("1"))], ) - exchange = MockPaperExchange(trade_fee_schema) + exchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()), + trade_fee_schema=trade_fee_schema) budget_checker: BudgetChecker = exchange.budget_checker exchange.set_balance(self.quote_asset, Decimal("12")) @@ -378,7 +394,9 @@ def test_adjust_candidate_insufficient_funds_for_flat_fees_and_percent_fees_thir taker_percent_fee_decimal=Decimal("0.01"), maker_fixed_fees=[TokenAmount(fc_token, Decimal("1"))] ) - exchange = MockPaperExchange(trade_fee_schema) + exchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()), + trade_fee_schema=trade_fee_schema) pfc_quote_pair = combine_to_hb_trading_pair(self.quote_asset, fc_token) exchange.set_balanced_order_book( # the quote to pfc price will be 1:2 trading_pair=pfc_quote_pair, diff --git a/test/hummingbot/connector/test_client_order_tracker.py b/test/hummingbot/connector/test_client_order_tracker.py index a3e488d73c..2a3b6e8222 100644 --- a/test/hummingbot/connector/test_client_order_tracker.py +++ b/test/hummingbot/connector/test_client_order_tracker.py @@ -4,6 +4,8 @@ from typing import Awaitable, Dict from unittest.mock import patch +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.client_order_tracker import ClientOrderTracker from hummingbot.connector.exchange_base import ExchangeBase from hummingbot.core.data_type.common import OrderType, TradeType @@ -46,7 +48,7 @@ def setUp(self) -> None: super().setUp() self.log_records = [] - self.connector = MockExchange() + self.connector = MockExchange(client_config_map=ClientConfigAdapter(ClientConfigMap())) self.connector._set_current_timestamp(1640000000.0) self.tracker = ClientOrderTracker(connector=self.connector) diff --git a/test/hummingbot/connector/test_connector_base.py b/test/hummingbot/connector/test_connector_base.py index 5fbb2ef8b9..44ac7687dc 100644 --- a/test/hummingbot/connector/test_connector_base.py +++ b/test/hummingbot/connector/test_connector_base.py @@ -2,6 +2,8 @@ import unittest.mock from decimal import Decimal +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.connector_base import ConnectorBase from hummingbot.connector.in_flight_order_base import InFlightOrderBase from hummingbot.core.data_type.common import OrderType, TradeType @@ -34,7 +36,7 @@ def tearDownClass(cls) -> None: cls._patcher.stop() def test_in_flight_asset_balances(self): - connector = ConnectorBase() + connector = ConnectorBase(client_config_map=ClientConfigAdapter(ClientConfigMap())) connector.real_time_balance_update = True print(connector._account_balances) orders = { diff --git a/test/hummingbot/connector/test_connector_metrics_collector.py b/test/hummingbot/connector/test_connector_metrics_collector.py index 28d9ded00a..712560c234 100644 --- a/test/hummingbot/connector/test_connector_metrics_collector.py +++ b/test/hummingbot/connector/test_connector_metrics_collector.py @@ -1,19 +1,16 @@ import asyncio import json import platform -from copy import deepcopy from decimal import Decimal from typing import Awaitable from unittest import TestCase from unittest.mock import AsyncMock, MagicMock, PropertyMock import hummingbot.connector.connector_metrics_collector -from hummingbot.client.config.config_var import ConfigVar -from hummingbot.client.config.global_config_map import global_config_map -from hummingbot.connector.connector_metrics_collector import DummyMetricsCollector, TradeVolumeMetricCollector -from hummingbot.core.data_type.common import TradeType, OrderType +from hummingbot.connector.connector_metrics_collector import TradeVolumeMetricCollector +from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee -from hummingbot.core.event.events import OrderFilledEvent, MarketEvent +from hummingbot.core.event.events import MarketEvent, OrderFilledEvent from hummingbot.core.rate_oracle.rate_oracle import RateOracle @@ -21,7 +18,6 @@ class TradeVolumeMetricCollectorTests(TestCase): def setUp(self) -> None: super().setUp() - self._global_config_backup = deepcopy(global_config_map) self.metrics_collector_url = "localhost" self.connector_name = "test_connector" @@ -38,75 +34,36 @@ def setUp(self) -> None: self.metrics_collector = TradeVolumeMetricCollector( connector=self.connector_mock, - activation_interval=10, - metrics_dispatcher=self.dispatcher_mock, + activation_interval=Decimal(10), rate_provider=self.rate_oracle, - instance_id=self.instance_id, - client_version=self.client_version) + instance_id=self.instance_id) + + self.metrics_collector._dispatcher = self.dispatcher_mock def tearDown(self) -> None: hummingbot.connector.connector_metrics_collector.CLIENT_VERSION = self.original_client_version - self._reset_global_config() super().tearDown() - def _reset_global_config(self): - global_config_map.clear() - for key, value in self._global_config_backup.items(): - global_config_map[key] = value - def async_run_with_timeout(self, coroutine: Awaitable, timeout: float = 1): ret = asyncio.get_event_loop().run_until_complete(asyncio.wait_for(coroutine, timeout)) return ret def test_instance_creation_using_configuration_parameters(self): - metrics_config = ConfigVar(key="anonymized_metrics_enabled", prompt="") - metrics_config.value = True - global_config_map["anonymized_metrics_enabled"] = metrics_config - - url_config = ConfigVar(key="log_server_url", prompt="") - url_config.value = "localhost/reporting-proxy" - global_config_map["log_server_url"] = url_config - interval_config = ConfigVar(key="anonymized_metrics_interval_min", prompt="") - interval_config.value = 5 - global_config_map["anonymized_metrics_interval_min"] = interval_config - - instance_id_config = ConfigVar(key="instance_id", prompt="") - instance_id_config.value = self.instance_id - global_config_map["instance_id"] = instance_id_config - - metrics_collector = TradeVolumeMetricCollector.from_configuration( - connector=MagicMock(), + metrics_collector = TradeVolumeMetricCollector( + connector=self.connector_mock, + activation_interval=300, rate_provider=self.rate_oracle, + instance_id=self.instance_id, valuation_token="USDT") self.assertEqual(5 * 60, metrics_collector._activation_interval) - self.assertEqual("localhost/reporting-proxy", metrics_collector._dispatcher.log_server_url) + self.assertEqual(TradeVolumeMetricCollector.DEFAULT_METRICS_SERVER_URL, + metrics_collector._dispatcher.log_server_url) self.assertEqual(self.instance_id, metrics_collector._instance_id) self.assertEqual(self.client_version, metrics_collector._client_version) self.assertEqual("USDT", metrics_collector._valuation_token) - def test_instance_creation_using_configuration_parameters_when_disabled(self): - metrics_config = ConfigVar(key="anonymized_metrics_enabled", prompt="") - metrics_config.value = False - global_config_map["anonymized_metrics_enabled"] = metrics_config - - metrics_collector = TradeVolumeMetricCollector.from_configuration( - connector=MagicMock(), - rate_provider=self.rate_oracle, - valuation_token="USDT") - - self.assertEqual(DummyMetricsCollector, type(metrics_collector)) - - del (global_config_map["anonymized_metrics_enabled"]) - - metrics_collector = TradeVolumeMetricCollector.from_configuration( - connector=MagicMock(), - rate_provider=self.rate_oracle, - valuation_token="USDT") - - self.assertEqual(DummyMetricsCollector, type(metrics_collector)) - def test_start_and_stop_are_forwarded_to_dispatcher(self): self.metrics_collector.start() self.dispatcher_mock.start.assert_called() @@ -221,10 +178,9 @@ def test_metrics_not_collected_when_convertion_rate_to_volume_token_not_found(se local_collector = TradeVolumeMetricCollector( connector=self.connector_mock, activation_interval=10, - metrics_dispatcher=self.dispatcher_mock, rate_provider=mock_rate_oracle, - instance_id=self.instance_id, - client_version=self.client_version) + instance_id=self.instance_id) + local_collector._dispatcher = self.dispatcher_mock event = OrderFilledEvent( timestamp=1000, diff --git a/test/hummingbot/connector/test_markets_recorder.py b/test/hummingbot/connector/test_markets_recorder.py index 07c409fe24..22317a21b7 100644 --- a/test/hummingbot/connector/test_markets_recorder.py +++ b/test/hummingbot/connector/test_markets_recorder.py @@ -24,7 +24,7 @@ class MarketsRecorderTests(TestCase): - @patch("hummingbot.model.sql_connection_manager.SQLConnectionManager.get_db_engine") + @patch("hummingbot.model.sql_connection_manager.create_engine") def setUp(self, engine_mock) -> None: super().setUp() self.display_name = "test_market" diff --git a/test/hummingbot/core/api_throttler/test_async_throttler.py b/test/hummingbot/core/api_throttler/test_async_throttler.py index 209a4035e6..a5442f287b 100644 --- a/test/hummingbot/core/api_throttler/test_async_throttler.py +++ b/test/hummingbot/core/api_throttler/test_async_throttler.py @@ -3,20 +3,16 @@ import math import time import unittest - from decimal import Decimal -from typing import ( - Dict, - List, -) +from typing import Dict, List +from unittest.mock import patch -from hummingbot.client.config.global_config_map import global_config_map +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.core.api_throttler.async_throttler import AsyncRequestContext, AsyncThrottler from hummingbot.core.api_throttler.data_types import LinkedLimitWeightPair, RateLimit, TaskLog - from hummingbot.logger.struct_logger import METRICS_LOG_LEVEL - TEST_PATH_URL = "/hummingbot" TEST_POOL_ID = "TEST" TEST_WEIGHTED_POOL_ID = "TEST_WEIGHTED" @@ -51,10 +47,7 @@ def setUp(self) -> None: super().setUp() self.throttler = AsyncThrottler(rate_limits=self.rate_limits) self._req_counters: Dict[str, int] = {limit.limit_id: 0 for limit in self.rate_limits} - - def tearDown(self) -> None: - global_config_map["rate_limits_share_pct"].value = None - super().tearDown() + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) async def execute_requests(self, no_request: int, limit_id: str, throttler: AsyncThrottler): for _ in range(no_request): @@ -67,10 +60,13 @@ def test_init_without_rate_limits_share_pct(self): self.assertEqual(1, self.throttler._id_to_limit_map[TEST_POOL_ID].limit) self.assertEqual(1, self.throttler._id_to_limit_map[TEST_PATH_URL].limit) - def test_init_with_rate_limits_share_pct(self): + @patch("hummingbot.core.api_throttler.async_throttler_base.AsyncThrottlerBase._client_config_map") + def test_init_with_rate_limits_share_pct(self, config_map_mock): rate_share_pct: Decimal = Decimal("55") - global_config_map["rate_limits_share_pct"].value = rate_share_pct + self.client_config_map.rate_limits_share_pct = rate_share_pct + config_map_mock.return_value = self.client_config_map + self.throttler = AsyncThrottler(rate_limits=self.rate_limits) rate_limits = self.rate_limits.copy() rate_limits.append(RateLimit(limit_id="ANOTHER_TEST", limit=10, time_interval=5)) diff --git a/test/hummingbot/core/gateway/debug_collect_gatewy_test_samples.py b/test/hummingbot/core/gateway/debug_collect_gatewy_test_samples.py index d7c6fdd10f..0d72fa82ab 100755 --- a/test/hummingbot/core/gateway/debug_collect_gatewy_test_samples.py +++ b/test/hummingbot/core/gateway/debug_collect_gatewy_test_samples.py @@ -7,21 +7,25 @@ the ETH address and nonce numbers. """ -from bin import path_util # noqa: F401 import asyncio from decimal import Decimal from os.path import join, realpath +from test.mock.http_recorder import HttpRecorder -from hummingbot.client.config.config_helpers import read_system_configs_from_yml -from hummingbot.client.config.global_config_map import global_config_map +from bin import path_util # noqa: F401 +from hummingbot.client.config.config_helpers import ( + ClientConfigAdapter, + load_client_config_map_from_file, + read_system_configs_from_yml, +) from hummingbot.core.event.events import TradeType -from hummingbot.core.gateway import GatewayHttpClient -from test.mock.http_recorder import HttpRecorder +from hummingbot.core.gateway.gateway_http_client import GatewayHttpClient async def main(): + client_config_map: ClientConfigAdapter = load_client_config_map_from_file() await read_system_configs_from_yml() - global_config_map["gateway_api_port"].value = 5000 + client_config_map.gateway.gateway_api_port = 5000 fixture_db_path: str = realpath(join(__file__, "../fixtures/gateway_http_client_fixture.db")) http_recorder: HttpRecorder = HttpRecorder(fixture_db_path) diff --git a/test/hummingbot/core/utils/test_ssl_cert.py b/test/hummingbot/core/utils/test_ssl_cert.py index e4551d47e3..a0717f8147 100644 --- a/test/hummingbot/core/utils/test_ssl_cert.py +++ b/test/hummingbot/core/utils/test_ssl_cert.py @@ -3,23 +3,30 @@ """ import os -from pathlib import Path import tempfile import unittest +from pathlib import Path from unittest.mock import patch +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.core.gateway import GatewayPaths from hummingbot.core.utils.ssl_cert import ( + certs_files_exist, + create_self_sign_certs, + generate_csr, generate_private_key, generate_public_key, - generate_csr, sign_csr, - create_self_sign_certs, - certs_files_exist, ) class SslCertTest(unittest.TestCase): + + def setUp(self) -> None: + super().setUp() + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) + def test_generate_private_key(self): """ Unit tests for generate_private_key @@ -114,8 +121,8 @@ def test_create_self_sign_certs(self, _): ) with patch("hummingbot.core.utils.ssl_cert.get_gateway_paths", return_value=mock_gateway_paths): - self.assertEqual(certs_files_exist(), False) + self.assertEqual(certs_files_exist(client_config_map=self.client_config_map), False) # generate all necessary certs then confirm they exist in the expected place - create_self_sign_certs("abc123") - self.assertEqual(certs_files_exist(), True) + create_self_sign_certs("abc123", client_config_map=self.client_config_map) + self.assertEqual(certs_files_exist(client_config_map=self.client_config_map), True) diff --git a/test/hummingbot/strategy/amm_arb/test_amm_arb.py b/test/hummingbot/strategy/amm_arb/test_amm_arb.py index b6cc169309..6650ec3dbf 100644 --- a/test/hummingbot/strategy/amm_arb/test_amm_arb.py +++ b/test/hummingbot/strategy/amm_arb/test_amm_arb.py @@ -1,16 +1,16 @@ -from aiounittest import async_test import asyncio import contextlib +import unittest from decimal import Decimal from typing import List -import unittest from unittest.mock import patch +from aiounittest import async_test + +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.connector_base import ConnectorBase -from hummingbot.core.clock import ( - Clock, - ClockMode -) +from hummingbot.core.clock import Clock, ClockMode from hummingbot.core.data_type.common import OrderType from hummingbot.core.data_type.trade_fee import TokenAmount, TradeFeeSchema from hummingbot.core.event.event_logger import EventLogger @@ -18,14 +18,15 @@ BuyOrderCompletedEvent, BuyOrderCreatedEvent, MarketEvent, + SellOrderCompletedEvent, SellOrderCreatedEvent, - SellOrderCompletedEvent) +) from hummingbot.core.network_iterator import NetworkStatus from hummingbot.core.utils.async_utils import safe_ensure_future from hummingbot.core.utils.fixed_rate_source import FixedRateSource from hummingbot.core.utils.tracking_nonce import get_tracking_nonce from hummingbot.strategy.amm_arb.amm_arb import AmmArbStrategy -from hummingbot.strategy.amm_arb.data_types import ArbProposalSide, ArbProposal +from hummingbot.strategy.amm_arb.data_types import ArbProposal, ArbProposalSide from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple TRADING_PAIR: str = "HBOT-USDT" @@ -37,9 +38,9 @@ class MockAMM(ConnectorBase): - def __init__(self, name): + def __init__(self, name, client_config_map: "ClientConfigAdapter"): self._name = name - super().__init__() + super().__init__(client_config_map) self._buy_prices = {} self._sell_prices = {} self._network_transaction_fee = TokenAmount("ETH", s_decimal_0) @@ -127,12 +128,16 @@ class AmmArbUnitTest(unittest.TestCase): def setUp(self): self.clock: Clock = Clock(ClockMode.REALTIME) self.stack: contextlib.ExitStack = contextlib.ExitStack() - self.amm_1: MockAMM = MockAMM("onion") + self.amm_1: MockAMM = MockAMM( + name="onion", + client_config_map=ClientConfigAdapter(ClientConfigMap())) self.amm_1.set_balance(BASE_ASSET, 500) self.amm_1.set_balance(QUOTE_ASSET, 500) self.market_info_1 = MarketTradingPairTuple(self.amm_1, TRADING_PAIR, BASE_ASSET, QUOTE_ASSET) - self.amm_2: MockAMM = MockAMM("garlic") + self.amm_2: MockAMM = MockAMM( + name="garlic", + client_config_map=ClientConfigAdapter(ClientConfigMap())) self.amm_2.set_balance(BASE_ASSET, 500) self.amm_2.set_balance(QUOTE_ASSET, 500) self.market_info_2 = MarketTradingPairTuple(self.amm_2, TRADING_PAIR, BASE_ASSET, QUOTE_ASSET) diff --git a/test/hummingbot/strategy/amm_arb/test_utils.py b/test/hummingbot/strategy/amm_arb/test_utils.py index b569c051b3..57123993f2 100644 --- a/test/hummingbot/strategy/amm_arb/test_utils.py +++ b/test/hummingbot/strategy/amm_arb/test_utils.py @@ -1,9 +1,11 @@ +import asyncio import unittest from decimal import Decimal -import asyncio -from hummingbot.strategy.amm_arb import utils +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.connector_base import ConnectorBase +from hummingbot.strategy.amm_arb import utils from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple trading_pair = "HBOT-USDT" @@ -39,8 +41,16 @@ def test_create_arb_proposals(self): asyncio.get_event_loop().run_until_complete(self._test_create_arb_proposals()) async def _test_create_arb_proposals(self): - market_info1 = MarketTradingPairTuple(MockConnector1(), trading_pair, base, quote) - market_info2 = MarketTradingPairTuple(MockConnector2(), trading_pair, base, quote) + market_info1 = MarketTradingPairTuple( + MockConnector1(client_config_map=ClientConfigAdapter(ClientConfigMap())), + trading_pair, + base, + quote) + market_info2 = MarketTradingPairTuple( + MockConnector2(client_config_map=ClientConfigAdapter(ClientConfigMap())), + trading_pair, + base, + quote) arb_proposals = await utils.create_arb_proposals(market_info1, market_info2, [], [], Decimal("1")) # there are 2 proposal combination possible - (buy_1, sell_2) and (buy_2, sell_1) self.assertEqual(2, len(arb_proposals)) diff --git a/test/hummingbot/strategy/arbitrage/test_arbitrage.py b/test/hummingbot/strategy/arbitrage/test_arbitrage.py index 07c180c7f0..89beaacf3c 100644 --- a/test/hummingbot/strategy/arbitrage/test_arbitrage.py +++ b/test/hummingbot/strategy/arbitrage/test_arbitrage.py @@ -5,6 +5,8 @@ import pandas as pd from nose.plugins.attrib import attr +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode @@ -28,8 +30,8 @@ class ArbitrageUnitTest(unittest.TestCase): def setUp(self): self.maxDiff = None self.clock: Clock = Clock(ClockMode.BACKTEST, 1.0, self.start_timestamp, self.end_timestamp) - self.market_1: MockPaperExchange = MockPaperExchange() - self.market_2: MockPaperExchange = MockPaperExchange() + self.market_1: MockPaperExchange = MockPaperExchange(client_config_map=ClientConfigAdapter(ClientConfigMap())) + self.market_2: MockPaperExchange = MockPaperExchange(client_config_map=ClientConfigAdapter(ClientConfigMap())) self.market_1.set_balanced_order_book(self.market_1_trading_pairs[0], 1.0, 0.5, 1.5, 0.01, 10) self.market_2.set_balanced_order_book(self.market_2_trading_pairs[0], 1.0, 0.5, 1.5, 0.005, 5) diff --git a/test/hummingbot/strategy/arbitrage/test_arbitrage_start.py b/test/hummingbot/strategy/arbitrage/test_arbitrage_start.py index 67746d9a4f..ec54da6229 100644 --- a/test/hummingbot/strategy/arbitrage/test_arbitrage_start.py +++ b/test/hummingbot/strategy/arbitrage/test_arbitrage_start.py @@ -1,10 +1,13 @@ -from decimal import Decimal import unittest.mock +from decimal import Decimal +from test.hummingbot.strategy import assign_config_default + import hummingbot.strategy.arbitrage.start as arbitrage_start +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.connector_base import ConnectorBase -from hummingbot.strategy.arbitrage.arbitrage_config_map import arbitrage_config_map from hummingbot.strategy.arbitrage.arbitrage import ArbitrageStrategy -from test.hummingbot.strategy import assign_config_default +from hummingbot.strategy.arbitrage.arbitrage_config_map import arbitrage_config_map class ArbitrageStartTest(unittest.TestCase): @@ -12,7 +15,10 @@ class ArbitrageStartTest(unittest.TestCase): def setUp(self) -> None: super().setUp() self.strategy: ArbitrageStrategy = None - self.markets = {"binance": ConnectorBase(), "balancer": ConnectorBase()} + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) + self.markets = { + "binance": ConnectorBase(client_config_map=self.client_config_map), + "balancer": ConnectorBase(client_config_map=self.client_config_map)} self.notifications = [] self.log_errors = [] assign_config_default(arbitrage_config_map) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py index 957afac2ed..56210f09d4 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py @@ -8,6 +8,7 @@ import numpy as np import pandas as pd +from hummingbot.client.config.client_config_map import ClientConfigMap from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange @@ -83,7 +84,9 @@ def setUp(self): trade_fee_schema = TradeFeeSchema( maker_percent_fee_decimal=Decimal("0.25"), taker_percent_fee_decimal=Decimal("0.25") ) - self.market: MockPaperExchange = MockPaperExchange(trade_fee_schema) + self.market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()), + trade_fee_schema=trade_fee_schema) self.market_info: MarketTradingPairTuple = MarketTradingPairTuple( self.market, self.trading_pair, *self.trading_pair.split("-") ) diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py index dce3f1dd33..605dc21985 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making_start.py @@ -4,6 +4,7 @@ from decimal import Decimal import hummingbot.strategy.avellaneda_market_making.start as strategy_start +from hummingbot.client.config.client_config_map import ClientConfigMap from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange_base import ExchangeBase from hummingbot.connector.utils import combine_to_hb_trading_pair @@ -22,7 +23,7 @@ class AvellanedaStartTest(unittest.TestCase): def setUp(self) -> None: super().setUp() self.strategy = None - self.markets = {"binance": ExchangeBase()} + self.markets = {"binance": ExchangeBase(client_config_map=ClientConfigAdapter(ClientConfigMap()))} self.notifications = [] self.log_records = [] self.base = "ETH" diff --git a/test/hummingbot/strategy/celo_arb/test_celo_arb.py b/test/hummingbot/strategy/celo_arb/test_celo_arb.py index d0fe524bf3..fd6b79b0da 100644 --- a/test/hummingbot/strategy/celo_arb/test_celo_arb.py +++ b/test/hummingbot/strategy/celo_arb/test_celo_arb.py @@ -7,6 +7,8 @@ import pandas as pd from nose.plugins.attrib import attr +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.connector.other.celo.celo_cli import CeloCLI @@ -53,7 +55,7 @@ def tearDownClass(cls) -> None: def setUp(self): self.maxDiff = None self.clock: Clock = Clock(ClockMode.BACKTEST, 1.0, self.start_timestamp, self.end_timestamp) - self.market: MockPaperExchange = MockPaperExchange() + self.market: MockPaperExchange = MockPaperExchange(client_config_map=ClientConfigAdapter(ClientConfigMap())) self.market.set_balanced_order_book(self.trading_pair, 10, 5, 15, 0.1, 1) diff --git a/test/hummingbot/strategy/celo_arb/test_celo_arb_start.py b/test/hummingbot/strategy/celo_arb/test_celo_arb_start.py index 8413460291..aca85ee459 100644 --- a/test/hummingbot/strategy/celo_arb/test_celo_arb_start.py +++ b/test/hummingbot/strategy/celo_arb/test_celo_arb_start.py @@ -1,9 +1,12 @@ -from decimal import Decimal import unittest.mock +from decimal import Decimal +from test.hummingbot.strategy import assign_config_default + import hummingbot.strategy.celo_arb.start as strategy_start +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange_base import ExchangeBase from hummingbot.strategy.celo_arb.celo_arb_config_map import celo_arb_config_map as strategy_cmap -from test.hummingbot.strategy import assign_config_default class CeloArbStartTest(unittest.TestCase): @@ -11,7 +14,7 @@ class CeloArbStartTest(unittest.TestCase): def setUp(self) -> None: super().setUp() self.strategy = None - self.markets = {"binance": ExchangeBase()} + self.markets = {"binance": ExchangeBase(client_config_map=ClientConfigAdapter(ClientConfigMap()))} self.notifications = [] self.log_errors = [] assign_config_default(strategy_cmap) diff --git a/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making.py b/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making.py index 1fcada0963..e7eeeb0aa8 100644 --- a/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making.py +++ b/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making.py @@ -6,6 +6,8 @@ import pandas as pd from nose.plugins.attrib import attr +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode @@ -41,8 +43,10 @@ class HedgedMarketMakingUnitTest(unittest.TestCase): def setUp(self): self.clock: Clock = Clock(ClockMode.BACKTEST, 1.0, self.start_timestamp, self.end_timestamp) self.min_profitbality = Decimal("0.005") - self.maker_market: MockPaperExchange = MockPaperExchange() - self.taker_market: MockPaperExchange = MockPaperExchange() + self.maker_market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap())) + self.taker_market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap())) self.maker_market.set_balanced_order_book(self.maker_trading_pairs[0], 1.0, 0.5, 1.5, 0.01, 10) self.taker_market.set_balanced_order_book(self.taker_trading_pairs[0], 1.0, 0.5, 1.5, 0.001, 4) self.maker_market.set_balance("COINALPHA", 5) @@ -494,7 +498,9 @@ def test_maker_price(self): def test_with_adjust_orders_enabled(self): self.clock.remove_iterator(self.strategy) self.clock.remove_iterator(self.maker_market) - self.maker_market: MockPaperExchange = MockPaperExchange() + self.maker_market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) self.maker_market.set_balanced_order_book(self.maker_trading_pairs[0], 1.0, 0.5, 1.5, 0.1, 10) self.market_pair: CrossExchangeMarketPair = CrossExchangeMarketPair( MarketTradingPairTuple(self.maker_market, *self.maker_trading_pairs), @@ -528,7 +534,9 @@ def test_with_adjust_orders_enabled(self): def test_with_adjust_orders_disabled(self): self.clock.remove_iterator(self.strategy) self.clock.remove_iterator(self.maker_market) - self.maker_market: MockPaperExchange = MockPaperExchange() + self.maker_market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) self.maker_market.set_balanced_order_book(self.maker_trading_pairs[0], 1.0, 0.5, 1.5, 0.1, 10) self.taker_market.set_balanced_order_book(self.taker_trading_pairs[0], 1.0, 0.5, 1.5, 0.001, 20) @@ -706,7 +714,9 @@ def test_check_if_sufficient_balance_adjusts_including_slippage(self): def test_empty_maker_orderbook(self): self.clock.remove_iterator(self.strategy) self.clock.remove_iterator(self.maker_market) - self.maker_market: MockPaperExchange = MockPaperExchange() + self.maker_market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) # Orderbook is empty self.maker_market.new_empty_order_book(self.maker_trading_pairs[0]) diff --git a/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_start.py b/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_start.py index 2930df6925..88c30e3260 100644 --- a/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_start.py +++ b/test/hummingbot/strategy/cross_exchange_market_making/test_cross_exchange_market_making_start.py @@ -1,12 +1,14 @@ -from decimal import Decimal import unittest.mock +from decimal import Decimal +from test.hummingbot.strategy import assign_config_default + import hummingbot.strategy.cross_exchange_market_making.start as strategy_start +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange_base import ExchangeBase from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_making_config_map import ( - cross_exchange_market_making_config_map as strategy_cmap + cross_exchange_market_making_config_map as strategy_cmap, ) -from hummingbot.client.config.global_config_map import global_config_map -from test.hummingbot.strategy import assign_config_default class XEMMStartTest(unittest.TestCase): @@ -14,7 +16,11 @@ class XEMMStartTest(unittest.TestCase): def setUp(self) -> None: super().setUp() self.strategy = None - self.markets = {"binance": ExchangeBase(), "kucoin": ExchangeBase()} + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) + self.client_config_map.strategy_report_interval = 60. + self.markets = { + "binance": ExchangeBase(client_config_map=self.client_config_map), + "kucoin": ExchangeBase(client_config_map=self.client_config_map)} self.notifications = [] self.log_errors = [] assign_config_default(strategy_cmap) @@ -24,7 +30,6 @@ def setUp(self) -> None: strategy_cmap.get("taker_market_trading_pair").value = "ETH-USDT" strategy_cmap.get("order_amount").value = Decimal("1") strategy_cmap.get("min_profitability").value = Decimal("2") - global_config_map.get("strategy_report_interval").value = 60. strategy_cmap.get("use_oracle_conversion_rate").value = False def _initialize_market_assets(self, market, trading_pairs): diff --git a/test/hummingbot/strategy/dev_0_hello_world/test_dev_0_hello_world.py b/test/hummingbot/strategy/dev_0_hello_world/test_dev_0_hello_world.py index f1352a74e9..ac989d295b 100644 --- a/test/hummingbot/strategy/dev_0_hello_world/test_dev_0_hello_world.py +++ b/test/hummingbot/strategy/dev_0_hello_world/test_dev_0_hello_world.py @@ -3,6 +3,8 @@ import pandas as pd +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode @@ -22,7 +24,9 @@ class Dev0HelloWorldUnitTest(unittest.TestCase): def setUpClass(cls): cls.ev_loop = asyncio.get_event_loop() cls.clock: Clock = Clock(ClockMode.BACKTEST, cls.tick_size, cls.start_timestamp, cls.end_timestamp) - cls.market: MockPaperExchange = MockPaperExchange() + cls.market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) cls.strategy: HelloWorldStrategy = HelloWorldStrategy( exchange=cls.market, trading_pair=cls.trading_pair, diff --git a/test/hummingbot/strategy/dev_1_get_order_book/test_dev_1_get_order_book.py b/test/hummingbot/strategy/dev_1_get_order_book/test_dev_1_get_order_book.py index e68313bdee..6e25a43c8b 100644 --- a/test/hummingbot/strategy/dev_1_get_order_book/test_dev_1_get_order_book.py +++ b/test/hummingbot/strategy/dev_1_get_order_book/test_dev_1_get_order_book.py @@ -4,6 +4,8 @@ import pandas as pd +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode @@ -23,7 +25,9 @@ class Dev1GetOrderBookUnitTest(unittest.TestCase): def setUpClass(cls): cls.ev_loop = asyncio.get_event_loop() cls.clock: Clock = Clock(ClockMode.BACKTEST, cls.tick_size, cls.start_timestamp, cls.end_timestamp) - cls.market: MockPaperExchange = MockPaperExchange() + cls.market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) cls.strategy: GetOrderBookStrategy = GetOrderBookStrategy( exchange=cls.market, trading_pair=cls.trading_pair, diff --git a/test/hummingbot/strategy/dev_2_perform_trade/test_dev_2_perform_trade.py b/test/hummingbot/strategy/dev_2_perform_trade/test_dev_2_perform_trade.py index 1c015759a0..7e4857f339 100644 --- a/test/hummingbot/strategy/dev_2_perform_trade/test_dev_2_perform_trade.py +++ b/test/hummingbot/strategy/dev_2_perform_trade/test_dev_2_perform_trade.py @@ -4,6 +4,8 @@ import pandas as pd +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode @@ -39,7 +41,9 @@ def setUp(self): self.time_delay = 15 self.cancel_order_wait_time = 45 - self.market: MockPaperExchange = MockPaperExchange() + self.market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) self.market.set_balanced_order_book(trading_pair=self.trading_pair, mid_price=self.mid_price, min_price=1, max_price=200, price_step_size=1, volume_step_size=10) diff --git a/test/hummingbot/strategy/dev_5_vwap/test_vwap.py b/test/hummingbot/strategy/dev_5_vwap/test_vwap.py index 7ae0673a27..6aa66a88c8 100644 --- a/test/hummingbot/strategy/dev_5_vwap/test_vwap.py +++ b/test/hummingbot/strategy/dev_5_vwap/test_vwap.py @@ -5,6 +5,8 @@ import pandas as pd +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode @@ -28,7 +30,9 @@ class TWAPUnitTest(unittest.TestCase): def setUp(self): self.clock: Clock = Clock(ClockMode.BACKTEST, self.clock_tick_size, self.start_timestamp, self.end_timestamp) - self.market: MockPaperExchange = MockPaperExchange() + self.market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) self.mid_price = 100 self.time_delay = 15 self.cancel_order_wait_time = 45 diff --git a/test/hummingbot/strategy/dev_simple_trade/test_simple_trade.py b/test/hummingbot/strategy/dev_simple_trade/test_simple_trade.py index 817cd8bd4e..7ec479633a 100644 --- a/test/hummingbot/strategy/dev_simple_trade/test_simple_trade.py +++ b/test/hummingbot/strategy/dev_simple_trade/test_simple_trade.py @@ -4,6 +4,8 @@ import pandas as pd +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode @@ -33,7 +35,9 @@ class SimpleTradeUnitTest(unittest.TestCase): def setUp(self): self.clock: Clock = Clock(ClockMode.BACKTEST, self.clock_tick_size, self.start_timestamp, self.end_timestamp) - self.market: MockPaperExchange = MockPaperExchange() + self.market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) self.mid_price = 100 self.time_delay = 15 self.cancel_order_wait_time = 45 diff --git a/test/hummingbot/strategy/liquidity_mining/test_liquidity_mining.py b/test/hummingbot/strategy/liquidity_mining/test_liquidity_mining.py index 5f3d17fff3..14da314d4f 100644 --- a/test/hummingbot/strategy/liquidity_mining/test_liquidity_mining.py +++ b/test/hummingbot/strategy/liquidity_mining/test_liquidity_mining.py @@ -4,6 +4,8 @@ import pandas as pd +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.client.hummingbot_application import HummingbotApplication from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange @@ -32,7 +34,9 @@ def create_market(trading_pairs: List[str], mid_price, balances: Dict[str, int]) """ Create a BacktestMarket and marketinfo dictionary to be used by the liquidity mining strategy """ - market: MockPaperExchange = MockPaperExchange() + market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) market_infos: Dict[str, MarketTradingPairTuple] = {} for trading_pair in trading_pairs: diff --git a/test/hummingbot/strategy/liquidity_mining/test_liquidity_mining_start.py b/test/hummingbot/strategy/liquidity_mining/test_liquidity_mining_start.py index e5c81a471d..bc0db46b73 100644 --- a/test/hummingbot/strategy/liquidity_mining/test_liquidity_mining_start.py +++ b/test/hummingbot/strategy/liquidity_mining/test_liquidity_mining_start.py @@ -1,11 +1,14 @@ -from decimal import Decimal import unittest.mock +from decimal import Decimal +from test.hummingbot.strategy import assign_config_default + import hummingbot.strategy.liquidity_mining.start as strategy_start +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange_base import ExchangeBase from hummingbot.strategy.liquidity_mining.liquidity_mining_config_map import ( - liquidity_mining_config_map as strategy_cmap + liquidity_mining_config_map as strategy_cmap, ) -from test.hummingbot.strategy import assign_config_default class LiquidityMiningStartTest(unittest.TestCase): @@ -13,7 +16,7 @@ class LiquidityMiningStartTest(unittest.TestCase): def setUp(self) -> None: super().setUp() self.strategy = None - self.markets = {"binance": ExchangeBase()} + self.markets = {"binance": ExchangeBase(client_config_map=ClientConfigAdapter(ClientConfigMap()))} self.notifications = [] self.log_errors = [] assign_config_default(strategy_cmap) diff --git a/test/hummingbot/strategy/perpetual_market_making/test_perpetual_market_making.py b/test/hummingbot/strategy/perpetual_market_making/test_perpetual_market_making.py index 6c0865d71c..d525c870fb 100644 --- a/test/hummingbot/strategy/perpetual_market_making/test_perpetual_market_making.py +++ b/test/hummingbot/strategy/perpetual_market_making/test_perpetual_market_making.py @@ -5,6 +5,8 @@ import pandas as pd +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.derivative.position import Position from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange @@ -54,7 +56,9 @@ def setUpClass(cls): def setUp(self): super().setUp() self.log_records = [] - self.market: MockPerpConnector = MockPerpConnector(self.trade_fee_schema) + self.market: MockPerpConnector = MockPerpConnector( + client_config_map=ClientConfigAdapter(ClientConfigMap()), + trade_fee_schema=self.trade_fee_schema) self.market.set_quantization_param( QuantizationParams( self.trading_pair, diff --git a/test/hummingbot/strategy/perpetual_market_making/test_perpetual_market_making_start.py b/test/hummingbot/strategy/perpetual_market_making/test_perpetual_market_making_start.py index 046a6aaccb..b2a1427d6f 100644 --- a/test/hummingbot/strategy/perpetual_market_making/test_perpetual_market_making_start.py +++ b/test/hummingbot/strategy/perpetual_market_making/test_perpetual_market_making_start.py @@ -1,11 +1,14 @@ -from decimal import Decimal import unittest.mock +from decimal import Decimal +from test.hummingbot.strategy import assign_config_default + import hummingbot.strategy.perpetual_market_making.start as strategy_start +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange_base import ExchangeBase from hummingbot.strategy.perpetual_market_making.perpetual_market_making_config_map import ( - perpetual_market_making_config_map as c_map + perpetual_market_making_config_map as c_map, ) -from test.hummingbot.strategy import assign_config_default class PerpetualMarketMakingStartTest(unittest.TestCase): @@ -13,7 +16,7 @@ class PerpetualMarketMakingStartTest(unittest.TestCase): def setUp(self) -> None: super().setUp() self.strategy = None - self.markets = {"binance": ExchangeBase()} + self.markets = {"binance": ExchangeBase(client_config_map=ClientConfigAdapter(ClientConfigMap()))} self.notifications = [] self.log_errors = [] assign_config_default(c_map) diff --git a/test/hummingbot/strategy/pure_market_making/test_pmm.py b/test/hummingbot/strategy/pure_market_making/test_pmm.py index e445fb2297..a669d9b404 100644 --- a/test/hummingbot/strategy/pure_market_making/test_pmm.py +++ b/test/hummingbot/strategy/pure_market_making/test_pmm.py @@ -55,7 +55,9 @@ class PMMUnitTest(unittest.TestCase): def setUp(self): self.clock_tick_size = 1 self.clock: Clock = Clock(ClockMode.BACKTEST, self.clock_tick_size, self.start_timestamp, self.end_timestamp) - self.market: MockPaperExchange = MockPaperExchange() + self.market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) self.mid_price = 100 self.bid_spread = 0.01 self.ask_spread = 0.01 @@ -140,7 +142,9 @@ def setUp(self): price_type="custom", ) - self.ext_market: MockPaperExchange = MockPaperExchange() + self.ext_market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) self.ext_market_info: MarketTradingPairTuple = MarketTradingPairTuple( self.ext_market, self.trading_pair, self.base_asset, self.quote_asset ) @@ -1209,7 +1213,9 @@ class PureMarketMakingMinimumSpreadUnitTest(unittest.TestCase): def setUp(self): self.clock_tick_size = 1 self.clock: Clock = Clock(ClockMode.BACKTEST, self.clock_tick_size, self.start_timestamp, self.end_timestamp) - self.market: MockPaperExchange = MockPaperExchange() + self.market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) self.mid_price = 100 self.market.set_balanced_order_book(trading_pair=self.trading_pair, mid_price=self.mid_price, min_price=1, diff --git a/test/hummingbot/strategy/pure_market_making/test_pmm_ping_pong.py b/test/hummingbot/strategy/pure_market_making/test_pmm_ping_pong.py index 9c273b10a2..e2b8d80c5f 100644 --- a/test/hummingbot/strategy/pure_market_making/test_pmm_ping_pong.py +++ b/test/hummingbot/strategy/pure_market_making/test_pmm_ping_pong.py @@ -4,6 +4,8 @@ import pandas as pd +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode @@ -39,7 +41,9 @@ def simulate_maker_market_trade(self, is_buy: bool, quantity: Decimal, price: De def setUp(self): self.clock_tick_size = 1 self.clock: Clock = Clock(ClockMode.BACKTEST, self.clock_tick_size, self.start_timestamp, self.end_timestamp) - self.market: MockPaperExchange = MockPaperExchange() + self.market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) self.mid_price = 100 self.bid_spread = 0.01 self.ask_spread = 0.01 diff --git a/test/hummingbot/strategy/pure_market_making/test_pmm_refresh_tolerance.py b/test/hummingbot/strategy/pure_market_making/test_pmm_refresh_tolerance.py index dedbd75540..a3c35a0ab8 100644 --- a/test/hummingbot/strategy/pure_market_making/test_pmm_refresh_tolerance.py +++ b/test/hummingbot/strategy/pure_market_making/test_pmm_refresh_tolerance.py @@ -5,6 +5,8 @@ import pandas as pd +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode @@ -41,7 +43,9 @@ def simulate_maker_market_trade(self, is_buy: bool, quantity: Decimal, price: De def setUp(self): self.clock_tick_size = 1 self.clock: Clock = Clock(ClockMode.BACKTEST, self.clock_tick_size, self.start_timestamp, self.end_timestamp) - self.market: MockPaperExchange = MockPaperExchange() + self.market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) self.mid_price = 100 self.bid_spread = 0.01 self.ask_spread = 0.01 diff --git a/test/hummingbot/strategy/pure_market_making/test_pmm_take_if_cross.py b/test/hummingbot/strategy/pure_market_making/test_pmm_take_if_cross.py index 193de936de..d675dbca63 100644 --- a/test/hummingbot/strategy/pure_market_making/test_pmm_take_if_cross.py +++ b/test/hummingbot/strategy/pure_market_making/test_pmm_take_if_cross.py @@ -5,6 +5,8 @@ import pandas as pd +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode @@ -51,7 +53,9 @@ class PureMMTakeIfCrossUnitTest(unittest.TestCase): def setUp(self): self.clock_tick_size = 1 self.clock: Clock = Clock(ClockMode.BACKTEST, self.clock_tick_size, self.start_timestamp, self.end_timestamp) - self.market: MockPaperExchange = MockPaperExchange() + self.market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) self.mid_price = 100 self.bid_spread = 0.01 self.ask_spread = 0.01 @@ -77,7 +81,9 @@ def setUp(self): self.market.add_listener(MarketEvent.OrderFilled, self.order_fill_logger) self.market.add_listener(MarketEvent.OrderCancelled, self.cancel_order_logger) - self.ext_market: MockPaperExchange = MockPaperExchange() + self.ext_market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) self.ext_market_info: MarketTradingPairTuple = MarketTradingPairTuple( self.ext_market, self.trading_pair, self.base_asset, self.quote_asset ) diff --git a/test/hummingbot/strategy/pure_market_making/test_pure_market_making_start.py b/test/hummingbot/strategy/pure_market_making/test_pure_market_making_start.py index fc27c6544a..7d0a778651 100644 --- a/test/hummingbot/strategy/pure_market_making/test_pure_market_making_start.py +++ b/test/hummingbot/strategy/pure_market_making/test_pure_market_making_start.py @@ -1,13 +1,13 @@ import unittest.mock from decimal import Decimal +from test.hummingbot.strategy import assign_config_default import hummingbot.strategy.pure_market_making.start as strategy_start +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange_base import ExchangeBase from hummingbot.core.data_type.common import PriceType -from hummingbot.strategy.pure_market_making.pure_market_making_config_map import ( - pure_market_making_config_map as c_map -) -from test.hummingbot.strategy import assign_config_default +from hummingbot.strategy.pure_market_making.pure_market_making_config_map import pure_market_making_config_map as c_map class PureMarketMakingStartTest(unittest.TestCase): @@ -15,7 +15,8 @@ class PureMarketMakingStartTest(unittest.TestCase): def setUp(self) -> None: super().setUp() self.strategy = None - self.markets = {"binance": ExchangeBase()} + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) + self.markets = {"binance": ExchangeBase(client_config_map=self.client_config_map)} self.notifications = [] self.log_errors = [] assign_config_default(c_map) diff --git a/test/hummingbot/strategy/spot_perpetual_arbitrage/test_spot_perpetual_arbitrage.py b/test/hummingbot/strategy/spot_perpetual_arbitrage/test_spot_perpetual_arbitrage.py index 800ec8a584..eb1b80db51 100644 --- a/test/hummingbot/strategy/spot_perpetual_arbitrage/test_spot_perpetual_arbitrage.py +++ b/test/hummingbot/strategy/spot_perpetual_arbitrage/test_spot_perpetual_arbitrage.py @@ -5,6 +5,8 @@ import pandas as pd +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.connector_base import ConnectorBase from hummingbot.connector.derivative.position import Position from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams @@ -45,7 +47,9 @@ def setUp(self): self.order_fill_logger: EventLogger = EventLogger() self.cancel_order_logger: EventLogger = EventLogger() self.clock: Clock = Clock(ClockMode.BACKTEST, 1, self.start_timestamp, self.end_timestamp) - self.spot_connector: MockPaperExchange = MockPaperExchange() + self.spot_connector: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) self.spot_connector.set_balanced_order_book(trading_pair=trading_pair, mid_price=100, min_price=1, @@ -62,7 +66,9 @@ def setUp(self): self.spot_market_info = MarketTradingPairTuple(self.spot_connector, trading_pair, base_asset, quote_asset) - self.perp_connector: MockPerpConnector = MockPerpConnector() + self.perp_connector: MockPerpConnector = MockPerpConnector( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) self.perp_connector.set_leverage(trading_pair, 5) self.perp_connector.set_balanced_order_book(trading_pair=trading_pair, mid_price=110, diff --git a/test/hummingbot/strategy/spot_perpetual_arbitrage/test_spot_perpetual_arbitrage_start.py b/test/hummingbot/strategy/spot_perpetual_arbitrage/test_spot_perpetual_arbitrage_start.py index a42c81c007..cbfff97b67 100644 --- a/test/hummingbot/strategy/spot_perpetual_arbitrage/test_spot_perpetual_arbitrage_start.py +++ b/test/hummingbot/strategy/spot_perpetual_arbitrage/test_spot_perpetual_arbitrage_start.py @@ -1,12 +1,15 @@ -from decimal import Decimal import unittest.mock +from decimal import Decimal +from test.hummingbot.strategy import assign_config_default +from test.mock.mock_perp_connector import MockPerpConnector + import hummingbot.strategy.spot_perpetual_arbitrage.start as strategy_start +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange_base import ExchangeBase from hummingbot.strategy.spot_perpetual_arbitrage.spot_perpetual_arbitrage_config_map import ( - spot_perpetual_arbitrage_config_map as strategy_cmap + spot_perpetual_arbitrage_config_map as strategy_cmap, ) -from test.hummingbot.strategy import assign_config_default -from test.mock.mock_perp_connector import MockPerpConnector class SpotPerpetualArbitrageStartTest(unittest.TestCase): @@ -14,7 +17,10 @@ class SpotPerpetualArbitrageStartTest(unittest.TestCase): def setUp(self) -> None: super().setUp() self.strategy = None - self.markets = {"binance": ExchangeBase(), "kucoin": MockPerpConnector()} + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) + self.markets = { + "binance": ExchangeBase(client_config_map=self.client_config_map), + "kucoin": MockPerpConnector(client_config_map=self.client_config_map)} self.notifications = [] self.log_errors = [] assign_config_default(strategy_cmap) diff --git a/test/hummingbot/strategy/test_market_trading_pair_tuple.py b/test/hummingbot/strategy/test_market_trading_pair_tuple.py index 470a9f28e1..a81c8bd767 100644 --- a/test/hummingbot/strategy/test_market_trading_pair_tuple.py +++ b/test/hummingbot/strategy/test_market_trading_pair_tuple.py @@ -6,6 +6,8 @@ import pandas as pd +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode @@ -41,7 +43,9 @@ class MarketTradingPairTupleUnitTest(unittest.TestCase): def setUp(self): self.clock: Clock = Clock(ClockMode.BACKTEST, self.clock_tick_size, self.start_timestamp, self.end_timestamp) - self.market: MockPaperExchange = MockPaperExchange() + self.market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) self.market.set_balanced_order_book(trading_pair=self.trading_pair, mid_price=100, min_price=50, diff --git a/test/hummingbot/strategy/test_order_tracker.py b/test/hummingbot/strategy/test_order_tracker.py index c497e90c83..2c83449b56 100644 --- a/test/hummingbot/strategy/test_order_tracker.py +++ b/test/hummingbot/strategy/test_order_tracker.py @@ -6,6 +6,8 @@ import pandas as pd +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode from hummingbot.core.data_type.limit_order import LimitOrder @@ -50,7 +52,9 @@ def setUpClass(cls): for i in range(20) ] - cls.market: MockPaperExchange = MockPaperExchange() + cls.market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) cls.market_info: MarketTradingPairTuple = MarketTradingPairTuple( cls.market, cls.trading_pair, *cls.trading_pair.split("-") ) diff --git a/test/hummingbot/strategy/test_script_strategy_base.py b/test/hummingbot/strategy/test_script_strategy_base.py index 1564d5283f..77a2a15bc6 100644 --- a/test/hummingbot/strategy/test_script_strategy_base.py +++ b/test/hummingbot/strategy/test_script_strategy_base.py @@ -4,6 +4,8 @@ import pandas as pd +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock @@ -41,7 +43,9 @@ def setUp(self): self.initial_mid_price: int = 100 self.clock_tick_size = 1 self.clock: Clock = Clock(ClockMode.BACKTEST, self.clock_tick_size, self.start_timestamp, self.end_timestamp) - self.connector: MockPaperExchange = MockPaperExchange() + self.connector: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) self.connector.set_balanced_order_book(trading_pair=self.trading_pair, mid_price=100, min_price=50, diff --git a/test/hummingbot/strategy/test_strategy_base.py b/test/hummingbot/strategy/test_strategy_base.py index e4da7a67d3..a57407b75f 100644 --- a/test/hummingbot/strategy/test_strategy_base.py +++ b/test/hummingbot/strategy/test_strategy_base.py @@ -7,6 +7,8 @@ from decimal import Decimal from typing import Any, Dict, List, Tuple, Union +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.client.hummingbot_application import HummingbotApplication from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.in_flight_order_base import InFlightOrderBase @@ -24,8 +26,8 @@ class ExtendedMockPaperExchange(MockPaperExchange): - def __init__(self): - super().__init__() + def __init__(self, client_config_map: "ClientConfigAdapter"): + super().__init__(client_config_map) self._in_flight_orders = {} @@ -61,7 +63,9 @@ def setUpClass(cls): cls.trading_pair = "COINALPHA-HBOT" def setUp(self): - self.market: ExtendedMockPaperExchange = ExtendedMockPaperExchange() + self.market: ExtendedMockPaperExchange = ExtendedMockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) self.market_info: MarketTradingPairTuple = MarketTradingPairTuple( self.market, self.trading_pair, *self.trading_pair.split("-") ) @@ -125,7 +129,9 @@ def test_add_markets(self): self.assertEqual(1, len(self.strategy.active_markets)) - new_market: MockPaperExchange = MockPaperExchange() + new_market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) self.strategy.add_markets([new_market]) self.assertEqual(2, len(self.strategy.active_markets)) diff --git a/test/hummingbot/strategy/test_strategy_py_base.py b/test/hummingbot/strategy/test_strategy_py_base.py index 06e8475e71..b45b6195fb 100644 --- a/test/hummingbot/strategy/test_strategy_py_base.py +++ b/test/hummingbot/strategy/test_strategy_py_base.py @@ -5,6 +5,8 @@ from decimal import Decimal from typing import Union +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.data_type.limit_order import LimitOrder @@ -69,7 +71,9 @@ def setUpClass(cls): cls.trading_pair = "COINALPHA-HBOT" def setUp(self): - self.market: MockPaperExchange = MockPaperExchange() + self.market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) self.market_info: MarketTradingPairTuple = MarketTradingPairTuple( self.market, self.trading_pair, *self.trading_pair.split("-") ) diff --git a/test/hummingbot/strategy/twap/test_twap.py b/test/hummingbot/strategy/twap/test_twap.py index c8d2228c9b..5738bac77a 100644 --- a/test/hummingbot/strategy/twap/test_twap.py +++ b/test/hummingbot/strategy/twap/test_twap.py @@ -6,6 +6,8 @@ import pandas as pd +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.paper_trade.paper_trade_exchange import QuantizationParams from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange from hummingbot.core.clock import Clock, ClockMode @@ -44,7 +46,9 @@ def setUp(self): self.log_records = [] self.clock: Clock = Clock(ClockMode.BACKTEST, self.clock_tick_size, self.start_timestamp, self.end_timestamp) - self.market: MockPaperExchange = MockPaperExchange() + self.market: MockPaperExchange = MockPaperExchange( + client_config_map=ClientConfigAdapter(ClientConfigMap()) + ) self.mid_price = 100 self.order_delay_time = 15 self.cancel_order_wait_time = 45 diff --git a/test/hummingbot/strategy/twap/test_twap_trade_strategy.py b/test/hummingbot/strategy/twap/test_twap_trade_strategy.py index 58b01b90c3..3362456bc7 100644 --- a/test/hummingbot/strategy/twap/test_twap_trade_strategy.py +++ b/test/hummingbot/strategy/twap/test_twap_trade_strategy.py @@ -1,17 +1,15 @@ import time -from decimal import Decimal from datetime import datetime +from decimal import Decimal +from test.hummingbot.strategy.twap.twap_test_support import MockExchange from unittest import TestCase -from hummingbot.core.clock import ( - Clock, - ClockMode) +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter +from hummingbot.core.clock import Clock, ClockMode from hummingbot.strategy.conditional_execution_state import RunInTimeConditionalExecutionState - -from hummingbot.strategy.twap import TwapTradeStrategy from hummingbot.strategy.market_trading_pair_tuple import MarketTradingPairTuple - -from test.hummingbot.strategy.twap.twap_test_support import MockExchange +from hummingbot.strategy.twap import TwapTradeStrategy class TwapTradeStrategyTest(TestCase): @@ -41,7 +39,7 @@ def test_creation_without_market_info_fails(self): self.assertEqual(str(ex_context.exception), "market_infos must not be empty.") def test_start(self): - exchange = MockExchange() + exchange = MockExchange(client_config_map=ClientConfigAdapter(ClientConfigMap())) marketTuple = MarketTradingPairTuple(exchange, "ETH-USDT", "ETH", "USDT") strategy = TwapTradeStrategy(market_infos=[marketTuple], is_buy=True, @@ -57,7 +55,7 @@ def test_start(self): self.assertTrue(self._is_logged('INFO', 'Waiting for 10.0 to place orders')) def test_tick_logs_warning_when_market_not_ready(self): - exchange = MockExchange() + exchange = MockExchange(client_config_map=ClientConfigAdapter(ClientConfigMap())) exchange.ready = False marketTuple = MarketTradingPairTuple(exchange, "ETH-USDT", "ETH", "USDT") strategy = TwapTradeStrategy(market_infos=[marketTuple], @@ -75,7 +73,7 @@ def test_tick_logs_warning_when_market_not_ready(self): self.assertTrue(self._is_logged('WARNING', "Markets are not ready. No market making trades are permitted.")) def test_tick_logs_warning_when_market_not_connected(self): - exchange = MockExchange() + exchange = MockExchange(client_config_map=ClientConfigAdapter(ClientConfigMap())) exchange.ready = True marketTuple = MarketTradingPairTuple(exchange, "ETH-USDT", "ETH", "USDT") strategy = TwapTradeStrategy(market_infos=[marketTuple], @@ -95,7 +93,7 @@ def test_tick_logs_warning_when_market_not_connected(self): "Market making may be dangerous when markets or networks are unstable."))) def test_status(self): - exchange = MockExchange() + exchange = MockExchange(client_config_map=ClientConfigAdapter(ClientConfigMap())) exchange.buy_price = Decimal("25100") exchange.sell_price = Decimal("24900") exchange.update_account_balance({"ETH": Decimal("100000"), "USDT": Decimal(10000)}) @@ -128,7 +126,7 @@ def test_status(self): self.assertEqual(expected_status, status) def test_status_with_time_span_execution(self): - exchange = MockExchange() + exchange = MockExchange(client_config_map=ClientConfigAdapter(ClientConfigMap())) exchange.buy_price = Decimal("25100") exchange.sell_price = Decimal("24900") exchange.update_account_balance({"ETH": Decimal("100000"), "USDT": Decimal(10000)}) @@ -166,7 +164,7 @@ def test_status_with_time_span_execution(self): self.assertEqual(expected_status, status) def test_status_with_delayed_start_execution(self): - exchange = MockExchange() + exchange = MockExchange(client_config_map=ClientConfigAdapter(ClientConfigMap())) exchange.buy_price = Decimal("25100") exchange.sell_price = Decimal("24900") exchange.update_account_balance({"ETH": Decimal("100000"), "USDT": Decimal(10000)}) diff --git a/test/hummingbot/strategy/twap/twap_test_support.py b/test/hummingbot/strategy/twap/twap_test_support.py index 548e434d67..2b29046396 100644 --- a/test/hummingbot/strategy/twap/twap_test_support.py +++ b/test/hummingbot/strategy/twap/twap_test_support.py @@ -1,21 +1,24 @@ from decimal import Decimal -from typing import Dict, List, Optional +from typing import TYPE_CHECKING, Dict, List, Optional from hummingbot.connector.exchange_base import ExchangeBase from hummingbot.connector.in_flight_order_base import InFlightOrderBase from hummingbot.core.data_type.cancellation_result import CancellationResult +from hummingbot.core.data_type.common import OrderType, TradeType from hummingbot.core.data_type.limit_order import LimitOrder from hummingbot.core.data_type.order_book import OrderBook from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee -from hummingbot.core.data_type.common import OrderType, TradeType + +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter s_decimal_NaN = Decimal("nan") class MockExchange(ExchangeBase): - def __init__(self): - super(MockExchange, self).__init__() + def __init__(self, client_config_map: "ClientConfigAdapter"): + super(MockExchange, self).__init__(client_config_map) self._buy_price = Decimal(1) self._sell_price = Decimal(1) diff --git a/test/hummingbot/test_hummingbot_application.py b/test/hummingbot/test_hummingbot_application.py index c330bed86c..5ac836ee46 100644 --- a/test/hummingbot/test_hummingbot_application.py +++ b/test/hummingbot/test_hummingbot_application.py @@ -1,5 +1,5 @@ import unittest -from unittest.mock import patch, MagicMock +from unittest.mock import MagicMock, patch from hummingbot.client.hummingbot_application import HummingbotApplication @@ -16,7 +16,7 @@ def test_set_strategy_file_name(self, mock: MagicMock): self.app.strategy_file_name = file_name self.assertEqual(file_name, self.app.strategy_file_name) - mock.assert_called_with(strategy_name) + mock.assert_called_with(self.app.client_config_map, strategy_name) @patch("hummingbot.model.sql_connection_manager.SQLConnectionManager.get_trade_fills_instance") def test_set_strategy_file_name_to_none(self, mock: MagicMock): diff --git a/test/mock/mock_cli.py b/test/mock/mock_cli.py index 17b498dc4a..fd68072c1f 100644 --- a/test/mock/mock_cli.py +++ b/test/mock/mock_cli.py @@ -1,12 +1,13 @@ import asyncio -from typing import Optional -from unittest.mock import patch, MagicMock, AsyncMock +from typing import TYPE_CHECKING, Optional +from unittest.mock import AsyncMock, MagicMock, patch -from hummingbot.client.ui.hummingbot_cli import HummingbotCLI +if TYPE_CHECKING: + from hummingbot.client.ui.hummingbot_cli import HummingbotCLI class CLIMockingAssistant: - def __init__(self, app: HummingbotCLI): + def __init__(self, app: "HummingbotCLI"): self._app = app self._prompt_patch = patch( "hummingbot.client.ui.hummingbot_cli.HummingbotCLI.prompt" diff --git a/test/mock/mock_perp_connector.py b/test/mock/mock_perp_connector.py index a22f866663..106fefb4ce 100644 --- a/test/mock/mock_perp_connector.py +++ b/test/mock/mock_perp_connector.py @@ -1,5 +1,5 @@ from decimal import Decimal -from typing import Optional +from typing import TYPE_CHECKING, Optional from hummingbot.connector.derivative.perpetual_budget_checker import PerpetualBudgetChecker from hummingbot.connector.mock.mock_paper_exchange.mock_paper_exchange import MockPaperExchange @@ -8,15 +8,22 @@ from hummingbot.core.data_type.trade_fee import AddedToCostTradeFee, TradeFeeSchema from hummingbot.core.utils.estimate_fee import build_perpetual_trade_fee +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + class MockPerpConnector(MockPaperExchange, PerpetualTrading): def __init__( self, + client_config_map: "ClientConfigAdapter", trade_fee_schema: Optional[TradeFeeSchema] = None, buy_collateral_token: Optional[str] = None, sell_collateral_token: Optional[str] = None, ): - MockPaperExchange.__init__(self, trade_fee_schema) + MockPaperExchange.__init__( + self, + client_config_map=client_config_map, + trade_fee_schema=trade_fee_schema) PerpetualTrading.__init__(self) self._budget_checker = PerpetualBudgetChecker(exchange=self) self._funding_payment_span = [0, 10] From 4d46754324027de5ef34b13526a33dcb45340d78 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Thu, 2 Jun 2022 12:25:29 +0300 Subject: [PATCH 128/152] (fix) Introduces several fixes following the merge - Adapts AMM to accommodate for the changes in the way the new trading intensity indicator works and is used. - Adds a check to not print the values for sub-models that are not modes but are simply a collection of settings. --- hummingbot/client/command/start_command.py | 2 +- hummingbot/client/config/client_config_map.py | 6 +++++ hummingbot/client/config/conf_migration.py | 2 +- hummingbot/client/config/config_crypt.py | 4 ++-- hummingbot/client/config/config_helpers.py | 18 ++++++++++---- hummingbot/client/hummingbot_application.py | 4 ++-- hummingbot/client/ui/__init__.py | 2 +- .../avellaneda_market_making.pyx | 24 +++++++++++-------- .../hummingbot/client/config/test_security.py | 2 +- 9 files changed, 42 insertions(+), 22 deletions(-) diff --git a/hummingbot/client/command/start_command.py b/hummingbot/client/command/start_command.py index cb7f5c1d57..c3fa098f79 100644 --- a/hummingbot/client/command/start_command.py +++ b/hummingbot/client/command/start_command.py @@ -210,7 +210,7 @@ async def start_market_making(self, # type: HummingbotApplication ) except ValueError as e: self.notify(f"Error: {e}") - else: + if self._pmm_script_iterator is not None: self.clock.add_iterator(self._pmm_script_iterator) self.notify(f"PMM script ({self.client_config_map.pmm_script_mode.pmm_script_file_path}) started.") self.strategy_task: asyncio.Task = safe_ensure_future(self._run_clock(), loop=self.ev_loop) diff --git a/hummingbot/client/config/client_config_map.py b/hummingbot/client/config/client_config_map.py index 935b373c6f..04078e01cb 100644 --- a/hummingbot/client/config/client_config_map.py +++ b/hummingbot/client/config/client_config_map.py @@ -453,6 +453,9 @@ class GlobalTokenConfigMap(BaseClientModel): ), ) + class Config: + title = "global_token" + # === post-validations === @root_validator() @@ -491,6 +494,9 @@ class CommandsTimeoutConfigMap(BaseClientModel): ), ) + class Config: + title = "commands_timeout" + @validator( "create_command_timeout", "other_commands_timeout", diff --git a/hummingbot/client/config/conf_migration.py b/hummingbot/client/config/conf_migration.py index 124dc2a275..c9577080c1 100644 --- a/hummingbot/client/config/conf_migration.py +++ b/hummingbot/client/config/conf_migration.py @@ -100,7 +100,7 @@ def migrate_global_config() -> List[str]: if key in client_config_map.keys(): migrate_global_config_field(client_config_map, data, key) for key in data: - logging.getLogger().warning(f"ConfigVar {key} was not migrated.") + logging.getLogger().warning(f"Global ConfigVar {key} was not migrated.") errors.extend(client_config_map.validate_model()) if len(errors) == 0: save_to_yml(CLIENT_CONFIG_PATH, client_config_map) diff --git a/hummingbot/client/config/config_crypt.py b/hummingbot/client/config/config_crypt.py index 18eae9d641..d8cc84ac10 100644 --- a/hummingbot/client/config/config_crypt.py +++ b/hummingbot/client/config/config_crypt.py @@ -19,10 +19,10 @@ ) from pydantic import SecretStr -from hummingbot import root_path +from hummingbot.client.settings import CONF_DIR_PATH PASSWORD_VERIFICATION_WORD = "HummingBot" -PASSWORD_VERIFICATION_PATH = root_path() / ".password_verification" +PASSWORD_VERIFICATION_PATH = CONF_DIR_PATH / ".password_verification" class BaseSecretsManager(ABC): diff --git a/hummingbot/client/config/config_helpers.py b/hummingbot/client/config/config_helpers.py index fb718fb0e1..d23016b5d0 100644 --- a/hummingbot/client/config/config_helpers.py +++ b/hummingbot/client/config/config_helpers.py @@ -169,7 +169,7 @@ def traverse(self, secure: bool = True) -> Generator[ConfigTraversalItem, None, type_ = field.type_ if hasattr(self, attr): value = getattr(self, attr) - printable_value = self._get_printable_value(value, secure) + printable_value = self._get_printable_value(attr, value, secure) client_field_data = field_info.extra.get("client_data") else: value = None @@ -210,6 +210,9 @@ def get_description(self, attr_name: str) -> str: def get_default(self, attr_name: str) -> Any: return self._hb_config.__fields__[attr_name].field_info.default + def get_type(self, attr_name: str) -> Type: + return self._hb_config.__fields__[attr_name].type_ + def generate_yml_output_str_with_comments(self) -> str: fragments_with_comments = [self._generate_title()] self._add_model_fragments(fragments_with_comments) @@ -242,16 +245,23 @@ def _disable_validation(self): yield self._hb_config.Config.validate_assignment = True - @staticmethod - def _get_printable_value(value: Any, secure: bool) -> str: + def _get_printable_value(self, attr: str, value: Any, secure: bool) -> str: if isinstance(value, ClientConfigAdapter): - printable_value = value.hb_config.Config.title + if self._is_union(self.get_type(attr)): # it is a union of modes + printable_value = value.hb_config.Config.title + else: # it is a collection of settings stored in a submodule + printable_value = "" elif isinstance(value, SecretStr) and not secure: printable_value = value.get_secret_value() else: printable_value = str(value) return printable_value + @staticmethod + def _is_union(t: Type) -> bool: + is_union = hasattr(t, "__origin__") and t.__origin__ == Union + return is_union + def _dict_in_conf_order(self) -> Dict[str, Any]: d = {} for attr in self._hb_config.__fields__.keys(): diff --git a/hummingbot/client/hummingbot_application.py b/hummingbot/client/hummingbot_application.py index 3911a35411..aa052ec631 100644 --- a/hummingbot/client/hummingbot_application.py +++ b/hummingbot/client/hummingbot_application.py @@ -281,7 +281,7 @@ def _initialize_markets(self, market_names: List[Tuple[str, List[str]]]): conn_setting = AllConnectorSettings.get_connector_settings()[connector_name] if connector_name.endswith("paper_trade") and conn_setting.type == ConnectorType.Exchange: - connector = create_paper_trade_market(conn_setting.parent_name, trading_pairs) + connector = create_paper_trade_market(conn_setting.parent_name, self.client_config_map, trading_pairs) paper_trade_account_balance = self.client_config_map.paper_trade.paper_trade_account_balance if paper_trade_account_balance is not None: for asset, balance in paper_trade_account_balance.items(): @@ -306,7 +306,7 @@ def _initialize_markets(self, market_names: List[Tuple[str, List[str]]]): def _initialize_notifiers(self): self.notifiers.extend( [ - notifier for notifier in self.client_config_map.telegram_mode.get_notifiers() + notifier for notifier in self.client_config_map.telegram_mode.get_notifiers(self) if notifier not in self.notifiers ] ) diff --git a/hummingbot/client/ui/__init__.py b/hummingbot/client/ui/__init__.py index ad2a2cb2ab..1ca0ad341b 100644 --- a/hummingbot/client/ui/__init__.py +++ b/hummingbot/client/ui/__init__.py @@ -107,7 +107,7 @@ def migrate_configs_prompt(secrets_manager_cls: Type[BaseSecretsManager], style: secrets_manager = secrets_manager_cls(password) errors = migrate_configs(secrets_manager) if len(errors) != 0: - _migration_errors_dialog(errors) + _migration_errors_dialog(errors, style) else: message_dialog( title='Configs Migration Success', diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx index 4cf609a22f..5221bbe789 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx @@ -126,8 +126,6 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): except FileNotFoundError: pass - self.update_from_config_map() - def all_markets_ready(self): return all([market.ready for market in self._sb_markets]) @@ -417,15 +415,20 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): if self._trading_intensity_buffer_size == 0 or self._trading_intensity_buffer_size != trading_intensity_buffer_size: self._trading_intensity_buffer_size = trading_intensity_buffer_size + self._trading_intensity = TradingIntensityIndicator( + order_book=self.market_info.order_book, + price_delegate=self._price_delegate, + sampling_length=self._trading_intensity_buffer_size, + ) - if self._trading_intensity is None: - self._trading_intensity = TradingIntensityIndicator( - order_book=self.market_info.order_book, - price_delegate=self._price_delegate, - sampling_length=self._trading_intensity_buffer_size, - ) - else: - self._trading_intensity.sampling_length = trading_intensity_buffer_size + if ( + ( + self._trading_intensity_buffer_size == 0 + or self._trading_intensity_buffer_size != trading_intensity_buffer_size + ) and self._trading_intensity is not None + ): + self._trading_intensity_buffer_size = trading_intensity_buffer_size + self._trading_intensity.sampling_length = trading_intensity_buffer_size self._ticks_to_be_ready += (ticks_to_be_ready_after - ticks_to_be_ready_before) if self._ticks_to_be_ready < 0: @@ -564,6 +567,7 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): cdef c_start(self, Clock clock, double timestamp): StrategyBase.c_start(self, clock, timestamp) + self.update_from_config_map() self._last_timestamp = timestamp self._hanging_orders_tracker.register_events(self.active_markets) diff --git a/test/hummingbot/client/config/test_security.py b/test/hummingbot/client/config/test_security.py index 84cde5cb28..a0d283cfdd 100644 --- a/test/hummingbot/client/config/test_security.py +++ b/test/hummingbot/client/config/test_security.py @@ -24,7 +24,7 @@ def setUp(self) -> None: self.default_pswrd_verification_path = security.PASSWORD_VERIFICATION_PATH self.default_connectors_conf_dir_path = config_helpers.CONNECTORS_CONF_DIR_PATH config_crypt.PASSWORD_VERIFICATION_PATH = ( - Path(self.new_conf_dir_path.name) / ".password_verification" + Path(self.new_conf_dir_path.name) / "conf" / ".password_verification" ) security.PASSWORD_VERIFICATION_PATH = config_crypt.PASSWORD_VERIFICATION_PATH config_helpers.CONNECTORS_CONF_DIR_PATH = ( From 55321501e2da6d489729a5f35b7826749320fb30 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Thu, 2 Jun 2022 14:03:20 +0300 Subject: [PATCH 129/152] (fix) Introduces several fixes following the merge - Adapts AMM to accommodate for the changes in the way the new trading intensity indicator works and is used. - Adds a check to not print the values for sub-models that are not modes but are simply a collection of settings. --- .../trailing_indicators/trading_intensity.pyx | 10 ++++++--- .../avellaneda_market_making.pyx | 17 ++++++-------- .../client/command/test_config_command.py | 21 +++++++++++------- .../client/config/test_config_data_types.py | 22 +++++++++++-------- .../hummingbot/client/config/test_security.py | 7 +++--- 5 files changed, 44 insertions(+), 33 deletions(-) diff --git a/hummingbot/strategy/__utils__/trailing_indicators/trading_intensity.pyx b/hummingbot/strategy/__utils__/trailing_indicators/trading_intensity.pyx index 599a9577e8..3adaf0cd3f 100644 --- a/hummingbot/strategy/__utils__/trailing_indicators/trading_intensity.pyx +++ b/hummingbot/strategy/__utils__/trailing_indicators/trading_intensity.pyx @@ -60,6 +60,10 @@ cdef class TradingIntensityIndicator: def sampling_length(self) -> int: return self._sampling_length + @sampling_length.setter + def sampling_length(self, new_len: int): + self._sampling_length = new_len + @property def last_quotes(self) -> list: """A helper method to be used in unit tests""" @@ -119,9 +123,6 @@ cdef class TradingIntensityIndicator: cdef c_register_trade(self, object trade): self._current_trade_sample.append(trade) - def _estimate_intensity(self): - self.c_estimate_intensity() - cdef c_estimate_intensity(self): cdef: dict trades_consolidated @@ -152,6 +153,9 @@ cdef class TradingIntensityIndicator: # Fit the probability density function; reuse previously calculated parameters as initial values try: + print(f"price_levels = {price_levels}") + print(f"lambdas_adj = {lambdas_adj}") + print(f"alpha = {self._alpha} :: kappa = {self._kappa}") params = curve_fit(lambda t, a, b: a*np.exp(-b*t), price_levels, lambdas_adj, diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx index 5221bbe789..8914b4c713 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx @@ -413,23 +413,20 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): else: self._avg_vol.sampling_length = volatility_buffer_size - if self._trading_intensity_buffer_size == 0 or self._trading_intensity_buffer_size != trading_intensity_buffer_size: + if ( + self._trading_intensity_buffer_size == 0 + or self._trading_intensity_buffer_size != trading_intensity_buffer_size + ): self._trading_intensity_buffer_size = trading_intensity_buffer_size + self._trading_intensity.sampling_length = trading_intensity_buffer_size + + if self._trading_intensity is None and self.market_info.market.ready: self._trading_intensity = TradingIntensityIndicator( order_book=self.market_info.order_book, price_delegate=self._price_delegate, sampling_length=self._trading_intensity_buffer_size, ) - if ( - ( - self._trading_intensity_buffer_size == 0 - or self._trading_intensity_buffer_size != trading_intensity_buffer_size - ) and self._trading_intensity is not None - ): - self._trading_intensity_buffer_size = trading_intensity_buffer_size - self._trading_intensity.sampling_length = trading_intensity_buffer_size - self._ticks_to_be_ready += (ticks_to_be_ready_after - ticks_to_be_ready_before) if self._ticks_to_be_ready < 0: self._ticks_to_be_ready = 0 diff --git a/test/hummingbot/client/command/test_config_command.py b/test/hummingbot/client/command/test_config_command.py index a8ac81c31b..a7d5aa2157 100644 --- a/test/hummingbot/client/command/test_config_command.py +++ b/test/hummingbot/client/command/test_config_command.py @@ -3,6 +3,7 @@ from collections import Awaitable from decimal import Decimal from test.mock.mock_cli import CLIMockingAssistant +from typing import Union from unittest.mock import MagicMock, patch from pydantic import Field @@ -66,14 +67,14 @@ def test_list_configs(self, notify_mock, get_strategy_config_map_mock): " | telegram_mode | telegram_disabled |\n" " | send_error_logs | True |\n" " | pmm_script_mode | pmm_script_disabled |\n" - " | gateway | gateway |\n" + " | gateway | |\n" " | ∟ gateway_api_host | localhost |\n" " | ∟ gateway_api_port | 5000 |\n" " | rate_oracle_source | binance |\n" - " | global_token | None |\n" + " | global_token | |\n" " | ∟ global_token_symbol | $ |\n" " | rate_limits_share_pct | 100 |\n" - " | commands_timeout | None |\n" + " | commands_timeout | |\n" " | ∟ create_command_timeout | 10 |\n" " | ∟ other_commands_timeout | 30 |\n" " | tables_format | psql |\n" @@ -121,16 +122,20 @@ class DoubleNestedModel(BaseClientModel): class Config: title = "double_nested_model" - class NestedModel(BaseClientModel): + class NestedModelOne(BaseClientModel): nested_attr: str = Field(default="some value") double_nested_model: DoubleNestedModel = Field(default=DoubleNestedModel()) class Config: - title = "nested_model" + title = "nested_mode_one" + + class NestedModelTwo(BaseClientModel): + class Config: + title = "nested_mode_two" class DummyModel(BaseClientModel): some_attr: int = Field(default=1) - nested_model: NestedModel = Field(default=NestedModel()) + nested_model: Union[NestedModelTwo, NestedModelOne] = Field(default=NestedModelOne()) another_attr: Decimal = Field(default=Decimal("1.0")) missing_no_default: int = Field(default=...) @@ -150,9 +155,9 @@ class Config: "\n | Key | Value |" "\n |------------------------+------------------------|" "\n | some_attr | 1 |" - "\n | nested_model | nested_model |" + "\n | nested_model | nested_mode_one |" "\n | ∟ nested_attr | some value |" - "\n | ∟ double_nested_model | double_nested_model |" + "\n | ∟ double_nested_model | |" "\n | ∟ double_nested_attr | 3.0 |" "\n | another_attr | 1.0 |" "\n | missing_no_default | &cMISSING_AND_REQUIRED |" diff --git a/test/hummingbot/client/config/test_config_data_types.py b/test/hummingbot/client/config/test_config_data_types.py index ff64f799f4..216ba6cc22 100644 --- a/test/hummingbot/client/config/test_config_data_types.py +++ b/test/hummingbot/client/config/test_config_data_types.py @@ -3,7 +3,7 @@ import unittest from datetime import date, datetime, time from decimal import Decimal -from typing import Awaitable, Dict +from typing import Awaitable, Dict, Union from unittest.mock import patch from pydantic import Field, SecretStr @@ -90,29 +90,33 @@ class DoubleNestedModel(BaseClientModel): class Config: title = "double_nested_model" - class NestedModel(BaseClientModel): + class NestedModelOne(BaseClientModel): nested_attr: str = Field(default="some value") double_nested_model: DoubleNestedModel = Field(default=DoubleNestedModel()) class Config: - title = "nested_model" + title = "nested_mode_one" + + class NestedModelTwo(BaseClientModel): + class Config: + title = "nested_mode_two" class DummyModel(BaseClientModel): some_attr: int = Field(default=1, client_data=ClientFieldData()) - nested_model: NestedModel = Field(default=NestedModel()) + nested_model: Union[NestedModelTwo, NestedModelOne] = Field(default=NestedModelOne()) another_attr: Decimal = Field(default=Decimal("1.0")) class Config: title = "dummy_model" - expected = [ + expected_values = [ ConfigTraversalItem(0, "some_attr", "some_attr", 1, "1", ClientFieldData(), None, int), ConfigTraversalItem( 0, "nested_model", "nested_model", - ClientConfigAdapter(NestedModel()), - "nested_model", + ClientConfigAdapter(NestedModelOne()), + "nested_mode_one", None, None, NestedModel, @@ -125,7 +129,7 @@ class Config: "nested_model.double_nested_model", "double_nested_model", ClientConfigAdapter(DoubleNestedModel()), - "double_nested_model", + "", None, None, DoubleNestedModel, @@ -143,7 +147,7 @@ class Config: ] cm = ClientConfigAdapter(DummyModel()) - for expected, actual in zip(expected, cm.traverse()): + for expected, actual in zip(expected_values, cm.traverse()): self.assertEqual(expected.depth, actual.depth) self.assertEqual(expected.config_path, actual.config_path) self.assertEqual(expected.attr, actual.attr) diff --git a/test/hummingbot/client/config/test_security.py b/test/hummingbot/client/config/test_security.py index a0d283cfdd..4712430b24 100644 --- a/test/hummingbot/client/config/test_security.py +++ b/test/hummingbot/client/config/test_security.py @@ -23,9 +23,10 @@ def setUp(self) -> None: self.new_conf_dir_path = TemporaryDirectory() self.default_pswrd_verification_path = security.PASSWORD_VERIFICATION_PATH self.default_connectors_conf_dir_path = config_helpers.CONNECTORS_CONF_DIR_PATH - config_crypt.PASSWORD_VERIFICATION_PATH = ( - Path(self.new_conf_dir_path.name) / "conf" / ".password_verification" - ) + mock_conf_dir = Path(self.new_conf_dir_path.name) / "conf" + mock_conf_dir.mkdir(parents=True, exist_ok=True) + config_crypt.PASSWORD_VERIFICATION_PATH = mock_conf_dir / ".password_verification" + security.PASSWORD_VERIFICATION_PATH = config_crypt.PASSWORD_VERIFICATION_PATH config_helpers.CONNECTORS_CONF_DIR_PATH = ( Path(self.new_conf_dir_path.name) / "connectors" From 21c6bca58d4babef115d72ca23bdd99951f9e8ea Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Thu, 2 Jun 2022 15:05:10 +0300 Subject: [PATCH 130/152] (fix) Fixes failing tests --- .../trailing_indicators/trading_intensity.pyx | 3 --- .../avellaneda_market_making.pyx | 6 +++++- .../test_avellaneda_market_making.py | 20 ++++++++++--------- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/hummingbot/strategy/__utils__/trailing_indicators/trading_intensity.pyx b/hummingbot/strategy/__utils__/trailing_indicators/trading_intensity.pyx index 3adaf0cd3f..9e9d13e332 100644 --- a/hummingbot/strategy/__utils__/trailing_indicators/trading_intensity.pyx +++ b/hummingbot/strategy/__utils__/trailing_indicators/trading_intensity.pyx @@ -153,9 +153,6 @@ cdef class TradingIntensityIndicator: # Fit the probability density function; reuse previously calculated parameters as initial values try: - print(f"price_levels = {price_levels}") - print(f"lambdas_adj = {lambdas_adj}") - print(f"alpha = {self._alpha} :: kappa = {self._kappa}") params = curve_fit(lambda t, a, b: a*np.exp(-b*t), price_levels, lambdas_adj, diff --git a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx index 8914b4c713..d1182fb1ea 100644 --- a/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx +++ b/hummingbot/strategy/avellaneda_market_making/avellaneda_market_making.pyx @@ -126,6 +126,9 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): except FileNotFoundError: pass + self.get_config_map_execution_mode() + self.get_config_map_hanging_orders() + def all_markets_ready(self): return all([market.ready for market in self._sb_markets]) @@ -418,7 +421,8 @@ cdef class AvellanedaMarketMakingStrategy(StrategyBase): or self._trading_intensity_buffer_size != trading_intensity_buffer_size ): self._trading_intensity_buffer_size = trading_intensity_buffer_size - self._trading_intensity.sampling_length = trading_intensity_buffer_size + if self._trading_intensity is not None: + self._trading_intensity.sampling_length = trading_intensity_buffer_size if self._trading_intensity is None and self.market_info.market.ready: self._trading_intensity = TradingIntensityIndicator( diff --git a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py index 3f8db48580..ef0d1a5337 100644 --- a/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py +++ b/test/hummingbot/strategy/avellaneda_market_making/test_avellaneda_market_making.py @@ -131,13 +131,15 @@ def setUp(self): sampling_length=20) self.strategy.avg_vol = self.avg_vol_indicator - self.strategy.trading_intensity = self.trading_intensity_indicator self.clock: Clock = Clock(ClockMode.BACKTEST, self.clock_tick_size, self.start_timestamp, self.end_timestamp) self.clock.add_iterator(self.market) self.clock.add_iterator(self.strategy) self.strategy.start(self.clock, self.start_timestamp) + + self.strategy.trading_intensity = self.trading_intensity_indicator + self.clock.backtest_til(self.start_timestamp) def tearDown(self) -> None: @@ -716,10 +718,10 @@ def test_calculate_reservation_price_and_optimal_spread_timeframe_constrained(se self.strategy.calculate_reservation_price_and_optimal_spread() # Check reservation_price, optimal_ask and optimal_bid - self.assertAlmostEqual(Decimal("100.0326907301569683694926933"), self.strategy.reservation_price, 2) - self.assertAlmostEqual(Decimal("0.5839922263432378129408529925"), self.strategy.optimal_spread, 2) - self.assertAlmostEqual(Decimal("100.3246868433285872759631198"), self.strategy.optimal_ask, 2) - self.assertAlmostEqual(Decimal("99.74069461698534946302226680"), self.strategy.optimal_bid, 2) + self.assertAlmostEqual(Decimal("100.035"), self.strategy.reservation_price, 2) + self.assertAlmostEqual(Decimal("0.592"), self.strategy.optimal_spread, 2) + self.assertAlmostEqual(Decimal("100.331"), self.strategy.optimal_ask, 2) + self.assertAlmostEqual(Decimal("99.739"), self.strategy.optimal_bid, 2) def test_calculate_reservation_price_and_optimal_spread_timeframe_infinite(self): # Init params @@ -737,10 +739,10 @@ def test_calculate_reservation_price_and_optimal_spread_timeframe_infinite(self) self.strategy.calculate_reservation_price_and_optimal_spread() # Check reservation_price, optimal_ask and optimal_bid - self.assertAlmostEqual(Decimal("100.0368705177791277118658666"), self.strategy.reservation_price, 2) - self.assertAlmostEqual(Decimal("0.5836641014510773004537887908"), self.strategy.optimal_spread, 2) - self.assertAlmostEqual(Decimal("100.3287025685046663620927610"), self.strategy.optimal_ask, 2) - self.assertAlmostEqual(Decimal("99.74503846705358906163897220"), self.strategy.optimal_bid, 2) + self.assertAlmostEqual(Decimal("100.040"), self.strategy.reservation_price, 2) + self.assertAlmostEqual(Decimal("0.594"), self.strategy.optimal_spread, 2) + self.assertAlmostEqual(Decimal("100.337"), self.strategy.optimal_ask, 2) + self.assertAlmostEqual(Decimal("99.743"), self.strategy.optimal_bid, 2) def test_create_proposal_based_on_order_override(self): # Initial check for empty order_override From de031ddb07f00bab5319de26ab87ee0bd8d034a5 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Thu, 2 Jun 2022 17:01:08 +0300 Subject: [PATCH 131/152] (feat) Adds a migration step for AMM --- hummingbot/client/config/conf_migration.py | 53 +++++++++++++++++++++- hummingbot/client/settings.py | 4 +- 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/hummingbot/client/config/conf_migration.py b/hummingbot/client/config/conf_migration.py index c9577080c1..71d02a4804 100644 --- a/hummingbot/client/config/conf_migration.py +++ b/hummingbot/client/config/conf_migration.py @@ -28,6 +28,9 @@ from hummingbot.client.config.config_helpers import ClientConfigAdapter, save_to_yml from hummingbot.client.config.security import Security from hummingbot.client.settings import CLIENT_CONFIG_PATH, CONF_DIR_PATH, STRATEGIES_CONF_DIR_PATH +from hummingbot.strategy.avellaneda_market_making.avellaneda_market_making_config_map_pydantic import ( + AvellanedaMarketMakingConfigMap, +) from hummingbot.strategy.cross_exchange_market_making.cross_exchange_market_making_config_map_pydantic import ( CrossExchangeMarketMakingConfigMap, ) @@ -242,12 +245,60 @@ def migrate_strategy_confs_paths(): if "strategy" in conf and _has_connector_field(conf): new_path = strategies_conf_dir_path / child.name child.rename(new_path) - if conf["strategy"] == "cross_exchange_market_making": + if conf["strategy"] == "avellaneda_market_making": + errors.extend(migrate_amm_confs(conf, new_path)) + elif conf["strategy"] == "cross_exchange_market_making": errors.extend(migrate_xemm_confs(conf, new_path)) logging.getLogger().info(f"Migrated conf for {conf['strategy']}") return errors +def migrate_amm_confs(conf, new_path) -> List[str]: + execution_timeframe = conf.pop("execution_timeframe") + if execution_timeframe == "infinite": + conf["execution_timeframe_mode"] = {} + conf.pop("start_time") + conf.pop("end_time") + elif execution_timeframe == "from_date_to_date": + conf["execution_timeframe_mode"] = { + "start_datetime": conf.pop("start_time"), + "end_datetime": conf.pop("end_time"), + } + else: + assert execution_timeframe == "daily_between_times" + conf["execution_timeframe_mode"] = { + "start_time": conf.pop("start_time"), + "end_time": conf.pop("end_time"), + } + order_levels = int(conf.pop("order_levels")) + if order_levels == 1: + conf["order_levels_mode"] = {} + conf.pop("level_distances") + else: + conf["order_levels_mode"] = { + "order_levels": order_levels, + "level_distances": conf.pop("level_distances") + } + hanging_orders_enabled = conf.pop("hanging_orders_enabled") + if not hanging_orders_enabled: + conf["hanging_orders_mode"] = {} + conf.pop("hanging_orders_cancel_pct") + else: + conf["hanging_orders_mode"] = { + "hanging_orders_cancel_pct": conf.pop("hanging_orders_cancel_pct") + } + if "template_version" in conf: + conf.pop("template_version") + try: + config_map = ClientConfigAdapter(AvellanedaMarketMakingConfigMap(**conf)) + save_to_yml(new_path, config_map) + errors = [] + except Exception as e: + logging.getLogger().error(str(e)) + errors = [str(e)] + return errors + + def migrate_xemm_confs(conf, new_path) -> List[str]: if "active_order_canceling" in conf: if conf["active_order_canceling"]: diff --git a/hummingbot/client/settings.py b/hummingbot/client/settings.py index 725e9fdae4..c1caaa4733 100644 --- a/hummingbot/client/settings.py +++ b/hummingbot/client/settings.py @@ -358,7 +358,9 @@ def update_connector_config_keys(cls, new_config_keys: "BaseConnectorConfigMap") @classmethod def get_exchange_names(cls) -> Set[str]: - return {cs.name for cs in cls.all_connector_settings.values() if cs.type is ConnectorType.Exchange} + return { + cs.name for cs in cls.all_connector_settings.values() if cs.type is ConnectorType.Exchange + }.union(set(PAPER_TRADE_EXCHANGES)) @classmethod def get_derivative_names(cls) -> Set[str]: From 42cde156d4c904acde087298474c311f52b40d5d Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Thu, 2 Jun 2022 17:10:17 +0300 Subject: [PATCH 132/152] (fix) Disables strategy configuration while the strategy is running --- hummingbot/client/command/config_command.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/hummingbot/client/command/config_command.py b/hummingbot/client/command/config_command.py index 8cbf46e0f6..37f2359cb4 100644 --- a/hummingbot/client/command/config_command.py +++ b/hummingbot/client/command/config_command.py @@ -220,15 +220,18 @@ async def _config_single_key(self, # type: HummingbotApplication ): await self._config_single_key_legacy(key, input_value) else: - if input_value is None: - self.notify("Please follow the prompt to complete configurations: ") client_config_key = key in self.client_config_map.config_paths() if client_config_key: config_map = self.client_config_map file_path = CLIENT_CONFIG_PATH + elif self.strategy is not None: + self.notify("Configuring the strategy while it is running is not currently supported.") + return else: config_map = self.strategy_config_map file_path = STRATEGIES_CONF_DIR_PATH / self.strategy_file_name + if input_value is None: + self.notify("Please follow the prompt to complete configurations: ") if key == "inventory_target_base_pct": await self.asset_ratio_maintenance_prompt(config_map, input_value) elif key == "inventory_price": From 2afc21013cb0a1b95c8e06c8963ce14853302ec4 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Fri, 3 Jun 2022 11:04:47 +0300 Subject: [PATCH 133/152] (fix) Fixes failing tests after the merge --- .../coinflex_perpetual_utils.py | 87 ++++++++++++------- .../exchange/binance/binance_utils.py | 6 +- .../test_coinflex_perpetual_auth.py | 2 +- .../exchange/coinflex/test_coinflex_auth.py | 5 +- 4 files changed, 61 insertions(+), 39 deletions(-) diff --git a/hummingbot/connector/derivative/coinflex_perpetual/coinflex_perpetual_utils.py b/hummingbot/connector/derivative/coinflex_perpetual/coinflex_perpetual_utils.py index 85a72a188a..e894d5fa82 100644 --- a/hummingbot/connector/derivative/coinflex_perpetual/coinflex_perpetual_utils.py +++ b/hummingbot/connector/derivative/coinflex_perpetual/coinflex_perpetual_utils.py @@ -1,9 +1,10 @@ from decimal import Decimal from typing import Any, Dict +from pydantic import Field, SecretStr + import hummingbot.connector.derivative.coinflex_perpetual.constants as CONSTANTS -from hummingbot.client.config.config_methods import using_exchange -from hummingbot.client.config.config_var import ConfigVar +from hummingbot.client.config.config_data_types import BaseConnectorConfigMap, ClientFieldData from hummingbot.core.data_type.trade_fee import TradeFeeSchema from hummingbot.core.utils.tracking_nonce import get_tracking_nonce @@ -42,43 +43,63 @@ def decimal_val_or_none(string_value: str): return Decimal(string_value) if string_value else None -KEYS = { - "coinflex_perpetual_api_key": ConfigVar( - key="coinflex_perpetual_api_key", - prompt="Enter your Coinflex Perpetual API key >>> ", - required_if=using_exchange("coinflex_perpetual"), - is_secure=True, - is_connect_key=True, - ), - "coinflex_perpetual_api_secret": ConfigVar( - key="coinflex_perpetual_api_secret", - prompt="Enter your Coinflex Perpetual API secret >>> ", - required_if=using_exchange("coinflex_perpetual"), - is_secure=True, - is_connect_key=True, - ), -} +class CoinflexPerpetulConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="coinflex_perpetual", const=True, client_data=None) + coinflex_perpetual_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Coinflex Perpetual API key", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ), + ) + coinflex_perpetual_api_secret: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Coinflex Perpetual API secret", + is_secure=True, + is_connect_key=True, + prompt_on_new=True, + ) + ) + + class Config: + title = "coinflex_perpetual" + + +KEYS = CoinflexPerpetulConfigMap.construct() + OTHER_DOMAINS = ["coinflex_perpetual_testnet"] OTHER_DOMAINS_PARAMETER = {"coinflex_perpetual_testnet": "coinflex_perpetual_testnet"} OTHER_DOMAINS_EXAMPLE_PAIR = {"coinflex_perpetual_testnet": "BTC-USDT"} -OTHER_DOMAINS_DEFAULT_FEES = {"coinflex_perpetual_testnet": [0.0, 0.08]} -OTHER_DOMAINS_KEYS = { - "coinflex_perpetual_testnet": { - # add keys for testnet - "coinflex_perpetual_testnet_api_key": ConfigVar( - key="coinflex_perpetual_testnet_api_key", - prompt="Enter your Coinflex Perpetual testnet API key >>> ", - required_if=using_exchange("coinflex_perpetual_testnet"), +OTHER_DOMAINS_DEFAULT_FEES = {"coinflex_perpetual_testnet": DEFAULT_FEES} + + +class CoinflexPerpetulTestnetConfigMap(BaseConnectorConfigMap): + connector: str = Field(default="coinflex_perpetual_testnet", const=True, client_data=None) + coinflex_perpetual_testnet_api_key: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Coinflex Perpetual testnet API key", is_secure=True, is_connect_key=True, + prompt_on_new=True, ), - "coinflex_perpetual_testnet_api_secret": ConfigVar( - key="coinflex_perpetual_testnet_api_secret", - prompt="Enter your Coinflex Perpetual testnet API secret >>> ", - required_if=using_exchange("coinflex_perpetual_testnet"), + ) + coinflex_perpetual_testnet_api_secret: SecretStr = Field( + default=..., + client_data=ClientFieldData( + prompt=lambda cm: "Enter your Coinflex Perpetual testnet API secret", is_secure=True, is_connect_key=True, - ), - } -} + prompt_on_new=True, + ) + ) + + class Config: + title = "coinflex_perpetual_testnet" + + +OTHER_DOMAINS_KEYS = {"coinflex_perpetual_testnet": CoinflexPerpetulTestnetConfigMap.construct()} diff --git a/hummingbot/connector/exchange/binance/binance_utils.py b/hummingbot/connector/exchange/binance/binance_utils.py index 43024ce5be..b0e47c4da0 100644 --- a/hummingbot/connector/exchange/binance/binance_utils.py +++ b/hummingbot/connector/exchange/binance/binance_utils.py @@ -26,7 +26,7 @@ def is_exchange_information_valid(exchange_info: Dict[str, Any]) -> bool: class BinanceConfigMap(BaseConnectorConfigMap): - connector: str = Field(default="binance", client_data=None) + connector: str = Field(default="binance", const=True, client_data=None) binance_api_key: SecretStr = Field( default=..., client_data=ClientFieldData( @@ -55,11 +55,11 @@ class Config: OTHER_DOMAINS = ["binance_us"] OTHER_DOMAINS_PARAMETER = {"binance_us": "us"} OTHER_DOMAINS_EXAMPLE_PAIR = {"binance_us": "BTC-USDT"} -OTHER_DOMAINS_DEFAULT_FEES = {"binance_us": [0.1, 0.1]} +OTHER_DOMAINS_DEFAULT_FEES = {"binance_us": DEFAULT_FEES} class BinanceUSConfigMap(BaseConnectorConfigMap): - connector: str = Field(default="binance_us", client_data=None) + connector: str = Field(default="binance_us", const=True, client_data=None) binance_api_key: SecretStr = Field( default=..., client_data=ClientFieldData( diff --git a/test/hummingbot/connector/derivative/coinflex_perpetual/test_coinflex_perpetual_auth.py b/test/hummingbot/connector/derivative/coinflex_perpetual/test_coinflex_perpetual_auth.py index b2532ed3f2..9c2c54d4a8 100644 --- a/test/hummingbot/connector/derivative/coinflex_perpetual/test_coinflex_perpetual_auth.py +++ b/test/hummingbot/connector/derivative/coinflex_perpetual/test_coinflex_perpetual_auth.py @@ -49,7 +49,7 @@ def test_rest_authenticate(self, time_mock): request = web_utils.CoinflexPerpetualRESTRequest(method=RESTMethod.GET, endpoint="", params=params, is_auth_required=True) configured_request = self.async_run_with_timeout(self.auth.rest_authenticate(request)) - str_timestamp = datetime.fromtimestamp(now).isoformat() + str_timestamp = datetime.utcfromtimestamp(now).isoformat() nonce = int(now * 1e3) encoded_params = "&".join([f"{key}={value}" for key, value in full_params.items()]) diff --git a/test/hummingbot/connector/exchange/coinflex/test_coinflex_auth.py b/test/hummingbot/connector/exchange/coinflex/test_coinflex_auth.py index 3f22f1347d..c29d8aad95 100644 --- a/test/hummingbot/connector/exchange/coinflex/test_coinflex_auth.py +++ b/test/hummingbot/connector/exchange/coinflex/test_coinflex_auth.py @@ -7,10 +7,11 @@ from unittest import TestCase from unittest.mock import patch +from typing_extensions import Awaitable + from hummingbot.connector.exchange.coinflex.coinflex_auth import CoinflexAuth from hummingbot.connector.exchange.coinflex.coinflex_web_utils import CoinflexRESTRequest from hummingbot.core.web_assistant.connections.data_types import RESTMethod -from typing_extensions import Awaitable class CoinflexAuthTests(TestCase): @@ -42,7 +43,7 @@ def test_rest_authenticate(self, time_mock): request = CoinflexRESTRequest(method=RESTMethod.GET, endpoint="", params=params, is_auth_required=True) configured_request = self.async_run_with_timeout(auth.rest_authenticate(request)) - str_timestamp = datetime.fromtimestamp(now).isoformat() + str_timestamp = datetime.utcfromtimestamp(now).isoformat() nonce = int(now * 1e3) encoded_params = "&".join([f"{key}={value}" for key, value in full_params.items()]) From 8df8987b49082aabc15fbe404687c16bb7a06859 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Fri, 3 Jun 2022 11:24:52 +0300 Subject: [PATCH 134/152] (fix) Fixes failing tests after the merge --- .../coinflex_perpetual/coinflex_perpetual_derivative.py | 8 ++++++-- .../test_coinflex_perpetual_derivative.py | 4 ++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/hummingbot/connector/derivative/coinflex_perpetual/coinflex_perpetual_derivative.py b/hummingbot/connector/derivative/coinflex_perpetual/coinflex_perpetual_derivative.py index c2a6f85e57..5be371ab29 100644 --- a/hummingbot/connector/derivative/coinflex_perpetual/coinflex_perpetual_derivative.py +++ b/hummingbot/connector/derivative/coinflex_perpetual/coinflex_perpetual_derivative.py @@ -3,7 +3,7 @@ import time import warnings from decimal import Decimal -from typing import Any, AsyncIterable, Dict, List, Optional +from typing import TYPE_CHECKING, Any, AsyncIterable, Dict, List, Optional from async_timeout import timeout @@ -50,6 +50,9 @@ from hummingbot.core.web_assistant.ws_assistant import WSAssistant from hummingbot.logger import HummingbotLogger +if TYPE_CHECKING: + from hummingbot.client.config.config_helpers import ClientConfigAdapter + bpm_logger = None s_float_NaN = float("nan") s_decimal_0 = Decimal("0") @@ -76,6 +79,7 @@ def logger(cls) -> HummingbotLogger: def __init__( self, + client_config_map: "ClientConfigAdapter", coinflex_perpetual_api_key: str = None, coinflex_perpetual_api_secret: str = None, trading_pairs: Optional[List[str]] = None, @@ -92,7 +96,7 @@ def __init__( self._rest_assistant: Optional[RESTAssistant] = None self._ws_assistant: Optional[WSAssistant] = None - ExchangeBase.__init__(self) + ExchangeBase.__init__(self, client_config_map) PerpetualTrading.__init__(self) self._user_stream_tracker = UserStreamTracker( diff --git a/test/hummingbot/connector/derivative/coinflex_perpetual/test_coinflex_perpetual_derivative.py b/test/hummingbot/connector/derivative/coinflex_perpetual/test_coinflex_perpetual_derivative.py index 2720085ec8..55e2efa0a7 100644 --- a/test/hummingbot/connector/derivative/coinflex_perpetual/test_coinflex_perpetual_derivative.py +++ b/test/hummingbot/connector/derivative/coinflex_perpetual/test_coinflex_perpetual_derivative.py @@ -14,6 +14,8 @@ import hummingbot.connector.derivative.coinflex_perpetual.coinflex_perpetual_web_utils as web_utils import hummingbot.connector.derivative.coinflex_perpetual.constants as CONSTANTS +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.derivative.coinflex_perpetual.coinflex_perpetual_api_order_book_data_source import ( CoinflexPerpetualAPIOrderBookDataSource, ) @@ -52,8 +54,10 @@ def setUp(self) -> None: self.ws_sent_messages = [] self.ws_incoming_messages = asyncio.Queue() self.resume_test_event = asyncio.Event() + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) self.exchange = CoinflexPerpetualDerivative( + client_config_map=self.client_config_map, coinflex_perpetual_api_key="testAPIKey", coinflex_perpetual_api_secret="testSecret", trading_pairs=[self.trading_pair], From be9e9b7993f7d9e837549b28c07770e240e1ec3f Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Fri, 3 Jun 2022 12:31:27 +0300 Subject: [PATCH 135/152] (fix) Addresses all QA comments --- hummingbot/client/command/connect_command.py | 2 -- hummingbot/client/config/client_config_map.py | 2 +- hummingbot/client/ui/custom_widgets.py | 4 ++-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/hummingbot/client/command/connect_command.py b/hummingbot/client/command/connect_command.py index 6684f99ab5..a19f36550f 100644 --- a/hummingbot/client/command/connect_command.py +++ b/hummingbot/client/command/connect_command.py @@ -186,5 +186,3 @@ async def _perform_connect(self, connector_config: ClientConfigAdapter, previous if previous_keys is not None: previous_config = ClientConfigAdapter(connector_config.hb_config.__class__(**previous_keys)) Security.update_secure_config(previous_config) - else: - Security.remove_secure_config(connector_name) diff --git a/hummingbot/client/config/client_config_map.py b/hummingbot/client/config/client_config_map.py index 04078e01cb..fcf4744e83 100644 --- a/hummingbot/client/config/client_config_map.py +++ b/hummingbot/client/config/client_config_map.py @@ -88,7 +88,7 @@ class ColorConfigMap(BaseClientModel): primary_label: str = Field( default="#5FFFD7", client_data=ClientFieldData( - prompt=lambda cm: "What is the background color for secondary label?", + prompt=lambda cm: "What is the background color for primary label?", ), ) secondary_label: str = Field( diff --git a/hummingbot/client/ui/custom_widgets.py b/hummingbot/client/ui/custom_widgets.py index 566eb1fcba..147a2b2190 100644 --- a/hummingbot/client/ui/custom_widgets.py +++ b/hummingbot/client/ui/custom_widgets.py @@ -45,9 +45,9 @@ def __init__(self, client_config_map: ClientConfigAdapter) -> None: style: css for style, css in load_style(client_config_map).style_rules } self.html_tag_css_style_map.update({ - ti.attr.replace("_", "-"): ti.value + ti.attr: ti.value for ti in client_config_map.color.traverse() - if ti.attr.replace("_", "-") not in self.html_tag_css_style_map + if ti.attr not in self.html_tag_css_style_map }) # Maps specific text to its corresponding UI styles From 8a6a7a90b7f91b49dedeb7b10c8136a245f810a5 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Fri, 3 Jun 2022 13:12:01 +0300 Subject: [PATCH 136/152] (fix) Fixes quickstart file --- bin/hummingbot_quickstart.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/hummingbot_quickstart.py b/bin/hummingbot_quickstart.py index 6867c0f486..4397e1f9c3 100755 --- a/bin/hummingbot_quickstart.py +++ b/bin/hummingbot_quickstart.py @@ -28,6 +28,7 @@ from hummingbot.client.hummingbot_application import HummingbotApplication from hummingbot.client.settings import STRATEGIES_CONF_DIR_PATH, AllConnectorSettings from hummingbot.client.ui import login_prompt +from hummingbot.client.ui.style import load_style from hummingbot.core.event.events import HummingbotUIEvent from hummingbot.core.gateway import start_existing_gateway_container from hummingbot.core.management.console import start_management_console @@ -136,8 +137,9 @@ def main(): # If no password is given from the command line, prompt for one. secrets_manager_cls = ETHKeyFileSecretManger + client_config_map = load_client_config_map_from_file() if args.config_password is None: - secrets_manager = login_prompt(secrets_manager_cls) + secrets_manager = login_prompt(secrets_manager_cls, style=load_style(client_config_map)) if not secrets_manager: return else: From b2f3991fa366cf2ad6d1d38a354adeacbde2d604 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 8 Jun 2022 11:12:14 +0300 Subject: [PATCH 137/152] (fix) Bug fix for the `gateway create` command --- hummingbot/client/command/gateway_command.py | 2 +- hummingbot/core/gateway/gateway_http_client.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hummingbot/client/command/gateway_command.py b/hummingbot/client/command/gateway_command.py index fcec32b320..e2c8a926ce 100644 --- a/hummingbot/client/command/gateway_command.py +++ b/hummingbot/client/command/gateway_command.py @@ -114,7 +114,7 @@ async def _generate_certs( pass_phase = Security.secrets_manager.password.get_secret_value() create_self_sign_certs(pass_phase, self.client_config_map) self.notify(f"Gateway SSL certification files are created in {cert_path}.") - self._get_gateway_instance().reload_certs() + self._get_gateway_instance().reload_certs(self.client_config_map) async def _generate_gateway_confs( self, # type: HummingbotApplication diff --git a/hummingbot/core/gateway/gateway_http_client.py b/hummingbot/core/gateway/gateway_http_client.py index 9720dc6701..a8f456e614 100644 --- a/hummingbot/core/gateway/gateway_http_client.py +++ b/hummingbot/core/gateway/gateway_http_client.py @@ -84,12 +84,12 @@ def _http_client(cls, client_config_map: "ClientConfigAdapter", re_init: bool = return cls._shared_client @classmethod - def reload_certs(cls): + def reload_certs(cls, client_config_map: "ClientConfigAdapter"): """ Re-initializes the aiohttp.ClientSession. This should be called whenever there is any updates to the Certificates used to secure a HTTPS connection to the Gateway service. """ - cls._http_client(re_init=True) + cls._http_client(client_config_map, re_init=True) @property def base_url(self) -> str: From 08e343ffb2234878021db375c95c0a2821efe598 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 8 Jun 2022 12:22:06 +0300 Subject: [PATCH 138/152] (fix) Fixes the migration of celo configs --- hummingbot/client/config/conf_migration.py | 62 +++++++++++-------- hummingbot/client/config/config_validators.py | 3 +- 2 files changed, 38 insertions(+), 27 deletions(-) diff --git a/hummingbot/client/config/conf_migration.py b/hummingbot/client/config/conf_migration.py index 71d02a4804..3c370cb0e7 100644 --- a/hummingbot/client/config/conf_migration.py +++ b/hummingbot/client/config/conf_migration.py @@ -39,6 +39,7 @@ encrypted_conf_postfix = ".json" conf_dir_path = CONF_DIR_PATH strategies_conf_dir_path = STRATEGIES_CONF_DIR_PATH +celo_address = None def migrate_configs(secrets_manager: BaseSecretsManager) -> List[str]: @@ -89,6 +90,8 @@ def backup_existing_dir() -> List[str]: def migrate_global_config() -> List[str]: + global celo_address + logging.getLogger().info("\nMigrating the global config...") global_config_path = CONF_DIR_PATH / "conf_global.yml" errors = [] @@ -97,11 +100,16 @@ def migrate_global_config() -> List[str]: data = yaml.safe_load(f) del data["template_version"] client_config_map = ClientConfigAdapter(ClientConfigMap()) - migrate_global_config_modes(client_config_map, data) + _migrate_global_config_modes(client_config_map, data) + "kraken_api_tier" in data and data.pop("kraken_api_tier") + "key_file_path" in data and data.pop("key_file_path") + celo_address = data.get("celo_address") + if celo_address is not None: + data.pop("celo_address") keys = list(data.keys()) for key in keys: if key in client_config_map.keys(): - migrate_global_config_field(client_config_map, data, key) + _migrate_global_config_field(client_config_map, data, key) for key in data: logging.getLogger().warning(f"Global ConfigVar {key} was not migrated.") errors.extend(client_config_map.validate_model()) @@ -115,7 +123,7 @@ def migrate_global_config() -> List[str]: return errors -def migrate_global_config_modes(client_config_map: ClientConfigAdapter, data: Dict): +def _migrate_global_config_modes(client_config_map: ClientConfigAdapter, data: Dict): client_config_map: Union[ClientConfigAdapter, ClientConfigMap] = client_config_map # for IDE autocomplete kill_switch_enabled = data.pop("kill_switch_enabled") @@ -125,10 +133,10 @@ def migrate_global_config_modes(client_config_map: ClientConfigAdapter, data: Di else: client_config_map.kill_switch_mode = KillSwitchDisabledMode() - migrate_global_config_field( + _migrate_global_config_field( client_config_map.paper_trade, data, "paper_trade_exchanges" ) - migrate_global_config_field( + _migrate_global_config_field( client_config_map.paper_trade, data, "paper_trade_account_balance" ) @@ -168,10 +176,10 @@ def migrate_global_config_modes(client_config_map: ClientConfigAdapter, data: Di else: client_config_map.pmm_script_mode = PMMScriptDisabledMode() - migrate_global_config_field( + _migrate_global_config_field( client_config_map.gateway, data, "gateway_api_host" ) - migrate_global_config_field( + _migrate_global_config_field( client_config_map.gateway, data, "gateway_api_port" ) @@ -184,33 +192,33 @@ def migrate_global_config_modes(client_config_map: ClientConfigAdapter, data: Di else: client_config_map.anonymized_metrics_mode = AnonymizedMetricsDisabledMode() - migrate_global_config_field( + _migrate_global_config_field( client_config_map.global_token, data, "global_token", "global_token_name" ) - migrate_global_config_field( + _migrate_global_config_field( client_config_map.global_token, data, "global_token_symbol" ) - migrate_global_config_field( + _migrate_global_config_field( client_config_map.commands_timeout, data, "create_command_timeout" ) - migrate_global_config_field( + _migrate_global_config_field( client_config_map.commands_timeout, data, "other_commands_timeout" ) color_map: Union[ClientConfigAdapter, ColorConfigMap] = client_config_map.color - migrate_global_config_field(color_map, data, "top-pane", "top_pane") - migrate_global_config_field(color_map, data, "bottom-pane", "bottom_pane") - migrate_global_config_field(color_map, data, "output-pane", "output_pane") - migrate_global_config_field(color_map, data, "input-pane", "input_pane") - migrate_global_config_field(color_map, data, "logs-pane", "logs_pane") - migrate_global_config_field(color_map, data, "terminal-primary", "terminal_primary") - migrate_global_config_field(color_map, data, "primary-label", "primary_label") - migrate_global_config_field(color_map, data, "secondary-label", "secondary_label") - migrate_global_config_field(color_map, data, "success-label", "success_label") - migrate_global_config_field(color_map, data, "warning-label", "warning_label") - migrate_global_config_field(color_map, data, "info-label", "info_label") - migrate_global_config_field(color_map, data, "error-label", "error_label") + _migrate_global_config_field(color_map, data, "top-pane", "top_pane") + _migrate_global_config_field(color_map, data, "bottom-pane", "bottom_pane") + _migrate_global_config_field(color_map, data, "output-pane", "output_pane") + _migrate_global_config_field(color_map, data, "input-pane", "input_pane") + _migrate_global_config_field(color_map, data, "logs-pane", "logs_pane") + _migrate_global_config_field(color_map, data, "terminal-primary", "terminal_primary") + _migrate_global_config_field(color_map, data, "primary-label", "primary_label") + _migrate_global_config_field(color_map, data, "secondary-label", "secondary_label") + _migrate_global_config_field(color_map, data, "success-label", "success_label") + _migrate_global_config_field(color_map, data, "warning-label", "warning_label") + _migrate_global_config_field(color_map, data, "info-label", "info_label") + _migrate_global_config_field(color_map, data, "error-label", "error_label") balance_asset_limit = data.pop("balance_asset_limit") if balance_asset_limit is not None: @@ -226,7 +234,7 @@ def migrate_global_config_modes(client_config_map: ClientConfigAdapter, data: Di client_config_map.balance_asset_limit = balance_asset_limit -def migrate_global_config_field( +def _migrate_global_config_field( cm: ClientConfigAdapter, global_config_data: Dict[str, Any], attr: str, cm_attr: Optional[str] = None ): value = global_config_data.pop(attr) @@ -367,8 +375,9 @@ def migrate_connector_confs(secrets_manager: BaseSecretsManager): if connector_dir.name.startswith("_") or connector_dir.name in connector_exceptions: continue try: + suffix = "data_types" if connector_dir.name == "celo" else "utils" util_module_path: str = ( - f"hummingbot.connector.{type_dir.name}.{connector_dir.name}.{connector_dir.name}_utils" + f"hummingbot.connector.{type_dir.name}.{connector_dir.name}.{connector_dir.name}_{suffix}" ) util_module = importlib.import_module(util_module_path) config_keys = getattr(util_module, "KEYS", None) @@ -391,6 +400,9 @@ def _maybe_migrate_encrypted_confs(config_keys: BaseConnectorConfigMap) -> List[ missing_fields = [] for el in cm.traverse(): if el.client_field_data is not None: + if el.attr == "celo_address" and celo_address is not None: + cm.setattr_no_validation(el.attr, celo_address) + continue key_path = conf_dir_path / f"{encrypted_conf_prefix}{el.attr}{encrypted_conf_postfix}" if key_path.exists(): with open(key_path, 'r') as f: diff --git a/hummingbot/client/config/config_validators.py b/hummingbot/client/config/config_validators.py index b3394b7f11..d80ece8334 100644 --- a/hummingbot/client/config/config_validators.py +++ b/hummingbot/client/config/config_validators.py @@ -5,7 +5,6 @@ """ import time - from datetime import datetime from decimal import Decimal from typing import Optional @@ -34,7 +33,7 @@ def validate_connector(value: str) -> Optional[str]: Restrict valid derivatives to the connector file names """ from hummingbot.client.settings import AllConnectorSettings - if value not in AllConnectorSettings.get_connector_settings(): + if value not in AllConnectorSettings.get_connector_settings() and value != "celo": return f"Invalid connector, please choose value from {AllConnectorSettings.get_connector_settings().keys()}" From 8a5011e5cdb92c1cca215f530b9f403054f7a4ef Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Fri, 10 Jun 2022 13:38:05 +0700 Subject: [PATCH 139/152] Update hummingbot/client/command/balance_command.py Co-authored-by: Abel Armoa <30988000+aarmoa@users.noreply.github.com> --- hummingbot/client/command/balance_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/client/command/balance_command.py b/hummingbot/client/command/balance_command.py index d9d1743a7f..6d68dec4b1 100644 --- a/hummingbot/client/command/balance_command.py +++ b/hummingbot/client/command/balance_command.py @@ -48,7 +48,7 @@ def balance(self, # type: HummingbotApplication exchange = args[0] asset = args[1].upper() amount = float(args[2]) - if exchange not in balance_asset_limit or balance_asset_limit[exchange] is None: + if balance_asset_limit.get(exchange) is None: balance_asset_limit[exchange] = {} if amount < 0 and asset in balance_asset_limit[exchange].keys(): balance_asset_limit[exchange].pop(asset) From d2e6b667417e5689abec912a4447c38e7adfdb4c Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Fri, 10 Jun 2022 13:28:25 +0300 Subject: [PATCH 140/152] (fix) Addresses @aarmoa's PR comments --- hummingbot/client/config/client_config_map.py | 2 +- hummingbot/client/ui/hummingbot_cli.py | 1 - hummingbot/connector/connector_base.pyx | 6 +----- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/hummingbot/client/config/client_config_map.py b/hummingbot/client/config/client_config_map.py index fcf4744e83..e5789b21fa 100644 --- a/hummingbot/client/config/client_config_map.py +++ b/hummingbot/client/config/client_config_map.py @@ -673,7 +673,7 @@ class ClientConfigMap(BaseClientModel): "\nPort need to match the final installation port for Gateway"), ) anonymized_metrics_mode: Union[tuple(METRICS_MODES.values())] = Field( - default=AnonymizedMetricsDisabledMode(), + default=AnonymizedMetricsEnabledMode(), description="Whether to enable aggregated order and trade data collection", client_data=ClientFieldData( prompt=lambda cm: f"Select the desired metrics mode ({'/'.join(list(METRICS_MODES.keys()))})", diff --git a/hummingbot/client/ui/hummingbot_cli.py b/hummingbot/client/ui/hummingbot_cli.py index 7209955a80..7bd5fff994 100644 --- a/hummingbot/client/ui/hummingbot_cli.py +++ b/hummingbot/client/ui/hummingbot_cli.py @@ -116,7 +116,6 @@ async def run(self): clipboard=PyperclipClipboard(), ) await self.app.run_async(pre_run=self.did_start_ui) - # await self.app.run_async(pre_run=partial(self.did_start_ui, self.app)) self._stdout_redirect_context.close() def accept(self, buff): diff --git a/hummingbot/connector/connector_base.pyx b/hummingbot/connector/connector_base.pyx index 3f06d6d7d0..af6b5686c1 100644 --- a/hummingbot/connector/connector_base.pyx +++ b/hummingbot/connector/connector_base.pyx @@ -5,7 +5,7 @@ from typing import Dict, List, Set, Tuple, TYPE_CHECKING from hummingbot.client.config.trade_fee_schema_loader import TradeFeeSchemaLoader from hummingbot.connector.in_flight_order_base import InFlightOrderBase from hummingbot.connector.utils import split_hb_trading_pair, TradeFillOrderDetails -from hummingbot.connector.constants import NaN, s_decimal_NaN, s_decimal_0 +from hummingbot.connector.constants import s_decimal_NaN, s_decimal_0 from hummingbot.core.clock cimport Clock from hummingbot.core.data_type.cancellation_result import CancellationResult from hummingbot.core.data_type.common import OrderType, TradeType @@ -19,10 +19,6 @@ if TYPE_CHECKING: from hummingbot.client.config.client_config_map import ClientConfigMap from hummingbot.client.config.config_helpers import ClientConfigAdapter -NaN = float("nan") -s_decimal_NaN = Decimal("nan") -s_decimal_0 = Decimal(0) - cdef class ConnectorBase(NetworkIterator): MARKET_EVENTS = [ From e3a59ae936db5bb733e70e771ac82f3ea193f307 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Fri, 10 Jun 2022 17:35:40 +0300 Subject: [PATCH 141/152] (fix) Quick bug fix for Celo connect command --- hummingbot/client/command/connect_command.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/hummingbot/client/command/connect_command.py b/hummingbot/client/command/connect_command.py index a19f36550f..4ceef00669 100644 --- a/hummingbot/client/command/connect_command.py +++ b/hummingbot/client/command/connect_command.py @@ -140,7 +140,10 @@ async def validate_n_connect_celo(self, to_reconnect: bool = False) -> Optional[ return "No Celo connection has been configured." if CeloCLI.unlocked and not to_reconnect: return None - err_msg = CeloCLI.validate_node_synced() + try: + err_msg = CeloCLI.validate_node_synced() + except FileNotFoundError: + err_msg = "Celo CLI not installed." if err_msg is not None: return err_msg err_msg = CeloCLI.unlock_account(celo_config.celo_address, celo_config.celo_password.get_secret_value()) From 1edc110d032d93b8195cb44e374ce3e4adaa5fcd Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Tue, 14 Jun 2022 15:54:33 +0300 Subject: [PATCH 142/152] (fix) Fixes `celo_address` not being properly migrated --- hummingbot/client/config/conf_migration.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/hummingbot/client/config/conf_migration.py b/hummingbot/client/config/conf_migration.py index 3c370cb0e7..f7ec83e8ba 100644 --- a/hummingbot/client/config/conf_migration.py +++ b/hummingbot/client/config/conf_migration.py @@ -101,11 +101,9 @@ def migrate_global_config() -> List[str]: del data["template_version"] client_config_map = ClientConfigAdapter(ClientConfigMap()) _migrate_global_config_modes(client_config_map, data) - "kraken_api_tier" in data and data.pop("kraken_api_tier") - "key_file_path" in data and data.pop("key_file_path") - celo_address = data.get("celo_address") - if celo_address is not None: - data.pop("celo_address") + data.pop("kraken_api_tier", None) + data.pop("key_file_path", None) + celo_address = data.pop("celo_address", None) keys = list(data.keys()) for key in keys: if key in client_config_map.keys(): From d9edf3ba3ea15b453fb04a7400dc6fd64a182ae6 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 15 Jun 2022 16:11:15 +0300 Subject: [PATCH 143/152] (fix) Fixes kill switch and scripts bugs --- hummingbot/client/command/start_command.py | 2 +- hummingbot/client/config/client_config_map.py | 4 ++-- hummingbot/core/utils/kill_switch.py | 23 +++++++++---------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/hummingbot/client/command/start_command.py b/hummingbot/client/command/start_command.py index c3fa098f79..b185ff5ac1 100644 --- a/hummingbot/client/command/start_command.py +++ b/hummingbot/client/command/start_command.py @@ -211,7 +211,7 @@ async def start_market_making(self, # type: HummingbotApplication except ValueError as e: self.notify(f"Error: {e}") if self._pmm_script_iterator is not None: - self.clock.add_iterator(self._pmm_script_iterator) + self.clock.add_tarterator(self._pmm_script_iterator) self.notify(f"PMM script ({self.client_config_map.pmm_script_mode.pmm_script_file_path}) started.") self.strategy_task: asyncio.Task = safe_ensure_future(self._run_clock(), loop=self.ev_loop) self.notify(f"\n'{self.strategy_name}' strategy started.\n" diff --git a/hummingbot/client/config/client_config_map.py b/hummingbot/client/config/client_config_map.py index e5789b21fa..c7a735615a 100644 --- a/hummingbot/client/config/client_config_map.py +++ b/hummingbot/client/config/client_config_map.py @@ -401,8 +401,8 @@ def get_iterator( pmm_script_iterator = PMMScriptIterator( pmm_script_file, markets, - self.strategy, - queue_check_interval=0.1 + strategy, + queue_check_interval=0.1, ) return pmm_script_iterator diff --git a/hummingbot/core/utils/kill_switch.py b/hummingbot/core/utils/kill_switch.py index 6a1b720021..27fb1d0024 100644 --- a/hummingbot/core/utils/kill_switch.py +++ b/hummingbot/core/utils/kill_switch.py @@ -41,18 +41,17 @@ def __init__(self, async def check_profitability_loop(self): while True: try: - if self._kill_switch_enabled: - self._profitability: Decimal = await self._hummingbot_application.calculate_profitability() - - # Stop the bot if losing too much money, or if gained a certain amount of profit - if (self._profitability <= self._kill_switch_rate < Decimal("0.0")) or \ - (self._profitability >= self._kill_switch_rate > Decimal("0.0")): - self.logger().info("Kill switch threshold reached. Stopping the bot...") - self._hummingbot_application.notify(f"\n[Kill switch triggered]\n" - f"Current profitability " - f"is {self._profitability}. Stopping the bot...") - self._hummingbot_application.stop() - break + self._profitability: Decimal = await self._hummingbot_application.calculate_profitability() + + # Stop the bot if losing too much money, or if gained a certain amount of profit + if (self._profitability <= self._kill_switch_rate < Decimal("0.0")) or \ + (self._profitability >= self._kill_switch_rate > Decimal("0.0")): + self.logger().info("Kill switch threshold reached. Stopping the bot...") + self._hummingbot_application.notify(f"\n[Kill switch triggered]\n" + f"Current profitability " + f"is {self._profitability}. Stopping the bot...") + self._hummingbot_application.stop() + break except asyncio.CancelledError: raise From e6f750060b2d3a92c85bdb1012e1b6858ae46538 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 22 Jun 2022 14:50:57 +0300 Subject: [PATCH 144/152] (fix) Test case fix --- .../hummingbot/connector/exchange/kucoin/test_kucoin_exchange.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/hummingbot/connector/exchange/kucoin/test_kucoin_exchange.py b/test/hummingbot/connector/exchange/kucoin/test_kucoin_exchange.py index dc67b8e3d5..27455e8851 100644 --- a/test/hummingbot/connector/exchange/kucoin/test_kucoin_exchange.py +++ b/test/hummingbot/connector/exchange/kucoin/test_kucoin_exchange.py @@ -372,6 +372,7 @@ def test_get_fee_returns_fee_from_exchange_if_available_and_default_if_not(self, @aioresponses() def test_fee_request_for_multiple_pairs(self, mocked_api): self.exchange = KucoinExchange( + self.client_config_map, self.api_key, self.api_passphrase, self.api_secret_key, From 892e32f71315161001412e6d2d446bd8c555d8c4 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 22 Jun 2022 15:44:02 +0300 Subject: [PATCH 145/152] (fix) Test case fix --- .../connector/exchange/bitmart/test_bitmar_exchange.py | 4 ++++ .../bitmart/test_bitmart_api_order_book_data_source.py | 4 ++++ .../bitmart/test_bitmart_api_user_stream_data_source.py | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/test/hummingbot/connector/exchange/bitmart/test_bitmar_exchange.py b/test/hummingbot/connector/exchange/bitmart/test_bitmar_exchange.py index 205be1e13c..0b219310a7 100644 --- a/test/hummingbot/connector/exchange/bitmart/test_bitmar_exchange.py +++ b/test/hummingbot/connector/exchange/bitmart/test_bitmar_exchange.py @@ -7,6 +7,8 @@ from aioresponses import aioresponses from aioresponses.core import RequestCall +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.bitmart import bitmart_constants as CONSTANTS, bitmart_web_utils as web_utils from hummingbot.connector.exchange.bitmart.bitmart_exchange import BitmartExchange from hummingbot.connector.test_support.exchange_connector_test import AbstractExchangeConnectorTests @@ -322,7 +324,9 @@ def exchange_symbol_for_tokens(self, base_token: str, quote_token: str) -> str: return base_token + "_" + quote_token def create_exchange_instance(self): + client_config_map = ClientConfigAdapter(ClientConfigMap()) return BitmartExchange( + client_config_map=client_config_map, bitmart_api_key="testAPIKey", bitmart_secret_key="testSecret", bitmart_memo="testMemo", diff --git a/test/hummingbot/connector/exchange/bitmart/test_bitmart_api_order_book_data_source.py b/test/hummingbot/connector/exchange/bitmart/test_bitmart_api_order_book_data_source.py index 48eeb06a24..216840dba4 100644 --- a/test/hummingbot/connector/exchange/bitmart/test_bitmart_api_order_book_data_source.py +++ b/test/hummingbot/connector/exchange/bitmart/test_bitmart_api_order_book_data_source.py @@ -10,6 +10,8 @@ from bidict import bidict import hummingbot.connector.exchange.bitmart.bitmart_constants as CONSTANTS +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.bitmart import bitmart_utils from hummingbot.connector.exchange.bitmart.bitmart_api_order_book_data_source import BitmartAPIOrderBookDataSource from hummingbot.connector.exchange.bitmart.bitmart_exchange import BitmartExchange @@ -42,8 +44,10 @@ def setUp(self) -> None: self.log_records = [] self.listening_task = None self.mocking_assistant = NetworkMockingAssistant() + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) self.connector = BitmartExchange( + client_config_map=self.client_config_map, bitmart_api_key="", bitmart_secret_key="", bitmart_memo="", diff --git a/test/hummingbot/connector/exchange/bitmart/test_bitmart_api_user_stream_data_source.py b/test/hummingbot/connector/exchange/bitmart/test_bitmart_api_user_stream_data_source.py index 4a3d77f78d..9c92c64d0a 100644 --- a/test/hummingbot/connector/exchange/bitmart/test_bitmart_api_user_stream_data_source.py +++ b/test/hummingbot/connector/exchange/bitmart/test_bitmart_api_user_stream_data_source.py @@ -8,6 +8,8 @@ from bidict import bidict import hummingbot.connector.exchange.bitmart.bitmart_constants as CONSTANTS +from hummingbot.client.config.client_config_map import ClientConfigMap +from hummingbot.client.config.config_helpers import ClientConfigAdapter from hummingbot.connector.exchange.bitmart import bitmart_utils from hummingbot.connector.exchange.bitmart.bitmart_api_user_stream_data_source import BitmartAPIUserStreamDataSource from hummingbot.connector.exchange.bitmart.bitmart_auth import BitmartAuth @@ -33,6 +35,7 @@ def setUp(self) -> None: self.log_records = [] self.listening_task: Optional[asyncio.Task] = None self.mocking_assistant = NetworkMockingAssistant() + self.client_config_map = ClientConfigAdapter(ClientConfigMap()) self.time_synchronizer = MagicMock() self.time_synchronizer.time.return_value = 1640001112.223 @@ -44,6 +47,7 @@ def setUp(self) -> None: time_provider=self.time_synchronizer) self.connector = BitmartExchange( + client_config_map=self.client_config_map, bitmart_api_key="test_api_key", bitmart_secret_key="test_secret_key", bitmart_memo="test_memo", From 820d08050f2c8ac1c33bbf5491210e711ccad3a7 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Thu, 23 Jun 2022 09:30:45 +0300 Subject: [PATCH 146/152] (fix) Adds the docker conf copy step to the arm dockerfile --- Dockerfile.arm | 3 ++- conf/.dockerignore | 12 ++---------- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/Dockerfile.arm b/Dockerfile.arm index baa881706e..bd6c12871d 100644 --- a/Dockerfile.arm +++ b/Dockerfile.arm @@ -112,9 +112,10 @@ VOLUME /conf /logs /data /pmm_scripts /scripts \ # Pre-populate pmm_scripts/ volume with default pmm_scripts COPY --chown=hummingbot:hummingbot pmm_scripts/ pmm_scripts/ - # Pre-populate scripts/ volume with default scripts COPY --chown=hummingbot:hummingbot scripts/ scripts/ +# Copy the conf folder structure +COPY --chown=hummingbot:hummingbot conf/ conf/ # Install packages required in runtime RUN apt-get update && \ diff --git a/conf/.dockerignore b/conf/.dockerignore index 1ac225396f..48c69e5063 100644 --- a/conf/.dockerignore +++ b/conf/.dockerignore @@ -1,10 +1,2 @@ -/config_local.py -/quote_api_secret.py -/web3_wallet_secret.py -/binance_secret.py -/api_secrets.py -/*secret* -*encrypted* -*key* -*.yml -/gateway_connections.json +.password_verification +*.yml \ No newline at end of file From ae62b8c2545c348c8e4f17afec7b9883d9e52603 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Mon, 27 Jun 2022 17:28:03 +0300 Subject: [PATCH 147/152] (fix) If there are no config keys, don't traverse in `settings` --- hummingbot/client/settings.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/hummingbot/client/settings.py b/hummingbot/client/settings.py index 79fdfebab8..36a47528af 100644 --- a/hummingbot/client/settings.py +++ b/hummingbot/client/settings.py @@ -228,9 +228,10 @@ def non_trading_connector_instance_with_default_configuration( trading_pairs = trading_pairs or [] connector_class = getattr(importlib.import_module(self.module_path()), self.class_name()) + kwargs = {} if isinstance(self.config_keys, Dict): kwargs = {key: (config.value or "") for key, config in self.config_keys.items()} # legacy - else: + elif self.config_keys is not None: kwargs = { traverse_item.attr: traverse_item.value or "" for traverse_item From 95f935663c5d605674f5c4160b247fba9edefc8a Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Tue, 28 Jun 2022 13:05:49 +0300 Subject: [PATCH 148/152] (cleanup) Adds friendlier outputs - Improves the printout for one of the migration messages. - New configs now add the default as a suggestion on the prompt. --- hummingbot/client/command/create_command.py | 2 ++ hummingbot/client/ui/__init__.py | 15 +++++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/hummingbot/client/command/create_command.py b/hummingbot/client/command/create_command.py index b5a57d5f3b..494a77bceb 100644 --- a/hummingbot/client/command/create_command.py +++ b/hummingbot/client/command/create_command.py @@ -159,6 +159,8 @@ async def prompt_a_config( if input_value is None: prompt = await model.get_client_prompt(config) if prompt is not None: + default = model.get_default(config) or "" + self.app.set_text(default) prompt = f"{prompt} >>> " client_data = model.get_client_data(config) input_value = await self.app.prompt(prompt=prompt, is_password=client_data.is_secure) diff --git a/hummingbot/client/ui/__init__.py b/hummingbot/client/ui/__init__.py index 1ca0ad341b..d30c8c5c5d 100644 --- a/hummingbot/client/ui/__init__.py +++ b/hummingbot/client/ui/__init__.py @@ -28,12 +28,15 @@ def login_prompt(secrets_manager_cls: Type[BaseSecretsManager], style: Style): show_welcome(style) password = input_dialog( title="Set Password", - text="Create a password to protect your sensitive data. " - "This password is not shared with us nor with anyone else, so please store it securely." - "\n\nIf you have used hummingbot before and already have secure configs stored," - " input your previous password in this prompt, then run the scripts/conf_migration_script.py script" - " to migrate your existing secure configs to the new management system." - "\n\nEnter your new password:", + text=""" + Create a password to protect your sensitive data. + This password is not shared with us nor with anyone else, so please store it securely. + + If you have used hummingbot before and already have secure configs stored, + input your previous password in this prompt. The next step will automatically + migrate your existing configs. + + Enter your new password:""", password=True, style=style).run() if password is None: From c048e5fce28b2df31c68037c88844ea9b6f991fa Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Tue, 28 Jun 2022 13:27:49 +0300 Subject: [PATCH 149/152] (fix) Bug fix. --- hummingbot/client/command/create_command.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hummingbot/client/command/create_command.py b/hummingbot/client/command/create_command.py index 494a77bceb..d981ac087a 100644 --- a/hummingbot/client/command/create_command.py +++ b/hummingbot/client/command/create_command.py @@ -160,7 +160,7 @@ async def prompt_a_config( prompt = await model.get_client_prompt(config) if prompt is not None: default = model.get_default(config) or "" - self.app.set_text(default) + self.app.set_text(str(default)) prompt = f"{prompt} >>> " client_data = model.get_client_data(config) input_value = await self.app.prompt(prompt=prompt, is_password=client_data.is_secure) From 863b7f4f4543ccf429b3921df4d7c13ec3eaf01f Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Tue, 28 Jun 2022 13:28:43 +0300 Subject: [PATCH 150/152] (cleanup) Adds precision to the treatment of default values --- hummingbot/client/command/create_command.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/hummingbot/client/command/create_command.py b/hummingbot/client/command/create_command.py index d981ac087a..76a1021d52 100644 --- a/hummingbot/client/command/create_command.py +++ b/hummingbot/client/command/create_command.py @@ -159,8 +159,9 @@ async def prompt_a_config( if input_value is None: prompt = await model.get_client_prompt(config) if prompt is not None: - default = model.get_default(config) or "" - self.app.set_text(str(default)) + default = model.get_default(config) + default = str(default) if default is not None else "" + self.app.set_text(default) prompt = f"{prompt} >>> " client_data = model.get_client_data(config) input_value = await self.app.prompt(prompt=prompt, is_password=client_data.is_secure) From 580ef26c3954d4a41ac02c525d3ab6225e11e97c Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Tue, 28 Jun 2022 17:01:10 +0300 Subject: [PATCH 151/152] (fix) Fixes the config default handling --- hummingbot/client/command/config_command.py | 2 +- hummingbot/client/command/create_command.py | 8 +++++--- hummingbot/client/config/config_helpers.py | 5 ++++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/hummingbot/client/command/config_command.py b/hummingbot/client/command/config_command.py index 37f2359cb4..b9564f38ca 100644 --- a/hummingbot/client/command/config_command.py +++ b/hummingbot/client/command/config_command.py @@ -237,7 +237,7 @@ async def _config_single_key(self, # type: HummingbotApplication elif key == "inventory_price": await self.inventory_price_prompt(config_map, input_value) else: - await self.prompt_a_config(config_map, key, input_value) + await self.prompt_a_config(config_map, key, input_value, assign_default=False) if self.app.to_stop_config: self.app.to_stop_config = False return diff --git a/hummingbot/client/command/create_command.py b/hummingbot/client/command/create_command.py index 76a1021d52..c91e62b004 100644 --- a/hummingbot/client/command/create_command.py +++ b/hummingbot/client/command/create_command.py @@ -150,6 +150,7 @@ async def prompt_a_config( model: ClientConfigAdapter, config: str, input_value=None, + assign_default=True, ): config_path = config.split(".") while len(config_path) != 1: @@ -159,9 +160,10 @@ async def prompt_a_config( if input_value is None: prompt = await model.get_client_prompt(config) if prompt is not None: - default = model.get_default(config) - default = str(default) if default is not None else "" - self.app.set_text(default) + if assign_default: + default = model.get_default(config) + default = str(default) if default is not None else "" + self.app.set_text(default) prompt = f"{prompt} >>> " client_data = model.get_client_data(config) input_value = await self.app.prompt(prompt=prompt, is_password=client_data.is_secure) diff --git a/hummingbot/client/config/config_helpers.py b/hummingbot/client/config/config_helpers.py index d23016b5d0..c87222ba83 100644 --- a/hummingbot/client/config/config_helpers.py +++ b/hummingbot/client/config/config_helpers.py @@ -208,7 +208,10 @@ def get_description(self, attr_name: str) -> str: return self._hb_config.__fields__[attr_name].field_info.description def get_default(self, attr_name: str) -> Any: - return self._hb_config.__fields__[attr_name].field_info.default + default = self._hb_config.__fields__[attr_name].field_info.default + if isinstance(default, type(Ellipsis)): + default = None + return default def get_type(self, attr_name: str) -> Type: return self._hb_config.__fields__[attr_name].type_ From 38fbbd89db9c513e96e31fbc2fcba5ff409221a0 Mon Sep 17 00:00:00 2001 From: Petio Petrov Date: Wed, 29 Jun 2022 07:24:47 +0300 Subject: [PATCH 152/152] (fix) Fixes issues with the connector settings class --- hummingbot/client/settings.py | 8 ++++-- test/hummingbot/client/test_settings.py | 38 +++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) create mode 100644 test/hummingbot/client/test_settings.py diff --git a/hummingbot/client/settings.py b/hummingbot/client/settings.py index 36a47528af..7a9a69c78d 100644 --- a/hummingbot/client/settings.py +++ b/hummingbot/client/settings.py @@ -6,6 +6,8 @@ from os.path import exists, join, realpath from typing import TYPE_CHECKING, Any, Dict, List, NamedTuple, Optional, Set, Union, cast +from pydantic import SecretStr + from hummingbot import get_strategy_list, root_path from hummingbot.core.data_type.trade_fee import TradeFeeSchema @@ -233,7 +235,9 @@ def non_trading_connector_instance_with_default_configuration( kwargs = {key: (config.value or "") for key, config in self.config_keys.items()} # legacy elif self.config_keys is not None: kwargs = { - traverse_item.attr: traverse_item.value or "" + traverse_item.attr: traverse_item.value.get_secret_value() + if isinstance(traverse_item.value, SecretStr) + else traverse_item.value or "" for traverse_item in ClientConfigAdapter(self.config_keys).traverse() if traverse_item.attr != "connector" @@ -361,7 +365,7 @@ def initialize_paper_trade_settings(cls, paper_trade_exchanges: List[str]): @classmethod def get_all_connectors(cls) -> List[str]: """Avoids circular import problems introduced by `create_connector_settings`.""" - connector_names = PAPER_TRADE_EXCHANGES + connector_names = PAPER_TRADE_EXCHANGES.copy() type_dirs: List[DirEntry] = [ cast(DirEntry, f) for f in scandir(f"{root_path() / 'hummingbot' / 'connector'}") diff --git a/test/hummingbot/client/test_settings.py b/test/hummingbot/client/test_settings.py new file mode 100644 index 0000000000..df07815feb --- /dev/null +++ b/test/hummingbot/client/test_settings.py @@ -0,0 +1,38 @@ +import unittest + +from pydantic import SecretStr + +from hummingbot.client.settings import ConnectorSetting, ConnectorType +from hummingbot.connector.exchange.binance.binance_utils import BinanceConfigMap +from hummingbot.core.data_type.trade_fee import TradeFeeSchema + + +class SettingsTest(unittest.TestCase): + def test_non_trading_connector_instance_with_default_configuration_secrets_revealed( + self + ): + api_key = "someKey" + api_secret = "someSecret" + config_map = BinanceConfigMap( + binance_api_key=api_key, + binance_api_secret=api_secret, + ) + conn_settings = ConnectorSetting( + name="binance", + type=ConnectorType.Exchange, + example_pair="BTC-USDT", + centralised=True, + use_ethereum_wallet=False, + trade_fee_schema=TradeFeeSchema(), + config_keys=config_map, + is_sub_domain=False, + parent_name=None, + domain_parameter=None, + use_eth_gas_lookup=False, + ) + connector = conn_settings.non_trading_connector_instance_with_default_configuration() + + self.assertNotIsInstance(connector.api_key, SecretStr) + self.assertEqual(api_key, connector.api_key) + self.assertNotIsInstance(connector.secret_key, SecretStr) + self.assertEqual(api_secret, connector.secret_key)