Merge pull request #5428 from CoinAlpha/epic/config_management_refact…

…oring

Epic/config management refactoring

.coveragerc

@@ -4,6 +4,7 @@ omit =
         hummingbot/core/gateway/*
         hummingbot/core/management/*
         hummingbot/client/config/config_helpers.py
+        hummingbot/client/config/conf_migration.py
         hummingbot/client/config/security.py
         hummingbot/client/hummingbot_application.py
         hummingbot/client/command/*
@@ -37,6 +38,7 @@ omit =
         hummingbot/core/utils/wallet_setup.py
         hummingbot/connector/mock*
         hummingbot/strategy/aroon_oscillator*
+        hummingbot/strategy/*/start.py
         hummingbot/strategy/dev*
         hummingbot/strategy/hedge*
         hummingbot/user/user_balances.py        

.dockerignore

@@ -1,4 +1,3 @@
-/conf
 /data
 /logs
 /build

.gitignore

@@ -111,3 +111,6 @@ coverage.xml
 
 # Debug console
 .debug_console_ssh_host_key
+
+# password file
+.password_verification

Dockerfile

@@ -114,6 +114,8 @@ VOLUME /conf /logs /data /pmm_scripts /scripts \
 COPY --chown=hummingbot:hummingbot pmm_scripts/ pmm_scripts/
 # Pre-populate scripts/ volume with default scripts
 COPY --chown=hummingbot:hummingbot scripts/ scripts/
+# Copy the conf folder structure
+COPY --chown=hummingbot:hummingbot conf/ conf/
 
 # Install packages required in runtime
 RUN apt-get update && \

Dockerfile.arm

@@ -112,9 +112,10 @@ VOLUME /conf /logs /data /pmm_scripts /scripts \
 
 # Pre-populate pmm_scripts/ volume with default pmm_scripts
 COPY --chown=hummingbot:hummingbot pmm_scripts/ pmm_scripts/
-
 # Pre-populate scripts/ volume with default scripts
 COPY --chown=hummingbot:hummingbot scripts/ scripts/
+# Copy the conf folder structure
+COPY --chown=hummingbot:hummingbot conf/ conf/
 
 # Install packages required in runtime
 RUN apt-get update && \

bin/conf_migration_script.py

@@ -0,0 +1,13 @@
+import argparse
+
+import path_util  # noqa: F401
+
+from hummingbot.client.config.conf_migration import migrate_configs
+from hummingbot.client.config.config_crypt import ETHKeyFileSecretManger
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Migrate the HummingBot confs")
+    parser.add_argument("password", type=str, help="Required to migrate all encrypted configs.")
+    args = parser.parse_args()
+    secrets_manager_ = ETHKeyFileSecretManger(args.password)
+    migrate_configs(secrets_manager_)

bin/hummingbot.py

@@ -8,8 +8,13 @@
 
 from bin.docker_connection import fork_and_start
 from hummingbot import chdir_to_data_directory, init_logging
-from hummingbot.client.config.config_helpers import create_yml_files, read_system_configs_from_yml, write_config_to_yml
-from hummingbot.client.config.global_config_map import global_config_map
+from hummingbot.client.config.config_crypt import ETHKeyFileSecretManger
+from hummingbot.client.config.config_helpers import (
+    ClientConfigAdapter,
+    create_yml_files_legacy,
+    load_client_config_map_from_file,
+    write_config_to_yml,
+)
 from hummingbot.client.hummingbot_application import HummingbotApplication
 from hummingbot.client.settings import AllConnectorSettings
 from hummingbot.client.ui import login_prompt
@@ -35,31 +40,28 @@ def hummingbot_app(self) -> HummingbotApplication:
 
     async def ui_start_handler(self):
         hb: HummingbotApplication = self.hummingbot_app
+        if hb.strategy_config_map is not None:
+            write_config_to_yml(hb.strategy_config_map, hb.strategy_file_name, hb.client_config_map)
+            hb.start(self._hb_ref.client_config_map.log_level)
 
-        if hb.strategy_file_name is not None and hb.strategy_name is not None:
-            await write_config_to_yml(hb.strategy_name, hb.strategy_file_name)
-            hb.start(global_config_map.get("log_level").value)
 
-
-async def main_async():
-    await create_yml_files()
+async def main_async(client_config_map: ClientConfigAdapter):
+    await create_yml_files_legacy()
 
     # This init_logging() call is important, to skip over the missing config warnings.
-    init_logging("hummingbot_logs.yml")
-
-    await read_system_configs_from_yml()
+    init_logging("hummingbot_logs.yml", client_config_map)
 
-    AllConnectorSettings.initialize_paper_trade_settings(global_config_map.get("paper_trade_exchanges").value)
+    AllConnectorSettings.initialize_paper_trade_settings(client_config_map.paper_trade.paper_trade_exchanges)
 
-    hb = HummingbotApplication.main_application()
+    hb = HummingbotApplication.main_application(client_config_map)
 
     # The listener needs to have a named variable for keeping reference, since the event listener system
     # uses weak references to remove unneeded listeners.
     start_listener: UIStartListener = UIStartListener(hb)
     hb.app.add_listener(HummingbotUIEvent.Start, start_listener)
 
-    tasks: List[Coroutine] = [hb.run(), start_existing_gateway_container()]
-    if global_config_map.get("debug_console").value:
+    tasks: List[Coroutine] = [hb.run(), start_existing_gateway_container(client_config_map)]
+    if client_config_map.debug_console:
         if not hasattr(__builtins__, "help"):
             import _sitebuiltins
             __builtins__.help = _sitebuiltins._Helper()
@@ -72,10 +74,11 @@ async def main_async():
 
 def main():
     chdir_to_data_directory()
+    secrets_manager_cls = ETHKeyFileSecretManger
     ev_loop: asyncio.AbstractEventLoop = asyncio.get_event_loop()
-    ev_loop.run_until_complete(read_system_configs_from_yml())
-    if login_prompt(style=load_style()):
-        ev_loop.run_until_complete(main_async())
+    client_config_map = load_client_config_map_from_file()
+    if login_prompt(secrets_manager_cls, style=load_style(client_config_map)):
+        ev_loop.run_until_complete(main_async(client_config_map))
 
 
 if __name__ == "__main__":

bin/hummingbot_quickstart.py

@@ -15,16 +15,18 @@
 from bin.docker_connection import fork_and_start
 from bin.hummingbot import UIStartListener, detect_available_port
 from hummingbot import init_logging
+from hummingbot.client.config.config_crypt import BaseSecretsManager, ETHKeyFileSecretManger
+from hummingbot.client.config.config_data_types import BaseStrategyConfigMap
 from hummingbot.client.config.config_helpers import (
     all_configs_complete,
-    create_yml_files,
+    create_yml_files_legacy,
+    load_client_config_map_from_file,
+    load_strategy_config_map_from_file,
     read_system_configs_from_yml,
-    update_strategy_config_map_from_file,
 )
-from hummingbot.client.config.global_config_map import global_config_map
 from hummingbot.client.config.security import Security
 from hummingbot.client.hummingbot_application import HummingbotApplication
-from hummingbot.client.settings import CONF_FILE_PATH, AllConnectorSettings
+from hummingbot.client.settings import STRATEGIES_CONF_DIR_PATH, AllConnectorSettings
 from hummingbot.client.ui import login_prompt
 from hummingbot.client.ui.style import load_style
 from hummingbot.core.event.events import HummingbotUIEvent
@@ -71,46 +73,51 @@ def autofix_permissions(user_group_spec: str):
     os.setuid(uid)
 
 
-async def quick_start(args: argparse.Namespace):
+async def quick_start(args: argparse.Namespace, secrets_manager: BaseSecretsManager):
     config_file_name = args.config_file_name
-    password = args.config_password
+    client_config = load_client_config_map_from_file()
 
     if args.auto_set_permissions is not None:
         autofix_permissions(args.auto_set_permissions)
 
-    if password is not None and not Security.login(password):
+    if not Security.login(secrets_manager):
         logging.getLogger().error("Invalid password.")
         return
 
     await Security.wait_til_decryption_done()
-    await create_yml_files()
-    init_logging("hummingbot_logs.yml")
+    await create_yml_files_legacy()
+    init_logging("hummingbot_logs.yml", client_config)
     await read_system_configs_from_yml()
 
-    AllConnectorSettings.initialize_paper_trade_settings(global_config_map.get("paper_trade_exchanges").value)
+    AllConnectorSettings.initialize_paper_trade_settings(client_config.paper_trade.paper_trade_exchanges)
 
     hb = HummingbotApplication.main_application()
     # Todo: validate strategy and config_file_name before assinging
 
+    strategy_config = None
     if config_file_name is not None:
         hb.strategy_file_name = config_file_name
-        hb.strategy_name = await update_strategy_config_map_from_file(os.path.join(CONF_FILE_PATH, config_file_name))
-
-    # To ensure quickstart runs with the default value of False for kill_switch_enabled if not present
-    if not global_config_map.get("kill_switch_enabled"):
-        global_config_map.get("kill_switch_enabled").value = False
-
-    if hb.strategy_name and hb.strategy_file_name:
-        if not all_configs_complete(hb.strategy_name):
+        strategy_config = await load_strategy_config_map_from_file(
+            STRATEGIES_CONF_DIR_PATH / config_file_name
+        )
+        hb.strategy_name = (
+            strategy_config.strategy
+            if isinstance(strategy_config, BaseStrategyConfigMap)
+            else strategy_config.get("strategy").value
+        )
+        hb.strategy_config_map = strategy_config
+
+    if strategy_config is not None:
+        if not all_configs_complete(strategy_config, hb.client_config_map):
             hb.status()
 
     # The listener needs to have a named variable for keeping reference, since the event listener system
     # uses weak references to remove unneeded listeners.
     start_listener: UIStartListener = UIStartListener(hb)
     hb.app.add_listener(HummingbotUIEvent.Start, start_listener)
 
-    tasks: List[Coroutine] = [hb.run(), start_existing_gateway_container()]
-    if global_config_map.get("debug_console").value:
+    tasks: List[Coroutine] = [hb.run(), start_existing_gateway_container(client_config)]
+    if client_config.debug_console:
         management_port: int = detect_available_port(8211)
         tasks.append(start_management_console(locals(), host="localhost", port=management_port))
 
@@ -129,12 +136,16 @@ def main():
         args.config_password = os.environ["CONFIG_PASSWORD"]
 
     # If no password is given from the command line, prompt for one.
-    asyncio.get_event_loop().run_until_complete(read_system_configs_from_yml())
+    secrets_manager_cls = ETHKeyFileSecretManger
+    client_config_map = load_client_config_map_from_file()
     if args.config_password is None:
-        if not login_prompt(style=load_style()):
+        secrets_manager = login_prompt(secrets_manager_cls, style=load_style(client_config_map))
+        if not secrets_manager:
             return
+    else:
+        secrets_manager = secrets_manager_cls(args.config_password)
 
-    asyncio.get_event_loop().run_until_complete(quick_start(args))
+    asyncio.get_event_loop().run_until_complete(quick_start(args, secrets_manager))
 
 
 if __name__ == "__main__":

conf/.dockerignore

@@ -0,0 +1,2 @@
+.password_verification
+*.yml

conf/__init__.py

@@ -3,8 +3,6 @@
 import logging as _logging
 import os
 
-from hummingbot.client.config.global_config_map import connector_keys
-
 _logger = _logging.getLogger(__name__)
 
 master_host = "***REMOVED***"
@@ -39,10 +37,6 @@
 # whether to enable api mocking in unit test cases
 mock_api_enabled = os.getenv("MOCK_API_ENABLED")
 
-# ALL TEST KEYS
-for key in connector_keys().keys():
-    locals()[key] = os.getenv(key.upper())
-
 """
 # AscendEX Tests
 ascend_ex_api_key = os.getenv("ASCEND_EX_KEY")

conf/connectors/.dockerignore

@@ -0,0 +1 @@
+*.yml

conf/connectors/.gitignore

@@ -0,0 +1 @@
+*.yml

conf/strategies/.dockerignore

@@ -0,0 +1 @@
+*.yml

conf/strategies/.gitignore

@@ -0,0 +1 @@
+*.yml

hummingbot/__init__.py

@@ -3,10 +3,14 @@
 import sys
 from concurrent.futures import ThreadPoolExecutor
 from os import listdir, path
-from typing import List, Optional
+from pathlib import Path
+from typing import TYPE_CHECKING, List, Optional
 
 from hummingbot.logger.struct_logger import StructLogger, StructLogRecord
 
+if TYPE_CHECKING:
+    from hummingbot.client.config.config_helpers import ClientConfigAdapter as _ClientConfigAdapter
+
 STRUCT_LOGGER_SET = False
 DEV_STRATEGY_PREFIX = "dev"
 _prefix_path = None
@@ -20,9 +24,9 @@
 _cert_path = None
 
 
-def root_path() -> str:
-    from os.path import realpath, join
-    return realpath(join(__file__, "../../"))
+def root_path() -> Path:
+    from os.path import join, realpath
+    return Path(realpath(join(__file__, "../../")))
 
 
 def get_executor() -> ThreadPoolExecutor:
@@ -35,26 +39,20 @@ def get_executor() -> ThreadPoolExecutor:
 def prefix_path() -> str:
     global _prefix_path
     if _prefix_path is None:
-        from os.path import (
-            realpath,
-            join
-        )
+        from os.path import join, realpath
         _prefix_path = realpath(join(__file__, "../../"))
     return _prefix_path
 
 
-def set_prefix_path(path: str):
+def set_prefix_path(p: str):
     global _prefix_path
-    _prefix_path = path
+    _prefix_path = p
 
 
 def data_path() -> str:
     global _data_path
     if _data_path is None:
-        from os.path import (
-            realpath,
-            join
-        )
+        from os.path import join, realpath
         _data_path = realpath(join(prefix_path(), "data"))
 
     import os
@@ -97,8 +95,9 @@ def chdir_to_data_directory():
         # Do nothing.
         return
 
-    import appdirs
     import os
+
+    import appdirs
     app_data_dir: str = appdirs.user_data_dir("Hummingbot", "hummingbot.io")
     os.makedirs(os.path.join(app_data_dir, "logs"), 0o711, exist_ok=True)
     os.makedirs(os.path.join(app_data_dir, "conf"), 0o711, exist_ok=True)
@@ -110,20 +109,18 @@ def chdir_to_data_directory():
 
 
 def init_logging(conf_filename: str,
+                 client_config_map: "_ClientConfigAdapter",
                  override_log_level: Optional[str] = None,
                  strategy_file_path: str = "hummingbot"):
     import io
     import logging.config
     from os.path import join
-    import pandas as pd
     from typing import Dict
+
+    import pandas as pd
     from ruamel.yaml import YAML
 
-    from hummingbot.client.config.global_config_map import global_config_map
-    from hummingbot.logger.struct_logger import (
-        StructLogRecord,
-        StructLogger
-    )
+    from hummingbot.logger.struct_logger import StructLogger, StructLogRecord
     global STRUCT_LOGGER_SET
     if not STRUCT_LOGGER_SET:
         logging.setLogRecordFactory(StructLogRecord)
@@ -144,8 +141,7 @@ def init_logging(conf_filename: str,
         config_dict: Dict = yaml_parser.load(io_stream)
         if override_log_level is not None and "loggers" in config_dict:
             for logger in config_dict["loggers"]:
-                if global_config_map["logger_override_whitelist"].value and \
-                        logger in global_config_map["logger_override_whitelist"].value:
+                if logger in client_config_map.logger_override_whitelist:
                     config_dict["loggers"][logger]["level"] = override_log_level
         logging.config.dictConfig(config_dict)