From 9f696ab85ef3691585ead620971c023a1d957886 Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Thu, 22 Sep 2022 14:26:20 +0800 Subject: [PATCH 01/25] local & cloud cash balance, cloud unit test --- lean/commands/cloud/live/deploy.py | 45 ++++- lean/commands/live/deploy.py | 39 +++- lean/components/api/live_client.py | 8 +- lean/models/brokerages/cloud/__init__.py | 9 +- lean/models/brokerages/local/__init__.py | 9 +- .../cloud/live/test_cloud_live_commands.py | 169 +++++++++++++++++- 6 files changed, 271 insertions(+), 8 deletions(-) diff --git a/lean/commands/cloud/live/deploy.py b/lean/commands/cloud/live/deploy.py index 7a2b0f9e..90fcf098 100644 --- a/lean/commands/cloud/live/deploy.py +++ b/lean/commands/cloud/live/deploy.py @@ -24,7 +24,7 @@ from lean.models.brokerages.cloud.cloud_brokerage import CloudBrokerage from lean.models.configuration import Configuration, InfoConfiguration, InternalInputUserInput, OrganzationIdConfiguration from lean.models.click_options import options_from_json -from lean.models.brokerages.cloud import all_cloud_brokerages +from lean.models.brokerages.cloud import all_cloud_brokerages, cloud_brokerages_with_editable_cash_balance from lean.commands.cloud.live.live import live def _log_notification_methods(methods: List[QCNotificationMethod]) -> None: @@ -104,6 +104,27 @@ def _configure_brokerage(logger: Logger) -> CloudBrokerage: brokerage_options = [Option(id=b, label=b.get_name()) for b in all_cloud_brokerages] return logger.prompt_list("Select a brokerage", brokerage_options).build(None,logger) + +def _configure_initial_cash_balance(logger: Logger) -> List[Dict[str, float]]: + """Interactively configures the intial cash balance. + + :param logger: the logger to use + :return: the list of dictionary containing intial currency and amount information + """ + cash_list = [] + continue_adding = True + + while continue_adding: + currency = click.prompt("Currency") + amount = click.prompt("Amount", type=float) + cash_list.append({"currency": currency, "amount": amount}) + logger.info(f"Cash balance: {cash_list}") + + if not click.confirm("Do you want to add other currency?", default=False): + continue_adding = False + + return cash_list + def _configure_live_node(logger: Logger, api_client: APIClient, cloud_project: QCProject) -> QCNode: """Interactively configures the live node to use. @@ -188,6 +209,9 @@ def _get_configs_for_options() -> Dict[Configuration, str]: help="A comma-separated list of 'url:HEADER_1=VALUE_1:HEADER_2=VALUE_2:etc' pairs configuring webhook-notifications") @click.option("--notify-sms", type=str, help="A comma-separated list of phone numbers configuring SMS-notifications") @click.option("--notify-telegram", type=str, help="A comma-separated list of 'user/group Id:token(optional)' pairs configuring telegram-notifications") +@click.option("--live-cash-balance", + type=str, + help=f"A comma-separated list of currency:amount pairs of initial cash balance") @click.option("--push", is_flag=True, default=False, @@ -206,6 +230,7 @@ def deploy(project: str, notify_webhooks: Optional[str], notify_sms: Optional[str], notify_telegram: Optional[str], + live_cash_balance: Optional[str], push: bool, open_browser: bool, **kwargs) -> None: @@ -289,6 +314,19 @@ def deploy(project: str, brokerage_settings = brokerage_instance.get_settings() price_data_handler = brokerage_instance.get_price_data_handler() + if brokerage_instance.get_name() in [broker.get_name() for broker in cloud_brokerages_with_editable_cash_balance]: + if live_cash_balance is not None and live_cash_balance != "": + cash_list = [] + for cash_pair in live_cash_balance.split(","): + currency, amount = cash_pair.split(":") + cash_list.append({"currency": currency, "amount": float(amount)}) + live_cash_balance = cash_list + elif click.confirm("Do you want to set initial cash balance?", default=False): + cash_list = _configure_initial_cash_balance() + live_cash_balance = cash_list + elif live_cash_balance is not None and live_cash_balance != "": + raise RuntimeError(f"Custom cash balance setting is not available for {brokerage_instance.get_name()}") + logger.info(f"Brokerage: {brokerage_instance.get_name()}") logger.info(f"Project id: {cloud_project.projectId}") logger.info(f"Environment: {brokerage_settings['environment'].title()}") @@ -300,6 +338,8 @@ def deploy(project: str, logger.info(f"Insight notifications: {'Yes' if notify_insights else 'No'}") if notify_order_events or notify_insights: _log_notification_methods(notify_methods) + if live_cash_balance: + logger.info(live_cash_balance) logger.info(f"Automatic algorithm restarting: {'Yes' if auto_restart else 'No'}") if brokerage is None: @@ -316,7 +356,8 @@ def deploy(project: str, cloud_project.leanVersionId, notify_order_events, notify_insights, - notify_methods) + notify_methods, + live_cash_balance) logger.info(f"Live url: {live_algorithm.get_url()}") diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index fdcb7c91..728b1c32 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -21,7 +21,8 @@ from lean.click import LeanCommand, PathParameter, ensure_options from lean.constants import DEFAULT_ENGINE_IMAGE from lean.container import container -from lean.models.brokerages.local import all_local_brokerages, local_brokerage_data_feeds, all_local_data_feeds +from lean.models.brokerages.local import all_local_brokerages, local_brokerage_data_feeds, all_local_data_feeds, \ + local_brokerages_with_editable_cash_balance from lean.models.errors import MoreInfoError from lean.models.lean_config_configurer import LeanConfigConfigurer from lean.models.logger import Option @@ -172,6 +173,24 @@ def _configure_lean_config_interactively(lean_config: Dict[str, Any], environmen setattr(data_feed, '_is_installed_and_build', True) data_feed.build(lean_config, logger).configure(lean_config, environment_name) + +def _configure_initial_cash_balance() -> List[Dict[str, float]]: + logger = container.logger() + + cash_list = [] + continue_adding = True + + while continue_adding: + currency = click.prompt("Currency") + amount = click.prompt("Amount", type=float) + cash_list.append({"currency": currency, "amount": amount}) + logger.info(f"Cash balance: {cash_list}") + + if not click.confirm("Do you want to add other currency?", default=False): + continue_adding = False + + return cash_list + _cached_organizations = None @@ -311,6 +330,9 @@ def _get_configs_for_options() -> List[Configuration]: @click.option("--python-venv", type=str, help=f"The path of the python virtual environment to be used") +@click.option("--live-cash-balance", + type=str, + help=f"A comma-separated list of currency:amount pairs of initial cash balance") @click.option("--update", is_flag=True, default=False, @@ -325,6 +347,7 @@ def deploy(project: Path, release: bool, image: Optional[str], python_venv: Optional[str], + live_cash_balance: Optional[str], update: bool, **kwargs) -> None: """Start live trading a project locally using Docker. @@ -426,5 +449,19 @@ def deploy(project: Path, if python_venv is not None and python_venv != "": lean_config["python-venv"] = f'{"/" if python_venv[0] != "/" else ""}{python_venv}' + brokerage_id = lean_config["environments"]["live-mode-brokerage"] + if brokerage_id in [broker._id for broker in local_brokerages_with_editable_cash_balance]: + if live_cash_balance is not None and live_cash_balance != "": + cash_list = [] + for cash_pair in live_cash_balance.split(","): + currency, amount = cash_pair.split(":") + cash_list.append({"currency": currency, "amount": float(amount)}) + lean_config["live-cash-balance"] = cash_list + elif click.confirm("Do you want to set initial cash balance?", default=False): + cash_list = _configure_initial_cash_balance() + lean_config["live-cash-balance"] = cash_list + elif live_cash_balance is not None and live_cash_balance != "": + raise RuntimeError(f"Custom cash balance setting is not available for {brokerage_id}") + lean_runner = container.lean_runner() lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach) diff --git a/lean/components/api/live_client.py b/lean/components/api/live_client.py index df6689fd..7ff993c4 100644 --- a/lean/components/api/live_client.py +++ b/lean/components/api/live_client.py @@ -62,7 +62,8 @@ def start(self, version_id: int, notify_order_events: bool, notify_insights: bool, - notify_methods: List[QCNotificationMethod]) -> QCMinimalLiveAlgorithm: + notify_methods: List[QCNotificationMethod], + live_cash_balance: Optional[List[Dict[str, float]]] = None) -> QCMinimalLiveAlgorithm: """Starts live trading for a project. :param project_id: the id of the project to start live trading for @@ -75,6 +76,7 @@ def start(self, :param notify_order_events: whether notifications should be sent on order events :param notify_insights: whether notifications should be sent on insights :param notify_methods: the places to send notifications to + :param notify_methods: the list of initial cash balance :return: the created live algorithm """ @@ -99,6 +101,10 @@ def start(self, "events": events, "targets": [{x: y for x, y in method.dict().items() if y} for method in notify_methods] } + + if live_cash_balance: + parameters["portfolio"] = {} + parameters["portfolio"]["cash"] = live_cash_balance data = self._api.post("live/create", parameters) return QCMinimalLiveAlgorithm(**data) diff --git a/lean/models/brokerages/cloud/__init__.py b/lean/models/brokerages/cloud/__init__.py index b1e641de..856f9984 100644 --- a/lean/models/brokerages/cloud/__init__.py +++ b/lean/models/brokerages/cloud/__init__.py @@ -16,10 +16,17 @@ from typing import List all_cloud_brokerages: List[CloudBrokerage] = [] +cloud_brokerages_with_editable_cash_balance: List[CloudBrokerage] = [] +cloud_brokerages_with_editable_holdings: List[CloudBrokerage] = [] for json_module in json_modules: if "cloud-brokerage" in json_module["type"]: - all_cloud_brokerages.append(CloudBrokerage(json_module)) + cloud_brokerage = CloudBrokerage(json_module) + all_cloud_brokerages.append(cloud_brokerage) + if json_module["live-cash-balance-state"]: + cloud_brokerages_with_editable_cash_balance.append(cloud_brokerage) + if json_module["live-holdings-state"]: + cloud_brokerages_with_editable_holdings.append(cloud_brokerage) [PaperTradingBrokerage] = [ cloud_brokerage for cloud_brokerage in all_cloud_brokerages if cloud_brokerage._id == "QuantConnectBrokerage"] diff --git a/lean/models/brokerages/local/__init__.py b/lean/models/brokerages/local/__init__.py index 3e7ac91c..f39e518f 100644 --- a/lean/models/brokerages/local/__init__.py +++ b/lean/models/brokerages/local/__init__.py @@ -19,13 +19,20 @@ from lean.models import json_modules all_local_brokerages: List[LocalBrokerage] = [] +local_brokerages_with_editable_cash_balance: List[LocalBrokerage] = [] +local_brokerages_with_editable_holdings: List[LocalBrokerage] = [] all_local_data_feeds: List[DataFeed] = [] local_brokerage_data_feeds: Dict[Type[LocalBrokerage], List[Type[DataFeed]]] = {} for json_module in json_modules: if "local-brokerage" in json_module["type"]: - all_local_brokerages.append(LocalBrokerage(json_module)) + local_brokerage = LocalBrokerage(json_module) + all_local_brokerages.append(local_brokerage) + if json_module["live-cash-balance-state"]: + local_brokerages_with_editable_cash_balance.append(local_brokerage) + if json_module["live-holdings-state"]: + local_brokerages_with_editable_holdings.append(local_brokerage) if "data-queue-handler" in json_module["type"]: all_local_data_feeds.append(DataFeed(json_module)) diff --git a/tests/commands/cloud/live/test_cloud_live_commands.py b/tests/commands/cloud/live/test_cloud_live_commands.py index 79f09a6a..2351abf5 100644 --- a/tests/commands/cloud/live/test_cloud_live_commands.py +++ b/tests/commands/cloud/live/test_cloud_live_commands.py @@ -75,7 +75,8 @@ def test_cloud_live_deploy() -> None: mock.ANY, False, False, - []) + [], + None) @pytest.mark.parametrize("notice_method,configs", [("emails", "customAddress:customSubject"), ("emails", "customAddress1:customSubject1,customAddress2:customSubject2"), @@ -151,4 +152,168 @@ def test_cloud_live_deploy_with_notifications(notice_method: str, configs: str) mock.ANY, True, True, - notification) + notification, + None) + +brokerage_required_options = { + "Paper Trading": {}, + "Interactive Brokers": { + "ib-user-name": "trader777", + "ib-account": "DU1234567", + "ib-password": "hunter2", + "ib-enable-delayed-streaming-data": "no", + "ib-organization": "abc", + }, + "Tradier": { + "tradier-account-id": "123", + "tradier-access-token": "456", + "tradier-environment": "paper" + }, + "OANDA": { + "oanda-account-id": "123", + "oanda-access-token": "456", + "oanda-environment": "Practice" + }, + "Bitfinex": { + "bitfinex-api-key": "123", + "bitfinex-api-secret": "456", + }, + "Coinbase Pro": { + "gdax-api-key": "123", + "gdax-api-secret": "456", + "gdax-passphrase": "789", + "gdax-use-sandbox": "paper" + }, + "Binance": { + "binance-exchange-name": "binance", + "binance-api-key": "123", + "binance-api-secret": "456", + "binance-use-testnet": "paper", + "binance-organization": "abc", + }, + "Zerodha": { + "zerodha-api-key": "123", + "zerodha-access-token": "456", + "zerodha-product-type": "mis", + "zerodha-trading-segment": "equity", + "zerodha-history-subscription": "false", + "zerodha-organization": "abc", + }, + "Samco": { + "samco-client-id": "123", + "samco-client-password": "456", + "samco-year-of-birth": "2000", + "samco-product-type": "mis", + "samco-trading-segment": "equity", + "samco-organization": "abc", + }, + "Atreyu": { + "atreyu-host": "abc", + "atreyu-req-port": "123", + "atreyu-sub-port": "456", + "atreyu-username": "abc", + "atreyu-password": "abc", + "atreyu-client-id": "abc", + "atreyu-broker-mpid": "abc", + "atreyu-locate-rqd": "abc", + "atreyu-organization": "abc", + }, + "Terminal Link": { + "terminal-link-environment": "Beta", + "terminal-link-server-host": "abc", + "terminal-link-server-port": "123", + "terminal-link-emsx-broker": "abc", + "terminal-link-allow-modification": "no", + "terminal-link-emsx-account": "abc", + "terminal-link-emsx-strategy": "abc", + "terminal-link-emsx-notes": "abc", + "terminal-link-emsx-handling": "abc", + "terminal-link-emsx-user-time-zone": "abc", + "terminal-link-organization": "abc", + }, + "Kraken": { + "kraken-api-key": "abc", + "kraken-api-secret": "abc", + "kraken-verification-tier": "starter", + "kraken-organization": "abc", + }, + "FTX": { + "ftxus-api-key": "abc", + "ftxus-api-secret": "abc", + "ftxus-account-tier": "tier1", + "ftx-api-key": "abc", + "ftx-api-secret": "abc", + "ftx-account-tier": "tier1", + "ftx-exchange-name": "FTX", + "ftx-organization": "abc", + }, + "Trading Technologies": { + "tt-organization": "abc", + "tt-user-name": "abc", + "tt-session-password": "abc", + "tt-account-name": "abc", + "tt-rest-app-key": "abc", + "tt-rest-app-secret": "abc", + "tt-rest-environment": "abc", + "tt-market-data-sender-comp-id": "123", + "tt-market-data-target-comp-id": "123", + "tt-market-data-host": "abc", + "tt-market-data-port": "123", + "tt-order-routing-sender-comp-id": "123", + "tt-order-routing-target-comp-id": "123", + "tt-order-routing-host": "abc", + "tt-order-routing-port": "123", + "tt-log-fix-messages": "abc" + } +} + +@pytest.mark.parametrize("brokerage,cash", [("Paper Trading", "USD:100"), + ("Paper Trading", "USD:100,EUR:200"), + ("Atreyu", "USD:100"), + ("Trading Technologies", "USD:100"), + ("Zerodha", "USD:100")]) +def test_cloud_live_deploy_with_initial_cash_balance(brokerage: str, cash: str) -> None: + create_fake_lean_cli_directory() + + api_client = mock.Mock() + api_client.nodes.get_all.return_value = create_qc_nodes() + container.api_client.override(providers.Object(api_client)) + + cloud_project_manager = mock.Mock() + container.cloud_project_manager.override(providers.Object(cloud_project_manager)) + + cloud_runner = mock.Mock() + container.cloud_runner.override(providers.Object(cloud_runner)) + + options = [] + for key, value in brokerage_required_options[brokerage].items(): + options.extend([f"--{key}", value]) + + result = CliRunner().invoke(lean, ["cloud", "live", "Python Project", "--brokerage", brokerage, "--live-cash-balance", cash, + "--node", "live", "--auto-restart", "yes", "--notify-order-events", "no", + "--notify-insights", "no", *options]) + + if brokerage not in ["Paper Trading", "Atreyu", "Trading Technologies"]: + assert result.exit_code != 0 + api_client.live.start.assert_not_called() + return + + assert result.exit_code == 0 + + cash_pairs = cash.split(",") + if len(cash_pairs) == 2: + cash_list = [{"currency": "USD", "amount": 100}, {"currency": "EUR", "amount": 200}] + else: + cash_list = [{"currency": "USD", "amount": 100}] + + api_client.live.start.assert_called_once_with(mock.ANY, + mock.ANY, + "3", + mock.ANY, + mock.ANY, + True, + mock.ANY, + False, + False, + [], + cash_list) From e25b07d92f3abcbaafcfab79dba6a2acef87c4d5 Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Thu, 22 Sep 2022 18:01:14 +0800 Subject: [PATCH 02/25] fix test --- tests/commands/cloud/live/test_cloud_live_commands.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/commands/cloud/live/test_cloud_live_commands.py b/tests/commands/cloud/live/test_cloud_live_commands.py index 2351abf5..3b04bd5e 100644 --- a/tests/commands/cloud/live/test_cloud_live_commands.py +++ b/tests/commands/cloud/live/test_cloud_live_commands.py @@ -271,6 +271,16 @@ def test_cloud_live_deploy_with_notifications(notice_method: str, configs: str) ("Paper Trading", "USD:100,EUR:200"), ("Atreyu", "USD:100"), ("Trading Technologies", "USD:100"), + ("Binance", "USD:100"), + ("Bitfinex", "USD:100"), + ("FTX", "USD:100"), + ("Coinbase Pro", "USD:100"), + ("Interactive Brokers", "USD:100"), + ("Kraken", "USD:100"), + ("OANDA", "USD:100"), + ("Samco", "USD:100"), + ("Terminal Link", "USD:100"), + ("Tradier", "USD:100"), ("Zerodha", "USD:100")]) def test_cloud_live_deploy_with_initial_cash_balance(brokerage: str, cash: str) -> None: create_fake_lean_cli_directory() From e0b4bc47f3899437b93cf12de68326db514c16df Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Thu, 22 Sep 2022 22:38:08 +0800 Subject: [PATCH 03/25] Add cloud unit test --- lean/commands/live/deploy.py | 4 +- .../cloud/live/test_cloud_live_commands.py | 2 +- tests/commands/test_live.py | 84 ++++++++++++++++++- 3 files changed, 86 insertions(+), 4 deletions(-) diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index 728b1c32..d639cfd9 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -449,8 +449,8 @@ def deploy(project: Path, if python_venv is not None and python_venv != "": lean_config["python-venv"] = f'{"/" if python_venv[0] != "/" else ""}{python_venv}' - brokerage_id = lean_config["environments"]["live-mode-brokerage"] - if brokerage_id in [broker._id for broker in local_brokerages_with_editable_cash_balance]: + brokerage_id = lean_config["environments"]["lean-cli"]["live-mode-brokerage"] + if brokerage_id in [broker.get_live_name("lean-cli") for broker in local_brokerages_with_editable_cash_balance]: if live_cash_balance is not None and live_cash_balance != "": cash_list = [] for cash_pair in live_cash_balance.split(","): diff --git a/tests/commands/cloud/live/test_cloud_live_commands.py b/tests/commands/cloud/live/test_cloud_live_commands.py index 3b04bd5e..5474383d 100644 --- a/tests/commands/cloud/live/test_cloud_live_commands.py +++ b/tests/commands/cloud/live/test_cloud_live_commands.py @@ -282,7 +282,7 @@ def test_cloud_live_deploy_with_notifications(notice_method: str, configs: str) ("Terminal Link", "USD:100"), ("Tradier", "USD:100"), ("Zerodha", "USD:100")]) -def test_cloud_live_deploy_with_initial_cash_balance(brokerage: str, cash: str) -> None: +def test_cloud_live_deploy_with_live_cash_balance(brokerage: str, cash: str) -> None: create_fake_lean_cli_directory() api_client = mock.Mock() diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index 93923f4d..5e5d7634 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -395,7 +395,24 @@ def test_live_aborts_when_lean_config_is_missing_properties(target: str, replace "ftx-exchange-name": "FTX", "ftx-organization": "abc", }, - + "Trading Technologies": { + "tt-organization": "abc", + "tt-user-name": "abc", + "tt-session-password": "abc", + "tt-account-name": "abc", + "tt-rest-app-key": "abc", + "tt-rest-app-secret": "abc", + "tt-rest-environment": "abc", + "tt-market-data-sender-comp-id": "abc", + "tt-market-data-target-comp-id": "abc", + "tt-market-data-host": "abc", + "tt-market-data-port": "abc", + "tt-order-routing-sender-comp-id": "abc", + "tt-order-routing-target-comp-id": "abc", + "tt-order-routing-host": "abc", + "tt-order-routing-port": "abc", + "tt-log-fix-messages": "no" + } } data_feed_required_options = { @@ -904,3 +921,68 @@ def test_live_passes_custom_python_venv_to_lean_runner_when_given_as_option(pyth assert args[0]["python-venv"] == "/Custom-venv" else: assert "python-venv" not in args[0] + + +@pytest.mark.parametrize("brokerage,cash", [("Paper Trading", "USD:100"), + ("Paper Trading", "USD:100,EUR:200"), + ("Atreyu", "USD:100"), + ("Trading Technologies", "USD:100"), + ("Binance", "USD:100"), + ("Bitfinex", "USD:100"), + ("FTX", "USD:100"), + ("Coinbase Pro", "USD:100"), + ("Interactive Brokers", "USD:100"), + ("Kraken", "USD:100"), + ("OANDA", "USD:100"), + ("Samco", "USD:100"), + ("Terminal Link", "USD:100"), + ("Tradier", "USD:100"), + ("Zerodha", "USD:100")]) +def test_live_passes_live_cash_balance_to_lean_runner_when_given_as_option(brokerage: str, cash: str) -> None: + create_fake_lean_cli_directory() + + docker_manager = mock.Mock() + container.docker_manager.override(providers.Object(docker_manager)) + + lean_runner = mock.Mock() + container.lean_runner.override(providers.Object(lean_runner)) + + api_client = mock.MagicMock() + api_client.organizations.get_all.return_value = [ + QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) + ] + container.api_client.override(providers.Object(api_client)) + + options = [] + required_options = brokerage_required_options[brokerage].items() + for key, value in required_options: + options.extend([f"--{key}", value]) + + options_config = {key: value for key, value in set(required_options)} + with (Path.cwd() / "lean.json").open("w+", encoding="utf-8") as file: + file.write(json.dumps({ + **options_config, + "data-folder": "data", + "job-organization-id": "abc" + })) + + result = CliRunner().invoke(lean, ["live", "Python Project", "--brokerage", brokerage, "--live-cash-balance", cash, + "--data-feed", "Custom data only", *options]) + + if brokerage not in ["Paper Trading", "Atreyu", "Trading Technologies"]: + assert result.exit_code != 0 + lean_runner.run_lean.start.assert_not_called() + return + + assert result.exit_code == 0 + + cash_pairs = cash.split(",") + if len(cash_pairs) == 2: + cash_list = [{"currency": "USD", "amount": 100}, {"currency": "EUR", "amount": 200}] + else: + cash_list = [{"currency": "USD", "amount": 100}] + + lean_runner.run_lean.assert_called_once() + args, _ = lean_runner.run_lean.call_args + + assert args[0]["live-cash-balance"] == cash_list From 29161bf1d034c4bdbd4b3846db5c6a36d18c3a50 Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Fri, 23 Sep 2022 01:09:35 +0800 Subject: [PATCH 04/25] readme --- README.md | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 035dd2eb..408cf4fc 100644 --- a/README.md +++ b/README.md @@ -239,7 +239,7 @@ Usage: lean cloud live deploy [OPTIONS] PROJECT --notify-insights. Options: - --brokerage [Paper Trading|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Pro|Binance|Zerodha|Samco|Kraken|FTX] + --brokerage [Paper Trading|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Pro|Binance|Zerodha|Samco|Atreyu|Trading Technologies|Kraken|FTX] The brokerage to use --ib-user-name TEXT Your Interactive Brokers username --ib-account TEXT Your Interactive Brokers account id @@ -288,6 +288,33 @@ Options: --samco-trading-segment [equity|commodity] EQUITY if you are trading equities on NSE or BSE, COMMODITY if you are trading commodities on MCX + --atreyu-host TEXT The host of the Atreyu server + --atreyu-req-port INTEGER The Atreyu request port + --atreyu-sub-port INTEGER The Atreyu subscribe port + --atreyu-username TEXT Your Atreyu username + --atreyu-password TEXT Your Atreyu password + --atreyu-client-id TEXT Your Atreyu client id + --atreyu-broker-mpid TEXT The broker MPID to use + --atreyu-locate-rqd TEXT The locate rqd to use + --tt-user-name TEXT Your Trading Technologies username + --tt-session-password TEXT Your Trading Technologies session password + --tt-account-name TEXT Your Trading Technologies account name + --tt-rest-app-key TEXT Your Trading Technologies REST app key + --tt-rest-app-secret TEXT Your Trading Technologies REST app secret + --tt-rest-environment TEXT The REST environment to run in + --tt-market-data-sender-comp-id TEXT + The market data sender comp id to use + --tt-market-data-target-comp-id TEXT + The market data target comp id to use + --tt-market-data-host TEXT The host of the market data server + --tt-market-data-port TEXT The port of the market data server + --tt-order-routing-sender-comp-id TEXT + The order routing sender comp id to use + --tt-order-routing-target-comp-id TEXT + The order routing target comp id to use + --tt-order-routing-host TEXT The host of the order routing server + --tt-order-routing-port TEXT The port of the order routing server + --tt-log-fix-messages BOOLEAN Whether FIX messages should be logged --kraken-api-key TEXT Your Kraken API key --kraken-api-secret TEXT Your Kraken API secret --kraken-verification-tier [Starter|Intermediate|Pro] @@ -312,6 +339,7 @@ Options: --notify-sms TEXT A comma-separated list of phone numbers configuring SMS-notifications --notify-telegram TEXT A comma-separated list of 'user/group Id:token(optional)' pairs configuring telegram- notifications + --live-cash-balance TEXT A comma-separated list of currency:amount pairs of initial cash balance --push Push local modifications to the cloud before starting live trading --open Automatically open the live results in the browser once the deployment starts --verbose Enable debug logging @@ -979,6 +1007,7 @@ Options: --release Compile C# projects in release configuration instead of debug --image TEXT The LEAN engine image to use (defaults to quantconnect/lean:latest) --python-venv TEXT The path of the python virtual environment to be used + --live-cash-balance TEXT A comma-separated list of currency:amount pairs of initial cash balance --update Pull the LEAN engine image before starting live trading --lean-config FILE The Lean configuration file that should be used (defaults to the nearest lean.json) --verbose Enable debug logging From e5f3aadab4ab5c87ce696910c9e41f6fcee3b1f1 Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Fri, 23 Sep 2022 10:00:59 +0800 Subject: [PATCH 05/25] as brokerage properties --- lean/commands/cloud/live/deploy.py | 6 +++--- lean/commands/live/deploy.py | 5 ++--- lean/models/brokerages/cloud/__init__.py | 9 +-------- lean/models/brokerages/cloud/cloud_brokerage.py | 2 ++ lean/models/brokerages/local/__init__.py | 9 +-------- lean/models/brokerages/local/local_brokerage.py | 2 ++ 6 files changed, 11 insertions(+), 22 deletions(-) diff --git a/lean/commands/cloud/live/deploy.py b/lean/commands/cloud/live/deploy.py index 90fcf098..a363d05d 100644 --- a/lean/commands/cloud/live/deploy.py +++ b/lean/commands/cloud/live/deploy.py @@ -24,7 +24,7 @@ from lean.models.brokerages.cloud.cloud_brokerage import CloudBrokerage from lean.models.configuration import Configuration, InfoConfiguration, InternalInputUserInput, OrganzationIdConfiguration from lean.models.click_options import options_from_json -from lean.models.brokerages.cloud import all_cloud_brokerages, cloud_brokerages_with_editable_cash_balance +from lean.models.brokerages.cloud import all_cloud_brokerages from lean.commands.cloud.live.live import live def _log_notification_methods(methods: List[QCNotificationMethod]) -> None: @@ -314,7 +314,7 @@ def deploy(project: str, brokerage_settings = brokerage_instance.get_settings() price_data_handler = brokerage_instance.get_price_data_handler() - if brokerage_instance.get_name() in [broker.get_name() for broker in cloud_brokerages_with_editable_cash_balance]: + if brokerage_instance in [broker for broker in all_cloud_brokerages if broker._editable_initial_cash_balance]: if live_cash_balance is not None and live_cash_balance != "": cash_list = [] for cash_pair in live_cash_balance.split(","): @@ -339,7 +339,7 @@ def deploy(project: str, if notify_order_events or notify_insights: _log_notification_methods(notify_methods) if live_cash_balance: - logger.info(live_cash_balance) + logger.info(f"Initial live cash balance: {live_cash_balance}") logger.info(f"Automatic algorithm restarting: {'Yes' if auto_restart else 'No'}") if brokerage is None: diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index d639cfd9..7ad80f47 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -21,8 +21,7 @@ from lean.click import LeanCommand, PathParameter, ensure_options from lean.constants import DEFAULT_ENGINE_IMAGE from lean.container import container -from lean.models.brokerages.local import all_local_brokerages, local_brokerage_data_feeds, all_local_data_feeds, \ - local_brokerages_with_editable_cash_balance +from lean.models.brokerages.local import all_local_brokerages, local_brokerage_data_feeds, all_local_data_feeds from lean.models.errors import MoreInfoError from lean.models.lean_config_configurer import LeanConfigConfigurer from lean.models.logger import Option @@ -450,7 +449,7 @@ def deploy(project: Path, lean_config["python-venv"] = f'{"/" if python_venv[0] != "/" else ""}{python_venv}' brokerage_id = lean_config["environments"]["lean-cli"]["live-mode-brokerage"] - if brokerage_id in [broker.get_live_name("lean-cli") for broker in local_brokerages_with_editable_cash_balance]: + if brokerage_id in [broker.get_live_name("lean-cli") for broker in all_local_brokerages if broker._editable_initial_cash_balance]: if live_cash_balance is not None and live_cash_balance != "": cash_list = [] for cash_pair in live_cash_balance.split(","): diff --git a/lean/models/brokerages/cloud/__init__.py b/lean/models/brokerages/cloud/__init__.py index 856f9984..b1e641de 100644 --- a/lean/models/brokerages/cloud/__init__.py +++ b/lean/models/brokerages/cloud/__init__.py @@ -16,17 +16,10 @@ from typing import List all_cloud_brokerages: List[CloudBrokerage] = [] -cloud_brokerages_with_editable_cash_balance: List[CloudBrokerage] = [] -cloud_brokerages_with_editable_holdings: List[CloudBrokerage] = [] for json_module in json_modules: if "cloud-brokerage" in json_module["type"]: - cloud_brokerage = CloudBrokerage(json_module) - all_cloud_brokerages.append(cloud_brokerage) - if json_module["live-cash-balance-state"]: - cloud_brokerages_with_editable_cash_balance.append(cloud_brokerage) - if json_module["live-holdings-state"]: - cloud_brokerages_with_editable_holdings.append(cloud_brokerage) + all_cloud_brokerages.append(CloudBrokerage(json_module)) [PaperTradingBrokerage] = [ cloud_brokerage for cloud_brokerage in all_cloud_brokerages if cloud_brokerage._id == "QuantConnectBrokerage"] diff --git a/lean/models/brokerages/cloud/cloud_brokerage.py b/lean/models/brokerages/cloud/cloud_brokerage.py index da8dbc0c..4f1aee4b 100644 --- a/lean/models/brokerages/cloud/cloud_brokerage.py +++ b/lean/models/brokerages/cloud/cloud_brokerage.py @@ -21,6 +21,8 @@ class CloudBrokerage(JsonModule): def __init__(self, json_cloud_brokerage_data: Dict[str, Any]) -> None: super().__init__(json_cloud_brokerage_data) + self._editable_initial_cash_balance = json_cloud_brokerage_data["live-cash-balance-state"] + self._editable_initial_holdings = json_cloud_brokerage_data["live-holdings-state"] def get_id(self) -> str: """Returns the id of the brokerage. diff --git a/lean/models/brokerages/local/__init__.py b/lean/models/brokerages/local/__init__.py index f39e518f..3e7ac91c 100644 --- a/lean/models/brokerages/local/__init__.py +++ b/lean/models/brokerages/local/__init__.py @@ -19,20 +19,13 @@ from lean.models import json_modules all_local_brokerages: List[LocalBrokerage] = [] -local_brokerages_with_editable_cash_balance: List[LocalBrokerage] = [] -local_brokerages_with_editable_holdings: List[LocalBrokerage] = [] all_local_data_feeds: List[DataFeed] = [] local_brokerage_data_feeds: Dict[Type[LocalBrokerage], List[Type[DataFeed]]] = {} for json_module in json_modules: if "local-brokerage" in json_module["type"]: - local_brokerage = LocalBrokerage(json_module) - all_local_brokerages.append(local_brokerage) - if json_module["live-cash-balance-state"]: - local_brokerages_with_editable_cash_balance.append(local_brokerage) - if json_module["live-holdings-state"]: - local_brokerages_with_editable_holdings.append(local_brokerage) + all_local_brokerages.append(LocalBrokerage(json_module)) if "data-queue-handler" in json_module["type"]: all_local_data_feeds.append(DataFeed(json_module)) diff --git a/lean/models/brokerages/local/local_brokerage.py b/lean/models/brokerages/local/local_brokerage.py index c9bee591..4db1de54 100644 --- a/lean/models/brokerages/local/local_brokerage.py +++ b/lean/models/brokerages/local/local_brokerage.py @@ -21,6 +21,8 @@ class LocalBrokerage(LeanConfigConfigurer): def __init__(self, json_brokerage_data: Dict[str, Any]) -> None: super().__init__(json_brokerage_data) + self._editable_initial_cash_balance = json_brokerage_data["live-cash-balance-state"] + self._editable_initial_holdings = json_brokerage_data["live-holdings-state"] def get_live_name(self, environment_name: str) -> str: live_name = self._id From 5093333b9ab3825f744e62aa6d511e33c06b6d20 Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Fri, 23 Sep 2022 13:46:28 +0800 Subject: [PATCH 06/25] remove duplicates and correct cloud payload --- lean/commands/cloud/live/deploy.py | 36 +++--- lean/commands/live/deploy.py | 31 +---- lean/components/api/live_client.py | 11 +- .../cloud/live/test_cloud_live_commands.py | 112 +----------------- 4 files changed, 27 insertions(+), 163 deletions(-) diff --git a/lean/commands/cloud/live/deploy.py b/lean/commands/cloud/live/deploy.py index a363d05d..cde97c5d 100644 --- a/lean/commands/cloud/live/deploy.py +++ b/lean/commands/cloud/live/deploy.py @@ -105,23 +105,31 @@ def _configure_brokerage(logger: Logger) -> CloudBrokerage: return logger.prompt_list("Select a brokerage", brokerage_options).build(None,logger) -def _configure_initial_cash_balance(logger: Logger) -> List[Dict[str, float]]: +def _configure_initial_cash_balance(logger: Logger, live_cash_balance: str) -> List[Dict[str, float]]: """Interactively configures the intial cash balance. :param logger: the logger to use + :param live_cash_balance: the initial cash balance option input :return: the list of dictionary containing intial currency and amount information """ cash_list = [] - continue_adding = True - while continue_adding: - currency = click.prompt("Currency") - amount = click.prompt("Amount", type=float) - cash_list.append({"currency": currency, "amount": amount}) - logger.info(f"Cash balance: {cash_list}") + if live_cash_balance is not None and live_cash_balance != "": + for cash_pair in live_cash_balance.split(","): + currency, amount = cash_pair.split(":") + cash_list.append({"currency": currency, "amount": float(amount)}) - if not click.confirm("Do you want to add other currency?", default=False): - continue_adding = False + elif click.confirm("Do you want to set initial cash balance?", default=False): + continue_adding = True + + while continue_adding: + currency = click.prompt("Currency") + amount = click.prompt("Amount", type=float) + cash_list.append({"currency": currency, "amount": amount}) + logger.info(f"Cash balance: {cash_list}") + + if not click.confirm("Do you want to add other currency?", default=False): + continue_adding = False return cash_list @@ -315,15 +323,7 @@ def deploy(project: str, price_data_handler = brokerage_instance.get_price_data_handler() if brokerage_instance in [broker for broker in all_cloud_brokerages if broker._editable_initial_cash_balance]: - if live_cash_balance is not None and live_cash_balance != "": - cash_list = [] - for cash_pair in live_cash_balance.split(","): - currency, amount = cash_pair.split(":") - cash_list.append({"currency": currency, "amount": float(amount)}) - live_cash_balance = cash_list - elif click.confirm("Do you want to set initial cash balance?", default=False): - cash_list = _configure_initial_cash_balance() - live_cash_balance = cash_list + live_cash_balance = _configure_initial_cash_balance(logger, live_cash_balance) elif live_cash_balance is not None and live_cash_balance != "": raise RuntimeError(f"Custom cash balance setting is not available for {brokerage_instance.get_name()}") diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index 7ad80f47..63278d54 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -28,6 +28,7 @@ from lean.models.configuration import Configuration, InfoConfiguration, InternalInputUserInput, OrganzationIdConfiguration from lean.models.click_options import options_from_json from lean.models.json_module import JsonModule +from lean.commands.cloud.live.deploy import _configure_initial_cash_balance from lean.commands.live.live import live from lean.models.data_providers import all_data_providers @@ -172,25 +173,6 @@ def _configure_lean_config_interactively(lean_config: Dict[str, Any], environmen setattr(data_feed, '_is_installed_and_build', True) data_feed.build(lean_config, logger).configure(lean_config, environment_name) - -def _configure_initial_cash_balance() -> List[Dict[str, float]]: - logger = container.logger() - - cash_list = [] - continue_adding = True - - while continue_adding: - currency = click.prompt("Currency") - amount = click.prompt("Amount", type=float) - cash_list.append({"currency": currency, "amount": amount}) - logger.info(f"Cash balance: {cash_list}") - - if not click.confirm("Do you want to add other currency?", default=False): - continue_adding = False - - return cash_list - - _cached_organizations = None @@ -450,15 +432,8 @@ def deploy(project: Path, brokerage_id = lean_config["environments"]["lean-cli"]["live-mode-brokerage"] if brokerage_id in [broker.get_live_name("lean-cli") for broker in all_local_brokerages if broker._editable_initial_cash_balance]: - if live_cash_balance is not None and live_cash_balance != "": - cash_list = [] - for cash_pair in live_cash_balance.split(","): - currency, amount = cash_pair.split(":") - cash_list.append({"currency": currency, "amount": float(amount)}) - lean_config["live-cash-balance"] = cash_list - elif click.confirm("Do you want to set initial cash balance?", default=False): - cash_list = _configure_initial_cash_balance() - lean_config["live-cash-balance"] = cash_list + logger = container.logger() + live_cash_balance = _configure_initial_cash_balance(logger, live_cash_balance) elif live_cash_balance is not None and live_cash_balance != "": raise RuntimeError(f"Custom cash balance setting is not available for {brokerage_id}") diff --git a/lean/components/api/live_client.py b/lean/components/api/live_client.py index 7ff993c4..21ee5899 100644 --- a/lean/components/api/live_client.py +++ b/lean/components/api/live_client.py @@ -56,7 +56,7 @@ def start(self, project_id: int, compile_id: str, node_id: str, - brokerage_settings: Dict[str, str], + brokerage_settings: Dict[str, Any], price_data_handler: str, automatic_redeploy: bool, version_id: int, @@ -76,10 +76,13 @@ def start(self, :param notify_order_events: whether notifications should be sent on order events :param notify_insights: whether notifications should be sent on insights :param notify_methods: the places to send notifications to - :param notify_methods: the list of initial cash balance + :param live_cash_balance: the list of initial cash balance :return: the created live algorithm """ + if live_cash_balance: + brokerage_settings["cash"] = live_cash_balance + parameters = { "projectId": project_id, "compileId": compile_id, @@ -101,10 +104,6 @@ def start(self, "events": events, "targets": [{x: y for x, y in method.dict().items() if y} for method in notify_methods] } - - if live_cash_balance: - parameters["portfolio"] = {} - parameters["portfolio"]["cash"] = live_cash_balance data = self._api.post("live/create", parameters) return QCMinimalLiveAlgorithm(**data) diff --git a/tests/commands/cloud/live/test_cloud_live_commands.py b/tests/commands/cloud/live/test_cloud_live_commands.py index 5474383d..11e49a39 100644 --- a/tests/commands/cloud/live/test_cloud_live_commands.py +++ b/tests/commands/cloud/live/test_cloud_live_commands.py @@ -21,6 +21,7 @@ from lean.container import container from lean.models.api import QCEmailNotificationMethod, QCWebhookNotificationMethod, QCSMSNotificationMethod, QCTelegramNotificationMethod from tests.test_helpers import create_fake_lean_cli_directory, create_qc_nodes +from tests.commands.test_live import brokerage_required_options def test_cloud_live_stop() -> None: create_fake_lean_cli_directory() @@ -155,117 +156,6 @@ def test_cloud_live_deploy_with_notifications(notice_method: str, configs: str) notification, None) -brokerage_required_options = { - "Paper Trading": {}, - "Interactive Brokers": { - "ib-user-name": "trader777", - "ib-account": "DU1234567", - "ib-password": "hunter2", - "ib-enable-delayed-streaming-data": "no", - "ib-organization": "abc", - }, - "Tradier": { - "tradier-account-id": "123", - "tradier-access-token": "456", - "tradier-environment": "paper" - }, - "OANDA": { - "oanda-account-id": "123", - "oanda-access-token": "456", - "oanda-environment": "Practice" - }, - "Bitfinex": { - "bitfinex-api-key": "123", - "bitfinex-api-secret": "456", - }, - "Coinbase Pro": { - "gdax-api-key": "123", - "gdax-api-secret": "456", - "gdax-passphrase": "789", - "gdax-use-sandbox": "paper" - }, - "Binance": { - "binance-exchange-name": "binance", - "binance-api-key": "123", - "binance-api-secret": "456", - "binance-use-testnet": "paper", - "binance-organization": "abc", - }, - "Zerodha": { - "zerodha-api-key": "123", - "zerodha-access-token": "456", - "zerodha-product-type": "mis", - "zerodha-trading-segment": "equity", - "zerodha-history-subscription": "false", - "zerodha-organization": "abc", - }, - "Samco": { - "samco-client-id": "123", - "samco-client-password": "456", - "samco-year-of-birth": "2000", - "samco-product-type": "mis", - "samco-trading-segment": "equity", - "samco-organization": "abc", - }, - "Atreyu": { - "atreyu-host": "abc", - "atreyu-req-port": "123", - "atreyu-sub-port": "456", - "atreyu-username": "abc", - "atreyu-password": "abc", - "atreyu-client-id": "abc", - "atreyu-broker-mpid": "abc", - "atreyu-locate-rqd": "abc", - "atreyu-organization": "abc", - }, - "Terminal Link": { - "terminal-link-environment": "Beta", - "terminal-link-server-host": "abc", - "terminal-link-server-port": "123", - "terminal-link-emsx-broker": "abc", - "terminal-link-allow-modification": "no", - "terminal-link-emsx-account": "abc", - "terminal-link-emsx-strategy": "abc", - "terminal-link-emsx-notes": "abc", - "terminal-link-emsx-handling": "abc", - "terminal-link-emsx-user-time-zone": "abc", - "terminal-link-organization": "abc", - }, - "Kraken": { - "kraken-api-key": "abc", - "kraken-api-secret": "abc", - "kraken-verification-tier": "starter", - "kraken-organization": "abc", - }, - "FTX": { - "ftxus-api-key": "abc", - "ftxus-api-secret": "abc", - "ftxus-account-tier": "tier1", - "ftx-api-key": "abc", - "ftx-api-secret": "abc", - "ftx-account-tier": "tier1", - "ftx-exchange-name": "FTX", - "ftx-organization": "abc", - }, - "Trading Technologies": { - "tt-organization": "abc", - "tt-user-name": "abc", - "tt-session-password": "abc", - "tt-account-name": "abc", - "tt-rest-app-key": "abc", - "tt-rest-app-secret": "abc", - "tt-rest-environment": "abc", - "tt-market-data-sender-comp-id": "123", - "tt-market-data-target-comp-id": "123", - "tt-market-data-host": "abc", - "tt-market-data-port": "123", - "tt-order-routing-sender-comp-id": "123", - "tt-order-routing-target-comp-id": "123", - "tt-order-routing-host": "abc", - "tt-order-routing-port": "123", - "tt-log-fix-messages": "abc" - } -} @pytest.mark.parametrize("brokerage,cash", [("Paper Trading", "USD:100"), ("Paper Trading", "USD:100,EUR:200"), From bd2bd6f2266d7903fd7d8c3e792010cedb2b5b29 Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Fri, 23 Sep 2022 23:26:44 +0800 Subject: [PATCH 07/25] Peer review and bug fix --- lean/commands/cloud/live/deploy.py | 45 ++---------- lean/commands/live/deploy.py | 23 ++---- lean/components/util/live_utils.py | 72 +++++++++++++++++++ .../cloud/live/test_cloud_live_commands.py | 37 +++++----- tests/commands/test_live.py | 10 +-- 5 files changed, 106 insertions(+), 81 deletions(-) create mode 100644 lean/components/util/live_utils.py diff --git a/lean/commands/cloud/live/deploy.py b/lean/commands/cloud/live/deploy.py index cde97c5d..da349bb1 100644 --- a/lean/commands/cloud/live/deploy.py +++ b/lean/commands/cloud/live/deploy.py @@ -12,7 +12,7 @@ # limitations under the License. import webbrowser -from typing import Dict, List, Tuple, Optional +from typing import List, Tuple, Optional import click from lean.click import LeanCommand, ensure_options from lean.components.api.api_client import APIClient @@ -22,10 +22,11 @@ QCWebhookNotificationMethod, QCTelegramNotificationMethod, QCProject) from lean.models.logger import Option from lean.models.brokerages.cloud.cloud_brokerage import CloudBrokerage -from lean.models.configuration import Configuration, InfoConfiguration, InternalInputUserInput, OrganzationIdConfiguration +from lean.models.configuration import InternalInputUserInput, OrganzationIdConfiguration from lean.models.click_options import options_from_json from lean.models.brokerages.cloud import all_cloud_brokerages from lean.commands.cloud.live.live import live +from lean.components.util.live_utils import _get_configs_for_options, _configure_initial_cash_balance def _log_notification_methods(methods: List[QCNotificationMethod]) -> None: """Logs a list of notification methods.""" @@ -104,35 +105,6 @@ def _configure_brokerage(logger: Logger) -> CloudBrokerage: brokerage_options = [Option(id=b, label=b.get_name()) for b in all_cloud_brokerages] return logger.prompt_list("Select a brokerage", brokerage_options).build(None,logger) - -def _configure_initial_cash_balance(logger: Logger, live_cash_balance: str) -> List[Dict[str, float]]: - """Interactively configures the intial cash balance. - - :param logger: the logger to use - :param live_cash_balance: the initial cash balance option input - :return: the list of dictionary containing intial currency and amount information - """ - cash_list = [] - - if live_cash_balance is not None and live_cash_balance != "": - for cash_pair in live_cash_balance.split(","): - currency, amount = cash_pair.split(":") - cash_list.append({"currency": currency, "amount": float(amount)}) - - elif click.confirm("Do you want to set initial cash balance?", default=False): - continue_adding = True - - while continue_adding: - currency = click.prompt("Currency") - amount = click.prompt("Amount", type=float) - cash_list.append({"currency": currency, "amount": amount}) - logger.info(f"Cash balance: {cash_list}") - - if not click.confirm("Do you want to add other currency?", default=False): - continue_adding = False - - return cash_list - def _configure_live_node(logger: Logger, api_client: APIClient, cloud_project: QCProject) -> QCNode: """Interactively configures the live node to use. @@ -189,22 +161,13 @@ def _configure_auto_restart(logger: Logger) -> bool: logger.info("This can help improve its resilience to temporary errors such as a brokerage API disconnection") return click.confirm("Do you want to enable automatic algorithm restarting?", default=True) -#TODO: same duplication present in commands\live.py -def _get_configs_for_options() -> Dict[Configuration, str]: - run_options: Dict[str, Configuration] = {} - for module in all_cloud_brokerages: - for config in module.get_all_input_configs([InternalInputUserInput, InfoConfiguration]): - if config._id in run_options: - raise ValueError(f'Options names should be unique. Duplicate key present: {config._id}') - run_options[config._id] = config - return list(run_options.values()) @live.command(cls=LeanCommand, default_command=True, name="deploy") @click.argument("project", type=str) @click.option("--brokerage", type=click.Choice([b.get_name() for b in all_cloud_brokerages], case_sensitive=False), help="The brokerage to use") -@options_from_json(_get_configs_for_options()) +@options_from_json(_get_configs_for_options("cloud")) @click.option("--node", type=str, help="The name or id of the live node to run on") @click.option("--auto-restart", type=bool, help="Whether automatic algorithm restarting must be enabled") @click.option("--notify-order-events", type=bool, help="Whether notifications must be sent for order events") diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index 63278d54..42b774c1 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -25,11 +25,11 @@ from lean.models.errors import MoreInfoError from lean.models.lean_config_configurer import LeanConfigConfigurer from lean.models.logger import Option -from lean.models.configuration import Configuration, InfoConfiguration, InternalInputUserInput, OrganzationIdConfiguration +from lean.models.configuration import InternalInputUserInput, OrganzationIdConfiguration from lean.models.click_options import options_from_json from lean.models.json_module import JsonModule -from lean.commands.cloud.live.deploy import _configure_initial_cash_balance from lean.commands.live.live import live +from lean.components.util.live_utils import _get_configs_for_options, _configure_initial_cash_balance from lean.models.data_providers import all_data_providers _environment_skeleton = { @@ -262,21 +262,6 @@ def _get_default_value(key: str) -> Optional[Any]: return value -def _get_configs_for_options() -> List[Configuration]: - run_options: Dict[str, Configuration] = {} - config_with_module_id: Dict[str, str] = {} - for module in all_local_brokerages + all_local_data_feeds + all_data_providers: - for config in module.get_all_input_configs([InternalInputUserInput, InfoConfiguration]): - if config._id in run_options: - if (config._id in config_with_module_id - and config_with_module_id[config._id] == module._id): - # config of same module - continue - else: - raise ValueError(f'Options names should be unique. Duplicate key present: {config._id}') - run_options[config._id] = config - config_with_module_id[config._id] = module._id - return list(run_options.values()) @live.command(cls=LeanCommand, requires_lean_config=True, requires_docker=True, default_command=True, name="deploy") @click.argument("project", type=PathParameter(exists=True, file_okay=True, dir_okay=True)) @@ -300,7 +285,7 @@ def _get_configs_for_options() -> List[Configuration]: @click.option("--data-provider", type=click.Choice([dp.get_name() for dp in all_data_providers], case_sensitive=False), help="Update the Lean configuration file to retrieve data from the given provider") -@options_from_json(_get_configs_for_options()) +@options_from_json(_get_configs_for_options("local")) @click.option("--release", is_flag=True, default=False, @@ -434,6 +419,8 @@ def deploy(project: Path, if brokerage_id in [broker.get_live_name("lean-cli") for broker in all_local_brokerages if broker._editable_initial_cash_balance]: logger = container.logger() live_cash_balance = _configure_initial_cash_balance(logger, live_cash_balance) + if live_cash_balance: + lean_config["live-cash-balance"] = live_cash_balance elif live_cash_balance is not None and live_cash_balance != "": raise RuntimeError(f"Custom cash balance setting is not available for {brokerage_id}") diff --git a/lean/components/util/live_utils.py b/lean/components/util/live_utils.py new file mode 100644 index 00000000..dd1d789d --- /dev/null +++ b/lean/components/util/live_utils.py @@ -0,0 +1,72 @@ +# QUANTCONNECT.COM - Democratizing Finance, Empowering Individuals. +# Lean CLI v1.0. Copyright 2021 QuantConnect Corporation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import click +from typing import Dict, List +from lean.components.util.logger import Logger +from lean.models.brokerages.cloud import all_cloud_brokerages +from lean.models.brokerages.local import all_local_brokerages, all_local_data_feeds +from lean.models.data_providers import all_data_providers +from lean.models.configuration import Configuration, InfoConfiguration, InternalInputUserInput + +def _get_configs_for_options(env: str) -> List[Configuration]: + if env == "cloud": + brokerage = all_cloud_brokerages + elif env == "local": + brokerage = all_local_brokerages + all_local_data_feeds + all_data_providers + else: + raise ValueError("Only 'cloud' and 'local' are accepted for the argument 'env'") + + run_options: Dict[str, Configuration] = {} + config_with_module_id: Dict[str, str] = {} + for module in brokerage: + for config in module.get_all_input_configs([InternalInputUserInput, InfoConfiguration]): + if config._id in run_options: + if (config._id in config_with_module_id + and config_with_module_id[config._id] == module._id): + # config of same module + continue + else: + raise ValueError(f'Options names should be unique. Duplicate key present: {config._id}') + run_options[config._id] = config + config_with_module_id[config._id] = module._id + return list(run_options.values()) + + +def _configure_initial_cash_balance(logger: Logger, live_cash_balance: str) -> List[Dict[str, float]]: + """Interactively configures the intial cash balance. + + :param logger: the logger to use + :param live_cash_balance: the initial cash balance option input + :return: the list of dictionary containing intial currency and amount information + """ + cash_list = [] + + if live_cash_balance is not None and live_cash_balance != "": + for cash_pair in live_cash_balance.split(","): + currency, amount = cash_pair.split(":") + cash_list.append({"currency": currency, "amount": float(amount)}) + + elif click.confirm("Do you want to set initial cash balance?", default=False): + continue_adding = True + + while continue_adding: + currency = click.prompt("Currency") + amount = click.prompt("Amount", type=float) + cash_list.append({"currency": currency, "amount": amount}) + logger.info(f"Cash balance: {cash_list}") + + if not click.confirm("Do you want to add other currency?", default=False): + continue_adding = False + + return cash_list \ No newline at end of file diff --git a/tests/commands/cloud/live/test_cloud_live_commands.py b/tests/commands/cloud/live/test_cloud_live_commands.py index 11e49a39..61b94072 100644 --- a/tests/commands/cloud/live/test_cloud_live_commands.py +++ b/tests/commands/cloud/live/test_cloud_live_commands.py @@ -77,7 +77,7 @@ def test_cloud_live_deploy() -> None: False, False, [], - None) + mock.ANY) @pytest.mark.parametrize("notice_method,configs", [("emails", "customAddress:customSubject"), ("emails", "customAddress1:customSubject1,customAddress2:customSubject2"), @@ -154,7 +154,7 @@ def test_cloud_live_deploy_with_notifications(notice_method: str, configs: str) True, True, notification, - None) + mock.ANY) @pytest.mark.parametrize("brokerage,cash", [("Paper Trading", "USD:100"), @@ -200,20 +200,23 @@ def test_cloud_live_deploy_with_live_cash_balance(brokerage: str, cash: str) -> assert result.exit_code == 0 - cash_pairs = cash.split(",") - if len(cash_pairs) == 2: - cash_list = [{"currency": "USD", "amount": 100}, {"currency": "EUR", "amount": 200}] + if cash: + cash_pairs = cash.split(",") + if len(cash_pairs) == 2: + cash_list = [{"currency": "USD", "amount": 100}, {"currency": "EUR", "amount": 200}] + else: + cash_list = [{"currency": "USD", "amount": 100}] else: - cash_list = [{"currency": "USD", "amount": 100}] - + cash_list = [] + api_client.live.start.assert_called_once_with(mock.ANY, - mock.ANY, - "3", - mock.ANY, - mock.ANY, - True, - mock.ANY, - False, - False, - [], - cash_list) + mock.ANY, + "3", + mock.ANY, + mock.ANY, + True, + mock.ANY, + False, + False, + [], + cash_list) diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index 5e5d7634..d4806ffc 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -63,7 +63,7 @@ def create_fake_environment(name: str, live_mode: bool) -> None: path.write_text(config, encoding="utf-8") - +''' def test_live_calls_lean_runner_with_correct_algorithm_file() -> None: # TODO: currently it is not using the live-paper envrionment create_fake_lean_cli_directory() @@ -301,7 +301,7 @@ def test_live_aborts_when_lean_config_is_missing_properties(target: str, replace assert result.exit_code != 0 lean_runner.run_lean.assert_not_called() - +''' brokerage_required_options = { "Paper Trading": {}, @@ -436,7 +436,7 @@ def test_live_aborts_when_lean_config_is_missing_properties(target: str, replace "Terminal Link": brokerage_required_options["Terminal Link"] } - +''' @pytest.mark.parametrize("data_provider", data_providers_required_options.keys()) def test_live_calls_lean_runner_with_data_provider(data_provider: str) -> None: create_fake_lean_cli_directory() @@ -921,7 +921,7 @@ def test_live_passes_custom_python_venv_to_lean_runner_when_given_as_option(pyth assert args[0]["python-venv"] == "/Custom-venv" else: assert "python-venv" not in args[0] - +''' @pytest.mark.parametrize("brokerage,cash", [("Paper Trading", "USD:100"), ("Paper Trading", "USD:100,EUR:200"), @@ -966,7 +966,7 @@ def test_live_passes_live_cash_balance_to_lean_runner_when_given_as_option(broke "job-organization-id": "abc" })) - result = CliRunner().invoke(lean, ["live", "Python Project", "--brokerage", brokerage, "--live-cash-balance", cash, + result = CliRunner().invoke(lean, ["live", "Python Project", "--brokerage", brokerage, "--live-cash-balance", cash, "--data-feed", "Custom data only", *options]) if brokerage not in ["Paper Trading", "Atreyu", "Trading Technologies"]: From aeaaf80a7277bb0ebcab87f7d87f58b9df9ad333 Mon Sep 17 00:00:00 2001 From: Louis Szeto <56447733+LouisSzeto@users.noreply.github.com> Date: Mon, 26 Sep 2022 13:27:15 +0800 Subject: [PATCH 08/25] update tests --- tests/commands/cloud/live/test_cloud_live_commands.py | 2 +- tests/commands/test_live.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/commands/cloud/live/test_cloud_live_commands.py b/tests/commands/cloud/live/test_cloud_live_commands.py index 61b94072..daabbe30 100644 --- a/tests/commands/cloud/live/test_cloud_live_commands.py +++ b/tests/commands/cloud/live/test_cloud_live_commands.py @@ -193,7 +193,7 @@ def test_cloud_live_deploy_with_live_cash_balance(brokerage: str, cash: str) -> "--node", "live", "--auto-restart", "yes", "--notify-order-events", "no", "--notify-insights", "no", *options]) - if brokerage not in ["Paper Trading", "Atreyu", "Trading Technologies"]: + if brokerage not in ["Paper Trading", "Trading Technologies"]: assert result.exit_code != 0 api_client.live.start.assert_not_called() return diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index d4806ffc..995f64db 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -969,6 +969,7 @@ def test_live_passes_live_cash_balance_to_lean_runner_when_given_as_option(broke result = CliRunner().invoke(lean, ["live", "Python Project", "--brokerage", brokerage, "--live-cash-balance", cash, "--data-feed", "Custom data only", *options]) + # TODO: remove Atreyu after the discontinuation of the brokerage support (when removed frommodule-*.json) if brokerage not in ["Paper Trading", "Atreyu", "Trading Technologies"]: assert result.exit_code != 0 lean_runner.run_lean.start.assert_not_called() From b4d217754523e443cae4eaff5aa24e2d4e849d32 Mon Sep 17 00:00:00 2001 From: Louis Szeto <56447733+LouisSzeto@users.noreply.github.com> Date: Mon, 26 Sep 2022 13:30:58 +0800 Subject: [PATCH 09/25] update reademe --- README.md | 10 +--------- tests/commands/test_live.py | 8 ++++---- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 408cf4fc..a1cf986b 100644 --- a/README.md +++ b/README.md @@ -239,7 +239,7 @@ Usage: lean cloud live deploy [OPTIONS] PROJECT --notify-insights. Options: - --brokerage [Paper Trading|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Pro|Binance|Zerodha|Samco|Atreyu|Trading Technologies|Kraken|FTX] + --brokerage [Paper Trading|Interactive Brokers|Tradier|Oanda|Bitfinex|Coinbase Pro|Binance|Zerodha|Samco|Trading Technologies|Kraken|FTX] The brokerage to use --ib-user-name TEXT Your Interactive Brokers username --ib-account TEXT Your Interactive Brokers account id @@ -288,14 +288,6 @@ Options: --samco-trading-segment [equity|commodity] EQUITY if you are trading equities on NSE or BSE, COMMODITY if you are trading commodities on MCX - --atreyu-host TEXT The host of the Atreyu server - --atreyu-req-port INTEGER The Atreyu request port - --atreyu-sub-port INTEGER The Atreyu subscribe port - --atreyu-username TEXT Your Atreyu username - --atreyu-password TEXT Your Atreyu password - --atreyu-client-id TEXT Your Atreyu client id - --atreyu-broker-mpid TEXT The broker MPID to use - --atreyu-locate-rqd TEXT The locate rqd to use --tt-user-name TEXT Your Trading Technologies username --tt-session-password TEXT Your Trading Technologies session password --tt-account-name TEXT Your Trading Technologies account name diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index 995f64db..07788298 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -63,7 +63,7 @@ def create_fake_environment(name: str, live_mode: bool) -> None: path.write_text(config, encoding="utf-8") -''' + def test_live_calls_lean_runner_with_correct_algorithm_file() -> None: # TODO: currently it is not using the live-paper envrionment create_fake_lean_cli_directory() @@ -301,7 +301,7 @@ def test_live_aborts_when_lean_config_is_missing_properties(target: str, replace assert result.exit_code != 0 lean_runner.run_lean.assert_not_called() -''' + brokerage_required_options = { "Paper Trading": {}, @@ -436,7 +436,7 @@ def test_live_aborts_when_lean_config_is_missing_properties(target: str, replace "Terminal Link": brokerage_required_options["Terminal Link"] } -''' + @pytest.mark.parametrize("data_provider", data_providers_required_options.keys()) def test_live_calls_lean_runner_with_data_provider(data_provider: str) -> None: create_fake_lean_cli_directory() @@ -921,7 +921,7 @@ def test_live_passes_custom_python_venv_to_lean_runner_when_given_as_option(pyth assert args[0]["python-venv"] == "/Custom-venv" else: assert "python-venv" not in args[0] -''' + @pytest.mark.parametrize("brokerage,cash", [("Paper Trading", "USD:100"), ("Paper Trading", "USD:100,EUR:200"), From 4481743b06907aa32a7a4f3ec02fb305eef6a164 Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Mon, 26 Sep 2022 23:12:59 +0800 Subject: [PATCH 10/25] interactive selection of cloud --- lean/models/configuration.py | 1 + lean/models/json_module.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/lean/models/configuration.py b/lean/models/configuration.py index ba1ab62d..b96b77e9 100644 --- a/lean/models/configuration.py +++ b/lean/models/configuration.py @@ -93,6 +93,7 @@ def __init__(self, config_json_object): self._id: str = config_json_object["id"] self._config_type: str = config_json_object["type"] self._value: str = config_json_object["value"] + self._is_cloud_property: bool = "cloud-id" in config_json_object self._is_required_from_user = False self._is_type_configurations_env: bool = type( self) is ConfigurationsEnvConfiguration diff --git a/lean/models/json_module.py b/lean/models/json_module.py index c146a3a1..cce5a8b8 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -163,6 +163,8 @@ def build(self, lean_config: Dict[str, Any], logger: Logger) -> 'JsonModule': continue if type(configuration) is InternalInputUserInput: continue + if self.__class__.__name__ == 'CloudBrokerage' and not configuration._is_cloud_property: + continue if configuration._log_message is not None: logger.info(configuration._log_message.strip()) if configuration.is_type_organization_id: From 7881020a56b665842779dab6fef28f8e8fe26bb1 Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Tue, 27 Sep 2022 23:54:55 +0800 Subject: [PATCH 11/25] fix unit test --- tests/commands/cloud/live/test_cloud_live_commands.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/commands/cloud/live/test_cloud_live_commands.py b/tests/commands/cloud/live/test_cloud_live_commands.py index daabbe30..b7e6f805 100644 --- a/tests/commands/cloud/live/test_cloud_live_commands.py +++ b/tests/commands/cloud/live/test_cloud_live_commands.py @@ -187,7 +187,8 @@ def test_cloud_live_deploy_with_live_cash_balance(brokerage: str, cash: str) -> options = [] for key, value in brokerage_required_options[brokerage].items(): - options.extend([f"--{key}", value]) + if "organization" not in key: + options.extend([f"--{key}", value]) result = CliRunner().invoke(lean, ["cloud", "live", "Python Project", "--brokerage", brokerage, "--live-cash-balance", cash, "--node", "live", "--auto-restart", "yes", "--notify-order-events", "no", From 51931a3b27ea4295c7813acf23db787d5b8681e0 Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Wed, 28 Sep 2022 01:07:39 +0800 Subject: [PATCH 12/25] Remove redundancy --- lean/models/brokerages/cloud/cloud_brokerage.py | 1 - lean/models/brokerages/local/local_brokerage.py | 1 - 2 files changed, 2 deletions(-) diff --git a/lean/models/brokerages/cloud/cloud_brokerage.py b/lean/models/brokerages/cloud/cloud_brokerage.py index 4f1aee4b..5899f3c6 100644 --- a/lean/models/brokerages/cloud/cloud_brokerage.py +++ b/lean/models/brokerages/cloud/cloud_brokerage.py @@ -22,7 +22,6 @@ class CloudBrokerage(JsonModule): def __init__(self, json_cloud_brokerage_data: Dict[str, Any]) -> None: super().__init__(json_cloud_brokerage_data) self._editable_initial_cash_balance = json_cloud_brokerage_data["live-cash-balance-state"] - self._editable_initial_holdings = json_cloud_brokerage_data["live-holdings-state"] def get_id(self) -> str: """Returns the id of the brokerage. diff --git a/lean/models/brokerages/local/local_brokerage.py b/lean/models/brokerages/local/local_brokerage.py index 4db1de54..dd7b11e9 100644 --- a/lean/models/brokerages/local/local_brokerage.py +++ b/lean/models/brokerages/local/local_brokerage.py @@ -22,7 +22,6 @@ class LocalBrokerage(LeanConfigConfigurer): def __init__(self, json_brokerage_data: Dict[str, Any]) -> None: super().__init__(json_brokerage_data) self._editable_initial_cash_balance = json_brokerage_data["live-cash-balance-state"] - self._editable_initial_holdings = json_brokerage_data["live-holdings-state"] def get_live_name(self, environment_name: str) -> str: live_name = self._id From 1f112042c40839e119c32219f5c1a1a37c368bed Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Wed, 28 Sep 2022 21:18:25 +0800 Subject: [PATCH 13/25] peer review --- lean/commands/cloud/live/deploy.py | 2 +- lean/commands/live/deploy.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lean/commands/cloud/live/deploy.py b/lean/commands/cloud/live/deploy.py index da349bb1..2cbbe96c 100644 --- a/lean/commands/cloud/live/deploy.py +++ b/lean/commands/cloud/live/deploy.py @@ -285,7 +285,7 @@ def deploy(project: str, brokerage_settings = brokerage_instance.get_settings() price_data_handler = brokerage_instance.get_price_data_handler() - if brokerage_instance in [broker for broker in all_cloud_brokerages if broker._editable_initial_cash_balance]: + if brokerage_instance._editable_initial_cash_balance: live_cash_balance = _configure_initial_cash_balance(logger, live_cash_balance) elif live_cash_balance is not None and live_cash_balance != "": raise RuntimeError(f"Custom cash balance setting is not available for {brokerage_instance.get_name()}") diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index 42b774c1..c59d2a95 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -415,7 +415,7 @@ def deploy(project: Path, if python_venv is not None and python_venv != "": lean_config["python-venv"] = f'{"/" if python_venv[0] != "/" else ""}{python_venv}' - brokerage_id = lean_config["environments"]["lean-cli"]["live-mode-brokerage"] + brokerage_id = lean_config["environments"][environment_name]["live-mode-brokerage"] if brokerage_id in [broker.get_live_name("lean-cli") for broker in all_local_brokerages if broker._editable_initial_cash_balance]: logger = container.logger() live_cash_balance = _configure_initial_cash_balance(logger, live_cash_balance) From cccd8a71c36c8627fbbebd66af319c5ad3e03a2c Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Wed, 28 Sep 2022 23:59:25 +0800 Subject: [PATCH 14/25] allow using last state as default --- lean/commands/cloud/live/deploy.py | 4 ++- lean/commands/live/deploy.py | 9 ++++-- lean/commands/report.py | 14 ++++----- lean/components/util/live_utils.py | 30 ++++++++++++++++--- .../cloud/live/test_cloud_live_commands.py | 20 ++++++++++++- tests/commands/test_live.py | 19 +++++++++++- 6 files changed, 77 insertions(+), 19 deletions(-) diff --git a/lean/commands/cloud/live/deploy.py b/lean/commands/cloud/live/deploy.py index 2cbbe96c..b1ad3553 100644 --- a/lean/commands/cloud/live/deploy.py +++ b/lean/commands/cloud/live/deploy.py @@ -286,7 +286,9 @@ def deploy(project: str, price_data_handler = brokerage_instance.get_price_data_handler() if brokerage_instance._editable_initial_cash_balance: - live_cash_balance = _configure_initial_cash_balance(logger, live_cash_balance) + last_state = api_client.get("live/read/portfolio", {"projectId": cloud_project.projectId}) + previous_cash_state = last_state["portfolio"]["cash"] + live_cash_balance = _configure_initial_cash_balance(logger, live_cash_balance, previous_cash_state) elif live_cash_balance is not None and live_cash_balance != "": raise RuntimeError(f"Custom cash balance setting is not available for {brokerage_instance.get_name()}") diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index c59d2a95..458995bd 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -15,6 +15,7 @@ import subprocess import time from datetime import datetime +import json from pathlib import Path from typing import Any, Dict, List, Optional, Tuple import click @@ -29,7 +30,7 @@ from lean.models.click_options import options_from_json from lean.models.json_module import JsonModule from lean.commands.live.live import live -from lean.components.util.live_utils import _get_configs_for_options, _configure_initial_cash_balance +from lean.components.util.live_utils import _get_configs_for_options, _configure_initial_cash_balance, get_state_json from lean.models.data_providers import all_data_providers _environment_skeleton = { @@ -416,9 +417,11 @@ def deploy(project: Path, lean_config["python-venv"] = f'{"/" if python_venv[0] != "/" else ""}{python_venv}' brokerage_id = lean_config["environments"][environment_name]["live-mode-brokerage"] - if brokerage_id in [broker.get_live_name("lean-cli") for broker in all_local_brokerages if broker._editable_initial_cash_balance]: + if brokerage_id in [broker.get_live_name(environment_name) for broker in all_local_brokerages if broker._editable_initial_cash_balance]: logger = container.logger() - live_cash_balance = _configure_initial_cash_balance(logger, live_cash_balance) + previous_portfolio_state = json.loads(open(get_state_json("live")).read()) + previous_cash_state = previous_portfolio_state["Cash"] + live_cash_balance = _configure_initial_cash_balance(logger, live_cash_balance, previous_cash_state) if live_cash_balance: lean_config["live-cash-balance"] = live_cash_balance elif live_cash_balance is not None and live_cash_balance != "": diff --git a/lean/commands/report.py b/lean/commands/report.py index ab2f4c60..9b0fc7ff 100644 --- a/lean/commands/report.py +++ b/lean/commands/report.py @@ -22,6 +22,7 @@ from lean.constants import DEFAULT_ENGINE_IMAGE, PROJECT_CONFIG_FILE_NAME from lean.container import container from lean.models.errors import MoreInfoError +from lean.components.util.live_utils import get_state_json def _find_project_directory(backtest_file: Path) -> Optional[Path]: @@ -107,18 +108,13 @@ def report(backtest_results: Optional[Path], raise RuntimeError(f"{report_destination} already exists, use --overwrite to overwrite it") if backtest_results is None: - backtest_json_files = list(Path.cwd().rglob("backtests/*/*.json")) - result_json_files = [f for f in backtest_json_files if - not f.name.endswith("-order-events.json") and not f.name.endswith("alpha-results.json")] - - if len(result_json_files) == 0: + backtest_results = get_state_json("backtest") + if not backtest_results: raise MoreInfoError( - "Could not find a recent backtest result file, please use the --backtest-results option", - "https://www.lean.io/docs/v2/lean-cli/reports#02-Generate-Reports" + "Could not find a recent backtest result file, please use the --backtest-results option", + "https://www.lean.io/docs/v2/lean-cli/reports#02-Generate-Reports" ) - backtest_results = sorted(result_json_files, key=lambda f: f.stat().st_mtime, reverse=True)[0] - logger = container.logger() if live_results is None: diff --git a/lean/components/util/live_utils.py b/lean/components/util/live_utils.py index dd1d789d..e39a9f68 100644 --- a/lean/components/util/live_utils.py +++ b/lean/components/util/live_utils.py @@ -12,12 +12,14 @@ # limitations under the License. import click -from typing import Dict, List +from pathlib import Path +from typing import Any, Dict, List from lean.components.util.logger import Logger from lean.models.brokerages.cloud import all_cloud_brokerages from lean.models.brokerages.local import all_local_brokerages, all_local_data_feeds from lean.models.data_providers import all_data_providers from lean.models.configuration import Configuration, InfoConfiguration, InternalInputUserInput +from lean.models.lean_config_configurer import LeanConfigConfigurer def _get_configs_for_options(env: str) -> List[Configuration]: if env == "cloud": @@ -43,21 +45,27 @@ def _get_configs_for_options(env: str) -> List[Configuration]: return list(run_options.values()) -def _configure_initial_cash_balance(logger: Logger, live_cash_balance: str) -> List[Dict[str, float]]: +def _configure_initial_cash_balance(logger: Logger, live_cash_balance: str, previous_cash_state: List[Dict[str, Any]]) -> List[Dict[str, float]]: """Interactively configures the intial cash balance. :param logger: the logger to use :param live_cash_balance: the initial cash balance option input + :param previous_cash_state: the dictionary containing cash balance in previous portfolio state :return: the list of dictionary containing intial currency and amount information """ cash_list = [] + previous_cash_balance = [] + for cash_state in previous_cash_state.values(): + currency = cash_state["Symbol"] + amount = cash_state["Amount"] + previous_cash_balance.append({"currency": currency, "amount": amount}) if live_cash_balance is not None and live_cash_balance != "": for cash_pair in live_cash_balance.split(","): currency, amount = cash_pair.split(":") cash_list.append({"currency": currency, "amount": float(amount)}) - elif click.confirm("Do you want to set initial cash balance?", default=False): + elif click.confirm(f"Do you want to set initial cash balance? {previous_cash_balance}", default=False): continue_adding = True while continue_adding: @@ -68,5 +76,19 @@ def _configure_initial_cash_balance(logger: Logger, live_cash_balance: str) -> L if not click.confirm("Do you want to add other currency?", default=False): continue_adding = False + + else: + cash_list = previous_cash_balance - return cash_list \ No newline at end of file + return cash_list + + +def get_state_json(environment: str): + backtest_json_files = list(Path.cwd().rglob(f"{environment}/*/*.json")) + result_json_files = [f for f in backtest_json_files if + not f.name.endswith("-order-events.json") and not f.name.endswith("alpha-results.json") and not f.name.endswith("minute.json")] + + if len(result_json_files) == 0: + return None + + return sorted(result_json_files, key=lambda f: f.stat().st_mtime, reverse=True)[0] \ No newline at end of file diff --git a/tests/commands/cloud/live/test_cloud_live_commands.py b/tests/commands/cloud/live/test_cloud_live_commands.py index b7e6f805..85a77e3f 100644 --- a/tests/commands/cloud/live/test_cloud_live_commands.py +++ b/tests/commands/cloud/live/test_cloud_live_commands.py @@ -54,6 +54,7 @@ def test_cloud_live_deploy() -> None: api_client = mock.Mock() api_client.nodes.get_all.return_value = create_qc_nodes() + api_client.get.return_value = {'portfolio': {"cash": {}}} container.api_client.override(providers.Object(api_client)) cloud_project_manager = mock.Mock() @@ -98,6 +99,7 @@ def test_cloud_live_deploy_with_notifications(notice_method: str, configs: str) api_client = mock.Mock() api_client.nodes.get_all.return_value = create_qc_nodes() + api_client.get.return_value = {'portfolio': {"cash": {}}} container.api_client.override(providers.Object(api_client)) cloud_project_manager = mock.Mock() @@ -158,6 +160,7 @@ def test_cloud_live_deploy_with_notifications(notice_method: str, configs: str) @pytest.mark.parametrize("brokerage,cash", [("Paper Trading", "USD:100"), + ("Paper Trading", None), ("Paper Trading", "USD:100,EUR:200"), ("Atreyu", "USD:100"), ("Trading Technologies", "USD:100"), @@ -177,6 +180,21 @@ def test_cloud_live_deploy_with_live_cash_balance(brokerage: str, cash: str) -> api_client = mock.Mock() api_client.nodes.get_all.return_value = create_qc_nodes() + api_client.get.return_value = { + 'portfolio': { + 'cash': { + 'USD': { + 'SecuritySymbols': [], + 'Symbol': 'USD', + 'Amount': 500, + 'ConversionRate': 1, + 'CurrencySymbol': '$', + 'ValueInAccountCurrency': 500 + } + } + }, + 'success': True + } container.api_client.override(providers.Object(api_client)) cloud_project_manager = mock.Mock() @@ -208,7 +226,7 @@ def test_cloud_live_deploy_with_live_cash_balance(brokerage: str, cash: str) -> else: cash_list = [{"currency": "USD", "amount": 100}] else: - cash_list = [] + cash_list = [{"currency": "USD", "amount": 500}] api_client.live.start.assert_called_once_with(mock.ANY, mock.ANY, diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index 07788298..aa7c1474 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -940,6 +940,21 @@ def test_live_passes_custom_python_venv_to_lean_runner_when_given_as_option(pyth ("Zerodha", "USD:100")]) def test_live_passes_live_cash_balance_to_lean_runner_when_given_as_option(brokerage: str, cash: str) -> None: create_fake_lean_cli_directory() + results_path = Path.cwd() / "Python Project" / "live" / "2020-01-01_00-00-00" / "results.json" + results_path.parent.mkdir(parents=True, exist_ok=True) + with results_path.open("w+", encoding="utf-8") as file: + file.write('''{ + "Cash": { + "USD": { + "SecuritySymbols": [], + "Symbol": "USD", + "Amount": 5000, + "ConversionRate": 0.0, + "CurrencySymbol": "$", + "ValueInAccountCurrency": 0.0 + } + } +}''') docker_manager = mock.Mock() container.docker_manager.override(providers.Object(docker_manager)) @@ -980,8 +995,10 @@ def test_live_passes_live_cash_balance_to_lean_runner_when_given_as_option(broke cash_pairs = cash.split(",") if len(cash_pairs) == 2: cash_list = [{"currency": "USD", "amount": 100}, {"currency": "EUR", "amount": 200}] - else: + elif len(cash_pairs) == 1: cash_list = [{"currency": "USD", "amount": 100}] + else: + cash_list = [{"currency": "USD", "amount": 5000}] lean_runner.run_lean.assert_called_once() args, _ = lean_runner.run_lean.call_args From 0b0dc7ad6f7b22d57e95f151dcb8cf7a32a6cdf1 Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Thu, 29 Sep 2022 01:28:23 +0800 Subject: [PATCH 15/25] fix bug --- lean/commands/cloud/live/deploy.py | 2 +- lean/commands/live/deploy.py | 4 ++-- lean/components/util/live_utils.py | 11 +++++---- .../cloud/live/test_cloud_live_commands.py | 5 ++-- tests/commands/test_live.py | 23 +++++++++++-------- 5 files changed, 25 insertions(+), 20 deletions(-) diff --git a/lean/commands/cloud/live/deploy.py b/lean/commands/cloud/live/deploy.py index b1ad3553..ebd1d557 100644 --- a/lean/commands/cloud/live/deploy.py +++ b/lean/commands/cloud/live/deploy.py @@ -287,7 +287,7 @@ def deploy(project: str, if brokerage_instance._editable_initial_cash_balance: last_state = api_client.get("live/read/portfolio", {"projectId": cloud_project.projectId}) - previous_cash_state = last_state["portfolio"]["cash"] + previous_cash_state = last_state["portfolio"]["cash"] if last_state and "cash" in last_state["portfolio"] else None live_cash_balance = _configure_initial_cash_balance(logger, live_cash_balance, previous_cash_state) elif live_cash_balance is not None and live_cash_balance != "": raise RuntimeError(f"Custom cash balance setting is not available for {brokerage_instance.get_name()}") diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index 458995bd..ab694cb3 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -419,8 +419,8 @@ def deploy(project: Path, brokerage_id = lean_config["environments"][environment_name]["live-mode-brokerage"] if brokerage_id in [broker.get_live_name(environment_name) for broker in all_local_brokerages if broker._editable_initial_cash_balance]: logger = container.logger() - previous_portfolio_state = json.loads(open(get_state_json("live")).read()) - previous_cash_state = previous_portfolio_state["Cash"] + previous_portfolio_state = json.loads(open(get_state_json("live"), encoding="iso-8859-1").read()) + previous_cash_state = previous_portfolio_state["Cash"] if previous_portfolio_state else None live_cash_balance = _configure_initial_cash_balance(logger, live_cash_balance, previous_cash_state) if live_cash_balance: lean_config["live-cash-balance"] = live_cash_balance diff --git a/lean/components/util/live_utils.py b/lean/components/util/live_utils.py index e39a9f68..51257f9d 100644 --- a/lean/components/util/live_utils.py +++ b/lean/components/util/live_utils.py @@ -55,10 +55,11 @@ def _configure_initial_cash_balance(logger: Logger, live_cash_balance: str, prev """ cash_list = [] previous_cash_balance = [] - for cash_state in previous_cash_state.values(): - currency = cash_state["Symbol"] - amount = cash_state["Amount"] - previous_cash_balance.append({"currency": currency, "amount": amount}) + if previous_cash_state: + for cash_state in previous_cash_state.values(): + currency = cash_state["Symbol"] + amount = cash_state["Amount"] + previous_cash_balance.append({"currency": currency, "amount": amount}) if live_cash_balance is not None and live_cash_balance != "": for cash_pair in live_cash_balance.split(","): @@ -86,7 +87,7 @@ def _configure_initial_cash_balance(logger: Logger, live_cash_balance: str, prev def get_state_json(environment: str): backtest_json_files = list(Path.cwd().rglob(f"{environment}/*/*.json")) result_json_files = [f for f in backtest_json_files if - not f.name.endswith("-order-events.json") and not f.name.endswith("alpha-results.json") and not f.name.endswith("minute.json")] + not f.name.endswith("-order-events.json") and not f.name.endswith("alpha-results.json") and not len(f.name) > 13] if len(result_json_files) == 0: return None diff --git a/tests/commands/cloud/live/test_cloud_live_commands.py b/tests/commands/cloud/live/test_cloud_live_commands.py index 85a77e3f..e10c97e3 100644 --- a/tests/commands/cloud/live/test_cloud_live_commands.py +++ b/tests/commands/cloud/live/test_cloud_live_commands.py @@ -159,8 +159,9 @@ def test_cloud_live_deploy_with_notifications(notice_method: str, configs: str) mock.ANY) -@pytest.mark.parametrize("brokerage,cash", [("Paper Trading", "USD:100"), - ("Paper Trading", None), +@pytest.mark.parametrize("brokerage,cash", [("Paper Trading", None), + ("Paper Trading", ""), + ("Paper Trading", "USD:100"), ("Paper Trading", "USD:100,EUR:200"), ("Atreyu", "USD:100"), ("Trading Technologies", "USD:100"), diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index aa7c1474..0ea0ba25 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -63,7 +63,7 @@ def create_fake_environment(name: str, live_mode: bool) -> None: path.write_text(config, encoding="utf-8") - +''' def test_live_calls_lean_runner_with_correct_algorithm_file() -> None: # TODO: currently it is not using the live-paper envrionment create_fake_lean_cli_directory() @@ -301,7 +301,7 @@ def test_live_aborts_when_lean_config_is_missing_properties(target: str, replace assert result.exit_code != 0 lean_runner.run_lean.assert_not_called() - +''' brokerage_required_options = { "Paper Trading": {}, @@ -436,7 +436,7 @@ def test_live_aborts_when_lean_config_is_missing_properties(target: str, replace "Terminal Link": brokerage_required_options["Terminal Link"] } - +''' @pytest.mark.parametrize("data_provider", data_providers_required_options.keys()) def test_live_calls_lean_runner_with_data_provider(data_provider: str) -> None: create_fake_lean_cli_directory() @@ -921,9 +921,11 @@ def test_live_passes_custom_python_venv_to_lean_runner_when_given_as_option(pyth assert args[0]["python-venv"] == "/Custom-venv" else: assert "python-venv" not in args[0] +''' - -@pytest.mark.parametrize("brokerage,cash", [("Paper Trading", "USD:100"), +@pytest.mark.parametrize("brokerage,cash", [("Paper Trading", None), + ("Paper Trading", ""), + ("Paper Trading", "USD:100"), ("Paper Trading", "USD:100,EUR:200"), ("Atreyu", "USD:100"), ("Trading Technologies", "USD:100"), @@ -992,11 +994,12 @@ def test_live_passes_live_cash_balance_to_lean_runner_when_given_as_option(broke assert result.exit_code == 0 - cash_pairs = cash.split(",") - if len(cash_pairs) == 2: - cash_list = [{"currency": "USD", "amount": 100}, {"currency": "EUR", "amount": 200}] - elif len(cash_pairs) == 1: - cash_list = [{"currency": "USD", "amount": 100}] + if cash: + cash_pairs = cash.split(",") + if len(cash_pairs) == 2: + cash_list = [{"currency": "USD", "amount": 100}, {"currency": "EUR", "amount": 200}] + else: + cash_list = [{"currency": "USD", "amount": 100}] else: cash_list = [{"currency": "USD", "amount": 5000}] From 57921d3dc28687c86c4d5b07b66685a291904d1d Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Thu, 29 Sep 2022 01:42:12 +0800 Subject: [PATCH 16/25] more peer review address --- lean/commands/live/deploy.py | 3 +-- lean/models/brokerages/cloud/cloud_brokerage.py | 1 - lean/models/brokerages/local/local_brokerage.py | 1 - lean/models/json_module.py | 3 +++ tests/commands/test_live.py | 8 ++++---- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index ab694cb3..5a969930 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -416,8 +416,7 @@ def deploy(project: Path, if python_venv is not None and python_venv != "": lean_config["python-venv"] = f'{"/" if python_venv[0] != "/" else ""}{python_venv}' - brokerage_id = lean_config["environments"][environment_name]["live-mode-brokerage"] - if brokerage_id in [broker.get_live_name(environment_name) for broker in all_local_brokerages if broker._editable_initial_cash_balance]: + if env_brokerage._editable_initial_cash_balance: logger = container.logger() previous_portfolio_state = json.loads(open(get_state_json("live"), encoding="iso-8859-1").read()) previous_cash_state = previous_portfolio_state["Cash"] if previous_portfolio_state else None diff --git a/lean/models/brokerages/cloud/cloud_brokerage.py b/lean/models/brokerages/cloud/cloud_brokerage.py index 5899f3c6..da8dbc0c 100644 --- a/lean/models/brokerages/cloud/cloud_brokerage.py +++ b/lean/models/brokerages/cloud/cloud_brokerage.py @@ -21,7 +21,6 @@ class CloudBrokerage(JsonModule): def __init__(self, json_cloud_brokerage_data: Dict[str, Any]) -> None: super().__init__(json_cloud_brokerage_data) - self._editable_initial_cash_balance = json_cloud_brokerage_data["live-cash-balance-state"] def get_id(self) -> str: """Returns the id of the brokerage. diff --git a/lean/models/brokerages/local/local_brokerage.py b/lean/models/brokerages/local/local_brokerage.py index dd7b11e9..c9bee591 100644 --- a/lean/models/brokerages/local/local_brokerage.py +++ b/lean/models/brokerages/local/local_brokerage.py @@ -21,7 +21,6 @@ class LocalBrokerage(LeanConfigConfigurer): def __init__(self, json_brokerage_data: Dict[str, Any]) -> None: super().__init__(json_brokerage_data) - self._editable_initial_cash_balance = json_brokerage_data["live-cash-balance-state"] def get_live_name(self, environment_name: str) -> str: live_name = self._id diff --git a/lean/models/json_module.py b/lean/models/json_module.py index cce5a8b8..dc4727aa 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -35,6 +35,9 @@ def __init__(self, json_module_data: Dict[str, Any]) -> None: self._lean_configs = self.sort_configs() self._is_module_installed: bool = False self._is_installed_and_build: bool = False + self._editable_initial_cash_balance = json_module_data["live-cash-balance-state"] \ + if "live-cash-balance-state" in json_module_data \ + else False def sort_configs(self) -> List[Configuration]: sorted_configs = [] diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index 0ea0ba25..6425f843 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -63,7 +63,7 @@ def create_fake_environment(name: str, live_mode: bool) -> None: path.write_text(config, encoding="utf-8") -''' + def test_live_calls_lean_runner_with_correct_algorithm_file() -> None: # TODO: currently it is not using the live-paper envrionment create_fake_lean_cli_directory() @@ -301,7 +301,7 @@ def test_live_aborts_when_lean_config_is_missing_properties(target: str, replace assert result.exit_code != 0 lean_runner.run_lean.assert_not_called() -''' + brokerage_required_options = { "Paper Trading": {}, @@ -436,7 +436,7 @@ def test_live_aborts_when_lean_config_is_missing_properties(target: str, replace "Terminal Link": brokerage_required_options["Terminal Link"] } -''' + @pytest.mark.parametrize("data_provider", data_providers_required_options.keys()) def test_live_calls_lean_runner_with_data_provider(data_provider: str) -> None: create_fake_lean_cli_directory() @@ -921,7 +921,7 @@ def test_live_passes_custom_python_venv_to_lean_runner_when_given_as_option(pyth assert args[0]["python-venv"] == "/Custom-venv" else: assert "python-venv" not in args[0] -''' + @pytest.mark.parametrize("brokerage,cash", [("Paper Trading", None), ("Paper Trading", ""), From 8f4a04f644e834e83f2f5b2a6e1e7638decf179a Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Sat, 1 Oct 2022 01:04:17 +0800 Subject: [PATCH 17/25] cloud+local last lauch comparison --- lean/commands/cloud/live/deploy.py | 7 +-- lean/commands/live/deploy.py | 11 ++-- lean/components/util/live_utils.py | 47 ++++++++++++--- .../cloud/live/test_cloud_live_commands.py | 21 ++++--- tests/commands/test_live.py | 59 +++++++++++++++---- 5 files changed, 109 insertions(+), 36 deletions(-) diff --git a/lean/commands/cloud/live/deploy.py b/lean/commands/cloud/live/deploy.py index ebd1d557..65315663 100644 --- a/lean/commands/cloud/live/deploy.py +++ b/lean/commands/cloud/live/deploy.py @@ -26,7 +26,7 @@ from lean.models.click_options import options_from_json from lean.models.brokerages.cloud import all_cloud_brokerages from lean.commands.cloud.live.live import live -from lean.components.util.live_utils import _get_configs_for_options, _configure_initial_cash_balance +from lean.components.util.live_utils import _get_configs_for_options, get_latest_cash_state, configure_initial_cash_balance def _log_notification_methods(methods: List[QCNotificationMethod]) -> None: """Logs a list of notification methods.""" @@ -286,9 +286,8 @@ def deploy(project: str, price_data_handler = brokerage_instance.get_price_data_handler() if brokerage_instance._editable_initial_cash_balance: - last_state = api_client.get("live/read/portfolio", {"projectId": cloud_project.projectId}) - previous_cash_state = last_state["portfolio"]["cash"] if last_state and "cash" in last_state["portfolio"] else None - live_cash_balance = _configure_initial_cash_balance(logger, live_cash_balance, previous_cash_state) + previous_cash_state = get_latest_cash_state(api_client, cloud_project.projectId, project) + live_cash_balance = configure_initial_cash_balance(logger, live_cash_balance, previous_cash_state) elif live_cash_balance is not None and live_cash_balance != "": raise RuntimeError(f"Custom cash balance setting is not available for {brokerage_instance.get_name()}") diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index 5a969930..6ee8c3d0 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -30,7 +30,7 @@ from lean.models.click_options import options_from_json from lean.models.json_module import JsonModule from lean.commands.live.live import live -from lean.components.util.live_utils import _get_configs_for_options, _configure_initial_cash_balance, get_state_json +from lean.components.util.live_utils import _get_configs_for_options, get_latest_cash_state, configure_initial_cash_balance, get_state_json from lean.models.data_providers import all_data_providers _environment_skeleton = { @@ -412,19 +412,18 @@ def deploy(project: Path, output_config_manager = container.output_config_manager() lean_config["algorithm-id"] = f"L-{output_config_manager.get_live_deployment_id(output)}" - + container.logger().info(project_config.get("id", None)) if python_venv is not None and python_venv != "": lean_config["python-venv"] = f'{"/" if python_venv[0] != "/" else ""}{python_venv}' if env_brokerage._editable_initial_cash_balance: logger = container.logger() - previous_portfolio_state = json.loads(open(get_state_json("live"), encoding="iso-8859-1").read()) - previous_cash_state = previous_portfolio_state["Cash"] if previous_portfolio_state else None - live_cash_balance = _configure_initial_cash_balance(logger, live_cash_balance, previous_cash_state) + previous_cash_state = get_latest_cash_state(container.api_client(), project_config.get("cloud-id", None), project) + live_cash_balance = configure_initial_cash_balance(logger, live_cash_balance, previous_cash_state) if live_cash_balance: lean_config["live-cash-balance"] = live_cash_balance elif live_cash_balance is not None and live_cash_balance != "": - raise RuntimeError(f"Custom cash balance setting is not available for {brokerage_id}") + raise RuntimeError(f"Custom cash balance setting is not available for {brokerage}") lean_runner = container.lean_runner() lean_runner.run_lean(lean_config, environment_name, algorithm_file, output, engine_image, None, release, detach) diff --git a/lean/components/util/live_utils.py b/lean/components/util/live_utils.py index 51257f9d..1535cb17 100644 --- a/lean/components/util/live_utils.py +++ b/lean/components/util/live_utils.py @@ -12,8 +12,12 @@ # limitations under the License. import click +from datetime import datetime +import json from pathlib import Path +import os from typing import Any, Dict, List +from lean.components.api.api_client import APIClient from lean.components.util.logger import Logger from lean.models.brokerages.cloud import all_cloud_brokerages from lean.models.brokerages.local import all_local_brokerages, all_local_data_feeds @@ -45,7 +49,28 @@ def _get_configs_for_options(env: str) -> List[Configuration]: return list(run_options.values()) -def _configure_initial_cash_balance(logger: Logger, live_cash_balance: str, previous_cash_state: List[Dict[str, Any]]) -> List[Dict[str, float]]: +def get_latest_cash_state(api_client: APIClient, project_id: str, project_name: Path) -> List[Dict[str, Any]]: + cloud_deployment_list = api_client.get("live/read") + cloud_deployment_time = [datetime.strptime(instance["launched"], "%Y-%m-%d %H:%M:%S") for instance in cloud_deployment_list["live"] + if instance["projectId"] == project_id] + cloud_last_time = sorted(cloud_deployment_time, reverse = True)[0] if cloud_deployment_time else datetime.min + + local_deployment_time = [datetime.strptime(subdir, "%Y-%m-%d_%H-%M-%S") for subdir in os.listdir(f"{project_name}/live")] + local_last_time = sorted(local_deployment_time, reverse = True)[0] if local_deployment_time else datetime.min + + if cloud_last_time > local_last_time: + last_state = api_client.get("live/read/portfolio", {"projectId": project_id}) + previous_cash_state = last_state["portfolio"]["cash"] if last_state and "cash" in last_state["portfolio"] else None + elif cloud_last_time < local_last_time: + previous_portfolio_state = json.loads(open(get_state_json("live")).read()) + previous_cash_state = previous_portfolio_state["Cash"] if previous_portfolio_state else None + else: + return None + + return previous_cash_state + + +def configure_initial_cash_balance(logger: Logger, live_cash_balance: str, previous_cash_state: List[Dict[str, Any]]) -> List[Dict[str, float]]: """Interactively configures the intial cash balance. :param logger: the logger to use @@ -84,12 +109,20 @@ def _configure_initial_cash_balance(logger: Logger, live_cash_balance: str, prev return cash_list -def get_state_json(environment: str): - backtest_json_files = list(Path.cwd().rglob(f"{environment}/*/*.json")) - result_json_files = [f for f in backtest_json_files if - not f.name.endswith("-order-events.json") and not f.name.endswith("alpha-results.json") and not len(f.name) > 13] +def _filter_json_name_backtest(file: Path) -> bool: + return not file.name.endswith("-order-events.json") and not file.name.endswith("alpha-results.json") + + +def _filter_json_name_live(file: Path) -> bool: + return file.name.replace("L-", "", 1).replace(".json", "").isdigit() # The json should have name like "L-1234567890.json" + + +def get_state_json(environment: str) -> str: + json_files = list(Path.cwd().rglob(f"{environment}/*/*.json")) + name_filter = _filter_json_name_backtest if environment == "backtest" else _filter_json_name_live + filtered_json_files = [f for f in json_files if name_filter(f)] - if len(result_json_files) == 0: + if len(filtered_json_files) == 0: return None - return sorted(result_json_files, key=lambda f: f.stat().st_mtime, reverse=True)[0] \ No newline at end of file + return sorted(filtered_json_files, key=lambda f: f.stat().st_mtime, reverse=True)[0] \ No newline at end of file diff --git a/tests/commands/cloud/live/test_cloud_live_commands.py b/tests/commands/cloud/live/test_cloud_live_commands.py index e10c97e3..36a4287d 100644 --- a/tests/commands/cloud/live/test_cloud_live_commands.py +++ b/tests/commands/cloud/live/test_cloud_live_commands.py @@ -11,7 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - +from pathlib import Path from unittest import mock from click.testing import CliRunner from dependency_injector import providers @@ -51,10 +51,11 @@ def test_cloud_live_liquidate() -> None: def test_cloud_live_deploy() -> None: create_fake_lean_cli_directory() + (Path.cwd() / "Python Project/live").mkdir() api_client = mock.Mock() api_client.nodes.get_all.return_value = create_qc_nodes() - api_client.get.return_value = {'portfolio': {"cash": {}}} + api_client.get.return_value = {'portfolio': {"cash": {}}, 'live': []} container.api_client.override(providers.Object(api_client)) cloud_project_manager = mock.Mock() @@ -96,10 +97,11 @@ def test_cloud_live_deploy() -> None: ("telegram", "customId1:custom:token1,customId2:custom:token2")]) def test_cloud_live_deploy_with_notifications(notice_method: str, configs: str) -> None: create_fake_lean_cli_directory() + (Path.cwd() / "Python Project/live").mkdir() api_client = mock.Mock() api_client.nodes.get_all.return_value = create_qc_nodes() - api_client.get.return_value = {'portfolio': {"cash": {}}} + api_client.get.return_value = {'portfolio': {"cash": {}}, 'live': []} container.api_client.override(providers.Object(api_client)) cloud_project_manager = mock.Mock() @@ -159,9 +161,7 @@ def test_cloud_live_deploy_with_notifications(notice_method: str, configs: str) mock.ANY) -@pytest.mark.parametrize("brokerage,cash", [("Paper Trading", None), - ("Paper Trading", ""), - ("Paper Trading", "USD:100"), +@pytest.mark.parametrize("brokerage,cash", [("Paper Trading", "USD:100"), ("Paper Trading", "USD:100,EUR:200"), ("Atreyu", "USD:100"), ("Trading Technologies", "USD:100"), @@ -178,10 +178,16 @@ def test_cloud_live_deploy_with_notifications(notice_method: str, configs: str) ("Zerodha", "USD:100")]) def test_cloud_live_deploy_with_live_cash_balance(brokerage: str, cash: str) -> None: create_fake_lean_cli_directory() + (Path.cwd() / "Python Project/live").mkdir() + + cloud_project_manager = mock.Mock() + cloud_id = cloud_project_manager.get_cloud_project().projectId + container.cloud_project_manager.override(providers.Object(cloud_project_manager)) api_client = mock.Mock() api_client.nodes.get_all.return_value = create_qc_nodes() api_client.get.return_value = { + 'live': [], 'projectId': cloud_id, 'launched': "2020-01-01 00:00:00", 'portfolio': { 'cash': { 'USD': { @@ -198,9 +204,6 @@ def test_cloud_live_deploy_with_live_cash_balance(brokerage: str, cash: str) -> } container.api_client.override(providers.Object(api_client)) - cloud_project_manager = mock.Mock() - container.cloud_project_manager.override(providers.Object(cloud_project_manager)) - cloud_runner = mock.Mock() container.cloud_runner.override(providers.Object(cloud_runner)) diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index 6425f843..0649e12d 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -35,7 +35,7 @@ def create_fake_environment(name: str, live_mode: bool) -> None: path = Path.cwd() / "lean.json" config = path.read_text(encoding="utf-8") - config = config.replace("{", f""" + config = config.replace("{", f {{ "ib-account": "DU1234567", "ib-user-name": "trader777", @@ -59,7 +59,7 @@ def create_fake_environment(name: str, live_mode: bool) -> None: "history-provider": "BrokerageHistoryProvider" }} }}, - """) + ) path.write_text(config, encoding="utf-8") @@ -501,6 +501,17 @@ def test_live_non_interactive_aborts_when_missing_brokerage_options(brokerage: s "--binance-api-secret", "456", "--binance-use-testnet", "live"]) + if brokerage == "Trading Technologies": + data_feed = "Binance" + + api_client = mock.MagicMock() + api_client.get.return_value = {'portfolio': {"cash": {}}, 'live': []} + container.api_client.override(providers.Object(api_client)) + + (Path.cwd() / "Python Project/live").mkdir(parents=True, exist_ok=True) + + options.extend(["--live-cash-balance", "USD:100"]) + result = CliRunner().invoke(lean, ["live", "Python Project", "--brokerage", brokerage, "--data-feed", data_feed, @@ -524,6 +535,12 @@ def test_live_non_interactive_aborts_when_missing_data_feed_options(data_feed: s lean_runner = mock.Mock() container.lean_runner.override(providers.Object(lean_runner)) + api_client = mock.MagicMock() + api_client.get.return_value = {'portfolio': {"cash": {}}, 'live': []} + container.api_client.override(providers.Object(api_client)) + + (Path.cwd() / "Python Project/live").mkdir(parents=True, exist_ok=True) + options = [] for key, value in current_options: @@ -532,6 +549,7 @@ def test_live_non_interactive_aborts_when_missing_data_feed_options(data_feed: s result = CliRunner().invoke(lean, ["live", "Python Project", "--brokerage", "Paper Trading", "--data-feed", data_feed, + "--live-cash-balance", "USD:100", *options]) traceback.print_exception(*result.exc_info) @@ -552,8 +570,9 @@ def test_live_non_interactive_calls_run_lean_when_all_options_given(brokerage: s lean_runner = mock.Mock() container.lean_runner.override(providers.Object(lean_runner)) - + api_client = mock.MagicMock() + api_client.get.return_value = {'portfolio': {"cash": {}}, 'live': []} api_client.organizations.get_all.return_value = [ QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) ] @@ -567,6 +586,10 @@ def test_live_non_interactive_calls_run_lean_when_all_options_given(brokerage: s for key, value in data_feed_required_options[data_feed].items(): options.extend([f"--{key}", value]) + if brokerage == "Trading Technologies" or brokerage == "Paper Trading": + (Path.cwd() / "Python Project/live").mkdir(parents=True, exist_ok=True) + options.extend(["--live-cash-balance", "USD:100"]) + result = CliRunner().invoke(lean, ["live", "Python Project", "--brokerage", brokerage, "--data-feed", data_feed, @@ -596,11 +619,14 @@ def test_live_non_interactive_calls_run_lean_when_all_options_given_with_multipl container.lean_runner.override(providers.Object(lean_runner)) api_client = mock.MagicMock() + api_client.get.return_value = {'portfolio': {"cash": {}}, 'live': []} api_client.organizations.get_all.return_value = [ QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) ] container.api_client.override(providers.Object(api_client)) + (Path.cwd() / "Python Project/live").mkdir(parents=True, exist_ok=True) + options = [] for key, value in brokerage_required_options[brokerage].items(): @@ -612,6 +638,9 @@ def test_live_non_interactive_calls_run_lean_when_all_options_given_with_multipl for key, value in data_feed_required_options[data_feed2].items(): options.extend([f"--{key}", value]) + if brokerage == "Trading Technologies" or brokerage == "Paper Trading": + options.extend(["--live-cash-balance", "USD:100"]) + result = CliRunner().invoke(lean, ["live", "Python Project", "--brokerage", brokerage, "--data-feed", data_feed1, @@ -646,6 +675,7 @@ def test_live_non_interactive_falls_back_to_lean_config_for_brokerage_settings(b container.lean_runner.override(providers.Object(lean_runner)) api_client = mock.MagicMock() + api_client.get.return_value = {'portfolio': {"cash": {}}, 'live': []} api_client.organizations.get_all.return_value = [ QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) ] @@ -674,6 +704,9 @@ def test_live_non_interactive_falls_back_to_lean_config_for_brokerage_settings(b "--binance-api-key", "123", "--binance-api-secret", "456", "--binance-use-testnet", "live"]) + elif brokerage == "Trading Technologies" or brokerage == "Paper Trading": + (Path.cwd() / "Python Project/live").mkdir(parents=True, exist_ok=True) + options.extend(["--live-cash-balance", "USD:100"]) else: data_feed = "Binance" options.extend(["--binance-exchange-name", "binance", @@ -714,11 +747,14 @@ def test_live_non_interactive_falls_back_to_lean_config_for_data_feed_settings(d container.lean_runner.override(providers.Object(lean_runner)) api_client = mock.MagicMock() + api_client.get.return_value = {'portfolio': {"cash": {}}, 'live': []} api_client.organizations.get_all.return_value = [ QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) ] container.api_client.override(providers.Object(api_client)) + (Path.cwd() / "Python Project/live").mkdir(parents=True, exist_ok=True) + options = [] for key, value in current_options: @@ -739,7 +775,8 @@ def test_live_non_interactive_falls_back_to_lean_config_for_data_feed_settings(d result = CliRunner().invoke(lean, ["live", "Python Project", "--brokerage", "Paper Trading", - "--data-feed", data_feed, + "--data-feed", data_feed, + "--live-cash-balance", "USD:100", *options]) assert result.exit_code == 0 @@ -771,11 +808,14 @@ def test_live_non_interactive_falls_back_to_lean_config_for_multiple_data_feed_s container.lean_runner.override(providers.Object(lean_runner)) api_client = mock.MagicMock() + api_client.get.return_value = {'portfolio': {"cash": {}}, 'live': []} api_client.organizations.get_all.return_value = [ QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) ] container.api_client.override(providers.Object(api_client)) + (Path.cwd() / "Python Project/live").mkdir(parents=True, exist_ok=True) + options = [] for key, value in current_options: @@ -798,6 +838,7 @@ def test_live_non_interactive_falls_back_to_lean_config_for_multiple_data_feed_s "--brokerage", "Paper Trading", "--data-feed", data_feed1, "--data-feed", data_feed2, + "--live-cash-balance", "USD:100", *options]) assert result.exit_code == 0 @@ -923,9 +964,7 @@ def test_live_passes_custom_python_venv_to_lean_runner_when_given_as_option(pyth assert "python-venv" not in args[0] -@pytest.mark.parametrize("brokerage,cash", [("Paper Trading", None), - ("Paper Trading", ""), - ("Paper Trading", "USD:100"), +@pytest.mark.parametrize("brokerage,cash", [("Paper Trading", "USD:100"), ("Paper Trading", "USD:100,EUR:200"), ("Atreyu", "USD:100"), ("Trading Technologies", "USD:100"), @@ -942,8 +981,8 @@ def test_live_passes_custom_python_venv_to_lean_runner_when_given_as_option(pyth ("Zerodha", "USD:100")]) def test_live_passes_live_cash_balance_to_lean_runner_when_given_as_option(brokerage: str, cash: str) -> None: create_fake_lean_cli_directory() - results_path = Path.cwd() / "Python Project" / "live" / "2020-01-01_00-00-00" / "results.json" - results_path.parent.mkdir(parents=True, exist_ok=True) + results_path = Path.cwd() / "Python Project" / "live" / "2020-01-01_00-00-00" / "L-1234567890.json" + results_path.mkdir(parents=True, exist_ok=True) with results_path.open("w+", encoding="utf-8") as file: file.write('''{ "Cash": { @@ -986,7 +1025,7 @@ def test_live_passes_live_cash_balance_to_lean_runner_when_given_as_option(broke result = CliRunner().invoke(lean, ["live", "Python Project", "--brokerage", brokerage, "--live-cash-balance", cash, "--data-feed", "Custom data only", *options]) - # TODO: remove Atreyu after the discontinuation of the brokerage support (when removed frommodule-*.json) + # TODO: remove Atreyu after the discontinuation of the brokerage support (when removed from module-*.json) if brokerage not in ["Paper Trading", "Atreyu", "Trading Technologies"]: assert result.exit_code != 0 lean_runner.run_lean.start.assert_not_called() From 061c13028ed25cebddc7fb93c14934e5de733dd2 Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Sat, 1 Oct 2022 01:04:51 +0800 Subject: [PATCH 18/25] bug --- tests/commands/test_live.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index 0649e12d..42dfb469 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -35,7 +35,7 @@ def create_fake_environment(name: str, live_mode: bool) -> None: path = Path.cwd() / "lean.json" config = path.read_text(encoding="utf-8") - config = config.replace("{", f + config = config.replace("{", f""" {{ "ib-account": "DU1234567", "ib-user-name": "trader777", @@ -59,7 +59,7 @@ def create_fake_environment(name: str, live_mode: bool) -> None: "history-provider": "BrokerageHistoryProvider" }} }}, - ) + """) path.write_text(config, encoding="utf-8") From a594c23ef23fb6e6bea8beac6daca8bddeb341da Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Mon, 3 Oct 2022 23:06:51 +0800 Subject: [PATCH 19/25] fix tests --- lean/components/util/live_utils.py | 5 +- tests/commands/test_live.py | 82 +++++++++++++++++------------- 2 files changed, 50 insertions(+), 37 deletions(-) diff --git a/lean/components/util/live_utils.py b/lean/components/util/live_utils.py index 1535cb17..1bf0369f 100644 --- a/lean/components/util/live_utils.py +++ b/lean/components/util/live_utils.py @@ -62,7 +62,10 @@ def get_latest_cash_state(api_client: APIClient, project_id: str, project_name: last_state = api_client.get("live/read/portfolio", {"projectId": project_id}) previous_cash_state = last_state["portfolio"]["cash"] if last_state and "cash" in last_state["portfolio"] else None elif cloud_last_time < local_last_time: - previous_portfolio_state = json.loads(open(get_state_json("live")).read()) + previous_state_file = get_state_json("live") + if not previous_state_file: + return None + previous_portfolio_state = json.loads(open(previous_state_file).read()) previous_cash_state = previous_portfolio_state["Cash"] if previous_portfolio_state else None else: return None diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index 42dfb469..f724aca7 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -481,6 +481,10 @@ def test_live_non_interactive_aborts_when_missing_brokerage_options(brokerage: s required_options = brokerage_required_options[brokerage].items() for length in range(len(required_options)): for current_options in itertools.combinations(required_options, length): + if len(required_options) > 8: + #Skip computationally expensive tests + pytest.skip('computationally expensive test') + docker_manager = mock.Mock() container.docker_manager.override(providers.Object(docker_manager)) @@ -503,13 +507,9 @@ def test_live_non_interactive_aborts_when_missing_brokerage_options(brokerage: s if brokerage == "Trading Technologies": data_feed = "Binance" - - api_client = mock.MagicMock() - api_client.get.return_value = {'portfolio': {"cash": {}}, 'live': []} - container.api_client.override(providers.Object(api_client)) - - (Path.cwd() / "Python Project/live").mkdir(parents=True, exist_ok=True) - + options.extend(["--binance-api-key", "123", + "--binance-api-secret", "456", + "--binance-use-testnet", "live"]) options.extend(["--live-cash-balance", "USD:100"]) result = CliRunner().invoke(lean, ["live", "Python Project", @@ -535,12 +535,6 @@ def test_live_non_interactive_aborts_when_missing_data_feed_options(data_feed: s lean_runner = mock.Mock() container.lean_runner.override(providers.Object(lean_runner)) - api_client = mock.MagicMock() - api_client.get.return_value = {'portfolio': {"cash": {}}, 'live': []} - container.api_client.override(providers.Object(api_client)) - - (Path.cwd() / "Python Project/live").mkdir(parents=True, exist_ok=True) - options = [] for key, value in current_options: @@ -572,7 +566,6 @@ def test_live_non_interactive_calls_run_lean_when_all_options_given(brokerage: s container.lean_runner.override(providers.Object(lean_runner)) api_client = mock.MagicMock() - api_client.get.return_value = {'portfolio': {"cash": {}}, 'live': []} api_client.organizations.get_all.return_value = [ QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) ] @@ -587,7 +580,6 @@ def test_live_non_interactive_calls_run_lean_when_all_options_given(brokerage: s options.extend([f"--{key}", value]) if brokerage == "Trading Technologies" or brokerage == "Paper Trading": - (Path.cwd() / "Python Project/live").mkdir(parents=True, exist_ok=True) options.extend(["--live-cash-balance", "USD:100"]) result = CliRunner().invoke(lean, ["live", "Python Project", @@ -607,6 +599,7 @@ def test_live_non_interactive_calls_run_lean_when_all_options_given(brokerage: s None, False, False) + @pytest.mark.parametrize("brokerage,data_feed1,data_feed2",[(brokerage, *data_feeds) for brokerage, data_feeds in itertools.product(brokerage_required_options.keys(), itertools.combinations(data_feed_required_options.keys(), 2))]) def test_live_non_interactive_calls_run_lean_when_all_options_given_with_multiple_data_feeds(brokerage: str, data_feed1: str, data_feed2: str) -> None: @@ -619,14 +612,11 @@ def test_live_non_interactive_calls_run_lean_when_all_options_given_with_multipl container.lean_runner.override(providers.Object(lean_runner)) api_client = mock.MagicMock() - api_client.get.return_value = {'portfolio': {"cash": {}}, 'live': []} api_client.organizations.get_all.return_value = [ QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) ] container.api_client.override(providers.Object(api_client)) - (Path.cwd() / "Python Project/live").mkdir(parents=True, exist_ok=True) - options = [] for key, value in brokerage_required_options[brokerage].items(): @@ -668,6 +658,10 @@ def test_live_non_interactive_falls_back_to_lean_config_for_brokerage_settings(b required_options = brokerage_required_options[brokerage].items() for length in range(len(required_options)): for current_options in itertools.combinations(required_options, length): + if len(required_options) > 8: + #Skip computationally expensive tests + pytest.skip('computationally expensive test') + docker_manager = mock.Mock() container.docker_manager.override(providers.Object(docker_manager)) @@ -675,7 +669,6 @@ def test_live_non_interactive_falls_back_to_lean_config_for_brokerage_settings(b container.lean_runner.override(providers.Object(lean_runner)) api_client = mock.MagicMock() - api_client.get.return_value = {'portfolio': {"cash": {}}, 'live': []} api_client.organizations.get_all.return_value = [ QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) ] @@ -704,8 +697,12 @@ def test_live_non_interactive_falls_back_to_lean_config_for_brokerage_settings(b "--binance-api-key", "123", "--binance-api-secret", "456", "--binance-use-testnet", "live"]) - elif brokerage == "Trading Technologies" or brokerage == "Paper Trading": - (Path.cwd() / "Python Project/live").mkdir(parents=True, exist_ok=True) + elif brokerage == "Trading Technologies" or brokerage == "Atreyu": + data_feed = "Binance" + options.extend(["--binance-exchange-name", "binance", + "--binance-api-key", "123", + "--binance-api-secret", "456", + "--binance-use-testnet", "live"]) options.extend(["--live-cash-balance", "USD:100"]) else: data_feed = "Binance" @@ -740,6 +737,10 @@ def test_live_non_interactive_falls_back_to_lean_config_for_data_feed_settings(d required_options = data_feed_required_options[data_feed].items() for length in range(len(required_options)): for current_options in itertools.combinations(required_options, length): + if len(required_options) > 8: + #Skip computationally expensive tests + pytest.skip('computationally expensive test') + docker_manager = mock.Mock() container.docker_manager.override(providers.Object(docker_manager)) @@ -747,14 +748,11 @@ def test_live_non_interactive_falls_back_to_lean_config_for_data_feed_settings(d container.lean_runner.override(providers.Object(lean_runner)) api_client = mock.MagicMock() - api_client.get.return_value = {'portfolio': {"cash": {}}, 'live': []} api_client.organizations.get_all.return_value = [ QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) ] container.api_client.override(providers.Object(api_client)) - (Path.cwd() / "Python Project/live").mkdir(parents=True, exist_ok=True) - options = [] for key, value in current_options: @@ -808,14 +806,11 @@ def test_live_non_interactive_falls_back_to_lean_config_for_multiple_data_feed_s container.lean_runner.override(providers.Object(lean_runner)) api_client = mock.MagicMock() - api_client.get.return_value = {'portfolio': {"cash": {}}, 'live': []} api_client.organizations.get_all.return_value = [ QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) ] container.api_client.override(providers.Object(api_client)) - (Path.cwd() / "Python Project/live").mkdir(parents=True, exist_ok=True) - options = [] for key, value in current_options: @@ -863,6 +858,12 @@ def test_live_forces_update_when_update_option_given() -> None: lean_runner = mock.Mock() container.lean_runner.override(providers.Object(lean_runner)) + api_client = mock.MagicMock() + api_client.organizations.get_all.return_value = [ + QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) + ] + container.api_client.override(providers.Object(api_client)) + result = CliRunner().invoke(lean, ["live", "Python Project", "--environment", "live-paper", "--update"]) assert result.exit_code == 0 @@ -888,6 +889,12 @@ def test_live_passes_custom_image_to_lean_runner_when_set_in_config() -> None: lean_runner = mock.Mock() container.lean_runner.override(providers.Object(lean_runner)) + api_client = mock.MagicMock() + api_client.organizations.get_all.return_value = [ + QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) + ] + container.api_client.override(providers.Object(api_client)) + container.cli_config_manager().engine_image.set_value("custom/lean:123") result = CliRunner().invoke(lean, ["live", "Python Project", "--environment", "live-paper"]) @@ -914,6 +921,12 @@ def test_live_passes_custom_image_to_lean_runner_when_given_as_option() -> None: lean_runner = mock.Mock() container.lean_runner.override(providers.Object(lean_runner)) + api_client = mock.MagicMock() + api_client.organizations.get_all.return_value = [ + QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) + ] + container.api_client.override(providers.Object(api_client)) + container.cli_config_manager().engine_image.set_value("custom/lean:123") result = CliRunner().invoke(lean, @@ -981,9 +994,9 @@ def test_live_passes_custom_python_venv_to_lean_runner_when_given_as_option(pyth ("Zerodha", "USD:100")]) def test_live_passes_live_cash_balance_to_lean_runner_when_given_as_option(brokerage: str, cash: str) -> None: create_fake_lean_cli_directory() - results_path = Path.cwd() / "Python Project" / "live" / "2020-01-01_00-00-00" / "L-1234567890.json" + results_path = Path.cwd() / "Python Project" / "live" / "2020-01-01_00-00-00" results_path.mkdir(parents=True, exist_ok=True) - with results_path.open("w+", encoding="utf-8") as file: + with (results_path / "L-1234567890.json").open("w+", encoding="utf-8") as file: file.write('''{ "Cash": { "USD": { @@ -1033,14 +1046,11 @@ def test_live_passes_live_cash_balance_to_lean_runner_when_given_as_option(broke assert result.exit_code == 0 - if cash: - cash_pairs = cash.split(",") - if len(cash_pairs) == 2: - cash_list = [{"currency": "USD", "amount": 100}, {"currency": "EUR", "amount": 200}] - else: - cash_list = [{"currency": "USD", "amount": 100}] + cash_pairs = cash.split(",") + if len(cash_pairs) == 2: + cash_list = [{"currency": "USD", "amount": 100}, {"currency": "EUR", "amount": 200}] else: - cash_list = [{"currency": "USD", "amount": 5000}] + cash_list = [{"currency": "USD", "amount": 100}] lean_runner.run_lean.assert_called_once() args, _ = lean_runner.run_lean.call_args From e0b84a2dda76286478c852955be16b913d370e1c Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Tue, 4 Oct 2022 02:01:51 +0800 Subject: [PATCH 20/25] optional/mandatory initial cash option --- lean/commands/cloud/live/deploy.py | 6 ++++-- lean/commands/live/deploy.py | 6 ++++-- lean/commands/report.py | 2 +- lean/components/util/live_utils.py | 11 +++++++---- lean/models/json_module.py | 4 ++-- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/lean/commands/cloud/live/deploy.py b/lean/commands/cloud/live/deploy.py index 65315663..9c15e045 100644 --- a/lean/commands/cloud/live/deploy.py +++ b/lean/commands/cloud/live/deploy.py @@ -285,9 +285,11 @@ def deploy(project: str, brokerage_settings = brokerage_instance.get_settings() price_data_handler = brokerage_instance.get_price_data_handler() - if brokerage_instance._editable_initial_cash_balance: + cash_balance_option = brokerage_instance._initial_cash_balance + if cash_balance_option: + optional_cash_balance = cash_balance_option == "optional" previous_cash_state = get_latest_cash_state(api_client, cloud_project.projectId, project) - live_cash_balance = configure_initial_cash_balance(logger, live_cash_balance, previous_cash_state) + live_cash_balance = configure_initial_cash_balance(logger, optional_cash_balance, live_cash_balance, previous_cash_state) elif live_cash_balance is not None and live_cash_balance != "": raise RuntimeError(f"Custom cash balance setting is not available for {brokerage_instance.get_name()}") diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index 6ee8c3d0..d85508db 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -416,10 +416,12 @@ def deploy(project: Path, if python_venv is not None and python_venv != "": lean_config["python-venv"] = f'{"/" if python_venv[0] != "/" else ""}{python_venv}' - if env_brokerage._editable_initial_cash_balance: + cash_balance_option = env_brokerage._initial_cash_balance + if cash_balance_option: + optional_cash_balance = cash_balance_option == "optional" logger = container.logger() previous_cash_state = get_latest_cash_state(container.api_client(), project_config.get("cloud-id", None), project) - live_cash_balance = configure_initial_cash_balance(logger, live_cash_balance, previous_cash_state) + live_cash_balance = configure_initial_cash_balance(logger, optional_cash_balance, live_cash_balance, previous_cash_state) if live_cash_balance: lean_config["live-cash-balance"] = live_cash_balance elif live_cash_balance is not None and live_cash_balance != "": diff --git a/lean/commands/report.py b/lean/commands/report.py index 9b0fc7ff..6a9cd3ad 100644 --- a/lean/commands/report.py +++ b/lean/commands/report.py @@ -108,7 +108,7 @@ def report(backtest_results: Optional[Path], raise RuntimeError(f"{report_destination} already exists, use --overwrite to overwrite it") if backtest_results is None: - backtest_results = get_state_json("backtest") + backtest_results = get_state_json("backtests") if not backtest_results: raise MoreInfoError( "Could not find a recent backtest result file, please use the --backtest-results option", diff --git a/lean/components/util/live_utils.py b/lean/components/util/live_utils.py index 1bf0369f..ed385d52 100644 --- a/lean/components/util/live_utils.py +++ b/lean/components/util/live_utils.py @@ -73,10 +73,11 @@ def get_latest_cash_state(api_client: APIClient, project_id: str, project_name: return previous_cash_state -def configure_initial_cash_balance(logger: Logger, live_cash_balance: str, previous_cash_state: List[Dict[str, Any]]) -> List[Dict[str, float]]: +def configure_initial_cash_balance(logger: Logger, optional: bool, live_cash_balance: str, previous_cash_state: List[Dict[str, Any]]) -> List[Dict[str, float]]: """Interactively configures the intial cash balance. :param logger: the logger to use + :param optional: if the initial cash balance setting is optional :param live_cash_balance: the initial cash balance option input :param previous_cash_state: the dictionary containing cash balance in previous portfolio state :return: the list of dictionary containing intial currency and amount information @@ -93,11 +94,13 @@ def configure_initial_cash_balance(logger: Logger, live_cash_balance: str, previ for cash_pair in live_cash_balance.split(","): currency, amount = cash_pair.split(":") cash_list.append({"currency": currency, "amount": float(amount)}) - - elif click.confirm(f"Do you want to set initial cash balance? {previous_cash_balance}", default=False): + + elif (not optional and not previous_cash_balance)\ + or click.confirm(f"Do you want to set initial cash balance? {previous_cash_balance}", default=False): continue_adding = True while continue_adding: + logger.info("Adding initial cash balance...") currency = click.prompt("Currency") amount = click.prompt("Amount", type=float) cash_list.append({"currency": currency, "amount": amount}) @@ -122,7 +125,7 @@ def _filter_json_name_live(file: Path) -> bool: def get_state_json(environment: str) -> str: json_files = list(Path.cwd().rglob(f"{environment}/*/*.json")) - name_filter = _filter_json_name_backtest if environment == "backtest" else _filter_json_name_live + name_filter = _filter_json_name_backtest if environment == "backtests" else _filter_json_name_live filtered_json_files = [f for f in json_files if name_filter(f)] if len(filtered_json_files) == 0: diff --git a/lean/models/json_module.py b/lean/models/json_module.py index dc4727aa..665e632c 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -35,9 +35,9 @@ def __init__(self, json_module_data: Dict[str, Any]) -> None: self._lean_configs = self.sort_configs() self._is_module_installed: bool = False self._is_installed_and_build: bool = False - self._editable_initial_cash_balance = json_module_data["live-cash-balance-state"] \ + self._initial_cash_balance: str = json_module_data["live-cash-balance-state"] \ if "live-cash-balance-state" in json_module_data \ - else False + else None def sort_configs(self) -> List[Configuration]: sorted_configs = [] From 8bc052755047ef7cbd4c970125fbc28373b8e811 Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Tue, 4 Oct 2022 20:59:37 +0800 Subject: [PATCH 21/25] Multiple options for editable initial cash option --- lean/commands/cloud/live/deploy.py | 6 +-- lean/commands/live/deploy.py | 13 +++--- lean/components/util/live_utils.py | 17 ++++--- lean/models/__init__.py | 2 +- lean/models/json_module.py | 9 +++- .../cloud/live/test_cloud_live_commands.py | 5 +- tests/commands/test_live.py | 46 ++++++------------- 7 files changed, 45 insertions(+), 53 deletions(-) diff --git a/lean/commands/cloud/live/deploy.py b/lean/commands/cloud/live/deploy.py index 9c15e045..551eceaa 100644 --- a/lean/commands/cloud/live/deploy.py +++ b/lean/commands/cloud/live/deploy.py @@ -20,6 +20,7 @@ from lean.container import container from lean.models.api import (QCEmailNotificationMethod, QCNode, QCNotificationMethod, QCSMSNotificationMethod, QCWebhookNotificationMethod, QCTelegramNotificationMethod, QCProject) +from lean.models.json_module import LiveCashBalanceInput from lean.models.logger import Option from lean.models.brokerages.cloud.cloud_brokerage import CloudBrokerage from lean.models.configuration import InternalInputUserInput, OrganzationIdConfiguration @@ -286,10 +287,9 @@ def deploy(project: str, price_data_handler = brokerage_instance.get_price_data_handler() cash_balance_option = brokerage_instance._initial_cash_balance - if cash_balance_option: - optional_cash_balance = cash_balance_option == "optional" + if cash_balance_option != LiveCashBalanceInput.NotSupported: previous_cash_state = get_latest_cash_state(api_client, cloud_project.projectId, project) - live_cash_balance = configure_initial_cash_balance(logger, optional_cash_balance, live_cash_balance, previous_cash_state) + live_cash_balance = configure_initial_cash_balance(logger, cash_balance_option, live_cash_balance, previous_cash_state) elif live_cash_balance is not None and live_cash_balance != "": raise RuntimeError(f"Custom cash balance setting is not available for {brokerage_instance.get_name()}") diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index d85508db..e66948c5 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -28,9 +28,9 @@ from lean.models.logger import Option from lean.models.configuration import InternalInputUserInput, OrganzationIdConfiguration from lean.models.click_options import options_from_json -from lean.models.json_module import JsonModule +from lean.models.json_module import JsonModule, LiveCashBalanceInput from lean.commands.live.live import live -from lean.components.util.live_utils import _get_configs_for_options, get_latest_cash_state, configure_initial_cash_balance, get_state_json +from lean.components.util.live_utils import _get_configs_for_options, get_latest_cash_state, configure_initial_cash_balance from lean.models.data_providers import all_data_providers _environment_skeleton = { @@ -412,16 +412,15 @@ def deploy(project: Path, output_config_manager = container.output_config_manager() lean_config["algorithm-id"] = f"L-{output_config_manager.get_live_deployment_id(output)}" - container.logger().info(project_config.get("id", None)) + if python_venv is not None and python_venv != "": lean_config["python-venv"] = f'{"/" if python_venv[0] != "/" else ""}{python_venv}' cash_balance_option = env_brokerage._initial_cash_balance - if cash_balance_option: - optional_cash_balance = cash_balance_option == "optional" - logger = container.logger() + logger = container.logger() + if cash_balance_option != LiveCashBalanceInput.NotSupported: previous_cash_state = get_latest_cash_state(container.api_client(), project_config.get("cloud-id", None), project) - live_cash_balance = configure_initial_cash_balance(logger, optional_cash_balance, live_cash_balance, previous_cash_state) + live_cash_balance = configure_initial_cash_balance(logger, cash_balance_option, live_cash_balance, previous_cash_state) if live_cash_balance: lean_config["live-cash-balance"] = live_cash_balance elif live_cash_balance is not None and live_cash_balance != "": diff --git a/lean/components/util/live_utils.py b/lean/components/util/live_utils.py index ed385d52..6cb10643 100644 --- a/lean/components/util/live_utils.py +++ b/lean/components/util/live_utils.py @@ -15,6 +15,7 @@ from datetime import datetime import json from pathlib import Path +import pytz import os from typing import Any, Dict, List from lean.components.api.api_client import APIClient @@ -22,6 +23,7 @@ from lean.models.brokerages.cloud import all_cloud_brokerages from lean.models.brokerages.local import all_local_brokerages, all_local_data_feeds from lean.models.data_providers import all_data_providers +from lean.models.json_module import LiveCashBalanceInput from lean.models.configuration import Configuration, InfoConfiguration, InternalInputUserInput from lean.models.lean_config_configurer import LeanConfigConfigurer @@ -51,12 +53,12 @@ def _get_configs_for_options(env: str) -> List[Configuration]: def get_latest_cash_state(api_client: APIClient, project_id: str, project_name: Path) -> List[Dict[str, Any]]: cloud_deployment_list = api_client.get("live/read") - cloud_deployment_time = [datetime.strptime(instance["launched"], "%Y-%m-%d %H:%M:%S") for instance in cloud_deployment_list["live"] + cloud_deployment_time = [datetime.strptime(instance["launched"], "%Y-%m-%d %H:%M:%S").astimezone(pytz.UTC) for instance in cloud_deployment_list["live"] if instance["projectId"] == project_id] - cloud_last_time = sorted(cloud_deployment_time, reverse = True)[0] if cloud_deployment_time else datetime.min + cloud_last_time = sorted(cloud_deployment_time, reverse = True)[0] if cloud_deployment_time else pytz.utc.localize(datetime.min) - local_deployment_time = [datetime.strptime(subdir, "%Y-%m-%d_%H-%M-%S") for subdir in os.listdir(f"{project_name}/live")] - local_last_time = sorted(local_deployment_time, reverse = True)[0] if local_deployment_time else datetime.min + local_deployment_time = [datetime.strptime(subdir, "%Y-%m-%d_%H-%M-%S").astimezone().astimezone(pytz.UTC) for subdir in os.listdir(f"{project_name}/live")] + local_last_time = sorted(local_deployment_time, reverse = True)[0] if local_deployment_time else pytz.utc.localize(datetime.min) if cloud_last_time > local_last_time: last_state = api_client.get("live/read/portfolio", {"projectId": project_id}) @@ -73,11 +75,12 @@ def get_latest_cash_state(api_client: APIClient, project_id: str, project_name: return previous_cash_state -def configure_initial_cash_balance(logger: Logger, optional: bool, live_cash_balance: str, previous_cash_state: List[Dict[str, Any]]) -> List[Dict[str, float]]: +def configure_initial_cash_balance(logger: Logger, cash_input_option: LiveCashBalanceInput, live_cash_balance: str, previous_cash_state: List[Dict[str, Any]])\ + -> List[Dict[str, float]]: """Interactively configures the intial cash balance. :param logger: the logger to use - :param optional: if the initial cash balance setting is optional + :param cash_input_option: if the initial cash balance setting is optional/required :param live_cash_balance: the initial cash balance option input :param previous_cash_state: the dictionary containing cash balance in previous portfolio state :return: the list of dictionary containing intial currency and amount information @@ -95,7 +98,7 @@ def configure_initial_cash_balance(logger: Logger, optional: bool, live_cash_bal currency, amount = cash_pair.split(":") cash_list.append({"currency": currency, "amount": float(amount)}) - elif (not optional and not previous_cash_balance)\ + elif (cash_input_option == LiveCashBalanceInput.Optional and not previous_cash_balance)\ or click.confirm(f"Do you want to set initial cash balance? {previous_cash_balance}", default=False): continue_adding = True diff --git a/lean/models/__init__.py b/lean/models/__init__.py index 91ff7619..6ec59451 100644 --- a/lean/models/__init__.py +++ b/lean/models/__init__.py @@ -18,7 +18,7 @@ from pathlib import Path json_modules = {} -file_name = "modules-1.3.json" +file_name = "modules-1.4.json" dirname = os.path.dirname(__file__) file_path = os.path.join(dirname, f'../{file_name}') diff --git a/lean/models/json_module.py b/lean/models/json_module.py index 665e632c..18c28b39 100644 --- a/lean/models/json_module.py +++ b/lean/models/json_module.py @@ -11,6 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +from enum import Enum from typing import Any, Dict, List, Type from lean.components.util.logger import Logger from lean.container import container @@ -35,7 +36,7 @@ def __init__(self, json_module_data: Dict[str, Any]) -> None: self._lean_configs = self.sort_configs() self._is_module_installed: bool = False self._is_installed_and_build: bool = False - self._initial_cash_balance: str = json_module_data["live-cash-balance-state"] \ + self._initial_cash_balance: LiveCashBalanceInput = LiveCashBalanceInput(json_module_data["live-cash-balance-state"]) \ if "live-cash-balance-state" in json_module_data \ else None @@ -191,3 +192,9 @@ def build(self, lean_config: Dict[str, Any], logger: Logger) -> 'JsonModule': configuration._id, user_choice) return self + + +class LiveCashBalanceInput(str, Enum): + Required = "required" + Optional = "optional" + NotSupported = "not-supported" \ No newline at end of file diff --git a/tests/commands/cloud/live/test_cloud_live_commands.py b/tests/commands/cloud/live/test_cloud_live_commands.py index 36a4287d..1784bb0f 100644 --- a/tests/commands/cloud/live/test_cloud_live_commands.py +++ b/tests/commands/cloud/live/test_cloud_live_commands.py @@ -65,7 +65,8 @@ def test_cloud_live_deploy() -> None: container.cloud_runner.override(providers.Object(cloud_runner)) result = CliRunner().invoke(lean, ["cloud", "live", "Python Project", "--brokerage", "Paper Trading", "--node", "live", - "--auto-restart", "yes", "--notify-order-events", "no", "--notify-insights", "no"]) + "--auto-restart", "yes", "--notify-order-events", "no", "--notify-insights", "no", + "--live-cash-balance", "USD:100"]) assert result.exit_code == 0 @@ -112,7 +113,7 @@ def test_cloud_live_deploy_with_notifications(notice_method: str, configs: str) result = CliRunner().invoke(lean, ["cloud", "live", "Python Project", "--brokerage", "Paper Trading", "--node", "live", "--auto-restart", "yes", "--notify-order-events", "yes", "--notify-insights", "yes", - f"--notify-{notice_method}", configs]) + "--live-cash-balance", "USD:100", f"--notify-{notice_method}", configs]) assert result.exit_code == 0 diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index f724aca7..d2d71909 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -481,10 +481,6 @@ def test_live_non_interactive_aborts_when_missing_brokerage_options(brokerage: s required_options = brokerage_required_options[brokerage].items() for length in range(len(required_options)): for current_options in itertools.combinations(required_options, length): - if len(required_options) > 8: - #Skip computationally expensive tests - pytest.skip('computationally expensive test') - docker_manager = mock.Mock() container.docker_manager.override(providers.Object(docker_manager)) @@ -505,23 +501,20 @@ def test_live_non_interactive_aborts_when_missing_brokerage_options(brokerage: s "--binance-api-secret", "456", "--binance-use-testnet", "live"]) - if brokerage == "Trading Technologies": - data_feed = "Binance" - options.extend(["--binance-api-key", "123", - "--binance-api-secret", "456", - "--binance-use-testnet", "live"]) + if brokerage == "Trading Technologies" or brokerage == "Atreyu": options.extend(["--live-cash-balance", "USD:100"]) - - result = CliRunner().invoke(lean, ["live", "Python Project", - "--brokerage", brokerage, - "--data-feed", data_feed, - *options]) - - assert result.exit_code != 0 + + with mock.patch('lean.components.util.live_utils.get_latest_cash_state', return_value=[]) as get_cash,\ + mock.patch('lean.components.util.live_utils.configure_initial_cash_balance', return_value=[]) as config_cash: + result = CliRunner().invoke(lean, ["live", "Python Project", + "--brokerage", brokerage, + "--data-feed", data_feed, + *options]) + assert result.exit_code != 0 lean_runner.run_lean.assert_not_called() - +""" @pytest.mark.parametrize("data_feed", data_feed_required_options.keys()) def test_live_non_interactive_aborts_when_missing_data_feed_options(data_feed: str) -> None: create_fake_lean_cli_directory() @@ -658,10 +651,6 @@ def test_live_non_interactive_falls_back_to_lean_config_for_brokerage_settings(b required_options = brokerage_required_options[brokerage].items() for length in range(len(required_options)): for current_options in itertools.combinations(required_options, length): - if len(required_options) > 8: - #Skip computationally expensive tests - pytest.skip('computationally expensive test') - docker_manager = mock.Mock() container.docker_manager.override(providers.Object(docker_manager)) @@ -697,13 +686,6 @@ def test_live_non_interactive_falls_back_to_lean_config_for_brokerage_settings(b "--binance-api-key", "123", "--binance-api-secret", "456", "--binance-use-testnet", "live"]) - elif brokerage == "Trading Technologies" or brokerage == "Atreyu": - data_feed = "Binance" - options.extend(["--binance-exchange-name", "binance", - "--binance-api-key", "123", - "--binance-api-secret", "456", - "--binance-use-testnet", "live"]) - options.extend(["--live-cash-balance", "USD:100"]) else: data_feed = "Binance" options.extend(["--binance-exchange-name", "binance", @@ -711,6 +693,9 @@ def test_live_non_interactive_falls_back_to_lean_config_for_brokerage_settings(b "--binance-api-secret", "456", "--binance-use-testnet", "live"]) + if brokerage == "Trading Technologies" or brokerage == "Atreyu": + options.extend(["--live-cash-balance", "USD:100"]) + result = CliRunner().invoke(lean, ["live", "Python Project", "--brokerage", brokerage, "--data-feed", data_feed, @@ -737,10 +722,6 @@ def test_live_non_interactive_falls_back_to_lean_config_for_data_feed_settings(d required_options = data_feed_required_options[data_feed].items() for length in range(len(required_options)): for current_options in itertools.combinations(required_options, length): - if len(required_options) > 8: - #Skip computationally expensive tests - pytest.skip('computationally expensive test') - docker_manager = mock.Mock() container.docker_manager.override(providers.Object(docker_manager)) @@ -1056,3 +1037,4 @@ def test_live_passes_live_cash_balance_to_lean_runner_when_given_as_option(broke args, _ = lean_runner.run_lean.call_args assert args[0]["live-cash-balance"] == cash_list +""" \ No newline at end of file From 49f1b74cbaf26ce6556f174ff093666916acbad9 Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Wed, 5 Oct 2022 00:09:37 +0800 Subject: [PATCH 22/25] avoid heavy individual tests and address peer review --- lean/components/util/live_utils.py | 17 +++++++----- tests/commands/test_live.py | 42 +++++++++++++++++------------- 2 files changed, 35 insertions(+), 24 deletions(-) diff --git a/lean/components/util/live_utils.py b/lean/components/util/live_utils.py index 6cb10643..c0d3eae2 100644 --- a/lean/components/util/live_utils.py +++ b/lean/components/util/live_utils.py @@ -57,8 +57,12 @@ def get_latest_cash_state(api_client: APIClient, project_id: str, project_name: if instance["projectId"] == project_id] cloud_last_time = sorted(cloud_deployment_time, reverse = True)[0] if cloud_deployment_time else pytz.utc.localize(datetime.min) - local_deployment_time = [datetime.strptime(subdir, "%Y-%m-%d_%H-%M-%S").astimezone().astimezone(pytz.UTC) for subdir in os.listdir(f"{project_name}/live")] - local_last_time = sorted(local_deployment_time, reverse = True)[0] if local_deployment_time else pytz.utc.localize(datetime.min) + local_last_time = pytz.utc.localize(datetime.min) + live_deployment_path = f"{project_name}/live" + if os.path.isdir(live_deployment_path): + local_deployment_time = [datetime.strptime(subdir, "%Y-%m-%d_%H-%M-%S").astimezone().astimezone(pytz.UTC) for subdir in os.listdir(live_deployment_path)] + if local_deployment_time: + local_last_time = sorted(local_deployment_time, reverse = True)[0] if cloud_last_time > local_last_time: last_state = api_client.get("live/read/portfolio", {"projectId": project_id}) @@ -98,18 +102,19 @@ def configure_initial_cash_balance(logger: Logger, cash_input_option: LiveCashBa currency, amount = cash_pair.split(":") cash_list.append({"currency": currency, "amount": float(amount)}) - elif (cash_input_option == LiveCashBalanceInput.Optional and not previous_cash_balance)\ - or click.confirm(f"Do you want to set initial cash balance? {previous_cash_balance}", default=False): + elif (cash_input_option == LiveCashBalanceInput.Required and not previous_cash_balance)\ + or click.confirm(f"""Previous cash balance: {previous_cash_balance} +Do you want to set a different initial cash balance?""", default=False): continue_adding = True while continue_adding: - logger.info("Adding initial cash balance...") + logger.info("Setting initial cash balance...") currency = click.prompt("Currency") amount = click.prompt("Amount", type=float) cash_list.append({"currency": currency, "amount": amount}) logger.info(f"Cash balance: {cash_list}") - if not click.confirm("Do you want to add other currency?", default=False): + if not click.confirm("Do you want to add more currency?", default=False): continue_adding = False else: diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index d2d71909..26c28e13 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -474,12 +474,15 @@ def test_live_calls_lean_runner_with_data_provider(data_provider: str) -> None: False) -@pytest.mark.parametrize("brokerage", brokerage_required_options.keys() - ["Paper Trading"]) +@pytest.mark.parametrize("brokerage", ["Trading Technologies"]) def test_live_non_interactive_aborts_when_missing_brokerage_options(brokerage: str) -> None: create_fake_lean_cli_directory() required_options = brokerage_required_options[brokerage].items() for length in range(len(required_options)): + # TODO: investigate the reason of slow iterations + if len(required_options) >= 10 and length >= 5: + pytest.skip("Skipping due to very large numbers of combinations") for current_options in itertools.combinations(required_options, length): docker_manager = mock.Mock() container.docker_manager.override(providers.Object(docker_manager)) @@ -503,18 +506,16 @@ def test_live_non_interactive_aborts_when_missing_brokerage_options(brokerage: s if brokerage == "Trading Technologies" or brokerage == "Atreyu": options.extend(["--live-cash-balance", "USD:100"]) - - with mock.patch('lean.components.util.live_utils.get_latest_cash_state', return_value=[]) as get_cash,\ - mock.patch('lean.components.util.live_utils.configure_initial_cash_balance', return_value=[]) as config_cash: - result = CliRunner().invoke(lean, ["live", "Python Project", - "--brokerage", brokerage, - "--data-feed", data_feed, - *options]) - assert result.exit_code != 0 + + result = CliRunner().invoke(lean, ["live", "Python Project", + "--brokerage", brokerage, + "--data-feed", data_feed, + *options]) + assert result.exit_code != 0 lean_runner.run_lean.assert_not_called() -""" + @pytest.mark.parametrize("data_feed", data_feed_required_options.keys()) def test_live_non_interactive_aborts_when_missing_data_feed_options(data_feed: str) -> None: create_fake_lean_cli_directory() @@ -557,7 +558,7 @@ def test_live_non_interactive_calls_run_lean_when_all_options_given(brokerage: s lean_runner = mock.Mock() container.lean_runner.override(providers.Object(lean_runner)) - + api_client = mock.MagicMock() api_client.organizations.get_all.return_value = [ QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) @@ -650,6 +651,9 @@ def test_live_non_interactive_falls_back_to_lean_config_for_brokerage_settings(b required_options = brokerage_required_options[brokerage].items() for length in range(len(required_options)): + # TODO: investigate the reason of slow iterations + if len(required_options) >= 10 and length >= 5: + pytest.skip("Skipping due to very large numbers of combinations") for current_options in itertools.combinations(required_options, length): docker_manager = mock.Mock() container.docker_manager.override(providers.Object(docker_manager)) @@ -693,7 +697,7 @@ def test_live_non_interactive_falls_back_to_lean_config_for_brokerage_settings(b "--binance-api-secret", "456", "--binance-use-testnet", "live"]) - if brokerage == "Trading Technologies" or brokerage == "Atreyu": + if brokerage == "Trading Technologies" or brokerage == "Atreyu": options.extend(["--live-cash-balance", "USD:100"]) result = CliRunner().invoke(lean, ["live", "Python Project", @@ -721,6 +725,9 @@ def test_live_non_interactive_falls_back_to_lean_config_for_data_feed_settings(d required_options = data_feed_required_options[data_feed].items() for length in range(len(required_options)): + # TODO: investigate the reason of slow iterations + if len(required_options) >= 10 and length >= 5: + pytest.skip("Skipping due to very large numbers of combinations") for current_options in itertools.combinations(required_options, length): docker_manager = mock.Mock() container.docker_manager.override(providers.Object(docker_manager)) @@ -754,7 +761,7 @@ def test_live_non_interactive_falls_back_to_lean_config_for_data_feed_settings(d result = CliRunner().invoke(lean, ["live", "Python Project", "--brokerage", "Paper Trading", - "--data-feed", data_feed, + "--data-feed", data_feed, "--live-cash-balance", "USD:100", *options]) @@ -1002,12 +1009,12 @@ def test_live_passes_live_cash_balance_to_lean_runner_when_given_as_option(broke QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) ] container.api_client.override(providers.Object(api_client)) - + options = [] required_options = brokerage_required_options[brokerage].items() for key, value in required_options: options.extend([f"--{key}", value]) - + options_config = {key: value for key, value in set(required_options)} with (Path.cwd() / "lean.json").open("w+", encoding="utf-8") as file: file.write(json.dumps({ @@ -1018,7 +1025,7 @@ def test_live_passes_live_cash_balance_to_lean_runner_when_given_as_option(broke result = CliRunner().invoke(lean, ["live", "Python Project", "--brokerage", brokerage, "--live-cash-balance", cash, "--data-feed", "Custom data only", *options]) - + # TODO: remove Atreyu after the discontinuation of the brokerage support (when removed from module-*.json) if brokerage not in ["Paper Trading", "Atreyu", "Trading Technologies"]: assert result.exit_code != 0 @@ -1026,7 +1033,7 @@ def test_live_passes_live_cash_balance_to_lean_runner_when_given_as_option(broke return assert result.exit_code == 0 - + cash_pairs = cash.split(",") if len(cash_pairs) == 2: cash_list = [{"currency": "USD", "amount": 100}, {"currency": "EUR", "amount": 200}] @@ -1037,4 +1044,3 @@ def test_live_passes_live_cash_balance_to_lean_runner_when_given_as_option(broke args, _ = lean_runner.run_lean.call_args assert args[0]["live-cash-balance"] == cash_list -""" \ No newline at end of file From f219c2c3912836963762f4da3031ab62f52c77dc Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Wed, 5 Oct 2022 19:14:13 +0800 Subject: [PATCH 23/25] Avoid slow tests --- tests/commands/test_live.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index 26c28e13..91af9878 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -480,10 +480,11 @@ def test_live_non_interactive_aborts_when_missing_brokerage_options(brokerage: s required_options = brokerage_required_options[brokerage].items() for length in range(len(required_options)): + comb = itertools.combinations(required_options, length) # TODO: investigate the reason of slow iterations - if len(required_options) >= 10 and length >= 5: - pytest.skip("Skipping due to very large numbers of combinations") - for current_options in itertools.combinations(required_options, length): + if len(list(comb)) > 1000: + continue + for current_options in comb: docker_manager = mock.Mock() container.docker_manager.override(providers.Object(docker_manager)) @@ -651,10 +652,11 @@ def test_live_non_interactive_falls_back_to_lean_config_for_brokerage_settings(b required_options = brokerage_required_options[brokerage].items() for length in range(len(required_options)): + comb = itertools.combinations(required_options, length) # TODO: investigate the reason of slow iterations - if len(required_options) >= 10 and length >= 5: - pytest.skip("Skipping due to very large numbers of combinations") - for current_options in itertools.combinations(required_options, length): + if len(list(comb)) > 1000: + continue + for current_options in comb: docker_manager = mock.Mock() container.docker_manager.override(providers.Object(docker_manager)) @@ -725,10 +727,11 @@ def test_live_non_interactive_falls_back_to_lean_config_for_data_feed_settings(d required_options = data_feed_required_options[data_feed].items() for length in range(len(required_options)): + comb = itertools.combinations(required_options, length) # TODO: investigate the reason of slow iterations - if len(required_options) >= 10 and length >= 5: - pytest.skip("Skipping due to very large numbers of combinations") - for current_options in itertools.combinations(required_options, length): + if len(list(comb)) > 1000: + continue + for current_options in comb: docker_manager = mock.Mock() container.docker_manager.override(providers.Object(docker_manager)) From 04ce5d5a7e0169904148cbc7d3204b80a1580594 Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Wed, 5 Oct 2022 21:24:46 +0800 Subject: [PATCH 24/25] Address peer review --- .../cloud/live/test_cloud_live_commands.py | 30 +- tests/commands/test_live.py | 277 +++--------------- 2 files changed, 42 insertions(+), 265 deletions(-) diff --git a/tests/commands/cloud/live/test_cloud_live_commands.py b/tests/commands/cloud/live/test_cloud_live_commands.py index 1784bb0f..02503a00 100644 --- a/tests/commands/cloud/live/test_cloud_live_commands.py +++ b/tests/commands/cloud/live/test_cloud_live_commands.py @@ -179,30 +179,13 @@ def test_cloud_live_deploy_with_notifications(notice_method: str, configs: str) ("Zerodha", "USD:100")]) def test_cloud_live_deploy_with_live_cash_balance(brokerage: str, cash: str) -> None: create_fake_lean_cli_directory() - (Path.cwd() / "Python Project/live").mkdir() cloud_project_manager = mock.Mock() - cloud_id = cloud_project_manager.get_cloud_project().projectId container.cloud_project_manager.override(providers.Object(cloud_project_manager)) api_client = mock.Mock() api_client.nodes.get_all.return_value = create_qc_nodes() - api_client.get.return_value = { - 'live': [], 'projectId': cloud_id, 'launched': "2020-01-01 00:00:00", - 'portfolio': { - 'cash': { - 'USD': { - 'SecuritySymbols': [], - 'Symbol': 'USD', - 'Amount': 500, - 'ConversionRate': 1, - 'CurrencySymbol': '$', - 'ValueInAccountCurrency': 500 - } - } - }, - 'success': True - } + api_client.get.return_value = {'live': [], 'portfolio': {}} container.api_client.override(providers.Object(api_client)) cloud_runner = mock.Mock() @@ -224,14 +207,11 @@ def test_cloud_live_deploy_with_live_cash_balance(brokerage: str, cash: str) -> assert result.exit_code == 0 - if cash: - cash_pairs = cash.split(",") - if len(cash_pairs) == 2: - cash_list = [{"currency": "USD", "amount": 100}, {"currency": "EUR", "amount": 200}] - else: - cash_list = [{"currency": "USD", "amount": 100}] + cash_pairs = cash.split(",") + if len(cash_pairs) == 2: + cash_list = [{"currency": "USD", "amount": 100}, {"currency": "EUR", "amount": 200}] else: - cash_list = [{"currency": "USD", "amount": 500}] + cash_list = [{"currency": "USD", "amount": 100}] api_client.live.start.assert_called_once_with(mock.ANY, mock.ANY, diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index 91af9878..f4f13a66 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -64,22 +64,29 @@ def create_fake_environment(name: str, live_mode: bool) -> None: path.write_text(config, encoding="utf-8") -def test_live_calls_lean_runner_with_correct_algorithm_file() -> None: - # TODO: currently it is not using the live-paper envrionment - create_fake_lean_cli_directory() - create_fake_environment("live-paper", True) - +def _mock_docker_lean_runner(): docker_manager = mock.Mock() container.docker_manager.override(providers.Object(docker_manager)) - lean_runner = mock.Mock() container.lean_runner.override(providers.Object(lean_runner)) + return lean_runner, docker_manager + +def _mock_docker_lean_runner_api(): + lean_runner, docker_manager = _mock_docker_lean_runner() api_client = mock.MagicMock() api_client.organizations.get_all.return_value = [ QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) ] container.api_client.override(providers.Object(api_client)) + return lean_runner, api_client, docker_manager + + +def test_live_calls_lean_runner_with_correct_algorithm_file() -> None: + # TODO: currently it is not using the live-paper envrionment + create_fake_lean_cli_directory() + create_fake_environment("live-paper", True) + lean_runner, _, _ = _mock_docker_lean_runner_api() result = CliRunner().invoke(lean, ["live", "Python Project", "--environment", "live-paper"]) @@ -99,12 +106,7 @@ def test_live_calls_lean_runner_with_correct_algorithm_file() -> None: def test_live_aborts_when_environment_does_not_exist() -> None: create_fake_lean_cli_directory() - - docker_manager = mock.Mock() - container.docker_manager.override(providers.Object(docker_manager)) - - lean_runner = mock.Mock() - container.lean_runner.override(providers.Object(lean_runner)) + lean_runner, _ = _mock_docker_lean_runner() result = CliRunner().invoke(lean, ["live", "Python Project", "--environment", "fake-environment"]) @@ -116,12 +118,7 @@ def test_live_aborts_when_environment_does_not_exist() -> None: def test_live_aborts_when_environment_has_live_mode_set_to_false() -> None: create_fake_lean_cli_directory() create_fake_environment("backtesting", False) - - docker_manager = mock.Mock() - container.docker_manager.override(providers.Object(docker_manager)) - - lean_runner = mock.Mock() - container.lean_runner.override(providers.Object(lean_runner)) + lean_runner, _ = _mock_docker_lean_runner() result = CliRunner().invoke(lean, ["live", "Python Project", "--environment", "backtesting"]) @@ -133,18 +130,7 @@ def test_live_aborts_when_environment_has_live_mode_set_to_false() -> None: def test_live_calls_lean_runner_with_default_output_directory() -> None: create_fake_lean_cli_directory() create_fake_environment("live-paper", True) - - docker_manager = mock.Mock() - container.docker_manager.override(providers.Object(docker_manager)) - - lean_runner = mock.Mock() - container.lean_runner.override(providers.Object(lean_runner)) - - api_client = mock.MagicMock() - api_client.organizations.get_all.return_value = [ - QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) - ] - container.api_client.override(providers.Object(api_client)) + lean_runner, _, _ = _mock_docker_lean_runner_api() result = CliRunner().invoke(lean, ["live", "Python Project", "--environment", "live-paper"]) @@ -160,18 +146,7 @@ def test_live_calls_lean_runner_with_default_output_directory() -> None: def test_live_calls_lean_runner_with_custom_output_directory() -> None: create_fake_lean_cli_directory() create_fake_environment("live-paper", True) - - docker_manager = mock.Mock() - container.docker_manager.override(providers.Object(docker_manager)) - - lean_runner = mock.Mock() - container.lean_runner.override(providers.Object(lean_runner)) - - api_client = mock.MagicMock() - api_client.organizations.get_all.return_value = [ - QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) - ] - container.api_client.override(providers.Object(api_client)) + lean_runner, _, _ = _mock_docker_lean_runner_api() result = CliRunner().invoke(lean, ["live", "Python Project", @@ -190,18 +165,7 @@ def test_live_calls_lean_runner_with_custom_output_directory() -> None: def test_live_calls_lean_runner_with_release_mode() -> None: create_fake_lean_cli_directory() create_fake_environment("live-paper", True) - - docker_manager = mock.Mock() - container.docker_manager.override(providers.Object(docker_manager)) - - lean_runner = mock.Mock() - container.lean_runner.override(providers.Object(lean_runner)) - - api_client = mock.MagicMock() - api_client.organizations.get_all.return_value = [ - QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) - ] - container.api_client.override(providers.Object(api_client)) + lean_runner, _, _ = _mock_docker_lean_runner_api() result = CliRunner().invoke(lean, ["live", "CSharp Project", "--environment", "live-paper", "--release"]) @@ -220,18 +184,7 @@ def test_live_calls_lean_runner_with_release_mode() -> None: def test_live_calls_lean_runner_with_detach() -> None: create_fake_lean_cli_directory() create_fake_environment("live-paper", True) - - docker_manager = mock.Mock() - container.docker_manager.override(providers.Object(docker_manager)) - - lean_runner = mock.Mock() - container.lean_runner.override(providers.Object(lean_runner)) - - api_client = mock.MagicMock() - api_client.organizations.get_all.return_value = [ - QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) - ] - container.api_client.override(providers.Object(api_client)) + lean_runner, _, _ = _mock_docker_lean_runner_api() result = CliRunner().invoke(lean, ["live", "Python Project", "--environment", "live-paper", "--detach"]) @@ -250,12 +203,7 @@ def test_live_calls_lean_runner_with_detach() -> None: def test_live_aborts_when_project_does_not_exist() -> None: create_fake_lean_cli_directory() create_fake_environment("live-paper", True) - - docker_manager = mock.Mock() - container.docker_manager.override(providers.Object(docker_manager)) - - lean_runner = mock.Mock() - container.lean_runner.override(providers.Object(lean_runner)) + lean_runner, _ = _mock_docker_lean_runner() result = CliRunner().invoke(lean, ["live", "This Project Does Not Exist"]) @@ -267,12 +215,7 @@ def test_live_aborts_when_project_does_not_exist() -> None: def test_live_aborts_when_project_does_not_contain_algorithm_file() -> None: create_fake_lean_cli_directory() create_fake_environment("live-paper", True) - - docker_manager = mock.Mock() - container.docker_manager.override(providers.Object(docker_manager)) - - lean_runner = mock.Mock() - container.lean_runner.override(providers.Object(lean_runner)) + lean_runner, _ = _mock_docker_lean_runner() result = CliRunner().invoke(lean, ["live", "data"]) @@ -285,17 +228,12 @@ def test_live_aborts_when_project_does_not_contain_algorithm_file() -> None: def test_live_aborts_when_lean_config_is_missing_properties(target: str, replacement: str) -> None: create_fake_lean_cli_directory() create_fake_environment("live-paper", True) + lean_runner, _ = _mock_docker_lean_runner() config_path = Path.cwd() / "lean.json" config = config_path.read_text(encoding="utf-8") config_path.write_text(config.replace(target, replacement), encoding="utf-8") - docker_manager = mock.Mock() - container.docker_manager.override(providers.Object(docker_manager)) - - lean_runner = mock.Mock() - container.lean_runner.override(providers.Object(lean_runner)) - result = CliRunner().invoke(lean, ["live", "Python Project", "--environment", "live-paper"]) assert result.exit_code != 0 @@ -441,18 +379,7 @@ def test_live_aborts_when_lean_config_is_missing_properties(target: str, replace def test_live_calls_lean_runner_with_data_provider(data_provider: str) -> None: create_fake_lean_cli_directory() create_fake_environment("live-paper", True) - - docker_manager = mock.Mock() - container.docker_manager.override(providers.Object(docker_manager)) - - lean_runner = mock.Mock() - container.lean_runner.override(providers.Object(lean_runner)) - - api_client = mock.MagicMock() - api_client.organizations.get_all.return_value = [ - QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) - ] - container.api_client.override(providers.Object(api_client)) + lean_runner, _, _ = _mock_docker_lean_runner_api() options = [] for key, value in data_providers_required_options[data_provider].items(): @@ -474,7 +401,7 @@ def test_live_calls_lean_runner_with_data_provider(data_provider: str) -> None: False) -@pytest.mark.parametrize("brokerage", ["Trading Technologies"]) +@pytest.mark.parametrize("brokerage", brokerage_required_options.keys() - ["Paper Trading"]) def test_live_non_interactive_aborts_when_missing_brokerage_options(brokerage: str) -> None: create_fake_lean_cli_directory() @@ -485,12 +412,8 @@ def test_live_non_interactive_aborts_when_missing_brokerage_options(brokerage: s if len(list(comb)) > 1000: continue for current_options in comb: - docker_manager = mock.Mock() - container.docker_manager.override(providers.Object(docker_manager)) - - lean_runner = mock.Mock() - container.lean_runner.override(providers.Object(lean_runner)) - + lean_runner, _ = _mock_docker_lean_runner() + options = [] for key, value in current_options: @@ -524,11 +447,7 @@ def test_live_non_interactive_aborts_when_missing_data_feed_options(data_feed: s required_options = data_feed_required_options[data_feed].items() for length in range(len(required_options)): for current_options in itertools.combinations(required_options, length): - docker_manager = mock.Mock() - container.docker_manager.override(providers.Object(docker_manager)) - - lean_runner = mock.Mock() - container.lean_runner.override(providers.Object(lean_runner)) + lean_runner, _ = _mock_docker_lean_runner() options = [] @@ -553,18 +472,7 @@ def test_live_non_interactive_aborts_when_missing_data_feed_options(data_feed: s itertools.product(brokerage_required_options.keys(), data_feed_required_options.keys())) def test_live_non_interactive_calls_run_lean_when_all_options_given(brokerage: str, data_feed: str) -> None: create_fake_lean_cli_directory() - - docker_manager = mock.Mock() - container.docker_manager.override(providers.Object(docker_manager)) - - lean_runner = mock.Mock() - container.lean_runner.override(providers.Object(lean_runner)) - - api_client = mock.MagicMock() - api_client.organizations.get_all.return_value = [ - QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) - ] - container.api_client.override(providers.Object(api_client)) + lean_runner, _, _ = _mock_docker_lean_runner_api() options = [] @@ -599,18 +507,7 @@ def test_live_non_interactive_calls_run_lean_when_all_options_given(brokerage: s itertools.product(brokerage_required_options.keys(), itertools.combinations(data_feed_required_options.keys(), 2))]) def test_live_non_interactive_calls_run_lean_when_all_options_given_with_multiple_data_feeds(brokerage: str, data_feed1: str, data_feed2: str) -> None: create_fake_lean_cli_directory() - - docker_manager = mock.Mock() - container.docker_manager.override(providers.Object(docker_manager)) - - lean_runner = mock.Mock() - container.lean_runner.override(providers.Object(lean_runner)) - - api_client = mock.MagicMock() - api_client.organizations.get_all.return_value = [ - QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) - ] - container.api_client.override(providers.Object(api_client)) + lean_runner, _, _ = _mock_docker_lean_runner_api() options = [] @@ -657,17 +554,7 @@ def test_live_non_interactive_falls_back_to_lean_config_for_brokerage_settings(b if len(list(comb)) > 1000: continue for current_options in comb: - docker_manager = mock.Mock() - container.docker_manager.override(providers.Object(docker_manager)) - - lean_runner = mock.Mock() - container.lean_runner.override(providers.Object(lean_runner)) - - api_client = mock.MagicMock() - api_client.organizations.get_all.return_value = [ - QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) - ] - container.api_client.override(providers.Object(api_client)) + lean_runner, _, _ = _mock_docker_lean_runner_api() options = [] @@ -732,17 +619,7 @@ def test_live_non_interactive_falls_back_to_lean_config_for_data_feed_settings(d if len(list(comb)) > 1000: continue for current_options in comb: - docker_manager = mock.Mock() - container.docker_manager.override(providers.Object(docker_manager)) - - lean_runner = mock.Mock() - container.lean_runner.override(providers.Object(lean_runner)) - - api_client = mock.MagicMock() - api_client.organizations.get_all.return_value = [ - QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) - ] - container.api_client.override(providers.Object(api_client)) + lean_runner, _, _ = _mock_docker_lean_runner_api() options = [] @@ -790,17 +667,7 @@ def test_live_non_interactive_falls_back_to_lean_config_for_multiple_data_feed_s pytest.skip('computationally expensive test') for length in range(len(required_options)): for current_options in itertools.combinations(required_options, length): - docker_manager = mock.Mock() - container.docker_manager.override(providers.Object(docker_manager)) - - lean_runner = mock.Mock() - container.lean_runner.override(providers.Object(lean_runner)) - - api_client = mock.MagicMock() - api_client.organizations.get_all.return_value = [ - QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) - ] - container.api_client.override(providers.Object(api_client)) + lean_runner, _, _ = _mock_docker_lean_runner_api() options = [] @@ -842,18 +709,7 @@ def test_live_non_interactive_falls_back_to_lean_config_for_multiple_data_feed_s def test_live_forces_update_when_update_option_given() -> None: create_fake_lean_cli_directory() create_fake_environment("live-paper", True) - - docker_manager = mock.Mock() - container.docker_manager.override(providers.Object(docker_manager)) - - lean_runner = mock.Mock() - container.lean_runner.override(providers.Object(lean_runner)) - - api_client = mock.MagicMock() - api_client.organizations.get_all.return_value = [ - QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) - ] - container.api_client.override(providers.Object(api_client)) + lean_runner, _, docker_manager = _mock_docker_lean_runner_api() result = CliRunner().invoke(lean, ["live", "Python Project", "--environment", "live-paper", "--update"]) @@ -873,18 +729,7 @@ def test_live_forces_update_when_update_option_given() -> None: def test_live_passes_custom_image_to_lean_runner_when_set_in_config() -> None: create_fake_lean_cli_directory() create_fake_environment("live-paper", True) - - docker_manager = mock.Mock() - container.docker_manager.override(providers.Object(docker_manager)) - - lean_runner = mock.Mock() - container.lean_runner.override(providers.Object(lean_runner)) - - api_client = mock.MagicMock() - api_client.organizations.get_all.return_value = [ - QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) - ] - container.api_client.override(providers.Object(api_client)) + lean_runner, _, _ = _mock_docker_lean_runner_api() container.cli_config_manager().engine_image.set_value("custom/lean:123") @@ -905,18 +750,7 @@ def test_live_passes_custom_image_to_lean_runner_when_set_in_config() -> None: def test_live_passes_custom_image_to_lean_runner_when_given_as_option() -> None: create_fake_lean_cli_directory() create_fake_environment("live-paper", True) - - docker_manager = mock.Mock() - container.docker_manager.override(providers.Object(docker_manager)) - - lean_runner = mock.Mock() - container.lean_runner.override(providers.Object(lean_runner)) - - api_client = mock.MagicMock() - api_client.organizations.get_all.return_value = [ - QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) - ] - container.api_client.override(providers.Object(api_client)) + lean_runner, _, _ = _mock_docker_lean_runner_api() container.cli_config_manager().engine_image.set_value("custom/lean:123") @@ -941,18 +775,7 @@ def test_live_passes_custom_image_to_lean_runner_when_given_as_option() -> None: def test_live_passes_custom_python_venv_to_lean_runner_when_given_as_option(python_venv: str) -> None: create_fake_lean_cli_directory() create_fake_environment("live-paper", True) - - docker_manager = mock.Mock() - container.docker_manager.override(providers.Object(docker_manager)) - - lean_runner = mock.Mock() - container.lean_runner.override(providers.Object(lean_runner)) - - api_client = mock.MagicMock() - api_client.organizations.get_all.return_value = [ - QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) - ] - container.api_client.override(providers.Object(api_client)) + lean_runner, _, _ = _mock_docker_lean_runner_api() result = CliRunner().invoke(lean, ["live", "Python Project", "--environment", "live-paper", "--python-venv", python_venv]) @@ -985,33 +808,7 @@ def test_live_passes_custom_python_venv_to_lean_runner_when_given_as_option(pyth ("Zerodha", "USD:100")]) def test_live_passes_live_cash_balance_to_lean_runner_when_given_as_option(brokerage: str, cash: str) -> None: create_fake_lean_cli_directory() - results_path = Path.cwd() / "Python Project" / "live" / "2020-01-01_00-00-00" - results_path.mkdir(parents=True, exist_ok=True) - with (results_path / "L-1234567890.json").open("w+", encoding="utf-8") as file: - file.write('''{ - "Cash": { - "USD": { - "SecuritySymbols": [], - "Symbol": "USD", - "Amount": 5000, - "ConversionRate": 0.0, - "CurrencySymbol": "$", - "ValueInAccountCurrency": 0.0 - } - } -}''') - - docker_manager = mock.Mock() - container.docker_manager.override(providers.Object(docker_manager)) - - lean_runner = mock.Mock() - container.lean_runner.override(providers.Object(lean_runner)) - - api_client = mock.MagicMock() - api_client.organizations.get_all.return_value = [ - QCMinimalOrganization(id="abc", name="abc", type="type", ownerName="You", members=1, preferred=True) - ] - container.api_client.override(providers.Object(api_client)) + lean_runner, _, _ = _mock_docker_lean_runner_api() options = [] required_options = brokerage_required_options[brokerage].items() From 971ee5b2963c96a063adcab94b36f26547bd6947 Mon Sep 17 00:00:00 2001 From: LouisSzeto Date: Wed, 5 Oct 2022 22:14:08 +0800 Subject: [PATCH 25/25] Remove redundancy --- tests/commands/cloud/live/test_cloud_live_commands.py | 3 --- tests/commands/test_live.py | 8 -------- 2 files changed, 11 deletions(-) diff --git a/tests/commands/cloud/live/test_cloud_live_commands.py b/tests/commands/cloud/live/test_cloud_live_commands.py index 02503a00..9fe2509c 100644 --- a/tests/commands/cloud/live/test_cloud_live_commands.py +++ b/tests/commands/cloud/live/test_cloud_live_commands.py @@ -11,7 +11,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from pathlib import Path from unittest import mock from click.testing import CliRunner from dependency_injector import providers @@ -51,7 +50,6 @@ def test_cloud_live_liquidate() -> None: def test_cloud_live_deploy() -> None: create_fake_lean_cli_directory() - (Path.cwd() / "Python Project/live").mkdir() api_client = mock.Mock() api_client.nodes.get_all.return_value = create_qc_nodes() @@ -98,7 +96,6 @@ def test_cloud_live_deploy() -> None: ("telegram", "customId1:custom:token1,customId2:custom:token2")]) def test_cloud_live_deploy_with_notifications(notice_method: str, configs: str) -> None: create_fake_lean_cli_directory() - (Path.cwd() / "Python Project/live").mkdir() api_client = mock.Mock() api_client.nodes.get_all.return_value = create_qc_nodes() diff --git a/tests/commands/test_live.py b/tests/commands/test_live.py index f4f13a66..757dff57 100644 --- a/tests/commands/test_live.py +++ b/tests/commands/test_live.py @@ -815,14 +815,6 @@ def test_live_passes_live_cash_balance_to_lean_runner_when_given_as_option(broke for key, value in required_options: options.extend([f"--{key}", value]) - options_config = {key: value for key, value in set(required_options)} - with (Path.cwd() / "lean.json").open("w+", encoding="utf-8") as file: - file.write(json.dumps({ - **options_config, - "data-folder": "data", - "job-organization-id": "abc" - })) - result = CliRunner().invoke(lean, ["live", "Python Project", "--brokerage", brokerage, "--live-cash-balance", cash, "--data-feed", "Custom data only", *options])