From f9a0280ffdc5650ed7b532177e257e1466560633 Mon Sep 17 00:00:00 2001 From: Romazes Date: Thu, 31 Jul 2025 18:43:53 +0300 Subject: [PATCH 1/4] fix: running IQFeed client locally --- lean/commands/data/download.py | 4 ++++ lean/commands/live/deploy.py | 17 +++++++++++------ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 6b21f402..86c5c143 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -17,6 +17,7 @@ from typing import Any, Dict, Iterable, List, Optional from click import command, option, confirm, pass_context, Context, Choice, prompt from lean.click import LeanCommand, ensure_options +from lean.commands.live.deploy import _start_iqconnect_if_necessary from lean.components.util.json_modules_handler import config_build_for_name from lean.constants import DEFAULT_ENGINE_IMAGE from lean.container import container @@ -721,6 +722,9 @@ def download(ctx: Context, read_only=True) ) + # Run IQConnect if using the IQFeed data downloader + _start_iqconnect_if_necessary(lean_config, data_downloader_provider.get_settings()["data-downloader"]) + success = container.docker_manager.run_image(engine_image, **run_options) if not success: diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index a7d41f83..28ecda9d 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -40,16 +40,21 @@ } -def _start_iqconnect_if_necessary(lean_config: Dict[str, Any], environment_name: str) -> None: - """Starts IQConnect if the given environment uses IQFeed as data queue handler. +def _start_iqconnect_if_necessary(lean_config: Dict[str, Any], data_provider_type: str) -> None: + """Starts IQConnect if the specified data provider is IQFeed. :param lean_config: the LEAN configuration that should be used - :param environment_name: the name of the environment + :param data_provider_type: The fully qualified name of the data provider or data downloader. """ from subprocess import Popen - environment = lean_config["environments"][environment_name] - if environment["data-queue-handler"] != "QuantConnect.ToolBox.IQFeed.IQFeedDataQueueHandler": + # Normalize and parse possible list + cleaned = data_provider_type.strip('[]') + types = [t.strip(' "\'') for t in cleaned.split(',')] + + if not any(t.startswith("QuantConnect.Lean.DataSource.IQFeed.") for t in types): + container.logger.debug( + f"Exiting _start_iqconnect_if_necessary because data_provider_type does not use IQFeed: {data_provider_type}") return args = [lean_config["iqfeed-iqconnect"], @@ -285,7 +290,7 @@ def deploy(project: Path, raise MoreInfoError(f"The '{environment_name}' is not a live trading environment (live-mode is set to false)", "https://www.lean.io/docs/v2/lean-cli/live-trading/brokerages/quantconnect-paper-trading") - _start_iqconnect_if_necessary(lean_config, environment_name) + _start_iqconnect_if_necessary(lean_config, lean_config["environments"][environment_name]["data-queue-handler"]) if python_venv is not None and python_venv != "": lean_config["python-venv"] = f'{"/" if python_venv[0] != "/" else ""}{python_venv}' From f62138148c4e01b6f3966ac3f86e0323df1917a5 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 5 Aug 2025 12:37:04 +0300 Subject: [PATCH 2/4] refactor: run IQFeed app based on packages everywhere remove: IQFeed run in other part of source code base --- lean/commands/data/download.py | 4 -- lean/commands/live/deploy.py | 39 ----------------- lean/components/docker/lean_runner.py | 63 ++++++++++++++++++++++++--- 3 files changed, 58 insertions(+), 48 deletions(-) diff --git a/lean/commands/data/download.py b/lean/commands/data/download.py index 86c5c143..6b21f402 100644 --- a/lean/commands/data/download.py +++ b/lean/commands/data/download.py @@ -17,7 +17,6 @@ from typing import Any, Dict, Iterable, List, Optional from click import command, option, confirm, pass_context, Context, Choice, prompt from lean.click import LeanCommand, ensure_options -from lean.commands.live.deploy import _start_iqconnect_if_necessary from lean.components.util.json_modules_handler import config_build_for_name from lean.constants import DEFAULT_ENGINE_IMAGE from lean.container import container @@ -722,9 +721,6 @@ def download(ctx: Context, read_only=True) ) - # Run IQConnect if using the IQFeed data downloader - _start_iqconnect_if_necessary(lean_config, data_downloader_provider.get_settings()["data-downloader"]) - success = container.docker_manager.run_image(engine_image, **run_options) if not success: diff --git a/lean/commands/live/deploy.py b/lean/commands/live/deploy.py index 28ecda9d..de7576ed 100644 --- a/lean/commands/live/deploy.py +++ b/lean/commands/live/deploy.py @@ -39,43 +39,6 @@ "transaction-handler": "QuantConnect.Lean.Engine.TransactionHandlers.BrokerageTransactionHandler" } - -def _start_iqconnect_if_necessary(lean_config: Dict[str, Any], data_provider_type: str) -> None: - """Starts IQConnect if the specified data provider is IQFeed. - - :param lean_config: the LEAN configuration that should be used - :param data_provider_type: The fully qualified name of the data provider or data downloader. - """ - from subprocess import Popen - - # Normalize and parse possible list - cleaned = data_provider_type.strip('[]') - types = [t.strip(' "\'') for t in cleaned.split(',')] - - if not any(t.startswith("QuantConnect.Lean.DataSource.IQFeed.") for t in types): - container.logger.debug( - f"Exiting _start_iqconnect_if_necessary because data_provider_type does not use IQFeed: {data_provider_type}") - return - - args = [lean_config["iqfeed-iqconnect"], - "-product", lean_config["iqfeed-productName"], - "-version", lean_config["iqfeed-version"]] - - username = lean_config.get("iqfeed-username", "") - if username != "": - args.extend(["-login", username]) - - password = lean_config.get("iqfeed-password", "") - if password != "": - args.extend(["-password", password]) - - Popen(args) - - container.logger.info("Waiting 10 seconds for IQFeed to start") - from time import sleep - sleep(10) - - def _get_history_provider_name(data_provider_live_names: [str]) -> [str]: """ Get name for history providers based on the live data providers @@ -290,8 +253,6 @@ def deploy(project: Path, raise MoreInfoError(f"The '{environment_name}' is not a live trading environment (live-mode is set to false)", "https://www.lean.io/docs/v2/lean-cli/live-trading/brokerages/quantconnect-paper-trading") - _start_iqconnect_if_necessary(lean_config, lean_config["environments"][environment_name]["data-queue-handler"]) - if python_venv is not None and python_venv != "": lean_config["python-venv"] = f'{"/" if python_venv[0] != "/" else ""}{python_venv}' diff --git a/lean/components/docker/lean_runner.py b/lean/components/docker/lean_runner.py index b452a4b7..a7d5de4f 100644 --- a/lean/components/docker/lean_runner.py +++ b/lean/components/docker/lean_runner.py @@ -233,7 +233,7 @@ def get_basic_docker_config(self, lean_config[key] = "host.docker.internal" # Set up modules - set_up_common_csharp_options_called = self._setup_installed_packages(run_options, image) + set_up_common_csharp_options_called = self._setup_installed_packages(run_options, image, lean_config) # Set up language-specific run options self.setup_language_specific_run_options(run_options, project_dir, algorithm_file, @@ -295,7 +295,7 @@ def get_basic_docker_config_without_algo(self, lean_config[key] = "host.docker.internal" # Set up modules - self._setup_installed_packages(run_options, image, target_path) + self._setup_installed_packages(run_options, image, lean_config, target_path) self._mount_lean_config_and_finalize(run_options, lean_config, None, config_local_path) @@ -340,7 +340,8 @@ def _mount_lean_config_and_finalize(self, run_options: Dict[str, Any], lean_conf if "live-mode-brokerage" in environment: output_config.set("brokerage", environment["live-mode-brokerage"].split(".")[-1]) - def _setup_installed_packages(self, run_options: Dict[str, Any], image: DockerImage, target_path: str = "/Lean/Launcher/bin/Debug"): + def _setup_installed_packages(self, run_options: Dict[str, Any], image: DockerImage, + lean_config: Dict[str, Any], target_path: str = "/Lean/Launcher/bin/Debug"): """Sets up installed packages.""" installed_packages = self._module_manager.get_installed_packages() if installed_packages: @@ -364,6 +365,7 @@ def _setup_installed_packages(self, run_options: Dict[str, Any], image: DockerIm for package in installed_packages: self._logger.debug(f"LeanRunner._setup_installed_packages(): Adding module {package} to the project") run_options["commands"].append(f"rm -rf /root/.nuget/packages/{package.name.lower()}") + self._ensure_iqconnect_running(lean_config, package.name) run_options["commands"].append(f"dotnet add /ModulesProject package {package.name} --version {package.version}") # Copy all module files to /Lean/Launcher/bin/Debug, but don't overwrite anything that already exists @@ -938,7 +940,7 @@ def parse_extra_docker_config(run_options: Dict[str, Any], extra_docker_config: if "name" in extra_docker_config: run_options["name"] = extra_docker_config["name"] - + if "environment" in extra_docker_config: target = run_options.get("environment") if not target: @@ -947,7 +949,7 @@ def parse_extra_docker_config(run_options: Dict[str, Any], extra_docker_config: target.update({item[0]: item[1] for item in [ item if not isinstance(item, str) else (item.split("=")[0], item.split("=")[1]) for item in extra_docker_config["environment"] ]}) - elif isinstance(extra_docker_config["environment"], dict): + elif isinstance(extra_docker_config["environment"], dict): target.update(extra_docker_config["environment"]) else: raise ValueError("Additional environment variables can be passed to the container in a dictionary, list of '{key}={value}' strings, or list of '(key, value)' tuples.") @@ -975,3 +977,54 @@ def parse_extra_docker_config(run_options: Dict[str, Any], extra_docker_config: if "read_only" in mount: read_only = mount["read_only"] target.append(Mount(target=mount["target"], source=mount["source"], type="bind", read_only=read_only)) + + def _ensure_iqconnect_running(self, lean_config: Dict[str, Any], data_provider_package_name: str) -> None: + """ + Starts the IQConnect client if the given data provider is IQFeed. + + :param lean_config: The LEAN configuration dictionary to use. + :param data_provider_package_name: The fully qualified name of the data provider or data downloader. + """ + + if data_provider_package_name != "QuantConnect.Lean.DataSource.IQFeed": + self._logger.debug( + f"Skipped starting IQConnect: data provider '{data_provider_package_name}' is not IQFeed." + ) + return + + args = [ + lean_config["iqfeed-iqconnect"], + "-product", lean_config["iqfeed-productName"], + "-version", lean_config["iqfeed-version"], + "-autoconnect" + ] + + username = lean_config.get("iqfeed-username", "") + if username != "": + args.extend(["-login", username]) + + password = lean_config.get("iqfeed-password", "") + if password != "": + args.extend(["-password", password]) + + from subprocess import Popen + + try: + process = Popen(args) + except FileNotFoundError: + raise RuntimeError( + "IQFeed executable not found. Please check:\n" + " - The path in 'args' is correct.\n" + " - IQFeed is installed.\n" + " - You have permission to access the file." + ) + + self._logger.info("Waiting 10 seconds for IQFeed to start") + from time import sleep + sleep(10) + + if process.poll() is not None: + raise RuntimeError( + f"IQFeed failed to start (exit code {process.returncode}). " + "Check if IQFeed is installed, path is correct, and no other instance is running." + ) From 6b18b66c603fdb92194e811188ff87bfa62ada4d Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 5 Aug 2025 16:14:47 +0300 Subject: [PATCH 3/4] refactor: use log warning instead of runtime error --- lean/components/docker/lean_runner.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lean/components/docker/lean_runner.py b/lean/components/docker/lean_runner.py index a7d5de4f..f46e2235 100644 --- a/lean/components/docker/lean_runner.py +++ b/lean/components/docker/lean_runner.py @@ -1012,7 +1012,7 @@ def _ensure_iqconnect_running(self, lean_config: Dict[str, Any], data_provider_p try: process = Popen(args) except FileNotFoundError: - raise RuntimeError( + self._logger.warn( "IQFeed executable not found. Please check:\n" " - The path in 'args' is correct.\n" " - IQFeed is installed.\n" @@ -1024,7 +1024,8 @@ def _ensure_iqconnect_running(self, lean_config: Dict[str, Any], data_provider_p sleep(10) if process.poll() is not None: - raise RuntimeError( + self._logger.warn( f"IQFeed failed to start (exit code {process.returncode}). " - "Check if IQFeed is installed, path is correct, and no other instance is running." + "It might already be running, or there was an error starting it. " + "Check if IQFeed is installed, the path is correct, and no issues with permissions." ) From 5447b3e0abd3d61c38f8da5487c56b8314ac4844 Mon Sep 17 00:00:00 2001 From: Romazes Date: Tue, 5 Aug 2025 16:16:56 +0300 Subject: [PATCH 4/4] fix: missed 'return' --- lean/components/docker/lean_runner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lean/components/docker/lean_runner.py b/lean/components/docker/lean_runner.py index f46e2235..a2db4962 100644 --- a/lean/components/docker/lean_runner.py +++ b/lean/components/docker/lean_runner.py @@ -1018,6 +1018,7 @@ def _ensure_iqconnect_running(self, lean_config: Dict[str, Any], data_provider_p " - IQFeed is installed.\n" " - You have permission to access the file." ) + return self._logger.info("Waiting 10 seconds for IQFeed to start") from time import sleep