diff --git a/docs/book/how-to/customize-docker-builds/specify-pip-dependencies-and-apt-packages.md b/docs/book/how-to/customize-docker-builds/specify-pip-dependencies-and-apt-packages.md index 584fdb26d8..96cb71a36a 100644 --- a/docs/book/how-to/customize-docker-builds/specify-pip-dependencies-and-apt-packages.md +++ b/docs/book/how-to/customize-docker-builds/specify-pip-dependencies-and-apt-packages.md @@ -102,9 +102,11 @@ You can combine these methods but do make sure that your list of requirements do Depending on the options specified in your Docker settings, ZenML installs the requirements in the following order (each step optional): -* The packages installed in your local Python environment -* The packages specified via the `requirements` attribute (step level overwrites pipeline level) -* The packages specified via the `required_integrations` and potentially stack requirements +* The packages installed in your local python environment +* The packages specified via the `required_hub_plugins` attribute +* The packages required by the stack unless this is disabled by setting `install_stack_requirements=False`. +* The packages specified via the `required_integrations` +* The packages specified via the `requirements` attribute * You can specify additional arguments for the installer used to install your Python packages as follows: ```python # This will result in a `pip install --timeout=1000 ...` call when installing packages in the diff --git a/src/zenml/config/docker_settings.py b/src/zenml/config/docker_settings.py index 2cb084a065..2066cd92d8 100644 --- a/src/zenml/config/docker_settings.py +++ b/src/zenml/config/docker_settings.py @@ -102,10 +102,11 @@ class DockerSettings(BaseSettings): Depending on the configuration of this object, requirements will be installed in the following order (each step optional): - The packages installed in your local python environment - - The packages specified via the `requirements` attribute - - The packages specified via the `required_integrations` and potentially - stack requirements - The packages specified via the `required_hub_plugins` attribute + - The packages required by the stack unless this is disabled by setting + `install_stack_requirements=False`. + - The packages specified via the `required_integrations` + - The packages specified via the `requirements` attribute Attributes: parent_image: Full name of the Docker image that should be diff --git a/src/zenml/utils/pipeline_docker_image_builder.py b/src/zenml/utils/pipeline_docker_image_builder.py index 47169406e1..e2d32d349e 100644 --- a/src/zenml/utils/pipeline_docker_image_builder.py +++ b/src/zenml/utils/pipeline_docker_image_builder.py @@ -460,69 +460,6 @@ def gather_requirements_files( "- Including python packages from local environment" ) - # Generate/Read requirements file for user-defined requirements - if isinstance(docker_settings.requirements, str): - path = os.path.abspath(docker_settings.requirements) - try: - user_requirements = io_utils.read_file_contents_as_string(path) - except FileNotFoundError as e: - raise FileNotFoundError( - f"Requirements file {path} does not exist." - ) from e - if log: - logger.info( - "- Including user-defined requirements from file `%s`", - path, - ) - elif isinstance(docker_settings.requirements, List): - user_requirements = "\n".join(docker_settings.requirements) - if log: - logger.info( - "- Including user-defined requirements: %s", - ", ".join(f"`{r}`" for r in docker_settings.requirements), - ) - else: - user_requirements = None - - if user_requirements: - requirements_files.append( - (".zenml_user_requirements", user_requirements, []) - ) - - # Generate requirements file for all required integrations - integration_requirements = set( - itertools.chain.from_iterable( - integration_registry.select_integration_requirements( - integration_name=integration, - target_os=OperatingSystemType.LINUX, - ) - for integration in docker_settings.required_integrations - ) - ) - - if docker_settings.install_stack_requirements: - integration_requirements.update(stack.requirements()) - if code_repository: - integration_requirements.update(code_repository.requirements) - - if integration_requirements: - integration_requirements_list = sorted(integration_requirements) - integration_requirements_file = "\n".join( - integration_requirements_list - ) - requirements_files.append( - ( - ".zenml_integration_requirements", - integration_requirements_file, - [], - ) - ) - if log: - logger.info( - "- Including integration requirements: %s", - ", ".join(f"`{r}`" for r in integration_requirements_list), - ) - # Generate requirements files for all ZenML Hub plugins if docker_settings.required_hub_plugins: ( @@ -560,6 +497,85 @@ def gather_requirements_files( ", ".join(f"`{r}`" for r in hub_pypi_requirements), ) + if docker_settings.install_stack_requirements: + stack_requirements = stack.requirements() + if code_repository: + stack_requirements.update(code_repository.requirements) + + if stack_requirements: + stack_requirements_list = sorted(stack_requirements) + stack_requirements_file = "\n".join(stack_requirements_list) + requirements_files.append( + ( + ".zenml_stack_integration_requirements", + stack_requirements_file, + [], + ) + ) + if log: + logger.info( + "- Including stack requirements: %s", + ", ".join(f"`{r}`" for r in stack_requirements_list), + ) + + # Generate requirements file for all required integrations + integration_requirements = set( + itertools.chain.from_iterable( + integration_registry.select_integration_requirements( + integration_name=integration, + target_os=OperatingSystemType.LINUX, + ) + for integration in docker_settings.required_integrations + ) + ) + + if integration_requirements: + integration_requirements_list = sorted(integration_requirements) + integration_requirements_file = "\n".join( + integration_requirements_list + ) + requirements_files.append( + ( + ".zenml_integration_requirements", + integration_requirements_file, + [], + ) + ) + if log: + logger.info( + "- Including integration requirements: %s", + ", ".join(f"`{r}`" for r in integration_requirements_list), + ) + + # Generate/Read requirements file for user-defined requirements + if isinstance(docker_settings.requirements, str): + path = os.path.abspath(docker_settings.requirements) + try: + user_requirements = io_utils.read_file_contents_as_string(path) + except FileNotFoundError as e: + raise FileNotFoundError( + f"Requirements file {path} does not exist." + ) from e + if log: + logger.info( + "- Including user-defined requirements from file `%s`", + path, + ) + elif isinstance(docker_settings.requirements, List): + user_requirements = "\n".join(docker_settings.requirements) + if log: + logger.info( + "- Including user-defined requirements: %s", + ", ".join(f"`{r}`" for r in docker_settings.requirements), + ) + else: + user_requirements = None + + if user_requirements: + requirements_files.append( + (".zenml_user_requirements", user_requirements, []) + ) + return requirements_files @staticmethod diff --git a/tests/unit/utils/test_pipeline_docker_image_builder.py b/tests/unit/utils/test_pipeline_docker_image_builder.py index 9bf0e57be2..1753bbcfad 100644 --- a/tests/unit/utils/test_pipeline_docker_image_builder.py +++ b/tests/unit/utils/test_pipeline_docker_image_builder.py @@ -112,26 +112,28 @@ def test_requirements_file_generation( files = PipelineDockerImageBuilder.gather_requirements_files( settings, stack=local_stack ) - assert len(files) == 5 + assert len(files) == 6 # first up the local python requirements assert files[0][1] == "local_requirements" - # then the user requirements - assert files[1][1] == "user_requirements" - # then the integration requirements - expected_integration_requirements = "\n".join( - sorted(SklearnIntegration.REQUIREMENTS + ["stack_requirements"]) - ) - assert files[2][1] == expected_integration_requirements - # last the hub requirements + # then the hub requirements expected_hub_internal_requirements = ( f"-i {sample_hub_plugin_response_model.index_url}\n" f"{sample_hub_plugin_response_model.package_name}" ) - assert files[3][1] == expected_hub_internal_requirements + assert files[1][1] == expected_hub_internal_requirements expected_hub_pypi_requirements = "\n".join( sample_hub_plugin_response_model.requirements ) - assert files[4][1] == expected_hub_pypi_requirements + assert files[2][1] == expected_hub_pypi_requirements + # then the stack requirements + assert files[3][1] == "stack_requirements" + # then the integration requirements + expected_integration_requirements = "\n".join( + sorted(SklearnIntegration.REQUIREMENTS) + ) + assert files[4][1] == expected_integration_requirements + # last the user requirements + assert files[5][1] == "user_requirements" def test_build_skipping():