Update order in which requirements are installed

docs/book/how-to/customize-docker-builds/specify-pip-dependencies-and-apt-packages.md

@@ -102,9 +102,11 @@ You can combine these methods but do make sure that your list of requirements do
 
 Depending on the options specified in your Docker settings, ZenML installs the requirements in the following order (each step optional):
 
-* The packages installed in your local Python environment
-* The packages specified via the `requirements` attribute (step level overwrites pipeline level)
-* The packages specified via the `required_integrations` and potentially stack requirements
+* The packages installed in your local python environment
+* The packages specified via the `required_hub_plugins` attribute
+* The packages required by the stack unless this is disabled by setting `install_stack_requirements=False`.
+* The packages specified via the `required_integrations`
+* The packages specified via the `requirements` attribute
 * You can specify additional arguments for the installer used to install your Python packages as follows:
 ```python
 # This will result in a `pip install --timeout=1000 ...` call when installing packages in the

src/zenml/config/docker_settings.py

@@ -102,10 +102,11 @@ class DockerSettings(BaseSettings):
     Depending on the configuration of this object, requirements will be
     installed in the following order (each step optional):
     - The packages installed in your local python environment
-    - The packages specified via the `requirements` attribute
-    - The packages specified via the `required_integrations` and potentially
-      stack requirements
     - The packages specified via the `required_hub_plugins` attribute
+    - The packages required by the stack unless this is disabled by setting
+      `install_stack_requirements=False`.
+    - The packages specified via the `required_integrations`
+    - The packages specified via the `requirements` attribute
 
     Attributes:
         parent_image: Full name of the Docker image that should be

src/zenml/utils/pipeline_docker_image_builder.py

@@ -460,69 +460,6 @@ def gather_requirements_files(
                     "- Including python packages from local environment"
                 )
 
-        # Generate/Read requirements file for user-defined requirements
-        if isinstance(docker_settings.requirements, str):
-            path = os.path.abspath(docker_settings.requirements)
-            try:
-                user_requirements = io_utils.read_file_contents_as_string(path)
-            except FileNotFoundError as e:
-                raise FileNotFoundError(
-                    f"Requirements file {path} does not exist."
-                ) from e
-            if log:
-                logger.info(
-                    "- Including user-defined requirements from file `%s`",
-                    path,
-                )
-        elif isinstance(docker_settings.requirements, List):
-            user_requirements = "\n".join(docker_settings.requirements)
-            if log:
-                logger.info(
-                    "- Including user-defined requirements: %s",
-                    ", ".join(f"`{r}`" for r in docker_settings.requirements),
-                )
-        else:
-            user_requirements = None
-
-        if user_requirements:
-            requirements_files.append(
-                (".zenml_user_requirements", user_requirements, [])
-            )
-
-        # Generate requirements file for all required integrations
-        integration_requirements = set(
-            itertools.chain.from_iterable(
-                integration_registry.select_integration_requirements(
-                    integration_name=integration,
-                    target_os=OperatingSystemType.LINUX,
-                )
-                for integration in docker_settings.required_integrations
-            )
-        )
-
-        if docker_settings.install_stack_requirements:
-            integration_requirements.update(stack.requirements())
-            if code_repository:
-                integration_requirements.update(code_repository.requirements)
-
-        if integration_requirements:
-            integration_requirements_list = sorted(integration_requirements)
-            integration_requirements_file = "\n".join(
-                integration_requirements_list
-            )
-            requirements_files.append(
-                (
-                    ".zenml_integration_requirements",
-                    integration_requirements_file,
-                    [],
-                )
-            )
-            if log:
-                logger.info(
-                    "- Including integration requirements: %s",
-                    ", ".join(f"`{r}`" for r in integration_requirements_list),
-                )
-
         # Generate requirements files for all ZenML Hub plugins
         if docker_settings.required_hub_plugins:
             (
@@ -560,6 +497,85 @@ def gather_requirements_files(
                         ", ".join(f"`{r}`" for r in hub_pypi_requirements),
                     )
 
+        if docker_settings.install_stack_requirements:
+            stack_requirements = stack.requirements()
+            if code_repository:
+                stack_requirements.update(code_repository.requirements)
+
+            if stack_requirements:
+                stack_requirements_list = sorted(stack_requirements)
+                stack_requirements_file = "\n".join(stack_requirements_list)
+                requirements_files.append(
+                    (
+                        ".zenml_stack_integration_requirements",
+                        stack_requirements_file,
+                        [],
+                    )
+                )
+                if log:
+                    logger.info(
+                        "- Including stack requirements: %s",
+                        ", ".join(f"`{r}`" for r in stack_requirements_list),
+                    )
+
+        # Generate requirements file for all required integrations
+        integration_requirements = set(
+            itertools.chain.from_iterable(
+                integration_registry.select_integration_requirements(
+                    integration_name=integration,
+                    target_os=OperatingSystemType.LINUX,
+                )
+                for integration in docker_settings.required_integrations
+            )
+        )
+
+        if integration_requirements:
+            integration_requirements_list = sorted(integration_requirements)
+            integration_requirements_file = "\n".join(
+                integration_requirements_list
+            )
+            requirements_files.append(
+                (
+                    ".zenml_integration_requirements",
+                    integration_requirements_file,
+                    [],
+                )
+            )
+            if log:
+                logger.info(
+                    "- Including integration requirements: %s",
+                    ", ".join(f"`{r}`" for r in integration_requirements_list),
+                )
+
+        # Generate/Read requirements file for user-defined requirements
+        if isinstance(docker_settings.requirements, str):
+            path = os.path.abspath(docker_settings.requirements)
+            try:
+                user_requirements = io_utils.read_file_contents_as_string(path)
+            except FileNotFoundError as e:
+                raise FileNotFoundError(
+                    f"Requirements file {path} does not exist."
+                ) from e
+            if log:
+                logger.info(
+                    "- Including user-defined requirements from file `%s`",
+                    path,
+                )
+        elif isinstance(docker_settings.requirements, List):
+            user_requirements = "\n".join(docker_settings.requirements)
+            if log:
+                logger.info(
+                    "- Including user-defined requirements: %s",
+                    ", ".join(f"`{r}`" for r in docker_settings.requirements),
+                )
+        else:
+            user_requirements = None
+
+        if user_requirements:
+            requirements_files.append(
+                (".zenml_user_requirements", user_requirements, [])
+            )
+
         return requirements_files
 
     @staticmethod

tests/unit/utils/test_pipeline_docker_image_builder.py

@@ -112,26 +112,28 @@ def test_requirements_file_generation(
     files = PipelineDockerImageBuilder.gather_requirements_files(
         settings, stack=local_stack
     )
-    assert len(files) == 5
+    assert len(files) == 6
     # first up the local python requirements
     assert files[0][1] == "local_requirements"
-    # then the user requirements
-    assert files[1][1] == "user_requirements"
-    # then the integration requirements
-    expected_integration_requirements = "\n".join(
-        sorted(SklearnIntegration.REQUIREMENTS + ["stack_requirements"])
-    )
-    assert files[2][1] == expected_integration_requirements
-    # last the hub requirements
+    # then the hub requirements
     expected_hub_internal_requirements = (
         f"-i {sample_hub_plugin_response_model.index_url}\n"
         f"{sample_hub_plugin_response_model.package_name}"
     )
-    assert files[3][1] == expected_hub_internal_requirements
+    assert files[1][1] == expected_hub_internal_requirements
     expected_hub_pypi_requirements = "\n".join(
         sample_hub_plugin_response_model.requirements
     )
-    assert files[4][1] == expected_hub_pypi_requirements
+    assert files[2][1] == expected_hub_pypi_requirements
+    # then the stack requirements
+    assert files[3][1] == "stack_requirements"
+    # then the integration requirements
+    expected_integration_requirements = "\n".join(
+        sorted(SklearnIntegration.REQUIREMENTS)
+    )
+    assert files[4][1] == expected_integration_requirements
+    # last the user requirements
+    assert files[5][1] == "user_requirements"
 
 
 def test_build_skipping():