Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Update order in which requirements are installed #2341

Open
wants to merge 5 commits into
base: develop
Choose a base branch
from
Open

Update order in which requirements are installed #2341

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
54a5832
Update order in which requirements are installed
schustmi Jan 23, 2024
459d896
Fix test
schustmi Jan 23, 2024
3511cf2
Merge branch 'develop' into feature/requirements-installation-order
schustmi Jan 25, 2024
b125405
Update containerization docs
schustmi Jan 25, 2024
03b9674
Merge branch 'develop' into feature/requirements-installation-order
strickvl Mar 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
7 changes: 4 additions & 3 deletions src/zenml/config/docker_settings.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,10 +76,11 @@ class DockerSettings(BaseSettings):
Depending on the configuration of this object, requirements will be
installed in the following order (each step optional):
- The packages installed in your local python environment
- The packages specified via the `requirements` attribute
- The packages specified via the `required_integrations` and potentially
stack requirements
- The packages specified via the `required_hub_plugins` attribute
- The packages required by the stack unless this is disabled by setting
`install_stack_requirements=False`.
- The packages specified via the `required_integrations`
- The packages specified via the `requirements` attribute

Attributes:
parent_image: Full name of the Docker image that should be
Expand Down
142 changes: 79 additions & 63 deletions src/zenml/utils/pipeline_docker_image_builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -423,69 +423,6 @@ def gather_requirements_files(
"- Including python packages from local environment"
)

# Generate/Read requirements file for user-defined requirements
if isinstance(docker_settings.requirements, str):
path = os.path.abspath(docker_settings.requirements)
try:
user_requirements = io_utils.read_file_contents_as_string(path)
except FileNotFoundError as e:
raise FileNotFoundError(
f"Requirements file {path} does not exist."
) from e
if log:
logger.info(
"- Including user-defined requirements from file `%s`",
path,
)
elif isinstance(docker_settings.requirements, List):
user_requirements = "\n".join(docker_settings.requirements)
if log:
logger.info(
"- Including user-defined requirements: %s",
", ".join(f"`{r}`" for r in docker_settings.requirements),
)
else:
user_requirements = None

if user_requirements:
requirements_files.append(
(".zenml_user_requirements", user_requirements, [])
)

# Generate requirements file for all required integrations
integration_requirements = set(
itertools.chain.from_iterable(
integration_registry.select_integration_requirements(
integration_name=integration,
target_os=OperatingSystemType.LINUX,
)
for integration in docker_settings.required_integrations
)
)

if docker_settings.install_stack_requirements:
integration_requirements.update(stack.requirements())
if code_repository:
integration_requirements.update(code_repository.requirements)

if integration_requirements:
integration_requirements_list = sorted(integration_requirements)
integration_requirements_file = "\n".join(
integration_requirements_list
)
requirements_files.append(
(
".zenml_integration_requirements",
integration_requirements_file,
[],
)
)
if log:
logger.info(
"- Including integration requirements: %s",
", ".join(f"`{r}`" for r in integration_requirements_list),
)

# Generate requirements files for all ZenML Hub plugins
if docker_settings.required_hub_plugins:
(
Expand Down Expand Up @@ -523,6 +460,85 @@ def gather_requirements_files(
", ".join(f"`{r}`" for r in hub_pypi_requirements),
)

if docker_settings.install_stack_requirements:
stack_requirements = stack.requirements()
if code_repository:
stack_requirements.update(code_repository.requirements)

if stack_requirements:
stack_requirements_list = sorted(stack_requirements)
stack_requirements_file = "\n".join(stack_requirements_list)
requirements_files.append(
(
".zenml_stack_integration_requirements",
stack_requirements_file,
[],
)
)
if log:
logger.info(
"- Including stack requirements: %s",
", ".join(f"`{r}`" for r in stack_requirements_list),
)

# Generate requirements file for all required integrations
integration_requirements = set(
itertools.chain.from_iterable(
integration_registry.select_integration_requirements(
integration_name=integration,
target_os=OperatingSystemType.LINUX,
)
for integration in docker_settings.required_integrations
)
)

if integration_requirements:
integration_requirements_list = sorted(integration_requirements)
integration_requirements_file = "\n".join(
integration_requirements_list
)
requirements_files.append(
(
".zenml_integration_requirements",
integration_requirements_file,
[],
)
)
if log:
logger.info(
"- Including integration requirements: %s",
", ".join(f"`{r}`" for r in integration_requirements_list),
)

# Generate/Read requirements file for user-defined requirements
if isinstance(docker_settings.requirements, str):
path = os.path.abspath(docker_settings.requirements)
try:
user_requirements = io_utils.read_file_contents_as_string(path)
except FileNotFoundError as e:
raise FileNotFoundError(
f"Requirements file {path} does not exist."
) from e
if log:
logger.info(
"- Including user-defined requirements from file `%s`",
path,
)
elif isinstance(docker_settings.requirements, List):
user_requirements = "\n".join(docker_settings.requirements)
if log:
logger.info(
"- Including user-defined requirements: %s",
", ".join(f"`{r}`" for r in docker_settings.requirements),
)
else:
user_requirements = None

if user_requirements:
requirements_files.append(
(".zenml_user_requirements", user_requirements, [])
)

return requirements_files

@staticmethod
Expand Down
24 changes: 13 additions & 11 deletions tests/unit/utils/test_pipeline_docker_image_builder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,26 +110,28 @@ def test_requirements_file_generation(
files = PipelineDockerImageBuilder.gather_requirements_files(
settings, stack=local_stack
)
assert len(files) == 5
assert len(files) == 6
# first up the local python requirements
assert files[0][1] == "local_requirements"
# then the user requirements
assert files[1][1] == "user_requirements"
# then the integration requirements
expected_integration_requirements = "\n".join(
sorted(SklearnIntegration.REQUIREMENTS + ["stack_requirements"])
)
assert files[2][1] == expected_integration_requirements
# last the hub requirements
# then the hub requirements
expected_hub_internal_requirements = (
f"-i {sample_hub_plugin_response_model.index_url}\n"
f"{sample_hub_plugin_response_model.package_name}"
)
assert files[3][1] == expected_hub_internal_requirements
assert files[1][1] == expected_hub_internal_requirements
expected_hub_pypi_requirements = "\n".join(
sample_hub_plugin_response_model.requirements
)
assert files[4][1] == expected_hub_pypi_requirements
assert files[2][1] == expected_hub_pypi_requirements
# then the stack requirements
assert files[3][1] == "stack_requirements"
# then the integration requirements
expected_integration_requirements = "\n".join(
sorted(SklearnIntegration.REQUIREMENTS)
)
assert files[4][1] == expected_integration_requirements
# last the user requirements
assert files[5][1] == "user_requirements"


def test_build_skipping():
Expand Down