From 74dce6dcf32e771ef7a9843469e717b65a1955c8 Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Mon, 20 Jun 2022 12:05:54 +0200 Subject: [PATCH 1/8] Expand file paths during interactive secret registration --- src/zenml/cli/secret.py | 10 ++++++++-- src/zenml/cli/utils.py | 4 ++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/zenml/cli/secret.py b/src/zenml/cli/secret.py index ec10b1fe863..4183392867e 100644 --- a/src/zenml/cli/secret.py +++ b/src/zenml/cli/secret.py @@ -23,6 +23,7 @@ from zenml.cli.utils import ( confirmation, error, + expand_argument_value_from_file, parse_unknown_options, pretty_print_secret, print_secrets, @@ -206,7 +207,9 @@ def register_secret( for k in secret_keys: v = getpass.getpass(f"Secret value for {k}:") if v: - secret_contents[k] = v + secret_contents[k] = expand_argument_value_from_file( + name=k, value=v + ) else: click.echo( "You have not supplied a secret schema with any " @@ -215,9 +218,12 @@ def register_secret( while True: k = click.prompt("Please enter a secret-key") if k not in secret_contents: - secret_contents[k] = getpass.getpass( + v = getpass.getpass( f"Please enter the secret_value for the key [{k}]:" ) + secret_contents[k] = expand_argument_value_from_file( + name=k, value=v + ) else: warning( f"Key {k} already in this secret. Please restart this " diff --git a/src/zenml/cli/utils.py b/src/zenml/cli/utils.py index 06d070d2b78..75beaf9f429 100644 --- a/src/zenml/cli/utils.py +++ b/src/zenml/cli/utils.py @@ -497,7 +497,7 @@ def format_date( MAX_ARGUMENT_VALUE_SIZE = 10240 -def _expand_argument_value_from_file(name: str, value: str) -> str: +def expand_argument_value_from_file(name: str, value: str) -> str: """Expands the value of an argument pointing to a file into the contents of that file. Args: @@ -572,7 +572,7 @@ def parse_unknown_options( if expand_args: args_dict = { - k: _expand_argument_value_from_file(k, v) + k: expand_argument_value_from_file(k, v) for k, v in args_dict.items() } From 0ac1f09e4099f1eb83896394cef6d4067ac84234 Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Mon, 20 Jun 2022 12:07:01 +0200 Subject: [PATCH 2/8] Fix type in github example --- examples/github_actions_orchestration/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/github_actions_orchestration/README.md b/examples/github_actions_orchestration/README.md index 2007a29a586..a7bfa1f91ad 100644 --- a/examples/github_actions_orchestration/README.md +++ b/examples/github_actions_orchestration/README.md @@ -48,7 +48,7 @@ zenml orchestrator register github_orchestrator --flavor=github --push=true # You can find the repository owner and repository name from the URL of your GitHub repository, # for example https://github.com/zenml-io/zenml -> The owner would be `zenml-io` and the repository name `zenml` -zenml secrets_manager register github_secrets_manager \ +zenml secrets-manager register github_secrets_manager \ --flavor=github \ --owner= \ --repository= From ef3e19fc95074d91ce32a1c70efc1a4e228264b6 Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Mon, 20 Jun 2022 12:07:57 +0200 Subject: [PATCH 3/8] Allow ~ and relative paths when passing files as secrets --- src/zenml/cli/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zenml/cli/utils.py b/src/zenml/cli/utils.py index 75beaf9f429..38fb86fa9f7 100644 --- a/src/zenml/cli/utils.py +++ b/src/zenml/cli/utils.py @@ -519,7 +519,7 @@ def expand_argument_value_from_file(name: str, value: str) -> str: return value[1:] if not value.startswith("@"): return value - filename = value[1:] + filename = os.path.abspath(value[1:]) logger.info( f"Expanding argument value `{name}` to contents of file `{filename}`." ) From 0702edd4d98abb10b1766937426a6b8ba02bb03f Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Mon, 20 Jun 2022 13:14:40 +0200 Subject: [PATCH 4/8] Quote pip requirements during docker install --- src/zenml/utils/docker_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/zenml/utils/docker_utils.py b/src/zenml/utils/docker_utils.py index 7ac5dbe8e9a..00405d4a198 100644 --- a/src/zenml/utils/docker_utils.py +++ b/src/zenml/utils/docker_utils.py @@ -87,6 +87,8 @@ def generate_dockerfile_contents( lines.append(f"ENV {key.upper()}={value}") if requirements: + # Quote the requirements to avoid problems with special characters + requirements = [f"'{r}'" for r in requirements] lines.append( f"RUN pip install --no-cache {' '.join(sorted(requirements))}" ) @@ -100,6 +102,7 @@ def generate_dockerfile_contents( if entrypoint: lines.append(f"ENTRYPOINT {entrypoint}") + print(lines) return "\n".join(lines) From c45834d4f45273896df8779230a3e5549a4aad9a Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Mon, 20 Jun 2022 13:15:26 +0200 Subject: [PATCH 5/8] Updating stack component now uses pydantic validation --- src/zenml/cli/stack_components.py | 19 +++++++++++++------ src/zenml/utils/docker_utils.py | 1 - 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/src/zenml/cli/stack_components.py b/src/zenml/cli/stack_components.py index e5da1ee450e..ab361a12c2f 100644 --- a/src/zenml/cli/stack_components.py +++ b/src/zenml/cli/stack_components.py @@ -643,9 +643,13 @@ def update_stack_component_command( else: continue - updated_component = component_wrapper.to_component().copy( - update=parsed_args - ) + # Initialize a new component object to make sure pydantic validation + # is used + new_attributes = { + **component_wrapper.to_component().dict(), + **parsed_args, + } + updated_component = component_class(**new_attributes) repo.update_stack_component( name, updated_component.TYPE, updated_component @@ -721,9 +725,12 @@ def remove_attribute_stack_component_command( f"'{', '.join(optional_attributes)}'." ) - updated_component = current_component.copy( - update={arg: None for arg in parsed_args} - ) + # Remove the attributes from the current component dict + new_attributes = current_component.dict() + for arg in parsed_args: + new_attributes.pop(arg, None) + + updated_component = current_component.__class__(**new_attributes) repo.update_stack_component( name, updated_component.TYPE, updated_component diff --git a/src/zenml/utils/docker_utils.py b/src/zenml/utils/docker_utils.py index 00405d4a198..df785829b11 100644 --- a/src/zenml/utils/docker_utils.py +++ b/src/zenml/utils/docker_utils.py @@ -102,7 +102,6 @@ def generate_dockerfile_contents( if entrypoint: lines.append(f"ENTRYPOINT {entrypoint}") - print(lines) return "\n".join(lines) From 674f5c2bbc8152feb75625aaa272a384ec5e9a93 Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Mon, 20 Jun 2022 14:03:38 +0200 Subject: [PATCH 6/8] Update interactive secret registration message --- src/zenml/cli/secret.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/zenml/cli/secret.py b/src/zenml/cli/secret.py index 4183392867e..e1b7a005200 100644 --- a/src/zenml/cli/secret.py +++ b/src/zenml/cli/secret.py @@ -216,10 +216,10 @@ def register_secret( "predefined keys. Entering interactive mode:" ) while True: - k = click.prompt("Please enter a secret-key") + k = click.prompt("Please enter a secret key") if k not in secret_contents: v = getpass.getpass( - f"Please enter the secret_value for the key [{k}]:" + f"Please enter the secret value for the key [{k}]:" ) secret_contents[k] = expand_argument_value_from_file( name=k, value=v From 568abb01178fae3dff640737f74705ec467e0762 Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Mon, 20 Jun 2022 14:08:16 +0200 Subject: [PATCH 7/8] Fix failing test --- src/zenml/cli/stack_components.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/zenml/cli/stack_components.py b/src/zenml/cli/stack_components.py index ab361a12c2f..ae71eed3f8a 100644 --- a/src/zenml/cli/stack_components.py +++ b/src/zenml/cli/stack_components.py @@ -726,9 +726,10 @@ def remove_attribute_stack_component_command( ) # Remove the attributes from the current component dict - new_attributes = current_component.dict() - for arg in parsed_args: - new_attributes.pop(arg, None) + new_attributes = { + **current_component.dict(), + **{arg: None for arg in parsed_args}, + } updated_component = current_component.__class__(**new_attributes) From 39789d148db0b2d065bbb4916e42f2975a947b38 Mon Sep 17 00:00:00 2001 From: Michael Schuster Date: Mon, 20 Jun 2022 15:01:59 +0200 Subject: [PATCH 8/8] Fix typing issue --- src/zenml/utils/docker_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/zenml/utils/docker_utils.py b/src/zenml/utils/docker_utils.py index df785829b11..09c837f8134 100644 --- a/src/zenml/utils/docker_utils.py +++ b/src/zenml/utils/docker_utils.py @@ -88,7 +88,7 @@ def generate_dockerfile_contents( if requirements: # Quote the requirements to avoid problems with special characters - requirements = [f"'{r}'" for r in requirements] + requirements = {f"'{r}'" for r in requirements} lines.append( f"RUN pip install --no-cache {' '.join(sorted(requirements))}" )