From f3a53f0b76954aa447ba4e2ec1eebff2475111ad Mon Sep 17 00:00:00 2001 From: floralatin Date: Fri, 4 Nov 2022 19:13:01 +0800 Subject: [PATCH] refactore: load secrets with --id and /run/secrets/id --- normalizer/core.py | 10 +++++----- normalizer/docker/parser.py | 6 ++---- normalizer/models.py | 2 +- server/routes/normalizer.py | 2 +- tests/cases/executor_7/Dockerfile.expect_file | 16 ++++++++++++++++ tests/test_core.py | 15 ++++++++++----- 6 files changed, 35 insertions(+), 16 deletions(-) create mode 100644 tests/cases/executor_7/Dockerfile.expect_file diff --git a/normalizer/core.py b/normalizer/core.py index eeca435..d33ffc7 100644 --- a/normalizer/core.py +++ b/normalizer/core.py @@ -371,7 +371,7 @@ def normalize( build_env: Dict = {}, dry_run: bool = False, dockerfile: Optional[str] = None, - build_env_path: Optional['pathlib.Path'] = None, + build_env_file: Optional[str] = None, **kwargs, ) -> ExecutorModel: """Normalize the executor package. @@ -623,8 +623,8 @@ def normalize( if build_env and isinstance(build_env, dict) and len(build_env.keys()): dockerfile.insert_build_env(build_env) - if build_env_path and build_env_path.exists(): - dockerfile.insert_build_env_path(build_env_path) + if build_env_file: + dockerfile.insert_build_env_file(build_env_file) # if dockerfile.is_multistage(): # # Don't support multi-stage Dockerfie Optimization # return @@ -657,8 +657,8 @@ def normalize( if build_env and isinstance(build_env, dict) and len(build_env.keys()): dockerfile.insert_build_env(build_env) - if build_env_path and build_env_path.exists(): - dockerfile.insert_build_env_path(build_env_path) + if build_env_file: + dockerfile.insert_build_env_file(build_env_file) # if len(test_glob) > 0: # dockerfile.add_unitest() diff --git a/normalizer/docker/parser.py b/normalizer/docker/parser.py index f014638..40b2406 100644 --- a/normalizer/docker/parser.py +++ b/normalizer/docker/parser.py @@ -61,12 +61,10 @@ def insert_build_env(self, build_env: Dict): replace_line = line.replace('RUN', f'RUN {build_env_str}') self._parser.content = self._parser.content.replace(line, replace_line) - def insert_build_env_path(self, build_env_path: pathlib.Path): + def insert_build_env_file(self, build_env_file: str): build_env_str = '' - build_env_path_str = str(build_env_path) - build_env_str += dedent( - f' --mount=type=secret,id={build_env_path.stem},dst={build_env_path} . {build_env_path_str} ' + f' --mount=type=secret,id={build_env_file} . run/secrets/{build_env_file} ' ) build_env_str += ' && ' diff --git a/normalizer/models.py b/normalizer/models.py index 264c01a..102535a 100644 --- a/normalizer/models.py +++ b/normalizer/models.py @@ -40,7 +40,7 @@ class PackagePayload(BaseModel): env: Optional[Dict] = {} build_env: Optional[Dict] = {} dockerfile: Optional[str] = None - build_env_path: Optional[Path] = None + build_env_file: Optional[str] = None class NormalizeResult(BaseModel): diff --git a/server/routes/normalizer.py b/server/routes/normalizer.py index bf4f5cf..59ccb56 100644 --- a/server/routes/normalizer.py +++ b/server/routes/normalizer.py @@ -33,7 +33,7 @@ def normalize( meta=block_data.meta, env=block_data.env, build_env=block_data.build_env, - build_env_path=block_data.build_env_path, + build_env_file=block_data.build_env_file, dockerfile=block_data.dockerfile ) diff --git a/tests/cases/executor_7/Dockerfile.expect_file b/tests/cases/executor_7/Dockerfile.expect_file new file mode 100644 index 0000000..1fb4a25 --- /dev/null +++ b/tests/cases/executor_7/Dockerfile.expect_file @@ -0,0 +1,16 @@ +# This file is automatically generated by Jina executor normalizer plugin. +# It is not intended for manual editing. + +FROM jinaai/jina:3.6.10-py39-perf + +# install the third-party requirements +RUN --mount=type=secret,id=build_env . run/secrets/build_env && pip install . +ENTRYPOINT ["jina", "executor", "--uses", "config.yml"] + + +FROM jinaai/jina:3.6.10-py39-perf + +# install the third-party requirements +RUN --mount=type=secret,id=build_env . run/secrets/build_env && pip install --no-cache-dir -r requirements.txt + +ENTRYPOINT ["jina", "executor", "--uses", "config.yml"] diff --git a/tests/test_core.py b/tests/test_core.py index 7c573aa..14379b2 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -211,29 +211,34 @@ def test_normalized_custom_dockerfile(package_path, build_env, dockerfile): @pytest.mark.parametrize( - 'package_path, build_env_path', + 'package_path, build_env_file', [ ( Path(__file__).parent / 'cases' / 'executor_7', - Path(__file__).parent / 'cases' / 'build_env', + 'build_env', ) ], ) -def test_compare_dockerfile_env_vars_form_path(package_path, build_env_path): +def test_compare_dockerfile_env_vars_form_path(package_path, build_env_file): dockerfile_path = Path(package_path / 'Dockerfile') + dockerfile_expected_path = Path(package_path / 'Dockerfile.expect_file') origin_dockerfile_str = None; if dockerfile_path.exists(): with open(dockerfile_path, 'r') as fp: origin_dockerfile_str = str(fp.read()) - core.normalize(package_path, build_env_path=build_env_path, dry_run=False) + core.normalize(package_path, build_env_file=build_env_file, dry_run=False) assert dockerfile_path.exists() == True; dockerfileStr = None with open(dockerfile_path, 'r') as fp: dockerfileStr = str(fp.read()) + + dockerfileExpectedStr = '' + with open(dockerfile_expected_path, 'r') as fp: + dockerfileExpectedStr = str(fp.read()) if origin_dockerfile_str: with open(dockerfile_path, 'w') as fp: @@ -241,4 +246,4 @@ def test_compare_dockerfile_env_vars_form_path(package_path, build_env_path): else: os.remove(dockerfile_path) - assert f'--mount=type=secret,id=build_env,dst={str(build_env_path)}' in dockerfileStr \ No newline at end of file + assert dockerfileExpectedStr == dockerfileStr \ No newline at end of file