From 0d2e5dffcd62aefc7b146fc415ee514be1f602a3 Mon Sep 17 00:00:00 2001 From: zhangkai Date: Mon, 7 Nov 2022 11:26:13 +0800 Subject: [PATCH] feat: use build env file to load secret environs dynamically (#61) * feat: load secret environs dynamically by using file src * style: improve the the style of code * style: improve the style of code * refactore: load secrets with --id and /run/secrets/id --- normalizer/core.py | 7 ++++ normalizer/docker/parser.py | 33 ++++++++++----- normalizer/models.py | 1 + server/routes/normalizer.py | 1 + tests/cases/build_env | 2 + tests/cases/executor_7/Dockerfile.expect_file | 16 ++++++++ tests/test_core.py | 41 ++++++++++++++++++- 7 files changed, 89 insertions(+), 12 deletions(-) create mode 100644 tests/cases/build_env create mode 100644 tests/cases/executor_7/Dockerfile.expect_file diff --git a/normalizer/core.py b/normalizer/core.py index d22cce0..d33ffc7 100644 --- a/normalizer/core.py +++ b/normalizer/core.py @@ -371,6 +371,7 @@ def normalize( build_env: Dict = {}, dry_run: bool = False, dockerfile: Optional[str] = None, + build_env_file: Optional[str] = None, **kwargs, ) -> ExecutorModel: """Normalize the executor package. @@ -621,6 +622,9 @@ def normalize( ) if build_env and isinstance(build_env, dict) and len(build_env.keys()): dockerfile.insert_build_env(build_env) + + if build_env_file: + dockerfile.insert_build_env_file(build_env_file) # if dockerfile.is_multistage(): # # Don't support multi-stage Dockerfie Optimization # return @@ -653,6 +657,9 @@ def normalize( if build_env and isinstance(build_env, dict) and len(build_env.keys()): dockerfile.insert_build_env(build_env) + if build_env_file: + dockerfile.insert_build_env_file(build_env_file) + # if len(test_glob) > 0: # dockerfile.add_unitest() diff --git a/normalizer/docker/parser.py b/normalizer/docker/parser.py index 89e4d9c..40b2406 100644 --- a/normalizer/docker/parser.py +++ b/normalizer/docker/parser.py @@ -1,13 +1,15 @@ import io -from re import template -from normalizer import docker +import pathlib +import re from pathlib import Path from posixpath import basename +from re import template from textwrap import dedent from typing import Dict, List -import re -RUN_VAR_RE = re.compile(r"(?P(?P^RUN))") +from normalizer import docker + +RUN_VAR_RE = re.compile(r'(?P(?P^RUN))') from dockerfile_parse import DockerfileParser @@ -44,16 +46,25 @@ def __init__( def __str__(self): return self.content - - def insert_build_env(self, build_env): + + def insert_build_env(self, build_env: Dict): build_env_str = '' for index, env in enumerate(build_env): - build_env_str += dedent( - f' --mount=type=secret,id={env} ' - ) + build_env_str += dedent(f' --mount=type=secret,id={env} ') for index, env in enumerate(build_env): - build_env_str += dedent( - f' export {env}=\"$(cat /run/secrets/{env})\" ' + build_env_str += dedent(f' export {env}=\"$(cat /run/secrets/{env})\" ') + build_env_str += " && " + + for index, line in enumerate(self._parser.lines): + strip_line = line.strip() + if RUN_VAR_RE.match(strip_line): + replace_line = line.replace('RUN', f'RUN {build_env_str}') + self._parser.content = self._parser.content.replace(line, replace_line) + + def insert_build_env_file(self, build_env_file: str): + build_env_str = '' + build_env_str += dedent( + f' --mount=type=secret,id={build_env_file} . run/secrets/{build_env_file} ' ) build_env_str += ' && ' diff --git a/normalizer/models.py b/normalizer/models.py index 9f428aa..102535a 100644 --- a/normalizer/models.py +++ b/normalizer/models.py @@ -40,6 +40,7 @@ class PackagePayload(BaseModel): env: Optional[Dict] = {} build_env: Optional[Dict] = {} dockerfile: Optional[str] = None + build_env_file: Optional[str] = None class NormalizeResult(BaseModel): diff --git a/server/routes/normalizer.py b/server/routes/normalizer.py index 01a63fa..59ccb56 100644 --- a/server/routes/normalizer.py +++ b/server/routes/normalizer.py @@ -33,6 +33,7 @@ def normalize( meta=block_data.meta, env=block_data.env, build_env=block_data.build_env, + build_env_file=block_data.build_env_file, dockerfile=block_data.dockerfile ) diff --git a/tests/cases/build_env b/tests/cases/build_env new file mode 100644 index 0000000..bfb9a9e --- /dev/null +++ b/tests/cases/build_env @@ -0,0 +1,2 @@ +export DOMAIN="DOMAIN" +export REPO="REPO" \ No newline at end of file diff --git a/tests/cases/executor_7/Dockerfile.expect_file b/tests/cases/executor_7/Dockerfile.expect_file new file mode 100644 index 0000000..1fb4a25 --- /dev/null +++ b/tests/cases/executor_7/Dockerfile.expect_file @@ -0,0 +1,16 @@ +# This file is automatically generated by Jina executor normalizer plugin. +# It is not intended for manual editing. + +FROM jinaai/jina:3.6.10-py39-perf + +# install the third-party requirements +RUN --mount=type=secret,id=build_env . run/secrets/build_env && pip install . +ENTRYPOINT ["jina", "executor", "--uses", "config.yml"] + + +FROM jinaai/jina:3.6.10-py39-perf + +# install the third-party requirements +RUN --mount=type=secret,id=build_env . run/secrets/build_env && pip install --no-cache-dir -r requirements.txt + +ENTRYPOINT ["jina", "executor", "--uses", "config.yml"] diff --git a/tests/test_core.py b/tests/test_core.py index ac6cb5c..14379b2 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -207,4 +207,43 @@ def test_normalized_custom_dockerfile(package_path, build_env, dockerfile): if originDockerfileStr: with open(dockerfile_path, 'w') as fp: - fp.write(originDockerfileStr) \ No newline at end of file + fp.write(originDockerfileStr) + + +@pytest.mark.parametrize( + 'package_path, build_env_file', + [ + ( + Path(__file__).parent / 'cases' / 'executor_7', + 'build_env', + ) + ], +) +def test_compare_dockerfile_env_vars_form_path(package_path, build_env_file): + + dockerfile_path = Path(package_path / 'Dockerfile') + dockerfile_expected_path = Path(package_path / 'Dockerfile.expect_file') + + origin_dockerfile_str = None; + if dockerfile_path.exists(): + with open(dockerfile_path, 'r') as fp: + origin_dockerfile_str = str(fp.read()) + + core.normalize(package_path, build_env_file=build_env_file, dry_run=False) + assert dockerfile_path.exists() == True; + + dockerfileStr = None + with open(dockerfile_path, 'r') as fp: + dockerfileStr = str(fp.read()) + + dockerfileExpectedStr = '' + with open(dockerfile_expected_path, 'r') as fp: + dockerfileExpectedStr = str(fp.read()) + + if origin_dockerfile_str: + with open(dockerfile_path, 'w') as fp: + fp.write(origin_dockerfile_str) + else: + os.remove(dockerfile_path) + + assert dockerfileExpectedStr == dockerfileStr \ No newline at end of file