feat: use build env file to load secret environs dynamically (#61)

* feat: load secret environs dynamically by using file src

* style: improve the the style of code

* style: improve the style of code

* refactore: load secrets with --id and /run/secrets/id

normalizer/core.py

@@ -371,6 +371,7 @@ def normalize(
     build_env: Dict = {},
     dry_run: bool = False,
     dockerfile: Optional[str] = None,
+    build_env_file: Optional[str] = None,
     **kwargs,
 ) -> ExecutorModel:
     """Normalize the executor package.
@@ -621,6 +622,9 @@ def normalize(
         )
         if build_env and isinstance(build_env, dict) and len(build_env.keys()):
             dockerfile.insert_build_env(build_env)
+
+        if build_env_file:
+            dockerfile.insert_build_env_file(build_env_file)
         # if dockerfile.is_multistage():
         #     # Don't support multi-stage Dockerfie Optimization
         #     return
@@ -653,6 +657,9 @@ def normalize(
         if build_env and isinstance(build_env, dict) and len(build_env.keys()):
             dockerfile.insert_build_env(build_env)
 
+        if build_env_file:
+            dockerfile.insert_build_env_file(build_env_file)
+
         # if len(test_glob) > 0:
         #     dockerfile.add_unitest()
 

normalizer/docker/parser.py

@@ -1,13 +1,15 @@
 import io
-from re import template
-from normalizer import docker
+import pathlib
+import re
 from pathlib import Path
 from posixpath import basename
+from re import template
 from textwrap import dedent
 from typing import Dict, List
-import re
 
-RUN_VAR_RE = re.compile(r"(?P<var>(?P<name>^RUN))")
+from normalizer import docker
+
+RUN_VAR_RE = re.compile(r'(?P<var>(?P<name>^RUN))')
 
 from dockerfile_parse import DockerfileParser
 
@@ -44,16 +46,25 @@ def __init__(
 
     def __str__(self):
         return self.content
-    
-    def insert_build_env(self, build_env):
+
+    def insert_build_env(self, build_env: Dict):
         build_env_str = ''
         for index, env in enumerate(build_env):
-            build_env_str += dedent(
-            f' --mount=type=secret,id={env} '
-        )
+            build_env_str += dedent(f' --mount=type=secret,id={env} ')
         for index, env in enumerate(build_env):
-            build_env_str += dedent(
-            f' export {env}=\"$(cat /run/secrets/{env})\" '
+            build_env_str += dedent(f' export {env}=\"$(cat /run/secrets/{env})\" ')
+        build_env_str += " && "
+
+        for index, line in enumerate(self._parser.lines):
+            strip_line = line.strip()
+            if RUN_VAR_RE.match(strip_line):
+                replace_line = line.replace('RUN', f'RUN {build_env_str}')
+                self._parser.content = self._parser.content.replace(line, replace_line)
+
+    def insert_build_env_file(self, build_env_file: str):
+        build_env_str = ''
+        build_env_str += dedent(
+            f' --mount=type=secret,id={build_env_file} . run/secrets/{build_env_file} '
         )
         build_env_str += ' && '
 

normalizer/models.py

@@ -40,6 +40,7 @@ class PackagePayload(BaseModel):
     env: Optional[Dict] = {}
     build_env: Optional[Dict] = {}
     dockerfile: Optional[str] = None
+    build_env_file: Optional[str] = None
 
 
 class NormalizeResult(BaseModel):

server/routes/normalizer.py

@@ -33,6 +33,7 @@ def normalize(
             meta=block_data.meta,
             env=block_data.env,
             build_env=block_data.build_env,
+            build_env_file=block_data.build_env_file,
             dockerfile=block_data.dockerfile
         )
 

tests/cases/build_env

@@ -0,0 +1,2 @@
+export DOMAIN="DOMAIN"
+export REPO="REPO"

tests/cases/executor_7/Dockerfile.expect_file

@@ -0,0 +1,16 @@
+# This file is automatically generated by Jina executor normalizer plugin.
+# It is not intended for manual editing.
+
+FROM jinaai/jina:3.6.10-py39-perf        
+
+# install the third-party requirements
+RUN --mount=type=secret,id=build_env . run/secrets/build_env  &&   pip install .
+ENTRYPOINT ["jina", "executor", "--uses", "config.yml"]
+
+
+FROM jinaai/jina:3.6.10-py39-perf     
+
+# install the third-party requirements
+RUN --mount=type=secret,id=build_env . run/secrets/build_env  &&   pip install --no-cache-dir -r requirements.txt
+
+ENTRYPOINT ["jina", "executor", "--uses", "config.yml"]

tests/test_core.py

@@ -207,4 +207,43 @@ def test_normalized_custom_dockerfile(package_path, build_env, dockerfile):
 
     if originDockerfileStr:
         with open(dockerfile_path, 'w') as fp:
-            fp.write(originDockerfileStr)
+            fp.write(originDockerfileStr)
+
+
+@pytest.mark.parametrize(
+    'package_path, build_env_file',
+    [   
+         (
+            Path(__file__).parent / 'cases' / 'executor_7',
+            'build_env',
+        )
+    ],
+)
+def test_compare_dockerfile_env_vars_form_path(package_path, build_env_file):
+
+    dockerfile_path = Path(package_path / 'Dockerfile') 
+    dockerfile_expected_path = Path(package_path / 'Dockerfile.expect_file')
+
+    origin_dockerfile_str = None;
+    if dockerfile_path.exists():
+        with open(dockerfile_path, 'r') as fp:
+            origin_dockerfile_str = str(fp.read())
+
+    core.normalize(package_path, build_env_file=build_env_file, dry_run=False)
+    assert dockerfile_path.exists() == True;
+
+    dockerfileStr = None
+    with open(dockerfile_path, 'r') as fp:
+        dockerfileStr = str(fp.read())
+
+    dockerfileExpectedStr = ''
+    with open(dockerfile_expected_path, 'r') as fp:
+        dockerfileExpectedStr = str(fp.read())
+
+    if origin_dockerfile_str:
+        with open(dockerfile_path, 'w') as fp:
+            fp.write(origin_dockerfile_str)
+    else:
+        os.remove(dockerfile_path)
+
+    assert dockerfileExpectedStr == dockerfileStr