feat: add docarray install instructions in dockerfile (#65)

* feat: add docarray install instructions in dockerfile

* fix: cleaning

* test: assert docarray version

normalizer/core.py

@@ -688,6 +688,11 @@ def normalize(
     )
     new_dockerfile.set_entrypoint(entrypoint_value)
 
+    if 'docarray' in meta and '__jina__.Dockerfile' not in str(dockerfile_path):
+        dockerfile.add_docarray_install(meta["docarray"])
+        if not dry_run:
+            dockerfile.dump(dockerfile_path)
+
     new_dockerfile_path = work_path / '__jina__.Dockerfile'
     if not dry_run:
         new_dockerfile.dump(new_dockerfile_path)

normalizer/docker/parser.py

@@ -91,6 +91,19 @@ def add_pip_install(self):
             """
         )
 
+    def add_docarray_install(self, docArrayVersion):
+        instruction_template = dedent(
+            """\
+            RUN pip install --default-timeout=1000 --compile --no-cache-dir docarray=={0} # generated
+
+            """
+        )
+        content = instruction_template.format(docArrayVersion)
+        for instruction in self._parser.structure:
+            if instruction['instruction'] == 'ENTRYPOINT':
+                self._parser.add_lines_at(instruction['startline']-1, content, after=True)
+                break
+
     @property
     def is_multistage(self):
         return self._parser.is_multistage()

normalizer/resources/templates/dockerfile.base

@@ -7,6 +7,7 @@ FROM ${ARG_BASE_IMAGE}
 
 ARG ARG_JINA_VERSION
 ARG ARG_PIP_JINA_VERSION
+ARG ARG_DOCARRAY_VERSION
 ARG ARG_BUILD_DATE
 # the following label use ARG hence will invalid the cache
 LABEL org.opencontainers.image.created=${ARG_BUILD_DATE} \
@@ -29,4 +30,7 @@ ENV JINA_PIP_INSTALL_PERF=1
 
 # Need to uninstall jina first when upgrading 2.x to 3.x
 # https://github.com/jina-ai/jina/issues/4194
-RUN pip uninstall -y jina && pip install --upgrade ${ARG_PIP_JINA_VERSION}
+RUN pip uninstall -y jina && pip install --upgrade ${ARG_PIP_JINA_VERSION}
+RUN if [ "$ARG_DOCARRAY_VERSION" != "undefined" ] ; then \
+    pip uninstall -y docarray && pip install --upgrade docarray==${ARG_DOCARRAY_VERSION} ; \
+    fi

tests/cases/executor_8/Dockerfile

@@ -0,0 +1,6 @@
+FROM jinaai/jina:3.16.0
+
+COPY . /workspace
+WORKDIR /workspace
+
+ENTRYPOINT ["jina", "executor", "--uses", "config.yml"]

tests/cases/executor_8/Dockerfile.expect

@@ -0,0 +1,7 @@
+FROM jinaai/jina:3.16.0
+
+COPY . /workspace
+WORKDIR /workspace
+
+RUN pip install --default-timeout=1000 --compile --no-cache-dir docarray==0.30.0 # generated
+ENTRYPOINT ["jina", "executor", "--uses", "config.yml"]

tests/cases/executor_8/README.md

@@ -0,0 +1,2 @@
+# executor_7
+

tests/cases/executor_8/__jina__.Dockerfile

@@ -0,0 +1,37 @@
+# This file is automatically generated by Jina executor normalizer plugin.
+# It is not intended for manual editing.
+
+# ATTENTION: ARG before FROM will be invalid in the statements after FROM tag
+ARG ARG_BASE_IMAGE
+FROM ${ARG_BASE_IMAGE}
+
+ARG ARG_JINA_VERSION
+ARG ARG_PIP_JINA_VERSION
+ARG ARG_DOCARRAY_VERSION
+ARG ARG_BUILD_DATE
+# the following label use ARG hence will invalid the cache
+LABEL org.opencontainers.image.created=${ARG_BUILD_DATE} \
+      org.opencontainers.image.source="https://github.com/jina-ai/jina/commit/refs/tags/${ARG_JINA_VERSION}" \
+      org.opencontainers.image.version=${ARG_JINA_VERSION} \
+      org.opencontainers.image.revision=refs/tags/${ARG_JINA_VERSION}
+
+# the following env use ARG hence will invalid the cache
+ENV JINA_VERSION=${ARG_JINA_VERSION} \
+    JINA_VCS_VERSION=refs/tags/${ARG_JINA_VERSION} \
+    JINA_BUILD_DATE=${ARG_BUILD_DATE}
+
+# There is a history bug in Jina core.
+# Both JINA_PIP_INSTALL_CORE and JINA_PIP_INSTALL_PERF were set no matter in any cases
+# https://github.com/jina-ai/jina/pull/3673
+RUN unset JINA_PIP_INSTALL_CORE && \
+    unset JINA_PIP_INSTALL_PERF
+
+ENV JINA_PIP_INSTALL_PERF=1
+
+# Need to uninstall jina first when upgrading 2.x to 3.x
+# https://github.com/jina-ai/jina/issues/4194
+RUN pip uninstall -y jina && pip install --upgrade ${ARG_PIP_JINA_VERSION}
+RUN if [ "$ARG_DOCARRAY_VERSION" != "undefined" ] ; then \
+    pip uninstall -y docarray && pip install --upgrade docarray==${ARG_DOCARRAY_VERSION} ; \
+    fi
+ENTRYPOINT ["jina", "executor", "--uses", "config.yml"]

tests/cases/executor_8/config.yml

@@ -0,0 +1,4 @@
+jtype: executor_7
+metas:
+  py_modules:
+    - executor.py

tests/cases/executor_8/executor.py

@@ -0,0 +1,7 @@
+from jina import Executor, DocumentArray, requests
+
+
+class executor_7(Executor):
+    @requests
+    def foo(self, docs: DocumentArray, **kwargs):
+        pass

tests/cases/executor_9/Dockerfile

@@ -0,0 +1,6 @@
+FROM jinaai/jina:3.16.0
+
+COPY . /workspace
+WORKDIR /workspace
+
+ENTRYPOINT ["jina", "executor", "--uses", "config.yml"]

tests/cases/executor_9/Dockerfile.expect

@@ -0,0 +1,37 @@
+# This file is automatically generated by Jina executor normalizer plugin.
+# It is not intended for manual editing.
+
+# ATTENTION: ARG before FROM will be invalid in the statements after FROM tag
+ARG ARG_BASE_IMAGE
+FROM ${ARG_BASE_IMAGE}
+
+ARG ARG_JINA_VERSION
+ARG ARG_PIP_JINA_VERSION
+ARG ARG_DOCARRAY_VERSION
+ARG ARG_BUILD_DATE
+# the following label use ARG hence will invalid the cache
+LABEL org.opencontainers.image.created=${ARG_BUILD_DATE} \
+      org.opencontainers.image.source="https://github.com/jina-ai/jina/commit/refs/tags/${ARG_JINA_VERSION}" \
+      org.opencontainers.image.version=${ARG_JINA_VERSION} \
+      org.opencontainers.image.revision=refs/tags/${ARG_JINA_VERSION}
+
+# the following env use ARG hence will invalid the cache
+ENV JINA_VERSION=${ARG_JINA_VERSION} \
+    JINA_VCS_VERSION=refs/tags/${ARG_JINA_VERSION} \
+    JINA_BUILD_DATE=${ARG_BUILD_DATE}
+
+# There is a history bug in Jina core.
+# Both JINA_PIP_INSTALL_CORE and JINA_PIP_INSTALL_PERF were set no matter in any cases
+# https://github.com/jina-ai/jina/pull/3673
+RUN unset JINA_PIP_INSTALL_CORE && \
+    unset JINA_PIP_INSTALL_PERF
+
+ENV JINA_PIP_INSTALL_PERF=1
+
+# Need to uninstall jina first when upgrading 2.x to 3.x
+# https://github.com/jina-ai/jina/issues/4194
+RUN pip uninstall -y jina && pip install --upgrade ${ARG_PIP_JINA_VERSION}
+RUN if [ "$ARG_DOCARRAY_VERSION" != "undefined" ] ; then \
+    pip uninstall -y docarray && pip install --upgrade docarray==${ARG_DOCARRAY_VERSION} ; \
+    fi
+ENTRYPOINT ["jina", "executor", "--uses", "config.yml"]

tests/cases/executor_9/README.md

@@ -0,0 +1,2 @@
+# executor_7
+

tests/cases/executor_9/__jina__.Dockerfile

@@ -0,0 +1,37 @@
+# This file is automatically generated by Jina executor normalizer plugin.
+# It is not intended for manual editing.
+
+# ATTENTION: ARG before FROM will be invalid in the statements after FROM tag
+ARG ARG_BASE_IMAGE
+FROM ${ARG_BASE_IMAGE}
+
+ARG ARG_JINA_VERSION
+ARG ARG_PIP_JINA_VERSION
+ARG ARG_DOCARRAY_VERSION
+ARG ARG_BUILD_DATE
+# the following label use ARG hence will invalid the cache
+LABEL org.opencontainers.image.created=${ARG_BUILD_DATE} \
+      org.opencontainers.image.source="https://github.com/jina-ai/jina/commit/refs/tags/${ARG_JINA_VERSION}" \
+      org.opencontainers.image.version=${ARG_JINA_VERSION} \
+      org.opencontainers.image.revision=refs/tags/${ARG_JINA_VERSION}
+
+# the following env use ARG hence will invalid the cache
+ENV JINA_VERSION=${ARG_JINA_VERSION} \
+    JINA_VCS_VERSION=refs/tags/${ARG_JINA_VERSION} \
+    JINA_BUILD_DATE=${ARG_BUILD_DATE}
+
+# There is a history bug in Jina core.
+# Both JINA_PIP_INSTALL_CORE and JINA_PIP_INSTALL_PERF were set no matter in any cases
+# https://github.com/jina-ai/jina/pull/3673
+RUN unset JINA_PIP_INSTALL_CORE && \
+    unset JINA_PIP_INSTALL_PERF
+
+ENV JINA_PIP_INSTALL_PERF=1
+
+# Need to uninstall jina first when upgrading 2.x to 3.x
+# https://github.com/jina-ai/jina/issues/4194
+RUN pip uninstall -y jina && pip install --upgrade ${ARG_PIP_JINA_VERSION}
+RUN if [ "$ARG_DOCARRAY_VERSION" != "undefined" ] ; then \
+    pip uninstall -y docarray && pip install --upgrade docarray==${ARG_DOCARRAY_VERSION} ; \
+    fi
+ENTRYPOINT ["jina", "executor", "--uses", "config.yml"]

tests/cases/executor_9/config.yml

@@ -0,0 +1,4 @@
+jtype: executor_7
+metas:
+  py_modules:
+    - executor.py

tests/cases/executor_9/executor.py

@@ -0,0 +1,7 @@
+from jina import Executor, DocumentArray, requests
+
+
+class executor_7(Executor):
+    @requests
+    def foo(self, docs: DocumentArray, **kwargs):
+        pass

tests/test_core.py

@@ -98,37 +98,57 @@ def test_get_executor_args(package_path, expected_path):
 
 
 @pytest.mark.parametrize(
-    'package_path, dockerfile_syntax',
+    'package_path, dockerfile_syntax, meta_dict, dockerfile_name',
     [
         (
             Path(__file__).parent / 'cases' / 'executor_1',
-            None
+            None,
+            {'jina': '2'},
+            'Dockerfile'
         ),
          (
             Path(__file__).parent / 'cases' / 'executor_2',
-            None
+            None,
+            {'jina': '2'},
+            'Dockerfile'
         ),
         (
             Path(__file__).parent / 'cases' / 'executor_6',
-            'jinahub/dockerfile:1.4.3-magic-shell'
+            'jinahub/dockerfile:1.4.3-magic-shell',
+            {'jina': '2'},
+            'Dockerfile'
         ),
         (
             Path(__file__).parent / 'cases' / 'executor_7',
-            'jinahub/dockerfile:1.4.3-magic-shell'
+            'jinahub/dockerfile:1.4.3-magic-shell',
+            {'jina': '2'},
+            'Dockerfile'
+        ),
+        (
+            Path(__file__).parent / 'cases' / 'executor_8',
+            None,
+            {'jina': '3.16.0', 'docarray': '0.30.0'},
+            'Dockerfile'
+        ),
+        (
+            Path(__file__).parent / 'cases' / 'executor_9',
+            None,
+            {'jina': '3.16.0', 'docarray': '0.30.0'},
+            '__jina__.Dockerfile'
         ),
     ],
 )
-def test_compare_dockerfile_syntax(package_path, dockerfile_syntax):
+def test_compare_dockerfile_syntax(package_path, dockerfile_syntax, meta_dict, dockerfile_name):
 
-    dockerfile_path = Path(package_path / 'Dockerfile')
+    dockerfile_path = Path(package_path / dockerfile_name)
     dockerfile_expected_path = Path(package_path / 'Dockerfile.expect')
 
     originDockerfileStr = None;
     if dockerfile_path.exists():
         with open(dockerfile_path, 'r') as fp:
             originDockerfileStr = str(fp.read())
 
-    core.normalize(package_path, dockerfile_syntax=dockerfile_syntax, dry_run=False)
+    core.normalize(package_path, dockerfile_syntax=dockerfile_syntax, dry_run=False, meta=meta_dict, dockerfile=dockerfile_name)
     assert dockerfile_path.exists() == True;
 
     dockerfileStr = None