diff --git a/normalizer/core.py b/normalizer/core.py index 522bce5..6053187 100644 --- a/normalizer/core.py +++ b/normalizer/core.py @@ -688,6 +688,11 @@ def normalize( ) new_dockerfile.set_entrypoint(entrypoint_value) + if 'docarray' in meta and '__jina__.Dockerfile' not in str(dockerfile_path): + dockerfile.add_docarray_install(meta["docarray"]) + if not dry_run: + dockerfile.dump(dockerfile_path) + new_dockerfile_path = work_path / '__jina__.Dockerfile' if not dry_run: new_dockerfile.dump(new_dockerfile_path) diff --git a/normalizer/docker/parser.py b/normalizer/docker/parser.py index 20b267b..8d5533f 100644 --- a/normalizer/docker/parser.py +++ b/normalizer/docker/parser.py @@ -91,6 +91,19 @@ def add_pip_install(self): """ ) + def add_docarray_install(self, docArrayVersion): + instruction_template = dedent( + """\ + RUN pip install --default-timeout=1000 --compile --no-cache-dir docarray=={0} # generated + + """ + ) + content = instruction_template.format(docArrayVersion) + for instruction in self._parser.structure: + if instruction['instruction'] == 'ENTRYPOINT': + self._parser.add_lines_at(instruction['startline']-1, content, after=True) + break + @property def is_multistage(self): return self._parser.is_multistage() diff --git a/normalizer/resources/templates/dockerfile.base b/normalizer/resources/templates/dockerfile.base index 9c78486..e6262c9 100644 --- a/normalizer/resources/templates/dockerfile.base +++ b/normalizer/resources/templates/dockerfile.base @@ -7,6 +7,7 @@ FROM ${ARG_BASE_IMAGE} ARG ARG_JINA_VERSION ARG ARG_PIP_JINA_VERSION +ARG ARG_DOCARRAY_VERSION ARG ARG_BUILD_DATE # the following label use ARG hence will invalid the cache LABEL org.opencontainers.image.created=${ARG_BUILD_DATE} \ @@ -29,4 +30,7 @@ ENV JINA_PIP_INSTALL_PERF=1 # Need to uninstall jina first when upgrading 2.x to 3.x # https://github.com/jina-ai/jina/issues/4194 -RUN pip uninstall -y jina && pip install --upgrade ${ARG_PIP_JINA_VERSION} \ No newline at end of file +RUN pip uninstall -y jina && pip install --upgrade ${ARG_PIP_JINA_VERSION} +RUN if [ "$ARG_DOCARRAY_VERSION" != "undefined" ] ; then \ + pip uninstall -y docarray && pip install --upgrade docarray==${ARG_DOCARRAY_VERSION} ; \ + fi \ No newline at end of file diff --git a/tests/cases/executor_8/Dockerfile b/tests/cases/executor_8/Dockerfile new file mode 100644 index 0000000..c361b7a --- /dev/null +++ b/tests/cases/executor_8/Dockerfile @@ -0,0 +1,6 @@ +FROM jinaai/jina:3.16.0 + +COPY . /workspace +WORKDIR /workspace + +ENTRYPOINT ["jina", "executor", "--uses", "config.yml"] \ No newline at end of file diff --git a/tests/cases/executor_8/Dockerfile.expect b/tests/cases/executor_8/Dockerfile.expect new file mode 100644 index 0000000..e31e830 --- /dev/null +++ b/tests/cases/executor_8/Dockerfile.expect @@ -0,0 +1,7 @@ +FROM jinaai/jina:3.16.0 + +COPY . /workspace +WORKDIR /workspace + +RUN pip install --default-timeout=1000 --compile --no-cache-dir docarray==0.30.0 # generated +ENTRYPOINT ["jina", "executor", "--uses", "config.yml"] \ No newline at end of file diff --git a/tests/cases/executor_8/README.md b/tests/cases/executor_8/README.md new file mode 100644 index 0000000..07e688e --- /dev/null +++ b/tests/cases/executor_8/README.md @@ -0,0 +1,2 @@ +# executor_7 + diff --git a/tests/cases/executor_8/__jina__.Dockerfile b/tests/cases/executor_8/__jina__.Dockerfile new file mode 100644 index 0000000..944b60f --- /dev/null +++ b/tests/cases/executor_8/__jina__.Dockerfile @@ -0,0 +1,37 @@ +# This file is automatically generated by Jina executor normalizer plugin. +# It is not intended for manual editing. + +# ATTENTION: ARG before FROM will be invalid in the statements after FROM tag +ARG ARG_BASE_IMAGE +FROM ${ARG_BASE_IMAGE} + +ARG ARG_JINA_VERSION +ARG ARG_PIP_JINA_VERSION +ARG ARG_DOCARRAY_VERSION +ARG ARG_BUILD_DATE +# the following label use ARG hence will invalid the cache +LABEL org.opencontainers.image.created=${ARG_BUILD_DATE} \ + org.opencontainers.image.source="https://github.com/jina-ai/jina/commit/refs/tags/${ARG_JINA_VERSION}" \ + org.opencontainers.image.version=${ARG_JINA_VERSION} \ + org.opencontainers.image.revision=refs/tags/${ARG_JINA_VERSION} + +# the following env use ARG hence will invalid the cache +ENV JINA_VERSION=${ARG_JINA_VERSION} \ + JINA_VCS_VERSION=refs/tags/${ARG_JINA_VERSION} \ + JINA_BUILD_DATE=${ARG_BUILD_DATE} + +# There is a history bug in Jina core. +# Both JINA_PIP_INSTALL_CORE and JINA_PIP_INSTALL_PERF were set no matter in any cases +# https://github.com/jina-ai/jina/pull/3673 +RUN unset JINA_PIP_INSTALL_CORE && \ + unset JINA_PIP_INSTALL_PERF + +ENV JINA_PIP_INSTALL_PERF=1 + +# Need to uninstall jina first when upgrading 2.x to 3.x +# https://github.com/jina-ai/jina/issues/4194 +RUN pip uninstall -y jina && pip install --upgrade ${ARG_PIP_JINA_VERSION} +RUN if [ "$ARG_DOCARRAY_VERSION" != "undefined" ] ; then \ + pip uninstall -y docarray && pip install --upgrade docarray==${ARG_DOCARRAY_VERSION} ; \ + fi +ENTRYPOINT ["jina", "executor", "--uses", "config.yml"] diff --git a/tests/cases/executor_8/config.yml b/tests/cases/executor_8/config.yml new file mode 100644 index 0000000..e62ba4a --- /dev/null +++ b/tests/cases/executor_8/config.yml @@ -0,0 +1,4 @@ +jtype: executor_7 +metas: + py_modules: + - executor.py diff --git a/tests/cases/executor_8/executor.py b/tests/cases/executor_8/executor.py new file mode 100644 index 0000000..a9ae4b1 --- /dev/null +++ b/tests/cases/executor_8/executor.py @@ -0,0 +1,7 @@ +from jina import Executor, DocumentArray, requests + + +class executor_7(Executor): + @requests + def foo(self, docs: DocumentArray, **kwargs): + pass diff --git a/tests/cases/executor_8/requirements.txt b/tests/cases/executor_8/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/tests/cases/executor_9/Dockerfile b/tests/cases/executor_9/Dockerfile new file mode 100644 index 0000000..c361b7a --- /dev/null +++ b/tests/cases/executor_9/Dockerfile @@ -0,0 +1,6 @@ +FROM jinaai/jina:3.16.0 + +COPY . /workspace +WORKDIR /workspace + +ENTRYPOINT ["jina", "executor", "--uses", "config.yml"] \ No newline at end of file diff --git a/tests/cases/executor_9/Dockerfile.expect b/tests/cases/executor_9/Dockerfile.expect new file mode 100644 index 0000000..944b60f --- /dev/null +++ b/tests/cases/executor_9/Dockerfile.expect @@ -0,0 +1,37 @@ +# This file is automatically generated by Jina executor normalizer plugin. +# It is not intended for manual editing. + +# ATTENTION: ARG before FROM will be invalid in the statements after FROM tag +ARG ARG_BASE_IMAGE +FROM ${ARG_BASE_IMAGE} + +ARG ARG_JINA_VERSION +ARG ARG_PIP_JINA_VERSION +ARG ARG_DOCARRAY_VERSION +ARG ARG_BUILD_DATE +# the following label use ARG hence will invalid the cache +LABEL org.opencontainers.image.created=${ARG_BUILD_DATE} \ + org.opencontainers.image.source="https://github.com/jina-ai/jina/commit/refs/tags/${ARG_JINA_VERSION}" \ + org.opencontainers.image.version=${ARG_JINA_VERSION} \ + org.opencontainers.image.revision=refs/tags/${ARG_JINA_VERSION} + +# the following env use ARG hence will invalid the cache +ENV JINA_VERSION=${ARG_JINA_VERSION} \ + JINA_VCS_VERSION=refs/tags/${ARG_JINA_VERSION} \ + JINA_BUILD_DATE=${ARG_BUILD_DATE} + +# There is a history bug in Jina core. +# Both JINA_PIP_INSTALL_CORE and JINA_PIP_INSTALL_PERF were set no matter in any cases +# https://github.com/jina-ai/jina/pull/3673 +RUN unset JINA_PIP_INSTALL_CORE && \ + unset JINA_PIP_INSTALL_PERF + +ENV JINA_PIP_INSTALL_PERF=1 + +# Need to uninstall jina first when upgrading 2.x to 3.x +# https://github.com/jina-ai/jina/issues/4194 +RUN pip uninstall -y jina && pip install --upgrade ${ARG_PIP_JINA_VERSION} +RUN if [ "$ARG_DOCARRAY_VERSION" != "undefined" ] ; then \ + pip uninstall -y docarray && pip install --upgrade docarray==${ARG_DOCARRAY_VERSION} ; \ + fi +ENTRYPOINT ["jina", "executor", "--uses", "config.yml"] diff --git a/tests/cases/executor_9/README.md b/tests/cases/executor_9/README.md new file mode 100644 index 0000000..07e688e --- /dev/null +++ b/tests/cases/executor_9/README.md @@ -0,0 +1,2 @@ +# executor_7 + diff --git a/tests/cases/executor_9/__jina__.Dockerfile b/tests/cases/executor_9/__jina__.Dockerfile new file mode 100644 index 0000000..944b60f --- /dev/null +++ b/tests/cases/executor_9/__jina__.Dockerfile @@ -0,0 +1,37 @@ +# This file is automatically generated by Jina executor normalizer plugin. +# It is not intended for manual editing. + +# ATTENTION: ARG before FROM will be invalid in the statements after FROM tag +ARG ARG_BASE_IMAGE +FROM ${ARG_BASE_IMAGE} + +ARG ARG_JINA_VERSION +ARG ARG_PIP_JINA_VERSION +ARG ARG_DOCARRAY_VERSION +ARG ARG_BUILD_DATE +# the following label use ARG hence will invalid the cache +LABEL org.opencontainers.image.created=${ARG_BUILD_DATE} \ + org.opencontainers.image.source="https://github.com/jina-ai/jina/commit/refs/tags/${ARG_JINA_VERSION}" \ + org.opencontainers.image.version=${ARG_JINA_VERSION} \ + org.opencontainers.image.revision=refs/tags/${ARG_JINA_VERSION} + +# the following env use ARG hence will invalid the cache +ENV JINA_VERSION=${ARG_JINA_VERSION} \ + JINA_VCS_VERSION=refs/tags/${ARG_JINA_VERSION} \ + JINA_BUILD_DATE=${ARG_BUILD_DATE} + +# There is a history bug in Jina core. +# Both JINA_PIP_INSTALL_CORE and JINA_PIP_INSTALL_PERF were set no matter in any cases +# https://github.com/jina-ai/jina/pull/3673 +RUN unset JINA_PIP_INSTALL_CORE && \ + unset JINA_PIP_INSTALL_PERF + +ENV JINA_PIP_INSTALL_PERF=1 + +# Need to uninstall jina first when upgrading 2.x to 3.x +# https://github.com/jina-ai/jina/issues/4194 +RUN pip uninstall -y jina && pip install --upgrade ${ARG_PIP_JINA_VERSION} +RUN if [ "$ARG_DOCARRAY_VERSION" != "undefined" ] ; then \ + pip uninstall -y docarray && pip install --upgrade docarray==${ARG_DOCARRAY_VERSION} ; \ + fi +ENTRYPOINT ["jina", "executor", "--uses", "config.yml"] diff --git a/tests/cases/executor_9/config.yml b/tests/cases/executor_9/config.yml new file mode 100644 index 0000000..e62ba4a --- /dev/null +++ b/tests/cases/executor_9/config.yml @@ -0,0 +1,4 @@ +jtype: executor_7 +metas: + py_modules: + - executor.py diff --git a/tests/cases/executor_9/executor.py b/tests/cases/executor_9/executor.py new file mode 100644 index 0000000..a9ae4b1 --- /dev/null +++ b/tests/cases/executor_9/executor.py @@ -0,0 +1,7 @@ +from jina import Executor, DocumentArray, requests + + +class executor_7(Executor): + @requests + def foo(self, docs: DocumentArray, **kwargs): + pass diff --git a/tests/cases/executor_9/requirements.txt b/tests/cases/executor_9/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_core.py b/tests/test_core.py index fa10fe7..9909fb4 100644 --- a/tests/test_core.py +++ b/tests/test_core.py @@ -98,29 +98,49 @@ def test_get_executor_args(package_path, expected_path): @pytest.mark.parametrize( - 'package_path, dockerfile_syntax', + 'package_path, dockerfile_syntax, meta_dict, dockerfile_name', [ ( Path(__file__).parent / 'cases' / 'executor_1', - None + None, + {'jina': '2'}, + 'Dockerfile' ), ( Path(__file__).parent / 'cases' / 'executor_2', - None + None, + {'jina': '2'}, + 'Dockerfile' ), ( Path(__file__).parent / 'cases' / 'executor_6', - 'jinahub/dockerfile:1.4.3-magic-shell' + 'jinahub/dockerfile:1.4.3-magic-shell', + {'jina': '2'}, + 'Dockerfile' ), ( Path(__file__).parent / 'cases' / 'executor_7', - 'jinahub/dockerfile:1.4.3-magic-shell' + 'jinahub/dockerfile:1.4.3-magic-shell', + {'jina': '2'}, + 'Dockerfile' + ), + ( + Path(__file__).parent / 'cases' / 'executor_8', + None, + {'jina': '3.16.0', 'docarray': '0.30.0'}, + 'Dockerfile' + ), + ( + Path(__file__).parent / 'cases' / 'executor_9', + None, + {'jina': '3.16.0', 'docarray': '0.30.0'}, + '__jina__.Dockerfile' ), ], ) -def test_compare_dockerfile_syntax(package_path, dockerfile_syntax): +def test_compare_dockerfile_syntax(package_path, dockerfile_syntax, meta_dict, dockerfile_name): - dockerfile_path = Path(package_path / 'Dockerfile') + dockerfile_path = Path(package_path / dockerfile_name) dockerfile_expected_path = Path(package_path / 'Dockerfile.expect') originDockerfileStr = None; @@ -128,7 +148,7 @@ def test_compare_dockerfile_syntax(package_path, dockerfile_syntax): with open(dockerfile_path, 'r') as fp: originDockerfileStr = str(fp.read()) - core.normalize(package_path, dockerfile_syntax=dockerfile_syntax, dry_run=False) + core.normalize(package_path, dockerfile_syntax=dockerfile_syntax, dry_run=False, meta=meta_dict, dockerfile=dockerfile_name) assert dockerfile_path.exists() == True; dockerfileStr = None