refactor: moving jinad into daemon module of the core (#1610)

* refactor: moving jinad into daemon module of the core
jina-ai · Jan 7, 2021 · a338e51 · a338e51
1 parent 7673ccf
commit a338e51
Show file tree

Hide file tree

Showing 119 changed files with 2,742 additions and 610 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -8,14 +8,5 @@
 !*requirements.txt
 !README.md
 !MANIFEST.in
-!tests/**
 !cli/**
-
-# Ignore unnecessary files inside allowed directories
-# This should go after the allowed directories
-
-**/*.log
-**/Thumbs.db
-**/.pyre/
-**/.eggs/
-__pycache__
+!daemon/**
diff --git a/.github/labeler.yml b/.github/labeler.yml
@@ -42,6 +42,9 @@ area/docker:
 area/cli:
   - cli/**/*
 
+area/daemon:
+  - daemon/**/*
+
 area/script:
   - script/**/*
 

diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
@@ -39,7 +39,7 @@ jobs:
         with:
           python-version: 3.7
       - run: |
-          pip install .  --no-cache-dir
+          pip install .[daemon] --no-cache-dir
           jina check --summary-exec ./docs/chapters/all_exec.md --summary-driver ./docs/chapters/all_driver.md
           echo "JINA_VERSION=$(sed -n '/^__version__/p' ./jina/__init__.py | cut -d \' -f2)-devel" >> $GITHUB_ENV
           pip install css-html-js-minify
@@ -49,6 +49,17 @@ jobs:
           cp ./jina/resources/hub-builder-success/style.min.css ./schema/hub
           cd schema
           jina export-api --yaml-path "${{env.JINA_VERSION}}.yml" devel.yml --json-path "${{env.JINA_VERSION}}.json" devel.json devel
+          mkdir -p jinad && cd jinad
+          python -c "from daemon import _write_openapi_schema; _write_openapi_schema()"
+          cd -
+      - name: redoc-cli-jinad
+        uses: seeebiii/redoc-cli-github-action@v10
+        with:
+          args: 'bundle schema/jinad/schema.json -o jinad.html'
+      - name: push-to-api-repo
+        run: |
+          cp jinad.html schema/jinad/${{env.JINA_VERSION}}-devel.html
+          cd schema
           git config --local user.email "dev-bot@jina.ai"
           git config --local user.name "Jina Dev Bot"
           git add . && git commit -m "update ${{env.JINA_VERSION}} due to ${{github.event_name}} on ${{github.repository}}" && git push
@@ -70,18 +81,19 @@ jobs:
   update-docker:
     needs: update-doc
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - tag_prefix: ""
+            build_option: "--target jina_base"
+          - tag_prefix: "daemon-"
+            build_option: "--target jina_daemon"
     steps:
       - uses: actions/checkout@v2
         with:
           submodules: true  # this job build devel image, devel image includes hub submodule
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: crazy-max/ghaction-docker-buildx@v1
-        with:
-          buildx-version: latest
       - name: Set envs and versions
         run: |
-          echo ${{ steps.buildx.outputs.platforms }}
           echo "VCS_REF=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
           echo "BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_ENV
           echo "INSTALL_DEV=true" >> $GITHUB_ENV
@@ -94,7 +106,8 @@ jobs:
           password: ${{ secrets.DOCKERHUB_DEVBOT_TOKEN }}
           dockerfile: Dockerfiles/debianx.Dockerfile
           buildargs: BUILD_DATE, VCS_REF, JINA_VERSION, INSTALL_DEV
-          tags: "devel, ${{env.JINA_VERSION}}"
+          buildoptions: ${{ matrix.build_option }}
+          tags: "${{ matrix.tag_prefix }}latest, ${{ matrix.tag_prefix }}${{env.JINA_VERSION}}, ${{ matrix.tag_prefix }}${{env.JINA_MINOR_VERSION}}"
       - name: Upload to Github Docker Registry
         uses: elgohr/Publish-Docker-Github-Action@master
         with:
@@ -104,7 +117,8 @@ jobs:
           registry: docker.pkg.github.com
           dockerfile: Dockerfiles/debianx.Dockerfile
           buildargs: BUILD_DATE, VCS_REF, JINA_VERSION, INSTALL_DEV
-          tags: "devel, ${{env.JINA_VERSION}}"
+          buildoptions: ${{ matrix.build_option }}
+          tags: "${{ matrix.tag_prefix }}latest, ${{ matrix.tag_prefix }}${{env.JINA_VERSION}}, ${{ matrix.tag_prefix }}${{env.JINA_MINOR_VERSION}}"
 
   unit-test:
     needs: update-docker

diff --git a/.github/workflows/tag.yml b/.github/workflows/tag.yml
@@ -24,10 +24,21 @@ jobs:
       - run: |
           echo "JINA_VERSION=${GITHUB_REF/refs\/tags\/v/}" >> $GITHUB_ENV
           echo "V_JINA_VERSION=${GITHUB_REF/refs\/tags\//}" >> $GITHUB_ENV
-          pip install .  --no-cache-dir
-      - run: |
+          pip install .[daemon] --no-cache-dir
           cd schema
           jina export-api --yaml-path "${{env.JINA_VERSION}}.yml" latest.yml --json-path "${{env.JINA_VERSION}}.json" latest.json latest
+          mkdir -p jinad && cd jinad
+          python -c "from daemon import _write_openapi_schema; _write_openapi_schema()"
+          cd -
+      - name: redoc-cli-jinad
+        uses: seeebiii/redoc-cli-github-action@v10
+        with:
+          args: 'bundle schema/jinad/schema.json -o jinad.html'
+      - name: push-to-api-repo
+        run: |
+          cp jinad.html schema/jinad/index.html
+          cp jinad.html schema/jinad/${{env.JINA_VERSION}}.html
+          cd schema
           git config --local user.email "dev-bot@jina.ai"
           git config --local user.name "Jina Dev Bot"
           git add . && git commit -m "update ${{env.JINA_VERSION}} due to ${{github.event_name}} on ${{github.repository}}" && git push
@@ -70,16 +81,17 @@ jobs:
   update-docker:
     needs: update-doc
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - tag_prefix: ""
+            build_option: "--target jina_base"
+          - tag_prefix: "daemon-"
+            build_option: "--target jina_daemon"
     steps:
       - uses: actions/checkout@v2
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: crazy-max/ghaction-docker-buildx@v1
-        with:
-          buildx-version: latest
       - name: Set envs and versions
         run: |
-          echo ${{ steps.buildx.outputs.platforms }}
           echo "VCS_REF=$(git rev-parse --short HEAD)" >> $GITHUB_ENV
           echo "BUILD_DATE=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_ENV
           echo "JINA_VERSION=${GITHUB_REF/refs\/tags\/v/}" >> $GITHUB_ENV
@@ -94,7 +106,8 @@ jobs:
           password: ${{ secrets.DOCKERHUB_DEVBOT_TOKEN }}
           dockerfile: Dockerfiles/debianx.Dockerfile
           buildargs: BUILD_DATE, VCS_REF, JINA_VERSION
-          tags: "latest, ${{env.JINA_VERSION}}, ${{env.JINA_MINOR_VERSION}}"
+          buildoptions: ${{ matrix.build_option }}
+          tags: "${{ matrix.tag_prefix }}latest, ${{ matrix.tag_prefix }}${{env.JINA_VERSION}}, ${{ matrix.tag_prefix }}${{env.JINA_MINOR_VERSION}}"
       - name: Upload to Github Docker Registry
         uses: elgohr/Publish-Docker-Github-Action@master
         with:
@@ -104,7 +117,8 @@ jobs:
           registry: docker.pkg.github.com
           dockerfile: Dockerfiles/debianx.Dockerfile
           buildargs: BUILD_DATE, VCS_REF, JINA_VERSION
-          tags: "latest, ${{env.JINA_VERSION}}, ${{env.JINA_MINOR_VERSION}}"
+          buildoptions: ${{ matrix.build_option }}
+          tags: "${{ matrix.tag_prefix }}latest, ${{ matrix.tag_prefix }}${{env.JINA_VERSION}}, ${{ matrix.tag_prefix }}${{env.JINA_MINOR_VERSION}}"
 
   create-release:
     needs: update-docker

diff --git a/.github/workflows/test-hubapp-hubpods.yml b/.github/workflows/test-hubapp-hubpods.yml
@@ -19,15 +19,6 @@ jobs:
         uses: actions/setup-python@v1
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Build jina test-pip
-        run: |
-          docker build --build-arg PIP_TAG="[devel]" -f Dockerfiles/pip.Dockerfile -t jinaai/jina:test-pip .
-      - name: Build hub pod image
-        run: |
-          docker build -f tests/integration/jinad/hub_mwu/Dockerfile tests/integration/jinad/hub_mwu -t hubpod:test
-      - name: Build hubapp image
-        run: |
-          docker build -f tests/integration/hubapp_hubpods/Dockerfile -t jinaai/test_hubapp_hubpods .
       - name: Test hubapp with hubpods
         run: |
           ./tests/integration/hubapp_hubpods/test_integration.sh

diff --git a/.github/workflows/test-jinad.yml b/.github/workflows/test-jinad.yml
diff --git a/Dockerfiles/alpinex.Dockerfile b/Dockerfiles/alpinex.Dockerfile
@@ -16,19 +16,18 @@ LABEL org.opencontainers.image.created=$BUILD_DATE \
       org.opencontainers.image.title="Jina" \
       org.opencontainers.image.description="Jina is the cloud-native neural search solution powered by state-of-the-art AI and deep learning technology"
 
-WORKDIR /jina/
-
-ADD setup.py MANIFEST.in requirements.txt extra-requirements.txt README.md ./
-ADD cli ./cli/
-ADD jina ./jina/
+COPY . /jina/
 
 ENV PYTHONPATH=$PYTHONPATH:/usr/lib/python3.8/dist-packages:/usr/local/lib/python3.8/site-packages:/usr/lib/python3/dist-packages:/usr/local/lib/python3/site-packages \
     PIP_NO_CACHE_DIR=1 \
-    PIP_DISABLE_PIP_VERSION_CHECK=1
+    PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    JINA_VCS_VERSION=$VCS_REF \
+    JINA_BUILD_DATE=$BUILD_DATE
 
 # py3-scipy
 RUN apk add --no-cache py3-pyzmq py3-numpy grpc && \
     ln -s locale.h /usr/include/xlocale.h && \
+    cd /jina && \
     pip install . --compile && \
     find /usr/lib/python3.8/ -name 'tests' -exec rm -r '{}' + && \
     find /usr/lib/python3.8/site-packages/ -name '*.so' -print -exec sh -c 'file "{}" | grep -q "not stripped" && strip -s "{}"' \; && \
@@ -37,9 +36,4 @@ RUN apk add --no-cache py3-pyzmq py3-numpy grpc && \
     rm -rf /jina && \
     rm -rf /var/cache/apk/*
 
-WORKDIR /
-
-ENV JINA_VCS_VERSION=$VCS_REF \
-    JINA_BUILD_DATE=$BUILD_DATE
-
 ENTRYPOINT ["jina"]
diff --git a/Dockerfiles/debianx-py39.Dockerfile b/Dockerfiles/debianx-py39.Dockerfile
diff --git a/Dockerfiles/debianx.Dockerfile b/Dockerfiles/debianx.Dockerfile
@@ -1,11 +1,17 @@
-ARG PY_VERSION=3.7
-
-FROM python:${PY_VERSION}-slim
+# NOTE: The strucutre of this file is optimized for our CICD
+# If you try to build Jina locally, please use
+# docker build --target jina_base ...
+# If you try to build JinaD locally, please use
+# docker build ...
 
+ARG PY_VERSION=3.7
 ARG VCS_REF
 ARG BUILD_DATE
 ARG JINA_VERSION
 ARG INSTALL_DEV
+ARG PIP_TAG
+
+FROM python:${PY_VERSION}-slim AS jina_base
 
 LABEL org.opencontainers.image.created=$BUILD_DATE \
       org.opencontainers.image.authors="dev-team@jina.ai" \
@@ -20,33 +26,37 @@ LABEL org.opencontainers.image.created=$BUILD_DATE \
       org.opencontainers.image.description="Jina is the cloud-native neural search solution powered by state-of-the-art AI and deep learning technology"
 
 ENV JINA_BUILD_BASE_DEP="python3-grpcio" \
-    JINA_BUILD_DEVEL_DEP="build-essential gcc libc-dev python3-gevent libmagic1"
-
-RUN apt-get update && apt-get install --no-install-recommends -y $JINA_BUILD_BASE_DEP && \
-    if [ -n "$INSTALL_DEV" ]; then apt-get install --no-install-recommends -y $JINA_BUILD_DEVEL_DEP; fi && \
-    apt-get autoremove && apt-get clean && rm -rf /var/lib/apt/lists/*
-
-ENV PYTHONPATH=$PYTHONPATH:/usr/lib/python${PY_VERSION}/dist-packages:/usr/local/lib/python${PY_VERSION}/site-packages:/usr/lib/python3/dist-packages:/usr/local/lib/python3/site-packages \
+    JINA_BUILD_DEVEL_DEP="build-essential gcc libc-dev python3-gevent libmagic1" \
+    PYTHONPATH=$PYTHONPATH:/usr/lib/python${PY_VERSION}/dist-packages:/usr/local/lib/python${PY_VERSION}/site-packages:/usr/lib/python3/dist-packages:/usr/local/lib/python3/site-packages \
     JINA_VERSION=$JINA_VERSION \
     JINA_VCS_VERSION=$VCS_REF \
     JINA_BUILD_DATE=$BUILD_DATE \
     PIP_NO_CACHE_DIR=1 \
     PIP_DISABLE_PIP_VERSION_CHECK=1
 
-WORKDIR /jina/
-
-ADD setup.py MANIFEST.in requirements.txt extra-requirements.txt README.md ./
-ADD cli ./cli/
-ADD jina ./jina/
+COPY . /jina/
 
-RUN ln -s locale.h /usr/include/xlocale.h && \
+RUN apt-get update && apt-get install --no-install-recommends -y $JINA_BUILD_BASE_DEP && \
+    if [ -n "$INSTALL_DEV" ]; then apt-get install --no-install-recommends -y $JINA_BUILD_DEVEL_DEP; fi && \
+    apt-get autoremove && apt-get clean && rm -rf /var/lib/apt/lists/* && \
+    ln -s locale.h /usr/include/xlocale.h && \
+    cd /jina && \
     pip install . --compile && \
     if [ -n "$INSTALL_DEV" ]; then pip install .[devel] --compile; fi && \
+    if [ -n "$PIP_TAG" ]; then pip install ".[$PIP_TAG]" --compile; fi && \
     rm -rf /tmp/* && rm -rf /jina && \
     rm /usr/include/xlocale.h
 
-WORKDIR /
+ENTRYPOINT ["jina"]
+
+FROM jina_base AS jina_daemon
 
+COPY . /jina/
 
+RUN apt-get update && apt-get install --no-install-recommends -y ruby-dev build-essential && \
+    gem install fluentd --no-doc && \
+    cd /jina && \
+    pip install .[daemon] --compile && \
+    rm -rf /tmp/* && rm -rf /jina
 
-ENTRYPOINT ["jina"]
+ENTRYPOINT ["jinad"]