refactor: parallelize cicd (#1633)

* refactor: parallelize cicd * fix(indexer): fix indexer keys assert
jina-ai · Jan 8, 2021 · 2891033 · 2891033
1 parent 8d63a8e
commit 2891033
Show file tree

Hide file tree

Showing 8 changed files with 83 additions and 29 deletions.
diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
@@ -119,13 +119,18 @@ jobs:
           buildargs: BUILD_DATE, VCS_REF, JINA_VERSION, INSTALL_DEV
           buildoptions: ${{ matrix.build_option }}
           tags: "${{ matrix.tag_prefix }}devel, ${{ matrix.tag_prefix }}${{env.JINA_VERSION}}"
+      - id: set-matrix
+        run: echo "::set-output name=matrix::$(bash scripts/get-test-paths.sh)"
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
 
   core-test:
+    needs: update-docker
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8]
-        test-path: ['unit', 'integration']
+        python-version: [3.7]
+        test-path: ${{fromJson(needs.update-docker.outputs.matrix)}}
 
     steps:
       - uses: actions/checkout@v2
@@ -143,17 +148,17 @@ jobs:
           jina check
           export JINA_LOG_VERBOSITY="ERROR"
           pip install .[test]
-          pytest --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml -n 1 --timeout=120 -v --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/${{ matrix.test-path }}
-        timeout-minutes: 40
+          pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml -n 1 --timeout=120 -v --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/${{ matrix.test-path }}
+        timeout-minutes: 20
         env:
           JINAHUB_USERNAME: ${{ secrets.JINAHUB_USERNAME }}
           JINAHUB_PASSWORD: ${{ secrets.JINAHUB_PASSWORD }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - name: Upload coverage from test to Codecov
         uses: codecov/codecov-action@v1
-        if: ${{ matrix.python-version }} == 3.7
+        if: ${{ matrix.python-version }} == '3.7'
         with:
           file: coverage.xml
           name: ${{ matrix.test-path }}-codecov
-          flags: ${{ matrix.test-path }}, cd
+          flags: ${{ matrix.test-path }}, cd, ${{ matrix.python-version }}
           fail_ci_if_error: false
diff --git a/.github/workflows/ci-hub.yml b/.github/workflows/ci-hub.yml
@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8]
+        python-version: [3.7]
 
     steps:
       - uses: actions/checkout@v2

diff --git a/.github/workflows/ci-jinad.yml b/.github/workflows/ci-jinad.yml
@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8]
+        python-version: [3.7]
         test-path: ['unit', 'integration']
 
     steps:

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -71,14 +71,52 @@ jobs:
           # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
           flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --exclude .git,__pycache__,docs/source/conf.py,old,build,dist,tests/,jina/hub/
 
-  core-test:
+
+  docker-build:
     needs: lint-flake-8
     runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: true
+      - run: |
+          echo "PIP_TAG='[devel]'" >> $GITHUB_ENV
+      - name: Upload to Github Docker Registry
+        uses: elgohr/Publish-Docker-Github-Action@master
+        with:
+          name: jina-ai/jina/jina
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+          registry: docker.pkg.github.com
+          dockerfile: Dockerfiles/pip.Dockerfile
+          buildargs: PIP_TAG
+          buildoptions: ${{ matrix.build_option }}
+          tags: "test-pip"
+      - id: set-matrix
+        run: |
+          sudo apt-get install jq
+          echo "::set-output name=matrix::$(bash scripts/get-test-paths.sh)"
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+
+  docker-image-test:
+    needs: docker-build
+    runs-on: ubuntu-latest
+    steps:
+      - run: |
+          docker login docker.pkg.github.com -u $GITHUB_ACTOR -p $GITHUB_TOKEN
+          docker pull docker.pkg.github.com/jina-ai/jina/jina:test-pip
+          docker run docker.pkg.github.com/jina-ai/jina/jina:test-pip hello-world
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
+
+  core-test:
+    needs: docker-build
+    runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8]
-        test-path: ['unit', 'integration']
-
+        python-version: [3.7]
+        test-path: ${{fromJson(needs.docker-build.outputs.matrix)}}
     steps:
       - uses: actions/checkout@v2
         with:
@@ -89,28 +127,29 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
+          docker login docker.pkg.github.com -u $GITHUB_ACTOR -p $GITHUB_TOKEN
+          docker pull docker.pkg.github.com/jina-ai/jina/jina:test-pip
+          docker tag docker.pkg.github.com/jina-ai/jina/jina:test-pip jinaai/jina:test-pip
           python -m pip install --upgrade pip
-          pip install .[cicd] --no-cache-dir
-      - name: Test docker install
-        run: |
-          docker build --build-arg PIP_TAG="[devel]" -f Dockerfiles/pip.Dockerfile -t jinaai/jina:test-pip .
-          docker run jinaai/jina:test-pip hello-world
-      - name: Test with pytest
-        run: |
+          pip install ".[cicd,test]" --no-cache-dir
           jina check
           export JINA_LOG_VERBOSITY="ERROR"
-          pip install .[test]
-          pytest --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml -n 1 --timeout=120 -v --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/${{ matrix.test-path }}
-        timeout-minutes: 40
+          pytest --suppress-no-test-exit-code --force-flaky --min-passes 1 --max-runs 5 --cov=jina --cov-report=xml -n 1 --timeout=120 -v --ignore-glob='tests/integration/hub_usage/dummyhub*' tests/${{ matrix.test-path }}
+        timeout-minutes: 20
         env:
           JINAHUB_USERNAME: ${{ secrets.JINAHUB_USERNAME }}
           JINAHUB_PASSWORD: ${{ secrets.JINAHUB_PASSWORD }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      - name: Check file existence
+        id: check_files
+        uses: andstor/file-existence-action@v1
+        with:
+          files: "coverage.xml"
       - name: Upload coverage from test to Codecov
         uses: codecov/codecov-action@v1
-        if: ${{ matrix.python-version }} == 3.7
+        if: steps.check_files.outputs.files_exists == 'true' && ${{ matrix.python-version }} == '3.7'
         with:
           file: coverage.xml
-          name: ${{ matrix.python-version }}-${{ matrix.test-path }}-codecov
-          flags: ${{ matrix.test-path }}, core, ci
+          name: ${{ matrix.test-path }}-codecov
+          flags: ${{ matrix.test-path }}, core, ci, ${{ matrix.python-version }}
           fail_ci_if_error: false
diff --git a/extra-requirements.txt b/extra-requirements.txt
@@ -68,4 +68,5 @@ prettytable:                devel, test
 sseclient-py:               test
 websockets:                 http, devel, test, daemon
 pydantic:                   http, devel, test, daemon
-python-multipart:           http, devel, test, daemon
+python-multipart:           http, devel, test, daemon
+pytest-custom_exit_code:    cicd, test
diff --git a/jina/drivers/index.py b/jina/drivers/index.py
@@ -31,7 +31,6 @@ def _apply_all(self, docs: 'DocumentSet', *args, **kwargs) -> None:
                 self.exec_fn(np.array([int(doc.id) for doc in docs_pts]), np.stack(embed_vecs))
 
 
-
 class KVIndexDriver(BaseIndexDriver):
     """Serialize the documents/chunks in the request to key-value JSON pairs and write it using the executor
     """

diff --git a/jina/executors/indexers/vector.py b/jina/executors/indexers/vector.py
@@ -129,15 +129,17 @@ def add(self, keys: 'np.ndarray', vectors: 'np.ndarray', *args, **kwargs) -> Non
     def update(self, keys: Sequence[int], values: Sequence[bytes], *args, **kwargs) -> None:
         keys = self._filter_nonexistent_keys(keys, self.ext2int_id.keys(), self.save_abspath)
         # could be empty
-        if keys:
+        # please do not use "if keys:", it wont work on both sequence and ndarray
+        if getattr(keys, 'size', len(keys)):
             # expects np array for computing shapes
             keys = np.array(list(keys))
             self._delete(keys)
             self.add(np.array(keys), np.array(values))
 
     def _delete(self, keys):
         # could be empty
-        if keys:
+        # please do not use "if keys:", it wont work on both sequence and ndarray
+        if getattr(keys, 'size', len(keys)):
             # expects np array for computing shapes
             keys = np.array(list(keys))
             for key in keys:

diff --git a/scripts/get-test-paths.sh b/scripts/get-test-paths.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+
+set -ex
+
+declare -a array1=( "unit/*.py" "integration/*.py" )
+declare -a array2=( $(ls -d tests/{unit,integration}/*/ | cut -f2,3 -d'/' | grep -v '__pycache__' ))
+dest=( "${array1[@]}" "${array2[@]}" )
+printf '%s\n' "${dest[@]}" | jq -R . | jq -cs .