Support to pre-release GraphScope(image + wheel package) nightly with…

… version like 'major.minor.alpha_patch' (#1949) * Fixes nightly CI on linux * Support to pre-release GraphScope(image + wheel package) nightly
alibaba · Aug 9, 2022 · 84e8de8 · 84e8de8
1 parent 5d9b629
commit 84e8de8
Show file tree

Hide file tree

Showing 5 changed files with 75 additions and 47 deletions.
diff --git a/.github/workflows/build-graphscope-wheels-linux.yml b/.github/workflows/build-graphscope-wheels-linux.yml
@@ -39,12 +39,18 @@ jobs:
         if [ -z "${r}" ];then export hn=$(hostname); sudo -E bash -c 'echo "127.0.0.1 ${hn}" >> /etc/hosts'; fi
         cat /etc/hosts
 
+        # change the version for nightly release
+        # 0.15.0 -> 0.15.0a20220808
+        time=$(date "+%Y%m%d")
+        version=$(cat ${GITHUB_WORKSPACE}/VERSION)
+        if [ "${{ GITHUB.REF }}" == "refs/heads/main" ];then echo "${version}a${time}" >> ${GITHUB_WORKSPACE}/VERSION; fi
+
         cd ${GITHUB_WORKSPACE}/k8s
         # build graphscope wheels
-        sudo -E -u graphscope make graphscope-py3-package
+        sudo -E -u runner make graphscope-py3-package
 
         # build client wheels
-        sudo -E -u graphscope make graphscope-client-py3-package
+        sudo -E -u runner make graphscope-client-py3-package
 
         # package
         cd ${GITHUB_WORKSPACE}
@@ -78,7 +84,6 @@ jobs:
         packages_dir: upload_pypi/
 
     - name: Publish distribution to PyPI
-      if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }}
       uses: pypa/gh-action-pypi-publish@v1.4.2
       with:
         user: __token__
@@ -117,6 +122,23 @@ jobs:
         # dataset image
         make dataset-image
 
+    - name: Release Nightly Image
+      if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }}
+      env:
+        docker_password: ${{ secrets.DOCKER_PASSWORD }}
+        docker_username: ${{ secrets.DOCKER_USER }}
+      run: |
+        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin
+        # graphscope image
+        sudo docker tag graphscope/graphscope:${SHORT_SHA} ${{ env.GS_IMAGE }}:latest
+        sudo docker push ${{ env.GS_IMAGE }}:latest
+        # jupyter image
+        sudo docker tag graphscope/jupyter:${SHORT_SHA} ${{ env.JUPYTER_IMAGE }}:latest
+        sudo docker push ${{ env.JUPYTER_IMAGE }}:latest
+        # dataset image
+        sudo docker tag graphscope/dataset:${SHORT_SHA} ${{ env.DATASET_IMAGE }}:latest
+        sudo dockler push ${{ env.DATASET_IMAGE }}:latest
+
     - name: Extract Tag Name
       if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }}
       id: tag
@@ -129,19 +151,14 @@ jobs:
         docker_username: ${{ secrets.DOCKER_USER }}
       run: |
         echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin
-        sudo docker tag graphscope/graphscope:${SHORT_SHA} \
-            ${{ env.GS_IMAGE }}:${{ steps.tag.outputs.TAG }}
+        # graphscope image
+        sudo docker tag graphscope/graphscope:${SHORT_SHA} ${{ env.GS_IMAGE }}:${{ steps.tag.outputs.TAG }}
         sudo docker push ${{ env.GS_IMAGE }}:${{ steps.tag.outputs.TAG }}
-        sudo docker tag graphscope/graphscope:${SHORT_SHA} \
-            ${{ env.GS_IMAGE }}:latest
-        sudo docker push ${{ env.GS_IMAGE }}:latest
         # jupyter image
-        sudo docker tag graphscope/jupyter:${SHORT_SHA} \
-            ${{ env.JUPYTER_IMAGE }}:${{ steps.tag.outputs.TAG }}
+        sudo docker tag graphscope/jupyter:${SHORT_SHA} ${{ env.JUPYTER_IMAGE }}:${{ steps.tag.outputs.TAG }}
         sudo docker push ${{ env.JUPYTER_IMAGE }}:${{ steps.tag.outputs.TAG }}
         # dataset image
-        sudo docker tag graphscope/dataset:${SHORT_SHA} \
-            ${{ env.DATASET_IMAGE }}:${{ steps.tag.outputs.TAG }}
+        sudo docker tag graphscope/dataset:${SHORT_SHA} ${{ env.DATASET_IMAGE }}:${{ steps.tag.outputs.TAG }}
         sudo docker push ${{ env.DATASET_IMAGE }}:${{ steps.tag.outputs.TAG }}
 
   ubuntu-python-test:

diff --git a/.github/workflows/build-graphscope-wheels-macos.yml b/.github/workflows/build-graphscope-wheels-macos.yml
@@ -38,6 +38,12 @@ jobs:
         source ~/.graphscope_env
         echo ${CC}
 
+        # change the version for nightly release
+        # 0.15.0 -> 0.15.0a20220808
+        time=$(date "+%Y%m%d")
+        version=$(cat ${GITHUB_WORKSPACE}/VERSION)
+        if [ "${{ GITHUB.REF }}" == "refs/heads/main" ];then echo "${version}a${time}" >> ${GITHUB_WORKSPACE}/VERSION; fi
+
         # build graphscope server wheel
         cd ${GITHUB_WORKSPACE}/k8s
         sudo -E make graphscope-py3-package
@@ -83,6 +89,12 @@ jobs:
         python3 -c "import sys; print(sys.version)"
         source ~/.graphscope_env
 
+        # change the version for nightly release
+        # 0.15.0 -> 0.15.0a20220808
+        time=$(date "+%Y%m%d")
+        version=$(cat ${GITHUB_WORKSPACE}/VERSION)
+        if [ "${{ GITHUB.REF }}" == "refs/heads/main" ];then echo "${version}a${time}" >> ${GITHUB_WORKSPACE}/VERSION; fi
+
         # build graphscope client wheel
         cd ${GITHUB_WORKSPACE}/k8s
         make graphscope-client-py3-package
@@ -100,7 +112,7 @@ jobs:
 
   # Action gh-action-pypi-publish not support non-linux os.
   publish-wheels:
-    if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }}
+    if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope')
     runs-on: ubuntu-20.04
     needs: [build-wheels, build-client-wheels]
     strategy:

diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml
@@ -55,19 +55,3 @@ jobs:
       with:
         file: ./python/coverage.xml
         fail_ci_if_error: true
-
-  build-gss-image:
-    if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }}
-    runs-on: ubuntu-20.04
-    steps:
-    - uses: actions/checkout@v2.3.2
-
-    - name: Add envs to GITHUB_ENV
-      run: |
-        short_sha=$(git rev-parse --short HEAD)
-        echo "SHORT_SHA=${short_sha}" >> $GITHUB_ENV
-
-    - name: Build GraphScope Store Image
-      run: |
-        cd ${GITHUB_WORKSPACE}
-        sudo make graphscope-store-image
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -1,6 +1,12 @@
 name: Cut Release
 
 on:
+  workflow_dispatch:
+  schedule:
+    # The notifications for scheduled workflows are sent to the user who
+    # last modified the cron syntax in the workflow file.
+    # Trigger the workflow at 03:00(CST) every day.
+    - cron: '00 19 * * *'
   push:
     tags:
       - 'v*'
@@ -14,7 +20,7 @@ env:
 
 jobs:
   release-gss-image:
-    if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }}
+    if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope')
     runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v2.3.2
@@ -29,22 +35,30 @@ jobs:
         cd ${GITHUB_WORKSPACE}
         sudo make graphscope-store-image
 
+    - name: Release Nightly Image
+      if: ${{ github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope' }}
+      env:
+        docker_password: ${{ secrets.DOCKER_PASSWORD }}
+        docker_username: ${{ secrets.DOCKER_USER }}
+      run: |
+        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin
+        sudo docker tag graphscope/graphscope-store:${SHORT_SHA} ${{ env.GSS_IMAGE }}:latest
+        sudo docker push ${{ env.GSS_IMAGE }}:latest
+
     - name: Extract Tag Name
+      if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }}
       id: tag
       run: echo ::set-output name=TAG::${GITHUB_REF#refs/tags/v}
 
     - name: Release Image
+      if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }}
       env:
         docker_password: ${{ secrets.DOCKER_PASSWORD }}
         docker_username: ${{ secrets.DOCKER_USER }}
       run: |
         echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.DOCKER_URL }} --password-stdin
-        sudo docker tag graphscope/graphscope-store:${SHORT_SHA} \
-          ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}
+        sudo docker tag graphscope/graphscope-store:${SHORT_SHA} ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}
         sudo docker push ${{ env.GSS_IMAGE }}:${{ steps.tag.outputs.TAG }}
-        sudo docker tag graphscope/graphscope-store:${SHORT_SHA} \
-          ${{ env.GSS_IMAGE }}:latest
-        sudo docker push ${{ env.GSS_IMAGE }}:latest
 
   release-helm-charts:
     if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }}

diff --git a/coordinator/setup.py b/coordinator/setup.py
@@ -45,17 +45,6 @@ def get_version(file):
     return __version__
 
 
-def get_lib_suffix():
-    suffix = ""
-    if platform.system() == "Linux":
-        suffix = "so"
-    elif platform.system() == "Darwin":
-        suffix = "dylib"
-    else:
-        raise RuntimeError("Get library suffix failed on {0}".format(platform.system()))
-    return suffix
-
-
 repo_root = os.path.dirname(os.path.abspath(__file__))
 version = get_version(os.path.join(repo_root, "..", "VERSION"))
 
@@ -101,6 +90,18 @@ def __get_openmpi_prefix():
             )
         return openmpi_prefix
 
+    def __get_lib_suffix():
+        suffix = ""
+        if platform.system() == "Linux":
+            suffix = "so"
+        elif platform.system() == "Darwin":
+            suffix = "dylib"
+        else:
+            raise RuntimeError(
+                "Get library suffix failed on {0}".format(platform.system())
+            )
+        return suffix
+
     name = os.environ.get("package_name", "gs-coordinator")
     RUNTIME_ROOT = "graphscope.runtime"
 
@@ -114,7 +115,7 @@ def __get_openmpi_prefix():
         data = {
             "/opt/graphscope/lib/": os.path.join(RUNTIME_ROOT, "lib"),
             "/usr/local/lib/libvineyard_internal_registry.{0}".format(
-                get_lib_suffix()
+                __get_lib_suffix()
             ): os.path.join(RUNTIME_ROOT, "lib"),
         }