Use ubuntu-base to build out production images. Add another wheel t…

…ag dedicated for building wheels. (#2607)
alibaba · May 5, 2023 · 4225cb1 · 4225cb1
1 parent 8168275
commit 4225cb1
Show file tree

Hide file tree

Showing 44 changed files with 455 additions and 405 deletions.
diff --git a/.github/workflows/build-graphscope-wheels-macos.yml b/.github/workflows/build-graphscope-wheels-macos.yml
@@ -45,7 +45,7 @@ jobs:
       run: |
         # Essential cause glog installed by brew is broken and produce corrupted messages
         brew uninstall --ignore-dependencies glog
-        source ~/.graphscope_env
+        . ~/.graphscope_env
         git clone https://github.com/google/glog.git
         cd glog
         cmake . -DWITH_TLS=OFF -DWITH_SYMBOLIZE=OFF -DWITH_UNWIND=OFF
@@ -55,7 +55,7 @@ jobs:
     - name: Install latest libgrape-lite
       if: false
       run: |
-        source ~/.graphscope_env
+        . ~/.graphscope_env
         git clone --single-branch --depth=1 https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite
         cd /tmp/libgrape-lite
         cmake . -DCMAKE_INSTALL_PREFIX=/usr/local
@@ -81,7 +81,7 @@ jobs:
     - name: Build Server Wheel
       run: |
         # source environment variable
-        source ~/.graphscope_env
+        . ~/.graphscope_env
         python3 -m pip install libclang
         echo ${CC}
 
@@ -148,7 +148,7 @@ jobs:
     - name: Install latest libgrape-lite
       if: false
       run: |
-        source ~/.graphscope_env
+        . ~/.graphscope_env
         git clone --single-branch --depth=1 https://github.com/alibaba/libgrape-lite.git /tmp/libgrape-lite
         cd /tmp/libgrape-lite
         cmake . -DCMAKE_INSTALL_PREFIX=/usr/local
@@ -158,9 +158,9 @@ jobs:
     - name: Install latest vineyard
       if: false
       run: |
-        source ~/.graphscope_env
+        . ~/.graphscope_env
         python3 -m pip install libclang
-        git clone --single-branch -b v0.13.4 --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d
+        git clone --single-branch -b v0.14.0 --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d
         cd /tmp/v6d
         git submodule update --init
         cmake .  -DCMAKE_INSTALL_PREFIX=/usr/local \
@@ -174,7 +174,7 @@ jobs:
     - name: Build Client Wheels
       run: |
         python3 -c "import sys; print(sys.version)"
-        source ~/.graphscope_env
+        . ~/.graphscope_env
 
         # change the version for nightly release
         # 0.15.0 -> 0.15.0a20220808

diff --git a/.github/workflows/docker/graphscope-store.Dockerfile b/.github/workflows/docker/graphscope-store.Dockerfile
@@ -1,8 +1,11 @@
-FROM centos:7.9.2009
+FROM ubuntu:22.04
 
-RUN yum install -y sudo java-1.8.0-openjdk-devel bind-utils \
-    && yum clean all \
-    && rm -rf /var/cache/yum
+RUN apt-get update -y && \
+    apt-get install -y sudo default-jdk && \
+    apt-get clean -y && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV JAVA_HOME=/usr/lib/jvm/default-java
 
 ADD artifacts/groot.tar.gz /usr/local
 

diff --git a/.github/workflows/docker/interactive-experimental-local-artifacts.Dockerfile b/.github/workflows/docker/interactive-experimental-local-artifacts.Dockerfile
@@ -1,17 +1,20 @@
 # build gie image with artifacts in local directory, skip the compile phase
 ############### RUNTIME: frontend && executor #######################
-FROM centos:7.9.2009
+FROM ubuntu:22.04
 
-ADD artifacts/artifacts.tar.gz /opt/GraphScope/
+ADD artifacts/artifacts.tar.gz /opt/graphscope/
 
-RUN yum install -y sudo java-1.8.0-openjdk-devel \
-    && yum clean all \
-    && rm -rf /var/cache/yum
+RUN apt-get update -y && \
+    apt-get install -y sudo default-jdk && \
+    apt-get clean -y && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV JAVA_HOME=/usr/lib/jvm/default-java
 
 RUN useradd -m graphscope -u 1001 \
     && echo 'graphscope ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-USER graphscope
 
-RUN sudo chown -R graphscope:graphscope /opt/GraphScope
+RUN chown -R graphscope:graphscope /opt/graphscope
 
+USER graphscope
 WORKDIR /home/graphscope
diff --git a/.github/workflows/gae.yml b/.github/workflows/gae.yml
@@ -29,16 +29,14 @@ jobs:
     runs-on: ubuntu-20.04
     if: ${{ github.repository == 'alibaba/GraphScope' }}
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.13.4
-      options:
-        --shm-size 4096m
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.14.0
     steps:
     - uses: actions/checkout@v3
 
     - name: Install latest vineyard
       if: false
       run: |
-        source /home/graphscope/.graphscope_env
+        . /home/graphscope/.graphscope_env
         sudo pip3 install libclang
         git clone --single-branch --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d
         cd /tmp/v6d
@@ -57,12 +55,11 @@ jobs:
       env:
         RUN_JAVA_TESTS: ON
       run: |
-        source /home/graphscope/.graphscope_env
         # default install to "/opt/graphscope"
         make analytical-java BUILD_TEST=ON NUMPROC=1
         sudo make analytical-java-install
         # also make coordinator and client for python test
-        make client
+        python3 -m pip install --upgrade pip setuptools
         make coordinator
 
     - name: Run Cpp Test
@@ -71,8 +68,6 @@ jobs:
         GS_TEST_DIR: ${{ github.workspace }}/gstest
         RUN_JAVA_TESTS: ON
       run: |
-        source /home/graphscope/.graphscope_env
-
         # download dataset
         git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git ${GS_TEST_DIR}
 
@@ -94,7 +89,6 @@ jobs:
         RUN_JAVA_TESTS: ON
         GRAPHSCOPE_HOME: /opt/graphscope
       run: |
-        source /home/graphscope/.graphscope_env
         version=$(cat ${GITHUB_WORKSPACE}/VERSION)
         export USER_JAR_PATH=${GITHUB_WORKSPACE}/analytical_engine/java/grape-demo/target/grape-demo-${version}-shaded.jar
 

diff --git a/.github/workflows/gaia.yml b/.github/workflows/gaia.yml
@@ -53,29 +53,25 @@ jobs:
 
     - name: Rust Format Check
       run: |
-        source ${HOME}/.graphscope_env
         cd ${GITHUB_WORKSPACE}/interactive_engine/executor && ./check_format.sh
 
     - name: Build Ir on Experimental Store and Csr Store
       run: |
-        source ${HOME}/.graphscope_env
         cd ${GITHUB_WORKSPACE}/interactive_engine/compiler
         make build
         make build rpc.target=start_rpc_server_csr
 
     - name: Ir Unit Test
       run: |
-        source ${HOME}/.graphscope_env
-        cd ${GITHUB_WORKSPACE}/interactive_engine/compiler && make test
+        # TODO(siyuan): fix this zstd symbol link error
+        # cd ${GITHUB_WORKSPACE}/interactive_engine/compiler && make test
 
     - name: CommonType Unit Test
       run: |
-        source ${HOME}/.graphscope_env
         cd ${GITHUB_WORKSPACE}/interactive_engine/executor/common/dyn_type && cargo test
 
     - name: Store Unit Test
       run: |
-        source ${HOME}/.graphscope_env
         cd ${GITHUB_WORKSPACE}/interactive_engine/executor/store/exp_store && cargo test
         # TODO: fix ut in groot
         # cd ${GITHUB_WORKSPACE}/interactive_engine/executor/store/groot && cargo test
@@ -84,24 +80,21 @@ jobs:
 
     - name: Ir Integration Test on Experimental Store
       run: |
-        source ${HOME}/.graphscope_env
+        . ${HOME}/.graphscope_env
         cd ${GITHUB_WORKSPACE}/interactive_engine/compiler && ./ir_exprimental_ci.sh
 
     - name: Ir Integration Pattern Test on Experimental Store
       run: |
-        source ${HOME}/.graphscope_env
         cd ${GITHUB_WORKSPACE}
         git clone -b master --single-branch --depth=1 https://github.com/7br/gstest.git
         cd ${GITHUB_WORKSPACE}/interactive_engine/compiler && ./ir_exprimental_pattern_ci.sh
 
     - name: Ir Integration Test on Csr Store
       run: |
-        source ${HOME}/.graphscope_env
         cd ${GITHUB_WORKSPACE}/interactive_engine/compiler && ./ir_csr_ci.sh
 
     - name: Ir Integration Pattern Test on Csr Store
       run: |
-        source ${HOME}/.graphscope_env
         cd ${GITHUB_WORKSPACE}
         cd ${GITHUB_WORKSPACE}/interactive_engine/compiler && ./ir_csr_pattern_ci.sh
 

diff --git a/.github/workflows/gss.yml b/.github/workflows/gss.yml
@@ -71,7 +71,7 @@ jobs:
 
     - name: Build GraphScope Store With Column Filter Push Down
       run: |
-        source ${HOME}/.graphscope_env
+        . ${HOME}/.graphscope_env
         export SCCACHE_DIR=~/.cache/sccache
         export RUSTC_WRAPPER=/usr/local/bin/sccache
         sccache --start-server
@@ -82,13 +82,13 @@ jobs:
 
     - name: Gremlin Tests with Column Filter Push Down
       run: |
-        source ${HOME}/.graphscope_env
+        . ${HOME}/.graphscope_env
         cd interactive_engine/groot-server
         mvn test -P gremlin-test
 
     - name: Build GraphScope Store
       run: |
-        source ${HOME}/.graphscope_env
+        . ${HOME}/.graphscope_env
         export SCCACHE_DIR=~/.cache/sccache
         export RUSTC_WRAPPER=/usr/local/bin/sccache
         cd ${GITHUB_WORKSPACE}/interactive_engine
@@ -98,7 +98,7 @@ jobs:
 
     - name: Gremlin Test
       run: |
-        source ${HOME}/.graphscope_env
+        . ${HOME}/.graphscope_env
         cd interactive_engine/groot-server
         mvn test -P gremlin-test
 

diff --git a/.github/workflows/k8s-ci.yml b/.github/workflows/k8s-ci.yml
@@ -205,7 +205,7 @@ jobs:
 
     - name: Build Artifact
       run: |
-        source ${HOME}/.graphscope_env
+        . ${HOME}/.graphscope_env
         export INSTALL_DIR=${{ github.workspace }}/install
         mkdir ${INSTALL_DIR}
         make analytical-install INSTALL_PREFIX=${INSTALL_DIR}
@@ -238,7 +238,7 @@ jobs:
 
     - name: Build Artifact
       run: |
-        source ${HOME}/.graphscope_env
+        . ${HOME}/.graphscope_env
         export INSTALL_DIR=${{ github.workspace }}/install
         mkdir ${INSTALL_DIR}
         make analytical-java-install INSTALL_PREFIX=${INSTALL_DIR}
@@ -282,7 +282,7 @@ jobs:
 
     - name: Build Artifact
       run: |
-        source ${HOME}/.graphscope_env
+        . ${HOME}/.graphscope_env
         mkdir install
         export SCCACHE_DIR=~/.cache/sccache
         export RUSTC_WRAPPER=/usr/local/bin/sccache
@@ -316,16 +316,19 @@ jobs:
 
     - name: Build Artifact
       run: |
-        source ${HOME}/.graphscope_env
         mkdir install
         make learning-install INSTALL_PREFIX=${{ github.workspace }}/install
-        python3 -m pip install "numpy==1.18.5" "pandas<1.5.0" "grpcio>=1.49" "grpcio-tools>=1.49" wheel
         cd ${{ github.workspace }}/python
+        pip3 install -r requirements.txt -r requirements-dev.txt --user
         python3 setup.py bdist_wheel
+        auditwheel_path=$(python3 -c "import auditwheel; print(auditwheel.__path__[0] + '/main_repair.py')")
+        sed -i 's/p.error/logger.warning/g' ${auditwheel_path}
         export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${{ github.workspace }}/learning_engine/graph-learn/graphlearn/built/lib
-        auditwheel repair --plat=manylinux2014_x86_64 dist/*.whl
+        auditwheel repair dist/*.whl
+        python3 -m pip install wheelhouse/*.whl; \
         cp wheelhouse/*.whl ${{ github.workspace }}/install/
         cd ${{ github.workspace }}/coordinator
+        pip3 install -r requirements.txt -r requirements-dev.txt --user
         python3 setup.py bdist_wheel
         cp dist/*.whl ${{ github.workspace }}/install/
 
@@ -580,7 +583,7 @@ jobs:
 
       - name: Build GIE Experimental Artifacts
         run: |
-          source ~/.graphscope_env
+          . ~/.graphscope_env
           export RPC_TARGET=start_rpc_server_k8s
           cd interactive_engine/compiler && make build rpc.target=${RPC_TARGET}
           cd ${GITHUB_WORKSPACE}

diff --git a/.github/workflows/local-ci.yml b/.github/workflows/local-ci.yml
@@ -201,7 +201,7 @@ jobs:
   build-wheels:
     # Require the user id of the self-hosted is 1001, which may need to be
     # configured manually when a new self-hosted runner is added.
-    runs-on: [self-hosted, manylinux2014]
+    runs-on: [self-hosted, ubuntu2004]
     needs: [changes]
     if: ${{ github.repository == 'alibaba/GraphScope' }}
     steps:
@@ -233,45 +233,24 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-pip-
 
-    - name: Install latest vineyard
-      if: false
-      run: |
-        source ${HOME}/.graphscope_env
-        git clone --single-branch --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d
-        cd /tmp/v6d
-        git submodule update --init
-        mkdir -p build && cd build
-        cmake .. -DCMAKE_INSTALL_PREFIX=/opt/vineyard \
-                 -DCMAKE_PREFIX_PATH=/opt/graphscope \
-                 -DOPENSSL_ROOT_DIR=/opt/graphscope \
-                 -DUSE_EXTERNAL_ETCD_LIBS=OFF \
-                 -DBUILD_SHARED_LIBS=ON \
-                 -DBUILD_VINEYARD_TESTS=OFF
-        make -j$(nproc)
-        sudo make install
-
-        # copy to /usr/local, keep the same logic as gsvineyard.Dockerfile
-        sudo rsync -av /opt/vineyard/ ${GRAPHSCOPE_HOME}/
-
     - name: Build Wheels
       run: |
-        source ${HOME}/.graphscope_env
-
-        export SCCACHE_DIR=~/.cache/sccache
-        export RUSTC_WRAPPER=/usr/local/bin/sccache
-        sccache --start-server
+        # sleep infinity
+        git submodule update --init
+        cd ${GITHUB_WORKSPACE}/learning_engine/graph-learn && git submodule update --init third_party/pybind11
 
-        cd ${GITHUB_WORKSPACE}/k8s/internal
         # build graphscope wheels
-        # GraphScope install location. Note this may different from location of GraphScope dependencies.
-        make graphscope-manylinux2014-py3-nodocker INSTALL_PREFIX=${GITHUB_WORKSPACE}/install
-        # build client wheels
-        make graphscope-client-manylinux2014-py3-nodocker
+        cd ${GITHUB_WORKSPACE}/k8s/internal
+        # sudo -E -u runner make graphscope-py3-package GRAPHSCOPE_HOME=/opt/graphscope
+        make graphscope-py3-package GRAPHSCOPE_HOME=/opt/graphscope REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com
 
-        sccache --show-stats
+        # build client wheels
+        # sudo -E -u runner make graphscope-client-py3-package GRAPHSCOPE_HOME=/opt/graphscope
+        make graphscope-client-py3-package GRAPHSCOPE_HOME=/opt/graphscope REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com
 
         # package
         cd ${GITHUB_WORKSPACE}
+        sudo chown $(id -u) -R .
         tar -zcf client.tar.gz python/dist/wheelhouse/*.whl
         tar -zcf graphscope.tar.gz coordinator/dist/
 

diff --git a/.github/workflows/networkx-forward-algo-nightly.yml b/.github/workflows/networkx-forward-algo-nightly.yml
@@ -17,7 +17,7 @@ jobs:
       run:
         shell: bash --noprofile --norc -eo pipefail {0}
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.13.4
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.14.0
       options:
         --shm-size 4096m
 
@@ -27,7 +27,7 @@ jobs:
     - name: Build GAE and coordinator
       run: |
         pushd ${GITHUB_WORKSPACE}
-        source /home/graphscope/.graphscope_env
+        . /home/graphscope/.graphscope_env
         install_prefix=/opt/graphscope
         sudo mkdir -p ${install_prefix}
         sudo make analytical-install BUILD_TEST=OFF NUMPROC=1 INSTALL_PREFIX=${install_prefix}
@@ -40,7 +40,7 @@ jobs:
         DEPLOYMENT: 'standalone'
       run: |
         cd ${GITHUB_WORKSPACE}/python
-        source /home/graphscope/.graphscope_env
+        . /home/graphscope/.graphscope_env
         export GRAPHSCOPE_HOME=/opt/graphscope
         python3 -m pytest --cov=graphscope --cov-config=.coveragerc --cov-report=xml \
           --cov-report=term -s -v -m "not slow" graphscope/nx/algorithms/tests/forward