Install GraphScope into /opt/graphscope and link to the /usr/local

.github/workflows/ci.yml

@@ -30,7 +30,7 @@ jobs:
       run:
         shell: scl enable devtoolset-7 -- bash --noprofile --norc -eo pipefail {0}
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.2.6
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.2.9
       options:
         --shm-size 4096m
     strategy:
@@ -92,14 +92,16 @@ jobs:
 
     - name: Build
       run: |
-        strip /opt/graphscope/lib/* || true
-        strip /opt/graphscope/lib64/* || true
+        strip /opt/vineyard/lib/* || true
+        strip /opt/vineyard/lib64/* || true
+        tar -cf vineyard.tar /opt/vineyard
+
+        sudo mkdir -p /opt/graphscope
+        sudo chown -R $(id -u):$(id -g) /opt/graphscope
 
-        export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/graphscope/lib:/opt/graphscope/lib64
         mkdir analytical_engine/build
         cd analytical_engine/build
-        cmake .. -DCMAKE_PREFIX_PATH=/opt/graphscope \
-                 -DCMAKE_INSTALL_PREFIX=/opt/graphscope \
+        cmake .. -DCMAKE_INSTALL_PREFIX=/opt/graphscope \
                  -DNETWORKX=${{ matrix.networkx }}
         make -j`nproc`
         make install
@@ -122,7 +124,9 @@ jobs:
       uses: actions/upload-artifact@v2
       with:
         name: gae-${{ github.sha }}
-        path: gae.tar
+        path: |
+          gae.tar
+          vineyard.tar
         retention-days: 5
 
   build-gie:
@@ -131,7 +135,7 @@ jobs:
       run:
         shell: scl enable devtoolset-7 -- bash --noprofile --norc -eo pipefail {0}
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.2.6
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.2.9
     steps:
     - name: Install Dependencies
       run: |
@@ -147,9 +151,6 @@ jobs:
     - name: Build
       run: |
         export PATH=${PATH}:/usr/local/go/bin
-        export CMAKE_PREFIX_PATH=/opt/graphscope
-        export LIBRARY_PATH=$LIBRARY_PATH:/opt/graphscope/lib
-        export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/graphscope/lib:/opt/graphscope/lib64
         source ~/.bashrc
 
         # install zetcd
@@ -177,23 +178,25 @@ jobs:
       run:
         shell: scl enable devtoolset-7 -- bash --noprofile --norc -eo pipefail {0}
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.2.6
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.2.9
     steps:
     - uses: actions/checkout@v2.3.2
       with:
         submodules: true
     - name: Build
       run: |
-        strip /opt/graphscope/lib/* || true
-        strip /opt/graphscope/lib64/* || true
-        export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/graphscope/lib:/opt/graphscope/lib64
+        strip /opt/vineyard/lib/* || true
+        strip /opt/vineyard/lib64/* || true
+
+        sudo mkdir -p /opt/graphscope
+        sudo chown -R $(id -u):$(id -g) /opt/graphscope
+
         export PATH=${PATH}:/opt/rh/devtoolset-7/root/usr/bin
         cd learning_engine/graph-learn
         git submodule update --init third_party/pybind11
         mkdir cmake-build
         cd cmake-build
-        cmake -DCMAKE_PREFIX_PATH=/opt/graphscope \
-              -DCMAKE_INSTALL_PREFIX=/opt/graphscope \
+        cmake -DCMAKE_INSTALL_PREFIX=/opt/graphscope \
               -DWITH_VINEYARD=ON \
               -DTESTING=OFF ..
         make graphlearn_shared install -j
@@ -210,6 +213,7 @@ jobs:
         python3 setup.py bdist_wheel
         pushd ./dist
         auditwheel repair --plat=manylinux2014_x86_64 ./*.whl
+        mkdir -p /opt/graphscope/dist
         cp ./wheelhouse/* /opt/graphscope/dist/
         popd
         popd
@@ -284,11 +288,25 @@ jobs:
         sudo cp $(go env GOPATH)/bin/zetcd /usr/local/bin/
 
         python3 -m pip install pytest-cov
+        tar -xf ./gae-${{ github.sha }}/vineyard.tar
         tar -xf ./gae-${{ github.sha }}/gae.tar
         tar -xf ./gle-${{ github.sha }}/gle.tar
-        sudo cp -R -n opt/graphscope/* /usr/local/
-        ls -la /usr/local/bin/
+
+        # install vineyard into /usr/local
+        sudo cp -r -n opt/vineyard/* /usr/local/
+
+        # Install to /opt/graphscope and link them to /usr/local
+        sudo mkdir -p /opt/graphscope
+        sudo chown -R $(id -u):$(id -g) /opt/graphscope
+        cp -r opt/graphscope/* /opt/graphscope/
+        sudo ln -sf /opt/graphscope/bin/* /usr/local/bin/
+        sudo ln -sfn /opt/graphscope/include/graphscope /usr/local/include/graphscope
+        sudo ln -sf /opt/graphscope/lib/*so* /usr/local/lib
+        sudo ln -sf /opt/graphscope/lib64/*so* /usr/local/lib64
+        sudo ln -sfn /opt/graphscope/lib64/cmake/graphscope-analytical /usr/local/lib64/cmake/graphscope-analytical
+
         pushd /usr/local/dist/ && pip3 install ./*.whl && popd
+        pushd /opt/graphscope/dist/ && pip3 install ./*.whl && popd
         python3 -m pip install -r python/requirements.txt
         python3 -m pip install -r python/requirements-dev.txt
         pushd python && python3 setup.py build_proto && popd
@@ -304,7 +322,7 @@ jobs:
         # prelaunch the etcd
         /usr/local/bin/etcd --data-dir=/tmp/default.etcd &
 
-        export GRAPHSCOPE_HOME='/usr/local'
+        export GRAPHSCOPE_HOME='/opt/graphscope'
         export GS_TEST_DIR='/home/graphscope/gstest'
         cd python
         python3 -m pytest --cov=graphscope --cov-config=.coveragerc \
@@ -325,7 +343,7 @@ jobs:
         # prelaunch the etcd
         /usr/local/bin/etcd --data-dir=/tmp/default.etcd &
 
-        export GRAPHSCOPE_HOME='/usr/local'
+        export GRAPHSCOPE_HOME='/opt/graphscope'
         export GS_TEST_DIR='/home/graphscope/gstest'
         python3 -m pytest --exitfirst -s -v python/graphscope/nx/tests \
                           --ignore=python/graphscope/nx/tests/algorithms/forward
@@ -367,6 +385,7 @@ jobs:
       run: |
         cp .github/workflows/*.Dockerfile artifacts/
         pushd artifacts
+        tar -xf ./gae-${{ github.sha }}/vineyard.tar
         tar -xf ./gae-${{ github.sha }}/gae.tar
         tar -xf ./gle-${{ github.sha }}/gle.tar
         tar -xf ./gie-${{ github.sha }}/gie.tar
@@ -380,8 +399,7 @@ jobs:
         python3 -m pip install -r python/requirements-dev.txt --user
         python3 -m pip install pytest-cov --user
         pushd python && sudo -E python3 setup.py develop && popd
-
-        pip3 install git+https://github.com/mars-project/mars.git@35b44ed56e031c252e50373b88b85bd9f454332e#egg=pymars[distributed]
+        # pip3 install git+https://github.com/mars-project/mars.git@35b44ed56e031c252e50373b88b85bd9f454332e#egg=pymars[distributed]
 
     - name: Kubernetes test
       env:
@@ -490,6 +508,7 @@ jobs:
       run: |
         cp .github/workflows/*.Dockerfile artifacts/
         pushd artifacts
+        tar -xf ./gae-${{ github.sha }}/vineyard.tar
         tar -xf ./gae-${{ github.sha }}/gae.tar
         tar -xf ./gle-${{ github.sha }}/gle.tar
         tar -xf ./gie-${{ github.sha }}/gie.tar
@@ -524,7 +543,7 @@ jobs:
       run:
         shell: scl enable devtoolset-7 -- bash --noprofile --norc -eo pipefail {0}
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.2.6
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.2.9
     steps:
 
     #- name: "Debug: Package dependancies for tmate"
@@ -552,22 +571,19 @@ jobs:
 
     - name: Build
       run: |
+        sudo mkdir -p /opt/graphscope
+        sudo chown -R $(id -u):$(id -g) /opt/graphscope
         export PATH=${PATH}:/usr/local/go/bin
-        export CMAKE_PREFIX_PATH=/opt/graphscope
-        export LIBRARY_PATH=$LIBRARY_PATH:/opt/graphscope/lib
-        export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/graphscope/lib:/opt/graphscope/lib64
         source ~/.bashrc
 
         cd interactive_engine
         for i in {1..3}; do mvn clean install -Pv2 -Drust.compile.mode=debug -DskipTests --quiet  && break || sleep 60; done
         cd ${GITHUB_WORKSPACE}
         mv interactive_engine/distribution/target/maxgraph.tar.gz ./
-        tar -zcf lib.tar.gz /opt/graphscope
+        tar -zcf vineyard.tar.gz /opt/vineyard
 
     - name: gremlin-test
       run: |
-        export LIBRARY_PATH=$LIBRARY_PATH:/opt/graphscope/lib
-        export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/opt/graphscope/lib:/opt/graphscope/lib64
         cd interactive_engine/src/v2
         mvn test -Pgremlin-test
 
@@ -576,7 +592,7 @@ jobs:
         name: gss-${{ github.sha }}
         path: |
           maxgraph.tar.gz
-          lib.tar.gz
+          vineyard.tar.gz
         retention-days: 5
 
   gss-test:
@@ -609,7 +625,7 @@ jobs:
         cp k8s/ready_probe.sh artifacts/
         cp .github/workflows/graphscope-store.Dockerfile artifacts/
         pushd artifacts
-        tar -zxf ./gss-${{ github.sha }}/lib.tar.gz
+        tar -zxf ./gss-${{ github.sha }}/vineyard.tar.gz
         tar -zxf ./gss-${{ github.sha }}/maxgraph.tar.gz
 
         sudo docker pull registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-runtime:latest
@@ -699,6 +715,7 @@ jobs:
         cp k8s/ready_probe.sh artifacts/
         pushd artifacts
 
+        tar -xf ./gae-${{ github.sha }}/vineyard.tar
         tar -xf ./gae-${{ github.sha }}/gae.tar
         tar -xf ./gle-${{ github.sha }}/gle.tar
         tar -xf ./gie-${{ github.sha }}/gie.tar

.github/workflows/graphscope-store.Dockerfile

@@ -1,6 +1,6 @@
 FROM registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-runtime:latest
 
-COPY ./opt/graphscope/ /usr/local/
+COPY ./opt/vineyard/ /usr/local/
 COPY ./maxgraph /usr/local/maxgraph
 COPY ./ready_probe.sh /tmp/ready_probe.sh
 

.github/workflows/graphscope.Dockerfile

@@ -2,22 +2,28 @@ FROM registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-runtime:latest
 
 ARG profile=release
 
-COPY ./opt/graphscope/ /usr/local/
-RUN cd /usr/local/dist/ && pip3 install ./*.whl
-
-COPY ./interactive_engine/src/executor/target/release/executor /usr/local/bin/executor
-COPY ./interactive_engine/src/executor/target/release/gaia_executor /usr/local/bin/gaia_executor
-COPY ./interactive_engine/bin/giectl /usr/local/bin/giectl
-COPY ./interactive_engine/bin/zetcd /usr/local/bin/zetcd
-COPY ./interactive_engine/src/assembly/target/0.0.1-SNAPSHOT.tar.gz /tmp/0.0.1-SNAPSHOT.tar.gz
-RUN sudo tar -xf /tmp/0.0.1-SNAPSHOT.tar.gz -C /usr/local
+COPY ./opt/vineyard/ /usr/local/
+COPY ./opt/graphscope /opt/graphscope
+COPY ./interactive_engine/conf/* /opt/graphscope/conf/
+COPY ./interactive_engine/bin/zetcd /opt/graphscope/bin/zetcd
+COPY ./interactive_engine/bin/giectl /opt/graphscope/bin/giectl
+COPY ./interactive_engine/src/executor/target/release/executor /opt/graphscope/bin/executor
+COPY ./interactive_engine/src/executor/target/release/gaia_executor /opt/graphscope/bin/gaia_executor
+COPY ./interactive_engine/src/assembly/target/0.0.1-SNAPSHOT.tar.gz /opt/graphscope/0.0.1-SNAPSHOT.tar.gz
 
 # install mars
-RUN pip3 install git+https://github.com/mars-project/mars.git@35b44ed56e031c252e50373b88b85bd9f454332e#egg=pymars[distributed]
+# RUN pip3 install git+https://github.com/mars-project/mars.git@35b44ed56e031c252e50373b88b85bd9f454332e#egg=pymars[distributed]
+
+RUN sudo tar -xf /opt/graphscope/0.0.1-SNAPSHOT.tar.gz -C /opt/graphscope \
+  && sudo chown -R $(id -u):$(id -g) /opt/graphscope \
+  && cd /usr/local/dist && pip3 install ./*.whl \
+  && cd /opt/graphscope/dist && pip3 install ./*.whl \
+  && sudo ln -sf /opt/graphscope/bin/* /usr/local/bin/ \
+  && sudo ln -sfn /opt/graphscope/include/graphscope /usr/local/include/graphscope \
+  && sudo ln -sf /opt/graphscope/lib/*so* /usr/local/lib \
+  && sudo ln -sf /opt/graphscope/lib64/*so* /usr/local/lib64 \
+  && sudo ln -sfn /opt/graphscope/lib64/cmake/graphscope-analytical /usr/local/lib64/cmake/graphscope-analytical
 
 # enable debugging
 ENV RUST_BACKTRACE=1
-
-# copy start script from builder
-COPY ./interactive_engine/conf/* /usr/local/conf/
-ENV GRAPHSCOPE_HOME=/usr/local
+ENV GRAPHSCOPE_HOME=/opt/graphscope

.github/workflows/local.yml

@@ -63,7 +63,7 @@ jobs:
         OS : ${{ matrix.os }}
       run: |
         source ~/.graphscope_env
-        export GRAPHSCOPE_HOME=/usr/local
+        export GRAPHSCOPE_HOME=/opt/graphscope
 
         # install tensorflow for graphlearn if os is ubuntu20.04
         if [ "${OS}" == "ubuntu-20.04" ];
@@ -92,7 +92,7 @@ jobs:
       if: matrix.os == 'ubuntu-20.04'
       run: |
         source ~/.graphscope_env
-        export GRAPHSCOPE_HOME=/usr/local
+        export GRAPHSCOPE_HOME=/opt/graphscope
         export GS_TEST_DIR=${GITHUB_WORKSPACE}/gstest
         # run test
         cd interactive_engine/src/gaia-adaptor && ./gaia_on_vineyard_test.sh

Makefile

@@ -3,7 +3,7 @@ WORKING_DIR 			:= $(dir $(MKFILE_PATH))
 NUM_PROC                := $( $(command -v nproc &> /dev/null) && echo $(nproc) || echo $(sysctl -n hw.physicalcpu) )
 
 VERSION                     ?= 0.1.0
-INSTALL_PREFIX              ?= /usr/local
+INSTALL_PREFIX              ?= /opt/graphscope
 
 # GAE build options
 NETWORKX                    ?= OFF
@@ -63,6 +63,17 @@ gae:
 	cmake -DCMAKE_INSTALL_PREFIX=$(INSTALL_PREFIX) -DNETWORKX=$(NETWORKX) -DBUILD_TESTS=${BUILD_TEST} .. && \
 	make -j$(NUM_PROC) && \
 	sudo make install
+ifneq ($(INSTALL_PREFIX), /usr/local)
+	sudo ln -sf $(INSTALL_PREFIX)/bin/* /usr/local/bin/ && \
+	sudo ln -sfn $(INSTALL_PREFIX)/include/graphscope /usr/local/include/graphscope && \
+	sudo ln -sf ${INSTALL_PREFIX}/lib/*so* /usr/local/lib && \
+	if [ -d "${INSTALL_PREFIX}/lib64/cmake/graphscope-analytical" ]; then \
+		sudo ln -sfn ${INSTALL_PREFIX}/lib64/cmake/graphscope-analytical /usr/local/lib64/cmake/graphscope-analytical; \
+	else \
+		sudo ln -sfn ${INSTALL_PREFIX}/lib/cmake/graphscope-analytical /usr/local/lib/cmake/graphscope-analytical; \
+	fi
+endif
+
 
 .PHONY: gie
 gie:
@@ -96,6 +107,9 @@ ifeq ($(WITH_LEARNING_ENGINE), ON)
 	cmake -DCMAKE_INSTALL_PREFIX=$(INSTALL_PREFIX) -DWITH_VINEYARD=ON -DTESTING=${BUILD_TEST} .. && \
 	make -j$(NUM_PROC) && \
 	sudo make install
+ifneq ($(INSTALL_PREFIX), /usr/local)
+	sudo ln -sf ${INSTALL_PREFIX}/lib/*so* /usr/local/lib
+endif
 endif
 
 .PHONY: prepare-client

analytical_engine/CMakeLists.txt

@@ -119,7 +119,7 @@ endif ()
 find_package(libgrapelite REQUIRED)
 include_directories(${LIBGRAPELITE_INCLUDE_DIRS})
 
-find_package(vineyard 0.2.6 REQUIRED)
+find_package(vineyard 0.2.9 REQUIRED)
 include_directories(${VINEYARD_INCLUDE_DIRS})
 add_compile_options(-DENABLE_SELECTOR)
 
@@ -301,9 +301,9 @@ endmacro()
 macro(install_gsa_dependency_modules cmake_target)
     get_filename_component(ABS_DIR ${cmake_target} REALPATH)
     install(DIRECTORY ${ABS_DIR}
-            DESTINATION lib                     # target directory
-            FILES_MATCHING                      # install only matched files
-            PATTERN "*.cmake"                   # select cmake files
+            DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/graphscope-analytical  # target directory
+            FILES_MATCHING                                                   # install only matched files
+            PATTERN "*.cmake"                                                # select cmake files
             )
 endmacro()
 

analytical_engine/benchmarks/apps/bfs/bfs.h

@@ -80,8 +80,8 @@ class BFS : public grape::ParallelAppBase<FRAG_T, BFSContext<FRAG_T>>,
     auto outer_vertices = frag.OuterVertices();
 
     // init double buffer which contains updated vertices using bitmap
-    ctx.curr_inner_updated.Init(inner_vertices, thread_num());
-    ctx.next_inner_updated.Init(inner_vertices, thread_num());
+    ctx.curr_inner_updated.Init(inner_vertices, GetThreadPool());
+    ctx.next_inner_updated.Init(inner_vertices, GetThreadPool());
 
     auto& channel_0 = messages.Channels()[0];
 
@@ -112,7 +112,7 @@ class BFS : public grape::ParallelAppBase<FRAG_T, BFSContext<FRAG_T>>,
 
     depth_type next_depth = ctx.current_depth + 1;
     int thrd_num = thread_num();
-    ctx.next_inner_updated.ParallelClear(thrd_num);
+    ctx.next_inner_updated.ParallelClear(GetThreadPool());
 
     // process received messages and update depth
     messages.ParallelProcess<fragment_t, grape::EmptyType>(