Adapt to latest vineyard for varint encoding in CSR (#2791)

## Related issue number

Part of #2628

Signed-off-by: Tao He <linzhu.ht@alibaba-inc.com>

.dockerignore

@@ -32,7 +32,9 @@ python/*.zip
 # Needed by build analytical image in CI, the COPY should copy the generated proto files
 # **/*.pb.*
 **/*_pb2.py
+**/*_pb2.pyi
 **/*_pb2_grpc.py
+**/*_pb2_grpc.pyi
 
 # Visual Studio Code
 .vscode/

.github/workflows/build-graphscope-wheels-macos.yml

@@ -92,11 +92,12 @@ jobs:
           echo "${version}a${time}" > ${GITHUB_WORKSPACE}/VERSION;
         fi
 
-        python3 -m pip install numpy pandas "grpcio>=1.49" "grpcio-tools>=1.49" wheel
+        python3 -m pip install numpy pandas "grpcio>=1.49" "grpcio-tools>=1.49" "mypy-protobuf>=3.4.0" wheel
 
+        export PATH=$PATH:/Users/runner/Library/Python/3.11/bin
         # build graphscope server wheel
         cd ${GITHUB_WORKSPACE}/k8s/internal
-        sudo -E make graphscope-py3-package GRAPHSCOPE_HOME=/usr/local
+        sudo -E env PATH=$PATH make graphscope-py3-package GRAPHSCOPE_HOME=/usr/local
 
         # package
         cd ${GITHUB_WORKSPACE}
@@ -156,7 +157,7 @@ jobs:
       run: |
         . ~/.graphscope_env
         python3 -m pip install libclang
-        git clone --single-branch -b v0.14.5 --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d
+        git clone --single-branch -b v0.15.0 --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d
         cd /tmp/v6d
         git submodule update --init
         cmake .  -DCMAKE_INSTALL_PREFIX=/usr/local \

.github/workflows/gae.yml

@@ -29,12 +29,12 @@ jobs:
     runs-on: ubuntu-20.04
     if: ${{ github.repository == 'alibaba/GraphScope' }}
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.14.5
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.15.0
     steps:
     - uses: actions/checkout@v3
 
     - name: Install latest vineyard
-      if: false
+      if: true
       run: |
         . /home/graphscope/.graphscope_env
         sudo pip3 install libclang

.github/workflows/local-ci.yml

@@ -242,11 +242,11 @@ jobs:
         # build graphscope wheels
         cd ${GITHUB_WORKSPACE}/k8s/internal
         # sudo -E -u runner make graphscope-py3-package GRAPHSCOPE_HOME=/opt/graphscope
-        make graphscope-py3-package GRAPHSCOPE_HOME=/opt/graphscope REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com
+        make graphscope-py3-package GRAPHSCOPE_HOME=/opt/graphscope INSTALL_PREFIX=/home/graphscope/graphscope-install REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com
 
         # build client wheels
         # sudo -E -u runner make graphscope-client-py3-package GRAPHSCOPE_HOME=/opt/graphscope
-        make graphscope-client-py3-package GRAPHSCOPE_HOME=/opt/graphscope REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com
+        make graphscope-client-py3-package GRAPHSCOPE_HOME=/opt/graphscope INSTALL_PREFIX=/home/graphscope/graphscope-install REGISTRY=registry-vpc.cn-hongkong.aliyuncs.com
 
         # package
         cd ${GITHUB_WORKSPACE}

.github/workflows/networkx-forward-algo-nightly.yml

@@ -17,7 +17,7 @@ jobs:
       run:
         shell: bash --noprofile --norc -eo pipefail {0}
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.14.5
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.15.0
       options:
         --shm-size 4096m
 

.github/workflows/nightly.yml

@@ -43,7 +43,7 @@ jobs:
         fi
         sudo mkdir /opt/graphscope
         sudo chown -R $(id -u):$(id -g) /opt/graphscope
-        ./gs install-deps dev --v6d-version v0.14.5
+        ./gs install-deps dev --v6d-version v0.15.0
 
     - name: Setup tmate session
       if: false 

.gitignore

@@ -21,7 +21,9 @@ python/proto
 # Protobuf GRPC
 *.pb.*
 *_pb2.py
+*_pb2.pyi
 *_pb2_grpc.py
+*_pb2_grpc.pyi
 
 # Visual Studio Code
 .vscode/

analytical_engine/CMakeLists.txt

@@ -261,7 +261,7 @@ else()
 endif()
 
 # find vineyard after arrow to avoid duplicate target names
-find_package(vineyard 0.13.0 REQUIRED)
+find_package(vineyard 0.14.8 REQUIRED)
 include_directories(${VINEYARD_INCLUDE_DIRS})
 add_compile_options(-DENABLE_SELECTOR)
 
@@ -413,6 +413,7 @@ if (BUILD_TESTS)
     endmacro()
 
     add_vineyard_app(run_vy_app SRCS test/run_vy_app.cc)
+    add_vineyard_app(run_vy_app_compact SRCS test/run_vy_app_compact.cc)
     add_vineyard_app(run_vy_app_local_vm SRCS test/run_vy_app_local_vm.cc)
 
     add_vineyard_app(run_load_from_stream SRCS test/run_load_from_stream.cc)
@@ -451,7 +452,7 @@ if (BUILD_TESTS)
     endif ()
 endif ()
 
-# Cpplint
+# cpplint
 file(GLOB_RECURSE FILES_NEED_LINT
         "apps/*.cc"
         "apps/*.h"

analytical_engine/benchmarks/property_graph_java_app_benchmarks.cc

@@ -146,13 +146,13 @@ int main(int argc, char** argv) {
   }
 
   VLOG(1) << "Property: " << run_property << ", app name: " << app_name
-          << ", thread num: " << threadNum << ", outprefix: " << output_prefix;
+          << ", thread num: " << threadNum << ", out prefix: " << output_prefix;
   {
     vineyard::Client client;
     VINEYARD_CHECK_OK(client.Connect(ipc_socket));
 
     VLOG(1) << "Connected to IPCServer: " << ipc_socket;
-    VLOG(1) << "Using ArrowFragment with objid: " << fragment_id;
+    VLOG(1) << "Using ArrowFragment with object id: " << fragment_id;
 
     std::shared_ptr<FragmentType> fragment =
         std::dynamic_pointer_cast<FragmentType>(client.GetObject(fragment_id));

analytical_engine/core/app/pregel/pregel_context.h

@@ -89,6 +89,7 @@ class PregelContext
  * @tparam COMPUTE_CONTEXT_T
  */
 template <typename OID_T, typename VID_T, typename VERTEX_MAP_T,
+          /* bool COMPACT, */  // TODO(tao): support compact CSR
           typename COMPUTE_CONTEXT_T>
 class PregelContext<vineyard::ArrowFragment<OID_T, VID_T, VERTEX_MAP_T>,
                     COMPUTE_CONTEXT_T>

analytical_engine/core/fragment/arrow_flattened_fragment.h

@@ -477,7 +477,8 @@ template <typename OID_T, typename VID_T, typename VDATA_T, typename EDATA_T,
               typename vineyard::InternalType<OID_T>::type, VID_T>>
 class ArrowFlattenedFragment {
  public:
-  using fragment_t = vineyard::ArrowFragment<OID_T, VID_T, VERTEX_MAP_T>;
+  // TODO(tao): ArrowFragment with compact edges cannot be flattened.
+  using fragment_t = vineyard::ArrowFragment<OID_T, VID_T, VERTEX_MAP_T, false>;
   using oid_t = OID_T;
   using vid_t = VID_T;
   using internal_oid_t = typename vineyard::InternalType<oid_t>::type;