Upgrade vineyard to v0.3.21. (#1389)

* Upgrade vineyard to v0.3.21. * Filter out invalid ids. * Disable networkx test under distributed settings, see also #1392. Signed-off-by: Tao He <linzhu.ht@alibaba-inc.com>
alibaba · Mar 22, 2022 · 0bf6686 · 0bf6686
1 parent 3b520ca
commit 0bf6686
Show file tree

Hide file tree

Showing 18 changed files with 35 additions and 19 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -301,7 +301,9 @@ jobs:
         shell: bash
     strategy:
       matrix:
-        deployment: ["standalone", "distributed"]
+        # the networkx implementation under distributed setting is broken
+        # deployment: ["standalone", "distributed"]
+        deployment: ["standalone"]
     steps:
     - uses: actions/checkout@v2.3.2
     - uses: dorny/paths-filter@v2

diff --git a/.github/workflows/gae.yml b/.github/workflows/gae.yml
@@ -33,7 +33,7 @@ jobs:
     runs-on: ubuntu-20.04
     if: ${{ github.repository == 'alibaba/GraphScope' }}
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.3.19
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.3.21
       options:
         --shm-size 4096m
     steps:

diff --git a/.github/workflows/gss.yml b/.github/workflows/gss.yml
@@ -44,7 +44,7 @@ jobs:
     runs-on: self-hosted
     if: ${{ github.repository == 'alibaba/GraphScope' }}
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.3.19
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.3.21
     defaults:
       run:
         shell: bash --noprofile --norc -eo pipefail {0}

diff --git a/.github/workflows/networkx-forward-algo-nightly.yml b/.github/workflows/networkx-forward-algo-nightly.yml
@@ -16,7 +16,7 @@ jobs:
       run:
         shell: bash --noprofile --norc -eo pipefail {0}
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.3.19
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.3.21
       options:
         --shm-size 4096m
 

diff --git a/analytical_engine/CMakeLists.txt b/analytical_engine/CMakeLists.txt
@@ -170,7 +170,7 @@ endif ()
 find_package(libgrapelite REQUIRED)
 include_directories(${LIBGRAPELITE_INCLUDE_DIRS})
 
-find_package(vineyard 0.3.19 REQUIRED)
+find_package(vineyard 0.3.21 REQUIRED)
 include_directories(${VINEYARD_INCLUDE_DIRS})
 add_compile_options(-DENABLE_SELECTOR)
 

diff --git a/analytical_engine/core/context/tensor_context.h b/analytical_engine/core/context/tensor_context.h
@@ -536,6 +536,8 @@ class TensorContextWrapper<
         *arc << static_cast<int>(vineyard::TypeToInt<int64_t>::value);
       } else if (data_type == dynamic::Type::kDoubleType) {
         *arc << static_cast<int>(vineyard::TypeToInt<double>::value);
+      } else if (data_type == dynamic::Type::kNullType) {
+        *arc << static_cast<int>(vineyard::TypeToInt<void>::value);
       } else {
         RETURN_GS_ERROR(vineyard::ErrorCode::kInvalidOperationError,
                         "Only support int64, double");

diff --git a/analytical_engine/core/fragment/dynamic_fragment.h b/analytical_engine/core/fragment/dynamic_fragment.h
@@ -976,6 +976,9 @@ class DynamicFragment
       std::fill(oe_degree_to_add.begin(), oe_degree_to_add.end(), 0);
       std::fill(ie_degree_to_add.begin(), ie_degree_to_add.end(), 0);
       for (auto& e : edges) {
+        if (e.src == invalid_vid) {
+          continue;
+        }
         if (updateOrAddEdgeOutIn(e)) {
           if (e.src < ivnum_) {
             ++oe_degree_to_add[e.src];
@@ -1006,6 +1009,9 @@ class DynamicFragment
       // add edges
       std::fill(oe_degree_to_add.begin(), oe_degree_to_add.end(), 0);
       for (auto& e : edges) {
+        if (e.src == invalid_vid) {
+          continue;
+        }
         if (updateOrAddEdgeOut(e)) {
           if (e.src < ivnum_) {
             ++oe_degree_to_add[e.src];
@@ -1042,6 +1048,9 @@ class DynamicFragment
       oe_degree_to_add.clear();
       ie_degree_to_add.clear();
       for (auto& e : edges) {
+        if (e.src == invalid_vid) {
+          continue;
+        }
         if (updateOrAddEdgeOutIn(e)) {
           if (e.src < ivnum_) {
             ++oe_degree_to_add[e.src];
@@ -1072,6 +1081,9 @@ class DynamicFragment
       // add edges
       oe_degree_to_add.clear();
       for (auto& e : edges) {
+        if (e.src == invalid_vid) {
+          continue;
+        }
         if (updateOrAddEdgeOut(e)) {
           if (e.src < ivnum_) {
             ++oe_degree_to_add[e.src];

diff --git a/analytical_engine/core/object/dynamic.cc b/analytical_engine/core/object/dynamic.cc
@@ -21,7 +21,7 @@
 #include "core/object/dynamic.h"
 
 /* static definition */
-gs::dynamic::AllocatorT gs::dynamic::Value::allocator_;
+gs::dynamic::AllocatorT gs::dynamic::Value::allocator_{};
 
 std::size_t gs::dynamic::Value::hash() const {
   switch (GetType()) {

diff --git a/coordinator/gscoordinator/template/CMakeLists.template b/coordinator/gscoordinator/template/CMakeLists.template
@@ -165,7 +165,7 @@ if (libgrapelite_FOUND)
 endif()
 
 # find vineyard-----------------------------------------
-find_package(vineyard 0.3.13 QUIET)
+find_package(vineyard 0.3.21 QUIET)
 if (vineyard_FOUND)
   include_directories(AFTER SYSTEM ${VINEYARD_INCLUDE_DIRS})
 endif()

diff --git a/coordinator/requirements.txt b/coordinator/requirements.txt
@@ -5,5 +5,5 @@ grpcio
 kubernetes
 protobuf
 PyYAML
-vineyard>=0.3.16; sys_platform != "win32"
-vineyard-io>=0.3.16; sys_platform != "win32"
+vineyard>=0.3.21; sys_platform != "win32"
+vineyard-io>=0.3.21; sys_platform != "win32"
diff --git a/interactive_engine/executor/runtime/native/CMakeLists.txt b/interactive_engine/executor/runtime/native/CMakeLists.txt
@@ -48,7 +48,7 @@ find_package(Threads REQUIRED)
 #  we need edge src/dst ids in etable.
 add_definitions(-DENDPOINT_LISTS)
 
-find_package(vineyard 0.3.1 REQUIRED)
+find_package(vineyard 0.3.21 REQUIRED)
 add_library(native_store global_store_ffi.cc
                          htap_ds_impl.cc
                          graph_builder_ffi.cc

diff --git a/k8s/Makefile b/k8s/Makefile
@@ -40,7 +40,7 @@ else
 endif
 
 VERSION ?= latest
-VINEYARD_VERSION ?= v0.3.19
+VINEYARD_VERSION ?= v0.3.21
 PROFILE ?= release
 
 

diff --git a/k8s/graphscope-dev.Dockerfile b/k8s/graphscope-dev.Dockerfile
@@ -4,7 +4,7 @@
 # the result image includes all runtime stuffs of graphscope, with analytical engine,
 # learning engine and interactive engine installed.
 
-ARG BASE_VERSION=v0.3.19
+ARG BASE_VERSION=v0.3.21
 FROM registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:$BASE_VERSION as builder
 
 ARG NETWORKX=ON

diff --git a/k8s/graphscope-store.Dockerfile b/k8s/graphscope-store.Dockerfile
@@ -1,4 +1,4 @@
-ARG BASE_VERSION=v0.3.19
+ARG BASE_VERSION=v0.3.21
 FROM registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:$BASE_VERSION as builder
 
 ARG CI=true

diff --git a/k8s/gsvineyard.Dockerfile b/k8s/gsvineyard.Dockerfile
@@ -27,7 +27,7 @@ RUN sudo mkdir -p /opt/vineyard && \
     make -j`nproc` && \
     make install && \
     cd /tmp && \
-    git clone -b v0.3.19 https://github.com/v6d-io/v6d.git --depth=1 && \
+    git clone -b v0.3.21 https://github.com/v6d-io/v6d.git --depth=1 && \
     cd v6d && \
     git submodule update --init && \
     mkdir -p /tmp/v6d/build && \

diff --git a/k8s/ubuntu/gsvineyard.Dockerfile b/k8s/ubuntu/gsvineyard.Dockerfile
@@ -14,7 +14,7 @@ RUN cd /tmp && \
     make -j`nproc` && \
     make install && \
     cd /tmp && \
-    git clone -b v0.3.19 https://github.com/v6d-io/v6d.git --depth=1 && \
+    git clone -b v0.3.21 https://github.com/v6d-io/v6d.git --depth=1 && \
     cd v6d && \
     git submodule update --init && \
     mkdir -p /tmp/v6d/build && \

diff --git a/python/requirements.txt b/python/requirements.txt
@@ -15,5 +15,5 @@ pyarrow
 PyYAML
 scipy
 tqdm
-vineyard>=0.3.16; sys_platform != "win32"
-vineyard-io>=0.3.16; sys_platform != "win32"
+vineyard>=0.3.21; sys_platform != "win32"
+vineyard-io>=0.3.21; sys_platform != "win32"
diff --git a/scripts/install_deps.sh b/scripts/install_deps.sh
@@ -14,8 +14,8 @@ readonly GREEN="\033[0;32m"
 readonly NC="\033[0m" # No Color
 
 readonly GRAPE_BRANCH="master" # libgrape-lite branch
-readonly V6D_VERSION="0.3.19"  # vineyard version
-readonly V6D_BRANCH="v0.3.19" # vineyard branch
+readonly V6D_VERSION="0.3.21"  # vineyard version
+readonly V6D_BRANCH="v0.3.21" # vineyard branch
 
 readonly OUTPUT_ENV_FILE="${HOME}/.graphscope_env"
 IS_IN_WSL=false && [[ ! -z "${IS_WSL}" || ! -z "${WSL_DISTRO_NAME}" ]] && IS_IN_WSL=true