Merge branch 'main' into wl/revised_gae_arch

.github/workflows/build-graphscope-wheels-macos.yml

@@ -132,7 +132,7 @@ jobs:
       run: |
         source ~/.graphscope_env
         python3 -m pip install libclang
-        git clone --single-branch -b v0.12.2 --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d
+        git clone --single-branch -b v0.13.0 --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d
         cd /tmp/v6d
         git submodule update --init
         cmake .  -DCMAKE_INSTALL_PREFIX=/usr/local \

.github/workflows/gae.yml

@@ -29,7 +29,7 @@ jobs:
     runs-on: ubuntu-20.04
     if: ${{ github.repository == 'alibaba/GraphScope' }}
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.12.2
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.13.0
       options:
         --shm-size 4096m
     steps:
@@ -38,20 +38,21 @@ jobs:
     - name: Install latest vineyard
       if: false
       run: |
+        source /home/graphscope/.graphscope_env
+        sudo pip3 install libclang
         git clone --single-branch --depth=1 https://github.com/v6d-io/v6d.git /tmp/v6d
         cd /tmp/v6d
         git submodule update --init
         mkdir -p build && cd build
-        cmake .. -DCMAKE_INSTALL_PREFIX=/opt/vineyard \
+        cmake .. -DCMAKE_PREFIX_PATH=/opt/graphscope \
+                 -DCMAKE_INSTALL_PREFIX=/opt/graphscope \
+                 -DOPENSSL_ROOT_DIR=/opt/graphscope \
                  -DUSE_EXTERNAL_ETCD_LIBS=OFF \
                  -DBUILD_SHARED_LIBS=ON \
                  -DBUILD_VINEYARD_TESTS=OFF
         make -j$(nproc)
         sudo make install
 
-        # copy to /usr/local, keep the same logic as gsvineyard.Dockerfile
-        sudo cp -r /opt/vineyard/* /usr/local/
-
     - name: Build
       env:
         RUN_JAVA_TESTS: ON
@@ -83,7 +84,7 @@ jobs:
         export USER_JAR_PATH=${GITHUB_WORKSPACE}/analytical_engine/java/grape-demo/target/grape-demo-${version}-shaded.jar
         # for giraph test
         export GIRAPH_JAR_PATH=${GITHUB_WORKSPACE}/analytical_engine/java/grape-giraph/target/grape-giraph-${version}-shaded.jar
-        
+
         cd ${GITHUB_WORKSPACE}/analytical_engine/build
         ../test/app_tests.sh --test_dir ${GS_TEST_DIR}
 

.github/workflows/local-ci.yml

@@ -232,14 +232,16 @@ jobs:
         git submodule update --init
         mkdir -p build && cd build
         cmake .. -DCMAKE_INSTALL_PREFIX=/opt/vineyard \
+                 -DCMAKE_PREFIX_PATH=/opt/graphscope \
+                 -DOPENSSL_ROOT_DIR=/opt/graphscope \
                  -DUSE_EXTERNAL_ETCD_LIBS=OFF \
                  -DBUILD_SHARED_LIBS=ON \
                  -DBUILD_VINEYARD_TESTS=OFF
         make -j$(nproc)
         sudo make install
 
         # copy to /usr/local, keep the same logic as gsvineyard.Dockerfile
-        sudo cp -r /opt/vineyard/* ${GRAPHSCOPE_HOME}/
+        sudo rsync -av /opt/vineyard/ ${GRAPHSCOPE_HOME}/
 
     - name: Build Wheels
       run: |

.github/workflows/networkx-forward-algo-nightly.yml

@@ -16,7 +16,7 @@ jobs:
       run:
         shell: bash --noprofile --norc -eo pipefail {0}
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.12.2
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.13.0
       options:
         --shm-size 4096m
 

.github/workflows/nightly.yml

@@ -39,7 +39,7 @@ jobs:
         fi
         sudo mkdir /opt/graphscope
         sudo chown -R $(id -u):$(id -g) /opt/graphscope
-        ./gs install-deps dev --v6d-version v0.12.2
+        ./gs install-deps dev --v6d-version v0.13.0
 
     - name: Setup tmate session
       if: false 

analytical_engine/CMakeLists.txt

@@ -240,7 +240,7 @@ else()
 endif()
 
 # find vineyard after arrow to avoid duplicate target names
-find_package(vineyard 0.12.2 REQUIRED)
+find_package(vineyard 0.13.0 REQUIRED)
 include_directories(${VINEYARD_INCLUDE_DIRS})
 add_compile_options(-DENABLE_SELECTOR)
 

analytical_engine/core/context/tensor_dataframe_builder.h

@@ -55,30 +55,32 @@ class MPIGlobalTensorBuilder : public vineyard::GlobalTensorBuilder,
 
   void AddChunks(std::vector<vineyard::ObjectID> const& chunk_ids) {
     for (auto& chunk_id : chunk_ids) {
-      this->local_chunk_ids_.emplace_back(chunk_id);
+      this->AddChunk(chunk_id);
     }
   }
 
-  std::shared_ptr<vineyard::Object> _Seal(vineyard::Client& client) override {
-    std::shared_ptr<vineyard::GlobalTensor> tensor;
+  vineyard::Status _Seal(vineyard::Client& client,
+                         std::shared_ptr<vineyard::Object>& object) override {
     vineyard::ObjectID id = vineyard::InvalidObjectID();
+    this->SetGlobal();
     if (comm_spec_.worker_id() == 0) {
-      tensor = std::dynamic_pointer_cast<vineyard::GlobalTensor>(
-          vineyard::GlobalTensorBuilder::_Seal(client));
-      id = tensor->id();
+      RETURN_ON_ERROR(vineyard::GlobalTensorBuilder::_Seal(client, object));
+      id = object->id();
+      RETURN_ON_ERROR(client_.Persist(id));
     } else {
-      VINEYARD_CHECK_OK(this->Build(client));
+      RETURN_ON_ERROR(this->Build(client));
     }
     SyncGlobalObjectID(comm_spec_, id);  // this sync can be seen as a barrier
     if (comm_spec_.worker_id() != 0) {
-      // FIXME: the aim of `Construct` is to fillup the ObjectSet, needs better
+      // FIXME: the aim of `Construct` is to fill up the ObjectSet, needs better
       // design.
-      tensor = std::make_shared<vineyard::GlobalTensor>();
+      auto tensor = std::make_shared<vineyard::GlobalTensor>();
       vineyard::ObjectMeta meta;
-      VINEYARD_CHECK_OK(client.GetMetaData(id, meta, true));
+      RETURN_ON_ERROR(client.GetMetaData(id, meta, true));
       tensor->Construct(meta);
+      object = std::static_pointer_cast<vineyard::Object>(tensor);
     }
-    return tensor;
+    return vineyard::Status::OK();
   }
 
   vineyard::Status Build(vineyard::Client& client) override {
@@ -114,28 +116,30 @@ class MPIGlobalDataFrameBuilder : public vineyard::GlobalDataFrameBuilder,
 
   void AddChunks(std::vector<vineyard::ObjectID> const& chunk_ids) {
     for (auto& chunk_id : chunk_ids) {
-      this->local_chunk_ids_.emplace_back(chunk_id);
+      this->AddChunk(chunk_id);
     }
   }
 
-  std::shared_ptr<vineyard::Object> _Seal(vineyard::Client& client) override {
-    std::shared_ptr<vineyard::GlobalDataFrame> df;
+  vineyard::Status _Seal(vineyard::Client& client,
+                         std::shared_ptr<vineyard::Object>& object) override {
     vineyard::ObjectID id = vineyard::InvalidObjectID();
+    this->SetGlobal();
     if (comm_spec_.worker_id() == 0) {
-      df = std::dynamic_pointer_cast<vineyard::GlobalDataFrame>(
-          vineyard::GlobalDataFrameBuilder::_Seal(client));
-      id = df->id();
+      RETURN_ON_ERROR(vineyard::GlobalDataFrameBuilder::_Seal(client, object));
+      id = object->id();
+      RETURN_ON_ERROR(client_.Persist(id));
     } else {
-      VINEYARD_CHECK_OK(this->Build(client));
+      RETURN_ON_ERROR(this->Build(client));
     }
     SyncGlobalObjectID(comm_spec_, id);  // this sync can be seen as a barrier
     if (comm_spec_.worker_id() != 0) {
-      df = std::make_shared<vineyard::GlobalDataFrame>();
+      auto df = std::make_shared<vineyard::GlobalDataFrame>();
       vineyard::ObjectMeta meta;
-      VINEYARD_CHECK_OK(client.GetMetaData(id, meta, true));
+      RETURN_ON_ERROR(client.GetMetaData(id, meta, true));
       df->Construct(meta);
+      object = std::static_pointer_cast<vineyard::Object>(df);
     }
-    return df;
+    return vineyard::Status::OK();
   }
 
   vineyard::Status Build(vineyard::Client& client) override {

analytical_engine/core/fragment/arrow_flattened_fragment.h

@@ -516,11 +516,16 @@ class ArrowFlattenedFragment {
                                   prop_id_t e_prop_id)
       : fragment_(frag), v_prop_id_(v_prop_id), e_prop_id_(e_prop_id) {
     ivnum_ = ovnum_ = tvnum_ = 0;
-    label_id_t vertex_label_num = fragment_->vertex_label_num();
+    auto& schema = fragment_->schema();
+    label_id_t vertex_label_num =
+        static_cast<label_id_t>(schema.AllVertexEntries().size());
     for (label_id_t v_label = 0; v_label < vertex_label_num; v_label++) {
-      vid_t ivnum = fragment_->GetInnerVerticesNum(v_label);
-      vid_t ovnum = fragment_->GetOuterVerticesNum(v_label);
-      vid_t tvnum = fragment_->GetVerticesNum(v_label);
+      vid_t ivnum = 0, ovnum = 0, tvnum = 0;
+      if (schema.IsVertexValid(v_label)) {
+        ivnum = fragment_->GetInnerVerticesNum(v_label);
+        ovnum = fragment_->GetOuterVerticesNum(v_label);
+        tvnum = fragment_->GetVerticesNum(v_label);
+      }
       ivnums_.push_back(ivnum);
       ovnums_.push_back(ovnum);
       tvnums_.push_back(tvnum);
@@ -535,13 +540,19 @@ class ArrowFlattenedFragment {
     union_vertex_range_offset_.resize(2 * vertex_label_num + 1, 0);
     for (label_id_t v_label = 0; v_label < vertex_label_num; v_label++) {
       union_vertex_range_offset_[v_label + 1] =
-          union_vertex_range_offset_[v_label] +
-          fragment_->GetInnerVerticesNum(v_label);
+          union_vertex_range_offset_[v_label];
+      if (schema.IsVertexValid(v_label)) {
+        union_vertex_range_offset_[v_label + 1] +=
+            fragment_->GetInnerVerticesNum(v_label);
+      }
     }
     for (label_id_t v_label = 0; v_label < vertex_label_num; v_label++) {
       union_vertex_range_offset_[v_label + vertex_label_num + 1] =
-          union_vertex_range_offset_[v_label + vertex_label_num] +
-          fragment_->GetOuterVerticesNum(v_label);
+          union_vertex_range_offset_[v_label + vertex_label_num];
+      if (schema.IsVertexValid(v_label)) {
+        union_vertex_range_offset_[v_label + vertex_label_num + 1] +=
+            fragment_->GetOuterVerticesNum(v_label);
+      }
     }
     // init id parser
     union_id_parser_.Init(fragment_->fnum(), vertex_label_num,
@@ -730,8 +741,13 @@ class ArrowFlattenedFragment {
     std::vector<vineyard::property_graph_utils::AdjList<vid_t, eid_t>>
         adj_lists;
     adj_lists.reserve(fragment_->edge_label_num());
-    for (label_id_t e_label = 0; e_label < fragment_->edge_label_num();
-         e_label++) {
+    auto& schema = fragment_->schema();
+    label_id_t edge_label_num =
+        static_cast<label_id_t>(schema.AllEdgeEntries().size());
+    for (label_id_t e_label = 0; e_label < edge_label_num; e_label++) {
+      if (!schema.IsEdgeValid(e_label)) {
+        continue;
+      }
       auto adj_list = fragment_->GetOutgoingAdjList(v_, e_label);
       if (adj_list.NotEmpty()) {
         adj_lists.push_back(adj_list);
@@ -745,8 +761,13 @@ class ArrowFlattenedFragment {
     std::vector<vineyard::property_graph_utils::AdjList<vid_t, eid_t>>
         adj_lists;
     adj_lists.reserve(fragment_->edge_label_num());
-    for (label_id_t e_label = 0; e_label < fragment_->edge_label_num();
-         e_label++) {
+    auto& schema = fragment_->schema();
+    label_id_t edge_label_num =
+        static_cast<label_id_t>(schema.AllEdgeEntries().size());
+    for (label_id_t e_label = 0; e_label < edge_label_num; e_label++) {
+      if (!schema.IsEdgeValid(e_label)) {
+        continue;
+      }
       auto adj_list = fragment_->GetIncomingAdjList(v_, e_label);
       if (adj_list.NotEmpty()) {
         adj_lists.push_back(adj_list);
@@ -758,8 +779,13 @@ class ArrowFlattenedFragment {
   inline int GetLocalOutDegree(const vertex_t& v) const {
     vertex_t v_(union_id_parser_.ParseContinuousLid(v.GetValue()));
     int local_out_degree = 0;
-    for (label_id_t e_label = 0; e_label < fragment_->edge_label_num();
-         e_label++) {
+    auto& schema = fragment_->schema();
+    label_id_t edge_label_num =
+        static_cast<label_id_t>(schema.AllEdgeEntries().size());
+    for (label_id_t e_label = 0; e_label < edge_label_num; e_label++) {
+      if (!schema.IsEdgeValid(e_label)) {
+        continue;
+      }
       local_out_degree += fragment_->GetLocalOutDegree(v_, e_label);
     }
     return local_out_degree;
@@ -768,8 +794,13 @@ class ArrowFlattenedFragment {
   inline int GetLocalInDegree(const vertex_t& v) const {
     vertex_t v_(union_id_parser_.ParseContinuousLid(v.GetValue()));
     int local_in_degree = 0;
-    for (label_id_t e_label = 0; e_label < fragment_->edge_label_num();
-         e_label++) {
+    auto& schema = fragment_->schema();
+    label_id_t edge_label_num =
+        static_cast<label_id_t>(schema.AllEdgeEntries().size());
+    for (label_id_t e_label = 0; e_label < edge_label_num; e_label++) {
+      if (!schema.IsEdgeValid(e_label)) {
+        continue;
+      }
       local_in_degree += fragment_->GetLocalInDegree(v_, e_label);
     }
     return local_in_degree;
@@ -779,8 +810,13 @@ class ArrowFlattenedFragment {
     vertex_t v_(union_id_parser_.ParseContinuousLid(v.GetValue()));
     std::vector<grape::DestList> dest_lists;
     dest_lists.reserve(fragment_->edge_label_num());
-    for (label_id_t e_label = 0; e_label < fragment_->edge_label_num();
-         e_label++) {
+    auto& schema = fragment_->schema();
+    label_id_t edge_label_num =
+        static_cast<label_id_t>(schema.AllEdgeEntries().size());
+    for (label_id_t e_label = 0; e_label < edge_label_num; e_label++) {
+      if (!schema.IsEdgeValid(e_label)) {
+        continue;
+      }
       dest_lists.push_back(fragment_->IEDests(v_, e_label));
     }
     return dest_list_t(dest_lists);
@@ -790,8 +826,13 @@ class ArrowFlattenedFragment {
     vertex_t v_(union_id_parser_.ParseContinuousLid(v.GetValue()));
     std::vector<grape::DestList> dest_lists;
     dest_lists.reserve(fragment_->edge_label_num());
-    for (label_id_t e_label = 0; e_label < fragment_->edge_label_num();
-         e_label++) {
+    auto& schema = fragment_->schema();
+    label_id_t edge_label_num =
+        static_cast<label_id_t>(schema.AllEdgeEntries().size());
+    for (label_id_t e_label = 0; e_label < edge_label_num; e_label++) {
+      if (!schema.IsEdgeValid(e_label)) {
+        continue;
+      }
       dest_lists.push_back(fragment_->OEDests(v_, e_label));
     }
     return dest_list_t(dest_lists);
@@ -801,8 +842,13 @@ class ArrowFlattenedFragment {
     vertex_t v_(union_id_parser_.ParseContinuousLid(v.GetValue()));
     std::vector<grape::DestList> dest_lists;
     dest_lists.reserve(fragment_->edge_label_num());
-    for (label_id_t e_label = 0; e_label < fragment_->edge_label_num();
-         e_label++) {
+    auto& schema = fragment_->schema();
+    label_id_t edge_label_num =
+        static_cast<label_id_t>(schema.AllEdgeEntries().size());
+    for (label_id_t e_label = 0; e_label < edge_label_num; e_label++) {
+      if (!schema.IsEdgeValid(e_label)) {
+        continue;
+      }
       dest_lists.push_back(fragment_->IOEDests(v_, e_label));
     }
     return dest_list_t(dest_lists);

analytical_engine/test/run_ctx.cc

@@ -177,15 +177,8 @@ void output_vineyard_tensor(vineyard::Client& client,
                             const std::string& prefix) {
   auto stored_tensor = std::dynamic_pointer_cast<vineyard::GlobalTensor>(
       client.GetObject(tensor_object));
-  auto const& shape = stored_tensor->shape();
-  auto const& partition_shape = stored_tensor->partition_shape();
   auto const& local_chunks = stored_tensor->LocalPartitions(client);
-  CHECK_EQ(shape.size(), 1);
-  CHECK_EQ(partition_shape.size(), 1);
   CHECK_EQ(local_chunks.size(), static_cast<size_t>(comm_spec.local_num()));
-  if (comm_spec.worker_id() == 0) {
-    LOG(INFO) << "tensor shape: " << shape[0] << ", " << partition_shape[0];
-  }
 
   if (comm_spec.local_id() == 0) {
     for (auto obj : local_chunks) {
@@ -223,15 +216,9 @@ void output_vineyard_dataframe(vineyard::Client& client,
                                const std::string& prefix) {
   auto stored_dataframe = std::dynamic_pointer_cast<vineyard::GlobalDataFrame>(
       client.GetObject(dataframe_object));
-  auto const& partition_shape = stored_dataframe->partition_shape();
   auto const& local_chunks = stored_dataframe->LocalPartitions(client);
   CHECK_EQ(local_chunks.size(), static_cast<size_t>(comm_spec.local_num()));
 
-  if (comm_spec.worker_id() == 0) {
-    LOG(INFO) << "dataframe shape: " << partition_shape.first << ", "
-              << partition_shape.second;
-  }
-
   if (comm_spec.local_id() == 0) {
     for (auto const& obj : local_chunks) {
       auto single_dataframe =

analytical_engine/test/run_java_app.cc

@@ -173,15 +173,8 @@ void output_vineyard_tensor(vineyard::Client& client,
                             vineyard::AnyType& expected_type) {
   auto stored_tensor = std::dynamic_pointer_cast<vineyard::GlobalTensor>(
       client.GetObject(tensor_object));
-  auto const& shape = stored_tensor->shape();
-  auto const& partition_shape = stored_tensor->partition_shape();
   auto const& local_chunks = stored_tensor->LocalPartitions(client);
-  CHECK_EQ(shape.size(), 1);
-  CHECK_EQ(partition_shape.size(), 1);
   CHECK_EQ(local_chunks.size(), static_cast<size_t>(comm_spec.local_num()));
-  if (comm_spec.worker_id() == 0) {
-    VLOG(1) << "tensor shape: " << shape[0] << ", " << partition_shape[0];
-  }
 
   if (comm_spec.local_id() == 0) {
     for (auto obj : local_chunks) {

charts/graphscope/values.yaml

@@ -89,7 +89,7 @@ vineyard:
   image:
     name: vineyardcloudnative/vineyardd
     # Overrides the image tag whose default is the chart appVersion.
-    tag: v0.12.2
+    tag: v0.13.0
   resources:
     requests:
       cpu: 0.5