Support combine multiple add vertice and edges operation.

.github/workflows/ci.yml

@@ -30,7 +30,7 @@ jobs:
       run:
         shell: scl enable devtoolset-7 -- bash --noprofile --norc -eo pipefail {0}
     container:
-      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.1.14
+      image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.1.15
       options:
         --shm-size 4096m
     strategy:
@@ -135,7 +135,7 @@ jobs:
     defaults:
       run:
         shell: scl enable devtoolset-7 -- bash --noprofile --norc -eo pipefail {0}
-    container: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.1.14
+    container: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.1.15
     steps:
     - name: Install Dependencies
       run: |
@@ -175,7 +175,7 @@ jobs:
     defaults:
       run:
         shell: scl enable devtoolset-7 -- bash --noprofile --norc -eo pipefail {0}
-    container: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.1.14
+    container: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-vineyard:v0.1.15
     steps:
     - name: Install Dependencies
       run: |

.github/workflows/local.yml

@@ -78,7 +78,7 @@ jobs:
 
     - name: Install libvineyard
       run: |
-        git clone -b v0.1.14 --single-branch --depth=1 https://github.com/alibaba/libvineyard.git
+        git clone -b v0.1.15 --single-branch --depth=1 https://github.com/alibaba/libvineyard.git
         cd libvineyard
         git submodule update --init
         mkdir build && pushd build

analytical_engine/CMakeLists.txt

@@ -109,7 +109,7 @@ endif ()
 find_package(libgrapelite REQUIRED)
 include_directories(${LIBGRAPELITE_INCLUDE_DIRS})
 
-find_package(vineyard 0.1.14 REQUIRED)
+find_package(vineyard 0.1.15 REQUIRED)
 include_directories(${VINEYARD_INCLUDE_DIRS})
 add_compile_options(-DENABLE_SELECTOR)
 

analytical_engine/core/grape_instance.cc

@@ -613,7 +613,7 @@ bl::result<rpc::GraphDef> GrapeInstance::copyGraph(
   return dst_wrapper->graph_def();
 }
 
-bl::result<rpc::GraphDef> GrapeInstance::addVertices(
+bl::result<rpc::GraphDef> GrapeInstance::addLabelsToGraph(
     const rpc::GSParams& params) {
   BOOST_LEAF_AUTO(graph_name, params.Get<std::string>(rpc::GRAPH_NAME));
   BOOST_LEAF_AUTO(
@@ -630,31 +630,7 @@ bl::result<rpc::GraphDef> GrapeInstance::addVertices(
   BOOST_LEAF_AUTO(graph_utils,
                   object_manager_.GetObject<PropertyGraphUtils>(type_sig));
   std::string dst_graph_name = "graph_" + generateId();
-  BOOST_LEAF_AUTO(dst_wrapper, graph_utils->AddVerticesToGraph(
-                                   src_frag_id, comm_spec_, *client_,
-                                   dst_graph_name, params));
-  BOOST_LEAF_CHECK(object_manager_.PutObject(dst_wrapper));
-
-  return dst_wrapper->graph_def();
-}
-
-bl::result<rpc::GraphDef> GrapeInstance::addEdges(const rpc::GSParams& params) {
-  BOOST_LEAF_AUTO(graph_name, params.Get<std::string>(rpc::GRAPH_NAME));
-  BOOST_LEAF_AUTO(
-      src_wrapper,
-      object_manager_.GetObject<ILabeledFragmentWrapper>(graph_name));
-  if (src_wrapper->graph_def().graph_type() != rpc::ARROW_PROPERTY) {
-    RETURN_GS_ERROR(vineyard::ErrorCode::kInvalidOperationError,
-                    "AddEdges is only avaiable for ArrowFragment");
-  }
-
-  auto src_frag_id =
-      std::static_pointer_cast<vineyard::Object>(src_wrapper->fragment())->id();
-  BOOST_LEAF_AUTO(type_sig, params.Get<std::string>(rpc::TYPE_SIGNATURE));
-  BOOST_LEAF_AUTO(graph_utils,
-                  object_manager_.GetObject<PropertyGraphUtils>(type_sig));
-  std::string dst_graph_name = "graph_" + generateId();
-  BOOST_LEAF_AUTO(dst_wrapper, graph_utils->AddEdgesToGraph(
+  BOOST_LEAF_AUTO(dst_wrapper, graph_utils->AddLabelsToGraph(
                                    src_frag_id, comm_spec_, *client_,
                                    dst_graph_name, params));
   BOOST_LEAF_CHECK(object_manager_.PutObject(dst_wrapper));
@@ -819,13 +795,8 @@ bl::result<std::shared_ptr<DispatchResult>> GrapeInstance::OnReceive(
     r->set_graph_def(graph_def);
     break;
   }
-  case rpc::ADD_VERTICES: {
-    BOOST_LEAF_AUTO(graph_def, addVertices(params));
-    r->set_graph_def(graph_def);
-    break;
-  }
-  case rpc::ADD_EDGES: {
-    BOOST_LEAF_AUTO(graph_def, addEdges(params));
+  case rpc::ADD_LABELS: {
+    BOOST_LEAF_AUTO(graph_def, addLabelsToGraph(params));
     r->set_graph_def(graph_def);
     break;
   }

analytical_engine/core/grape_instance.h

@@ -118,9 +118,7 @@ class GrapeInstance : public Subscriber {
 
   bl::result<rpc::GraphDef> copyGraph(const rpc::GSParams& params);
 
-  bl::result<rpc::GraphDef> addVertices(const rpc::GSParams& params);
-
-  bl::result<rpc::GraphDef> addEdges(const rpc::GSParams& params);
+  bl::result<rpc::GraphDef> addLabelsToGraph(const rpc::GSParams& params);
 
   bl::result<std::shared_ptr<grape::InArchive>> graphToNumpy(
       const rpc::GSParams& params);

analytical_engine/core/loader/arrow_fragment_loader.h

@@ -140,7 +140,107 @@ class ArrowFragmentLoader {
     return e_tables;
   }
 
-  boost::leaf::result<vineyard::ObjectID> AddVertices(
+  boost::leaf::result<vineyard::ObjectID> AddLabelsToGraph(
+      vineyard::ObjectID frag_id) {
+    if (!graph_info_->vertices.empty() && !graph_info_->edges.empty()) {
+      return addVerticesAndEdges(frag_id);
+    } else if (!graph_info_->vertices.empty()) {
+      return addVertices(frag_id);
+    } else {
+      return addEdges(frag_id);
+    }
+    return vineyard::InvalidObjectID();
+  }
+
+  boost::leaf::result<vineyard::ObjectID> addVerticesAndEdges(
+      vineyard::ObjectID frag_id) {
+    BOOST_LEAF_AUTO(partitioner, initPartitioner());
+    BOOST_LEAF_AUTO(partial_v_tables, LoadVertexTables());
+    BOOST_LEAF_AUTO(partial_e_tables, LoadEdgeTables());
+
+    auto basic_fragment_loader = std::make_shared<
+        vineyard::BasicEVFragmentLoader<OID_T, VID_T, partitioner_t>>(
+        client_, comm_spec_, partitioner, directed_, true, generate_eid_);
+    auto frag = std::static_pointer_cast<vineyard::ArrowFragment<oid_t, vid_t>>(
+        client_.GetObject(frag_id));
+    for (auto table : partial_v_tables) {
+      auto meta = table->schema()->metadata();
+      if (meta == nullptr) {
+        RETURN_GS_ERROR(vineyard::ErrorCode::kInvalidValueError,
+                        "Metadata of input vertex tables shouldn't be empty.");
+      }
+      int label_meta_index = meta->FindKey(LABEL_TAG);
+      if (label_meta_index == -1) {
+        RETURN_GS_ERROR(
+            vineyard::ErrorCode::kInvalidValueError,
+            "Metadata of input vertex tables should contain label name.");
+      }
+      std::string label_name = meta->value(label_meta_index);
+      BOOST_LEAF_CHECK(
+          basic_fragment_loader->AddVertexTable(label_name, table));
+    }
+    partial_v_tables.clear();
+    auto old_vm_ptr = frag->GetVertexMap();
+    BOOST_LEAF_CHECK(
+        basic_fragment_loader->ConstructVertices(old_vm_ptr->id()));
+
+    label_id_t pre_label_num = old_vm_ptr->label_num();
+    auto schema = frag->schema();
+    std::map<std::string, label_id_t> vertex_label_to_index;
+    for (auto& entry : schema.vertex_entries()) {
+      vertex_label_to_index[entry.label] = entry.id;
+    }
+    auto new_labels_index = basic_fragment_loader->get_vertex_label_to_index();
+    for (auto& pair : new_labels_index) {
+      vertex_label_to_index[pair.first] = pair.second + pre_label_num;
+    }
+    basic_fragment_loader->set_vertex_label_to_index(
+        std::move(vertex_label_to_index));
+    for (auto& table_vec : partial_e_tables) {
+      for (auto table : table_vec) {
+        auto meta = table->schema()->metadata();
+        if (meta == nullptr) {
+          RETURN_GS_ERROR(vineyard::ErrorCode::kInvalidValueError,
+                          "Metadata of input edge tables shouldn't be empty.");
+        }
+
+        int label_meta_index = meta->FindKey(LABEL_TAG);
+        if (label_meta_index == -1) {
+          RETURN_GS_ERROR(
+              vineyard::ErrorCode::kInvalidValueError,
+              "Metadata of input edge tables should contain label name.");
+        }
+        std::string label_name = meta->value(label_meta_index);
+
+        int src_label_meta_index = meta->FindKey(SRC_LABEL_TAG);
+        if (src_label_meta_index == -1) {
+          RETURN_GS_ERROR(
+              vineyard::ErrorCode::kInvalidValueError,
+              "Metadata of input edge tables should contain src label name.");
+        }
+        std::string src_label_name = meta->value(src_label_meta_index);
+
+        int dst_label_meta_index = meta->FindKey(DST_LABEL_TAG);
+        if (dst_label_meta_index == -1) {
+          RETURN_GS_ERROR(
+              vineyard::ErrorCode::kInvalidValueError,
+              "Metadata of input edge tables should contain dst label name.");
+        }
+        std::string dst_label_name = meta->value(dst_label_meta_index);
+
+        BOOST_LEAF_CHECK(basic_fragment_loader->AddEdgeTable(
+            src_label_name, dst_label_name, label_name, table));
+      }
+    }
+
+    partial_e_tables.clear();
+
+    BOOST_LEAF_CHECK(basic_fragment_loader->ConstructEdges(
+        schema.all_edge_label_num(), schema.all_vertex_label_num()));
+    return basic_fragment_loader->AddVerticesAndEdgesToFragment(frag);
+  }
+
+  boost::leaf::result<vineyard::ObjectID> addVertices(
       vineyard::ObjectID frag_id) {
     BOOST_LEAF_AUTO(partitioner, initPartitioner());
     BOOST_LEAF_AUTO(partial_v_tables, LoadVertexTables());
@@ -175,7 +275,7 @@ class ArrowFragmentLoader {
     return basic_fragment_loader->AddVerticesToFragment(frag);
   }
 
-  boost::leaf::result<vineyard::ObjectID> AddEdges(vineyard::ObjectID frag_id) {
+  boost::leaf::result<vineyard::ObjectID> addEdges(vineyard::ObjectID frag_id) {
     BOOST_LEAF_AUTO(partitioner, initPartitioner());
     BOOST_LEAF_AUTO(partial_e_tables, LoadEdgeTables());
 
@@ -366,16 +466,9 @@ class ArrowFragmentLoader {
     }
   }
 
-  boost::leaf::result<vineyard::ObjectID> AddVerticesAsFragmentGroup(
-      vineyard::ObjectID frag_id) {
-    BOOST_LEAF_AUTO(new_frag_id, AddVertices(frag_id));
-    VY_OK_OR_RAISE(client_.Persist(new_frag_id));
-    return vineyard::ConstructFragmentGroup(client_, new_frag_id, comm_spec_);
-  }
-
-  boost::leaf::result<vineyard::ObjectID> AddEdgesAsFragmentGroup(
+  boost::leaf::result<vineyard::ObjectID> AddLabelsToGraphAsFragmentGroup(
       vineyard::ObjectID frag_id) {
-    BOOST_LEAF_AUTO(new_frag_id, AddEdges(frag_id));
+    BOOST_LEAF_AUTO(new_frag_id, AddLabelsToGraph(frag_id));
     VY_OK_OR_RAISE(client_.Persist(new_frag_id));
     return vineyard::ConstructFragmentGroup(client_, new_frag_id, comm_spec_);
   }

analytical_engine/core/object/graph_utils.h

@@ -37,13 +37,7 @@ typedef void LoadGraphT(
     const std::string& graph_name, const rpc::GSParams& params,
     bl::result<std::shared_ptr<IFragmentWrapper>>& fragment_wrapper);
 
-typedef void AddVerticesToGraphT(
-    vineyard::ObjectID frag_id, const grape::CommSpec& comm_spec,
-    vineyard::Client& client, const std::string& graph_name,
-    const rpc::GSParams& params,
-    bl::result<std::shared_ptr<IFragmentWrapper>>& fragment_wrapper);
-
-typedef void AddEdgesToGraphT(
+typedef void AddLabelsToGraphT(
     vineyard::ObjectID frag_id, const grape::CommSpec& comm_spec,
     vineyard::Client& client, const std::string& graph_name,
     const rpc::GSParams& params,
@@ -73,8 +67,7 @@ class PropertyGraphUtils : public GSObject {
         lib_path_(std::move(lib_path)),
         dl_handle_(nullptr),
         load_graph_(nullptr),
-        add_vertices_to_graph_(nullptr),
-        add_edges_to_graph_(nullptr),
+        add_labels_to_graph_(nullptr),
         to_arrow_fragment_(nullptr),
         to_dynamic_fragment_(nullptr) {}
 
@@ -84,15 +77,10 @@ class PropertyGraphUtils : public GSObject {
       BOOST_LEAF_AUTO(p_fun, get_func_ptr(lib_path_, dl_handle_, "LoadGraph"));
       load_graph_ = reinterpret_cast<LoadGraphT*>(p_fun);
     }
-    {
-      BOOST_LEAF_AUTO(
-          p_fun, get_func_ptr(lib_path_, dl_handle_, "AddVerticesToGraph"));
-      add_vertices_to_graph_ = reinterpret_cast<AddVerticesToGraphT*>(p_fun);
-    }
     {
       BOOST_LEAF_AUTO(p_fun,
-                      get_func_ptr(lib_path_, dl_handle_, "AddEdgesToGraph"));
-      add_edges_to_graph_ = reinterpret_cast<AddEdgesToGraphT*>(p_fun);
+                      get_func_ptr(lib_path_, dl_handle_, "AddLabelsToGraph"));
+      add_labels_to_graph_ = reinterpret_cast<AddLabelsToGraphT*>(p_fun);
     }
     {
       BOOST_LEAF_AUTO(p_fun,
@@ -116,25 +104,14 @@ class PropertyGraphUtils : public GSObject {
     return wrapper;
   }
 
-  bl::result<std::shared_ptr<IFragmentWrapper>> AddVerticesToGraph(
-      vineyard::ObjectID frag_id, const grape::CommSpec& comm_spec,
-      vineyard::Client& client, const std::string& graph_name,
-      const rpc::GSParams& params) {
-    bl::result<std::shared_ptr<IFragmentWrapper>> wrapper;
-
-    add_vertices_to_graph_(frag_id, comm_spec, client, graph_name, params,
-                           wrapper);
-    return wrapper;
-  }
-
-  bl::result<std::shared_ptr<IFragmentWrapper>> AddEdgesToGraph(
+  bl::result<std::shared_ptr<IFragmentWrapper>> AddLabelsToGraph(
       vineyard::ObjectID frag_id, const grape::CommSpec& comm_spec,
       vineyard::Client& client, const std::string& graph_name,
       const rpc::GSParams& params) {
     bl::result<std::shared_ptr<IFragmentWrapper>> wrapper;
 
-    add_edges_to_graph_(frag_id, comm_spec, client, graph_name, params,
-                        wrapper);
+    add_labels_to_graph_(frag_id, comm_spec, client, graph_name, params,
+                         wrapper);
     return wrapper;
   }
 
@@ -172,8 +149,7 @@ class PropertyGraphUtils : public GSObject {
   std::string lib_path_;
   void* dl_handle_;
   LoadGraphT* load_graph_;
-  AddVerticesToGraphT* add_vertices_to_graph_;
-  AddEdgesToGraphT* add_edges_to_graph_;
+  AddLabelsToGraphT* add_labels_to_graph_;
   ToArrowFragmentT* to_arrow_fragment_;
   ToDynamicFragmentT* to_dynamic_fragment_;
 };