Cherry pick 3.3.0 (10.13) (#4720)

* Use vertex key only when use_vertex_key is on (#4716) * Revert "insert vertex key when only vertex and flag is set (#4685)" This reverts commit fff82a6. * Revert "only write vertex key when flag is set or explictly insert vertex (#4680)" This reverts commit a5bed33. * Revert "fix bug (#4675)" This reverts commit cccc014. * Revert "Rollback vertex key (#4629)" This reverts commit 545137a. * wtf * tck format * fix job status when all finished (#4717) * fix job name from zone balance to data balance * address wenhui's comment * fix create index if not exists behavior (#4705) Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com> Co-authored-by: Doodle <13706157+critical27@users.noreply.github.com> Co-authored-by: Doodle <13706157+critical27@users.noreply.github.com> Co-authored-by: Alex Xing <90179377+SuperYoko@users.noreply.github.com> Co-authored-by: kyle.cao <kyle.cao@vesoft.com>
vesoft-inc · Oct 13, 2022 · 8d63052 · 8d63052
1 parent 2af23ed
commit 8d63052
Show file tree

Hide file tree

Showing 33 changed files with 2,919 additions and 66 deletions.
diff --git a/docker/README.md b/docker/README.md
@@ -5,4 +5,4 @@ Following docker images will be ready in production.
 - [vesoft/nebula-graphd](https://hub.docker.com/r/vesoft/nebula-graphd): nebula-graphd service built with `Dockerfile.graphd`
 - [vesoft/nebula-metad](https://hub.docker.com/r/vesoft/nebula-metad): nebula-metad service built with `Dockerfile.metad`
 - [vesoft/nebula-storaged](https://hub.docker.com/r/vesoft/nebula-storaged): nebula-storaged service built with `Dockerfile.storaged`
-- [vesoft/nebula-tools](https://hub.docker.com/r/vesoft/nebula-tools): nebula tools built with `Dockerfile.tools`, including db_dump and meta_dump
+- [vesoft/nebula-tools](https://hub.docker.com/r/vesoft/nebula-tools): nebula tools built with `Dockerfile.tools`, including db_dump, meta_dump and db_upgrader
diff --git a/src/common/utils/NebulaKeyUtils.cpp b/src/common/utils/NebulaKeyUtils.cpp
@@ -253,6 +253,7 @@ std::vector<std::string> NebulaKeyUtils::snapshotPrefix(PartitionID partId) {
   if (partId == 0) {
     result.emplace_back("");
   } else {
+    result.emplace_back(vertexPrefix(partId));
     result.emplace_back(tagPrefix(partId));
     result.emplace_back(edgePrefix(partId));
     result.emplace_back(IndexKeyUtils::indexPrefix(partId));

diff --git a/src/graph/executor/admin/SubmitJobExecutor.cpp b/src/graph/executor/admin/SubmitJobExecutor.cpp
@@ -79,9 +79,14 @@ StatusOr<DataSet> SubmitJobExecutor::buildResult(meta::cpp2::JobOp jobOp,
       }
       const auto &jobsDesc = *resp.job_desc_ref();
       for (const auto &jobDesc : jobsDesc) {
+        // show zone balance as data balance
+        auto type = jobDesc.get_type();
+        if (type == meta::cpp2::JobType::ZONE_BALANCE) {
+          type = meta::cpp2::JobType::DATA_BALANCE;
+        }
         v.emplace_back(nebula::Row({
             jobDesc.get_job_id(),
-            apache::thrift::util::enumNameSafe(jobDesc.get_type()),
+            apache::thrift::util::enumNameSafe(type),
             apache::thrift::util::enumNameSafe(jobDesc.get_status()),
             convertJobTimestampToDateTime(jobDesc.get_start_time()),
             convertJobTimestampToDateTime(jobDesc.get_stop_time()),
@@ -124,7 +129,7 @@ nebula::DataSet SubmitJobExecutor::buildShowResultData(
     uint32_t total = paras.size() - index - 1, succeeded = 0, failed = 0, inProgress = 0,
              invalid = 0;
     v.emplace_back(Row({jd.get_job_id(),
-                        apache::thrift::util::enumNameSafe(meta::cpp2::JobType::ZONE_BALANCE),
+                        apache::thrift::util::enumNameSafe(meta::cpp2::JobType::DATA_BALANCE),
                         apache::thrift::util::enumNameSafe(jd.get_status()),
                         convertJobTimestampToDateTime(jd.get_start_time()).toString(),
                         convertJobTimestampToDateTime(jd.get_stop_time()).toString(),

diff --git a/src/graph/service/GraphFlags.cpp b/src/graph/service/GraphFlags.cpp
@@ -103,5 +103,3 @@ DEFINE_uint32(
     gc_worker_size,
     0,
     "Background garbage clean workers, default number is 0 which means using hardware core size.");
-
-DEFINE_bool(graph_use_vertex_key, false, "whether allow insert or query the vertex key");
diff --git a/src/graph/service/GraphFlags.h b/src/graph/service/GraphFlags.h
@@ -65,7 +65,4 @@ DECLARE_int32(max_job_size);
 
 DECLARE_bool(enable_async_gc);
 DECLARE_uint32(gc_worker_size);
-
-DECLARE_bool(graph_use_vertex_key);
-
 #endif  // GRAPH_GRAPHFLAGS_H_
diff --git a/src/graph/validator/MutateValidator.cpp b/src/graph/validator/MutateValidator.cpp
@@ -46,9 +46,6 @@ Status InsertVerticesValidator::check() {
   }
 
   auto tagItems = sentence->tagItems();
-  if (!FLAGS_graph_use_vertex_key && tagItems.empty()) {
-    return Status::SemanticError("Insert vertex is forbidden, please speicify the tag");
-  }
 
   schemas_.reserve(tagItems.size());
 

diff --git a/src/meta/processors/index/CreateEdgeIndexProcessor.cpp b/src/meta/processors/index/CreateEdgeIndexProcessor.cpp
@@ -16,6 +16,7 @@ void CreateEdgeIndexProcessor::process(const cpp2::CreateEdgeIndexReq& req) {
   const auto& indexName = req.get_index_name();
   auto& edgeName = req.get_edge_name();
   const auto& fields = req.get_fields();
+  auto ifNotExists = req.get_if_not_exists();
 
   std::set<std::string> columnSet;
   for (const auto& field : fields) {
@@ -41,7 +42,7 @@ void CreateEdgeIndexProcessor::process(const cpp2::CreateEdgeIndexReq& req) {
   // check if the space already exist index has the same index name
   auto ret = getIndexID(space, indexName);
   if (nebula::ok(ret)) {
-    if (req.get_if_not_exists()) {
+    if (ifNotExists) {
       handleErrorCode(nebula::cpp2::ErrorCode::SUCCEEDED);
     } else {
       LOG(INFO) << "Create Edge Index Failed: " << indexName << " has existed";
@@ -96,7 +97,15 @@ void CreateEdgeIndexProcessor::process(const cpp2::CreateEdgeIndexReq& req) {
     }
 
     if (checkIndexExist(fields, item)) {
-      resp_.code_ref() = nebula::cpp2::ErrorCode::E_EXISTED;
+      if (ifNotExists) {
+        resp_.code_ref() = nebula::cpp2::ErrorCode::SUCCEEDED;
+        cpp2::ID thriftID;
+        // Fill index id to avoid broken promise
+        thriftID.index_id_ref() = item.get_index_id();
+        resp_.id_ref() = thriftID;
+      } else {
+        resp_.code_ref() = nebula::cpp2::ErrorCode::E_EXISTED;
+      }
       onFinished();
       return;
     }

diff --git a/src/meta/processors/index/CreateTagIndexProcessor.cpp b/src/meta/processors/index/CreateTagIndexProcessor.cpp
@@ -16,6 +16,7 @@ void CreateTagIndexProcessor::process(const cpp2::CreateTagIndexReq& req) {
   const auto& indexName = req.get_index_name();
   auto& tagName = req.get_tag_name();
   const auto& fields = req.get_fields();
+  auto ifNotExists = req.get_if_not_exists();
 
   std::set<std::string> columnSet;
   for (const auto& field : fields) {
@@ -42,7 +43,7 @@ void CreateTagIndexProcessor::process(const cpp2::CreateTagIndexReq& req) {
   // check if the space has the index with the same name
   auto ret = getIndexID(space, indexName);
   if (nebula::ok(ret)) {
-    if (req.get_if_not_exists()) {
+    if (ifNotExists) {
       handleErrorCode(nebula::cpp2::ErrorCode::SUCCEEDED);
     } else {
       LOG(INFO) << "Create Tag Index Failed: " << indexName << " has existed";
@@ -96,7 +97,15 @@ void CreateTagIndexProcessor::process(const cpp2::CreateTagIndexReq& req) {
     }
 
     if (checkIndexExist(fields, item)) {
-      resp_.code_ref() = nebula::cpp2::ErrorCode::E_EXISTED;
+      if (ifNotExists) {
+        resp_.code_ref() = nebula::cpp2::ErrorCode::SUCCEEDED;
+        cpp2::ID thriftID;
+        // Fill index id to avoid broken promise
+        thriftID.index_id_ref() = item.get_index_id();
+        resp_.id_ref() = thriftID;
+      } else {
+        resp_.code_ref() = nebula::cpp2::ErrorCode::E_EXISTED;
+      }
       onFinished();
       return;
     }

diff --git a/src/storage/exec/GetPropNode.h b/src/storage/exec/GetPropNode.h
@@ -42,8 +42,10 @@ class GetTagPropNode : public QueryNode<VertexID> {
       return ret;
     }
 
-    // If none of the tag node valid, will check vertex key if use_vertex_key is true,
-    // do not emplace the row if the flag is false
+    // If none of the tag node valid, will check if vertex exists:
+    // 1. if use_vertex_key is true, check it by vertex key
+    // 2. if use_vertex_key is false, check it by scanning vertex prefix
+    // If vertex does not exists, do not emplace the row.
     if (!std::any_of(tagNodes_.begin(), tagNodes_.end(), [](const auto& tagNode) {
           return tagNode->valid();
         })) {
@@ -58,7 +60,16 @@ class GetTagPropNode : public QueryNode<VertexID> {
           return ret;
         }
       } else {
-        return nebula::cpp2::ErrorCode::SUCCEEDED;
+        // check if vId has any valid tag by prefix scan
+        std::unique_ptr<kvstore::KVIterator> iter;
+        auto tagPrefix = NebulaKeyUtils::tagPrefix(context_->vIdLen(), partId, vId);
+        ret = context_->env()->kvstore_->prefix(context_->spaceId(), partId, tagPrefix, &iter);
+        if (ret != nebula::cpp2::ErrorCode::SUCCEEDED) {
+          return ret;
+        } else if (!iter->valid()) {
+          return nebula::cpp2::ErrorCode::SUCCEEDED;
+        }
+        // if has any tag, will emplace a row with vId
       }
     }
 

diff --git a/src/storage/exec/UpdateNode.h b/src/storage/exec/UpdateNode.h
@@ -488,6 +488,9 @@ class UpdateTagNode : public UpdateNode<VertexID> {
       }
     }
     // step 3, insert new vertex data
+    if (FLAGS_use_vertex_key) {
+      batchHolder->put(NebulaKeyUtils::vertexKey(context_->vIdLen(), partId, vId), "");
+    }
     batchHolder->put(std::move(key_), std::move(nVal));
     return encodeBatchValue(batchHolder->getBatch());
   }

diff --git a/src/storage/mutate/AddVerticesProcessor.cpp b/src/storage/mutate/AddVerticesProcessor.cpp
@@ -62,7 +62,6 @@ void AddVerticesProcessor::process(const cpp2::AddVerticesRequest& req) {
 void AddVerticesProcessor::doProcess(const cpp2::AddVerticesRequest& req) {
   const auto& partVertices = req.get_parts();
   const auto& propNamesMap = req.get_prop_names();
-  bool onlyVertex = propNamesMap.empty();
   for (auto& part : partVertices) {
     auto partId = part.first;
     const auto& vertices = part.second;
@@ -82,7 +81,7 @@ void AddVerticesProcessor::doProcess(const cpp2::AddVerticesRequest& req) {
         code = nebula::cpp2::ErrorCode::E_INVALID_VID;
         break;
       }
-      if (onlyVertex && FLAGS_use_vertex_key) {
+      if (FLAGS_use_vertex_key) {
         data.emplace_back(NebulaKeyUtils::vertexKey(spaceVidLen_, partId, vid), "");
       }
       for (auto& newTag : newTags) {
@@ -140,7 +139,6 @@ void AddVerticesProcessor::doProcess(const cpp2::AddVerticesRequest& req) {
 void AddVerticesProcessor::doProcessWithIndex(const cpp2::AddVerticesRequest& req) {
   const auto& partVertices = req.get_parts();
   const auto& propNamesMap = req.get_prop_names();
-  bool onlyVertex = propNamesMap.empty();
 
   for (const auto& part : partVertices) {
     auto partId = part.first;
@@ -163,7 +161,8 @@ void AddVerticesProcessor::doProcessWithIndex(const cpp2::AddVerticesRequest& re
         code = nebula::cpp2::ErrorCode::E_INVALID_VID;
         break;
       }
-      if (onlyVertex && FLAGS_use_vertex_key) {
+
+      if (FLAGS_use_vertex_key) {
         verticeData.emplace_back(NebulaKeyUtils::vertexKey(spaceVidLen_, partId, vid));
       }
       for (const auto& newTag : newTags) {

diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt
@@ -69,3 +69,4 @@ nebula_add_subdirectory(simple-kv-verify)
 endif()
 nebula_add_subdirectory(meta-dump)
 nebula_add_subdirectory(db-dump)
+nebula_add_subdirectory(db-upgrade)
diff --git a/src/tools/db-upgrade/CMakeLists.txt b/src/tools/db-upgrade/CMakeLists.txt
@@ -0,0 +1,33 @@
+# Copyright (c) 2022 vesoft inc. All rights reserved.
+#
+# This source code is licensed under Apache 2.0 License.
+
+nebula_add_executable(
+    NAME
+        db_upgrader
+    SOURCES
+        DbUpgraderTool.cpp
+        NebulaKeyUtilsV1.cpp
+        NebulaKeyUtilsV2.cpp
+        NebulaKeyUtilsV3.cpp
+        DbUpgrader.cpp
+    OBJECTS
+        ${tools_test_deps}
+    LIBRARIES
+        ${ROCKSDB_LIBRARIES}
+        ${THRIFT_LIBRARIES}
+        ${PROXYGEN_LIBRARIES}
+)
+
+install(
+    TARGETS
+        db_upgrader
+    PERMISSIONS
+        OWNER_EXECUTE OWNER_WRITE OWNER_READ
+        GROUP_EXECUTE GROUP_READ
+        WORLD_EXECUTE WORLD_READ
+    DESTINATION
+        bin
+    COMPONENT
+        tool
+)