From 12ceb15954bdf737e68145ad6c6f791fbedcf7ef Mon Sep 17 00:00:00 2001 From: Doodle <13706157+critical27@users.noreply.github.com> Date: Wed, 21 Jul 2021 06:41:50 -0500 Subject: [PATCH 1/2] fix task in recovered balance plan is still marked as FAILED (#528) * fix task in recovered balance plan is still marked as FAILED * fix bugs --- src/meta/processors/admin/AdminClient.cpp | 4 ++++ src/meta/processors/admin/Balancer.cpp | 7 ++----- src/meta/processors/partsMan/ListHostsProcessor.cpp | 4 ++-- src/storage/StorageFlags.cpp | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/meta/processors/admin/AdminClient.cpp b/src/meta/processors/admin/AdminClient.cpp index 0da7be80e..181e29502 100644 --- a/src/meta/processors/admin/AdminClient.cpp +++ b/src/meta/processors/admin/AdminClient.cpp @@ -36,6 +36,10 @@ folly::Future AdminClient::transLeader(GraphSpaceID spaceId, if (it == peers.end()) { return Status::PartNotFound(); } + if (peers.size() == 1 && peers.front() == leader) { + // if there is only one replica, skip transfer leader phase + return Status::OK(); + } auto target = dst; if (dst == kRandomPeer) { for (auto& p : peers) { diff --git a/src/meta/processors/admin/Balancer.cpp b/src/meta/processors/admin/Balancer.cpp index 94b7c8e78..bee329817 100644 --- a/src/meta/processors/admin/Balancer.cpp +++ b/src/meta/processors/admin/Balancer.cpp @@ -174,7 +174,8 @@ nebula::cpp2::ErrorCode Balancer::recovery() { return recRet; } } - return nebula::cpp2::ErrorCode::SUCCEEDED; + // save the balance plan again because FAILED tasks would be marked as IN_PROGRESS again + return plan_->saveInStore(); } nebula::cpp2::ErrorCode @@ -303,10 +304,6 @@ Balancer::genTasks(GraphSpaceID spaceId, } } - if (confirmedHostParts.size() < 2) { - LOG(INFO) << "Too few hosts, no need for balance!"; - return nebula::cpp2::ErrorCode::E_NO_VALID_HOST; - } // 2. Make all hosts in confirmedHostParts balanced if (balanceParts(plan_->id_, spaceId, confirmedHostParts, totalParts, tasks)) { return tasks; diff --git a/src/meta/processors/partsMan/ListHostsProcessor.cpp b/src/meta/processors/partsMan/ListHostsProcessor.cpp index 82bcc792c..dc0d3ccd6 100644 --- a/src/meta/processors/partsMan/ListHostsProcessor.cpp +++ b/src/meta/processors/partsMan/ListHostsProcessor.cpp @@ -200,7 +200,7 @@ nebula::cpp2::ErrorCode ListHostsProcessor::fillLeaders() { } auto it = std::find(activeHosts.begin(), activeHosts.end(), host); if (it == activeHosts.end()) { - LOG(INFO) << "skip inactive host: " << host; + VLOG(1) << "skip inactive host: " << host; continue; // skip inactive host } @@ -209,7 +209,7 @@ nebula::cpp2::ErrorCode ListHostsProcessor::fillLeaders() { }); if (hostIt == hostItems_.end()) { - LOG(INFO) << "skip inactive host"; + VLOG(1) << "skip inactive host"; continue; } diff --git a/src/storage/StorageFlags.cpp b/src/storage/StorageFlags.cpp index d37191251..b1aacb080 100644 --- a/src/storage/StorageFlags.cpp +++ b/src/storage/StorageFlags.cpp @@ -15,7 +15,7 @@ DEFINE_int32(waiting_catch_up_retry_times, 30, "retry times when waiting for cat DEFINE_int32(waiting_catch_up_interval_in_secs, 30, "interval between two requests for catching up state"); -DEFINE_int32(waiting_new_leader_retry_times, 30, "retry times when waiting for catching up data"); +DEFINE_int32(waiting_new_leader_retry_times, 5, "retry times when waiting for new leader"); DEFINE_int32(waiting_new_leader_interval_in_secs, 5, "interval between two requests for catching up state"); From 249bc46d011340c8eacd84d57b78c2edf6f9a51a Mon Sep 17 00:00:00 2001 From: laura-ding <48548375+laura-ding@users.noreply.github.com> Date: Fri, 23 Jul 2021 17:39:17 +0800 Subject: [PATCH 2/2] Fix db_dump print int vid (#533) * fix db_dump print int vid * fix --vids * fix partId * use new common --- .../mutate/AddEdgesAtomicProcessor.cpp | 7 +--- src/storage/test/TossEnvironment.h | 4 +- .../transaction/TransactionManager.cpp | 13 ++---- src/tools/db-dump/DbDumper.cpp | 42 +++++++++++++++---- src/tools/db-dump/DbDumper.h | 3 ++ 5 files changed, 41 insertions(+), 28 deletions(-) diff --git a/src/storage/mutate/AddEdgesAtomicProcessor.cpp b/src/storage/mutate/AddEdgesAtomicProcessor.cpp index 23e893a43..7f06c3f70 100644 --- a/src/storage/mutate/AddEdgesAtomicProcessor.cpp +++ b/src/storage/mutate/AddEdgesAtomicProcessor.cpp @@ -49,13 +49,8 @@ void AddEdgesAtomicProcessor::processByChain(const cpp2::AddEdgesRequest& req) { for (auto& part : *req.parts_ref()) { auto localPart = part.first; for (auto& edge : part.second) { - auto stPartId = env_->metaClient_->partId(spaceId_, + auto remotePart = env_->metaClient_->partId(spaceId_, (*(*edge.key_ref()).dst_ref()).getStr()); - if (!stPartId.ok()) { - failedPart[localPart] = nebula::cpp2::ErrorCode::E_SPACE_NOT_FOUND; - break; - } - auto remotePart = stPartId.value(); ChainId cid{localPart, remotePart}; if (FLAGS_trace_toss) { auto& ekey = *edge.key_ref(); diff --git a/src/storage/test/TossEnvironment.h b/src/storage/test/TossEnvironment.h index 7f473dedc..f4df2d2fd 100644 --- a/src/storage/test/TossEnvironment.h +++ b/src/storage/test/TossEnvironment.h @@ -545,9 +545,7 @@ struct TossEnvironment { int32_t getPartId(const std::string& src) { // auto stPart = mClient_->partId(spaceId_, edgeKey.src.getStr()); - auto stPart = mClient_->partId(spaceId_, src); - LOG_IF(FATAL, !stPart.ok()) << "mClient_->partId failed"; - return stPart.value(); + return mClient_->partId(spaceId_, src); } /** diff --git a/src/storage/transaction/TransactionManager.cpp b/src/storage/transaction/TransactionManager.cpp index 545f87191..f6c63e387 100644 --- a/src/storage/transaction/TransactionManager.cpp +++ b/src/storage/transaction/TransactionManager.cpp @@ -231,11 +231,7 @@ TransactionManager::updateEdgeAtomic(size_t vIdLen, PartitionID partId, const cpp2::EdgeKey& edgeKey, GetBatchFunc batchGetter) { - auto stRemotePart = env_->metaClient_->partId(spaceId, (*edgeKey.dst_ref()).getStr()); - if (!stRemotePart.ok()) { - return folly::makeFuture(nebula::cpp2::ErrorCode::E_UNKNOWN); - } - auto remotePart = stRemotePart.value(); + auto remotePart = env_->metaClient_->partId(spaceId, (*edgeKey.dst_ref()).getStr()); auto localKey = TransactionUtils::edgeKey(vIdLen, partId, edgeKey); std::vector data{std::make_pair(localKey, "")}; @@ -265,11 +261,8 @@ TransactionManager::resumeTransaction(size_t vIdLen, auto c = folly::makePromiseContract(); auto dst = NebulaKeyUtils::getDstId(vIdLen, localKey); - auto stRemotePartId = env_->metaClient_->partId(spaceId, dst.str()); - if (!stRemotePartId.ok()) { - return nebula::cpp2::ErrorCode::E_SPACE_NOT_FOUND; - } - auto remoteKey = TransactionUtils::reverseRawKey(vIdLen, stRemotePartId.value(), localKey); + auto remotePartId = env_->metaClient_->partId(spaceId, dst.str()); + auto remoteKey = TransactionUtils::reverseRawKey(vIdLen, remotePartId, localKey); LOG_IF(INFO, FLAGS_trace_toss) << "try to get remote key=" << folly::hexlify(remoteKey) << ", according to lock=" << folly::hexlify(lockKey); diff --git a/src/tools/db-dump/DbDumper.cpp b/src/tools/db-dump/DbDumper.cpp index 0e151deab..7d2300e33 100644 --- a/src/tools/db-dump/DbDumper.cpp +++ b/src/tools/db-dump/DbDumper.cpp @@ -82,6 +82,12 @@ Status DbDumper::initSpace() { } spaceVidLen_ = spaceVidLen.value(); + auto vidTypeStatus = metaClient_->getSpaceVidType(spaceId_); + if (!vidTypeStatus) { + return vidTypeStatus.status(); + } + spaceVidType_ = std::move(vidTypeStatus).value(); + auto partNum = metaClient_->partsNum(spaceId_); if (!partNum.ok()) { return Status::Error("Get partition number from '%s' failed.", FLAGS_space_name.c_str()); @@ -94,7 +100,15 @@ Status DbDumper::initParams() { std::vector tags, edges; try { folly::splitTo(',', FLAGS_parts, std::inserter(parts_, parts_.begin()), true); - folly::splitTo(',', FLAGS_vids, std::inserter(vids_, vids_.begin()), true); + if (spaceVidType_ == meta::cpp2::PropertyType::INT64) { + std::vector intVids; + folly::splitTo(',', FLAGS_vids, std::inserter(intVids, intVids.begin()), true); + for (auto vid : intVids) { + vids_.emplace(std::string(reinterpret_cast(&vid), 8)); + } + } else { + folly::splitTo(',', FLAGS_vids, std::inserter(vids_, vids_.begin()), true); + } folly::splitTo(',', FLAGS_tags, std::inserter(tags, tags.begin()), true); folly::splitTo(',', FLAGS_edges, std::inserter(edges, edges.begin()), true); } catch (const std::exception& e) { @@ -209,7 +223,7 @@ void DbDumper::run() { if (!isValidVidLen(vid)) { continue; } - auto partId = std::hash()(vid) % partNum_ + 1; + auto partId = metaClient_->partId(partNum_, vid); auto prefix = NebulaKeyUtils::vertexPrefix(spaceVidLen_, partId, vid); seek(prefix); } @@ -221,7 +235,7 @@ void DbDumper::run() { if (!isValidVidLen(vid)) { continue; } - auto partId = std::hash()(vid) % partNum_ + 1; + auto partId = metaClient_->partId(partNum_, vid); for (auto edgeType : edgeTypes_) { auto prefix = NebulaKeyUtils::edgePrefix(spaceVidLen_, partId, vid, edgeType); seek(prefix); @@ -235,7 +249,7 @@ void DbDumper::run() { if (!isValidVidLen(vid)) { continue; } - auto partId = std::hash()(vid) % partNum_ + 1; + auto partId = metaClient_->partId(partNum_, vid); for (auto tagId : tagIds_) { auto prefix = NebulaKeyUtils::vertexPrefix(spaceVidLen_, partId, vid, tagId); seek(prefix); @@ -249,7 +263,7 @@ void DbDumper::run() { if (!isValidVidLen(vid)) { continue; } - auto partId = std::hash()(vid) % partNum_ + 1; + auto partId = metaClient_->partId(partNum_, vid); for (auto edgeType : edgeTypes_) { auto prefix = NebulaKeyUtils::edgePrefix(spaceVidLen_, partId, vid, edgeType); seek(prefix); @@ -260,7 +274,7 @@ void DbDumper::run() { if (!isValidVidLen(vid)) { continue; } - auto partId = std::hash()(vid) % partNum_ + 1; + auto partId = metaClient_->partId(partNum_, vid); for (auto tagId : tagIds_) { auto prefix = NebulaKeyUtils::vertexPrefix(spaceVidLen_, partId, vid, tagId); seek(prefix); @@ -515,7 +529,7 @@ void DbDumper::iterates(kvstore::RocksPrefixIter* it) { inline void DbDumper::printTagKey(const folly::StringPiece& key) { auto part = NebulaKeyUtils::getPart(key); - auto vid = NebulaKeyUtils::getVertexId(spaceVidLen_, key); + auto vid = getVertexId(NebulaKeyUtils::getVertexId(spaceVidLen_, key)); auto tagId = NebulaKeyUtils::getTagId(spaceVidLen_, key); std::cout << "[vertex] key: " << part << ", " << vid << ", " << getTagName(tagId); } @@ -523,8 +537,8 @@ inline void DbDumper::printTagKey(const folly::StringPiece& key) { inline void DbDumper::printEdgeKey(const folly::StringPiece& key) { auto part = NebulaKeyUtils::getPart(key); auto edgeType = NebulaKeyUtils::getEdgeType(spaceVidLen_, key); - auto src = NebulaKeyUtils::getSrcId(spaceVidLen_, key); - auto dst = NebulaKeyUtils::getDstId(spaceVidLen_, key); + auto src = getVertexId(NebulaKeyUtils::getSrcId(spaceVidLen_, key)); + auto dst = getVertexId(NebulaKeyUtils::getDstId(spaceVidLen_, key)); auto rank = NebulaKeyUtils::getRank(spaceVidLen_, key); std::cout << "[edge] key: " << part << ", " << src << ", " << getEdgeName(edgeType) << ", " << rank << ", " << dst; @@ -569,5 +583,15 @@ std::string DbDumper::getEdgeName(const EdgeType edgeType) { return name.value(); } } + +Value DbDumper::getVertexId(const folly::StringPiece &vidStr) { + if (spaceVidType_ == meta::cpp2::PropertyType::INT64) { + int64_t val; + memcpy(reinterpret_cast(&val), vidStr.begin(), sizeof(int64_t)); + return val; + } else { + return vidStr.str(); + } +} } // namespace storage } // namespace nebula diff --git a/src/tools/db-dump/DbDumper.h b/src/tools/db-dump/DbDumper.h index 5add06ae7..ce29a2938 100644 --- a/src/tools/db-dump/DbDumper.h +++ b/src/tools/db-dump/DbDumper.h @@ -65,6 +65,8 @@ class DbDumper { bool isValidVidLen(VertexID vid); + Value getVertexId(const folly::StringPiece &vidStr); + private: std::unique_ptr db_; rocksdb::Options options_; @@ -72,6 +74,7 @@ class DbDumper { std::unique_ptr schemaMng_; GraphSpaceID spaceId_; int32_t spaceVidLen_; + meta::cpp2::PropertyType spaceVidType_; int32_t partNum_; std::unordered_set parts_; std::unordered_set vids_;