diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 85df78b2182..ff1d72c25f0 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -132,6 +132,7 @@ jobs: -DCMAKE_CXX_COMPILER=$TOOLSET_CLANG_DIR/bin/clang++ \ -DCMAKE_C_COMPILER=$TOOLSET_CLANG_DIR/bin/clang \ -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DENABLE_MEMORY_TRACKER=off \ -DENABLE_ASAN=on \ -DENABLE_TESTING=on \ -GNinja \ diff --git a/CMakeLists.txt b/CMakeLists.txt index d5c55196392..6fdb7b827d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -65,6 +65,21 @@ if (ENABLE_NATIVE) add_compile_options(-fPIC) endif() +if(ENABLE_MEMORY_TRACKER) + if(ENABLE_JEMALLOC) + if(NOT ENABLE_ASAN) + add_definitions(-DENABLE_MEMORY_TRACKER) + message(STATUS "MemoryTracker is ENABLED") + else() + message(FATAL_ERROR "MemoryTracker need -DENABLE_ASAN=off") + endif() + else() + message(FATAL_ERROR "MemoryTracker need -DENABLE_JEMALLOC=on") + endif() +else() + message(WARNING "MemoryTracker is DISABLED") +endif() + include_directories(AFTER ${CMAKE_SOURCE_DIR}/src) include_directories(AFTER ${CMAKE_CURRENT_BINARY_DIR}/src) diff --git a/cmake/nebula/GeneralCMakeOptions.cmake b/cmake/nebula/GeneralCMakeOptions.cmake index aaf3f1ea0e6..0e3c855b3e9 100644 --- a/cmake/nebula/GeneralCMakeOptions.cmake +++ b/cmake/nebula/GeneralCMakeOptions.cmake @@ -20,6 +20,7 @@ option(ENABLE_GDB_SCRIPT_SECTION "Add .debug_gdb_scripts section" OFF) option(DISABLE_CXX11_ABI "Whether to disable cxx11 abi" OFF) option(ENABLE_BREAKPAD "Whether to enable breakpad" OFF) option(ENABLE_STANDALONE_VERSION "Enable standalone version build" OFF) +option(ENABLE_MEMORY_TRACKER "Enable memory tracker" ON) get_cmake_property(variable_list VARIABLES) foreach(_varname ${variable_list}) diff --git a/src/clients/meta/MetaClient.cpp b/src/clients/meta/MetaClient.cpp index e04cb91214e..8d0125099e1 100644 --- a/src/clients/meta/MetaClient.cpp +++ b/src/clients/meta/MetaClient.cpp @@ -254,6 +254,7 @@ bool MetaClient::loadUsersAndRoles() { } bool MetaClient::loadData() { + memory::MemoryCheckOffGuard g; // UNKNOWN role will skip heartbeat if (options_.role_ != cpp2::HostRole::UNKNOWN && localDataLastUpdateTime_ == metadLastUpdateTime_) { @@ -425,6 +426,7 @@ bool MetaClient::loadData() { } TagSchemas MetaClient::buildTagSchemas(std::vector tagItemVec) { + memory::MemoryCheckOffGuard g; TagSchemas tagSchemas; for (auto& tagIt : tagItemVec) { // meta will return the different version from new to old @@ -447,6 +449,7 @@ TagSchemas MetaClient::buildTagSchemas(std::vector tagItemVec) { } EdgeSchemas MetaClient::buildEdgeSchemas(std::vector edgeItemVec) { + memory::MemoryCheckOffGuard g; EdgeSchemas edgeSchemas; std::unordered_set> edges; for (auto& edgeIt : edgeItemVec) { @@ -470,6 +473,7 @@ EdgeSchemas MetaClient::buildEdgeSchemas(std::vector edgeItemVec } void MetaClient::addSchemaField(NebulaSchemaProvider* schema, const cpp2::ColumnDef& col) { + memory::MemoryCheckOffGuard g; bool hasDef = col.default_value_ref().has_value(); auto& colType = col.get_type(); size_t len = colType.type_length_ref().has_value() ? *colType.get_type_length() : 0; @@ -493,6 +497,7 @@ bool MetaClient::loadSchemas(GraphSpaceID spaceId, SpaceNewestTagVerMap& newestTagVerMap, SpaceNewestEdgeVerMap& newestEdgeVerMap, SpaceAllEdgeMap& allEdgeMap) { + memory::MemoryCheckOffGuard g; auto tagRet = listTagSchemas(spaceId).get(); if (!tagRet.ok()) { LOG(ERROR) << "Get tag schemas failed for spaceId " << spaceId << ", " << tagRet.status(); @@ -559,6 +564,7 @@ bool MetaClient::loadSchemas(GraphSpaceID spaceId, } Indexes buildIndexes(std::vector indexItemVec) { + memory::MemoryCheckOffGuard g; Indexes indexes; for (auto index : indexItemVec) { auto indexName = index.get_index_name(); @@ -570,6 +576,7 @@ Indexes buildIndexes(std::vector indexItemVec) { } bool MetaClient::loadIndexes(GraphSpaceID spaceId, std::shared_ptr cache) { + memory::MemoryCheckOffGuard g; auto tagIndexesRet = listTagIndexes(spaceId).get(); if (!tagIndexesRet.ok()) { LOG(ERROR) << "Get tag indexes failed for spaceId " << spaceId << ", " @@ -607,6 +614,7 @@ bool MetaClient::loadIndexes(GraphSpaceID spaceId, std::shared_ptr cache) { + memory::MemoryCheckOffGuard g; auto listenerRet = listListener(spaceId).get(); if (!listenerRet.ok()) { LOG(ERROR) << "Get listeners failed for spaceId " << spaceId << ", " << listenerRet.status(); @@ -622,6 +630,7 @@ bool MetaClient::loadListeners(GraphSpaceID spaceId, std::shared_ptr> MetaClient::reverse( const PartsAlloc& parts) { + memory::MemoryCheckOffGuard g; std::unordered_map> hosts; for (auto& partHost : parts) { for (auto& h : partHost.second) { @@ -694,6 +707,7 @@ void MetaClient::getResponse(Request req, bool toLeader, int32_t retry, int32_t retryLimit) { + memory::MemoryCheckOffGuard g; stats::StatsManager::addValue(kNumRpcSentToMetad); auto* evb = ioThreadPool_->getEventBase(); HostAddr host; @@ -803,6 +817,7 @@ void MetaClient::getResponse(Request req, } std::vector MetaClient::toSpaceIdName(const std::vector& tIdNames) { + memory::MemoryCheckOffGuard g; std::vector idNames; idNames.resize(tIdNames.size()); std::transform(tIdNames.begin(), tIdNames.end(), idNames.begin(), [](const auto& tin) { @@ -813,6 +828,7 @@ std::vector MetaClient::toSpaceIdName(const std::vector Status MetaClient::handleResponse(const RESP& resp) { + memory::MemoryCheckOffGuard g; switch (resp.get_code()) { case nebula::cpp2::ErrorCode::SUCCEEDED: return Status::OK(); @@ -979,6 +995,7 @@ Status MetaClient::handleResponse(const RESP& resp) { } PartsMap MetaClient::doGetPartsMap(const HostAddr& host, const LocalCache& localCache) { + memory::MemoryCheckOffGuard g; PartsMap partMap; for (const auto& it : localCache) { auto spaceId = it.first; @@ -999,6 +1016,7 @@ PartsMap MetaClient::doGetPartsMap(const HostAddr& host, const LocalCache& local } void MetaClient::diff(const LocalCache& oldCache, const LocalCache& newCache) { + memory::MemoryCheckOffGuard g; folly::SharedMutex::WriteHolder holder(listenerLock_); if (listener_ == nullptr) { VLOG(3) << "Listener is null!"; @@ -1060,6 +1078,7 @@ void MetaClient::diff(const LocalCache& oldCache, const LocalCache& newCache) { } void MetaClient::listenerDiff(const LocalCache& oldCache, const LocalCache& newCache) { + memory::MemoryCheckOffGuard g; folly::SharedMutex::WriteHolder holder(listenerLock_); if (listener_ == nullptr) { VLOG(3) << "Listener is null!"; @@ -1169,6 +1188,7 @@ void MetaClient::listenerDiff(const LocalCache& oldCache, const LocalCache& newC } void MetaClient::loadRemoteListeners() { + memory::MemoryCheckOffGuard g; folly::SharedMutex::WriteHolder holder(listenerLock_); if (listener_ == nullptr) { VLOG(3) << "Listener is null!"; @@ -1194,6 +1214,7 @@ void MetaClient::loadRemoteListeners() { /// ================================== public methods ================================= PartitionID MetaClient::partId(int32_t numParts, const VertexID id) const { + memory::MemoryCheckOffGuard g; // If the length of the id is 8, we will treat it as int64_t to be compatible // with the version 1.0 uint64_t vid = 0; @@ -1210,6 +1231,7 @@ PartitionID MetaClient::partId(int32_t numParts, const VertexID id) const { folly::Future> MetaClient::submitJob( GraphSpaceID spaceId, cpp2::JobOp op, cpp2::JobType type, std::vector paras) { + memory::MemoryCheckOffGuard g; cpp2::AdminJobReq req; req.space_id_ref() = spaceId; req.op_ref() = op; @@ -1227,6 +1249,7 @@ folly::Future> MetaClient::submitJob( folly::Future> MetaClient::createSpace(meta::cpp2::SpaceDesc spaceDesc, bool ifNotExists) { + memory::MemoryCheckOffGuard g; cpp2::CreateSpaceReq req; req.properties_ref() = std::move(spaceDesc); req.if_not_exists_ref() = ifNotExists; @@ -1242,6 +1265,7 @@ folly::Future> MetaClient::createSpace(meta::cpp2::SpaceD folly::Future> MetaClient::createSpaceAs(const std::string& oldSpaceName, const std::string& newSpaceName) { + memory::MemoryCheckOffGuard g; cpp2::CreateSpaceAsReq req; req.old_space_name_ref() = oldSpaceName; req.new_space_name_ref() = newSpaceName; @@ -1256,6 +1280,7 @@ folly::Future> MetaClient::createSpaceAs(const std::strin } folly::Future>> MetaClient::listSpaces() { + memory::MemoryCheckOffGuard g; cpp2::ListSpacesReq req; folly::Promise>> promise; auto future = promise.getFuture(); @@ -1270,6 +1295,7 @@ folly::Future>> MetaClient::listSpaces() { } folly::Future> MetaClient::getSpace(std::string name) { + memory::MemoryCheckOffGuard g; cpp2::GetSpaceReq req; req.space_name_ref() = std::move(name); folly::Promise> promise; @@ -1283,6 +1309,7 @@ folly::Future> MetaClient::getSpace(std::string name) } folly::Future> MetaClient::dropSpace(std::string name, const bool ifExists) { + memory::MemoryCheckOffGuard g; cpp2::DropSpaceReq req; req.space_name_ref() = std::move(name); req.if_exists_ref() = ifExists; @@ -1299,6 +1326,7 @@ folly::Future> MetaClient::dropSpace(std::string name, const bool } folly::Future> MetaClient::clearSpace(std::string name, const bool ifExists) { + memory::MemoryCheckOffGuard g; cpp2::ClearSpaceReq req; req.space_name_ref() = std::move(name); req.if_exists_ref() = ifExists; @@ -1315,6 +1343,7 @@ folly::Future> MetaClient::clearSpace(std::string name, const boo } folly::Future>> MetaClient::listHosts(cpp2::ListHostType tp) { + memory::MemoryCheckOffGuard g; cpp2::ListHostsReq req; req.type_ref() = tp; @@ -1331,6 +1360,7 @@ folly::Future>> MetaClient::listHosts(cpp2: folly::Future> MetaClient::alterSpace(const std::string& spaceName, meta::cpp2::AlterSpaceOp op, const std::vector& paras) { + memory::MemoryCheckOffGuard g; cpp2::AlterSpaceReq req; req.op_ref() = op; req.space_name_ref() = spaceName; @@ -1349,6 +1379,7 @@ folly::Future> MetaClient::alterSpace(const std::string& spaceNam folly::Future>> MetaClient::listParts( GraphSpaceID spaceId, std::vector partIds) { + memory::MemoryCheckOffGuard g; cpp2::ListPartsReq req; req.space_id_ref() = spaceId; req.part_ids_ref() = std::move(partIds); @@ -1364,6 +1395,7 @@ folly::Future>> MetaClient::listParts( folly::Future>>> MetaClient::getPartsAlloc(GraphSpaceID spaceId, PartTerms* partTerms) { + memory::MemoryCheckOffGuard g; cpp2::GetPartsAllocReq req; req.space_id_ref() = spaceId; folly::Promise>>> promise; @@ -1388,6 +1420,7 @@ MetaClient::getPartsAlloc(GraphSpaceID spaceId, PartTerms* partTerms) { } StatusOr MetaClient::getSpaceIdByNameFromCache(const std::string& name) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -1401,6 +1434,7 @@ StatusOr MetaClient::getSpaceIdByNameFromCache(const std::string& } StatusOr MetaClient::getSpaceNameByIdFromCache(GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -1416,6 +1450,7 @@ StatusOr MetaClient::getSpaceNameByIdFromCache(GraphSpaceID spaceId StatusOr MetaClient::getTagIDByNameFromCache(const GraphSpaceID& space, const std::string& name) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -1430,6 +1465,7 @@ StatusOr MetaClient::getTagIDByNameFromCache(const GraphSpaceID& space, StatusOr MetaClient::getTagNameByIdFromCache(const GraphSpaceID& space, const TagID& tagId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -1444,6 +1480,7 @@ StatusOr MetaClient::getTagNameByIdFromCache(const GraphSpaceID& sp StatusOr MetaClient::getEdgeTypeByNameFromCache(const GraphSpaceID& space, const std::string& name) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -1458,6 +1495,7 @@ StatusOr MetaClient::getEdgeTypeByNameFromCache(const GraphSpaceID& sp StatusOr MetaClient::getEdgeNameByTypeFromCache(const GraphSpaceID& space, const EdgeType edgeType) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -1471,6 +1509,7 @@ StatusOr MetaClient::getEdgeNameByTypeFromCache(const GraphSpaceID& } StatusOr> MetaClient::getAllEdgeFromCache(const GraphSpaceID& space) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -1484,12 +1523,14 @@ StatusOr> MetaClient::getAllEdgeFromCache(const GraphSp } PartsMap MetaClient::getPartsMapFromCache(const HostAddr& host) { + memory::MemoryCheckOffGuard g; folly::rcu_reader guard; const auto& metadata = *metadata_.load(); return doGetPartsMap(host, metadata.localCache_); } StatusOr MetaClient::getPartHostsFromCache(GraphSpaceID spaceId, PartitionID partId) { + memory::MemoryCheckOffGuard g; folly::rcu_reader guard; const auto& metadata = *metadata_.load(); auto it = metadata.localCache_.find(spaceId); @@ -1511,6 +1552,7 @@ StatusOr MetaClient::getPartHostsFromCache(GraphSpaceID spaceId, Part Status MetaClient::checkPartExistInCache(const HostAddr& host, GraphSpaceID spaceId, PartitionID partId) { + memory::MemoryCheckOffGuard g; folly::rcu_reader guard; const auto& metadata = *metadata_.load(); auto it = metadata.localCache_.find(spaceId); @@ -1531,6 +1573,7 @@ Status MetaClient::checkPartExistInCache(const HostAddr& host, } Status MetaClient::checkSpaceExistInCache(const HostAddr& host, GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; folly::rcu_reader guard; const auto& metadata = *metadata_.load(); auto it = metadata.localCache_.find(spaceId); @@ -1546,6 +1589,7 @@ Status MetaClient::checkSpaceExistInCache(const HostAddr& host, GraphSpaceID spa } StatusOr MetaClient::partsNum(GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; folly::rcu_reader guard; const auto& metadata = *metadata_.load(); auto it = metadata.localCache_.find(spaceId); @@ -1559,6 +1603,7 @@ folly::Future> MetaClient::createTagSchema(GraphSpaceID spaceId, std::string name, cpp2::Schema schema, bool ifNotExists) { + memory::MemoryCheckOffGuard g; cpp2::CreateTagReq req; req.space_id_ref() = spaceId; req.tag_name_ref() = std::move(name); @@ -1578,6 +1623,7 @@ folly::Future> MetaClient::alterTagSchema(GraphSpaceID spaceId, std::string name, std::vector items, cpp2::SchemaProp schemaProp) { + memory::MemoryCheckOffGuard g; cpp2::AlterTagReq req; req.space_id_ref() = spaceId; req.tag_name_ref() = std::move(name); @@ -1597,6 +1643,7 @@ folly::Future> MetaClient::alterTagSchema(GraphSpaceID spaceId, folly::Future>> MetaClient::listTagSchemas( GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; cpp2::ListTagsReq req; req.space_id_ref() = spaceId; folly::Promise>> promise; @@ -1612,6 +1659,7 @@ folly::Future>> MetaClient::listTagSchemas( folly::Future> MetaClient::dropTagSchema(GraphSpaceID spaceId, std::string tagName, const bool ifExists) { + memory::MemoryCheckOffGuard g; cpp2::DropTagReq req; req.space_id_ref() = spaceId; req.tag_name_ref() = std::move(tagName); @@ -1631,6 +1679,7 @@ folly::Future> MetaClient::dropTagSchema(GraphSpaceID spaceId, folly::Future> MetaClient::getTagSchema(GraphSpaceID spaceId, std::string name, int64_t version) { + memory::MemoryCheckOffGuard g; cpp2::GetTagReq req; req.space_id_ref() = spaceId; req.tag_name_ref() = std::move(name); @@ -1649,6 +1698,7 @@ folly::Future> MetaClient::createEdgeSchema(GraphSpaceID spac std::string name, cpp2::Schema schema, bool ifNotExists) { + memory::MemoryCheckOffGuard g; cpp2::CreateEdgeReq req; req.space_id_ref() = spaceId; req.edge_name_ref() = std::move(name); @@ -1669,6 +1719,7 @@ folly::Future> MetaClient::alterEdgeSchema(GraphSpaceID spaceId, std::string name, std::vector items, cpp2::SchemaProp schemaProp) { + memory::MemoryCheckOffGuard g; cpp2::AlterEdgeReq req; req.space_id_ref() = spaceId; req.edge_name_ref() = std::move(name); @@ -1688,6 +1739,7 @@ folly::Future> MetaClient::alterEdgeSchema(GraphSpaceID spaceId, folly::Future>> MetaClient::listEdgeSchemas( GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; cpp2::ListEdgesReq req; req.space_id_ref() = spaceId; folly::Promise>> promise; @@ -1703,6 +1755,7 @@ folly::Future>> MetaClient::listEdgeSchemas folly::Future> MetaClient::getEdgeSchema(GraphSpaceID spaceId, std::string name, SchemaVer version) { + memory::MemoryCheckOffGuard g; cpp2::GetEdgeReq req; req.space_id_ref() = spaceId; req.edge_name_ref() = std::move(name); @@ -1720,6 +1773,7 @@ folly::Future> MetaClient::getEdgeSchema(GraphSpaceID spa folly::Future> MetaClient::dropEdgeSchema(GraphSpaceID spaceId, std::string name, const bool ifExists) { + memory::MemoryCheckOffGuard g; cpp2::DropEdgeReq req; req.space_id_ref() = spaceId; req.edge_name_ref() = std::move(name); @@ -1743,6 +1797,7 @@ folly::Future> MetaClient::createTagIndex(GraphSpaceID spaceID bool ifNotExists, const cpp2::IndexParams* indexParams, const std::string* comment) { + memory::MemoryCheckOffGuard g; cpp2::CreateTagIndexReq req; req.space_id_ref() = spaceID; req.index_name_ref() = std::move(indexName); @@ -1769,6 +1824,7 @@ folly::Future> MetaClient::createTagIndex(GraphSpaceID spaceID folly::Future> MetaClient::dropTagIndex(GraphSpaceID spaceID, std::string name, bool ifExists) { + memory::MemoryCheckOffGuard g; cpp2::DropTagIndexReq req; req.space_id_ref() = (spaceID); req.index_name_ref() = (std::move(name)); @@ -1788,6 +1844,7 @@ folly::Future> MetaClient::dropTagIndex(GraphSpaceID spaceID, folly::Future> MetaClient::getTagIndex(GraphSpaceID spaceID, std::string name) { + memory::MemoryCheckOffGuard g; cpp2::GetTagIndexReq req; req.space_id_ref() = spaceID; req.index_name_ref() = std::move(name); @@ -1804,6 +1861,7 @@ folly::Future> MetaClient::getTagIndex(GraphSpaceID sp folly::Future>> MetaClient::listTagIndexes( GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; cpp2::ListTagIndexesReq req; req.space_id_ref() = spaceId; @@ -1818,6 +1876,7 @@ folly::Future>> MetaClient::listTagIndexes } folly::Future> MetaClient::rebuildTagIndex(GraphSpaceID spaceID, std::string name) { + memory::MemoryCheckOffGuard g; cpp2::RebuildIndexReq req; req.space_id_ref() = spaceID; req.index_name_ref() = std::move(name); @@ -1836,6 +1895,7 @@ folly::Future> MetaClient::rebuildTagIndex(GraphSpaceID spaceID, folly::Future>> MetaClient::listTagIndexStatus( GraphSpaceID spaceID) { + memory::MemoryCheckOffGuard g; cpp2::ListIndexStatusReq req; req.space_id_ref() = spaceID; @@ -1859,6 +1919,7 @@ folly::Future> MetaClient::createEdgeIndex( bool ifNotExists, const cpp2::IndexParams* indexParams, const std::string* comment) { + memory::MemoryCheckOffGuard g; cpp2::CreateEdgeIndexReq req; req.space_id_ref() = spaceID; req.index_name_ref() = std::move(indexName); @@ -1886,6 +1947,7 @@ folly::Future> MetaClient::createEdgeIndex( folly::Future> MetaClient::dropEdgeIndex(GraphSpaceID spaceId, std::string name, bool ifExists) { + memory::MemoryCheckOffGuard g; cpp2::DropEdgeIndexReq req; req.space_id_ref() = spaceId; req.index_name_ref() = std::move(name); @@ -1905,6 +1967,7 @@ folly::Future> MetaClient::dropEdgeIndex(GraphSpaceID spaceId, folly::Future> MetaClient::getEdgeIndex(GraphSpaceID spaceId, std::string name) { + memory::MemoryCheckOffGuard g; cpp2::GetEdgeIndexReq req; req.space_id_ref() = spaceId; req.index_name_ref() = std::move(name); @@ -1921,6 +1984,7 @@ folly::Future> MetaClient::getEdgeIndex(GraphSpaceID s folly::Future>> MetaClient::listEdgeIndexes( GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; cpp2::ListEdgeIndexesReq req; req.space_id_ref() = spaceId; @@ -1937,6 +2001,7 @@ folly::Future>> MetaClient::listEdgeIndexe } StatusOr MetaClient::getSpaceVidLen(const GraphSpaceID& spaceId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -1956,6 +2021,7 @@ StatusOr MetaClient::getSpaceVidLen(const GraphSpaceID& spaceId) { } StatusOr MetaClient::getSpaceVidType(const GraphSpaceID& spaceId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -1978,6 +2044,7 @@ StatusOr MetaClient::getSpaceVidType(const GraphSpac } StatusOr MetaClient::getSpaceDesc(const GraphSpaceID& spaceId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -1992,6 +2059,7 @@ StatusOr MetaClient::getSpaceDesc(const GraphSpaceID& spaceId) } StatusOr MetaClient::getIsolationLevel(GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; auto spaceDescStatus = getSpaceDesc(spaceId); if (!spaceDescStatus.ok()) { return spaceDescStatus.status(); @@ -2002,6 +2070,7 @@ StatusOr MetaClient::getIsolationLevel(GraphSpaceID StatusOr> MetaClient::getTagSchemaFromCache( GraphSpaceID spaceId, TagID tagID, SchemaVer ver) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2023,6 +2092,7 @@ StatusOr> MetaClient::getTagSchemaFr StatusOr> MetaClient::getEdgeSchemaFromCache( GraphSpaceID spaceId, EdgeType edgeType, SchemaVer ver) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2043,6 +2113,7 @@ StatusOr> MetaClient::getEdgeSchemaF } StatusOr MetaClient::getAllVerTagSchema(GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2056,6 +2127,7 @@ StatusOr MetaClient::getAllVerTagSchema(GraphSpaceID spaceId) { } StatusOr MetaClient::getAllLatestVerTagSchema(const GraphSpaceID& spaceId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2075,6 +2147,7 @@ StatusOr MetaClient::getAllLatestVerTagSchema(const GraphSpaceID& spa } StatusOr MetaClient::getAllVerEdgeSchema(GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2088,6 +2161,7 @@ StatusOr MetaClient::getAllVerEdgeSchema(GraphSpaceID spaceId) { } StatusOr MetaClient::getAllLatestVerEdgeSchemaFromCache(const GraphSpaceID& spaceId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2107,6 +2181,7 @@ StatusOr MetaClient::getAllLatestVerEdgeSchemaFromCache(const GraphS } folly::Future> MetaClient::rebuildEdgeIndex(GraphSpaceID spaceID, std::string name) { + memory::MemoryCheckOffGuard g; cpp2::RebuildIndexReq req; req.space_id_ref() = spaceID; req.index_name_ref() = std::move(name); @@ -2125,6 +2200,7 @@ folly::Future> MetaClient::rebuildEdgeIndex(GraphSpaceID spaceID, folly::Future>> MetaClient::listEdgeIndexStatus( GraphSpaceID spaceID) { + memory::MemoryCheckOffGuard g; cpp2::ListIndexStatusReq req; req.space_id_ref() = spaceID; @@ -2142,6 +2218,7 @@ folly::Future>> MetaClient::listEdgeInde StatusOr> MetaClient::getTagIndexByNameFromCache( const GraphSpaceID space, const std::string& name) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2160,6 +2237,7 @@ StatusOr> MetaClient::getTagIndexByNameFromCach StatusOr> MetaClient::getEdgeIndexByNameFromCache( const GraphSpaceID space, const std::string& name) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2178,6 +2256,7 @@ StatusOr> MetaClient::getEdgeIndexByNameFromCac StatusOr> MetaClient::getTagIndexFromCache(GraphSpaceID spaceId, IndexID indexID) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2201,6 +2280,7 @@ StatusOr> MetaClient::getTagIndexFromCache(Grap StatusOr MetaClient::getRelatedTagIDByIndexNameFromCache(const GraphSpaceID space, const std::string& indexName) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2216,6 +2296,7 @@ StatusOr MetaClient::getRelatedTagIDByIndexNameFromCache(const GraphSpace StatusOr> MetaClient::getEdgeIndexFromCache(GraphSpaceID spaceId, IndexID indexId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2239,6 +2320,7 @@ StatusOr> MetaClient::getEdgeIndexFromCache(Gra StatusOr MetaClient::getRelatedEdgeTypeByIndexNameFromCache( const GraphSpaceID space, const std::string& indexName) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2254,6 +2336,7 @@ StatusOr MetaClient::getRelatedEdgeTypeByIndexNameFromCache( StatusOr>> MetaClient::getTagIndexesFromCache( GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2278,6 +2361,7 @@ StatusOr>> MetaClient::getTagIndexe StatusOr>> MetaClient::getEdgeIndexesFromCache( GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2301,6 +2385,7 @@ StatusOr>> MetaClient::getEdgeIndex } StatusOr MetaClient::getStorageLeaderFromCache(GraphSpaceID spaceId, PartitionID partId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2332,18 +2417,21 @@ StatusOr MetaClient::getStorageLeaderFromCache(GraphSpaceID spaceId, P void MetaClient::updateStorageLeader(GraphSpaceID spaceId, PartitionID partId, const HostAddr& leader) { + memory::MemoryCheckOffGuard g; VLOG(1) << "Update the leader for [" << spaceId << ", " << partId << "] to " << leader; folly::SharedMutex::WriteHolder holder(leadersLock_); leadersInfo_.leaderMap_[{spaceId, partId}] = leader; } void MetaClient::invalidStorageLeader(GraphSpaceID spaceId, PartitionID partId) { + memory::MemoryCheckOffGuard g; VLOG(1) << "Invalidate the leader for [" << spaceId << ", " << partId << "]"; folly::SharedMutex::WriteHolder holder(leadersLock_); leadersInfo_.leaderMap_.erase({spaceId, partId}); } StatusOr MetaClient::getLeaderInfo() { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2356,6 +2444,7 @@ const std::vector& MetaClient::getAddresses() { } std::vector MetaClient::getRolesByUserFromCache(const std::string& user) { + memory::MemoryCheckOffGuard g; if (!ready_) { return std::vector(0); } @@ -2369,6 +2458,7 @@ std::vector MetaClient::getRolesByUserFromCache(const std::strin } Status MetaClient::authCheckFromCache(const std::string& account, const std::string& password) { + memory::MemoryCheckOffGuard g; // Check meta service status if (!ready_) { return Status::Error("Meta Service not ready"); @@ -2437,6 +2527,7 @@ Status MetaClient::authCheckFromCache(const std::string& account, const std::str } bool MetaClient::checkShadowAccountFromCache(const std::string& account) { + memory::MemoryCheckOffGuard g; if (!ready_) { return false; } @@ -2450,6 +2541,7 @@ bool MetaClient::checkShadowAccountFromCache(const std::string& account) { } StatusOr MetaClient::getTermFromCache(GraphSpaceID spaceId, PartitionID partId) { + memory::MemoryCheckOffGuard g; folly::rcu_reader guard; const auto& metadata = *metadata_.load(); auto spaceInfo = metadata.localCache_.find(spaceId); @@ -2466,6 +2558,7 @@ StatusOr MetaClient::getTermFromCache(GraphSpaceID spaceId, PartitionID } StatusOr> MetaClient::getStorageHosts() { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2477,6 +2570,7 @@ StatusOr> MetaClient::getStorageHosts() { StatusOr MetaClient::getLatestTagVersionFromCache(const GraphSpaceID& space, const TagID& tagId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2491,6 +2585,7 @@ StatusOr MetaClient::getLatestTagVersionFromCache(const GraphSpaceID& StatusOr MetaClient::getLatestEdgeVersionFromCache(const GraphSpaceID& space, const EdgeType& edgeType) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2504,6 +2599,7 @@ StatusOr MetaClient::getLatestEdgeVersionFromCache(const GraphSpaceID } folly::Future> MetaClient::heartbeat() { + memory::MemoryCheckOffGuard g; cpp2::HBReq req; req.host_ref() = options_.localHost_; req.role_ref() = options_.role_; @@ -2596,6 +2692,7 @@ folly::Future> MetaClient::heartbeat() { folly::Future> MetaClient::createUser(std::string account, std::string password, bool ifNotExists) { + memory::MemoryCheckOffGuard g; cpp2::CreateUserReq req; req.account_ref() = std::move(account); req.encoded_pwd_ref() = std::move(password); @@ -2613,6 +2710,7 @@ folly::Future> MetaClient::createUser(std::string account, } folly::Future> MetaClient::dropUser(std::string account, bool ifExists) { + memory::MemoryCheckOffGuard g; cpp2::DropUserReq req; req.account_ref() = std::move(account); req.if_exists_ref() = ifExists; @@ -2629,6 +2727,7 @@ folly::Future> MetaClient::dropUser(std::string account, bool ifE } folly::Future> MetaClient::alterUser(std::string account, std::string password) { + memory::MemoryCheckOffGuard g; cpp2::AlterUserReq req; req.account_ref() = std::move(account); req.encoded_pwd_ref() = std::move(password); @@ -2645,6 +2744,7 @@ folly::Future> MetaClient::alterUser(std::string account, std::st } folly::Future> MetaClient::grantToUser(cpp2::RoleItem roleItem) { + memory::MemoryCheckOffGuard g; cpp2::GrantRoleReq req; req.role_item_ref() = std::move(roleItem); folly::Promise> promise; @@ -2660,6 +2760,7 @@ folly::Future> MetaClient::grantToUser(cpp2::RoleItem roleItem) { } folly::Future> MetaClient::revokeFromUser(cpp2::RoleItem roleItem) { + memory::MemoryCheckOffGuard g; cpp2::RevokeRoleReq req; req.role_item_ref() = std::move(roleItem); folly::Promise> promise; @@ -2675,6 +2776,7 @@ folly::Future> MetaClient::revokeFromUser(cpp2::RoleItem roleItem } folly::Future>> MetaClient::listUsers() { + memory::MemoryCheckOffGuard g; cpp2::ListUsersReq req; folly::Promise>> promise; auto future = promise.getFuture(); @@ -2687,6 +2789,7 @@ folly::Future>> MetaClient } folly::Future>> MetaClient::listRoles(GraphSpaceID space) { + memory::MemoryCheckOffGuard g; cpp2::ListRolesReq req; req.space_id_ref() = std::move(space); folly::Promise>> promise; @@ -2702,6 +2805,7 @@ folly::Future>> MetaClient::listRoles(Graph folly::Future> MetaClient::changePassword(std::string account, std::string newPwd, std::string oldPwd) { + memory::MemoryCheckOffGuard g; cpp2::ChangePasswordReq req; req.account_ref() = std::move(account); req.new_encoded_pwd_ref() = std::move(newPwd); @@ -2719,6 +2823,7 @@ folly::Future> MetaClient::changePassword(std::string account, } folly::Future>> MetaClient::getUserRoles(std::string account) { + memory::MemoryCheckOffGuard g; cpp2::GetUserRolesReq req; req.account_ref() = std::move(account); folly::Promise>> promise; @@ -2732,6 +2837,7 @@ folly::Future>> MetaClient::getUserRoles(st } folly::Future> MetaClient::regConfig(const std::vector& items) { + memory::MemoryCheckOffGuard g; cpp2::RegConfigReq req; req.items_ref() = items; folly::Promise> promise; @@ -2748,6 +2854,7 @@ folly::Future> MetaClient::regConfig(const std::vector>> MetaClient::getConfig( const cpp2::ConfigModule& module, const std::string& name) { + memory::MemoryCheckOffGuard g; if (!configReady_) { return Status::Error("Not ready!"); } @@ -2769,6 +2876,7 @@ folly::Future>> MetaClient::getConfig( folly::Future> MetaClient::setConfig(const cpp2::ConfigModule& module, const std::string& name, const Value& value) { + memory::MemoryCheckOffGuard g; cpp2::ConfigItem item; item.module_ref() = module; item.name_ref() = name; @@ -2790,6 +2898,7 @@ folly::Future> MetaClient::setConfig(const cpp2::ConfigModule& mo folly::Future>> MetaClient::listConfigs( const cpp2::ConfigModule& module) { + memory::MemoryCheckOffGuard g; cpp2::ListConfigsReq req; req.module_ref() = module; folly::Promise>> promise; @@ -2803,6 +2912,7 @@ folly::Future>> MetaClient::listConfigs( } folly::Future> MetaClient::createSnapshot() { + memory::MemoryCheckOffGuard g; cpp2::CreateSnapshotReq req; folly::Promise> promise; auto future = promise.getFuture(); @@ -2820,6 +2930,7 @@ folly::Future> MetaClient::createSnapshot() { } folly::Future> MetaClient::dropSnapshot(const std::string& name) { + memory::MemoryCheckOffGuard g; cpp2::DropSnapshotReq req; std::vector names{name}; req.names_ref() = names; @@ -2836,6 +2947,7 @@ folly::Future> MetaClient::dropSnapshot(const std::string& name) } folly::Future>> MetaClient::listSnapshots() { + memory::MemoryCheckOffGuard g; cpp2::ListSnapshotsReq req; folly::Promise>> promise; auto future = promise.getFuture(); @@ -2852,6 +2964,7 @@ folly::Future>> MetaClient::listSnapshots() folly::Future> MetaClient::addListener(GraphSpaceID spaceId, cpp2::ListenerType type, std::vector hosts) { + memory::MemoryCheckOffGuard g; cpp2::AddListenerReq req; req.space_id_ref() = spaceId; req.type_ref() = type; @@ -2870,6 +2983,7 @@ folly::Future> MetaClient::addListener(GraphSpaceID spaceId, folly::Future> MetaClient::removeListener(GraphSpaceID spaceId, cpp2::ListenerType type) { + memory::MemoryCheckOffGuard g; cpp2::RemoveListenerReq req; req.space_id_ref() = spaceId; req.type_ref() = type; @@ -2887,6 +3001,7 @@ folly::Future> MetaClient::removeListener(GraphSpaceID spaceId, folly::Future>> MetaClient::listListener( GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; cpp2::ListListenerReq req; req.space_id_ref() = spaceId; folly::Promise>> promise; @@ -2902,6 +3017,7 @@ folly::Future>> MetaClient::listListene } bool MetaClient::registerCfg() { + memory::MemoryCheckOffGuard g; auto ret = regConfig(gflagsDeclared_).get(); if (ret.ok()) { LOG(INFO) << "Register gflags ok " << gflagsDeclared_.size(); @@ -2912,6 +3028,7 @@ bool MetaClient::registerCfg() { StatusOr>> MetaClient::getListenersBySpaceHostFromCache(GraphSpaceID spaceId, const HostAddr& host) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2932,6 +3049,7 @@ MetaClient::getListenersBySpaceHostFromCache(GraphSpaceID spaceId, const HostAdd } StatusOr MetaClient::getListenersByHostFromCache(const HostAddr& host) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2941,6 +3059,7 @@ StatusOr MetaClient::getListenersByHostFromCache(const HostAddr& h } ListenersMap MetaClient::doGetListenersMap(const HostAddr& host, const LocalCache& localCache) { + memory::MemoryCheckOffGuard g; ListenersMap listenersMap; for (const auto& space : localCache) { auto spaceId = space.first; @@ -2970,6 +3089,7 @@ ListenersMap MetaClient::doGetListenersMap(const HostAddr& host, const LocalCach StatusOr MetaClient::getListenerHostsBySpacePartType(GraphSpaceID spaceId, PartitionID partId, cpp2::ListenerType type) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -2992,6 +3112,7 @@ StatusOr MetaClient::getListenerHostsBySpacePartType(GraphSpaceID spac StatusOr> MetaClient::getListenerHostTypeBySpacePartType( GraphSpaceID spaceId, PartitionID partId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -3017,6 +3138,7 @@ StatusOr> MetaClient::getListenerHostTypeBySpace } bool MetaClient::loadCfg() { + memory::MemoryCheckOffGuard g; // UNKNOWN role will skip heartbeat if (options_.skipConfig_ || (options_.role_ != cpp2::HostRole::UNKNOWN && localCfgLastUpdateTime_ == metadLastUpdateTime_)) { @@ -3057,6 +3179,7 @@ bool MetaClient::loadCfg() { } void MetaClient::updateGflagsValue(const cpp2::ConfigItem& item) { + memory::MemoryCheckOffGuard g; if (item.get_mode() != cpp2::ConfigMode::MUTABLE) { return; } @@ -3085,6 +3208,7 @@ void MetaClient::updateGflagsValue(const cpp2::ConfigItem& item) { } void MetaClient::updateNestedGflags(const std::unordered_map& nameValues) { + memory::MemoryCheckOffGuard g; std::unordered_map optionMap; for (const auto& value : nameValues) { optionMap.emplace(value.first, value.second.toString()); @@ -3098,12 +3222,14 @@ void MetaClient::updateNestedGflags(const std::unordered_map } Status MetaClient::refreshCache() { + memory::MemoryCheckOffGuard g; auto ret = bgThread_->addTask(&MetaClient::loadData, this).get(); return ret ? Status::OK() : Status::Error("Load data failed"); } void MetaClient::loadLeader(const std::vector& hostItems, const SpaceNameIdMap& spaceIndexByName) { + memory::MemoryCheckOffGuard g; LeaderInfo leaderInfo; for (auto& item : hostItems) { for (auto& spaceEntry : item.get_leader_parts()) { @@ -3143,6 +3269,7 @@ void MetaClient::loadLeader(const std::vector& hostItems, } folly::Future> MetaClient::addHosts(std::vector hosts) { + memory::MemoryCheckOffGuard g; cpp2::AddHostsReq req; req.hosts_ref() = std::move(hosts); @@ -3159,6 +3286,7 @@ folly::Future> MetaClient::addHosts(std::vector hosts) } folly::Future> MetaClient::dropHosts(std::vector hosts) { + memory::MemoryCheckOffGuard g; cpp2::DropHostsReq req; req.hosts_ref() = std::move(hosts); @@ -3176,6 +3304,7 @@ folly::Future> MetaClient::dropHosts(std::vector hosts) folly::Future> MetaClient::mergeZone(std::vector zones, std::string zoneName) { + memory::MemoryCheckOffGuard g; cpp2::MergeZoneReq req; req.zone_name_ref() = std::move(zoneName); req.zones_ref() = std::move(zones); @@ -3193,6 +3322,7 @@ folly::Future> MetaClient::mergeZone(std::vector zon folly::Future> MetaClient::divideZone( std::string zoneName, std::unordered_map> zoneItems) { + memory::MemoryCheckOffGuard g; cpp2::DivideZoneReq req; req.zone_name_ref() = std::move(zoneName); req.zone_items_ref() = std::move(zoneItems); @@ -3210,6 +3340,7 @@ folly::Future> MetaClient::divideZone( folly::Future> MetaClient::renameZone(std::string originalZoneName, std::string zoneName) { + memory::MemoryCheckOffGuard g; cpp2::RenameZoneReq req; req.original_zone_name_ref() = std::move(originalZoneName); req.zone_name_ref() = std::move(zoneName); @@ -3226,6 +3357,7 @@ folly::Future> MetaClient::renameZone(std::string originalZoneNam } folly::Future> MetaClient::dropZone(std::string zoneName) { + memory::MemoryCheckOffGuard g; cpp2::DropZoneReq req; req.zone_name_ref() = std::move(zoneName); @@ -3244,6 +3376,7 @@ folly::Future> MetaClient::dropZone(std::string zoneName) { folly::Future> MetaClient::addHostsIntoZone(std::vector hosts, std::string zoneName, bool isNew) { + memory::MemoryCheckOffGuard g; cpp2::AddHostsIntoZoneReq req; req.hosts_ref() = hosts; req.zone_name_ref() = zoneName; @@ -3262,6 +3395,7 @@ folly::Future> MetaClient::addHostsIntoZone(std::vector } folly::Future>> MetaClient::getZone(std::string zoneName) { + memory::MemoryCheckOffGuard g; cpp2::GetZoneReq req; req.zone_name_ref() = std::move(zoneName); @@ -3276,6 +3410,7 @@ folly::Future>> MetaClient::getZone(std::string z } folly::Future>> MetaClient::listZones() { + memory::MemoryCheckOffGuard g; cpp2::ListZonesReq req; folly::Promise>> promise; auto future = promise.getFuture(); @@ -3288,6 +3423,7 @@ folly::Future>> MetaClient::listZones() { } folly::Future> MetaClient::getStats(GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; cpp2::GetStatsReq req; req.space_id_ref() = (spaceId); folly::Promise> promise; @@ -3307,6 +3443,7 @@ folly::Future> MetaClient::reportTaskFinish( int32_t taskId, nebula::cpp2::ErrorCode taskErrCode, cpp2::StatsItem* statisticItem) { + memory::MemoryCheckOffGuard g; cpp2::ReportTaskReq req; req.code_ref() = taskErrCode; req.space_id_ref() = spaceId; @@ -3328,6 +3465,7 @@ folly::Future> MetaClient::reportTaskFinish( folly::Future> MetaClient::signInService( const cpp2::ExternalServiceType& type, const std::vector& clients) { + memory::MemoryCheckOffGuard g; cpp2::SignInServiceReq req; req.type_ref() = type; req.clients_ref() = clients; @@ -3345,6 +3483,7 @@ folly::Future> MetaClient::signInService( } folly::Future> MetaClient::signOutService(const cpp2::ExternalServiceType& type) { + memory::MemoryCheckOffGuard g; cpp2::SignOutServiceReq req; req.type_ref() = type; folly::Promise> promise; @@ -3362,6 +3501,7 @@ folly::Future> MetaClient::signOutService(const cpp2::ExternalSer folly::Future> MetaClient::listServiceClients( const cpp2::ExternalServiceType& type) { + memory::MemoryCheckOffGuard g; cpp2::ListServiceClientsReq req; req.type_ref() = type; folly::Promise> promise; @@ -3378,6 +3518,7 @@ folly::Future> MetaClient::listServiceClients( StatusOr> MetaClient::getServiceClientsFromCache( const cpp2::ExternalServiceType& type) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -3394,6 +3535,7 @@ StatusOr> MetaClient::getServiceClientsFromCach folly::Future> MetaClient::createFTIndex(const std::string& name, const cpp2::FTIndex& index) { + memory::MemoryCheckOffGuard g; cpp2::CreateFTIndexReq req; req.fulltext_index_name_ref() = name; req.index_ref() = index; @@ -3412,6 +3554,7 @@ folly::Future> MetaClient::createFTIndex(const std::string& name, folly::Future> MetaClient::dropFTIndex(GraphSpaceID spaceId, const std::string& name) { + memory::MemoryCheckOffGuard g; cpp2::DropFTIndexReq req; req.fulltext_index_name_ref() = name; req.space_id_ref() = spaceId; @@ -3430,6 +3573,7 @@ folly::Future> MetaClient::dropFTIndex(GraphSpaceID spaceId, folly::Future>> MetaClient::listFTIndexes() { + memory::MemoryCheckOffGuard g; cpp2::ListFTIndexesReq req; folly::Promise>> promise; auto future = promise.getFuture(); @@ -3444,6 +3588,7 @@ MetaClient::listFTIndexes() { } StatusOr> MetaClient::getFTIndexesFromCache() { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -3454,6 +3599,7 @@ StatusOr> MetaClient::getFTIndexe StatusOr> MetaClient::getFTIndexBySpaceFromCache( GraphSpaceID spaceId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -3470,6 +3616,7 @@ StatusOr> MetaClient::getFTIndexB StatusOr> MetaClient::getFTIndexFromCache( GraphSpaceID spaceId, int32_t schemaId, const std::string& field) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -3490,6 +3637,7 @@ StatusOr> MetaClient::getFTIndexFromCache( StatusOr> MetaClient::getFTIndexFromCache( GraphSpaceID spaceId, int32_t schemaId) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -3509,6 +3657,7 @@ StatusOr> MetaClient::getFTIndexF StatusOr MetaClient::getFTIndexByNameFromCache(GraphSpaceID spaceId, const std::string& name) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -3523,6 +3672,7 @@ StatusOr MetaClient::getFTIndexByNameFromCache(GraphSpaceID space folly::Future> MetaClient::createSession( const std::string& userName, const HostAddr& graphAddr, const std::string& clientIp) { + memory::MemoryCheckOffGuard g; cpp2::CreateSessionReq req; req.user_ref() = userName; req.graph_addr_ref() = graphAddr; @@ -3540,6 +3690,7 @@ folly::Future> MetaClient::createSession( folly::Future> MetaClient::updateSessions( const std::vector& sessions) { + memory::MemoryCheckOffGuard g; cpp2::UpdateSessionsReq req; req.sessions_ref() = sessions; folly::Promise> promise; @@ -3554,6 +3705,7 @@ folly::Future> MetaClient::updateSessions( } folly::Future> MetaClient::listSessions() { + memory::MemoryCheckOffGuard g; cpp2::ListSessionsReq req; folly::Promise> promise; auto future = promise.getFuture(); @@ -3566,6 +3718,7 @@ folly::Future> MetaClient::listSessions() { } folly::Future> MetaClient::getSession(SessionID sessionId) { + memory::MemoryCheckOffGuard g; cpp2::GetSessionReq req; req.session_id_ref() = sessionId; folly::Promise> promise; @@ -3580,6 +3733,7 @@ folly::Future> MetaClient::getSession(SessionID s folly::Future> MetaClient::removeSessions( const std::vector& sessionIds) { + memory::MemoryCheckOffGuard g; cpp2::RemoveSessionReq req; req.session_ids_ref() = sessionIds; folly::Promise> promise; @@ -3595,6 +3749,7 @@ folly::Future> MetaClient::removeSessions( folly::Future> MetaClient::killQuery( std::unordered_map> killQueries) { + memory::MemoryCheckOffGuard g; cpp2::KillQueryReq req; req.kill_queries_ref() = std::move(killQueries); folly::Promise> promise; @@ -3609,6 +3764,7 @@ folly::Future> MetaClient::killQuery( } folly::Future> MetaClient::getWorkerId(std::string ipAddr) { + memory::MemoryCheckOffGuard g; cpp2::GetWorkerIdReq req; req.host_ref() = std::move(ipAddr); @@ -3624,6 +3780,7 @@ folly::Future> MetaClient::getWorkerId(std::string ipAddr) { } folly::Future> MetaClient::getSegmentId(int64_t length) { + memory::MemoryCheckOffGuard g; auto req = cpp2::GetSegmentIdReq(); req.length_ref() = length; @@ -3639,6 +3796,7 @@ folly::Future> MetaClient::getSegmentId(int64_t length) { } bool MetaClient::loadSessions() { + memory::MemoryCheckOffGuard g; auto session_list = listSessions().get(); if (!session_list.ok()) { LOG(ERROR) << "List sessions failed, status:" << session_list.status(); @@ -3658,6 +3816,7 @@ bool MetaClient::loadSessions() { } StatusOr MetaClient::getSessionFromCache(const nebula::SessionID& session_id) { + memory::MemoryCheckOffGuard g; if (!ready_) { return Status::Error("Not ready!"); } @@ -3671,6 +3830,7 @@ StatusOr MetaClient::getSessionFromCache(const nebula::SessionID& } bool MetaClient::checkIsPlanKilled(SessionID sessionId, ExecutionPlanID planId) { + memory::MemoryCheckOffGuard g; static thread_local int check_counter = 0; // Inaccurate in a multi-threaded environment, but it is not important check_counter = (check_counter + 1) & ((1 << FLAGS_check_plan_killed_frequency) - 1); @@ -3682,6 +3842,7 @@ bool MetaClient::checkIsPlanKilled(SessionID sessionId, ExecutionPlanID planId) } Status MetaClient::verifyVersion() { + memory::MemoryCheckOffGuard g; auto req = cpp2::VerifyClientVersionReq(); req.build_version_ref() = getOriginVersion(); req.host_ref() = options_.localHost_; @@ -3705,6 +3866,7 @@ Status MetaClient::verifyVersion() { } Status MetaClient::saveVersionToMeta() { + memory::MemoryCheckOffGuard g; auto req = cpp2::SaveGraphVersionReq(); req.build_version_ref() = getOriginVersion(); req.host_ref() = options_.localHost_; diff --git a/src/clients/storage/StorageClientBase-inl.h b/src/clients/storage/StorageClientBase-inl.h index beb8d25d958..c9b530595a5 100644 --- a/src/clients/storage/StorageClientBase-inl.h +++ b/src/clients/storage/StorageClientBase-inl.h @@ -76,6 +76,7 @@ StorageClientBase::collectResponse( folly::EventBase* evb, std::unordered_map requests, RemoteFunc&& remoteFunc) { + memory::MemoryCheckOffGuard offGuard; std::vector>> respFutures; respFutures.reserve(requests.size()); diff --git a/src/codec/RowReader.cpp b/src/codec/RowReader.cpp index fa5c190dcbf..ff365f0970d 100644 --- a/src/codec/RowReader.cpp +++ b/src/codec/RowReader.cpp @@ -15,7 +15,7 @@ namespace nebula { * class RowReader::Cell * ********************************************/ -Value RowReader::Cell::value() const noexcept { +Value RowReader::Cell::value() const { return iter_->reader_->getValueByIndex(iter_->index_); } @@ -50,7 +50,7 @@ RowReader::Iterator& RowReader::Iterator::operator++() { * ********************************************/ -bool RowReader::resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) noexcept { +bool RowReader::resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) { schema_ = schema; data_ = row; diff --git a/src/codec/RowReader.h b/src/codec/RowReader.h index 91f09896a39..364f5dae57b 100644 --- a/src/codec/RowReader.h +++ b/src/codec/RowReader.h @@ -25,7 +25,7 @@ class RowReader { friend class Iterator; public: - Value value() const noexcept; + Value value() const; private: const Iterator* iter_; @@ -74,7 +74,7 @@ class RowReader { * @param prop Property name * @return Value Property value */ - virtual Value getValueByName(const std::string& prop) const noexcept = 0; + virtual Value getValueByName(const std::string& prop) const = 0; /** * @brief Get the property value by index in schema @@ -82,7 +82,7 @@ class RowReader { * @param index Index in Schema * @return Value Property value */ - virtual Value getValueByIndex(const int64_t index) const noexcept = 0; + virtual Value getValueByIndex(const int64_t index) const = 0; /** * @brief Get the timestamp in value @@ -108,7 +108,7 @@ class RowReader { * * @return Iterator */ - virtual Iterator begin() const noexcept { + virtual Iterator begin() const { return Iterator(this, 0); } @@ -117,7 +117,7 @@ class RowReader { * * @return const Iterator& */ - virtual const Iterator& end() const noexcept { + virtual const Iterator& end() const { return endIter_; } @@ -170,7 +170,7 @@ class RowReader { * @param row * @return Whether reset succeed */ - virtual bool resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) noexcept; + virtual bool resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row); private: Iterator endIter_; diff --git a/src/codec/RowReaderV1.cpp b/src/codec/RowReaderV1.cpp index a8f9023c258..f39af174003 100644 --- a/src/codec/RowReaderV1.cpp +++ b/src/codec/RowReaderV1.cpp @@ -27,7 +27,7 @@ using nebula::cpp2::PropertyType; * class RowReaderV1 * ********************************************/ -bool RowReaderV1::resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) noexcept { +bool RowReaderV1::resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) { RowReader::resetImpl(schema, row); DCHECK(schema_ != nullptr) << "A schema must be provided"; @@ -95,7 +95,7 @@ bool RowReaderV1::processHeader(folly::StringPiece row) { return true; } -int64_t RowReaderV1::skipToNext(int64_t index, int64_t offset) const noexcept { +int64_t RowReaderV1::skipToNext(int64_t index, int64_t offset) const { const PropertyType& vType = getSchema()->getFieldType(index); if (offsets_[index + 1] >= 0) { return offsets_[index + 1]; @@ -160,7 +160,7 @@ int64_t RowReaderV1::skipToNext(int64_t index, int64_t offset) const noexcept { return offset; } -int64_t RowReaderV1::skipToField(int64_t index) const noexcept { +int64_t RowReaderV1::skipToField(int64_t index) const { DCHECK_GE(index, 0); if (index >= static_cast(schema_->getNumFields())) { // Index is out of range @@ -191,12 +191,12 @@ int64_t RowReaderV1::skipToField(int64_t index) const noexcept { * Get the property value * ***********************************************************/ -Value RowReaderV1::getValueByName(const std::string& prop) const noexcept { +Value RowReaderV1::getValueByName(const std::string& prop) const { int64_t index = getSchema()->getFieldIndex(prop); return getValueByIndex(index); } -Value RowReaderV1::getValueByIndex(const int64_t index) const noexcept { +Value RowReaderV1::getValueByIndex(const int64_t index) const { if (index < 0 || static_cast(index) >= schema_->getNumFields()) { return Value(NullType::UNKNOWN_PROP); } @@ -233,7 +233,7 @@ int64_t RowReaderV1::getTimestamp() const noexcept { * Get the property value from the serialized binary string * ***********************************************************/ -Value RowReaderV1::getBool(int64_t index) const noexcept { +Value RowReaderV1::getBool(int64_t index) const { RR_GET_OFFSET() Value v; switch (getSchema()->getFieldType(index)) { @@ -272,7 +272,7 @@ Value RowReaderV1::getBool(int64_t index) const noexcept { return v; } -Value RowReaderV1::getInt(int64_t index) const noexcept { +Value RowReaderV1::getInt(int64_t index) const { RR_GET_OFFSET() Value v; switch (getSchema()->getFieldType(index)) { @@ -296,7 +296,7 @@ Value RowReaderV1::getInt(int64_t index) const noexcept { return v; } -Value RowReaderV1::getFloat(int64_t index) const noexcept { +Value RowReaderV1::getFloat(int64_t index) const { RR_GET_OFFSET() Value v; switch (getSchema()->getFieldType(index)) { @@ -334,7 +334,7 @@ Value RowReaderV1::getFloat(int64_t index) const noexcept { return v; } -Value RowReaderV1::getDouble(int64_t index) const noexcept { +Value RowReaderV1::getDouble(int64_t index) const { RR_GET_OFFSET() Value v; switch (getSchema()->getFieldType(index)) { @@ -368,7 +368,7 @@ Value RowReaderV1::getDouble(int64_t index) const noexcept { return v; } -Value RowReaderV1::getString(int64_t index) const noexcept { +Value RowReaderV1::getString(int64_t index) const { RR_GET_OFFSET() Value v; switch (getSchema()->getFieldType(index)) { @@ -391,7 +391,7 @@ Value RowReaderV1::getString(int64_t index) const noexcept { return v; } -Value RowReaderV1::getInt64(int64_t index) const noexcept { +Value RowReaderV1::getInt64(int64_t index) const { RR_GET_OFFSET() Value v; int64_t val; @@ -425,7 +425,7 @@ Value RowReaderV1::getInt64(int64_t index) const noexcept { return v; } -Value RowReaderV1::getVid(int64_t index) const noexcept { +Value RowReaderV1::getVid(int64_t index) const { auto fieldType = getSchema()->getFieldType(index); if (fieldType == PropertyType::INT64 || fieldType == PropertyType::VID) { // Since 2.0, vid has been defined as a binary array. So we need to convert @@ -445,7 +445,7 @@ Value RowReaderV1::getVid(int64_t index) const noexcept { * Low-level functions to read from the bytes * ***********************************************************/ -int32_t RowReaderV1::readInteger(int64_t offset, int64_t& v) const noexcept { +int32_t RowReaderV1::readInteger(int64_t offset, int64_t& v) const { const uint8_t* start = reinterpret_cast(&(buffer_[offset])); folly::ByteRange range(start, buffer_.size() - offset); @@ -457,7 +457,7 @@ int32_t RowReaderV1::readInteger(int64_t offset, int64_t& v) const noexcept { return range.begin() - start; } -int32_t RowReaderV1::readFloat(int64_t offset, float& v) const noexcept { +int32_t RowReaderV1::readFloat(int64_t offset, float& v) const { if (offset + sizeof(float) > buffer_.size()) { return -1; } @@ -467,7 +467,7 @@ int32_t RowReaderV1::readFloat(int64_t offset, float& v) const noexcept { return sizeof(float); } -int32_t RowReaderV1::readDouble(int64_t offset, double& v) const noexcept { +int32_t RowReaderV1::readDouble(int64_t offset, double& v) const { if (offset + sizeof(double) > buffer_.size()) { return -1; } @@ -477,7 +477,7 @@ int32_t RowReaderV1::readDouble(int64_t offset, double& v) const noexcept { return sizeof(double); } -int32_t RowReaderV1::readString(int64_t offset, folly::StringPiece& v) const noexcept { +int32_t RowReaderV1::readString(int64_t offset, folly::StringPiece& v) const { int64_t strLen = 0; int32_t intLen = readInteger(offset, strLen); CHECK_GT(intLen, 0) << "Invalid string length"; @@ -489,7 +489,7 @@ int32_t RowReaderV1::readString(int64_t offset, folly::StringPiece& v) const noe return intLen + strLen; } -int32_t RowReaderV1::readInt64(int64_t offset, int64_t& v) const noexcept { +int32_t RowReaderV1::readInt64(int64_t offset, int64_t& v) const { if (offset + sizeof(int64_t) > buffer_.size()) { return -1; } @@ -500,7 +500,7 @@ int32_t RowReaderV1::readInt64(int64_t offset, int64_t& v) const noexcept { return sizeof(int64_t); } -int32_t RowReaderV1::readVid(int64_t offset, int64_t& v) const noexcept { +int32_t RowReaderV1::readVid(int64_t offset, int64_t& v) const { return readInt64(offset, v); } diff --git a/src/codec/RowReaderV1.h b/src/codec/RowReaderV1.h index 473e338fdbe..5197f3e5761 100644 --- a/src/codec/RowReaderV1.h +++ b/src/codec/RowReaderV1.h @@ -27,8 +27,8 @@ class RowReaderV1 : public RowReader { public: ~RowReaderV1() = default; - Value getValueByName(const std::string& prop) const noexcept override; - Value getValueByIndex(const int64_t index) const noexcept override; + Value getValueByName(const std::string& prop) const override; + Value getValueByIndex(const int64_t index) const override; int64_t getTimestamp() const noexcept override; int32_t readerVer() const noexcept override { @@ -40,7 +40,7 @@ class RowReaderV1 : public RowReader { } protected: - bool resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) noexcept override; + bool resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) override; private: int32_t headerLen_ = 0; @@ -72,31 +72,31 @@ class RowReaderV1 : public RowReader { // When succeeded, the method returns the offset pointing to the // next field // When failed, the method returns a negative number - int64_t skipToNext(int64_t index, int64_t offset) const noexcept; + int64_t skipToNext(int64_t index, int64_t offset) const; // Skip to the {index}Th field // The method returns the offset of the field // It returns a negative number when the data corrupts - int64_t skipToField(int64_t index) const noexcept; + int64_t skipToField(int64_t index) const; // Following methods assume the parameters index are valid // When succeeded, offset will advance - Value getBool(int64_t index) const noexcept; - Value getInt(int64_t index) const noexcept; - Value getFloat(int64_t index) const noexcept; - Value getDouble(int64_t index) const noexcept; - Value getString(int64_t index) const noexcept; - Value getInt64(int64_t index) const noexcept; - Value getVid(int64_t index) const noexcept; + Value getBool(int64_t index) const; + Value getInt(int64_t index) const; + Value getFloat(int64_t index) const; + Value getDouble(int64_t index) const; + Value getString(int64_t index) const; + Value getInt64(int64_t index) const; + Value getVid(int64_t index) const; // The following methods all return the number of bytes read // A negative number will be returned if an error occurs - int32_t readInteger(int64_t offset, int64_t& v) const noexcept; - int32_t readFloat(int64_t offset, float& v) const noexcept; - int32_t readDouble(int64_t offset, double& v) const noexcept; - int32_t readString(int64_t offset, folly::StringPiece& v) const noexcept; - int32_t readInt64(int64_t offset, int64_t& v) const noexcept; - int32_t readVid(int64_t offset, int64_t& v) const noexcept; + int32_t readInteger(int64_t offset, int64_t& v) const; + int32_t readFloat(int64_t offset, float& v) const; + int32_t readDouble(int64_t offset, double& v) const; + int32_t readString(int64_t offset, folly::StringPiece& v) const; + int32_t readInt64(int64_t offset, int64_t& v) const; + int32_t readVid(int64_t offset, int64_t& v) const; }; } // namespace nebula diff --git a/src/codec/RowReaderV2.cpp b/src/codec/RowReaderV2.cpp index 384a171494f..9e662a27677 100644 --- a/src/codec/RowReaderV2.cpp +++ b/src/codec/RowReaderV2.cpp @@ -9,7 +9,7 @@ namespace nebula { using nebula::cpp2::PropertyType; -bool RowReaderV2::resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) noexcept { +bool RowReaderV2::resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) { RowReader::resetImpl(schema, row); DCHECK(!!schema_); @@ -45,12 +45,12 @@ bool RowReaderV2::isNull(size_t pos) const { return flag != 0; } -Value RowReaderV2::getValueByName(const std::string& prop) const noexcept { +Value RowReaderV2::getValueByName(const std::string& prop) const { int64_t index = schema_->getFieldIndex(prop); return getValueByIndex(index); } -Value RowReaderV2::getValueByIndex(const int64_t index) const noexcept { +Value RowReaderV2::getValueByIndex(const int64_t index) const { if (index < 0 || static_cast(index) >= schema_->getNumFields()) { return Value(NullType::UNKNOWN_PROP); } diff --git a/src/codec/RowReaderV2.h b/src/codec/RowReaderV2.h index 9bad0bf7f05..9003793e1e9 100644 --- a/src/codec/RowReaderV2.h +++ b/src/codec/RowReaderV2.h @@ -28,8 +28,8 @@ class RowReaderV2 : public RowReader { public: ~RowReaderV2() override = default; - Value getValueByName(const std::string& prop) const noexcept override; - Value getValueByIndex(const int64_t index) const noexcept override; + Value getValueByName(const std::string& prop) const override; + Value getValueByIndex(const int64_t index) const override; int64_t getTimestamp() const noexcept override; int32_t readerVer() const noexcept override { @@ -41,7 +41,7 @@ class RowReaderV2 : public RowReader { } protected: - bool resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) noexcept override; + bool resetImpl(meta::SchemaProviderIf const* schema, folly::StringPiece row) override; private: size_t headerLen_; diff --git a/src/codec/RowReaderWrapper.cpp b/src/codec/RowReaderWrapper.cpp index 5a100fabcfb..0a1b818c066 100644 --- a/src/codec/RowReaderWrapper.cpp +++ b/src/codec/RowReaderWrapper.cpp @@ -97,7 +97,7 @@ RowReaderWrapper::RowReaderWrapper(const meta::SchemaProviderIf* schema, bool RowReaderWrapper::reset(meta::SchemaProviderIf const* schema, folly::StringPiece row, - int32_t readerVer) noexcept { + int32_t readerVer) { CHECK_NOTNULL(schema); readerVer_ = readerVer; if (readerVer_ == 1) { @@ -115,8 +115,7 @@ bool RowReaderWrapper::reset(meta::SchemaProviderIf const* schema, } } -bool RowReaderWrapper::reset(meta::SchemaProviderIf const* schema, - folly::StringPiece row) noexcept { +bool RowReaderWrapper::reset(meta::SchemaProviderIf const* schema, folly::StringPiece row) { currReader_ = nullptr; if (schema == nullptr) { return false; @@ -132,7 +131,7 @@ bool RowReaderWrapper::reset(meta::SchemaProviderIf const* schema, bool RowReaderWrapper::reset( const std::vector>& schemas, - folly::StringPiece row) noexcept { + folly::StringPiece row) { currReader_ = nullptr; SchemaVer schemaVer; int32_t readerVer; diff --git a/src/codec/RowReaderWrapper.h b/src/codec/RowReaderWrapper.h index a619d6d3490..2181670de42 100644 --- a/src/codec/RowReaderWrapper.h +++ b/src/codec/RowReaderWrapper.h @@ -137,9 +137,7 @@ class RowReaderWrapper : public RowReader { * @param readVer * @return Whether reset succeed */ - bool reset(meta::SchemaProviderIf const* schema, - folly::StringPiece row, - int32_t readVer) noexcept; + bool reset(meta::SchemaProviderIf const* schema, folly::StringPiece row, int32_t readVer); /** * @brief Reset current row reader wrapper to of given schema and data @@ -148,7 +146,7 @@ class RowReaderWrapper : public RowReader { * @param row * @return Whether reset succeed */ - bool reset(meta::SchemaProviderIf const* schema, folly::StringPiece row) noexcept; + bool reset(meta::SchemaProviderIf const* schema, folly::StringPiece row); /** * @brief Reset current row reader wrapper of given schemas and data, the schemas are stored in @@ -159,14 +157,14 @@ class RowReaderWrapper : public RowReader { * @return Whether reset succeed */ bool reset(const std::vector>& schemas, - folly::StringPiece row) noexcept; + folly::StringPiece row); - Value getValueByName(const std::string& prop) const noexcept override { + Value getValueByName(const std::string& prop) const override { DCHECK(!!currReader_); return currReader_->getValueByName(prop); } - Value getValueByIndex(const int64_t index) const noexcept override { + Value getValueByIndex(const int64_t index) const override { DCHECK(!!currReader_); return currReader_->getValueByIndex(index); } @@ -187,12 +185,12 @@ class RowReaderWrapper : public RowReader { return currReader_->headerLen(); } - Iterator begin() const noexcept override { + Iterator begin() const override { DCHECK(!!currReader_); return currReader_->begin(); } - const Iterator& end() const noexcept override { + const Iterator& end() const override { DCHECK(!!currReader_); return currReader_->end(); } diff --git a/src/codec/RowWriterV2.cpp b/src/codec/RowWriterV2.cpp index 17bc7c3a6d4..5fcd457f30a 100644 --- a/src/codec/RowWriterV2.cpp +++ b/src/codec/RowWriterV2.cpp @@ -147,7 +147,7 @@ RowWriterV2::RowWriterV2(RowReader& reader) : RowWriterV2(reader.getSchema()) { } } -void RowWriterV2::processV2EncodedStr() noexcept { +void RowWriterV2::processV2EncodedStr() { CHECK_EQ(0x08, buf_[0] & 0x18); int32_t verBytes = buf_[0] & 0x07; SchemaVer ver = 0; @@ -172,21 +172,21 @@ void RowWriterV2::processV2EncodedStr() noexcept { isSet_.resize(schema_->getNumFields(), true); } -void RowWriterV2::setNullBit(ssize_t pos) noexcept { +void RowWriterV2::setNullBit(ssize_t pos) { static const uint8_t orBits[] = {0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01}; size_t offset = headerLen_ + (pos >> 3); buf_[offset] = buf_[offset] | orBits[pos & 0x0000000000000007L]; } -void RowWriterV2::clearNullBit(ssize_t pos) noexcept { +void RowWriterV2::clearNullBit(ssize_t pos) { static const uint8_t andBits[] = {0x7F, 0xBF, 0xDF, 0xEF, 0xF7, 0xFB, 0xFD, 0xFE}; size_t offset = headerLen_ + (pos >> 3); buf_[offset] = buf_[offset] & andBits[pos & 0x0000000000000007L]; } -bool RowWriterV2::checkNullBit(ssize_t pos) const noexcept { +bool RowWriterV2::checkNullBit(ssize_t pos) const { static const uint8_t bits[] = {0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01}; size_t offset = headerLen_ + (pos >> 3); @@ -194,7 +194,7 @@ bool RowWriterV2::checkNullBit(ssize_t pos) const noexcept { return flag != 0; } -WriteResult RowWriterV2::setValue(ssize_t index, const Value& val) noexcept { +WriteResult RowWriterV2::setValue(ssize_t index, const Value& val) { CHECK(!finished_) << "You have called finish()"; if (index < 0 || static_cast(index) >= schema_->getNumFields()) { return WriteResult::UNKNOWN_FIELD; @@ -226,13 +226,13 @@ WriteResult RowWriterV2::setValue(ssize_t index, const Value& val) noexcept { } } -WriteResult RowWriterV2::setValue(const std::string& name, const Value& val) noexcept { +WriteResult RowWriterV2::setValue(const std::string& name, const Value& val) { CHECK(!finished_) << "You have called finish()"; int64_t index = schema_->getFieldIndex(name); return setValue(index, val); } -WriteResult RowWriterV2::setNull(ssize_t index) noexcept { +WriteResult RowWriterV2::setNull(ssize_t index) { CHECK(!finished_) << "You have called finish()"; if (index < 0 || static_cast(index) >= schema_->getNumFields()) { return WriteResult::UNKNOWN_FIELD; @@ -249,13 +249,13 @@ WriteResult RowWriterV2::setNull(ssize_t index) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::setNull(const std::string& name) noexcept { +WriteResult RowWriterV2::setNull(const std::string& name) { CHECK(!finished_) << "You have called finish()"; int64_t index = schema_->getFieldIndex(name); return setNull(index); } -WriteResult RowWriterV2::write(ssize_t index, bool v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, bool v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); switch (field->type()) { @@ -285,7 +285,7 @@ WriteResult RowWriterV2::write(ssize_t index, bool v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, float v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, float v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); switch (field->type()) { @@ -342,7 +342,7 @@ WriteResult RowWriterV2::write(ssize_t index, float v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, double v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, double v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); switch (field->type()) { @@ -401,11 +401,11 @@ WriteResult RowWriterV2::write(ssize_t index, double v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, uint8_t v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, uint8_t v) { return write(index, static_cast(v)); } -WriteResult RowWriterV2::write(ssize_t index, int8_t v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, int8_t v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); switch (field->type()) { @@ -452,11 +452,11 @@ WriteResult RowWriterV2::write(ssize_t index, int8_t v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, uint16_t v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, uint16_t v) { return write(index, static_cast(v)); } -WriteResult RowWriterV2::write(ssize_t index, int16_t v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, int16_t v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); switch (field->type()) { @@ -506,11 +506,11 @@ WriteResult RowWriterV2::write(ssize_t index, int16_t v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, uint32_t v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, uint32_t v) { return write(index, static_cast(v)); } -WriteResult RowWriterV2::write(ssize_t index, int32_t v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, int32_t v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); switch (field->type()) { @@ -573,11 +573,11 @@ WriteResult RowWriterV2::write(ssize_t index, int32_t v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, uint64_t v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, uint64_t v) { return write(index, static_cast(v)); } -WriteResult RowWriterV2::write(ssize_t index, int64_t v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, int64_t v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); switch (field->type()) { @@ -643,15 +643,15 @@ WriteResult RowWriterV2::write(ssize_t index, int64_t v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, const std::string& v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, const std::string& v) { return write(index, folly::StringPiece(v)); } -WriteResult RowWriterV2::write(ssize_t index, const char* v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, const char* v) { return write(index, folly::StringPiece(v)); } -WriteResult RowWriterV2::write(ssize_t index, folly::StringPiece v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, folly::StringPiece v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); switch (field->type()) { @@ -701,7 +701,7 @@ WriteResult RowWriterV2::write(ssize_t index, folly::StringPiece v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, const Date& v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, const Date& v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); switch (field->type()) { @@ -720,7 +720,7 @@ WriteResult RowWriterV2::write(ssize_t index, const Date& v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, const Time& v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, const Time& v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); switch (field->type()) { @@ -742,7 +742,7 @@ WriteResult RowWriterV2::write(ssize_t index, const Time& v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, const DateTime& v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, const DateTime& v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); int16_t year = v.year; @@ -774,7 +774,7 @@ WriteResult RowWriterV2::write(ssize_t index, const DateTime& v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, const Duration& v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, const Duration& v) { auto field = schema_->field(index); auto offset = headerLen_ + numNullBytes_ + field->offset(); switch (field->type()) { @@ -797,7 +797,7 @@ WriteResult RowWriterV2::write(ssize_t index, const Duration& v) noexcept { return WriteResult::SUCCEEDED; } -WriteResult RowWriterV2::write(ssize_t index, const Geography& v) noexcept { +WriteResult RowWriterV2::write(ssize_t index, const Geography& v) { auto field = schema_->field(index); auto geoShape = field->geoShape(); if (geoShape != meta::cpp2::GeoShape::ANY && @@ -808,7 +808,7 @@ WriteResult RowWriterV2::write(ssize_t index, const Geography& v) noexcept { return write(index, folly::StringPiece(wkb)); } -WriteResult RowWriterV2::checkUnsetFields() noexcept { +WriteResult RowWriterV2::checkUnsetFields() { DefaultValueContext expCtx; for (size_t i = 0; i < schema_->getNumFields(); i++) { if (!isSet_[i]) { @@ -875,7 +875,7 @@ WriteResult RowWriterV2::checkUnsetFields() noexcept { return WriteResult::SUCCEEDED; } -std::string RowWriterV2::processOutOfSpace() noexcept { +std::string RowWriterV2::processOutOfSpace() { std::string temp; // Reserve enough space to avoid memory re-allocation temp.reserve(headerLen_ + numNullBytes_ + schema_->size() + approxStrLen_ + sizeof(int64_t)); @@ -919,7 +919,7 @@ std::string RowWriterV2::processOutOfSpace() noexcept { return temp; } -WriteResult RowWriterV2::finish() noexcept { +WriteResult RowWriterV2::finish() { CHECK(!finished_) << "You have called finish()"; // First to check whether all fields are set. If not, to check whether diff --git a/src/codec/RowWriterV2.h b/src/codec/RowWriterV2.h index 6b79c00a0ca..6dca13bbafa 100644 --- a/src/codec/RowWriterV2.h +++ b/src/codec/RowWriterV2.h @@ -118,7 +118,7 @@ class RowWriterV2 { * * @return const std::string& */ - const std::string& getEncodedStr() const noexcept { + const std::string& getEncodedStr() const { CHECK(finished_) << "You need to call finish() first"; return buf_; } @@ -128,7 +128,7 @@ class RowWriterV2 { * * @return std::string */ - std::string moveEncodedStr() noexcept { + std::string moveEncodedStr() { CHECK(finished_) << "You need to call finish() first"; return std::move(buf_); } @@ -138,7 +138,7 @@ class RowWriterV2 { * * @return WriteResult Whether encode succeed */ - WriteResult finish() noexcept; + WriteResult finish(); // Data write /** @@ -150,7 +150,7 @@ class RowWriterV2 { * @return WriteResult */ template - WriteResult set(size_t index, T&& v) noexcept { + WriteResult set(size_t index, T&& v) { CHECK(!finished_) << "You have called finish()"; if (index >= schema_->getNumFields()) { return WriteResult::UNKNOWN_FIELD; @@ -168,7 +168,7 @@ class RowWriterV2 { * @return WriteResult */ template - WriteResult set(const std::string& name, T&& v) noexcept { + WriteResult set(const std::string& name, T&& v) { CHECK(!finished_) << "You have called finish()"; int64_t index = schema_->getFieldIndex(name); if (index >= 0) { @@ -185,7 +185,7 @@ class RowWriterV2 { * @param val * @return WriteResult */ - WriteResult setValue(ssize_t index, const Value& val) noexcept; + WriteResult setValue(ssize_t index, const Value& val); /** * @brief Set the value by index @@ -194,7 +194,7 @@ class RowWriterV2 { * @param val * @return WriteResult */ - WriteResult setValue(const std::string& name, const Value& val) noexcept; + WriteResult setValue(const std::string& name, const Value& val); /** * @brief Set null by index @@ -202,7 +202,7 @@ class RowWriterV2 { * @param index * @return WriteResult */ - WriteResult setNull(ssize_t index) noexcept; + WriteResult setNull(ssize_t index); /** * @brief Set null by property name @@ -210,7 +210,7 @@ class RowWriterV2 { * @param name * @return WriteResult */ - WriteResult setNull(const std::string& name) noexcept; + WriteResult setNull(const std::string& name); private: const meta::SchemaProviderIf* schema_; @@ -229,40 +229,40 @@ class RowWriterV2 { bool outOfSpaceStr_; std::vector strList_; - WriteResult checkUnsetFields() noexcept; - std::string processOutOfSpace() noexcept; + WriteResult checkUnsetFields(); + std::string processOutOfSpace(); - void processV2EncodedStr() noexcept; + void processV2EncodedStr(); - void setNullBit(ssize_t pos) noexcept; - void clearNullBit(ssize_t pos) noexcept; + void setNullBit(ssize_t pos); + void clearNullBit(ssize_t pos); // Return true if the flag at the given position is NULL; // otherwise, return false - bool checkNullBit(ssize_t pos) const noexcept; - - WriteResult write(ssize_t index, bool v) noexcept; - WriteResult write(ssize_t index, float v) noexcept; - WriteResult write(ssize_t index, double v) noexcept; - - WriteResult write(ssize_t index, int8_t v) noexcept; - WriteResult write(ssize_t index, int16_t v) noexcept; - WriteResult write(ssize_t index, int32_t v) noexcept; - WriteResult write(ssize_t index, int64_t v) noexcept; - WriteResult write(ssize_t index, uint8_t v) noexcept; - WriteResult write(ssize_t index, uint16_t v) noexcept; - WriteResult write(ssize_t index, uint32_t v) noexcept; - WriteResult write(ssize_t index, uint64_t v) noexcept; - - WriteResult write(ssize_t index, const std::string& v) noexcept; - WriteResult write(ssize_t index, folly::StringPiece v) noexcept; - WriteResult write(ssize_t index, const char* v) noexcept; - - WriteResult write(ssize_t index, const Date& v) noexcept; - WriteResult write(ssize_t index, const Time& v) noexcept; - WriteResult write(ssize_t index, const DateTime& v) noexcept; - WriteResult write(ssize_t index, const Duration& v) noexcept; - - WriteResult write(ssize_t index, const Geography& v) noexcept; + bool checkNullBit(ssize_t pos) const; + + WriteResult write(ssize_t index, bool v); + WriteResult write(ssize_t index, float v); + WriteResult write(ssize_t index, double v); + + WriteResult write(ssize_t index, int8_t v); + WriteResult write(ssize_t index, int16_t v); + WriteResult write(ssize_t index, int32_t v); + WriteResult write(ssize_t index, int64_t v); + WriteResult write(ssize_t index, uint8_t v); + WriteResult write(ssize_t index, uint16_t v); + WriteResult write(ssize_t index, uint32_t v); + WriteResult write(ssize_t index, uint64_t v); + + WriteResult write(ssize_t index, const std::string& v); + WriteResult write(ssize_t index, folly::StringPiece v); + WriteResult write(ssize_t index, const char* v); + + WriteResult write(ssize_t index, const Date& v); + WriteResult write(ssize_t index, const Time& v); + WriteResult write(ssize_t index, const DateTime& v); + WriteResult write(ssize_t index, const Duration& v); + + WriteResult write(ssize_t index, const Geography& v); }; } // namespace nebula diff --git a/src/common/base/ObjectPool.h b/src/common/base/ObjectPool.h index acd20d73986..ce0f420c365 100644 --- a/src/common/base/ObjectPool.h +++ b/src/common/base/ObjectPool.h @@ -38,9 +38,12 @@ class ObjectPool final : private boost::noncopyable, private cpp::NonMovable { template T *makeAndAdd(Args &&... args) { - lock_.lock(); - void *ptr = arena_.allocateAligned(sizeof(T)); - lock_.unlock(); + void *ptr; + { + // alloc happens here(may throw bad_alloc), use guard to guarantee unlock + SLGuard g(lock_); + ptr = arena_.allocateAligned(sizeof(T)); + } return add(new (ptr) T(std::forward(args)...)); } diff --git a/src/common/datatypes/DataSet.h b/src/common/datatypes/DataSet.h index b1230001ce8..600ef231ec8 100644 --- a/src/common/datatypes/DataSet.h +++ b/src/common/datatypes/DataSet.h @@ -26,7 +26,7 @@ struct DataSet { DataSet() = default; explicit DataSet(std::vector columns) : colNames(std::move(columns)) {} - DataSet(const DataSet& ds) noexcept { + DataSet(const DataSet& ds) { colNames = ds.colNames; rows = ds.rows; } @@ -34,7 +34,7 @@ struct DataSet { colNames = std::move(ds.colNames); rows = std::move(ds.rows); } - DataSet& operator=(const DataSet& ds) noexcept { + DataSet& operator=(const DataSet& ds) { if (&ds != this) { colNames = ds.colNames; rows = ds.rows; diff --git a/src/common/datatypes/Edge.cpp b/src/common/datatypes/Edge.cpp index c5d2f3464b4..c734db5aa4d 100644 --- a/src/common/datatypes/Edge.cpp +++ b/src/common/datatypes/Edge.cpp @@ -167,7 +167,7 @@ std::string Edge::id() const { namespace std { // Inject a customized hash function -std::size_t hash::operator()(const nebula::Edge& h) const noexcept { +std::size_t hash::operator()(const nebula::Edge& h) const { const auto& src = h.type > 0 ? h.src.toString() : h.dst.toString(); const auto& dst = h.type > 0 ? h.dst.toString() : h.src.toString(); auto type = h.type > 0 ? h.type : -h.type; diff --git a/src/common/datatypes/Edge.h b/src/common/datatypes/Edge.h index e873c62689a..dcdef8a58c7 100644 --- a/src/common/datatypes/Edge.h +++ b/src/common/datatypes/Edge.h @@ -100,7 +100,7 @@ namespace std { // Inject a customized hash function template <> struct hash { - std::size_t operator()(const nebula::Edge& h) const noexcept; + std::size_t operator()(const nebula::Edge& h) const; }; } // namespace std diff --git a/src/common/datatypes/Geography.cpp b/src/common/datatypes/Geography.cpp index 05885d294db..01eb56e6a7b 100644 --- a/src/common/datatypes/Geography.cpp +++ b/src/common/datatypes/Geography.cpp @@ -387,7 +387,7 @@ bool Geography::operator<(const Geography& rhs) const { namespace std { // Inject a customized hash function -std::size_t hash::operator()(const nebula::Geography& v) const noexcept { +std::size_t hash::operator()(const nebula::Geography& v) const { std::string wkb = v.asWKB(); return hash{}(wkb); } diff --git a/src/common/datatypes/Geography.h b/src/common/datatypes/Geography.h index 6d819750af4..2f93c157b3d 100644 --- a/src/common/datatypes/Geography.h +++ b/src/common/datatypes/Geography.h @@ -238,7 +238,7 @@ namespace std { // Inject a customized hash function template <> struct hash { - std::size_t operator()(const nebula::Geography& h) const noexcept; + std::size_t operator()(const nebula::Geography& h) const; }; } // namespace std diff --git a/src/common/datatypes/List.h b/src/common/datatypes/List.h index 28b55caf5cf..d2509d9b687 100644 --- a/src/common/datatypes/List.h +++ b/src/common/datatypes/List.h @@ -103,7 +103,7 @@ inline std::ostream& operator<<(std::ostream& os, const List& l) { namespace std { template <> struct hash { - std::size_t operator()(const nebula::List& h) const noexcept { + std::size_t operator()(const nebula::List& h) const { if (h.values.size() == 1) { return std::hash()(h.values[0]); } diff --git a/src/common/datatypes/Map.cpp b/src/common/datatypes/Map.cpp index 095d001cc4a..7cb1de2b80a 100644 --- a/src/common/datatypes/Map.cpp +++ b/src/common/datatypes/Map.cpp @@ -91,7 +91,7 @@ Map::Map(const folly::dynamic& obj) { } // namespace nebula namespace std { -std::size_t hash::operator()(const nebula::Map& m) const noexcept { +std::size_t hash::operator()(const nebula::Map& m) const { size_t seed = 0; for (auto& v : m.kvs) { seed ^= hash()(v.first) + 0x9e3779b9 + (seed << 6) + (seed >> 2); diff --git a/src/common/datatypes/Map.h b/src/common/datatypes/Map.h index 91134a48387..15a0dd9099e 100644 --- a/src/common/datatypes/Map.h +++ b/src/common/datatypes/Map.h @@ -88,7 +88,7 @@ inline std::ostream& operator<<(std::ostream& os, const Map& m) { namespace std { template <> struct hash { - std::size_t operator()(const nebula::Map& m) const noexcept; + std::size_t operator()(const nebula::Map& m) const; }; } // namespace std diff --git a/src/common/datatypes/Path.cpp b/src/common/datatypes/Path.cpp index b752239ed12..9ebb33be243 100644 --- a/src/common/datatypes/Path.cpp +++ b/src/common/datatypes/Path.cpp @@ -87,7 +87,7 @@ std::size_t hash::operator()(const nebula::Step& h) const noexcept return folly::hash::fnv64_buf(reinterpret_cast(&h.ranking), sizeof(h.ranking), hv); } -std::size_t hash::operator()(const nebula::Path& h) const noexcept { +std::size_t hash::operator()(const nebula::Path& h) const { size_t hv = hash()(h.src); for (auto& s : h.steps) { hv += (hv << 1) + (hv << 4) + (hv << 5) + (hv << 7) + (hv << 8) + (hv << 40); diff --git a/src/common/datatypes/Path.h b/src/common/datatypes/Path.h index 5dbedb3b9d0..cfc9c07c629 100644 --- a/src/common/datatypes/Path.h +++ b/src/common/datatypes/Path.h @@ -28,11 +28,7 @@ struct Step { name(std::move(s.name)), ranking(std::move(s.ranking)), props(std::move(s.props)) {} - Step(Vertex d, - EdgeType t, - std::string n, - EdgeRanking r, - std::unordered_map p) noexcept + Step(Vertex d, EdgeType t, std::string n, EdgeRanking r, std::unordered_map p) : dst(std::move(d)), type(t), name(std::move(n)), ranking(r), props(std::move(p)) {} void clear() { @@ -71,7 +67,7 @@ struct Step { return *this; } - Step& operator=(const Step& rhs) noexcept { + Step& operator=(const Step& rhs) { if (&rhs != this) { dst = rhs.dst; type = rhs.type; @@ -190,7 +186,7 @@ struct Path { return *this; } - Path& operator=(const Path& rhs) noexcept { + Path& operator=(const Path& rhs) { if (&rhs != this) { src = rhs.src; steps = rhs.steps; @@ -247,7 +243,7 @@ struct hash { template <> struct hash { - std::size_t operator()(const nebula::Path& h) const noexcept; + std::size_t operator()(const nebula::Path& h) const; }; } // namespace std diff --git a/src/common/datatypes/Set.cpp b/src/common/datatypes/Set.cpp index 103170546e4..92f116f2c10 100644 --- a/src/common/datatypes/Set.cpp +++ b/src/common/datatypes/Set.cpp @@ -44,7 +44,7 @@ folly::dynamic Set::getMetaData() const { } // namespace nebula namespace std { -std::size_t hash::operator()(const nebula::Set& s) const noexcept { +std::size_t hash::operator()(const nebula::Set& s) const { size_t seed = 0; for (auto& v : s.values) { seed ^= hash()(v) + 0x9e3779b9 + (seed << 6) + (seed >> 2); diff --git a/src/common/datatypes/Set.h b/src/common/datatypes/Set.h index 107cf3aee5f..5e3e30870b8 100644 --- a/src/common/datatypes/Set.h +++ b/src/common/datatypes/Set.h @@ -72,7 +72,7 @@ inline std::ostream& operator<<(std::ostream& os, const Set& s) { namespace std { template <> struct hash { - std::size_t operator()(const nebula::Set& s) const noexcept; + std::size_t operator()(const nebula::Set& s) const; }; } // namespace std diff --git a/src/common/datatypes/Value.cpp b/src/common/datatypes/Value.cpp index f17d9a4642b..12d2b143ada 100644 --- a/src/common/datatypes/Value.cpp +++ b/src/common/datatypes/Value.cpp @@ -1185,265 +1185,265 @@ Value& Value::operator=(const Value& rhs) { } void Value::setN(const NullType& v) { - type_ = Type::NULLVALUE; new (std::addressof(value_.nVal)) NullType(v); + type_ = Type::NULLVALUE; } void Value::setN(NullType&& v) { - type_ = Type::NULLVALUE; new (std::addressof(value_.nVal)) NullType(std::move(v)); + type_ = Type::NULLVALUE; } void Value::setB(const bool& v) { - type_ = Type::BOOL; new (std::addressof(value_.bVal)) bool(v); // NOLINT + type_ = Type::BOOL; } void Value::setB(bool&& v) { - type_ = Type::BOOL; new (std::addressof(value_.bVal)) bool(std::move(v)); // NOLINT + type_ = Type::BOOL; } void Value::setI(const int64_t& v) { - type_ = Type::INT; new (std::addressof(value_.iVal)) int64_t(v); // NOLINT + type_ = Type::INT; } void Value::setI(int64_t&& v) { - type_ = Type::INT; new (std::addressof(value_.iVal)) int64_t(std::move(v)); // NOLINT + type_ = Type::INT; } void Value::setF(const double& v) { - type_ = Type::FLOAT; new (std::addressof(value_.fVal)) double(v); // NOLINT + type_ = Type::FLOAT; } void Value::setF(double&& v) { - type_ = Type::FLOAT; new (std::addressof(value_.fVal)) double(std::move(v)); // NOLINT + type_ = Type::FLOAT; } void Value::setS(std::unique_ptr v) { - type_ = Type::STRING; new (std::addressof(value_.sVal)) std::unique_ptr(std::move(v)); + type_ = Type::STRING; } void Value::setS(const std::string& v) { - type_ = Type::STRING; new (std::addressof(value_.sVal)) std::unique_ptr(new std::string(v)); + type_ = Type::STRING; } void Value::setS(std::string&& v) { - type_ = Type::STRING; new (std::addressof(value_.sVal)) std::unique_ptr(new std::string(std::move(v))); + type_ = Type::STRING; } void Value::setS(const char* v) { - type_ = Type::STRING; new (std::addressof(value_.sVal)) std::unique_ptr(new std::string(v)); + type_ = Type::STRING; } void Value::setD(const Date& v) { - type_ = Type::DATE; new (std::addressof(value_.dVal)) Date(v); + type_ = Type::DATE; } void Value::setD(Date&& v) { - type_ = Type::DATE; new (std::addressof(value_.dVal)) Date(std::move(v)); + type_ = Type::DATE; } void Value::setT(const Time& v) { - type_ = Type::TIME; new (std::addressof(value_.tVal)) Time(v); + type_ = Type::TIME; } void Value::setT(Time&& v) { - type_ = Type::TIME; new (std::addressof(value_.tVal)) Time(std::move(v)); + type_ = Type::TIME; } void Value::setDT(const DateTime& v) { - type_ = Type::DATETIME; new (std::addressof(value_.dtVal)) DateTime(v); + type_ = Type::DATETIME; } void Value::setDT(DateTime&& v) { - type_ = Type::DATETIME; new (std::addressof(value_.dtVal)) DateTime(std::move(v)); + type_ = Type::DATETIME; } void Value::setV(Vertex* v) { - type_ = Type::VERTEX; value_.vVal = v; value_.vVal->ref(); + type_ = Type::VERTEX; } void Value::setV(const Vertex& v) { - type_ = Type::VERTEX; new (std::addressof(value_.vVal)) Vertex*(new Vertex(v)); + type_ = Type::VERTEX; } void Value::setV(Vertex&& v) { - type_ = Type::VERTEX; new (std::addressof(value_.vVal)) Vertex*(new Vertex(std::move(v))); + type_ = Type::VERTEX; } void Value::setE(Edge* v) { - type_ = Type::EDGE; value_.eVal = v; value_.eVal->ref(); + type_ = Type::EDGE; } void Value::setE(const Edge& v) { - type_ = Type::EDGE; new (std::addressof(value_.eVal)) Edge*(new Edge(v)); + type_ = Type::EDGE; } void Value::setE(Edge&& v) { - type_ = Type::EDGE; new (std::addressof(value_.eVal)) Edge*(new Edge(std::move(v))); + type_ = Type::EDGE; } void Value::setP(const std::unique_ptr& v) { - type_ = Type::PATH; new (std::addressof(value_.pVal)) std::unique_ptr(new Path(*v)); + type_ = Type::PATH; } void Value::setP(std::unique_ptr&& v) { - type_ = Type::PATH; new (std::addressof(value_.pVal)) std::unique_ptr(std::move(v)); + type_ = Type::PATH; } void Value::setP(const Path& v) { - type_ = Type::PATH; new (std::addressof(value_.pVal)) std::unique_ptr(new Path(v)); + type_ = Type::PATH; } void Value::setP(Path&& v) { - type_ = Type::PATH; new (std::addressof(value_.pVal)) std::unique_ptr(new Path(std::move(v))); + type_ = Type::PATH; } void Value::setL(const std::unique_ptr& v) { - type_ = Type::LIST; new (std::addressof(value_.lVal)) std::unique_ptr(new List(*v)); + type_ = Type::LIST; } void Value::setL(std::unique_ptr&& v) { - type_ = Type::LIST; new (std::addressof(value_.lVal)) std::unique_ptr(std::move(v)); + type_ = Type::LIST; } void Value::setL(const List& v) { - type_ = Type::LIST; new (std::addressof(value_.lVal)) std::unique_ptr(new List(v)); + type_ = Type::LIST; } void Value::setL(List&& v) { - type_ = Type::LIST; new (std::addressof(value_.lVal)) std::unique_ptr(new List(std::move(v))); + type_ = Type::LIST; } void Value::setM(const std::unique_ptr& v) { - type_ = Type::MAP; new (std::addressof(value_.mVal)) std::unique_ptr(new Map(*v)); + type_ = Type::MAP; } void Value::setM(std::unique_ptr&& v) { - type_ = Type::MAP; new (std::addressof(value_.mVal)) std::unique_ptr(std::move(v)); + type_ = Type::MAP; } void Value::setM(const Map& v) { - type_ = Type::MAP; new (std::addressof(value_.mVal)) std::unique_ptr(new Map(v)); + type_ = Type::MAP; } void Value::setM(Map&& v) { - type_ = Type::MAP; new (std::addressof(value_.mVal)) std::unique_ptr(new Map(std::move(v))); + type_ = Type::MAP; } void Value::setU(const std::unique_ptr& v) { - type_ = Type::SET; new (std::addressof(value_.uVal)) std::unique_ptr(new Set(*v)); + type_ = Type::SET; } void Value::setU(std::unique_ptr&& v) { - type_ = Type::SET; new (std::addressof(value_.uVal)) std::unique_ptr(std::move(v)); + type_ = Type::SET; } void Value::setU(const Set& v) { - type_ = Type::SET; new (std::addressof(value_.uVal)) std::unique_ptr(new Set(v)); + type_ = Type::SET; } void Value::setU(Set&& v) { - type_ = Type::SET; new (std::addressof(value_.vVal)) std::unique_ptr(new Set(std::move(v))); + type_ = Type::SET; } void Value::setG(const std::unique_ptr& v) { - type_ = Type::DATASET; new (std::addressof(value_.gVal)) std::unique_ptr(new DataSet(*v)); + type_ = Type::DATASET; } void Value::setG(std::unique_ptr&& v) { - type_ = Type::DATASET; new (std::addressof(value_.gVal)) std::unique_ptr(std::move(v)); + type_ = Type::DATASET; } void Value::setG(const DataSet& v) { - type_ = Type::DATASET; new (std::addressof(value_.gVal)) std::unique_ptr(new DataSet(v)); + type_ = Type::DATASET; } void Value::setG(DataSet&& v) { - type_ = Type::DATASET; new (std::addressof(value_.gVal)) std::unique_ptr(new DataSet(std::move(v))); + type_ = Type::DATASET; } void Value::setGG(const std::unique_ptr& v) { - type_ = Type::GEOGRAPHY; new (std::addressof(value_.ggVal)) std::unique_ptr(new Geography(*v)); + type_ = Type::GEOGRAPHY; } void Value::setGG(std::unique_ptr&& v) { - type_ = Type::GEOGRAPHY; new (std::addressof(value_.ggVal)) std::unique_ptr(std::move(v)); + type_ = Type::GEOGRAPHY; } void Value::setGG(const Geography& v) { - type_ = Type::GEOGRAPHY; new (std::addressof(value_.ggVal)) std::unique_ptr(new Geography(v)); + type_ = Type::GEOGRAPHY; } void Value::setGG(Geography&& v) { - type_ = Type::GEOGRAPHY; new (std::addressof(value_.ggVal)) std::unique_ptr(new Geography(std::move(v))); + type_ = Type::GEOGRAPHY; } void Value::setDU(const std::unique_ptr& v) { - type_ = Type::DURATION; new (std::addressof(value_.duVal)) std::unique_ptr(new Duration(*v)); + type_ = Type::DURATION; } void Value::setDU(std::unique_ptr&& v) { - type_ = Type::DURATION; new (std::addressof(value_.duVal)) std::unique_ptr(std::move(v)); + type_ = Type::DURATION; } void Value::setDU(const Duration& v) { - type_ = Type::DURATION; new (std::addressof(value_.duVal)) std::unique_ptr(new Duration(v)); + type_ = Type::DURATION; } void Value::setDU(Duration&& v) { - type_ = Type::DURATION; new (std::addressof(value_.duVal)) std::unique_ptr(new Duration(std::move(v))); + type_ = Type::DURATION; } // Convert Nebula::Value to a value compatible with Json standard diff --git a/src/common/datatypes/Value.h b/src/common/datatypes/Value.h index 7047ccd0f53..6d693533995 100644 --- a/src/common/datatypes/Value.h +++ b/src/common/datatypes/Value.h @@ -557,7 +557,7 @@ namespace std { // Inject a customized hash function template <> struct hash { - std::size_t operator()(const nebula::Value& h) const noexcept { + std::size_t operator()(const nebula::Value& h) const { if (h.isInt()) { return h.getInt(); } else if (h.isStr()) { @@ -569,14 +569,14 @@ struct hash { template <> struct hash { - std::size_t operator()(const nebula::Value* h) const noexcept { + std::size_t operator()(const nebula::Value* h) const { return h == nullptr ? 0 : hash()(*h); } }; template <> struct hash { - std::size_t operator()(const nebula::Value* h) const noexcept { + std::size_t operator()(const nebula::Value* h) const { return h == nullptr ? 0 : hash()(*h); } }; diff --git a/src/common/datatypes/ValueOps-inl.h b/src/common/datatypes/ValueOps-inl.h index c58cb0a5261..08ff4205f2c 100644 --- a/src/common/datatypes/ValueOps-inl.h +++ b/src/common/datatypes/ValueOps-inl.h @@ -369,10 +369,10 @@ void Cpp2Ops::read(Protocol* proto, nebula::Value* obj) { } case 9: { if (readState.fieldType == apache::thrift::protocol::T_STRUCT) { - auto* ptr = new nebula::Vertex(); - Cpp2Ops::read(proto, ptr); - obj->setVertex(ptr); + auto ptr = std::make_unique(); ptr->unref(); + Cpp2Ops::read(proto, ptr.get()); + obj->setVertex(ptr.release()); } else { proto->skip(readState.fieldType); } @@ -380,10 +380,10 @@ void Cpp2Ops::read(Protocol* proto, nebula::Value* obj) { } case 10: { if (readState.fieldType == apache::thrift::protocol::T_STRUCT) { - auto* ptr = new nebula::Edge(); - Cpp2Ops::read(proto, ptr); - obj->setEdge(ptr); + auto ptr = std::make_unique(); ptr->unref(); + Cpp2Ops::read(proto, ptr.get()); + obj->setEdge(ptr.release()); } else { proto->skip(readState.fieldType); } diff --git a/src/common/datatypes/Vertex.cpp b/src/common/datatypes/Vertex.cpp index 291e6cfd6f1..41eef5208a4 100644 --- a/src/common/datatypes/Vertex.cpp +++ b/src/common/datatypes/Vertex.cpp @@ -133,7 +133,7 @@ std::size_t hash::operator()(const nebula::Tag& h) const noexcept { return folly::hash::fnv64(h.name); } -std::size_t hash::operator()(const nebula::Vertex& h) const noexcept { +std::size_t hash::operator()(const nebula::Vertex& h) const { size_t hv = folly::hash::fnv64(h.vid.toString()); for (auto& t : h.tags) { hv += (hv << 1) + (hv << 4) + (hv << 5) + (hv << 7) + (hv << 8) + (hv << 40); diff --git a/src/common/datatypes/Vertex.h b/src/common/datatypes/Vertex.h index 29ef6f83753..539d256f28a 100644 --- a/src/common/datatypes/Vertex.h +++ b/src/common/datatypes/Vertex.h @@ -130,7 +130,7 @@ struct hash { template <> struct hash { - std::size_t operator()(const nebula::Vertex& h) const noexcept; + std::size_t operator()(const nebula::Vertex& h) const; }; } // namespace std diff --git a/src/common/expression/Expression.cpp b/src/common/expression/Expression.cpp index ec36dab0b67..478bdbe3c71 100644 --- a/src/common/expression/Expression.cpp +++ b/src/common/expression/Expression.cpp @@ -50,12 +50,12 @@ std::string Expression::Encoder::moveStr() { return std::move(buf_); } -Expression::Encoder& Expression::Encoder::operator<<(Kind kind) noexcept { +Expression::Encoder& Expression::Encoder::operator<<(Kind kind) { buf_.append(reinterpret_cast(&kind), sizeof(uint8_t)); return *this; } -Expression::Encoder& Expression::Encoder::operator<<(const std::string& str) noexcept { +Expression::Encoder& Expression::Encoder::operator<<(const std::string& str) { size_t sz = str.size(); buf_.append(reinterpret_cast(&sz), sizeof(size_t)); if (sz > 0) { @@ -64,22 +64,22 @@ Expression::Encoder& Expression::Encoder::operator<<(const std::string& str) noe return *this; } -Expression::Encoder& Expression::Encoder::operator<<(const Value& val) noexcept { +Expression::Encoder& Expression::Encoder::operator<<(const Value& val) { serializer::serialize(val, &buf_); return *this; } -Expression::Encoder& Expression::Encoder::operator<<(size_t size) noexcept { +Expression::Encoder& Expression::Encoder::operator<<(size_t size) { buf_.append(reinterpret_cast(&size), sizeof(size_t)); return *this; } -Expression::Encoder& Expression::Encoder::operator<<(Value::Type vType) noexcept { +Expression::Encoder& Expression::Encoder::operator<<(Value::Type vType) { buf_.append(reinterpret_cast(&vType), sizeof(Value::Type)); return *this; } -Expression::Encoder& Expression::Encoder::operator<<(const Expression& exp) noexcept { +Expression::Encoder& Expression::Encoder::operator<<(const Expression& exp) { exp.writeTo(*this); return *this; } @@ -100,7 +100,7 @@ std::string Expression::Decoder::getHexStr() const { return toHexStr(encoded_); } -Expression::Kind Expression::Decoder::readKind() noexcept { +Expression::Kind Expression::Decoder::readKind() { CHECK_LE(ptr_ + sizeof(uint8_t), encoded_.end()); Expression::Kind kind; @@ -110,7 +110,7 @@ Expression::Kind Expression::Decoder::readKind() noexcept { return kind; } -std::string Expression::Decoder::readStr() noexcept { +std::string Expression::Decoder::readStr() { CHECK_LE(ptr_ + sizeof(size_t), encoded_.end()); size_t sz = 0; @@ -127,7 +127,7 @@ std::string Expression::Decoder::readStr() noexcept { return std::string(ptr, sz); } -Value Expression::Decoder::readValue() noexcept { +Value Expression::Decoder::readValue() { Value val; size_t len = serializer::deserialize(folly::StringPiece(ptr_, encoded_.end()), val); ptr_ += len; @@ -148,7 +148,7 @@ Value::Type Expression::Decoder::readValueType() noexcept { return type; } -Expression* Expression::Decoder::readExpression(ObjectPool* pool) noexcept { +Expression* Expression::Decoder::readExpression(ObjectPool* pool) { return Expression::decode(pool, *this); } diff --git a/src/common/expression/Expression.h b/src/common/expression/Expression.h index 0b21287a5fd..a23340589fe 100644 --- a/src/common/expression/Expression.h +++ b/src/common/expression/Expression.h @@ -177,12 +177,12 @@ class Expression { explicit Encoder(size_t bufSizeHint = 2048); std::string moveStr(); - Encoder& operator<<(Kind kind) noexcept; - Encoder& operator<<(const std::string& str) noexcept; - Encoder& operator<<(const Value& val) noexcept; - Encoder& operator<<(size_t size) noexcept; - Encoder& operator<<(Value::Type vType) noexcept; - Encoder& operator<<(const Expression& exp) noexcept; + Encoder& operator<<(Kind kind); + Encoder& operator<<(const std::string& str); + Encoder& operator<<(const Value& val); + Encoder& operator<<(size_t size); + Encoder& operator<<(Value::Type vType); + Encoder& operator<<(const Expression& exp); private: std::string buf_; @@ -194,12 +194,12 @@ class Expression { bool finished() const; - Kind readKind() noexcept; - std::string readStr() noexcept; - Value readValue() noexcept; + Kind readKind(); + std::string readStr(); + Value readValue(); size_t readSize() noexcept; Value::Type readValueType() noexcept; - Expression* readExpression(ObjectPool* pool) noexcept; + Expression* readExpression(ObjectPool* pool); // Convert the unprocessed part into the hex string std::string getHexStr() const; diff --git a/src/common/geo/GeoIndex.cpp b/src/common/geo/GeoIndex.cpp index 0995afc99eb..b041557216e 100644 --- a/src/common/geo/GeoIndex.cpp +++ b/src/common/geo/GeoIndex.cpp @@ -57,7 +57,7 @@ nebula::storage::cpp2::IndexColumnHint ScanRange::toIndexColumnHint() const { return hint; } -std::vector GeoIndex::indexCells(const Geography& g) const noexcept { +std::vector GeoIndex::indexCells(const Geography& g) const { auto r = g.asS2(); if (UNLIKELY(!r)) { return {}; @@ -72,7 +72,7 @@ std::vector GeoIndex::indexCells(const Geography& g) const noexcept { return cellIds; } -std::vector GeoIndex::intersects(const Geography& g) const noexcept { +std::vector GeoIndex::intersects(const Geography& g) const { auto r = g.asS2(); if (UNLIKELY(!r)) { return {}; @@ -82,16 +82,16 @@ std::vector GeoIndex::intersects(const Geography& g) const noexcept { } // covers degenerates to intersects currently -std::vector GeoIndex::covers(const Geography& g) const noexcept { +std::vector GeoIndex::covers(const Geography& g) const { return intersects(g); } // coveredBy degenerates to intersects currently -std::vector GeoIndex::coveredBy(const Geography& g) const noexcept { +std::vector GeoIndex::coveredBy(const Geography& g) const { return intersects(g); } -std::vector GeoIndex::dWithin(const Geography& g, double distance) const noexcept { +std::vector GeoIndex::dWithin(const Geography& g, double distance) const { auto r = g.asS2(); if (UNLIKELY(!r)) { return {}; @@ -124,7 +124,7 @@ std::vector GeoIndex::dWithin(const Geography& g, double distance) co } } -std::vector GeoIndex::intersects(const S2Region& r, bool isPoint) const noexcept { +std::vector GeoIndex::intersects(const S2Region& r, bool isPoint) const { auto cells = coveringCells(r, isPoint); std::vector scanRanges; for (const S2CellId& cellId : cells) { @@ -147,7 +147,7 @@ std::vector GeoIndex::intersects(const S2Region& r, bool isPoint) con return scanRanges; } -std::vector GeoIndex::coveringCells(const S2Region& r, bool isPoint) const noexcept { +std::vector GeoIndex::coveringCells(const S2Region& r, bool isPoint) const { // Currently we don't apply region coverer params to point, because it's useless. // Point always use level 30. if (isPoint) { @@ -166,7 +166,7 @@ std::vector GeoIndex::coveringCells(const S2Region& r, bool isPoint) c return covering; } -std::vector GeoIndex::ancestorCells(const std::vector& cells) const noexcept { +std::vector GeoIndex::ancestorCells(const std::vector& cells) const { // DCHECK(rc.IsCanonical(cells)); std::vector ancestors; std::unordered_set seen; diff --git a/src/common/geo/GeoIndex.h b/src/common/geo/GeoIndex.h index 3b1c7e618f4..cdd5032ed97 100644 --- a/src/common/geo/GeoIndex.h +++ b/src/common/geo/GeoIndex.h @@ -64,24 +64,24 @@ class GeoIndex { : rcParams_(params), pointsOnly_(pointsOnly) {} // Build geo index for the geography g - std::vector indexCells(const Geography& g) const noexcept; + std::vector indexCells(const Geography& g) const; // Query the geo index // ST_Intersects(g, x), x is the indexed geography column - std::vector intersects(const Geography& g) const noexcept; + std::vector intersects(const Geography& g) const; // ST_Covers(g, x), x is the indexed geography column - std::vector covers(const Geography& g) const noexcept; + std::vector covers(const Geography& g) const; // ST_CoveredBy(g, x), x is the indexed geography column - std::vector coveredBy(const Geography& g) const noexcept; + std::vector coveredBy(const Geography& g) const; // ST_Distance(g, x, distance), x is the indexed geography column - std::vector dWithin(const Geography& g, double distance) const noexcept; + std::vector dWithin(const Geography& g, double distance) const; private: - std::vector intersects(const S2Region& r, bool isPoint = false) const noexcept; + std::vector intersects(const S2Region& r, bool isPoint = false) const; - std::vector coveringCells(const S2Region& r, bool isPoint = false) const noexcept; + std::vector coveringCells(const S2Region& r, bool isPoint = false) const; - std::vector ancestorCells(const std::vector& cells) const noexcept; + std::vector ancestorCells(const std::vector& cells) const; private: RegionCoverParams rcParams_; diff --git a/src/common/memory/Memory.h b/src/common/memory/Memory.h index 418685900bd..c0a43ba2c17 100644 --- a/src/common/memory/Memory.h +++ b/src/common/memory/Memory.h @@ -105,11 +105,21 @@ inline ALWAYS_INLINE size_t getActualAllocationSize(size_t size, std::align_val_ } inline ALWAYS_INLINE void trackMemory(std::size_t size) { + std::size_t actual_size = getActualAllocationSize(size); + MemoryTracker::alloc(actual_size); +} + +inline ALWAYS_INLINE void trackMemoryNoThrow(std::size_t size) { std::size_t actual_size = getActualAllocationSize(size); MemoryTracker::allocNoThrow(actual_size); } inline ALWAYS_INLINE void trackMemory(std::size_t size, std::align_val_t align) { + std::size_t actual_size = getActualAllocationSize(size, align); + MemoryTracker::alloc(actual_size); +} + +inline ALWAYS_INLINE void trackMemoryNoThrow(std::size_t size, std::align_val_t align) { std::size_t actual_size = getActualAllocationSize(size, align); MemoryTracker::allocNoThrow(actual_size); } diff --git a/src/common/memory/MemoryTracker.cpp b/src/common/memory/MemoryTracker.cpp index a22a0aabf0d..cae9c756b72 100644 --- a/src/common/memory/MemoryTracker.cpp +++ b/src/common/memory/MemoryTracker.cpp @@ -41,8 +41,8 @@ bool MemoryTracker::isOn() { return MemoryStats::instance().throwOnMemoryExceeded(); } -void MemoryTracker::allocImpl(int64_t size, bool) { - MemoryStats::instance().alloc(size); +void MemoryTracker::allocImpl(int64_t size, bool throw_if_memory_exceeded) { + MemoryStats::instance().alloc(size, throw_if_memory_exceeded); } } // namespace memory diff --git a/src/common/memory/MemoryTracker.h b/src/common/memory/MemoryTracker.h index f42d5223ad9..a1c4a646c6a 100644 --- a/src/common/memory/MemoryTracker.h +++ b/src/common/memory/MemoryTracker.h @@ -63,7 +63,7 @@ class MemoryStats { } /// Inform size of memory allocation - inline ALWAYS_INLINE void alloc(int64_t size) { + inline ALWAYS_INLINE void alloc(int64_t size, bool throw_if_memory_exceeded) { int64_t willBe = threadMemoryStats_.reserved - size; if (UNLIKELY(willBe < 0)) { @@ -74,7 +74,7 @@ class MemoryStats { } // allocGlobal() may end with bad_alloc, only invoke allocGlobal() once (ALL_OR_NOTHING // semantic) - allocGlobal(getFromGlobal); + allocGlobal(getFromGlobal, throw_if_memory_exceeded); willBe += getFromGlobal; } // Only update after successful allocations, failed allocations should not be taken into @@ -103,12 +103,19 @@ class MemoryStats { /// Set limit (maximum usable bytes) of memory void setLimit(int64_t limit) { if (this->limit_ != limit) { - LOG(INFO) << fmt::format( + DLOG(INFO) << fmt::format( "MemoryTracker update limit {} -> {}", ReadableSize(this->limit_), ReadableSize(limit)); this->limit_ = limit; } } + /// update limit (maximum usable bytes) of memory + /// limit will be set to "used memory + available" + void updateLimit(int64_t available) { + int64_t newLimit = used_ + available; + setLimit(newLimit); + } + /// Get limit (maximum usable bytes) of memory int64_t getLimit() { return this->limit_; @@ -143,12 +150,17 @@ class MemoryStats { return threadMemoryStats_.throwOnMemoryExceeded; } + static bool setThrowOnMemoryExceeded(bool value) { + return threadMemoryStats_.throwOnMemoryExceeded = value; + } + private: - inline ALWAYS_INLINE void allocGlobal(int64_t size) { + inline ALWAYS_INLINE void allocGlobal(int64_t size, bool throw_if_memory_exceeded) { int64_t willBe = size + used_.fetch_add(size, std::memory_order_relaxed); - if (threadMemoryStats_.throwOnMemoryExceeded && willBe > limit_) { + if (threadMemoryStats_.throwOnMemoryExceeded && throw_if_memory_exceeded && willBe > limit_) { // revert used_.fetch_sub(size, std::memory_order_relaxed); + threadMemoryStats_.throwOnMemoryExceeded = false; throw std::bad_alloc(); } } @@ -165,13 +177,27 @@ class MemoryStats { // A guard to only enable memory check (throw when memory exceed) during its lifetime. struct MemoryCheckGuard { + bool previous; MemoryCheckGuard() { + previous = MemoryStats::throwOnMemoryExceeded(); MemoryStats::turnOnThrow(); } ~MemoryCheckGuard() { + MemoryStats::setThrowOnMemoryExceeded(previous); + } +}; + +struct MemoryCheckOffGuard { + bool previous; + MemoryCheckOffGuard() { + previous = MemoryStats::throwOnMemoryExceeded(); MemoryStats::turnOffThrow(); } + + ~MemoryCheckOffGuard() { + MemoryStats::setThrowOnMemoryExceeded(previous); + } }; // A global static memory tracker enable tracking every memory allocation and deallocation. diff --git a/src/common/memory/MemoryUtils.cpp b/src/common/memory/MemoryUtils.cpp index f8950fd3d21..52d832af937 100644 --- a/src/common/memory/MemoryUtils.cpp +++ b/src/common/memory/MemoryUtils.cpp @@ -6,6 +6,9 @@ #include "common/memory/MemoryUtils.h" #include + +#include + #if ENABLE_JEMALLOC #include @@ -55,7 +58,11 @@ DEFINE_double(memory_tracker_untracked_reserved_memory_mb, "memory (direct call malloc/free)"); DEFINE_double(memory_tracker_limit_ratio, 0.8, - "memory tacker usable memory ratio to (total_available - untracked_reserved_memory)"); + "memory tacker usable memory ratio to (total - untracked_reserved_memory)"); + +DEFINE_double(memory_tracker_available_ratio, + 0.8, + "memory tacker available memory ratio to (available - untracked_reserved_memory)"); using nebula::fs::FileUtils; @@ -70,6 +77,8 @@ static const std::regex reTotalCache(R"(^total_(cache|inactive_file)\s+(\d+)$)") std::atomic_bool MemoryUtils::kHitMemoryHighWatermark{false}; int64_t MemoryUtils::kLastPurge_{0}; int64_t MemoryUtils::kLastPrintMemoryTrackerStats_{0}; +int64_t MemoryUtils::kCurrentTotal_{0}; +double MemoryUtils::kCurrentLimitRatio_{0.0}; StatusOr MemoryUtils::hitsHighWatermark() { if (FLAGS_system_memory_high_watermark_ratio >= 1.0) { @@ -118,18 +127,77 @@ StatusOr MemoryUtils::hitsHighWatermark() { } } - // MemoryStats depends on jemalloc -#ifdef ENABLE_JEMALLOC -#ifndef ENABLE_ASAN - // set MemoryStats limit (MemoryTracker track-able memory) - int64_t trackable = total - FLAGS_memory_tracker_untracked_reserved_memory_mb * MiB; - if (trackable > 0) { - MemoryStats::instance().setLimit(trackable * FLAGS_memory_tracker_limit_ratio); + handleMemoryTracker(total, available); + + auto hits = (1 - available / total) > FLAGS_system_memory_high_watermark_ratio; + LOG_IF_EVERY_N(WARNING, hits, 100) + << "Memory usage has hit the high watermark of system, available: " << available + << " vs. total: " << total << " in bytes."; + return hits; +} + +StatusOr MemoryUtils::readSysContents(const std::string& path) { + std::ifstream ifs(path); + if (!ifs) { + return Status::Error("Could not open the file: %s", path.c_str()); + } + uint64_t value = 0; + ifs >> value; + return value; +} + +void MemoryUtils::handleMemoryTracker(int64_t total, int64_t available) { +#ifdef ENABLE_MEMORY_TRACKER + double limitRatio = FLAGS_memory_tracker_limit_ratio; + double availableRatio = FLAGS_memory_tracker_available_ratio; + + double untrackedMb = FLAGS_memory_tracker_untracked_reserved_memory_mb; + + // update MemoryTracker limit memory by Flags + if (limitRatio > 0.0 && limitRatio <= 1.0) { + /** + * (<= 1.0): Normal limit is set to ratio of trackable memory; + * limit = limitRatio * (total - untrackedMb) + */ + if (kCurrentTotal_ != total || kCurrentLimitRatio_ != limitRatio) { + double trackable = total - (untrackedMb * MiB); + if (trackable > 0) { + MemoryStats::instance().setLimit(trackable * limitRatio); + } else { + LOG(ERROR) << "Total memory less than untracked " << untrackedMb << " Mib," + << " MemoryTracker limit is not set properly, " + << " Current memory stats:" << MemoryStats::instance().toString(); + } + LOG(INFO) << "MemoryTracker set static ratio: " << limitRatio; + } + } else if (std::fabs(limitRatio - 2.0) < std::numeric_limits::epsilon()) { + /** + * (== 2.0): Special value for dynamic self adaptive; + * limit = current_used_memory + (available - untracked) * availableRatio + */ + double trackableAvailable = available - (untrackedMb * MiB); + if (trackableAvailable > 0) { + MemoryStats::instance().updateLimit(trackableAvailable * availableRatio); + } else { + MemoryStats::instance().updateLimit(0); + } + DLOG(INFO) << "MemoryTracker set to dynamic self adaptive"; + } else if (std::fabs(limitRatio - 3.0) < std::numeric_limits::epsilon()) { + /** + * (== 3.0): Special value for disable memory_tracker; + * limit is set to max + */ + if (kCurrentLimitRatio_ != limitRatio) { + MemoryStats::instance().setLimit(std::numeric_limits::max()); + LOG(INFO) << "MemoryTracker disabled"; + } } else { - // Do not set limit, keep previous set limit or default limit - LOG(ERROR) << "Total available memory less than " - << FLAGS_memory_tracker_untracked_reserved_memory_mb << " Mib"; + if (kCurrentTotal_ != limitRatio) { + LOG(ERROR) << "Invalid memory_tracker_limit_ratio: " << limitRatio; + } } + kCurrentTotal_ = total; + kCurrentLimitRatio_ = limitRatio; // purge if enabled if (FLAGS_memory_purge_enabled) { @@ -142,24 +210,26 @@ StatusOr MemoryUtils::hitsHighWatermark() { } } - // print system & application level memory stats - // sys: read from system environment, varies depends on environment: - // container: controlled by cgroup, - // used: read from memory.current in cgroup path - // total: read from memory.max in cgroup path - // physical machine: judge by system level memory consumption - // used: current used memory of the system - // total: all physical memory installed - // usr: record by current process's MemoryStats - // used: bytes allocated by new operator - // total: sys_total * FLAGS_system_memory_high_watermark_ratio + /** + * print system & application level memory stats + * sys: read from system environment, varies depends on environment: + * container: controlled by cgroup, + * used: read from memory.current in cgroup path + * total: read from memory.max in cgroup path + * physical machine: judge by system level memory consumption + * used: current used memory of the system + * total: all physical memory installed + * usr: record by current process's MemoryStats + * used: bytes allocated by new operator + * total: sys_total * FLAGS_system_memory_high_watermark_ratio + */ int64_t now = time::WallClock::fastNowInMilliSec(); if (FLAGS_memory_tracker_detail_log) { if (now - kLastPrintMemoryTrackerStats_ >= FLAGS_memory_tracker_detail_log_interval_ms) { LOG(INFO) << fmt::format("sys:{}/{} {:.2f}%", - ReadableSize(static_cast(total - available)), + ReadableSize(total - available), ReadableSize(total), - (1 - available / total) * 100) + (1 - available / static_cast(total)) * 100) << fmt::format(" usr:{}/{} {:.2f}%", ReadableSize(MemoryStats::instance().used()), ReadableSize(MemoryStats::instance().getLimit()), @@ -167,25 +237,12 @@ StatusOr MemoryUtils::hitsHighWatermark() { kLastPrintMemoryTrackerStats_ = now; } } +#else + LOG_FIRST_N(WARNING, 1) << "WARNING: MemoryTracker was disabled at compile time"; + UNUSED(total); + UNUSED(available); #endif -#endif - - auto hits = (1 - available / total) > FLAGS_system_memory_high_watermark_ratio; - LOG_IF_EVERY_N(WARNING, hits, 100) - << "Memory usage has hit the high watermark of system, available: " << available - << " vs. total: " << total << " in bytes."; - return hits; -} - -StatusOr MemoryUtils::readSysContents(const std::string& path) { - std::ifstream ifs(path); - if (!ifs) { - return Status::Error("Could not open the file: %s", path.c_str()); - } - uint64_t value = 0; - ifs >> value; - return value; -} +} // namespace memory } // namespace memory } // namespace nebula diff --git a/src/common/memory/MemoryUtils.h b/src/common/memory/MemoryUtils.h index 90a471bb2c3..1fdf09d1c9b 100644 --- a/src/common/memory/MemoryUtils.h +++ b/src/common/memory/MemoryUtils.h @@ -33,9 +33,15 @@ class MemoryUtils final { static StatusOr readSysContents(const std::string &path); + // Handle memory tracker related logic by inform system's total memory and current available + // memory in bytes + static void handleMemoryTracker(int64_t total, int64_t available); + private: static int64_t kLastPurge_; static int64_t kLastPrintMemoryTrackerStats_; + static int64_t kCurrentTotal_; + static double kCurrentLimitRatio_; }; } // namespace memory diff --git a/src/common/memory/NewDelete.cpp b/src/common/memory/NewDelete.cpp index 5bc67076553..930cfe1f8e0 100644 --- a/src/common/memory/NewDelete.cpp +++ b/src/common/memory/NewDelete.cpp @@ -7,18 +7,12 @@ #include "common/memory/Memory.h" /// Replace default new/delete with memory tracking versions. -/// -/// Two condition need check before MemoryTracker is on +/// ENABLE_MEMORY_TRACKER is defined by cmake only following two condition satisfied: /// 1. jemalloc is used /// MemoryTracker need jemalloc API to get accurate size of alloc/free memory. /// 2. address_sanitizer is off /// sanitizer has already override the new/delete operator, /// only override new/delete operator only when address_sanitizer is off -#ifdef ENABLE_JEMALLOC -#ifndef ENABLE_ASAN -#define ENABLE_MEMORY_TRACKER -#endif -#endif #ifdef ENABLE_MEMORY_TRACKER /// new @@ -43,22 +37,22 @@ void *operator new[](std::size_t size, std::align_val_t align) { } void *operator new(std::size_t size, const std::nothrow_t &) noexcept { - nebula::memory::trackMemory(size); + nebula::memory::trackMemoryNoThrow(size); return nebula::memory::newNoException(size); } void *operator new[](std::size_t size, const std::nothrow_t &) noexcept { - nebula::memory::trackMemory(size); + nebula::memory::trackMemoryNoThrow(size); return nebula::memory::newNoException(size); } void *operator new(std::size_t size, std::align_val_t align, const std::nothrow_t &) noexcept { - nebula::memory::trackMemory(size, align); + nebula::memory::trackMemoryNoThrow(size, align); return nebula::memory::newNoException(size, align); } void *operator new[](std::size_t size, std::align_val_t align, const std::nothrow_t &) noexcept { - nebula::memory::trackMemory(size, align); + nebula::memory::trackMemoryNoThrow(size, align); return nebula::memory::newNoException(size, align); } @@ -103,4 +97,6 @@ void operator delete[](void *ptr, std::size_t size, std::align_val_t align) noex nebula::memory::deleteSized(ptr, size, align); } +#else +#pragma message("WARNING: MemoryTracker was disabled at compile time") #endif diff --git a/src/graph/context/ExecutionContext.cpp b/src/graph/context/ExecutionContext.cpp index 1f1baae0429..bca879a329b 100644 --- a/src/graph/context/ExecutionContext.cpp +++ b/src/graph/context/ExecutionContext.cpp @@ -21,12 +21,16 @@ void ExecutionContext::setValue(const std::string& name, Value&& val) { } void ExecutionContext::setResult(const std::string& name, Result&& result) { + folly::RWSpinLock::WriteHolder holder(lock_); auto& hist = valueMap_[name]; hist.emplace_back(std::move(result)); } void ExecutionContext::dropResult(const std::string& name) { - DCHECK_EQ(valueMap_.count(name), 1); + folly::RWSpinLock::WriteHolder holder(lock_); + if (valueMap_.count(name) == 0) { + return; + } auto& val = valueMap_[name]; if (FLAGS_enable_async_gc) { GC::instance().clear(std::move(val)); @@ -36,6 +40,7 @@ void ExecutionContext::dropResult(const std::string& name) { } size_t ExecutionContext::numVersions(const std::string& name) const { + folly::RWSpinLock::ReadHolder holder(lock_); auto it = valueMap_.find(name); CHECK(it != valueMap_.end()); return it->second.size(); @@ -43,6 +48,7 @@ size_t ExecutionContext::numVersions(const std::string& name) const { // Only keep the last several versions of the Value void ExecutionContext::truncHistory(const std::string& name, size_t numVersionsToKeep) { + folly::RWSpinLock::WriteHolder holder(lock_); auto it = valueMap_.find(name); if (it != valueMap_.end()) { if (it->second.size() <= numVersionsToKeep) { @@ -59,6 +65,7 @@ const Value& ExecutionContext::getValue(const std::string& name) const { } Value ExecutionContext::moveValue(const std::string& name) { + folly::RWSpinLock::WriteHolder holder(lock_); auto it = valueMap_.find(name); if (it != valueMap_.end() && !it->second.empty()) { return it->second.back().moveValue(); @@ -68,6 +75,7 @@ Value ExecutionContext::moveValue(const std::string& name) { } const Result& ExecutionContext::getResult(const std::string& name) const { + folly::RWSpinLock::ReadHolder holder(lock_); auto it = valueMap_.find(name); if (it != valueMap_.end() && !it->second.empty()) { return it->second.back(); @@ -79,6 +87,7 @@ const Result& ExecutionContext::getResult(const std::string& name) const { void ExecutionContext::setVersionedResult(const std::string& name, Result&& result, int64_t version) { + folly::RWSpinLock::WriteHolder holder(lock_); auto it = valueMap_.find(name); if (it != valueMap_.end()) { auto& hist = it->second; @@ -101,6 +110,7 @@ const Result& ExecutionContext::getVersionedResult(const std::string& name, int6 } const std::vector& ExecutionContext::getHistory(const std::string& name) const { + folly::RWSpinLock::ReadHolder holder(lock_); auto it = valueMap_.find(name); if (it != valueMap_.end()) { return it->second; diff --git a/src/graph/context/ExecutionContext.h b/src/graph/context/ExecutionContext.h index d34acddc6a9..64cc02dfd5e 100644 --- a/src/graph/context/ExecutionContext.h +++ b/src/graph/context/ExecutionContext.h @@ -41,6 +41,7 @@ class ExecutionContext { virtual ~ExecutionContext() = default; void initVar(const std::string& name) { + folly::RWSpinLock::WriteHolder holder(lock_); valueMap_[name]; } @@ -69,6 +70,7 @@ class ExecutionContext { void truncHistory(const std::string& name, size_t numVersionsToKeep); bool exist(const std::string& name) const { + folly::RWSpinLock::ReadHolder holder(lock_); return valueMap_.find(name) != valueMap_.end(); } @@ -77,6 +79,7 @@ class ExecutionContext { Value moveValue(const std::string& name); // name -> Value with multiple versions + mutable folly::RWSpinLock lock_; std::unordered_map> valueMap_; }; diff --git a/src/graph/context/Symbols.cpp b/src/graph/context/Symbols.cpp index 8e2c1b60969..77602ed3f8d 100644 --- a/src/graph/context/Symbols.cpp +++ b/src/graph/context/Symbols.cpp @@ -22,6 +22,7 @@ std::string Variable::toString() const { } std::string SymbolTable::toString() const { + folly::RWSpinLock::ReadHolder holder(lock_); std::stringstream ss; ss << "SymTable: ["; for (const auto& p : vars_) { @@ -35,12 +36,19 @@ std::string SymbolTable::toString() const { } bool SymbolTable::existsVar(const std::string& varName) const { + folly::RWSpinLock::ReadHolder holder(lock_); return vars_.find(varName) != vars_.end(); } Variable* SymbolTable::newVariable(const std::string& name) { - VLOG(1) << "New variable for: " << name; - DCHECK(vars_.find(name) == vars_.end()); +#ifdef NDEBUG + { + // addVar has a write lock, should warp this read lock block + folly::RWSpinLock::ReadHolder holder(lock_); + VLOG(1) << "New variable for: " << name; + DCHECK(vars_.find(name) == vars_.end()); + } +#endif auto* variable = objPool_->makeAndAdd(name); addVar(name, variable); // Initialize all variable in variable map (output of node, inner variable etc.) @@ -50,10 +58,12 @@ Variable* SymbolTable::newVariable(const std::string& name) { } void SymbolTable::addVar(std::string varName, Variable* variable) { + folly::RWSpinLock::WriteHolder holder(lock_); vars_.emplace(std::move(varName), variable); } bool SymbolTable::readBy(const std::string& varName, PlanNode* node) { + folly::RWSpinLock::WriteHolder holder(lock_); auto var = vars_.find(varName); if (var == vars_.end()) { return false; @@ -63,6 +73,7 @@ bool SymbolTable::readBy(const std::string& varName, PlanNode* node) { } bool SymbolTable::writtenBy(const std::string& varName, PlanNode* node) { + folly::RWSpinLock::WriteHolder holder(lock_); auto var = vars_.find(varName); if (var == vars_.end()) { return false; @@ -72,6 +83,7 @@ bool SymbolTable::writtenBy(const std::string& varName, PlanNode* node) { } bool SymbolTable::deleteReadBy(const std::string& varName, PlanNode* node) { + folly::RWSpinLock::WriteHolder holder(lock_); auto var = vars_.find(varName); if (var == vars_.end()) { return false; @@ -81,6 +93,7 @@ bool SymbolTable::deleteReadBy(const std::string& varName, PlanNode* node) { } bool SymbolTable::deleteWrittenBy(const std::string& varName, PlanNode* node) { + folly::RWSpinLock::WriteHolder holder(lock_); auto var = vars_.find(varName); if (var == vars_.end()) { return false; @@ -102,6 +115,7 @@ bool SymbolTable::updateWrittenBy(const std::string& oldVar, } Variable* SymbolTable::getVar(const std::string& varName) { + folly::RWSpinLock::ReadHolder holder(lock_); DCHECK(!varName.empty()) << "the variable name is empty"; auto var = vars_.find(varName); if (var == vars_.end()) { diff --git a/src/graph/context/Symbols.h b/src/graph/context/Symbols.h index 84a497e11ba..3ced502d418 100644 --- a/src/graph/context/Symbols.h +++ b/src/graph/context/Symbols.h @@ -6,6 +6,7 @@ #ifndef GRAPH_CONTEXT_SYMBOLS_H_ #define GRAPH_CONTEXT_SYMBOLS_H_ +#include #include #include @@ -82,6 +83,7 @@ class SymbolTable final { ObjectPool* objPool_{nullptr}; ExecutionContext* ectx_{nullptr}; // var name -> variable + mutable folly::RWSpinLock lock_; std::unordered_map vars_; }; diff --git a/src/graph/executor/admin/SwitchSpaceExecutor.cpp b/src/graph/executor/admin/SwitchSpaceExecutor.cpp index d777742ab7e..f7a22d04fac 100644 --- a/src/graph/executor/admin/SwitchSpaceExecutor.cpp +++ b/src/graph/executor/admin/SwitchSpaceExecutor.cpp @@ -5,6 +5,7 @@ #include "graph/executor/admin/SwitchSpaceExecutor.h" #include "clients/meta/MetaClient.h" +#include "common/memory/MemoryTracker.h" #include "graph/planner/plan/Query.h" #include "graph/service/PermissionManager.h" #include "interface/gen-cpp2/meta_types.h" @@ -13,6 +14,8 @@ namespace nebula { namespace graph { folly::Future SwitchSpaceExecutor::execute() { + // TODO: need check if this MemoryCheckOff is necessary + memory::MemoryCheckOffGuard guard; SCOPED_TIMER(&execTime_); auto *spaceToNode = asNode(node()); @@ -25,6 +28,11 @@ folly::Future SwitchSpaceExecutor::execute() { } auto spaceId = resp.value().get_space_id(); + // SwitchSpace can be mixed with normal query, if the corresponding query failed by other + // Executors, QueryContext may already be released. + if (!qctx() || !qctx()->rctx() || qctx_->rctx()->session() == nullptr) { + return Status::Error("Session not found"); + } auto *session = qctx_->rctx()->session(); NG_RETURN_IF_ERROR(PermissionManager::canReadSpace(session, spaceId)); const auto &properties = resp.value().get_properties(); diff --git a/src/graph/executor/algo/BFSShortestPathExecutor.cpp b/src/graph/executor/algo/BFSShortestPathExecutor.cpp index 098ad840df5..b098145fbe4 100644 --- a/src/graph/executor/algo/BFSShortestPathExecutor.cpp +++ b/src/graph/executor/algo/BFSShortestPathExecutor.cpp @@ -11,7 +11,6 @@ namespace nebula { namespace graph { folly::Future BFSShortestPathExecutor::execute() { // MemoryTrackerVerified - SCOPED_TIMER(&execTime_); pathNode_ = asNode(node()); terminateEarlyVar_ = pathNode_->terminateEarlyVar(); diff --git a/src/graph/gc/GC.cpp b/src/graph/gc/GC.cpp index de3428861a0..0d6b2899c52 100644 --- a/src/graph/gc/GC.cpp +++ b/src/graph/gc/GC.cpp @@ -23,12 +23,15 @@ GC::GC() { } void GC::clear(std::vector&& garbage) { + memory::MemoryCheckOffGuard guard; + // do not bother folly queue_.enqueue(std::move(garbage)); } void GC::periodicTask() { - // TODO: maybe could release by batch - queue_.try_dequeue(); + while (!queue_.empty()) { + queue_.try_dequeue(); + } } } // namespace graph } // namespace nebula diff --git a/src/graph/optimizer/rule/IndexScanRule.cpp b/src/graph/optimizer/rule/IndexScanRule.cpp index badab2b4ac6..bae416e36c8 100644 --- a/src/graph/optimizer/rule/IndexScanRule.cpp +++ b/src/graph/optimizer/rule/IndexScanRule.cpp @@ -144,7 +144,7 @@ Status IndexScanRule::createMultipleIQC(IndexQueryCtx& iqctx, return Status::OK(); } -size_t IndexScanRule::hintCount(const FilterItems& items) const noexcept { +size_t IndexScanRule::hintCount(const FilterItems& items) const { std::unordered_set hintCols; for (const auto& i : items.items) { hintCols.emplace(i.col_); diff --git a/src/graph/optimizer/rule/IndexScanRule.h b/src/graph/optimizer/rule/IndexScanRule.h index 8f8fb016c7d..2a334b84405 100644 --- a/src/graph/optimizer/rule/IndexScanRule.h +++ b/src/graph/optimizer/rule/IndexScanRule.h @@ -121,7 +121,7 @@ class IndexScanRule final : public OptRule { const FilterItems& items, const meta::cpp2::ColumnDef& col) const; - size_t hintCount(const FilterItems& items) const noexcept; + size_t hintCount(const FilterItems& items) const; bool isEdge(const OptGroupNode* groupNode) const; diff --git a/src/graph/scheduler/AsyncMsgNotifyBasedScheduler.cpp b/src/graph/scheduler/AsyncMsgNotifyBasedScheduler.cpp index d2d121378bf..5c869a0ab0a 100644 --- a/src/graph/scheduler/AsyncMsgNotifyBasedScheduler.cpp +++ b/src/graph/scheduler/AsyncMsgNotifyBasedScheduler.cpp @@ -15,6 +15,19 @@ AsyncMsgNotifyBasedScheduler::AsyncMsgNotifyBasedScheduler(QueryContext* qctx) : query_ = qctx->rctx()->query(); } +void AsyncMsgNotifyBasedScheduler::waitFinish() { + std::unique_lock lck(emtx_); + cv_.wait(lck, [this] { + if (executing_ != 0) { + DLOG(INFO) << "executing: " << executing_; + return false; + } else { + DLOG(INFO) << " wait finish"; + return true; + } + }); +} + folly::Future AsyncMsgNotifyBasedScheduler::schedule() { auto root = qctx_->plan()->root(); if (FLAGS_enable_lifetime_optimize) { @@ -24,6 +37,7 @@ folly::Future AsyncMsgNotifyBasedScheduler::schedule() { analyzeLifetime(root); } auto executor = Executor::create(root, qctx_); + DLOG(INFO) << formatPrettyDependencyTree(executor); return doSchedule(executor); } @@ -90,7 +104,7 @@ folly::Future AsyncMsgNotifyBasedScheduler::doSchedule(Executor* root) c return folly::makeFuture(Status::GraphMemoryExceeded( "(%d)", static_cast(nebula::cpp2::ErrorCode::E_GRAPH_MEMORY_EXCEEDED))); }) - .thenTry([this, pros = std::move(currentExePromises)](auto&& t) mutable { + .thenTry([this, exe, pros = std::move(currentExePromises)](auto&& t) mutable { // any exception or status not ok handled with notifyError if (t.hasException()) { notifyError(pros, Status::Error(std::move(t).exception().what())); @@ -99,6 +113,8 @@ folly::Future AsyncMsgNotifyBasedScheduler::doSchedule(Executor* root) c if (v.ok()) { notifyOK(pros); } else { + DLOG(INFO) << "[" << exe->name() << "," << exe->id() << "]" + << " fail with: " << v.toString(); notifyError(pros, v); } } @@ -143,7 +159,6 @@ folly::Future AsyncMsgNotifyBasedScheduler::runSelect( return execute(select); }) .thenValue([select, this](auto&& selectStatus) mutable -> folly::Future { - memory::MemoryCheckGuard guard; NG_RETURN_IF_ERROR(selectStatus); auto val = qctx_->ectx()->getValue(select->node()->outputVar()); if (!val.isBool()) { @@ -185,7 +200,6 @@ folly::Future AsyncMsgNotifyBasedScheduler::runLoop( return execute(loop); }) .thenValue([loop, runner, this](auto&& loopStatus) mutable -> folly::Future { - memory::MemoryCheckGuard guard; NG_RETURN_IF_ERROR(loopStatus); auto val = qctx_->ectx()->getValue(loop->node()->outputVar()); if (!val.isBool()) { @@ -225,17 +239,94 @@ void AsyncMsgNotifyBasedScheduler::notifyError(std::vector AsyncMsgNotifyBasedScheduler::execute(Executor* executor) const { - memory::MemoryCheckGuard guard1; auto status = executor->open(); if (!status.ok()) { return executor->error(std::move(status)); } - return executor->execute().thenValue([executor](Status s) { - memory::MemoryCheckGuard guard2; - NG_RETURN_IF_ERROR(s); - return executor->close(); + + auto exeStatus = runExecute(executor); + + return std::move(exeStatus).thenValue([this, executor](Status s) { + if (!s.ok()) { + DLOG(INFO) << formatPrettyId(executor) << " failed with: " << s.toString(); + setFailStatus(s); + removeExecuting(executor); + return Status::from(s); + } + auto ret = executor->close(); + removeExecuting(executor); + return ret; }); } +folly::Future AsyncMsgNotifyBasedScheduler::runExecute(Executor* executor) const { + // catch Executor::execute here, upward call stack should only get Status, no exceptions. + try { + addExecuting(executor); + if (hasFailStatus()) return failedStatus_.value(); + folly::Future status = Status::OK(); + { + memory::MemoryCheckGuard guard; + status = executor->execute(); + } + return std::move(status).thenError(folly::tag_t{}, [](const std::bad_alloc&) { + return folly::makeFuture(Status::GraphMemoryExceeded( + "(%d)", static_cast(nebula::cpp2::ErrorCode::E_GRAPH_MEMORY_EXCEEDED))); + }); + } catch (std::bad_alloc& e) { + return folly::makeFuture(Status::GraphMemoryExceeded( + "(%d)", static_cast(nebula::cpp2::ErrorCode::E_GRAPH_MEMORY_EXCEEDED))); + } catch (std::exception& e) { + return folly::makeFuture(Status::Error("%s", e.what())); + } catch (...) { + return folly::makeFuture(Status::Error("unknown error")); + } +} + +void AsyncMsgNotifyBasedScheduler::addExecuting(Executor* executor) const { + std::unique_lock lck(emtx_); + executing_++; + DLOG(INFO) << formatPrettyId(executor) << " add " << executing_; +} + +void AsyncMsgNotifyBasedScheduler::removeExecuting(Executor* executor) const { + std::unique_lock lck(emtx_); + executing_--; + DLOG(INFO) << formatPrettyId(executor) << "remove: " << executing_; + cv_.notify_one(); +} + +void AsyncMsgNotifyBasedScheduler::setFailStatus(Status status) const { + std::unique_lock lck(smtx_); + if (!failedStatus_.has_value()) { + failedStatus_ = status; + } +} + +bool AsyncMsgNotifyBasedScheduler::hasFailStatus() const { + std::unique_lock lck(smtx_); + return failedStatus_.has_value(); +} + +std::string AsyncMsgNotifyBasedScheduler::formatPrettyId(Executor* executor) { + return fmt::format("[{},{}]", executor->name(), executor->id()); +} + +std::string AsyncMsgNotifyBasedScheduler::formatPrettyDependencyTree(Executor* root) { + std::stringstream ss; + size_t spaces = 0; + appendExecutor(spaces, root, ss); + return ss.str(); +} + +void AsyncMsgNotifyBasedScheduler::appendExecutor(size_t spaces, + Executor* executor, + std::stringstream& ss) { + ss << std::string(spaces, ' ') << formatPrettyId(executor) << std::endl; + for (auto depend : executor->depends()) { + appendExecutor(spaces + 1, depend, ss); + } +} + } // namespace graph } // namespace nebula diff --git a/src/graph/scheduler/AsyncMsgNotifyBasedScheduler.h b/src/graph/scheduler/AsyncMsgNotifyBasedScheduler.h index 5698d2fcce3..a9788fe8c15 100644 --- a/src/graph/scheduler/AsyncMsgNotifyBasedScheduler.h +++ b/src/graph/scheduler/AsyncMsgNotifyBasedScheduler.h @@ -27,6 +27,8 @@ class AsyncMsgNotifyBasedScheduler final : public Scheduler { folly::Future schedule() override; + void waitFinish() override; + private: folly::Future doSchedule(Executor* root) const; @@ -62,6 +64,29 @@ class AsyncMsgNotifyBasedScheduler final : public Scheduler { folly::Future execute(Executor* executor) const; + folly::Future runExecute(Executor* executor) const; + + void addExecuting(Executor* executor) const; + + void removeExecuting(Executor* executor) const; + + void setFailStatus(Status status) const; + + bool hasFailStatus() const; + + static std::string formatPrettyId(Executor* executor); + + static std::string formatPrettyDependencyTree(Executor* root); + + static void appendExecutor(size_t tabs, Executor* executor, std::stringstream& ss); + + private: + // set if some Executor failed, then other Executors no need to do Executor::execute() further + mutable std::mutex smtx_; + mutable std::optional failedStatus_; + mutable std::mutex emtx_; + mutable std::condition_variable cv_; + mutable size_t executing_{0}; QueryContext* qctx_{nullptr}; }; } // namespace graph diff --git a/src/graph/scheduler/Scheduler.h b/src/graph/scheduler/Scheduler.h index 43e765de292..34b13b8bbca 100644 --- a/src/graph/scheduler/Scheduler.h +++ b/src/graph/scheduler/Scheduler.h @@ -22,6 +22,8 @@ class Scheduler : private boost::noncopyable, private cpp::NonMovable { virtual folly::Future schedule() = 0; + virtual void waitFinish() = 0; + static void analyzeLifetime(const PlanNode *node, std::size_t loopLayers = 0); protected: diff --git a/src/graph/service/QueryEngine.cpp b/src/graph/service/QueryEngine.cpp index 4f6927dbcd1..f7045f7fd50 100644 --- a/src/graph/service/QueryEngine.cpp +++ b/src/graph/service/QueryEngine.cpp @@ -46,7 +46,6 @@ Status QueryEngine::init(std::shared_ptr ioExecutor // Create query context and query instance and execute it void QueryEngine::execute(RequestContextPtr rctx) { - memory::MemoryCheckGuard guard; auto qctx = std::make_unique(std::move(rctx), schemaManager_.get(), indexManager_.get(), diff --git a/src/graph/service/QueryInstance.cpp b/src/graph/service/QueryInstance.cpp index 9e4d9f6406c..cc1f6cd457b 100644 --- a/src/graph/service/QueryInstance.cpp +++ b/src/graph/service/QueryInstance.cpp @@ -38,7 +38,6 @@ QueryInstance::QueryInstance(std::unique_ptr qctx, Optimizer *opti void QueryInstance::execute() { try { - memory::MemoryCheckGuard guard1; Status status = validateAndOptimize(); if (!status.ok()) { onError(std::move(status)); @@ -55,7 +54,6 @@ void QueryInstance::execute() { // series of Executors through the Scheduler to drive the execution of the Executors. scheduler_->schedule() .thenValue([this](Status s) { - memory::MemoryCheckGuard guard2; if (s.ok()) { this->onFinish(); } else { @@ -139,6 +137,7 @@ void QueryInstance::onFinish() { rctx->finish(); rctx->session()->deleteQuery(qctx_.get()); + scheduler_->waitFinish(); // The `QueryInstance' is the root node holding all resources during the // execution. When the whole query process is done, it's safe to release this // object, as long as no other contexts have chances to access these resources diff --git a/src/graph/service/RequestContext.h b/src/graph/service/RequestContext.h index 80348480680..4f659ac96d6 100644 --- a/src/graph/service/RequestContext.h +++ b/src/graph/service/RequestContext.h @@ -55,7 +55,7 @@ class RequestContext final : public boost::noncopyable, public cpp::NonMovable { } void setSession(std::shared_ptr session) { - session_ = std::move(session); + session_ = session; if (session_ != nullptr) { // Keep the session active session_->charge(); diff --git a/src/kvstore/RocksEngine.cpp b/src/kvstore/RocksEngine.cpp index eadacf20092..94bab5723eb 100644 --- a/src/kvstore/RocksEngine.cpp +++ b/src/kvstore/RocksEngine.cpp @@ -143,6 +143,7 @@ nebula::cpp2::ErrorCode RocksEngine::commitBatchWrite(std::unique_ptr(snapshot); @@ -161,6 +162,7 @@ nebula::cpp2::ErrorCode RocksEngine::get(const std::string& key, std::vector RocksEngine::multiGet(const std::vector& keys, std::vector* values) { + memory::MemoryCheckOffGuard guard; rocksdb::ReadOptions options; std::vector slices; for (size_t index = 0; index < keys.size(); index++) { @@ -184,23 +186,25 @@ std::vector RocksEngine::multiGet(const std::vector& keys, nebula::cpp2::ErrorCode RocksEngine::range(const std::string& start, const std::string& end, std::unique_ptr* storageIter) { + memory::MemoryCheckOffGuard guard; rocksdb::ReadOptions options; if (!isPlainTable_) { options.total_order_seek = FLAGS_enable_rocksdb_prefix_filtering; } else { options.prefix_same_as_start = true; } - rocksdb::Iterator* iter = db_->NewIterator(options); + std::unique_ptr iter(db_->NewIterator(options)); if (iter) { iter->Seek(rocksdb::Slice(start)); } - storageIter->reset(new RocksRangeIter(iter, start, end)); + storageIter->reset(new RocksRangeIter(std::move(iter), start, end)); return nebula::cpp2::ErrorCode::SUCCEEDED; } nebula::cpp2::ErrorCode RocksEngine::prefix(const std::string& prefix, std::unique_ptr* storageIter, const void* snapshot) { + memory::MemoryCheckOffGuard guard; // In fact, we don't need to check prefix.size() >= extractorLen_, which is caller's duty to make // sure the prefix bloom filter exists. But this is quite error-prone, so we do a check here. if (FLAGS_enable_rocksdb_prefix_filtering && prefix.size() >= extractorLen_) { @@ -213,58 +217,62 @@ nebula::cpp2::ErrorCode RocksEngine::prefix(const std::string& prefix, nebula::cpp2::ErrorCode RocksEngine::prefixWithExtractor(const std::string& prefix, const void* snapshot, std::unique_ptr* storageIter) { + memory::MemoryCheckOffGuard guard; rocksdb::ReadOptions options; if (UNLIKELY(snapshot != nullptr)) { options.snapshot = reinterpret_cast(snapshot); } options.prefix_same_as_start = true; - rocksdb::Iterator* iter = db_->NewIterator(options); + std::unique_ptr iter(db_->NewIterator(options)); if (iter) { iter->Seek(rocksdb::Slice(prefix)); } - storageIter->reset(new RocksPrefixIter(iter, prefix)); + storageIter->reset(new RocksPrefixIter(std::move(iter), prefix)); return nebula::cpp2::ErrorCode::SUCCEEDED; } nebula::cpp2::ErrorCode RocksEngine::prefixWithoutExtractor( const std::string& prefix, const void* snapshot, std::unique_ptr* storageIter) { + memory::MemoryCheckOffGuard guard; rocksdb::ReadOptions options; if (snapshot != nullptr) { options.snapshot = reinterpret_cast(snapshot); } // prefix_same_as_start is false by default options.total_order_seek = FLAGS_enable_rocksdb_prefix_filtering; - rocksdb::Iterator* iter = db_->NewIterator(options); + std::unique_ptr iter(db_->NewIterator(options)); if (iter) { iter->Seek(rocksdb::Slice(prefix)); } - storageIter->reset(new RocksPrefixIter(iter, prefix)); + storageIter->reset(new RocksPrefixIter(std::move(iter), prefix)); return nebula::cpp2::ErrorCode::SUCCEEDED; } nebula::cpp2::ErrorCode RocksEngine::rangeWithPrefix(const std::string& start, const std::string& prefix, std::unique_ptr* storageIter) { + memory::MemoryCheckOffGuard guard; rocksdb::ReadOptions options; if (!isPlainTable_) { options.total_order_seek = FLAGS_enable_rocksdb_prefix_filtering; } else { options.prefix_same_as_start = true; } - rocksdb::Iterator* iter = db_->NewIterator(options); + std::unique_ptr iter(db_->NewIterator(options)); if (iter) { iter->Seek(rocksdb::Slice(start)); } - storageIter->reset(new RocksPrefixIter(iter, prefix)); + storageIter->reset(new RocksPrefixIter(std::move(iter), prefix)); return nebula::cpp2::ErrorCode::SUCCEEDED; } nebula::cpp2::ErrorCode RocksEngine::scan(std::unique_ptr* storageIter) { + memory::MemoryCheckOffGuard guard; rocksdb::ReadOptions options; options.total_order_seek = true; - rocksdb::Iterator* iter = db_->NewIterator(options); + std::unique_ptr iter(db_->NewIterator(options)); iter->SeekToFirst(); - storageIter->reset(new RocksCommonIter(iter)); + storageIter->reset(new RocksCommonIter(std::move(iter))); return nebula::cpp2::ErrorCode::SUCCEEDED; } diff --git a/src/kvstore/RocksEngine.h b/src/kvstore/RocksEngine.h index e4be3f1a5b9..1a529686ffd 100644 --- a/src/kvstore/RocksEngine.h +++ b/src/kvstore/RocksEngine.h @@ -11,6 +11,8 @@ #include #include +#include + #include "common/base/Base.h" #include "kvstore/KVEngine.h" #include "kvstore/KVIterator.h" @@ -27,6 +29,9 @@ class RocksRangeIter : public KVIterator { RocksRangeIter(rocksdb::Iterator* iter, rocksdb::Slice start, rocksdb::Slice end) : iter_(iter), start_(start), end_(end) {} + RocksRangeIter(std::unique_ptr iter, rocksdb::Slice start, rocksdb::Slice end) + : iter_(std::move(iter)), start_(start), end_(end) {} + ~RocksRangeIter() = default; bool valid() const override { @@ -62,6 +67,9 @@ class RocksPrefixIter : public KVIterator { public: RocksPrefixIter(rocksdb::Iterator* iter, rocksdb::Slice prefix) : iter_(iter), prefix_(prefix) {} + RocksPrefixIter(std::unique_ptr iter, rocksdb::Slice prefix) + : iter_(std::move(iter)), prefix_(prefix) {} + ~RocksPrefixIter() = default; bool valid() const override { @@ -96,6 +104,8 @@ class RocksCommonIter : public KVIterator { public: explicit RocksCommonIter(rocksdb::Iterator* iter) : iter_(iter) {} + explicit RocksCommonIter(std::unique_ptr iter) : iter_(std::move(iter)) {} + ~RocksCommonIter() = default; bool valid() const override { diff --git a/src/kvstore/listener/elasticsearch/ESListener.cpp b/src/kvstore/listener/elasticsearch/ESListener.cpp index 92435e6ec22..1f40cba4a22 100644 --- a/src/kvstore/listener/elasticsearch/ESListener.cpp +++ b/src/kvstore/listener/elasticsearch/ESListener.cpp @@ -220,9 +220,7 @@ bool ESListener::writeAppliedId(LogID lastId, TermID lastTerm, LogID lastApplyLo return true; } -std::string ESListener::encodeAppliedId(LogID lastId, - TermID lastTerm, - LogID lastApplyLogId) const noexcept { +std::string ESListener::encodeAppliedId(LogID lastId, TermID lastTerm, LogID lastApplyLogId) const { std::string val; val.reserve(sizeof(LogID) * 2 + sizeof(TermID)); val.append(reinterpret_cast(&lastId), sizeof(LogID)) diff --git a/src/kvstore/listener/elasticsearch/ESListener.h b/src/kvstore/listener/elasticsearch/ESListener.h index b46cc58f830..4edbb30437e 100644 --- a/src/kvstore/listener/elasticsearch/ESListener.h +++ b/src/kvstore/listener/elasticsearch/ESListener.h @@ -102,7 +102,7 @@ class ESListener : public Listener { * @param lastApplyLogId Last apply id * @return Encoded string */ - std::string encodeAppliedId(LogID lastId, TermID lastTerm, LogID lastApplyLogId) const noexcept; + std::string encodeAppliedId(LogID lastId, TermID lastTerm, LogID lastApplyLogId) const; private: meta::SchemaManager* schemaMan_{nullptr}; diff --git a/src/kvstore/raftex/RaftPart.cpp b/src/kvstore/raftex/RaftPart.cpp index ff3909b3970..7d66e518b76 100644 --- a/src/kvstore/raftex/RaftPart.cpp +++ b/src/kvstore/raftex/RaftPart.cpp @@ -87,16 +87,14 @@ class AppendLogsIterator final : public LogIterator { ~AppendLogsIterator() { if (!logs_.empty()) { - size_t notFulfilledPromise = 0; for (auto& log : logs_) { auto& promiseRef = std::get<4>(log); if (!promiseRef.isFulfilled()) { - ++notFulfilledPromise; + // When a AppendLogsIterator destruct before calling commit, the promise it not fulfilled. + // It only happens when exception is thrown, which make `commit` is never invoked. + promiseRef.setValue(nebula::cpp2::ErrorCode::E_STORAGE_MEMORY_EXCEEDED); } } - if (notFulfilledPromise > 0) { - LOG(FATAL) << "notFulfilledPromise == " << notFulfilledPromise; - } } } diff --git a/src/storage/BaseProcessor.h b/src/storage/BaseProcessor.h index b94fc723f08..1721be49cf3 100644 --- a/src/storage/BaseProcessor.h +++ b/src/storage/BaseProcessor.h @@ -39,6 +39,7 @@ class BaseProcessor { } virtual void onFinished() { + memory::MemoryCheckOffGuard guard; if (counters_) { stats::StatsManager::addValue(counters_->numCalls_); if (!this->result_.get_failed_parts().empty()) { @@ -62,6 +63,7 @@ class BaseProcessor { } virtual void onError() { + memory::MemoryCheckOffGuard guard; if (counters_) { stats::StatsManager::addValue(counters_->numCalls_); if (!this->result_.get_failed_parts().empty()) { @@ -171,7 +173,6 @@ struct MemoryCheckScope { : processor_(processor), f_(std::move(f)) {} ~MemoryCheckScope() { - memory::MemoryCheckGuard guard; try { f_(); } catch (std::bad_alloc& e) { diff --git a/src/storage/GraphStorageServiceHandler.cpp b/src/storage/GraphStorageServiceHandler.cpp index 7d0d18971d5..72c386148e0 100644 --- a/src/storage/GraphStorageServiceHandler.cpp +++ b/src/storage/GraphStorageServiceHandler.cpp @@ -31,20 +31,20 @@ // Processors DO NOT NEED handle error in their logic. // else (do some work in another thread) // Processors need handle error in that thread by itself -#define RETURN_FUTURE(processor) \ - memory::MemoryCheckGuard guard; \ - auto f = processor->getFuture(); \ - try { \ - processor->process(req); \ - } catch (std::bad_alloc & e) { \ - processor->memoryExceeded(); \ - processor->onError(); \ - } catch (std::exception & e) { \ - LOG(ERROR) << e.what(); \ - processor->onError(); \ - } catch (...) { \ - processor->onError(); \ - } \ +#define RETURN_FUTURE(processor) \ + auto f = processor->getFuture(); \ + try { \ + processor->process(req); \ + } catch (std::bad_alloc & e) { \ + LOG(ERROR) << processor << " bad_alloc"; \ + processor->memoryExceeded(); \ + processor->onError(); \ + } catch (std::exception & e) { \ + LOG(ERROR) << e.what(); \ + processor->onError(); \ + } catch (...) { \ + processor->onError(); \ + } \ return f; namespace nebula { diff --git a/src/storage/index/LookupProcessor.cpp b/src/storage/index/LookupProcessor.cpp index 975cc348794..46b79c6be4d 100644 --- a/src/storage/index/LookupProcessor.cpp +++ b/src/storage/index/LookupProcessor.cpp @@ -218,6 +218,7 @@ ErrorOr> LookupProcessor::bu void LookupProcessor::runInSingleThread(const std::vector& parts, std::unique_ptr plan) { + memory::MemoryCheckGuard guard; // printPlan(plan.get()); std::vector> datasetList; std::vector<::nebula::cpp2::ErrorCode> codeList; @@ -265,6 +266,7 @@ void LookupProcessor::runInSingleThread(const std::vector& parts, void LookupProcessor::runInMultipleThread(const std::vector& parts, std::unique_ptr plan) { + memory::MemoryCheckOffGuard offGuard; std::vector> planCopy = reproducePlan(plan.get(), parts.size()); using ReturnType = std::tuple, Row>; std::vector> futures; @@ -298,49 +300,47 @@ void LookupProcessor::runInMultipleThread(const std::vector& parts, } return {part, code, dataset, statResult}; }) - .thenError( - folly::tag_t{}, - [this](const std::bad_alloc&) { - memoryExceeded_ = true; - return folly::makeFuture< - std::tuple, Row>>( - std::runtime_error("Memory Limit Exceeded, " + - memory::MemoryStats::instance().toString())); - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - LOG(ERROR) << e.what(); + .thenError(folly::tag_t{}, [this](const std::bad_alloc&) { + memoryExceeded_ = true; return folly::makeFuture< std::tuple, Row>>( - std::runtime_error(e.what())); + std::runtime_error("Memory Limit Exceeded, " + + memory::MemoryStats::instance().toString())); })); } - folly::collectAll(futures).via(executor_).thenTry([this](auto&& t) { - memory::MemoryCheckGuard guard; - CHECK(!t.hasException()); - const auto& tries = t.value(); - std::vector statResults; - for (size_t j = 0; j < tries.size(); j++) { - if (tries[j].hasException()) { - onError(); - return; - } - auto& [partId, code, dataset, statResult] = tries[j].value(); - if (code == ::nebula::cpp2::ErrorCode::SUCCEEDED) { - for (auto& row : dataset) { - resultDataSet_.emplace_back(std::move(row)); + folly::collectAll(futures) + .via(executor_) + .thenTry([this](auto&& t) { + memory::MemoryCheckGuard guard; + CHECK(!t.hasException()); + const auto& tries = t.value(); + std::vector statResults; + for (size_t j = 0; j < tries.size(); j++) { + if (tries[j].hasException()) { + onError(); + return; + } + auto& [partId, code, dataset, statResult] = tries[j].value(); + if (code == ::nebula::cpp2::ErrorCode::SUCCEEDED) { + for (auto& row : dataset) { + resultDataSet_.emplace_back(std::move(row)); + } + } else { + handleErrorCode(code, context_->spaceId(), partId); + } + statResults.emplace_back(std::move(statResult)); } - } else { - handleErrorCode(code, context_->spaceId(), partId); - } - statResults.emplace_back(std::move(statResult)); - } - DLOG(INFO) << "finish"; - // IndexAggregateNode has been copied and each part get it's own aggregate info, - // we need to merge it - this->mergeStatsResult(statResults); - this->onProcessFinished(); - this->onFinished(); - }); + DLOG(INFO) << "finish"; + // IndexAggregateNode has been copied and each part get it's own aggregate info, + // we need to merge it + this->mergeStatsResult(statResults); + this->onProcessFinished(); + this->onFinished(); + }) + .thenError(folly::tag_t{}, [this](const std::bad_alloc&) { + memoryExceeded_ = true; + onError(); + }); } ErrorOr>> diff --git a/src/storage/query/GetDstBySrcProcessor.cpp b/src/storage/query/GetDstBySrcProcessor.cpp index f4a1ce38029..b6ee5f2a4a2 100644 --- a/src/storage/query/GetDstBySrcProcessor.cpp +++ b/src/storage/query/GetDstBySrcProcessor.cpp @@ -66,6 +66,7 @@ void GetDstBySrcProcessor::doProcess(const cpp2::GetDstBySrcRequest& req) { } void GetDstBySrcProcessor::runInSingleThread(const cpp2::GetDstBySrcRequest& req) { + memory::MemoryCheckGuard guard; contexts_.emplace_back(RuntimeContext(planContext_.get())); auto plan = buildPlan(&contexts_.front(), &flatResult_); std::unordered_set failedParts; @@ -101,6 +102,7 @@ void GetDstBySrcProcessor::runInSingleThread(const cpp2::GetDstBySrcRequest& req } void GetDstBySrcProcessor::runInMultipleThread(const cpp2::GetDstBySrcRequest& req) { + memory::MemoryCheckOffGuard offGuard; for (size_t i = 0; i < req.get_parts().size(); i++) { partResults_.emplace_back(); contexts_.emplace_back(RuntimeContext(planContext_.get())); @@ -112,39 +114,45 @@ void GetDstBySrcProcessor::runInMultipleThread(const cpp2::GetDstBySrcRequest& r i++; } - folly::collectAll(futures).via(executor_).thenTry([this](auto&& t) mutable { - memory::MemoryCheckGuard guard; - CHECK(!t.hasException()); - const auto& tries = t.value(); - - // size_t sum = 0; - // for (size_t j = 0; j < tries.size(); j++) { - // const auto& [code, partId] = tries[j].value(); - // if (code == nebula::cpp2::ErrorCode::SUCCEEDED) { - // sum += partResults_[j].size(); - // } - // } - // flatResult_.reserve(sum); - - for (size_t j = 0; j < tries.size(); j++) { - if (tries[j].hasException()) { - onError(); - return; - } - const auto& [code, partId] = tries[j].value(); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - handleErrorCode(code, spaceId_, partId); - } else { - for (auto& v : partResults_[j]) { - flatResult_.emplace_back(std::move(v)); + folly::collectAll(futures) + .via(executor_) + .thenTry([this](auto&& t) mutable { + memory::MemoryCheckGuard guard; + CHECK(!t.hasException()); + const auto& tries = t.value(); + + // size_t sum = 0; + // for (size_t j = 0; j < tries.size(); j++) { + // const auto& [code, partId] = tries[j].value(); + // if (code == nebula::cpp2::ErrorCode::SUCCEEDED) { + // sum += partResults_[j].size(); + // } + // } + // flatResult_.reserve(sum); + + for (size_t j = 0; j < tries.size(); j++) { + if (tries[j].hasException()) { + onError(); + return; + } + const auto& [code, partId] = tries[j].value(); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + handleErrorCode(code, spaceId_, partId); + } else { + for (auto& v : partResults_[j]) { + flatResult_.emplace_back(std::move(v)); + } + std::deque().swap(partResults_[j]); + } } - std::deque().swap(partResults_[j]); - } - } - this->onProcessFinished(); - this->onFinished(); - }); + this->onProcessFinished(); + this->onFinished(); + }) + .thenError(folly::tag_t{}, [this](const std::bad_alloc&) { + memoryExceeded_ = true; + onError(); + }); } folly::Future> GetDstBySrcProcessor::runInExecutor( @@ -155,6 +163,10 @@ folly::Future> GetDstBySrcProces return folly::via(executor_, [this, context, result, partId, input = std::move(srcIds)]() mutable { memory::MemoryCheckGuard guard; + if (memoryExceeded_) { + return std::make_pair(nebula::cpp2::ErrorCode::E_STORAGE_MEMORY_EXCEEDED, + partId); + } auto plan = buildPlan(context, result); for (const auto& src : input) { auto& vId = src.getStr(); @@ -176,17 +188,9 @@ folly::Future> GetDstBySrcProces } return std::make_pair(nebula::cpp2::ErrorCode::SUCCEEDED, partId); }) - .thenError(folly::tag_t{}, - [this](const std::bad_alloc&) { - memoryExceeded_ = true; - return folly::makeFuture>( - std::runtime_error("Memory Limit Exceeded, " + - memory::MemoryStats::instance().toString())); - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - LOG(ERROR) << e.what(); - return folly::makeFuture>( - std::runtime_error(e.what())); + .thenError(folly::tag_t{}, [this, partId](const std::bad_alloc&) { + memoryExceeded_ = true; + return std::make_pair(nebula::cpp2::ErrorCode::E_STORAGE_MEMORY_EXCEEDED, partId); }); } diff --git a/src/storage/query/GetNeighborsProcessor.cpp b/src/storage/query/GetNeighborsProcessor.cpp index 07c68d6a62a..b362ee6d60a 100644 --- a/src/storage/query/GetNeighborsProcessor.cpp +++ b/src/storage/query/GetNeighborsProcessor.cpp @@ -77,6 +77,7 @@ void GetNeighborsProcessor::doProcess(const cpp2::GetNeighborsRequest& req) { void GetNeighborsProcessor::runInSingleThread(const cpp2::GetNeighborsRequest& req, int64_t limit, bool random) { + memory::MemoryCheckGuard guard; contexts_.emplace_back(RuntimeContext(planContext_.get())); expCtxs_.emplace_back(StorageExpressionContext(spaceVidLen_, isIntId_)); auto plan = buildPlan(&contexts_.front(), &expCtxs_.front(), &resultDataSet_, limit, random); @@ -115,6 +116,7 @@ void GetNeighborsProcessor::runInSingleThread(const cpp2::GetNeighborsRequest& r void GetNeighborsProcessor::runInMultipleThread(const cpp2::GetNeighborsRequest& req, int64_t limit, bool random) { + memory::MemoryCheckOffGuard offGuard; for (size_t i = 0; i < req.get_parts().size(); i++) { nebula::DataSet result = resultDataSet_; results_.emplace_back(std::move(result)); @@ -129,30 +131,36 @@ void GetNeighborsProcessor::runInMultipleThread(const cpp2::GetNeighborsRequest& i++; } - folly::collectAll(futures).via(executor_).thenTry([this](auto&& t) mutable { - memory::MemoryCheckGuard guard; - CHECK(!t.hasException()); - const auto& tries = t.value(); - size_t sum = 0; - for (size_t j = 0; j < tries.size(); j++) { - if (tries[j].hasException()) { + folly::collectAll(futures) + .via(executor_) + .thenTry([this](auto&& t) mutable { + memory::MemoryCheckGuard guard; + CHECK(!t.hasException()); + const auto& tries = t.value(); + size_t sum = 0; + for (size_t j = 0; j < tries.size(); j++) { + if (tries[j].hasException()) { + onError(); + return; + } + sum += results_[j].size(); + } + resultDataSet_.rows.reserve(sum); + for (size_t j = 0; j < tries.size(); j++) { + const auto& [code, partId] = tries[j].value(); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + handleErrorCode(code, spaceId_, partId); + } else { + resultDataSet_.append(std::move(results_[j])); + } + } + this->onProcessFinished(); + this->onFinished(); + }) + .thenError(folly::tag_t{}, [this](const std::bad_alloc&) { + memoryExceeded_ = true; onError(); - return; - } - sum += results_[j].size(); - } - resultDataSet_.rows.reserve(sum); - for (size_t j = 0; j < tries.size(); j++) { - const auto& [code, partId] = tries[j].value(); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - handleErrorCode(code, spaceId_, partId); - } else { - resultDataSet_.append(std::move(results_[j])); - } - } - this->onProcessFinished(); - this->onFinished(); - }); + }); } folly::Future> GetNeighborsProcessor::runInExecutor( @@ -167,6 +175,9 @@ folly::Future> GetNeighborsProce executor_, [this, context, expCtx, result, partId, input = std::move(vids), limit, random]() { memory::MemoryCheckGuard guard; + if (memoryExceeded_) { + return std::make_pair(nebula::cpp2::ErrorCode::E_STORAGE_MEMORY_EXCEEDED, partId); + } auto plan = buildPlan(context, expCtx, result, limit, random); for (const auto& vid : input) { auto vId = vid.getStr(); @@ -188,17 +199,9 @@ folly::Future> GetNeighborsProce } return std::make_pair(nebula::cpp2::ErrorCode::SUCCEEDED, partId); }) - .thenError(folly::tag_t{}, - [this](const std::bad_alloc&) { - memoryExceeded_ = true; - return folly::makeFuture>( - std::runtime_error("Memory Limit Exceeded, " + - memory::MemoryStats::instance().toString())); - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - LOG(ERROR) << e.what(); - return folly::makeFuture>( - std::runtime_error(e.what())); + .thenError(folly::tag_t{}, [this, partId](const std::bad_alloc&) { + memoryExceeded_ = true; + return std::make_pair(nebula::cpp2::ErrorCode::E_STORAGE_MEMORY_EXCEEDED, partId); }); } diff --git a/src/storage/query/GetPropProcessor.cpp b/src/storage/query/GetPropProcessor.cpp index 81dfb5ef7e2..b5dfaf35437 100644 --- a/src/storage/query/GetPropProcessor.cpp +++ b/src/storage/query/GetPropProcessor.cpp @@ -56,6 +56,7 @@ void GetPropProcessor::doProcess(const cpp2::GetPropRequest& req) { } void GetPropProcessor::runInSingleThread(const cpp2::GetPropRequest& req) { + memory::MemoryCheckGuard guard; contexts_.emplace_back(RuntimeContext(planContext_.get())); std::unordered_set failedParts; if (!isEdge_) { @@ -116,6 +117,7 @@ void GetPropProcessor::runInSingleThread(const cpp2::GetPropRequest& req) { } void GetPropProcessor::runInMultipleThread(const cpp2::GetPropRequest& req) { + memory::MemoryCheckOffGuard offGuard; for (size_t i = 0; i < req.get_parts().size(); i++) { nebula::DataSet result = resultDataSet_; results_.emplace_back(std::move(result)); @@ -128,30 +130,36 @@ void GetPropProcessor::runInMultipleThread(const cpp2::GetPropRequest& req) { i++; } - folly::collectAll(futures).via(executor_).thenTry([this](auto&& t) mutable { - memory::MemoryCheckGuard guard; - CHECK(!t.hasException()); - const auto& tries = t.value(); - size_t sum = 0; - for (size_t j = 0; j < tries.size(); j++) { - if (tries[j].hasException()) { + folly::collectAll(futures) + .via(executor_) + .thenTry([this](auto&& t) mutable { + memory::MemoryCheckGuard guard; + CHECK(!t.hasException()); + const auto& tries = t.value(); + size_t sum = 0; + for (size_t j = 0; j < tries.size(); j++) { + if (tries[j].hasException()) { + onError(); + return; + } + sum += results_[j].size(); + } + resultDataSet_.rows.reserve(sum); + for (size_t j = 0; j < tries.size(); j++) { + const auto& [code, partId] = tries[j].value(); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + handleErrorCode(code, spaceId_, partId); + } else { + resultDataSet_.append(std::move(results_[j])); + } + } + this->onProcessFinished(); + this->onFinished(); + }) + .thenError(folly::tag_t{}, [this](const std::bad_alloc&) { + memoryExceeded_ = true; onError(); - return; - } - sum += results_[j].size(); - } - resultDataSet_.rows.reserve(sum); - for (size_t j = 0; j < tries.size(); j++) { - const auto& [code, partId] = tries[j].value(); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - handleErrorCode(code, spaceId_, partId); - } else { - resultDataSet_.append(std::move(results_[j])); - } - } - this->onProcessFinished(); - this->onFinished(); - }); + }); } folly::Future> GetPropProcessor::runInExecutor( @@ -162,6 +170,10 @@ folly::Future> GetPropProcessor: return folly::via(executor_, [this, context, result, partId, input = std::move(rows)]() { memory::MemoryCheckGuard guard; + if (memoryExceeded_) { + return std::make_pair(nebula::cpp2::ErrorCode::E_STORAGE_MEMORY_EXCEEDED, + partId); + } if (!isEdge_) { auto plan = buildTagPlan(context, result); for (const auto& row : input) { @@ -206,17 +218,11 @@ folly::Future> GetPropProcessor: return std::make_pair(nebula::cpp2::ErrorCode::SUCCEEDED, partId); } }) - .thenError(folly::tag_t{}, - [this](const std::bad_alloc&) { - memoryExceeded_ = true; - return folly::makeFuture>( - std::runtime_error("Memory Limit Exceeded, " + - memory::MemoryStats::instance().toString())); - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - LOG(ERROR) << e.what(); + .thenError(folly::tag_t{}, [this](const std::bad_alloc&) { + memoryExceeded_ = true; return folly::makeFuture>( - std::runtime_error(e.what())); + std::runtime_error("Memory Limit Exceeded, " + + memory::MemoryStats::instance().toString())); }); } diff --git a/src/storage/query/ScanEdgeProcessor.cpp b/src/storage/query/ScanEdgeProcessor.cpp index c49d72906f5..a193ad7da7c 100644 --- a/src/storage/query/ScanEdgeProcessor.cpp +++ b/src/storage/query/ScanEdgeProcessor.cpp @@ -128,6 +128,10 @@ folly::Future> ScanEdgeProcessor return folly::via(executor_, [this, context, result, cursors, partId, input = std::move(cursor), expCtx]() { memory::MemoryCheckGuard guard; + if (memoryExceeded_) { + return std::make_pair(nebula::cpp2::ErrorCode::E_STORAGE_MEMORY_EXCEEDED, + partId); + } auto plan = buildPlan(context, result, cursors, expCtx); auto ret = plan.go(partId, input); @@ -136,21 +140,14 @@ folly::Future> ScanEdgeProcessor } return std::make_pair(nebula::cpp2::ErrorCode::SUCCEEDED, partId); }) - .thenError(folly::tag_t{}, - [this](const std::bad_alloc&) { - memoryExceeded_ = true; - return folly::makeFuture>( - std::runtime_error("Memory Limit Exceeded, " + - memory::MemoryStats::instance().toString())); - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - LOG(ERROR) << e.what(); - return folly::makeFuture>( - std::runtime_error(e.what())); + .thenError(folly::tag_t{}, [this, partId](const std::bad_alloc&) { + memoryExceeded_ = true; + return std::make_pair(nebula::cpp2::ErrorCode::E_STORAGE_MEMORY_EXCEEDED, partId); }); } void ScanEdgeProcessor::runInSingleThread(const cpp2::ScanEdgeRequest& req) { + memory::MemoryCheckGuard guard; contexts_.emplace_back(RuntimeContext(planContext_.get())); expCtxs_.emplace_back(StorageExpressionContext(spaceVidLen_, isIntId_)); std::unordered_set failedParts; @@ -172,6 +169,7 @@ void ScanEdgeProcessor::runInSingleThread(const cpp2::ScanEdgeRequest& req) { } void ScanEdgeProcessor::runInMultipleThread(const cpp2::ScanEdgeRequest& req) { + memory::MemoryCheckOffGuard offGuard; cursorsOfPart_.resize(req.get_parts().size()); for (size_t i = 0; i < req.get_parts().size(); i++) { nebula::DataSet result = resultDataSet_; @@ -192,31 +190,37 @@ void ScanEdgeProcessor::runInMultipleThread(const cpp2::ScanEdgeRequest& req) { i++; } - folly::collectAll(futures).via(executor_).thenTry([this](auto&& t) mutable { - memory::MemoryCheckGuard guard; - CHECK(!t.hasException()); - const auto& tries = t.value(); - size_t sum = 0; - for (size_t j = 0; j < tries.size(); j++) { - if (tries[j].hasException()) { + folly::collectAll(futures) + .via(executor_) + .thenTry([this](auto&& t) mutable { + memory::MemoryCheckGuard guard; + CHECK(!t.hasException()); + const auto& tries = t.value(); + size_t sum = 0; + for (size_t j = 0; j < tries.size(); j++) { + if (tries[j].hasException()) { + onError(); + return; + } + sum += results_[j].size(); + } + resultDataSet_.rows.reserve(sum); + for (size_t j = 0; j < tries.size(); j++) { + const auto& [code, partId] = tries[j].value(); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + handleErrorCode(code, spaceId_, partId); + } else { + resultDataSet_.append(std::move(results_[j])); + cursors_.merge(std::move(cursorsOfPart_[j])); + } + } + this->onProcessFinished(); + this->onFinished(); + }) + .thenError(folly::tag_t{}, [this](const std::bad_alloc&) { + memoryExceeded_ = true; onError(); - return; - } - sum += results_[j].size(); - } - resultDataSet_.rows.reserve(sum); - for (size_t j = 0; j < tries.size(); j++) { - const auto& [code, partId] = tries[j].value(); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - handleErrorCode(code, spaceId_, partId); - } else { - resultDataSet_.append(std::move(results_[j])); - cursors_.merge(std::move(cursorsOfPart_[j])); - } - } - this->onProcessFinished(); - this->onFinished(); - }); + }); } } // namespace storage diff --git a/src/storage/query/ScanVertexProcessor.cpp b/src/storage/query/ScanVertexProcessor.cpp index 6465b11de4b..cb5754aa527 100644 --- a/src/storage/query/ScanVertexProcessor.cpp +++ b/src/storage/query/ScanVertexProcessor.cpp @@ -133,6 +133,9 @@ folly::Future> ScanVertexProcess executor_, [this, context, result, cursorsOfPart, partId, input = std::move(cursor), expCtx]() { memory::MemoryCheckGuard guard; + if (memoryExceeded_) { + return std::make_pair(nebula::cpp2::ErrorCode::E_STORAGE_MEMORY_EXCEEDED, partId); + } auto plan = buildPlan(context, result, cursorsOfPart, expCtx); auto ret = plan.go(partId, input); @@ -141,21 +144,14 @@ folly::Future> ScanVertexProcess } return std::make_pair(nebula::cpp2::ErrorCode::SUCCEEDED, partId); }) - .thenError(folly::tag_t{}, - [this](const std::bad_alloc&) { - memoryExceeded_ = true; - return folly::makeFuture>( - std::runtime_error("Memory Limit Exceeded, " + - memory::MemoryStats::instance().toString())); - }) - .thenError(folly::tag_t{}, [](const std::exception& e) { - LOG(ERROR) << e.what(); - return folly::makeFuture>( - std::runtime_error(e.what())); + .thenError(folly::tag_t{}, [this, partId](const std::bad_alloc&) { + memoryExceeded_ = true; + return std::make_pair(nebula::cpp2::ErrorCode::E_STORAGE_MEMORY_EXCEEDED, partId); }); } void ScanVertexProcessor::runInSingleThread(const cpp2::ScanVertexRequest& req) { + memory::MemoryCheckGuard guard; contexts_.emplace_back(RuntimeContext(planContext_.get())); expCtxs_.emplace_back(StorageExpressionContext(spaceVidLen_, isIntId_)); std::unordered_set failedParts; @@ -177,6 +173,7 @@ void ScanVertexProcessor::runInSingleThread(const cpp2::ScanVertexRequest& req) } void ScanVertexProcessor::runInMultipleThread(const cpp2::ScanVertexRequest& req) { + memory::MemoryCheckOffGuard offGuard; cursorsOfPart_.resize(req.get_parts().size()); for (size_t i = 0; i < req.get_parts().size(); i++) { nebula::DataSet result = resultDataSet_; @@ -197,31 +194,37 @@ void ScanVertexProcessor::runInMultipleThread(const cpp2::ScanVertexRequest& req i++; } - folly::collectAll(futures).via(executor_).thenTry([this](auto&& t) mutable { - memory::MemoryCheckGuard guard; - CHECK(!t.hasException()); - const auto& tries = t.value(); - size_t sum = 0; - for (size_t j = 0; j < tries.size(); j++) { - if (tries[j].hasException()) { + folly::collectAll(futures) + .via(executor_) + .thenTry([this](auto&& t) mutable { + memory::MemoryCheckGuard guard; + CHECK(!t.hasException()); + const auto& tries = t.value(); + size_t sum = 0; + for (size_t j = 0; j < tries.size(); j++) { + if (tries[j].hasException()) { + onError(); + return; + } + sum += results_[j].size(); + } + resultDataSet_.rows.reserve(sum); + for (size_t j = 0; j < tries.size(); j++) { + const auto& [code, partId] = tries[j].value(); + if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { + handleErrorCode(code, spaceId_, partId); + } else { + resultDataSet_.append(std::move(results_[j])); + cursors_.merge(std::move(cursorsOfPart_[j])); + } + } + this->onProcessFinished(); + this->onFinished(); + }) + .thenError(folly::tag_t{}, [this](const std::bad_alloc&) { + memoryExceeded_ = true; onError(); - return; - } - sum += results_[j].size(); - } - resultDataSet_.rows.reserve(sum); - for (size_t j = 0; j < tries.size(); j++) { - const auto& [code, partId] = tries[j].value(); - if (code != nebula::cpp2::ErrorCode::SUCCEEDED) { - handleErrorCode(code, spaceId_, partId); - } else { - resultDataSet_.append(std::move(results_[j])); - cursors_.merge(std::move(cursorsOfPart_[j])); - } - } - this->onProcessFinished(); - this->onFinished(); - }); + }); } } // namespace storage