Skip to content

Commit

Permalink
curvebs/mds: add poolset rules to support assigning poolset based on …
Browse files Browse the repository at this point in the history
…directories

Signed-off-by: Hanqing Wu <wuhanqing@corp.netease.com>
  • Loading branch information
wu-hanqing authored and opencurveadmin committed Jun 12, 2023
1 parent bd6b5cf commit fea8cd4
Show file tree
Hide file tree
Showing 8 changed files with 188 additions and 27 deletions.
12 changes: 12 additions & 0 deletions conf/mds.conf
Original file line number Diff line number Diff line change
Expand Up @@ -202,3 +202,15 @@ mds.common.logDir=./ # __CURVEADM_TEMPLATE__ ${prefix}/logs __CURVEADM_TEMPLATE
# 单元测试情况下
# mds.common.logDir=./runlog/

#
## poolset rules
#
# for backward compatibility, rules are applied for select poolset when creating file
#
# for example
# mds.poolset.rules=/dir1/:poolset1;/dir2/:poolset2;/dir1/sub/:sub
#
# when creating file reqeust doesn't have poolset, above rules are used to select poolset
# - if filename is /dir1/file, then poolset1 is select
# - if filename is /dir1/sub/file, then sub is select
mds.poolset.rules=
38 changes: 19 additions & 19 deletions src/mds/nameserver2/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -15,26 +15,26 @@
#

cc_library(
name="nameserver2",
hdrs=glob(["*.h"]),
srcs=glob(["*.cpp"]),
name = "nameserver2",
srcs = glob(["*.cpp"]),
hdrs = glob(["*.h"]),
visibility = ["//visibility:public"],
deps = [
"//external:glog",
"//external:gflags",
"//external:leveldb",
"//external:brpc",
"//proto:nameserver2_cc_proto",
"//src/common:curve_common",
"//src/mds/topology:topology",
"//src/mds/nameserver2/allocstatistic:alloc_statistic",
"//src/mds/chunkserverclient:chunkserverclient",
"//src/mds/snapshotcloneclient:snapshotcloneclient",
"//src/mds/common:mds_common",
"//src/mds/nameserver2/helper:helper",
"//src/mds/nameserver2/idgenerator:idgenerator",
"//src/common:curve_auth",
"//src/kvstorageclient:kvstorage_client",
"//external:brpc",
"//external:gflags",
"//external:glog",
"//external:leveldb",
"//proto:nameserver2_cc_proto",
"//src/common:curve_auth",
"//src/common:curve_common",
"//src/kvstorageclient:kvstorage_client",
"//src/mds/chunkserverclient",
"//src/mds/common:mds_common",
"//src/mds/nameserver2/allocstatistic:alloc_statistic",
"//src/mds/nameserver2/helper",
"//src/mds/nameserver2/idgenerator",
"//src/mds/snapshotcloneclient",
"//src/mds/topology",
"@com_google_absl//absl/strings",
],
)

39 changes: 33 additions & 6 deletions src/mds/nameserver2/curvefs.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@
#include "src/mds/common/mds_define.h"
#include "src/mds/nameserver2/helper/namespace_helper.h"

#include "absl/strings/match.h"

using curve::common::TimeUtility;
using curve::common::kDefaultPoolsetName;
using curve::mds::topology::LogicalPool;
Expand Down Expand Up @@ -122,6 +124,7 @@ bool CurveFS::Init(std::shared_ptr<NameServerStorage> storage,
maxFileLength_ = curveFSOptions.maxFileLength;
topology_ = topology;
snapshotCloneClient_ = snapshotCloneClient;
poolsetRules_ = curveFSOptions.poolsetRules;

InitRootFile();
bool ret = InitRecycleBinDir();
Expand Down Expand Up @@ -245,7 +248,7 @@ StatusCode CurveFS::CreateFile(const std::string& fileName,

// check param
if (filetype == FileType::INODE_PAGEFILE) {
StatusCode retCode = CheckOrAssignPoolset(&poolset);
StatusCode retCode = CheckOrAssignPoolset(fileName, &poolset);
if (retCode != StatusCode::kOK) {
return retCode;
}
Expand Down Expand Up @@ -738,17 +741,17 @@ StatusCode CurveFS::DeleteFile(const std::string & filename, uint64_t fileId,
}
}

StatusCode CurveFS::CheckOrAssignPoolset(std::string* poolset) const {
StatusCode CurveFS::CheckOrAssignPoolset(const std::string& filename,
std::string* poolset) const {
const auto names = topology_->GetPoolsetNameInCluster();
if (names.empty()) {
LOG(WARNING) << "Cluster doesn't have poolsets";
return StatusCode::kPoolsetNotExist;
}

if (poolset->empty()) {
*poolset = kDefaultPoolsetName;
LOG(INFO) << "Poolset is empty, set to: " << kDefaultPoolsetName;
return StatusCode::kOK;
*poolset = SelectPoolsetByRules(filename, poolsetRules_);
LOG(INFO) << "Poolset is empty, set to: " << *poolset;
}

auto it = std::find(names.begin(), names.end(), *poolset);
Expand Down Expand Up @@ -1725,7 +1728,7 @@ StatusCode CurveFS::CreateCloneFile(const std::string &fileName,
return ret;
}

ret = CheckOrAssignPoolset(&poolset);
ret = CheckOrAssignPoolset(fileName, &poolset);
if (ret != StatusCode::kOK) {
return ret;
}
Expand Down Expand Up @@ -2479,5 +2482,29 @@ uint64_t GetOpenFileNum(void *varg) {
bvar::PassiveStatus<uint64_t> g_open_file_num_bvar(
CURVE_MDS_CURVEFS_METRIC_PREFIX, "open_file_num",
GetOpenFileNum, &kCurveFS);

std::string SelectPoolsetByRules(
const std::string& filename,
const std::map<std::string, std::string>& rules) {
if (rules.empty()) {
return kDefaultPoolsetName;
}

// using reverse order, so that we support subdir rules
//
// for example
// /A/ -> poolset1
// /A/B/ -> poolset2
//
// if filename is /A/B/C, then we select `poolset2`
for (auto it = rules.rbegin(); it != rules.rend(); ++it) {
if (absl::StartsWith(filename, it->first)) {
return it->second;
}
}

return kDefaultPoolsetName;
}

} // namespace mds
} // namespace curve
11 changes: 10 additions & 1 deletion src/mds/nameserver2/curvefs.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
#include <thread> //NOLINT
#include <chrono> //NOLINT
#include <unordered_map>
#include <map>
#include "proto/nameserver2.pb.h"
#include "src/mds/nameserver2/namespace_storage.h"
#include "src/mds/common/mds_define.h"
Expand Down Expand Up @@ -62,6 +63,7 @@ struct CurveFSOption {
uint64_t maxFileLength;
RootAuthOption authOptions;
FileRecordOptions fileRecordOptions;
std::map<std::string, std::string> poolsetRules;
};

struct AllocatedSize {
Expand Down Expand Up @@ -222,7 +224,8 @@ class CurveFS {
* @param[in|out] poolset: poolset name
* @return StatusCode::kOK if success
*/
StatusCode CheckOrAssignPoolset(std::string* poolset) const;
StatusCode CheckOrAssignPoolset(const std::string& filename,
std::string* poolset) const;

/**
* @brief get information of all files in the directory
Expand Down Expand Up @@ -791,6 +794,8 @@ class CurveFS {
uint64_t minFileLength_;
uint64_t maxFileLength_;
std::chrono::steady_clock::time_point startTime_;

std::map<std::string, std::string> poolsetRules_;
};
extern CurveFS &kCurveFS;

Expand All @@ -813,6 +818,10 @@ StatusCode CheckStripeParam(uint64_t segmentSize,
uint64_t stripeUnit,
uint64_t stripeCount);

std::string SelectPoolsetByRules(
const std::string& filename,
const std::map<std::string, std::string>& rules);

} // namespace mds
} // namespace curve
#endif // SRC_MDS_NAMESERVER2_CURVEFS_H_
35 changes: 34 additions & 1 deletion src/mds/server/mds.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,13 @@
*/

#include <glog/logging.h>
#include <map>
#include "src/mds/server/mds.h"
#include "src/mds/nameserver2/helper/namespace_helper.h"
#include "src/mds/topology/topology_storge_etcd.h"

#include "absl/strings/str_split.h"

using ::curve::mds::topology::TopologyStorageEtcd;
using ::curve::mds::topology::TopologyStorageCodec;
using ::curve::mds::topology::ChunkServerRegistInfoBuilder;
Expand Down Expand Up @@ -470,8 +473,11 @@ void MDS::InitCurveFSOptions(CurveFSOption *curveFSOptions) {
FileRecordOptions fileRecordOptions;
InitFileRecordOptions(&curveFSOptions->fileRecordOptions);

RootAuthOption authOptions;
InitAuthOptions(&curveFSOptions->authOptions);

LOG_IF(FATAL, !ParsePoolsetRules(conf_->GetStringValue("mds.poolset.rules"),
&curveFSOptions->poolsetRules))
<< "Fail to parse poolset rules";
}

void MDS::InitCleanManager() {
Expand Down Expand Up @@ -580,5 +586,32 @@ void MDS::InitHeartbeatOption(HeartbeatOption* heartbeatOption) {
conf_->GetValueFatalIfFail("mds.heartbeat.clean_follower_afterMs",
&heartbeatOption->cleanFollowerAfterMs);
}

bool ParsePoolsetRules(const std::string& str,
std::map<std::string, std::string>* rules) {
rules->clear();

if (str.empty()) {
return true;
}

for (absl::string_view sp : absl::StrSplit(str, ';')) {
rules->insert(absl::StrSplit(sp, ':'));
}

for (const auto& rule : *rules) {
const auto& key = rule.first;

if (key.empty() || key.front() != '/' || key.back() != '/') {
LOG(ERROR) << "Invalid poolset rules, key must starts and ends "
"with `/`, rules: `"
<< str << "`";
return false;
}
}

return true;
}

} // namespace mds
} // namespace curve
4 changes: 4 additions & 0 deletions src/mds/server/mds.h
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
#include <brpc/server.h>
#include <string>
#include <memory>
#include <map>

#include "src/mds/nameserver2/namespace_storage.h"
#include "src/mds/nameserver2/namespace_service.h"
Expand Down Expand Up @@ -230,6 +231,9 @@ class MDS {
std::shared_ptr<SnapshotCloneClient> snapshotCloneClient_;
};

bool ParsePoolsetRules(const std::string& str,
std::map<std::string, std::string>* rules);

} // namespace mds
} // namespace curve

Expand Down
39 changes: 39 additions & 0 deletions test/mds/nameserver2/curvefs_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,45 @@ TEST_F(CurveFSTest, testCreateFileWithPoolset) {
}
}

TEST(TestSelectPoolsetByRules, Test) {
ASSERT_EQ(kDefaultPoolsetName, SelectPoolsetByRules("/filename", {}));

{
std::map<std::string, std::string> rules{
{"/system/", "system"}
};
ASSERT_EQ("system", SelectPoolsetByRules("/system/file", rules));
}

{
std::map<std::string, std::string> rules{
{"/system/", "system"}
};
ASSERT_EQ(kDefaultPoolsetName, SelectPoolsetByRules("/systems", rules));
}

{
std::map<std::string, std::string> rules{
{"/system/", "system"},
{"/systems/", "system1"},
};
ASSERT_EQ("system1", SelectPoolsetByRules("/systems/file", rules));
}

// subdir rules
{
std::map<std::string, std::string> rules{
{"/system/", "system"},
{"/system/sub/", "system-sub"}
};
ASSERT_EQ("system-sub",
SelectPoolsetByRules("/system/sub/file", rules));

ASSERT_EQ("system-sub",
SelectPoolsetByRules("/system/sub/sub/file", rules));
}
}

TEST_F(CurveFSTest, testGetFileInfo) {
// test parm error
FileInfo fileInfo;
Expand Down
37 changes: 37 additions & 0 deletions test/mds/server/mds_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -217,5 +217,42 @@ TEST_F(MDSTest, common) {
ASSERT_LE(stopTime - startTime, 100);
}

TEST(TestParsePoolsetRules, Test) {
std::map<std::string, std::string> rules;

{
ASSERT_TRUE(ParsePoolsetRules("", &rules));
ASSERT_TRUE(rules.empty());
}

{
ASSERT_TRUE(ParsePoolsetRules("/:hello", &rules));
ASSERT_EQ(1, rules.size());
ASSERT_EQ("hello", rules["/"]);
}

{
ASSERT_TRUE(ParsePoolsetRules("/system/:system;/data/:data", &rules));
ASSERT_EQ(2, rules.size());
ASSERT_EQ("system", rules["/system/"]);
ASSERT_EQ("data", rules["/data/"]);
}

{
// key must starts and ends with '/'
ASSERT_FALSE(ParsePoolsetRules("/system:system;/data/:data", &rules));
}

{
// subdir rules
ASSERT_TRUE(ParsePoolsetRules(
"/system/:system;/data/:data;/system/sub/:system-sub", &rules));
ASSERT_EQ(3, rules.size());
ASSERT_EQ("system", rules["/system/"]);
ASSERT_EQ("data", rules["/data/"]);
ASSERT_EQ("system-sub", rules["/system/sub/"]);
}
}

} // namespace mds
} // namespace curve

0 comments on commit fea8cd4

Please sign in to comment.