Skip to content

Commit

Permalink
optimize copyset creation
Browse files Browse the repository at this point in the history
  • Loading branch information
@cw123
cw123 committed Mar 28, 2022
1 parent d13d4b0 commit 137bb4e
Show file tree
Hide file tree
Showing 13 changed files with 679 additions and 115 deletions.
1 change: 1 addition & 0 deletions curvefs/proto/topology.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ enum TopoStatusCode {
TOPO_PARTITION_NOT_FOUND = 20;
TOPO_CREATE_PARTITION_FAIL = 21;
TOPO_METASERVER_EXIST = 22;
TOPO_GET_AVAILABLE_COPYSET_ERROR = 23;
}

enum OnlineState {
Expand Down
17 changes: 7 additions & 10 deletions curvefs/src/mds/metaserverclient/metaserver_client.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -322,18 +322,15 @@ FSStatusCode MetaserverClient::DeletePartition(
}
}

FSStatusCode MetaserverClient::CreateCopySet(
uint32_t poolId, std::set<uint32_t> copysetIds,
const std::set<std::string> &addrs) {
FSStatusCode MetaserverClient::CreateCopySet(uint32_t poolId,
uint32_t copysetId, const std::set<std::string> &addrs) {
CreateCopysetRequest request;
CreateCopysetResponse response;
for (auto id : copysetIds) {
auto copyset = request.add_copysets();
copyset->set_poolid(poolId);
copyset->set_copysetid(id);
for (auto item : addrs) {
copyset->add_peers()->set_address(BuildPeerIdWithAddr(item));
}
auto copyset = request.add_copysets();
copyset->set_poolid(poolId);
copyset->set_copysetid(copysetId);
for (auto item : addrs) {
copyset->add_peers()->set_address(BuildPeerIdWithAddr(item));
}

for (const std::string &item : addrs) {
Expand Down
2 changes: 1 addition & 1 deletion curvefs/src/mds/metaserverclient/metaserver_client.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ class MetaserverClient {
const std::set<std::string> &addrs);

virtual FSStatusCode CreateCopySet(uint32_t poolId,
std::set<uint32_t> copysetIds,
uint32_t copysetId,
const std::set<std::string> &addrs);

virtual FSStatusCode CreateCopySetOnOneMetaserver(uint32_t poolId,
Expand Down
198 changes: 183 additions & 15 deletions curvefs/src/mds/topology/topology.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -661,34 +661,58 @@ bool TopologyImpl::GetCopysetOfPartition(PartitionIdType id,
return false;
}

bool TopologyImpl::GetAvailableCopyset(CopySetInfo *out) const {
std::list<CopySetKey> TopologyImpl::GetAvailableCopysetList() const {
ReadLockGuard rlockCopySet(copySetMutex_);
uint64_t maxPartitionNum = option_.partitionNumberInCopyset;
CopySetKey key;
uint64_t minPartitionNum = maxPartitionNum;

std::list<CopySetKey> result;
for (auto const &it : copySetMap_) {
uint64_t partitionNum = it.second.GetPartitionNum();
if (partitionNum < maxPartitionNum) {
if (partitionNum < minPartitionNum) {
minPartitionNum = partitionNum;
key = it.first;
}
if (it.second.GetPartitionNum() >= option_.partitionNumberInCopyset) {
continue;
}

result.push_back(it.first);
}

if (minPartitionNum == maxPartitionNum) {
return result;
}

// choose random
int TopologyImpl::GetOneRandomNumber(int start, int end) const {
static std::random_device rd;
static std::mt19937 gen(rd());
std::uniform_int_distribution<> dis(start, end);
return dis(gen);
}

bool TopologyImpl::GetAvailableCopyset(CopySetInfo *out) const {
std::list<CopySetKey> copysetList = GetAvailableCopysetList();
if (copysetList.size() == 0) {
return false;
}

auto it = copySetMap_.find(key);
// random select one copyset
int randomValue = GetOneRandomNumber(0, copysetList.size() - 1);
auto iter = copysetList.begin();
std::advance(iter, randomValue);
auto it = copySetMap_.find(*iter);
if (it != copySetMap_.end()) {
*out = it->second;
return true;
}
return false;
}

int TopologyImpl::GetAvailableCopysetNum() const {
ReadLockGuard rlockCopySet(copySetMutex_);
int num = 0;
for (auto const &it : copySetMap_) {
if (it.second.GetPartitionNum() >= option_.partitionNumberInCopyset) {
continue;
}
num++;
}
return num;
}

std::list<Partition> TopologyImpl::GetPartitionOfFs(
FsIdType id, PartitionFilter filter) const {
std::list<Partition> ret;
Expand Down Expand Up @@ -1257,8 +1281,152 @@ TopoStatusCode TopologyImpl::ChooseNewMetaServerForCopyset(
return TopoStatusCode::TOPO_OK;
}

TopoStatusCode TopologyImpl::ChooseAvailableMetaServers(
std::set<MetaServerIdType> *metaServers, PoolIdType *poolId) {
TopoStatusCode TopologyImpl::GenCopysetAddrRandomBatch(
int needCreateNum, std::list<CopysetCreateInfo>* copysetList) {
ReadLockGuard rlockPool(poolMutex_);
ReadLockGuard rlockMetaserver(metaServerMutex_);

if (needCreateNum == 0) {
LOG(WARNING) << "GenCopysetAddrRandomBatch want to create 0 copyset"
<< "return TOPO_OK";
return TopoStatusCode::TOPO_OK;
}

if (needCreateNum < 0) {
LOG(ERROR) << "GenCopysetAddrRandomBatch needCreateNum = "
<< needCreateNum << ", the num is error";
return TopoStatusCode::TOPO_INTERNAL_ERROR;
}

LOG(INFO) << "GenCopysetAddrRandomBatch needCreateNum = "
<< needCreateNum << " begin";

// 1. genarate replicaMap
std::map<PoolIdType, uint16_t> replicaMap;
for (const auto &it : poolMap_) {
replicaMap.emplace(it.first, it.second.GetReplicaNum());
}

// 2. get all meet the conditions metaserver
std::vector<const MetaServer *> vec;
for (const auto &it : metaServerMap_) {
if (it.second.GetOnlineState() == OnlineState::ONLINE
&& it.second.GetMetaServerSpace().IsMetaserverResourceAvailable()) {
vec.emplace_back(&(it.second));
}
}

// 3. genarate candidateMap
std::map<PoolIdType, std::map<ZoneIdType, std::list<MetaServerIdType>>>
candidateMap;
for (const auto &it : vec) {
Server server;
ServerIdType serverId = it->GetServerId();
if (!GetServer(serverId, &server)) {
LOG(ERROR) << "get server failed when choose metaservers,"
<< " the serverId = " << serverId;
return TopoStatusCode::TOPO_SERVER_NOT_FOUND;
}

PoolIdType poolId = server.GetPoolId();
ZoneIdType zoneId = server.GetZoneId();
auto iter = candidateMap.find(poolId);
if (iter == candidateMap.end()) {
std::map<ZoneIdType, std::list<MetaServerIdType>> tmpZoneMap;
iter = candidateMap.emplace(poolId, tmpZoneMap).first;
}

auto iter2 = iter->second.find(zoneId);
if (iter2 == iter->second.end()) {
std::list<MetaServerIdType> tempMetaseverList;
iter2 = iter->second.emplace(zoneId, tempMetaseverList).first;
}

iter2->second.push_back(it->GetId());
}

// 4. remove pools that do not meet the conditions
for (auto it = candidateMap.begin(); it != candidateMap.end();) {
auto iter = replicaMap.find(it->first);
if (iter == replicaMap.end()) {
LOG(WARNING) << "poolId = " << it->first
<< " can not find in replicaMap";
it = candidateMap.erase(it);
continue;
}

uint16_t replicaNum = iter->second;
if (it->second.size() < replicaNum) {
LOG(WARNING) << "poolId = " << it->first << " need replica num = "
<< replicaNum << ", but only has available zone num = "
<< it->second.size();
it = candidateMap.erase(it);
continue;
}

LOG(INFO) << "find pool available, poolId = " << it->first
<< ", zone num = " << it->second.size()
<< ", replica num = " << replicaNum;
it++;
}

// 5. generate result
if (candidateMap.size() == 0) {
LOG(WARNING) << "can not find available for copyset.";
return TopoStatusCode::TOPO_METASERVER_NOT_FOUND;
}

int createCount = 0;
while (createCount < needCreateNum) {
for (const auto &it : candidateMap) {
CopysetCreateInfo copysetInfo;
copysetInfo.poolId = it.first;
copysetInfo.copysetId = UNINITIALIZE_ID;
uint16_t replicaNum = replicaMap[copysetInfo.poolId];
copysetInfo.metaServerIds =
RandomGetMetaserverIds(it.second, replicaNum);
copysetList->push_back(copysetInfo);

createCount++;
if (needCreateNum == createCount) {
return TopoStatusCode::TOPO_OK;
}
}
}

LOG(ERROR) << "can not find available for copyset.";
return TopoStatusCode::TOPO_METASERVER_NOT_FOUND;
}

std::set<MetaServerIdType> TopologyImpl::RandomGetMetaserverIds(
const std::map<ZoneIdType, std::list<MetaServerIdType>>& zoneMap,
uint16_t num) {
std::vector<MetaServerIdType> metaseverList;
for (const auto &it : zoneMap) {
int randomValue = GetOneRandomNumber(0, it.second.size() - 1);
auto iter = it.second.begin();
std::advance(iter, randomValue);
metaseverList.push_back(*iter);
}

static std::random_device rd;
static std::mt19937 g(rd());
std::shuffle(metaseverList.begin(), metaseverList.end(), g);

std::set<MetaServerIdType> ids;
int count = 0;
for (MetaServerIdType it : metaseverList) {
ids.insert(it);
count++;
if (count == num) {
break;
}
}
return ids;
}

TopoStatusCode TopologyImpl::GenCopysetAddrByResourceUsage(
std::set<MetaServerIdType> *metaServers, PoolIdType *poolId) {
ReadLockGuard rlockMetaserver(metaServerMutex_);
std::vector<const MetaServer *> vec;
for (const auto &it : metaServerMap_) {
Expand Down
54 changes: 47 additions & 7 deletions curvefs/src/mds/topology/topology.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#ifndef CURVEFS_SRC_MDS_TOPOLOGY_TOPOLOGY_H_
#define CURVEFS_SRC_MDS_TOPOLOGY_TOPOLOGY_H_

#include <algorithm>
#include <list>
#include <map>
#include <memory>
Expand All @@ -30,7 +31,6 @@
#include <string>
#include <unordered_map>
#include <vector>
#include <algorithm>

#include "curvefs/proto/mds.pb.h"
#include "curvefs/proto/topology.pb.h"
Expand Down Expand Up @@ -58,6 +58,30 @@ using ServerFilter = std::function<bool(const Server &)>;
using ZoneFilter = std::function<bool(const Zone &)>;
using PoolFilter = std::function<bool(const Pool &)>;

class CopysetCreateInfo {
public:
CopysetCreateInfo() = default;
CopysetCreateInfo(uint32_t poolid, uint32_t copysetid,
std::set<MetaServerIdType> ids) :
poolId(poolid), copysetId(copysetid), metaServerIds(ids) {}

std::string ToString() const {
std::string str;
str = "poolId = " + std::to_string(poolId) + ", copysetId = "
+ std::to_string(copysetId) + ", metaserver list = [";
for (auto it : metaServerIds) {
str = str + std::to_string(it) + ", ";
}
str = str + "]";
return str;
}

public:
uint32_t poolId;
uint32_t copysetId;
std::set<MetaServerIdType> metaServerIds;
};

class Topology {
public:
Topology() {}
Expand Down Expand Up @@ -131,6 +155,8 @@ class Topology {
virtual bool GetCopysetOfPartition(PartitionIdType id,
CopySetInfo *out) const = 0;
virtual bool GetAvailableCopyset(CopySetInfo *out) const = 0;
virtual int GetAvailableCopysetNum() const = 0;
virtual std::list<CopySetKey> GetAvailableCopysetList() const = 0;
virtual bool GetPartition(PartitionIdType partitionId, Partition *out) = 0;

virtual bool GetPool(const std::string &poolName, Pool *out) const = 0;
Expand Down Expand Up @@ -220,9 +246,11 @@ class Topology {
virtual std::list<Partition> GetPartitionInfosInCopyset(
CopySetIdType copysetId) const = 0;

virtual TopoStatusCode ChooseAvailableMetaServers(
std::set<MetaServerIdType> *metaServers,
PoolIdType *poolId) = 0;
virtual TopoStatusCode GenCopysetAddrRandomBatch(
int createNum, std::list<CopysetCreateInfo>* copysetList) = 0;

virtual TopoStatusCode GenCopysetAddrByResourceUsage(
std::set<MetaServerIdType> *replicas, PoolIdType *poolId) = 0;

virtual uint32_t GetPartitionNumberOfFs(FsIdType fsId) = 0;

Expand Down Expand Up @@ -321,6 +349,10 @@ class TopologyImpl : public Topology {
bool GetCopysetOfPartition(PartitionIdType id,
CopySetInfo *out) const override;
bool GetAvailableCopyset(CopySetInfo *out) const override;
int GetAvailableCopysetNum() const override;

std::list<CopySetKey> GetAvailableCopysetList() const override;

bool GetPartition(PartitionIdType partitionId, Partition *out) override;

bool GetPool(const std::string &poolName, Pool *out) const override {
Expand Down Expand Up @@ -432,9 +464,11 @@ class TopologyImpl : public Topology {
const std::set<MetaServerIdType> &unavailableMs,
MetaServerIdType *target) override;

TopoStatusCode ChooseAvailableMetaServers(
std::set<MetaServerIdType> *metaServers,
PoolIdType *poolId) override;
TopoStatusCode GenCopysetAddrRandomBatch(
int createNum, std::list<CopysetCreateInfo>* copysetList) override;

TopoStatusCode GenCopysetAddrByResourceUsage(
std::set<MetaServerIdType> *metaServers, PoolIdType *poolId) override;

uint32_t GetPartitionNumberOfFs(FsIdType fsId);

Expand Down Expand Up @@ -462,6 +496,12 @@ class TopologyImpl : public Topology {

void FlushMetaServerToStorage();

int GetOneRandomNumber(int start, int end) const;

std::set<MetaServerIdType> RandomGetMetaserverIds(
const std::map<ZoneIdType, std::list<MetaServerIdType>>& zoneMap,
uint16_t num);

private:
std::unordered_map<PoolIdType, Pool> poolMap_;
std::unordered_map<ZoneIdType, Zone> zoneMap_;
Expand Down
3 changes: 2 additions & 1 deletion curvefs/src/mds/topology/topology_item.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -288,7 +288,8 @@ class MetaServerSpace {
}

bool IsMetaserverResourceAvailable() {
if (diskThresholdByte_ < (diskCopysetMinRequireByte_ + diskUsedByte_)) {
if (diskThresholdByte_ <= diskUsedByte_ ||
diskThresholdByte_ < (diskCopysetMinRequireByte_ + diskUsedByte_)) {
return false;
}

Expand Down
Loading

0 comments on commit 137bb4e

Please sign in to comment.