Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[feat]tools-v2: Implement bs list poolset command #2984

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

[feat]tools-v2: Implement bs list poolset command #2984

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 3 additions & 0 deletions proto/nameserver2.proto
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -539,12 +539,15 @@ message GetAllocatedSizeResponse {

message GetFileSizeRequest {
required string fileName = 1;
optional bool groupByPoolset = 2 [default = false];
}

message GetFileSizeResponse {
required StatusCode statusCode = 1;
// 文件或目录的file length
optional uint64 fileSize = 2;
// key is poolset name and value is total file size on this poolset
map<string, uint64> fileSizeMap = 3;
}

message ClientInfo {
Expand Down
21 changes: 11 additions & 10 deletions src/mds/nameserver2/curvefs.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -512,9 +512,9 @@ StatusCode CurveFS::GetDirAllocSize(const std::string& fileName,
return StatusCode::kOK;
}

StatusCode CurveFS::GetFileSize(const std::string& fileName, uint64_t* size) {
assert(size != nullptr);
*size = 0;
StatusCode CurveFS::GetFileSize(const std::string& fileName,
std::map<std::string, uint64_t>* fileSizeMap) {
assert(fileSizeMap != nullptr);
FileInfo fileInfo;
auto ret = GetFileInfo(fileName, &fileInfo);
if (ret != StatusCode::kOK) {
Expand All @@ -527,21 +527,24 @@ StatusCode CurveFS::GetFileSize(const std::string& fileName, uint64_t* size) {
<< fileInfo.filetype() << ", fileName = " << fileName;
return StatusCode::kNotSupported;
}
return GetFileSize(fileName, fileInfo, size);
return GetFileSize(fileName, fileInfo, fileSizeMap);
}

StatusCode CurveFS::GetFileSize(const std::string& fileName,
const FileInfo& fileInfo,
uint64_t* fileSize) {
std::map<std::string, uint64_t>* fileSizeMap) {
// return file length if it is a file
switch (fileInfo.filetype()) {
case FileType::INODE_PAGEFILE: {
*fileSize = fileInfo.length();
if (fileInfo.has_poolset()) {
(*fileSizeMap)[fileInfo.poolset()] += fileInfo.length();
} else {
(*fileSizeMap)[kDefaultPoolsetName] += fileInfo.length();
}
return StatusCode::kOK;
}
case FileType::INODE_SNAPSHOT_PAGEFILE: {
// Do not count snapshot file size, set fileSize=0
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe this comment can be deleted.

*fileSize = 0;
return StatusCode::kOK;
}
case FileType::INODE_DIRECTORY: {
Expand Down Expand Up @@ -569,14 +572,12 @@ StatusCode CurveFS::GetFileSize(const std::string& fileName,
} else {
fullPathName = fileName + "/" + file.filename();
}
uint64_t size = 0;
ret = GetFileSize(fullPathName, file, &size);
ret = GetFileSize(fullPathName, file, fileSizeMap);
if (ret != StatusCode::kOK) {
LOG(ERROR) << "Get file size of " << fullPathName
<< " fail, error code: " << ret;
return ret;
}
*fileSize += size;
}
return StatusCode::kOK;
}
Expand Down
9 changes: 5 additions & 4 deletions src/mds/nameserver2/curvefs.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,10 +187,11 @@ class CurveFS {
/**
* @brief get size of the file or directory
* @brief fileName
* @param[out]: size: the fileLength of the file or directory
* @param[out]: fileSizeMap: total file size of corresponding poolset
* @return StatusCode::kOK if succeeded
*/
StatusCode GetFileSize(const std::string& fileName, uint64_t* size);
StatusCode GetFileSize(const std::string& fileName,
std::map<std::string, uint64_t>* fileSizeMap);

/**
* @brief delete file
Expand Down Expand Up @@ -764,12 +765,12 @@ class CurveFS {
* @brief get the size of file or directory
* @param: fileName
* @param: fileInfo
* @param[out]: fileSize: the size of file or directory
* @param[out]: fileSizeMap: total file size of corresponding poolset
* @return StatusCode::kOK if succeeded
*/
StatusCode GetFileSize(const std::string& fileName,
const FileInfo& fileInfo,
uint64_t* fileSize);
std::map<std::string, uint64_t>* fileSizeMap);

/**
* @brief check file has rely dest file
Expand Down
24 changes: 18 additions & 6 deletions src/mds/nameserver2/namespace_service.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
#include <algorithm>
#include <vector>
#include <string>
#include <map>
#include <memory>
#include <utility>
#include "src/mds/nameserver2/curvefs.h"
Expand Down Expand Up @@ -2002,13 +2003,12 @@ void NameSpaceService::GetFileSize(
brpc::ClosureGuard doneGuard(done);
brpc::Controller* cntl = static_cast<brpc::Controller*>(controller);


LOG(INFO) << "logid = " << cntl->log_id()
<< ", GetFileSize request, fileName = " << request->filename();

StatusCode retCode;
uint64_t fileSize = 0;
retCode = kCurveFS.GetFileSize(request->filename(), &fileSize);
std::map<std::string, uint64_t> fileSizeMap;
retCode = kCurveFS.GetFileSize(request->filename(), &fileSizeMap);
if (retCode != StatusCode::kOK) {
response->set_statuscode(retCode);
LOG(ERROR) << "logid = " << cntl->log_id()
Expand All @@ -2018,10 +2018,22 @@ void NameSpaceService::GetFileSize(
return;
} else {
response->set_statuscode(StatusCode::kOK);
response->set_filesize(fileSize);

if (request->groupbypoolset()) {
auto* map = response->mutable_filesizemap();
for (const auto& p : fileSizeMap) {
(*map)[p.first] = p.second;
}
} else {
uint64_t totalSize = 0;
for (const auto& p : fileSizeMap) {
totalSize += p.second;
}
response->set_filesize(totalSize);
}
LOG(INFO) << "logid = " << cntl->log_id()
<< ", GetFileSize ok, fileName = " << request->filename()
<< ", fileSize = " << response->filesize() / kGB << "GB";
<< ", GetFileSize ok, response: "
<< response->ShortDebugString();
}
return;
}
Expand Down
100 changes: 92 additions & 8 deletions test/mds/nameserver2/curvefs_test.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1221,8 +1221,17 @@ TEST_F(CurveFSTest, testGetAllocatedSize) {
}
}

namespace {
uint64_t TotalSize(const std::map<std::string, uint64_t>& fileSizeMap) {
uint64_t total = 0;
for (const auto& p : fileSizeMap) {
total += p.second;
}
return total;
}
} // namespace

TEST_F(CurveFSTest, testGetFileSize) {
uint64_t fileSize;
FileInfo fileInfo;
fileInfo.set_id(0);
fileInfo.set_filetype(FileType::INODE_PAGEFILE);
Expand All @@ -1234,9 +1243,10 @@ TEST_F(CurveFSTest, testGetFileSize) {
.Times(1)
.WillOnce(DoAll(SetArgPointee<2>(fileInfo),
Return(StoreStatus::OK)));
std::map<std::string, uint64_t> fileSizeMap;
ASSERT_EQ(StatusCode::kOK,
curvefs_->GetFileSize("/tests", &fileSize));
ASSERT_EQ(10 * kGB, fileSize);
curvefs_->GetFileSize("/tests", &fileSizeMap));
ASSERT_EQ(10 * kGB, TotalSize(fileSizeMap));
}
// test directory normal
{
Expand All @@ -1254,17 +1264,19 @@ TEST_F(CurveFSTest, testGetFileSize) {
.Times(1)
.WillOnce(DoAll(SetArgPointee<2>(files),
Return(StoreStatus::OK)));
std::map<std::string, uint64_t> fileSizeMap;
ASSERT_EQ(StatusCode::kOK,
curvefs_->GetFileSize("/tests", &fileSize));
ASSERT_EQ(30 * kGB, fileSize);
curvefs_->GetFileSize("/tests", &fileSizeMap));
ASSERT_EQ(30 * kGB, TotalSize(fileSizeMap));
}
// test GetFile fail
{
EXPECT_CALL(*storage_, GetFile(_, _, _))
.Times(1)
.WillOnce(Return(StoreStatus::KeyNotExist));
std::map<std::string, uint64_t> fileSizeMap;
ASSERT_EQ(StatusCode::kFileNotExists,
curvefs_->GetFileSize("/tests", &fileSize));
curvefs_->GetFileSize("/tests", &fileSizeMap));
}
// test file type not supported
{
Expand All @@ -1274,8 +1286,9 @@ TEST_F(CurveFSTest, testGetFileSize) {
.Times(1)
.WillOnce(DoAll(SetArgPointee<2>(appendFileInfo),
Return(StoreStatus::OK)));
std::map<std::string, uint64_t> fileSizeMap;
ASSERT_EQ(StatusCode::kNotSupported,
curvefs_->GetFileSize("/tests", &fileSize));
curvefs_->GetFileSize("/tests", &fileSizeMap));
}
// test list directory fail
{
Expand All @@ -1288,11 +1301,82 @@ TEST_F(CurveFSTest, testGetFileSize) {
EXPECT_CALL(*storage_, ListFile(_, _, _))
.Times(1)
.WillOnce(Return(StoreStatus::InternalError));
std::map<std::string, uint64_t> fileSizeMap;
ASSERT_EQ(StatusCode::kStorageError,
curvefs_->GetFileSize("/tests", &fileSize));
curvefs_->GetFileSize("/tests", &fileSizeMap));
}
}

TEST_F(CurveFSTest, testGetFileSizeGroupByPoolset) {
// /
// ├──A (no poolset info)
// └──B
// ├── C (poolset a)
// ├── D (poolset b)
// └── E (poolset a)
FileInfo root;
root.set_id(0);
root.set_filename("/");
root.set_filetype(FileType::INODE_DIRECTORY);

FileInfo fileInfoA;
fileInfoA.set_id(1);
fileInfoA.set_filename("A");
fileInfoA.set_filetype(FileType::INODE_PAGEFILE);
fileInfoA.set_length(10 * kGB);

FileInfo dirB;
dirB.set_id(2);
dirB.set_filename("B");
dirB.set_filetype(FileType::INODE_DIRECTORY);

FileInfo fileInfoC;
fileInfoC.set_id(3);
fileInfoC.set_filename("C");
fileInfoC.set_filetype(FileType::INODE_PAGEFILE);
fileInfoC.set_length(10 * kGB);
fileInfoC.set_poolset("poolset-a");

FileInfo fileInfoD;
fileInfoD.set_id(4);
fileInfoD.set_filename("D");
fileInfoD.set_filetype(FileType::INODE_PAGEFILE);
fileInfoD.set_length(15 * kGB);
fileInfoD.set_poolset("poolset-b");

FileInfo fileInfoE;
fileInfoE.set_id(5);
fileInfoE.set_filename("E");
fileInfoE.set_filetype(FileType::INODE_PAGEFILE);
fileInfoE.set_length(20 * kGB);
fileInfoE.set_poolset("poolset-a");

EXPECT_CALL(*storage_, GetFile(_, _, _))
.WillOnce(DoAll(SetArgPointee<2>(root), Return(StoreStatus::OK)));

std::vector<FileInfo> listRootResults;
listRootResults.push_back(fileInfoA);
listRootResults.push_back(dirB);

std::vector<FileInfo> listDirBResults;
listDirBResults.push_back(fileInfoC);
listDirBResults.push_back(fileInfoD);
listDirBResults.push_back(fileInfoE);

EXPECT_CALL(*storage_, ListFile(_, _, _))
.WillOnce(
DoAll(SetArgPointee<2>(listRootResults), Return(StoreStatus::OK)))
.WillOnce(
DoAll(SetArgPointee<2>(listDirBResults), Return(StoreStatus::OK)));

std::map<std::string, uint64_t> fileSizeMap;
ASSERT_EQ(StatusCode::kOK, curvefs_->GetFileSize("/", &fileSizeMap));
ASSERT_EQ(3, fileSizeMap.size());
ASSERT_EQ(10 * kGB, fileSizeMap[kDefaultPoolsetName]);
ASSERT_EQ(30 * kGB, fileSizeMap["poolset-a"]);
ASSERT_EQ(15 * kGB, fileSizeMap["poolset-b"]);
}

TEST_F(CurveFSTest, testReadDir) {
FileInfo fileInfo;
std::vector<FileInfo> items;
Expand Down
2 changes: 2 additions & 0 deletions tools-v2/internal/utils/row.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ const (
ROW_PEER_ID = "peerId"
ROW_PEER_NUMBER = "peerNumber"
ROW_PHYPOOL = "phyPool"
ROW_PHYPOOLS = "phyPools"
ROW_PATH = "path"
ROW_POOL = "pool"
ROW_POOL_ID = "poolId"
Expand Down Expand Up @@ -138,6 +139,7 @@ const (
ROW_NEXTSTEP = "nextStep"
ROW_TIME = "time"
ROW_SNAPSHOTNAME = "snapshotName"
ROW_ALLOC_PERCENT = "alloc percent(%)"

ROW_RW_STATUS = "rwStatus"
ROW_DISK_STATE = "diskState"
Expand Down
2 changes: 2 additions & 0 deletions tools-v2/pkg/cli/command/curvebs/list/list.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (
"github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/list/formatstatus"
logicalpool "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/list/logicalPool"
may_broken_vol "github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/list/may-broken-vol"
"github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/list/poolset"
"github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/list/scanstatus"
"github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/list/server"
"github.com/opencurve/curve/tools-v2/pkg/cli/command/curvebs/list/snapshot"
Expand All @@ -54,6 +55,7 @@ func (listCmd *ListCommand) AddSubCommands() {
scanstatus.NewScanStatusCommand(),
may_broken_vol.NewMayBrokenVolCommand(),
formatstatus.NewFormatStatusCommand(),
poolset.NewPoolsetCommand(),
snapshot.NewSnapShotCommand(),
)
}
Expand Down