CreateVolumeWithPoolsetSpecified

Signed-off-by: jolly-sy <757050468@qq.com>
opencurve · Oct 19, 2022 · a46f82b · a46f82b
1 parent 271a05f
commit a46f82b
Show file tree

Hide file tree

Showing 30 changed files with 220 additions and 110 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -1,6 +1,6 @@
 [submodule "nbd"]
 	path = nbd
-	url = https://gitee.com/NetEase_Curve/curve-nbd
+	url = https://github.com/opencurve/curve-nbd
 [submodule "thirdparties/aws/aws-sdk-cpp"]
 	path = thirdparties/aws/aws-sdk-cpp
 	url = https://github.com/opencurve/aws-sdk-cpp
diff --git a/curvefs/proto/BUILD b/curvefs/proto/BUILD
@@ -125,4 +125,4 @@ cc_proto_library(
 proto_library(
     name = "curvefs_schedule_proto",
     srcs = ["schedule.proto"],
-)
+)
diff --git a/include/client/libcurve.h b/include/client/libcurve.h
@@ -138,7 +138,8 @@ int Create(const char* filename,
  */
 int Create2(const char* filename,
            const C_UserInfo_t* userinfo,
-           size_t size, uint64_t stripeUnit, uint64_t stripeCount);
+           size_t size, uint64_t stripeUnit,
+           uint64_t stripeCount, const char* poolsetName);
 
 /**
  * 同步模式读

diff --git a/proto/nameserver2.proto b/proto/nameserver2.proto
@@ -93,6 +93,7 @@ message FileInfo {
 
     optional    FileThrottleParams throttleParams = 17;
     optional    uint64      epoch = 18;
+    optional    string      poolsetName = 19;
 }
 
 // status code
@@ -202,6 +203,7 @@ message CreateFileRequest {
     required uint64     date = 6;
     optional uint64     stripeUnit = 7;
     optional uint64     stripeCount = 8;
+    optional string     poolsetName = 9;
 };
 
 message CreateFileResponse {

diff --git a/src/client/libcurve_file.cpp b/src/client/libcurve_file.cpp
@@ -319,7 +319,7 @@ int FileClient::Create(const std::string& filename,
     const UserInfo_t& userinfo, size_t size) {
     LIBCURVE_ERROR ret;
     if (mdsClient_ != nullptr) {
-        ret = mdsClient_->CreateFile(filename, userinfo, size);
+        ret = mdsClient_->CreateFile(filename, userinfo, "", size);
         LOG_IF(ERROR, ret != LIBCURVE_ERROR::OK)
             << "Create file failed, filename: " << filename << ", ret: " << ret;
     } else {
@@ -331,11 +331,12 @@ int FileClient::Create(const std::string& filename,
 
 int FileClient::Create2(const std::string& filename,
     const UserInfo_t& userinfo, size_t size,
-    uint64_t stripeUnit, uint64_t stripeCount) {
+    uint64_t stripeUnit, uint64_t stripeCount,
+    const std::string& poolsetName) {
     LIBCURVE_ERROR ret;
     if (mdsClient_ != nullptr) {
-        ret = mdsClient_->CreateFile(filename, userinfo, size, true,
-                                     stripeUnit, stripeCount);
+        ret = mdsClient_->CreateFile(filename, userinfo, poolsetName,
+                                     size, true, stripeUnit, stripeCount);
         LOG_IF(ERROR, ret != LIBCURVE_ERROR::OK)
             << "Create file failed, filename: " << filename << ", ret: " << ret;
     } else {
@@ -575,7 +576,7 @@ int FileClient::Listdir(const std::string& dirpath,
 int FileClient::Mkdir(const std::string& dirpath, const UserInfo_t& userinfo) {
     LIBCURVE_ERROR ret;
     if (mdsClient_ != nullptr) {
-        ret = mdsClient_->CreateFile(dirpath, userinfo, 0, false);
+        ret = mdsClient_->CreateFile(dirpath, userinfo, "", 0, false);
         if (ret != LIBCURVE_ERROR::OK) {
             if (ret == LIBCURVE_ERROR::EXISTS) {
                 LOG(WARNING) << "Create directory failed, " << dirpath
@@ -897,15 +898,16 @@ int Create(const char* filename, const C_UserInfo_t* userinfo, size_t size) {
 }
 
 int Create2(const char* filename, const C_UserInfo_t* userinfo, size_t size,
-                                uint64_t stripeUnit, uint64_t stripeCount) {
+                                uint64_t stripeUnit, uint64_t stripeCount,
+                                const char* poolsetName) {
     if (globalclient == nullptr) {
         LOG(ERROR) << "not inited!";
         return -LIBCURVE_ERROR::FAILED;
     }
 
     return globalclient->Create2(filename,
             UserInfo(userinfo->owner, userinfo->password),
-                       size, stripeUnit, stripeCount);
+                       size, stripeUnit, stripeCount, poolsetName);
 }
 
 int Rename(const C_UserInfo_t* userinfo,

diff --git a/src/client/libcurve_file.h b/src/client/libcurve_file.h
@@ -41,6 +41,7 @@ namespace curve {
 namespace client {
 
 using curve::common::BthreadRWLock;
+using PoolsetType = ::curve::mds::topology::PoolsetType;
 
 class FileClient {
  public:
@@ -111,7 +112,8 @@ class FileClient {
     virtual int Create2(const std::string& filename,
                         const UserInfo_t& userinfo,
                         size_t size, uint64_t stripeUnit,
-                        uint64_t stripeCount);
+                        uint64_t stripeCount,
+                        const std::string& poolsetName);
 
     /**
      * 同步模式读

diff --git a/src/client/mds_client.cpp b/src/client/mds_client.cpp
@@ -308,16 +308,18 @@ LIBCURVE_ERROR MDSClient::OpenFile(const std::string &filename,
 }
 
 LIBCURVE_ERROR MDSClient::CreateFile(const std::string &filename,
-                                     const UserInfo_t &userinfo, size_t size,
+                                     const UserInfo_t &userinfo,
+                                     const std::string &poolsetName,
+                                     size_t size,
                                      bool normalFile, uint64_t stripeUnit,
                                      uint64_t stripeCount) {
     auto task = RPCTaskDefine {
         CreateFileResponse response;
         mdsClientMetric_.createFile.qps.count << 1;
         LatencyGuard lg(&mdsClientMetric_.createFile.latency);
-        MDSClientBase::CreateFile(filename, userinfo, size, normalFile,
-                                  stripeUnit, stripeCount, &response, cntl,
-                                  channel);
+        MDSClientBase::CreateFile(filename, poolsetName, userinfo, size,
+                                  normalFile, stripeUnit, stripeCount,
+                                  &response, cntl, channel);
 
         if (cntl->Failed()) {
             mdsClientMetric_.createFile.eps.count << 1;

diff --git a/src/client/mds_client.h b/src/client/mds_client.h
@@ -181,7 +181,9 @@ class MDSClient : public MDSClientBase,
      *          如果认证失败返回LIBCURVE_ERROR::AUTHFAIL，
      */
     LIBCURVE_ERROR CreateFile(const std::string &filename,
-                              const UserInfo_t &userinfo, size_t size = 0,
+                              const UserInfo_t &userinfo,
+                              const std::string &poolsetName = "",
+                              size_t size = 0,
                               bool normalFile = true, uint64_t stripeUnit = 0,
                               uint64_t stripeCount = 0);
     /**

diff --git a/src/client/mds_client_base.cpp b/src/client/mds_client_base.cpp
@@ -61,6 +61,7 @@ void MDSClientBase::OpenFile(const std::string& filename,
 }
 
 void MDSClientBase::CreateFile(const std::string& filename,
+                               const std::string& poolsetName,
                                const UserInfo_t& userinfo,
                                size_t size,
                                bool normalFile,
@@ -80,14 +81,16 @@ void MDSClientBase::CreateFile(const std::string& filename,
 
     request.set_stripeunit(stripeUnit);
     request.set_stripecount(stripeCount);
+    request.set_poolsetname(poolsetName);
     FillUserInfo(&request, userinfo);
 
     LOG(INFO) << "CreateFile: filename = " << filename
                 << ", owner = " << userinfo.owner
                 << ", is normalfile: " << normalFile
                 << ", log id = " << cntl->log_id()
                 << ", stripeUnit = " << stripeUnit
-                << ", stripeCount = " << stripeCount;
+                << ", stripeCount = " << stripeCount
+                << ", poolsetName = " << poolsetName;
 
     curve::mds::CurveFSService_Stub stub(channel);
     stub.CreateFile(cntl, &request, response, NULL);

diff --git a/src/client/mds_client_base.h b/src/client/mds_client_base.h
@@ -120,6 +120,7 @@ class MDSClientBase {
      * @param[in]:channel是当前与mds建立的通道
      */
     void CreateFile(const std::string& filename,
+                    const std::string& poolsetName,
                     const UserInfo_t& userinfo,
                     size_t size,
                     bool normalFile,

diff --git a/src/mds/nameserver2/chunk_allocator.cpp b/src/mds/nameserver2/chunk_allocator.cpp
@@ -29,7 +29,8 @@ namespace curve {
 namespace mds {
 bool ChunkSegmentAllocatorImpl::AllocateChunkSegment(FileType type,
         SegmentSizeType segmentSize, ChunkSizeType chunkSize,
-        offset_t offset, PageFileSegment *segment)  {
+        PoolsetType pType, offset_t offset,
+        PageFileSegment *segment)  {
         if (segment == nullptr) {
             LOG(ERROR) << "segment pointer is null";
             return false;
@@ -54,7 +55,7 @@ bool ChunkSegmentAllocatorImpl::AllocateChunkSegment(FileType type,
         std::vector<CopysetIdInfo> copysets;
         if (!topologyChunkAllocator_->
                 AllocateChunkRoundRobinInSingleLogicalPool(
-                type, chunkNum, chunkSize, &copysets)) {
+                type, pType, chunkNum, chunkSize, &copysets)) {
             LOG(ERROR) << "AllocateChunkRoundRobinInSingleLogicalPool error";
             return false;
         }

diff --git a/src/mds/nameserver2/chunk_allocator.h b/src/mds/nameserver2/chunk_allocator.h
@@ -32,6 +32,7 @@
 #include "src/mds/topology/topology_chunk_allocator.h"
 
 using ::curve::mds::topology::TopologyChunkAllocator;
+using ::curve::mds::topology::PoolsetType;
 
 namespace curve {
 namespace mds {
@@ -42,10 +43,13 @@ class ChunkSegmentAllocator {
 
     virtual bool AllocateChunkSegment(FileType type,
         SegmentSizeType segmentSize, ChunkSizeType chunkSize,
-        offset_t offset, PageFileSegment *segment) = 0;
+        PoolsetType pType, offset_t offset,
+        PageFileSegment *segment) = 0;
+
     virtual void GetRemainingSpaceInLogicalPool(
         const std::vector<PoolIdType>& logicalPools,
-        std::map<PoolIdType, double>* remianingSpace) = 0;
+        std::map<PoolIdType, double>* remianingSpace,
+        curve::mds::topology::PoolsetType pType) = 0;
 };
 
 
@@ -67,13 +71,15 @@ class ChunkSegmentAllocatorImpl: public ChunkSegmentAllocator {
 
     bool AllocateChunkSegment(FileType type,
         SegmentSizeType segmentSize, ChunkSizeType chunkSize,
-        offset_t offset, PageFileSegment *segment) override;
+        PoolsetType pType, offset_t offset,
+        PageFileSegment *segment) override;
 
     void GetRemainingSpaceInLogicalPool(
         const std::vector<PoolIdType>& logicalPools,
-        std::map<PoolIdType, double>* remianingSpace) {
+        std::map<PoolIdType, double>* remianingSpace,
+        curve::mds::topology::PoolsetType pType) {
             return topologyChunkAllocator_->GetRemainingSpaceInLogicalPool(
-                            logicalPools, remianingSpace);
+                            logicalPools, remianingSpace, pType);
         }
 
  private:

diff --git a/src/mds/nameserver2/curveadm b/src/mds/nameserver2/curveadm
diff --git a/src/mds/nameserver2/curvefs.cpp b/src/mds/nameserver2/curvefs.cpp
@@ -45,6 +45,7 @@ using curve::mds::topology::CopySetIdType;
 using curve::mds::topology::ChunkServer;
 using curve::mds::topology::ChunkServerStatus;
 using curve::mds::topology::OnlineState;
+using curve::mds::topology::Poolset;
 
 namespace curve {
 namespace mds {
@@ -245,6 +246,7 @@ StatusCode CurveFS::SnapShotFile(const FileInfo * origFileInfo,
 }
 
 StatusCode CurveFS::CreateFile(const std::string & fileName,
+                               const std::string & poolsetName,
                                const std::string& owner,
                                FileType filetype, uint64_t length,
                                uint64_t stripeUnit, uint64_t stripeCount) {
@@ -281,13 +283,17 @@ StatusCode CurveFS::CreateFile(const std::string & fileName,
             return StatusCode::kFileLengthNotSupported;
         }
     }
-
+     
     std::vector<PoolIdType> logicalPools =
                 topology_->GetLogicalPoolInCluster();
     std::map<PoolIdType, double> remianingSpace;
     uint64_t allRemianingSpace = 0;
+
+    Poolset poolset;
+    topology_->GetPoolset(poolsetName, &poolset);
+    PoolsetType pType = poolset.GetType();
     chunkSegAllocator_->GetRemainingSpaceInLogicalPool(logicalPools,
-                                &remianingSpace);
+                                &remianingSpace, pType);
 
     for (auto it = remianingSpace.begin(); it != remianingSpace.end(); it++) {
         allRemianingSpace +=it->second;
@@ -327,6 +333,7 @@ StatusCode CurveFS::CreateFile(const std::string & fileName,
 
         fileInfo.set_id(inodeID);
         fileInfo.set_filename(lastEntry);
+        fileInfo.set_poolsetname(poolsetName);
         fileInfo.set_parentid(parentFileInfo.id());
         fileInfo.set_filetype(filetype);
         fileInfo.set_owner(owner);
@@ -1251,9 +1258,15 @@ StatusCode CurveFS::GetOrAllocateSegment(const std::string & filename,
             return  StatusCode::kSegmentNotAllocated;
         } else {
             // TODO(hzsunjianliang): check the user and define the logical pool
+            if (!fileInfo.has_poolsetname()) {
+                fileInfo.set_poolsetname("ssdPoolset1");
+            }
+            Poolset poolset;
+            topology_->GetPoolset(fileInfo.poolsetname(), &poolset);
             auto ifok = chunkSegAllocator_->AllocateChunkSegment(
                             fileInfo.filetype(), fileInfo.segmentsize(),
-                            fileInfo.chunksize(), offset, segment);
+                            fileInfo.chunksize(), poolset.GetType(),
+                            offset, segment);
             if (ifok == false) {
                 LOG(ERROR) << "AllocateChunkSegment error";
                 return StatusCode::kSegmentAllocateError;

diff --git a/src/mds/nameserver2/curvefs.h b/src/mds/nameserver2/curvefs.h
@@ -31,6 +31,7 @@
 #include <chrono>  //NOLINT
 #include <unordered_map>
 #include "proto/nameserver2.pb.h"
+// #include "proto/topology.pb.h"
 #include "src/mds/nameserver2/namespace_storage.h"
 #include "src/mds/common/mds_define.h"
 #include "src/mds/nameserver2/chunk_allocator.h"
@@ -43,6 +44,7 @@
 #include "src/mds/snapshotcloneclient/snapshotclone_client.h"
 #include "src/mds/topology/topology_service_manager.h"
 
+using curve::mds::topology::PoolsetType;
 using curve::common::Authenticator;
 using curve::mds::snapshotcloneclient::SnapshotCloneClient;
 using curve::common::ChunkServerLocation;
@@ -141,6 +143,7 @@ class CurveFS {
      *  @return return StatusCode::kOK if succeeded
      */
     StatusCode CreateFile(const std::string & fileName,
+                          const std::string & poolsetName,
                           const std::string& owner,
                           FileType filetype,
                           uint64_t length,

diff --git a/src/mds/nameserver2/namespace_service.cpp b/src/mds/nameserver2/namespace_service.cpp
@@ -85,9 +85,9 @@ void NameSpaceService::CreateFile(::google::protobuf::RpcController* controller,
         return;
     }
 
-    retCode = kCurveFS.CreateFile(request->filename(), request->owner(),
-            request->filetype(), request->filelength(), request->stripeunit(),
-                                              request->stripecount());
+    retCode = kCurveFS.CreateFile(request->filename(), request->poolsetname(),
+            request->owner(), request->filetype(), request->filelength(),
+            request->stripeunit(), request->stripecount());
     if (retCode != StatusCode::kOK)  {
         response->set_statuscode(retCode);
         // TODO(hzsunjianliang): check if we should really print error here

diff --git a/src/mds/topology/topology.h b/src/mds/topology/topology.h
@@ -216,7 +216,8 @@ class Topology {
         const std::string &hostIp,
         uint32_t port) const = 0;
 
-    virtual bool GetPoolset(PoolsetIdType poolsetId, Poolset *out) const = 0;
+    virtual bool GetPoolset(PoolsetIdType poolsetId,
+        Poolset *out) const = 0;
 
     virtual bool GetLogicalPool(PoolIdType poolId,
                                 LogicalPool *out) const = 0;