curvefs: support space deallocate for curvebs volume as backend

Signed-off-by: ilixiaocui <ilixiaocui@163.com>

curvefs/proto/common.proto

@@ -31,6 +31,13 @@ enum BitmapLocation {
     AtEnd = 2;
 }
 
+message EmptyMsg {}
+
+message BlockGroupID {
+    required uint64 fsId = 1;
+    required uint64 offset = 2;
+}
+
 // When creating fs, `volumeSize` and `extendAlignment` are fetched from the bs cluster
 message Volume {
     optional uint64 volumeSize = 1;

curvefs/proto/metaserver.proto

@@ -167,6 +167,51 @@ enum FsFileType {
     TYPE_S3 = 4;
 };
 
+enum DeallocatableBlockGroupAction {
+    TYPE_INCREASE = 1;
+    TYPE_DECREASE = 2;
+    TYPE_MARK = 3;
+}
+
+message DeallocatableBlockGroup {
+    required uint64 volumeOffset = 1;
+    optional uint64 deallocatableSize = 2;
+    repeated uint64 inodeIdlist = 3;
+    repeated uint64 inodeIdUnderDeallocate = 4;
+
+    oneof type {
+        IncreaseDeallocatableBlockGroup increase = 5;
+        DecreaseDeallocatableBlockGroup decrease = 6;
+        MarkDeallocatableBlockGroup mark = 7;
+    }
+}
+
+message IncreaseDeallocatableBlockGroup {
+    required uint64 increaseDeallocatableSize = 1;
+    repeated uint64 inodeIdlistAdd = 2;
+}
+
+message DecreaseDeallocatableBlockGroup {
+    required uint64 decreaseDeallocatableSize = 1;
+}
+
+message MarkDeallocatableBlockGroup {
+    repeated uint64 inodeIdUnderDeallocate =2;
+}
+
+message UpdateDeallocatableBlockGroupRequest {
+    required uint32 poolId = 1;
+    required uint32 copysetId = 2;
+    required uint32 partitionId = 3;
+    required uint64 fsId = 4;
+    repeated DeallocatableBlockGroup update = 5;
+}
+
+message UpdateDeallocatableBlockGroupResponse {
+    required MetaStatusCode statusCode = 1; 
+    optional uint64 appliedIndex = 2;
+}
+
 message VolumeExtent {
     required uint64 fsOffset = 1;
     required uint64 volumeOffset = 2;
@@ -179,7 +224,7 @@ message VolumeExtentSlice {
     repeated VolumeExtent extents = 2;
 }
 
-message VolumeExtentList {
+message VolumeExtenSlicetList {
     repeated VolumeExtentSlice slices = 1;
 }
 
@@ -317,7 +362,7 @@ message UpdateInodeRequest {
     map<string, bytes> xattr = 20;
     repeated uint64 parent = 21;
     map<uint64, S3ChunkInfoList> s3ChunkInfoAdd = 22;
-    optional VolumeExtentList volumeExtents = 23;
+    optional VolumeExtenSlicetList volumeExtents = 23;
 }
 
 message UpdateInodeResponse {
@@ -464,7 +509,7 @@ message GetVolumeExtentRequest {
 message GetVolumeExtentResponse {
     required MetaStatusCode statusCode = 1;
     optional uint64 appliedIndex = 2;
-    optional VolumeExtentList slices = 3;
+    optional VolumeExtenSlicetList slices = 3;
 }
 
 message UpdateVolumeExtentRequest {
@@ -473,7 +518,7 @@ message UpdateVolumeExtentRequest {
     required uint32 partitionId = 3;
     required uint32 fsId = 4;
     required uint64 inodeId = 5;
-    required VolumeExtentList extents = 6;
+    required VolumeExtenSlicetList extents = 6;
 }
 
 message UpdateVolumeExtentResponse {
@@ -508,4 +553,7 @@ service MetaServerService {
     // volume extent interface
     rpc GetVolumeExtent(GetVolumeExtentRequest) returns (GetVolumeExtentResponse);
     rpc UpdateVolumeExtent(UpdateVolumeExtentRequest) returns (UpdateVolumeExtentResponse);
+
+    // block group with deallocatable inode list interface
+    rpc UpdateDeallocatableBlockGroup(UpdateDeallocatableBlockGroupRequest) returns (UpdateDeallocatableBlockGroupResponse);
 }

curvefs/src/client/inode_wrapper.cpp

@@ -582,7 +582,7 @@ void InodeWrapper::AsyncS3(MetaServerClientDone *done, bool internal) {
 }
 
 CURVEFS_ERROR InodeWrapper::RefreshVolumeExtent() {
-    VolumeExtentList extents;
+    VolumeExtenSlicetList extents;
     auto st = metaClient_->GetVolumeExtent(inode_.fsid(), inode_.inodeid(),
                                            true, &extents);
     VLOG(9) << "RefreshVolumeExtent, ino: " << inode_.inodeid()

curvefs/src/client/inode_wrapper.h

@@ -52,7 +52,7 @@ constexpr int kAccessTime = 1 << 0;
 constexpr int kChangeTime = 1 << 1;
 constexpr int kModifyTime = 1 << 2;
 
-using ::curvefs::metaserver::VolumeExtentList;
+using ::curvefs::metaserver::VolumeExtenSlicetList;
 
 enum class InodeStatus {
     kNormal = 0,

curvefs/src/client/rpcclient/metaserver_client.cpp

@@ -1459,13 +1459,13 @@ void UpdateVolumeExtentRpcDone::Run() {
 void MetaServerClientImpl::AsyncUpdateVolumeExtent(
     uint32_t fsId,
     uint64_t inodeId,
-    const VolumeExtentList &extents,
+    const VolumeExtenSlicetList &extents,
     MetaServerClientDone *done) {
     auto task = AsyncRPCTask {
         metric_.updateVolumeExtent.qps.count << 1;
         metaserver::UpdateVolumeExtentRequest request;
         SET_COMMON_FIELDS;
-        request.set_allocated_extents(new VolumeExtentList{extents});
+        request.set_allocated_extents(new VolumeExtenSlicetList{extents});
 
         auto *rpcDone =
             new UpdateVolumeExtentRpcDone(taskExecutorDone, &metric_);
@@ -1485,7 +1485,7 @@ void MetaServerClientImpl::AsyncUpdateVolumeExtent(
 namespace {
 
 struct ParseVolumeExtentCallBack {
-    explicit ParseVolumeExtentCallBack(VolumeExtentList *ext) : extents(ext) {}
+    explicit ParseVolumeExtentCallBack(VolumeExtenSlicetList *ext) : extents(ext) {}
 
     bool operator()(butil::IOBuf *data) const {
         metaserver::VolumeExtentSlice slice;
@@ -1498,7 +1498,7 @@ struct ParseVolumeExtentCallBack {
         return true;
     }
 
-    VolumeExtentList *extents;
+    VolumeExtenSlicetList *extents;
 };
 
 }  // namespace
@@ -1507,7 +1507,7 @@ MetaStatusCode MetaServerClientImpl::GetVolumeExtent(
     uint32_t fsId,
     uint64_t inodeId,
     bool streaming,
-    VolumeExtentList *extents) {
+    VolumeExtenSlicetList *extents) {
     auto task = RPCTask {
         (void)txId;
         (void)applyIndex;

curvefs/src/client/rpcclient/metaserver_client.h

@@ -49,21 +49,24 @@ using ::curvefs::metaserver::InodeAttr;
 using ::curvefs::metaserver::XAttr;
 using ::curvefs::metaserver::MetaStatusCode;
 using ::curvefs::metaserver::S3ChunkInfoList;
+using ::curvefs::metaserver::DeallocatableBlockGroup;
 using ::curvefs::common::StreamStatus;
 using ::curvefs::common::StreamClient;
-using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
 using ::curvefs::metaserver::Time;
 
+using DeallocatableBlockGroupMap = std::map<uint64_t, DeallocatableBlockGroup>;
+using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
+
 namespace curvefs {
 namespace client {
 namespace rpcclient {
 
 using S3ChunkInfoMap = google::protobuf::Map<uint64_t, S3ChunkInfoList>;
-using ::curvefs::metaserver::VolumeExtentList;
+using ::curvefs::metaserver::VolumeExtenSlicetList;
 
 struct DataIndices {
     absl::optional<S3ChunkInfoMap> s3ChunkInfoMap;
-    absl::optional<VolumeExtentList> volumeExtents;
+    absl::optional<VolumeExtenSlicetList> volumeExtents;
 };
 
 class MetaServerClient {
@@ -161,13 +164,17 @@ class MetaServerClient {
 
     virtual void AsyncUpdateVolumeExtent(uint32_t fsId,
                                          uint64_t inodeId,
-                                         const VolumeExtentList &extents,
+                                         const VolumeExtenSlicetList &extents,
                                          MetaServerClientDone *done) = 0;
 
     virtual MetaStatusCode GetVolumeExtent(uint32_t fsId,
                                            uint64_t inodeId,
                                            bool streaming,
-                                           VolumeExtentList *extents) = 0;
+                                           VolumeExtenSlicetList *extents) = 0;
+
+    virtual MetaStatusCode UpdateDeallocatableBlockGroup(
+        uint32_t fsId,
+        DeallocatableBlockGroupMap *statistic) = 0;
 };
 
 class MetaServerClientImpl : public MetaServerClient {
@@ -266,13 +273,17 @@ class MetaServerClientImpl : public MetaServerClient {
 
     void AsyncUpdateVolumeExtent(uint32_t fsId,
                                  uint64_t inodeId,
-                                 const VolumeExtentList &extents,
+                                 const VolumeExtenSlicetList &extents,
                                  MetaServerClientDone *done) override;
 
     MetaStatusCode GetVolumeExtent(uint32_t fsId,
                                    uint64_t inodeId,
                                    bool streaming,
-                                   VolumeExtentList *extents) override;
+                                   VolumeExtenSlicetList *extents) override;
+
+    MetaStatusCode UpdateDeallocatableBlockGroup(
+        uint32_t fsId,
+        DeallocatableBlockGroupMap *statistic) override;
 
  private:
     MetaStatusCode UpdateInode(const UpdateInodeRequest &request,

curvefs/src/client/volume/extent_cache.cpp

@@ -228,7 +228,7 @@ void ExtentCache::SetOption(const ExtentCacheOption& option) {
     LOG(INFO) << "ExtentCacheOption: [" << option_ << "]";
 }
 
-void ExtentCache::Build(const VolumeExtentList &extents) {
+void ExtentCache::Build(const VolumeExtenSlicetList &extents) {
     WriteLockGuard lk(lock_);
     slices_.clear();
     dirties_.clear();
@@ -238,8 +238,8 @@ void ExtentCache::Build(const VolumeExtentList &extents) {
     }
 }
 
-VolumeExtentList ExtentCache::GetDirtyExtents() {
-    VolumeExtentList result;
+VolumeExtenSlicetList ExtentCache::GetDirtyExtents() {
+    VolumeExtenSlicetList result;
     WriteLockGuard lk(lock_);
     for (const auto* slice : dirties_) {
         *result.add_slices() = slice->ToVolumeExtentSlice();

curvefs/src/client/volume/extent_cache.h

@@ -43,7 +43,7 @@ namespace client {
 using ::curvefs::volume::ReadPart;
 using ::curvefs::volume::WritePart;
 using ::curvefs::metaserver::VolumeExtentSlice;
-using ::curvefs::metaserver::VolumeExtentList;
+using ::curvefs::metaserver::VolumeExtenSlicetList;
 
 struct ExtentCacheOption {
     // preallocation size if offset ~ length is not allocated
@@ -61,7 +61,7 @@ class ExtentCache {
 
     static void SetOption(const ExtentCacheOption& option);
 
-    void Build(const VolumeExtentList& extents);
+    void Build(const VolumeExtenSlicetList& extents);
 
     void DivideForWrite(uint64_t offset,
                         uint64_t len,
@@ -81,7 +81,7 @@ class ExtentCache {
 
     bool HasDirtyExtents() const;
 
-    VolumeExtentList GetDirtyExtents();
+    VolumeExtenSlicetList GetDirtyExtents();
 
     std::unordered_map<uint64_t, std::map<uint64_t, PExtent>>
     GetExtentsForTesting() const;

curvefs/src/mds/heartbeat/heartbeat_manager.h

@@ -140,6 +140,10 @@ class HeartbeatManager {
 
     void UpdateMetaServerSpace(const MetaServerHeartbeatRequest &request);
 
+    // According to the free space of the BlockGroup in the heartbeat, update
+    // the statistics of the current free space
+    void UpdataBlockGroupSpaceStatis(const BlockGroupStatisticInfo &info);
+
  private:
     // Dependencies of heartbeat
     std::shared_ptr<Topology> topology_;

curvefs/src/metaserver/BUILD

@@ -29,6 +29,10 @@ cc_library(
         ["storage/*.cpp"],
     ) + glob(
         ["streaming/*.cpp"],
+    ) + glob(
+        ["space/*.cpp"],
+    ) + glob(
+        ["mds/*.cpp"],
     ),
     hdrs = glob(
         ["*.h"],
@@ -38,6 +42,10 @@ cc_library(
         ["storage/*.h"],
     ) + glob(
         ["streaming/*.h"],
+    ) + glob(
+        ["space/*.h"],
+    ) + glob(
+        ["mds/*.h"],
     ),
     copts = CURVE_DEFAULT_COPTS,
     visibility = ["//visibility:public"],
@@ -51,6 +59,7 @@ cc_library(
         "//curvefs/proto:mds_cc_proto",
         "//curvefs/src/common:curvefs_common",
         "//curvefs/src/metaserver/common:fs_metaserver_common",
+        "//curvefs/src/volume:volume",
         "//external:braft",
         "//src/common:curve_common",
         "//src/fs:lfs",
@@ -61,7 +70,7 @@ cc_library(
         "@com_google_absl//absl/container:btree",
         "@com_google_absl//absl/memory",
         "@com_google_absl//absl/utility",
-        "//external:gtest",
+        "//external:glog",
     ],
 )
 

curvefs/src/metaserver/copyset/meta_operator.cpp

@@ -193,7 +193,8 @@ OPERATOR_ON_APPLY(CreateManageInode);
 OPERATOR_ON_APPLY(CreatePartition);
 OPERATOR_ON_APPLY(DeletePartition);
 OPERATOR_ON_APPLY(PrepareRenameTx);
-OPERATOR_ON_APPLY(UpdateVolumeExtent);;
+OPERATOR_ON_APPLY(UpdateVolumeExtent);
+OPERATOR_ON_APPLY(UpdateDeallocatableBlockGroup);
 
 #undef OPERATOR_ON_APPLY
 
@@ -273,7 +274,7 @@ void GetVolumeExtentOperator::OnApply(int64_t index,
     }
 
     // in streaming mode, swap slices out and send them by streaming
-    VolumeExtentList extents;
+    VolumeExtenSlicetList extents;
     response->mutable_slices()->Swap(&extents);
     response->clear_slices();
 
@@ -320,6 +321,7 @@ OPERATOR_ON_APPLY_FROM_LOG(CreatePartition);
 OPERATOR_ON_APPLY_FROM_LOG(DeletePartition);
 OPERATOR_ON_APPLY_FROM_LOG(PrepareRenameTx);
 OPERATOR_ON_APPLY_FROM_LOG(UpdateVolumeExtent);
+OPERATOR_ON_APPLY_FROM_LOG(UpdateDeallocatableBlockGroup);
 
 #undef OPERATOR_ON_APPLY_FROM_LOG
 
@@ -377,6 +379,7 @@ OPERATOR_REDIRECT(DeletePartition);
 OPERATOR_REDIRECT(PrepareRenameTx);
 OPERATOR_REDIRECT(GetVolumeExtent);
 OPERATOR_REDIRECT(UpdateVolumeExtent);
+OPERATOR_REDIRECT(UpdateDeallocatableBlockGroup);
 
 #undef OPERATOR_REDIRECT
 
@@ -403,6 +406,7 @@ OPERATOR_ON_FAILED(DeletePartition);
 OPERATOR_ON_FAILED(PrepareRenameTx);
 OPERATOR_ON_FAILED(GetVolumeExtent);
 OPERATOR_ON_FAILED(UpdateVolumeExtent);
+OPERATOR_ON_FAILED(UpdateDeallocatableBlockGroup);
 
 #undef OPERATOR_ON_FAILED
 
@@ -428,6 +432,8 @@ OPERATOR_HASH_CODE(PrepareRenameTx);
 OPERATOR_HASH_CODE(DeletePartition);
 OPERATOR_HASH_CODE(GetVolumeExtent);
 OPERATOR_HASH_CODE(UpdateVolumeExtent);
+OPERATOR_HASH_CODE(UpdateDeallocatableBlockGroup);
+
 
 #undef OPERATOR_HASH_CODE
 
@@ -465,6 +471,7 @@ OPERATOR_TYPE(CreatePartition);
 OPERATOR_TYPE(DeletePartition);
 OPERATOR_TYPE(GetVolumeExtent);
 OPERATOR_TYPE(UpdateVolumeExtent);
+OPERATOR_TYPE(UpdateDeallocatableBlockGroup);
 
 #undef OPERATOR_TYPE