curvefs: support curvebs as data backend

curvefs/conf/client.conf

@@ -57,7 +57,13 @@ spaceserver.spaceaddr=127.0.0.1:19999  # __ANSIBLE_TEMPLATE__ {{ groups.space |
 spaceserver.rpcTimeoutMs=1000
 
 #### bdev
+# curve client's config file
 bdev.confpath=/etc/curve/client.conf
+# currently, we only use synchronous read/write interface of curve client
+# so, use a dedicated threadpool to issue sync io requests to curve client to
+# avoid performance degradation if single fs IO reqeust forms more than one requests to curve client
+# TODO(wuhanqing): remove this threadpool and use asynchronous interface
+bdev.threadnum=10
 
 #### extentManager
 extentManager.preAllocSize=65536
@@ -86,6 +92,19 @@ volume.bigFileSize=1048576
 volume.volBlockSize=4096
 volume.fsBlockSize=4096
 
+# alloator type, supported {bitmap}
+volume.allocator.type=bitmap
+
+## for bitmap allocator
+# size of each bit, default is 4MiB
+volume.bitmapallocator.size_per_bit=4194304
+
+# small allocation proportion [0-1]
+volume.bitmapallocator.small_alloc_proportion=0.2
+
+# number of block groups that allocated once
+volume.blockgroup.allocate_once=4
+
 #### s3
 # the max size that fuse send
 s3.fuseMaxSize=131072

curvefs/conf/tools.conf

@@ -14,11 +14,14 @@ etcdAddr=127.0.0.1:12379  # __CURVEADM_TEMPLATE__ ${cluster_etcd_addr} __CURVEAD
 blockSize=1048576
 fsType=s3
 # volume
-volumeSize=1048576
+volumeSize=0
 volumeBlockSize=4096
 volumeName=volume
 volumeUser=user
 volumePassword=password
+volumeBlockGroupSize=134217728
+# support |AtStart| and |AtEnd|
+volumeBitmapLocation=AtStart
 # s3
 s3.ak=ak
 s3.sk=sk

curvefs/proto/common.proto

@@ -23,12 +23,22 @@ enum FSType {
     TYPE_S3 = 2;
 }
 
+// bitmap location for each block group
+enum BitmapLocation {
+    AtStart = 1;
+    AtEnd = 2;
+}
+
 message Volume {
     required uint64 volumeSize = 1;
     required uint64 blockSize = 2;
     required string volumeName = 3;
     required string user = 4;
     optional string password = 5;
+    required uint64 blockGroupSize = 6;
+    required BitmapLocation bitmapLocation = 7;
+
+    // TODO(all): maybe we need curvebs cluster's mds ip and port in here
 }
 
 message S3Info {

curvefs/proto/metaserver.proto

@@ -146,10 +146,11 @@ message GetInodeRequest {
 }
 
 enum FsFileType {
-    TYPE_DIRECTORY = 1;
-    TYPE_FILE = 2;
-    TYPE_SYM_LINK = 3;
-    TYPE_S3 = 4;
+    TYPE_DIRECTORY = 1;   // 1 << 0
+    TYPE_FILE = 2;        // 1 << 1
+    TYPE_SYM_LINK = 4;    // 1 << 2
+    TYPE_S3 = 18;         // (1 << 4) | TYPE_FILE
+    TYPE_VOLUME = 34;     // (1 << 5) | TYPE_FILE
 };
 
 message VolumeExtent {
@@ -194,7 +195,7 @@ message Inode {
     required FsFileType type = 14;
     optional string symlink = 15;   // TYPE_SYM_LINK only
     optional uint64 rdev = 16;
-    optional VolumeExtentList volumeExtentList = 17;  // TYPE_FILE only
+    map<uint64, VolumeExtentList> volumeExtentMap = 17;  // TYPE_FILE only, key is file's offset
     map<uint64, S3ChunkInfoList> s3ChunkInfoMap = 18; // TYPE_S3 only, first is chunk index
     optional uint32 dtime = 19;
     optional uint32 openmpcount = 20; // openmpcount mount points had the file open
@@ -266,7 +267,7 @@ message UpdateInodeRequest {
     optional uint32 uid = 13;
     optional uint32 gid = 14;
     optional uint32 mode = 15;
-    optional VolumeExtentList volumeExtentList = 16;
+    map<uint64, VolumeExtentList> volumeExtentMap = 16;
     map<uint64, S3ChunkInfoList> s3ChunkInfoMap = 17;
     optional uint32 nlink = 18;
     optional InodeOpenStatusChange inodeOpenstatusChange = 19;

curvefs/proto/space.proto

@@ -1,5 +1,5 @@
 /*
- *  Copyright (c) 2021 NetEase Inc.
+ *  Copyright (c) 2022 NetEase Inc.
  *
  *  Licensed under the Apache License, Version 2.0 (the "License");
  *  you may not use this file except in compliance with the License.
@@ -15,115 +15,103 @@
  */
 
 syntax="proto2";
-import "curvefs/proto/mds.proto";
+package curvefs.mds.space;
 import "curvefs/proto/common.proto";
-package curvefs.space;
-option cc_generic_services = true;
-
-enum SpaceStatusCode {
-    SPACE_OK = 0;
-    SPACE_UNKNOWN_ERROR = 1;
-    SPACE_NO_SPACE = 2;
-    SPACE_NOT_FOUND = 3;
-    SPACE_EXISTS = 4;
-    SPACE_RELOAD_ERROR = 5;
-    SPACE_DEALLOC_ERROR = 6;
-}
-
-// space interface
-message Extent {
-    required uint64 offset = 1;  // the offset in the volume
-    required uint64 length = 2;
-}
-
-enum AllocateType {
-    NONE = 0;
-    SMALL = 1;     // small type file allocate
-    BIG = 2;       // big type file allocate
-    S3 = 3;
-}
-
-message AllocateHint {
-    optional AllocateType allocType = 1;
-    optional uint64 leftOffset = 2;        // The starting position of the address space expected
-    optional uint64 rightOffset = 3;       // The end position of the address space expected
-}
-
-message InitSpaceRequest {
-    required mds.FsInfo fsInfo = 1;
+option cc_generic_services=true;
+
+enum SpaceErrCode {
+    SpaceOk = 0;
+    SpaceErrUnknown = 1;
+    SpaceErrNoSpace = 2;
+    SpaceErrNotFound = 3;
+    SpaceErrConflict = 4;
+    SpaceErrEncode = 5;
+    SpaceErrDecode = 6;
+    SpaceErrCreate = 7;
+    SpaceErrExist = 8;
+    SpaceErrStorage = 9;
+    SpaceErrParam = 10;
+}
+
+message BlockGroup {
+    // start offset on volume
+    required uint64 offset = 1;
+    // total bytes of each block group
+    // this must be fixed when create fs or mkfs, so, this field maybe not needed
+    required uint64 size = 2;
+    // available bytes
+    required uint64 available = 3;
+    // bitmap location
+    required common.BitmapLocation bitmaplocation = 4;
+    // owner, who owns this block group
+    optional string owner = 5;
+}
+
+message AllocateBlockGroupRequest {
+    required uint32 fsId = 1;
+    required uint64 count = 2;    // number of block groups
+    required string owner = 3;    // or just ip:port
 }
 
-message InitSpaceResponse {
-    required SpaceStatusCode status = 1;
+message AllocateBlockGroupResponse {
+    required SpaceErrCode status = 1;
+    repeated BlockGroup blockGroups = 2;
 }
 
-message AllocateSpaceRequest {
+message AcquireBlockGroupRequest {
     required uint32 fsId = 1;
-    required uint32 size = 2;
-    optional AllocateHint allocHint = 3;
+    required string owner = 2;
+    required uint64 blockGroupOffset = 3;
 }
 
-message AllocateSpaceResponse {
-    required SpaceStatusCode status = 1;
-    repeated Extent extents = 2;     // The applied address space may not be continuous, so it is represented by repeated
+message AcquireBlockGroupResponse {
+    required SpaceErrCode status = 1;
+    optional BlockGroup blockGroups = 2;
 }
 
-message DeallocateSpaceRequest {
+message ReleaseBlockGroupRequest {
     required uint32 fsId = 1;
-    repeated Extent extents = 2;     // the space deallocate
+    required string owner = 2;
+    repeated BlockGroup blockGroups = 3;
 }
 
-message DeallocateSpaceResponse {
-    required SpaceStatusCode status = 1;
+message ReleaseBlockGroupResponse {
+    required SpaceErrCode status = 1;
 }
 
 message StatSpaceRequest {
     required uint32 fsId = 1;
 }
 
-message StatSpaceResponse {
-    required SpaceStatusCode status = 1;
-    optional uint64 blockSize = 2;
-    optional uint64 totalBlock = 3;
-    optional uint64 availableBlock = 4;
-    optional uint64 usedBlock = 5;
-}
-
-message UnInitSpaceRequest {
-    required uint32 fsId = 1;
+message SpaceInfo {
+    required uint64 size = 1;
+    required uint64 available = 2;
+    required uint32 totalBlockGroups = 3;
+    required uint32 allocatedBlockGroups = 4;
 }
 
-message UnInitSpaceResponse {
-    required SpaceStatusCode status = 1;
+message StatSpaceResponse {
+    required SpaceErrCode status = 1;
+    optional SpaceInfo spaceInfo = 2;
 }
 
-message AllocateS3ChunkRequest {
+message UpdateUsageRequest {
     required uint32 fsId = 1;
-}
 
-message AllocateS3ChunkResponse {
-   required SpaceStatusCode status = 1;  // 状态码
-   required uint64 chunkId = 2;
+    // TODO(wuhanqing): maybe only need some fields of BlockGroup
+    repeated BlockGroup blockGroups = 2;
 }
 
-message StatClusterSpaceRequest{
+message UpdateUsageResponse {
+    required SpaceErrCode status = 1;
 }
 
-message StatClusterSpaceResponse{
-    required SpaceStatusCode status = 1;
-    required common.FSType fsType = 2;
-    optional uint32 total = 3;
-    optional uint32 used = 4;
-    optional uint32 left = 5;
-}
+service SpaceService {
+    rpc AllocateBlockGroup(AllocateBlockGroupRequest) returns(AllocateBlockGroupResponse);
+    rpc AcquireBlockGroup(AcquireBlockGroupRequest) returns(AcquireBlockGroupResponse);
+    rpc ReleaseBlockGroup(ReleaseBlockGroupRequest) returns(ReleaseBlockGroupResponse);
+    rpc StatSpace(StatSpaceRequest) returns(StatSpaceResponse);
 
-service SpaceAllocService {
-    // space interface
-    rpc InitSpace(InitSpaceRequest) returns (InitSpaceResponse);
-    rpc AllocateSpace(AllocateSpaceRequest) returns (AllocateSpaceResponse);
-    rpc DeallocateSpace(DeallocateSpaceRequest) returns (DeallocateSpaceResponse);
-    rpc StatSpace(StatSpaceRequest) returns (StatSpaceResponse);
-    rpc UnInitSpace(UnInitSpaceRequest) returns (UnInitSpaceResponse);
-    rpc AllocateS3Chunk(AllocateS3ChunkRequest) returns (AllocateS3ChunkResponse);
-    rpc StatClusterSpace (StatClusterSpaceRequest) returns (StatClusterSpaceResponse);
+    // TODO(wuhanqing): client should update usage periodicly
+    rpc UpdateUsage(UpdateUsageRequest) returns(UpdateUsageResponse);
 }

curvefs/src/client/BUILD

@@ -40,6 +40,8 @@ cc_library(
             "*.h",
             "s3/*.cpp",
             "s3/*.h",
+            "volume/*.cpp",
+            "volume/*.h",
         ],
         exclude = ["main.c"],
     ),
@@ -62,5 +64,11 @@ cc_library(
         "//src/client:curve_client",
         "//src/common:curve_common",
         "//src/common:curve_s3_adapter",
+        "//curvefs/src/volume",
+        "//curvefs/src/common:metric_utils",
+        "@com_google_absl//absl/memory",
+        "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/meta:type_traits",
+        "@com_google_absl//absl/types:optional",
     ],
 )