From a155b564fee151a1ce92d1ad2a2b23b862c9aeb6 Mon Sep 17 00:00:00 2001 From: wuhanqing Date: Sun, 27 Feb 2022 15:46:27 +0800 Subject: [PATCH] curvefs: support curvebs as data backend --- WORKSPACE | 21 +- curvefs/conf/client.conf | 19 + curvefs/conf/tools.conf | 5 +- curvefs/proto/common.proto | 10 + curvefs/proto/metaserver.proto | 13 +- curvefs/proto/space.proto | 152 ++-- curvefs/src/client/BUILD | 8 + curvefs/src/client/base_client.cpp | 66 -- curvefs/src/client/base_client.h | 67 -- curvefs/src/client/common/config.cpp | 27 +- curvefs/src/client/common/config.h | 18 +- curvefs/src/client/curve_fuse_op.cpp | 15 +- curvefs/src/client/error_code.h | 1 + curvefs/src/client/extent_manager.cpp | 422 ----------- curvefs/src/client/extent_manager.h | 156 ---- curvefs/src/client/fuse_client.cpp | 17 +- curvefs/src/client/fuse_client.h | 7 +- curvefs/src/client/fuse_s3_client.cpp | 8 +- curvefs/src/client/fuse_volume_client.cpp | 350 +++++---- curvefs/src/client/fuse_volume_client.h | 46 +- curvefs/src/client/inode_cache_manager.cpp | 1 + curvefs/src/client/inode_wrapper.cpp | 54 ++ curvefs/src/client/inode_wrapper.h | 74 +- curvefs/src/client/main.c | 1 + curvefs/src/client/rpcclient/base_client.cpp | 58 +- curvefs/src/client/rpcclient/base_client.h | 37 +- curvefs/src/client/rpcclient/mds_client.cpp | 108 +++ curvefs/src/client/rpcclient/mds_client.h | 43 ++ curvefs/src/client/rpcclient/metacache.h | 2 +- .../client/rpcclient/metaserver_client.cpp | 20 +- .../src/client/rpcclient/metaserver_client.h | 1 - curvefs/src/client/s3/client_s3_adaptor.h | 3 - curvefs/src/client/space_client.cpp | 139 ---- curvefs/src/client/space_client.h | 101 --- .../client/volume/default_volume_storage.cpp | 194 +++++ .../client/volume/default_volume_storage.h | 82 ++ .../src/client/{common => volume}/extent.h | 22 +- curvefs/src/client/volume/extent_cache.cpp | 704 ++++++++++++++++++ curvefs/src/client/volume/extent_cache.h | 126 ++++ curvefs/src/client/volume/metric.h | 48 ++ curvefs/src/client/volume/utils.cpp | 164 ++++ curvefs/src/client/volume/utils.h | 77 ++ curvefs/src/client/volume/volume_storage.h | 56 ++ curvefs/src/common/BUILD | 20 +- .../common/metric_utils.h} | 37 +- curvefs/src/mds/BUILD | 2 +- curvefs/src/mds/codec/BUILD | 1 + curvefs/src/mds/codec/codec.cpp | 21 + curvefs/src/mds/codec/codec.h | 2 + curvefs/src/mds/common/storage_key.h | 2 + curvefs/src/mds/fs_manager.cpp | 79 +- curvefs/src/mds/fs_manager.h | 33 +- curvefs/src/mds/fs_storage.cpp | 1 + curvefs/src/mds/mds.cpp | 23 +- curvefs/src/mds/mds.h | 9 +- curvefs/src/mds/{spaceclient => space}/BUILD | 8 +- curvefs/src/mds/space/block_group_storage.cpp | 131 ++++ curvefs/src/mds/space/block_group_storage.h | 88 +++ curvefs/src/mds/space/manager.cpp | 109 +++ curvefs/src/mds/space/manager.h | 83 +++ curvefs/src/mds/space/reloader.cpp | 78 ++ curvefs/src/mds/space/reloader.h | 60 ++ curvefs/src/mds/space/service.cpp | 157 ++++ curvefs/src/mds/space/service.h | 70 ++ curvefs/src/mds/space/volume_space.cpp | 395 ++++++++++ curvefs/src/mds/space/volume_space.h | 179 +++++ curvefs/src/mds/spaceclient/space_client.cpp | 110 --- curvefs/src/mds/spaceclient/space_client.h | 66 -- curvefs/src/metaserver/inode_manager.cpp | 6 +- curvefs/src/metaserver/metaserver.cpp | 2 +- curvefs/src/metaserver/metastore.cpp | 3 +- curvefs/src/space/BUILD | 83 --- curvefs/src/space/common.h | 103 --- curvefs/src/space/main.cpp | 138 ---- curvefs/src/space/metaserver_client.cpp | 127 ---- curvefs/src/space/metaserver_client.h | 69 -- curvefs/src/space/reloader.h | 63 -- curvefs/src/space/space_manager.cpp | 141 ---- curvefs/src/space/space_manager.h | 89 --- curvefs/src/space/space_service.cpp | 150 ---- curvefs/src/space/space_service.h | 71 -- .../src/tools/create/curvefs_create_fs.cpp | 44 +- curvefs/src/tools/curvefs_tool_define.cpp | 22 + curvefs/src/tools/curvefs_tool_define.h | 6 + curvefs/src/volume/BUILD | 44 ++ curvefs/src/{space => volume}/allocator.cpp | 12 +- curvefs/src/{space => volume}/allocator.h | 36 +- .../{space => volume}/bitmap_allocator.cpp | 126 ++-- .../src/{space => volume}/bitmap_allocator.h | 68 +- .../block_device_client.cpp | 227 ++++-- .../{client => volume}/block_device_client.h | 82 +- curvefs/src/volume/block_group_loader.cpp | 137 ++++ curvefs/src/volume/block_group_loader.h | 88 +++ curvefs/src/volume/block_group_manager.cpp | 96 +++ curvefs/src/volume/block_group_manager.h | 82 ++ curvefs/src/volume/block_group_updater.cpp | 68 ++ curvefs/src/volume/block_group_updater.h | 89 +++ curvefs/src/volume/common.h | 107 +++ .../src/{space => volume}/free_extents.cpp | 57 +- curvefs/src/{space => volume}/free_extents.h | 52 +- .../src/{space/config.h => volume/option.h} | 33 +- curvefs/src/volume/space_manager.cpp | 259 +++++++ curvefs/src/volume/space_manager.h | 136 ++++ curvefs/src/{space => volume}/utils.cpp | 75 +- curvefs/src/{space => volume}/utils.h | 24 +- curvefs/test/client/BUILD | 32 +- curvefs/test/client/base_client_test.cpp | 176 ----- .../test/client/client_s3_adaptor_test.cpp | 16 +- curvefs/test/client/common/BUILD | 1 + curvefs/test/client/common/test_config.cpp | 69 ++ curvefs/test/client/fs_cache_manager_test.cpp | 2 +- curvefs/test/client/mock_extent_manager.h | 73 -- curvefs/test/client/mock_mds_client.h | 18 + curvefs/test/client/mock_space_client.h | 59 -- .../test/client/mock_spacealloc_base_client.h | 55 -- curvefs/test/client/mock_spacealloc_service.h | 57 -- curvefs/test/client/mock_volume_storage.h | 44 ++ curvefs/test/client/rpcclient/BUILD | 1 + .../client/rpcclient/base_client_test.cpp | 10 +- .../test/client/rpcclient/mds_client_test.cpp | 129 ++++ .../client/rpcclient/mock_mds_base_client.h | 25 + .../test/client/rpcclient/mock_mds_client.h | 18 + .../client/rpcclient/mock_space_service.h | 71 ++ .../rpcclient/mock_spacealloc_service.h | 58 -- curvefs/test/client/space_client_test.cpp | 127 ---- curvefs/test/client/test_extent_manager.cpp | 207 ----- curvefs/test/client/test_fuse_client.cpp | 327 ++------ curvefs/test/client/test_inodeWrapper.cpp | 2 + .../test/client/test_inode_cache_manager.cpp | 2 +- curvefs/test/client/volume/BUILD | 34 + .../test/client/volume/alloc_space_test.cpp | 244 ++++++ curvefs/test/client/volume/common.h | 56 ++ .../volume/default_volume_storage_test.cpp | 283 +++++++ .../volume/extent_cache_mark_written_test.cpp | 683 +++++++++++++++++ .../client/volume/extent_cache_merge_test.cpp | 414 ++++++++++ .../volume/extent_cache_read_divide_test.cpp | 458 ++++++++++++ .../volume/extent_cache_to_inode_pb_test.cpp | 244 ++++++ .../volume/extent_cache_write_divide_test.cpp | 693 +++++++++++++++++ .../volume/prepare_write_request_test.cpp | 96 +++ curvefs/test/mds/BUILD | 3 +- curvefs/test/mds/chunkid_allocator_test.cpp | 3 +- curvefs/test/mds/codec/codec_test.cpp | 4 + curvefs/test/mds/fake_space.cpp | 89 --- curvefs/test/mds/fake_space.h | 70 -- curvefs/test/mds/fs_info_wrapper_test.cpp | 2 + curvefs/test/mds/fs_manager_test.cpp | 58 +- curvefs/test/mds/fs_manager_test2.cpp | 29 +- curvefs/test/mds/mds_service_test.cpp | 33 +- .../test/mds/mock/mock_block_group_storage.h | 49 ++ .../mock_etcd_client.h} | 15 +- curvefs/test/mds/mock/mock_space.h | 60 -- .../mds/mock/mock_space_manager.h} | 42 +- curvefs/test/mds/mock/mock_volume_space.h | 57 ++ curvefs/test/mds/persist_kvstorage_test.cpp | 2 + curvefs/test/mds/space/BUILD | 36 + .../mds/space/block_group_storage_test.cpp | 146 ++++ curvefs/test/mds/space/space_manager_test.cpp | 149 ++++ curvefs/test/mds/space/space_service_test.cpp | 213 ++++++ curvefs/test/mds/space/volume_space_test.cpp | 335 +++++++++ curvefs/test/mds/space_client_test.cpp | 197 ----- .../metaserver/metaserver_s3_adaptor_test.cpp | 3 - .../metaserver/metaserver_s3_adaptor_test.h | 1 - curvefs/test/metaserver/metastore_test.cpp | 2 +- curvefs/test/metaserver/test_helper.cpp | 5 +- curvefs/test/space/BUILD | 163 ---- curvefs/test/space/fake_user.cpp | 173 ----- curvefs/test/space/mock/mock_metaserver.h | 49 -- curvefs/test/space/mock/mock_space_manager.h | 44 -- curvefs/test/space/space_manager_test.cpp | 169 ----- curvefs/test/space/space_service_test.cpp | 393 ---------- curvefs/test/volume/BUILD | 113 +++ .../test/{space => volume}/allocator_test.cpp | 14 +- .../bitmap_allocator_brute_test.cpp | 27 +- ...tmap_allocator_multi_thread_brute_test.cpp | 36 +- .../bitmap_allocator_test.cpp | 104 ++- .../block_device_client_test.cpp | 293 ++++++-- .../test/volume/block_group_loader_test.cpp | 97 +++ .../test/volume/block_group_updater_test.cpp | 106 +++ curvefs/test/{space => volume}/common.h | 45 +- .../test/{space => volume}/extents_test.cpp | 48 +- curvefs/test/{space => volume}/mock/BUILD | 13 +- .../mock}/mock_block_device_client.h | 37 +- curvefs/test/volume/mock/mock_space_manager.h | 46 ++ include/curve_compiler_specific.h | 2 + src/client/copyset_client.h | 13 +- src/client/lease_executor.h | 2 +- src/client/libcurve_file.cpp | 2 + src/client/mds_client.cpp | 2 +- src/client/request_scheduler.h | 4 +- src/common/bitmap.cpp | 16 + src/common/bitmap.h | 8 +- src/common/encode.h | 8 +- src/common/fast_align.h | 4 + .../allocstatistic/alloc_statistic.cpp | 13 +- .../allocstatistic/alloc_statistic_helper.cpp | 1 + test/integration/common/chunkservice_op.cpp | 2 + 196 files changed, 10963 insertions(+), 5793 deletions(-) delete mode 100644 curvefs/src/client/base_client.cpp delete mode 100644 curvefs/src/client/base_client.h delete mode 100644 curvefs/src/client/extent_manager.cpp delete mode 100644 curvefs/src/client/extent_manager.h delete mode 100644 curvefs/src/client/space_client.cpp delete mode 100644 curvefs/src/client/space_client.h create mode 100644 curvefs/src/client/volume/default_volume_storage.cpp create mode 100644 curvefs/src/client/volume/default_volume_storage.h rename curvefs/src/client/{common => volume}/extent.h (70%) create mode 100644 curvefs/src/client/volume/extent_cache.cpp create mode 100644 curvefs/src/client/volume/extent_cache.h create mode 100644 curvefs/src/client/volume/metric.h create mode 100644 curvefs/src/client/volume/utils.cpp create mode 100644 curvefs/src/client/volume/utils.h create mode 100644 curvefs/src/client/volume/volume_storage.h rename curvefs/{test/space/reloader_test.cpp => src/common/metric_utils.h} (54%) rename curvefs/src/mds/{spaceclient => space}/BUILD (78%) create mode 100644 curvefs/src/mds/space/block_group_storage.cpp create mode 100644 curvefs/src/mds/space/block_group_storage.h create mode 100644 curvefs/src/mds/space/manager.cpp create mode 100644 curvefs/src/mds/space/manager.h create mode 100644 curvefs/src/mds/space/reloader.cpp create mode 100644 curvefs/src/mds/space/reloader.h create mode 100644 curvefs/src/mds/space/service.cpp create mode 100644 curvefs/src/mds/space/service.h create mode 100644 curvefs/src/mds/space/volume_space.cpp create mode 100644 curvefs/src/mds/space/volume_space.h delete mode 100644 curvefs/src/mds/spaceclient/space_client.cpp delete mode 100644 curvefs/src/mds/spaceclient/space_client.h delete mode 100644 curvefs/src/space/BUILD delete mode 100644 curvefs/src/space/common.h delete mode 100644 curvefs/src/space/main.cpp delete mode 100644 curvefs/src/space/metaserver_client.cpp delete mode 100644 curvefs/src/space/metaserver_client.h delete mode 100644 curvefs/src/space/reloader.h delete mode 100644 curvefs/src/space/space_manager.cpp delete mode 100644 curvefs/src/space/space_manager.h delete mode 100644 curvefs/src/space/space_service.cpp delete mode 100644 curvefs/src/space/space_service.h create mode 100644 curvefs/src/volume/BUILD rename curvefs/src/{space => volume}/allocator.cpp (80%) rename curvefs/src/{space => volume}/allocator.h (74%) rename curvefs/src/{space => volume}/bitmap_allocator.cpp (69%) rename curvefs/src/{space => volume}/bitmap_allocator.h (61%) rename curvefs/src/{client => volume}/block_device_client.cpp (52%) rename curvefs/src/{client => volume}/block_device_client.h (62%) create mode 100644 curvefs/src/volume/block_group_loader.cpp create mode 100644 curvefs/src/volume/block_group_loader.h create mode 100644 curvefs/src/volume/block_group_manager.cpp create mode 100644 curvefs/src/volume/block_group_manager.h create mode 100644 curvefs/src/volume/block_group_updater.cpp create mode 100644 curvefs/src/volume/block_group_updater.h create mode 100644 curvefs/src/volume/common.h rename curvefs/src/{space => volume}/free_extents.cpp (85%) rename curvefs/src/{space => volume}/free_extents.h (67%) rename curvefs/src/{space/config.h => volume/option.h} (73%) create mode 100644 curvefs/src/volume/space_manager.cpp create mode 100644 curvefs/src/volume/space_manager.h rename curvefs/src/{space => volume}/utils.cpp (53%) rename curvefs/src/{space => volume}/utils.h (63%) delete mode 100644 curvefs/test/client/base_client_test.cpp create mode 100644 curvefs/test/client/common/test_config.cpp delete mode 100644 curvefs/test/client/mock_extent_manager.h delete mode 100644 curvefs/test/client/mock_space_client.h delete mode 100644 curvefs/test/client/mock_spacealloc_base_client.h delete mode 100644 curvefs/test/client/mock_spacealloc_service.h create mode 100644 curvefs/test/client/mock_volume_storage.h create mode 100644 curvefs/test/client/rpcclient/mock_space_service.h delete mode 100644 curvefs/test/client/rpcclient/mock_spacealloc_service.h delete mode 100644 curvefs/test/client/space_client_test.cpp delete mode 100644 curvefs/test/client/test_extent_manager.cpp create mode 100644 curvefs/test/client/volume/BUILD create mode 100644 curvefs/test/client/volume/alloc_space_test.cpp create mode 100644 curvefs/test/client/volume/common.h create mode 100644 curvefs/test/client/volume/default_volume_storage_test.cpp create mode 100644 curvefs/test/client/volume/extent_cache_mark_written_test.cpp create mode 100644 curvefs/test/client/volume/extent_cache_merge_test.cpp create mode 100644 curvefs/test/client/volume/extent_cache_read_divide_test.cpp create mode 100644 curvefs/test/client/volume/extent_cache_to_inode_pb_test.cpp create mode 100644 curvefs/test/client/volume/extent_cache_write_divide_test.cpp create mode 100644 curvefs/test/client/volume/prepare_write_request_test.cpp delete mode 100644 curvefs/test/mds/fake_space.cpp delete mode 100644 curvefs/test/mds/fake_space.h create mode 100644 curvefs/test/mds/mock/mock_block_group_storage.h rename curvefs/test/mds/{mock_etcdclient.h => mock/mock_etcd_client.h} (88%) delete mode 100644 curvefs/test/mds/mock/mock_space.h rename curvefs/{src/space/reloader.cpp => test/mds/mock/mock_space_manager.h} (52%) create mode 100644 curvefs/test/mds/mock/mock_volume_space.h create mode 100644 curvefs/test/mds/space/BUILD create mode 100644 curvefs/test/mds/space/block_group_storage_test.cpp create mode 100644 curvefs/test/mds/space/space_manager_test.cpp create mode 100644 curvefs/test/mds/space/space_service_test.cpp create mode 100644 curvefs/test/mds/space/volume_space_test.cpp delete mode 100644 curvefs/test/mds/space_client_test.cpp delete mode 100644 curvefs/test/space/BUILD delete mode 100644 curvefs/test/space/fake_user.cpp delete mode 100644 curvefs/test/space/mock/mock_metaserver.h delete mode 100644 curvefs/test/space/mock/mock_space_manager.h delete mode 100644 curvefs/test/space/space_manager_test.cpp delete mode 100644 curvefs/test/space/space_service_test.cpp create mode 100644 curvefs/test/volume/BUILD rename curvefs/test/{space => volume}/allocator_test.cpp (84%) rename curvefs/test/{space => volume}/bitmap_allocator_brute_test.cpp (90%) rename curvefs/test/{space => volume}/bitmap_allocator_multi_thread_brute_test.cpp (89%) rename curvefs/test/{space => volume}/bitmap_allocator_test.cpp (72%) rename curvefs/test/{client => volume}/block_device_client_test.cpp (54%) create mode 100644 curvefs/test/volume/block_group_loader_test.cpp create mode 100644 curvefs/test/volume/block_group_updater_test.cpp rename curvefs/test/{space => volume}/common.h (67%) rename curvefs/test/{space => volume}/extents_test.cpp (92%) rename curvefs/test/{space => volume}/mock/BUILD (72%) rename curvefs/test/{client => volume/mock}/mock_block_device_client.h (58%) create mode 100644 curvefs/test/volume/mock/mock_space_manager.h diff --git a/WORKSPACE b/WORKSPACE index 5c4a351f16..62af6e19a9 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -20,7 +20,7 @@ load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") git_repository( name = "com_github_baidu_braft", - remote = "https://github.com/baidu/braft", + remote = "https://gitee.com/baidu/braft", commit = "e255c0e4b18d1a8a5d484d4b647f41ff1385ef1e", ) @@ -36,7 +36,7 @@ http_archive( name = "com_google_protobuf", sha256 = "cef7f1b5a7c5fba672bec2a319246e8feba471f04dcebfe362d55930ee7c1c30", strip_prefix = "protobuf-3.5.0", - urls = ["https://github.com/google/protobuf/archive/v3.5.0.zip"], + urls = ["https://curve-build.nos-eastchina1.126.net/protobuf-3.5.0.zip"], ) bind( @@ -48,7 +48,7 @@ bind( new_git_repository( name = "com_google_googletest", build_file = "bazel/gmock.BUILD", - remote = "https://github.com/google/googletest", + remote = "https://gitee.com/mirrors/googletest", tag = "release-1.8.0", ) @@ -61,7 +61,7 @@ bind( # brpc内BUILD文件在依赖glog时, 直接指定的依赖是"@com_github_google_glog//:glog" git_repository( name = "com_github_google_glog", - remote = "https://github.com/google/glog", + remote = "https://gitee.com/mirrors/glog", commit = "4cc89c9e2b452db579397887c37f302fb28f6ca1", patch_args = ["-p1"], patches = ["//:thirdparties/glog/glog.patch"], @@ -77,7 +77,7 @@ http_archive( name = "com_github_gflags_gflags", strip_prefix = "gflags-2.2.2", urls = [ - "https://mirror.bazel.build/github.com/gflags/gflags/archive/v2.2.2.tar.gz", + "https://curve-build.nos-eastchina1.126.net/gflags-2.2.2.tar.gz", "https://github.com/gflags/gflags/archive/v2.2.2.tar.gz", ], ) @@ -91,7 +91,7 @@ new_http_archive( name = "com_github_google_leveldb", build_file = "bazel/leveldb.BUILD", strip_prefix = "leveldb-a53934a3ae1244679f812d998a4f16f2c7f309a6", - url = "https://github.com/google/leveldb/archive/a53934a3ae1244679f812d998a4f16f2c7f309a6.tar.gz", + url = "https://curve-build.nos-eastchina1.126.net/leveldb-a53934a3ae1244679f812d998a4f16f2c7f309a6.tar.gz", ) bind( @@ -101,7 +101,7 @@ bind( git_repository( name = "com_github_apache_brpc", - remote = "https://github.com/apache/incubator-brpc", + remote = "https://gitee.com/baidu/BRPC", commit = "1b9e00641cbec1c8803da6a1f7f555398c954cb0", patches = ["//:thirdparties/brpc/brpc.patch"], patch_args = ["-p1"], @@ -131,7 +131,7 @@ bind( new_git_repository( name = "jsoncpp", build_file = "bazel/jsoncpp.BUILD", - remote = "https://github.com/open-source-parsers/jsoncpp.git", + remote = "https://gitee.com/mirrors/jsoncpp", tag = "1.8.4", ) @@ -149,7 +149,7 @@ new_local_repository( new_http_archive( name = "aws", urls = [ - "https://github.com/aws/aws-sdk-cpp/archive/1.7.340.tar.gz", + "https://curve-build.nos-eastchina1.126.net/aws-sdk-cpp-1.7.340.tar.gz", "https://mirror.bazel.build/github.com/aws/aws-sdk-cpp/archive/1.7.340.tar.gz", ], sha256 = "2e82517045efb55409cff1408c12829d9e8aea22c1e2888529cb769b7473b0bf", @@ -160,6 +160,7 @@ new_http_archive( new_http_archive( name = "aws_c_common", urls = [ + "https://curve-build.nos-eastchina1.126.net/aws-c-common-0.4.29.tar.gz", "https://github.com/awslabs/aws-c-common/archive/v0.4.29.tar.gz", "https://mirror.tensorflow.org/github.com/awslabs/aws-c-common/archive/v0.4.29.tar.gz", ], @@ -171,6 +172,7 @@ new_http_archive( new_http_archive( name = "aws_c_event_stream", urls = [ + "https://curve-build.nos-eastchina1.126.net/aws-c-event-stream-0.1.4.tar.gz", "https://github.com/awslabs/aws-c-event-stream/archive/v0.1.4.tar.gz", "https://mirror.tensorflow.org/github.com/awslabs/aws-c-event-stream/archive/v0.1.4.tar.gz", ], @@ -182,6 +184,7 @@ new_http_archive( new_http_archive( name = "aws_checksums", urls = [ + "https://curve-build.nos-eastchina1.126.net/aws-checksums-0.1.5.tar.gz", "https://github.com/awslabs/aws-checksums/archive/v0.1.5.tar.gz", "https://mirror.tensorflow.org/github.com/awslabs/aws-checksums/archive/v0.1.5.tar.gz", ], diff --git a/curvefs/conf/client.conf b/curvefs/conf/client.conf index 4e6052b480..aef9325eb6 100644 --- a/curvefs/conf/client.conf +++ b/curvefs/conf/client.conf @@ -50,7 +50,13 @@ spaceserver.spaceaddr=127.0.0.1:19999 # __ANSIBLE_TEMPLATE__ {{ groups.space | spaceserver.rpcTimeoutMs=1000 #### bdev +# curve client's config file bdev.confpath=/etc/curve/client.conf +# currently, we only use synchronous read/write interface of curve client +# so, use a dedicated threadpool to issue sync io requests to curve client to +# avoid performance degradation if single fs IO reqeust forms more than one requests to curve client +# TODO(wuhanqing): remove this threadpool and use asynchronous interface +bdev.threadnum=10 #### extentManager extentManager.preAllocSize=65536 @@ -79,6 +85,19 @@ volume.bigFileSize=1048576 volume.volBlockSize=4096 volume.fsBlockSize=4096 +# alloator type, supported {bitmap} +volume.allocator.type=bitmap + +## for bitmap allocator +# size of each bit, default is 4MiB +volume.bitmapallocator.size_per_bit=4194304 + +# small allocation proportion [0-1] +volume.bitmapallocator.small_alloc_proportion=0.2 + +# number of block groups that allocated once +volume.blockgroup.allocate_once=4 + #### s3 # the max size that fuse send s3.fuseMaxSize=131072 diff --git a/curvefs/conf/tools.conf b/curvefs/conf/tools.conf index cf43d5a190..b5b11caf43 100644 --- a/curvefs/conf/tools.conf +++ b/curvefs/conf/tools.conf @@ -14,11 +14,14 @@ etcdAddr=127.0.0.1:12379 # __CURVEADM_TEMPLATE__ ${cluster_etcd_addr} __CURVEAD blockSize=1048576 fsType=s3 # volume -volumeSize=1048576 +volumeSize=0 volumeBlockSize=4096 volumeName=volume volumeUser=user volumePassword=password +volumeBlockGroupSize=134217728 +# support |AtStart| and |AtEnd| +volumeBitmapLocation=AtStart # s3 s3.ak=ak s3.sk=sk diff --git a/curvefs/proto/common.proto b/curvefs/proto/common.proto index 837527df16..5c84e7b2a7 100644 --- a/curvefs/proto/common.proto +++ b/curvefs/proto/common.proto @@ -23,12 +23,22 @@ enum FSType { TYPE_S3 = 2; } +// bitmap location for each block group +enum BitmapLocation { + AtStart = 1; + AtEnd = 2; +} + message Volume { required uint64 volumeSize = 1; required uint64 blockSize = 2; required string volumeName = 3; required string user = 4; optional string password = 5; + required uint64 blockGroupSize = 6; + required BitmapLocation bitmapLocation = 7; + + // TODO(all): maybe we need curvebs cluster's mds ip and port in here } message S3Info { diff --git a/curvefs/proto/metaserver.proto b/curvefs/proto/metaserver.proto index 13b66d129d..0c89183a22 100644 --- a/curvefs/proto/metaserver.proto +++ b/curvefs/proto/metaserver.proto @@ -146,10 +146,11 @@ message GetInodeRequest { } enum FsFileType { - TYPE_DIRECTORY = 1; - TYPE_FILE = 2; - TYPE_SYM_LINK = 3; - TYPE_S3 = 4; + TYPE_DIRECTORY = 1; // 1 << 0 + TYPE_FILE = 2; // 1 << 1 + TYPE_SYM_LINK = 4; // 1 << 2 + TYPE_S3 = 18; // (1 << 4) | TYPE_FILE + TYPE_VOLUME = 34; // (1 << 5) | TYPE_FILE }; message VolumeExtent { @@ -194,7 +195,7 @@ message Inode { required FsFileType type = 14; optional string symlink = 15; // TYPE_SYM_LINK only optional uint64 rdev = 16; - optional VolumeExtentList volumeExtentList = 17; // TYPE_FILE only + map volumeExtentMap = 17; // TYPE_FILE only, key is file's offset map s3ChunkInfoMap = 18; // TYPE_S3 only, first is chunk index optional uint32 dtime = 19; optional uint32 openmpcount = 20; // openmpcount mount points had the file open @@ -266,7 +267,7 @@ message UpdateInodeRequest { optional uint32 uid = 13; optional uint32 gid = 14; optional uint32 mode = 15; - optional VolumeExtentList volumeExtentList = 16; + map volumeExtentMap = 16; map s3ChunkInfoMap = 17; optional uint32 nlink = 18; optional InodeOpenStatusChange inodeOpenstatusChange = 19; diff --git a/curvefs/proto/space.proto b/curvefs/proto/space.proto index 45f50712e8..36435fc532 100644 --- a/curvefs/proto/space.proto +++ b/curvefs/proto/space.proto @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 NetEase Inc. + * Copyright (c) 2022 NetEase Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,115 +15,103 @@ */ syntax="proto2"; -import "curvefs/proto/mds.proto"; +package curvefs.mds.space; import "curvefs/proto/common.proto"; -package curvefs.space; -option cc_generic_services = true; - -enum SpaceStatusCode { - SPACE_OK = 0; - SPACE_UNKNOWN_ERROR = 1; - SPACE_NO_SPACE = 2; - SPACE_NOT_FOUND = 3; - SPACE_EXISTS = 4; - SPACE_RELOAD_ERROR = 5; - SPACE_DEALLOC_ERROR = 6; -} - -// space interface -message Extent { - required uint64 offset = 1; // the offset in the volume - required uint64 length = 2; -} - -enum AllocateType { - NONE = 0; - SMALL = 1; // small type file allocate - BIG = 2; // big type file allocate - S3 = 3; -} - -message AllocateHint { - optional AllocateType allocType = 1; - optional uint64 leftOffset = 2; // The starting position of the address space expected - optional uint64 rightOffset = 3; // The end position of the address space expected -} - -message InitSpaceRequest { - required mds.FsInfo fsInfo = 1; +option cc_generic_services=true; + +enum SpaceErrCode { + SpaceOk = 0; + SpaceErrUnknown = 1; + SpaceErrNoSpace = 2; + SpaceErrNotFound = 3; + SpaceErrConflict = 4; + SpaceErrEncode = 5; + SpaceErrDecode = 6; + SpaceErrCreate = 7; + SpaceErrExist = 8; + SpaceErrStorage = 9; + SpaceErrParam = 10; +} + +message BlockGroup { + // start offset on volume + required uint64 offset = 1; + // total bytes of each block group + // this must be fixed when create fs or mkfs, so, this field maybe not needed + required uint64 size = 2; + // available bytes + required uint64 available = 3; + // bitmap location + required common.BitmapLocation bitmaplocation = 4; + // owner, who owns this block group + optional string owner = 5; +} + +message AllocateBlockGroupRequest { + required uint32 fsId = 1; + required uint64 count = 2; // number of block groups + required string owner = 3; // or just ip:port } -message InitSpaceResponse { - required SpaceStatusCode status = 1; +message AllocateBlockGroupResponse { + required SpaceErrCode status = 1; + repeated BlockGroup blockGroups = 2; } -message AllocateSpaceRequest { +message AcquireBlockGroupRequest { required uint32 fsId = 1; - required uint32 size = 2; - optional AllocateHint allocHint = 3; + required string owner = 2; + required uint64 blockGroupOffset = 3; } -message AllocateSpaceResponse { - required SpaceStatusCode status = 1; - repeated Extent extents = 2; // The applied address space may not be continuous, so it is represented by repeated +message AcquireBlockGroupResponse { + required SpaceErrCode status = 1; + optional BlockGroup blockGroups = 2; } -message DeallocateSpaceRequest { +message ReleaseBlockGroupRequest { required uint32 fsId = 1; - repeated Extent extents = 2; // the space deallocate + required string owner = 2; + repeated BlockGroup blockGroups = 3; } -message DeallocateSpaceResponse { - required SpaceStatusCode status = 1; +message ReleaseBlockGroupResponse { + required SpaceErrCode status = 1; } message StatSpaceRequest { required uint32 fsId = 1; } -message StatSpaceResponse { - required SpaceStatusCode status = 1; - optional uint64 blockSize = 2; - optional uint64 totalBlock = 3; - optional uint64 availableBlock = 4; - optional uint64 usedBlock = 5; -} - -message UnInitSpaceRequest { - required uint32 fsId = 1; +message SpaceInfo { + required uint64 size = 1; + required uint64 available = 2; + required uint32 totalBlockGroups = 3; + required uint32 allocatedBlockGroups = 4; } -message UnInitSpaceResponse { - required SpaceStatusCode status = 1; +message StatSpaceResponse { + required SpaceErrCode status = 1; + optional SpaceInfo spaceInfo = 2; } -message AllocateS3ChunkRequest { +message UpdateUsageRequest { required uint32 fsId = 1; -} -message AllocateS3ChunkResponse { - required SpaceStatusCode status = 1; // 状态码 - required uint64 chunkId = 2; + // TODO(wuhanqing): maybe only need some fields of BlockGroup + repeated BlockGroup blockGroups = 2; } -message StatClusterSpaceRequest{ +message UpdateUsageResponse { + required SpaceErrCode status = 1; } -message StatClusterSpaceResponse{ - required SpaceStatusCode status = 1; - required common.FSType fsType = 2; - optional uint32 total = 3; - optional uint32 used = 4; - optional uint32 left = 5; -} +service SpaceService { + rpc AllocateBlockGroup(AllocateBlockGroupRequest) returns(AllocateBlockGroupResponse); + rpc AcquireBlockGroup(AcquireBlockGroupRequest) returns(AcquireBlockGroupResponse); + rpc ReleaseBlockGroup(ReleaseBlockGroupRequest) returns(ReleaseBlockGroupResponse); + rpc StatSpace(StatSpaceRequest) returns(StatSpaceResponse); -service SpaceAllocService { - // space interface - rpc InitSpace(InitSpaceRequest) returns (InitSpaceResponse); - rpc AllocateSpace(AllocateSpaceRequest) returns (AllocateSpaceResponse); - rpc DeallocateSpace(DeallocateSpaceRequest) returns (DeallocateSpaceResponse); - rpc StatSpace(StatSpaceRequest) returns (StatSpaceResponse); - rpc UnInitSpace(UnInitSpaceRequest) returns (UnInitSpaceResponse); - rpc AllocateS3Chunk(AllocateS3ChunkRequest) returns (AllocateS3ChunkResponse); - rpc StatClusterSpace (StatClusterSpaceRequest) returns (StatClusterSpaceResponse); + // TODO(wuhanqing): client should update usage periodicly + rpc UpdateUsage(UpdateUsageRequest) returns(UpdateUsageResponse); } diff --git a/curvefs/src/client/BUILD b/curvefs/src/client/BUILD index a56d967a2c..8e236b9ff0 100644 --- a/curvefs/src/client/BUILD +++ b/curvefs/src/client/BUILD @@ -40,6 +40,8 @@ cc_library( "*.h", "s3/*.cpp", "s3/*.h", + "volume/*.cpp", + "volume/*.h", ], exclude = ["main.c"], ), @@ -61,5 +63,11 @@ cc_library( "//src/client:curve_client", "//src/common:curve_common", "//src/common:curve_s3_adapter", + "//curvefs/src/volume", + "//curvefs/src/common:metric_utils", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/meta:type_traits", + "@com_google_absl//absl/types:optional", ], ) diff --git a/curvefs/src/client/base_client.cpp b/curvefs/src/client/base_client.cpp deleted file mode 100644 index 7ac2c9067a..0000000000 --- a/curvefs/src/client/base_client.cpp +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"){} - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Project: curve - * Created Date: Fri Jun 11 2021 - * Author: lixiaocui - */ - -#include "curvefs/src/client/base_client.h" -namespace curvefs { -namespace client { - -void SpaceBaseClient::AllocExtents(uint32_t fsId, - const ExtentAllocInfo &toAllocExtent, - curvefs::space::AllocateType type, - AllocateSpaceResponse *response, - brpc::Controller *cntl, - brpc::Channel *channel) { - AllocateSpaceRequest request; - request.set_fsid(fsId); - request.set_size(toAllocExtent.len); - auto allochint = new curvefs::space::AllocateHint(); - allochint->set_alloctype(type); - if (toAllocExtent.leftHintAvailable) { - allochint->set_leftoffset(toAllocExtent.pOffsetLeft); - } - if (toAllocExtent.rightHintAvailable) { - allochint->set_rightoffset(toAllocExtent.pOffsetRight); - } - request.set_allocated_allochint(allochint); - curvefs::space::SpaceAllocService_Stub stub(channel); - stub.AllocateSpace(cntl, &request, response, nullptr); -} - -void SpaceBaseClient::DeAllocExtents(uint32_t fsId, - const std::list &allocatedExtents, - DeallocateSpaceResponse *response, - brpc::Controller *cntl, - brpc::Channel *channel) { - DeallocateSpaceRequest request; - request.set_fsid(fsId); - auto iter = allocatedExtents.begin(); - while (iter != allocatedExtents.end()) { - auto extent = request.add_extents(); - extent->CopyFrom(*iter); - ++iter; - } - curvefs::space::SpaceAllocService_Stub stub(channel); - stub.DeallocateSpace(cntl, &request, response, nullptr); -} -} // namespace client -} // namespace curvefs diff --git a/curvefs/src/client/base_client.h b/curvefs/src/client/base_client.h deleted file mode 100644 index c24f2cdea6..0000000000 --- a/curvefs/src/client/base_client.h +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Project: curve - * Created Date: Fri Jun 11 2021 - * Author: lixiaocui - */ -#ifndef CURVEFS_SRC_CLIENT_BASE_CLIENT_H_ -#define CURVEFS_SRC_CLIENT_BASE_CLIENT_H_ - -#include -#include - -#include -#include -#include - -#include "curvefs/proto/mds.pb.h" -#include "curvefs/proto/metaserver.pb.h" -#include "curvefs/proto/space.pb.h" -#include "curvefs/src/client/common/extent.h" - -namespace curvefs { -namespace client { - -using curvefs::space::AllocateSpaceRequest; -using curvefs::space::AllocateSpaceResponse; -using curvefs::space::DeallocateSpaceRequest; -using curvefs::space::DeallocateSpaceResponse; -using curvefs::space::Extent; - -using common::ExtentAllocInfo; - -class SpaceBaseClient { - public: - virtual ~SpaceBaseClient() = default; - - virtual void AllocExtents(uint32_t fsId, - const ExtentAllocInfo &toAllocExtent, - curvefs::space::AllocateType type, - AllocateSpaceResponse *response, - brpc::Controller *cntl, brpc::Channel *channel); - - virtual void DeAllocExtents(uint32_t fsId, - const std::list &allocatedExtents, - DeallocateSpaceResponse *response, - brpc::Controller *cntl, brpc::Channel *channel); -}; - -} // namespace client -} // namespace curvefs - -#endif // CURVEFS_SRC_CLIENT_BASE_CLIENT_H_ diff --git a/curvefs/src/client/common/config.cpp b/curvefs/src/client/common/config.cpp index 1404864b1f..706140ec6c 100644 --- a/curvefs/src/client/common/config.cpp +++ b/curvefs/src/client/common/config.cpp @@ -89,16 +89,10 @@ void InitExcutorOption(Configuration *conf, ExcutorOpt *opts) { conf->GetValueFatalIfFail("excutorOpt.batchLimit", &opts->batchLimit); } -void InitSpaceServerOption(Configuration *conf, - SpaceAllocServerOption *spaceOpt) { - conf->GetValueFatalIfFail("spaceserver.spaceaddr", &spaceOpt->spaceaddr); - conf->GetValueFatalIfFail("spaceserver.rpcTimeoutMs", - &spaceOpt->rpcTimeoutMs); -} - void InitBlockDeviceOption(Configuration *conf, BlockDeviceClientOptions *bdevOpt) { conf->GetValueFatalIfFail("bdev.confpath", &bdevOpt->configPath); + conf->GetValueFatalIfFail("bdev.threadnum", &bdevOpt->threadnum); } void InitDiskCacheOption(Configuration *conf, @@ -166,6 +160,24 @@ void InitVolumeOption(Configuration *conf, VolumeOption *volumeOpt) { conf->GetValueFatalIfFail("volume.bigFileSize", &volumeOpt->bigFileSize); conf->GetValueFatalIfFail("volume.volBlockSize", &volumeOpt->volBlockSize); conf->GetValueFatalIfFail("volume.fsBlockSize", &volumeOpt->fsBlockSize); + conf->GetValueFatalIfFail("volume.allocator.type", + &volumeOpt->allocatorOption.type); + + conf->GetValueFatalIfFail( + "volume.blockgroup.allocate_once", + &volumeOpt->allocatorOption.blockGroupOption.allocateOnce); + + if (volumeOpt->allocatorOption.type == "bitmap") { + conf->GetValueFatalIfFail( + "volume.bitmapallocator.size_per_bit", + &volumeOpt->allocatorOption.bitmapAllocatorOption.sizePerBit); + conf->GetValueFatalIfFail( + "volume.bitmapallocator.small_alloc_proportion", + &volumeOpt->allocatorOption.bitmapAllocatorOption + .smallAllocProportion); + } else { + CHECK(false) << "only support bitmap allocator"; + } } void InitExtentManagerOption(Configuration *conf, @@ -186,7 +198,6 @@ void InitFuseClientOption(Configuration *conf, FuseClientOption *clientOption) { InitMdsOption(conf, &clientOption->mdsOpt); InitMetaCacheOption(conf, &clientOption->metaCacheOpt); InitExcutorOption(conf, &clientOption->excutorOpt); - InitSpaceServerOption(conf, &clientOption->spaceOpt); InitBlockDeviceOption(conf, &clientOption->bdevOpt); InitS3Option(conf, &clientOption->s3Opt); InitExtentManagerOption(conf, &clientOption->extentManagerOpt); diff --git a/curvefs/src/client/common/config.h b/curvefs/src/client/common/config.h index 94162c7a90..c92292daae 100644 --- a/curvefs/src/client/common/config.h +++ b/curvefs/src/client/common/config.h @@ -43,8 +43,8 @@ namespace common { using MdsOption = ::curve::client::MetaServerOption; struct BlockDeviceClientOptions { - // config path std::string configPath; + uint32_t threadnum; }; struct MetaCacheOpt { @@ -125,10 +125,26 @@ struct S3Option { S3AdapterOption s3AdaptrOpt; }; +struct BlockGroupOption { + uint32_t allocateOnce; +}; + +struct BitmapAllocatorOption { + uint64_t sizePerBit; + double smallAllocProportion; +}; + +struct VolumeAllocatorOption { + std::string type; + BitmapAllocatorOption bitmapAllocatorOption; + BlockGroupOption blockGroupOption; +}; + struct VolumeOption { uint64_t bigFileSize; uint64_t volBlockSize; uint64_t fsBlockSize; + VolumeAllocatorOption allocatorOption; }; struct ExtentManagerOption { diff --git a/curvefs/src/client/curve_fuse_op.cpp b/curvefs/src/client/curve_fuse_op.cpp index d304de8af4..d2b694279d 100644 --- a/curvefs/src/client/curve_fuse_op.cpp +++ b/curvefs/src/client/curve_fuse_op.cpp @@ -69,13 +69,13 @@ void EnableSplice(struct fuse_conn_info* conn) { } } -int GetFsInfo(const char* fsName, std::shared_ptr fsInfo) { +int GetFsInfo(const char* fsName, FsInfo* fsInfo) { MdsClientImpl mdsClient; MDSBaseClient mdsBase; mdsClient.Init(g_fuseClientOption->mdsOpt, &mdsBase); std::string fn = (fsName == nullptr) ? "" : fsName; - FSStatusCode ret = mdsClient.GetFsInfo(fn, fsInfo.get()); + FSStatusCode ret = mdsClient.GetFsInfo(fn, fsInfo); if (ret != FSStatusCode::OK) { if (FSStatusCode::NOT_FOUND == ret) { LOG(ERROR) << "The fsName not exist, fsName = " << fsName; @@ -132,7 +132,7 @@ int InitFuseClient(const char *confPath, const char* fsName, curvefs::client::common::InitFuseClientOption(&conf, g_fuseClientOption); std::shared_ptr fsInfo = std::make_shared(); - if (GetFsInfo(fsName, fsInfo) != 0) { + if (GetFsInfo(fsName, fsInfo.get()) != 0) { return -1; } @@ -165,6 +165,7 @@ int InitFuseClient(const char *confPath, const char* fsName, if (ret != CURVEFS_ERROR::OK) { return -1; } + return 0; } @@ -469,3 +470,11 @@ void FuseOpFlush(fuse_req_t req, fuse_ino_t ino, CURVEFS_ERROR ret = g_ClientInstance->FuseOpFlush(req, ino, fi); FuseReplyErrByErrCode(req, ret); } + +void FuseOpBmap(fuse_req_t req, + fuse_ino_t /*ino*/, + size_t /*blocksize*/, + uint64_t /*idx*/) { + // TODO(wuhanqing): implement for volume storage + FuseReplyErrByErrCode(req, CURVEFS_ERROR::NOTSUPPORT); +} diff --git a/curvefs/src/client/error_code.h b/curvefs/src/client/error_code.h index 5c28bd7fc1..a9eabee53b 100644 --- a/curvefs/src/client/error_code.h +++ b/curvefs/src/client/error_code.h @@ -48,6 +48,7 @@ enum class CURVEFS_ERROR { MOUNT_FAILED = -14, OUT_OF_RANGE = -15, NODATA = -16, + IO_ERROR = -17, }; diff --git a/curvefs/src/client/extent_manager.cpp b/curvefs/src/client/extent_manager.cpp deleted file mode 100644 index 8a35f10bda..0000000000 --- a/curvefs/src/client/extent_manager.cpp +++ /dev/null @@ -1,422 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Project: curve - * Created Date: Thur May 27 2021 - * Author: xuchaojie - */ -#include "curvefs/src/client/extent_manager.h" - -#include "glog/logging.h" - -namespace curvefs { -namespace client { - -CURVEFS_ERROR SimpleExtentManager::GetToAllocExtents( - const VolumeExtentList &extents, uint64_t offset, uint64_t len, - std::list *toAllocExtents) { - toAllocExtents->clear(); - uint64_t pOffsetLeft = 0; - bool leftHintAvailable = false; - for (int i = 0; i < extents.volumeextents_size(); i++) { - uint64_t left = extents.volumeextents(i).fsoffset(); - uint64_t right = left + extents.volumeextents(i).length(); - if (offset >= left) { - // - // extent of [offset, len] |------| - // volumeextents ... |-------| ... - // - if (offset > right) { - pOffsetLeft = 0; - leftHintAvailable = false; - continue; - // - // extent of [offset, len] |------| - // volumeextents ... |-------| ... - // - } else if ((offset + len) > right) { - len = offset + len - right; - offset = right; - pOffsetLeft = extents.volumeextents(i).volumeoffset() + - extents.volumeextents(i).length(); - leftHintAvailable = true; - continue; - // - // extent of [offset, len] |------| - // volumeextents ... |----------| ... - // - } else { - return CURVEFS_ERROR::OK; - } - } else if ((offset + len) >= left) { - ExtentAllocInfo allocInfo; - allocInfo.lOffset = offset; - allocInfo.len = left - offset; - allocInfo.leftHintAvailable = leftHintAvailable; - allocInfo.pOffsetLeft = pOffsetLeft; - allocInfo.rightHintAvailable = true; - allocInfo.pOffsetRight = extents.volumeextents(i).volumeoffset(); - toAllocExtents->push_back(allocInfo); - // - // extent of [offset, len] |-------------| - // volumeextents ... |-------| ... - // - if ((offset + len) > right) { - len = offset + len - right; - offset = right; - pOffsetLeft = extents.volumeextents(i).volumeoffset() + - extents.volumeextents(i).length(); - leftHintAvailable = true; - continue; - // - // extent of [offset, len] |-------| - // volumeextents ... |-------| ... - // - } else { - return CURVEFS_ERROR::OK; - } - // - // extent of [offset, len] |-------| - // volumeextents ... |-------| ... - // - } else { - ExtentAllocInfo allocInfo; - allocInfo.lOffset = offset; - allocInfo.len = len; - allocInfo.leftHintAvailable = leftHintAvailable; - allocInfo.pOffsetLeft = pOffsetLeft; - allocInfo.rightHintAvailable = false; - allocInfo.pOffsetRight = 0; - toAllocExtents->push_back(allocInfo); - return CURVEFS_ERROR::OK; - } - } - // the extent of [offset, len] is at the right of all volumeextents. - if (len > 0) { - // prealloc - if (len > preAllocSize_) { - len = (len / preAllocSize_ + 1) * preAllocSize_; - } else { - len = preAllocSize_; - } - ExtentAllocInfo allocInfo; - allocInfo.lOffset = offset; - allocInfo.len = len; - allocInfo.leftHintAvailable = leftHintAvailable; - allocInfo.pOffsetLeft = pOffsetLeft; - allocInfo.rightHintAvailable = false; - allocInfo.pOffsetRight = 0; - toAllocExtents->push_back(allocInfo); - } - return CURVEFS_ERROR::OK; -} - -CURVEFS_ERROR SimpleExtentManager::MergeAllocedExtents( - const std::list &toAllocExtents, - const std::list &allocatedExtents, VolumeExtentList *extents) { - CURVEFS_ERROR ret = CURVEFS_ERROR::OK; - VolumeExtentList newExtents; - auto it = toAllocExtents.begin(); - auto ix = allocatedExtents.begin(); - int i = 0; - while (i < extents->volumeextents_size() || it != toAllocExtents.end()) { - if ((it == toAllocExtents.end()) || - ((i < extents->volumeextents_size()) && - (extents->volumeextents(i).fsoffset() <= it->lOffset))) { - ret = MergeToTheLastOrAdd(&newExtents, &extents->volumeextents(i)); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "MergeToTheLastOrAdd failed, ret = " << ret; - return ret; - } - i++; - } else { - uint64_t len = 0; - uint64_t lOffset = it->lOffset; - while (len < it->len && ix != allocatedExtents.end()) { - VolumeExtent temp; - temp.set_fsoffset(lOffset); - temp.set_volumeoffset(ix->offset()); - temp.set_length(ix->length()); - temp.set_isused(false); - ret = MergeToTheLastOrAdd(&newExtents, &temp); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "MergeToTheLastOrAdd failed, ret = " << ret; - return ret; - } - len += ix->length(); - lOffset += ix->length(); - ix++; - } - if (len != it->len) { - LOG(ERROR) << "MergeAllocedExtents find toAllocExtents and" - << " allocatedExtents not match, " - << "need len = " << it->len - << ", actual len = " << len; - return CURVEFS_ERROR::INTERNAL; - } - it++; - } - } - extents->Swap(&newExtents); - return CURVEFS_ERROR::OK; -} - -CURVEFS_ERROR -SimpleExtentManager::MergeToTheLastOrAdd(VolumeExtentList *extents, - const VolumeExtent *extent) { - VLOG(6) << "MergeToTheLastOrAdd newExtent {" - << "lOffset = " << extent->fsoffset() - << ", pOffset = " << extent->volumeoffset() - << ", length = " << extent->length() - << ", isused = " << extent->isused() << "}"; - - if (extents->volumeextents_size() == 0) { - VolumeExtent *ext = extents->add_volumeextents(); - ext->CopyFrom(*extent); - return CURVEFS_ERROR::OK; - } - int last = extents->volumeextents_size() - 1; - uint64_t lLeft = extents->volumeextents(last).fsoffset(); - uint64_t lRight = lLeft + extents->volumeextents(last).length(); - uint64_t pLeft = extents->volumeextents(last).volumeoffset(); - uint64_t pRight = pLeft + extents->volumeextents(last).length(); - if (extent->fsoffset() < lRight) { - LOG(ERROR) << "The new extent is overlaped, " - << " fsoffset = " << extent->fsoffset() - << " is < the last extent lRight = " << lRight; - return CURVEFS_ERROR::INTERNAL; - } - if (extent->fsoffset() == lRight && extent->volumeoffset() == pRight) { - if (extent->isused() == extents->volumeextents(last).isused()) { - uint64_t newLen = extent->fsoffset() + extent->length() - - extents->volumeextents(last).fsoffset(); - extents->mutable_volumeextents(last)->set_length(newLen); - return CURVEFS_ERROR::OK; - } - } - VolumeExtent *ext = extents->add_volumeextents(); - ext->CopyFrom(*extent); - return CURVEFS_ERROR::OK; -} - -CURVEFS_ERROR -SimpleExtentManager::MarkExtentsWritten(uint64_t offset, uint64_t len, - VolumeExtentList *extents) { - CURVEFS_ERROR ret = CURVEFS_ERROR::OK; - VolumeExtentList newExtents; - int i = 0; - for (; i < extents->volumeextents_size() && len != 0; i++) { - uint64_t lLeft = extents->volumeextents(i).fsoffset(); - uint64_t length = extents->volumeextents(i).length(); - uint64_t lRight = lLeft + length; - uint64_t pLeft = extents->volumeextents(i).volumeoffset(); - bool isused = extents->volumeextents(i).isused(); - // - // extent of [offset, len] |------ ... - // volumeextents ... |-------| ... - // - if (offset < lLeft) { - LOG(ERROR) << "MarkExtentsWritten find write [offset len]" - << "is not allocated, offset = " << offset - << ", len = " << len - << ", right neighbor lLeft = " << lLeft - << ", lRight = " << lRight; - return CURVEFS_ERROR::INTERNAL; - // - // extent of [offset, len] |-------| - // volumeextents ... |-------| ... - // - } else if (offset >= lRight) { - ret = MergeToTheLastOrAdd(&newExtents, &extents->volumeextents(i)); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "MergeToTheLastOrAdd failed, ret = " << ret; - return ret; - } - // - // extent of [offset, len] |--- ... - // volumeextents ... |-------| ... - // - } else { - VolumeExtent currentExtent; - currentExtent.CopyFrom(extents->volumeextents(i)); - if (offset > lLeft) { - if (!isused) { - // cut currentExtent's left part - VolumeExtent left; - left.set_fsoffset(lLeft); - left.set_volumeoffset(pLeft); - left.set_length(offset - lLeft); - left.set_isused(false); - ret = MergeToTheLastOrAdd(&newExtents, &left); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) - << "MergeToTheLastOrAdd failed, ret = " << ret; - return ret; - } - - pLeft = pLeft + (offset - lLeft); - currentExtent.set_volumeoffset(pLeft); - lLeft = offset; - currentExtent.set_fsoffset(lLeft); - length = lRight - lLeft; - currentExtent.set_length(length); - currentExtent.set_isused(isused); - } - } - // - // extent of [offset, len] |-------| - // volumeextents ... |-------| ... - // - if (offset + len >= lRight) { - currentExtent.set_isused(true); - ret = MergeToTheLastOrAdd(&newExtents, ¤tExtent); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "MergeToTheLastOrAdd failed, ret = " << ret; - return ret; - } - // cut [offset len] left part - len = offset + len - lRight; - offset = lRight; - // - // extent of [offset, len] |-------| - // volumeextents ... |-----------| ... - // - } else { - // cut currentExtent's right part - currentExtent.set_length(len); - currentExtent.set_isused(true); - - VolumeExtent right; - right.set_fsoffset(lLeft + len); - right.set_volumeoffset(pLeft + len); - right.set_length(length - len); - right.set_isused(isused); - ret = MergeToTheLastOrAdd(&newExtents, ¤tExtent); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "MergeToTheLastOrAdd failed, ret = " << ret; - return ret; - } - ret = MergeToTheLastOrAdd(&newExtents, &right); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "MergeToTheLastOrAdd failed, ret = " << ret; - return ret; - } - len = 0; - } - } - } - for (; i < extents->volumeextents_size(); i++) { - VolumeExtent *ext = newExtents.add_volumeextents(); - ext->CopyFrom(extents->volumeextents(i)); - } - extents->Swap(&newExtents); - return CURVEFS_ERROR::OK; -} - -CURVEFS_ERROR -SimpleExtentManager::DivideExtents(const VolumeExtentList &extents, - uint64_t offset, uint64_t len, - std::list *pExtents) { - CURVEFS_ERROR ret = CURVEFS_ERROR::OK; - pExtents->clear(); - for (int i = 0; i < extents.volumeextents_size() && len != 0; i++) { - uint64_t lLeft = extents.volumeextents(i).fsoffset(); - uint64_t length = extents.volumeextents(i).length(); - uint64_t lRight = lLeft + length; - uint64_t pLeft = extents.volumeextents(i).volumeoffset(); - uint64_t pRight = pLeft + length; - bool isused = extents.volumeextents(i).isused(); - // - // extent of [offset, len] |------ ... - // volumeextents ... |-------| ... - // - if (offset < lLeft) { - // TODO(xuchaojie) : fix when have hole - LOG(WARNING) << "DivideExtents find [offset len]" - << "is not allocated, offset = " << offset - << ", len = " << len - << ", right neighbor lLeft = " << lLeft - << ", lRight = " << lRight; - return CURVEFS_ERROR::INTERNAL; - // - // extent of [offset, len] |-------| - // volumeextents ... |-------| ... - // - } else if (offset >= lRight) { - continue; - } else { - // - // extent of [offset, len] |-------| - // volumeextents ... |-------| ... - // - if (offset + len >= lRight) { - PExtent pExt; - pExt.pOffset = pLeft + (offset - lLeft); - pExt.len = pRight - pExt.pOffset; - pExt.UnWritten = !isused; - ret = MergeToTheLastOrAdd(pExtents, pExt); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "MergeToTheLastOrAdd failed, ret = " << ret; - return ret; - } - len = offset + len - lRight; - offset = lRight; - // - // extent of [offset, len] |-------| - // volumeextents ... |-----------| ... - // - } else { - PExtent pExt; - pExt.pOffset = pLeft + (offset - lLeft); - pExt.len = len; - pExt.UnWritten = !isused; - ret = MergeToTheLastOrAdd(pExtents, pExt); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "MergeToTheLastOrAdd failed, ret = " << ret; - return ret; - } - break; - } - } - } - return CURVEFS_ERROR::OK; -} - -CURVEFS_ERROR -SimpleExtentManager::MergeToTheLastOrAdd(std::list *pExtents, - const PExtent &pExt) { - VLOG(6) << "MergeToTheLastOrAdd pExtent {" - << "pOffset = " << pExt.pOffset << ", len = " << pExt.len - << ", UnWritten = " << pExt.UnWritten; - if (pExtents->empty()) { - pExtents->push_back(pExt); - return CURVEFS_ERROR::OK; - } - auto last = pExtents->back(); - if (pExt.pOffset == last.pOffset + last.len) { - if (pExt.UnWritten == last.UnWritten) { - last.len += pExt.len; - return CURVEFS_ERROR::OK; - } - } - pExtents->push_back(pExt); - return CURVEFS_ERROR::OK; -} - -} // namespace client -} // namespace curvefs diff --git a/curvefs/src/client/extent_manager.h b/curvefs/src/client/extent_manager.h deleted file mode 100644 index 10f9a63ba2..0000000000 --- a/curvefs/src/client/extent_manager.h +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Project: curve - * Created Date: Thur May 27 2021 - * Author: xuchaojie - */ - -#ifndef CURVEFS_SRC_CLIENT_EXTENT_MANAGER_H_ -#define CURVEFS_SRC_CLIENT_EXTENT_MANAGER_H_ - -#include -#include - -#include "curvefs/proto/metaserver.pb.h" -#include "curvefs/proto/space.pb.h" -#include "curvefs/src/client/common/extent.h" -#include "curvefs/src/client/error_code.h" -#include "curvefs/src/client/common/config.h" - -using ::curvefs::metaserver::VolumeExtentList; -using ::curvefs::metaserver::VolumeExtent; -using ::curvefs::space::Extent; - -namespace curvefs { -namespace client { - -using common::ExtentManagerOption; -using common::ExtentAllocInfo; -using common::PExtent; - -class ExtentManager { - public: - ExtentManager() {} - virtual ~ExtentManager() {} - - virtual CURVEFS_ERROR Init(const ExtentManagerOption &options) = 0; - - /** - * @brief According to the extents in the file, - * divide the [offset, len] to be written, - * and get the extents that need to be allocated - * - * @param[in] extents extents currently allocated in the file - * @param[in] offset offset to be written - * @param[in] len len to be written - * @param[out] toAllocExtents extents that need to be allocated - * - * @return errcode - */ - virtual CURVEFS_ERROR GetToAllocExtents(const VolumeExtentList &extents, - uint64_t offset, - uint64_t len, - std::list *toAllocExtents) = 0; - - /** - * @brief Merge extents allocated to extents in the file - * - * @param[in] allocedExtents extents allocated - * @param[in] toAllocExtents extents that need to be allocated - * @param[in,out] extents extents in the file - * - * @return errcode - */ - virtual CURVEFS_ERROR MergeAllocedExtents( - const std::list &toAllocExtents, - const std::list &allocatedExtents, - VolumeExtentList *extents) = 0; - - /** - * @brief mark unwritten extents to written in the file extents - * - * @param[in] offset offset to be written - * @param[in] len len to be written - * @param[in,out] extents extents in the file - * - * @return errcode - */ - virtual CURVEFS_ERROR MarkExtentsWritten(uint64_t offset, uint64_t len, - VolumeExtentList *extents) = 0; - - /** - * @brief According to the extents in the file, - * divide the [lOffset, len] to be read/write, - * and get the physical extents - * - * @param extents extents in the file - * @param offset logical offset to be read/write - * @param len len to be read/write - * @param PExtent physical extents - * - * @return errcode - */ - virtual CURVEFS_ERROR DivideExtents(const VolumeExtentList &extents, - uint64_t offset, - uint64_t len, - std::list *pExtents) = 0; -}; - -class SimpleExtentManager : public ExtentManager { - public: - SimpleExtentManager() {} - - CURVEFS_ERROR Init(const ExtentManagerOption &options) { - preAllocSize_ = options.preAllocSize; - return CURVEFS_ERROR::OK; - } - - CURVEFS_ERROR GetToAllocExtents(const VolumeExtentList &extents, - uint64_t offset, - uint64_t len, - std::list *toAllocExtents) override; - - CURVEFS_ERROR MergeAllocedExtents( - const std::list &toAllocExtents, - const std::list &allocatedExtents, - VolumeExtentList *extents) override; - - CURVEFS_ERROR MarkExtentsWritten(uint64_t offset, uint64_t len, - VolumeExtentList *extents) override; - - CURVEFS_ERROR DivideExtents(const VolumeExtentList &extents, - uint64_t offset, - uint64_t len, - std::list *pExtents) override; - - private: - CURVEFS_ERROR MergeToTheLastOrAdd(VolumeExtentList *extents, - const VolumeExtent *extent); - - CURVEFS_ERROR MergeToTheLastOrAdd(std::list *pExtents, - const PExtent &pExt); - - private: - uint64_t preAllocSize_; -}; - -} // namespace client -} // namespace curvefs - -#endif // CURVEFS_SRC_CLIENT_EXTENT_MANAGER_H_ diff --git a/curvefs/src/client/fuse_client.cpp b/curvefs/src/client/fuse_client.cpp index 6226d2da93..fcfd362166 100644 --- a/curvefs/src/client/fuse_client.cpp +++ b/curvefs/src/client/fuse_client.cpp @@ -34,7 +34,7 @@ #include "curvefs/proto/mds.pb.h" #include "curvefs/src/client/fuse_common.h" -#include "curvefs/src/client/extent_manager.h" +#include "curvefs/src/client/client_operator.h" #include "src/common/timeutility.h" #include "src/common/dummyserver.h" #include "src/client/client_common.h" @@ -155,33 +155,34 @@ CURVEFS_ERROR FuseClient::FuseOpInit(void *userdata, (mOpts->mountPoint == nullptr) ? "" : mOpts->mountPoint; std::string fsName = (mOpts->fsName == nullptr) ? "" : mOpts->fsName; - std::string mountPointWithHost; - int retVal = AddHostNameToMountPointStr(mountPointStr, &mountPointWithHost); + int retVal = AddHostNameToMountPointStr(mountPointStr, &mountpoint_); if (retVal < 0) { LOG(ERROR) << "AddHostNameToMountPointStr failed, ret = " << retVal; return CURVEFS_ERROR::INTERNAL; } auto find = std::find(fsInfo_->mountpoints().begin(), - fsInfo_->mountpoints().end(), mountPointWithHost); + fsInfo_->mountpoints().end(), mountpoint_); if (find != fsInfo_->mountpoints().end()) { LOG(ERROR) << "MountFs found mountPoint exist"; return CURVEFS_ERROR::MOUNT_POINT_EXIST; } - auto ret = mdsClient_->MountFs(fsName, mountPointWithHost, fsInfo_.get()); + + auto ret = mdsClient_->MountFs(fsName, mountpoint_, fsInfo_.get()); if (ret != FSStatusCode::OK && ret != FSStatusCode::MOUNT_POINT_EXIST) { LOG(ERROR) << "MountFs failed, FSStatusCode = " << ret << ", FSStatusCode_Name = " << FSStatusCode_Name(ret) << ", fsName = " << fsName - << ", mountPoint = " << mountPointWithHost; + << ", mountPoint = " << mountpoint_; return CURVEFS_ERROR::MOUNT_FAILED; } + inodeManager_->SetFsId(fsInfo_->fsid()); dentryManager_->SetFsId(fsInfo_->fsid()); enableSumInDir_ = fsInfo_->enablesumindir() && !FLAGS_enableCto; - LOG(INFO) << "Mount " << fsName << " on " << mountPointWithHost - << " success!" << " enableSumInDir = " << enableSumInDir_; + LOG(INFO) << "Mount " << fsName << " on " << mountpoint_ << " success!" + << " enableSumInDir = " << enableSumInDir_; fsMetric_ = std::make_shared(fsName); diff --git a/curvefs/src/client/fuse_client.h b/curvefs/src/client/fuse_client.h index bd820056b7..2f3663f648 100644 --- a/curvefs/src/client/fuse_client.h +++ b/curvefs/src/client/fuse_client.h @@ -33,7 +33,6 @@ #include "curvefs/proto/common.pb.h" #include "curvefs/proto/mds.pb.h" -#include "curvefs/src/client/block_device_client.h" #include "curvefs/src/client/common/config.h" #include "curvefs/src/client/dentry_cache_manager.h" #include "curvefs/src/client/dir_buffer.h" @@ -49,7 +48,7 @@ #include "curvefs/src/client/common/common.h" #include "curvefs/src/client/client_operator.h" -#define DirectIOAlignemnt 512 +#define DirectIOAlignment 512 using ::curve::common::Atomic; using ::curve::common::InterruptibleSleeper; @@ -204,7 +203,7 @@ class FuseClient { return CURVEFS_ERROR::OK; } - void SetFsInfo(std::shared_ptr fsInfo) { + void SetFsInfo(const std::shared_ptr& fsInfo) { fsInfo_ = fsInfo; init_ = true; } @@ -297,6 +296,8 @@ class FuseClient { std::shared_ptr fsMetric_; + std::string mountpoint_; + private: MDSBaseClient* mdsBase_; diff --git a/curvefs/src/client/fuse_s3_client.cpp b/curvefs/src/client/fuse_s3_client.cpp index 159fd57ec9..a9d65b6864 100644 --- a/curvefs/src/client/fuse_s3_client.cpp +++ b/curvefs/src/client/fuse_s3_client.cpp @@ -78,8 +78,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpWrite(fuse_req_t req, fuse_ino_t ino, size_t *wSize) { // check align if (fi->flags & O_DIRECT) { - if (!(is_aligned(off, DirectIOAlignemnt) && - is_aligned(size, DirectIOAlignemnt))) + if (!(is_aligned(off, DirectIOAlignment) && + is_aligned(size, DirectIOAlignment))) return CURVEFS_ERROR::INVALIDPARAM; } uint64_t start = butil::cpuwide_time_us(); @@ -149,8 +149,8 @@ CURVEFS_ERROR FuseS3Client::FuseOpRead(fuse_req_t req, fuse_ino_t ino, size_t *rSize) { // check align if (fi->flags & O_DIRECT) { - if (!(is_aligned(off, DirectIOAlignemnt) && - is_aligned(size, DirectIOAlignemnt))) + if (!(is_aligned(off, DirectIOAlignment) && + is_aligned(size, DirectIOAlignment))) return CURVEFS_ERROR::INVALIDPARAM; } diff --git a/curvefs/src/client/fuse_volume_client.cpp b/curvefs/src/client/fuse_volume_client.cpp index b98ff10cf3..7a41b3f895 100644 --- a/curvefs/src/client/fuse_volume_client.cpp +++ b/curvefs/src/client/fuse_volume_client.cpp @@ -21,253 +21,210 @@ * Author: xuchaojie */ -#include -#include +#include "curvefs/src/client/fuse_volume_client.h" + +#include +#include #include +#include -#include "curvefs/src/client/fuse_volume_client.h" +#include "absl/cleanup/cleanup.h" +#include "absl/memory/memory.h" +#include "curvefs/src/client/volume/default_volume_storage.h" +#include "curvefs/src/volume/common.h" +#include "curvefs/src/volume/option.h" namespace curvefs { namespace client { +using ::curvefs::volume::SpaceManagerImpl; +using ::curvefs::volume::SpaceManagerOption; +using ::curvefs::volume::BlockDeviceClientOptions; +using ::curvefs::volume::BlockDeviceClientImpl; + CURVEFS_ERROR FuseVolumeClient::Init(const FuseClientOption &option) { volOpts_ = option.volumeOpt; CURVEFS_ERROR ret = FuseClient::Init(option); - if (ret != CURVEFS_ERROR::OK) { - return ret; - } - spaceBase_ = std::make_shared(); - ret = spaceClient_->Init(option.spaceOpt, spaceBase_.get()); + if (ret != CURVEFS_ERROR::OK) { return ret; } - ret = extManager_->Init(option.extentManagerOpt); - if (ret != CURVEFS_ERROR::OK) { - return ret; + BlockDeviceClientOptions opts; + opts.configPath = option.bdevOpt.configPath; + opts.threadnum = option.bdevOpt.threadnum; + + bool ret2 = blockDeviceClient_->Init(opts); + + if (!ret2) { + LOG(ERROR) << "Init block device client failed"; + return CURVEFS_ERROR::INTERNAL; } - ret = blockDeviceClient_->Init(option.bdevOpt); + return ret; } void FuseVolumeClient::UnInit() { + storage_->Shutdown(); + spaceManager_->Shutdown(); blockDeviceClient_->UnInit(); + FuseClient::UnInit(); } -CURVEFS_ERROR FuseVolumeClient::FuseOpInit( - void *userdata, struct fuse_conn_info *conn) { - struct MountOption *mOpts = (struct MountOption *) userdata; - std::string volName = (mOpts->volume == nullptr) ? "" : mOpts->volume; - std::string user = (mOpts->user == nullptr) ? "" : mOpts->user; - CURVEFS_ERROR ret = blockDeviceClient_->Open(volName, user); +CURVEFS_ERROR FuseVolumeClient::FuseOpInit(void *userdata, + struct fuse_conn_info *conn) { + auto ret = FuseClient::FuseOpInit(userdata, conn); if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "BlockDeviceClientImpl open failed, ret = " << ret - << ", volName = " << volName - << ", user = " << user; + LOG(ERROR) << "fuse op init failed, error: " << ret; return ret; } - return FuseClient::FuseOpInit(userdata, conn); -} -void FuseVolumeClient::FuseOpDestroy(void *userdata) { - FuseClient::FuseOpDestroy(userdata); - CURVEFS_ERROR ret = blockDeviceClient_->Close(); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "BlockDeviceClientImpl close failed, ret = " << ret; - return; + const auto &volName = fsInfo_->detail().volume().volumename(); + const auto &user = fsInfo_->detail().volume().user(); + auto ret2 = blockDeviceClient_->Open(volName, user); + if (!ret2) { + LOG(ERROR) << "BlockDeviceClientImpl open failed, ret = " << ret + << ", volName = " << volName << ", user = " << user; + return CURVEFS_ERROR::INTERNAL; } - return; + + SpaceManagerOption option; + option.blockGroupManagerOption.fsId = fsInfo_->fsid(); + option.blockGroupManagerOption.owner = mountpoint_; + option.blockGroupManagerOption.blockGroupAllocateOnce = + volOpts_.allocatorOption.blockGroupOption.allocateOnce; + option.blockGroupManagerOption.blockGroupSize = + fsInfo_->detail().volume().blockgroupsize(); + option.blockGroupManagerOption.blockSize = + fsInfo_->detail().volume().blocksize(); + + option.allocatorOption.type = volOpts_.allocatorOption.type; + option.allocatorOption.bitmapAllocatorOption.sizePerBit = + volOpts_.allocatorOption.bitmapAllocatorOption.sizePerBit; + option.allocatorOption.bitmapAllocatorOption.smallAllocProportion = + volOpts_.allocatorOption.bitmapAllocatorOption.smallAllocProportion; + + spaceManager_ = absl::make_unique(option, mdsClient_, + blockDeviceClient_); + + storage_ = absl::make_unique( + spaceManager_.get(), blockDeviceClient_.get(), inodeManager_.get()); + + return CURVEFS_ERROR::OK; } -CURVEFS_ERROR FuseVolumeClient::FuseOpWrite(fuse_req_t req, fuse_ino_t ino, - const char *buf, size_t size, +CURVEFS_ERROR FuseVolumeClient::FuseOpWrite(fuse_req_t req, + fuse_ino_t ino, + const char *buf, + size_t size, off_t off, struct fuse_file_info *fi, size_t *wSize) { - // check align + VLOG(9) << "write start, ino: " << ino << ", offset: " << off + << ", length: " << size; + if (fi->flags & O_DIRECT) { - if (!(is_aligned(off, DirectIOAlignemnt) && - is_aligned(size, DirectIOAlignemnt))) + if (!(is_aligned(off, DirectIOAlignment) && + is_aligned(size, DirectIOAlignment))) { + fsMetric_->userWrite.eps.count << 1; return CURVEFS_ERROR::INVALIDPARAM; - } - - std::shared_ptr inodeWrapper; - CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "inodeManager get inode fail, ret = " << ret - << ", inodeid = " << ino; - return ret; - } - - ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock(); - Inode inode = inodeWrapper->GetInodeUnlocked(); - - std::list toAllocExtents; - // get the extent need to be allocate - ret = extManager_->GetToAllocExtents(inode.volumeextentlist(), off, size, - &toAllocExtents); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "GetToAllocExtents fail, ret = " << ret; - return ret; - } - if (toAllocExtents.size() != 0) { - AllocateType type = AllocateType::NONE; - if (inode.length() >= volOpts_.bigFileSize || - size >= volOpts_.bigFileSize) { - type = AllocateType::BIG; - } else { - type = AllocateType::SMALL; - } - std::list allocatedExtents; - // to alloc extents - ret = spaceClient_->AllocExtents(fsInfo_->fsid(), toAllocExtents, type, - &allocatedExtents); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "metaClient alloc extents fail, ret = " << ret; - return ret; - } - // merge the allocated extent to inode - ret = extManager_->MergeAllocedExtents( - toAllocExtents, allocatedExtents, inode.mutable_volumeextentlist()); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "toAllocExtents and allocatedExtents not match, " - << "ret = " << ret; - CURVEFS_ERROR ret2 = - spaceClient_->DeAllocExtents(fsInfo_->fsid(), allocatedExtents); - if (ret2 != CURVEFS_ERROR::OK) { - LOG(ERROR) << "DeAllocExtents fail, ret = " << ret; - } - return ret; } } - // divide the extents which is write or not - std::list pExtents; - ret = extManager_->DivideExtents(inode.volumeextentlist(), off, size, - &pExtents); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "DivideExtents fail, ret = " << ret; - return ret; - } + butil::Timer timer; + timer.start(); - // write the physical extents - uint64_t writeLen = 0; - for (const auto &ext : pExtents) { - ret = blockDeviceClient_->Write(buf + writeLen, ext.pOffset, ext.len); - writeLen += ext.len; - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "block device write fail, ret = " << ret; - return ret; + ssize_t nr = storage_->Write(ino, off, size, buf); + if (nr < 0) { + if (fsMetric_) { + fsMetric_->userWrite.eps.count << 1; } + LOG(ERROR) << "write error, ino: " << ino << ", offset: " << off + << ", len: " << size; + return CURVEFS_ERROR::IO_ERROR; } - // make the unwritten flag in the inode. - ret = extManager_->MarkExtentsWritten(off, size, - inode.mutable_volumeextentlist()); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "MarkExtentsWritten fail, ret = " << ret; - return ret; - } *wSize = size; - // update file len - if (inode.length() < off + *wSize) { - inode.set_length(off + *wSize); - } - - struct timespec now; - clock_gettime(CLOCK_REALTIME, &now); - inode.set_mtime(now.tv_sec); - inode.set_mtime_ns(now.tv_nsec); - inode.set_ctime(now.tv_sec); - inode.set_ctime_ns(now.tv_nsec); - - inodeWrapper->SwapInode(&inode); - inodeManager_->ShipToFlush(inodeWrapper); + // NOTE: O_DIRECT/O_SYNC/O_DSYNC have simillar semantic, but not exactly the + // same, see `man 2 open` for more details if (fi->flags & O_DIRECT || fi->flags & O_SYNC || fi->flags & O_DSYNC) { // Todo: do some cache flush later } - return ret; + + timer.stop(); + + if (fsMetric_) { + fsMetric_->userWrite.bps.count << size; + fsMetric_->userWrite.qps.count << 1; + fsMetric_->userWrite.latency << timer.u_elapsed(); + } + + VLOG(9) << "write end, ino: " << ino << ", offset: " << off + << ", length: " << size << ", written: " << *wSize; + + return CURVEFS_ERROR::OK; } -CURVEFS_ERROR FuseVolumeClient::FuseOpRead(fuse_req_t req, fuse_ino_t ino, - size_t size, off_t off, +CURVEFS_ERROR FuseVolumeClient::FuseOpRead(fuse_req_t req, + fuse_ino_t ino, + size_t size, + off_t off, struct fuse_file_info *fi, - char *buffer, size_t *rSize) { + char *buffer, + size_t *rSize) { + VLOG(3) << "read start, ino: " << ino << ", offset: " << off + << ", length: " << size; + // check align if (fi->flags & O_DIRECT) { - if (!(is_aligned(off, DirectIOAlignemnt) && - is_aligned(size, DirectIOAlignemnt))) - return CURVEFS_ERROR::INVALIDPARAM; - } + if (!(is_aligned(off, DirectIOAlignment) && + is_aligned(size, DirectIOAlignment))) { + fsMetric_->userRead.eps.count << 1; - std::shared_ptr inodeWrapper; - CURVEFS_ERROR ret = inodeManager_->GetInode(ino, inodeWrapper); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "inodeManager get inode fail, ret = " << ret - << ", inodeid = " << ino; - return ret; + return CURVEFS_ERROR::INVALIDPARAM; + } } - ::curve::common::UniqueLock lgGuard = inodeWrapper->GetUniqueLock(); - Inode inode = inodeWrapper->GetInodeUnlocked(); + butil::Timer timer; + timer.start(); - size_t len = 0; - if (inode.length() <= off) { - *rSize = 0; - return CURVEFS_ERROR::OK; - } else if (inode.length() < off + size) { - len = inode.length() - off; - } else { - len = size; - } - std::list pExtents; - ret = extManager_->DivideExtents(inode.volumeextentlist(), off, len, - &pExtents); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "DivideExtents fail, ret = " << ret; - return ret; - } - uint64_t readOff = 0; - for (const auto &ext : pExtents) { - if (!ext.UnWritten) { - ret = blockDeviceClient_->Read(buffer + readOff, ext.pOffset, - ext.len); - if (ret != CURVEFS_ERROR::OK) { - LOG(ERROR) << "block device read fail, ret = " << ret; - return ret; - } + ssize_t nr = storage_->Read(ino, off, size, buffer); + if (nr < 0) { + if (fsMetric_) { + fsMetric_->userRead.eps.count << 1; } - readOff += ext.len; + LOG(ERROR) << "read error, ino: " << ino << ", offset: " << off + << ", len: " << size; + return CURVEFS_ERROR::IO_ERROR; } - *rSize = len; - struct timespec now; - clock_gettime(CLOCK_REALTIME, &now); - inode.set_ctime(now.tv_sec); - inode.set_ctime_ns(now.tv_nsec); - inode.set_atime(now.tv_sec); - inode.set_atime_ns(now.tv_nsec); + if (fsMetric_) { + fsMetric_->userRead.bps.count << size; + fsMetric_->userRead.qps.count << 1; + fsMetric_->userRead.latency << timer.u_elapsed(); + } - inodeWrapper->SwapInode(&inode); - inodeManager_->ShipToFlush(inodeWrapper); + *rSize = size; - VLOG(6) << "read end, read size = " << *rSize; - return ret; + VLOG(3) << "read end, ino: " << ino << ", offset: " << off + << ", length: " << size << ", rsize: " << *rSize; + + return CURVEFS_ERROR::OK; } CURVEFS_ERROR FuseVolumeClient::FuseOpCreate(fuse_req_t req, fuse_ino_t parent, const char *name, mode_t mode, struct fuse_file_info *fi, fuse_entry_param *e) { - LOG(INFO) << "FuseOpCreate, parent: " << parent + VLOG(3) << "FuseOpCreate, parent: " << parent << ", name: " << name << ", mode: " << mode; CURVEFS_ERROR ret = - MakeNode(req, parent, name, mode, FsFileType::TYPE_FILE, 0, e); + MakeNode(req, parent, name, mode, FsFileType::TYPE_VOLUME, 0, e); if (ret != CURVEFS_ERROR::OK) { return ret; } @@ -277,18 +234,37 @@ CURVEFS_ERROR FuseVolumeClient::FuseOpCreate(fuse_req_t req, fuse_ino_t parent, CURVEFS_ERROR FuseVolumeClient::FuseOpMkNod(fuse_req_t req, fuse_ino_t parent, const char *name, mode_t mode, dev_t rdev, fuse_entry_param *e) { - LOG(INFO) << "FuseOpMkNod, parent: " << parent - << ", name: " << name - << ", mode: " << mode - << ", rdev: " << rdev; - return MakeNode(req, parent, name, mode, FsFileType::TYPE_FILE, rdev, e); + VLOG(3) << "FuseOpMkNod, parent: " << parent << ", name: " << name + << ", mode: " << mode << ", rdev: " << rdev; + return MakeNode(req, parent, name, mode, FsFileType::TYPE_VOLUME, rdev, e); } CURVEFS_ERROR FuseVolumeClient::FuseOpFsync(fuse_req_t req, fuse_ino_t ino, int datasync, struct fuse_file_info *fi) { - LOG(INFO) << "FuseOpFsync, ino: " << ino << ", datasync: " << datasync; - return CURVEFS_ERROR::NOTSUPPORT; + VLOG(3) << "FuseOpFsync start, ino: " << ino << ", datasync: " << datasync; + + auto ret = storage_->Flush(ino); + if (!ret) { + LOG(ERROR) << "Storage flush ino: " << ino << " failed"; + return CURVEFS_ERROR::IO_ERROR; + } + + if (datasync) { + VLOG(3) << "FuseOpFsync end, ino: " << ino + << ", datasync: " << datasync; + return CURVEFS_ERROR::OK; + } + + std::shared_ptr inodeWrapper; + auto ret2 = inodeManager_->GetInode(ino, inodeWrapper); + if (ret2 != CURVEFS_ERROR::OK) { + LOG(ERROR) << "Get inode fail, ino: " << ino << ", ret: " << ret; + return ret2; + } + + auto lk = inodeWrapper->GetUniqueLock(); + return inodeWrapper->Sync(); } CURVEFS_ERROR FuseVolumeClient::Truncate(Inode *inode, uint64_t length) { @@ -306,5 +282,13 @@ void FuseVolumeClient::FlushData() { // TODO(xuchaojie) : flush volume data } +void FuseVolumeClient::SetSpaceManagerForTesting(SpaceManager *manager) { + spaceManager_.reset(manager); +} + +void FuseVolumeClient::SetVolumeStorageForTesting(VolumeStorage *storage) { + storage_.reset(storage); +} + } // namespace client } // namespace curvefs diff --git a/curvefs/src/client/fuse_volume_client.h b/curvefs/src/client/fuse_volume_client.h index 8da1f77c49..e898ba64e1 100644 --- a/curvefs/src/client/fuse_volume_client.h +++ b/curvefs/src/client/fuse_volume_client.h @@ -27,36 +27,33 @@ #include #include "curvefs/src/client/fuse_client.h" -#include "curvefs/src/client/space_client.h" -#include "curvefs/src/client/extent_manager.h" +#include "curvefs/src/client/volume/volume_storage.h" +#include "curvefs/src/volume/block_device_client.h" +#include "curvefs/src/volume/space_manager.h" namespace curvefs { namespace client { using common::VolumeOption; +using ::curvefs::volume::BlockDeviceClient; +using ::curvefs::volume::BlockDeviceClientImpl; +using ::curvefs::volume::SpaceManager; class FuseVolumeClient : public FuseClient { public: FuseVolumeClient() - : FuseClient(), - spaceClient_(std::make_shared()), - extManager_(std::make_shared()), - blockDeviceClient_(std::make_shared()), - spaceBase_(nullptr) {} + : FuseClient(), + blockDeviceClient_(std::make_shared()) {} - FuseVolumeClient(const std::shared_ptr &mdsClient, + // for UNIT_TEST + FuseVolumeClient( + const std::shared_ptr &mdsClient, const std::shared_ptr &metaClient, const std::shared_ptr &inodeManager, const std::shared_ptr &dentryManager, - const std::shared_ptr &spaceClient, - const std::shared_ptr &extManager, const std::shared_ptr &blockDeviceClient) - : FuseClient(mdsClient, metaClient, - inodeManager, dentryManager), - spaceClient_(spaceClient), - extManager_(extManager), - blockDeviceClient_(blockDeviceClient), - spaceBase_(nullptr) {} + : FuseClient(mdsClient, metaClient, inodeManager, dentryManager), + blockDeviceClient_(blockDeviceClient) {} CURVEFS_ERROR Init(const FuseClientOption &option) override; @@ -65,8 +62,6 @@ class FuseVolumeClient : public FuseClient { CURVEFS_ERROR FuseOpInit( void *userdata, struct fuse_conn_info *conn) override; - void FuseOpDestroy(void *userdata) override; - CURVEFS_ERROR FuseOpWrite(fuse_req_t req, fuse_ino_t ino, const char *buf, size_t size, off_t off, struct fuse_file_info *fi, size_t *wSize) override; @@ -90,22 +85,19 @@ class FuseVolumeClient : public FuseClient { CURVEFS_ERROR FuseOpFlush(fuse_req_t req, fuse_ino_t ino, struct fuse_file_info *fi) override; + void SetSpaceManagerForTesting(SpaceManager *manager); + + void SetVolumeStorageForTesting(VolumeStorage *storage); + private: CURVEFS_ERROR Truncate(Inode *inode, uint64_t length) override; void FlushData() override; private: - // space client - std::shared_ptr spaceClient_; - - // extent manager - std::shared_ptr extManager_; - - // curve client std::shared_ptr blockDeviceClient_; - - std::shared_ptr spaceBase_; + std::unique_ptr spaceManager_; + std::unique_ptr storage_; VolumeOption volOpts_; }; diff --git a/curvefs/src/client/inode_cache_manager.cpp b/curvefs/src/client/inode_cache_manager.cpp index 7389577ac9..bb0aab85db 100644 --- a/curvefs/src/client/inode_cache_manager.cpp +++ b/curvefs/src/client/inode_cache_manager.cpp @@ -61,6 +61,7 @@ CURVEFS_ERROR InodeCacheManagerImpl::GetInode(uint64_t inodeid, } Inode inode; + MetaStatusCode ret2 = metaClient_->GetInode(fsId_, inodeid, &inode); if (ret2 != MetaStatusCode::OK) { LOG_IF(ERROR, ret2 != MetaStatusCode::NOT_FOUND) diff --git a/curvefs/src/client/inode_wrapper.cpp b/curvefs/src/client/inode_wrapper.cpp index a2326ccab8..1e8315a1b2 100644 --- a/curvefs/src/client/inode_wrapper.cpp +++ b/curvefs/src/client/inode_wrapper.cpp @@ -136,6 +136,28 @@ class GetOrModifyS3ChunkInfoAsyncDone : public MetaServerClientDone { std::shared_ptr inodeWrapper_; }; +CURVEFS_ERROR InodeWrapper::SyncFullInode() { + std::lock_guard lk(syncingXattrMtx_); + std::lock_guard lk2(syncingVolumeExtentsMtx_); + + if (!dirty_) { + return CURVEFS_ERROR::OK; + } + + auto tmp = extentCache_.ToInodePb(); + inode_.mutable_volumeextentmap()->swap(tmp); + VLOG(9) << "Update inode: " << inode_.ShortDebugString(); + auto ret = metaClient_->UpdateInode(inode_); + if (ret != MetaStatusCode::OK) { + LOG(ERROR) << "update inode failed, error: " << MetaStatusCode_Name(ret) + << ", inodeid: " << inode_.inodeid(); + return MetaStatusCodeToCurvefsErrCode(ret); + } + + dirty_ = false; + return CURVEFS_ERROR::OK; +} + CURVEFS_ERROR InodeWrapper::SyncAttr() { curve::common::UniqueLock lock = GetSyncingInodeUniqueLock(); if (dirty_) { @@ -173,6 +195,12 @@ CURVEFS_ERROR InodeWrapper::SyncS3ChunkInfo() { void InodeWrapper::FlushAttrAsync() { if (dirty_) { LockSyncingInode(); + + if (inode_.type() == FsFileType::TYPE_VOLUME) { + auto tmp = extentCache_.ToInodePb(); + inode_.mutable_volumeextentmap()->swap(tmp); + } + auto *done = new UpdateInodeAsyncDone(shared_from_this()); metaClient_->UpdateInodeAsync(inode_, done); dirty_ = false; @@ -288,6 +316,23 @@ CURVEFS_ERROR InodeWrapper::UnLinkLocked(uint64_t parent) { } } + // TODO(all): Maybe we need separate unlink from UpdateInode, because + // it's wired that we need to update extents when unlinking + // inode. + // Currently, the reason is when unlinking an inode, + // we delete the inode from InodeCacheManager, so next time + // read the inode again, InodeCacheManager will fetch inode + // from metaserver. If we don't update extents here, read + // will read nothing. + // scenario: pjdtest/tests/unlink/14.t + // 1. open a file with O_RDWR + // 2. write "hello, world" + // 3. unlink this file + // 4. pread from 0 to 13, expected "hello, world" + auto extents = extentCache_.ToInodePb(); + if (!extents.empty()) { + inode_.mutable_volumeextentmap()->swap(extents); + } MetaStatusCode ret = metaClient_->UpdateInode(inode_); VLOG(6) << "UnLinkInode, inodeid = " << inode_.inodeid() << ", nlink = " << inode_.nlink(); @@ -390,5 +435,14 @@ CURVEFS_ERROR InodeWrapper::UpdateParentLocked( return CURVEFS_ERROR::OK; } +void InodeWrapper::BuildExtentCache() { + if (inode_.type() != FsFileType::TYPE_VOLUME) { + return; + } + + VLOG(9) << "Build extent for inode: " << inode_.ShortDebugString(); + extentCache_.Build(inode_.volumeextentmap()); +} + } // namespace client } // namespace curvefs diff --git a/curvefs/src/client/inode_wrapper.h b/curvefs/src/client/inode_wrapper.h index 12a3e416af..dc4c2c8ccc 100644 --- a/curvefs/src/client/inode_wrapper.h +++ b/curvefs/src/client/inode_wrapper.h @@ -35,6 +35,7 @@ #include "curvefs/src/client/error_code.h" #include "curvefs/src/client/rpcclient/metaserver_client.h" #include "src/common/concurrent/concurrent.h" +#include "curvefs/src/client/volume/extent_cache.h" using ::curvefs::metaserver::Inode; using ::curvefs::metaserver::InodeOpenStatusChange; @@ -68,7 +69,9 @@ class InodeWrapper : public std::enable_shared_from_this { status_(InodeStatus::Normal), metaClient_(metaClient), openCount_(0), - dirty_(false) {} + dirty_(false) { + BuildExtentCache(); + } InodeWrapper(Inode &&inode, const std::shared_ptr &metaClient) @@ -76,7 +79,9 @@ class InodeWrapper : public std::enable_shared_from_this { status_(InodeStatus::Normal), metaClient_(metaClient), openCount_(0), - dirty_(false) {} + dirty_(false) { + BuildExtentCache(); + } ~InodeWrapper() {} @@ -106,6 +111,11 @@ class InodeWrapper : public std::enable_shared_from_this { dirty_ = true; } + void SetType(FsFileType type) { + inode_.set_type(type); + dirty_ = true; + } + uint64_t GetLength() const { curve::common::UniqueLock lg(mtx_); return inode_.length(); @@ -249,20 +259,39 @@ class InodeWrapper : public std::enable_shared_from_this { CURVEFS_ERROR DecreaseNLink(); CURVEFS_ERROR Sync() { - CURVEFS_ERROR ret = SyncAttr(); - if (ret != CURVEFS_ERROR::OK) { - return ret; + VLOG(9) << "sync inode: " << inode_.ShortDebugString(); + + switch (inode_.type()) { + case FsFileType::TYPE_S3: { + auto ret = SyncAttr(); + if (ret != CURVEFS_ERROR::OK) { + return ret; + } + + return SyncS3ChunkInfo(); + } + + case FsFileType::TYPE_VOLUME: { + return SyncFullInode(); + } + + default: + return CURVEFS_ERROR::INVALIDPARAM; } - return SyncS3ChunkInfo(); } + CURVEFS_ERROR SyncFullInode(); + CURVEFS_ERROR SyncAttr(); CURVEFS_ERROR SyncS3ChunkInfo(); void FlushAsync() { FlushAttrAsync(); - FlushS3ChunkInfoAsync(); + + if (inode_.type() == FsFileType::TYPE_S3) { + FlushS3ChunkInfoAsync(); + } } void FlushAttrAsync(); @@ -333,23 +362,34 @@ class InodeWrapper : public std::enable_shared_from_this { void SetOpenCount(uint32_t openCount) { openCount_ = openCount; } + ExtentCache* GetMutableExtentCache() { + curve::common::UniqueLock lk(mtx_); + dirty_ = true; + return &extentCache_; + } + private: CURVEFS_ERROR UpdateInodeStatus(InodeOpenStatusChange statusChange); + void BuildExtentCache(); + private: - Inode inode_; - uint32_t openCount_; - InodeStatus status_; + Inode inode_; + uint32_t openCount_; + InodeStatus status_; + + google::protobuf::Map s3ChunkInfoAdd_; - google::protobuf::Map s3ChunkInfoAdd_; + std::shared_ptr metaClient_; + bool dirty_; + mutable ::curve::common::Mutex mtx_; - std::shared_ptr metaClient_; - bool dirty_; - mutable ::curve::common::Mutex mtx_; + mutable ::curve::common::Mutex syncingInodeMtx_; + mutable ::curve::common::Mutex syncingXattrMtx_; + mutable ::curve::common::Mutex syncingS3ChunkInfoMtx_; - mutable ::curve::common::Mutex syncingInodeMtx_; - mutable ::curve::common::Mutex syncingXattrMtx_; - mutable ::curve::common::Mutex syncingS3ChunkInfoMtx_; + mutable ::curve::common::Mutex syncingVolumeExtentsMtx_; + ExtentCache extentCache_; }; } // namespace client diff --git a/curvefs/src/client/main.c b/curvefs/src/client/main.c index 6e70d997cb..0ec9fa3a97 100644 --- a/curvefs/src/client/main.c +++ b/curvefs/src/client/main.c @@ -50,6 +50,7 @@ static const struct fuse_lowlevel_ops curve_ll_oper = { .fsync = FuseOpFsync, .releasedir = FuseOpReleaseDir, .flush = FuseOpFlush, + .bmap = FuseOpBmap, }; int main(int argc, char *argv[]) { diff --git a/curvefs/src/client/rpcclient/base_client.cpp b/curvefs/src/client/rpcclient/base_client.cpp index 209bfd9a88..38812ebb18 100644 --- a/curvefs/src/client/rpcclient/base_client.cpp +++ b/curvefs/src/client/rpcclient/base_client.cpp @@ -22,12 +22,12 @@ #include "curvefs/src/client/rpcclient/base_client.h" -#include "curvefs/src/client/common/extent.h" - namespace curvefs { namespace client { namespace rpcclient { +using ::curvefs::mds::space::SpaceService_Stub; + void MDSBaseClient::MountFs(const std::string& fsName, const std::string& mountPt, MountFsResponse* response, brpc::Controller* cntl, @@ -148,6 +148,60 @@ void MDSBaseClient::AllocS3ChunkId(uint32_t fsId, curvefs::mds::MdsService_Stub stub(channel); stub.AllocateS3Chunk(cntl, &request, response, nullptr); } + +// TODO(all): do we really need pass `fsId` all the time? +// each curve-fuse process only mount one filesystem +void MDSBaseClient::AllocateVolumeBlockGroup( + uint32_t fsId, + uint32_t count, + const std::string& owner, + AllocateBlockGroupResponse* response, + brpc::Controller* cntl, + brpc::Channel* channel) { + AllocateBlockGroupRequest request; + request.set_fsid(fsId); + request.set_count(count); + request.set_owner(owner); + + SpaceService_Stub stub(channel); + stub.AllocateBlockGroup(cntl, &request, response, nullptr); +} + +void MDSBaseClient::AcquireVolumeBlockGroup(uint32_t fsId, + uint64_t blockGroupOffset, + const std::string& owner, + AcquireBlockGroupResponse* response, + brpc::Controller* cntl, + brpc::Channel* channel) { + AcquireBlockGroupRequest request; + request.set_fsid(fsId); + request.set_owner(owner); + request.set_blockgroupoffset(blockGroupOffset); + + SpaceService_Stub stub(channel); + stub.AcquireBlockGroup(cntl, &request, response, nullptr); +} + +void MDSBaseClient::ReleaseVolumeBlockGroup( + uint32_t fsId, + const std::string& owner, + const std::vector& blockGroups, + ReleaseBlockGroupResponse* response, + brpc::Controller* cntl, + brpc::Channel* channel) { + ReleaseBlockGroupRequest request; + request.set_fsid(fsId); + request.set_owner(owner); + + google::protobuf::RepeatedPtrField groups( + blockGroups.begin(), blockGroups.end()); + + request.mutable_blockgroups()->Swap(&groups); + + SpaceService_Stub stub(channel); + stub.ReleaseBlockGroup(cntl, &request, response, nullptr); +} + } // namespace rpcclient } // namespace client } // namespace curvefs diff --git a/curvefs/src/client/rpcclient/base_client.h b/curvefs/src/client/rpcclient/base_client.h index a3ea85af40..941b289320 100644 --- a/curvefs/src/client/rpcclient/base_client.h +++ b/curvefs/src/client/rpcclient/base_client.h @@ -33,7 +33,6 @@ #include "curvefs/proto/metaserver.pb.h" #include "curvefs/proto/space.pb.h" #include "curvefs/proto/topology.pb.h" -#include "curvefs/src/client/common/extent.h" #include "src/client/client_common.h" namespace curvefs { @@ -79,12 +78,6 @@ using curvefs::mds::MountFsResponse; using curvefs::mds::UmountFsRequest; using curvefs::mds::UmountFsResponse; -using curvefs::space::AllocateSpaceRequest; -using curvefs::space::AllocateSpaceResponse; -using curvefs::space::DeallocateSpaceRequest; -using curvefs::space::DeallocateSpaceResponse; -using curvefs::space::Extent; - using ::curve::client::CopysetID; using ::curve::client::LogicPoolID; @@ -103,6 +96,14 @@ using curvefs::mds::topology::ListPartitionRequest; using curvefs::mds::topology::ListPartitionResponse; using curvefs::mds::topology::PartitionTxId; using curvefs::mds::topology::TopoStatusCode; + +using ::curvefs::mds::space::AllocateBlockGroupRequest; +using ::curvefs::mds::space::AllocateBlockGroupResponse; +using ::curvefs::mds::space::AcquireBlockGroupRequest; +using ::curvefs::mds::space::AcquireBlockGroupResponse; +using ::curvefs::mds::space::ReleaseBlockGroupRequest; +using ::curvefs::mds::space::ReleaseBlockGroupResponse; + struct InodeParam { uint64_t fsId; uint64_t length; @@ -173,6 +174,28 @@ class MDSBaseClient { virtual void AllocS3ChunkId(uint32_t fsId, AllocateS3ChunkResponse* response, brpc::Controller* cntl, brpc::Channel* channel); + + virtual void AllocateVolumeBlockGroup(uint32_t fsId, + uint32_t count, + const std::string& owner, + AllocateBlockGroupResponse* response, + brpc::Controller* cntl, + brpc::Channel* channel); + + virtual void AcquireVolumeBlockGroup(uint32_t fsId, + uint64_t blockGroupOffset, + const std::string& owner, + AcquireBlockGroupResponse* response, + brpc::Controller* cntl, + brpc::Channel* channel); + + virtual void ReleaseVolumeBlockGroup( + uint32_t fsId, + const std::string& owner, + const std::vector& blockGroups, + ReleaseBlockGroupResponse* response, + brpc::Controller* cntl, + brpc::Channel* channel); }; } // namespace rpcclient diff --git a/curvefs/src/client/rpcclient/mds_client.cpp b/curvefs/src/client/rpcclient/mds_client.cpp index cf96bfaec1..c6d7f3fb29 100644 --- a/curvefs/src/client/rpcclient/mds_client.cpp +++ b/curvefs/src/client/rpcclient/mds_client.cpp @@ -23,11 +23,18 @@ #include "curvefs/src/client/rpcclient/mds_client.h" #include +#include #include + +#include "curvefs/proto/space.pb.h" + namespace curvefs { namespace client { namespace rpcclient { +using ::curvefs::mds::space::SpaceErrCode; +using ::curvefs::mds::space::SpaceErrCode_Name; + FSStatusCode MdsClientImpl::Init(const ::curve::client::MetaServerOption &mdsOpt, MDSBaseClient *baseclient) { @@ -446,6 +453,107 @@ FSStatusCode MdsClientImpl::ReturnError(int retcode) { return static_cast(retcode); } +static SpaceErrCode ToSpaceErrCode(int err) { + if (err < 0) { + return SpaceErrCode::SpaceErrUnknown; + } + + return static_cast(err); +} + +#define CHECK_RPC_AND_RETRY_IF_ERROR(msg) \ + do { \ + if (cntl->Failed()) { \ + LOG(WARNING) << msg << " failed, error: " << cntl->ErrorText() \ + << ", log id: " << cntl->log_id(); \ + return -cntl->ErrorCode(); \ + } \ + } while (0) + +SpaceErrCode MdsClientImpl::AllocateVolumeBlockGroup( + uint32_t fsId, + uint32_t count, + const std::string &owner, + std::vector *groups) { + auto task = RPCTask { + AllocateBlockGroupResponse response; + mdsbasecli_->AllocateVolumeBlockGroup(fsId, count, owner, &response, + cntl, channel); + + CHECK_RPC_AND_RETRY_IF_ERROR("AllocateVolumeBlockGroup"); + + auto status = response.status(); + if (status != SpaceErrCode::SpaceOk) { + LOG(WARNING) << "Allocate volume block group failed, err: " + << SpaceErrCode_Name(status); + } else if (response.blockgroups_size() == 0) { + LOG(WARNING) << "Allocate volume block group failed, no block " + "group allcoated"; + return SpaceErrCode::SpaceErrNoSpace; + } else { + VLOG(9) << "AllocateVolumeBlockGroup, response: " + << response.ShortDebugString(); + groups->reserve(response.blockgroups_size()); + for (int i = 0; i < response.blockgroups_size(); ++i) { + groups->push_back(std::move(*response.mutable_blockgroups(i))); + } + } + + return status; + }; + + return ToSpaceErrCode(rpcexcutor_.DoRPCTask(task, mdsOpt_.mdsMaxRetryMS)); +} + +SpaceErrCode MdsClientImpl::AcquireVolumeBlockGroup( + uint32_t fsId, + uint64_t blockGroupOffset, + const std::string &owner, + curvefs::mds::space::BlockGroup *groups) { + auto task = RPCTask { + AcquireBlockGroupResponse response; + mdsbasecli_->AcquireVolumeBlockGroup(fsId, blockGroupOffset, owner, + &response, cntl, channel); + + CHECK_RPC_AND_RETRY_IF_ERROR("AcquireVolumeBlockGroup"); + + auto status = response.status(); + if (status != SpaceErrCode::SpaceOk) { + LOG(WARNING) << "Acquire volume block group failed, err: " + << SpaceErrCode_Name(status); + } else { + groups->Swap(response.mutable_blockgroups()); + } + + return status; + }; + + return ToSpaceErrCode(rpcexcutor_.DoRPCTask(task, mdsOpt_.mdsMaxRetryMS)); +} + +SpaceErrCode MdsClientImpl::ReleaseVolumeBlockGroup( + uint32_t fsId, + const std::string &owner, + const std::vector &blockGroups) { + auto task = RPCTask { + ReleaseBlockGroupResponse response; + mdsbasecli_->ReleaseVolumeBlockGroup(fsId, owner, blockGroups, + &response, cntl, channel); + + CHECK_RPC_AND_RETRY_IF_ERROR("ReleaseVolumeBlockGroup"); + + LOG_IF(WARNING, SpaceErrCode::SpaceOk != response.status()) + << "Release volume block group failed, err: " + << SpaceErrCode_Name(response.status()); + + return response.status(); + }; + + return ToSpaceErrCode(rpcexcutor_.DoRPCTask(task, mdsOpt_.mdsMaxRetryMS)); +} + +#undef CHECK_RPC_AND_RETRY_IF_ERROR + } // namespace rpcclient } // namespace client } // namespace curvefs diff --git a/curvefs/src/client/rpcclient/mds_client.h b/curvefs/src/client/rpcclient/mds_client.h index 1760e84b2b..ef91a0a1a6 100644 --- a/curvefs/src/client/rpcclient/mds_client.h +++ b/curvefs/src/client/rpcclient/mds_client.h @@ -35,6 +35,7 @@ #include "src/client/mds_client.h" #include "src/client/metacache_struct.h" #include "curvefs/src/client/metric/client_metric.h" +#include "curvefs/proto/space.pb.h" using ::curve::client::CopysetID; using ::curve::client::CopysetInfo; @@ -49,6 +50,7 @@ using ::curvefs::common::Peer; using ::curvefs::common::Volume; using ::curvefs::mds::FsInfo; using ::curvefs::mds::FSStatusCode; +using ::curvefs::mds::space::SpaceErrCode; using ::curvefs::mds::topology::Copyset; using ::curvefs::mds::topology::TopoStatusCode; @@ -99,6 +101,26 @@ class MdsClient { virtual bool ListPartition(uint32_t fsID, std::vector *partitionInfos) = 0; virtual FSStatusCode AllocS3ChunkId(uint32_t fsId, uint64_t *chunkId) = 0; + + // allocate block group + virtual SpaceErrCode AllocateVolumeBlockGroup( + uint32_t fsId, + uint32_t count, + const std::string &owner, + std::vector *groups) = 0; + + // acquire block group + virtual SpaceErrCode AcquireVolumeBlockGroup( + uint32_t fsId, + uint64_t blockGroupOffset, + const std::string &owner, + curvefs::mds::space::BlockGroup *groups) = 0; + + // release block group + virtual SpaceErrCode ReleaseVolumeBlockGroup( + uint32_t fsId, + const std::string &owner, + const std::vector &blockGroups) = 0; }; class MdsClientImpl : public MdsClient { @@ -142,6 +164,27 @@ class MdsClientImpl : public MdsClient { FSStatusCode AllocS3ChunkId(uint32_t fsId, uint64_t *chunkId) override; + // allocate block group + SpaceErrCode AllocateVolumeBlockGroup( + uint32_t fsId, + uint32_t size, + const std::string &owner, + std::vector *groups) override; + + // acquire block group + SpaceErrCode AcquireVolumeBlockGroup( + uint32_t fsId, + uint64_t blockGroupOffset, + const std::string &owner, + curvefs::mds::space::BlockGroup *groups) override; + + // release block group + SpaceErrCode ReleaseVolumeBlockGroup( + uint32_t fsId, + const std::string &owner, + const std::vector &blockGroups) + override; + private: FSStatusCode ReturnError(int retcode); diff --git a/curvefs/src/client/rpcclient/metacache.h b/curvefs/src/client/rpcclient/metacache.h index 337e02e6f3..508585b9d6 100644 --- a/curvefs/src/client/rpcclient/metacache.h +++ b/curvefs/src/client/rpcclient/metacache.h @@ -61,7 +61,7 @@ struct CopysetGroupID { CopysetGroupID() = default; CopysetGroupID(const LogicPoolID &poolid, const CopysetID ©setid) - : copysetID(copysetid), poolID(poolid) {} + : poolID(poolid), copysetID(copysetid) {} std::string ToString() const { return "{" + std::to_string(poolID) + "," + std::to_string(copysetID) + diff --git a/curvefs/src/client/rpcclient/metaserver_client.cpp b/curvefs/src/client/rpcclient/metaserver_client.cpp index 951ab0a3b7..a70a085645 100644 --- a/curvefs/src/client/rpcclient/metaserver_client.cpp +++ b/curvefs/src/client/rpcclient/metaserver_client.cpp @@ -626,7 +626,6 @@ MetaStatusCode MetaServerClientImpl::BatchGetXAttr(uint32_t fsId, return MetaStatusCode::OK; } - MetaStatusCode MetaServerClientImpl::UpdateInode(const Inode &inode, InodeOpenStatusChange statusChange) { @@ -649,12 +648,12 @@ MetaServerClientImpl::UpdateInode(const Inode &inode, request.set_nlink(inode.nlink()); request.set_inodeopenstatuschange(statusChange); *(request.mutable_parent()) = inode.parent(); - if (inode.has_volumeextentlist()) { - curvefs::metaserver::VolumeExtentList *vlist = - new curvefs::metaserver::VolumeExtentList; - vlist->CopyFrom(inode.volumeextentlist()); - request.set_allocated_volumeextentlist(vlist); + + if (!inode.volumeextentmap().empty()) { + auto *exts = request.mutable_volumeextentmap(); + *exts = inode.volumeextentmap(); } + curvefs::metaserver::MetaServerService_Stub stub(channel); stub.UpdateInode(cntl, &request, &response, nullptr); @@ -765,11 +764,10 @@ void MetaServerClientImpl::UpdateInodeAsync( request.set_nlink(inode.nlink()); request.set_inodeopenstatuschange(statusChange); *(request.mutable_parent()) = inode.parent(); - if (inode.has_volumeextentlist()) { - curvefs::metaserver::VolumeExtentList *vlist = - new curvefs::metaserver::VolumeExtentList; - vlist->CopyFrom(inode.volumeextentlist()); - request.set_allocated_volumeextentlist(vlist); + + if (!inode.volumeextentmap().empty()) { + auto *exts = request.mutable_volumeextentmap(); + *exts = inode.volumeextentmap(); } auto *rpcDone = new UpdateInodeRpcDone(taskExecutorDone, diff --git a/curvefs/src/client/rpcclient/metaserver_client.h b/curvefs/src/client/rpcclient/metaserver_client.h index 8c550caf54..5f533d29ab 100644 --- a/curvefs/src/client/rpcclient/metaserver_client.h +++ b/curvefs/src/client/rpcclient/metaserver_client.h @@ -45,7 +45,6 @@ using ::curvefs::metaserver::InodeOpenStatusChange; using ::curvefs::metaserver::InodeAttr; using ::curvefs::metaserver::XAttr; using ::curvefs::metaserver::MetaStatusCode; -using ::curvefs::space::AllocateType; using ::curvefs::metaserver::S3ChunkInfoList; namespace curvefs { diff --git a/curvefs/src/client/s3/client_s3_adaptor.h b/curvefs/src/client/s3/client_s3_adaptor.h index 17a236e1e0..a74a38f3c9 100644 --- a/curvefs/src/client/s3/client_s3_adaptor.h +++ b/curvefs/src/client/s3/client_s3_adaptor.h @@ -31,7 +31,6 @@ #include "curvefs/proto/common.pb.h" #include "curvefs/proto/mds.pb.h" #include "curvefs/proto/metaserver.pb.h" -#include "curvefs/proto/space.pb.h" #include "curvefs/src/client/common/common.h" #include "curvefs/src/client/common/config.h" #include "curvefs/src/client/error_code.h" @@ -50,8 +49,6 @@ using curvefs::client::common::DiskCacheType; using curvefs::metaserver::Inode; using curvefs::metaserver::S3ChunkInfo; using curvefs::metaserver::S3ChunkInfoList; -using curvefs::space::AllocateS3ChunkRequest; -using curvefs::space::AllocateS3ChunkResponse; using rpcclient::MdsClient; using curvefs::client::metric::S3Metric; diff --git a/curvefs/src/client/space_client.cpp b/curvefs/src/client/space_client.cpp deleted file mode 100644 index 7c04d7dc1f..0000000000 --- a/curvefs/src/client/space_client.cpp +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Project: curve - * Created Date: Thur Jun 17 2021 - * Author: lixiaocui - */ - - -#include "curvefs/src/client/space_client.h" - -namespace curvefs { -namespace client { - -CURVEFS_ERROR -SpaceAllocServerClientImpl::SpaceAllocRPCExcutor::DoRPCTask(RPCFunc task) { - brpc::Channel channel; - int ret = channel.Init(opt_.spaceaddr.c_str(), nullptr); - if (ret != 0) { - LOG(WARNING) << "Init channel failed, addr = " << opt_.spaceaddr; - return static_cast(-EHOSTDOWN); - } - - brpc::Controller cntl; - cntl.set_timeout_ms(opt_.rpcTimeoutMs); - - return task(&channel, &cntl); -} - -CURVEFS_ERROR -SpaceAllocServerClientImpl::Init(const SpaceAllocServerOption &spaceopt, - SpaceBaseClient *baseclient) { - basecli_ = baseclient; - excutor_.SetOption(spaceopt); - return CURVEFS_ERROR::OK; -} - -#define RPCTaskDefine \ - [&](brpc::Channel * channel, brpc::Controller * cntl) -> CURVEFS_ERROR - -CURVEFS_ERROR -SpaceAllocServerClientImpl::AllocExtents( - uint32_t fsId, const std::list &toAllocExtents, - curvefs::space::AllocateType type, std::list *allocatedExtents) { - auto task = RPCTaskDefine { - auto iter = toAllocExtents.begin(); - CURVEFS_ERROR retcode = CURVEFS_ERROR::UNKNOWN; - while (iter != toAllocExtents.end()) { - AllocateSpaceResponse response; - basecli_->AllocExtents(fsId, *iter, type, &response, cntl, channel); - if (cntl->Failed()) { - LOG(WARNING) - << "AllocExtents Failed, errorcode = " << cntl->ErrorCode() - << ", error content:" << cntl->ErrorText() - << ", log id = " << cntl->log_id(); - return static_cast(-cntl->ErrorCode()); - } - - SpaceStatusCode stcode = response.status(); - SpaceStatusCode2CurveFSErr(stcode, &retcode); - LOG_IF(WARNING, retcode != CURVEFS_ERROR::OK) - << "AllocExtents: fsId = " << fsId << ", errcode = " << retcode - << ", errmsg = " << SpaceStatusCode_Name(stcode); - if (retcode != CURVEFS_ERROR::OK) { - return static_cast(stcode); - } - - for (int i = 0; i < response.extents_size(); ++i) { - allocatedExtents->push_back(response.extents(i)); - } - ++iter; - } - - return retcode; - }; - - return excutor_.DoRPCTask(task); -} - -CURVEFS_ERROR -SpaceAllocServerClientImpl::DeAllocExtents(uint32_t fsId, - std::list allocatedExtents) { - auto task = RPCTaskDefine { - DeallocateSpaceResponse response; - basecli_->DeAllocExtents(fsId, allocatedExtents, &response, cntl, - channel); - if (cntl->Failed()) { - LOG(WARNING) << "AllocExtents Failed, errorcode = " - << cntl->ErrorCode() - << ", error content:" << cntl->ErrorText() - << ", log id = " << cntl->log_id(); - return static_cast(-cntl->ErrorCode()); - } - - CURVEFS_ERROR retcode = CURVEFS_ERROR::UNKNOWN; - SpaceStatusCode stcode = response.status(); - SpaceStatusCode2CurveFSErr(stcode, &retcode); - LOG_IF(WARNING, retcode != CURVEFS_ERROR::OK) - << "AllocExtents: fsId = " << fsId << ", errcode = " << retcode - << ", errmsg = " << SpaceStatusCode_Name(stcode); - return retcode; - }; - return excutor_.DoRPCTask(task); -} - - -void SpaceAllocServerClientImpl::SpaceStatusCode2CurveFSErr( - const SpaceStatusCode &statcode, CURVEFS_ERROR *errcode) { - switch (statcode) { - case SpaceStatusCode::SPACE_OK: - *errcode = CURVEFS_ERROR::OK; - break; - case SpaceStatusCode::SPACE_NO_SPACE: - *errcode = CURVEFS_ERROR::NO_SPACE; - break; - default: - *errcode = CURVEFS_ERROR::UNKNOWN; - break; - } -} - - -} // namespace client -} // namespace curvefs diff --git a/curvefs/src/client/space_client.h b/curvefs/src/client/space_client.h deleted file mode 100644 index 724f4cbc3e..0000000000 --- a/curvefs/src/client/space_client.h +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Project: curve - * Created Date: Thur Jun 17 2021 - * Author: lixiaocui - */ - -#ifndef CURVEFS_SRC_CLIENT_SPACE_CLIENT_H_ -#define CURVEFS_SRC_CLIENT_SPACE_CLIENT_H_ - -#include - -#include "curvefs/proto/space.pb.h" -#include "curvefs/src/client/base_client.h" -#include "curvefs/src/client/common/config.h" -#include "curvefs/src/client/error_code.h" -#include "curvefs/src/client/common/extent.h" - -namespace curvefs { -namespace client { - -using curvefs::space::Extent; -using curvefs::space::SpaceStatusCode; - -using common::SpaceAllocServerOption; -using common::ExtentAllocInfo; - -class SpaceClient { - public: - SpaceClient() {} - virtual ~SpaceClient() {} - - virtual CURVEFS_ERROR Init(const SpaceAllocServerOption &spaceopt, - SpaceBaseClient *baseclient) = 0; - - virtual CURVEFS_ERROR - AllocExtents(uint32_t fsId, - const std::list &toAllocExtents, - curvefs::space::AllocateType type, - std::list *allocatedExtents) = 0; - - virtual CURVEFS_ERROR - DeAllocExtents(uint32_t fsId, std::list allocatedExtents) = 0; -}; - -class SpaceAllocServerClientImpl : public SpaceClient { - public: - using RPCFunc = - std::function; - - CURVEFS_ERROR Init(const SpaceAllocServerOption &spaceopt, - SpaceBaseClient *baseclient) override; - - virtual CURVEFS_ERROR AllocExtents( - uint32_t fsId, const std::list &toAllocExtents, - curvefs::space::AllocateType type, std::list *allocatedExtents); - - virtual CURVEFS_ERROR DeAllocExtents(uint32_t fsId, - std::list allocatedExtents); - - void SpaceStatusCode2CurveFSErr(const SpaceStatusCode &statcode, - CURVEFS_ERROR *errcode); - - protected: - class SpaceAllocRPCExcutor { - public: - SpaceAllocRPCExcutor() : opt_() {} - ~SpaceAllocRPCExcutor() {} - void SetOption(const SpaceAllocServerOption &option) { opt_ = option; } - - CURVEFS_ERROR DoRPCTask(RPCFunc task); - - private: - SpaceAllocServerOption opt_; - }; - - private: - SpaceBaseClient *basecli_; - SpaceAllocRPCExcutor excutor_; -}; - -} // namespace client -} // namespace curvefs - -#endif // CURVEFS_SRC_CLIENT_SPACE_CLIENT_H_ diff --git a/curvefs/src/client/volume/default_volume_storage.cpp b/curvefs/src/client/volume/default_volume_storage.cpp new file mode 100644 index 0000000000..4d05793983 --- /dev/null +++ b/curvefs/src/client/volume/default_volume_storage.cpp @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Monday Mar 14 17:40:21 CST 2022 + * Author: wuhanqing + */ + +#include "curvefs/src/client/volume/default_volume_storage.h" + +#include +#include +#include + +#include +#include +#include +#include + +#include "absl/meta/type_traits.h" +#include "curvefs/src/client/inode_cache_manager.h" +#include "curvefs/src/client/inode_wrapper.h" +#include "curvefs/src/client/volume/extent_cache.h" +#include "curvefs/src/client/volume/utils.h" +#include "curvefs/src/common/metric_utils.h" + +namespace curvefs { +namespace client { + +namespace common { +DECLARE_bool(enableCto); +} // namespace common + +using ::curvefs::common::LatencyUpdater; + +namespace { + +template ().offset), + decltype(std::declval().length)>> +std::ostream& operator<<(std::ostream& os, const std::vector& iov) { + if (iov.empty()) { + os << "empty"; + return os; + } + + std::ostringstream oss; + oss << "{"; + for (const auto& io : iov) { + oss << io.offset << "~" << io.length << ","; + } + + oss << "}"; + os << oss.str(); + + return os; +} + +} // namespace + +ssize_t DefaultVolumeStorage::Write(uint64_t ino, + off_t offset, + size_t len, + const char* data) { + std::shared_ptr inodeWrapper; + LatencyUpdater updater(&metric_.writeLatency); + auto ret = inodeCacheManager_->GetInode(ino, inodeWrapper); + if (ret != CURVEFS_ERROR::OK) { + LOG(ERROR) << "Get inode error, ino: " << ino << ", ret: " << ret; + return static_cast(ret); + } + + auto extentCache = inodeWrapper->GetMutableExtentCache(); + + std::vector writes; + if (!PrepareWriteRequest(offset, len, data, extentCache, spaceManager_, + &writes)) { + LOG(ERROR) << "Prepare write requests error, ino: " << ino + << ", offset: " << offset << ", len: " << len; + return static_cast(CURVEFS_ERROR::IO_ERROR); + } + + VLOG(9) << "write ino: " << ino << ", offset: " << offset + << ", len: " << len << ", block write requests: " << writes; + + ssize_t nr = blockDeviceClient_->Writev(writes); + if (nr < 0 /*|| nr != len*/) { + LOG(ERROR) << "Block device write error, ino: " << ino + << ", offset: " << offset << ", length: " << len + << ", nr: " << nr; + return static_cast(CURVEFS_ERROR::IO_ERROR); + } + + extentCache->MarkWritten(offset, len); + + { + auto lk = inodeWrapper->GetUniqueLock(); + auto* inode = inodeWrapper->GetMutableInodeUnlocked(); + UpdateInodeTimestamp(inode, kModifyTime | kChangeTime); + + if (offset + len > inode->length()) { + inode->set_length(offset + len); + } + } + + inodeCacheManager_->ShipToFlush(inodeWrapper); + + return nr; +} + +ssize_t DefaultVolumeStorage::Read(uint64_t ino, + off_t offset, + size_t len, + char* data) { + std::shared_ptr inodeWrapper; + LatencyUpdater updater(&metric_.readLatency); + auto ret = inodeCacheManager_->GetInode(ino, inodeWrapper); + if (ret != CURVEFS_ERROR::OK) { + LOG(ERROR) << "Get inode error, ino: " << ino << ", ret: " << ret; + return static_cast(ret); + } + + auto extentCache = inodeWrapper->GetMutableExtentCache(); + std::vector reads; + std::vector holes; + extentCache->DivideForRead(offset, len, data, &reads, &holes); + + VLOG(9) << "read ino: " << ino << ", offest: " << offset << ", len: " << len + << ", read holes: " << holes; + + for (auto& hole : holes) { + memset(hole.data, 0, hole.length); + } + + if (!reads.empty()) { + VLOG(9) << "read ino: " << ino << ", offset: " << offset + << ", len: " << len << ", block read requests: " << reads; + + ssize_t nr = blockDeviceClient_->Readv(reads); + if (nr < 0 /*|| (nr + total) != len*/) { + LOG(ERROR) << "Block device read error, ino: " << ino + << ", offset: " << offset << ", length: " << len; + return static_cast(CURVEFS_ERROR::IO_ERROR); + } + } + + // TODO(all): check whether inode is opened with 'NO_ATIME' + auto* inode = inodeWrapper->GetMutableInodeUnlocked(); + UpdateInodeTimestamp(inode, kAccessTime | kChangeTime); + inodeCacheManager_->ShipToFlush(inodeWrapper); + + return len; +} + +bool DefaultVolumeStorage::Flush(uint64_t ino) { + if (!common::FLAGS_enableCto) { + return true; + } + + LatencyUpdater updater(&metric_.flushLatency); + std::shared_ptr inodeWrapper; + auto ret = inodeCacheManager_->GetInode(ino, inodeWrapper); + if (ret != CURVEFS_ERROR::OK) { + LOG(ERROR) << "Get inode error, ino: " << ino << ", ret: " << ret; + return false; + } + + auto lk = inodeWrapper->GetUniqueLock(); + ret = inodeWrapper->Sync(); + LOG_IF(ERROR, ret != CURVEFS_ERROR::OK) + << "Flush sync inode error, ino: " << ino << ", ret: " << ret; + return ret == CURVEFS_ERROR::OK; +} + +bool DefaultVolumeStorage::Shutdown() { + return true; +} + +} // namespace client +} // namespace curvefs diff --git a/curvefs/src/client/volume/default_volume_storage.h b/curvefs/src/client/volume/default_volume_storage.h new file mode 100644 index 0000000000..4efe7b070f --- /dev/null +++ b/curvefs/src/client/volume/default_volume_storage.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Wednesday Mar 16 15:44:00 CST 2022 + * Author: wuhanqing + */ + +#ifndef CURVEFS_SRC_CLIENT_VOLUME_DEFAULT_VOLUME_STORAGE_H_ +#define CURVEFS_SRC_CLIENT_VOLUME_DEFAULT_VOLUME_STORAGE_H_ + +#include + +#include + +#include "curvefs/src/client/volume/metric.h" +#include "curvefs/src/client/volume/volume_storage.h" +#include "curvefs/src/volume/block_device_client.h" +#include "curvefs/src/volume/space_manager.h" + +namespace curvefs { +namespace client { + +using ::curvefs::volume::BlockDeviceClient; +using ::curvefs::volume::SpaceManager; + +class InodeCacheManager; + +// `DefaultVolumeStorage` implements from `VolumeStorage` +// for write operation, data is written directly to the backend storage, +// meta-data is cached +class DefaultVolumeStorage final : public VolumeStorage { + public: + DefaultVolumeStorage(SpaceManager* spaceManager, + BlockDeviceClient* blockDeviceClient, + InodeCacheManager* inodeCacheManager) + : spaceManager_(spaceManager), + blockDeviceClient_(blockDeviceClient), + inodeCacheManager_(inodeCacheManager), + metric_("default_volume_storage") {} + + DefaultVolumeStorage(const DefaultVolumeStorage&) = delete; + DefaultVolumeStorage& operator=(const DefaultVolumeStorage&) = delete; + + ~DefaultVolumeStorage() override = default; + + ssize_t Read(uint64_t ino, off_t offset, size_t len, char* data) override; + + ssize_t Write(uint64_t ino, + off_t offset, + size_t len, + const char* data) override; + + bool Flush(uint64_t ino) override; + + bool Shutdown() override; + + private: + SpaceManager* spaceManager_; + BlockDeviceClient* blockDeviceClient_; + InodeCacheManager* inodeCacheManager_; + VolumeStorageMetric metric_; +}; + +} // namespace client +} // namespace curvefs + +#endif // CURVEFS_SRC_CLIENT_VOLUME_DEFAULT_VOLUME_STORAGE_H_ diff --git a/curvefs/src/client/common/extent.h b/curvefs/src/client/volume/extent.h similarity index 70% rename from curvefs/src/client/common/extent.h rename to curvefs/src/client/volume/extent.h index 06925b0c54..6684ee26b7 100644 --- a/curvefs/src/client/common/extent.h +++ b/curvefs/src/client/volume/extent.h @@ -21,14 +21,16 @@ * Author: xuchaojie */ -#ifndef CURVEFS_SRC_CLIENT_COMMON_EXTENT_H_ -#define CURVEFS_SRC_CLIENT_COMMON_EXTENT_H_ +#ifndef CURVEFS_SRC_CLIENT_VOLUME_EXTENT_H_ +#define CURVEFS_SRC_CLIENT_VOLUME_EXTENT_H_ + +#include namespace curvefs { namespace client { -namespace common { struct ExtentAllocInfo { + // logical offset within single inode/file uint64_t lOffset; uint64_t len; uint64_t pOffsetLeft; @@ -39,13 +41,17 @@ struct ExtentAllocInfo { // physical extent struct PExtent { - uint64_t pOffset; - uint64_t len; - bool UnWritten; + uint64_t len = 0; + uint64_t pOffset = 0; + bool UnWritten = true; + + PExtent() = default; + + PExtent(uint64_t len, uint64_t poffset, bool unwritten) + : len(len), pOffset(poffset), UnWritten(unwritten) {} }; -} // namespace common } // namespace client } // namespace curvefs -#endif // CURVEFS_SRC_CLIENT_COMMON_EXTENT_H_ +#endif // CURVEFS_SRC_CLIENT_VOLUME_EXTENT_H_ diff --git a/curvefs/src/client/volume/extent_cache.cpp b/curvefs/src/client/volume/extent_cache.cpp new file mode 100644 index 0000000000..e03cabe8ad --- /dev/null +++ b/curvefs/src/client/volume/extent_cache.cpp @@ -0,0 +1,704 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Monday Mar 14 19:42:09 CST 2022 + * Author: wuhanqing + */ + +#include "curvefs/src/client/volume/extent_cache.h" + +#include +#include +#include + +#include +#include + +#include "absl/types/optional.h" +#include "curvefs/src/client/volume/metric.h" +#include "curvefs/src/common/metric_utils.h" +#include "src/common/fast_align.h" + +namespace curvefs { +namespace client { + +using ::curve::common::align_down; +using ::curve::common::align_up; +using ::curve::common::is_aligned; +using ::curve::common::is_alignment; +using ::curve::common::ReadLockGuard; +using ::curve::common::WriteLockGuard; +using ::curvefs::common::LatencyUpdater; +using ::curvefs::metaserver::VolumeExtent; + +namespace { + +// TODO(wuhanqing): this value should align with backend storage's block size +constexpr uint64_t kMinAllocGranularity = 4096; + +bvar::LatencyRecorder g_write_divide_latency("extent_cache_write_divide"); +bvar::LatencyRecorder g_read_divide_latency_("extent_cache_read_divide"); +bvar::LatencyRecorder g_merge_latency("extent_cache_merge"); +bvar::LatencyRecorder g_mark_written_latency("extent_cache_mark_written"); + +} // namespace + +ExtentCacheOption ExtentCache::option_; + +// TODO(wuhanqing): loffset and pext can span on two ranges +void ExtentCache::Merge(uint64_t loffset, const PExtent& pExt) { + VLOG(9) << "merge extnet, loffset: " << loffset + << ", physical offset: " << pExt.pOffset << ", len: " << pExt.len + << ", writtern: " << !pExt.UnWritten; + LatencyUpdater updater(&g_merge_latency); + WriteLockGuard lk(lock_); + MergeWithinRange(&extents_[align_down(loffset, option_.rangeSize)], loffset, + pExt); +} + +void ExtentCache::MergeWithinRange(std::map* range, + uint64_t loffset, + const PExtent& extent) { + if (range->empty()) { + range->emplace(loffset, extent); + return; + } + + // try merge with leftside + auto it = range->lower_bound(loffset); + auto inserted = range->end(); + + if (it != range->begin()) { + --it; + } + + if (!it->second.UnWritten && it->first + it->second.len == loffset && + it->second.pOffset + it->second.len == extent.pOffset) { + it->second.len += extent.len; + inserted = it; + } else { + auto r = range->emplace(loffset, extent); + inserted = r.first; + } + + // try merge with rightside + const auto endOff = inserted->first + inserted->second.len; + it = range->lower_bound(endOff); + + if (!it->second.UnWritten && it != range->end() && it->first == endOff && + inserted->second.pOffset + inserted->second.len == it->second.pOffset) { + inserted->second.len += it->second.len; + range->erase(it); + } +} + +void ExtentCache::DivideForWrite(uint64_t offset, + uint64_t len, + const char* data, + std::vector* allocated, + std::vector* needAlloc) { + LatencyUpdater updater(&g_write_divide_latency); + ReadLockGuard lk(lock_); + + const auto end = offset + len; + const char* datap = data; + + while (offset < end) { + const auto length = std::min( + end - offset, option_.rangeSize - (offset & ~option_.rangeSize)); + + auto range = extents_.find(align_down(offset, option_.rangeSize)); + if (range != extents_.end()) { + DivideForWriteWithinRange(range->second, offset, length, datap, + allocated, needAlloc); + } else { + AllocPart part; + part.data = datap; + part.allocInfo.lOffset = offset; + + // TODO(wuhanqing): only prealloc for inode that big enough + part.allocInfo.len = align_up( + std::max(length, option_.preallocSize), kMinAllocGranularity); + + const auto rangeEnd = align_up(offset + 1, option_.rangeSize); + if (part.allocInfo.lOffset + part.allocInfo.len > rangeEnd) { + part.allocInfo.len = rangeEnd - part.allocInfo.lOffset; + } + + part.writelength = length; + needAlloc->push_back(part); + } + + datap += length; + offset += length; + } +} + +void ExtentCache::DivideForWriteWithinRange( + const std::map& range, + uint64_t offset, + uint64_t len, + const char* data, + std::vector* allocated, + std::vector* needAlloc) { + uint64_t curOff = offset; + const char* datap = data; + const uint64_t curEnd = offset + len; + + auto lower = range.lower_bound(curOff); + const auto upper = range.upper_bound(curEnd); + + absl::optional leftHintOffset; + absl::optional rightHintOffset; + + auto setAllocHint = [&leftHintOffset, &rightHintOffset](AllocPart* part) { + if (leftHintOffset) { + part->allocInfo.leftHintAvailable = true; + part->allocInfo.pOffsetLeft = leftHintOffset.value(); + } else if (rightHintOffset) { + part->allocInfo.rightHintAvailable = true; + part->allocInfo.pOffsetRight = rightHintOffset.value(); + } + }; + + if (lower != range.begin()) { + --lower; + } + + while (curOff < curEnd && lower != upper) { + const auto extStart = lower->first; + const auto extEnd = lower->first + lower->second.len; + + if (curOff < extStart) { + AllocPart part; + part.data = datap; + if (curEnd <= extStart) { + // write |----| |----| + // extent |----| |----| + part.allocInfo.lOffset = curOff; + part.allocInfo.len = curEnd - curOff; + part.writelength = part.allocInfo.len; + + // only set one side hint + if (curEnd == extStart && !leftHintOffset) { + rightHintOffset = lower->second.pOffset; + } + + ++lower; + } else { + // write |-----| |-------| |--------| + // extent |----| |----| |----| + part.allocInfo.lOffset = curOff; + part.allocInfo.len = extStart - curOff; + part.writelength = part.allocInfo.len; + + // only set one side hint + if (!leftHintOffset) { + rightHintOffset = lower->second.pOffset; + } + } + + curOff += part.allocInfo.len; + datap += part.allocInfo.len; + setAllocHint(&part); + needAlloc->push_back(part); + } else if (curOff == extStart) { + WritePart part; + part.data = datap; + + // write |----| |----| |-------| + // extent |----| |--------| |----| + if (curEnd <= extEnd) { + // write |----| |----| + // extent |----| |--------| + part.offset = lower->second.pOffset; + part.length = (curEnd - curOff); + } else { + // write |-------| + // extent |----| + part.offset = lower->second.pOffset; + part.length = (extEnd - extStart); + leftHintOffset = lower->second.pOffset + lower->second.len; + ++lower; + } + + curOff += part.length; + datap += part.length; + allocated->push_back(part); + } else { // curOff > extStart + WritePart part; + part.data = datap; + // write |----| |----| |----| |-----| |----| + // extents |----| |----| |------| |------| + // |--------| + if (curOff >= extEnd) { + // write |----| |----| + // extents |----| |----| + if (curOff == extEnd) { + leftHintOffset = lower->second.pOffset; + } + + ++lower; + continue; + } else if (curEnd <= extEnd) { + // write |----| |----| + // extents |------| |--------| + part.offset = lower->second.pOffset + (curOff - extStart); + part.length = curEnd - curOff; + } else { // curend > extExt + // write |----| + // extents |----| + part.offset = lower->second.pOffset + (curOff - extStart); + part.length = extEnd - curOff; + leftHintOffset = part.offset; + ++lower; + } + + curOff += part.length; + datap += part.length; + allocated->push_back(part); + } + } + + if (curOff < curEnd) { + AllocPart part; + part.data = datap; + part.allocInfo.lOffset = curOff; + part.allocInfo.len = + align_up(std::max(curEnd - curOff, option_.preallocSize), + kMinAllocGranularity); + + if (upper != range.end()) { + part.allocInfo.len = + std::min(upper->first - curOff, part.allocInfo.len); + assert(is_aligned(part.allocInfo.len, kMinAllocGranularity)); + } + + part.writelength = curEnd - curOff; + setAllocHint(&part); + needAlloc->push_back(part); + } +} + +void ExtentCache::MarkWritten(uint64_t offset, uint64_t len) { + LatencyUpdater updater(&g_mark_written_latency); + WriteLockGuard lk(lock_); + + auto cur = align_down(offset, option_.blocksize); + const auto end = align_up(offset + len, option_.blocksize); + + while (cur < end) { + const auto length = std::min( + end - offset, option_.rangeSize - (offset & ~option_.rangeSize)); + MarkWrittenWithinRange(&extents_[align_down(cur, option_.rangeSize)], + cur, length); + + cur += length; + } +} + +// TODO(wuhanqing): merge continuous extents +void ExtentCache::MarkWrittenWithinRange(std::map* range, + uint64_t offset, + uint64_t len) { + uint64_t curOff = offset; + const uint64_t curEnd = offset + len; + + auto lower = range->lower_bound(curOff); + const auto upper = range->upper_bound(curEnd); + auto prev = range->end(); + + if (lower != range->begin()) { + --lower; + } + + auto nonoverlap = [](uint64_t off1, uint64_t len1, uint64_t off2, + uint64_t len2) { + return off1 + len1 <= off2 || off2 + len2 <= off1; + }; + + // merge written extents + auto mergeable = + [&prev, &range]( + std::map::iterator current) { + return prev != range->end() && + !prev->second.UnWritten && + !current->second.UnWritten && + // logical is continuous + prev->first + prev->second.len == current->first && + // physical is continuous + prev->second.pOffset + prev->second.len == + current->second.pOffset; + }; + + while (curOff < curEnd && lower != upper) { + if (nonoverlap(curOff, curEnd, lower->first, lower->second.len)) { + prev = lower; + ++lower; + continue; + } + + if (!lower->second.UnWritten) { + if (mergeable(lower)) { + prev->second.len += lower->second.len; + lower = range->erase(lower); + } else { + prev = lower; + ++lower; + } + + continue; + } + + const auto extStart = lower->first; + const auto extEnd = lower->first + lower->second.len; + + if (curOff < extStart) { + if (curEnd >= extEnd) { + // write |-------| |--------| + // extent |----| |----| + lower->second.UnWritten = false; + if (mergeable(lower)) { + prev->second.len += lower->second.len; + lower = range->erase(lower); + } else { + prev = lower; + ++lower; + } + + curOff = extEnd; + } else { + // write |----| + // extent |----| + uint64_t overlap = curEnd - extStart; + PExtent sep; + sep.pOffset = lower->second.pOffset + overlap; + sep.len = lower->second.len - overlap; + sep.UnWritten = true; + + lower->second.len = overlap; + lower->second.UnWritten = false; + + range->emplace(extStart + overlap, sep); + return; + } + } else if (curOff == extStart) { + // write |----| |----| |-------| + // extent |----| |--------| |----| + if (extEnd <= curEnd) { + // write |----| |-------| + // extent |----| |----| + lower->second.UnWritten = false; + if (mergeable(lower)) { + prev->second.len += lower->second.len; + lower = range->erase(lower); + } else { + prev = lower; + ++lower; + } + + curOff = extEnd; + } else { + // write |----| + // extent |--------| + uint64_t overlap = curEnd - curOff; + PExtent sep; + sep.pOffset = lower->second.pOffset + overlap; + sep.len = lower->second.len - overlap; + sep.UnWritten = true; + + lower->second.len = overlap; + lower->second.UnWritten = false; + + if (mergeable(lower)) { + prev->second.len += lower->second.len; + range->erase(lower); + } + + range->emplace(curEnd, sep); + return; + } + } else { + // write |----| |----| |----| + // extents |------| |--------| |----| + if (curEnd == extEnd) { + // write |----| + // extents |------| + uint64_t overlap = curEnd - curOff; + PExtent sep; + sep.pOffset = lower->second.pOffset + (curOff - extStart); + sep.len = overlap; + sep.UnWritten = false; + + lower->second.len -= overlap; + + range->emplace(curOff, sep); + + // 这里是不是可以直接跳出了 + return; + } else if (curEnd < extEnd) { + // write |----| + // extents |--------| + uint64_t overlap = curEnd - curOff; + + PExtent middle; + middle.pOffset = lower->second.pOffset + (curOff - extStart); + middle.len = overlap; + middle.UnWritten = false; + + PExtent right; + right.pOffset = middle.pOffset + middle.len; + right.len = extEnd - curEnd; + right.UnWritten = true; + + lower->second.len = curOff - extStart; + + range->emplace(curOff, middle); + range->emplace(curEnd, right); + + return; + } else { + // write |----| + // extents |----| + + uint64_t overlap = extEnd - curOff; + PExtent sep; + sep.pOffset = lower->second.pOffset + (curOff - extStart); + sep.len = overlap; + sep.UnWritten = false; + + lower->second.len -= overlap; + + range->emplace(curOff, sep); + + curOff = extEnd; + prev = lower; + ++lower; + } + } + } +} + +std::unordered_map> +ExtentCache::GetExtentsForTesting() const { + WriteLockGuard lk(lock_); + return extents_; +} + +google::protobuf::Map +ExtentCache::ToInodePb() const { + google::protobuf::Map res; + + auto mergeable = + [](const VolumeExtent* prev, + const std::pair& ext) { + if (!prev) { + return false; + } + + return prev->isused() != ext.second.UnWritten && + prev->fsoffset() + prev->length() == ext.first && + prev->volumeoffset() + prev->length() == ext.second.pOffset; + }; + + ReadLockGuard lk(lock_); + + for (const auto& range : extents_) { + curvefs::metaserver::VolumeExtentList& lis = res[range.first]; + auto* pbExt = lis.mutable_volumeextents(); + pbExt->Reserve(range.second.size()); + + VolumeExtent* prev = nullptr; + + for (const auto& ext : range.second) { + if (mergeable(prev, ext)) { + prev->set_length(prev->length() + ext.second.len); + continue;; + } + + auto* t = pbExt->Add(); + t->set_fsoffset(ext.first); + t->set_volumeoffset(ext.second.pOffset); + t->set_length(ext.second.len); + t->set_isused(!ext.second.UnWritten); + prev = t; + } + } + + return res; +} + +void ExtentCache::DivideForRead(uint64_t offset, + uint64_t len, + char* data, + std::vector* reads, + std::vector* holes) { + LatencyUpdater updater(&g_read_divide_latency_); + ReadLockGuard lk(lock_); + + const auto end = offset + len; + char* datap = data; + + while (offset < end) { + const auto length = std::min( + end - offset, option_.rangeSize - (offset & ~option_.rangeSize)); + + auto range = extents_.find(align_down(offset, option_.rangeSize)); + if (range != extents_.end()) { + DivideForReadWithinRange(range->second, offset, length, datap, + reads, holes); + } else { + holes->emplace_back(offset, length, datap); + } + + datap += length; + offset += length; + } +} + +void ExtentCache::DivideForReadWithinRange( + const std::map& range, + uint64_t offset, + uint64_t len, + char* data, + std::vector* reads, + std::vector* holes) { + uint64_t curOff = offset; + const uint64_t curEnd = offset + len; + char* datap = data; + + auto lower = range.lower_bound(curOff); + const auto upper = range.upper_bound(curEnd); + + if (lower != range.begin()) { + --lower; + } + + while (curOff < curEnd && lower != upper) { + const auto extStart = lower->first; + const auto extEnd = lower->first + lower->second.len; + + if (curOff < extStart) { + if (curEnd <= extStart) { + // read |----| |----| + // extent |----| |----| + holes->emplace_back(curOff, curEnd - curOff, datap); + return; + } else { + // read |----| |-------| |--------| + // extent |----| |----| |----| + holes->emplace_back(curOff, extStart - curOff, datap); + } + + datap += (extStart - curOff); + curOff = extStart; + // ++lower; + } else if (curOff == extStart) { + if (curEnd <= extEnd) { + // read |----| |----| + // extent |----| |--------| + if (lower->second.UnWritten) { + holes->emplace_back(curOff, curEnd - curOff, datap); + return; + } else { + reads->emplace_back(lower->second.pOffset, curEnd - curOff, + datap); + return; + } + } else { + // read |-------| + // extent |----| + if (lower->second.UnWritten) { + holes->emplace_back(curOff, extEnd - curOff, datap); + } else { + reads->emplace_back(lower->second.pOffset, extEnd - curOff, + datap); + } + + datap += (extEnd - curOff); + curOff = extEnd; + ++lower; + } + } else { + if (curOff >= extEnd) { + // read |----| |----| + // extents |----| |----| + // do nothing, try next one + ++lower; + continue; + } else if (curEnd <= extEnd) { + // read |----| |----| + // extents |------| |--------| + if (lower->second.UnWritten) { + holes->emplace_back(curOff, curEnd - curOff, datap); + } else { + reads->emplace_back( + lower->second.pOffset + (curOff - extStart), + curEnd - curOff, datap); + } + + return; + } else { + // read |----| + // extents |----| + if (lower->second.UnWritten) { + holes->emplace_back(curOff, extEnd - curOff, datap); + } else { + reads->emplace_back( + lower->second.pOffset + (curOff - extStart), + extEnd - curOff, datap); + } + + datap += (extEnd - curOff); + curOff = extEnd; + ++lower; + } + } + } + + if (curOff < curEnd) { + holes->emplace_back(curOff, curEnd - curOff, datap); + } +} + +void ExtentCache::Build( + const google::protobuf::Map& + fromInode) { + WriteLockGuard lk(lock_); + extents_.clear(); + + for (const auto& l : fromInode) { + auto& range = extents_[l.first]; + + for (const auto& ext : l.second.volumeextents()) { + range.emplace( + ext.fsoffset(), + PExtent{ext.length(), ext.volumeoffset(), !ext.isused()}); + } + } +} + +void ExtentCache::SetOption(const ExtentCacheOption& option) { + CHECK(is_alignment(option_.preallocSize)) << option_.preallocSize; + CHECK(is_alignment(option_.rangeSize)) << option_.rangeSize; + option_ = option; +} + +} // namespace client +} // namespace curvefs diff --git a/curvefs/src/client/volume/extent_cache.h b/curvefs/src/client/volume/extent_cache.h new file mode 100644 index 0000000000..04501fb851 --- /dev/null +++ b/curvefs/src/client/volume/extent_cache.h @@ -0,0 +1,126 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Monday Mar 14 19:27:02 CST 2022 + * Author: wuhanqing + */ + +#ifndef CURVEFS_SRC_CLIENT_VOLUME_EXTENT_CACHE_H_ +#define CURVEFS_SRC_CLIENT_VOLUME_EXTENT_CACHE_H_ + +#include + +#include +#include +#include +#include + +#include "curvefs/proto/metaserver.pb.h" +#include "curvefs/src/client/volume/extent.h" +#include "curvefs/src/volume/common.h" +#include "src/common/concurrent/rw_lock.h" + +namespace curvefs { +namespace client { + +using ::curvefs::volume::ReadPart; +using ::curvefs::volume::WritePart; + +struct AllocPart { + ExtentAllocInfo allocInfo; + size_t writelength = 0; + const char* data = nullptr; +}; + +struct ExtentCacheOption { + uint64_t preallocSize = 64ULL * 1024; + uint64_t rangeSize = 1ULL * 1024 * 1024 * 1024; + uint32_t blocksize = 4096; +}; + +class ExtentCache { + public: + ExtentCache() = default; + + static void SetOption(const ExtentCacheOption& option); + + // build extent cache from inode + void Build( + const google::protobuf::Map& + fromInode); + + void DivideForWrite(uint64_t offset, + uint64_t len, + const char* data, + std::vector* allocated, + std::vector* needAlloc); + + void DivideForRead(uint64_t offset, + uint64_t len, + char* data, + std::vector* reads, + std::vector* holes); + + void Merge(uint64_t loffset, const PExtent& pExt); + + void MarkWritten(uint64_t offset, uint64_t len); + + google::protobuf::Map + ToInodePb() const; + + std::unordered_map> + GetExtentsForTesting() const; + + private: + void DivideForWriteWithinRange(const std::map& range, + uint64_t offset, + uint64_t len, + const char* data, + std::vector* allocated, + std::vector* needAlloc); + + void DivideForReadWithinRange(const std::map& range, + uint64_t offset, + uint64_t len, + char* data, + std::vector* reads, + std::vector* holes); + + void MergeWithinRange(std::map* range, + uint64_t loffset, + const PExtent& extent); + + void MarkWrittenWithinRange(std::map* range, + uint64_t offset, + uint64_t len); + + private: + mutable curve::common::RWLock lock_; + + // key is offset + std::unordered_map> extents_; + + private: + static ExtentCacheOption option_; +}; + +} // namespace client +} // namespace curvefs + +#endif // CURVEFS_SRC_CLIENT_VOLUME_EXTENT_CACHE_H_ diff --git a/curvefs/src/client/volume/metric.h b/curvefs/src/client/volume/metric.h new file mode 100644 index 0000000000..c8b6db206f --- /dev/null +++ b/curvefs/src/client/volume/metric.h @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Monday Mar 21 16:05:24 CST 2022 + * Author: wuhanqing + */ + +#ifndef CURVEFS_SRC_CLIENT_VOLUME_METRIC_H_ +#define CURVEFS_SRC_CLIENT_VOLUME_METRIC_H_ + +#include + +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" + +namespace curvefs { +namespace client { + +struct VolumeStorageMetric { + bvar::LatencyRecorder readLatency; + bvar::LatencyRecorder writeLatency; + bvar::LatencyRecorder flushLatency; + + explicit VolumeStorageMetric(absl::string_view prefix) + : readLatency(absl::StrCat(prefix, "_read")), + writeLatency(absl::StrCat(prefix, "_write")), + flushLatency(absl::StrCat(prefix, "_flush")) {} +}; + +} // namespace client +} // namespace curvefs + +#endif // CURVEFS_SRC_CLIENT_VOLUME_METRIC_H_ diff --git a/curvefs/src/client/volume/utils.cpp b/curvefs/src/client/volume/utils.cpp new file mode 100644 index 0000000000..fb4b8e5146 --- /dev/null +++ b/curvefs/src/client/volume/utils.cpp @@ -0,0 +1,164 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Friday Mar 18 11:31:14 CST 2022 + * Author: wuhanqing + */ + +#include "curvefs/src/client/volume/utils.h" + +#include + +#include +#include + +#include "curvefs/proto/metaserver.pb.h" +#include "curvefs/src/volume/common.h" + +namespace curvefs { +namespace client { + +using ::curvefs::metaserver::Inode; +using ::curvefs::volume::AllocateHint; + +void UpdateInodeTimestamp(Inode* inode, int flags) { + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + + if (flags & kAccessTime) { + inode->set_atime(now.tv_sec); + inode->set_atime_ns(now.tv_nsec); + } + + if (flags & kChangeTime) { + inode->set_ctime(now.tv_sec); + inode->set_ctime_ns(now.tv_nsec); + } + + if (flags & kModifyTime) { + inode->set_mtime(now.tv_sec); + inode->set_mtime_ns(now.tv_nsec); + } +} + +bool AllocSpace(SpaceManager* space, + const AllocPart& part, + std::map* parts, + std::map* alloc) { + AllocateHint hint; + if (part.allocInfo.leftHintAvailable) { + hint.leftOffset = part.allocInfo.pOffsetLeft; + } + + if (part.allocInfo.rightHintAvailable) { + hint.rightOffset = part.allocInfo.pOffsetRight; + } + + std::vector exts; + auto ret = space->Alloc(part.allocInfo.len, hint, &exts); + if (!ret) { + LOG(ERROR) << "allocate space error: " << ret; + return false; + } + + // may allocate more than one extents for one alloc request + if (exts.size() == 1) { + WritePart newPart; + newPart.offset = exts[0].offset; + + newPart.length = part.writelength; + newPart.data = part.data; + + parts->emplace(part.allocInfo.lOffset, newPart); + alloc->emplace(part.allocInfo.lOffset, exts[0]); + + return true; + } + + const char* datap = part.data; + + uint64_t loffset = part.allocInfo.lOffset; + uint64_t totalsize = 0; + + uint64_t writelength = part.writelength; + + for (const auto& ext : exts) { + if (writelength > 0) { + WritePart newPart; + newPart.offset = ext.offset; + newPart.length = std::min(writelength, ext.len); + newPart.data = datap; + + parts->emplace(loffset, newPart); + datap += newPart.length; + writelength -= newPart.length; + } + + alloc->emplace(loffset, ext); + + loffset += ext.len; + totalsize += ext.len; + } + + CHECK(part.allocInfo.len == totalsize); + + return true; +} + +bool PrepareWriteRequest(off_t off, + size_t size, + const char* data, + ExtentCache* extentCache, + SpaceManager* spaceManager, + std::vector* writes) { + std::vector needalloc; + + extentCache->DivideForWrite(off, size, data, writes, &needalloc); + + if (needalloc.empty()) { + return true; + } + + std::map newalloc; + std::map newextents; + + // alloc enough space for write + for (const auto& alloc : needalloc) { + auto ret = AllocSpace(spaceManager, alloc, &newalloc, &newextents); + if (!ret) { + LOG(ERROR) << "Alloc space error"; + return false; + } + } + + // insert allocated space into extent cache + for (const auto& ext : newextents) { + PExtent pext{ext.second.len, ext.second.offset, true}; + extentCache->Merge(ext.first, pext); + } + + for (const auto& alloc : newalloc) { + writes->emplace_back(alloc.second.offset, alloc.second.length, + alloc.second.data); + } + + return true; +} + +} // namespace client +} // namespace curvefs diff --git a/curvefs/src/client/volume/utils.h b/curvefs/src/client/volume/utils.h new file mode 100644 index 0000000000..3fe28b5af4 --- /dev/null +++ b/curvefs/src/client/volume/utils.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Friday Mar 18 11:29:08 CST 2022 + * Author: wuhanqing + */ + +#ifndef CURVEFS_SRC_CLIENT_VOLUME_UTILS_H_ +#define CURVEFS_SRC_CLIENT_VOLUME_UTILS_H_ + +#include +#include + +#include "curvefs/src/client/volume/extent_cache.h" +#include "curvefs/src/volume/common.h" +#include "curvefs/src/volume/space_manager.h" + +namespace curvefs { + +namespace metaserver { +class Inode; +} // namespace metaserver + +namespace client { + +using ::curvefs::metaserver::Inode; +using ::curvefs::volume::Extent; +using ::curvefs::volume::SpaceManager; +using ::curvefs::volume::WritePart; + +enum { + kAccessTime = 1 << 0, + kChangeTime = 1 << 1, + kModifyTime = 1 << 2, +}; + +void UpdateInodeTimestamp(Inode* inode, int flags); + +/** + * @brief Allocate space for a write request + * @param space space manager that used for allocating space + * @param part allocate info + * @param[out] parts generated write request, key is file offset + * @param[out] alloc allocated extents, key is file offset + * @return return true on success, otherwise return false + */ +bool AllocSpace(SpaceManager* space, + const AllocPart& part, + std::map* parts, + std::map* alloc); + +bool PrepareWriteRequest(off_t off, + size_t size, + const char* data, + ExtentCache* extentCache, + SpaceManager* spaceManager, + std::vector* writes); + +} // namespace client +} // namespace curvefs + +#endif // CURVEFS_SRC_CLIENT_VOLUME_UTILS_H_ diff --git a/curvefs/src/client/volume/volume_storage.h b/curvefs/src/client/volume/volume_storage.h new file mode 100644 index 0000000000..8b8dd567c8 --- /dev/null +++ b/curvefs/src/client/volume/volume_storage.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Monday Mar 14 17:40:12 CST 2022 + * Author: wuhanqing + */ + +#ifndef CURVEFS_SRC_CLIENT_VOLUME_VOLUME_STORAGE_H_ +#define CURVEFS_SRC_CLIENT_VOLUME_VOLUME_STORAGE_H_ + +#include + +#include +#include + +namespace curvefs { +namespace client { + +class VolumeStorage { + public: + virtual ~VolumeStorage() = default; + + virtual ssize_t Read(uint64_t ino, + off_t offset, + size_t len, + char* data) = 0; + + virtual ssize_t Write(uint64_t ino, + off_t offset, + size_t len, + const char* data) = 0; + + virtual bool Flush(uint64_t ino) = 0; + + virtual bool Shutdown() = 0; +}; + +} // namespace client +} // namespace curvefs + +#endif // CURVEFS_SRC_CLIENT_VOLUME_VOLUME_STORAGE_H_ diff --git a/curvefs/src/common/BUILD b/curvefs/src/common/BUILD index 2edf4d3345..340698435f 100644 --- a/curvefs/src/common/BUILD +++ b/curvefs/src/common/BUILD @@ -19,12 +19,28 @@ load("//:copts.bzl", "CURVE_DEFAULT_COPTS") cc_library( name = "curvefs_common", srcs = glob(["*.cpp"]), - hdrs = glob(["*.h"]), + hdrs = glob( + ["*.h"], + exclude = ["metric_utils.h"], + ), copts = CURVE_DEFAULT_COPTS, visibility = ["//visibility:public"], deps = [ - "//src/common:curve_common", "//external:gflags", "//external:glog", + "//src/common:curve_common", ], ) + +cc_library( + name = "metric_utils", + hdrs = [ + "metric_utils.h" + ], + copts = CURVE_DEFAULT_COPTS, + visibility = ["//visibility:public"], + deps = [ + "//external:bvar", + "//external:butil", + ] +) diff --git a/curvefs/test/space/reloader_test.cpp b/curvefs/src/common/metric_utils.h similarity index 54% rename from curvefs/test/space/reloader_test.cpp rename to curvefs/src/common/metric_utils.h index a00a1193e2..79b08d2a24 100644 --- a/curvefs/test/space/reloader_test.cpp +++ b/curvefs/src/common/metric_utils.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 NetEase Inc. + * Copyright (c) 2022 NetEase Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,24 +14,33 @@ * limitations under the License. */ -#include "curvefs/src/space/reloader.h" +/* + * Project: curve + * Date: Tuesday Apr 05 15:42:46 CST 2022 + * Author: wuhanqing + */ + -#include +#include +#include namespace curvefs { -namespace space { +namespace common { -class ReloaderTest : public ::testing::Test { - protected: - void SetUp() override {} +struct LatencyUpdater { + explicit LatencyUpdater(bvar::LatencyRecorder* recorder) + : recorder(recorder) { + timer.start(); + } - void TearDown() override {} + ~LatencyUpdater() { + timer.stop(); + (*recorder) << timer.u_elapsed(); + } + + bvar::LatencyRecorder* recorder; + butil::Timer timer; }; -} // namespace space +} // namespace common } // namespace curvefs - -int main(int argc, char* argv[]) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/curvefs/src/mds/BUILD b/curvefs/src/mds/BUILD index ee6030c48e..e77056b1ec 100644 --- a/curvefs/src/mds/BUILD +++ b/curvefs/src/mds/BUILD @@ -33,7 +33,7 @@ cc_library( "//curvefs/src/mds/idgenerator:fs_mds_idgenerator", "//curvefs/src/mds/metaserverclient:curvefs_mds_metaserverclient", "//curvefs/src/mds/metric:fs_mds_metric", - "//curvefs/src/mds/spaceclient:curvefs_mds_spaceclient", + "//curvefs/src/mds/space:curvefs_mds_space", "//curvefs/src/mds/topology:curvefs_topology", "//external:brpc", "//external:gflags", diff --git a/curvefs/src/mds/codec/BUILD b/curvefs/src/mds/codec/BUILD index 057f3cb008..3bcd6b1388 100644 --- a/curvefs/src/mds/codec/BUILD +++ b/curvefs/src/mds/codec/BUILD @@ -25,5 +25,6 @@ cc_library( deps = [ "//curvefs/src/mds/common:fs_mds_common", "//external:protobuf", + "//src/common:curve_common", ], ) diff --git a/curvefs/src/mds/codec/codec.cpp b/curvefs/src/mds/codec/codec.cpp index dd2f84ba79..0cfc4cfc12 100644 --- a/curvefs/src/mds/codec/codec.cpp +++ b/curvefs/src/mds/codec/codec.cpp @@ -25,12 +25,18 @@ #include #include +#include "src/common/encode.h" + namespace curvefs { namespace mds { namespace codec { using ::curvefs::mds::COMMON_PREFIX_LENGTH; using ::curvefs::mds::FS_NAME_KEY_PREFIX; +using ::curvefs::mds::BLOCKGROUP_KEY_PREFIX; +using ::curvefs::mds::BLOCKGROUP_KEY_END; +using ::curve::common::EncodeBigEndian; +using ::curve::common::EncodeBigEndian_uint32; std::string EncodeFsName(const std::string& fsName) { std::string key; @@ -43,6 +49,21 @@ std::string EncodeFsName(const std::string& fsName) { return key; } +std::string EncodeBlockGroupKey(uint32_t fsId, uint64_t offset) { + static const size_t size = + COMMON_PREFIX_LENGTH + sizeof(fsId) + sizeof(offset); + + std::string key; + key.resize(size); + + memcpy(&key[0], BLOCKGROUP_KEY_PREFIX, COMMON_PREFIX_LENGTH); + + EncodeBigEndian_uint32(&key[COMMON_PREFIX_LENGTH], fsId); + EncodeBigEndian(&key[COMMON_PREFIX_LENGTH + sizeof(fsId)], offset); + + return key; +} + } // namespace codec } // namespace mds } // namespace curvefs diff --git a/curvefs/src/mds/codec/codec.h b/curvefs/src/mds/codec/codec.h index 020ee5fab8..9f10c0628a 100644 --- a/curvefs/src/mds/codec/codec.h +++ b/curvefs/src/mds/codec/codec.h @@ -62,6 +62,8 @@ inline bool DecodeProtobufMessage(const std::string& encode, Message* message) { return message->ParseFromString(encode); } +std::string EncodeBlockGroupKey(uint32_t fsId, uint64_t offset); + } // namespace codec } // namespace mds } // namespace curvefs diff --git a/curvefs/src/mds/common/storage_key.h b/curvefs/src/mds/common/storage_key.h index 2d38477524..198fa850b3 100644 --- a/curvefs/src/mds/common/storage_key.h +++ b/curvefs/src/mds/common/storage_key.h @@ -41,6 +41,8 @@ const char FS_ID_KEY_PREFIX[] = "fs_02"; const char CHUNKID_NAME_KEY_PREFIX[] = "fs_03"; const char CHUNKID_NAME_KEY_END[] = "fs_04"; +const char BLOCKGROUP_KEY_PREFIX[] = "fs_04"; +const char BLOCKGROUP_KEY_END[] = "fs_05"; constexpr uint32_t COMMON_PREFIX_LENGTH = 5; diff --git a/curvefs/src/mds/fs_manager.cpp b/curvefs/src/mds/fs_manager.cpp index 6a6d61c790..a9ddc77f65 100644 --- a/curvefs/src/mds/fs_manager.cpp +++ b/curvefs/src/mds/fs_manager.cpp @@ -31,6 +31,7 @@ #include "curvefs/src/mds/common/types.h" #include "curvefs/src/mds/metric/fs_metric.h" +#include "curvefs/src/mds/space/reloader.h" namespace curvefs { namespace mds { @@ -38,13 +39,17 @@ namespace mds { using ::curvefs::common::FSType; using NameLockGuard = ::curve::common::GenericNameLockGuard; using curvefs::mds::topology::TopoStatusCode; +using ::curvefs::mds::space::Reloader; bool FsManager::Init() { - LOG_IF(FATAL, !spaceClient_->Init()) << "spaceClient Init fail"; LOG_IF(FATAL, !fsStorage_->Init()) << "fsStorage Init fail"; - s3Adapter_->Init(s3AdapterOption_); + s3Adapter_->Init(option_.s3AdapterOption); + auto ret = ReloadMountedFsVolumeSpace(); + if (ret != FSStatusCode::OK) { + LOG(ERROR) << "Reload mounted fs volume space error"; + } - return true; + return ret == FSStatusCode::OK; } void FsManager::Run() { @@ -67,7 +72,6 @@ void FsManager::Uninit() { } fsStorage_->Uninit(); - spaceClient_->Uninit(); LOG(INFO) << "FsManager Uninit ok."; } @@ -123,12 +127,24 @@ void FsManager::ScanFs(const FsInfoWrapper& wrapper) { if (partitionList.empty()) { LOG(INFO) << "fs has no partition, delete fs record, fsName = " << wrapper.GetFsName() << ", fsId = " << wrapper.GetFsId(); + + if (wrapper.GetFsType() == FSType::TYPE_VOLUME) { + auto err = spaceManager_->DeleteVolume(wrapper.GetFsId()); + if (err != space::SpaceOk && err != space::SpaceErrNotFound) { + LOG(ERROR) << "Delete volume space failed, fsId: " + << wrapper.GetFsId() + << ", err: " << space::SpaceErrCode_Name(err); + return; + } + } + FSStatusCode ret = fsStorage_->Delete(wrapper.GetFsName()); if (ret != FSStatusCode::OK) { LOG(ERROR) << "delete fs record fail, fsName = " << wrapper.GetFsName() << ", errCode = " << FSStatusCode_Name(ret); } + return; } @@ -143,7 +159,8 @@ void FsManager::ScanFs(const FsInfoWrapper& wrapper) { } void FsManager::BackEndFunc() { - while (sleeper_.wait_for(std::chrono::seconds(backEndThreadRunInterSec_))) { + while (sleeper_.wait_for( + std::chrono::seconds(option_.backEndThreadRunInterSec))) { std::vector wrapperVec; fsStorage_->GetAll(&wrapperVec); for (const FsInfoWrapper& wrapper : wrapperVec) { @@ -189,11 +206,11 @@ FSStatusCode FsManager::CreateFs(const std::string& fsName, FSType fsType, // check s3info if (detail.has_s3info()) { const auto& s3Info = detail.s3info(); - s3AdapterOption_.ak = s3Info.ak(); - s3AdapterOption_.sk = s3Info.sk(); - s3AdapterOption_.s3Address = s3Info.endpoint(); - s3AdapterOption_.bucketName = s3Info.bucketname(); - s3Adapter_->Reinit(s3AdapterOption_); + option_.s3AdapterOption.ak = s3Info.ak(); + option_.s3AdapterOption.sk = s3Info.sk(); + option_.s3AdapterOption.s3Address = s3Info.endpoint(); + option_.s3AdapterOption.bucketName = s3Info.bucketname(); + s3Adapter_->Reinit(option_.s3AdapterOption); if (!s3Adapter_->BucketExist()) { LOG(ERROR) << "CreateFs " << fsName << " error, s3info is not available!"; @@ -391,12 +408,12 @@ FSStatusCode FsManager::MountFs(const std::string& fsName, if (wrapper.GetFsType() == FSType::TYPE_VOLUME && wrapper.IsMountPointEmpty()) { FsInfo tempFsInfo = wrapper.ProtoFsInfo(); - ret = spaceClient_->InitSpace(tempFsInfo); - if (ret != FSStatusCode::OK) { + auto ret = spaceManager_->AddVolume(tempFsInfo); + if (ret != space::SpaceOk) { LOG(ERROR) << "MountFs fail, init space fail, fsName = " << fsName << ", mountpoint = " << mountpoint - << ", errCode = " << FSStatusCode_Name(ret); - return ret; + << ", errCode = " << FSStatusCode_Name(INIT_SPACE_ERROR); + return INIT_SPACE_ERROR; } } @@ -452,12 +469,13 @@ FSStatusCode FsManager::UmountFs(const std::string& fsName, // 3. if no mount point exist, uninit space if (wrapper.GetFsType() == FSType::TYPE_VOLUME && wrapper.IsMountPointEmpty()) { - ret = spaceClient_->UnInitSpace(wrapper.GetFsId()); - if (ret != FSStatusCode::OK) { + auto ret = spaceManager_->RemoveVolume(wrapper.GetFsId()); + if (ret != space::SpaceOk) { LOG(ERROR) << "UmountFs fail, uninit space fail, fsName = " << fsName << ", mountpoint = " << mountpoint - << ", errCode = " << FSStatusCode_Name(ret); - return ret; + << ", errCode = " + << FSStatusCode_Name(UNINIT_SPACE_ERROR); + return UNINIT_SPACE_ERROR; } } @@ -563,5 +581,30 @@ void FsManager::GetAllFsInfo( return; } +FSStatusCode FsManager::ReloadMountedFsVolumeSpace() { + std::vector allfs; + fsStorage_->GetAll(&allfs); + + Reloader reloader(spaceManager_.get(), option_.spaceReloadConcurrency); + for (auto& fs : allfs) { + if (fs.GetFsType() != FSType::TYPE_VOLUME) { + continue; + } + + if (!fs.MountPoints().empty()) { + reloader.Add(fs.ProtoFsInfo()); + } + } + + auto err = reloader.Wait(); + if (err != space::SpaceOk) { + LOG(ERROR) << "Reload volume space failed, err: " + << space::SpaceErrCode_Name(err); + return FSStatusCode::INIT_SPACE_ERROR; + } + + return FSStatusCode::OK; +} + } // namespace mds } // namespace curvefs diff --git a/curvefs/src/mds/fs_manager.h b/curvefs/src/mds/fs_manager.h index fcf4b2e117..310851a15d 100644 --- a/curvefs/src/mds/fs_manager.h +++ b/curvefs/src/mds/fs_manager.h @@ -28,6 +28,7 @@ #include #include #include + #include "curvefs/proto/mds.pb.h" #include "curvefs/proto/topology.pb.h" #include "curvefs/src/common/define.h" @@ -35,7 +36,7 @@ #include "curvefs/src/mds/fs_info_wrapper.h" #include "curvefs/src/mds/fs_storage.h" #include "curvefs/src/mds/metaserverclient/metaserver_client.h" -#include "curvefs/src/mds/spaceclient/space_client.h" +#include "curvefs/src/mds/space/manager.h" #include "curvefs/src/mds/topology/topology.h" #include "curvefs/src/mds/topology/topology_manager.h" #include "curvefs/src/mds/topology/topology_storage_codec.h" @@ -48,8 +49,10 @@ namespace curvefs { namespace mds { -using ::curvefs::mds::topology::TopologyManager; +using ::curvefs::mds::space::SpaceManager; using ::curvefs::mds::topology::Topology; +using ::curvefs::mds::topology::TopologyManager; + using ::curve::common::Thread; using ::curve::common::InterruptibleSleeper; using ::curve::common::Atomic; @@ -57,27 +60,26 @@ using ::curve::common::S3Adapter; struct FsManagerOption { uint32_t backEndThreadRunInterSec; + uint32_t spaceReloadConcurrency; curve::common::S3AdapterOption s3AdapterOption; }; class FsManager { public: - FsManager(std::shared_ptr fsStorage, - std::shared_ptr spaceClient, - std::shared_ptr metaserverClient, - std::shared_ptr topoManager, - std::shared_ptr s3Adapter, + FsManager(const std::shared_ptr& fsStorage, + const std::shared_ptr& spaceManager, + const std::shared_ptr& metaserverClient, + const std::shared_ptr& topoManager, + const std::shared_ptr& s3Adapter, const FsManagerOption& option) : fsStorage_(fsStorage), - spaceClient_(spaceClient), + spaceManager_(spaceManager), metaserverClient_(metaserverClient), topoManager_(topoManager), s3Adapter_(s3Adapter), nameLock_(), - s3AdapterOption_(option.s3AdapterOption) { - isStop_ = true; - backEndThreadRunInterSec_ = option.backEndThreadRunInterSec; - } + isStop_(true), + option_(option) {} bool Init(); void Run(); @@ -191,12 +193,14 @@ class FsManager { // set partition status to DELETING in topology bool SetPartitionToDeleting(const PartitionInfo& partition); + FSStatusCode ReloadMountedFsVolumeSpace(); + private: uint64_t GetRootId(); private: std::shared_ptr fsStorage_; - std::shared_ptr spaceClient_; + std::shared_ptr spaceManager_; std::shared_ptr metaserverClient_; curve::common::GenericNameLock nameLock_; std::shared_ptr topoManager_; @@ -206,8 +210,7 @@ class FsManager { Thread backEndThread_; Atomic isStop_; InterruptibleSleeper sleeper_; - int backEndThreadRunInterSec_; - curve::common::S3AdapterOption s3AdapterOption_; + FsManagerOption option_; }; } // namespace mds } // namespace curvefs diff --git a/curvefs/src/mds/fs_storage.cpp b/curvefs/src/mds/fs_storage.cpp index 854b2c58f0..cd21fcf0bf 100644 --- a/curvefs/src/mds/fs_storage.cpp +++ b/curvefs/src/mds/fs_storage.cpp @@ -358,6 +358,7 @@ bool PersisKVStorage::LoadAllFs() { << "' success, detail: " << fsInfo.ShortDebugString(); idToName_.emplace(fsInfo.fsid(), fsInfo.fsname()); + fs_.emplace(fsInfo.fsname(), std::move(fsInfo)); } diff --git a/curvefs/src/mds/mds.cpp b/curvefs/src/mds/mds.cpp index 76f739278a..5597422b27 100644 --- a/curvefs/src/mds/mds.cpp +++ b/curvefs/src/mds/mds.cpp @@ -41,6 +41,8 @@ namespace mds { using ::curve::election::LeaderElection; using ::curve::idgenerator::EtcdIdGenerator; using ::curve::kvstorage::EtcdClientImp; +using ::curvefs::mds::space::SpaceManagerImpl; +using ::curvefs::mds::space::SpaceServiceImpl; MDS::MDS() : conf_(), @@ -48,7 +50,6 @@ MDS::MDS() running_(false), fsManager_(), fsStorage_(), - spaceClient_(), metaserverClient_(), topology_(), options_(), @@ -65,18 +66,11 @@ void MDS::InitOptions(std::shared_ptr conf) { conf_->GetValueFatalIfFail("mds.listen.addr", &options_.mdsListenAddr); conf_->GetValueFatalIfFail("mds.dummy.port", &options_.dummyPort); - InitSpaceOption(&options_.spaceOptions); InitMetaServerOption(&options_.metaserverOptions); InitTopologyOption(&options_.topologyOptions); InitScheduleOption(&options_.scheduleOption); } -void MDS::InitSpaceOption(SpaceOptions* spaceOption) { - conf_->GetValueFatalIfFail("space.addr", &spaceOption->spaceAddr); - conf_->GetValueFatalIfFail("space.rpcTimeoutMs", - &spaceOption->rpcTimeoutMs); -} - void MDS::InitMetaServerOption(MetaserverOptions* metaserverOption) { conf_->GetValueFatalIfFail("metaserver.addr", &metaserverOption->metaserverAddr); @@ -145,7 +139,7 @@ void MDS::Init() { InitEtcdClient(); fsStorage_ = std::make_shared(etcdClient_); - spaceClient_ = std::make_shared(options_.spaceOptions); + spaceManager_ = std::make_shared(etcdClient_); metaserverClient_ = std::make_shared(options_.metaserverOptions); @@ -159,9 +153,9 @@ void MDS::Init() { InitFsManagerOptions(&fsManagerOption); s3Adapter_ = std::make_shared(); - fsManager_ = - std::make_shared(fsStorage_, spaceClient_, metaserverClient_, - topologyManager_, s3Adapter_, fsManagerOption); + fsManager_ = std::make_shared( + fsStorage_, spaceManager_, metaserverClient_, topologyManager_, + s3Adapter_, fsManagerOption); LOG_IF(FATAL, !fsManager_->Init()) << "fsManager Init fail"; chunkIdAllocator_ = std::make_shared(etcdClient_); @@ -242,6 +236,11 @@ void MDS::Run() { brpc::SERVER_DOESNT_OWN_SERVICE) != 0) << "add topologyService error"; + SpaceServiceImpl spaceService(spaceManager_.get()); + LOG_IF(FATAL, server.AddService(&spaceService, + brpc::SERVER_DOESNT_OWN_SERVICE) != 0) + << "add space service error"; + // start rpc server brpc::ServerOptions option; LOG_IF(FATAL, server.Start(options_.mdsListenAddr.c_str(), &option) != 0) diff --git a/curvefs/src/mds/mds.h b/curvefs/src/mds/mds.h index e3152f3ac9..6bfe6aa038 100644 --- a/curvefs/src/mds/mds.h +++ b/curvefs/src/mds/mds.h @@ -38,10 +38,12 @@ #include "curvefs/src/mds/topology/topology_metric.h" #include "curvefs/src/mds/topology/topology_service.h" #include "curvefs/src/mds/topology/topology_storge_etcd.h" +#include "curvefs/src/mds/space/manager.h" #include "src/common/configuration.h" #include "src/kvstorageclient/etcd_client.h" #include "src/leader_election/leader_election.h" #include "src/common/s3_adapter.h" +#include "curvefs/src/mds/space/service.h" using ::curve::common::Configuration; using ::curvefs::mds::topology::TopologyOption; @@ -70,12 +72,14 @@ using ::curve::election::LeaderElection; using ::curve::election::LeaderElectionOptions; using curve::kvstorage::EtcdClientImp; using ::curve::kvstorage::KVStorageClient; + // TODO(split InitEtcdConf): split this InitEtcdConf to a single module +using ::curvefs::mds::space::SpaceManager; + struct MDSOptions { int dummyPort; std::string mdsListenAddr; - SpaceOptions spaceOptions; MetaserverOptions metaserverOptions; // TODO(add EtcdConf): add etcd configure @@ -115,7 +119,6 @@ class MDS { void InitScheduleOption(ScheduleOption* scheduleOption); private: - void InitSpaceOption(SpaceOptions* spaceOption); void InitMetaServerOption(MetaserverOptions* metaserverOption); void InitTopologyOption(TopologyOption* topologyOption); @@ -140,7 +143,7 @@ class MDS { bool running_; std::shared_ptr fsManager_; std::shared_ptr fsStorage_; - std::shared_ptr spaceClient_; + std::shared_ptr spaceManager_; std::shared_ptr metaserverClient_; std::shared_ptr chunkIdAllocator_; std::shared_ptr topology_; diff --git a/curvefs/src/mds/spaceclient/BUILD b/curvefs/src/mds/space/BUILD similarity index 78% rename from curvefs/src/mds/spaceclient/BUILD rename to curvefs/src/mds/space/BUILD index f08179c6cc..e4f9153799 100644 --- a/curvefs/src/mds/spaceclient/BUILD +++ b/curvefs/src/mds/space/BUILD @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 NetEase Inc. +# Copyright (c) 2022 NetEase Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -17,7 +17,7 @@ load("//:copts.bzl", "CURVE_DEFAULT_COPTS") cc_library( - name = "curvefs_mds_spaceclient", + name = "curvefs_mds_space", srcs = glob([ "*.cpp", "*.h", @@ -27,6 +27,10 @@ cc_library( deps = [ "//curvefs/proto:mds_cc_proto", "//curvefs/proto:space_cc_proto", + "//curvefs/src/mds/codec:fs_mds_codec", "//external:brpc", + "//src/common:curve_common", + "//src/kvstorageclient:kvstorage_client", + "@com_google_absl//absl/memory", ], ) diff --git a/curvefs/src/mds/space/block_group_storage.cpp b/curvefs/src/mds/space/block_group_storage.cpp new file mode 100644 index 0000000000..aef23244d7 --- /dev/null +++ b/curvefs/src/mds/space/block_group_storage.cpp @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Monday Feb 28 14:52:29 CST 2022 + * Author: wuhanqing + */ + +#include "curvefs/src/mds/space/block_group_storage.h" + +#include + +#include +#include + +#include "curvefs/src/mds/codec/codec.h" + +namespace curvefs { +namespace mds { +namespace space { + +using ::curve::kvstorage::KVStorageClient; +using ::curvefs::mds::codec::DecodeProtobufMessage; +using ::curvefs::mds::codec::EncodeBlockGroupKey; +using ::curvefs::mds::codec::EncodeProtobufMessage; + +namespace { + +SpaceErrCode StoreErrCodeToSpaceErrCode(int code) { + switch (code) { + case EtcdErrCode::EtcdOK: + return SpaceOk; + + case EtcdErrCode::EtcdKeyNotExist: + return SpaceErrNotFound; + + default: + return SpaceErrStorage; + } +} + +} // namespace + +SpaceErrCode BlockGroupStorageImpl::PutBlockGroup( + uint32_t fsId, + uint64_t offset, + const BlockGroup& blockGroup) { + std::string key = EncodeBlockGroupKey(fsId, offset); + std::string value; + + if (!blockGroup.IsInitialized()) { + LOG(WARNING) << "Block group is not initialized, fsId: " << fsId + << ", offset: " << offset; + return SpaceErrEncode; + } + + if (!EncodeProtobufMessage(blockGroup, &value)) { + LOG(WARNING) << "Encode block group failed, fsId: " << fsId + << ", block group offset: " << offset; + return SpaceErrEncode; + } + + int err = store_->Put(key, value); + if (err != EtcdErrCode::EtcdOK) { + LOG(ERROR) << "Put block group failed, fsId: " << fsId + << ", block group offset: " << offset << ", err: " << err; + } + + return StoreErrCodeToSpaceErrCode(err); +} + +SpaceErrCode BlockGroupStorageImpl::RemoveBlockGroup(uint32_t fsId, + uint64_t offset) { + std::string key = EncodeBlockGroupKey(fsId, offset); + + int err = store_->Delete(key); + if (err != EtcdErrCode::EtcdOK) { + LOG(WARNING) << "Remove block group failed, fsId: " << fsId + << ", block group offset: " << offset << ", err: " << err; + } + + return StoreErrCodeToSpaceErrCode(err); +} + +SpaceErrCode BlockGroupStorageImpl::ListBlockGroups( + uint32_t fsId, + std::vector* blockGroups) { + std::string startKey = EncodeBlockGroupKey(fsId, 0); + std::string endKey = EncodeBlockGroupKey(fsId + 1, 0); + + std::vector raw; + + int err = store_->List(startKey, endKey, &raw); + if (err != EtcdErrCode::EtcdOK) { + LOG(WARNING) << "List block groups failed, fsId: " << fsId + << ", err: " << err; + return StoreErrCodeToSpaceErrCode(err); + } + + for (size_t i = 0; i < raw.size(); ++i) { + BlockGroup group; + bool ok = DecodeProtobufMessage(raw[i], &group); + if (!ok) { + blockGroups->clear(); + LOG(WARNING) << "Decode block group failed"; + return SpaceErrDecode; + } + + blockGroups->push_back(std::move(group)); + } + + return SpaceOk; +} + +} // namespace space +} // namespace mds +} // namespace curvefs diff --git a/curvefs/src/mds/space/block_group_storage.h b/curvefs/src/mds/space/block_group_storage.h new file mode 100644 index 0000000000..303f7b59cd --- /dev/null +++ b/curvefs/src/mds/space/block_group_storage.h @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Monday Feb 28 14:30:05 CST 2022 + * Author: wuhanqing + */ + +#ifndef CURVEFS_SRC_MDS_SPACE_BLOCK_GROUP_STORAGE_H_ +#define CURVEFS_SRC_MDS_SPACE_BLOCK_GROUP_STORAGE_H_ + +#include +#include + +#include "curvefs/proto/space.pb.h" +#include "src/kvstorageclient/etcd_client.h" + +namespace curvefs { +namespace mds { +namespace space { + +class BlockGroupStorage { + public: + virtual ~BlockGroupStorage() = default; + + virtual SpaceErrCode PutBlockGroup(uint32_t fsId, + uint64_t offset, + const BlockGroup& blockGroup) = 0; + + virtual SpaceErrCode RemoveBlockGroup(uint32_t fsId, uint64_t offset) = 0; + + virtual SpaceErrCode ListBlockGroups( + uint32_t fsId, + std::vector* blockGroups) = 0; +}; + +class BlockGroupStorageImpl final : public BlockGroupStorage { + public: + explicit BlockGroupStorageImpl( + const std::shared_ptr& store) + : store_(store) {} + + BlockGroupStorageImpl(const BlockGroupStorageImpl&) = delete; + BlockGroupStorageImpl& operator=(const BlockGroupStorageImpl&) = delete; + + /** + * @brief Insert one block group into storage + * @return if success return SpaceOk, otherwise return error code + */ + SpaceErrCode PutBlockGroup(uint32_t fsId, + uint64_t offset, + const BlockGroup& blockGroup) override; + + /** + * @brief Remove a block group from storage + * @return if success return SpaceOk, otherwise return error code + */ + SpaceErrCode RemoveBlockGroup(uint32_t fsId, uint64_t offset) override; + + /** + * @brief List all block groups from storage that belong to a single fs + */ + SpaceErrCode ListBlockGroups(uint32_t fsId, + std::vector* blockGroups) override; + + private: + std::shared_ptr store_; +}; + +} // namespace space +} // namespace mds +} // namespace curvefs + +#endif // CURVEFS_SRC_MDS_SPACE_BLOCK_GROUP_STORAGE_H_ diff --git a/curvefs/src/mds/space/manager.cpp b/curvefs/src/mds/space/manager.cpp new file mode 100644 index 0000000000..b5edfc2d94 --- /dev/null +++ b/curvefs/src/mds/space/manager.cpp @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Friday Feb 25 17:21:10 CST 2022 + * Author: wuhanqing + */ + +#include "curvefs/src/mds/space/manager.h" + +#include + +#include "curvefs/proto/mds.pb.h" +#include "curvefs/src/mds/space/volume_space.h" + +namespace curvefs { +namespace mds { +namespace space { + +using ::curve::common::ReadLockGuard; +using ::curve::common::WriteLockGuard; + +using NameLockGuard = ::curve::common::GenericNameLockGuard; + +AbstractVolumeSpace* SpaceManagerImpl::GetVolumeSpace(uint32_t fsId) const { + ReadLockGuard lk(rwlock_); + auto it = volumes_.find(fsId); + if (it == volumes_.end()) { + return nullptr; + } + + return it->second.get(); +} + +SpaceErrCode SpaceManagerImpl::AddVolume(const FsInfo& fsInfo) { + NameLockGuard lock(namelock_, fsInfo.fsname()); + + { + ReadLockGuard lk(rwlock_); + if (volumes_.count(fsInfo.fsid())) { + return SpaceErrCode::SpaceErrExist; + } + } + + const auto& volumeInfo = fsInfo.detail().volume(); + auto space = + VolumeSpace::Create(fsInfo.fsid(), volumeInfo.volumesize(), + volumeInfo.blocksize(), volumeInfo.blockgroupsize(), + volumeInfo.bitmaplocation(), storage_.get()); + + if (!space) { + LOG(ERROR) << "Create volume space failed, fsId: " << fsInfo.fsid(); + return SpaceErrCreate; + } + + { + WriteLockGuard lk(rwlock_); + volumes_.emplace(fsInfo.fsid(), std::move(space)); + } + + return SpaceOk; +} + +SpaceErrCode SpaceManagerImpl::RemoveVolume(uint32_t fsId) { + WriteLockGuard lk(rwlock_); + auto it = volumes_.find(fsId); + if (it == volumes_.end()) { + return SpaceErrNotFound; + } + + // TODO(wuhanqing): move this out of lock + volumes_.erase(it); + return SpaceOk; +} + +SpaceErrCode SpaceManagerImpl::DeleteVolume(uint32_t fsId) { + // remove all persist block groups + WriteLockGuard lk(rwlock_); + auto it = volumes_.find(fsId); + if (it == volumes_.end()) { + return SpaceErrNotFound; + } + + auto err = it->second->RemoveAllBlockGroups(); + if (err != SpaceOk) { + return err; + } + + volumes_.erase(it); + return SpaceOk; +} + +} // namespace space +} // namespace mds +} // namespace curvefs diff --git a/curvefs/src/mds/space/manager.h b/curvefs/src/mds/space/manager.h new file mode 100644 index 0000000000..8dd3a9cd12 --- /dev/null +++ b/curvefs/src/mds/space/manager.h @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Friday Feb 25 17:21:10 CST 2022 + * Author: wuhanqing + */ + +#ifndef CURVEFS_SRC_MDS_SPACE_MANAGER_H_ +#define CURVEFS_SRC_MDS_SPACE_MANAGER_H_ + +#include +#include +#include + +#include "curvefs/proto/space.pb.h" +#include "curvefs/proto/mds.pb.h" +#include "curvefs/src/mds/common/types.h" +#include "curvefs/src/mds/space/block_group_storage.h" +#include "curvefs/src/mds/space/volume_space.h" +#include "src/common/concurrent/name_lock.h" + +namespace curvefs { +namespace mds { +namespace space { + +class SpaceManager { + public: + virtual ~SpaceManager() = default; + + virtual AbstractVolumeSpace* GetVolumeSpace(uint32_t fsId) const = 0; + virtual SpaceErrCode AddVolume(const FsInfo& fsInfo) = 0; + virtual SpaceErrCode RemoveVolume(uint32_t fsId) = 0; + virtual SpaceErrCode DeleteVolume(uint32_t fsId) = 0; +}; + +class SpaceManagerImpl final : public SpaceManager { + public: + explicit SpaceManagerImpl( + const std::shared_ptr& kvstore) + : storage_(new BlockGroupStorageImpl(kvstore)) {} + + SpaceManagerImpl(const SpaceManagerImpl&) = delete; + SpaceManagerImpl& operator=(const SpaceManagerImpl&) = delete; + + AbstractVolumeSpace* GetVolumeSpace(uint32_t fsId) const override; + + SpaceErrCode AddVolume(const FsInfo& fsInfo) override; + + SpaceErrCode RemoveVolume(uint32_t fsId) override; + + SpaceErrCode DeleteVolume(uint32_t fsId) override; + + private: + mutable RWLock rwlock_; + + // key is fs id + std::unordered_map> volumes_; + + std::unique_ptr storage_; + + curve::common::GenericNameLock namelock_; +}; + +} // namespace space +} // namespace mds +} // namespace curvefs + +#endif // CURVEFS_SRC_MDS_SPACE_MANAGER_H_ diff --git a/curvefs/src/mds/space/reloader.cpp b/curvefs/src/mds/space/reloader.cpp new file mode 100644 index 0000000000..319e758c7c --- /dev/null +++ b/curvefs/src/mds/space/reloader.cpp @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Tuesday Mar 01 12:40:43 CST 2022 + * Author: wuhanqing + */ + +#include "curvefs/src/mds/space/reloader.h" + +#include +#include + +#include "absl/memory/memory.h" +#include "curvefs/proto/mds.pb.h" + +namespace curvefs { +namespace mds { +namespace space { + +using ::curve::common::TaskThreadPool; + +Reloader::Reloader(SpaceManager* manager, int concurrency) + : manager_(manager), + taskPool_(absl::make_unique>()), + error_(SpaceOk) { + taskPool_->Start(concurrency); +} + +void Reloader::Add(const FsInfo& fsInfo) { + taskPool_->Enqueue(&Reloader::ReloadOne, this, fsInfo); +} + +void Reloader::ReloadOne(const FsInfo& fsInfo) { + butil::Timer timer; + timer.start(); + auto err = manager_->AddVolume(fsInfo); + timer.stop(); + + if (err != SpaceOk) { + LOG(WARNING) << "Reload volume space failed, fsId: " << fsInfo.fsid() + << ", err: " << SpaceErrCode_Name(err); + + // only record the first occurred error + SpaceErrCode expected = SpaceOk; + error_.compare_exchange_strong(expected, err, + std::memory_order_relaxed); + } else { + LOG(INFO) << "Reload volume space succeeded, fsId: " << fsInfo.fsid() + << ", elapsed " << timer.m_elapsed(.0) << " ms"; + } +} + +SpaceErrCode Reloader::Wait() { + while (taskPool_->QueueSize() != 0) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + } + + return error_; +} + +} // namespace space +} // namespace mds +} // namespace curvefs diff --git a/curvefs/src/mds/space/reloader.h b/curvefs/src/mds/space/reloader.h new file mode 100644 index 0000000000..dee438234f --- /dev/null +++ b/curvefs/src/mds/space/reloader.h @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Tuesday Mar 01 12:40:35 CST 2022 + * Author: wuhanqing + */ + +#ifndef CURVEFS_SRC_MDS_SPACE_RELOADER_H_ +#define CURVEFS_SRC_MDS_SPACE_RELOADER_H_ + +#include +#include + +#include "curvefs/src/mds/space/manager.h" +#include "src/common/concurrent/task_thread_pool.h" + +namespace curvefs { +namespace mds { + +class FsInfo; + +namespace space { + +class Reloader { + public: + Reloader(SpaceManager* manager, int concurrency); + + void Add(const FsInfo& fsInfo); + + SpaceErrCode Wait(); + + private: + void ReloadOne(const FsInfo& fsInfo); + + private: + SpaceManager* manager_; + std::unique_ptr> taskPool_; + std::atomic error_; +}; + +} // namespace space +} // namespace mds +} // namespace curvefs + +#endif // CURVEFS_SRC_MDS_SPACE_RELOADER_H_ diff --git a/curvefs/src/mds/space/service.cpp b/curvefs/src/mds/space/service.cpp new file mode 100644 index 0000000000..550bfa9199 --- /dev/null +++ b/curvefs/src/mds/space/service.cpp @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Friday Feb 25 17:18:47 CST 2022 + * Author: wuhanqing + */ + +#include "curvefs/src/mds/space/service.h" + +#include + +#include + +#include "curvefs/src/mds/space/volume_space.h" + +namespace curvefs { +namespace mds { +namespace space { + +using brpc::ClosureGuard; + +void SpaceServiceImpl::AllocateBlockGroup( + google::protobuf::RpcController* /*controller*/, + const AllocateBlockGroupRequest* request, + AllocateBlockGroupResponse* response, + google::protobuf::Closure* done) { + ClosureGuard doneGuard(done); + + auto* space = spaceMgr_->GetVolumeSpace(request->fsid()); + if (!space) { + LOG(WARNING) << "Volume space not found, fsid: " << request->fsid(); + response->set_status(SpaceErrNotFound); + return; + } + + std::vector groups; + auto err = + space->AllocateBlockGroups(request->count(), request->owner(), &groups); + if (err != SpaceOk) { + LOG(ERROR) << "Allocate block groups failed, err: " + << SpaceErrCode_Name(err); + } else { + for (auto& group : groups) { + response->add_blockgroups()->Swap(&group); + } + } + + response->set_status(err); +} + +void SpaceServiceImpl::AcquireBlockGroup( + google::protobuf::RpcController* /*controller*/, + const AcquireBlockGroupRequest* request, + AcquireBlockGroupResponse* response, + google::protobuf::Closure* done) { + ClosureGuard doneGuard(done); + + auto* space = spaceMgr_->GetVolumeSpace(request->fsid()); + if (!space) { + LOG(WARNING) << "Volume space not found, fsid: " << request->fsid() + << ", block group offset: " << request->blockgroupoffset(); + response->set_status(SpaceErrNotFound); + return; + } + + auto err = + space->AcquireBlockGroup(request->blockgroupoffset(), request->owner(), + response->mutable_blockgroups()); + if (err != SpaceOk) { + LOG(WARNING) << "Acquire block group failed, fsId: " << request->fsid() + << ", block group offset: " << request->blockgroupoffset() + << ", err: " << SpaceErrCode_Name(err); + response->clear_blockgroups(); + } + + response->set_status(err); +} + +void SpaceServiceImpl::ReleaseBlockGroup( + google::protobuf::RpcController* /*controller*/, + const ReleaseBlockGroupRequest* request, + ReleaseBlockGroupResponse* response, + google::protobuf::Closure* done) { + ClosureGuard doneGuard(done); + + auto* space = spaceMgr_->GetVolumeSpace(request->fsid()); + if (!space) { + LOG(WARNING) << "Volume space not found, fsId: " << request->fsid(); + response->set_status(SpaceErrNotFound); + return; + } + + std::vector groups(request->blockgroups().begin(), + request->blockgroups().end()); + auto err = space->ReleaseBlockGroups(groups); + if (err != SpaceOk) { + LOG(WARNING) << "Release block group failed, fsId: " << request->fsid() + << ", err: " << SpaceErrCode_Name(err); + } + + response->set_status(err); +} + +void SpaceServiceImpl::StatSpace( + google::protobuf::RpcController* /*controller*/, + const StatSpaceRequest* request, + StatSpaceResponse* response, + google::protobuf::Closure* done) { + ClosureGuard doneGuard(done); + + auto* space = spaceMgr_->GetVolumeSpace(request->fsid()); + if (!space) { + LOG(WARNING) << "Volume space not found, fsid: " << request->fsid(); + response->set_status(SpaceErrNotFound); + return; + } + + // TODO(wuhanqing): implement stat space + response->set_status(SpaceOk); +} + +void SpaceServiceImpl::UpdateUsage( + google::protobuf::RpcController* /*controller*/, + const UpdateUsageRequest* request, + UpdateUsageResponse* response, + google::protobuf::Closure* done) { + ClosureGuard doneGuard(done); + + auto* space = spaceMgr_->GetVolumeSpace(request->fsid()); + if (!space) { + LOG(WARNING) << "Volume space not found, fsid: " << request->fsid(); + response->set_status(SpaceErrNotFound); + return; + } + + // TODO(wuhanqing): update usage + response->set_status(SpaceOk); +} + +} // namespace space +} // namespace mds +} // namespace curvefs diff --git a/curvefs/src/mds/space/service.h b/curvefs/src/mds/space/service.h new file mode 100644 index 0000000000..91ebbb1917 --- /dev/null +++ b/curvefs/src/mds/space/service.h @@ -0,0 +1,70 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Friday Feb 25 17:13:33 CST 2022 + * Author: wuhanqing + */ + +#ifndef CURVEFS_SRC_MDS_SPACE_SERVICE_H_ +#define CURVEFS_SRC_MDS_SPACE_SERVICE_H_ + +#include "curvefs/proto/space.pb.h" +#include "curvefs/src/mds/space/manager.h" + +namespace curvefs { +namespace mds { +namespace space { + +class SpaceServiceImpl : public SpaceService { + public: + explicit SpaceServiceImpl(SpaceManager* spaceMgr) : spaceMgr_(spaceMgr) {} + + void AllocateBlockGroup(google::protobuf::RpcController* controller, + const AllocateBlockGroupRequest* request, + AllocateBlockGroupResponse* response, + google::protobuf::Closure* done) override; + + void AcquireBlockGroup(google::protobuf::RpcController* controller, + const AcquireBlockGroupRequest* request, + AcquireBlockGroupResponse* response, + google::protobuf::Closure* done) override; + + void ReleaseBlockGroup(google::protobuf::RpcController* controller, + const ReleaseBlockGroupRequest* request, + ReleaseBlockGroupResponse* response, + google::protobuf::Closure* done) override; + + void StatSpace(google::protobuf::RpcController* controller, + const StatSpaceRequest* request, + StatSpaceResponse* response, + google::protobuf::Closure* done) override; + + void UpdateUsage(google::protobuf::RpcController* controller, + const UpdateUsageRequest* request, + UpdateUsageResponse* response, + google::protobuf::Closure* done) override; + + private: + SpaceManager* spaceMgr_; +}; + +} // namespace space +} // namespace mds +} // namespace curvefs + +#endif // CURVEFS_SRC_MDS_SPACE_SERVICE_H_ diff --git a/curvefs/src/mds/space/volume_space.cpp b/curvefs/src/mds/space/volume_space.cpp new file mode 100644 index 0000000000..dc740af6f5 --- /dev/null +++ b/curvefs/src/mds/space/volume_space.cpp @@ -0,0 +1,395 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Friday Feb 25 17:45:50 CST 2022 + * Author: wuhanqing + */ + +#include "curvefs/src/mds/space/volume_space.h" + +#include + +#include +#include +#include + +#include "absl/memory/memory.h" + +namespace curvefs { +namespace mds { +namespace space { + +using LockGuard = std::lock_guard; + +namespace { + +BlockGroup BuildBlockGroupFromClean(uint64_t offset, + uint64_t size, + BitmapLocation location, + const std::string& owner) { + BlockGroup group; + group.set_offset(offset); + group.set_size(size); + group.set_available(size); + group.set_bitmaplocation(location); + group.set_owner(owner); + + return group; +} + +} // namespace + +std::unique_ptr VolumeSpace::Create(uint32_t fsId, + uint64_t size, + uint32_t blockSize, + uint64_t blockGroupSize, + BitmapLocation location, + BlockGroupStorage* storage) { + auto space = absl::WrapUnique(new VolumeSpace( + fsId, size, blockSize, blockGroupSize, location, storage)); + + // reload from storage + std::vector groups; + auto err = storage->ListBlockGroups(fsId, &groups); + if (err != SpaceOk) { + LOG(WARNING) << "List block groups from storage failed, fsId: " << fsId + << ", err: " << SpaceErrCode_Name(err); + return nullptr; + } + + // only record availabe block groups and clean groups + // for allocated groups, client will send heartbeat to update usage + uint64_t availableSize = 0; + std::set usedGroupOffsets; + for (auto& group : groups) { + usedGroupOffsets.insert(group.offset()); + // availableSize += group.available(); + + auto offset = group.offset(); + assert(offset % blockGroupSize == 0); + assert(group.size() == blockGroupSize); + assert(group.bitmaplocation() == location); + if (group.has_owner()) { + space->allocatedGroups_.emplace(offset, std::move(group)); + } else { + space->availableGroups_.emplace(offset, std::move(group)); + availableSize += group.available(); + } + } + + std::set allOffsets; + for (uint64_t off = 0; off < size; off += blockGroupSize) { + allOffsets.insert(off); + } + + std::unordered_set cleanGroupOffsets; + std::set_difference( + allOffsets.begin(), allOffsets.end(), usedGroupOffsets.begin(), + usedGroupOffsets.end(), + std::inserter(cleanGroupOffsets, cleanGroupOffsets.end())); + + space->cleanGroups_ = std::move(cleanGroupOffsets); + + LOG(INFO) << "Init volume space success, fsId: " << fsId + << ", size: " << size << ", available: " << availableSize + << ", block size: " << blockSize + << ", block group size: " << blockGroupSize + << ", total groups: " << size / blockGroupSize + << ", allocated groups: " << space->allocatedGroups_.size() + << ", available groups: " << space->availableGroups_.size() + << ", clean groups: " << space->cleanGroups_.size(); + + return space; +} + +VolumeSpace::VolumeSpace(uint32_t fsId, + uint64_t size, + uint32_t blockSize, + uint64_t blockGroupSize, + BitmapLocation location, + BlockGroupStorage* storage) + : fsId_(fsId), + blockSize_(blockSize), + blockGroupSize_(blockGroupSize), + bitmapLocation_(location), + volumeSize_(size), + storage_(storage) {} + +SpaceErrCode VolumeSpace::AllocateBlockGroups( + uint32_t count, + const std::string& owner, + std::vector* blockGroups) { + LockGuard lk(mtx_); + auto err = AllocateBlockGroupsInternal(count, owner, blockGroups); + if (err != SpaceOk) { + LOG(WARNING) << "Allocate block groups failed, fsId: " << fsId_ + << ", err: " << SpaceErrCode_Name(err); + return err; + } + + for (auto& group : *blockGroups) { + allocatedGroups_.emplace(group.offset(), group); + } + + err = PersistBlockGroups(*blockGroups); + if (err != SpaceOk) { + LOG(WARNING) << "Mark group allocated failed, fsId: " << fsId_ + << ", err: " << SpaceErrCode_Name(err); + return err; + } + + return SpaceOk; +} + +SpaceErrCode VolumeSpace::AllocateBlockGroupsInternal( + uint32_t count, + const std::string& owner, + std::vector* blockGroups) { + uint32_t allocated = 0; + allocated += AllocateFromCleanGroups(count, owner, blockGroups); + + if (allocated >= count) { + return SpaceOk; + } + + allocated += + AllocateFromAvailableGroups(count - allocated, owner, blockGroups); + if (allocated >= count) { + return SpaceOk; + } + + // TODO(wuhanqing): extent the volume + return SpaceErrNoSpace; +} + +uint32_t VolumeSpace::AllocateFromCleanGroups(uint32_t count, + const std::string& owner, + std::vector* groups) { + uint32_t allocated = 0; + auto it = cleanGroups_.begin(); + while (allocated < count && it != cleanGroups_.end()) { + auto offset = *it; + it = cleanGroups_.erase(it); + + ++allocated; + groups->push_back(BuildBlockGroupFromClean(offset, blockGroupSize_, + bitmapLocation_, owner)); + } + + return allocated; +} + +uint32_t VolumeSpace::AllocateFromAvailableGroups( + uint32_t count, + const std::string& owner, + std::vector* groups) { + uint32_t allocated = 0; + auto it = availableGroups_.begin(); + while (allocated < count && it != availableGroups_.end()) { + assert(!it->second.has_owner()); + ++allocated; + it->second.set_owner(owner); + groups->push_back(std::move(it->second)); + it = availableGroups_.erase(it); + } + + return allocated; +} + +SpaceErrCode VolumeSpace::AcquireBlockGroup(uint64_t blockGroupOffset, + const std::string& owner, + BlockGroup* group) { + LockGuard lk(mtx_); + auto err = AcquireBlockGroupInternal(blockGroupOffset, owner, group); + if (err != SpaceOk) { + LOG(WARNING) << "Acquire block group failed, fsId: " << fsId_ + << ", block group offset: " << blockGroupOffset + << ", err: " << SpaceErrCode_Name(err); + return err; + } + + allocatedGroups_.emplace(blockGroupOffset, *group); + + err = PersistBlockGroup(*group); + if (err != SpaceOk) { + LOG(WARNING) << "Persist block group failed, fsId: " << fsId_ + << ", block group offset: " << blockGroupOffset + << ", err: " << SpaceErrCode_Name(err); + } + + return err; +} + +SpaceErrCode VolumeSpace::AcquireBlockGroupInternal(uint64_t blockGroupOffset, + const std::string& owner, + BlockGroup* group) { + // find in availables + { + auto it = availableGroups_.find(blockGroupOffset); + if (it != availableGroups_.end()) { + assert(!it->second.has_owner()); + *group = std::move(it->second); + group->set_owner(owner); + availableGroups_.erase(it); + return SpaceOk; + } + } + + // find from allocated + { + auto it = allocatedGroups_.find(blockGroupOffset); + if (it != allocatedGroups_.end()) { + assert(it->second.has_owner()); + if (it->second.owner() == owner) { + *group = it->second; + return SpaceOk; + } else { + return SpaceErrConflict; + } + } + } + + // this shouldn't happen, because currently only delete inode needs acquire + // block group and in this case, block group must not be emtpy + { + LOG(WARNING) << "unexpected acquire block group, fsid: " << fsId_ + << ", block group offset: " << blockGroupOffset + << ", owner: " << owner; + + if (cleanGroups_.count(blockGroupOffset)) { + *group = BuildBlockGroupFromClean(blockGroupOffset, blockGroupSize_, + bitmapLocation_, owner); + allocatedGroups_.emplace(blockGroupOffset, *group); + return SpaceOk; + } + } + + return SpaceErrNotFound; +} + +SpaceErrCode VolumeSpace::ReleaseBlockGroups( + const std::vector& blockGroups) { + LockGuard lk(mtx_); + + for (auto& group : blockGroups) { + auto it = allocatedGroups_.find(group.offset()); + if (it != allocatedGroups_.end()) { + if (it->second.owner() != group.owner()) { + LOG(WARNING) + << "Owner is not identical, block group may " + "assign to others, fsId: " + << fsId_ << ", block group offset: " << group.offset(); + return SpaceErrConflict; + } + + // space is total available + if (group.available() == group.size()) { + auto err = ClearBlockGroup(group); + if (err != SpaceOk) { + LOG(WARNING) << "Clear block group failed, fsId: " << fsId_ + << ", block group offset: " << group.offset(); + return err; + } + + cleanGroups_.insert(group.offset()); + } else { + auto copy = group; + copy.clear_owner(); + auto err = PersistBlockGroup(copy); + if (err != SpaceOk) { + LOG(WARNING) + << "Persist block group failed, fsId: " << fsId_ + << ", block group offset: " << group.offset() + << ", err: " << SpaceErrCode_Name(err); + return err; + } + + availableGroups_.emplace(group.offset(), std::move(copy)); + } + + allocatedGroups_.erase(group.offset()); + } + + // and if it's not allocated, this request must be a retry request + } + + return SpaceOk; +} + +SpaceErrCode VolumeSpace::PersistBlockGroup(const BlockGroup& group) { + return storage_->PutBlockGroup(fsId_, group.offset(), group); +} + +SpaceErrCode VolumeSpace::PersistBlockGroups( + const std::vector& blockGroups) { + SpaceErrCode err = SpaceOk; + for (auto& group : blockGroups) { + err = PersistBlockGroup(group); + + // TODO(wuhanqing): handle error, and rollback if necessary + if (err != SpaceOk) { + LOG(ERROR) << "Put block group failed, fsId: " << fsId_ + << ", offset: " << group.offset() + << ", err: " << SpaceErrCode_Name(err); + return err; + } + } + + return err; +} + +SpaceErrCode VolumeSpace::ClearBlockGroup(const BlockGroup& group) { + auto err = storage_->RemoveBlockGroup(fsId_, group.offset()); + if (err != SpaceOk) { + LOG(WARNING) << "Remove block group failed, fsId: " << fsId_ + << ", offset: " << group.offset() + << ", err: " << SpaceErrCode_Name(err); + } + + return err; +} + +SpaceErrCode VolumeSpace::RemoveAllBlockGroups() { + LOG(INFO) << "Going to remove all block groups from backend storage, fsId: " + << fsId_; + + LockGuard lk(mtx_); + for (auto& group : allocatedGroups_) { + auto err = ClearBlockGroup(group.second); + if (err != SpaceOk && err != SpaceErrNotFound) { + return err; + } + } + + for (auto& group : availableGroups_) { + auto err = ClearBlockGroup(group.second); + if (err != SpaceOk && err != SpaceErrNotFound) { + return err; + } + } + + allocatedGroups_.clear(); + availableGroups_.clear(); + cleanGroups_.clear(); + + return SpaceOk; +} + +} // namespace space +} // namespace mds +} // namespace curvefs diff --git a/curvefs/src/mds/space/volume_space.h b/curvefs/src/mds/space/volume_space.h new file mode 100644 index 0000000000..25e1450164 --- /dev/null +++ b/curvefs/src/mds/space/volume_space.h @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Friday Feb 25 17:45:46 CST 2022 + * Author: wuhanqing + */ + +#ifndef CURVEFS_SRC_MDS_SPACE_VOLUME_SPACE_H_ +#define CURVEFS_SRC_MDS_SPACE_VOLUME_SPACE_H_ + +#include + +#include +#include +#include +#include +#include +#include + +#include "curvefs/proto/space.pb.h" +#include "curvefs/src/mds/space/block_group_storage.h" + +namespace curvefs { +namespace mds { +namespace space { + +class AbstractVolumeSpace { + public: + virtual ~AbstractVolumeSpace() = default; + + virtual SpaceErrCode AllocateBlockGroups( + uint32_t count, + const std::string& owner, + std::vector* blockGroups) = 0; + + virtual SpaceErrCode AcquireBlockGroup(uint64_t blockGroupOffset, + const std::string& owner, + BlockGroup* group) = 0; + + virtual SpaceErrCode ReleaseBlockGroups( + const std::vector& blockGroups) = 0; +}; + +using ::curvefs::common::BitmapLocation; + +class VolumeSpace final : public AbstractVolumeSpace { + public: + static std::unique_ptr Create(uint32_t fsId, + uint64_t size, + uint32_t blockSize, + uint64_t blockGroupSize, + BitmapLocation location, + BlockGroupStorage* storage); + + VolumeSpace(const VolumeSpace&) = delete; + VolumeSpace& operator=(const VolumeSpace&) = delete; + + /** + * @brief Allocate block groups + */ + SpaceErrCode AllocateBlockGroups( + uint32_t count, + const std::string& owner, + std::vector* blockGroups) override; + + /** + * @brief Acquire a designated block group by offset + */ + SpaceErrCode AcquireBlockGroup(uint64_t blockGroupOffset, + const std::string& owner, + BlockGroup* group) override; + + /** + * @brief Release block groups + */ + SpaceErrCode ReleaseBlockGroups( + const std::vector& blockGroups) override; + + /** + * @brief Remove all block groups and persistent records that belong to + * current volume + * @return return SpaceOk if success, otherwise return error code + */ + SpaceErrCode RemoveAllBlockGroups(); + + private: + VolumeSpace(uint32_t fsId, + uint64_t size, + uint32_t blockSize, + uint64_t blockGroupSize, + BitmapLocation location, + BlockGroupStorage* storage); + + private: + SpaceErrCode AllocateBlockGroupsInternal( + uint32_t count, + const std::string& owner, + std::vector* blockGroups); + + uint32_t AllocateFromAvailableGroups(uint32_t count, + const std::string& owner, + std::vector* groups); + + uint32_t AllocateFromCleanGroups(uint32_t count, + const std::string& owner, + std::vector* groups); + + SpaceErrCode AcquireBlockGroupInternal(uint64_t blockGroupOffset, + const std::string& owner, + BlockGroup* group); + + private: + // persist block group to backend storage + SpaceErrCode PersistBlockGroup(const BlockGroup& group); + SpaceErrCode PersistBlockGroups(const std::vector& blockGroups); + + // remove corresponding block group from backend storage + SpaceErrCode ClearBlockGroup(const BlockGroup& group); + + private: + const uint32_t fsId_; + const uint32_t blockSize_; + const uint64_t blockGroupSize_; + const BitmapLocation bitmapLocation_; + + uint64_t volumeSize_; + + mutable bthread::Mutex mtx_; + + // block groups are divided into three types + // 1. allocated + // these block groups are now owned by clients(curve-fuse or + // curvefs-metaserver), these block groups are forbidden to reallocate to + // other clients. + // allocated block groups are persisted into storage. + // 2. available + // these block groups had been used by some clinets, but now they don't + // have owners, and their space is partial used. so they can be + // reallocated to other clients. + // available block groups are persisted into storage, but doesn't have to + // do this, we persist these block groups for two reasons, first, we can + // statistics space usage more convenient, second is we can reallocate + // these block groups to its previous owner. + // 3. clean + // these block groups' space is never used, and they can be allocated to + // other clients. + + // key is block group offset + std::unordered_map allocatedGroups_; + + // key is block group offset + std::unordered_map availableGroups_; + + // stores clean block groups' offset + std::unordered_set cleanGroups_; + + BlockGroupStorage* storage_; +}; + +} // namespace space +} // namespace mds +} // namespace curvefs + +#endif // CURVEFS_SRC_MDS_SPACE_VOLUME_SPACE_H_ diff --git a/curvefs/src/mds/spaceclient/space_client.cpp b/curvefs/src/mds/spaceclient/space_client.cpp deleted file mode 100644 index 1be0cb4457..0000000000 --- a/curvefs/src/mds/spaceclient/space_client.cpp +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * @Project: curve - * @Date: 2021-06-02 17:05:43 - * @Author: chenwei - */ - -#include "curvefs/src/mds/spaceclient/space_client.h" - -using curvefs::space::InitSpaceRequest; -using curvefs::space::InitSpaceResponse; -using curvefs::space::UnInitSpaceRequest; -using curvefs::space::UnInitSpaceResponse; -using curvefs::space::SpaceStatusCode; -using curvefs::space::SpaceAllocService_Stub; - -namespace curvefs { -namespace mds { -bool SpaceClient::Init() { - if (channel_.Init(options_.spaceAddr.c_str(), nullptr) != 0) { - LOG(ERROR) << "Init channel to space: " << options_.spaceAddr - << " failed!"; - return false; - } - inited_ = true; - return true; -} - -void SpaceClient::Uninit() { - inited_ = false; -} - -FSStatusCode SpaceClient::InitSpace(const FsInfo& fsInfo) { - if (!inited_) { - LOG(ERROR) << "SpaceClient not Init, init first"; - return FSStatusCode::SPACE_CLIENT_NOT_INITED; - } - - InitSpaceRequest request; - InitSpaceResponse response; - - brpc::Controller cntl; - cntl.set_timeout_ms(options_.rpcTimeoutMs); - - SpaceAllocService_Stub stub(&channel_); - FsInfo* fsInfoPtr = request.mutable_fsinfo(); - fsInfoPtr->CopyFrom(fsInfo); - - stub.InitSpace(&cntl, &request, &response, nullptr); - - if (cntl.Failed()) { - LOG(ERROR) << "InitSpace failed, Rpc error = " << cntl.ErrorText(); - return FSStatusCode::RPC_ERROR; - } - - if (response.status() != SpaceStatusCode::SPACE_OK) { - LOG(ERROR) << "InitSpace failed, ret = " - << FSStatusCode::INIT_SPACE_ERROR; - return FSStatusCode::INIT_SPACE_ERROR; - } - return FSStatusCode::OK; -} - -FSStatusCode SpaceClient::UnInitSpace(uint32_t fsId) { - if (!inited_) { - LOG(ERROR) << "SpaceClient not Init, init first"; - return FSStatusCode::SPACE_CLIENT_NOT_INITED; - } - - UnInitSpaceRequest request; - UnInitSpaceResponse response; - - brpc::Controller cntl; - cntl.set_timeout_ms(options_.rpcTimeoutMs); - - SpaceAllocService_Stub stub(&channel_); - - request.set_fsid(fsId); - - stub.UnInitSpace(&cntl, &request, &response, nullptr); - - if (cntl.Failed()) { - LOG(ERROR) << "UnInitSpace failed, Rpc error = " << cntl.ErrorText(); - return FSStatusCode::RPC_ERROR; - } - - if (response.status() != SpaceStatusCode::SPACE_OK) { - LOG(ERROR) << "UnInitSpace failed, ret = " - << FSStatusCode::UNINIT_SPACE_ERROR; - return FSStatusCode::UNINIT_SPACE_ERROR; - } - return FSStatusCode::OK; -} -} // namespace mds -} // namespace curvefs diff --git a/curvefs/src/mds/spaceclient/space_client.h b/curvefs/src/mds/spaceclient/space_client.h deleted file mode 100644 index 402af101ae..0000000000 --- a/curvefs/src/mds/spaceclient/space_client.h +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * @Project: curve - * @Date: 2021-06-02 16:38:47 - * @Author: chenwei - */ - -#ifndef CURVEFS_SRC_MDS_SPACECLIENT_SPACE_CLIENT_H_ -#define CURVEFS_SRC_MDS_SPACECLIENT_SPACE_CLIENT_H_ - -#include -#include -#include "curvefs/proto/mds.pb.h" -#include "curvefs/proto/space.pb.h" - -using curvefs::common::Volume; - -namespace curvefs { -namespace mds { -struct SpaceOptions { - std::string spaceAddr; - uint32_t rpcTimeoutMs; -}; - -struct SpaceStat { - uint64_t blockSize; - uint64_t totalBlock; - uint64_t availabalBlock; - uint64_t usedBlock; -}; - -class SpaceClient { - public: - explicit SpaceClient(const SpaceOptions& option) { options_ = option; } - - bool Init(); - - void Uninit(); - - FSStatusCode InitSpace(const FsInfo& fsInfo); - - FSStatusCode UnInitSpace(uint32_t fsId); - - private: - SpaceOptions options_; - bool inited_; - brpc::Channel channel_; -}; -} // namespace mds -} // namespace curvefs -#endif // CURVEFS_SRC_MDS_SPACECLIENT_SPACE_CLIENT_H_ diff --git a/curvefs/src/metaserver/inode_manager.cpp b/curvefs/src/metaserver/inode_manager.cpp index e9ecd5fe4e..33235741c6 100644 --- a/curvefs/src/metaserver/inode_manager.cpp +++ b/curvefs/src/metaserver/inode_manager.cpp @@ -247,9 +247,9 @@ MetaStatusCode InodeManager::UpdateInode(const UpdateInodeRequest &request) { old->set_nlink(request.nlink()); } - if (request.has_volumeextentlist()) { - VLOG(1) << "update inode has extent"; - old->mutable_volumeextentlist()->CopyFrom(request.volumeextentlist()); + // FIXME(wuhanqing): turncate + if (!request.volumeextentmap().empty()) { + *old->mutable_volumeextentmap() = request.volumeextentmap(); } if (!request.xattr().empty()) { diff --git a/curvefs/src/metaserver/metaserver.cpp b/curvefs/src/metaserver/metaserver.cpp index 551248a76b..a858e22877 100644 --- a/curvefs/src/metaserver/metaserver.cpp +++ b/curvefs/src/metaserver/metaserver.cpp @@ -312,7 +312,7 @@ int Metaserver::PersistDataToLocalFile(std::shared_ptr fs, } int writtenCount = fs->Write(fd, data.c_str(), 0, data.size()); - if (writtenCount < data.size()) { + if (writtenCount < 0 || static_cast(writtenCount) != data.size()) { LOG(ERROR) << "Failed to write data to file, path = " << localPath << ", writtenCount = " << writtenCount << ", data size = " << data.size(); diff --git a/curvefs/src/metaserver/metastore.cpp b/curvefs/src/metaserver/metastore.cpp index 2f95c844fa..c9adcd6a37 100644 --- a/curvefs/src/metaserver/metastore.cpp +++ b/curvefs/src/metaserver/metastore.cpp @@ -616,7 +616,8 @@ MetaStatusCode MetaStoreImpl::UpdateInode(const UpdateInodeRequest* request, UpdateInodeResponse* response) { uint32_t fsId = request->fsid(); uint64_t inodeId = request->inodeid(); - if (request->has_volumeextentlist() && !request->s3chunkinfomap().empty()) { + if (!request->volumeextentmap().empty() && + !request->s3chunkinfomap().empty()) { LOG(ERROR) << "only one of type space info, choose volume or s3"; response->set_statuscode(MetaStatusCode::PARAM_ERROR); return MetaStatusCode::PARAM_ERROR; diff --git a/curvefs/src/space/BUILD b/curvefs/src/space/BUILD deleted file mode 100644 index 12ba3b33b2..0000000000 --- a/curvefs/src/space/BUILD +++ /dev/null @@ -1,83 +0,0 @@ -# -# Copyright (c) 2021 NetEase Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library") -load("//:copts.bzl", "CURVE_DEFAULT_COPTS") - -config_setting( - name = "absl_btree", - values = { - "define": "use_absl_btree=true", - }, - visibility = ["//visibility:public"], -) - -config_setting( - name = "absl_vector", - values = { - "define": "use_absl_vector=true", - }, - visibility = ["//visibility:public"], -) - -ABSL_COPTS = select({ - ":absl_btree": ["-DUSE_ABSL_BTREE"], - "//conditions:default": [], -}) + select({ - ":absl_vector": ["-DUSE_ABSL_VECTOR"], - "//conditions:default": [], -}) - -ABSL_DEPS = select({ - ":absl_btree": ["@com_google_absl//absl/container:btree"], - "//conditions:default": [], -}) + select({ - ":absl_vector": ["@com_google_absl//absl/container:inlined_vector"], - "//conditions:default": [], -}) - -cc_library( - name = "curve_fs_space", - srcs = glob( - ["*.cpp"], - exclude = ["main.cpp"], - ), - hdrs = glob(["*.h"]), - copts = CURVE_DEFAULT_COPTS + ABSL_COPTS, - visibility = ["//visibility:public"], - deps = [ - "//curvefs/proto:space_cc_proto", - "//curvefs/proto:mds_cc_proto", - "//curvefs/proto:metaserver_cc_proto", - "//curvefs/src/common:curvefs_common", - "//src/common:bitmap", - "//src/common/concurrent:rw_lock", - "//external:brpc", - "//external:bthread", - "//external:butil", - "//external:bvar", - ] + ABSL_DEPS, -) - -cc_binary( - name = "curvefs-space", - srcs = ["main.cpp"], - copts = CURVE_DEFAULT_COPTS + ABSL_COPTS, - deps = [ - ":curve_fs_space", - "//src/common:curve_common", - ] + ABSL_DEPS, -) diff --git a/curvefs/src/space/common.h b/curvefs/src/space/common.h deleted file mode 100644 index 06c957d58f..0000000000 --- a/curvefs/src/space/common.h +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Project: curve - * File Created: Fri Jul 16 21:22:40 CST 2021 - * Author: wuhanqing - */ - -#ifndef CURVEFS_SRC_SPACE_COMMON_H_ -#define CURVEFS_SRC_SPACE_COMMON_H_ - -#include -#include - -#ifdef USE_ABSL_BTREE -#include "absl/container/btree_map.h" -#error "There's exist some cases that insert/delete an element into/from a btree_map may invalidate outstanding iterator" // NOLINT -#else -#include -#endif // USE_ABSL_BTREE - -#ifdef USE_ABSL_VECTOR -#include "absl/container/inlined_vector.h" -#else -#include -#endif // USE_ABSL_VECTOR - -#include "curvefs/proto/space.pb.h" - -namespace curvefs { -namespace space { - -#ifdef USE_ABSL_BTREE -using ExtentMap = absl::btree_map; -#else -using ExtentMap = std::map; -#endif // USE_ABSL_BTREE - -struct PExtent { - uint64_t offset; - uint64_t len; - - PExtent() : PExtent(0, 0) {} - PExtent(uint64_t o, uint64_t l) : offset(o), len(l) {} - - bool operator==(const PExtent& e) const { - return offset == e.offset && len == e.len; - } -}; - -#ifdef USE_ABSL_VECTOR -constexpr size_t kDefaultAbslInlinedVectorSize = 8; -using Extents = absl::InlinedVector; -#else -using Extents = std::vector; -#endif // USE_ABSL_VECTOR - -using ProtoExtents = - google::protobuf::RepeatedPtrField<::curvefs::space::Extent>; - -constexpr uint64_t kKiB = 1024ull; -constexpr uint64_t kMiB = 1024ull * kKiB; -constexpr uint64_t kGiB = 1024ull * kMiB; -constexpr uint64_t kTiB = 1024ull * kGiB; - -struct SpaceAllocateHint { - enum - { INVALID_OFFSET = std::numeric_limits::max() }; - - AllocateType allocType = AllocateType::NONE; - uint64_t leftOffset = INVALID_OFFSET; - uint64_t rightOffset = INVALID_OFFSET; -}; - -struct SpaceStat { - uint64_t total = 0; - uint64_t available = 0; - uint32_t blockSize = 0; - - SpaceStat() = default; - - SpaceStat(uint64_t total, uint64_t available, uint64_t blockSize) - : total(total), available(available), blockSize(blockSize) {} -}; - -} // namespace space -} // namespace curvefs - -#endif // CURVEFS_SRC_SPACE_COMMON_H_ diff --git a/curvefs/src/space/main.cpp b/curvefs/src/space/main.cpp deleted file mode 100644 index 27a63c61e2..0000000000 --- a/curvefs/src/space/main.cpp +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Project: curve - * File Created: Fri Jul 16 21:22:40 CST 2021 - * Author: wuhanqing - */ - -#include -#include -#include - -#include "curvefs/src/space/config.h" -#include "curvefs/src/space/space_manager.h" -#include "curvefs/src/space/space_service.h" -#include "src/common/configuration.h" - -using curve::common::Configuration; -using curvefs::space::SpaceAllocServiceImpl; -using curvefs::space::SpaceManagerImpl; -using curvefs::space::SpaceManagerOption; - -DEFINE_string(conf, "/etc/curve/space.conf", "space config file path"); -DEFINE_string(listenAddr, "127.0.0.1:9500", "space listen address"); - -void LoadConfigFromGflags(Configuration* conf) { - if (FLAGS_log_dir.empty()) { - if (!conf->GetStringValue("common.log_dir", &FLAGS_log_dir)) { - LOG(WARNING) << "no common.log_dir found in " - << conf->GetConfigPath() << ", will log to /tmp"; - } - } - - google::CommandLineFlagInfo info; - if (google::GetCommandLineFlagInfo("listenAddr", &info) && - !info.is_default) { - conf->SetStringValue("common.listen_address", FLAGS_listenAddr); - } -} - -void LoadSpaceManagerOption(Configuration* conf, SpaceManagerOption* opt) { - LOG_IF(FATAL, !conf->GetUInt32Value("common.block_size", &opt->blockSize)) - << "common.block_size not found in " << conf->GetConfigPath(); - LOG_IF(FATAL, !conf->GetStringValue("allocator.type", &opt->allocatorType)) - << "allocator.type not found in " << conf->GetConfigPath(); - - LOG_IF(FATAL, - !conf->GetStringValue("metaserver.address", - &opt->reloaderOption.metaServerOption.addr)) - << "metaserver.address not found in " << conf->GetConfigPath(); - - if (opt->allocatorType == "bitmap") { - LOG_IF(FATAL, - !conf->GetUInt64Value( - "bitmapallocator.size_per_bit", - &opt->allocatorOption.bitmapAllocatorOption.sizePerBit)) - << "bitmapallocator.size_per_bit not found in " - << conf->GetConfigPath(); - LOG_IF(FATAL, - !conf->GetDoubleValue("bitmapallocator.small_alloc_proportion", - &opt->allocatorOption.bitmapAllocatorOption - .smallAllocProportion)) - << "bitmapallocator.small_alloc_proportion not found in " - << conf->GetConfigPath(); - } -} - -int Run(Configuration* conf) { - SpaceManagerOption spaceManagerOpt; - LoadSpaceManagerOption(conf, &spaceManagerOpt); - - SpaceManagerImpl space(spaceManagerOpt); - SpaceAllocServiceImpl spaceAllocService(&space); - - brpc::Server server; - int rv = - server.AddService(&spaceAllocService, brpc::SERVER_DOESNT_OWN_SERVICE); - if (rv != 0) { - LOG(ERROR) << "AddService failed"; - return -1; - } - - std::string listenAddr; - LOG_IF(FATAL, !conf->GetStringValue("common.listen_address", &listenAddr)) - << "common.listen_address not found in " << conf->GetConfigPath(); - - butil::EndPoint listenPoint; - rv = butil::str2endpoint(listenAddr.c_str(), &listenPoint); - if (rv != 0) { - LOG(ERROR) << "parse endpoint from " << FLAGS_listenAddr << " failed"; - return -1; - } - - rv = server.Start(listenPoint, nullptr); - if (rv != 0) { - LOG(ERROR) << "Start server failed"; - return -1; - } - - LOG(INFO) << "Server started at " << butil::endpoint2str(listenPoint); - server.RunUntilAskedToQuit(); - LOG(INFO) << "Server stopped"; - - return 0; -} - -int main(int argc, char* argv[]) { - google::ParseCommandLineFlags(&argc, &argv, true); - - Configuration conf; - conf.SetConfigPath(FLAGS_conf); - - LOG_IF(FATAL, !conf.LoadConfig()) - << "load space configuration failed, conf path: " - << conf.GetConfigPath(); - - LoadConfigFromGflags(&conf); - - google::InitGoogleLogging(argv[0]); - int ret = Run(&conf); - google::ShutdownGoogleLogging(); - - return ret; -} diff --git a/curvefs/src/space/metaserver_client.cpp b/curvefs/src/space/metaserver_client.cpp deleted file mode 100644 index 2afa50a749..0000000000 --- a/curvefs/src/space/metaserver_client.cpp +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Project: curve - * File Created: Fri Jul 16 21:22:40 CST 2021 - * Author: wuhanqing - */ - -#include "curvefs/src/space/metaserver_client.h" - -#include - -#include - -namespace curvefs { -namespace space { - -bool MetaServerClient::Init(const MetaServerClientOption &opt) { - opt_ = opt; - - if (opt_.addr.empty()) { - LOG(ERROR) << "meta server address is empty"; - return false; - } - - int ret = channel_.Init(opt_.addr.c_str(), nullptr); - if (ret != 0) { - LOG(ERROR) << "Init channel to metaserver failed"; - return false; - } - - return true; -} - -bool MetaServerClient::GetAllInodeExtents(uint32_t fsId, uint64_t rootInodeId, - Extents *exts) { - return RecursiveListDentry(fsId, rootInodeId, exts); -} - -// TODO(wuhanqing): Using recursion, if the file system has a deep layer, will -// blow up the stack? -bool MetaServerClient::RecursiveListDentry(uint32_t fsId, uint64_t inodeId, - Extents *exts) { - metaserver::MetaServerService_Stub stub(&channel_); - metaserver::ListDentryRequest request; - metaserver::ListDentryResponse response; - - brpc::Controller cntl; - - // TODO(wuhanqing): add partition - request.set_poolid(0); - request.set_copysetid(0); - request.set_partitionid(0); - // TODO(wuhanqing): end add partition - request.set_fsid(fsId); - request.set_dirinodeid(inodeId); - // TODO(@汉卿) 适配新的proto - request.set_copysetid(1); - request.set_poolid(1); - request.set_partitionid(1); - request.set_txid(100); - - stub.ListDentry(&cntl, &request, &response, nullptr); - - if (cntl.Failed() || (response.statuscode() != metaserver::OK && - response.statuscode() != metaserver::NOT_FOUND)) { - LOG(ERROR) << "ListDentry rpc failed, rpc errro: " << cntl.ErrorCode() - << "response status: " - << curvefs::metaserver::MetaStatusCode_Name( - response.statuscode()); - return false; - } - - for (auto &d : response.dentrys()) { - metaserver::MetaServerService_Stub stub(&channel_); - metaserver::GetInodeRequest request; - metaserver::GetInodeResponse response; - - brpc::Controller cntl; - request.set_fsid(fsId); - request.set_inodeid(d.inodeid()); - - stub.GetInode(&cntl, &request, &response, nullptr); - - if (cntl.Failed() || response.statuscode() != metaserver::OK) { - return false; - } - - switch (response.inode().type()) { - case metaserver::FsFileType::TYPE_FILE: - AppendExtents(exts, response.inode().volumeextentlist()); - break; - case metaserver::FsFileType::TYPE_DIRECTORY: - return RecursiveListDentry(fsId, response.inode().inodeid(), exts); - default: - break; - } - } - - return true; -} - -void MetaServerClient::AppendExtents( - Extents *exts, - const curvefs::metaserver::VolumeExtentList &protoExts) const { - const auto &volumeextents = protoExts.volumeextents(); - for (auto &e : volumeextents) { - exts->emplace_back(e.volumeoffset(), e.length()); - } -} - -} // namespace space -} // namespace curvefs diff --git a/curvefs/src/space/metaserver_client.h b/curvefs/src/space/metaserver_client.h deleted file mode 100644 index 6b3a4b39a2..0000000000 --- a/curvefs/src/space/metaserver_client.h +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Project: curve - * File Created: Fri Jul 16 21:22:40 CST 2021 - * Author: wuhanqing - */ - -#ifndef CURVEFS_SRC_SPACE_METASERVER_CLIENT_H_ -#define CURVEFS_SRC_SPACE_METASERVER_CLIENT_H_ - -#include - -#include - -#include "curvefs/proto/metaserver.pb.h" -#include "curvefs/src/space/common.h" -#include "curvefs/src/space/config.h" - -namespace curvefs { -namespace space { - -class MetaServerClient { - public: - MetaServerClient() = default; - - ~MetaServerClient() = default; - - bool Init(const MetaServerClientOption& opt); - - /** - * @brief Get all file extents that belongs to `fsId` - */ - bool GetAllInodeExtents(uint32_t fsId, uint64_t rootInodeId, Extents* exts); - - private: - /** - * @brief Recursive traverse all files in the filesystem, and store files' - * extents - */ - bool RecursiveListDentry(uint32_t fsId, uint64_t inodeId, Extents* exts); - - void AppendExtents( - Extents* exts, - const curvefs::metaserver::VolumeExtentList& protoExts) const; - - private: - MetaServerClientOption opt_; - brpc::Channel channel_; -}; - -} // namespace space -} // namespace curvefs - -#endif // CURVEFS_SRC_SPACE_METASERVER_CLIENT_H_ diff --git a/curvefs/src/space/reloader.h b/curvefs/src/space/reloader.h deleted file mode 100644 index 3cfc9917c0..0000000000 --- a/curvefs/src/space/reloader.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Project: curve - * File Created: Fri Jul 16 21:22:40 CST 2021 - * Author: wuhanqing - */ - -#ifndef CURVEFS_SRC_SPACE_RELOADER_H_ -#define CURVEFS_SRC_SPACE_RELOADER_H_ - -#include - -#include "curvefs/src/space/allocator.h" -#include "curvefs/src/space/config.h" - -namespace curvefs { -namespace space { - -class Reloader { - public: - Reloader(Allocator* allocator, uint32_t fsId, uint64_t rootInodeId, - const ReloaderOption& option) - : fsId_(fsId), - rootInodeId_(rootInodeId), - option_(option), - allocator_(allocator) {} - - ~Reloader() = default; - - Reloader(const Reloader&) = delete; - - Reloader& operator=(const Reloader&) = delete; - - bool Reload(); - - private: - const uint32_t fsId_; - const uint64_t rootInodeId_; - - ReloaderOption option_; - - Allocator* allocator_; -}; - -} // namespace space -} // namespace curvefs - -#endif // CURVEFS_SRC_SPACE_RELOADER_H_ diff --git a/curvefs/src/space/space_manager.cpp b/curvefs/src/space/space_manager.cpp deleted file mode 100644 index 3397f0aa0d..0000000000 --- a/curvefs/src/space/space_manager.cpp +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * Project: curve - * File Created: Fri Jul 16 21:22:40 CST 2021 - * Author: wuhanqing - */ - -#include "curvefs/src/space/space_manager.h" - -#include - -#include - -#include "curvefs/proto/mds.pb.h" -#include "curvefs/proto/space.pb.h" -#include "curvefs/src/space/reloader.h" -#include "curvefs/src/space/utils.h" - -namespace curvefs { -namespace space { - -using curve::common::ReadLockGuard; -using curve::common::WriteLockGuard; - -SpaceStatusCode SpaceManagerImpl::InitSpace(const mds::FsInfo& fsInfo) { - WriteLockGuard lock(blkRwlock_); - if (allocators_.count(fsInfo.fsid()) != 0) { - LOG(ERROR) << "allocator already exists, fsId: " << fsInfo.fsid(); - return SPACE_EXISTS; - } - - AllocatorOption allocOpt(opt_.allocatorOption); - allocOpt.bitmapAllocatorOption.startOffset = 0; - allocOpt.bitmapAllocatorOption.length = fsInfo.capacity(); - - auto allocator = - Allocator::Create(opt_.allocatorType, allocOpt); - if (!allocator) { - LOG(ERROR) << "unknown allocator type: " << opt_.allocatorType; - return SPACE_UNKNOWN_ERROR; - } - - Reloader reloader(allocator.get(), fsInfo.fsid(), fsInfo.rootinodeid(), - opt_.reloaderOption); - if (reloader.Reload() == false) { - LOG(ERROR) << "reload extents failed"; - return SPACE_RELOAD_ERROR; - } - - allocators_.emplace(fsInfo.fsid(), std::move(allocator)); - LOG(INFO) << "Init allocator success, fsId: " << fsInfo.fsid() - << ", volume size: " << fsInfo.capacity() - << ", block size: " << fsInfo.blocksize(); - return SPACE_OK; -} - -SpaceStatusCode SpaceManagerImpl::UnInitSpace(uint32_t fsId) { - WriteLockGuard lock(blkRwlock_); - auto iter = allocators_.find(fsId); - if (iter == allocators_.end()) { - return SPACE_NOT_FOUND; - } - - allocators_.erase(iter); - return SPACE_OK; -} - -SpaceStatusCode SpaceManagerImpl::StatSpace(uint32_t fsId, SpaceStat* stat) { - ReadLockGuard lock(blkRwlock_); - auto iter = allocators_.find(fsId); - if (iter == allocators_.end()) { - return SPACE_NOT_FOUND; - } - - stat->blockSize = opt_.blockSize; - stat->total = iter->second->Total(); - stat->available = iter->second->AvailableSize(); - return SPACE_OK; -} - -SpaceStatusCode SpaceManagerImpl::AllocateSpace(uint32_t fsId, uint64_t size, - const SpaceAllocateHint& hint, - Extents* exts) { - ReadLockGuard lock(blkRwlock_); - auto iter = allocators_.find(fsId); - if (iter == allocators_.end()) { - return SPACE_NOT_FOUND; - } - - if (iter->second->AvailableSize() < size) { - return SPACE_NO_SPACE; - } - - Extents tmp; - uint64_t allocated = iter->second->Alloc(size, hint, &tmp); - if (allocated < size) { - iter->second->DeAlloc(tmp); - return SPACE_NO_SPACE; - } else { - std::move(tmp.begin(), tmp.end(), std::back_inserter(*exts)); - return SPACE_OK; - } -} - -SpaceStatusCode SpaceManagerImpl::DeallocateSpace(uint32_t fsId, - const ProtoExtents& extents) { - ReadLockGuard lock(blkRwlock_); - auto iter = allocators_.find(fsId); - if (iter == allocators_.end()) { - return SPACE_NOT_FOUND; - } - - for (auto& e : extents) { - bool r = iter->second->DeAlloc(e.offset(), e.length()); - if (!r) { - LOG(ERROR) << "Dealloc " << e << " failed"; - return SPACE_DEALLOC_ERROR; - } - } - - return SPACE_OK; -} - -SpaceManagerImpl::SpaceManagerImpl(const SpaceManagerOption& opt) : opt_(opt) {} - -} // namespace space -} // namespace curvefs diff --git a/curvefs/src/space/space_manager.h b/curvefs/src/space/space_manager.h deleted file mode 100644 index 9d86c4de09..0000000000 --- a/curvefs/src/space/space_manager.h +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Project: curve - * File Created: Fri Jul 16 21:22:40 CST 2021 - * Author: wuhanqing - */ - -#ifndef CURVEFS_SRC_SPACE_SPACE_MANAGER_H_ -#define CURVEFS_SRC_SPACE_SPACE_MANAGER_H_ - -#include -#include - -#include "curvefs/proto/space.pb.h" -#include "curvefs/src/space/allocator.h" -#include "curvefs/src/space/config.h" -#include "src/common/concurrent/rw_lock.h" - -namespace curvefs { -namespace space { - -class SpaceManager { - public: - SpaceManager() = default; - - virtual ~SpaceManager() = default; - - SpaceManager(const SpaceManager&) = delete; - - SpaceManager& operator=(const SpaceManager&) = delete; - - virtual SpaceStatusCode InitSpace(const mds::FsInfo& fsInfo) = 0; - - virtual SpaceStatusCode UnInitSpace(uint32_t fsId) = 0; - - virtual SpaceStatusCode StatSpace(uint32_t fsId, SpaceStat* stat) = 0; - - virtual SpaceStatusCode AllocateSpace(uint32_t fsId, uint64_t size, - const SpaceAllocateHint& hint, - Extents* exts) = 0; - - virtual SpaceStatusCode DeallocateSpace(uint32_t fsId, - const ProtoExtents& extents) = 0; -}; - -class SpaceManagerImpl : public SpaceManager { - public: - explicit SpaceManagerImpl(const SpaceManagerOption& opt); - - SpaceStatusCode InitSpace(const mds::FsInfo& fsInfo) override; - - SpaceStatusCode UnInitSpace(uint32_t fsId) override; - - SpaceStatusCode StatSpace(uint32_t fsId, SpaceStat* stat) override; - - SpaceStatusCode AllocateSpace(uint32_t fsId, uint64_t size, - const SpaceAllocateHint& hint, - Extents* exts) override; - - SpaceStatusCode DeallocateSpace(uint32_t fsId, - const ProtoExtents& extents) override; - - private: - SpaceManagerOption opt_; - - ::curve::common::BthreadRWLock blkRwlock_; - - std::unordered_map> allocators_; -}; - -} // namespace space -} // namespace curvefs - -#endif // CURVEFS_SRC_SPACE_SPACE_MANAGER_H_ diff --git a/curvefs/src/space/space_service.cpp b/curvefs/src/space/space_service.cpp deleted file mode 100644 index d3db46c36c..0000000000 --- a/curvefs/src/space/space_service.cpp +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Project: curve - * File Created: Fri Jul 16 21:22:40 CST 2021 - * Author: wuhanqing - */ - -#include "curvefs/src/space/space_service.h" - -#include -#include - -#include "curvefs/src/space/utils.h" - -namespace curvefs { -namespace space { - -SpaceAllocServiceImpl::SpaceAllocServiceImpl(SpaceManager* space) - : space_(space) {} - -void SpaceAllocServiceImpl::InitSpace( - ::google::protobuf::RpcController* controller, - const ::curvefs::space::InitSpaceRequest* request, - ::curvefs::space::InitSpaceResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard guard(done); - brpc::Controller* cntl = static_cast(controller); - - auto status = space_->InitSpace(request->fsinfo()); - - response->set_status(status); - if (status != SPACE_OK) { - LOG(ERROR) << "InitSpace failure, logid: " << cntl->log_id() - << ", request: " << request->ShortDebugString() - << ", error: " << SpaceStatusCode_Name(status); - } else { - LOG(INFO) << "InitSpace success, logid: " << cntl->log_id() - << ", request: " << request->ShortDebugString(); - } -} - -void SpaceAllocServiceImpl::UnInitSpace( - ::google::protobuf::RpcController* controller, - const ::curvefs::space::UnInitSpaceRequest* request, - ::curvefs::space::UnInitSpaceResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard guard(done); - brpc::Controller* cntl = static_cast(controller); - - auto status = space_->UnInitSpace(request->fsid()); - response->set_status(status); - if (status != SPACE_OK) { - LOG(ERROR) << "UnInitSpace failure, " << cntl->log_id() - << ", request: " << request->ShortDebugString() - << ", error: " << SpaceStatusCode_Name(status); - } else { - LOG(INFO) << "UnInitSpace success, " << cntl->log_id() - << ", request: " << request->ShortDebugString(); - } -} - -void SpaceAllocServiceImpl::AllocateSpace( - ::google::protobuf::RpcController* controller, - const ::curvefs::space::AllocateSpaceRequest* request, - ::curvefs::space::AllocateSpaceResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard guard(done); - - Extents exts; - SpaceAllocateHint hint; - if (request->has_allochint()) { - hint = ConvertToSpaceAllocateHint(request->allochint()); - } - - auto status = - space_->AllocateSpace(request->fsid(), request->size(), hint, &exts); - - response->set_status(status); - if (status != SPACE_OK) { - LOG(ERROR) << "AllocateSpace failure, request: " - << request->ShortDebugString() - << ", error: " << SpaceStatusCode_Name(status); - } else { - *response->mutable_extents() = ConvertToProtoExtents(exts); - - LOG(INFO) << "AllocateSpace success, request: " - << request->ShortDebugString() - << ", response: " << response->ShortDebugString(); - } -} - -void SpaceAllocServiceImpl::DeallocateSpace( - ::google::protobuf::RpcController* controller, - const ::curvefs::space::DeallocateSpaceRequest* request, - ::curvefs::space::DeallocateSpaceResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard guard(done); - - auto status = space_->DeallocateSpace(request->fsid(), request->extents()); - - response->set_status(status); - if (status != SPACE_OK) { - LOG(ERROR) << "DeallocateSpace failure, request: " - << request->ShortDebugString() - << ", error: " << SpaceStatusCode_Name(status); - } else { - LOG(INFO) << "DeallocateSpace success, request: " - << request->ShortDebugString(); - } -} - -void SpaceAllocServiceImpl::StatSpace( - ::google::protobuf::RpcController* controller, - const ::curvefs::space::StatSpaceRequest* request, - ::curvefs::space::StatSpaceResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard guard(done); - - SpaceStat stat; - auto status = space_->StatSpace(request->fsid(), &stat); - - response->set_status(status); - if (status != SPACE_OK) { - LOG(ERROR) << "StatSpace failure, request: " - << request->ShortDebugString() - << ", error: " << SpaceStatusCode_Name(status); - } else { - response->set_totalblock(stat.total / stat.blockSize); - response->set_availableblock(stat.available / stat.blockSize); - response->set_usedblock((stat.total - stat.available) / stat.blockSize); - } -} - -} // namespace space -} // namespace curvefs diff --git a/curvefs/src/space/space_service.h b/curvefs/src/space/space_service.h deleted file mode 100644 index 4a63a49daf..0000000000 --- a/curvefs/src/space/space_service.h +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Project: curve - * File Created: Fri Jul 16 21:22:40 CST 2021 - * Author: wuhanqing - */ - -#ifndef CURVEFS_SRC_SPACE_SPACE_SERVICE_H_ -#define CURVEFS_SRC_SPACE_SPACE_SERVICE_H_ - -#include "curvefs/proto/space.pb.h" -#include "curvefs/src/space/space_manager.h" - -namespace curvefs { -namespace space { - -class SpaceAllocServiceImpl : public curvefs::space::SpaceAllocService { - public: - explicit SpaceAllocServiceImpl(SpaceManager* space); - - ~SpaceAllocServiceImpl() = default; - - void InitSpace(::google::protobuf::RpcController* controller, - const ::curvefs::space::InitSpaceRequest* request, - ::curvefs::space::InitSpaceResponse* response, - ::google::protobuf::Closure* done) override; - - void UnInitSpace(::google::protobuf::RpcController* controller, - const ::curvefs::space::UnInitSpaceRequest* request, - ::curvefs::space::UnInitSpaceResponse* response, - ::google::protobuf::Closure* done) override; - - void AllocateSpace(::google::protobuf::RpcController* controller, - const ::curvefs::space::AllocateSpaceRequest* request, - ::curvefs::space::AllocateSpaceResponse* response, - ::google::protobuf::Closure* done) override; - - void DeallocateSpace( - ::google::protobuf::RpcController* controller, - const ::curvefs::space::DeallocateSpaceRequest* request, - ::curvefs::space::DeallocateSpaceResponse* response, - ::google::protobuf::Closure* done) override; - - void StatSpace(::google::protobuf::RpcController* controller, - const ::curvefs::space::StatSpaceRequest* request, - ::curvefs::space::StatSpaceResponse* response, - ::google::protobuf::Closure* done) override; - - private: - SpaceManager* space_; -}; - -} // namespace space -} // namespace curvefs - -#endif // CURVEFS_SRC_SPACE_SPACE_SERVICE_H_ diff --git a/curvefs/src/tools/create/curvefs_create_fs.cpp b/curvefs/src/tools/create/curvefs_create_fs.cpp index 49b14d3712..1e2107e72c 100644 --- a/curvefs/src/tools/create/curvefs_create_fs.cpp +++ b/curvefs/src/tools/create/curvefs_create_fs.cpp @@ -24,6 +24,7 @@ #include "curvefs/proto/common.pb.h" #include "curvefs/src/tools/curvefs_tool_define.h" #include "src/common/string_util.h" +#include "src/common/fast_align.h" DECLARE_string(fsName); DECLARE_string(confPath); @@ -35,6 +36,8 @@ DECLARE_uint64(volumeBlockSize); DECLARE_string(volumeName); DECLARE_string(volumeUser); DECLARE_string(volumePassword); +DECLARE_uint64(volumeBlockGroupSize); +DECLARE_string(volumeBitmapLocation); DECLARE_string(s3_ak); DECLARE_string(s3_sk); DECLARE_string(s3_endpoint); @@ -49,16 +52,22 @@ namespace curvefs { namespace tools { namespace create { +using ::curve::common::is_aligned; +using ::curvefs::common::BitmapLocation; +using ::curvefs::common::BitmapLocation_Parse; + void CreateFsTool::PrintHelp() { CurvefsToolRpc::PrintHelp(); std::cout << " -fsName=" << FLAGS_fsName << " [-blockSize=" << FLAGS_blockSize << "] [-enableSumInDir=" << FLAGS_enableSumInDir << "] [-fsType=volume -volumeSize=" << FLAGS_volumeSize + << " -volumeBlockGroupSize=" << FLAGS_volumeBlockGroupSize << " -volumeBlockSize=" << FLAGS_volumeBlockSize << " -volumeName=" << FLAGS_volumeName << " -volumeUser=" << FLAGS_volumeUser << " -volumePassword=" << FLAGS_volumePassword + << " -volumeBitmapLocation=AtStart|AtEnd" << "]|[-fsType=s3 -s3_ak=" << FLAGS_s3_ak << " -s3_sk=" << FLAGS_s3_sk << " -s3_endpoint=" << FLAGS_s3_endpoint @@ -79,6 +88,8 @@ void CreateFsTool::AddUpdateFlags() { AddUpdateFlagsFunc(curvefs::tools::SetVolumeName); AddUpdateFlagsFunc(curvefs::tools::SetVolumeUser); AddUpdateFlagsFunc(curvefs::tools::SetVolumePassword); + AddUpdateFlagsFunc(curvefs::tools::SetVolumeBlockSize); + AddUpdateFlagsFunc(curvefs::tools::SetVolumeBitmapLocation); AddUpdateFlagsFunc(curvefs::tools::SetS3_ak); AddUpdateFlagsFunc(curvefs::tools::SetS3_sk); AddUpdateFlagsFunc(curvefs::tools::SetS3_endpoint); @@ -115,14 +126,45 @@ int CreateFsTool::Init() { s3->set_chunksize(FLAGS_s3_chunksize); request.mutable_fsdetail()->set_allocated_s3info(s3); } else if (FLAGS_fsType == kFsTypeVolume) { + // precheck + if (!is_aligned(FLAGS_volumeBlockSize, 4096)) { + std::cerr << "volumeBlockSize should align with 4096"; + return -1; + } + + if (!is_aligned(FLAGS_volumeBlockGroupSize, FLAGS_volumeBlockSize)) { + std::cerr + << "volumeBlockGroupSize should align with volumeBlockSize"; + return -1; + } + + if (!is_aligned(FLAGS_volumeBlockGroupSize, 128ULL * 1024 * 1024)) { + std::cerr << "volumeBlockGroupSize should align with 128MiB"; + return -1; + } + + if (!is_aligned(FLAGS_volumeSize, FLAGS_volumeBlockGroupSize)) { + std::cerr << "volumeSize should align with volumeBlockGroupSize"; + return -1; + } + + BitmapLocation location; + if (!BitmapLocation_Parse(FLAGS_volumeBitmapLocation, &location)) { + std::cerr << "Parse volumeBitmapLocation error, only support " + "|AtStart| and |AtEnd|"; + return -1; + } + // volume request.set_fstype(common::FSType::TYPE_VOLUME); - auto volume = new common::Volume(); + auto* volume = new common::Volume(); volume->set_volumesize(FLAGS_volumeSize); volume->set_blocksize(FLAGS_volumeBlockSize); volume->set_volumename(FLAGS_volumeName); volume->set_user(FLAGS_volumeUser); volume->set_password(FLAGS_volumePassword); + volume->set_blockgroupsize(FLAGS_volumeBlockGroupSize); + volume->set_bitmaplocation(location); request.mutable_fsdetail()->set_allocated_volume(volume); } else { std::cerr << "-fsType should be " << kFsTypeS3 << " or " diff --git a/curvefs/src/tools/curvefs_tool_define.cpp b/curvefs/src/tools/curvefs_tool_define.cpp index 1a4e8ede9c..8da2c53b73 100644 --- a/curvefs/src/tools/curvefs_tool_define.cpp +++ b/curvefs/src/tools/curvefs_tool_define.cpp @@ -67,6 +67,14 @@ DEFINE_string(mds_addr, "127.0.0.1:6700", "mds ip and port, separated by \",\""); // NOLINT DEFINE_string(cluster_map, "topo_example.json", "cluster topology map."); +DEFINE_uint64(volumeBlockGroupSize, + 128ULL * 1024 * 1024, + "volume block group size"); + +DEFINE_string(volumeBitmapLocation, + "AtStart", + "volume space bitmap location, support |AtStart| and |AtEnd|"); + namespace curvefs { namespace tools { @@ -160,6 +168,20 @@ std::function std::placeholders::_1, std::placeholders::_2, "volumePassword", &FLAGS_volumePassword); +std::function + SetVolumeBlockGroupSize = std::bind(&SetFlagInfo, + std::placeholders::_1, + std::placeholders::_2, + "volumeBlockGroupSize", + &FLAGS_volumeBlockGroupSize); + +std::function + SetVolumeBitmapLocation = std::bind(&SetFlagInfo, + std::placeholders::_1, + std::placeholders::_2, + "volumeBitmapLocation", + &FLAGS_volumeBitmapLocation); + std::function SetS3_ak = std::bind(&SetDiffFlagInfo, std::placeholders::_1, std::placeholders::_2, "s3_ak", "s3.ak", &FLAGS_s3_ak); diff --git a/curvefs/src/tools/curvefs_tool_define.h b/curvefs/src/tools/curvefs_tool_define.h index 68f0e3e0dd..f6f70967c5 100644 --- a/curvefs/src/tools/curvefs_tool_define.h +++ b/curvefs/src/tools/curvefs_tool_define.h @@ -233,6 +233,12 @@ extern std::function SetVolumePassword; +extern std::function + SetVolumeBlockSize; +extern std::function + SetVolumeBitmapLocation; extern std::function SetS3_ak; diff --git a/curvefs/src/volume/BUILD b/curvefs/src/volume/BUILD new file mode 100644 index 0000000000..7b2fc527ae --- /dev/null +++ b/curvefs/src/volume/BUILD @@ -0,0 +1,44 @@ +# +# Copyright (c) 2022 NetEase Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("//:copts.bzl", "CURVE_DEFAULT_COPTS") + +cc_library( + name = "volume", + srcs = glob([ + "*.cpp", + ]), + hdrs = glob([ + "*.h", + ]), + copts = CURVE_DEFAULT_COPTS, + linkopts = [ + "-pthread", + ], + visibility = ["//visibility:public"], + deps = [ + "//curvefs/proto:space_cc_proto", + "//curvefs/src/client/common", + "//curvefs/src/client/rpcclient", + "//curvefs/src/common:metric_utils", + "//external:glog", + "//src/client:curve_client", + "//src/common:curve_common", + "//src/common/concurrent:curve_concurrent", + "@com_google_absl//absl/cleanup", + "@com_google_absl//absl/memory", + ], +) diff --git a/curvefs/src/space/allocator.cpp b/curvefs/src/volume/allocator.cpp similarity index 80% rename from curvefs/src/space/allocator.cpp rename to curvefs/src/volume/allocator.cpp index 862dce0cf0..0ef437b583 100644 --- a/curvefs/src/space/allocator.cpp +++ b/curvefs/src/volume/allocator.cpp @@ -20,24 +20,24 @@ * Author: wuhanqing */ -#include "curvefs/src/space/allocator.h" +#include "curvefs/src/volume/allocator.h" #include -#include "curvefs/src/space/bitmap_allocator.h" +#include "absl/memory/memory.h" +#include "curvefs/src/volume/bitmap_allocator.h" namespace curvefs { -namespace space { +namespace volume { std::unique_ptr Allocator::Create(const std::string& type, const AllocatorOption& option) { if (type == "bitmap") { - return std::unique_ptr( - new BitmapAllocator(option.bitmapAllocatorOption)); + return absl::make_unique(option.bitmapAllocatorOption); } else { return nullptr; } } -} // namespace space +} // namespace volume } // namespace curvefs diff --git a/curvefs/src/space/allocator.h b/curvefs/src/volume/allocator.h similarity index 74% rename from curvefs/src/space/allocator.h rename to curvefs/src/volume/allocator.h index ddc15cd033..89d4f5bd19 100644 --- a/curvefs/src/space/allocator.h +++ b/curvefs/src/volume/allocator.h @@ -20,28 +20,26 @@ * Author: wuhanqing */ -#ifndef CURVEFS_SRC_SPACE_ALLOCATOR_H_ -#define CURVEFS_SRC_SPACE_ALLOCATOR_H_ +#ifndef CURVEFS_SRC_VOLUME_ALLOCATOR_H_ +#define CURVEFS_SRC_VOLUME_ALLOCATOR_H_ #include #include +#include -#include "curvefs/src/space/common.h" -#include "curvefs/src/space/config.h" +#include "curvefs/src/volume/common.h" +#include "curvefs/src/volume/option.h" namespace curvefs { -namespace space { +namespace volume { class Allocator { public: - Allocator() = default; + static std::unique_ptr Create(const std::string& type, + const AllocatorOption& option); virtual ~Allocator() = default; - Allocator(const Allocator&) = delete; - - Allocator& operator=(const Allocator&) = delete; - /** * @brief Allocate space * @@ -50,8 +48,9 @@ class Allocator { * @param[out] exts store allocated extents * @return return allocated size */ - virtual uint64_t Alloc(const uint64_t size, const SpaceAllocateHint& hint, - Extents* exts) = 0; + virtual uint64_t Alloc(const uint64_t size, + const AllocateHint& hint, + std::vector* exts) = 0; /** * @brief DeAllocate space @@ -65,7 +64,7 @@ class Allocator { * * @return return true if succeeded, otherwise return false */ - virtual bool DeAlloc(const Extents& exts) = 0; + virtual bool DeAlloc(const std::vector& exts) = 0; /** * @brief Total space size @@ -80,18 +79,15 @@ class Allocator { /** * @brief Mark extents are used */ - virtual bool MarkUsed(const Extents& extents) = 0; + virtual bool MarkUsed(const std::vector& extents) = 0; /** * @brief Mark extents are available */ - virtual bool MarkUsable(const Extents& extents) = 0; - - static std::unique_ptr Create(const std::string& type, - const AllocatorOption& option); + virtual bool MarkUsable(const std::vector& extents) = 0; }; -} // namespace space +} // namespace volume } // namespace curvefs -#endif // CURVEFS_SRC_SPACE_ALLOCATOR_H_ +#endif // CURVEFS_SRC_VOLUME_ALLOCATOR_H_ diff --git a/curvefs/src/space/bitmap_allocator.cpp b/curvefs/src/volume/bitmap_allocator.cpp similarity index 69% rename from curvefs/src/space/bitmap_allocator.cpp rename to curvefs/src/volume/bitmap_allocator.cpp index 321e71cbf8..6b35effaa1 100644 --- a/curvefs/src/space/bitmap_allocator.cpp +++ b/curvefs/src/volume/bitmap_allocator.cpp @@ -20,31 +20,31 @@ * Author: wuhanqing */ -#include "curvefs/src/space/bitmap_allocator.h" +#include "curvefs/src/volume/bitmap_allocator.h" #include #include #include -#include "curvefs/src/common/fast_align.h" +#include "curvefs/src/volume/common.h" +#include "curvefs/src/volume/utils.h" +#include "src/common/fast_align.h" namespace curvefs { -namespace space { +namespace volume { -using curvefs::common::align_up; -using curvefs::common::align_down; +using curve::common::align_down; +using curve::common::align_up; +using curve::common::is_aligned; namespace { -inline bool HasAllocOffsetHint(const SpaceAllocateHint& hint) { - return hint.leftOffset != hint.INVALID_OFFSET || - hint.rightOffset != hint.INVALID_OFFSET; +inline bool IsAllocTypeBig(const AllocateHint& hint) { + return hint.allocType == AllocateType::Big; } -inline bool IsAllocTypeBig(const SpaceAllocateHint& hint) { - return hint.allocType == AllocateType::BIG; -} +constexpr uint64_t kAlignment = 4096; } // namespace @@ -58,8 +58,8 @@ uint64_t BitmapAllocator::CalcBitmapAreaLength( struct BitmapAllocator::AllocOrder { using Func = uint64_t (BitmapAllocator::*)(const uint64_t size, - const SpaceAllocateHint& hint, - Extents* exts); + const AllocateHint& hint, + std::vector* exts); AllocOrder(Func first, Func second, Func third) : funs({first, second, third}) {} @@ -76,7 +76,7 @@ BitmapAllocator::BitmapAllocator(const BitmapAllocatorOption& opt) available_(opt_.length), bitmapAllocIdx_(0), bitmap_(bitmapAreaLength_ / opt_.sizePerBit), - bitmapExtent_(opt_.sizePerBit), + bitmapExtent_(opt_.sizePerBit, bitmapAreaOffset_, bitmapAreaLength_), smallExtent_(opt_.startOffset, smallAreaLength_) { CHECK(bitmap_.Size() * opt_.sizePerBit + smallExtent_.AvailableSize() == opt_.length) @@ -84,12 +84,21 @@ BitmapAllocator::BitmapAllocator(const BitmapAllocatorOption& opt) << ", opt.sizePerBit: " << opt_.sizePerBit << ", smallExtent.avail: " << smallExtent_.AvailableSize() << ", opt.length: " << opt.length; + + VLOG(9) << "offset: " << opt_.startOffset << ", len: " << opt_.length + << ", size_per_bit: " << opt_.sizePerBit << "bitmapAreaLength_ " + << bitmapAreaLength_ << ", bitmapAreaOffset_: " << bitmapAreaOffset_ + << ", smallAreaLength_: " << smallAreaLength_ + << ", available: " << available_; } BitmapAllocator::~BitmapAllocator() {} -uint64_t BitmapAllocator::Alloc(uint64_t size, const SpaceAllocateHint& hint, - Extents* exts) { +uint64_t BitmapAllocator::Alloc(uint64_t size, + const AllocateHint& hint, + std::vector* exts) { + assert(is_aligned(size, kAlignment)); + static const BitmapAllocator::AllocOrder kAllocBig( &BitmapAllocator::AllocFromBitmap, &BitmapAllocator::AllocFromBitmapExtent, @@ -114,7 +123,7 @@ uint64_t BitmapAllocator::Alloc(uint64_t size, const SpaceAllocateHint& hint, if (size < opt_.sizePerBit && !IsAllocTypeBig(hint)) { alloc = AllocInternal(kAllocSmall, size, hint, exts); - } else if (HasAllocOffsetHint(hint)) { + } else if (hint.HasHint()) { alloc = AllocInternal(kAllocBigWithHint, size, hint, exts); } else { alloc = AllocInternal(kAllocBig, size, hint, exts); @@ -127,6 +136,13 @@ uint64_t BitmapAllocator::Alloc(uint64_t size, const SpaceAllocateHint& hint, } bool BitmapAllocator::DeAlloc(const uint64_t off, const uint64_t len) { + assert(is_aligned(off, kAlignment) && is_aligned(len, kAlignment)); + + assert(off >= opt_.startOffset && + off + len <= opt_.startOffset + opt_.length); + + VLOG(9) << "Dealloc off: " << off << ", len: " << len; + std::lock_guard lock(mtx_); if (len > opt_.length - available_) { @@ -155,7 +171,7 @@ bool BitmapAllocator::DeAlloc(const uint64_t off, const uint64_t len) { return true; } -bool BitmapAllocator::DeAlloc(const Extents& exts) { +bool BitmapAllocator::DeAlloc(const std::vector& exts) { for (const auto& e : exts) { if (!DeAlloc(e.offset, e.len)) { return false; @@ -165,7 +181,7 @@ bool BitmapAllocator::DeAlloc(const Extents& exts) { return true; } -bool BitmapAllocator::MarkUsed(const Extents& extents) { +bool BitmapAllocator::MarkUsed(const std::vector& extents) { for (auto& e : extents) { MarkUsedInternal(e.offset, e.len); } @@ -173,14 +189,14 @@ bool BitmapAllocator::MarkUsed(const Extents& extents) { return true; } -bool BitmapAllocator::MarkUsable(const Extents& extents) { +bool BitmapAllocator::MarkUsable(const std::vector& /*extents*/) { return false; } uint64_t BitmapAllocator::AllocInternal(const AllocOrder& order, const uint64_t size, - const SpaceAllocateHint& hint, - Extents* exts) { + const AllocateHint& hint, + std::vector* exts) { uint64_t need = size; for (auto& fn : order.funs) { need -= (this->*fn)(need, hint, exts); @@ -193,8 +209,8 @@ uint64_t BitmapAllocator::AllocInternal(const AllocOrder& order, } uint64_t BitmapAllocator::AllocFromBitmap(uint64_t size, - const SpaceAllocateHint& hint, - Extents* exts) { + const AllocateHint& /*hint*/, + std::vector* exts) { uint64_t need = size; while (need > 0) { @@ -227,14 +243,14 @@ uint64_t BitmapAllocator::AllocFromBitmap(uint64_t size, } uint64_t BitmapAllocator::AllocFromBitmapExtent(uint64_t size, - const SpaceAllocateHint& hint, - Extents* exts) { + const AllocateHint& hint, + std::vector* exts) { return bitmapExtent_.Alloc(size, hint, exts); } uint64_t BitmapAllocator::AllocFromSmallExtent(uint64_t size, - const SpaceAllocateHint& hint, - Extents* exts) { + const AllocateHint& hint, + std::vector* exts) { return smallExtent_.Alloc(size, hint, exts); } @@ -273,7 +289,7 @@ void BitmapAllocator::DeAllocToBitmap(const uint64_t off, const uint64_t len) { void BitmapAllocator::DeAllocToBitmapExtent(const uint64_t off, const uint64_t len) { - ExtentMap blocks; + std::map blocks; bitmapExtent_.DeAlloc(off, len); auto size = bitmapExtent_.AvailableBlocks(&blocks); (void)size; @@ -285,6 +301,11 @@ void BitmapAllocator::DeAllocToBitmapExtent(const uint64_t off, } void BitmapAllocator::MarkUsedInternal(const uint64_t off, const uint64_t len) { + assert(is_aligned(off, kAlignment) && is_aligned(len, kAlignment)); + + assert(off >= opt_.startOffset && + off + len <= opt_.startOffset + opt_.length); + std::lock_guard lock(mtx_); uint64_t offInSmallExtent = 0; @@ -317,36 +338,53 @@ void BitmapAllocator::MarkUsedForBitmap(const uint64_t off, // if it's a aligned block, mark slot used // otherwise call bitmapExtent::MarkUsed - uint64_t alignedLeftOff = align_up(off, opt_.sizePerBit); + uint64_t alignedLeftOff = align_down(off, opt_.sizePerBit); uint64_t unalignedLeftLen = alignedLeftOff - off; - uint64_t alignedRightOff = align_down(off + len, opt_.sizePerBit); + uint64_t alignedRightOff = align_up(off + len, opt_.sizePerBit); uint64_t unalignedRightLen = off + len - alignedRightOff; // bitmap - if (alignedRightOff > alignedLeftOff) { - auto curOff = alignedLeftOff; - while (curOff < alignedRightOff) { - auto idx = (curOff - bitmapAreaOffset_) / opt_.sizePerBit; - bitmap_.Set(idx); - - curOff += opt_.sizePerBit; + if (alignedRightOff >= alignedLeftOff) { + if (alignedRightOff > alignedLeftOff) { + auto curOff = alignedLeftOff; + while (curOff < alignedRightOff) { + auto idx = (curOff - bitmapAreaOffset_) / opt_.sizePerBit; + bitmap_.Set(idx); + + curOff += opt_.sizePerBit; + } } if (unalignedLeftLen != 0) { - bitmapExtent_.MarkUsed(off, unalignedLeftLen); + auto idx = (off - bitmapAreaOffset_) / opt_.sizePerBit; + bitmap_.Set(idx); + bitmapExtent_.DeAlloc(idx * opt_.sizePerBit + bitmapAreaOffset_, + off % opt_.sizePerBit); } + if (unalignedRightLen != 0) { - bitmapExtent_.MarkUsed(alignedRightOff, unalignedRightLen); + auto idx = (off - bitmapAreaOffset_) / opt_.sizePerBit; + bitmap_.Set(idx); + bitmapExtent_.DeAlloc( + off + len, + (idx + 1) * opt_.sizePerBit + bitmapAreaOffset_ - (off + len)); } } else { - bitmapExtent_.MarkUsed(off, len); + auto idx = (off - bitmapAreaOffset_) / opt_.sizePerBit; + bitmap_.Set(idx); + bitmapExtent_.DeAlloc(idx * opt_.sizePerBit + bitmapAreaOffset_, + off % opt_.sizePerBit); + bitmapExtent_.DeAlloc(off + len, (idx + 1) * opt_.sizePerBit + + bitmapAreaOffset_ - (off + len)); } } -void BitmapAllocator::Split(const uint64_t off, const uint64_t len, +void BitmapAllocator::Split(const uint64_t off, + const uint64_t len, uint64_t* offInSmallExtent, - uint64_t* lenInSmallExtent, uint64_t* offInBitmap, + uint64_t* lenInSmallExtent, + uint64_t* offInBitmap, uint64_t* lenInBitmap) const { if (off >= bitmapAreaOffset_) { *lenInSmallExtent = 0; @@ -377,5 +415,5 @@ std::ostream& operator<<(std::ostream& os, const BitmapAllocator& alloc) { return os; } -} // namespace space +} // namespace volume } // namespace curvefs diff --git a/curvefs/src/space/bitmap_allocator.h b/curvefs/src/volume/bitmap_allocator.h similarity index 61% rename from curvefs/src/space/bitmap_allocator.h rename to curvefs/src/volume/bitmap_allocator.h index f9314064d1..dbc93114b4 100644 --- a/curvefs/src/space/bitmap_allocator.h +++ b/curvefs/src/volume/bitmap_allocator.h @@ -20,49 +20,48 @@ * Author: wuhanqing */ -#ifndef CURVEFS_SRC_SPACE_BITMAP_ALLOCATOR_H_ -#define CURVEFS_SRC_SPACE_BITMAP_ALLOCATOR_H_ +#ifndef CURVEFS_SRC_VOLUME_BITMAP_ALLOCATOR_H_ +#define CURVEFS_SRC_VOLUME_BITMAP_ALLOCATOR_H_ #include -#include "curvefs/src/space/allocator.h" -#include "curvefs/src/space/common.h" -#include "curvefs/src/space/config.h" -#include "curvefs/src/space/free_extents.h" +#include +#include + +#include "curvefs/src/volume/allocator.h" +#include "curvefs/src/volume/common.h" +#include "curvefs/src/volume/free_extents.h" #include "src/common/bitmap.h" namespace curvefs { -namespace space { +namespace volume { class BitmapAllocator : public Allocator { public: explicit BitmapAllocator(const BitmapAllocatorOption& opt); - ~BitmapAllocator(); + ~BitmapAllocator() override; - uint64_t Alloc(const uint64_t size, const SpaceAllocateHint& hint, - Extents* exts) override; + uint64_t Alloc(const uint64_t size, + const AllocateHint& hint, + std::vector* exts) override; bool DeAlloc(const uint64_t off, const uint64_t len) override; - bool DeAlloc(const Extents& exts) override; + bool DeAlloc(const std::vector& exts) override; - uint64_t StartOffset() const { - return opt_.startOffset; - } + uint64_t StartOffset() const { return opt_.startOffset; } - uint64_t Total() const override { - return opt_.length; - } + uint64_t Total() const override { return opt_.length; } uint64_t AvailableSize() const override { std::lock_guard lock(mtx_); return available_; } - bool MarkUsed(const Extents& extents) override; + bool MarkUsed(const std::vector& extents) override; - bool MarkUsable(const Extents& extents) override; + bool MarkUsable(const std::vector& extents) override; static uint64_t CalcBitmapAreaLength(const BitmapAllocatorOption& opt); @@ -76,17 +75,21 @@ class BitmapAllocator : public Allocator { * @brief call real allocate functions that stored in AllocOrder one by one, * untilsatisfy size or all functions has been called */ - uint64_t AllocInternal(const AllocOrder& order, const uint64_t size, - const SpaceAllocateHint& hint, Extents* exts); + uint64_t AllocInternal(const AllocOrder& order, + const uint64_t size, + const AllocateHint& hint, + std::vector* exts); // allocate staffs - uint64_t AllocFromBitmap(const uint64_t size, const SpaceAllocateHint& hint, - Extents* exts); + uint64_t AllocFromBitmap(const uint64_t size, + const AllocateHint& hint, + std::vector* exts); uint64_t AllocFromBitmapExtent(const uint64_t size, - const SpaceAllocateHint& hint, - Extents* exts); + const AllocateHint& hint, + std::vector* exts); uint64_t AllocFromSmallExtent(const uint64_t size, - const SpaceAllocateHint& hint, Extents* exts); + const AllocateHint& hint, + std::vector* exts); // deallocate staffs void DeAllocToSmallExtent(const uint64_t off, const uint64_t len); @@ -99,9 +102,12 @@ class BitmapAllocator : public Allocator { void MarkUsedForBitmap(const uint64_t off, const uint64_t len); // split off and len by bitmapAreaOffset - void Split(const uint64_t off, const uint64_t len, - uint64_t* offInSmallExtent, uint64_t* lenInSmallExtent, - uint64_t* offInBitmap, uint64_t* lenInBitmap) const; + void Split(const uint64_t off, + const uint64_t len, + uint64_t* offInSmallExtent, + uint64_t* lenInSmallExtent, + uint64_t* offInBitmap, + uint64_t* lenInBitmap) const; private: const BitmapAllocatorOption opt_; @@ -128,7 +134,7 @@ class BitmapAllocator : public Allocator { FreeExtents smallExtent_; }; -} // namespace space +} // namespace volume } // namespace curvefs -#endif // CURVEFS_SRC_SPACE_BITMAP_ALLOCATOR_H_ +#endif // CURVEFS_SRC_VOLUME_BITMAP_ALLOCATOR_H_ diff --git a/curvefs/src/client/block_device_client.cpp b/curvefs/src/volume/block_device_client.cpp similarity index 52% rename from curvefs/src/client/block_device_client.cpp rename to curvefs/src/volume/block_device_client.cpp index b62766813d..27784d79d6 100644 --- a/curvefs/src/client/block_device_client.cpp +++ b/curvefs/src/volume/block_device_client.cpp @@ -14,24 +14,39 @@ * limitations under the License. */ - /* * Project: curve * Created Date: Thur May 27 2021 * Author: xuchaojie */ +#include "curvefs/src/volume/block_device_client.h" + +#include + +#include +#include #include #include -#include +#include "absl/cleanup/cleanup.h" +#include "curvefs/src/common/metric_utils.h" #include "src/client/service_helper.h" -#include "curvefs/src/client/block_device_client.h" +#include "src/common/concurrent/count_down_event.h" namespace curvefs { -namespace client { +namespace volume { using ::curve::client::UserInfo; +using ::curve::common::CountDownEvent; +using ::curvefs::common::LatencyUpdater; + +namespace { + +bvar::LatencyRecorder g_write_latency("block_device_write"); +bvar::LatencyRecorder g_read_latency("block_device_read"); + +} // namespace BlockDeviceClientImpl::BlockDeviceClientImpl() : fileClient_(std::make_shared()), @@ -42,52 +57,60 @@ BlockDeviceClientImpl::BlockDeviceClientImpl( : fileClient_(fileClient), fd_(-1) {} -CURVEFS_ERROR BlockDeviceClientImpl::Init( - const BlockDeviceClientOptions& options) { - if (fileClient_->Init(options.configPath) != LIBCURVE_ERROR::OK) { - return CURVEFS_ERROR::INTERNAL; +bool BlockDeviceClientImpl::Init(const BlockDeviceClientOptions& options) { + auto ret = fileClient_->Init(options.configPath); + if (ret != LIBCURVE_ERROR::OK) { + LOG(ERROR) << "Init file client error: " << ret; + return false; } - return CURVEFS_ERROR::OK; + ret = taskpool_.Start(options.threadnum); + if (ret != 0) { + LOG(ERROR) << "Start task pool failed"; + return false; + } + + return true; } void BlockDeviceClientImpl::UnInit() { + taskpool_.Stop(); fileClient_->UnInit(); } -CURVEFS_ERROR BlockDeviceClientImpl::Open(const std::string& filename, - const std::string& owner) { +bool BlockDeviceClientImpl::Open(const std::string& filename, + const std::string& owner) { UserInfo userInfo(owner); curve::client::OpenFlags flags; auto retCode = fileClient_->Open(filename, userInfo, flags); if (retCode < 0) { LOG(ERROR) << "Open file failed, filename = " << filename << ", retCode = " << retCode; - return CURVEFS_ERROR::INTERNAL; + return false; } fd_ = retCode; filename_ = filename; owner_ = owner; - return CURVEFS_ERROR::OK; + return true; } -CURVEFS_ERROR BlockDeviceClientImpl::Close() { +bool BlockDeviceClientImpl::Close() { if (fd_ < 0) { - return CURVEFS_ERROR::OK; + return true; } int retCode; if ((retCode = fileClient_->Close(fd_)) != LIBCURVE_ERROR::OK) { LOG(ERROR) << "Close file failed, retCode = " << retCode; - return CURVEFS_ERROR::INTERNAL; + return false; } fd_ = -1; - return CURVEFS_ERROR::OK; + return true; } -CURVEFS_ERROR BlockDeviceClientImpl::Stat(const std::string& filename, +bool BlockDeviceClientImpl::Stat(const std::string& filename, const std::string& owner, BlockDeviceStat* statInfo) { FileStatInfo fileStatInfo; @@ -95,26 +118,28 @@ CURVEFS_ERROR BlockDeviceClientImpl::Stat(const std::string& filename, auto retCode = fileClient_->StatFile(filename, userInfo, &fileStatInfo); if (retCode != LIBCURVE_ERROR::OK) { LOG(ERROR) << "Stat file failed, retCode = " << retCode; - return CURVEFS_ERROR::INTERNAL; + return false; } statInfo->length = fileStatInfo.length; if (!ConvertFileStatus(fileStatInfo.fileStatus, &statInfo->status)) { LOG(ERROR) << "Stat file failed, unknown file status: " << fileStatInfo.fileStatus; - return CURVEFS_ERROR::INTERNAL; + return false; } - return CURVEFS_ERROR::OK; + return true; } -CURVEFS_ERROR BlockDeviceClientImpl::Read(char* buf, - off_t offset, - size_t length) { +ssize_t BlockDeviceClientImpl::Read(char* buf, off_t offset, size_t length) { + VLOG(9) << "read request, offset: " << offset << ", length: " << length; + + LatencyUpdater updater(&g_read_latency); + if (fd_ < 0) { - return CURVEFS_ERROR::BAD_FD; + return -1; } else if (0 == length) { - return CURVEFS_ERROR::OK; + return length; } else if (IsAligned(offset, length)) { return AlignRead(buf, offset, length); } @@ -126,20 +151,68 @@ CURVEFS_ERROR BlockDeviceClientImpl::Read(char* buf, std::unique_ptr readBuffer(new (std::nothrow) char[readLength]); auto retCode = AlignRead(readBuffer.get(), readStart, readLength); - if (retCode == CURVEFS_ERROR::OK) { + if (retCode >= 0) { memcpy(buf, readBuffer.get() + (offset - readStart), length); } return retCode; } -CURVEFS_ERROR BlockDeviceClientImpl::Write(const char* buf, - off_t offset, - size_t length) { +ssize_t BlockDeviceClientImpl::Readv(const std::vector& iov) { + if (iov.size() == 1) { + VLOG(9) << "read block offset: " << iov[0].offset + << ", length: " << iov[0].length; + return Read(iov[0].data, iov[0].offset, iov[0].length); + } + + CountDownEvent counter(iov.size()); + std::atomic res(0); + + for (auto& io : iov) { + auto task = [this, &counter, &io, &res]() { + VLOG(9) << "read block offset: " << io.offset + << ", length: " << io.length; + auto nr = this->Read(io.data, io.offset, io.length); + if (nr < 0) { + LOG(ERROR) << "IO Error, offseet: " << io.offset + << ", len: " << io.length; + res.store(nr, std::memory_order_relaxed); + } else { + auto old = res.load(std::memory_order_release); + if (old < 0) { + // already error + } else { + while (!res.compare_exchange_strong( + old, old + nr, std::memory_order_relaxed)) { + if (old < 0) { + // another request is error + break; + } + } + } + } + + counter.Signal(); + }; + + taskpool_.Enqueue(std::move(task)); + } + + counter.Wait(); + return res.load(std::memory_order_relaxed); +} + +ssize_t BlockDeviceClientImpl::Write(const char* buf, + off_t offset, + size_t length) { + VLOG(9) << "write request, offset: " << offset << ", length: " << length; + + LatencyUpdater updater(&g_write_latency); + if (fd_ < 0) { - return CURVEFS_ERROR::BAD_FD; + return -1; } else if (0 == length) { - return CURVEFS_ERROR::OK; + return length; } else if (IsAligned(offset, length)) { return AlignWrite(buf, offset, length); } @@ -152,19 +225,59 @@ CURVEFS_ERROR BlockDeviceClientImpl::Write(const char* buf, auto retCode = WritePadding( writeBuffer.get(), writeStart, writeEnd, offset, length); - if (retCode == CURVEFS_ERROR::OK) { - memcpy(writeBuffer.get() + (offset - writeStart), buf, length); - retCode = AlignWrite(writeBuffer.get(), writeStart, writeLength); + if (!retCode) { + return -1; } - return retCode; + memcpy(writeBuffer.get() + (offset - writeStart), buf, length); + return AlignWrite(writeBuffer.get(), writeStart, writeLength); +} + +ssize_t BlockDeviceClientImpl::Writev(const std::vector& writes) { + if (writes.size() == 1) { + return Write(writes[0].data, writes[0].offset, writes[0].length); + } + + CountDownEvent counter(writes.size()); + std::atomic res(0); + + for (const auto& io : writes) { + auto task = [this, &counter, &io, &res]() { + auto nr = this->Write(io.data, io.offset, io.length); + if (nr < 0) { + LOG(ERROR) << "IO Error, offseet: " << io.offset + << ", len: " << io.length; + res.store(nr, std::memory_order_relaxed); + } else { + auto old = res.load(std::memory_order_release); + if (old < 0) { + // already error + } else { + while (!res.compare_exchange_strong( + old, old + io.length, std::memory_order_relaxed)) { + if (old < 0) { + // another request is error + break; + } + } + } + } + + counter.Signal(); + }; + + taskpool_.Enqueue(std::move(task)); + } + + counter.Wait(); + return res.load(std::memory_order_relaxed); } -CURVEFS_ERROR BlockDeviceClientImpl::WritePadding(char* writeBuffer, - off_t writeStart, - off_t writeEnd, - off_t offset, - size_t length) { +bool BlockDeviceClientImpl::WritePadding(char* writeBuffer, + off_t writeStart, + off_t writeEnd, + off_t offset, + size_t length) { std::vector> readvec; // Align reads off_t readEnd = 0; @@ -187,44 +300,44 @@ CURVEFS_ERROR BlockDeviceClientImpl::WritePadding(char* writeBuffer, for (const auto& item : readvec) { auto retCode = AlignRead(writeBuffer + item.first - writeStart, item.first, item.second); - if (retCode != CURVEFS_ERROR::OK) { - return retCode; + if (retCode != item.second) { + return false; } } - return CURVEFS_ERROR::OK; + return true; } -CURVEFS_ERROR BlockDeviceClientImpl::AlignRead(char* buf, - off_t offset, - size_t length) { +ssize_t BlockDeviceClientImpl::AlignRead(char* buf, + off_t offset, + size_t length) { auto ret = fileClient_->Read(fd_, buf, offset, length); if (ret < 0) { LOG(ERROR) << "Read file failed, retCode = " << ret; - return CURVEFS_ERROR::INTERNAL; + return -1; } else if (ret != length) { LOG(ERROR) << "Read file failed, expect read " << length << " bytes, actual read " << ret << " bytes"; - return CURVEFS_ERROR::INTERNAL; + return -1; } - return CURVEFS_ERROR::OK; + return length; } -CURVEFS_ERROR BlockDeviceClientImpl::AlignWrite(const char* buf, - off_t offset, - size_t length) { +ssize_t BlockDeviceClientImpl::AlignWrite(const char* buf, + off_t offset, + size_t length) { auto ret = fileClient_->Write(fd_, buf, offset, length); if (ret < 0) { LOG(ERROR) << "Write file failed, retCode = " << ret; - return CURVEFS_ERROR::INTERNAL; + return -1; } else if (ret != length) { LOG(ERROR) << "Write file failed, expect write " << length << " bytes, actual write " << ret << " bytes"; - return CURVEFS_ERROR::INTERNAL; + return -1; } - return CURVEFS_ERROR::OK; + return length; } bool BlockDeviceClientImpl::ConvertFileStatus(int fileStatus, @@ -261,5 +374,5 @@ inline Range BlockDeviceClientImpl::CalcAlignRange(off_t start, off_t end) { Align(end, IO_ALIGNED_BLOCK_SIZE)); } -} // namespace client +} // namespace volume } // namespace curvefs diff --git a/curvefs/src/client/block_device_client.h b/curvefs/src/volume/block_device_client.h similarity index 62% rename from curvefs/src/client/block_device_client.h rename to curvefs/src/volume/block_device_client.h index 105113d42b..97d8fd0391 100644 --- a/curvefs/src/client/block_device_client.h +++ b/curvefs/src/volume/block_device_client.h @@ -14,33 +14,38 @@ * limitations under the License. */ - /* * Project: curve * Created Date: Thur May 27 2021 * Author: xuchaojie */ -#ifndef CURVEFS_SRC_CLIENT_BLOCK_DEVICE_CLIENT_H_ -#define CURVEFS_SRC_CLIENT_BLOCK_DEVICE_CLIENT_H_ +#ifndef CURVEFS_SRC_VOLUME_BLOCK_DEVICE_CLIENT_H_ +#define CURVEFS_SRC_VOLUME_BLOCK_DEVICE_CLIENT_H_ #include -#include #include +#include +#include +#include "curvefs/src/volume/common.h" #include "src/client/client_common.h" #include "src/client/libcurve_file.h" -#include "curvefs/src/client/common/config.h" -#include "curvefs/src/client/error_code.h" +#include "src/common/concurrent/task_thread_pool.h" namespace curvefs { -namespace client { +namespace volume { using ::curve::client::FileClient; -using common::BlockDeviceClientOptions; using Range = std::pair; +struct BlockDeviceClientOptions { + // config path + std::string configPath; + uint32_t threadnum; +}; + enum class BlockDeviceStatus { CREATED, DELETING, @@ -57,18 +62,17 @@ struct BlockDeviceStat { class BlockDeviceClient { public: - BlockDeviceClient() {} - virtual ~BlockDeviceClient() {} + virtual ~BlockDeviceClient() = default; /** - * @brief Initailize client + * @brief Initialize client * @param[in] options the options for client * @return error code (CURVEFS_ERROR:*) */ - virtual CURVEFS_ERROR Init(const BlockDeviceClientOptions& options) = 0; + virtual bool Init(const BlockDeviceClientOptions& options) = 0; /** - * @brief Uninitailize client + * @brief Deinitialize client */ virtual void UnInit() = 0; @@ -78,14 +82,14 @@ class BlockDeviceClient { * @param[in] owner owner for filename * @return error code (CURVEFS_ERROR:*) */ - virtual CURVEFS_ERROR Open(const std::string& filename, - const std::string& owner) = 0; + virtual bool Open(const std::string& filename, + const std::string& owner) = 0; /** * @brief Close the fd which init by Open() * @return error code (CURVEFS_ERROR:*) */ - virtual CURVEFS_ERROR Close() = 0; + virtual bool Close() = 0; /** * @brief Get file status @@ -94,7 +98,7 @@ class BlockDeviceClient { * @param[out] statInfo the struct for file status * @return error code (CURVEFS_ERROR:*) */ - virtual CURVEFS_ERROR Stat(const std::string& filename, + virtual bool Stat(const std::string& filename, const std::string& owner, BlockDeviceStat* statInfo) = 0; @@ -106,7 +110,7 @@ class BlockDeviceClient { * @return error code (CURVEFS_ERROR:*) * @note the offset and length maybe not aligned */ - virtual CURVEFS_ERROR Read(char* buf, off_t offset, size_t length) = 0; + virtual ssize_t Read(char* buf, off_t offset, size_t length) = 0; /** * @brief Write to fd which init by Open() @@ -116,8 +120,11 @@ class BlockDeviceClient { * @return error code (CURVEFS_ERROR:*) * @note the offset and length maybe not aligned */ - virtual CURVEFS_ERROR Write( - const char* buf, off_t offset, size_t length) = 0; + virtual ssize_t Write(const char* buf, off_t offset, size_t length) = 0; + + virtual ssize_t Readv(const std::vector& iov) = 0; + + virtual ssize_t Writev(const std::vector& iov) = 0; }; class BlockDeviceClientImpl : public BlockDeviceClient { @@ -127,32 +134,39 @@ class BlockDeviceClientImpl : public BlockDeviceClient { explicit BlockDeviceClientImpl( const std::shared_ptr& fileClient); - ~BlockDeviceClientImpl() = default; + ~BlockDeviceClientImpl() override = default; - CURVEFS_ERROR Init(const BlockDeviceClientOptions& options) override; + bool Init(const BlockDeviceClientOptions& options) override; void UnInit() override; - CURVEFS_ERROR Open(const std::string& filename, + bool Open(const std::string& filename, const std::string& owner) override; - CURVEFS_ERROR Close() override; + bool Close() override; - CURVEFS_ERROR Stat(const std::string& filename, + bool Stat(const std::string& filename, const std::string& owner, BlockDeviceStat* statInfo) override; - CURVEFS_ERROR Read(char* buf, off_t offset, size_t length) override; + ssize_t Read(char* buf, off_t offset, size_t length) override; + + ssize_t Write(const char* buf, off_t offset, size_t length) override; - CURVEFS_ERROR Write(const char* buf, off_t offset, size_t length) override; + ssize_t Readv(const std::vector& iov) override; + + ssize_t Writev(const std::vector& writes) override; private: - CURVEFS_ERROR WritePadding(char* writeBuffer, off_t writeStart, - off_t writeEnd, off_t offset, size_t length); + bool WritePadding(char* writeBuffer, + off_t writeStart, + off_t writeEnd, + off_t offset, + size_t length); - CURVEFS_ERROR AlignRead(char* buf, off_t offset, size_t length); + ssize_t AlignRead(char* buf, off_t offset, size_t length); - CURVEFS_ERROR AlignWrite(const char* buf, off_t offset, size_t length); + ssize_t AlignWrite(const char* buf, off_t offset, size_t length); bool IsAligned(off_t offset, size_t length); @@ -170,9 +184,11 @@ class BlockDeviceClientImpl : public BlockDeviceClient { std::string owner_; std::shared_ptr fileClient_; + + curve::common::TaskThreadPool<> taskpool_; }; -} // namespace client +} // namespace volume } // namespace curvefs -#endif // CURVEFS_SRC_CLIENT_BLOCK_DEVICE_CLIENT_H_ +#endif // CURVEFS_SRC_VOLUME_BLOCK_DEVICE_CLIENT_H_ diff --git a/curvefs/src/volume/block_group_loader.cpp b/curvefs/src/volume/block_group_loader.cpp new file mode 100644 index 0000000000..8006f29c7c --- /dev/null +++ b/curvefs/src/volume/block_group_loader.cpp @@ -0,0 +1,137 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Friday Mar 04 22:59:44 CST 2022 + * Author: wuhanqing + */ + +#include "curvefs/src/volume/block_group_loader.h" + +#include + +#include +#include + +#include "absl/memory/memory.h" +#include "curvefs/proto/common.pb.h" +#include "curvefs/src/volume/allocator.h" +#include "curvefs/src/volume/block_device_client.h" +#include "curvefs/src/volume/block_group_updater.h" +#include "curvefs/src/volume/utils.h" +#include "src/common/bitmap.h" + +namespace curvefs { +namespace volume { + +using ::curve::common::BITMAP_UNIT_SIZE; +using ::curve::common::BitRange; + +BitmapRange BlockGroupBitmapLoader::CalcBitmapRange() const { + BitmapRange range; + + range.length = blockGroupSize_ / blockSize_ / BITMAP_UNIT_SIZE; + + switch (bitmapLocation_) { + case BitmapLocation::AtStart: { + range.offset = offset_; + break; + } + + case BitmapLocation::AtEnd: { + range.offset = offset_ + blockGroupSize_ - range.length; + break; + } + + default: + CHECK(false) << "not implemented"; + } + + return range; +} + +std::ostream& operator<<(std::ostream& os, BitmapRange range) { + os << "bitmap range offset: " << range.offset + << ", length: " << range.length; + + return os; +} + +bool BlockGroupBitmapLoader::Load(AllocatorAndBitmapUpdater* out) { + BitmapRange bitmapRange = CalcBitmapRange(); + std::unique_ptr data(new char[bitmapRange.length]); + + VLOG(9) << bitmapRange; + + auto err = + blockDev_->Read(data.get(), bitmapRange.offset, bitmapRange.length); + if (!err) { + LOG(ERROR) << "Read bitmap from block device failed"; + return false; + } + + AllocatorOption option = allocatorOption_; + option.bitmapAllocatorOption.startOffset = offset_; + option.bitmapAllocatorOption.length = blockGroupSize_; + // TODO(wuhanqing): load from config + option.bitmapAllocatorOption.sizePerBit = 4ull * 1024 * 1024; + option.bitmapAllocatorOption.smallAllocProportion = 0; + + auto allocator = Allocator::Create(option.type, option); + if (!allocator) { + LOG(ERROR) << "Create allocator failed"; + return false; + } + + Bitmap bitmap(bitmapRange.length * BITMAP_UNIT_SIZE, data.release()); + std::vector clearRange; + std::vector setRange; + bitmap.Divide(0, bitmapRange.length * BITMAP_UNIT_SIZE, &clearRange, + &setRange); + + // mark used + // physical offset in volume + std::vector usedExtents; + for (auto& range : setRange) { + usedExtents.emplace_back( + offset_ + range.beginIndex * blockSize_, + (range.endIndex + 1 - range.endIndex) * blockSize_); + } + + // bitmap location is also used + usedExtents.emplace_back(bitmapRange.offset, // physical offset + bitmapRange.length); // physical length + + VLOG(9) << "markused, " << usedExtents + << ", bitmap range offset: " << bitmapRange.offset + << ", len: " << bitmapRange.length; + + if (!allocator->MarkUsed(usedExtents)) { + LOG(ERROR) << "Init allocator from bitmap failed"; + return false; + } + + out->allocator = std::move(allocator); + out->bitmapUpdater = absl::make_unique( + std::move(bitmap), blockSize_, blockGroupSize_, offset_, bitmapRange, + blockDev_); + + return true; +} + +} // namespace volume +} // namespace curvefs diff --git a/curvefs/src/volume/block_group_loader.h b/curvefs/src/volume/block_group_loader.h new file mode 100644 index 0000000000..c4e769d3ba --- /dev/null +++ b/curvefs/src/volume/block_group_loader.h @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Friday Mar 04 22:59:29 CST 2022 + * Author: wuhanqing + */ + +#ifndef CURVEFS_SRC_VOLUME_BLOCK_GROUP_LOADER_H_ +#define CURVEFS_SRC_VOLUME_BLOCK_GROUP_LOADER_H_ + +#include + +#include "curvefs/proto/space.pb.h" +#include "curvefs/src/volume/allocator.h" +#include "curvefs/src/volume/block_group_updater.h" +#include "curvefs/src/volume/common.h" +#include "curvefs/src/volume/option.h" + +namespace curvefs { +namespace volume { + +using ::curvefs::common::BitmapLocation; + +class BlockDeviceClient; + +struct AllocatorAndBitmapUpdater { + uint64_t blockGroupOffset; + std::unique_ptr allocator; + std::unique_ptr bitmapUpdater; +}; + +// load bitmap for each block group +class BlockGroupBitmapLoader { + public: + BlockGroupBitmapLoader(BlockDeviceClient* client, + uint32_t blockSize, + uint64_t offset, + uint64_t blockGroupSize, + BitmapLocation location, + const AllocatorOption& option) + : blockDev_(client), + offset_(offset), + blockGroupSize_(blockGroupSize), + blockSize_(blockSize), + bitmapLocation_(location), + allocatorOption_(option) {} + + BlockGroupBitmapLoader(const BlockGroupBitmapLoader&) = delete; + BlockGroupBitmapLoader& operator=(const BlockGroupBitmapLoader&) = delete; + + /** + * @brief Create a allocator and bitmap updater that corresponding to + * current block group + * @return return true if success, otherwise, return false + */ + bool Load(AllocatorAndBitmapUpdater* out); + + private: + BitmapRange CalcBitmapRange() const; + + private: + BlockDeviceClient* blockDev_; + uint64_t offset_; + uint64_t blockGroupSize_; + uint32_t blockSize_; + BitmapLocation bitmapLocation_; + const AllocatorOption& allocatorOption_; +}; + +} // namespace volume +} // namespace curvefs + +#endif // CURVEFS_SRC_VOLUME_BLOCK_GROUP_LOADER_H_ diff --git a/curvefs/src/volume/block_group_manager.cpp b/curvefs/src/volume/block_group_manager.cpp new file mode 100644 index 0000000000..c82f4aeec6 --- /dev/null +++ b/curvefs/src/volume/block_group_manager.cpp @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Wednesday Mar 02 20:39:51 CST 2022 + * Author: wuhanqing + */ + +#include "curvefs/src/volume/block_group_manager.h" + +#include + +#include + +#include "absl/memory/memory.h" +#include "curvefs/proto/space.pb.h" + +namespace curvefs { +namespace volume { + +using ::curve::common::Bitmap; +using ::curve::common::BITMAP_UNIT_SIZE; +using ::curve::common::BitRange; +using ::curvefs::common::BitmapLocation; +using ::curvefs::mds::space::BlockGroup; +using ::curvefs::mds::space::SpaceErrCode_Name; + +BlockGroupManagerImpl::BlockGroupManagerImpl( + SpaceManager* spaceManager, + const std::shared_ptr& mdsClient, + const std::shared_ptr& blockDevice, + const BlockGroupManagerOption& managerOption, + const AllocatorOption& allocatorOption) + : spaceManager_(spaceManager), + mdsClient_(mdsClient), + blockDeviceClient_(blockDevice), + option_(managerOption), + allocatorOption_(allocatorOption) {} + +bool BlockGroupManagerImpl::AllocateBlockGroup( + std::vector* out) { + std::vector groups; + auto err = mdsClient_->AllocateVolumeBlockGroup( + option_.fsId, option_.blockGroupAllocateOnce, option_.owner, &groups); + if (err != SpaceErrCode::SpaceOk) { + LOG(ERROR) << "Allocate volume block group failed, err: " + << SpaceErrCode_Name(err); + return false; + } else if (groups.empty()) { + LOG(ERROR) + << "Allocate volume block group failed, no block group allocated"; + return false; + } + + for (auto& group : groups) { + VLOG(9) << "load group: " << group.ShortDebugString(); + AllocatorAndBitmapUpdater res; + res.blockGroupOffset = group.offset(); + BlockGroupBitmapLoader loader( + blockDeviceClient_.get(), option_.blockSize, group.offset(), + group.size(), group.bitmaplocation(), allocatorOption_); + auto ret = loader.Load(&res); + if (!ret) { + LOG(ERROR) << "Create allocator for block group failed"; + return false; + } else { + out->push_back(std::move(res)); + } + } + + return true; +} + +// TODO(wuhanqing): implement this function +bool BlockGroupManagerImpl::ReleaseAllBlockGroups() { + return true; +} + +void BlockGroupManagerImpl::AllocateBlockGroupAsync() {} + +} // namespace volume +} // namespace curvefs diff --git a/curvefs/src/volume/block_group_manager.h b/curvefs/src/volume/block_group_manager.h new file mode 100644 index 0000000000..a398a7c2c7 --- /dev/null +++ b/curvefs/src/volume/block_group_manager.h @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Wednesday Mar 02 20:31:16 CST 2022 + * Author: wuhanqing + */ + +#ifndef CURVEFS_SRC_VOLUME_BLOCK_GROUP_MANAGER_H_ +#define CURVEFS_SRC_VOLUME_BLOCK_GROUP_MANAGER_H_ + +#include +#include + +#include "curvefs/proto/common.pb.h" +#include "curvefs/src/client/rpcclient/mds_client.h" +#include "curvefs/src/volume/block_device_client.h" +#include "curvefs/src/volume/block_group_loader.h" +#include "curvefs/src/volume/common.h" +#include "curvefs/src/volume/option.h" +#include "src/common/bitmap.h" + +namespace curvefs { +namespace volume { + +using ::curvefs::client::rpcclient::MdsClient; + +class SpaceManager; + +class BlockGroupManager { + public: + virtual ~BlockGroupManager() = default; + + virtual bool AllocateBlockGroup( + std::vector* out) = 0; + + virtual bool ReleaseAllBlockGroups() = 0; +}; + +class BlockGroupManagerImpl final : public BlockGroupManager { + public: + BlockGroupManagerImpl(SpaceManager* spaceManager, + const std::shared_ptr& mdsClient, + const std::shared_ptr& blockDevice, + const BlockGroupManagerOption& managerOption, + const AllocatorOption& allocatorOption); + + bool AllocateBlockGroup(std::vector* out); + + void AcquireBlockGroup(); + + void AllocateBlockGroupAsync(); + + bool ReleaseAllBlockGroups(); + + private: + SpaceManager* spaceManager_; + std::shared_ptr mdsClient_; + std::shared_ptr blockDeviceClient_; + + BlockGroupManagerOption option_; + AllocatorOption allocatorOption_; +}; + +} // namespace volume +} // namespace curvefs + +#endif // CURVEFS_SRC_VOLUME_BLOCK_GROUP_MANAGER_H_ diff --git a/curvefs/src/volume/block_group_updater.cpp b/curvefs/src/volume/block_group_updater.cpp new file mode 100644 index 0000000000..326b8eb8a7 --- /dev/null +++ b/curvefs/src/volume/block_group_updater.cpp @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Friday Mar 04 23:05:02 CST 2022 + * Author: wuhanqing + */ + +#include "curvefs/src/volume/block_group_updater.h" + +#include + +#include "curvefs/src/volume/block_device_client.h" + +namespace curvefs { +namespace volume { + +void BlockGroupBitmapUpdater::Update(const Extent& ext, Op op) { + std::lock_guard lk(lock_); + + uint64_t startOffset = ext.offset - groupOffset_; + uint64_t endOffset = ext.offset + ext.len - groupOffset_; + uint64_t startIdx = startOffset / blockSize_; + uint64_t endIdx = endOffset / blockSize_ - 1; + + if (op == Op::Set) { + bitmap_.Set(startIdx, endIdx); + } else { + bitmap_.Clear(startIdx, endIdx); + } + + dirty_ = true; +} + +bool BlockGroupBitmapUpdater::Sync() { + std::lock_guard lk(lock_); + + if (!dirty_) { + return true; + } + + bool ret = blockDev_->Write(bitmap_.GetBitmap(), bitmapRange_.offset, + bitmapRange_.length); + + LOG_IF(ERROR, !ret) << "Sync block group bitmap failed, err: " << ret + << ", block group offset: " << groupOffset_; + + dirty_ = !ret; + + return ret; +} + +} // namespace volume +} // namespace curvefs diff --git a/curvefs/src/volume/block_group_updater.h b/curvefs/src/volume/block_group_updater.h new file mode 100644 index 0000000000..c8adc48af1 --- /dev/null +++ b/curvefs/src/volume/block_group_updater.h @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Friday Mar 04 23:04:55 CST 2022 + * Author: wuhanqing + */ + +#ifndef CURVEFS_SRC_VOLUME_BLOCK_GROUP_UPDATER_H_ +#define CURVEFS_SRC_VOLUME_BLOCK_GROUP_UPDATER_H_ + +#include +#include + +#include "curvefs/src/volume/common.h" +#include "src/common/bitmap.h" + +namespace curvefs { +namespace volume { + +using ::curve::common::Bitmap; + +class BlockDeviceClient; + +struct BitmapRange { + uint64_t offset; + uint64_t length; +}; + +// bitmap updater for each block group +class BlockGroupBitmapUpdater { + public: + BlockGroupBitmapUpdater(Bitmap bitmap, + uint32_t blockSize, + uint32_t groupSize, + uint64_t groupOffset, + const BitmapRange& range, + BlockDeviceClient* blockDev) + : dirty_(false), + bitmap_(std::move(bitmap)), + blockSize_(blockSize), + groupSize_(groupSize), + groupOffset_(groupOffset), + bitmapRange_(range), + blockDev_(blockDev) {} + + enum Op { Set, Clear }; + + /** + * @brief Update corresponding bit that covered by exts + * @param op set or clear bit + */ + void Update(const Extent& ext, Op op); + + /** + * @brief Sync bitmap to backend storage if dirty + * @return return true if success, otherwise, return false + */ + bool Sync(); + + private: + std::mutex lock_; + bool dirty_; + Bitmap bitmap_; + uint32_t blockSize_; + uint32_t groupSize_; + uint64_t groupOffset_; + BitmapRange bitmapRange_; + BlockDeviceClient* blockDev_; +}; + +} // namespace volume +} // namespace curvefs + +#endif // CURVEFS_SRC_VOLUME_BLOCK_GROUP_UPDATER_H_ diff --git a/curvefs/src/volume/common.h b/curvefs/src/volume/common.h new file mode 100644 index 0000000000..cedae440d6 --- /dev/null +++ b/curvefs/src/volume/common.h @@ -0,0 +1,107 @@ +/* + * Copyright (c) 2021 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Project: curve + * File Created: Fri Jul 16 21:22:40 CST 2021 + * Author: wuhanqing + */ + +#ifndef CURVEFS_SRC_VOLUME_COMMON_H_ +#define CURVEFS_SRC_VOLUME_COMMON_H_ + +#include + +#include +#include +#include + +namespace curvefs { +namespace volume { + +constexpr uint64_t kKiB = 1024ULL; +constexpr uint64_t kMiB = 1024ULL * kKiB; +constexpr uint64_t kGiB = 1024ULL * kMiB; +constexpr uint64_t kTiB = 1024ULL * kGiB; + +struct Extent { + uint64_t offset = 0; + uint64_t len = 0; + + Extent() = default; + Extent(uint64_t o, uint64_t l) : offset(o), len(l) {} + + bool operator==(const Extent& e) const { + return offset == e.offset && len == e.len; + } +}; + +enum class AllocateType { + None, + Small, + Big, +}; + +struct AllocateHint { + enum { INVALID_OFFSET = std::numeric_limits::max() }; + + AllocateType allocType = AllocateType::None; + uint64_t leftOffset = INVALID_OFFSET; + uint64_t rightOffset = INVALID_OFFSET; + + bool HasLeftHint() const { return leftOffset != INVALID_OFFSET; } + + bool HasRightHint() const { return rightOffset != INVALID_OFFSET; } + + bool HasHint() const { return HasRightHint() || HasLeftHint(); } +}; + +struct SpaceStat { + uint64_t total = 0; + uint64_t available = 0; + uint32_t blockSize = 0; + + SpaceStat() = default; + SpaceStat(uint64_t total, uint64_t available, uint64_t blockSize) + : total(total), available(available), blockSize(blockSize) {} +}; + +struct WritePart { + off_t offset = 0; + size_t length = 0; + const char* data = nullptr; + + WritePart() = default; + + WritePart(off_t offset, size_t length, const char* data) + : offset(offset), length(length), data(data) {} +}; + +struct ReadPart { + off_t offset = 0; + size_t length = 0; + char* data = nullptr; + + ReadPart() = default; + + ReadPart(off_t offset, size_t length, char* data) + : offset(offset), length(length), data(data) {} +}; + +} // namespace volume +} // namespace curvefs + +#endif // CURVEFS_SRC_VOLUME_COMMON_H_ diff --git a/curvefs/src/space/free_extents.cpp b/curvefs/src/volume/free_extents.cpp similarity index 85% rename from curvefs/src/space/free_extents.cpp rename to curvefs/src/volume/free_extents.cpp index c3fdddae88..b94a08c6d6 100644 --- a/curvefs/src/space/free_extents.cpp +++ b/curvefs/src/volume/free_extents.cpp @@ -20,7 +20,7 @@ * Author: wuhanqing */ -#include "curvefs/src/space/free_extents.h" +#include "curvefs/src/volume/free_extents.h" #include @@ -29,18 +29,20 @@ #include #include -#include "curvefs/src/space/common.h" -#include "curvefs/src/common/fast_align.h" -#include "curvefs/src/space/utils.h" +#include "curvefs/src/volume/common.h" +#include "curvefs/src/volume/utils.h" +#include "src/common/fast_align.h" namespace curvefs { -namespace space { +namespace volume { -using curvefs::common::align_up; -using curvefs::common::align_down; +using curve::common::align_down; +using curve::common::align_up; FreeExtents::FreeExtents(const uint64_t off, const uint64_t len) - : maxLength_(len), + : startOffset_(off), + length_(len), + maxLength_(len), maxExtentSize_(0), available_(len), extents_(), @@ -51,24 +53,37 @@ FreeExtents::FreeExtents(const uint64_t off, const uint64_t len) } FreeExtents::FreeExtents(const uint64_t maxExtentSize) - : maxLength_(0), + : startOffset_(0), + length_(0), + maxLength_(0), + maxExtentSize_(maxExtentSize), + available_(0), + extents_(), + blocks_() {} + +FreeExtents::FreeExtents(const uint64_t maxExtentSize, + const uint64_t off, + const uint64_t len) + : startOffset_(off), + length_(len), + maxLength_(0), maxExtentSize_(maxExtentSize), available_(0), extents_(), blocks_() {} uint64_t FreeExtents::AllocInternal(const uint64_t size, - const SpaceAllocateHint& hint, - Extents* exts) { + const AllocateHint& hint, + std::vector* exts) { if (available_ == 0) { return 0; } uint64_t need = size; - ExtentMap::const_iterator iter; + std::map::const_iterator iter; // 1. find extents that satisfy hint.leftOffset - if (hint.leftOffset != SpaceAllocateHint::INVALID_OFFSET) { + if (hint.leftOffset != AllocateHint::INVALID_OFFSET) { iter = extents_.lower_bound(hint.leftOffset); if (iter != extents_.end() && iter->first == hint.leftOffset) { @@ -95,7 +110,7 @@ uint64_t FreeExtents::AllocInternal(const uint64_t size, } // 2. find extents that satisfy hint.rightOffset - if (hint.rightOffset != SpaceAllocateHint::INVALID_OFFSET && + if (hint.rightOffset != AllocateHint::INVALID_OFFSET && hint.rightOffset >= need) { iter = extents_.lower_bound(hint.rightOffset - need); if (iter != extents_.end() && @@ -170,7 +185,7 @@ uint64_t FreeExtents::AllocInternal(const uint64_t size, return size - need; } -uint64_t FreeExtents::AvailableBlocks(ExtentMap* blocks) { +uint64_t FreeExtents::AvailableBlocks(std::map* blocks) { uint64_t size = 0; // TODO(wuhanqing): move this calc to DeAlloc @@ -187,18 +202,21 @@ std::ostream& operator<<(std::ostream& os, const FreeExtents& e) { os << "avail: " << e.available_ << ", extents: "; for (auto& ee : e.extents_) { - os << PExtent(ee.first, ee.second) << " "; + os << Extent{ee.first, ee.second} << " "; } os << ", blocks: "; for (auto& ee : e.blocks_) { - os << PExtent(ee.first, ee.second) << " "; + os << Extent{ee.first, ee.second} << " "; } return os; } void FreeExtents::DeAllocInternal(const uint64_t off, const uint64_t len) { + assert((length_ == 0) || + (off >= startOffset_ && off + len <= startOffset_ + length_)); + if (available_ == 0) { // FIXME: off/len may need recycle to blocks extents_.emplace(off, len); @@ -265,6 +283,9 @@ void FreeExtents::DeAllocInternal(const uint64_t off, const uint64_t len) { } void FreeExtents::MarkUsedInternal(const uint64_t off, const uint64_t len) { + assert((length_ != 0) || + (off >= startOffset_ && off + len <= startOffset_ + length_)); + CHECK(available_ >= len) << "MarkUsedInternal: [off: " << off << " ~ len: " << len << "] " << *this; @@ -298,5 +319,5 @@ void FreeExtents::MarkUsedInternal(const uint64_t off, const uint64_t len) { } } -} // namespace space +} // namespace volume } // namespace curvefs diff --git a/curvefs/src/space/free_extents.h b/curvefs/src/volume/free_extents.h similarity index 67% rename from curvefs/src/space/free_extents.h rename to curvefs/src/volume/free_extents.h index b63453644e..e4fb71ad0f 100644 --- a/curvefs/src/space/free_extents.h +++ b/curvefs/src/volume/free_extents.h @@ -20,15 +20,18 @@ * Author: wuhanqing */ -#ifndef CURVEFS_SRC_SPACE_FREE_EXTENTS_H_ -#define CURVEFS_SRC_SPACE_FREE_EXTENTS_H_ +#ifndef CURVEFS_SRC_VOLUME_FREE_EXTENTS_H_ +#define CURVEFS_SRC_VOLUME_FREE_EXTENTS_H_ #include +#include +#include +#include -#include "curvefs/src/space/common.h" +#include "curvefs/src/volume/common.h" namespace curvefs { -namespace space { +namespace volume { // Two roles: // 1. As a small space allocator, the initial state maintains a space. @@ -38,7 +41,8 @@ namespace space { // Alloc is used for allocate space // DeAlloc has two purpose: // recycle space -// store smaller space from BitmapAllocator that doesn't satisfy `maxExtentSize` // NOLINT +// store smaller space from BitmapAllocator that doesn't satisfy +// `maxExtentSize` class FreeExtents { public: // Role 1 @@ -47,6 +51,10 @@ class FreeExtents { // Role 2 explicit FreeExtents(const uint64_t maxExtentSize); + FreeExtents(const uint64_t maxExtentSize, + const uint64_t off, + const uint64_t len); + FreeExtents(const FreeExtents&) = delete; FreeExtents& operator=(const FreeExtents&) = delete; @@ -57,14 +65,18 @@ class FreeExtents { void DeAlloc(const uint64_t off, const uint64_t len) { DeAllocInternal(off, len); available_ += len; + assert(length_ == 0 || available_ <= length_); } /** * @brief Allocate space */ - uint64_t Alloc(const uint64_t size, const SpaceAllocateHint& hint, - Extents* exts) { + uint64_t Alloc(const uint64_t size, + const AllocateHint& hint, + std::vector* exts) { auto alloc = AllocInternal(size, hint, exts); + assert(length_ == 0 || available_ <= length_); + available_ -= alloc; return alloc; @@ -76,45 +88,45 @@ class FreeExtents { void MarkUsed(const uint64_t off, const uint64_t len) { MarkUsedInternal(off, len); available_ -= len; + assert(length_ == 0 || available_ <= length_); } - uint64_t AvailableSize() const { - return available_; - } + uint64_t AvailableSize() const { return available_; } /** * @brief Get current available extents */ - const ExtentMap& AvailableExtents() const { - return extents_; - } + std::map AvailableExtents() const { return extents_; } /** * @brief Get currnet available blocks * @return Total size of available blocks */ - uint64_t AvailableBlocks(ExtentMap* blocks); + uint64_t AvailableBlocks(std::map* blocks); friend std::ostream& operator<<(std::ostream& os, const FreeExtents& e); private: - uint64_t AllocInternal(uint64_t size, const SpaceAllocateHint& hint, - Extents* exts); + uint64_t AllocInternal(uint64_t size, + const AllocateHint& hint, + std::vector* exts); void DeAllocInternal(const uint64_t off, const uint64_t len); void MarkUsedInternal(const uint64_t off, const uint64_t len); private: + const uint64_t startOffset_; + const uint64_t length_; const uint64_t maxLength_; const uint64_t maxExtentSize_; uint64_t available_; - ExtentMap extents_; - ExtentMap blocks_; + std::map extents_; + std::map blocks_; }; -} // namespace space +} // namespace volume } // namespace curvefs -#endif // CURVEFS_SRC_SPACE_FREE_EXTENTS_H_ +#endif // CURVEFS_SRC_VOLUME_FREE_EXTENTS_H_ diff --git a/curvefs/src/space/config.h b/curvefs/src/volume/option.h similarity index 73% rename from curvefs/src/space/config.h rename to curvefs/src/volume/option.h index c08dca54ff..01d6754253 100644 --- a/curvefs/src/space/config.h +++ b/curvefs/src/volume/option.h @@ -20,14 +20,16 @@ * Author: wuhanqing */ -#ifndef CURVEFS_SRC_SPACE_CONFIG_H_ -#define CURVEFS_SRC_SPACE_CONFIG_H_ +#ifndef CURVEFS_SRC_VOLUME_OPTION_H_ +#define CURVEFS_SRC_VOLUME_OPTION_H_ #include #include +#include "curvefs/proto/common.pb.h" + namespace curvefs { -namespace space { +namespace volume { struct BitmapAllocatorOption { uint64_t startOffset; @@ -36,26 +38,25 @@ struct BitmapAllocatorOption { double smallAllocProportion; }; -struct AllocatorOption { - BitmapAllocatorOption bitmapAllocatorOption; -}; - -struct MetaServerClientOption { - std::string addr; +struct BlockGroupManagerOption { + uint32_t fsId; + uint32_t blockGroupAllocateOnce; + uint32_t blockSize; + uint64_t blockGroupSize; + std::string owner; }; -struct ReloaderOption { - MetaServerClientOption metaServerOption; +struct AllocatorOption { + std::string type; + BitmapAllocatorOption bitmapAllocatorOption; }; struct SpaceManagerOption { - uint32_t blockSize; - std::string allocatorType; - ReloaderOption reloaderOption; AllocatorOption allocatorOption; + BlockGroupManagerOption blockGroupManagerOption; }; -} // namespace space +} // namespace volume } // namespace curvefs -#endif // CURVEFS_SRC_SPACE_CONFIG_H_ +#endif // CURVEFS_SRC_VOLUME_OPTION_H_ diff --git a/curvefs/src/volume/space_manager.cpp b/curvefs/src/volume/space_manager.cpp new file mode 100644 index 0000000000..0fd1dd4b53 --- /dev/null +++ b/curvefs/src/volume/space_manager.cpp @@ -0,0 +1,259 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Wednesday Mar 02 19:56:31 CST 2022 + * Author: wuhanqing + */ + +#include "curvefs/src/volume/space_manager.h" + +#include +#include + +#include +#include + +#include "absl/cleanup/cleanup.h" +#include "curvefs/src/volume/utils.h" +#include "src/common/fast_align.h" + +namespace curvefs { +namespace volume { + +using ::curve::common::align_down; +using ::curve::common::ReadLockGuard; +using ::curve::common::WriteLockGuard; + +SpaceManagerImpl::SpaceManagerImpl( + const SpaceManagerOption& option, + const std::shared_ptr& mdsClient, + const std::shared_ptr& blockDev) + : totalBytes_(0), + availableBytes_(0), + blockSize_(option.blockGroupManagerOption.blockGroupSize), + blockGroupSize_(option.blockGroupManagerOption.blockGroupSize), + blockGroupManager_( + new BlockGroupManagerImpl(this, + mdsClient, + blockDev, + option.blockGroupManagerOption, + option.allocatorOption)), + allocating_(false) {} + +bool SpaceManagerImpl::Alloc(uint32_t size, + const AllocateHint& hint, + std::vector* extents) { + VLOG(9) << "Alloc size: " << size << ", hint: " << hint; + + butil::Timer timer; + timer.start(); + + if (availableBytes_.load(std::memory_order_acquire) < size) { + auto ret = AllocateBlockGroup(); + if (!ret) { + LOG(ERROR) << "Allocate block group error"; + metric_.errorCount << 1; + return false; + } + } + + int64_t left = size; + while (left > 0) { + auto allocated = AllocInternal(left, hint, extents); + if (allocated < left) { + auto ret = AllocateBlockGroup(); + if (!ret) { + LOG(ERROR) << "Allocate block group error"; + metric_.errorCount << 1; + return false; + } + } + left -= allocated; + } + + auto ret = UpdateBitmap(*extents); + if (!ret) { + LOG(ERROR) << "Update bitmap failed"; + metric_.errorCount << 1; + return false; + } + + timer.stop(); + metric_.allocLatency << timer.u_elapsed(); + metric_.allocSize << size; + + VLOG(9) << "Alloc success, " << *extents; + + return true; +} + +bool SpaceManagerImpl::DeAlloc(const std::vector& extents) { + // TODO(wuhanqing): fix + (void)extents; + return true; +} + +std::map>::iterator +SpaceManagerImpl::FindAllocator(const AllocateHint& hint) { + if (hint.HasRightHint()) { + auto it = allocators_.lower_bound( + align_down(hint.rightOffset, blockGroupSize_)); + if (it != allocators_.end()) { + return it; + } + } + + if (hint.HasLeftHint()) { + auto it = allocators_.lower_bound( + align_down(hint.leftOffset - 1, blockGroupSize_)); + if (it != allocators_.end()) { + return it; + } + } + + static thread_local unsigned int seed = time(nullptr); + auto it = allocators_.begin(); + std::advance(it, rand_r(&seed) % allocators_.size()); + + return it; +} + +int64_t SpaceManagerImpl::AllocInternal(int64_t size, + const AllocateHint& hint, + std::vector* exts) { + ReadLockGuard lk(allocatorsLock_); + int64_t left = size; + auto it = FindAllocator(hint); + const auto beginIt = it; + + do { + left -= it->second->Alloc(left, hint, exts); + if (left <= 0) { + break; + } + + ++it; + if (it == allocators_.end()) { + it = allocators_.begin(); + } + + if (it == beginIt) { + break; + } + } while (true); + + return size - left; +} + +bool SpaceManagerImpl::UpdateBitmap(const std::vector& exts) { + ReadLockGuard lk(updatersLock_); + + std::unordered_set dirty; + for (auto& ext : exts) { + BlockGroupBitmapUpdater* updater = FindBitmapUpdater(ext); + updater->Update(ext, BlockGroupBitmapUpdater::Set); + } + + for (auto d : dirty) { + d->Sync(); + } + + return true; +} + +BlockGroupBitmapUpdater* SpaceManagerImpl::FindBitmapUpdater( + const Extent& ext) { + uint64_t blockGroupOffset = align_down(ext.offset, blockGroupSize_); + VLOG(9) << "block group offset: " << blockGroupOffset << ", ext: " << ext + << ", group block size: " << blockGroupSize_; + auto it = bitmapUpdaters_.find(blockGroupOffset); + CHECK(it != bitmapUpdaters_.end()) + << "block group offset: " << blockGroupOffset; + + return it->second.get(); +} + +bool SpaceManagerImpl::Shutdown() { + WriteLockGuard allocLk(allocatorsLock_); + WriteLockGuard updaterLk(updatersLock_); + + // sync all bitmap updater + bool ret = false; + for (auto& updater : bitmapUpdaters_) { + ret = updater.second->Sync(); + if (!ret) { + LOG(ERROR) << "Sync bitmap updater failed"; + return false; + } + } + + // release all block group + ret = blockGroupManager_->ReleaseAllBlockGroups(); + LOG_IF(ERROR, !ret) << "Release all block groups failed"; + return ret; +} + +bool SpaceManagerImpl::AllocateBlockGroup() { + { + std::unique_lock lk(mtx_); + if (allocating_) { + cond_.wait(lk); + return true; + } + } + + { + std::lock_guard lk(mtx_); + allocating_ = true; + } + + auto wakeup = absl::MakeCleanup([this]() { + std::unique_lock lk(mtx_); + allocating_ = false; + cond_.notify_all(); + }); + + std::vector out; + auto ret = blockGroupManager_->AllocateBlockGroup(&out); + if (!ret) { + LOG(ERROR) << "Allocate block group failed"; + return false; + } + + uint64_t available = 0; + uint64_t total = 0; + WriteLockGuard allocLk(allocatorsLock_); + WriteLockGuard updaterLk(updatersLock_); + for (auto& d : out) { + VLOG(9) << "add allocator, offset: " << d.blockGroupOffset + << ", available: " << d.allocator->AvailableSize() + << ", total: " << d.allocator->Total(); + available += d.allocator->AvailableSize(); + total += d.allocator->Total(); + allocators_.emplace(d.blockGroupOffset, std::move(d.allocator)); + bitmapUpdaters_.emplace(d.blockGroupOffset, std::move(d.bitmapUpdater)); + } + + availableBytes_.fetch_add(available, std::memory_order_release); + totalBytes_.fetch_add(total, std::memory_order_release); + + return true; +} + +} // namespace volume +} // namespace curvefs diff --git a/curvefs/src/volume/space_manager.h b/curvefs/src/volume/space_manager.h new file mode 100644 index 0000000000..cb7d136aa3 --- /dev/null +++ b/curvefs/src/volume/space_manager.h @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Wednesday Mar 02 19:56:23 CST 2022 + * Author: wuhanqing + */ + +#ifndef CURVEFS_SRC_VOLUME_SPACE_MANAGER_H_ +#define CURVEFS_SRC_VOLUME_SPACE_MANAGER_H_ + +#include +#include +#include + +#include "curvefs/proto/common.pb.h" +#include "curvefs/src/client/rpcclient/mds_client.h" +#include "curvefs/src/volume/allocator.h" +#include "curvefs/src/volume/block_device_client.h" +#include "curvefs/src/volume/block_group_manager.h" +#include "curvefs/src/volume/common.h" +#include "src/common/concurrent/rw_lock.h" + +namespace curvefs { +namespace volume { + +using ::curvefs::client::rpcclient::MdsClient; + +class SpaceManager { + public: + virtual ~SpaceManager() = default; + + virtual bool Alloc(uint32_t size, + const AllocateHint& hint, + std::vector* extents) = 0; + + virtual bool DeAlloc(const std::vector& extents) = 0; + + virtual bool Shutdown() = 0; +}; + +class SpaceManagerImpl final : public SpaceManager { + public: + SpaceManagerImpl( + const SpaceManagerOption& option, + const std::shared_ptr& mdsClient, + const std::shared_ptr& blockDeviceClient); + + SpaceManagerImpl(const SpaceManagerImpl&) = delete; + SpaceManagerImpl& operator=(const SpaceManagerImpl&) = delete; + + bool Alloc(uint32_t size, + const AllocateHint& hint, + std::vector* extents) override; + + bool DeAlloc(const std::vector& extents) override; + + /** + * @brief Shutdown space manager, release all blockgroups' rights + */ + bool Shutdown() override; + + private: + int64_t AllocInternal(int64_t size, + const AllocateHint& hint, + std::vector* exts); + + std::map>::iterator FindAllocator( + const AllocateHint& hint); + + /** + * @brief Find corresponding bitmap updater by extent + */ + BlockGroupBitmapUpdater* FindBitmapUpdater(const Extent& ext); + + bool UpdateBitmap(const std::vector& exts); + + private: + bool AllocateBlockGroup(); + + bool AcquireBlockGroup(uint64_t blockGroupOffset); + + private: + curve::common::RWLock allocatorsLock_; + std::map> allocators_; + + curve::common::RWLock updatersLock_; + std::map> + bitmapUpdaters_; + + std::atomic totalBytes_; + std::atomic availableBytes_; + + // from fileinfo + uint32_t blockSize_; + uint64_t blockGroupSize_; + + std::unique_ptr blockGroupManager_; + + bool allocating_; + std::mutex mtx_; + std::condition_variable cond_; + + private: + struct Metric { + bvar::LatencyRecorder allocLatency; + bvar::LatencyRecorder allocSize; + bvar::Adder errorCount; + + Metric() + : allocLatency("space_alloc_latency"), + allocSize("space_alloc_size"), + errorCount("space_alloc_error") {} + }; + + Metric metric_; +}; + +} // namespace volume +} // namespace curvefs + +#endif // CURVEFS_SRC_VOLUME_SPACE_MANAGER_H_ diff --git a/curvefs/src/space/utils.cpp b/curvefs/src/volume/utils.cpp similarity index 53% rename from curvefs/src/space/utils.cpp rename to curvefs/src/volume/utils.cpp index 1402d771fe..6b94aa6f39 100644 --- a/curvefs/src/space/utils.cpp +++ b/curvefs/src/volume/utils.cpp @@ -20,64 +20,65 @@ * Author: wuhanqing */ -#include "curvefs/src/space/utils.h" +#include "curvefs/src/volume/utils.h" #include namespace curvefs { -namespace space { +namespace volume { -SpaceAllocateHint ConvertToSpaceAllocateHint(const AllocateHint& hint) { - SpaceAllocateHint spaceHint; - - if (hint.has_alloctype()) { - spaceHint.allocType = hint.alloctype(); - } +std::ostream& operator<<(std::ostream& os, const Extent& e) { + os << "[off: " << e.offset << " ~ len: " << e.len << "]"; + return os; +} - if (hint.has_leftoffset()) { - spaceHint.leftOffset = hint.leftoffset(); - } +std::ostream& operator<<(std::ostream& os, const std::vector& es) { + std::ostringstream oss; - if (hint.has_rightoffset()) { - spaceHint.rightOffset = hint.rightoffset(); + for (const auto& e : es) { + oss << e << " "; } - return spaceHint; + os << oss.str(); + return os; } -// TODO(wuhanqing): sort and merge before convert to proto ? -ProtoExtents ConvertToProtoExtents(const Extents& exts) { - ProtoExtents result; +std::ostream& operator<<(std::ostream& os, const AllocateHint& hint) { + std::ostringstream oss; - for (const auto& e : exts) { - auto* p = result.Add(); - p->set_offset(e.offset); - p->set_length(e.len); - } + oss << "[type: " << hint.allocType; - return result; -} + if (hint.HasLeftHint()) { + oss << ", left hint: " << hint.leftOffset; + } + if (hint.HasRightHint()) { + oss << ", right hint: " << hint.rightOffset; + } -std::ostream& operator<<(std::ostream& os, const Extent& e) { - os << "[off: " << e.offset() << " ~ len: " << e.length() << "]"; - return os; -} + oss << "]"; -std::ostream& operator<<(std::ostream& os, const PExtent& e) { - os << "[off: " << e.offset << " ~ len: " << e.len << "]"; + os << oss.str(); return os; } -std::ostream& operator<<(std::ostream& os, const Extents& es) { - std::ostringstream oss; - - for (const auto& e : es) { - oss << e << " "; +std::ostream& operator<<(std::ostream& os, AllocateType type) { + switch (type) { + case AllocateType::None: + os << "none"; + break; + case AllocateType::Big: + os << "big"; + break; + case AllocateType::Small: + os << "small"; + break; + default: + os << "unknown"; + break; } - os << oss.str(); return os; } -} // namespace space +} // namespace volume } // namespace curvefs diff --git a/curvefs/src/space/utils.h b/curvefs/src/volume/utils.h similarity index 63% rename from curvefs/src/space/utils.h rename to curvefs/src/volume/utils.h index 952bc39a25..1dc8a6424a 100644 --- a/curvefs/src/space/utils.h +++ b/curvefs/src/volume/utils.h @@ -20,28 +20,26 @@ * Author: wuhanqing */ -#ifndef CURVEFS_SRC_SPACE_UTILS_H_ -#define CURVEFS_SRC_SPACE_UTILS_H_ +#ifndef CURVEFS_SRC_VOLUME_UTILS_H_ +#define CURVEFS_SRC_VOLUME_UTILS_H_ #include +#include -#include "curvefs/proto/space.pb.h" -#include "curvefs/src/space/common.h" +#include "curvefs/src/volume/common.h" namespace curvefs { -namespace space { - -SpaceAllocateHint ConvertToSpaceAllocateHint(const AllocateHint& hint); - -ProtoExtents ConvertToProtoExtents(const Extents& exts); +namespace volume { std::ostream& operator<<(std::ostream& os, const Extent& e); -std::ostream& operator<<(std::ostream& os, const PExtent& e); +std::ostream& operator<<(std::ostream& os, const std::vector& es); + +std::ostream& operator<<(std::ostream& os, const AllocateHint& hint); -std::ostream& operator<<(std::ostream& os, const Extents& es); +std::ostream& operator<<(std::ostream& os, AllocateType type); -} // namespace space +} // namespace volume } // namespace curvefs -#endif // CURVEFS_SRC_SPACE_UTILS_H_ +#endif // CURVEFS_SRC_VOLUME_UTILS_H_ diff --git a/curvefs/test/client/BUILD b/curvefs/test/client/BUILD index 813b7ea931..3584db8b62 100644 --- a/curvefs/test/client/BUILD +++ b/curvefs/test/client/BUILD @@ -16,22 +16,6 @@ load("//:copts.bzl", "CURVE_TEST_COPTS") -cc_test( - name = "block_device_client_test", - srcs = glob([ - "block_device_client_test.cpp" - ]), - copts = CURVE_TEST_COPTS, - defines = ["UNIT_TEST"], - visibility = ["//visibility:public"], - deps = [ - "//include/client:include_client", - "//curvefs/src/client:fuse_client_lib", - "//test/client/mock:client_mock_lib", - "@com_google_googletest//:gtest", - "@com_google_googletest//:gtest_main", - ], -) cc_library( name = "client_s3_test_lib", srcs = glob([ @@ -52,7 +36,7 @@ cc_library( linkopts = ["-L/usr/local/lib/x86_64-linux-gnu"], ) -cc_binary( +cc_test( name = "client_s3_test", srcs = glob([ "main.cpp", @@ -97,6 +81,7 @@ cc_test( "//curvefs/proto:metaserver_cc_proto", "//curvefs/proto:mds_cc_proto", "//curvefs/proto:space_cc_proto", + "//curvefs/test/volume/mock", ], linkopts = ["-lfuse3", "-L/usr/local/lib/x86_64-linux-gnu"], @@ -117,3 +102,16 @@ cc_test( ], copts = CURVE_TEST_COPTS, ) + +cc_library( + name = "mock", + hdrs = glob([ + "mock_*.h" + ]), + deps = [ + "//curvefs/src/client:fuse_client_lib", + "@com_google_googletest//:gtest", + ], + copts = CURVE_TEST_COPTS, + visibility = ["//visibility:public"], +) diff --git a/curvefs/test/client/base_client_test.cpp b/curvefs/test/client/base_client_test.cpp deleted file mode 100644 index 47dba13ba8..0000000000 --- a/curvefs/test/client/base_client_test.cpp +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Project: curve - * Created Date: Thur Jun 15 2021 - * Author: lixiaocui - */ - -#include -#include -#include - -#include "curvefs/src/client/base_client.h" -#include "curvefs/test/client/mock_mds_service.h" -#include "curvefs/test/client/mock_metaserver_service.h" -#include "curvefs/test/client/mock_spacealloc_service.h" - -namespace curvefs { -namespace client { -using ::testing::_; -using ::testing::DoAll; -using ::testing::Invoke; -using ::testing::Return; -using ::testing::SetArgPointee; - - -template -void RpcService(google::protobuf::RpcController *cntl_base, - const RpcRequestType *request, RpcResponseType *response, - google::protobuf::Closure *done) { - if (RpcFailed) { - brpc::Controller *cntl = static_cast(cntl_base); - cntl->SetFailed(112, "Not connected to"); - } - done->Run(); -} - -class BaseClientTest : public testing::Test { - protected: - void SetUp() override { - ASSERT_EQ(0, server_.AddService(&mockSpaceAllocService_, - brpc::SERVER_DOESNT_OWN_SERVICE)); - ASSERT_EQ(0, server_.Start(addr_.c_str(), nullptr)); - } - - void TearDown() override { - server_.Stop(0); - server_.Join(); - } - - protected: - MockSpaceAllocService mockSpaceAllocService_; - SpaceBaseClient spacebasecli_; - - std::string addr_ = "127.0.0.1:5700"; - brpc::Server server_; -}; - -TEST_F(BaseClientTest, test_AllocExtents) { - uint32_t fsId = 1; - ExtentAllocInfo info; - info.lOffset = 0; - info.len = 1024; - info.leftHintAvailable = true; - info.pOffsetLeft = 0; - info.rightHintAvailable = true; - info.pOffsetRight = 0; - curvefs::space::AllocateSpaceResponse resp; - brpc::Controller cntl; - cntl.set_timeout_ms(1000); - brpc::Channel ch; - ASSERT_EQ(0, ch.Init(addr_.c_str(), nullptr)); - - curvefs::space::AllocateSpaceResponse response; - auto extent = response.add_extents(); - extent->set_offset(0); - extent->set_length(1024); - response.set_status(curvefs::space::SpaceStatusCode::SPACE_OK); - EXPECT_CALL(mockSpaceAllocService_, AllocateSpace(_, _, _, _)) - .WillOnce(DoAll( - SetArgPointee<2>(response), - Invoke(RpcService))); - - spacebasecli_.AllocExtents(fsId, info, curvefs::space::AllocateType::NONE, - &resp, &cntl, &ch); - ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); - ASSERT_TRUE( - google::protobuf::util::MessageDifferencer::Equals(resp, response)) - << "resp:\n" - << resp.ShortDebugString() << "response:\n" - << response.ShortDebugString(); -} - -TEST_F(BaseClientTest, test_AllocExtents1) { - uint32_t fsId = 1; - ExtentAllocInfo info; - info.lOffset = 0; - info.len = 1024; - info.leftHintAvailable = false; - info.pOffsetLeft = 0; - info.rightHintAvailable = false; - info.pOffsetRight = 0; - curvefs::space::AllocateSpaceResponse resp; - brpc::Controller cntl; - cntl.set_timeout_ms(1000); - brpc::Channel ch; - ASSERT_EQ(0, ch.Init(addr_.c_str(), nullptr)); - - curvefs::space::AllocateSpaceResponse response; - auto extent = response.add_extents(); - extent->set_offset(0); - extent->set_length(1024); - response.set_status(curvefs::space::SpaceStatusCode::SPACE_OK); - EXPECT_CALL(mockSpaceAllocService_, AllocateSpace(_, _, _, _)) - .WillOnce(DoAll( - SetArgPointee<2>(response), - Invoke(RpcService))); - - spacebasecli_.AllocExtents(fsId, info, curvefs::space::AllocateType::NONE, - &resp, &cntl, &ch); - ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); - ASSERT_TRUE( - google::protobuf::util::MessageDifferencer::Equals(resp, response)) - << "resp:\n" - << resp.ShortDebugString() << "response:\n" - << response.ShortDebugString(); -} - -TEST_F(BaseClientTest, test_DeAllocExtents) { - uint32_t fsId = 1; - Extent extent; - extent.set_offset(0); - extent.set_length(1024); - std::list allocatedExtents; - allocatedExtents.push_back(extent); - curvefs::space::DeallocateSpaceResponse resp; - brpc::Controller cntl; - cntl.set_timeout_ms(1000); - brpc::Channel ch; - ASSERT_EQ(0, ch.Init(addr_.c_str(), nullptr)); - - curvefs::space::DeallocateSpaceResponse response; - response.set_status(curvefs::space::SpaceStatusCode::SPACE_OK); - EXPECT_CALL(mockSpaceAllocService_, DeallocateSpace(_, _, _, _)) - .WillOnce(DoAll( - SetArgPointee<2>(response), - Invoke( - RpcService))); - - spacebasecli_.DeAllocExtents(fsId, allocatedExtents, &resp, &cntl, &ch); - ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); - ASSERT_TRUE( - google::protobuf::util::MessageDifferencer::Equals(resp, response)) - << "resp:\n" - << resp.ShortDebugString() << "response:\n" - << response.ShortDebugString(); -} - -} // namespace client -} // namespace curvefs diff --git a/curvefs/test/client/client_s3_adaptor_test.cpp b/curvefs/test/client/client_s3_adaptor_test.cpp index 2385efd812..508d32682a 100644 --- a/curvefs/test/client/client_s3_adaptor_test.cpp +++ b/curvefs/test/client/client_s3_adaptor_test.cpp @@ -30,7 +30,6 @@ #include "curvefs/test/client/mock_inode_cache_manager.h" #include "curvefs/test/client/mock_mds_client.h" #include "curvefs/test/client/mock_metaserver_service.h" -#include "curvefs/test/client/mock_spacealloc_service.h" #include "src/common/curve_define.h" namespace curvefs { @@ -1147,14 +1146,7 @@ TEST_F(ClientS3AdaptorTest, test_truncate_big1) { uint64_t len = 1 * 1024 * 1024; CURVEFS_ERROR ret; uint64_t chunkIndex = offset / s3ClientAdaptor_->GetChunkSize(); - ::curvefs::space::AllocateS3ChunkResponse response; - response.set_status(::curvefs::space::SpaceStatusCode::SPACE_OK); - response.set_chunkid(25); - // mock - /*EXPECT_CALL(mockSpaceAllocService_, AllocateS3Chunk(_, _, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(response), - Invoke(S3RpcService)));*/ + auto inodeWrapper = std::make_shared(inode, nullptr); EXPECT_CALL(mockInodeManager_, GetInode(_, _)) .WillOnce( @@ -1199,9 +1191,6 @@ TEST_F(ClientS3AdaptorTest, test_truncate_big2) { memset(buf, 'a', len); uint64_t chunkIndex = offset / s3ClientAdaptor_->GetChunkSize(); - ::curvefs::space::AllocateS3ChunkResponse response; - response.set_status(::curvefs::space::SpaceStatusCode::SPACE_OK); - response.set_chunkid(25); auto inodeWrapper = std::make_shared(inode, nullptr); EXPECT_CALL(mockInodeManager_, GetInode(_, _)) .WillOnce( @@ -1244,9 +1233,6 @@ TEST_F(ClientS3AdaptorTest, test_truncate_big3) { uint64_t len = 1 * 1024 * 1024; CURVEFS_ERROR ret; uint64_t chunkIndex = offset / s3ClientAdaptor_->GetChunkSize(); - ::curvefs::space::AllocateS3ChunkResponse response; - response.set_status(::curvefs::space::SpaceStatusCode::SPACE_OK); - response.set_chunkid(25); // mock auto inodeWrapper = std::make_shared(inode, nullptr); EXPECT_CALL(mockInodeManager_, GetInode(_, _)) diff --git a/curvefs/test/client/common/BUILD b/curvefs/test/client/common/BUILD index 1616e69559..b24cd9fb59 100644 --- a/curvefs/test/client/common/BUILD +++ b/curvefs/test/client/common/BUILD @@ -26,5 +26,6 @@ cc_test( "//curvefs/src/common:curvefs_common", "@com_google_googletest//:gtest", "@com_google_googletest//:gtest_main", + "//curvefs/src/client/common", ], ) \ No newline at end of file diff --git a/curvefs/test/client/common/test_config.cpp b/curvefs/test/client/common/test_config.cpp new file mode 100644 index 0000000000..9f7d297190 --- /dev/null +++ b/curvefs/test/client/common/test_config.cpp @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Thursday Mar 24 16:14:48 CST 2022 + * Author: wuhanqing + */ + +#include + +#include "curvefs/src/client/common/config.h" + +namespace curvefs { +namespace client { +namespace common { + +using curve::common::Configuration; + +void InitVolumeOption(Configuration *conf, VolumeOption *volumeOpt); + +TEST(TestInitVolumeOption, Common) { + Configuration conf; + VolumeOption volopt; + + conf.SetUInt64Value("volume.bigFileSize", 1ULL * 1024 * 1024); + conf.SetUInt64Value("volume.volBlockSize", 4096); + conf.SetUInt64Value("volume.fsBlockSize", 4096); + conf.SetUInt32Value("volume.blockgroup.allocate_once", 4); + conf.SetStringValue("volume.allocator.type", "bitmap"); + conf.SetUInt64Value("volume.bitmapallocator.size_per_bit", + 4ULL * 1024 * 1024); + conf.SetDoubleValue("volume.bitmapallocator.small_alloc_proportion", 0.0); + + ASSERT_NO_FATAL_FAILURE({ InitVolumeOption(&conf, &volopt); }); +} + +TEST(TestInitVolumeOption, TypeError) { + Configuration conf; + VolumeOption volopt; + + conf.SetUInt64Value("volume.bigFileSize", 1ULL * 1024 * 1024); + conf.SetUInt64Value("volume.volBlockSize", 4096); + conf.SetUInt64Value("volume.fsBlockSize", 4096); + conf.SetUInt32Value("volume.blockgroup.allocate_once", 4); + conf.SetStringValue("volume.allocator.type", "xxx"); + conf.SetUInt64Value("volume.bitmapallocator.size_per_bit", + 4ULL * 1024 * 1024); + conf.SetDoubleValue("volume.bitmapallocator.small_alloc_proportion", 0.0); + + ASSERT_DEATH({ InitVolumeOption(&conf, &volopt); }, ""); +} + +} // namespace common +} // namespace client +} // namespace curvefs diff --git a/curvefs/test/client/fs_cache_manager_test.cpp b/curvefs/test/client/fs_cache_manager_test.cpp index 2ac49e6a88..93cb10c26f 100644 --- a/curvefs/test/client/fs_cache_manager_test.cpp +++ b/curvefs/test/client/fs_cache_manager_test.cpp @@ -33,7 +33,7 @@ namespace client { using ::testing::_; using ::testing::Invoke; -TEST(FsCacheManagerTest, test_read_lru_cache_size) { +TEST(FsCacheManagerTest, DISABLED_test_read_lru_cache_size) { uint64_t smallDataCacheByte = 128ull * 1024; // 128KiB uint64_t dataCacheByte = 4ull * 1024 * 1024; // 4MiB uint64_t maxReadCacheByte = 16ull * 1024 * 1024; // 16MiB diff --git a/curvefs/test/client/mock_extent_manager.h b/curvefs/test/client/mock_extent_manager.h deleted file mode 100644 index 840e692de3..0000000000 --- a/curvefs/test/client/mock_extent_manager.h +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Project: curve - * Created Date: Thur May 27 2021 - * Author: xuchaojie - */ - - -#ifndef CURVEFS_TEST_CLIENT_MOCK_EXTENT_MANAGER_H_ -#define CURVEFS_TEST_CLIENT_MOCK_EXTENT_MANAGER_H_ - -#include -#include - -#include - -#include "curvefs/src/client/extent_manager.h" - -using ::testing::Return; -using ::testing::_; - -namespace curvefs { -namespace client { - -class MockExtentManager : public ExtentManager { - public: - MockExtentManager() {} - ~MockExtentManager() {} - - MOCK_METHOD1(Init, CURVEFS_ERROR(const ExtentManagerOption &options)); - - MOCK_METHOD4(GetToAllocExtents, CURVEFS_ERROR( - const VolumeExtentList &extents, - uint64_t offset, - uint64_t len, - std::list *toAllocExtents)); - - MOCK_METHOD3(MergeAllocedExtents, CURVEFS_ERROR( - const std::list &toAllocExtents, - const std::list &allocatedExtents, - VolumeExtentList *extents)); - - MOCK_METHOD3(MarkExtentsWritten, CURVEFS_ERROR( - uint64_t offset, uint64_t len, - VolumeExtentList *extents)); - - MOCK_METHOD4(DivideExtents, CURVEFS_ERROR( - const VolumeExtentList &extents, - uint64_t offset, - uint64_t len, - std::list *pExtents)); -}; - -} // namespace client -} // namespace curvefs - -#endif // CURVEFS_TEST_CLIENT_MOCK_EXTENT_MANAGER_H_ diff --git a/curvefs/test/client/mock_mds_client.h b/curvefs/test/client/mock_mds_client.h index 5e2b1d909e..fef5815ce0 100644 --- a/curvefs/test/client/mock_mds_client.h +++ b/curvefs/test/client/mock_mds_client.h @@ -86,6 +86,24 @@ class MockMdsClient : public MdsClient { MOCK_METHOD2(ListPartition, bool(uint32_t fsID, std::vector* partitionInfos)); + + MOCK_METHOD4(AllocateVolumeBlockGroup, + SpaceErrCode(uint32_t, + uint32_t, + const std::string&, + std::vector*)); + + MOCK_METHOD4(AcquireVolumeBlockGroup, + SpaceErrCode(uint32_t, + uint64_t, + const std::string&, + curvefs::mds::space::BlockGroup*)); + + MOCK_METHOD3( + ReleaseVolumeBlockGroup, + SpaceErrCode(uint32_t, + const std::string&, + const std::vector&)); }; } // namespace rpcclient diff --git a/curvefs/test/client/mock_space_client.h b/curvefs/test/client/mock_space_client.h deleted file mode 100644 index bfe7d7826d..0000000000 --- a/curvefs/test/client/mock_space_client.h +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (c) 2020 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Project: curve - * Created Date: Thur May 27 2021 - * Author: xuchaojie - */ - -#ifndef CURVEFS_TEST_CLIENT_MOCK_SPACE_CLIENT_H_ -#define CURVEFS_TEST_CLIENT_MOCK_SPACE_CLIENT_H_ - - -#include -#include - -#include - -#include "curvefs/src/client/space_client.h" - - -namespace curvefs { -namespace client { - -class MockSpaceClient : public SpaceClient { - public: - MockSpaceClient() {} - ~MockSpaceClient() {} - - MOCK_METHOD2(Init, CURVEFS_ERROR(const SpaceAllocServerOption &spaceopt, - SpaceBaseClient *baseclient)); - - MOCK_METHOD4(AllocExtents, CURVEFS_ERROR(uint32_t fsId, - const std::list &toAllocExtents, - AllocateType type, - std::list *allocatedExtents)); - - MOCK_METHOD2(DeAllocExtents, CURVEFS_ERROR(uint32_t fsId, - std::list allocatedExtents)); -}; - - -} // namespace client -} // namespace curvefs - -#endif // CURVEFS_TEST_CLIENT_MOCK_SPACE_CLIENT_H_ diff --git a/curvefs/test/client/mock_spacealloc_base_client.h b/curvefs/test/client/mock_spacealloc_base_client.h deleted file mode 100644 index 29b4507975..0000000000 --- a/curvefs/test/client/mock_spacealloc_base_client.h +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Project: curve - * Created Date: Thur Jun 17 2021 - * Author: lixiaocui - */ - - -#ifndef CURVEFS_TEST_CLIENT_MOCK_SPACEALLOC_BASE_CLIENT_H_ -#define CURVEFS_TEST_CLIENT_MOCK_SPACEALLOC_BASE_CLIENT_H_ - -#include -#include -#include "curvefs/src/client/base_client.h" - -namespace curvefs { -namespace client { - -class MockSpaceBaseClient : public SpaceBaseClient { - public: - MockSpaceBaseClient() : SpaceBaseClient() {} - ~MockSpaceBaseClient() = default; - - MOCK_METHOD6(AllocExtents, - void(uint32_t fsId, const ExtentAllocInfo &toAllocExtent, - curvefs::space::AllocateType type, - AllocateSpaceResponse *response, brpc::Controller *cntl, - brpc::Channel *channel)); - - MOCK_METHOD5(DeAllocExtents, - void(uint32_t fsId, const std::list &allocatedExtents, - DeallocateSpaceResponse *response, brpc::Controller *cntl, - brpc::Channel *channel)); -}; - - -} // namespace client -} // namespace curvefs - -#endif // CURVEFS_TEST_CLIENT_MOCK_SPACEALLOC_BASE_CLIENT_H_ diff --git a/curvefs/test/client/mock_spacealloc_service.h b/curvefs/test/client/mock_spacealloc_service.h deleted file mode 100644 index 43e01ad85a..0000000000 --- a/curvefs/test/client/mock_spacealloc_service.h +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Project: curve - * Created Date: Thur Jun 17 2021 - * Author: lixiaocui - */ - - -#ifndef CURVEFS_TEST_CLIENT_MOCK_SPACEALLOC_SERVICE_H_ -#define CURVEFS_TEST_CLIENT_MOCK_SPACEALLOC_SERVICE_H_ - -#include -#include "curvefs/proto/space.pb.h" - -namespace curvefs { -namespace client { - -class MockSpaceAllocService : public curvefs::space::SpaceAllocService { - public: - MockSpaceAllocService() : SpaceAllocService() {} - ~MockSpaceAllocService() = default; - - MOCK_METHOD4(AllocateSpace, - void(::google::protobuf::RpcController *controller, - const ::curvefs::space::AllocateSpaceRequest *request, - ::curvefs::space::AllocateSpaceResponse *response, - ::google::protobuf::Closure *done)); - MOCK_METHOD4(DeallocateSpace, - void(::google::protobuf::RpcController *controller, - const ::curvefs::space::DeallocateSpaceRequest *request, - ::curvefs::space::DeallocateSpaceResponse *response, - ::google::protobuf::Closure *done)); - MOCK_METHOD4(AllocateS3Chunk, void( - ::google::protobuf::RpcController *controller, - const ::curvefs::space::AllocateS3ChunkRequest *request, - ::curvefs::space::AllocateS3ChunkResponse *response, - ::google::protobuf::Closure *done)); -}; -} // namespace client -} // namespace curvefs - -#endif // CURVEFS_TEST_CLIENT_MOCK_SPACEALLOC_SERVICE_H_ diff --git a/curvefs/test/client/mock_volume_storage.h b/curvefs/test/client/mock_volume_storage.h new file mode 100644 index 0000000000..eb5ca6a4db --- /dev/null +++ b/curvefs/test/client/mock_volume_storage.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Thursday Mar 24 09:53:32 CST 2022 + * Author: wuhanqing + */ + +#ifndef CURVEFS_TEST_CLIENT_MOCK_VOLUME_STORAGE_H_ +#define CURVEFS_TEST_CLIENT_MOCK_VOLUME_STORAGE_H_ + +#include + +#include "curvefs/src/client/volume/volume_storage.h" + +namespace curvefs { +namespace client { + +class MockVolumeStorage : public VolumeStorage { + public: + MOCK_METHOD4(Read, ssize_t(uint64_t, off_t, size_t, char*)); + MOCK_METHOD4(Write, ssize_t(uint64_t, off_t, size_t, const char*)); + MOCK_METHOD1(Flush, bool(uint64_t)); + MOCK_METHOD0(Shutdown, bool()); +}; + +} // namespace client +} // namespace curvefs + +#endif // CURVEFS_TEST_CLIENT_MOCK_VOLUME_STORAGE_H_ diff --git a/curvefs/test/client/rpcclient/BUILD b/curvefs/test/client/rpcclient/BUILD index d795982cea..51e580cf46 100644 --- a/curvefs/test/client/rpcclient/BUILD +++ b/curvefs/test/client/rpcclient/BUILD @@ -29,6 +29,7 @@ cc_test( "//curvefs/src/client/rpcclient:rpcclient", "//curvefs/proto:curvefs_topology_cc_proto", "//curvefs/src/common:curvefs_common", + "//curvefs/test/utils:curvefs_test_utils", ], visibility = ["//visibility:public"], ) diff --git a/curvefs/test/client/rpcclient/base_client_test.cpp b/curvefs/test/client/rpcclient/base_client_test.cpp index 8dd7d98fc4..de3dfac7ff 100644 --- a/curvefs/test/client/rpcclient/base_client_test.cpp +++ b/curvefs/test/client/rpcclient/base_client_test.cpp @@ -27,8 +27,8 @@ #include "curvefs/src/client/rpcclient/base_client.h" #include "curvefs/test/client/rpcclient/mock_mds_service.h" #include "curvefs/test/client/rpcclient/mock_metaserver_service.h" -#include "curvefs/test/client/rpcclient/mock_spacealloc_service.h" #include "curvefs/test/client/rpcclient/mock_topology_service.h" +#include "curvefs/test/client/rpcclient/mock_space_service.h" namespace curvefs { namespace client { @@ -38,6 +38,7 @@ using ::testing::DoAll; using ::testing::Invoke; using ::testing::Return; using ::testing::SetArgPointee; +using ::curvefs::mds::space::MockSpaceService; template @@ -58,8 +59,6 @@ class BaseClientTest : public testing::Test { brpc::SERVER_DOESNT_OWN_SERVICE)); ASSERT_EQ(0, server_.AddService(&mockMdsService_, brpc::SERVER_DOESNT_OWN_SERVICE)); - ASSERT_EQ(0, server_.AddService(&mockSpaceAllocService_, - brpc::SERVER_DOESNT_OWN_SERVICE)); ASSERT_EQ(0, server_.AddService(&mockTopologyService_, brpc::SERVER_DOESNT_OWN_SERVICE)); ASSERT_EQ(0, server_.Start(addr_.c_str(), nullptr)); @@ -74,7 +73,6 @@ class BaseClientTest : public testing::Test { MockMetaServerService mockMetaServerService_; MockMdsService mockMdsService_; MockTopologyService mockTopologyService_; - MockSpaceAllocService mockSpaceAllocService_; MDSBaseClient mdsbasecli_; // TODO(lixiaocui): add base client for curve block storage // SpaceBaseClient spacebasecli_; @@ -156,6 +154,8 @@ TEST_F(BaseClientTest, test_GetFsInfo_by_fsName) { vresp->set_volumename("test1"); vresp->set_user("test"); vresp->set_password("test"); + vresp->set_blockgroupsize(128ULL * 1024 * 1024); + vresp->set_bitmaplocation(curvefs::common::BitmapLocation::AtStart); auto detail = new curvefs::mds::FsDetail(); detail->set_allocated_volume(vresp); fsinfo->set_allocated_detail(detail); @@ -202,6 +202,8 @@ TEST_F(BaseClientTest, test_GetFsInfo_by_fsId) { vresp->set_volumename("test1"); vresp->set_user("test"); vresp->set_password("test"); + vresp->set_blockgroupsize(128ULL * 1024 * 1024); + vresp->set_bitmaplocation(curvefs::common::BitmapLocation::AtStart); auto detail = new curvefs::mds::FsDetail(); detail->set_allocated_volume(vresp); fsinfo->set_allocated_detail(detail); diff --git a/curvefs/test/client/rpcclient/mds_client_test.cpp b/curvefs/test/client/rpcclient/mds_client_test.cpp index 1fbc23702a..5d2e6e1404 100644 --- a/curvefs/test/client/rpcclient/mds_client_test.cpp +++ b/curvefs/test/client/rpcclient/mds_client_test.cpp @@ -29,6 +29,7 @@ #include "curvefs/test/client/rpcclient/mock_mds_service.h" #include "src/client/mds_client.h" #include "curvefs/proto/topology.pb.h" +#include "curvefs/test/utils/protobuf_message_utils.h" namespace curvefs { namespace client { @@ -615,6 +616,134 @@ TEST_F(MdsClientImplTest, GetCopysetOfPartition) { ASSERT_FALSE(mdsclient_.GetCopysetOfPartitions(partitionIDList, &out)); } +TEST_F(MdsClientImplTest, TestAllocateVolumeBlockGroup) { + // rpc error + { + EXPECT_CALL(mockmdsbasecli_, AllocateVolumeBlockGroup(_, _, _, _, _, _)) + .WillRepeatedly(Invoke( + [](uint32_t, uint32_t, const std::string &, + AllocateBlockGroupResponse *, brpc::Controller *cntl, + brpc::Channel *) { cntl->SetFailed(ENOENT, "Not Found"); })); + + std::vector groups; + + ASSERT_EQ(SpaceErrCode::SpaceErrUnknown, + mdsclient_.AllocateVolumeBlockGroup(1, 1, "hello", &groups)); + } + + // response error + { + AllocateBlockGroupResponse response; + response.set_status(SpaceErrCode::SpaceErrNoSpace); + + EXPECT_CALL(mockmdsbasecli_, AllocateVolumeBlockGroup(_, _, _, _, _, _)) + .WillOnce(SetArgPointee<3>(response)); + + std::vector groups; + + ASSERT_EQ(SpaceErrCode::SpaceErrNoSpace, + mdsclient_.AllocateVolumeBlockGroup(1, 1, "hello", &groups)); + } + + // no block groups + { + AllocateBlockGroupResponse response; + response.set_status(SpaceErrCode::SpaceOk); + + EXPECT_CALL(mockmdsbasecli_, AllocateVolumeBlockGroup(_, _, _, _, _, _)) + .WillOnce(SetArgPointee<3>(response)); + + std::vector groups; + + ASSERT_EQ(SpaceErrCode::SpaceErrNoSpace, + mdsclient_.AllocateVolumeBlockGroup(1, 1, "hello", &groups)); + } + + // success + { + AllocateBlockGroupResponse response; + response.set_status(SpaceErrCode::SpaceOk); + + auto blockgroup = curvefs::test::GenerateAnDefaultInitializedMessage( + "curvefs.mds.space.BlockGroup"); + *response.add_blockgroups() = + static_cast(*blockgroup); + + EXPECT_CALL(mockmdsbasecli_, AllocateVolumeBlockGroup(_, _, _, _, _, _)) + .WillOnce(SetArgPointee<3>(response)); + + std::vector groups; + + ASSERT_EQ(SpaceErrCode::SpaceOk, + mdsclient_.AllocateVolumeBlockGroup(1, 1, "hello", &groups)); + ASSERT_EQ(1, groups.size()); + } +} + +TEST_F(MdsClientImplTest, TestAcquireVolumeBlockGroup) { + // rpc error + { + EXPECT_CALL(mockmdsbasecli_, AcquireVolumeBlockGroup(_, _, _, _, _, _)) + .WillRepeatedly(Invoke( + [](uint32_t, uint64_t, const std::string &, + AcquireBlockGroupResponse *, brpc::Controller *cntl, + brpc::Channel *) { cntl->SetFailed(ENOENT, "Not Found"); })); + + curvefs::mds::space::BlockGroup blockGroup; + + ASSERT_EQ( + SpaceErrCode::SpaceErrUnknown, + mdsclient_.AcquireVolumeBlockGroup(1, 1, "hello", &blockGroup)); + } + + // response error + { + AcquireBlockGroupResponse response; + response.set_status(SpaceErrCode::SpaceErrNotFound); + EXPECT_CALL(mockmdsbasecli_, AcquireVolumeBlockGroup(_, _, _, _, _, _)) + .WillOnce(SetArgPointee<3>(response)); + + curvefs::mds::space::BlockGroup blockGroup; + + ASSERT_EQ( + SpaceErrCode::SpaceErrNotFound, + mdsclient_.AcquireVolumeBlockGroup(1, 1, "hello", &blockGroup)); + } +} + +TEST_F(MdsClientImplTest, TestReleaseVolumeBlockGroup) { + // rpc error + { + EXPECT_CALL(mockmdsbasecli_, ReleaseVolumeBlockGroup(_, _, _, _, _, _)) + .WillRepeatedly(Invoke( + [](uint32_t, const std::string &, + const std::vector &, + ReleaseBlockGroupResponse *, brpc::Controller *cntl, + brpc::Channel *) { cntl->SetFailed(ENOENT, "Not Found"); })); + + std::vector blockGroups; + + ASSERT_EQ(SpaceErrCode::SpaceErrUnknown, + mdsclient_.ReleaseVolumeBlockGroup(1, "hello", blockGroups)); + } + + { + for (auto err : + {SpaceErrCode::SpaceOk, SpaceErrCode::SpaceErrNoSpace}) { + ReleaseBlockGroupResponse response; + response.set_status(err); + EXPECT_CALL(mockmdsbasecli_, + ReleaseVolumeBlockGroup(_, _, _, _, _, _)) + .WillOnce(SetArgPointee<3>(response)); + + std::vector blockGroups; + + ASSERT_EQ(err, mdsclient_.ReleaseVolumeBlockGroup(1, "hello", + blockGroups)); + } + } +} + } // namespace rpcclient } // namespace client } // namespace curvefs diff --git a/curvefs/test/client/rpcclient/mock_mds_base_client.h b/curvefs/test/client/rpcclient/mock_mds_base_client.h index dce04e9959..22b2f9cf3c 100644 --- a/curvefs/test/client/rpcclient/mock_mds_base_client.h +++ b/curvefs/test/client/rpcclient/mock_mds_base_client.h @@ -83,6 +83,31 @@ class MockMDSBaseClient : public MDSBaseClient { MOCK_METHOD4(ListPartition, void(uint32_t fsID, ListPartitionResponse *response, brpc::Controller *cntl, brpc::Channel *channel)); + + MOCK_METHOD6(AllocateVolumeBlockGroup, + void(uint32_t fsId, + uint32_t count, + const std::string &owner, + AllocateBlockGroupResponse *response, + brpc::Controller *cntl, + brpc::Channel *channel)); + + MOCK_METHOD6(AcquireVolumeBlockGroup, + void(uint32_t fsId, + uint64_t blockGroupOffset, + const std::string &owner, + AcquireBlockGroupResponse *response, + brpc::Controller *cntl, + brpc::Channel *channel)); + + MOCK_METHOD6( + ReleaseVolumeBlockGroup, + void(uint32_t fsId, + const std::string &owner, + const std::vector &blockGroups, + ReleaseBlockGroupResponse *response, + brpc::Controller *cntl, + brpc::Channel *channel)); }; } // namespace rpcclient } // namespace client diff --git a/curvefs/test/client/rpcclient/mock_mds_client.h b/curvefs/test/client/rpcclient/mock_mds_client.h index 4cc110a334..0d5fda5ed7 100644 --- a/curvefs/test/client/rpcclient/mock_mds_client.h +++ b/curvefs/test/client/rpcclient/mock_mds_client.h @@ -86,6 +86,24 @@ class MockMdsClient : public MdsClient { MOCK_METHOD2(ListPartition, bool(uint32_t fsID, std::vector* partitionInfos)); + + MOCK_METHOD4(AllocateVolumeBlockGroup, + SpaceErrCode(uint32_t, + uint32_t, + const std::string&, + std::vector*)); + + MOCK_METHOD4(AcquireVolumeBlockGroup, + SpaceErrCode(uint32_t, + uint64_t, + const std::string&, + curvefs::mds::space::BlockGroup*)); + + MOCK_METHOD3( + ReleaseVolumeBlockGroup, + SpaceErrCode(uint32_t, + const std::string&, + const std::vector&)); }; } // namespace rpcclient } // namespace client diff --git a/curvefs/test/client/rpcclient/mock_space_service.h b/curvefs/test/client/rpcclient/mock_space_service.h new file mode 100644 index 0000000000..fcf6909a6e --- /dev/null +++ b/curvefs/test/client/rpcclient/mock_space_service.h @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Wednesday Mar 23 19:20:53 CST 2022 + * Author: wuhanqing + */ + +#ifndef CURVEFS_TEST_CLIENT_RPCCLIENT_MOCK_SPACE_SERVICE_H_ +#define CURVEFS_TEST_CLIENT_RPCCLIENT_MOCK_SPACE_SERVICE_H_ + +#include + +#include "curvefs/proto/space.pb.h" + +namespace curvefs { +namespace mds { +namespace space { + +class MockSpaceService : public SpaceService { + public: + MOCK_METHOD4(AllocateBlockGroup, + void(google::protobuf::RpcController* controller, + const AllocateBlockGroupRequest* request, + AllocateBlockGroupResponse* response, + google::protobuf::Closure* done)); + + MOCK_METHOD4(AcquireBlockGroup, + void(google::protobuf::RpcController* controller, + const AcquireBlockGroupRequest* request, + AcquireBlockGroupResponse* response, + google::protobuf::Closure* done)); + + MOCK_METHOD4(ReleaseBlockGroup, + void(google::protobuf::RpcController* controller, + const ReleaseBlockGroupRequest* request, + ReleaseBlockGroupResponse* response, + google::protobuf::Closure* done)); + + MOCK_METHOD4(StatSpace, + void(google::protobuf::RpcController* controller, + const StatSpaceRequest* request, + StatSpaceResponse* response, + google::protobuf::Closure* done)); + + MOCK_METHOD4(UpdateUsage, + void(google::protobuf::RpcController* controller, + const UpdateUsageRequest* request, + UpdateUsageResponse* response, + google::protobuf::Closure* done)); +}; + +} // namespace space +} // namespace mds +} // namespace curvefs + +#endif // CURVEFS_TEST_CLIENT_RPCCLIENT_MOCK_SPACE_SERVICE_H_ diff --git a/curvefs/test/client/rpcclient/mock_spacealloc_service.h b/curvefs/test/client/rpcclient/mock_spacealloc_service.h deleted file mode 100644 index 90bcc4eda9..0000000000 --- a/curvefs/test/client/rpcclient/mock_spacealloc_service.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Project: curve - * Created Date: Thur Jun 17 2021 - * Author: lixiaocui - */ - - -#ifndef CURVEFS_TEST_CLIENT_RPCCLIENT_MOCK_SPACEALLOC_SERVICE_H_ -#define CURVEFS_TEST_CLIENT_RPCCLIENT_MOCK_SPACEALLOC_SERVICE_H_ - -#include -#include "curvefs/proto/space.pb.h" - -namespace curvefs { -namespace client { -namespace rpcclient { -class MockSpaceAllocService : public curvefs::space::SpaceAllocService { - public: - MockSpaceAllocService() : SpaceAllocService() {} - ~MockSpaceAllocService() = default; - - MOCK_METHOD4(AllocateSpace, - void(::google::protobuf::RpcController *controller, - const ::curvefs::space::AllocateSpaceRequest *request, - ::curvefs::space::AllocateSpaceResponse *response, - ::google::protobuf::Closure *done)); - MOCK_METHOD4(DeallocateSpace, - void(::google::protobuf::RpcController *controller, - const ::curvefs::space::DeallocateSpaceRequest *request, - ::curvefs::space::DeallocateSpaceResponse *response, - ::google::protobuf::Closure *done)); - MOCK_METHOD4(AllocateS3Chunk, - void(::google::protobuf::RpcController *controller, - const ::curvefs::space::AllocateS3ChunkRequest *request, - ::curvefs::space::AllocateS3ChunkResponse *response, - ::google::protobuf::Closure *done)); -}; -} // namespace rpcclient -} // namespace client -} // namespace curvefs - -#endif // CURVEFS_TEST_CLIENT_RPCCLIENT_MOCK_SPACEALLOC_SERVICE_H_ diff --git a/curvefs/test/client/space_client_test.cpp b/curvefs/test/client/space_client_test.cpp deleted file mode 100644 index ffeacc5399..0000000000 --- a/curvefs/test/client/space_client_test.cpp +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -/* - * Project: curve - * Created Date: Thur Jun 17 2021 - * Author: lixiaocui - */ - -#include -#include -#include - -#include -#include - -#include "curvefs/src/client/space_client.h" -#include "curvefs/test/client/mock_spacealloc_base_client.h" -#include "curvefs/test/client/mock_spacealloc_service.h" - -namespace curvefs { -namespace client { -using ::testing::_; -using ::testing::DoAll; -using ::testing::Invoke; -using ::testing::Return; -using ::testing::SetArgPointee; - -class SpaceAllocServerClientImplTest : public testing::Test { - protected: - void SetUp() override { - SpaceAllocServerOption opt; - opt.spaceaddr = addr_; - opt.rpcTimeoutMs = 1000; - - ASSERT_EQ(CURVEFS_ERROR::OK, - spaceclient_.Init(opt, &mockspacebasecli_)); - - ASSERT_EQ(0, server_.AddService(&mocspaceallocService_, - brpc::SERVER_DOESNT_OWN_SERVICE)); - ASSERT_EQ(0, server_.Start(addr_.c_str(), nullptr)); - } - - void TearDown() override { - server_.Stop(0); - server_.Join(); - } - - protected: - SpaceAllocServerClientImpl spaceclient_; - MockSpaceBaseClient mockspacebasecli_; - - MockSpaceAllocService mocspaceallocService_; - std::string addr_ = "127.0.0.1:5703"; - brpc::Server server_; -}; - -TEST_F(SpaceAllocServerClientImplTest, test_AllocExtents) { - uint32_t fsid = 1; - std::list toallocatedExtents; - ExtentAllocInfo info; - info.lOffset = 0; - info.len = 1024; - info.leftHintAvailable = true; - info.pOffsetLeft = 0; - info.rightHintAvailable = true; - info.pOffsetRight = 0; - toallocatedExtents.push_back(info); - info.lOffset = 1024; - toallocatedExtents.push_back(info); - std::list out; - - curvefs::space::AllocateSpaceResponse response1; - response1.set_status(curvefs::space::SpaceStatusCode::SPACE_OK); - auto extent1 = response1.add_extents(); - extent1->set_offset(0); - extent1->set_length(1024); - curvefs::space::AllocateSpaceResponse response2; - response2.set_status(curvefs::space::SpaceStatusCode::SPACE_OK); - auto extent2 = response2.add_extents(); - extent2->set_offset(1024); - extent2->set_length(1024); - EXPECT_CALL(mockspacebasecli_, AllocExtents(_, _, _, _, _, _)) - .Times(2) - .WillOnce(SetArgPointee<3>(response1)) - .WillOnce(SetArgPointee<3>(response2)); - ASSERT_EQ(CURVEFS_ERROR::OK, spaceclient_.AllocExtents( - fsid, toallocatedExtents, - curvefs::space::AllocateType::NONE, &out)); - ASSERT_EQ(out.size(), 2); -} - -TEST_F(SpaceAllocServerClientImplTest, test_DeAllocExtents) { - uint32_t fsid = 1; - std::list allocatedExtents; - Extent extent; - extent.set_offset(0); - extent.set_length(1024); - allocatedExtents.push_back(extent); - extent.set_offset(1024); - allocatedExtents.push_back(extent); - - curvefs::space::DeallocateSpaceResponse response; - response.set_status(curvefs::space::SpaceStatusCode::SPACE_OK); - EXPECT_CALL(mockspacebasecli_, DeAllocExtents(_, _, _, _, _)) - .WillOnce(SetArgPointee<2>(response)); - ASSERT_EQ(CURVEFS_ERROR::OK, - spaceclient_.DeAllocExtents(fsid, allocatedExtents)); -} - - -} // namespace client -} // namespace curvefs diff --git a/curvefs/test/client/test_extent_manager.cpp b/curvefs/test/client/test_extent_manager.cpp deleted file mode 100644 index d8d6bddea7..0000000000 --- a/curvefs/test/client/test_extent_manager.cpp +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Copyright (c) 2020 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * Project: curve - * Created Date: Thur May 27 2021 - * Author: xuchaojie - */ - -#include -#include -#include - -#include "curvefs/src/client/extent_manager.h" - -namespace curvefs { -namespace client { - -using ::testing::Return; -using ::testing::_; -using ::testing::Contains; -using ::testing::SetArgPointee; - -class TestSimpleExtentManager : public ::testing::Test { - protected: - TestSimpleExtentManager() {} - ~TestSimpleExtentManager() {} - - virtual void SetUp() { - extent_manager_ = std::make_shared(); - extManagerOption_.preAllocSize = 65536; - extent_manager_->Init(extManagerOption_); - } - - virtual void TearDown() { - } - - protected: - std::shared_ptr extent_manager_; - ExtentManagerOption extManagerOption_; -}; - -TEST_F(TestSimpleExtentManager, allocateExtentAndMerge) { - VolumeExtentList extents; - uint64_t offset = 0; - uint64_t len = 4096; - std::list toAllocExtents; - - CURVEFS_ERROR ret = extent_manager_->GetToAllocExtents(extents, - offset, len, &toAllocExtents); - ASSERT_EQ(CURVEFS_ERROR::OK, ret); - ASSERT_EQ(1, toAllocExtents.size()); - ASSERT_EQ(0, toAllocExtents.front().lOffset); - ASSERT_EQ(extManagerOption_.preAllocSize, toAllocExtents.front().len); - ASSERT_EQ(false, toAllocExtents.front().leftHintAvailable); - ASSERT_EQ(false, toAllocExtents.front().rightHintAvailable); - - uint64_t pOffset = 10240; - std::list allocatedExtents; - Extent ext; - ext.set_offset(pOffset); - ext.set_length(extManagerOption_.preAllocSize); - allocatedExtents.push_back(ext); - - ret = extent_manager_->MergeAllocedExtents( - toAllocExtents, allocatedExtents, &extents); - ASSERT_EQ(CURVEFS_ERROR::OK, ret); - ASSERT_EQ(1, extents.volumeextents_size()); - ASSERT_EQ(0, extents.volumeextents(0).fsoffset()); - ASSERT_EQ(pOffset, extents.volumeextents(0).volumeoffset()); - ASSERT_EQ(extManagerOption_.preAllocSize, - extents.volumeextents(0).length()); - ASSERT_EQ(false, extents.volumeextents(0).isused()); - - for (int i = 0; i < extManagerOption_.preAllocSize/len; i++) { - ret = extent_manager_->GetToAllocExtents(extents, - offset, len, &toAllocExtents); - ASSERT_EQ(CURVEFS_ERROR::OK, ret); - ASSERT_EQ(0, toAllocExtents.size()) << "failed when i = " << i; - offset += len; - } - - ret = extent_manager_->GetToAllocExtents(extents, - offset, len, &toAllocExtents); - ASSERT_EQ(CURVEFS_ERROR::OK, ret); - ASSERT_EQ(1, toAllocExtents.size()); - ASSERT_EQ(offset, toAllocExtents.front().lOffset); - ASSERT_EQ(extManagerOption_.preAllocSize, toAllocExtents.front().len); - ASSERT_EQ(true, toAllocExtents.front().leftHintAvailable); - ASSERT_EQ(pOffset + extManagerOption_.preAllocSize, - toAllocExtents.front().pOffsetLeft); - ASSERT_EQ(false, toAllocExtents.front().rightHintAvailable); - - std::list allocatedExtents2; - Extent ext2; - ext2.set_offset(pOffset + extManagerOption_.preAllocSize); - ext2.set_length(extManagerOption_.preAllocSize); - allocatedExtents2.push_back(ext2); - ret = extent_manager_->MergeAllocedExtents( - toAllocExtents, allocatedExtents2, &extents); - ASSERT_EQ(CURVEFS_ERROR::OK, ret); - ASSERT_EQ(1, extents.volumeextents_size()); - ASSERT_EQ(0, extents.volumeextents(0).fsoffset()); - ASSERT_EQ(pOffset, extents.volumeextents(0).volumeoffset()); - ASSERT_EQ(extManagerOption_.preAllocSize * 2, - extents.volumeextents(0).length()); - ASSERT_EQ(false, extents.volumeextents(0).isused()); -} - -TEST_F(TestSimpleExtentManager, MarkExtentsWritten) { - uint64_t pOffset = 10240; - VolumeExtentList extents; - VolumeExtent *vext = extents.add_volumeextents(); - vext->set_fsoffset(0); - vext->set_volumeoffset(pOffset); - vext->set_length(extManagerOption_.preAllocSize); - vext->set_isused(false); - - uint64_t offset = 0; - uint64_t len = 4096; - CURVEFS_ERROR ret = CURVEFS_ERROR::OK; - - for (int i = 1; i < extManagerOption_.preAllocSize/len; i++) { - LOG(INFO) << "MarkExtentsWritten i = " << i; - ret = extent_manager_->MarkExtentsWritten( - offset, len, &extents); - ASSERT_EQ(CURVEFS_ERROR::OK, ret); - ASSERT_EQ(2, extents.volumeextents_size()); - - ASSERT_EQ(0, extents.volumeextents(0).fsoffset()); - ASSERT_EQ(pOffset, extents.volumeextents(0).volumeoffset()); - ASSERT_EQ(i * len, extents.volumeextents(0).length()); - ASSERT_EQ(true, extents.volumeextents(0).isused()); - - ASSERT_EQ(i * len, extents.volumeextents(1).fsoffset()); - ASSERT_EQ(pOffset + i * len, - extents.volumeextents(1).volumeoffset()); - ASSERT_EQ(extManagerOption_.preAllocSize - i * len, - extents.volumeextents(1).length()); - ASSERT_EQ(false, extents.volumeextents(1).isused()); - offset += len; - } - - ret = extent_manager_->MarkExtentsWritten( - offset, len, &extents); - ASSERT_EQ(CURVEFS_ERROR::OK, ret); - ASSERT_EQ(1, extents.volumeextents_size()); - ASSERT_EQ(0, extents.volumeextents(0).fsoffset()); - ASSERT_EQ(pOffset, extents.volumeextents(0).volumeoffset()); - ASSERT_EQ(extManagerOption_.preAllocSize, - extents.volumeextents(0).length()); - ASSERT_EQ(true, extents.volumeextents(0).isused()); -} - -TEST_F(TestSimpleExtentManager, DivideExtents) { - uint64_t pOffset = 10240; - VolumeExtentList extents; - VolumeExtent *vext1 = extents.add_volumeextents(); - vext1->set_fsoffset(0); - vext1->set_volumeoffset(pOffset); - vext1->set_length(4096); - vext1->set_isused(true); - - VolumeExtent *vext2 = extents.add_volumeextents(); - vext2->set_fsoffset(4096); - vext2->set_volumeoffset(pOffset + 4096); - vext2->set_length(4096); - vext2->set_isused(false); - - uint64_t offset = 0; - uint64_t len = 4096 * 2; - CURVEFS_ERROR ret = CURVEFS_ERROR::OK; - - std::list pExtents; - ret = extent_manager_->DivideExtents(extents, offset, len, &pExtents); - - ASSERT_EQ(CURVEFS_ERROR::OK, ret); - ASSERT_EQ(2, pExtents.size()); - - auto it = pExtents.begin(); - ASSERT_EQ(pOffset, it->pOffset); - ASSERT_EQ(4096, it->len); - ASSERT_EQ(false, it->UnWritten); - - it++; - ASSERT_EQ(pOffset + 4096, it->pOffset); - ASSERT_EQ(4096 , it->len); - ASSERT_EQ(true, it->UnWritten); -} - - - -} // namespace client -} // namespace curvefs diff --git a/curvefs/test/client/test_fuse_client.cpp b/curvefs/test/client/test_fuse_client.cpp index 586815877e..1d802489c6 100644 --- a/curvefs/test/client/test_fuse_client.cpp +++ b/curvefs/test/client/test_fuse_client.cpp @@ -23,17 +23,19 @@ #include #include +#include "absl/memory/memory.h" #include "curvefs/src/client/fuse_s3_client.h" #include "curvefs/src/client/fuse_volume_client.h" -#include "curvefs/test/client/mock_block_device_client.h" +#include "curvefs/src/common/define.h" #include "curvefs/test/client/mock_client_s3_adaptor.h" #include "curvefs/test/client/mock_dentry_cache_mamager.h" -#include "curvefs/test/client/mock_extent_manager.h" #include "curvefs/test/client/mock_inode_cache_manager.h" #include "curvefs/test/client/mock_mds_client.h" #include "curvefs/test/client/mock_metaserver_client.h" -#include "curvefs/test/client/mock_space_client.h" -#include "curvefs/src/common/define.h" +#include "curvefs/test/client/mock_volume_storage.h" +#include "curvefs/test/volume/mock/mock_block_device_client.h" +#include "curvefs/test/volume/mock/mock_space_manager.h" + struct fuse_req { struct fuse_ctx *ctx; }; @@ -61,10 +63,12 @@ using ::testing::SetArgReferee; using rpcclient::MockMdsClient; using rpcclient::MockMetaServerClient; using rpcclient::MetaServerClientDone; +using ::curvefs::volume::MockBlockDeviceClient; +using ::curvefs::volume::MockSpaceManager; #define EQUAL(a) (lhs.a() == rhs.a()) -bool operator==(const Dentry &lhs, const Dentry &rhs) { +static bool operator==(const Dentry &lhs, const Dentry &rhs) { return EQUAL(fsid) && EQUAL(parentinodeid) && EQUAL(name) && EQUAL(txid) && EQUAL(inodeid) && EQUAL(flag); } @@ -77,11 +81,9 @@ class TestFuseVolumeClient : public ::testing::Test { virtual void SetUp() { mdsClient_ = std::make_shared(); metaClient_ = std::make_shared(); - spaceClient_ = std::make_shared(); blockDeviceClient_ = std::make_shared(); inodeManager_ = std::make_shared(); dentryManager_ = std::make_shared(); - extManager_ = std::make_shared(); preAllocSize_ = 65536; bigFileSize_ = 1048576; listDentryLimit_ = 100; @@ -89,9 +91,13 @@ class TestFuseVolumeClient : public ::testing::Test { fuseClientOption_.volumeOpt.bigFileSize = bigFileSize_; fuseClientOption_.listDentryLimit = listDentryLimit_; fuseClientOption_.maxNameLength = 20u; + + spaceManager_ = new MockSpaceManager(); + volumeStorage_ = new MockVolumeStorage(); client_ = std::make_shared( mdsClient_, metaClient_, inodeManager_, dentryManager_, - spaceClient_, extManager_, blockDeviceClient_); + blockDeviceClient_); + client_->Init(fuseClientOption_); PrepareFsInfo(); } @@ -99,9 +105,12 @@ class TestFuseVolumeClient : public ::testing::Test { virtual void TearDown() { mdsClient_ = nullptr; metaClient_ = nullptr; - spaceClient_ = nullptr; blockDeviceClient_ = nullptr; - extManager_ = nullptr; + } + + void PrepareEnv() { + client_->SetSpaceManagerForTesting(spaceManager_); + client_->SetVolumeStorageForTesting(volumeStorage_); } void PrepareFsInfo() { @@ -129,13 +138,14 @@ class TestFuseVolumeClient : public ::testing::Test { std::shared_ptr mdsClient_; std::shared_ptr metaClient_; - std::shared_ptr spaceClient_; std::shared_ptr blockDeviceClient_; std::shared_ptr inodeManager_; std::shared_ptr dentryManager_; - std::shared_ptr extManager_; std::shared_ptr client_; + MockVolumeStorage *volumeStorage_; + MockSpaceManager *spaceManager_; + uint64_t preAllocSize_; uint64_t bigFileSize_; uint32_t listDentryLimit_; @@ -147,6 +157,8 @@ class TestFuseVolumeClient : public ::testing::Test { }; TEST_F(TestFuseVolumeClient, FuseOpInit_when_fs_exist) { + LOG(INFO) << "Entering " << __PRETTY_FUNCTION__; + MountOption mOpts; memset(&mOpts, 0, sizeof(mOpts)); mOpts.mountPoint = "host1:/test"; @@ -165,8 +177,8 @@ TEST_F(TestFuseVolumeClient, FuseOpInit_when_fs_exist) { EXPECT_CALL(*mdsClient_, MountFs(fsName, _, _)) .WillOnce(DoAll(SetArgPointee<2>(fsInfoExp), Return(FSStatusCode::OK))); - EXPECT_CALL(*blockDeviceClient_, Open(volName, user)) - .WillOnce(Return(CURVEFS_ERROR::OK)); + EXPECT_CALL(*blockDeviceClient_, Open(_, _)) + .WillOnce(Return(true)); CURVEFS_ERROR ret = client_->FuseOpInit(&mOpts, nullptr); ASSERT_EQ(CURVEFS_ERROR::OK, ret); @@ -191,9 +203,6 @@ TEST_F(TestFuseVolumeClient, FuseOpDestroy) { EXPECT_CALL(*mdsClient_, UmountFs(fsName, _)) .WillOnce(Return(FSStatusCode::OK)); - EXPECT_CALL(*blockDeviceClient_, Close()) - .WillOnce(Return(CURVEFS_ERROR::OK)); - client_->FuseOpDestroy(&mOpts); } @@ -263,76 +272,8 @@ TEST_F(TestFuseVolumeClient, FuseOpLookupNameTooLong) { } TEST_F(TestFuseVolumeClient, FuseOpWrite) { - fuse_req_t req; - fuse_ino_t ino = 1; - const char *buf = "xxx"; - size_t size = 4; - off_t off = 0; - struct fuse_file_info fi; - fi.flags = O_WRONLY; - size_t wSize = 0; + PrepareEnv(); - Inode inode; - inode.set_inodeid(ino); - inode.set_length(0); - - auto inodeWrapper = std::make_shared(inode, metaClient_); - EXPECT_CALL(*inodeManager_, GetInode(ino, _)) - .WillOnce( - DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK))); - - std::list toAllocExtents; - ExtentAllocInfo allocInfo; - allocInfo.lOffset = 0; - allocInfo.pOffsetLeft = 0; - allocInfo.len = preAllocSize_; - toAllocExtents.push_back(allocInfo); - EXPECT_CALL(*extManager_, GetToAllocExtents(_, off, size, _)) - .WillOnce( - DoAll(SetArgPointee<3>(toAllocExtents), Return(CURVEFS_ERROR::OK))); - - std::list allocatedExtents; - Extent ext; - ext.set_offset(0); - ext.set_length(preAllocSize_); - EXPECT_CALL(*spaceClient_, AllocExtents(fsId, _, AllocateType::SMALL, _)) - .WillOnce(DoAll(SetArgPointee<3>(allocatedExtents), - Return(CURVEFS_ERROR::OK))); - - VolumeExtentList *vlist = new VolumeExtentList(); - VolumeExtent *vext = vlist->add_volumeextents(); - vext->set_fsoffset(0); - vext->set_volumeoffset(0); - vext->set_length(preAllocSize_); - vext->set_isused(false); - inode.set_allocated_volumeextentlist(vlist); - - EXPECT_CALL(*extManager_, MergeAllocedExtents(_, _, _)) - .WillOnce(DoAll(SetArgPointee<2>(*vlist), Return(CURVEFS_ERROR::OK))); - - std::list pExtents; - PExtent pext; - pext.pOffset = 0; - pext.len = preAllocSize_; - pExtents.push_back(pext); - EXPECT_CALL(*extManager_, DivideExtents(_, off, size, _)) - .WillOnce(DoAll(SetArgPointee<3>(pExtents), Return(CURVEFS_ERROR::OK))); - - EXPECT_CALL(*blockDeviceClient_, Write(_, 0, preAllocSize_)) - .WillOnce(Return(CURVEFS_ERROR::OK)); - - EXPECT_CALL(*extManager_, MarkExtentsWritten(off, size, _)) - .WillOnce(Return(CURVEFS_ERROR::OK)); - - CURVEFS_ERROR ret = - client_->FuseOpWrite(req, ino, buf, size, off, &fi, &wSize); - - ASSERT_EQ(CURVEFS_ERROR::OK, ret); - ASSERT_EQ(size, wSize); - ASSERT_EQ(true, inodeWrapper->isDirty()); -} - -TEST_F(TestFuseVolumeClient, FuseOpWriteFailed) { fuse_req_t req; fuse_ino_t ino = 1; const char *buf = "xxx"; @@ -345,208 +286,49 @@ TEST_F(TestFuseVolumeClient, FuseOpWriteFailed) { Inode inode; inode.set_inodeid(ino); inode.set_length(0); - auto inodeWrapper = std::make_shared(inode, metaClient_); - - EXPECT_CALL(*inodeManager_, GetInode(ino, _)) - .WillOnce(Return(CURVEFS_ERROR::INTERNAL)) - .WillOnce( - DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK))) - .WillOnce( - DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK))) - .WillOnce( - DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK))) - .WillOnce( - DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK))) - .WillOnce( - DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK))) - .WillOnce( - DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK))); - - std::list toAllocExtents; - ExtentAllocInfo allocInfo; - allocInfo.lOffset = 0; - allocInfo.pOffsetLeft = 0; - allocInfo.len = preAllocSize_; - toAllocExtents.push_back(allocInfo); - EXPECT_CALL(*extManager_, GetToAllocExtents(_, off, size, _)) - .WillOnce(Return(CURVEFS_ERROR::INTERNAL)) - .WillOnce( - DoAll(SetArgPointee<3>(toAllocExtents), Return(CURVEFS_ERROR::OK))) - .WillOnce( - DoAll(SetArgPointee<3>(toAllocExtents), Return(CURVEFS_ERROR::OK))) - .WillOnce( - DoAll(SetArgPointee<3>(toAllocExtents), Return(CURVEFS_ERROR::OK))) - .WillOnce( - DoAll(SetArgPointee<3>(toAllocExtents), Return(CURVEFS_ERROR::OK))) - .WillOnce( - DoAll(SetArgPointee<3>(toAllocExtents), Return(CURVEFS_ERROR::OK))); - - std::list allocatedExtents; - Extent ext; - ext.set_offset(0); - ext.set_length(preAllocSize_); - EXPECT_CALL(*spaceClient_, AllocExtents(fsId, _, _, _)) - .WillOnce(Return(CURVEFS_ERROR::INTERNAL)) - .WillOnce(DoAll(SetArgPointee<3>(allocatedExtents), - Return(CURVEFS_ERROR::OK))) - .WillOnce(DoAll(SetArgPointee<3>(allocatedExtents), - Return(CURVEFS_ERROR::OK))) - .WillOnce(DoAll(SetArgPointee<3>(allocatedExtents), - Return(CURVEFS_ERROR::OK))) - .WillOnce(DoAll(SetArgPointee<3>(allocatedExtents), - Return(CURVEFS_ERROR::OK))); - - VolumeExtentList *vlist = new VolumeExtentList(); - VolumeExtent *vext = vlist->add_volumeextents(); - vext->set_fsoffset(0); - vext->set_volumeoffset(0); - vext->set_length(preAllocSize_); - vext->set_isused(false); - inode.set_allocated_volumeextentlist(vlist); - - EXPECT_CALL(*extManager_, MergeAllocedExtents(_, _, _)) - .WillOnce(Return(CURVEFS_ERROR::INTERNAL)) - .WillOnce(DoAll(SetArgPointee<2>(*vlist), Return(CURVEFS_ERROR::OK))) - .WillOnce(DoAll(SetArgPointee<2>(*vlist), Return(CURVEFS_ERROR::OK))) - .WillOnce(DoAll(SetArgPointee<2>(*vlist), Return(CURVEFS_ERROR::OK))); - - EXPECT_CALL(*spaceClient_, DeAllocExtents(_, _)) - .WillOnce(Return(CURVEFS_ERROR::OK)); - - std::list pExtents; - PExtent pext; - pext.pOffset = 0; - pext.len = preAllocSize_; - pExtents.push_back(pext); - EXPECT_CALL(*extManager_, DivideExtents(_, off, size, _)) - .WillOnce(Return(CURVEFS_ERROR::INTERNAL)) - .WillOnce(DoAll(SetArgPointee<3>(pExtents), Return(CURVEFS_ERROR::OK))) - .WillOnce(DoAll(SetArgPointee<3>(pExtents), Return(CURVEFS_ERROR::OK))); - - EXPECT_CALL(*blockDeviceClient_, Write(_, 0, preAllocSize_)) - .WillOnce(Return(CURVEFS_ERROR::INTERNAL)) - .WillOnce(Return(CURVEFS_ERROR::OK)); - - EXPECT_CALL(*extManager_, MarkExtentsWritten(off, size, _)) - .WillOnce(Return(CURVEFS_ERROR::INTERNAL)); - - CURVEFS_ERROR ret = - client_->FuseOpWrite(req, ino, buf, size, off, &fi, &wSize); - ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret); - - ret = client_->FuseOpWrite(req, ino, buf, size, off, &fi, &wSize); - ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret); - - ret = client_->FuseOpWrite(req, ino, buf, size, off, &fi, &wSize); - ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret); - - ret = client_->FuseOpWrite(req, ino, buf, size, off, &fi, &wSize); - ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret); - ret = client_->FuseOpWrite(req, ino, buf, size, off, &fi, &wSize); - ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret); + for (auto ret : std::initializer_list{size, -1}) { + EXPECT_CALL(*volumeStorage_, Write(_, _, _, _)) + .WillOnce(Return(ret)); - ret = client_->FuseOpWrite(req, ino, buf, size, off, &fi, &wSize); - ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret); + ASSERT_EQ(ret == size ? CURVEFS_ERROR::OK : CURVEFS_ERROR::IO_ERROR, + client_->FuseOpWrite(req, ino, buf, size, off, &fi, &wSize)); - ret = client_->FuseOpWrite(req, ino, buf, size, off, &fi, &wSize); - ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret); + if (ret == size) { + ASSERT_EQ(size, wSize); + } + } } TEST_F(TestFuseVolumeClient, FuseOpRead) { - fuse_req_t req; - fuse_ino_t ino = 1; - size_t size = 4; - off_t off = 0; - struct fuse_file_info fi; - fi.flags = O_RDONLY; - std::unique_ptr buffer(new char[size]); - size_t rSize = 0; - - Inode inode; - inode.set_fsid(fsId); - inode.set_inodeid(ino); - inode.set_length(4096); - auto inodeWrapper = std::make_shared(inode, metaClient_); + PrepareEnv(); - EXPECT_CALL(*inodeManager_, GetInode(ino, _)) - .WillOnce( - DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK))); - - std::list pExtents; - PExtent pext1, pext2; - pext1.pOffset = 0; - pext1.len = 4; - pext1.UnWritten = false; - pext2.pOffset = 4; - pext2.len = 4096; - pext2.UnWritten = true; - pExtents.push_back(pext1); - pExtents.push_back(pext2); - - EXPECT_CALL(*extManager_, DivideExtents(_, off, size, _)) - .WillOnce(DoAll(SetArgPointee<3>(pExtents), Return(CURVEFS_ERROR::OK))); - - EXPECT_CALL(*blockDeviceClient_, Read(_, 0, 4)) - .WillOnce(Return(CURVEFS_ERROR::OK)); - - CURVEFS_ERROR ret = - client_->FuseOpRead(req, ino, size, off, &fi, buffer.get(), &rSize); - ASSERT_EQ(CURVEFS_ERROR::OK, ret); - ASSERT_EQ(size, rSize); - ASSERT_EQ(true, inodeWrapper->isDirty()); -} - -TEST_F(TestFuseVolumeClient, FuseOpReadFailed) { fuse_req_t req; fuse_ino_t ino = 1; size_t size = 4; off_t off = 0; struct fuse_file_info fi; fi.flags = O_RDONLY; - std::unique_ptr buffer(new char[size]); + std::unique_ptr buf(new char[size]); size_t rSize = 0; Inode inode; inode.set_fsid(fsId); inode.set_inodeid(ino); inode.set_length(4096); - auto inodeWrapper = std::make_shared(inode, metaClient_); - EXPECT_CALL(*inodeManager_, GetInode(ino, _)) - .WillOnce(Return(CURVEFS_ERROR::INTERNAL)) - .WillOnce( - DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK))) - .WillOnce( - DoAll(SetArgReferee<1>(inodeWrapper), Return(CURVEFS_ERROR::OK))); + for (auto ret : std::initializer_list{size, -1}) { + EXPECT_CALL(*volumeStorage_, Read(_, _, _, _)) + .WillOnce(Return(ret)); - std::list pExtents; - PExtent pext1, pext2; - pext1.pOffset = 0; - pext1.len = 4; - pext1.UnWritten = false; - pext2.pOffset = 4; - pext2.len = 4096; - pext2.UnWritten = true; - pExtents.push_back(pext1); - pExtents.push_back(pext2); - - EXPECT_CALL(*extManager_, DivideExtents(_, off, size, _)) - .WillOnce(Return(CURVEFS_ERROR::INTERNAL)) - .WillOnce(DoAll(SetArgPointee<3>(pExtents), Return(CURVEFS_ERROR::OK))); + ASSERT_EQ( + ret == size ? CURVEFS_ERROR::OK : CURVEFS_ERROR::IO_ERROR, + client_->FuseOpRead(req, ino, size, off, &fi, buf.get(), &rSize)); - EXPECT_CALL(*blockDeviceClient_, Read(_, 0, 4)) - .WillOnce(Return(CURVEFS_ERROR::INTERNAL)); - - CURVEFS_ERROR ret = - client_->FuseOpRead(req, ino, size, off, &fi, buffer.get(), &rSize); - ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret); - - ret = client_->FuseOpRead(req, ino, size, off, &fi, buffer.get(), &rSize); - ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret); - - ret = client_->FuseOpRead(req, ino, size, off, &fi, buffer.get(), &rSize); - ASSERT_EQ(CURVEFS_ERROR::INTERNAL, ret); + if (ret == size) { + ASSERT_EQ(size, rSize); + } + } } TEST_F(TestFuseVolumeClient, FuseOpOpen) { @@ -1384,6 +1166,7 @@ TEST_F(TestFuseVolumeClient, FuseOpSetAttr) { Inode inode; inode.set_inodeid(ino); inode.set_length(0); + inode.set_type(FsFileType::TYPE_VOLUME); auto inodeWrapper = std::make_shared(inode, metaClient_); EXPECT_CALL(*inodeManager_, GetInode(ino, _)) @@ -1429,6 +1212,7 @@ TEST_F(TestFuseVolumeClient, FuseOpSetAttrFailed) { Inode inode; inode.set_inodeid(ino); inode.set_length(0); + inode.set_type(FsFileType::TYPE_VOLUME); auto inodeWrapper = std::make_shared(inode, metaClient_); EXPECT_CALL(*inodeManager_, GetInode(ino, _)) @@ -1749,9 +1533,7 @@ class TestFuseS3Client : public ::testing::Test { virtual void TearDown() { mdsClient_ = nullptr; metaClient_ = nullptr; - spaceClient_ = nullptr; s3ClientAdaptor_ = nullptr; - extManager_ = nullptr; } void PrepareFsInfo() { @@ -1767,11 +1549,9 @@ class TestFuseS3Client : public ::testing::Test { std::shared_ptr mdsClient_; std::shared_ptr metaClient_; - std::shared_ptr spaceClient_; std::shared_ptr s3ClientAdaptor_; std::shared_ptr inodeManager_; std::shared_ptr dentryManager_; - std::shared_ptr extManager_; std::shared_ptr client_; FuseClientOption fuseClientOption_; }; @@ -1943,6 +1723,7 @@ TEST_F(TestFuseS3Client, FuseOpFsync) { Inode inode; inode.set_inodeid(ino); inode.set_length(0); + inode.set_type(FsFileType::TYPE_S3); EXPECT_CALL(*s3ClientAdaptor_, Flush(_)) .WillOnce(Return(CURVEFS_ERROR::OK)) @@ -1968,10 +1749,11 @@ TEST_F(TestFuseS3Client, FuseOpFlush) { fuse_req_t req; fuse_ino_t ino = 1; struct fuse_file_info *fi; - Inode inode; inode.set_inodeid(ino); inode.set_length(0); + inode.set_type(FsFileType::TYPE_S3); + auto inodeWrapper = std::make_shared(inode, metaClient_); inodeWrapper->SetUid(32); inodeWrapper->SetOpenCount(1); @@ -2762,9 +2544,10 @@ TEST_F(TestFuseS3Client, FuseOpOpen_Trunc_EnableSummary) { inode.set_fsid(1); inode.set_inodeid(1); inode.set_length(4096); - inode.set_type(FsFileType::TYPE_FILE); inode.set_openmpcount(0); inode.add_parent(0); + inode.set_type(FsFileType::TYPE_S3); + auto inodeWrapper = std::make_shared(inode, metaClient_); Inode parentInode; diff --git a/curvefs/test/client/test_inodeWrapper.cpp b/curvefs/test/client/test_inodeWrapper.cpp index 56f71fb819..e5f9b3e526 100644 --- a/curvefs/test/client/test_inodeWrapper.cpp +++ b/curvefs/test/client/test_inodeWrapper.cpp @@ -121,6 +121,7 @@ TEST(TestAppendS3ChunkInfoToMap, testAppendS3ChunkInfoToMap) { TEST_F(TestInodeWrapper, testSyncSuccess) { inodeWrapper_->MarkDirty(); inodeWrapper_->SetLength(1024); + inodeWrapper_->SetType(FsFileType::TYPE_S3); S3ChunkInfo info1; info1.set_chunkid(1); @@ -145,6 +146,7 @@ TEST_F(TestInodeWrapper, testSyncSuccess) { TEST_F(TestInodeWrapper, testSyncFailed) { inodeWrapper_->MarkDirty(); inodeWrapper_->SetLength(1024); + inodeWrapper_->SetType(FsFileType::TYPE_S3); S3ChunkInfo info1; info1.set_chunkid(1); diff --git a/curvefs/test/client/test_inode_cache_manager.cpp b/curvefs/test/client/test_inode_cache_manager.cpp index 2aa7f5c3b4..aef119ed24 100644 --- a/curvefs/test/client/test_inode_cache_manager.cpp +++ b/curvefs/test/client/test_inode_cache_manager.cpp @@ -184,7 +184,7 @@ TEST_F(TestInodeCacheManager, ShipToFlushAndFlushAll) { Inode inode; inode.set_inodeid(inodeId); inode.set_fsid(fsId_); - inode.set_type(FsFileType::TYPE_FILE); + inode.set_type(FsFileType::TYPE_S3); std::shared_ptr inodeWrapper = std::make_shared(inode, metaClient_); diff --git a/curvefs/test/client/volume/BUILD b/curvefs/test/client/volume/BUILD new file mode 100644 index 0000000000..9d4b821cfc --- /dev/null +++ b/curvefs/test/client/volume/BUILD @@ -0,0 +1,34 @@ +# +# Copyright (c) 2022 NetEase Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("//:copts.bzl", "CURVE_TEST_COPTS") + +cc_test( + name = "curvefs_client_volume_test", + srcs = glob([ + "*.cpp", + "*.h", + ]), + copts = CURVE_TEST_COPTS, + deps = [ + "//curvefs/src/client:fuse_client_lib", + "//curvefs/test/volume/mock", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + "//curvefs/test/client:mock", + ], +) diff --git a/curvefs/test/client/volume/alloc_space_test.cpp b/curvefs/test/client/volume/alloc_space_test.cpp new file mode 100644 index 0000000000..b20ad5f0e4 --- /dev/null +++ b/curvefs/test/client/volume/alloc_space_test.cpp @@ -0,0 +1,244 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Friday Mar 18 12:30:42 CST 2022 + * Author: wuhanqing + */ + +#include +#include + +#include "curvefs/src/client/volume/utils.h" +#include "curvefs/test/client/volume/common.h" +#include "curvefs/test/volume/mock/mock_space_manager.h" + +namespace curvefs { +namespace client { + +using ::curvefs::volume::AllocateHint; +using ::curvefs::volume::MockSpaceManager; + +using ::testing::_; +using ::testing::Invoke; +using ::testing::Return; + +class AllocSpaceTest : public ::testing::Test { + protected: + MockSpaceManager mockSpaceManager_; +}; + +TEST_F(AllocSpaceTest, TestAllocError) { + AllocPart part; + part.allocInfo.len = 4 * kKiB; + part.allocInfo.lOffset = 4 * kKiB; + + EXPECT_CALL(mockSpaceManager_, Alloc(_, _, _)) + .WillOnce(Return(false)); + + std::map writes; + std::map allocates; + + ASSERT_FALSE(AllocSpace(&mockSpaceManager_, part, &writes, &allocates)); +} + +TEST_F(AllocSpaceTest, TestAllocOneExtent) { + AllocPart part; + part.writelength = 4 * kKiB; + part.allocInfo.len = 4 * kKiB; + part.allocInfo.lOffset = 4 * kKiB; + + std::unique_ptr data(new char[part.writelength]); + + part.data = data.get(); + + EXPECT_CALL(mockSpaceManager_, Alloc(_, _, _)) + .WillOnce(Invoke( + [](uint64_t, const AllocateHint&, std::vector* exts) { + exts->emplace_back(32 * kKiB, 4 * kKiB); + return exts; + })); + + std::map writes; + std::map allocates; + + ASSERT_TRUE(AllocSpace(&mockSpaceManager_, part, &writes, &allocates)); + + ASSERT_EQ(1, writes.size()); + ASSERT_EQ(1, allocates.size()); + + ASSERT_EQ(1, writes.count(4 * kKiB)); + ASSERT_EQ(1, allocates.count(4 * kKiB)); + + ASSERT_EQ(32 * kKiB, writes[4 * kKiB].offset); + ASSERT_EQ(4 * kKiB, writes[4 * kKiB].length); + ASSERT_EQ(data.get(), writes[4 * kKiB].data); + + ASSERT_EQ(32 * kKiB, allocates[4 * kKiB].offset); + ASSERT_EQ(4 * kKiB, allocates[4 * kKiB].len); +} + +TEST_F(AllocSpaceTest, TestPreAllocOneExtent) { + AllocPart part; + part.writelength = 4 * kKiB; + part.allocInfo.len = 64 * kKiB; + part.allocInfo.lOffset = 4 * kKiB; + + std::unique_ptr data(new char[part.writelength]); + + part.data = data.get(); + + EXPECT_CALL(mockSpaceManager_, Alloc(_, _, _)) + .WillOnce(Invoke( + [](uint64_t size, const AllocateHint&, std::vector* exts) { + exts->emplace_back(32 * kKiB, size); + return exts; + })); + + std::map writes; + std::map allocates; + + ASSERT_TRUE(AllocSpace(&mockSpaceManager_, part, &writes, &allocates)); + + ASSERT_EQ(1, writes.size()); + ASSERT_EQ(1, allocates.size()); + + ASSERT_EQ(1, writes.count(4 * kKiB)); + ASSERT_EQ(1, allocates.count(4 * kKiB)); + + ASSERT_EQ(32 * kKiB, writes[4 * kKiB].offset); + ASSERT_EQ(4 * kKiB, writes[4 * kKiB].length); + ASSERT_EQ(data.get(), writes[4 * kKiB].data); + + ASSERT_EQ(32 * kKiB, allocates[4 * kKiB].offset); + ASSERT_EQ(64 * kKiB, allocates[4 * kKiB].len); +} + +TEST_F(AllocSpaceTest, TestAllocOneExtentAndWriteIsNotAligned) { + AllocPart part; + part.writelength = 6; // 6 bytes + part.allocInfo.len = 64 * kKiB; + part.allocInfo.lOffset = 0 * kKiB; + + std::unique_ptr data(new char[part.writelength]); + + part.data = data.get(); + + EXPECT_CALL(mockSpaceManager_, Alloc(_, _, _)) + .WillOnce(Invoke( + [](uint64_t size, const AllocateHint&, std::vector* exts) { + exts->emplace_back(32 * kKiB, size); + return exts; + })); + + std::map writes; + std::map allocates; + + ASSERT_TRUE(AllocSpace(&mockSpaceManager_, part, &writes, &allocates)); + + ASSERT_EQ(1, writes.size()); + ASSERT_EQ(1, allocates.size()); + + ASSERT_EQ(1, writes.count(0 * kKiB)); + ASSERT_EQ(1, allocates.count(0 * kKiB)); + + ASSERT_EQ(32 * kKiB, writes[0 * kKiB].offset); + ASSERT_EQ(6, writes[0 * kKiB].length); + ASSERT_EQ(data.get(), writes[0 * kKiB].data); + + ASSERT_EQ(32 * kKiB, allocates[0 * kKiB].offset); + ASSERT_EQ(64 * kKiB, allocates[0 * kKiB].len); +} + +TEST_F(AllocSpaceTest, TestAllocMultiExtents) { + AllocPart part; + part.writelength = 16 * kKiB; + part.allocInfo.len = 64 * kKiB; + part.allocInfo.lOffset = 4 * kKiB; + + std::unique_ptr data(new char[part.writelength]); + + part.data = data.get(); + + EXPECT_CALL(mockSpaceManager_, Alloc(_, _, _)) + .WillOnce(Invoke( + [](uint64_t, const AllocateHint&, std::vector* exts) { + // first 16KiB is discontinuous + exts->emplace_back(32 * kKiB, 4 * kKiB); + exts->emplace_back(64 * kKiB, 4 * kKiB); + exts->emplace_back(128 * kKiB, 4 * kKiB); + exts->emplace_back(256 * kKiB, 4 * kKiB); + + // last 48KiB is continuous + exts->emplace_back(512 * kKiB, 48 * kKiB); + + return exts; + })); + + std::map writes; + std::map allocates; + + ASSERT_TRUE(AllocSpace(&mockSpaceManager_, part, &writes, &allocates)); + + ASSERT_EQ(4, writes.size()); + ASSERT_EQ(5, allocates.size()); + + ASSERT_EQ(1, writes.count(4 * kKiB)); + ASSERT_EQ(1, writes.count(8 * kKiB)); + ASSERT_EQ(1, writes.count(12 * kKiB)); + ASSERT_EQ(1, writes.count(16 * kKiB)); + + ASSERT_EQ(1, allocates.count(4 * kKiB)); + ASSERT_EQ(1, allocates.count(8 * kKiB)); + ASSERT_EQ(1, allocates.count(12 * kKiB)); + ASSERT_EQ(1, allocates.count(16 * kKiB)); + ASSERT_EQ(1, allocates.count(20 * kKiB)); + + ASSERT_EQ(32 * kKiB, writes[4 * kKiB].offset); + ASSERT_EQ(4 * kKiB, writes[4 * kKiB].length); + ASSERT_EQ(data.get(), writes[4 * kKiB].data); + + ASSERT_EQ(64 * kKiB, writes[8 * kKiB].offset); + ASSERT_EQ(4 * kKiB, writes[8 * kKiB].length); + ASSERT_EQ(data.get() + 4 * kKiB, writes[8 * kKiB].data); + + ASSERT_EQ(128 * kKiB, writes[12 * kKiB].offset); + ASSERT_EQ(4 * kKiB, writes[12 * kKiB].length); + ASSERT_EQ(data.get() + 8 * kKiB, writes[12 * kKiB].data); + + ASSERT_EQ(256 * kKiB, writes[16 * kKiB].offset); + ASSERT_EQ(4 * kKiB, writes[16 * kKiB].length); + ASSERT_EQ(data.get() + 12 * kKiB, writes[16 * kKiB].data); + + ASSERT_EQ(32 * kKiB, allocates[4 * kKiB].offset); + ASSERT_EQ(4 * kKiB, allocates[4 * kKiB].len); + + ASSERT_EQ(64 * kKiB, allocates[8 * kKiB].offset); + ASSERT_EQ(4 * kKiB, allocates[8 * kKiB].len); + + ASSERT_EQ(128 * kKiB, allocates[12 * kKiB].offset); + ASSERT_EQ(4 * kKiB, allocates[12 * kKiB].len); + + ASSERT_EQ(256 * kKiB, allocates[16 * kKiB].offset); + ASSERT_EQ(4 * kKiB, allocates[16 * kKiB].len); + + ASSERT_EQ(512 * kKiB, allocates[20 * kKiB].offset); + ASSERT_EQ(48 * kKiB, allocates[20 * kKiB].len); +} + +} // namespace client +} // namespace curvefs diff --git a/curvefs/test/client/volume/common.h b/curvefs/test/client/volume/common.h new file mode 100644 index 0000000000..1c61b86b4e --- /dev/null +++ b/curvefs/test/client/volume/common.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Tuesday Mar 15 20:06:07 CST 2022 + * Author: wuhanqing + */ + +#ifndef CURVEFS_TEST_CLIENT_VOLUME_COMMON_H_ +#define CURVEFS_TEST_CLIENT_VOLUME_COMMON_H_ + +#include +#include +#include + +#include "curvefs/src/client/volume/extent.h" +#include "curvefs/src/volume/common.h" + +namespace curvefs { +namespace client { + +using ::curvefs::volume::kKiB; +using ::curvefs::volume::kMiB; +using ::curvefs::volume::kGiB; +using ::curvefs::volume::kTiB; + +inline bool operator==(const PExtent& p1, const PExtent& p2) { + return p1.pOffset == p2.pOffset && p1.len == p2.len && + p1.UnWritten == p2.UnWritten; +} + +inline std::ostream& operator<<(std::ostream& os, const PExtent& p) { + os << "[physical offset: " << p.pOffset << ", length: " << p.len + << ", written: " << std::boolalpha << (!p.UnWritten) << "]"; + + return os; +} + +} // namespace client +} // namespace curvefs + +#endif // CURVEFS_TEST_CLIENT_VOLUME_COMMON_H_ diff --git a/curvefs/test/client/volume/default_volume_storage_test.cpp b/curvefs/test/client/volume/default_volume_storage_test.cpp new file mode 100644 index 0000000000..d7aa61b60c --- /dev/null +++ b/curvefs/test/client/volume/default_volume_storage_test.cpp @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Thursday Mar 24 16:57:03 CST 2022 + * Author: wuhanqing + */ + +#include "curvefs/src/client/volume/default_volume_storage.h" + +#include +#include + +#include "absl/memory/memory.h" +#include "curvefs/test/client/mock_inode_cache_manager.h" +#include "curvefs/test/client/mock_metaserver_client.h" +#include "curvefs/test/volume/mock/mock_block_device_client.h" +#include "curvefs/test/volume/mock/mock_space_manager.h" +#include "gtest/gtest.h" + +namespace curvefs { +namespace client { + +using ::curvefs::client::rpcclient::MockMetaServerClient; +using ::curvefs::volume::AllocateHint; +using ::curvefs::volume::Extent; +using ::curvefs::volume::MockBlockDeviceClient; +using ::curvefs::volume::MockSpaceManager; +using ::testing::_; +using ::testing::DoAll; +using ::testing::Invoke; +using ::testing::Return; +using ::testing::SetArgPointee; + +class DefaultVolumeStorageTest : public ::testing::Test { + protected: + DefaultVolumeStorageTest() + : storage_(&spaceMgr_, &blockDev_, &inodeCacheMgr_), + metaServerCli_(std::make_shared()) {} + + protected: + MockSpaceManager spaceMgr_; + MockBlockDeviceClient blockDev_; + MockInodeCacheManager inodeCacheMgr_; + DefaultVolumeStorage storage_; + std::shared_ptr metaServerCli_; +}; + +TEST_F(DefaultVolumeStorageTest, WriteAndReadTest_InodeNotFound) { + EXPECT_CALL(inodeCacheMgr_, GetInode(_, _)) + .Times(2) + .WillRepeatedly(Return(CURVEFS_ERROR::NOTEXIST)); + + uint64_t ino = 1; + off_t offset = 0; + size_t len = 4096; + std::unique_ptr data(new char[4096]); + + ASSERT_GT(0, storage_.Read(ino, offset, len, data.get())); + ASSERT_GT(0, storage_.Write(ino, offset, len, data.get())); +} + +TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadError) { + Inode inode; + inode.set_type(FsFileType::TYPE_VOLUME); + + VolumeExtentList exts; + auto* ext = exts.add_volumeextents(); + ext->set_fsoffset(0); + ext->set_length(4096); + ext->set_volumeoffset(8192); + ext->set_isused(true); + + inode.mutable_volumeextentmap()->insert({0, exts}); + + auto inodeWrapper = std::make_shared(inode, metaServerCli_); + + EXPECT_CALL(inodeCacheMgr_, GetInode(_, _)) + .WillOnce(Invoke([&](uint64_t, std::shared_ptr& out) { + out = inodeWrapper; + return CURVEFS_ERROR::OK; + })); + + EXPECT_CALL(blockDev_, Readv(_)) + .WillOnce(Return(-1)); + + uint64_t ino = 1; + off_t offset = 0; + size_t len = 4096; + std::unique_ptr data(new char[4096]); + + ASSERT_GT(0, storage_.Read(ino, offset, len, data.get())); +} + +TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadSuccess) { + Inode inode; + VolumeExtentList exts; + inode.set_type(FsFileType::TYPE_VOLUME); + + auto* ext = exts.add_volumeextents(); + ext->set_fsoffset(0); + ext->set_length(4096); + ext->set_volumeoffset(8192); + ext->set_isused(true); + + inode.mutable_volumeextentmap()->insert({0, exts}); + + auto inodeWrapper = std::make_shared(inode, metaServerCli_); + + EXPECT_CALL(inodeCacheMgr_, GetInode(_, _)) + .WillOnce(Invoke([&](uint64_t, std::shared_ptr& out) { + out = inodeWrapper; + return CURVEFS_ERROR::OK; + })); + + uint64_t ino = 1; + off_t offset = 0; + size_t len = 4096; + std::unique_ptr data(new char[4096]); + + EXPECT_CALL(blockDev_, Readv(_)) + .WillOnce(Return(len)); + + EXPECT_CALL(inodeCacheMgr_, ShipToFlush(inodeWrapper)) + .Times(1); + + ASSERT_EQ(len, storage_.Read(ino, offset, len, data.get())); +} + +TEST_F(DefaultVolumeStorageTest, ReadTest_BlockDevReadHoleSuccess) { + Inode inode; + VolumeExtentList exts; + inode.set_type(FsFileType::TYPE_VOLUME); + + auto* ext = exts.add_volumeextents(); + ext->set_fsoffset(0); + ext->set_length(4096); + ext->set_volumeoffset(8192); + ext->set_isused(false); + + inode.mutable_volumeextentmap()->insert({0, exts}); + + auto inodeWrapper = std::make_shared(inode, metaServerCli_); + + EXPECT_CALL(inodeCacheMgr_, GetInode(_, _)) + .WillOnce(Invoke([&](uint64_t, std::shared_ptr& out) { + out = inodeWrapper; + return CURVEFS_ERROR::OK; + })); + + uint64_t ino = 1; + off_t offset = 0; + size_t len = 4096; + std::unique_ptr data(new char[4096]); + + memset(data.get(), 'x', len); + + EXPECT_CALL(blockDev_, Readv(_)) + .Times(0); + + EXPECT_CALL(inodeCacheMgr_, ShipToFlush(inodeWrapper)) + .Times(1); + + ASSERT_EQ(len, storage_.Read(ino, offset, len, data.get())); + + for (size_t i = 0; i < len; ++i) { + ASSERT_EQ(data[i], 0); + } +} + +TEST_F(DefaultVolumeStorageTest, WriteTest_PrepareError) { + Inode inode; + inode.set_type(FsFileType::TYPE_VOLUME); + + auto inodeWrapper = std::make_shared(inode, metaServerCli_); + + EXPECT_CALL(inodeCacheMgr_, GetInode(_, _)) + .WillOnce(Invoke([&](uint64_t, std::shared_ptr& out) { + out = inodeWrapper; + return CURVEFS_ERROR::OK; + })); + + EXPECT_CALL(spaceMgr_, Alloc(_, _, _)) + .WillOnce(Return(false)); + + uint64_t ino = 1; + off_t offset = 0; + size_t len = 4096; + std::unique_ptr data(new char[4096]); + + ASSERT_GT(0, storage_.Write(ino, offset, len, data.get())); +} + +TEST_F(DefaultVolumeStorageTest, WriteTest_BlockDevWriteError) { + Inode inode; + inode.set_type(FsFileType::TYPE_VOLUME); + + auto inodeWrapper = std::make_shared(inode, metaServerCli_); + + EXPECT_CALL(inodeCacheMgr_, GetInode(_, _)) + .WillOnce(Invoke([&](uint64_t, std::shared_ptr& out) { + out = inodeWrapper; + return CURVEFS_ERROR::OK; + })); + + std::vector parts; + parts.push_back({}); + parts.push_back({}); + + EXPECT_CALL(spaceMgr_, Alloc(_, _, _)) + .WillOnce(Invoke( + [](uint32_t size, const AllocateHint&, std::vector* exts) { + exts->emplace_back(size, size); + return true; + })); + + uint64_t ino = 1; + off_t offset = 0; + size_t len = 4096; + std::unique_ptr data(new char[4096]); + + EXPECT_CALL(blockDev_, Writev(_)) + .WillOnce(Return(-1)); + + ASSERT_GT(0, storage_.Write(ino, offset, len, data.get())); +} + +TEST_F(DefaultVolumeStorageTest, WriteTest_BlockDevWriteSuccess) { + Inode inode; + inode.set_type(FsFileType::TYPE_VOLUME); + + auto inodeWrapper = std::make_shared(inode, metaServerCli_); + + EXPECT_CALL(inodeCacheMgr_, GetInode(_, _)) + .WillOnce(Invoke([&](uint64_t, std::shared_ptr& out) { + out = inodeWrapper; + return CURVEFS_ERROR::OK; + })); + + std::vector parts; + parts.push_back({}); + parts.push_back({}); + + EXPECT_CALL(spaceMgr_, Alloc(_, _, _)) + .WillOnce(Invoke( + [](uint32_t size, const AllocateHint&, std::vector* exts) { + exts->emplace_back(size, size); + return true; + })); + + uint64_t ino = 1; + off_t offset = 0; + size_t len = 4096; + std::unique_ptr data(new char[4096]); + + EXPECT_CALL(blockDev_, Writev(_)) + .WillOnce(Return(len)); + + EXPECT_CALL(inodeCacheMgr_, ShipToFlush(inodeWrapper)) + .Times(1); + + ASSERT_EQ(len, storage_.Write(ino, offset, len, data.get())); + + auto internal = inodeWrapper->GetInodeUnlocked(); + ASSERT_EQ(offset + len, internal.length()); +} + +} // namespace client +} // namespace curvefs diff --git a/curvefs/test/client/volume/extent_cache_mark_written_test.cpp b/curvefs/test/client/volume/extent_cache_mark_written_test.cpp new file mode 100644 index 0000000000..caf54def34 --- /dev/null +++ b/curvefs/test/client/volume/extent_cache_mark_written_test.cpp @@ -0,0 +1,683 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Wednesday Mar 16 10:32:58 CST 2022 + * Author: wuhanqing + */ + +#include + +#include "curvefs/src/client/volume/extent_cache.h" +#include "curvefs/test/client/volume/common.h" + +namespace curvefs { +namespace client { + +// write |----| |----| |----| |----| +// extent |----| |----| |----| |----| +TEST(ExtentCacheMarkWrittenTest, Case1_NonOverlap) { + using MarkWrittenRange = std::pair; + using ExtentRange = std::pair; + + bool unwritten = true; + + std::vector> tests{ + { + {0, 8 * kMiB}, + {8 * kMiB, 4 * kMiB}, + }, + { + {0, 8 * kMiB}, + {12 * kMiB, 4 * kMiB}, + }, + { + {4 * kMiB, 8 * kMiB}, + {0, 4 * kMiB}, + }, + { + {5 * kMiB, 8 * kMiB}, + {0, 4 * kMiB}, + }, + }; + + for (auto& test : tests) { + ExtentCache cache; + + PExtent pext; + pext.pOffset = 100 * kMiB; + pext.len = test.second.second; + pext.UnWritten = unwritten; + + cache.Merge(test.second.first, pext); + + cache.MarkWritten(test.first.first, test.first.second); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_EQ(1, extents.size()); + ASSERT_EQ(1, extents.count(0)); + + auto& range = extents[0]; + ASSERT_EQ(1, range.count(test.second.first)); + + auto& ext = range[test.second.first]; + ASSERT_EQ(test.second.second, ext.len); + ASSERT_EQ(pext.pOffset, ext.pOffset); + ASSERT_TRUE(ext.UnWritten); + } +} + +// write |-------| |--------| |--------| |--------| +// extent |----| |----| |----| |--------| +TEST(ExtentCacheMarkWrittenTest, Case2_Overlap) { + using MarkWrittenRange = std::pair; + using ExtentRange = std::pair; + + bool unwritten = true; + + std::vector> tests{ + { + {0, 8 * kMiB}, + {4 * kMiB, 4 * kMiB}, + }, + { + {0, 8 * kMiB}, + {2 * kMiB, 4 * kMiB}, + }, + { + {4 * kMiB, 8 * kMiB}, + {4 * kMiB, 2 * kMiB}, + }, + { + {0, 4 * kMiB}, + {0, 4 * kMiB}, + }, + }; + + for (auto& test : tests) { + ExtentCache cache; + + PExtent pext; + pext.pOffset = 100 * kMiB; + pext.len = test.second.second; + pext.UnWritten = unwritten; + + cache.Merge(test.second.first, pext); + + cache.MarkWritten(test.first.first, test.first.second); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_EQ(1, extents.size()); + ASSERT_EQ(1, extents.count(0)); + + auto& range = extents[0]; + ASSERT_EQ(1, range.count(test.second.first)); + + auto& ext = range[test.second.first]; + ASSERT_EQ(test.second.second, ext.len) << ext; + ASSERT_EQ(pext.pOffset, ext.pOffset) << ext; + ASSERT_FALSE(ext.UnWritten) << ext; + } +} + +// write |-------| |-----| |--------| +// extent |--------| |---------| |------------| +TEST(ExtentCacheMarkWrittenTest, Case3_Overlap) { + { + ExtentCache cache; + + PExtent pext; + pext.pOffset = 100 * kMiB; + pext.len = 4 * kMiB; + pext.UnWritten = true; + + cache.Merge(0, pext); + + cache.MarkWritten(0, 3 * kMiB); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_EQ(1, extents.size()); + ASSERT_EQ(1, extents.count(0)); + + auto& range = extents[0]; + ASSERT_EQ(2, range.size()); + + ASSERT_EQ(1, range.count(0 * kMiB)); + ASSERT_EQ(1, range.count(3 * kMiB)); + + auto& first = range[0 * kMiB]; + auto& second = range[3 * kMiB]; + + ASSERT_EQ(3 * kMiB, first.len); + ASSERT_EQ(100 * kMiB, first.pOffset); + ASSERT_FALSE(first.UnWritten); + + ASSERT_EQ(1 * kMiB, second.len); + ASSERT_EQ(103 * kMiB, second.pOffset); + ASSERT_TRUE(second.UnWritten); + } + + { + ExtentCache cache; + + PExtent pext; + pext.pOffset = 100 * kMiB, pext.len = 4 * kMiB; + pext.UnWritten = true; + + cache.Merge(0, pext); + + cache.MarkWritten(1 * kMiB, 3 * kMiB); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_EQ(1, extents.size()); + ASSERT_EQ(1, extents.count(0)); + + auto& range = extents[0]; + ASSERT_EQ(2, range.size()); + + ASSERT_EQ(1, range.count(0 * kMiB)); + ASSERT_EQ(1, range.count(1 * kMiB)); + + auto& first = range[0 * kMiB]; + auto& second = range[1 * kMiB]; + + ASSERT_EQ(1 * kMiB, first.len); + ASSERT_EQ(100 * kMiB, first.pOffset); + ASSERT_TRUE(first.UnWritten); + + ASSERT_EQ(3 * kMiB, second.len); + ASSERT_EQ(101 * kMiB, second.pOffset); + ASSERT_FALSE(second.UnWritten); + } + + { + ExtentCache cache; + + PExtent pext; + pext.pOffset = 100 * kMiB, pext.len = 4 * kMiB; + pext.UnWritten = true; + + cache.Merge(0, pext); + + cache.MarkWritten(1 * kMiB, 2 * kMiB); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_EQ(1, extents.size()); + ASSERT_EQ(1, extents.count(0)); + + auto& range = extents[0]; + ASSERT_EQ(3, range.size()); + + ASSERT_EQ(1, range.count(0 * kMiB)); + ASSERT_EQ(1, range.count(1 * kMiB)); + ASSERT_EQ(1, range.count(3 * kMiB)); + + auto& first = range[0 * kMiB]; + auto& second = range[1 * kMiB]; + auto& third = range[3 * kMiB]; + + ASSERT_EQ(1 * kMiB, first.len); + ASSERT_EQ(100 * kMiB, first.pOffset); + ASSERT_TRUE(first.UnWritten); + + ASSERT_EQ(2 * kMiB, second.len); + ASSERT_EQ(101 * kMiB, second.pOffset); + ASSERT_FALSE(second.UnWritten); + + ASSERT_EQ(1 * kMiB, third.len); + ASSERT_EQ(103 * kMiB, third.pOffset); + ASSERT_TRUE(third.UnWritten); + } +} + +// write |----| |-----| +// extent |----| |----| +TEST(ExtentCacheMarkWrittenTest, Case4_Overlap) { + { + ExtentCache cache; + + PExtent pext; + pext.len = 4 * kMiB; + pext.pOffset = 100 * kMiB; + pext.UnWritten = true; + + cache.Merge(8 * kMiB, pext); + + cache.MarkWritten(0, 10 * kMiB); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_EQ(1, extents.size()); + ASSERT_EQ(1, extents.count(0)); + + auto& range = extents[0]; + + ASSERT_EQ(2, range.size()); + ASSERT_EQ(1, range.count(8 * kMiB)); + ASSERT_EQ(1, range.count(10 * kMiB)); + + auto& first = range[8 * kMiB]; + auto& second = range[10 * kMiB]; + + ASSERT_EQ(2 * kMiB, first.len); + ASSERT_EQ(100 * kMiB, first.pOffset); + ASSERT_FALSE(first.UnWritten); + + ASSERT_EQ(2 * kMiB, second.len); + ASSERT_EQ(102 * kMiB, second.pOffset); + ASSERT_TRUE(second.UnWritten); + } + + { + ExtentCache cache; + + PExtent pext; + pext.len = 8 * kMiB; + pext.pOffset = 100 * kMiB; + pext.UnWritten = true; + + cache.Merge(0 * kMiB, pext); + + cache.MarkWritten(4 * kMiB, 10 * kMiB); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_EQ(1, extents.size()); + ASSERT_EQ(1, extents.count(0)); + + auto& range = extents[0]; + + ASSERT_EQ(2, range.size()); + ASSERT_EQ(1, range.count(0 * kMiB)); + ASSERT_EQ(1, range.count(4 * kMiB)); + + auto& first = range[0 * kMiB]; + auto& second = range[4 * kMiB]; + + ASSERT_EQ(4 * kMiB, first.len); + ASSERT_EQ(100 * kMiB, first.pOffset); + ASSERT_TRUE(first.UnWritten); + + ASSERT_EQ(4 * kMiB, second.len); + ASSERT_EQ(104 * kMiB, second.pOffset); + ASSERT_FALSE(second.UnWritten); + } +} + +// write |----| |---------| +// extent |----|----| |----|----| +TEST(ExtentCacheMarkWrittenTest, Case5_Mergeable) { + { + ExtentCache cache; + + // 8MiB ~ 12MiB + PExtent pext; + pext.len = 4 * kMiB; + pext.pOffset = 100 * kMiB; + pext.UnWritten = true; + + cache.Merge(8 * kMiB, pext); + + // 12MiB ~ 16MiB + pext.len = 4 * kMiB; + pext.pOffset = 104 * kMiB; + pext.UnWritten = true; + + cache.Merge(12 * kMiB, pext); + + // mark written 10MiB ~ 14MiB + cache.MarkWritten(10 * kMiB, 4 * kMiB); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_EQ(1, extents.size()); + ASSERT_EQ(1, extents.count(0)); + + auto& range = extents[0]; + + ASSERT_EQ(3, range.size()); + ASSERT_EQ(1, range.count(8 * kMiB)); + ASSERT_EQ(1, range.count(10 * kMiB)); + ASSERT_EQ(1, range.count(14 * kMiB)); + + auto& first = range[8 * kMiB]; + auto& second = range[10 * kMiB]; + auto& third = range[14 * kMiB]; + + ASSERT_EQ(2 * kMiB, first.len); + ASSERT_EQ(100 * kMiB, first.pOffset); + ASSERT_TRUE(first.UnWritten); + + ASSERT_EQ(4 * kMiB, second.len); + ASSERT_EQ(102 * kMiB, second.pOffset); + ASSERT_FALSE(second.UnWritten); + + ASSERT_EQ(2 * kMiB, third.len); + ASSERT_EQ(106 * kMiB, third.pOffset); + ASSERT_TRUE(third.UnWritten); + } + + { + ExtentCache cache; + + // 8MiB ~ 12MiB + PExtent pext; + pext.len = 4 * kMiB; + pext.pOffset = 100 * kMiB; + pext.UnWritten = true; + + cache.Merge(8 * kMiB, pext); + + // 12MiB ~ 16MiB + pext.len = 4 * kMiB; + pext.pOffset = 104 * kMiB; + pext.UnWritten = true; + + cache.Merge(12 * kMiB, pext); + + // mark written 8MiB ~ 16MiB + cache.MarkWritten(8 * kMiB, 8 * kMiB); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_EQ(1, extents.size()); + ASSERT_EQ(1, extents.count(0)); + + auto& range = extents[0]; + + ASSERT_EQ(1, range.size()); + + auto& first = range[8 * kMiB]; + + ASSERT_EQ(8 * kMiB, first.len); + ASSERT_EQ(100 * kMiB, first.pOffset); + ASSERT_FALSE(first.UnWritten); + } +} + +// write |----| |---------| +// extent |----|----| |----|----| +// physical offset is not continuous +TEST(ExtentCacheMarkWrittenTest, Case5_NotMerge) { + { + ExtentCache cache; + + // 8MiB ~ 12MiB + PExtent pext; + pext.len = 4 * kMiB; + pext.pOffset = 99 * kMiB; + pext.UnWritten = true; + + cache.Merge(8 * kMiB, pext); + + // 12MiB ~ 16MiB + pext.len = 4 * kMiB; + pext.pOffset = 104 * kMiB; + pext.UnWritten = true; + + cache.Merge(12 * kMiB, pext); + + // mark written 10MiB ~ 14MiB + cache.MarkWritten(10 * kMiB, 4 * kMiB); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_EQ(1, extents.size()); + ASSERT_EQ(1, extents.count(0)); + + auto& range = extents[0]; + + ASSERT_EQ(4, range.size()); + ASSERT_EQ(1, range.count(8 * kMiB)); + ASSERT_EQ(1, range.count(10 * kMiB)); + ASSERT_EQ(1, range.count(12 * kMiB)); + ASSERT_EQ(1, range.count(14 * kMiB)); + + auto& first = range[8 * kMiB]; + auto& second = range[10 * kMiB]; + auto& third = range[12 * kMiB]; + auto& fourth = range[14 * kMiB]; + + ASSERT_EQ(2 * kMiB, first.len); + ASSERT_EQ(99 * kMiB, first.pOffset); + ASSERT_TRUE(first.UnWritten); + + ASSERT_EQ(2 * kMiB, second.len); + ASSERT_EQ(101 * kMiB, second.pOffset); + ASSERT_FALSE(second.UnWritten); + + ASSERT_EQ(2 * kMiB, third.len); + ASSERT_EQ(104 * kMiB, third.pOffset); + ASSERT_FALSE(third.UnWritten); + + ASSERT_EQ(2 * kMiB, fourth.len); + ASSERT_EQ(106 * kMiB, fourth.pOffset); + ASSERT_TRUE(fourth.UnWritten); + } + + { + ExtentCache cache; + + // 8MiB ~ 12MiB + PExtent pext; + pext.len = 4 * kMiB; + pext.pOffset = 99 * kMiB; + pext.UnWritten = true; + + cache.Merge(8 * kMiB, pext); + + // 12MiB ~ 16MiB + pext.len = 4 * kMiB; + pext.pOffset = 104 * kMiB; + pext.UnWritten = true; + + cache.Merge(12 * kMiB, pext); + + // mark written 8MiB ~ 16MiB + cache.MarkWritten(8 * kMiB, 8 * kMiB); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_EQ(1, extents.size()); + ASSERT_EQ(1, extents.count(0)); + + auto& range = extents[0]; + + ASSERT_EQ(2, range.size()); + + auto& first = range[8 * kMiB]; + auto& second = range[12 * kMiB]; + + ASSERT_EQ(4 * kMiB, first.len); + ASSERT_EQ(99 * kMiB, first.pOffset); + ASSERT_FALSE(first.UnWritten); + + ASSERT_EQ(4 * kMiB, second.len); + ASSERT_EQ(104 * kMiB, second.pOffset); + ASSERT_FALSE(second.UnWritten); + } +} + +// write |-------------------| +// extent |----|----|----|----| +// ^^^^ ^^^^ +// written written +TEST(ExtentCacheMarkWrittenTest, Case6_Mergeable) { + ExtentCache cache; + + // 8MiB ~ 12MiB + PExtent pext; + pext.len = 4 * kMiB; + pext.pOffset = 100 * kMiB; + pext.UnWritten = true; + + cache.Merge(8 * kMiB, pext); + + // 12MiB ~ 16MiB + pext.len = 4 * kMiB; + pext.pOffset = 104 * kMiB; + pext.UnWritten = true; + + cache.Merge(12 * kMiB, pext); + + // 16MiB ~ 20MiB + pext.len = 4 * kMiB; + pext.pOffset = 108 * kMiB; + pext.UnWritten = true; + + cache.Merge(16 * kMiB, pext); + + // 20MiB ~ 24MiB + pext.len = 4 * kMiB; + pext.pOffset = 112 * kMiB; + pext.UnWritten = true; + + cache.Merge(20 * kMiB, pext); + + // mark written 8MiB ~ 12MiB + cache.MarkWritten(8 * kMiB, 4 * kMiB); + + // mark written 20MiB ~ 24MiB + cache.MarkWritten(20 * kMiB, 4 * kMiB); + + // mark written 8MiB ~ 24MiB + cache.MarkWritten(8 * kMiB, 16 * kMiB); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_EQ(1, extents.size()); + ASSERT_EQ(1, extents.count(0)); + + auto& range = extents[0]; + + ASSERT_EQ(1, range.size()); + + auto& first = range[8 * kMiB]; + + ASSERT_EQ(16 * kMiB, first.len); + ASSERT_EQ(100 * kMiB, first.pOffset); + ASSERT_FALSE(first.UnWritten); +} + +// write |--------------| +// extent |----|----|----|----| +// ^^^^ ^^^^ +// written written +// in this case, the rightmost extent is not merged +TEST(ExtentCacheMarkWrittenTest, Case7) { + ExtentCache cache; + + // 8MiB ~ 12MiB + PExtent pext; + pext.len = 4 * kMiB; + pext.pOffset = 100 * kMiB; + pext.UnWritten = true; + + cache.Merge(8 * kMiB, pext); + + // 12MiB ~ 16MiB + pext.len = 4 * kMiB; + pext.pOffset = 104 * kMiB; + pext.UnWritten = true; + + cache.Merge(12 * kMiB, pext); + + // 16MiB ~ 20MiB + pext.len = 4 * kMiB; + pext.pOffset = 108 * kMiB; + pext.UnWritten = true; + + cache.Merge(16 * kMiB, pext); + + // 20MiB ~ 24MiB + pext.len = 4 * kMiB; + pext.pOffset = 112 * kMiB; + pext.UnWritten = true; + + cache.Merge(20 * kMiB, pext); + + // mark written 8MiB ~ 12MiB + cache.MarkWritten(8 * kMiB, 4 * kMiB); + + // mark written 20MiB ~ 24MiB + cache.MarkWritten(20 * kMiB, 4 * kMiB); + + // mark written 8MiB ~ 20MiB + cache.MarkWritten(8 * kMiB, 12 * kMiB); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_EQ(1, extents.size()); + ASSERT_EQ(1, extents.count(0)); + + auto& range = extents[0]; + + ASSERT_EQ(2, range.size()); + + auto& first = range[8 * kMiB]; + auto& second = range[20 * kMiB]; + + ASSERT_EQ(12 * kMiB, first.len); + ASSERT_EQ(100 * kMiB, first.pOffset); + ASSERT_FALSE(first.UnWritten); + + ASSERT_EQ(4 * kMiB, second.len); + ASSERT_EQ(112 * kMiB, second.pOffset); + ASSERT_FALSE(second.UnWritten); +} + +// write |-------| |-----| |--------| +// extent |--------| |---------| |------------| +TEST(ExtentCacheMarkWrittenTest, Case3_Overlap_Unaligned) { + ExtentCache cache; + + PExtent pext; + pext.pOffset = 100 * kMiB; + pext.len = 4 * kKiB; + pext.UnWritten = true; + + cache.Merge(0, pext); + + // only first 6 bytes are written + cache.MarkWritten(0, 6); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_EQ(1, extents.size()); + ASSERT_EQ(1, extents.count(0)); + + auto& range = extents[0]; + ASSERT_EQ(1, range.size()); + + ASSERT_EQ(1, range.count(0)); + + auto& first = range[0]; + + ASSERT_EQ(4 * kKiB, first.len); + ASSERT_EQ(100 * kMiB, first.pOffset); + ASSERT_FALSE(first.UnWritten); + + // ASSERT_EQ(2, range.size()); + + // ASSERT_EQ(1, range.count(0)); + // ASSERT_EQ(1, range.count(6)); + + // auto& first = range[0]; + // auto& second = range[6]; + + // ASSERT_EQ(6, first.len); + // ASSERT_EQ(100 * kMiB, first.pOffset); + // ASSERT_FALSE(first.UnWritten); + + // ASSERT_EQ(4 * kKiB - 6, second.len); + // ASSERT_EQ(100 * kMiB + 6, second.pOffset); + // ASSERT_TRUE(second.UnWritten); +} + +} // namespace client +} // namespace curvefs diff --git a/curvefs/test/client/volume/extent_cache_merge_test.cpp b/curvefs/test/client/volume/extent_cache_merge_test.cpp new file mode 100644 index 0000000000..4c43f2d0e3 --- /dev/null +++ b/curvefs/test/client/volume/extent_cache_merge_test.cpp @@ -0,0 +1,414 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Tuesday Mar 15 20:07:11 CST 2022 + * Author: wuhanqing + */ + +#include + +#include "curvefs/src/client/volume/extent_cache.h" +#include "curvefs/test/client/volume/common.h" + +namespace curvefs { +namespace client { + +// no extents +TEST(ExtentCacheMergeTest, MergeCase1) { + ExtentCache cache; + + PExtent pext; + pext.pOffset = 16 * kMiB; + pext.len = 4 * kMiB; + pext.UnWritten = false; + + cache.Merge(32 * kMiB, pext); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_TRUE(extents.count(0)); + + auto& range = extents[0]; + ASSERT_EQ(1, range.size()); + ASSERT_TRUE(range.count(32 * kMiB)); + + ASSERT_EQ(pext, range[32 * kMiB]); +} + +// adding extent |----| +// existing extents |----| +TEST(ExtentCacheMergeTest, MergeCase2) { + ExtentCache cache; + + PExtent existing; + existing.pOffset = 16 * kMiB; + existing.len = 4 * kMiB; + existing.UnWritten = false; + + cache.Merge(32 * kMiB, existing); + + PExtent adding; + adding.pOffset = 100 * kMiB; + adding.len = 4 * kMiB; + adding.UnWritten = false; + + cache.Merge(16 * kMiB, adding); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_TRUE(extents.count(0)); + + auto& range = extents[0]; + ASSERT_EQ(2, range.size()); + ASSERT_TRUE(range.count(16 * kMiB)); + ASSERT_TRUE(range.count(32 * kMiB)); + + ASSERT_EQ(existing, range[32 * kMiB]); + ASSERT_EQ(adding, range[16 * kMiB]); +} + +// adding extent |----| +// existing extents |----| +TEST(ExtentCacheMergeTest, MergeCase3_PhysicalOffsetNotContinuous) { + ExtentCache cache; + + PExtent existing; + existing.pOffset = 16 * kMiB; + existing.len = 4 * kMiB; + existing.UnWritten = false; + + cache.Merge(20 * kMiB, existing); + + PExtent adding; + adding.pOffset = 100 * kMiB; + adding.len = 4 * kMiB; + adding.UnWritten = false; + + cache.Merge(16 * kMiB, adding); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_TRUE(extents.count(0)); + + auto& range = extents[0]; + ASSERT_EQ(2, range.size()); + ASSERT_TRUE(range.count(16 * kMiB)); + ASSERT_TRUE(range.count(20 * kMiB)); + + ASSERT_EQ(existing, range[20 * kMiB]); + ASSERT_EQ(adding, range[16 * kMiB]); +} + +// adding extent |----| +// existing extents |----| +TEST(ExtentCacheMergeTest, MergeCase3_PhysicalOffsetContinuous) { + ExtentCache cache; + + PExtent existing; + existing.pOffset = 104 * kMiB; + existing.len = 4 * kMiB; + existing.UnWritten = false; + + cache.Merge(20 * kMiB, existing); + + PExtent adding; + adding.pOffset = 100 * kMiB; + adding.len = 4 * kMiB; + adding.UnWritten = false; + + cache.Merge(16 * kMiB, adding); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_TRUE(extents.count(0)); + + auto& range = extents[0]; + ASSERT_EQ(1, range.size()); + ASSERT_TRUE(range.count(16 * kMiB)); + ASSERT_FALSE(range.count(20 * kMiB)); + + ASSERT_EQ(100 * kMiB, range[16 * kMiB].pOffset); + ASSERT_EQ(8 * kMiB, range[16 * kMiB].len); + ASSERT_FALSE(range[16 * kMiB].UnWritten); +} + +// adding extent |----| +// existing extents |----| +TEST(ExtentCacheMergeTest, MergeCase3_PhysicalOffsetContinuousButNoWritten) { + ExtentCache cache; + + PExtent existing; + existing.pOffset = 104 * kMiB; + existing.len = 4 * kMiB; + existing.UnWritten = true; + + cache.Merge(20 * kMiB, existing); + + PExtent adding; + adding.pOffset = 100 * kMiB; + adding.len = 4 * kMiB; + adding.UnWritten = false; + + cache.Merge(16 * kMiB, adding); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_TRUE(extents.count(0)); + + auto& range = extents[0]; + ASSERT_EQ(2, range.size()); + ASSERT_TRUE(range.count(16 * kMiB)); + ASSERT_TRUE(range.count(20 * kMiB)); + + ASSERT_EQ(100 * kMiB, range[16 * kMiB].pOffset); + ASSERT_EQ(4 * kMiB, range[16 * kMiB].len); + ASSERT_FALSE(range[16 * kMiB].UnWritten); + + ASSERT_EQ(104 * kMiB, range[20 * kMiB].pOffset); + ASSERT_EQ(4 * kMiB, range[20 * kMiB].len); + ASSERT_TRUE(range[20 * kMiB].UnWritten); +} + +// adding extent |----| +// existing extents |----| +TEST(ExtentCacheMergeTest, MergeCase4_PhysicalOffsetContinuous) { + ExtentCache cache; + + PExtent existing; + existing.pOffset = 96 * kMiB; + existing.len = 4 * kMiB; + existing.UnWritten = false; + + cache.Merge(16 * kMiB, existing); + + PExtent adding; + adding.pOffset = 100 * kMiB; + adding.len = 4 * kMiB; + adding.UnWritten = false; + + cache.Merge(20 * kMiB, adding); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_TRUE(extents.count(0)); + + auto& range = extents[0]; + ASSERT_EQ(1, range.size()); + ASSERT_TRUE(range.count(16 * kMiB)); + + ASSERT_EQ(96 * kMiB, range[16 * kMiB].pOffset); + ASSERT_EQ(8 * kMiB, range[16 * kMiB].len); + ASSERT_FALSE(range[16 * kMiB].UnWritten); + + // ASSERT_EQ(104 * kMiB, range[20 * kMiB].pOffset); + // ASSERT_EQ(4 * kMiB, range[20 * kMiB].len); + // ASSERT_TRUE(range[20 * kMiB].UnWritten); +} + +// adding extent |----| +// existing extents |----| +TEST(ExtentCacheMergeTest, MergeCase4_PhysicalNotNotContinuous) { + ExtentCache cache; + + PExtent existing; + existing.pOffset = 96 * kMiB; + existing.len = 4 * kMiB; + existing.UnWritten = false; + + cache.Merge(16 * kMiB, existing); + + PExtent adding; + adding.pOffset = 200 * kMiB; + adding.len = 4 * kMiB; + adding.UnWritten = false; + + cache.Merge(20 * kMiB, adding); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_EQ(1, extents.count(0)); + + auto& range = extents[0]; + ASSERT_EQ(2, range.size()); + ASSERT_EQ(1, range.count(16 * kMiB)); + ASSERT_EQ(1, range.count(20 * kMiB)); + + ASSERT_EQ(96 * kMiB, range[16 * kMiB].pOffset); + ASSERT_EQ(4 * kMiB, range[16 * kMiB].len); + ASSERT_FALSE(range[16 * kMiB].UnWritten); + + ASSERT_EQ(200 * kMiB, range[20 * kMiB].pOffset); + ASSERT_EQ(4 * kMiB, range[20 * kMiB].len); + ASSERT_FALSE(range[20 * kMiB].UnWritten); +} + +// adding extent |----| +// existing extents |----| |----| +TEST(ExtentCacheMergeTest, MergeCase5_PhysicalOffsetContinuous) { + ExtentCache cache; + + PExtent existing; + existing.pOffset = 96 * kMiB; + existing.len = 4 * kMiB; + existing.UnWritten = false; + cache.Merge(16 * kMiB, existing); + existing.pOffset = 104 * kMiB; + existing.len = 4 * kMiB; + existing.UnWritten = false; + cache.Merge(24 * kMiB, existing); + + PExtent adding; + adding.pOffset = 100 * kMiB; + adding.len = 4 * kMiB; + adding.UnWritten = false; + + cache.Merge(20 * kMiB, adding); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_TRUE(extents.count(0)); + + auto& range = extents[0]; + ASSERT_EQ(1, range.size()); + ASSERT_EQ(1, range.count(16 * kMiB)); + ASSERT_EQ(0, range.count(20 * kMiB)); + ASSERT_EQ(0, range.count(24 * kMiB)); + + ASSERT_EQ(96 * kMiB, range[16 * kMiB].pOffset); + ASSERT_EQ(12 * kMiB, range[16 * kMiB].len); + ASSERT_FALSE(range[16 * kMiB].UnWritten); +} + +// adding extent |----| +// existing extents |----| |----| +TEST(ExtentCacheMergeTest, MergeCase5_RightExtentPhysicalOffsetNotContinuous) { + ExtentCache cache; + + PExtent existing; + existing.pOffset = 96 * kMiB; + existing.len = 4 * kMiB; + existing.UnWritten = false; + cache.Merge(16 * kMiB, existing); + existing.pOffset = 105 * kMiB; + existing.len = 4 * kMiB; + existing.UnWritten = false; + cache.Merge(24 * kMiB, existing); + + PExtent adding; + adding.pOffset = 100 * kMiB; + adding.len = 4 * kMiB; + adding.UnWritten = false; + + cache.Merge(20 * kMiB, adding); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_EQ(1, extents.count(0)); + + auto& range = extents[0]; + ASSERT_EQ(2, range.size()); + ASSERT_EQ(1, range.count(16 * kMiB)); + ASSERT_EQ(0, range.count(20 * kMiB)); + ASSERT_EQ(1, range.count(24 * kMiB)); + + ASSERT_EQ(96 * kMiB, range[16 * kMiB].pOffset); + ASSERT_EQ(8 * kMiB, range[16 * kMiB].len); + ASSERT_FALSE(range[16 * kMiB].UnWritten); + + ASSERT_EQ(105 * kMiB, range[24 * kMiB].pOffset); + ASSERT_EQ(4 * kMiB, range[24 * kMiB].len); + ASSERT_FALSE(range[24 * kMiB].UnWritten); +} + +// adding extent |----| +// existing extents |----| |----| +TEST(ExtentCacheMergeTest, MergeCase5_LeftExtentPhysicalOffsetNotContinuous) { + ExtentCache cache; + + PExtent existing; + existing.pOffset = 95 * kMiB; + existing.len = 4 * kMiB; + existing.UnWritten = false; + cache.Merge(16 * kMiB, existing); + existing.pOffset = 104 * kMiB; + existing.len = 4 * kMiB; + existing.UnWritten = false; + cache.Merge(24 * kMiB, existing); + + PExtent adding; + adding.pOffset = 100 * kMiB; + adding.len = 4 * kMiB; + adding.UnWritten = false; + + cache.Merge(20 * kMiB, adding); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_EQ(1, extents.count(0)); + + auto& range = extents[0]; + ASSERT_EQ(2, range.size()); + ASSERT_EQ(1, range.count(16 * kMiB)); + ASSERT_EQ(1, range.count(20 * kMiB)); + ASSERT_EQ(0, range.count(24 * kMiB)); + + ASSERT_EQ(95 * kMiB, range[16 * kMiB].pOffset); + ASSERT_EQ(4 * kMiB, range[16 * kMiB].len); + ASSERT_FALSE(range[16 * kMiB].UnWritten); + + ASSERT_EQ(100 * kMiB, range[20 * kMiB].pOffset); + ASSERT_EQ(8 * kMiB, range[20 * kMiB].len); + ASSERT_FALSE(range[20 * kMiB].UnWritten); +} + +// adding extent |----| +// existing extents |----| |----| +TEST(ExtentCacheMergeTest, MergeCase5_PhysicalOffsetContinuousButNotWritten) { + ExtentCache cache; + + PExtent existing; + existing.pOffset = 96 * kMiB; + existing.len = 4 * kMiB; + existing.UnWritten = true; + cache.Merge(16 * kMiB, existing); + existing.pOffset = 104 * kMiB; + existing.len = 4 * kMiB; + existing.UnWritten = true; + cache.Merge(24 * kMiB, existing); + + PExtent adding; + adding.pOffset = 100 * kMiB; + adding.len = 4 * kMiB; + adding.UnWritten = false; + + cache.Merge(20 * kMiB, adding); + + auto extents = cache.GetExtentsForTesting(); + ASSERT_EQ(1, extents.count(0)); + + auto& range = extents[0]; + ASSERT_EQ(3, range.size()); + ASSERT_EQ(1, range.count(16 * kMiB)); + ASSERT_EQ(1, range.count(20 * kMiB)); + ASSERT_EQ(1, range.count(24 * kMiB)); + + ASSERT_EQ(96 * kMiB, range[16 * kMiB].pOffset); + ASSERT_EQ(4 * kMiB, range[16 * kMiB].len); + ASSERT_TRUE(range[16 * kMiB].UnWritten); + + ASSERT_EQ(100 * kMiB, range[20 * kMiB].pOffset); + ASSERT_EQ(4 * kMiB, range[20 * kMiB].len); + ASSERT_FALSE(range[20 * kMiB].UnWritten); + + ASSERT_EQ(104 * kMiB, range[24 * kMiB].pOffset); + ASSERT_EQ(4 * kMiB, range[24 * kMiB].len); + ASSERT_TRUE(range[24 * kMiB].UnWritten); +} + +} // namespace client +} // namespace curvefs diff --git a/curvefs/test/client/volume/extent_cache_read_divide_test.cpp b/curvefs/test/client/volume/extent_cache_read_divide_test.cpp new file mode 100644 index 0000000000..d2119cca3d --- /dev/null +++ b/curvefs/test/client/volume/extent_cache_read_divide_test.cpp @@ -0,0 +1,458 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Wednesday Mar 18 16:32:59 CST 2022 + * Author: wuhanqing + */ + +#include + +#include + +#include "curvefs/src/client/volume/extent_cache.h" +#include "curvefs/test/client/volume/common.h" + +namespace curvefs { +namespace client { + +TEST(ExtentCacheReadDivideTest, DivideWhenNoExtents) { + ExtentCache cache; + std::vector reads; + std::vector holes; + + off_t offset = 4 * kKiB; + size_t length = 4 * kMiB; + + std::unique_ptr data(new char[length]); + + cache.DivideForRead(offset, length, data.get(), &reads, &holes); + + ASSERT_TRUE(reads.empty()); + ASSERT_EQ(1, holes.size()); + + ASSERT_EQ(offset, holes[0].offset); + ASSERT_EQ(length, holes[0].length); + ASSERT_EQ(data.get(), holes[0].data); +} + +// read |----| |----| +// extent |----| |----| +TEST(ExtentCacheReadDivideTest, DivideCase1) { + for (auto& ext : {std::make_pair(4 * kKiB, 4 * kKiB), + std::make_pair(8 * kKiB, 4 * kKiB)}) { + ExtentCache cache; + + PExtent pext; + pext.pOffset = 10 * kMiB; + pext.len = ext.second; + + cache.Merge(ext.first, pext); + + off_t offset = 0 * kKiB; + size_t length = 4 * kKiB; + + std::unique_ptr data(new char[length]); + + std::vector reads; + std::vector holes; + cache.DivideForRead(offset, length, data.get(), &reads, &holes); + + ASSERT_TRUE(reads.empty()); + ASSERT_EQ(1, holes.size()); + + ASSERT_EQ(offset, holes[0].offset); + ASSERT_EQ(length, holes[0].length); + ASSERT_EQ(data.get(), holes[0].data); + } +} + +// read |----| |----| +// extent |----| |----| +TEST(ExtentCacheReadDivideTest, DivideCase2) { + for (auto& ext : {std::make_pair(4 * kKiB, 4 * kKiB), + std::make_pair(8 * kKiB, 4 * kKiB)}) { + ExtentCache cache; + + PExtent pext; + pext.pOffset = 10 * kMiB; + pext.len = 4 * kKiB; + + cache.Merge(0 * kKiB, pext); + + off_t offset = ext.first; + size_t length = ext.second; + + std::unique_ptr data(new char[length]); + + std::vector reads; + std::vector holes; + cache.DivideForRead(offset, length, data.get(), &reads, &holes); + + ASSERT_TRUE(reads.empty()); + ASSERT_EQ(1, holes.size()); + + ASSERT_EQ(offset, holes[0].offset); + ASSERT_EQ(length, holes[0].length); + ASSERT_EQ(data.get(), holes[0].data); + } +} + +// read |----| +// extent |----| +TEST(ExtentCacheReadDivideTest, DivideCase3) { + ExtentCache cache; + + PExtent pext; + pext.len = 4 * kKiB; + pext.UnWritten = false; + pext.pOffset = 4 * kMiB; + + cache.Merge(4 * kKiB, pext); + + off_t offset = 0 * kKiB; + size_t length = 8 * kKiB; + std::unique_ptr data(new char[length]); + + std::vector reads; + std::vector holes; + cache.DivideForRead(offset, length, data.get(), &reads, &holes); + + ASSERT_EQ(1, reads.size()); + ASSERT_EQ(1, holes.size()); + + ASSERT_EQ(4 * kMiB, reads[0].offset); + ASSERT_EQ(4 * kKiB, reads[0].length); + ASSERT_EQ(data.get() + 4 * kKiB, reads[0].data); + + ASSERT_EQ(offset, holes[0].offset); + ASSERT_EQ(4 * kKiB, holes[0].length); + ASSERT_EQ(data.get(), holes[0].data); +} + +// read |----| +// extent |----| +TEST(ExtentCacheReadDivideTest, DivideCase3_UnWritten) { + ExtentCache cache; + + PExtent pext; + pext.len = 4 * kKiB; + pext.UnWritten = true; // existing extent is unwritten + pext.pOffset = 4 * kMiB; + + cache.Merge(4 * kKiB, pext); + + off_t offset = 0 * kKiB; + size_t length = 8 * kKiB; + std::unique_ptr data(new char[length]); + + std::vector reads; + std::vector holes; + cache.DivideForRead(offset, length, data.get(), &reads, &holes); + + ASSERT_EQ(0, reads.size()); + ASSERT_EQ(2, holes.size()); + + ASSERT_EQ(0 * kKiB, holes[0].offset); + ASSERT_EQ(4 * kKiB, holes[0].length); + ASSERT_EQ(data.get(), holes[0].data); + + ASSERT_EQ(4 * kKiB, holes[1].offset); + ASSERT_EQ(4 * kKiB, holes[1].length); + ASSERT_EQ(data.get() + 4 * kKiB, holes[1].data); +} + +// read |-------| +// extent |----| +TEST(ExtentCacheReadDivideTest, DivideCase4) { + ExtentCache cache; + + PExtent pext; + pext.len = 8 * kKiB; + pext.UnWritten = false; + pext.pOffset = 4 * kMiB; + + cache.Merge(4 * kKiB, pext); + + off_t offset = 0 * kKiB; + size_t length = 12 * kKiB; + std::unique_ptr data(new char[length]); + + std::vector reads; + std::vector holes; + cache.DivideForRead(offset, length, data.get(), &reads, &holes); + + ASSERT_EQ(1, reads.size()); + ASSERT_EQ(1, holes.size()); + + ASSERT_EQ(4 * kMiB, reads[0].offset); + ASSERT_EQ(8 * kKiB, reads[0].length); + ASSERT_EQ(data.get() + 4 * kKiB, reads[0].data); + + ASSERT_EQ(0 * kKiB, holes[0].offset); + ASSERT_EQ(4 * kKiB, holes[0].length); + ASSERT_EQ(data.get(), holes[0].data); +} + +// read |-------| +// extent |----| +TEST(ExtentCacheReadDivideTest, DivideCase4_UnWritten) { + ExtentCache cache; + + PExtent pext; + pext.len = 8 * kKiB; + pext.UnWritten = true; + pext.pOffset = 4 * kMiB; + + cache.Merge(4 * kKiB, pext); + + off_t offset = 0 * kKiB; + size_t length = 12 * kKiB; + std::unique_ptr data(new char[length]); + + std::vector reads; + std::vector holes; + cache.DivideForRead(offset, length, data.get(), &reads, &holes); + + ASSERT_EQ(0, reads.size()); + ASSERT_EQ(2, holes.size()); + + ASSERT_EQ(0 * kKiB, holes[0].offset); + ASSERT_EQ(4 * kKiB, holes[0].length); + ASSERT_EQ(data.get(), holes[0].data); + + ASSERT_EQ(4 * kKiB, holes[1].offset); + ASSERT_EQ(8 * kKiB, holes[1].length); + ASSERT_EQ(data.get() + 4 * kKiB, holes[1].data); +} + +// read |---------| +// extent |----| +TEST(ExtentCacheReadDivideTest, DivideCase5) { + ExtentCache cache; + + PExtent pext; + pext.len = 8 * kKiB; + pext.UnWritten = false; + pext.pOffset = 4 * kMiB; + + cache.Merge(4 * kKiB, pext); + + off_t offset = 0 * kKiB; + size_t length = 16 * kKiB; + std::unique_ptr data(new char[length]); + + std::vector reads; + std::vector holes; + cache.DivideForRead(offset, length, data.get(), &reads, &holes); + + ASSERT_EQ(1, reads.size()); + ASSERT_EQ(2, holes.size()); + + ASSERT_EQ(0 * kKiB, holes[0].offset); + ASSERT_EQ(4 * kKiB, holes[0].length); + ASSERT_EQ(data.get(), holes[0].data); + + ASSERT_EQ(4 * kMiB, reads[0].offset); + ASSERT_EQ(8 * kKiB, reads[0].length); + ASSERT_EQ(data.get() + 4 * kKiB, reads[0].data); + + ASSERT_EQ(12 * kKiB, holes[1].offset); + ASSERT_EQ(4 * kKiB, holes[1].length); + ASSERT_EQ(data.get() + 12 * kKiB, holes[1].data); +} + +// read |---------| +// extent |----| +TEST(ExtentCacheReadDivideTest, DivideCase5_UnWritten) { + ExtentCache cache; + + PExtent pext; + pext.len = 8 * kKiB; + pext.UnWritten = true; + pext.pOffset = 4 * kMiB; + + cache.Merge(4 * kKiB, pext); + + off_t offset = 0 * kKiB; + size_t length = 16 * kKiB; + std::unique_ptr data(new char[length]); + + std::vector reads; + std::vector holes; + cache.DivideForRead(offset, length, data.get(), &reads, &holes); + + ASSERT_EQ(0, reads.size()); + ASSERT_EQ(3, holes.size()); + + ASSERT_EQ(0 * kKiB, holes[0].offset); + ASSERT_EQ(4 * kKiB, holes[0].length); + ASSERT_EQ(data.get(), holes[0].data); + + ASSERT_EQ(4 * kKiB, holes[1].offset); + ASSERT_EQ(8 * kKiB, holes[1].length); + ASSERT_EQ(data.get() + 4 * kKiB, holes[1].data); + + ASSERT_EQ(12 * kKiB, holes[2].offset); + ASSERT_EQ(4 * kKiB, holes[2].length); + ASSERT_EQ(data.get() + 12 * kKiB, holes[2].data); +} + +// read |----| +// extents |------| +TEST(ExtentCacheReadDivideTest, DivideCase6) { + for (auto written : {true, false}) { + ExtentCache cache; + + PExtent pext; + pext.len = 8 * kKiB; + pext.UnWritten = !written; + pext.pOffset = 4 * kMiB; + + cache.Merge(4 * kKiB, pext); + + off_t offset = 8 * kKiB; + size_t length = 4 * kKiB; + std::unique_ptr data(new char[length]); + + std::vector reads; + std::vector holes; + cache.DivideForRead(offset, length, data.get(), &reads, &holes); + + if (written) { + ASSERT_EQ(1, reads.size()); + ASSERT_TRUE(holes.empty()); + + ASSERT_EQ(4 * kMiB + 4 * kKiB, reads[0].offset); + ASSERT_EQ(4 * kKiB, reads[0].length); + ASSERT_EQ(data.get(), reads[0].data); + } else { + ASSERT_TRUE(reads.empty()); + ASSERT_EQ(1, holes.size()); + + ASSERT_EQ(offset, holes[0].offset); + ASSERT_EQ(length, holes[0].length); + ASSERT_EQ(data.get(), holes[0].data); + } + } +} + +// read |----| +// extents |--------| +TEST(ExtentCacheReadDivideTest, DivideCase7) { + for (auto written : {true, false}) { + ExtentCache cache; + + PExtent pext; + pext.len = 12 * kKiB; + pext.UnWritten = !written; + pext.pOffset = 4 * kMiB; + + cache.Merge(4 * kKiB, pext); + + off_t offset = 8 * kKiB; + size_t length = 4 * kKiB; + std::unique_ptr data(new char[length]); + + std::vector reads; + std::vector holes; + cache.DivideForRead(offset, length, data.get(), &reads, &holes); + + if (written) { + ASSERT_EQ(1, reads.size()); + ASSERT_TRUE(holes.empty()); + + ASSERT_EQ(4 * kMiB + 4 * kKiB, reads[0].offset); + ASSERT_EQ(4 * kKiB, reads[0].length); + ASSERT_EQ(data.get(), reads[0].data); + } else { + ASSERT_TRUE(reads.empty()); + ASSERT_EQ(1, holes.size()); + + ASSERT_EQ(offset, holes[0].offset); + ASSERT_EQ(length, holes[0].length); + ASSERT_EQ(data.get(), holes[0].data); + } + } +} + +// read |----| +// extents |----| +TEST(ExtentCacheReadDivideTest, DivideCase8) { + ExtentCache cache; + + PExtent pext; + pext.len = 8 * kKiB; + pext.UnWritten = false; + pext.pOffset = 4 * kMiB; + + cache.Merge(0 * kKiB, pext); + + off_t offset = 4 * kKiB; + size_t length = 8 * kKiB; + std::unique_ptr data(new char[length]); + + std::vector reads; + std::vector holes; + cache.DivideForRead(offset, length, data.get(), &reads, &holes); + + ASSERT_EQ(1, reads.size()); + ASSERT_EQ(1, holes.size()); + + ASSERT_EQ(4 * kMiB + 4 * kKiB, reads[0].offset); + ASSERT_EQ(4 * kKiB, reads[0].length); + ASSERT_EQ(data.get(), reads[0].data); + + ASSERT_EQ(8 * kKiB, holes[0].offset); + ASSERT_EQ(4 * kKiB, holes[0].length); + ASSERT_EQ(data.get() + 4 * kKiB, holes[0].data); +} + +// read |----| +// extents |----| +TEST(ExtentCacheReadDivideTest, DivideCase8_UnWritten) { + ExtentCache cache; + + PExtent pext; + pext.len = 8 * kKiB; + pext.UnWritten = true; + pext.pOffset = 4 * kMiB; + + cache.Merge(0 * kKiB, pext); + + off_t offset = 4 * kKiB; + size_t length = 8 * kKiB; + std::unique_ptr data(new char[length]); + + std::vector reads; + std::vector holes; + cache.DivideForRead(offset, length, data.get(), &reads, &holes); + + ASSERT_EQ(0, reads.size()); + ASSERT_EQ(2, holes.size()); + + ASSERT_EQ(4 * kKiB, holes[0].offset); + ASSERT_EQ(4 * kKiB, holes[0].length); + ASSERT_EQ(data.get(), holes[0].data); + + ASSERT_EQ(8 * kKiB, holes[1].offset); + ASSERT_EQ(4 * kKiB, holes[1].length); + ASSERT_EQ(data.get() + 4 * kKiB, holes[1].data); +} + +} // namespace client +} // namespace curvefs diff --git a/curvefs/test/client/volume/extent_cache_to_inode_pb_test.cpp b/curvefs/test/client/volume/extent_cache_to_inode_pb_test.cpp new file mode 100644 index 0000000000..84c05f9cbf --- /dev/null +++ b/curvefs/test/client/volume/extent_cache_to_inode_pb_test.cpp @@ -0,0 +1,244 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Friday Mar 25 17:24:18 CST 2022 + * Author: wuhanqing + */ + +#include + +#include "curvefs/src/client/volume/extent_cache.h" +#include "curvefs/test/client/volume/common.h" + +namespace curvefs { +namespace client { + +TEST(ExtentCacheToInodePbTest, EmptyTest) { + ExtentCache cache; + auto pb = cache.ToInodePb(); + ASSERT_TRUE(pb.empty()); +} + +TEST(ExtentCacheToInodePbTest, Case1) { + for (auto written : {true, false}) { + ExtentCache cache; + + PExtent pext; + pext.len = 1 * kMiB; + pext.pOffset = 2 * kMiB; + pext.UnWritten = !written; + + cache.Merge(4 * kMiB, pext); + + auto pb = cache.ToInodePb(); + ASSERT_EQ(1, pb.size()); + + ASSERT_TRUE(pb.count(0)); + + const auto& range = pb[0]; + ASSERT_EQ(1, range.volumeextents_size()); + + auto& first = range.volumeextents(0); + ASSERT_EQ(1 * kMiB, first.length()); + ASSERT_EQ(2 * kMiB, first.volumeoffset()); + ASSERT_EQ(4 * kMiB, first.fsoffset()); + ASSERT_EQ(written, first.isused()); + } +} + +// |----|----|----|----| +// ^^^^ ^^^^ +// written written +TEST(ExtentCacheToInodePbTest, Case2) { + ExtentCache cache; + + PExtent pext; + pext.len = 4 * kMiB; + pext.pOffset = 0 * kMiB; + pext.UnWritten = true; + cache.Merge(0 * kMiB, pext); + + pext.len = 4 * kMiB; + pext.pOffset = 4 * kMiB; + pext.UnWritten = true; + cache.Merge(4 * kMiB, pext); + + pext.len = 4 * kMiB; + pext.pOffset = 8 * kMiB; + pext.UnWritten = true; + cache.Merge(8 * kMiB, pext); + + pext.len = 4 * kMiB; + pext.pOffset = 12 * kMiB; + pext.UnWritten = true; + cache.Merge(12 * kMiB, pext); + + cache.MarkWritten(0 * kMiB, 4 * kMiB); + cache.MarkWritten(8 * kMiB, 4 * kMiB); + + auto pb = cache.ToInodePb(); + ASSERT_EQ(1, pb.size()); + + ASSERT_TRUE(pb.count(0)); + + const auto& range = pb[0]; + ASSERT_EQ(4, range.volumeextents_size()); + + auto& first = range.volumeextents(0); + auto& second = range.volumeextents(1); + auto& third = range.volumeextents(2); + auto& fourth = range.volumeextents(3); + + ASSERT_EQ(4 * kMiB, first.length()); + ASSERT_EQ(0 * kMiB, first.volumeoffset()); + ASSERT_EQ(0 * kMiB, first.fsoffset()); + ASSERT_EQ(true, first.isused()); + + ASSERT_EQ(4 * kMiB, second.length()); + ASSERT_EQ(4 * kMiB, second.volumeoffset()); + ASSERT_EQ(4 * kMiB, second.fsoffset()); + ASSERT_EQ(false, second.isused()); + + ASSERT_EQ(4 * kMiB, third.length()); + ASSERT_EQ(8 * kMiB, third.volumeoffset()); + ASSERT_EQ(8 * kMiB, third.fsoffset()); + ASSERT_EQ(true, third.isused()); + + ASSERT_EQ(4 * kMiB, fourth.length()); + ASSERT_EQ(12 * kMiB, fourth.volumeoffset()); + ASSERT_EQ(12 * kMiB, fourth.fsoffset()); + ASSERT_EQ(false, fourth.isused()); +} + +// |----|----|----|----| +// ^^^^ ^^^^ +// written +TEST(ExtentCacheToInodePbTest, Case3) { + ExtentCache cache; + + PExtent pext; + pext.len = 4 * kMiB; + pext.pOffset = 0 * kMiB; + pext.UnWritten = true; + cache.Merge(0 * kMiB, pext); + + pext.len = 4 * kMiB; + pext.pOffset = 4 * kMiB; + pext.UnWritten = true; + cache.Merge(4 * kMiB, pext); + + pext.len = 4 * kMiB; + pext.pOffset = 8 * kMiB; + pext.UnWritten = true; + cache.Merge(8 * kMiB, pext); + + pext.len = 4 * kMiB; + pext.pOffset = 12 * kMiB; + pext.UnWritten = true; + cache.Merge(12 * kMiB, pext); + + cache.MarkWritten(4 * kMiB, 8 * kMiB); + + auto pb = cache.ToInodePb(); + ASSERT_EQ(1, pb.size()); + + ASSERT_TRUE(pb.count(0)); + + const auto& range = pb[0]; + ASSERT_EQ(3, range.volumeextents_size()); + + auto& first = range.volumeextents(0); + auto& second = range.volumeextents(1); + auto& third = range.volumeextents(2); + + ASSERT_EQ(4 * kMiB, first.length()); + ASSERT_EQ(0 * kMiB, first.volumeoffset()); + ASSERT_EQ(0 * kMiB, first.fsoffset()); + ASSERT_EQ(false, first.isused()); + + ASSERT_EQ(8 * kMiB, second.length()); + ASSERT_EQ(4 * kMiB, second.volumeoffset()); + ASSERT_EQ(4 * kMiB, second.fsoffset()); + ASSERT_EQ(true, second.isused()); + + ASSERT_EQ(4 * kMiB, third.length()); + ASSERT_EQ(12 * kMiB, third.volumeoffset()); + ASSERT_EQ(12 * kMiB, third.fsoffset()); + ASSERT_EQ(false, third.isused()); +} + +// |----|----|----|----| +// ^^^^ ^^^^ +// written written +TEST(ExtentCacheToInodePbTest, Case4) { + ExtentCache cache; + + PExtent pext; + pext.len = 4 * kMiB; + pext.pOffset = 0 * kMiB; + pext.UnWritten = true; + cache.Merge(0 * kMiB, pext); + + pext.len = 4 * kMiB; + pext.pOffset = 4 * kMiB; + pext.UnWritten = true; + cache.Merge(4 * kMiB, pext); + + pext.len = 4 * kMiB; + pext.pOffset = 8 * kMiB; + pext.UnWritten = true; + cache.Merge(8 * kMiB, pext); + + pext.len = 4 * kMiB; + pext.pOffset = 12 * kMiB; + pext.UnWritten = true; + cache.Merge(12 * kMiB, pext); + + cache.MarkWritten(0 * kMiB, 4 * kMiB); + cache.MarkWritten(12 * kMiB, 4 * kMiB); + + auto pb = cache.ToInodePb(); + ASSERT_EQ(1, pb.size()); + + ASSERT_TRUE(pb.count(0)); + + const auto& range = pb[0]; + ASSERT_EQ(3, range.volumeextents_size()); + + auto& first = range.volumeextents(0); + auto& second = range.volumeextents(1); + auto& third = range.volumeextents(2); + + ASSERT_EQ(4 * kMiB, first.length()); + ASSERT_EQ(0 * kMiB, first.volumeoffset()); + ASSERT_EQ(0 * kMiB, first.fsoffset()); + ASSERT_EQ(true, first.isused()); + + ASSERT_EQ(8 * kMiB, second.length()); + ASSERT_EQ(4 * kMiB, second.volumeoffset()); + ASSERT_EQ(4 * kMiB, second.fsoffset()); + ASSERT_EQ(false, second.isused()); + + ASSERT_EQ(4 * kMiB, third.length()); + ASSERT_EQ(12 * kMiB, third.volumeoffset()); + ASSERT_EQ(12 * kMiB, third.fsoffset()); + ASSERT_EQ(true, third.isused()); +} + +} // namespace client +} // namespace curvefs diff --git a/curvefs/test/client/volume/extent_cache_write_divide_test.cpp b/curvefs/test/client/volume/extent_cache_write_divide_test.cpp new file mode 100644 index 0000000000..2c4445d02b --- /dev/null +++ b/curvefs/test/client/volume/extent_cache_write_divide_test.cpp @@ -0,0 +1,693 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Tuesday Mar 15 17:00:34 CST 2022 + * Author: wuhanqing + */ + +#include + +#include + +#include "curvefs/src/client/volume/extent_cache.h" +#include "curvefs/test/client/volume/common.h" + +namespace curvefs { +namespace client { + +class ExtentCacheWriteDivideTest : public ::testing::Test { + protected: + void SetUp() override { + option_.preallocSize = 32 * kKiB; + option_.rangeSize = 1 * kGiB; + + ExtentCache::SetOption(option_); + } + + protected: + ExtentCacheOption option_; +}; + +TEST_F(ExtentCacheWriteDivideTest, DivideWhenHasNoExtents) { + ExtentCache cache; + std::vector allocated; + std::vector needAlloc; + + off_t offset = 4 * kMiB; + size_t length = 4 * kMiB; + + std::unique_ptr data(new char[length]); + + cache.DivideForWrite(offset, length, data.get(), &allocated, &needAlloc); + + ASSERT_EQ(0, allocated.size()); + ASSERT_EQ(1, needAlloc.size()); + ASSERT_EQ(data.get(), needAlloc[0].data); + ASSERT_EQ(offset, needAlloc[0].allocInfo.lOffset); + ASSERT_EQ(length, needAlloc[0].allocInfo.len); + ASSERT_FALSE(needAlloc[0].allocInfo.leftHintAvailable); + ASSERT_FALSE(needAlloc[0].allocInfo.rightHintAvailable); +} + +TEST_F(ExtentCacheWriteDivideTest, DivideWhenHasNoExtents1) { + ExtentCache cache; + std::vector allocated; + std::vector needAlloc; + + off_t offset = 0; + size_t length = 128 * kKiB; + + std::unique_ptr data(new char[length]); + + cache.DivideForWrite(offset, length, data.get(), &allocated, &needAlloc); + + ASSERT_EQ(0, allocated.size()); + ASSERT_EQ(1, needAlloc.size()); + ASSERT_EQ(data.get(), needAlloc[0].data); + ASSERT_EQ(offset, needAlloc[0].allocInfo.lOffset); + ASSERT_EQ(length, needAlloc[0].allocInfo.len); + ASSERT_EQ(length, needAlloc[0].writelength); + ASSERT_FALSE(needAlloc[0].allocInfo.leftHintAvailable); + ASSERT_FALSE(needAlloc[0].allocInfo.rightHintAvailable); +} + +// write |-----| +// extents |-----| +TEST_F(ExtentCacheWriteDivideTest, DivideCase1) { + ExtentCache cache; + std::vector allocated; + std::vector needAlloc; + + PExtent pext; + pext.pOffset = 12 * kMiB; + pext.len = 4 * kMiB; + cache.Merge(12 * kMiB, pext); + + off_t offset = 4 * kMiB; + size_t length = 4 * kMiB; + + std::unique_ptr data(new char[length]); + + cache.DivideForWrite(offset, length, data.get(), &allocated, &needAlloc); + + ASSERT_EQ(0, allocated.size()); + ASSERT_EQ(1, needAlloc.size()); + ASSERT_EQ(data.get(), needAlloc[0].data); + ASSERT_EQ(offset, needAlloc[0].allocInfo.lOffset); + ASSERT_EQ(length, needAlloc[0].allocInfo.len); + ASSERT_FALSE(needAlloc[0].allocInfo.leftHintAvailable); + ASSERT_FALSE(needAlloc[0].allocInfo.rightHintAvailable); +} + +// write |-----| +// extents |-----| +TEST_F(ExtentCacheWriteDivideTest, DivideCase2) { + ExtentCache cache; + std::vector allocated; + std::vector needAlloc; + + PExtent pext; + pext.pOffset = 12 * kMiB; + pext.len = 4 * kMiB; + cache.Merge(8 * kMiB, pext); + + off_t offset = 4 * kMiB; + size_t length = 4 * kMiB; + + std::unique_ptr data(new char[length]); + + cache.DivideForWrite(offset, length, data.get(), &allocated, &needAlloc); + + ASSERT_EQ(0, allocated.size()); + ASSERT_EQ(1, needAlloc.size()); + ASSERT_EQ(data.get(), needAlloc[0].data); + ASSERT_EQ(offset, needAlloc[0].allocInfo.lOffset); + ASSERT_EQ(length, needAlloc[0].allocInfo.len); + ASSERT_TRUE(needAlloc[0].allocInfo.rightHintAvailable); + ASSERT_EQ(12 * kMiB, needAlloc[0].allocInfo.pOffsetRight); +} + +// write |-----| +// extents |-----| +TEST_F(ExtentCacheWriteDivideTest, DivideCase3) { + ExtentCache cache; + std::vector allocated; + std::vector needAlloc; + + PExtent pext; + + // 6MiB ~ 10MiB is allocated + pext.pOffset = 12 * kMiB; + pext.len = 4 * kMiB; + cache.Merge(6 * kMiB, pext); + + // write 4MiB ~ 8MiB + off_t offset = 4 * kMiB; + size_t length = 4 * kMiB; + + std::unique_ptr data(new char[length]); + + cache.DivideForWrite(offset, length, data.get(), &allocated, &needAlloc); + + ASSERT_EQ(1, allocated.size()); + ASSERT_EQ(1, needAlloc.size()); + ASSERT_EQ(data.get(), needAlloc[0].data); + ASSERT_EQ(offset, needAlloc[0].allocInfo.lOffset); + ASSERT_EQ(length / 2, needAlloc[0].allocInfo.len); + ASSERT_TRUE(needAlloc[0].allocInfo.rightHintAvailable); + ASSERT_EQ(12 * kMiB, needAlloc[0].allocInfo.pOffsetRight); + + ASSERT_EQ(pext.pOffset, allocated[0].offset); + ASSERT_EQ(length / 2, allocated[0].length); + ASSERT_EQ(data.get() + length / 2, allocated[0].data); +} + +// write |---------| +// extents |-----| +TEST_F(ExtentCacheWriteDivideTest, DivideCase4) { + ExtentCache cache; + std::vector allocated; + std::vector needAlloc; + + PExtent pext; + + // 6MiB ~ 10MiB is allocated + pext.pOffset = 12 * kMiB; + pext.len = 4 * kMiB; + cache.Merge(6 * kMiB, pext); + + // write 4MiB ~ 10MiB + off_t offset = 4 * kMiB; + size_t length = 6 * kMiB; + + std::unique_ptr data(new char[length]); + + cache.DivideForWrite(offset, length, data.get(), &allocated, &needAlloc); + + ASSERT_EQ(1, allocated.size()); + ASSERT_EQ(1, needAlloc.size()); + + ASSERT_EQ(data.get(), needAlloc[0].data); + ASSERT_EQ(4 * kMiB, needAlloc[0].allocInfo.lOffset); + ASSERT_EQ(2 * kMiB, needAlloc[0].allocInfo.len); + ASSERT_TRUE(needAlloc[0].allocInfo.rightHintAvailable); + ASSERT_EQ(12 * kMiB, needAlloc[0].allocInfo.pOffsetRight); + + ASSERT_EQ(12 * kMiB, allocated[0].offset); + ASSERT_EQ(4 * kMiB, allocated[0].length); + ASSERT_EQ(data.get() + 2 * kMiB, allocated[0].data); +} + +// write |-----------| +// extents |-----| +TEST_F(ExtentCacheWriteDivideTest, DivideCase5) { + ExtentCache cache; + std::vector allocated; + std::vector needAlloc; + + PExtent pext; + + // 6MiB ~ 10MiB is allocated + pext.pOffset = 12 * kMiB; + pext.len = 4 * kMiB; + cache.Merge(6 * kMiB, pext); + + // write 4MiB ~ 12MiB + off_t offset = 4 * kMiB; + size_t length = 8 * kMiB; + + std::unique_ptr data(new char[length]); + + cache.DivideForWrite(offset, length, data.get(), &allocated, &needAlloc); + + ASSERT_EQ(1, allocated.size()); + ASSERT_EQ(2, needAlloc.size()); + + ASSERT_EQ(data.get(), needAlloc[0].data); + ASSERT_EQ(4 * kMiB, needAlloc[0].allocInfo.lOffset); + ASSERT_EQ(2 * kMiB, needAlloc[0].allocInfo.len); + ASSERT_TRUE(needAlloc[0].allocInfo.rightHintAvailable); + ASSERT_EQ(12 * kMiB, needAlloc[0].allocInfo.pOffsetRight); + + ASSERT_EQ(data.get() + 6 * kMiB, needAlloc[1].data); + ASSERT_EQ(10 * kMiB, needAlloc[1].allocInfo.lOffset); + ASSERT_EQ(2 * kMiB, needAlloc[1].allocInfo.len); + ASSERT_TRUE(needAlloc[1].allocInfo.leftHintAvailable); + ASSERT_EQ(16 * kMiB, needAlloc[1].allocInfo.pOffsetLeft); + + ASSERT_EQ(data.get() + 2 * kMiB, allocated[0].data); + ASSERT_EQ(12 * kMiB, allocated[0].offset); + ASSERT_EQ(4 * kMiB, allocated[0].length); +} + +// write |---| +// extents |-----| +TEST_F(ExtentCacheWriteDivideTest, DivideCase6) { + ExtentCache cache; + std::vector allocated; + std::vector needAlloc; + + PExtent pext; + + // 6MiB ~ 10MiB is allocated + pext.pOffset = 12 * kMiB; + pext.len = 4 * kMiB; + cache.Merge(6 * kMiB, pext); + + // write 6MiB ~ 8MiB + off_t offset = 6 * kMiB; + size_t length = 2 * kMiB; + + std::unique_ptr data(new char[length]); + + cache.DivideForWrite(offset, length, data.get(), &allocated, &needAlloc); + + ASSERT_EQ(1, allocated.size()); + ASSERT_EQ(0, needAlloc.size()); + + ASSERT_EQ(data.get(), allocated[0].data); + ASSERT_EQ(12 * kMiB, allocated[0].offset); + ASSERT_EQ(2 * kMiB, allocated[0].length); +} + +// write |-----| +// extents |-----| +TEST_F(ExtentCacheWriteDivideTest, DivideCase7) { + ExtentCache cache; + std::vector allocated; + std::vector needAlloc; + + PExtent pext; + + // 6MiB ~ 10MiB is allocated + pext.pOffset = 12 * kMiB; + pext.len = 4 * kMiB; + cache.Merge(6 * kMiB, pext); + + // write 6MiB ~ 10MiB + off_t offset = 6 * kMiB; + size_t length = 4 * kMiB; + + std::unique_ptr data(new char[length]); + + cache.DivideForWrite(offset, length, data.get(), &allocated, &needAlloc); + + ASSERT_EQ(1, allocated.size()); + ASSERT_EQ(0, needAlloc.size()); + + ASSERT_EQ(data.get(), allocated[0].data); + ASSERT_EQ(12 * kMiB, allocated[0].offset); + ASSERT_EQ(4 * kMiB, allocated[0].length); +} + +// write |-------| +// extents |-----| +TEST_F(ExtentCacheWriteDivideTest, DivideCase8) { + ExtentCache cache; + std::vector allocated; + std::vector needAlloc; + + PExtent pext; + + // 6MiB ~ 10MiB is allocated + pext.pOffset = 12 * kMiB; + pext.len = 4 * kMiB; + cache.Merge(6 * kMiB, pext); + + // write 6MiB ~ 12MiB + off_t offset = 6 * kMiB; + size_t length = 6 * kMiB; + + std::unique_ptr data(new char[length]); + + cache.DivideForWrite(offset, length, data.get(), &allocated, &needAlloc); + + ASSERT_EQ(1, allocated.size()); + ASSERT_EQ(1, needAlloc.size()); + + ASSERT_EQ(data.get(), allocated[0].data); + ASSERT_EQ(12 * kMiB, allocated[0].offset); + ASSERT_EQ(4 * kMiB, allocated[0].length); + + ASSERT_EQ(data.get() + 4 * kMiB, needAlloc[0].data); + ASSERT_EQ(10 * kMiB, needAlloc[0].allocInfo.lOffset); + ASSERT_EQ(2 * kMiB, needAlloc[0].allocInfo.len); + ASSERT_TRUE(needAlloc[0].allocInfo.leftHintAvailable); + ASSERT_EQ(16 * kMiB, needAlloc[0].allocInfo.pOffsetLeft); +} + +// write |-----| +// extents |-----| +TEST_F(ExtentCacheWriteDivideTest, DivideCase9) { + ExtentCache cache; + std::vector allocated; + std::vector needAlloc; + + PExtent pext; + + // 6MiB ~ 10MiB is allocated + pext.pOffset = 12 * kMiB; + pext.len = 4 * kMiB; + cache.Merge(6 * kMiB, pext); + + // write 12MiB ~ 14MiB + off_t offset = 12 * kMiB; + size_t length = 2 * kMiB; + + std::unique_ptr data(new char[length]); + + cache.DivideForWrite(offset, length, data.get(), &allocated, &needAlloc); + + ASSERT_EQ(0, allocated.size()); + ASSERT_EQ(1, needAlloc.size()); + + ASSERT_EQ(data.get(), needAlloc[0].data); + ASSERT_EQ(12 * kMiB, needAlloc[0].allocInfo.lOffset); + ASSERT_EQ(2 * kMiB, needAlloc[0].allocInfo.len); + ASSERT_FALSE(needAlloc[0].allocInfo.leftHintAvailable); + ASSERT_FALSE(needAlloc[0].allocInfo.rightHintAvailable); +} + +// write |-----| +// extents |-----| +TEST_F(ExtentCacheWriteDivideTest, DivideCase10) { + ExtentCache cache; + std::vector allocated; + std::vector needAlloc; + + PExtent pext; + + // 6MiB ~ 10MiB is allocated + pext.pOffset = 12 * kMiB; + pext.len = 4 * kMiB; + cache.Merge(6 * kMiB, pext); + + // write 10MiB ~ 14MiB + off_t offset = 10 * kMiB; + size_t length = 4 * kMiB; + + std::unique_ptr data(new char[length]); + + cache.DivideForWrite(offset, length, data.get(), &allocated, &needAlloc); + + ASSERT_EQ(0, allocated.size()); + ASSERT_EQ(1, needAlloc.size()); + + ASSERT_EQ(data.get(), needAlloc[0].data); + ASSERT_EQ(10 * kMiB, needAlloc[0].allocInfo.lOffset); + ASSERT_EQ(4 * kMiB, needAlloc[0].allocInfo.len); + ASSERT_TRUE(needAlloc[0].allocInfo.leftHintAvailable); + ASSERT_EQ(12 * kMiB, needAlloc[0].allocInfo.pOffsetLeft); +} + +// write |-----| +// extents |------| +TEST_F(ExtentCacheWriteDivideTest, DivideCase11) { + ExtentCache cache; + std::vector allocated; + std::vector needAlloc; + + PExtent pext; + + // 0 ~ 5MiB is allocated + pext.pOffset = 100 * kMiB; + pext.len = 5 * kMiB; + cache.Merge(0, pext); + + // write 4MiB ~ 8MiB + off_t offset = 4 * kMiB; + size_t length = 4 * kMiB; + + std::unique_ptr data(new char[length]); + + cache.DivideForWrite(offset, length, data.get(), &allocated, &needAlloc); + + ASSERT_EQ(1, allocated.size()); + ASSERT_EQ(1, needAlloc.size()); + + ASSERT_EQ(5 * kMiB, needAlloc[0].allocInfo.lOffset); + ASSERT_EQ(3 * kMiB, needAlloc[0].allocInfo.len); + ASSERT_EQ(data.get() + 1 * kMiB, needAlloc[0].data); + ASSERT_TRUE(needAlloc[0].allocInfo.leftHintAvailable); + ASSERT_EQ(104 * kMiB, needAlloc[0].allocInfo.pOffsetLeft); + + ASSERT_EQ(104 * kMiB, allocated[0].offset); + ASSERT_EQ(1 * kMiB, allocated[0].length); + ASSERT_EQ(data.get(), allocated[0].data); +} + +// write |-----| +// extents |----------| +TEST_F(ExtentCacheWriteDivideTest, DivideCase12) { + ExtentCache cache; + std::vector allocated; + std::vector needAlloc; + + PExtent pext; + + // 0 ~ 5MiB is allocated + pext.pOffset = 100 * kMiB; + pext.len = 5 * kMiB; + cache.Merge(0, pext); + + // write 4MiB ~ 5MiB + off_t offset = 4 * kMiB; + size_t length = 1 * kMiB; + + std::unique_ptr data(new char[length]); + + cache.DivideForWrite(offset, length, data.get(), &allocated, &needAlloc); + + ASSERT_EQ(1, allocated.size()); + ASSERT_EQ(0, needAlloc.size()); + + ASSERT_EQ(data.get(), allocated[0].data); + ASSERT_EQ(104 * kMiB, allocated[0].offset); + ASSERT_EQ(1 * kMiB, allocated[0].length); +} + +// write |-----| +// extents |--------------| +TEST_F(ExtentCacheWriteDivideTest, DivideCase13) { + ExtentCache cache; + std::vector allocated; + std::vector needAlloc; + + PExtent pext; + + // 0 ~ 5MiB is allocated + pext.pOffset = 100 * kMiB; + pext.len = 5 * kMiB; + cache.Merge(0, pext); + + // write 2MiB ~ 3MiB + off_t offset = 2 * kMiB; + size_t length = 1 * kMiB; + + std::unique_ptr data(new char[length]); + + cache.DivideForWrite(offset, length, data.get(), &allocated, &needAlloc); + + ASSERT_EQ(1, allocated.size()); + ASSERT_EQ(0, needAlloc.size()); + + ASSERT_EQ(data.get(), allocated[0].data); + ASSERT_EQ(102 * kMiB, allocated[0].offset); + ASSERT_EQ(1 * kMiB, allocated[0].length); +} + +// write |-----| +// extents |-----| |-----| +TEST_F(ExtentCacheWriteDivideTest, DivideCase14) { + ExtentCache cache; + std::vector allocated; + std::vector needAlloc; + + PExtent pext; + // 6MiB ~ 10MiB is allocated + pext.pOffset = 12 * kMiB; + pext.len = 4 * kMiB; + cache.Merge(6 * kMiB, pext); + + // 0 ~ 5MiB is allocated + pext.pOffset = 100 * kMiB; + pext.len = 5 * kMiB; + cache.Merge(0, pext); + + // write 4MiB ~ 8MiB + off_t offset = 4 * kMiB; + size_t length = 4 * kMiB; + + std::unique_ptr data(new char[length]); + + cache.DivideForWrite(offset, length, data.get(), &allocated, &needAlloc); + + ASSERT_EQ(2, allocated.size()); + ASSERT_EQ(1, needAlloc.size()); + + ASSERT_EQ(5 * kMiB, needAlloc[0].allocInfo.lOffset); + ASSERT_EQ(1 * kMiB, needAlloc[0].allocInfo.len); + ASSERT_EQ(data.get() + 1 * kMiB, needAlloc[0].data); + ASSERT_TRUE(needAlloc[0].allocInfo.leftHintAvailable); + ASSERT_EQ(104 * kMiB, needAlloc[0].allocInfo.pOffsetLeft); + ASSERT_FALSE(needAlloc[0].allocInfo.rightHintAvailable); + + ASSERT_EQ(104 * kMiB, allocated[0].offset); + ASSERT_EQ(1 * kMiB, allocated[0].length); + ASSERT_EQ(data.get(), allocated[0].data); + + ASSERT_EQ(12 * kMiB, allocated[1].offset); + ASSERT_EQ(2 * kMiB, allocated[1].length); + ASSERT_EQ(data.get() + 2 * kMiB, allocated[1].data); +} + +// write |----------------------| +// extents |-----| |-----| |-----| +TEST_F(ExtentCacheWriteDivideTest, DivideCase15) { + ExtentCache cache; + std::vector allocated; + std::vector needAlloc; + + PExtent pext; + // 6MiB ~ 10MiB is allocated + pext.pOffset = 12 * kMiB; + pext.len = 4 * kMiB; + cache.Merge(6 * kMiB, pext); + + // 12 ~ 14MiB is allocated + pext.pOffset = 100 * kMiB; + pext.len = 2 * kMiB; + cache.Merge(12 * kMiB, pext); + + // 15 ~ 18MiB is allocated + pext.pOffset = 1 * kGiB; + pext.len = 3 * kMiB; + cache.Merge(15 * kMiB, pext); + + // write 6MiB ~ 16MiB + off_t offset = 6 * kMiB; + size_t length = 10 * kMiB; + + std::unique_ptr data(new char[length]); + + cache.DivideForWrite(offset, length, data.get(), &allocated, &needAlloc); + + ASSERT_EQ(3, allocated.size()); + ASSERT_EQ(2, needAlloc.size()); + + ASSERT_EQ(data.get(), allocated[0].data); + ASSERT_EQ(12 * kMiB, allocated[0].offset); + ASSERT_EQ(4 * kMiB, allocated[0].length); + + ASSERT_EQ(data.get() + 4 * kMiB, needAlloc[0].data); + ASSERT_EQ(10 * kMiB, needAlloc[0].allocInfo.lOffset); + ASSERT_EQ(2 * kMiB, needAlloc[0].allocInfo.len); + ASSERT_TRUE(needAlloc[0].allocInfo.leftHintAvailable); + ASSERT_EQ(16 * kMiB, needAlloc[0].allocInfo.pOffsetLeft); + + ASSERT_EQ(data.get() + 6 * kMiB, allocated[1].data); + ASSERT_EQ(100 * kMiB, allocated[1].offset); + ASSERT_EQ(2 * kMiB, allocated[1].length); + + ASSERT_EQ(data.get() + 8 * kMiB, needAlloc[1].data); + ASSERT_EQ(14 * kMiB, needAlloc[1].allocInfo.lOffset); + ASSERT_EQ(1 * kMiB, needAlloc[1].allocInfo.len); + ASSERT_TRUE(needAlloc[1].allocInfo.leftHintAvailable); + ASSERT_EQ(102 * kMiB, needAlloc[1].allocInfo.pOffsetLeft); + + ASSERT_EQ(data.get() + 9 * kMiB, allocated[2].data); + ASSERT_EQ(1 * kGiB, allocated[2].offset); + ASSERT_EQ(1 * kMiB, allocated[2].length); +} + +// write |----------| +// extents |----| |-----| +// We can prealloc more space, but it can't overlap with next allocated extent +TEST_F(ExtentCacheWriteDivideTest, DivideCase16_BoundaryTest) { + option_.preallocSize = 64 * kKiB; + ExtentCache::SetOption(option_); + ExtentCache cache; + + std::vector allocated; + std::vector needAlloc; + + PExtent pext; + // 32KiB ~ 64KiB is allocated + pext.pOffset = 12 * kMiB; + pext.len = 32 * kKiB; + cache.Merge(32 * kKiB, pext); + + // 0KiB ~ 4KiB is allocated + pext.pOffset = 15 * kMiB; + pext.len = 4 * kKiB; + cache.Merge(0 * kKiB, pext); + + // write 0 ~ 8KiB + off_t offset = 0 * kKiB; + size_t length = 8 * kKiB; + + std::unique_ptr data(new char[length]); + + cache.DivideForWrite(offset, length, data.get(), &allocated, &needAlloc); + + ASSERT_EQ(1, allocated.size()); + ASSERT_EQ(1, needAlloc.size()); + + ASSERT_EQ(data.get(), allocated[0].data); + ASSERT_EQ(15 * kMiB, allocated[0].offset); + ASSERT_EQ(4 * kKiB, allocated[0].length); + + ASSERT_EQ(data.get() + 4 * kKiB, needAlloc[0].data); + ASSERT_EQ(4 * kKiB, needAlloc[0].allocInfo.lOffset); + ASSERT_EQ(28 * kKiB, needAlloc[0].allocInfo.len); + ASSERT_TRUE(needAlloc[0].allocInfo.leftHintAvailable); + ASSERT_EQ(15 * kMiB + 4 * kKiB, needAlloc[0].allocInfo.pOffsetLeft); +} + +// | +// write |-----| +// extents | range boundary +// | +// We can prealloc more space, but it can't overlap with next range +TEST_F(ExtentCacheWriteDivideTest, DivideCase17_BoundaryTest) { + option_.preallocSize = 64 * kKiB; + ExtentCache::SetOption(option_); + ExtentCache cache; + + std::vector allocated; + std::vector needAlloc; + + // write 0 ~ 8KiB + off_t offset = option_.rangeSize - 4 * kKiB; + size_t length = 4 * kKiB; + + std::unique_ptr data(new char[length]); + + cache.DivideForWrite(offset, length, data.get(), &allocated, &needAlloc); + + ASSERT_EQ(0, allocated.size()); + ASSERT_EQ(1, needAlloc.size()); + + ASSERT_EQ(data.get(), needAlloc[0].data); + ASSERT_EQ(option_.rangeSize - 4 * kKiB, needAlloc[0].allocInfo.lOffset); + ASSERT_EQ(4 * kKiB, needAlloc[0].allocInfo.len); + ASSERT_FALSE(needAlloc[0].allocInfo.leftHintAvailable); +} + +} // namespace client +} // namespace curvefs diff --git a/curvefs/test/client/volume/prepare_write_request_test.cpp b/curvefs/test/client/volume/prepare_write_request_test.cpp new file mode 100644 index 0000000000..fea18c1bea --- /dev/null +++ b/curvefs/test/client/volume/prepare_write_request_test.cpp @@ -0,0 +1,96 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Friday Mar 18 16:31:01 CST 2022 + * Author: wuhanqing + */ + +#include + +#include "curvefs/src/client/volume/utils.h" +#include "curvefs/test/client/volume/common.h" +#include "curvefs/test/volume/mock/mock_space_manager.h" + +namespace curvefs { +namespace client { + +using ::curvefs::volume::AllocateHint; +using ::curvefs::volume::MockSpaceManager; + +using ::testing::_; +using ::testing::Invoke; +using ::testing::Return; + +class PrepareWriteRequestTest : public ::testing::Test { + protected: + ExtentCache extentCache_; + MockSpaceManager mockSpaceManager_; +}; + +TEST_F(PrepareWriteRequestTest, Case1) { + off_t offset = 0 * kKiB; + size_t length = 128 * kKiB; + + std::unique_ptr data(new char[length]); + + EXPECT_CALL(mockSpaceManager_, Alloc(_, _, _)) + .WillOnce(Invoke( + [](uint64_t size, const AllocateHint&, std::vector* exts) { + exts->emplace_back(1 * kMiB, size); + return true; + })); + + std::vector writes; + ASSERT_TRUE(PrepareWriteRequest(offset, length, data.get(), &extentCache_, + &mockSpaceManager_, &writes)); + + ASSERT_EQ(1, writes.size()); + + ASSERT_EQ(1 * kMiB, writes[0].offset); + ASSERT_EQ(128 * kKiB, writes[0].length); + ASSERT_EQ(data.get(), writes[0].data); +} + +TEST_F(PrepareWriteRequestTest, Case2) { + off_t offset = 0 * kKiB; + size_t length = 128 * kKiB; + + std::unique_ptr data(new char[length]); + + PExtent pext; + pext.pOffset = 1 * kMiB; + pext.len = 128 * kKiB; + pext.UnWritten = false; + extentCache_.Merge(0 * kKiB, pext); + + EXPECT_CALL(mockSpaceManager_, Alloc(_, _, _)) + .Times(0); + + std::vector writes; + ASSERT_TRUE(PrepareWriteRequest(offset, length, data.get(), &extentCache_, + &mockSpaceManager_, &writes)); + + ASSERT_EQ(1, writes.size()); + + ASSERT_EQ(1 * kMiB, writes[0].offset); + ASSERT_EQ(128 * kKiB, writes[0].length); + ASSERT_EQ(data.get(), writes[0].data); +} + +} // namespace client +} // namespace curvefs diff --git a/curvefs/test/mds/BUILD b/curvefs/test/mds/BUILD index ba75c9cce6..5e9457cbe9 100644 --- a/curvefs/test/mds/BUILD +++ b/curvefs/test/mds/BUILD @@ -21,7 +21,7 @@ cc_test( srcs = glob(["*.cpp", "*.h"], exclude = [ "chunkid_allocator_test.cpp", - "mock_etcdclient.h" + "mock_etcdclient.h", ]), copts = CURVE_TEST_COPTS, deps = [ @@ -50,5 +50,6 @@ cc_test( "//external:brpc", "//src/common:curve_common", "//curvefs/proto:space_cc_proto", + "//curvefs/test/mds/mock:curvefs_mds_mock", ], ) diff --git a/curvefs/test/mds/chunkid_allocator_test.cpp b/curvefs/test/mds/chunkid_allocator_test.cpp index f237b0d6ba..bf50a04237 100644 --- a/curvefs/test/mds/chunkid_allocator_test.cpp +++ b/curvefs/test/mds/chunkid_allocator_test.cpp @@ -30,12 +30,11 @@ #include #include -#include "curvefs/test/mds/mock_etcdclient.h" +#include "curvefs/test/mds/mock/mock_etcd_client.h" namespace curvefs { namespace mds { -using curve::kvstorage::KVStorageClient; using std::make_shared; using std::set; using std::shared_ptr; diff --git a/curvefs/test/mds/codec/codec_test.cpp b/curvefs/test/mds/codec/codec_test.cpp index 6730642dab..f1c391e531 100644 --- a/curvefs/test/mds/codec/codec_test.cpp +++ b/curvefs/test/mds/codec/codec_test.cpp @@ -56,6 +56,8 @@ TEST(CodecTest, TestEncodeProtobufMessage) { volume.set_blocksize(4096); volume.set_volumename("/curvefs"); volume.set_user("test"); + volume.set_blockgroupsize(4096); + volume.set_bitmaplocation(curvefs::common::BitmapLocation::AtEnd); fsinfo.mutable_detail()->set_allocated_volume(new Volume(volume)); @@ -85,6 +87,8 @@ TEST(CodecTest, TestDecodeProtobufMessage) { volume.set_blocksize(4096); volume.set_volumename("/curvefs"); volume.set_user("test"); + volume.set_blockgroupsize(4096); + volume.set_bitmaplocation(curvefs::common::BitmapLocation::AtEnd); fsinfo.mutable_detail()->set_allocated_volume(new Volume(volume)); diff --git a/curvefs/test/mds/fake_space.cpp b/curvefs/test/mds/fake_space.cpp deleted file mode 100644 index 89b4b88435..0000000000 --- a/curvefs/test/mds/fake_space.cpp +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * @Project: curve - * @Date: 2021-06-11 16:53:49 - * @Author: chenwei - */ -#include "curvefs/test/mds/fake_space.h" - -namespace curvefs { -namespace space { -void FakeSpaceImpl::InitSpace(::google::protobuf::RpcController* controller, - const ::curvefs::space::InitSpaceRequest* request, - ::curvefs::space::InitSpaceResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); - // brpc::Controller* cntl = static_cast(controller); - initCount++; - SpaceStatusCode status = SpaceStatusCode::SPACE_OK; - response->set_status(status); - return; -} - -void FakeSpaceImpl::AllocateSpace( - ::google::protobuf::RpcController* controller, - const ::curvefs::space::AllocateSpaceRequest* request, - ::curvefs::space::AllocateSpaceResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); - // brpc::Controller* cntl = static_cast(controller); - allocCount++; - SpaceStatusCode status = SpaceStatusCode::SPACE_OK; - response->set_status(status); - return; -} - -void FakeSpaceImpl::DeallocateSpace( - ::google::protobuf::RpcController* controller, // NOLINT - const ::curvefs::space::DeallocateSpaceRequest* request, - ::curvefs::space::DeallocateSpaceResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); - // brpc::Controller* cntl = static_cast(controller); - deallocateCount++; - SpaceStatusCode status = SpaceStatusCode::SPACE_OK; - response->set_status(status); - return; -} - -void FakeSpaceImpl::StatSpace(::google::protobuf::RpcController* controller, - const ::curvefs::space::StatSpaceRequest* request, - ::curvefs::space::StatSpaceResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); - // brpc::Controller* cntl = static_cast(controller); - statCount++; - SpaceStatusCode status = SpaceStatusCode::SPACE_OK; - response->set_status(status); - return; -} - -void FakeSpaceImpl::UnInitSpace( - ::google::protobuf::RpcController* controller, - const ::curvefs::space::UnInitSpaceRequest* request, - ::curvefs::space::UnInitSpaceResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); - // brpc::Controller* cntl = static_cast(controller); - uninitCount++; - SpaceStatusCode status = SpaceStatusCode::SPACE_OK; - response->set_status(status); - return; -} -} // namespace space -} // namespace curvefs diff --git a/curvefs/test/mds/fake_space.h b/curvefs/test/mds/fake_space.h deleted file mode 100644 index 04f0773888..0000000000 --- a/curvefs/test/mds/fake_space.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * @Project: curve - * @Date: 2021-06-11 16:53:49 - * @Author: chenwei - */ -#ifndef CURVEFS_TEST_MDS_FAKE_SPACE_H_ -#define CURVEFS_TEST_MDS_FAKE_SPACE_H_ - -#include -#include -#include "curvefs/proto/space.pb.h" - -namespace curvefs { -namespace space { -class FakeSpaceImpl : public SpaceAllocService { - public: - FakeSpaceImpl() { - initCount = 0; - allocCount = 0; - deallocateCount = 0; - statCount = 0; - uninitCount = 0; - } - void InitSpace(::google::protobuf::RpcController* controller, - const ::curvefs::space::InitSpaceRequest* request, - ::curvefs::space::InitSpaceResponse* response, - ::google::protobuf::Closure* done); - void AllocateSpace(::google::protobuf::RpcController* controller, - const ::curvefs::space::AllocateSpaceRequest* request, - ::curvefs::space::AllocateSpaceResponse* response, - ::google::protobuf::Closure* done); - void DeallocateSpace( - ::google::protobuf::RpcController* controller, - const ::curvefs::space::DeallocateSpaceRequest* request, - ::curvefs::space::DeallocateSpaceResponse* response, - ::google::protobuf::Closure* done); - void StatSpace(::google::protobuf::RpcController* controller, - const ::curvefs::space::StatSpaceRequest* request, - ::curvefs::space::StatSpaceResponse* response, - ::google::protobuf::Closure* done); - void UnInitSpace(::google::protobuf::RpcController* controller, - const ::curvefs::space::UnInitSpaceRequest* request, - ::curvefs::space::UnInitSpaceResponse* response, - ::google::protobuf::Closure* done); - uint32_t initCount; - uint32_t allocCount; - uint32_t deallocateCount; - uint32_t statCount; - uint32_t uninitCount; -}; - -} // namespace space -} // namespace curvefs -#endif // CURVEFS_TEST_MDS_FAKE_SPACE_H_ diff --git a/curvefs/test/mds/fs_info_wrapper_test.cpp b/curvefs/test/mds/fs_info_wrapper_test.cpp index cb28a3d080..d91fdf74ac 100644 --- a/curvefs/test/mds/fs_info_wrapper_test.cpp +++ b/curvefs/test/mds/fs_info_wrapper_test.cpp @@ -64,6 +64,8 @@ TEST(FsInfoWrapperTest, TestGenerateFsInfoWrapper) { volume.set_blocksize(blocksize); volume.set_volumename("/curvefs"); volume.set_user("test"); + volume.set_blockgroupsize(128ull * 1024 * 1024); + volume.set_bitmaplocation(common::BitmapLocation::AtStart); detail.set_allocated_volume(new Volume(volume)); diff --git a/curvefs/test/mds/fs_manager_test.cpp b/curvefs/test/mds/fs_manager_test.cpp index 0734dfbef9..f1e2426641 100644 --- a/curvefs/test/mds/fs_manager_test.cpp +++ b/curvefs/test/mds/fs_manager_test.cpp @@ -28,9 +28,9 @@ #include "curvefs/test/mds/mock/mock_cli2.h" #include "curvefs/test/mds/mock/mock_fs_stroage.h" #include "curvefs/test/mds/mock/mock_metaserver.h" -#include "curvefs/test/mds/mock/mock_space.h" #include "curvefs/test/mds/mock/mock_topology.h" #include "test/common/mock_s3_adapter.h" +#include "curvefs/test/mds/mock/mock_space_manager.h" using ::testing::AtLeast; using ::testing::StrEq; @@ -42,18 +42,12 @@ using ::testing::SetArgPointee; using ::testing::SaveArg; using ::testing::Mock; using ::testing::Invoke; -using ::curvefs::space::MockSpaceService; using ::curvefs::metaserver::MockMetaserverService; using curvefs::metaserver::CreateRootInodeRequest; using curvefs::metaserver::CreateRootInodeResponse; using curvefs::metaserver::DeletePartitionRequest; using curvefs::metaserver::DeletePartitionResponse; -using curvefs::space::InitSpaceRequest; -using curvefs::space::InitSpaceResponse; -using curvefs::space::UnInitSpaceRequest; -using curvefs::space::UnInitSpaceResponse; using curvefs::metaserver::MetaStatusCode; -using curvefs::space::SpaceStatusCode; using ::google::protobuf::util::MessageDifferencer; using ::curvefs::common::S3Info; using ::curvefs::common::Volume; @@ -78,6 +72,7 @@ using ::curvefs::mds::topology::TopoStatusCode; using ::curvefs::metaserver::copyset::MockCliService2; using ::curvefs::metaserver::copyset::GetLeaderResponse2; using ::curve::common::MockS3Adapter; +using ::curvefs::mds::space::MockSpaceManager; namespace curvefs { namespace mds { @@ -85,14 +80,12 @@ class FSManagerTest : public ::testing::Test { protected: void SetUp() override { std::string addr = "127.0.0.1:6704"; - SpaceOptions spaceOptions; - spaceOptions.spaceAddr = addr; - spaceOptions.rpcTimeoutMs = 500; + MetaserverOptions metaserverOptions; metaserverOptions.metaserverAddr = addr; metaserverOptions.rpcTimeoutMs = 500; fsStorage_ = std::make_shared(); - spaceClient_ = std::make_shared(spaceOptions); + spaceManager_ = std::make_shared(); metaserverClient_ = std::make_shared(metaserverOptions); // init mock topology manager @@ -113,14 +106,12 @@ class FSManagerTest : public ::testing::Test { FsManagerOption fsManagerOption; fsManagerOption.backEndThreadRunInterSec = 1; s3Adapter_ = std::make_shared(); - fsManager_ = std::make_shared(fsStorage_, spaceClient_, + fsManager_ = std::make_shared(fsStorage_, spaceManager_, metaserverClient_, topoManager_, s3Adapter_, fsManagerOption); ASSERT_TRUE(fsManager_->Init()); - ASSERT_EQ(0, server_.AddService(&mockSpaceService_, - brpc::SERVER_DOESNT_OWN_SERVICE)); ASSERT_EQ(0, server_.AddService(&mockMetaserverService_, brpc::SERVER_DOESNT_OWN_SERVICE)); ASSERT_EQ(0, server_.AddService(&mockCliService2_, @@ -171,9 +162,8 @@ class FSManagerTest : public ::testing::Test { protected: std::shared_ptr fsManager_; std::shared_ptr fsStorage_; - std::shared_ptr spaceClient_; + std::shared_ptr spaceManager_; std::shared_ptr metaserverClient_; - MockSpaceService mockSpaceService_; MockMetaserverService mockMetaserverService_; MockCliService2 mockCliService2_; std::shared_ptr topoManager_; @@ -393,21 +383,14 @@ TEST_F(FSManagerTest, test1) { FsInfo fsInfo3; // mount volumefs initspace fail - InitSpaceResponse initSpaceResponse; - initSpaceResponse.set_status(SpaceStatusCode::SPACE_UNKNOWN_ERROR); - EXPECT_CALL(mockSpaceService_, InitSpace(_, _, _, _)) - .WillOnce( - DoAll(SetArgPointee<2>(initSpaceResponse), - Invoke(RpcService))); + EXPECT_CALL(*spaceManager_, AddVolume(_)) + .WillOnce(Return(space::SpaceErrCreate)); ret = fsManager_->MountFs(fsName1, mountPoint, &fsInfo3); ASSERT_EQ(ret, FSStatusCode::INIT_SPACE_ERROR); // mount volumefs success - initSpaceResponse.set_status(SpaceStatusCode::SPACE_OK); - EXPECT_CALL(mockSpaceService_, InitSpace(_, _, _, _)) - .WillOnce( - DoAll(SetArgPointee<2>(initSpaceResponse), - Invoke(RpcService))); + EXPECT_CALL(*spaceManager_, AddVolume(_)) + .WillOnce(Return(space::SpaceOk)); ret = fsManager_->MountFs(fsName1, mountPoint, &fsInfo3); ASSERT_EQ(ret, FSStatusCode::OK); ASSERT_TRUE(CompareVolumeFs(volumeFsInfo1, fsInfo3)); @@ -419,7 +402,6 @@ TEST_F(FSManagerTest, test1) { // mount s3 fs success FsInfo fsInfo4; - initSpaceResponse.set_status(SpaceStatusCode::SPACE_OK); ret = fsManager_->MountFs(fsName2, mountPoint, &fsInfo4); ASSERT_EQ(ret, FSStatusCode::OK); ASSERT_TRUE(CompareS3Fs(s3FsInfo, fsInfo4)); @@ -431,22 +413,15 @@ TEST_F(FSManagerTest, test1) { // TEST UmountFs // umount UnInitSpace fail - UnInitSpaceResponse uninitSpaceResponse; - uninitSpaceResponse.set_status(SpaceStatusCode::SPACE_UNKNOWN_ERROR); - EXPECT_CALL(mockSpaceService_, UnInitSpace(_, _, _, _)) - .WillOnce( - DoAll(SetArgPointee<2>(uninitSpaceResponse), - Invoke(RpcService))); + EXPECT_CALL(*spaceManager_, RemoveVolume(_)) + .WillOnce(Return(space::SpaceErrNotFound)); ret = fsManager_->UmountFs(fsName1, mountPoint); ASSERT_EQ(ret, FSStatusCode::UNINIT_SPACE_ERROR); // for persistence consider // umount UnInitSpace success - uninitSpaceResponse.set_status(SpaceStatusCode::SPACE_OK); - EXPECT_CALL(mockSpaceService_, UnInitSpace(_, _, _, _)) - .WillOnce( - DoAll(SetArgPointee<2>(uninitSpaceResponse), - Invoke(RpcService))); + EXPECT_CALL(*spaceManager_, RemoveVolume(_)) + .WillOnce(Return(space::SpaceOk)); ret = fsManager_->UmountFs(fsName1, mountPoint); ASSERT_EQ(ret, FSStatusCode::OK); @@ -481,7 +456,7 @@ TEST_F(FSManagerTest, backgroud_thread_test) { fsManager_->Uninit(); } -TEST_F(FSManagerTest, backgroud_thread_deletefs_test) { +TEST_F(FSManagerTest, background_thread_deletefs_test) { fsManager_->Run(); std::string addr = "127.0.0.1:6704"; std::string leader = "127.0.0.1:6704:0"; @@ -636,10 +611,13 @@ TEST_F(FSManagerTest, backgroud_thread_deletefs_test) { EXPECT_CALL(*topoManager_, UpdatePartitionStatus(_, _)) .WillOnce(Return(TopoStatusCode::TOPO_OK)); + EXPECT_CALL(*spaceManager_, DeleteVolume(_)) + .WillOnce(Return(space::SpaceOk)); ret = fsManager_->DeleteFs(fsName1); ASSERT_EQ(ret, FSStatusCode::OK); sleep(4); + // query fs deleted FsInfo fsInfo3; ret = fsManager_->GetFsInfo(fsInfo1.fsid(), &fsInfo3); diff --git a/curvefs/test/mds/fs_manager_test2.cpp b/curvefs/test/mds/fs_manager_test2.cpp index 5f785fd3e9..bda90fbbe6 100644 --- a/curvefs/test/mds/fs_manager_test2.cpp +++ b/curvefs/test/mds/fs_manager_test2.cpp @@ -27,14 +27,13 @@ #include "curvefs/proto/common.pb.h" #include "curvefs/src/mds/fs_manager.h" #include "curvefs/src/mds/metaserverclient/metaserver_client.h" -#include "curvefs/src/mds/spaceclient/space_client.h" #include "curvefs/test/mds/mock/mock_fs_stroage.h" #include "curvefs/test/mds/mock/mock_metaserver.h" -#include "curvefs/test/mds/mock/mock_space.h" #include "curvefs/test/mds/mock/mock_topology.h" #include "curvefs/test/mds/mock/mock_cli2.h" #include "test/common/mock_s3_adapter.h" +#include "curvefs/test/mds/mock/mock_space_manager.h" using ::curvefs::mds::topology::TopologyManager; using ::curvefs::mds::topology::MockTopologyManager; @@ -61,8 +60,8 @@ using ::curve::common::MockS3Adapter; namespace curvefs { namespace mds { +using ::curvefs::mds::space::MockSpaceManager; using ::curvefs::metaserver::MockMetaserverService; -using ::curvefs::space::MockSpaceService; const char* kFsManagerTest2ServerAddress = "0.0.0.0:22000"; @@ -72,7 +71,6 @@ using ::testing::Matcher; using ::testing::Return; using ::testing::SaveArg; using ::testing::SetArgPointee; - using ::curvefs::common::S3Info; template (); - SpaceOptions spaceOpts; - spaceOpts.spaceAddr = kFsManagerTest2ServerAddress; - spaceOpts.rpcTimeoutMs = 1000; - spaceClient_ = std::make_shared(spaceOpts); - MetaserverOptions metaSvrOpts; metaSvrOpts.metaserverAddr = kFsManagerTest2ServerAddress; metaSvrOpts.rpcTimeoutMs = 1000; @@ -115,18 +108,19 @@ class FsManagerTest2 : public testing::Test { std::make_shared(idGenerator_, tokenGenerator_, topoStorage_), metaServerClient_); s3Adapter_ = std::make_shared(); + + spaceManager_ = std::make_shared(); + // init fsmanager FsManagerOption fsManagerOption; fsManagerOption.backEndThreadRunInterSec = 1; - fsManager_ = std::make_shared(storage_, spaceClient_, - metaServerClient_, topoManager_, - s3Adapter_, fsManagerOption); + fsManager_ = std::make_shared( + storage_, spaceManager_, metaServerClient_, topoManager_, + s3Adapter_, fsManagerOption); - spaceService_ = std::make_shared(); + // spaceService_ = std::make_shared(); metaserverService_ = std::make_shared(); - ASSERT_EQ(0, server_.AddService(spaceService_.get(), - brpc::SERVER_DOESNT_OWN_SERVICE)); ASSERT_EQ(0, server_.AddService(metaserverService_.get(), brpc::SERVER_DOESNT_OWN_SERVICE)); ASSERT_EQ(0, server_.AddService(&mockCliService2_, @@ -149,11 +143,10 @@ class FsManagerTest2 : public testing::Test { protected: std::shared_ptr storage_; - std::shared_ptr spaceClient_; std::shared_ptr metaServerClient_; + std::shared_ptr spaceManager_; std::shared_ptr topoManager_; - std::shared_ptr spaceService_; std::shared_ptr metaserverService_; MockCliService2 mockCliService2_; @@ -162,7 +155,7 @@ class FsManagerTest2 : public testing::Test { std::shared_ptr s3Adapter_; }; -TEST_F(FsManagerTest2, CreateFoundConflictFsNameAndNotIdenticialToPreviousOne) { +TEST_F(FsManagerTest2, CreateFoundConflictFsNameAndNotIdenticalToPreviousOne) { std::string fsname = "hello"; FSType type = FSType::TYPE_S3; uint64_t blocksize = 4 * 1024; diff --git a/curvefs/test/mds/mds_service_test.cpp b/curvefs/test/mds/mds_service_test.cpp index 5cbfeaf38e..7ad08bfd26 100644 --- a/curvefs/test/mds/mds_service_test.cpp +++ b/curvefs/test/mds/mds_service_test.cpp @@ -28,11 +28,11 @@ #include #include "curvefs/test/mds/fake_metaserver.h" -#include "curvefs/test/mds/fake_space.h" #include "curvefs/test/mds/mock/mock_kvstorage_client.h" #include "curvefs/test/mds/mock/mock_topology.h" #include "curvefs/test/mds/mock/mock_cli2.h" #include "test/common/mock_s3_adapter.h" +#include "curvefs/test/mds/mock/mock_space_manager.h" using ::testing::_; using ::testing::AtLeast; @@ -48,9 +48,6 @@ using ::testing::Matcher; using ::curvefs::common::S3Info; using ::curvefs::common::Volume; using ::curvefs::metaserver::FakeMetaserverImpl; -using ::curvefs::space::FakeSpaceImpl; -using ::curvefs::space::InitSpaceResponse; -using ::curvefs::space::SpaceStatusCode; using ::curvefs::mds::topology::TopologyManager; using ::curvefs::mds::topology::MockTopologyManager; using ::curvefs::mds::topology::MockTopology; @@ -72,6 +69,7 @@ using ::curvefs::mds::topology::TopoStatusCode; using ::curvefs::metaserver::copyset::MockCliService2; using ::curvefs::metaserver::copyset::GetLeaderResponse2; using ::curve::common::MockS3Adapter; +using ::curvefs::mds::space::MockSpaceManager; namespace curvefs { namespace mds { @@ -80,14 +78,10 @@ class MdsServiceTest : public ::testing::Test { void SetUp() override { kvstorage_ = std::make_shared(); - SpaceOptions spaceOptions; - spaceOptions.spaceAddr = "127.0.0.1:6703"; - spaceOptions.rpcTimeoutMs = 5000; MetaserverOptions metaserverOptions; metaserverOptions.metaserverAddr = "127.0.0.1:6703"; metaserverOptions.rpcTimeoutMs = 5000; fsStorage_ = std::make_shared(); - spaceClient_ = std::make_shared(spaceOptions); metaserverClient_ = std::make_shared(metaserverOptions); // init mock topology manager @@ -96,6 +90,7 @@ class MdsServiceTest : public ::testing::Test { std::shared_ptr tokenGenerator_ = std::make_shared(); + spaceManager_ = std::make_shared(); auto etcdClient_ = std::make_shared(); auto codec = std::make_shared(); auto topoStorage_ = @@ -107,7 +102,7 @@ class MdsServiceTest : public ::testing::Test { FsManagerOption fsManagerOption; fsManagerOption.backEndThreadRunInterSec = 1; s3Adapter_ = std::make_shared(); - fsManager_ = std::make_shared(fsStorage_, spaceClient_, + fsManager_ = std::make_shared(fsStorage_, spaceManager_, metaserverClient_, topoManager_, s3Adapter_, fsManagerOption); ASSERT_TRUE(fsManager_->Init()); @@ -137,7 +132,7 @@ class MdsServiceTest : public ::testing::Test { std::shared_ptr fsManager_; std::shared_ptr fsStorage_; - std::shared_ptr spaceClient_; + std::shared_ptr spaceManager_; std::shared_ptr metaserverClient_; std::shared_ptr kvstorage_; std::shared_ptr topoManager_; @@ -163,11 +158,6 @@ TEST_F(MdsServiceTest, test1) { ASSERT_EQ(server.AddService(&mdsService, brpc::SERVER_DOESNT_OWN_SERVICE), 0); - // MockSpaceService spaceService; - FakeSpaceImpl spaceService; - ASSERT_EQ(server.AddService(&spaceService, brpc::SERVER_DOESNT_OWN_SERVICE), - 0); - FakeMetaserverImpl metaserverService; ASSERT_EQ( server.AddService(&metaserverService, brpc::SERVER_DOESNT_OWN_SERVICE), @@ -217,6 +207,8 @@ TEST_F(MdsServiceTest, test1) { volume.set_blocksize(4096); volume.set_volumename("volume1"); volume.set_user("user1"); + volume.set_blockgroupsize(128ull * 1024 * 1024); + volume.set_bitmaplocation(common::BitmapLocation::AtStart); createRequest.set_fsname("fs1"); createRequest.set_blocksize(4096); @@ -332,7 +324,6 @@ TEST_F(MdsServiceTest, test1) { ASSERT_EQ(mountResponse.fsinfo().mountnum(), 1); ASSERT_EQ(mountResponse.fsinfo().mountpoints_size(), 1); ASSERT_EQ(mountResponse.fsinfo().mountpoints(0), mountPoint); - ASSERT_EQ(spaceService.initCount, 1); } else { LOG(ERROR) << "error = " << cntl.ErrorText(); ASSERT_TRUE(false); @@ -342,7 +333,6 @@ TEST_F(MdsServiceTest, test1) { stub.MountFs(&cntl, &mountRequest, &mountResponse, NULL); if (!cntl.Failed()) { ASSERT_EQ(mountResponse.statuscode(), FSStatusCode::MOUNT_POINT_EXIST); - ASSERT_EQ(spaceService.initCount, 1); } else { LOG(ERROR) << "error = " << cntl.ErrorText(); ASSERT_TRUE(false); @@ -354,7 +344,6 @@ TEST_F(MdsServiceTest, test1) { stub.MountFs(&cntl, &mountRequest, &mountResponse, NULL); if (!cntl.Failed()) { ASSERT_EQ(mountResponse.statuscode(), FSStatusCode::OK); - ASSERT_EQ(spaceService.initCount, 1); ASSERT_TRUE(mountResponse.has_fsinfo()); ASSERT_TRUE(CompareFs(mountResponse.fsinfo(), fsinfo1)); ASSERT_EQ(mountResponse.fsinfo().mountnum(), 2); @@ -370,7 +359,6 @@ TEST_F(MdsServiceTest, test1) { stub.MountFs(&cntl, &mountRequest, &mountResponse, NULL); if (!cntl.Failed()) { ASSERT_EQ(mountResponse.statuscode(), FSStatusCode::OK); - ASSERT_EQ(spaceService.initCount, 1); ASSERT_TRUE(mountResponse.has_fsinfo()); ASSERT_TRUE(CompareFs(mountResponse.fsinfo(), fsinfo1)); ASSERT_EQ(mountResponse.fsinfo().mountnum(), 3); @@ -386,7 +374,6 @@ TEST_F(MdsServiceTest, test1) { stub.MountFs(&cntl, &mountRequest, &mountResponse, NULL); if (!cntl.Failed()) { ASSERT_EQ(mountResponse.statuscode(), FSStatusCode::OK); - ASSERT_EQ(spaceService.initCount, 1); ASSERT_TRUE(mountResponse.has_fsinfo()); ASSERT_TRUE(CompareFs(mountResponse.fsinfo(), fsinfo1)); ASSERT_EQ(mountResponse.fsinfo().mountnum(), 4); @@ -415,7 +402,6 @@ TEST_F(MdsServiceTest, test1) { stub.GetFsInfo(&cntl, &getRequest, &getResponse, NULL); if (!cntl.Failed()) { ASSERT_EQ(getResponse.statuscode(), FSStatusCode::OK); - ASSERT_EQ(spaceService.initCount, 1); ASSERT_TRUE(getResponse.has_fsinfo()); ASSERT_TRUE(CompareFs(getResponse.fsinfo(), fsinfo1)); ASSERT_EQ(getResponse.fsinfo().mountnum(), 4); @@ -431,7 +417,6 @@ TEST_F(MdsServiceTest, test1) { stub.GetFsInfo(&cntl, &getRequest, &getResponse, NULL); if (!cntl.Failed()) { ASSERT_EQ(getResponse.statuscode(), FSStatusCode::OK); - ASSERT_EQ(spaceService.initCount, 1); ASSERT_TRUE(getResponse.has_fsinfo()); ASSERT_TRUE(CompareFs(getResponse.fsinfo(), fsinfo2)); ASSERT_EQ(getResponse.fsinfo().mountnum(), 0); @@ -459,7 +444,6 @@ TEST_F(MdsServiceTest, test1) { stub.GetFsInfo(&cntl, &getRequest, &getResponse, NULL); if (!cntl.Failed()) { ASSERT_EQ(getResponse.statuscode(), FSStatusCode::OK); - ASSERT_EQ(spaceService.initCount, 1); ASSERT_TRUE(getResponse.has_fsinfo()); ASSERT_TRUE(CompareFs(getResponse.fsinfo(), fsinfo1)); ASSERT_EQ(getResponse.fsinfo().mountnum(), 4); @@ -476,7 +460,6 @@ TEST_F(MdsServiceTest, test1) { stub.GetFsInfo(&cntl, &getRequest, &getResponse, NULL); if (!cntl.Failed()) { ASSERT_EQ(getResponse.statuscode(), FSStatusCode::OK); - ASSERT_EQ(spaceService.initCount, 1); ASSERT_TRUE(getResponse.has_fsinfo()); ASSERT_TRUE(CompareFs(getResponse.fsinfo(), fsinfo2)); ASSERT_EQ(getResponse.fsinfo().mountnum(), 0); @@ -505,7 +488,6 @@ TEST_F(MdsServiceTest, test1) { stub.GetFsInfo(&cntl, &getRequest, &getResponse, NULL); if (!cntl.Failed()) { ASSERT_EQ(getResponse.statuscode(), FSStatusCode::OK); - ASSERT_EQ(spaceService.initCount, 1); ASSERT_TRUE(getResponse.has_fsinfo()); ASSERT_TRUE(CompareFs(getResponse.fsinfo(), fsinfo2)); ASSERT_EQ(getResponse.fsinfo().mountnum(), 0); @@ -581,7 +563,6 @@ TEST_F(MdsServiceTest, test1) { stub.GetFsInfo(&cntl, &getRequest, &getResponse, NULL); if (!cntl.Failed()) { ASSERT_EQ(getResponse.statuscode(), FSStatusCode::OK); - ASSERT_EQ(spaceService.initCount, 1); ASSERT_TRUE(getResponse.has_fsinfo()); ASSERT_TRUE(CompareFs(getResponse.fsinfo(), fsinfo1)); ASSERT_EQ(getResponse.fsinfo().mountnum(), 2); diff --git a/curvefs/test/mds/mock/mock_block_group_storage.h b/curvefs/test/mds/mock/mock_block_group_storage.h new file mode 100644 index 0000000000..659097cf82 --- /dev/null +++ b/curvefs/test/mds/mock/mock_block_group_storage.h @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Tuesday Mar 01 19:47:08 CST 2022 + * Author: wuhanqing + */ + +#ifndef CURVEFS_TEST_MDS_MOCK_MOCK_BLOCK_GROUP_STORAGE_H_ +#define CURVEFS_TEST_MDS_MOCK_MOCK_BLOCK_GROUP_STORAGE_H_ + +#include + +#include + +#include "curvefs/src/mds/space/block_group_storage.h" + +namespace curvefs { +namespace mds { +namespace space { + +class MockBlockGroupStorage : public BlockGroupStorage { + public: + MOCK_METHOD3(PutBlockGroup, + SpaceErrCode(uint32_t, uint64_t, const BlockGroup&)); + MOCK_METHOD2(RemoveBlockGroup, SpaceErrCode(uint32_t, uint64_t)); + MOCK_METHOD2(ListBlockGroups, + SpaceErrCode(uint32_t, std::vector*)); +}; + +} // namespace space +} // namespace mds +} // namespace curvefs + +#endif // CURVEFS_TEST_MDS_MOCK_MOCK_BLOCK_GROUP_STORAGE_H_ diff --git a/curvefs/test/mds/mock_etcdclient.h b/curvefs/test/mds/mock/mock_etcd_client.h similarity index 88% rename from curvefs/test/mds/mock_etcdclient.h rename to curvefs/test/mds/mock/mock_etcd_client.h index afba2f5b6b..573c6028b3 100644 --- a/curvefs/test/mds/mock_etcdclient.h +++ b/curvefs/test/mds/mock/mock_etcd_client.h @@ -19,8 +19,8 @@ * @Author: chengyi */ -#ifndef CURVEFS_TEST_MDS_MOCK_ETCDCLIENT_H_ -#define CURVEFS_TEST_MDS_MOCK_ETCDCLIENT_H_ +#ifndef CURVEFS_TEST_MDS_MOCK_MOCK_ETCD_CLIENT_H_ +#define CURVEFS_TEST_MDS_MOCK_MOCK_ETCD_CLIENT_H_ #include @@ -33,13 +33,10 @@ namespace curvefs { namespace mds { -using curve::kvstorage::EtcdClientImp; - -class MockEtcdClientImpl : public KVStorageClient { +class MockEtcdClientImpl : public curve::kvstorage::KVStorageClient { public: - virtual ~MockEtcdClientImpl() { - // LOG(INFO) << "test"; - } + ~MockEtcdClientImpl() override = default; + MOCK_METHOD0(CloseClient, void()); MOCK_METHOD2(Put, int(const std::string&, const std::string&)); MOCK_METHOD3(PutRewithRevision, @@ -69,4 +66,4 @@ class MockEtcdClientImpl : public KVStorageClient { } // namespace mds } // namespace curvefs -#endif // CURVEFS_TEST_MDS_MOCK_ETCDCLIENT_H_ +#endif // CURVEFS_TEST_MDS_MOCK_MOCK_ETCD_CLIENT_H_ diff --git a/curvefs/test/mds/mock/mock_space.h b/curvefs/test/mds/mock/mock_space.h deleted file mode 100644 index 5a05011560..0000000000 --- a/curvefs/test/mds/mock/mock_space.h +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * @Project: curve - * @Date: 2021-06-11 14:43:05 - * @Author: chenwei - */ - -#ifndef CURVEFS_TEST_MDS_MOCK_MOCK_SPACE_H_ -#define CURVEFS_TEST_MDS_MOCK_MOCK_SPACE_H_ -#include -#include "curvefs/proto/space.pb.h" - -namespace curvefs { -namespace space { -class MockSpaceService : public SpaceAllocService { - public: - MOCK_METHOD4(InitSpace, - void(::google::protobuf::RpcController* controller, - const ::curvefs::space::InitSpaceRequest* request, - ::curvefs::space::InitSpaceResponse* response, - ::google::protobuf::Closure* done)); - MOCK_METHOD4(AllocateSpace, - void(::google::protobuf::RpcController* controller, - const ::curvefs::space::AllocateSpaceRequest* request, - ::curvefs::space::AllocateSpaceResponse* response, - ::google::protobuf::Closure* done)); - MOCK_METHOD4(DeallocateSpace, - void(::google::protobuf::RpcController* controller, - const ::curvefs::space::DeallocateSpaceRequest* request, - ::curvefs::space::DeallocateSpaceResponse* response, - ::google::protobuf::Closure* done)); - MOCK_METHOD4(StatSpace, - void(::google::protobuf::RpcController* controller, - const ::curvefs::space::StatSpaceRequest* request, - ::curvefs::space::StatSpaceResponse* response, - ::google::protobuf::Closure* done)); - MOCK_METHOD4(UnInitSpace, - void(::google::protobuf::RpcController* controller, - const ::curvefs::space::UnInitSpaceRequest* request, - ::curvefs::space::UnInitSpaceResponse* response, - ::google::protobuf::Closure* done)); -}; -} // namespace space -} // namespace curvefs -#endif // CURVEFS_TEST_MDS_MOCK_MOCK_SPACE_H_ diff --git a/curvefs/src/space/reloader.cpp b/curvefs/test/mds/mock/mock_space_manager.h similarity index 52% rename from curvefs/src/space/reloader.cpp rename to curvefs/test/mds/mock/mock_space_manager.h index bc18ab2783..54143b2b01 100644 --- a/curvefs/src/space/reloader.cpp +++ b/curvefs/test/mds/mock/mock_space_manager.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2021 NetEase Inc. + * Copyright (c) 2022 NetEase Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,39 +14,33 @@ * limitations under the License. */ -/** +/* * Project: curve - * File Created: Fri Jul 16 21:22:40 CST 2021 + * Date: Tuesday Mar 01 16:24:17 CST 2022 * Author: wuhanqing */ -#include "curvefs/src/space/reloader.h" +#ifndef CURVEFS_TEST_MDS_MOCK_MOCK_SPACE_MANAGER_H_ +#define CURVEFS_TEST_MDS_MOCK_MOCK_SPACE_MANAGER_H_ -#include -#include +#include -#include "curvefs/src/space/metaserver_client.h" +#include "curvefs/src/mds/space/manager.h" namespace curvefs { +namespace mds { namespace space { -bool Reloader::Reload() { - MetaServerClient client; - if (client.Init(option_.metaServerOption) == false) { - LOG(ERROR) << "init metaserver client failed"; - return false; - } - - Extents exts; - if (!client.GetAllInodeExtents(fsId_, rootInodeId_, &exts)) { - LOG(ERROR) << "get all inode extents failed"; - return false; - } - - allocator_->MarkUsed(exts); - - return true; -} +class MockSpaceManager : public SpaceManager { + public: + MOCK_CONST_METHOD1(GetVolumeSpace, AbstractVolumeSpace*(uint32_t)); + MOCK_METHOD1(AddVolume, SpaceErrCode(const FsInfo&)); + MOCK_METHOD1(RemoveVolume, SpaceErrCode(uint32_t)); + MOCK_METHOD1(DeleteVolume, SpaceErrCode(uint32_t)); +}; } // namespace space +} // namespace mds } // namespace curvefs + +#endif // CURVEFS_TEST_MDS_MOCK_MOCK_SPACE_MANAGER_H_ diff --git a/curvefs/test/mds/mock/mock_volume_space.h b/curvefs/test/mds/mock/mock_volume_space.h new file mode 100644 index 0000000000..cd40d3b671 --- /dev/null +++ b/curvefs/test/mds/mock/mock_volume_space.h @@ -0,0 +1,57 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Wednesday Mar 23 16:21:11 CST 2022 + * Author: wuhanqing + */ + +#ifndef CURVEFS_TEST_MDS_MOCK_MOCK_VOLUME_SPACE_H_ +#define CURVEFS_TEST_MDS_MOCK_MOCK_VOLUME_SPACE_H_ + +#include + +#include +#include + +#include "curvefs/src/mds/space/volume_space.h" + +namespace curvefs { +namespace mds { +namespace space { + +class MockVolumeSpace : public AbstractVolumeSpace { + public: + MOCK_METHOD3(AllocateBlockGroups, + SpaceErrCode(uint32_t, + const std::string&, + std::vector*)); + + MOCK_METHOD3(AcquireBlockGroup, + SpaceErrCode(uint64_t blockGroupOffset, + const std::string& owner, + BlockGroup* group)); + + MOCK_METHOD1(ReleaseBlockGroups, + SpaceErrCode(const std::vector& blockGroups)); +}; + +} // namespace space +} // namespace mds +} // namespace curvefs + +#endif // CURVEFS_TEST_MDS_MOCK_MOCK_VOLUME_SPACE_H_ diff --git a/curvefs/test/mds/persist_kvstorage_test.cpp b/curvefs/test/mds/persist_kvstorage_test.cpp index af5c8e11c5..513504e9d8 100644 --- a/curvefs/test/mds/persist_kvstorage_test.cpp +++ b/curvefs/test/mds/persist_kvstorage_test.cpp @@ -64,6 +64,8 @@ class PersistKVStorageTest : public ::testing::Test { volume.set_volumesize(4096); volume.set_volumename("/hello"); volume.set_user("test"); + volume.set_blockgroupsize(128ull * 1024 * 1024); + volume.set_bitmaplocation(common::BitmapLocation::AtStart); FsDetail helloDetail; helloDetail.set_allocated_volume(new common::Volume(volume)); diff --git a/curvefs/test/mds/space/BUILD b/curvefs/test/mds/space/BUILD new file mode 100644 index 0000000000..c117862116 --- /dev/null +++ b/curvefs/test/mds/space/BUILD @@ -0,0 +1,36 @@ +# +# Copyright (c) 2022 NetEase Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("//:copts.bzl", "CURVE_TEST_COPTS") + +cc_test( + name = "curvefs_mds_space_unittest", + srcs = glob([ + "*.cpp", + "*.h", + ]), + copts = CURVE_TEST_COPTS, + deps = [ + "//curvefs/proto:curvefs_common_cc_proto", + "//curvefs/src/mds/space:curvefs_mds_space", + "//curvefs/test/mds/mock:curvefs_mds_mock", + "//curvefs/test/utils:curvefs_test_utils", + "//external:brpc", + "//external:butil", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) diff --git a/curvefs/test/mds/space/block_group_storage_test.cpp b/curvefs/test/mds/space/block_group_storage_test.cpp new file mode 100644 index 0000000000..a28a5d43d2 --- /dev/null +++ b/curvefs/test/mds/space/block_group_storage_test.cpp @@ -0,0 +1,146 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Tuesday Mar 01 18:44:29 CST 2022 + * Author: wuhanqing + */ + +#include "curvefs/src/mds/space/block_group_storage.h" + +#include + +#include "absl/memory/memory.h" +#include "curvefs/test/mds/mock/mock_etcd_client.h" +#include "curvefs/test/utils/protobuf_message_utils.h" + +namespace curvefs { +namespace mds { +namespace space { + +using ::testing::_; +using ::testing::Invoke; +using ::testing::Matcher; +using ::testing::Return; + +static constexpr uint32_t kFsId = 1; +static constexpr uint64_t kOffset = 10ULL * 1024 * 1024 * 1024; + +class BlockGroupStorageTest : public ::testing::Test { + protected: + void SetUp() override { + etcdclient_ = std::make_shared(); + storage_ = absl::make_unique(etcdclient_); + + auto message = test::GenerateAnDefaultInitializedMessage( + "curvefs.mds.space.BlockGroup"); + + ASSERT_TRUE(message); + + auto* blockgroup = dynamic_cast(message.get()); + ASSERT_NE(nullptr, blockgroup); + + blockGroup_.reset(blockgroup); + message.release(); + } + + protected: + std::shared_ptr etcdclient_; + std::unique_ptr blockGroup_; + std::unique_ptr storage_; +}; + +TEST_F(BlockGroupStorageTest, TestPutBlockGroup_EncodeError) { + // missing required fields + blockGroup_->clear_size(); + + EXPECT_CALL(*etcdclient_, Put(_, _)).Times(0); + + ASSERT_EQ(SpaceErrEncode, + storage_->PutBlockGroup(kFsId, kOffset, *blockGroup_)); +} + +TEST_F(BlockGroupStorageTest, TestPutBlockGroup_PutError) { + EXPECT_CALL(*etcdclient_, Put(_, _)) + .WillOnce(Return(EtcdErrCode::EtcdInternal)); + + ASSERT_EQ(SpaceErrStorage, + storage_->PutBlockGroup(kFsId, kOffset, *blockGroup_)); +} + +TEST_F(BlockGroupStorageTest, TestPutBlockGroup_Success) { + EXPECT_CALL(*etcdclient_, Put(_, _)).WillOnce(Return(EtcdErrCode::EtcdOK)); + + ASSERT_EQ(SpaceOk, storage_->PutBlockGroup(kFsId, kOffset, *blockGroup_)); +} + +TEST_F(BlockGroupStorageTest, TestRemoveBlockGroup_RemoveSuccess) { + EXPECT_CALL(*etcdclient_, Delete(_)).WillOnce(Return(EtcdErrCode::EtcdOK)); + + ASSERT_EQ(SpaceOk, storage_->RemoveBlockGroup(kFsId, kOffset)); +} + +TEST_F(BlockGroupStorageTest, TestRemoveBlockGroup_KeyNotExist) { + EXPECT_CALL(*etcdclient_, Delete(_)) + .WillOnce(Return(EtcdErrCode::EtcdKeyNotExist)); + + ASSERT_EQ(SpaceErrNotFound, storage_->RemoveBlockGroup(kFsId, kOffset)); +} + +TEST_F(BlockGroupStorageTest, TestListBlockGroup_ListError) { + using type = std::vector*; + EXPECT_CALL(*etcdclient_, List(_, _, Matcher(_))) + .WillOnce(Return(EtcdErrCode::EtcdInternal)); + + std::vector groups; + ASSERT_EQ(SpaceErrStorage, storage_->ListBlockGroups(kFsId, &groups)); +} + +TEST_F(BlockGroupStorageTest, TestListBlockGroup_DecodeError) { + using type = std::vector*; + EXPECT_CALL(*etcdclient_, List(_, _, Matcher(_))) + .WillOnce(Invoke([](const std::string&, const std::string&, + std::vector* values) { + values->push_back("hello, world"); + return 0; + })); + + std::vector groups; + ASSERT_EQ(SpaceErrDecode, storage_->ListBlockGroups(kFsId, &groups)); +} + +TEST_F(BlockGroupStorageTest, TestListBlockGroup_Success) { + using type = std::vector*; + EXPECT_CALL(*etcdclient_, List(_, _, Matcher(_))) + .WillOnce(Invoke([this](const std::string&, const std::string&, + std::vector* values) { + std::string value; + blockGroup_->SerializeToString(&value); + values->push_back(std::move(value)); + return 0; + })); + + std::vector groups; + ASSERT_EQ(SpaceOk, storage_->ListBlockGroups(kFsId, &groups)); + ASSERT_EQ(1, groups.size()); + ASSERT_TRUE(google::protobuf::util::MessageDifferencer::Equals( + groups[0], *blockGroup_)); +} + +} // namespace space +} // namespace mds +} // namespace curvefs diff --git a/curvefs/test/mds/space/space_manager_test.cpp b/curvefs/test/mds/space/space_manager_test.cpp new file mode 100644 index 0000000000..cc646f769b --- /dev/null +++ b/curvefs/test/mds/space/space_manager_test.cpp @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Tuesday Mar 01 16:43:16 CST 2022 + * Author: wuhanqing + */ + +#include + +#include "absl/memory/memory.h" +#include "curvefs/proto/mds.pb.h" +#include "curvefs/src/mds/space/manager.h" +#include "curvefs/test/mds/mock/mock_etcd_client.h" + +namespace curvefs { +namespace mds { +namespace space { + +using ::curve::kvstorage::KVStorageClient; +using ::curvefs::common::Volume; +using ::curvefs::mds::FsInfo; +using ::testing::_; +using ::testing::Invoke; +using ::testing::Matcher; +using ::testing::Return; + +namespace { + +constexpr uint64_t kVolumeSize = 10ULL * 1024 * 1024 * 1024; // 10 GiB +constexpr uint32_t kBlockSize = 4096; +constexpr uint32_t kBlockGroupSize = 128ULL * 1024 * 1024; // 128 MiB +constexpr BitmapLocation kBitmapLocation = BitmapLocation::AtStart; +constexpr uint32_t kFsId = 1; + +FsInfo GenerateVolumeFsInfo() { + FsInfo fsInfo; + fsInfo.set_fsid(kFsId); + Volume volume; + volume.set_volumesize(kVolumeSize); + volume.set_blocksize(kBlockSize); + volume.set_blockgroupsize(kBlockGroupSize); + volume.set_bitmaplocation(kBitmapLocation); + auto* detail = fsInfo.mutable_detail(); + detail->mutable_volume()->Swap(&volume); + + return fsInfo; +} + +} // namespace + +class SpaceManagerTest : public ::testing::Test { + protected: + void SetUp() override { + etcdclient_ = std::make_shared(); + spaceManager_ = absl::make_unique(etcdclient_); + } + + void TearDown() override {} + + void AddOneNewVolume() { + FsInfo fsInfo = GenerateVolumeFsInfo(); + + using type = std::vector*; + + EXPECT_CALL(*etcdclient_, List(_, _, Matcher(_))) + .WillOnce(Invoke([](const std::string&, const std::string&, + std::vector* values) { + values->clear(); + return 0; + })); + + ASSERT_EQ(SpaceOk, spaceManager_->AddVolume(fsInfo)); + } + + protected: + std::shared_ptr etcdclient_; + std::unique_ptr spaceManager_; +}; + +TEST_F(SpaceManagerTest, TestGetVolumeSpace) { + ASSERT_EQ(nullptr, spaceManager_->GetVolumeSpace(0)); + + // add one volume space + { + AddOneNewVolume(); + ASSERT_NE(nullptr, spaceManager_->GetVolumeSpace(kFsId)); + } +} + +TEST_F(SpaceManagerTest, TestAddVolume_AlreadyExists) { + AddOneNewVolume(); + ASSERT_NE(nullptr, spaceManager_->GetVolumeSpace(kFsId)); + + FsInfo fsInfo = GenerateVolumeFsInfo(); + + using type = std::vector*; + EXPECT_CALL(*etcdclient_, List(_, _, Matcher(_))).Times(0); + + ASSERT_EQ(SpaceErrExist, spaceManager_->AddVolume(fsInfo)); +} + +TEST_F(SpaceManagerTest, TestAddVolume_LoadBlockGroupsError) { + using type = std::vector*; + EXPECT_CALL(*etcdclient_, List(_, _, Matcher(_))) + .WillOnce(Return(EtcdErrCode::EtcdInternal)); + + FsInfo fsInfo = GenerateVolumeFsInfo(); + ASSERT_EQ(SpaceErrCreate, spaceManager_->AddVolume(fsInfo)); + ASSERT_EQ(nullptr, spaceManager_->GetVolumeSpace(fsInfo.fsid())); +} + +TEST_F(SpaceManagerTest, TestRemoveVolume_SpaceNotExist) { + ASSERT_EQ(SpaceErrNotFound, spaceManager_->RemoveVolume(kFsId + 1)); +} + +TEST_F(SpaceManagerTest, TestRemoveVolume_Success) { + AddOneNewVolume(); + ASSERT_EQ(SpaceOk, spaceManager_->RemoveVolume(kFsId)); + ASSERT_EQ(nullptr, spaceManager_->GetVolumeSpace(kFsId)); +} + +TEST_F(SpaceManagerTest, TestDeleteVolume_SpaceNotExist) { + ASSERT_EQ(SpaceErrNotFound, spaceManager_->DeleteVolume(kFsId)); +} + +TEST_F(SpaceManagerTest, TestDeleteVolume_Success) { + AddOneNewVolume(); + ASSERT_EQ(SpaceOk, spaceManager_->DeleteVolume(kFsId)); + ASSERT_EQ(nullptr, spaceManager_->GetVolumeSpace(kFsId)); +} + +} // namespace space +} // namespace mds +} // namespace curvefs diff --git a/curvefs/test/mds/space/space_service_test.cpp b/curvefs/test/mds/space/space_service_test.cpp new file mode 100644 index 0000000000..f058d6ccb8 --- /dev/null +++ b/curvefs/test/mds/space/space_service_test.cpp @@ -0,0 +1,213 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Wednesday Mar 23 14:47:45 CST 2022 + * Author: wuhanqing + */ + +#include +#include +#include +#include +#include + +#include "absl/memory/memory.h" +#include "curvefs/src/mds/space/service.h" +#include "curvefs/test/mds/mock/mock_space_manager.h" +#include "curvefs/test/mds/mock/mock_volume_space.h" +#include "curvefs/test/utils/protobuf_message_utils.h" +#include "src/common/macros.h" + +namespace curvefs { +namespace mds { +namespace space { + +using ::curvefs::test::GenerateAnDefaultInitializedMessage; +using ::testing::_; +using ::testing::Invoke; +using ::testing::Return; + +static constexpr uint32_t kRetryTimes = 10; +static unsigned int seed = time(nullptr); + +class SpaceServiceTest : public ::testing::Test { + protected: + void SetUp() override { + service_ = absl::make_unique(&spaceMgr_); + server_.AddService(service_.get(), brpc::SERVER_DOESNT_OWN_SERVICE); + + bool success = false; + int retry = 0; + + do { + listening_.port = rand_r(&seed) % 20000 + 20000; // [20000 ~ 39999) + + int ret = server_.Start(listening_, nullptr); + if (!ret) { + LOG(INFO) << "Listening on " << listening_; + success = true; + break; + } + } while (retry++ < kRetryTimes); + + ASSERT_TRUE(success); + } + + void TearDown() override { + server_.Stop(0); + server_.Join(); + } + + protected: + int port_; + brpc::Server server_; + MockSpaceManager spaceMgr_; + std::unique_ptr service_; + butil::EndPoint listening_; +}; + +TEST_F(SpaceServiceTest, FsNotFound) { + brpc::Channel channel; + ASSERT_EQ(0, channel.Init(listening_, nullptr)); + + EXPECT_CALL(spaceMgr_, GetVolumeSpace(_)) + .WillRepeatedly(Return(nullptr)); + +#define NotFoundTest(type) \ + do { \ + auto request = GenerateAnDefaultInitializedMessage( \ + "curvefs.mds.space." STRINGIFY(type) "Request"); \ + ASSERT_TRUE(request); \ + type##Response response; \ + brpc::Controller cntl; \ + SpaceService_Stub stub(&channel); \ + stub.type(&cntl, static_cast(request.get()), \ + &response, nullptr); \ + ASSERT_FALSE(cntl.Failed()); \ + ASSERT_EQ(SpaceErrCode::SpaceErrNotFound, response.status()); \ + } while (0) + + NotFoundTest(AllocateBlockGroup); + NotFoundTest(AcquireBlockGroup); + NotFoundTest(ReleaseBlockGroup); + NotFoundTest(StatSpace); + NotFoundTest(UpdateUsage); + +#undef NotFoundTest +} + +TEST_F(SpaceServiceTest, AllcoateBlockGroupTest) { + brpc::Channel channel; + ASSERT_EQ(0, channel.Init(listening_, nullptr)); + + for (auto err : {SpaceErrNoSpace, SpaceOk}) { + MockVolumeSpace volumeSpace; + EXPECT_CALL(spaceMgr_, GetVolumeSpace(_)) + .WillOnce(Return(&volumeSpace)); + + EXPECT_CALL(volumeSpace, AllocateBlockGroups(_, _, _)) + .WillOnce(Return(err)); + + auto request = GenerateAnDefaultInitializedMessage( + "curvefs.mds.space.AllocateBlockGroupRequest"); + ASSERT_TRUE(request); + + AllocateBlockGroupResponse response; + brpc::Controller cntl; + SpaceService_Stub stub(&channel); + + stub.AllocateBlockGroup( + &cntl, static_cast(request.get()), + &response, nullptr); + ASSERT_FALSE(cntl.Failed()); + ASSERT_EQ(err, response.status()); + } +} + +TEST_F(SpaceServiceTest, AcquireBlockGroupTest) { + brpc::Channel channel; + ASSERT_EQ(0, channel.Init(listening_, nullptr)); + + for (auto err : {SpaceErrNoSpace, SpaceOk}) { + MockVolumeSpace volumeSpace; + EXPECT_CALL(spaceMgr_, GetVolumeSpace(_)) + .WillOnce(Return(&volumeSpace)); + + EXPECT_CALL(volumeSpace, AcquireBlockGroup(_, _, _)) + .WillOnce( + Invoke([err](uint64_t blockGroupOffset, + const std::string& owner, BlockGroup* group) { + if (err != SpaceOk) { + return err; + } + + group->set_offset(blockGroupOffset); + group->set_size(0); + group->set_available(0); + group->set_bitmaplocation(BitmapLocation::AtEnd); + + return err; + })); + + auto request = GenerateAnDefaultInitializedMessage( + "curvefs.mds.space.AcquireBlockGroupRequest"); + ASSERT_TRUE(request); + + AcquireBlockGroupResponse response; + brpc::Controller cntl; + SpaceService_Stub stub(&channel); + + stub.AcquireBlockGroup( + &cntl, static_cast(request.get()), + &response, nullptr); + ASSERT_FALSE(cntl.Failed()) << cntl.ErrorText(); + ASSERT_EQ(err, response.status()); + } +} + +TEST_F(SpaceServiceTest, ReleaseBlockGroupTest) { + brpc::Channel channel; + ASSERT_EQ(0, channel.Init(listening_, nullptr)); + + for (auto err : {SpaceErrNoSpace, SpaceOk}) { + MockVolumeSpace volumeSpace; + EXPECT_CALL(spaceMgr_, GetVolumeSpace(_)) + .WillOnce(Return(&volumeSpace)); + + EXPECT_CALL(volumeSpace, ReleaseBlockGroups(_)) + .WillOnce(Return(err)); + + auto request = GenerateAnDefaultInitializedMessage( + "curvefs.mds.space.ReleaseBlockGroupRequest"); + ASSERT_TRUE(request); + + ReleaseBlockGroupResponse response; + brpc::Controller cntl; + SpaceService_Stub stub(&channel); + + stub.ReleaseBlockGroup( + &cntl, static_cast(request.get()), + &response, nullptr); + ASSERT_FALSE(cntl.Failed()); + ASSERT_EQ(err, response.status()); + } +} + +} // namespace space +} // namespace mds +} // namespace curvefs diff --git a/curvefs/test/mds/space/volume_space_test.cpp b/curvefs/test/mds/space/volume_space_test.cpp new file mode 100644 index 0000000000..42dfeebb1d --- /dev/null +++ b/curvefs/test/mds/space/volume_space_test.cpp @@ -0,0 +1,335 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Tuesday Mar 01 19:30:56 CST 2022 + * Author: wuhanqing + */ + +#include "curvefs/src/mds/space/volume_space.h" + +#include + +#include "absl/memory/memory.h" +#include "curvefs/src/mds/space/block_group_storage.h" +#include "curvefs/test/mds/mock/mock_block_group_storage.h" + +namespace curvefs { +namespace mds { +namespace space { + +using ::testing::_; +using ::testing::Invoke; +using ::testing::Return; + +static constexpr uint32_t kFsId = 1; +static constexpr uint64_t kVolumeSize = 10ULL * 1024 * 1024 * 1024; +static constexpr uint32_t kBlockSize = 4096; +static constexpr uint32_t kBlockGroupSize = 128ULL * 1024 * 1024; +static constexpr curvefs::common::BitmapLocation kBitmapLocation = + curvefs::common::BitmapLocation::AtStart; +static constexpr uint32_t kAllocateOnce = 4; +static const char* kOwner = "test"; + +static unsigned int seed = time(nullptr); + +class VolumeSpaceTest : public ::testing::Test { + protected: + void SetUp() override { + storage_ = absl::make_unique(); + } + + std::unique_ptr CreateOneEmptyVolumeSpace() { + EXPECT_CALL(*storage_, ListBlockGroups(_, _)) + .WillOnce(Invoke([](uint32_t, std::vector* groups) { + groups->clear(); + return SpaceOk; + })); + + auto space = + VolumeSpace::Create(kFsId, kVolumeSize, kBlockSize, kBlockGroupSize, + kBitmapLocation, storage_.get()); + EXPECT_NE(nullptr, space); + return space; + } + + std::unique_ptr CreateVolumeSpaceWithTwoGroups( + bool allocated = true, std::vector* out = nullptr) { + BlockGroup group1; + group1.set_offset(0); + group1.set_size(kBlockGroupSize); + group1.set_available(kBlockGroupSize / 2); + group1.set_bitmaplocation(kBitmapLocation); + if (allocated) { + group1.set_owner(kOwner); + } + + BlockGroup group2; + group2.set_offset(kBlockGroupSize); + group2.set_size(kBlockGroupSize); + group2.set_available(kBlockGroupSize / 2); + group2.set_bitmaplocation(kBitmapLocation); + group2.set_owner(kOwner); + if (allocated) { + group2.set_owner(kOwner); + } + + std::vector exist{group1, group2}; + + if (out) { + *out = exist; + } + + EXPECT_CALL(*storage_, ListBlockGroups(_, _)) + .WillOnce( + Invoke([&exist](uint32_t, std::vector* groups) { + *groups = exist; + return SpaceOk; + })); + + auto space = + VolumeSpace::Create(kFsId, kVolumeSize, kBlockSize, kBlockGroupSize, + kBitmapLocation, storage_.get()); + EXPECT_NE(nullptr, space); + return space; + } + + protected: + std::unique_ptr storage_; +}; + +TEST_F(VolumeSpaceTest, TestCreate_ListError) { + EXPECT_CALL(*storage_, ListBlockGroups(_, _)) + .WillOnce(Return(SpaceErrStorage)); + + auto space = + VolumeSpace::Create(kFsId, kVolumeSize, kBlockSize, kBlockGroupSize, + kBitmapLocation, storage_.get()); + EXPECT_EQ(nullptr, space); +} + +TEST_F(VolumeSpaceTest, TestCreate_Success) { + EXPECT_CALL(*storage_, ListBlockGroups(_, _)) + .WillOnce(Invoke([](uint32_t, std::vector* groups) { + groups->clear(); + return SpaceOk; + })); + + auto space = + VolumeSpace::Create(kFsId, kVolumeSize, kBlockSize, kBlockGroupSize, + kBitmapLocation, storage_.get()); + EXPECT_NE(nullptr, space); +} + +TEST_F(VolumeSpaceTest, TestAllocateBlockGroups_PersistBlockGroupError) { + auto space = CreateOneEmptyVolumeSpace(); + + EXPECT_CALL(*storage_, PutBlockGroup(_, _, _)) + .WillOnce(Return(SpaceErrStorage)); + + std::vector groups; + EXPECT_EQ(SpaceErrStorage, + space->AllocateBlockGroups(kAllocateOnce, kOwner, &groups)); +} + +TEST_F(VolumeSpaceTest, TestAllocateBlockGroups) { + auto space = CreateOneEmptyVolumeSpace(); + constexpr auto totalGroups = kVolumeSize / kBlockGroupSize; + + EXPECT_CALL(*storage_, PutBlockGroup(_, _, _)) + .Times(totalGroups) + .WillRepeatedly(Return(SpaceOk)); + + std::vector groups; + + uint64_t i = 0; + while (i < totalGroups) { + uint64_t count = rand_r(&seed) % 10; + count = std::min(totalGroups - i, count); + + std::vector newGroups; + ASSERT_EQ(SpaceOk, + space->AllocateBlockGroups(count, kOwner, &newGroups)); + groups.insert(groups.end(), std::make_move_iterator(newGroups.begin()), + std::make_move_iterator(newGroups.end())); + + i += count; + } + + std::set offsets; + for (auto& group : groups) { + ASSERT_EQ(kOwner, group.owner()); + offsets.insert(group.offset()); + } + ASSERT_EQ(totalGroups, offsets.size()); + ASSERT_EQ(0, *offsets.begin()); + ASSERT_EQ(kVolumeSize - kBlockGroupSize, *offsets.rbegin()); + + ASSERT_EQ(SpaceErrNoSpace, + space->AllocateBlockGroups(kAllocateOnce, kOwner, &groups)); +} + +TEST_F(VolumeSpaceTest, TestAcquireBlockGroups) { + auto space = CreateOneEmptyVolumeSpace(); + + EXPECT_CALL(*storage_, PutBlockGroup(_, _, _)).WillOnce(Return(SpaceOk)); + + uint64_t offset = 0; + + BlockGroup group; + ASSERT_EQ(SpaceOk, space->AcquireBlockGroup(offset, kOwner, &group)); + ASSERT_EQ(kOwner, group.owner()); + ASSERT_EQ(offset, group.offset()); +} + +TEST_F(VolumeSpaceTest, TestAcquireBlockGroups_Retry) { + auto space = CreateOneEmptyVolumeSpace(); + + EXPECT_CALL(*storage_, PutBlockGroup(_, _, _)) + .Times(2) + .WillRepeatedly(Return(SpaceOk)); + + uint64_t offset = 0; + + BlockGroup group; + ASSERT_EQ(SpaceOk, space->AcquireBlockGroup(offset, kOwner, &group)); + ASSERT_EQ(kOwner, group.owner()); + ASSERT_EQ(offset, group.offset()); + + BlockGroup group2; + ASSERT_EQ(SpaceOk, space->AcquireBlockGroup(offset, kOwner, &group2)); + ASSERT_EQ(kOwner, group2.owner()); + ASSERT_EQ(offset, group2.offset()); +} + +TEST_F(VolumeSpaceTest, TestAcquireBlockGroups_Conflict) { + auto space = CreateOneEmptyVolumeSpace(); + + EXPECT_CALL(*storage_, PutBlockGroup(_, _, _)) + .Times(1) + .WillOnce(Return(SpaceOk)); + + uint64_t offset = 0; + + BlockGroup group; + ASSERT_EQ(SpaceOk, space->AcquireBlockGroup(offset, kOwner, &group)); + ASSERT_EQ(kOwner, group.owner()); + ASSERT_EQ(offset, group.offset()); + + BlockGroup group2; + ASSERT_EQ(SpaceErrConflict, + space->AcquireBlockGroup(offset, "another owner", &group2)); +} + +TEST_F(VolumeSpaceTest, TestRemoveAllBlockGroups_EmptyVolumeSpace) { + auto space = CreateOneEmptyVolumeSpace(); + ASSERT_EQ(SpaceOk, space->RemoveAllBlockGroups()); +} + +TEST_F(VolumeSpaceTest, TestRemoveAllBlockGroups_ClearBlockGroupError) { + for (auto allocated : {true, false}) { + auto space = CreateVolumeSpaceWithTwoGroups(allocated); + + EXPECT_CALL(*storage_, RemoveBlockGroup(_, _)) + .WillOnce(Return(SpaceErrStorage)); + + ASSERT_NE(SpaceOk, space->RemoveAllBlockGroups()); + } +} + +TEST_F(VolumeSpaceTest, TestRemoveAllBlockGroups_Success) { + for (auto allocated : {true, false}) { + auto space = CreateVolumeSpaceWithTwoGroups(allocated); + + EXPECT_CALL(*storage_, RemoveBlockGroup(_, _)) + .Times(2) + .WillRepeatedly(Return(SpaceOk)); + + ASSERT_EQ(SpaceOk, space->RemoveAllBlockGroups()); + } +} + +TEST_F(VolumeSpaceTest, TestReleaseBlockGroup_OwnerNotIdentical) { + std::vector exists; + auto space = CreateVolumeSpaceWithTwoGroups(true, &exists); + + exists[0].set_owner("hello"); + + ASSERT_EQ(SpaceErrConflict, space->ReleaseBlockGroups(exists)); +} + +TEST_F(VolumeSpaceTest, TestReleaseBlockGroup_SpaceIsNotUsed) { + std::vector exists; + auto space = CreateVolumeSpaceWithTwoGroups(true, &exists); + + exists[0].set_available(exists[0].size()); + exists[1].set_available(exists[1].size()); + + EXPECT_CALL(*storage_, RemoveBlockGroup(_, _)) + .Times(2) + .WillRepeatedly(Return(SpaceOk)); + + ASSERT_EQ(SpaceOk, space->ReleaseBlockGroups(exists)); +} + +TEST_F(VolumeSpaceTest, TestReleaseBlockGroup_SpaceIsNotUsed_ButClearError) { + std::vector exists; + auto space = CreateVolumeSpaceWithTwoGroups(true, &exists); + + exists[0].set_available(exists[0].size()); + exists[1].set_available(exists[1].size()); + + EXPECT_CALL(*storage_, RemoveBlockGroup(_, _)) + .WillOnce(Return(SpaceErrStorage)); + + ASSERT_EQ(SpaceErrStorage, space->ReleaseBlockGroups(exists)); +} + +MATCHER(NoOwner, "") { + return !arg.has_owner(); +} + +TEST_F(VolumeSpaceTest, TestReleaseBlockGroup_SpaceIsPartialUsed) { + std::vector exists; + auto space = CreateVolumeSpaceWithTwoGroups(true, &exists); + + exists[0].set_available(exists[0].size() / 2); + exists[1].set_available(exists[1].size() / 2); + + EXPECT_CALL(*storage_, PutBlockGroup(_, _, NoOwner())) + .Times(2) + .WillRepeatedly(Return(SpaceOk)); + + ASSERT_EQ(SpaceOk, space->ReleaseBlockGroups(exists)); +} + +TEST_F(VolumeSpaceTest, TestReleaseBlockGroup_SpaceIsPartialUsed_PutError) { + std::vector exists; + auto space = CreateVolumeSpaceWithTwoGroups(true, &exists); + + exists[0].set_available(exists[0].size() / 2); + exists[1].set_available(exists[1].size() / 2); + + EXPECT_CALL(*storage_, PutBlockGroup(_, _, NoOwner())) + .WillOnce(Return(SpaceErrStorage)); + + ASSERT_EQ(SpaceErrStorage, space->ReleaseBlockGroups(exists)); +} + +} // namespace space +} // namespace mds +} // namespace curvefs diff --git a/curvefs/test/mds/space_client_test.cpp b/curvefs/test/mds/space_client_test.cpp deleted file mode 100644 index 65302adb61..0000000000 --- a/curvefs/test/mds/space_client_test.cpp +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* - * @Project: curve - * @Date: 2021-07-07 14:46:28 - * @Author: chenwei - */ - -#include "curvefs/src/mds/spaceclient/space_client.h" -#include -#include -#include -#include -#include "curvefs/test/mds/mock/mock_space.h" - -using ::testing::AtLeast; -using ::testing::StrEq; -using ::testing::_; -using ::testing::Return; -using ::testing::ReturnArg; -using ::testing::DoAll; -using ::testing::SetArgPointee; -using ::testing::SaveArg; -using ::testing::Mock; -using ::testing::Invoke; -using ::curvefs::space::MockSpaceService; -using curvefs::space::InitSpaceRequest; -using curvefs::space::InitSpaceResponse; -using curvefs::space::UnInitSpaceRequest; -using curvefs::space::UnInitSpaceResponse; -using curvefs::space::SpaceStatusCode; - -namespace curvefs { -namespace mds { -class SpaceClientTest : public ::testing::Test { - protected: - void SetUp() override { - addr_ = "127.0.0.1:6704"; - ASSERT_EQ(0, server_.AddService(&mockSpaceService_, - brpc::SERVER_DOESNT_OWN_SERVICE)); - ASSERT_EQ(0, server_.Start(addr_.c_str(), nullptr)); - - return; - } - - void TearDown() override { - server_.Stop(0); - server_.Join(); - return; - } - - protected: - MockSpaceService mockSpaceService_; - brpc::Server server_; - std::string addr_; -}; - -template -void RpcService(google::protobuf::RpcController *cntl_base, - const RpcRequestType *request, RpcResponseType *response, - google::protobuf::Closure *done) { - if (RpcFailed) { - brpc::Controller *cntl = static_cast(cntl_base); - cntl->SetFailed(112, "Not connected to"); - } - done->Run(); -} - -TEST_F(SpaceClientTest, InitFailTest) { - SpaceOptions options; - SpaceClient client(options); - ASSERT_FALSE(client.Init()); -} - -TEST_F(SpaceClientTest, InitSuccess) { - SpaceOptions options; - options.spaceAddr = addr_; - SpaceClient client(options); - ASSERT_TRUE(client.Init()); -} - -TEST_F(SpaceClientTest, InitSpaceNotInitFail) { - SpaceOptions options; - options.spaceAddr = addr_; - SpaceClient client(options); - FsInfo fsinfo; - ASSERT_EQ(client.InitSpace(fsinfo), FSStatusCode::SPACE_CLIENT_NOT_INITED); -} - -TEST_F(SpaceClientTest, UnInitSpaceNotInitFail) { - SpaceOptions options; - options.spaceAddr = addr_; - SpaceClient client(options); - uint32_t fsId = 0; - ASSERT_EQ(client.UnInitSpace(fsId), FSStatusCode::SPACE_CLIENT_NOT_INITED); -} - -TEST_F(SpaceClientTest, InitSpaceSuccess) { - SpaceOptions options; - options.spaceAddr = addr_; - SpaceClient client(options); - ASSERT_TRUE(client.Init()); - FsInfo fsinfo; - fsinfo.set_fsid(1); - fsinfo.set_fsname("fs1"); - fsinfo.set_status(FsStatus::INITED); - fsinfo.set_rootinodeid(1); - fsinfo.set_capacity(0); - fsinfo.set_blocksize(0); - fsinfo.set_mountnum(0); - fsinfo.set_fstype(::curvefs::common::FSType::TYPE_VOLUME); - fsinfo.set_enablesumindir(false); - fsinfo.mutable_detail(); - - InitSpaceResponse response; - response.set_status(SpaceStatusCode::SPACE_OK); - EXPECT_CALL(mockSpaceService_, InitSpace(_, _, _, _)) - .WillOnce( - DoAll(SetArgPointee<2>(response), - Invoke(RpcService))); - ASSERT_EQ(client.InitSpace(fsinfo), FSStatusCode::OK); -} - -TEST_F(SpaceClientTest, InitSpaceFail) { - SpaceOptions options; - options.spaceAddr = addr_; - SpaceClient client(options); - ASSERT_TRUE(client.Init()); - FsInfo fsinfo; - fsinfo.set_fsid(1); - fsinfo.set_fsname("fs1"); - fsinfo.set_status(FsStatus::INITED); - fsinfo.set_rootinodeid(1); - fsinfo.set_capacity(0); - fsinfo.set_blocksize(0); - fsinfo.set_mountnum(0); - fsinfo.set_fstype(::curvefs::common::FSType::TYPE_VOLUME); - fsinfo.set_enablesumindir(false); - fsinfo.mutable_detail(); - - InitSpaceResponse response; - response.set_status(SpaceStatusCode::SPACE_UNKNOWN_ERROR); - EXPECT_CALL(mockSpaceService_, InitSpace(_, _, _, _)) - .WillOnce( - DoAll(SetArgPointee<2>(response), - Invoke(RpcService))); - ASSERT_EQ(client.InitSpace(fsinfo), FSStatusCode::INIT_SPACE_ERROR); -} - -TEST_F(SpaceClientTest, UnInitSpaceSuccess) { - SpaceOptions options; - options.spaceAddr = addr_; - SpaceClient client(options); - ASSERT_TRUE(client.Init()); - uint32_t fsId = 0; - - UnInitSpaceResponse response; - response.set_status(SpaceStatusCode::SPACE_OK); - EXPECT_CALL(mockSpaceService_, UnInitSpace(_, _, _, _)) - .WillOnce( - DoAll(SetArgPointee<2>(response), - Invoke(RpcService))); - ASSERT_EQ(client.UnInitSpace(fsId), FSStatusCode::OK); -} - -TEST_F(SpaceClientTest, UnInitSpaceFail) { - SpaceOptions options; - options.spaceAddr = addr_; - SpaceClient client(options); - ASSERT_TRUE(client.Init()); - uint32_t fsId = 0; - - UnInitSpaceResponse response; - response.set_status(SpaceStatusCode::SPACE_UNKNOWN_ERROR); - EXPECT_CALL(mockSpaceService_, UnInitSpace(_, _, _, _)) - .WillOnce( - DoAll(SetArgPointee<2>(response), - Invoke(RpcService))); - ASSERT_EQ(client.UnInitSpace(fsId), FSStatusCode::UNINIT_SPACE_ERROR); -} -} // namespace mds -} // namespace curvefs diff --git a/curvefs/test/metaserver/metaserver_s3_adaptor_test.cpp b/curvefs/test/metaserver/metaserver_s3_adaptor_test.cpp index c48f4101db..9d21562c86 100644 --- a/curvefs/test/metaserver/metaserver_s3_adaptor_test.cpp +++ b/curvefs/test/metaserver/metaserver_s3_adaptor_test.cpp @@ -29,8 +29,6 @@ class MetaserverS3AdaptorTest : public testing::Test { void SetUp() override { ASSERT_EQ(0, server_.AddService(&mockMetaServerService_, brpc::SERVER_DOESNT_OWN_SERVICE)); - ASSERT_EQ(0, server_.AddService(&mockSpaceAllocService_, - brpc::SERVER_DOESNT_OWN_SERVICE)); ASSERT_EQ(0, server_.Start(addr_.c_str(), nullptr)); S3ClientAdaptorOption option; @@ -59,7 +57,6 @@ class MetaserverS3AdaptorTest : public testing::Test { client::MockS3Client mockClientS3Client_; client::MockMetaServerService mockMetaServerService_; - client::MockSpaceAllocService mockSpaceAllocService_; std::string addr_ = "127.0.0.1:5629"; brpc::Server server_; }; diff --git a/curvefs/test/metaserver/metaserver_s3_adaptor_test.h b/curvefs/test/metaserver/metaserver_s3_adaptor_test.h index f9576ef856..692af4ae9d 100644 --- a/curvefs/test/metaserver/metaserver_s3_adaptor_test.h +++ b/curvefs/test/metaserver/metaserver_s3_adaptor_test.h @@ -32,7 +32,6 @@ #include "curvefs/src/metaserver/s3/metaserver_s3_adaptor.h" #include "curvefs/test/client/mock_client_s3.h" #include "curvefs/test/client/mock_metaserver_service.h" -#include "curvefs/test/client/mock_spacealloc_service.h" #include "curvefs/test/metaserver/mock_metaserver_s3.h" namespace curvefs { diff --git a/curvefs/test/metaserver/metastore_test.cpp b/curvefs/test/metaserver/metastore_test.cpp index 808a10e5a6..374039d038 100644 --- a/curvefs/test/metaserver/metastore_test.cpp +++ b/curvefs/test/metaserver/metastore_test.cpp @@ -440,7 +440,7 @@ TEST_F(MetastoreTest, test_inode) { updateRequest3.set_fsid(fsId); updateRequest3.set_inodeid(createResponse.inode().inodeid()); VolumeExtentList volumeExtentList; - updateRequest3.mutable_volumeextentlist()->CopyFrom(volumeExtentList); + updateRequest3.mutable_volumeextentmap()->insert({0, volumeExtentList}); S3ChunkInfoList s3ChunkInfoList; updateRequest3.mutable_s3chunkinfomap()->insert({0, s3ChunkInfoList}); ret = metastore.UpdateInode(&updateRequest3, &updateResponse3); diff --git a/curvefs/test/metaserver/test_helper.cpp b/curvefs/test/metaserver/test_helper.cpp index cbdf20f301..636646009d 100644 --- a/curvefs/test/metaserver/test_helper.cpp +++ b/curvefs/test/metaserver/test_helper.cpp @@ -43,10 +43,7 @@ UpdateInodeRequest MakeUpdateInodeRequestFromInode(const Inode &inode, request.set_uid(inode.uid()); request.set_gid(inode.gid()); request.set_mode(inode.mode()); - VolumeExtentList *vlist = - new VolumeExtentList; - vlist->CopyFrom(inode.volumeextentlist()); - request.set_allocated_volumeextentlist(vlist); + *(request.mutable_volumeextentmap()) = inode.volumeextentmap(); *(request.mutable_s3chunkinfomap()) = inode.s3chunkinfomap(); *(request.mutable_xattr()) = inode.xattr(); request.set_nlink(inode.nlink()); diff --git a/curvefs/test/space/BUILD b/curvefs/test/space/BUILD deleted file mode 100644 index 17f9c85ba6..0000000000 --- a/curvefs/test/space/BUILD +++ /dev/null @@ -1,163 +0,0 @@ -# -# Copyright (c) 2021 NetEase Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") -load("//:copts.bzl", "CURVE_TEST_COPTS") - -LINKOPTS = [ -] - -config_setting( - name = "absl_btree", - values = { - "define": "use_absl_btree=true", - }, - visibility = ["//visibility:public"], -) - -config_setting( - name = "absl_vector", - values = { - "define": "use_absl_vector=true", - }, - visibility = ["//visibility:public"], -) - -ABSL_COPTS = select({ - ":absl_btree": ["-DUSE_ABSL_BTREE"], - "//conditions:default": [], -}) + select({ - ":absl_vector": ["-DUSE_ABSL_VECTOR"], - "//conditions:default": [], -}) - -ABSL_DEPS = select({ - ":absl_btree": ["@com_google_absl//absl/container:btree"], - "//conditions:default": [], -}) + select({ - ":absl_vector": ["@com_google_absl//absl/container:inlined_vector"], - "//conditions:default": [], -}) - -cc_test( - name = "extents_test", - srcs = ["extents_test.cpp"], - copts = CURVE_TEST_COPTS + ABSL_COPTS, - linkopts = LINKOPTS, - deps = [ - "//curvefs/src/space:curve_fs_space", - "@com_google_googletest//:gtest", - ] + ABSL_DEPS, -) - -cc_test( - name = "bitmap_allocator_test", - srcs = [ - "bitmap_allocator_test.cpp", - "common.h", - ], - copts = CURVE_TEST_COPTS + ABSL_COPTS, - linkopts = LINKOPTS, - deps = [ - "//curvefs/src/space:curve_fs_space", - "@com_google_googletest//:gtest", - ] + ABSL_DEPS, -) - -cc_test( - name = "bitmap_allocator_brute_test", - srcs = [ - "bitmap_allocator_brute_test.cpp", - "common.h", - ], - copts = CURVE_TEST_COPTS + ABSL_COPTS, - linkopts = LINKOPTS, - deps = [ - "//curvefs/src/space:curve_fs_space", - "@com_google_googletest//:gtest", - ] + ABSL_DEPS, -) - -cc_test( - name = "bitmap_allocator_multi_thread_brute_test", - srcs = [ - "bitmap_allocator_multi_thread_brute_test.cpp", - "common.h", - ], - copts = CURVE_TEST_COPTS + ABSL_COPTS, - linkopts = LINKOPTS, - deps = [ - "//curvefs/src/space:curve_fs_space", - "@com_google_googletest//:gtest", - ] + ABSL_DEPS, -) - -cc_test( - name = "space_service_test", - srcs = [ - "common.h", - "space_service_test.cpp", - ], - copts = CURVE_TEST_COPTS + ABSL_COPTS, - linkopts = LINKOPTS, - deps = [ - "//curvefs/src/space:curve_fs_space", - "//curvefs/test/space/mock:space_test_mock", - "@com_google_googletest//:gtest", - ] + ABSL_DEPS, -) - -cc_test( - name = "space_manager_test", - srcs = [ - "common.h", - "space_manager_test.cpp", - ], - copts = CURVE_TEST_COPTS + ABSL_COPTS, - linkopts = LINKOPTS, - deps = [ - "//curvefs/src/space:curve_fs_space", - "//curvefs/test/space/mock:space_test_mock", - "@com_google_googletest//:gtest", - ] + ABSL_DEPS, -) - -cc_test( - name = "allocator_test", - srcs = [ - "allocator_test.cpp", - "common.h", - ], - copts = CURVE_TEST_COPTS + ABSL_COPTS, - linkopts = LINKOPTS, - deps = [ - "//curvefs/src/space:curve_fs_space", - "@com_google_googletest//:gtest", - ] + ABSL_DEPS, -) - -# https://docs.bazel.build/versions/master/be/c-cpp.html#cc_binary -cc_binary( - name = "fake_user", - srcs = [ - "fake_user.cpp", - ], - copts = CURVE_TEST_COPTS + ABSL_COPTS, - deps = [ - "//curvefs/proto:space_cc_proto", - "//curvefs/src/space:curve_fs_space", - ] + ABSL_DEPS, -) diff --git a/curvefs/test/space/fake_user.cpp b/curvefs/test/space/fake_user.cpp deleted file mode 100644 index b6fc5d7065..0000000000 --- a/curvefs/test/space/fake_user.cpp +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "curvefs/proto/mds.pb.h" -#include "curvefs/proto/space.pb.h" -#include "curvefs/src/space/common.h" - -using ::curvefs::space::kGiB; -using ::curvefs::space::kKiB; -using ::curvefs::space::kMiB; - -brpc::Channel channel; - -DEFINE_string(address, "0.0.0.0:19999", "space server address"); - -DEFINE_int32(fsId, -1, "filesystem"); -DEFINE_uint64(capacity, 100 * kGiB, "filesystem size"); - -using ::curvefs::space::PExtent; - -const std::vector sizes = { - 4 * kKiB, 8 * kKiB, 16 * kKiB, 32 * kKiB, 512 * kKiB, // NOLINT - 1 * kMiB, 4 * kMiB, 8 * kMiB, 16 * kMiB, 32 * kMiB // NOLINT -}; - -std::deque exts; - -std::atomic noSpace(false); - -void Alloc() { - ::curvefs::space::AllocateSpaceRequest request; - ::curvefs::space::AllocateSpaceResponse response; - - request.set_fsid(FLAGS_fsId); - request.set_size(sizes[rand() % sizes.size()]); - - brpc::Controller cntl; - ::curvefs::space::SpaceAllocService_Stub stub(&channel); - stub.AllocateSpace(&cntl, &request, &response, nullptr); - - if (cntl.Failed()) { - LOG(FATAL) << "rpc cntl failed, error: " << cntl.ErrorText(); - } - - if (response.status() == curvefs::space::SPACE_NO_SPACE) { - noSpace.store(true, std::memory_order_relaxed); - LOG(INFO) << "NO SPACE"; - } else { - uint64_t allocated = 0; - for (auto& e : response.extents()) { - exts.emplace_back(e.offset(), e.length()); - allocated += e.length(); - - LOG_IF(FATAL, e.offset() >= FLAGS_capacity || - e.offset() + e.length() > FLAGS_capacity) - << "allocate exceed capacity"; - } - - LOG_IF(FATAL, allocated != request.size()) - << "allocate success, but size is not as expected"; - } -} - -void Dealloc() { - if (exts.empty()) { - return; - } - - ::curvefs::space::DeallocateSpaceRequest request; - ::curvefs::space::DeallocateSpaceResponse response; - - request.set_fsid(FLAGS_fsId); - auto* e = request.add_extents(); - auto p = exts.front(); - exts.pop_front(); - e->set_offset(p.offset); - e->set_length(p.len); - - brpc::Controller cntl; - ::curvefs::space::SpaceAllocService_Stub stub(&channel); - stub.DeallocateSpace(&cntl, &request, &response, nullptr); - - if (cntl.Failed() || response.status() != ::curvefs::space::SPACE_OK) { - LOG(FATAL) << "deallocate failed"; - } - - noSpace.store(false, std::memory_order_relaxed); -} - -void Init() { - ::curvefs::space::InitSpaceRequest request; - ::curvefs::space::InitSpaceResponse response; - - auto* fsInfo = request.mutable_fsinfo(); - fsInfo->set_fsid(FLAGS_fsId); - fsInfo->set_fsname(std::to_string(FLAGS_fsId)); - fsInfo->set_rootinodeid(0); - fsInfo->set_capacity(FLAGS_capacity); - fsInfo->set_blocksize(4 * kKiB); - fsInfo->set_mountnum(0); - - auto* vol = fsInfo->mutable_detail()->mutable_volume(); - vol->set_volumesize(FLAGS_capacity); - vol->set_blocksize(4 * kKiB); - vol->set_volumename(std::to_string(FLAGS_fsId)); - vol->set_user("hello"); - - brpc::Controller cntl; - int rv = channel.Init(FLAGS_address.c_str(), nullptr); - if (rv != 0) { - LOG(FATAL) << "init channel to space server failed"; - } - - ::curvefs::space::SpaceAllocService_Stub stub(&channel); - stub.InitSpace(&cntl, &request, &response, nullptr); - - if (cntl.Failed() || response.status() != ::curvefs::space::SPACE_OK) { - LOG(FATAL) << "init space failed"; - } -} - -void Run() { - thread_local unsigned int seed = time(nullptr); - while (true) { - if (rand_r(&seed) % 4 == 0 || - noSpace.load(std::memory_order_relaxed) == true) { - Dealloc(); - } else { - Alloc(); - } - - std::this_thread::sleep_for(std::chrono::milliseconds(50)); - } -} - -int main(int argc, char* argv[]) { - ::google::ParseCommandLineFlags(&argc, &argv, true); - FLAGS_log_dir = "/home/wuhanqing/log"; - - ::google::InitGoogleLogging(argv[0]); - - srand(time(nullptr)); - - Init(); - Run(); - - return 0; -} diff --git a/curvefs/test/space/mock/mock_metaserver.h b/curvefs/test/space/mock/mock_metaserver.h deleted file mode 100644 index 51217468f6..0000000000 --- a/curvefs/test/space/mock/mock_metaserver.h +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Project: curve - * File Created: Fri Jul 30 10:25:00 CST 2021 - * Author: wuhanqing - */ - -#ifndef CURVEFS_TEST_SPACE_MOCK_MOCK_METASERVER_H_ -#define CURVEFS_TEST_SPACE_MOCK_MOCK_METASERVER_H_ - -#include -#include - -#include "curvefs/proto/metaserver.pb.h" - -namespace curvefs { -namespace space { - -class MockMetaServerService : public curvefs::metaserver::MetaServerService { - public: - MockMetaServerService() = default; - ~MockMetaServerService() = default; - - MOCK_METHOD4(ListDentry, - void(::google::protobuf::RpcController* controller, - const ::curvefs::metaserver::ListDentryRequest* request, - ::curvefs::metaserver::ListDentryResponse* response, - ::google::protobuf::Closure* done)); -}; - -} // namespace space -} // namespace curvefs - -#endif // CURVEFS_TEST_SPACE_MOCK_MOCK_METASERVER_H_ diff --git a/curvefs/test/space/mock/mock_space_manager.h b/curvefs/test/space/mock/mock_space_manager.h deleted file mode 100644 index 3444c1154f..0000000000 --- a/curvefs/test/space/mock/mock_space_manager.h +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef CURVEFS_TEST_SPACE_MOCK_MOCK_SPACE_MANAGER_H_ -#define CURVEFS_TEST_SPACE_MOCK_MOCK_SPACE_MANAGER_H_ - -#include - -#include - -#include "curvefs/src/space/space_manager.h" - -namespace curvefs { -namespace space { - -class MockSpaceManager : public SpaceManager { - public: - MOCK_METHOD1(InitSpace, SpaceStatusCode(const mds::FsInfo&)); - MOCK_METHOD1(UnInitSpace, SpaceStatusCode(uint32_t)); - MOCK_METHOD2(StatSpace, SpaceStatusCode(uint32_t, SpaceStat*)); - MOCK_METHOD4(AllocateSpace, - SpaceStatusCode(uint32_t fsId, uint64_t size, - const SpaceAllocateHint&, Extents*)); - MOCK_METHOD2(DeallocateSpace, - SpaceStatusCode(uint32_t, const ProtoExtents&)); -}; - -} // namespace space -} // namespace curvefs - -#endif // CURVEFS_TEST_SPACE_MOCK_MOCK_SPACE_MANAGER_H_ diff --git a/curvefs/test/space/space_manager_test.cpp b/curvefs/test/space/space_manager_test.cpp deleted file mode 100644 index 0554454a36..0000000000 --- a/curvefs/test/space/space_manager_test.cpp +++ /dev/null @@ -1,169 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "curvefs/src/space/space_manager.h" - -#include -#include - -#include "curvefs/src/space/utils.h" -#include "curvefs/test/space/common.h" -#include "curvefs/test/space/mock/mock_metaserver.h" - -namespace curvefs { -namespace space { - -using ::testing::_; -using ::testing::Invoke; - -void ListDentryService(::google::protobuf::RpcController* controller, - const ::curvefs::metaserver::ListDentryRequest* request, - ::curvefs::metaserver::ListDentryResponse* response, - ::google::protobuf::Closure* done) { - brpc::ClosureGuard doneGuard(done); - response->set_statuscode(metaserver::MetaStatusCode::OK); -} - -const char kMetaServerAddress[] = "127.0.0.1:12345"; - -class SpaceManagerTest : public ::testing::Test { - protected: - void SetUp() override { - SpaceManagerOption option; - option.allocatorType = "bitmap"; - option.allocatorOption.bitmapAllocatorOption.length = 10 * kGiB; - option.allocatorOption.bitmapAllocatorOption.sizePerBit = 4 * kMiB; - option.reloaderOption.metaServerOption.addr = kMetaServerAddress; - - space_.reset(new SpaceManagerImpl(option)); - - mockMetaServer_.reset(new MockMetaServerService()); - server_.reset(new brpc::Server()); - - ASSERT_EQ(0, server_->AddService(mockMetaServer_.get(), - brpc::SERVER_DOESNT_OWN_SERVICE)); - ASSERT_EQ(0, server_->Start(kMetaServerAddress, nullptr)); - } - - void TearDown() override { - server_->Stop(0); - server_->Join(); - } - - void PrepareEnv() { - mds::FsInfo blockFsInfo; - blockFsInfo.set_fsid(1); - blockFsInfo.set_fsname("block"); - blockFsInfo.set_fstype(::curvefs::common::FSType::TYPE_VOLUME); - blockFsInfo.set_capacity(10 * kGiB); - - EXPECT_CALL(*mockMetaServer_, ListDentry(_, _, _, _)) - .WillRepeatedly(Invoke(ListDentryService)); - - EXPECT_EQ(SPACE_OK, space_->InitSpace(blockFsInfo)); - } - - protected: - std::unique_ptr space_; - std::unique_ptr server_; - std::unique_ptr mockMetaServer_; -}; - -TEST_F(SpaceManagerTest, TestInitSpace) { - mds::FsInfo fsInfo; - fsInfo.set_fsid(1); - fsInfo.set_fsname("block"); - fsInfo.set_fstype(::curvefs::common::FSType::TYPE_VOLUME); - - EXPECT_CALL(*mockMetaServer_, ListDentry(_, _, _, _)) - .WillRepeatedly(Invoke(ListDentryService)); - - EXPECT_EQ(SPACE_OK, space_->InitSpace(fsInfo)); - - // if fsid is already exists, init return exists - EXPECT_EQ(SPACE_EXISTS, space_->InitSpace(fsInfo)); -} - -TEST_F(SpaceManagerTest, TestUnInitSpace) { - EXPECT_EQ(SPACE_NOT_FOUND, space_->UnInitSpace(1)); - - PrepareEnv(); - - EXPECT_EQ(SPACE_OK, space_->UnInitSpace(1)); - - EXPECT_EQ(SPACE_NOT_FOUND, space_->UnInitSpace(1)); -} - -TEST_F(SpaceManagerTest, TestAllocateSpace) { - PrepareEnv(); - - Extents exts; - EXPECT_EQ(SPACE_NOT_FOUND, space_->AllocateSpace(100, 1 * kGiB, {}, &exts)); - EXPECT_TRUE(exts.empty()); - - EXPECT_EQ(SPACE_OK, space_->AllocateSpace(1, 1 * kGiB, {}, &exts)); - EXPECT_EQ(1 * kGiB, TotalLength(exts)); - - Extents exts2; - EXPECT_EQ(SPACE_NO_SPACE, space_->AllocateSpace(1, 10 * kGiB, {}, &exts2)); - EXPECT_TRUE(exts2.empty()); - - SpaceStat stat; - EXPECT_EQ(SPACE_OK, space_->StatSpace(1, &stat)); - EXPECT_EQ(9 * kGiB, stat.available); -} - -TEST_F(SpaceManagerTest, TestDeallocateSpace) { - PrepareEnv(); - - EXPECT_EQ(SPACE_NOT_FOUND, space_->DeallocateSpace(100, {})); - - Extents exts; - EXPECT_EQ(SPACE_OK, space_->AllocateSpace(1, 5 * kGiB, {}, &exts)); - EXPECT_EQ(5 * kGiB, TotalLength(exts)); - - EXPECT_EQ(SPACE_OK, - space_->DeallocateSpace(1, ConvertToProtoExtents(exts))); - - SpaceStat stat; - EXPECT_EQ(SPACE_OK, space_->StatSpace(1, &stat)); - EXPECT_EQ(stat.total, stat.available); - EXPECT_EQ(10 * kGiB, stat.available); - - Extents dealloc(exts); - EXPECT_EQ(SPACE_DEALLOC_ERROR, - space_->DeallocateSpace(1, ConvertToProtoExtents(exts))); -} - -TEST_F(SpaceManagerTest, TestStatSpace) { - PrepareEnv(); - - SpaceStat stat; - EXPECT_EQ(SPACE_NOT_FOUND, space_->StatSpace(100, &stat)); - - EXPECT_EQ(SPACE_OK, space_->StatSpace(1, &stat)); - - EXPECT_EQ(stat.total, stat.available); - EXPECT_EQ(stat.total, 10 * kGiB); -} - -} // namespace space -} // namespace curvefs - -int main(int argc, char* argv[]) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/curvefs/test/space/space_service_test.cpp b/curvefs/test/space/space_service_test.cpp deleted file mode 100644 index 0772e7eb18..0000000000 --- a/curvefs/test/space/space_service_test.cpp +++ /dev/null @@ -1,393 +0,0 @@ -/* - * Copyright (c) 2021 NetEase Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "curvefs/src/space/space_service.h" - -#include -#include -#include -#include - -#include - -#include "curvefs/proto/space.pb.h" -#include "curvefs/test/space/common.h" -#include "curvefs/test/space/mock/mock_space_manager.h" - -namespace curvefs { -namespace space { - -using ::testing::_; -using ::testing::DoAll; -using ::testing::Return; -using ::testing::SaveArg; -using ::testing::SetArgPointee; - -const char* kTestIpAddr = "127.0.0.1:56789"; // NOLINT - -class SpaceAllocServiceTest : public ::testing::Test { - protected: - void SetUp() override { - service_.reset(new SpaceAllocServiceImpl(&space_)); - - ASSERT_EQ(0, server_.AddService(service_.get(), - brpc::SERVER_DOESNT_OWN_SERVICE)); - - ASSERT_EQ(0, server_.Start(kTestIpAddr, nullptr)); - - ASSERT_EQ(0, channel_.Init(kTestIpAddr, nullptr)); - } - - void TearDown() override { - server_.Stop(0); - server_.Join(); - } - - protected: - MockSpaceManager space_; - std::unique_ptr service_; - brpc::Server server_; - brpc::Channel channel_; -}; - -TEST_F(SpaceAllocServiceTest, TestInitSpace) { - // test failed - { - InitSpaceRequest request; - InitSpaceResponse response; - auto* fsInfo = request.mutable_fsinfo(); - fsInfo->set_fsid(100); - fsInfo->set_fsname("TestInitSpace"); - fsInfo->set_status(mds::FsStatus::NEW); - fsInfo->set_rootinodeid(0); - fsInfo->set_capacity(100 * kGiB); - fsInfo->set_blocksize(4 * kKiB); - auto* volume = fsInfo->mutable_detail()->mutable_volume(); - volume->set_volumesize(100 * kGiB); - volume->set_blocksize(4 * kKiB); - volume->set_volumename("TestInitSpaceVolume"); - volume->set_user("test"); - fsInfo->set_mountnum(1); - fsInfo->set_fstype(::curvefs::common::FSType::TYPE_VOLUME); - fsInfo->set_enablesumindir(false); - - EXPECT_CALL(space_, InitSpace(_)) - .WillOnce(Return(SPACE_RELOAD_ERROR)); - - brpc::Controller cntl; - SpaceAllocService_Stub stub(&channel_); - stub.InitSpace(&cntl, &request, &response, nullptr); - - EXPECT_FALSE(cntl.Failed()) << cntl.ErrorText(); - EXPECT_EQ(SPACE_RELOAD_ERROR, response.status()); - } - - // test succeeded - { - InitSpaceRequest request; - InitSpaceResponse response; - auto* fsInfo = request.mutable_fsinfo(); - fsInfo->set_fsid(100); - fsInfo->set_fsname("TestInitSpace"); - fsInfo->set_status(mds::FsStatus::NEW); - fsInfo->set_rootinodeid(0); - fsInfo->set_capacity(100 * kGiB); - fsInfo->set_blocksize(4 * kKiB); - auto* volume = fsInfo->mutable_detail()->mutable_volume(); - volume->set_volumesize(100 * kGiB); - volume->set_blocksize(4 * kKiB); - volume->set_volumename("TestInitSpaceVolume"); - volume->set_user("test"); - fsInfo->set_mountnum(1); - fsInfo->set_fstype(::curvefs::common::FSType::TYPE_VOLUME); - fsInfo->set_enablesumindir(false); - - EXPECT_CALL(space_, InitSpace(_)) - .WillOnce(Return(SPACE_OK)); - - brpc::Controller cntl; - SpaceAllocService_Stub stub(&channel_); - stub.InitSpace(&cntl, &request, &response, nullptr); - - EXPECT_FALSE(cntl.Failed()); - EXPECT_EQ(SPACE_OK, response.status()); - } -} - -TEST_F(SpaceAllocServiceTest, TestUnInitSpace) { - // test failed - { - UnInitSpaceRequest request; - UnInitSpaceResponse response; - - request.set_fsid(1); - - EXPECT_CALL(space_, UnInitSpace(_)) - .WillOnce(Return(SPACE_NOT_FOUND)); - - brpc::Controller cntl; - SpaceAllocService_Stub stub(&channel_); - stub.UnInitSpace(&cntl, &request, &response, nullptr); - - EXPECT_FALSE(cntl.Failed()); - EXPECT_EQ(SPACE_NOT_FOUND, response.status()); - } - - // test succeeded - { - UnInitSpaceRequest request; - UnInitSpaceResponse response; - - request.set_fsid(1); - - EXPECT_CALL(space_, UnInitSpace(_)) - .WillOnce(Return(SPACE_OK)); - - brpc::Controller cntl; - SpaceAllocService_Stub stub(&channel_); - stub.UnInitSpace(&cntl, &request, &response, nullptr); - - EXPECT_FALSE(cntl.Failed()); - EXPECT_EQ(SPACE_OK, response.status()); - } -} - -TEST_F(SpaceAllocServiceTest, TestStatSpace) { - // test failure - { - StatSpaceRequest request; - StatSpaceResponse response; - - request.set_fsid(1); - - EXPECT_CALL(space_, StatSpace(_, _)) - .WillOnce(Return(SPACE_NOT_FOUND)); - - brpc::Controller cntl; - SpaceAllocService_Stub stub(&channel_); - stub.StatSpace(&cntl, &request, &response, nullptr); - - EXPECT_FALSE(cntl.Failed()); - EXPECT_EQ(SPACE_NOT_FOUND, response.status()); - } - - // test success - { - StatSpaceRequest request; - StatSpaceResponse response; - - request.set_fsid(1); - - SpaceStat stat(100 * kGiB, 50 * kGiB, 4 * kKiB); - EXPECT_CALL(space_, StatSpace(_, _)) - .WillOnce(DoAll(SetArgPointee<1>(stat), Return(SPACE_OK))); - - brpc::Controller cntl; - SpaceAllocService_Stub stub(&channel_); - stub.StatSpace(&cntl, &request, &response, nullptr); - - EXPECT_FALSE(cntl.Failed()); - EXPECT_EQ(SPACE_OK, response.status()); - EXPECT_EQ(100 * kGiB / (4 * kKiB), response.totalblock()); - EXPECT_EQ(50 * kGiB / (4 * kKiB), response.availableblock()); - EXPECT_EQ(50 * kGiB / (4 * kKiB), response.usedblock()); - } -} - -TEST_F(SpaceAllocServiceTest, TestAllocateSpace) { - // test failure - { - AllocateSpaceRequest request; - AllocateSpaceResponse response; - - request.set_fsid(1); - request.set_size(8 * kKiB); - - EXPECT_CALL(space_, AllocateSpace(_, _, _, _)) - .WillOnce(Return(SPACE_NO_SPACE)); - - brpc::Controller cntl; - SpaceAllocService_Stub stub(&channel_); - stub.AllocateSpace(&cntl, &request, &response, nullptr); - - EXPECT_FALSE(cntl.Failed()); - EXPECT_EQ(SPACE_NO_SPACE, response.status()); - } - - // test success - { - AllocateSpaceRequest request; - AllocateSpaceResponse response; - - request.set_fsid(1); - request.set_size(8 * kKiB); - - Extents exts{{12 * kKiB, 8 * kKiB}}; - - EXPECT_CALL(space_, AllocateSpace(_, _, _, _)) - .WillOnce(DoAll(SetArgPointee<3>(exts), Return(SPACE_OK))); - - brpc::Controller cntl; - SpaceAllocService_Stub stub(&channel_); - stub.AllocateSpace(&cntl, &request, &response, nullptr); - - EXPECT_FALSE(cntl.Failed()); - EXPECT_EQ(SPACE_OK, response.status()); - EXPECT_EQ(1, response.extents_size()); - EXPECT_TRUE(IsEqual(response.extents(), exts)); - } -} - -TEST_F(SpaceAllocServiceTest, TestAllocateSpaceWithHint) { - // no hint - { - AllocateSpaceRequest request; - AllocateSpaceResponse response; - - request.set_fsid(1); - request.set_size(8 * kKiB); - - Extents exts{{12 * kKiB, 8 * kKiB}}; - - SpaceAllocateHint hint; - EXPECT_CALL(space_, AllocateSpace(1, 8 * kKiB, hint, _)) - .WillOnce(DoAll(SetArgPointee<3>(exts), Return(SPACE_OK))); - - brpc::Controller cntl; - SpaceAllocService_Stub stub(&channel_); - stub.AllocateSpace(&cntl, &request, &response, nullptr); - - EXPECT_FALSE(cntl.Failed()); - EXPECT_EQ(SPACE_OK, response.status()); - EXPECT_EQ(1, response.extents_size()); - EXPECT_TRUE(IsEqual(response.extents(), exts)); - } - - // hint with allocate type - { - AllocateSpaceRequest request; - AllocateSpaceResponse response; - - request.set_fsid(1); - request.set_size(8 * kKiB); - auto* protoHint = request.mutable_allochint(); - protoHint->set_alloctype(AllocateType::BIG); - - Extents exts{{12 * kKiB, 8 * kKiB}}; - - SpaceAllocateHint hint; - hint.allocType = AllocateType::BIG; - EXPECT_CALL(space_, AllocateSpace(1, 8 * kKiB, hint, _)) - .WillOnce(DoAll(SetArgPointee<3>(exts), Return(SPACE_OK))); - - brpc::Controller cntl; - SpaceAllocService_Stub stub(&channel_); - stub.AllocateSpace(&cntl, &request, &response, nullptr); - - EXPECT_FALSE(cntl.Failed()); - EXPECT_EQ(SPACE_OK, response.status()); - EXPECT_EQ(1, response.extents_size()); - EXPECT_TRUE(IsEqual(response.extents(), exts)); - } - - // hint with allocate type and left/right offset - { - AllocateSpaceRequest request; - AllocateSpaceResponse response; - - request.set_fsid(1); - request.set_size(8 * kKiB); - auto* protoHint = request.mutable_allochint(); - protoHint->set_alloctype(AllocateType::BIG); - protoHint->set_leftoffset(0 * kKiB); - protoHint->set_rightoffset(8 * kKiB); - - Extents exts{{12 * kKiB, 8 * kKiB}}; - - SpaceAllocateHint hint; - hint.allocType = AllocateType::BIG; - hint.leftOffset = 0; - hint.rightOffset = 8 * kKiB; - EXPECT_CALL(space_, AllocateSpace(1, 8 * kKiB, hint, _)) - .WillOnce(DoAll(SetArgPointee<3>(exts), Return(SPACE_OK))); - - brpc::Controller cntl; - SpaceAllocService_Stub stub(&channel_); - stub.AllocateSpace(&cntl, &request, &response, nullptr); - - EXPECT_FALSE(cntl.Failed()); - EXPECT_EQ(SPACE_OK, response.status()); - EXPECT_EQ(1, response.extents_size()); - EXPECT_TRUE(IsEqual(response.extents(), exts)); - } -} - -TEST_F(SpaceAllocServiceTest, TestDeallocateSpace) { - // test failure - { - DeallocateSpaceRequest request; - DeallocateSpaceResponse response; - - request.set_fsid(1); - auto* e = request.add_extents(); - e->set_offset(12 * kKiB); - e->set_length(8 * kKiB); - - EXPECT_CALL(space_, DeallocateSpace(_, _)) - .WillOnce(Return(SPACE_NOT_FOUND)); - - brpc::Controller cntl; - SpaceAllocService_Stub stub(&channel_); - stub.DeallocateSpace(&cntl, &request, &response, nullptr); - - EXPECT_FALSE(cntl.Failed()); - EXPECT_EQ(SPACE_NOT_FOUND, response.status()); - } - - // test success - { - DeallocateSpaceRequest request; - DeallocateSpaceResponse response; - - request.set_fsid(1); - auto* e = request.add_extents(); - e->set_offset(12 * kKiB); - e->set_length(8 * kKiB); - - google::protobuf::RepeatedPtrField protoExts; - - EXPECT_CALL(space_, DeallocateSpace(_, _)) - .WillOnce(DoAll(SaveArg<1>(&protoExts), Return(SPACE_OK))); - - brpc::Controller cntl; - SpaceAllocService_Stub stub(&channel_); - stub.DeallocateSpace(&cntl, &request, &response, nullptr); - - EXPECT_FALSE(cntl.Failed()); - EXPECT_EQ(SPACE_OK, response.status()); - EXPECT_EQ(1, protoExts.size()); - EXPECT_EQ(12 * kKiB, protoExts[0].offset()); - EXPECT_EQ(8 * kKiB, protoExts[0].length()); - } -} - -} // namespace space -} // namespace curvefs - -int main(int argc, char* argv[]) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/curvefs/test/volume/BUILD b/curvefs/test/volume/BUILD new file mode 100644 index 0000000000..96d13bab9d --- /dev/null +++ b/curvefs/test/volume/BUILD @@ -0,0 +1,113 @@ +# +# Copyright (c) 2022 NetEase Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +load("//:copts.bzl", "CURVE_TEST_COPTS") + +LINKOPTS = [ + "-pthread", +] + +cc_test( + name = "volume_unittest", + srcs = glob([ + "*.cpp", + "*.h", + ]), + copts = CURVE_TEST_COPTS, + linkopts = LINKOPTS, + deps = [ + "//curvefs/src/volume", + "//curvefs/test/volume/mock", + "//test/client/mock:client_mock_lib", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/types:optional", + "@com_google_googletest//:gtest", + "@com_google_googletest//:gtest_main", + ], +) + +# cc_test( +# name = "extents_test", +# srcs = ["extents_test.cpp"], +# copts = CURVE_TEST_COPTS, +# linkopts = LINKOPTS, +# deps = [ +# "//curvefs/src/volume", +# "@com_google_googletest//:gtest", +# "@com_google_googletest//:gtest_main", +# ], +# ) + +# cc_test( +# name = "bitmap_allocator_test", +# srcs = [ +# "bitmap_allocator_test.cpp", +# "common.h", +# ], +# copts = CURVE_TEST_COPTS, +# linkopts = LINKOPTS, +# deps = [ +# "//curvefs/src/volume", +# "@com_google_googletest//:gtest", +# "@com_google_googletest//:gtest_main", +# ], +# ) + +# cc_test( +# name = "bitmap_allocator_brute_test", +# srcs = [ +# "bitmap_allocator_brute_test.cpp", +# "common.h", +# ], +# copts = CURVE_TEST_COPTS, +# linkopts = LINKOPTS, +# deps = [ +# "//curvefs/src/volume", +# "@com_google_googletest//:gtest", +# "@com_google_googletest//:gtest_main", +# ], +# ) + +# cc_test( +# name = "bitmap_allocator_multi_thread_brute_test", +# srcs = [ +# "bitmap_allocator_multi_thread_brute_test.cpp", +# "common.h", +# ], +# copts = CURVE_TEST_COPTS, +# linkopts = LINKOPTS, +# deps = [ +# "//curvefs/src/volume", +# "@com_google_googletest//:gtest", +# "@com_google_googletest//:gtest_main", +# ], +# ) + +# cc_test( +# name = "block_device_test", +# srcs = [ +# "block_device_client_test.cpp", +# ], +# copts = CURVE_TEST_COPTS, +# deps = [ +# "//curvefs/src/volume", +# "//curvefs/test/volume/mock", +# "//test/client/mock:client_mock_lib", +# "@com_google_googletest//:gtest", +# "@com_google_googletest//:gtest_main", +# ], +# ) diff --git a/curvefs/test/space/allocator_test.cpp b/curvefs/test/volume/allocator_test.cpp similarity index 84% rename from curvefs/test/space/allocator_test.cpp rename to curvefs/test/volume/allocator_test.cpp index b7e25a08af..8b83673f97 100644 --- a/curvefs/test/space/allocator_test.cpp +++ b/curvefs/test/volume/allocator_test.cpp @@ -14,16 +14,17 @@ * limitations under the License. */ -#include "curvefs/src/space/allocator.h" +#include "curvefs/src/volume/allocator.h" #include #include -#include "curvefs/src/space/bitmap_allocator.h" +#include "curvefs/src/volume/bitmap_allocator.h" +#include "curvefs/test/volume/common.h" namespace curvefs { -namespace space { +namespace volume { TEST(AllocatorTest, Common) { auto allocator = Allocator::Create("", {}); @@ -43,10 +44,5 @@ TEST(AllocatorTest, Common) { EXPECT_TRUE(p); } -} // namespace space +} // namespace volume } // namespace curvefs - -int main(int argc, char* argv[]) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/curvefs/test/space/bitmap_allocator_brute_test.cpp b/curvefs/test/volume/bitmap_allocator_brute_test.cpp similarity index 90% rename from curvefs/test/space/bitmap_allocator_brute_test.cpp rename to curvefs/test/volume/bitmap_allocator_brute_test.cpp index ae23259621..9d9a5dfb47 100644 --- a/curvefs/test/space/bitmap_allocator_brute_test.cpp +++ b/curvefs/test/volume/bitmap_allocator_brute_test.cpp @@ -21,20 +21,19 @@ #include #include -#include "curvefs/src/space/bitmap_allocator.h" -#include "curvefs/src/space/config.h" -#include "curvefs/test/space/common.h" +#include "curvefs/src/volume/bitmap_allocator.h" +#include "curvefs/test/volume/common.h" namespace curvefs { -namespace space { +namespace volume { -std::vector allocSizes = { +static const std::vector kAllocSizes = { 4 * kKiB, 16 * kKiB, 32 * kKiB, 128 * kKiB, 512 * kKiB, 1 * kMiB, 4 * kMiB, 32 * kMiB, 128 * kMiB, 512 * kMiB, 1 * kGiB, 4 * kGiB, 32 * kGiB, 128 * kGiB, 512 * kGiB}; -std::vector allocTypes = {AllocateType::NONE, AllocateType::SMALL, - AllocateType::BIG}; +static const std::vector kAllocTypes = { + AllocateType::None, AllocateType::Small, AllocateType::Big}; class BitmapAllocatorBruteTests : public ::testing::TestWithParam { @@ -56,9 +55,9 @@ TEST_P(BitmapAllocatorBruteTests, TestAllocAllSpace) { // alloc all available space while (allocator_->AvailableSize() > 0) { - SpaceAllocateHint hint; - auto allocSize = allocSizes[rand() % allocSizes.size()]; - hint.allocType = allocTypes[rand() % allocTypes.size()]; + AllocateHint hint; + auto allocSize = kAllocSizes[rand() % kAllocSizes.size()]; + hint.allocType = kAllocTypes[rand() % kAllocTypes.size()]; allocSize = std::min(allocSize, allocator_->AvailableSize()); ASSERT_EQ(allocSize, allocator_->Alloc(allocSize, hint, &exts)); @@ -146,11 +145,5 @@ INSTANTIATE_TEST_CASE_P( .sizePerBit = 4 * kMiB, .smallAllocProportion = 1})); -} // namespace space +} // namespace volume } // namespace curvefs - -int main(int argc, char* argv[]) { - ::testing::InitGoogleTest(&argc, argv); - srand(time(nullptr)); - return RUN_ALL_TESTS(); -} diff --git a/curvefs/test/space/bitmap_allocator_multi_thread_brute_test.cpp b/curvefs/test/volume/bitmap_allocator_multi_thread_brute_test.cpp similarity index 89% rename from curvefs/test/space/bitmap_allocator_multi_thread_brute_test.cpp rename to curvefs/test/volume/bitmap_allocator_multi_thread_brute_test.cpp index 6584b595e6..7a403ef7ca 100644 --- a/curvefs/test/space/bitmap_allocator_multi_thread_brute_test.cpp +++ b/curvefs/test/volume/bitmap_allocator_multi_thread_brute_test.cpp @@ -22,21 +22,21 @@ #include #include -#include "curvefs/src/space/bitmap_allocator.h" -#include "curvefs/test/space/common.h" +#include "curvefs/src/volume/bitmap_allocator.h" +#include "curvefs/test/volume/common.h" namespace curvefs { -namespace space { +namespace volume { -std::vector allocSizes = { +static const std::vector kAllocSizes = { 4 * kKiB, 16 * kKiB, 32 * kKiB, 128 * kKiB, 512 * kKiB, 1 * kMiB, 4 * kMiB, 32 * kMiB, 128 * kMiB, 512 * kMiB, 1 * kGiB, 4 * kGiB, 32 * kGiB, 128 * kGiB, 512 * kGiB}; -std::vector allocTypes = {AllocateType::NONE, AllocateType::SMALL, - AllocateType::BIG}; +static const std::vector kAllocTypes = { + AllocateType::None, AllocateType::Small, AllocateType::Big}; -class BitmapAllocatorBruteTests +class BitmapAllocatorMultiThreadTests : public ::testing::TestWithParam> { protected: void SetUp() override { @@ -52,15 +52,16 @@ class BitmapAllocatorBruteTests exts.resize(threadNum_); for (int i = 0; i < threadNum_; ++i) { - ths.emplace_back(&BitmapAllocatorBruteTests::ThreadTask, this, i); + ths.emplace_back(&BitmapAllocatorMultiThreadTests::ThreadTask, this, + i); } } void ThreadTask(int idx) { while (allocator_->AvailableSize() > 0) { - SpaceAllocateHint hint; - hint.allocType = allocTypes[rand() % allocTypes.size()]; - auto allocSize = allocSizes[rand() % allocSizes.size()]; + AllocateHint hint; + hint.allocType = kAllocTypes[rand() % kAllocTypes.size()]; + auto allocSize = kAllocSizes[rand() % kAllocSizes.size()]; auto allocated = allocator_->Alloc(allocSize, hint, &exts[idx]); if (allocated == 0) { @@ -81,7 +82,7 @@ class BitmapAllocatorBruteTests std::vector exts; }; -TEST_P(BitmapAllocatorBruteTests, TestMultiThreadAllocAllSpace) { +TEST_P(BitmapAllocatorMultiThreadTests, TestMultiThreadAllocAllSpace) { for (int i = 0; i < 3; ++i) { LOG(INFO) << "loop " << i << ", " << *allocator_; ASSERT_EQ(opt_.length, allocator_->AvailableSize()); @@ -108,8 +109,8 @@ TEST_P(BitmapAllocatorBruteTests, TestMultiThreadAllocAllSpace) { } INSTANTIATE_TEST_CASE_P( - BitmapAllocatorTests, - BitmapAllocatorBruteTests, + BitmapAllocatorMultiThreadTests, + BitmapAllocatorMultiThreadTests, ::testing::Values( std::make_pair(4, BitmapAllocatorOption{.startOffset = 0, .length = 10 * kGiB, @@ -179,11 +180,6 @@ INSTANTIATE_TEST_CASE_P( .sizePerBit = 16 * kMiB, .smallAllocProportion = 1}))); -} // namespace space +} // namespace volume } // namespace curvefs -int main(int argc, char* argv[]) { - ::testing::InitGoogleTest(&argc, argv); - srand(time(nullptr)); - return RUN_ALL_TESTS(); -} diff --git a/curvefs/test/space/bitmap_allocator_test.cpp b/curvefs/test/volume/bitmap_allocator_test.cpp similarity index 72% rename from curvefs/test/space/bitmap_allocator_test.cpp rename to curvefs/test/volume/bitmap_allocator_test.cpp index 8ed2c257b2..4bc90d72bb 100644 --- a/curvefs/test/space/bitmap_allocator_test.cpp +++ b/curvefs/test/volume/bitmap_allocator_test.cpp @@ -14,14 +14,18 @@ * limitations under the License. */ -#include "curvefs/src/space/bitmap_allocator.h" +#include "curvefs/src/volume/bitmap_allocator.h" #include -#include "curvefs/test/space/common.h" +#include "curvefs/test/volume/common.h" + +#include "absl/memory/memory.h" namespace curvefs { -namespace space { +namespace volume { + +static unsigned int seed = time(nullptr); class BitmapAllocatorTest : public ::testing::Test { protected: @@ -52,8 +56,8 @@ TEST_F(BitmapAllocatorTest, AllocFromSmallExtentTest) { { Extents exts; - SpaceAllocateHint hint; - hint.allocType = AllocateType::SMALL; + AllocateHint hint; + hint.allocType = AllocateType::Small; ASSERT_EQ(allocSize, allocator_->Alloc(allocSize, hint, &exts)); ASSERT_EQ(1, exts.size()); @@ -68,8 +72,8 @@ TEST_F(BitmapAllocatorTest, AllocFromSmallExtentTest) { { Extents exts; - SpaceAllocateHint hint; - hint.allocType = AllocateType::NONE; + AllocateHint hint; + hint.allocType = AllocateType::None; ASSERT_EQ(allocSize, allocator_->Alloc(allocSize, hint, &exts)); ASSERT_EQ(1, exts.size()); @@ -87,15 +91,15 @@ TEST_F(BitmapAllocatorTest, AllocFromBitmap) { // alloc size >= opt_.sizePerBit { Extents exts; - SpaceAllocateHint hint; - hint.allocType = AllocateType::SMALL; + AllocateHint hint; + hint.allocType = AllocateType::Small; uint64_t allocSize = opt_.sizePerBit; ASSERT_EQ(allocSize, allocator_->Alloc(allocSize, hint, &exts)); ASSERT_EQ(allocSize / opt_.sizePerBit, exts.size()); Extents expected = { - PExtent(opt_.startOffset + opt_.length * opt_.smallAllocProportion, + Extent(opt_.startOffset + opt_.length * opt_.smallAllocProportion, allocSize)}; ASSERT_EQ(expected, exts); @@ -108,8 +112,8 @@ TEST_F(BitmapAllocatorTest, AllocFromBitmap) { { allocator_.reset(new BitmapAllocator(opt_)); Extents exts; - SpaceAllocateHint hint; - hint.allocType = AllocateType::BIG; + AllocateHint hint; + hint.allocType = AllocateType::Big; uint64_t allocSize = opt_.length * (1 - opt_.smallAllocProportion) / 2; ASSERT_EQ(allocSize, allocator_->Alloc(allocSize, hint, &exts)); @@ -138,12 +142,12 @@ TEST_F(BitmapAllocatorTest, AllocSmallTest3) { allocator_.reset(new BitmapAllocator(opt_)); Extents exts; - SpaceAllocateHint hint; - hint.allocType = AllocateType::BIG; + AllocateHint hint; + hint.allocType = AllocateType::Big; auto allocSize = allocator_->Alloc(128 * kKiB, hint, &exts); ASSERT_EQ(allocSize, 128 * kKiB); - hint.allocType = AllocateType::SMALL; + hint.allocType = AllocateType::Small; hint.leftOffset = exts.back().len + exts.back().offset; Extents ext2; allocSize = allocator_->Alloc(128 * kKiB, hint, &ext2); @@ -194,7 +198,7 @@ TEST_F(BitmapAllocatorTest, Alloc4) { } TEST_F(BitmapAllocatorTest, TestMarkUsed) { - // [0, 2kGiB] for small allocate + // [0, 2kGiB] for Small allocate // [2kGiB, 8, kGiB] for bit allocate Extents used = {{1 * kGiB, 2 * kGiB}}; allocator_->MarkUsed(used); @@ -205,14 +209,65 @@ TEST_F(BitmapAllocatorTest, TestMarkUsed) { auto sorted = SortAndMerge(exts); EXPECT_EQ(2, sorted.size()); - EXPECT_EQ(PExtent(0, 1 * kGiB), sorted[0]); - EXPECT_EQ(PExtent(3 * kGiB, 7 * kGiB), sorted[1]); + EXPECT_EQ(Extent(0, 1 * kGiB), sorted[0]); + EXPECT_EQ(Extent(3 * kGiB, 7 * kGiB), sorted[1]); +} + +TEST_F(BitmapAllocatorTest, TestMarkUsedRandom) { + opt_.startOffset = 128 * kMiB; + opt_.length = 128 * kMiB; + opt_.smallAllocProportion = 0; + opt_.sizePerBit = 4 * kMiB; + + allocator_ = absl::make_unique(opt_); + + Extents used; + uint64_t off = opt_.startOffset; + uint64_t usedSize = 0; + + // 对于每一个 size per bit，随机其中一部分设置 + auto select = [this, &usedSize](uint64_t startOffset) { + auto off = rand_r(&seed) * 4096 % opt_.sizePerBit; + auto len = rand_r(&seed) * 4096 % opt_.sizePerBit; + + if (off + len > opt_.sizePerBit) { + len = opt_.sizePerBit - off; + } + + usedSize += len; + return Extent{off + startOffset, len}; + }; + + while (off < opt_.startOffset + opt_.length) { + used.push_back(select(off)); + off += opt_.sizePerBit; + } + + auto left = opt_.length - usedSize; + allocator_->MarkUsed(used); + EXPECT_EQ(left, allocator_->AvailableSize()); + + Extents exts; + EXPECT_EQ(left, allocator_->Alloc(left, {}, &exts)); + + auto sorted = SortAndMerge(exts); + Extents out; + std::merge(used.begin(), used.end(), sorted.begin(), sorted.end(), + std::back_inserter(out), [](const Extent& e1, const Extent& e2) { + return e1.offset < e2.offset || + (e1.offset == e2.offset && e1.len < e2.len); + }); + + sorted = SortAndMerge(out); + EXPECT_EQ(1, sorted.size()); + EXPECT_EQ(opt_.startOffset, sorted[0].offset); + EXPECT_EQ(opt_.length, sorted[0].len); } TEST_F(BitmapAllocatorTest, TestAllocSmallWithHint) { - for (auto t : {AllocateType::SMALL, AllocateType::BIG}) { + for (auto t : {AllocateType::Small, AllocateType::Big}) { Extents exts; - SpaceAllocateHint hint; + AllocateHint hint; hint.allocType = t; uint64_t size = 512 * kKiB; @@ -221,7 +276,7 @@ TEST_F(BitmapAllocatorTest, TestAllocSmallWithHint) { EXPECT_EQ(alloc, size); EXPECT_EQ(1, exts.size()); - if (t == AllocateType::SMALL) { + if (t == AllocateType::Small) { EXPECT_EQ(opt_.startOffset, exts[0].offset); } else { EXPECT_EQ( @@ -247,10 +302,5 @@ TEST_F(BitmapAllocatorTest, TestAllocSmallWithHint) { } } -} // namespace space +} // namespace volume } // namespace curvefs - -int main(int argc, char* argv[]) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/curvefs/test/client/block_device_client_test.cpp b/curvefs/test/volume/block_device_client_test.cpp similarity index 54% rename from curvefs/test/client/block_device_client_test.cpp rename to curvefs/test/volume/block_device_client_test.cpp index 871a428123..9c6d10998a 100644 --- a/curvefs/test/client/block_device_client_test.cpp +++ b/curvefs/test/volume/block_device_client_test.cpp @@ -23,16 +23,19 @@ #include #include #include +#include "absl/memory/memory.h" -#include "curvefs/src/client/block_device_client.h" +#include "curvefs/src/volume/block_device_client.h" #include "test/client/mock/mock_file_client.h" +#include "curvefs/test/volume/common.h" namespace curvefs { -namespace client { +namespace volume { using ::testing::_; using ::testing::Return; using ::testing::Invoke; +using ::testing::NiceMock; using ::curve::client::UserInfo; using ::curve::client::MockFileClient; using AlignRead = std::pair; @@ -42,14 +45,22 @@ class BlockDeviceClientTest : public ::testing::Test { protected: void SetUp() override { options_.configPath = "/etc/curvefs/client.conf"; + options_.threadnum = 10; fileClient_ = std::make_shared(); - client_ = BlockDeviceClientImpl(fileClient_); + client_ = absl::make_unique(fileClient_); + + ON_CALL(*fileClient_, Init(_)) + .WillByDefault(Return(LIBCURVE_ERROR::OK)); + ASSERT_TRUE(client_->Init(options_)); } void TearDown() override {} - static int ReadCallback(int fd, char* buf, off_t offset, size_t length) { + static ssize_t ReadCallback(int fd, + char* buf, + off_t offset, + size_t length) { for (auto i = 0; i < length; i++) { buf[i] = '1'; } @@ -58,7 +69,7 @@ class BlockDeviceClientTest : public ::testing::Test { protected: BlockDeviceClientOptions options_; - BlockDeviceClientImpl client_; + std::unique_ptr client_; std::shared_ptr fileClient_; }; @@ -66,56 +77,56 @@ TEST_F(BlockDeviceClientTest, TestInit) { // CASE 1: init success EXPECT_CALL(*fileClient_, Init(options_.configPath)) .WillOnce(Return(LIBCURVE_ERROR::OK)); - ASSERT_EQ(client_.Init(options_), CURVEFS_ERROR::OK); + ASSERT_TRUE(client_->Init(options_)); // CASE 2: init failed EXPECT_CALL(*fileClient_, Init(options_.configPath)) .WillOnce(Return(LIBCURVE_ERROR::FAILED)); - ASSERT_EQ(client_.Init(options_), CURVEFS_ERROR::INTERNAL); + ASSERT_FALSE(client_->Init(options_)); } TEST_F(BlockDeviceClientTest, TestUnInit) { EXPECT_CALL(*fileClient_, UnInit()) - .WillOnce(Return()); - client_.UnInit(); + .Times(1); + client_->UnInit(); } TEST_F(BlockDeviceClientTest, TestOpen) { UserInfo userInfo("owner"); // CASE 1: open return fd (-1) - EXPECT_CALL(*fileClient_, Open("/filename", userInfo, _)) + EXPECT_CALL(*fileClient_, Open(_, _, _)) .WillOnce(Return(-1)); - ASSERT_EQ(client_.Open("/filename", "owner"), CURVEFS_ERROR::INTERNAL); + ASSERT_FALSE(client_->Open("/filename", "owner")); // CASE 2: open return fd (0) - EXPECT_CALL(*fileClient_, Open("/filename", userInfo, _)) + EXPECT_CALL(*fileClient_, Open(_, _, _)) .WillOnce(Return(0)); - ASSERT_EQ(client_.Open("/filename", "owner"), CURVEFS_ERROR::OK); + ASSERT_TRUE(client_->Open("/filename", "owner")); // CASE 3: open return fd (1) - EXPECT_CALL(*fileClient_, Open("/filename", userInfo, _)) + EXPECT_CALL(*fileClient_, Open(_, _, _)) .WillOnce(Return(10)); - ASSERT_EQ(client_.Open("/filename", "owner"), CURVEFS_ERROR::OK); + ASSERT_TRUE(client_->Open("/filename", "owner")); } TEST_F(BlockDeviceClientTest, TestClose) { // CASE 1: close failed with file not open - ASSERT_EQ(client_.Close(), CURVEFS_ERROR::OK); + ASSERT_TRUE(client_->Close()); // CASE 2: close failed - EXPECT_CALL(*fileClient_, Open("/filename", _, _)) + EXPECT_CALL(*fileClient_, Open(_, _, _)) .WillOnce(Return(10)); - ASSERT_EQ(client_.Open("/filename", "owner"), CURVEFS_ERROR::OK); + ASSERT_TRUE(client_->Open("/filename", "owner")); EXPECT_CALL(*fileClient_, Close(10)) .WillOnce(Return(-LIBCURVE_ERROR::FAILED)); - ASSERT_EQ(client_.Close(), CURVEFS_ERROR::INTERNAL); + ASSERT_FALSE(client_->Close()); // CASE 3: close success EXPECT_CALL(*fileClient_, Close(10)) .WillOnce(Return(LIBCURVE_ERROR::OK)); - ASSERT_EQ(client_.Close(), CURVEFS_ERROR::OK); + ASSERT_TRUE(client_->Close()); } TEST_F(BlockDeviceClientTest, TestStat) { @@ -125,8 +136,7 @@ TEST_F(BlockDeviceClientTest, TestStat) { // CASE 1: stat failed EXPECT_CALL(*fileClient_, StatFile("/filename", userInfo, _)) .WillOnce(Return(-LIBCURVE_ERROR::FAILED)); - ASSERT_EQ(client_.Stat("/filename", "owner", &stat), - CURVEFS_ERROR::INTERNAL); + ASSERT_FALSE(client_->Stat("/filename", "owner", &stat)); // CASE 2: stat success EXPECT_CALL(*fileClient_, StatFile("/filename", userInfo, _)) @@ -137,7 +147,7 @@ TEST_F(BlockDeviceClientTest, TestStat) { finfo->fileStatus = 1; return LIBCURVE_ERROR::OK; })); - ASSERT_EQ(client_.Stat("/filename", "owner", &stat), CURVEFS_ERROR::OK); + ASSERT_TRUE(client_->Stat("/filename", "owner", &stat)); ASSERT_EQ(stat.length, 1000); ASSERT_EQ(stat.status, BlockDeviceStatus::DELETING); } @@ -146,31 +156,31 @@ TEST_F(BlockDeviceClientTest, TestReadBasic) { char buf[4096]; // CASE 1: read failed with file not open - ASSERT_EQ(client_.Read(buf, 0, 4096), CURVEFS_ERROR::BAD_FD); + ASSERT_LT(client_->Read(buf, 0, 4096), 0); // CASE 2: read failed - EXPECT_CALL(*fileClient_, Open("/filename", _, _)) + EXPECT_CALL(*fileClient_, Open(_, _, _)) .WillOnce(Return(10)); - ASSERT_EQ(client_.Open("/filename", "owner"), CURVEFS_ERROR::OK); + ASSERT_TRUE(client_->Open("/filename", "owner")); EXPECT_CALL(*fileClient_, Read(10, buf, 0, 4096)) .WillOnce(Return(-1)); - ASSERT_EQ(client_.Read(buf, 0, 4096), CURVEFS_ERROR::INTERNAL); + ASSERT_LT(client_->Read(buf, 0, 4096), 0); // CASE 3: read failed with read not complete EXPECT_CALL(*fileClient_, Read(10, buf, 0, 4096)) .WillOnce(Return(4095)); - ASSERT_EQ(client_.Read(buf, 0, 4096), CURVEFS_ERROR::INTERNAL); + ASSERT_LT(client_->Read(buf, 0, 4096), 0); // CASE 4: read success with length is zero EXPECT_CALL(*fileClient_, Read(_, _, _, _)) .Times(0); - ASSERT_EQ(client_.Read(buf, 0, 0), CURVEFS_ERROR::OK); + ASSERT_EQ(client_->Read(buf, 0, 0), 0); // CASE 5: read success with aligned offset and length EXPECT_CALL(*fileClient_, Read(10, buf, 0, 4096)) .WillOnce(Return(4096)); - ASSERT_EQ(client_.Read(buf, 0, 4096), CURVEFS_ERROR::OK); + ASSERT_EQ(client_->Read(buf, 0, 4096), 4096); } TEST_F(BlockDeviceClientTest, TestReadWithUnAligned) { @@ -182,16 +192,16 @@ TEST_F(BlockDeviceClientTest, TestReadWithUnAligned) { EXPECT_CALL(*fileClient_, Read(10, _, alignOffset, alignLength)) .WillOnce(Invoke(ReadCallback)); - ASSERT_EQ(client_.Read(buf, offset, length), CURVEFS_ERROR::OK); + ASSERT_GT(client_->Read(buf, offset, length), 0); for (auto i = 0; i < 40960; i++) { ASSERT_EQ(buf[i], i < length ? '1' : '0'); } }; // Prepare: open file - EXPECT_CALL(*fileClient_, Open("/filename", _, _)) + EXPECT_CALL(*fileClient_, Open(_, _, _)) .WillOnce(Return(10)); - ASSERT_EQ(client_.Open("/filename", "owner"), CURVEFS_ERROR::OK); + ASSERT_TRUE(client_->Open("/filename", "owner")); // Test Cases: read success { @@ -210,7 +220,7 @@ TEST_F(BlockDeviceClientTest, TestReadWithUnAligned) { EXPECT_CALL(*fileClient_, Read(10, _, 0, 4096)) .WillOnce(Return(0)); - ASSERT_EQ(client_.Read(buf, 0, 1), CURVEFS_ERROR::INTERNAL); + ASSERT_LT(client_->Read(buf, 0, 1), 0); for (auto i = 0; i < 4096; i++) { ASSERT_EQ(buf[i], '0'); } @@ -221,31 +231,31 @@ TEST_F(BlockDeviceClientTest, TestWriteBasic) { char buf[4096]; // CASE 1: write failed with file not open - ASSERT_EQ(client_.Write(buf, 0, 4096), CURVEFS_ERROR::BAD_FD); + ASSERT_LT(client_->Write(buf, 0, 4096), 0); // CASE 2: write failed - EXPECT_CALL(*fileClient_, Open("/filename", _, _)) + EXPECT_CALL(*fileClient_, Open(_, _, _)) .WillOnce(Return(10)); - ASSERT_EQ(client_.Open("/filename", "owner"), CURVEFS_ERROR::OK); + ASSERT_TRUE(client_->Open("/filename", "owner")); EXPECT_CALL(*fileClient_, Write(10, buf, 0, 4096)) .WillOnce(Return(-1)); - ASSERT_EQ(client_.Write(buf, 0, 4096), CURVEFS_ERROR::INTERNAL); + ASSERT_LT(client_->Write(buf, 0, 4096), 0); // CASE 3: write failed with write not complete EXPECT_CALL(*fileClient_, Write(10, buf, 0, 4096)) .WillOnce(Return(4095)); - ASSERT_EQ(client_.Write(buf, 0, 4096), CURVEFS_ERROR::INTERNAL); + ASSERT_LT(client_->Write(buf, 0, 4096), 0); // CASE 4: write success with length is zero EXPECT_CALL(*fileClient_, Write(10, buf, 0, 4096)) .Times(0); - ASSERT_EQ(client_.Write(buf, 0, 0), CURVEFS_ERROR::OK); + ASSERT_EQ(client_->Write(buf, 0, 0), 0); // CASE 5: write success with aligned offset and length EXPECT_CALL(*fileClient_, Write(10, buf, 0, 4096)) .WillOnce(Return(4096)); - ASSERT_EQ(client_.Write(buf, 0, 4096), CURVEFS_ERROR::OK); + ASSERT_EQ(client_->Write(buf, 0, 4096), 4096); } TEST_F(BlockDeviceClientTest, TestWriteWithUnAligned) { @@ -276,7 +286,7 @@ TEST_F(BlockDeviceClientTest, TestWriteWithUnAligned) { return alignLength; })); - ASSERT_EQ(client_.Write(buf, offset, length), CURVEFS_ERROR::OK); + ASSERT_GT(client_->Write(buf, offset, length), 0); // Check write buffer auto count = 0; @@ -294,9 +304,9 @@ TEST_F(BlockDeviceClientTest, TestWriteWithUnAligned) { }; // Prepare: open file - EXPECT_CALL(*fileClient_, Open("/filename", _, _)) + EXPECT_CALL(*fileClient_, Open(_, _, _)) .WillOnce(Return(10)); - ASSERT_EQ(client_.Open("/filename", "owner"), CURVEFS_ERROR::OK); + ASSERT_TRUE(client_->Open("/filename", "owner")); // Test Cases: write success { @@ -319,14 +329,14 @@ TEST_F(BlockDeviceClientTest, TestWriteWithUnAligned) { .WillOnce(Return(-1)); EXPECT_CALL(*fileClient_, Write(_, _, _, _)) .Times(0); - ASSERT_EQ(client_.Write(buf, 0, 1), CURVEFS_ERROR::INTERNAL); + ASSERT_LT(client_->Write(buf, 0, 1), 0); // CASE 2: read unexpected bytes -> write failed EXPECT_CALL(*fileClient_, Read(10, _, 0, 8192)) .WillOnce(Return(8191)); EXPECT_CALL(*fileClient_, Write(_, _, _, _)) .Times(0); - ASSERT_EQ(client_.Write(buf, 1000, 5000), CURVEFS_ERROR::INTERNAL); + ASSERT_LT(client_->Write(buf, 1000, 5000), 0); // CASE 3: read failed once -> write failed EXPECT_CALL(*fileClient_, Read(10, _, 8192, 4096)) @@ -335,23 +345,198 @@ TEST_F(BlockDeviceClientTest, TestWriteWithUnAligned) { .WillOnce(Return(4095)); EXPECT_CALL(*fileClient_, Write(_, _, _, _)) .Times(0); - ASSERT_EQ(client_.Write(buf, 10000, 10000), CURVEFS_ERROR::INTERNAL); + ASSERT_LT(client_->Write(buf, 10000, 10000), 0); // CASE 4: write failed EXPECT_CALL(*fileClient_, Read(10, _, 0, 4096)) .WillOnce(Return(4096)); EXPECT_CALL(*fileClient_, Write(_, _, _, _)) .WillOnce(Return(-1)); - ASSERT_EQ(client_.Write(buf, 0, 1), CURVEFS_ERROR::INTERNAL); + ASSERT_LT(client_->Write(buf, 0, 1), 0); } } -} // namespace client -} // namespace curvefs +TEST_F(BlockDeviceClientTest, ReadvTest_AllSuccess) { + ON_CALL(*fileClient_, Open(_, _, _)) + .WillByDefault(Return(1)); + + char data[4 * kKiB]; + + std::vector iov{ + { 0 * kMiB, 4 * kKiB, data}, + { 4 * kMiB, 4 * kKiB, data}, + { 8 * kMiB, 4 * kKiB, data}, + {12 * kMiB, 4 * kKiB, data}, + }; + + EXPECT_CALL(*fileClient_, Read(_, _, _, _)) + .Times(4) + .WillRepeatedly( + Invoke([](int, char*, off_t, size_t length) { return length; })); + + ASSERT_TRUE(client_->Open({}, {})); + ASSERT_EQ(4 * (4 * kKiB), client_->Readv(iov)); +} + +TEST_F(BlockDeviceClientTest, ReadvTest_AllFailed) { + ON_CALL(*fileClient_, Open(_, _, _)) + .WillByDefault(Return(1)); + + char data[4 * kKiB]; + + std::vector iov{ + { 0 * kMiB, 4 * kKiB, data}, + { 4 * kMiB, 4 * kKiB, data}, + { 8 * kMiB, 4 * kKiB, data}, + {12 * kMiB, 4 * kKiB, data}, + }; + + EXPECT_CALL(*fileClient_, Read(_, _, _, _)) + .Times(4) + .WillRepeatedly( + Invoke([](int, char*, off_t, size_t length) { return -1; })); + + ASSERT_TRUE(client_->Open({}, {})); + ASSERT_GT(0, client_->Readv(iov)); +} + +TEST_F(BlockDeviceClientTest, ReadvTest_PartialFailed) { + ON_CALL(*fileClient_, Open(_, _, _)) + .WillByDefault(Return(1)); + + char data[4 * kKiB]; + + std::vector iov{ + { 0 * kMiB, 4 * kKiB, data}, + { 4 * kMiB, 4 * kKiB, data}, + { 8 * kMiB, 4 * kKiB, data}, + {12 * kMiB, 4 * kKiB, data}, + }; + + unsigned int seed = time(nullptr); + int count = rand_r(&seed) % (iov.size() - 1) + 1; + std::atomic counter(iov.size()); + + EXPECT_CALL(*fileClient_, Read(_, _, _, _)) + .Times(4) + .WillRepeatedly( + Invoke([&count, &counter](int, char*, off_t, size_t length) -> int { + auto c = counter.fetch_sub(1); + if (c <= count) { + return -1; + } + + return length; + })); + + ASSERT_TRUE(client_->Open({}, {})); + ASSERT_GT(0, client_->Readv(iov)); +} + + +TEST_F(BlockDeviceClientTest, WritevTest_AllSuccess) { + ON_CALL(*fileClient_, Open(_, _, _)) + .WillByDefault(Return(1)); + + char data[4 * kKiB]; -int main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - google::ParseCommandLineFlags(&argc, &argv, false); + std::vector iov{ + { 0 * kMiB, 4 * kKiB, data}, + { 4 * kMiB, 4 * kKiB, data}, + { 8 * kMiB, 4 * kKiB, data}, + {12 * kMiB, 4 * kKiB, data}, + }; + + EXPECT_CALL(*fileClient_, Write(_, _, _, _)) + .Times(4) + .WillRepeatedly(Invoke( + [](int, const char*, off_t, size_t length) { return length; })); + + ASSERT_TRUE(client_->Open({}, {})); + ASSERT_EQ(4 * (4 * kKiB), client_->Writev(iov)); +} + +TEST_F(BlockDeviceClientTest, WritevTest_AllFailed) { + ON_CALL(*fileClient_, Open(_, _, _)) + .WillByDefault(Return(1)); + + char data[4 * kKiB]; + + std::vector iov{ + { 0 * kMiB, 4 * kKiB, data}, + { 4 * kMiB, 4 * kKiB, data}, + { 8 * kMiB, 4 * kKiB, data}, + {12 * kMiB, 4 * kKiB, data}, + }; + + EXPECT_CALL(*fileClient_, Write(_, _, _, _)) + .Times(4) + .WillRepeatedly( + Invoke([](int, const char*, off_t, size_t length) { return -1; })); + + ASSERT_TRUE(client_->Open({}, {})); + ASSERT_GT(0, client_->Writev(iov)); +} + +TEST_F(BlockDeviceClientTest, WritevTest_PartialFailed) { + ON_CALL(*fileClient_, Open(_, _, _)) + .WillByDefault(Return(1)); + + char data[4 * kKiB]; + + std::vector iov{ + { 0 * kMiB, 4 * kKiB, data}, + { 4 * kMiB, 4 * kKiB, data}, + { 8 * kMiB, 4 * kKiB, data}, + {12 * kMiB, 4 * kKiB, data}, + }; - return RUN_ALL_TESTS(); + unsigned int seed = time(nullptr); + int count = rand_r(&seed) % (iov.size() - 1) + 1; + std::atomic counter(iov.size()); + + EXPECT_CALL(*fileClient_, Write(_, _, _, _)) + .Times(4) + .WillRepeatedly(Invoke( + [&count, &counter](int, const char*, off_t, size_t length) -> int { + auto c = counter.fetch_sub(1); + if (c <= count) { + return -1; + } + + return length; + })); + + ASSERT_TRUE(client_->Open({}, {})); + ASSERT_GT(0, client_->Writev(iov)); } + +TEST_F(BlockDeviceClientTest, WritevTest_AllUnAlignedSuccess) { + ON_CALL(*fileClient_, Open(_, _, _)) + .WillByDefault(Return(1)); + + char data[4 * kKiB]; + + std::vector iov{ + { 0 * kMiB, 2 * kKiB, data}, + { 4 * kMiB, 2 * kKiB, data}, + { 8 * kMiB, 2 * kKiB, data}, + {12 * kMiB, 2 * kKiB, data}, + }; + + EXPECT_CALL(*fileClient_, Read(_, _, _, _)) + .Times(4) + .WillRepeatedly(Invoke( + [](int, const char*, off_t, size_t length) { return length; })); + + EXPECT_CALL(*fileClient_, Write(_, _, _, _)) + .Times(4) + .WillRepeatedly(Invoke( + [](int, const char*, off_t, size_t length) { return length; })); + + ASSERT_TRUE(client_->Open({}, {})); + ASSERT_EQ(4 * (2 * kKiB), client_->Writev(iov)); +} + +} // namespace volume +} // namespace curvefs diff --git a/curvefs/test/volume/block_group_loader_test.cpp b/curvefs/test/volume/block_group_loader_test.cpp new file mode 100644 index 0000000000..5052f21fe8 --- /dev/null +++ b/curvefs/test/volume/block_group_loader_test.cpp @@ -0,0 +1,97 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Monday Mar 07 10:14:21 CST 2022 + * Author: wuhanqing + */ + +#include "curvefs/src/volume/block_group_loader.h" + +#include +#include + +#include "absl/memory/memory.h" +#include "curvefs/test/volume/common.h" +#include "curvefs/test/volume/mock/mock_block_device_client.h" + +namespace curvefs { +namespace volume { + +using ::testing::_; +using ::testing::Return; +using ::testing::Invoke; + +// TODO(wuhanqing): random choose values +static constexpr uint32_t kBlockSize = 4 * kKiB; +static constexpr uint64_t kBlockGroupSize = 128 * kMiB; // 128 MiB +static constexpr uint64_t kBlockGroupOffset = 10 * kGiB; // 10 GiB +static constexpr BitmapLocation kBitmapLocation = BitmapLocation::AtStart; + +class BlockGroupBitmapLoaderTest : public ::testing::Test { + protected: + void SetUp() override { + option_.type = "bitmap"; + option_.bitmapAllocatorOption.sizePerBit = 4 * kMiB; + option_.bitmapAllocatorOption.smallAllocProportion = 0; + + loader_ = absl::make_unique( + &blockDev_, kBlockSize, kBlockGroupOffset, kBlockGroupSize, + kBitmapLocation, option_); + } + + protected: + MockBlockDeviceClient blockDev_; + AllocatorOption option_; + std::unique_ptr loader_; +}; + +TEST_F(BlockGroupBitmapLoaderTest, LoadTest_ReadBitmapError) { + EXPECT_CALL(blockDev_, Read(_, _, _)) + .WillOnce(Return(false)); + + AllocatorAndBitmapUpdater dummy; + ASSERT_FALSE(loader_->Load(&dummy)); +} + +TEST_F(BlockGroupBitmapLoaderTest, LoadTest_CreataAllocatorError) { + option_.type = "none"; + + EXPECT_CALL(blockDev_, Read(_, _, _)) + .WillOnce(Return(true)); + + AllocatorAndBitmapUpdater dummy; + ASSERT_FALSE(loader_->Load(&dummy)); +} + +TEST_F(BlockGroupBitmapLoaderTest, LoadTest_Success) { + EXPECT_CALL(blockDev_, Read(_, _, _)) + .WillOnce(Invoke([](char* buf, off_t /*offset*/, size_t length) { + memset(buf, 0, length); + return true; + })); + + AllocatorAndBitmapUpdater dummy; + ASSERT_TRUE(loader_->Load(&dummy)); + + ASSERT_EQ(kBlockGroupSize - kBlockGroupSize / kBlockSize / 8, + dummy.allocator->AvailableSize()); + ASSERT_EQ(kBlockGroupSize, dummy.allocator->Total()); +} + +} // namespace volume +} // namespace curvefs diff --git a/curvefs/test/volume/block_group_updater_test.cpp b/curvefs/test/volume/block_group_updater_test.cpp new file mode 100644 index 0000000000..e28ec9e991 --- /dev/null +++ b/curvefs/test/volume/block_group_updater_test.cpp @@ -0,0 +1,106 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Saturday Mar 05 23:36:02 CST 2022 + * Author: wuhanqing + */ + +#include "curvefs/src/volume/block_group_updater.h" + +#include +#include + +#include "absl/memory/memory.h" +#include "curvefs/test/volume/mock/mock_block_device_client.h" + +namespace curvefs { +namespace volume { + +using ::testing::_; +using ::testing::Return; + +static constexpr uint32_t kBlockSize = 4 * kKiB; +static constexpr uint64_t kBlockGroupSize = 128 * kMiB; // 128 MiB +static constexpr uint64_t kBlockGroupOffset = 10 * kGiB; // 10 GiB + +class BlockGroupBitmapUpdaterTest : public ::testing::Test { + protected: + void SetUp() override { + mockBlockDev_ = absl::make_unique(); + + Bitmap bitmap(kBlockGroupSize / kBlockSize); + bitmap.Clear(); + + BitmapRange range{kBlockGroupOffset, kBlockGroupSize / kBlockSize}; + + updater_ = absl::make_unique( + std::move(bitmap), kBlockSize, kBlockGroupSize, kBlockGroupOffset, + range, mockBlockDev_.get()); + } + + protected: + std::unique_ptr mockBlockDev_; + std::unique_ptr updater_; +}; + +TEST_F(BlockGroupBitmapUpdaterTest, SyncTest_NoDirty) { + EXPECT_CALL(*mockBlockDev_, Write(_, _, _)). + Times(0); + + ASSERT_TRUE(updater_->Sync()); +} + +TEST_F(BlockGroupBitmapUpdaterTest, SyncTest_Dirty) { + auto start = kBlockGroupOffset; + auto end = kBlockGroupOffset + kBlockGroupSize; + updater_->Update({start, end}, BlockGroupBitmapUpdater::Set); + + EXPECT_CALL(*mockBlockDev_, Write(_, _, _)) + .WillOnce(Return(true)); + + ASSERT_TRUE(updater_->Sync()); +} + +TEST_F(BlockGroupBitmapUpdaterTest, SyncTest_OnlySyncOnce) { + auto start1 = kBlockGroupOffset; + auto end1 = kBlockGroupOffset + kBlockGroupSize; + updater_->Update({start1, end1}, BlockGroupBitmapUpdater::Set); + + auto start2 = kBlockGroupOffset; + auto end2 = kBlockGroupOffset + kBlockGroupSize; + updater_->Update({start2, end2}, BlockGroupBitmapUpdater::Clear); + + EXPECT_CALL(*mockBlockDev_, Write(_, _, _)) + .WillOnce(Return(true)); + + ASSERT_TRUE(updater_->Sync()); +} + +TEST_F(BlockGroupBitmapUpdaterTest, SyncTest_WriteFailed) { + auto start = kBlockGroupOffset; + auto end = kBlockGroupOffset + kBlockGroupSize; + updater_->Update({start, end}, BlockGroupBitmapUpdater::Set); + + EXPECT_CALL(*mockBlockDev_, Write(_, _, _)) + .WillOnce(Return(false)); + + ASSERT_FALSE(updater_->Sync()); +} + +} // namespace volume +} // namespace curvefs diff --git a/curvefs/test/space/common.h b/curvefs/test/volume/common.h similarity index 67% rename from curvefs/test/space/common.h rename to curvefs/test/volume/common.h index e5abd20cc6..9bae01ec6a 100644 --- a/curvefs/test/space/common.h +++ b/curvefs/test/volume/common.h @@ -14,8 +14,8 @@ * limitations under the License. */ -#ifndef CURVEFS_TEST_SPACE_COMMON_H_ -#define CURVEFS_TEST_SPACE_COMMON_H_ +#ifndef CURVEFS_TEST_VOLUME_COMMON_H_ +#define CURVEFS_TEST_VOLUME_COMMON_H_ #include @@ -23,20 +23,22 @@ #include #include -#include "curvefs/src/space/free_extents.h" +#include "curvefs/src/volume/free_extents.h" namespace curvefs { -namespace space { +namespace volume { + +using Extents = std::vector; inline uint64_t TotalLength(const Extents& exts) { return std::accumulate( exts.begin(), exts.end(), 0ull, - [](uint64_t v, const PExtent& e) { return v + e.len; }); + [](uint64_t v, const Extent& e) { return v + e.len; }); } inline bool ExtentsNotOverlap(Extents exts) { std::sort(exts.begin(), exts.end(), - [](const PExtent& e1, const PExtent& e2) { + [](const Extent& e1, const Extent& e2) { return e1.offset < e2.offset; }); @@ -51,7 +53,7 @@ inline bool ExtentsNotOverlap(Extents exts) { inline bool ExtentsContinuous(Extents exts) { std::sort(exts.begin(), exts.end(), - [](const PExtent& e1, const PExtent& e2) { + [](const Extent& e1, const Extent& e2) { return e1.offset < e2.offset; }); @@ -72,11 +74,11 @@ inline Extents SortAndMerge(Extents exts) { Extents tmp; std::sort(exts.begin(), exts.end(), - [](const PExtent& e1, const PExtent& e2) { + [](const Extent& e1, const Extent& e2) { return e1.offset < e2.offset; }); - PExtent current = exts[0]; + Extent current = exts[0]; for (size_t i = 1; i < exts.size(); ++i) { if (current.offset + current.len == exts[i].offset) { current.len += exts[i].len; @@ -91,30 +93,13 @@ inline Extents SortAndMerge(Extents exts) { return tmp; } -inline bool IsEqual(const ::google::protobuf::RepeatedPtrField< - ::curvefs::space::Extent>& protoExts, - const Extents& phyExtents) { - if (static_cast(protoExts.size()) != phyExtents.size()) { - return false; - } - - for (int i = 0; i < protoExts.size(); ++i) { - if (protoExts[i].offset() != phyExtents[i].offset || - protoExts[i].length() != phyExtents[i].len) { - return false; - } - } - - return true; -} - -inline bool operator==(const SpaceAllocateHint& h1, - const SpaceAllocateHint& h2) { +inline bool operator==(const AllocateHint& h1, + const AllocateHint& h2) { return h1.allocType == h2.allocType && h1.leftOffset == h2.leftOffset && h1.rightOffset && h2.rightOffset; } -} // namespace space +} // namespace volume } // namespace curvefs -#endif // CURVEFS_TEST_SPACE_COMMON_H_ +#endif // CURVEFS_TEST_VOLUME_COMMON_H_ diff --git a/curvefs/test/space/extents_test.cpp b/curvefs/test/volume/extents_test.cpp similarity index 92% rename from curvefs/test/space/extents_test.cpp rename to curvefs/test/volume/extents_test.cpp index 0199ab03cc..39895b913a 100644 --- a/curvefs/test/space/extents_test.cpp +++ b/curvefs/test/volume/extents_test.cpp @@ -16,15 +16,22 @@ #include -#include "curvefs/src/space/common.h" -#include "curvefs/src/space/free_extents.h" +#include +#include + +#include "curvefs/src/volume/common.h" +#include "curvefs/src/volume/free_extents.h" +#include "curvefs/test/volume/common.h" namespace curvefs { -namespace space { +namespace volume { + +using Extents = std::vector; +using ExtentMap = std::map; TEST(ExtentTest, TestAsSmallAllocator) { FreeExtents freeExt(0, 16 * kMiB); - SpaceAllocateHint hint; + AllocateHint hint; ASSERT_EQ(16 * kMiB, freeExt.AvailableSize()); ASSERT_EQ(1, freeExt.AvailableExtents().size()); @@ -130,7 +137,7 @@ TEST(ExtentTest, TestAsSmallAllocatorWithHint) { // left offset hint { FreeExtents freeExt(16 * kMiB, 16 * kMiB); - SpaceAllocateHint hint; + AllocateHint hint; Extents ext1; ASSERT_EQ(4 * kMiB, freeExt.Alloc(4 * kMiB, hint, &ext1)); @@ -145,7 +152,7 @@ TEST(ExtentTest, TestAsSmallAllocatorWithHint) { ext1.clear(); hint.leftOffset = 20 * kMiB; ASSERT_EQ(4 * kMiB, freeExt.Alloc(4 * kMiB, hint, &ext1)); - ASSERT_EQ(ext1[0], PExtent(hint.leftOffset, 4 * kMiB)); + ASSERT_EQ(ext1[0], Extent(hint.leftOffset, 4 * kMiB)); ASSERT_EQ(4 * kMiB, freeExt.AvailableSize()); } @@ -153,7 +160,7 @@ TEST(ExtentTest, TestAsSmallAllocatorWithHint) { // right offset hint { FreeExtents freeExt(16 * kMiB, 16 * kMiB); - SpaceAllocateHint hint; + AllocateHint hint; Extents ext1; ASSERT_EQ(4 * kMiB, freeExt.Alloc(4 * kMiB, hint, &ext1)); @@ -165,7 +172,7 @@ TEST(ExtentTest, TestAsSmallAllocatorWithHint) { ext1.clear(); hint.rightOffset = 24 * kMiB; ASSERT_EQ(4 * kMiB, freeExt.Alloc(4 * kMiB, hint, &ext1)); - ASSERT_EQ(ext1[0], PExtent(hint.rightOffset - 4 * kMiB, 4 * kMiB)); + ASSERT_EQ(ext1[0], Extent(hint.rightOffset - 4 * kMiB, 4 * kMiB)); ASSERT_EQ(4 * kMiB, freeExt.AvailableSize()); } @@ -173,7 +180,7 @@ TEST(ExtentTest, TestAsSmallAllocatorWithHint) { // right offset hint 2 { FreeExtents freeExt(16 * kMiB, 16 * kMiB); - SpaceAllocateHint hint; + AllocateHint hint; Extents ext1; ASSERT_EQ(16 * kMiB, freeExt.Alloc(16 * kMiB, hint, &ext1)); @@ -196,7 +203,7 @@ TEST(ExtentTest, TestAsSmallAllocatorWithHint) { // right offset hint 3 { FreeExtents freeExt(0, 16 * kMiB); - SpaceAllocateHint hint; + AllocateHint hint; Extents ext1; ASSERT_EQ(16 * kMiB, freeExt.Alloc(16 * kMiB, hint, &ext1)); @@ -219,7 +226,7 @@ TEST(ExtentTest, TestAsSmallAllocatorWithHint) { // found a extent that satisfy left offset, but its smaller { FreeExtents freeExt(16 * kMiB, 16 * kMiB); - SpaceAllocateHint hint; + AllocateHint hint; Extents ext1; ASSERT_EQ(4 * kMiB, freeExt.Alloc(4 * kMiB, hint, &ext1)); @@ -236,8 +243,8 @@ TEST(ExtentTest, TestAsSmallAllocatorWithHint) { ext1.clear(); hint.leftOffset = 20 * kMiB; ASSERT_EQ(8 * kMiB, freeExt.Alloc(8 * kMiB, hint, &ext1)); - ASSERT_EQ(ext1[0], PExtent(hint.leftOffset, 4 * kMiB)); - ASSERT_EQ(ext1[1], PExtent(28 * kMiB, 4 * kMiB)); + ASSERT_EQ(ext1[0], Extent(hint.leftOffset, 4 * kMiB)); + ASSERT_EQ(ext1[1], Extent(28 * kMiB, 4 * kMiB)); ASSERT_EQ(0, freeExt.AvailableSize()); } @@ -245,7 +252,7 @@ TEST(ExtentTest, TestAsSmallAllocatorWithHint) { // found a extent that satisfy right offset, but its smaller { FreeExtents freeExt(16 * kMiB, 16 * kMiB); - SpaceAllocateHint hint; + AllocateHint hint; Extents ext1; ASSERT_EQ(4 * kMiB, freeExt.Alloc(4 * kMiB, hint, &ext1)); @@ -262,8 +269,8 @@ TEST(ExtentTest, TestAsSmallAllocatorWithHint) { ext1.clear(); hint.rightOffset = 20 * kMiB; ASSERT_EQ(8 * kMiB, freeExt.Alloc(8 * kMiB, hint, &ext1)); - ASSERT_EQ(ext1[0], PExtent(16 * kMiB, 4 * kMiB)); - ASSERT_EQ(ext1[1], PExtent(24 * kMiB, 4 * kMiB)); + ASSERT_EQ(ext1[0], Extent(16 * kMiB, 4 * kMiB)); + ASSERT_EQ(ext1[1], Extent(24 * kMiB, 4 * kMiB)); ASSERT_EQ(0, freeExt.AvailableSize()); } @@ -271,7 +278,7 @@ TEST(ExtentTest, TestAsSmallAllocatorWithHint) { // extents are fragment { FreeExtents freeExt(16 * kMiB, 16 * kMiB); - SpaceAllocateHint hint; + AllocateHint hint; Extents ext1; ASSERT_EQ(16 * kMiB, freeExt.Alloc(16 * kMiB, hint, &ext1)); @@ -332,10 +339,5 @@ TEST(ExtentTest, TestMarkUsed) { EXPECT_EQ(0, freeExt.AvailableExtents().size()); } -} // namespace space +} // namespace volume } // namespace curvefs - -int main(int argc, char* argv[]) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/curvefs/test/space/mock/BUILD b/curvefs/test/volume/mock/BUILD similarity index 72% rename from curvefs/test/space/mock/BUILD rename to curvefs/test/volume/mock/BUILD index 16765c7e7b..466b772238 100644 --- a/curvefs/test/space/mock/BUILD +++ b/curvefs/test/volume/mock/BUILD @@ -1,5 +1,5 @@ # -# Copyright (c) 2021 NetEase Inc. +# Copyright (c) 2022 NetEase Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,15 +16,14 @@ load("//:copts.bzl", "CURVE_TEST_COPTS") -# https://docs.bazel.build/versions/master/be/c-cpp.html#cc_library cc_library( - name = "space_test_mock", - srcs = [], - hdrs = glob(["*.h"]), + name = "mock", + hdrs = glob([ + "*.h", + ]), copts = CURVE_TEST_COPTS, visibility = ["//visibility:public"], deps = [ - "//curvefs/proto:metaserver_cc_proto", - "//curvefs/src/space:curve_fs_space", + "//curvefs/src/volume", ], ) diff --git a/curvefs/test/client/mock_block_device_client.h b/curvefs/test/volume/mock/mock_block_device_client.h similarity index 58% rename from curvefs/test/client/mock_block_device_client.h rename to curvefs/test/volume/mock/mock_block_device_client.h index af13c9f73a..d572aa2e61 100644 --- a/curvefs/test/client/mock_block_device_client.h +++ b/curvefs/test/volume/mock/mock_block_device_client.h @@ -20,42 +20,43 @@ * Author: xuchaojie */ -#ifndef CURVEFS_TEST_CLIENT_MOCK_BLOCK_DEVICE_CLIENT_H_ -#define CURVEFS_TEST_CLIENT_MOCK_BLOCK_DEVICE_CLIENT_H_ +#ifndef CURVEFS_TEST_VOLUME_MOCK_MOCK_BLOCK_DEVICE_CLIENT_H_ +#define CURVEFS_TEST_VOLUME_MOCK_MOCK_BLOCK_DEVICE_CLIENT_H_ -#include #include +#include #include +#include -#include "curvefs/src/client/block_device_client.h" - -using ::testing::Return; -using ::testing::_; +#include "curvefs/src/volume/block_device_client.h" namespace curvefs { -namespace client { +namespace volume { class MockBlockDeviceClient : public BlockDeviceClient { public: MockBlockDeviceClient() {} ~MockBlockDeviceClient() {} - MOCK_METHOD1(Init, CURVEFS_ERROR(const BlockDeviceClientOptions &options)); + MOCK_METHOD1(Init, bool(const BlockDeviceClientOptions& options)); MOCK_METHOD0(UnInit, void()); - MOCK_METHOD2(Open, CURVEFS_ERROR(const std::string& filename, + MOCK_METHOD2(Open, + bool(const std::string& filename, const std::string& owner)); - MOCK_METHOD0(Close, CURVEFS_ERROR()); - MOCK_METHOD3(Stat, CURVEFS_ERROR(const std::string& filename, + MOCK_METHOD0(Close, bool()); + MOCK_METHOD3(Stat, + bool(const std::string& filename, const std::string& owner, BlockDeviceStat* statInfo)); - MOCK_METHOD3(Read, CURVEFS_ERROR(char* buf, off_t offset, size_t length)); - MOCK_METHOD3(Write, CURVEFS_ERROR( - const char* buf, off_t offset, size_t length)); -}; + MOCK_METHOD3(Read, ssize_t(char* buf, off_t offset, size_t length)); + MOCK_METHOD3(Write, ssize_t(const char* buf, off_t offset, size_t length)); + MOCK_METHOD1(Readv, ssize_t(const std::vector&)); + MOCK_METHOD1(Writev, ssize_t(const std::vector&)); +}; -} // namespace client +} // namespace volume } // namespace curvefs -#endif // CURVEFS_TEST_CLIENT_MOCK_BLOCK_DEVICE_CLIENT_H_ +#endif // CURVEFS_TEST_VOLUME_MOCK_MOCK_BLOCK_DEVICE_CLIENT_H_ diff --git a/curvefs/test/volume/mock/mock_space_manager.h b/curvefs/test/volume/mock/mock_space_manager.h new file mode 100644 index 0000000000..25fa899ff3 --- /dev/null +++ b/curvefs/test/volume/mock/mock_space_manager.h @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2022 NetEase Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Project: curve + * Date: Friday Mar 18 12:34:16 CST 2022 + * Author: wuhanqing + */ + +#ifndef CURVEFS_TEST_VOLUME_MOCK_MOCK_SPACE_MANAGER_H_ +#define CURVEFS_TEST_VOLUME_MOCK_MOCK_SPACE_MANAGER_H_ + +#include + +#include + +#include "curvefs/src/volume/space_manager.h" + +namespace curvefs { +namespace volume { + +class MockSpaceManager : public SpaceManager { + public: + MOCK_METHOD3(Alloc, + bool(uint32_t, const AllocateHint&, std::vector*)); + MOCK_METHOD1(DeAlloc, bool(const std::vector&)); + MOCK_METHOD0(Shutdown, bool()); +}; + +} // namespace volume +} // namespace curvefs + +#endif // CURVEFS_TEST_VOLUME_MOCK_MOCK_SPACE_MANAGER_H_ diff --git a/include/curve_compiler_specific.h b/include/curve_compiler_specific.h index 8c0c159661..aa7219c504 100644 --- a/include/curve_compiler_specific.h +++ b/include/curve_compiler_specific.h @@ -47,6 +47,8 @@ #define CURVE_MOCK #endif // UNIT_TEST +// FALLTHROUGH_INTENDED + #if defined(__GNUC__) && __GNUC__ >= 7 #define FALLTHROUGH_INTENDED __attribute__((fallthrough)) #else diff --git a/src/client/copyset_client.h b/src/client/copyset_client.h index ac72b36277..b3ab30346e 100644 --- a/src/client/copyset_client.h +++ b/src/client/copyset_client.h @@ -52,12 +52,13 @@ class RequestScheduler; */ class CopysetClient { public: - CopysetClient() : - sessionNotValid_(false), - metaCache_(nullptr), - senderManager_(nullptr), - scheduler_(nullptr), - exitFlag_(false) {} + CopysetClient() + : metaCache_(nullptr), + senderManager_(nullptr), + sessionNotValid_(false), + scheduler_(nullptr), + fileMetric_(nullptr), + exitFlag_(false) {} CopysetClient(const CopysetClient&) = delete; CopysetClient& operator=(const CopysetClient&) = delete; diff --git a/src/client/lease_executor.h b/src/client/lease_executor.h index 143e28edd9..4bce5c8453 100644 --- a/src/client/lease_executor.h +++ b/src/client/lease_executor.h @@ -98,7 +98,7 @@ class LeaseExecutor { * 测试使用，主动失效增加刷新失败 */ void InvalidLease() { - for (int i = 0; i <= leaseoption_.mdsRefreshTimesPerLease; i++) { + for (uint32_t i = 0; i <= leaseoption_.mdsRefreshTimesPerLease; i++) { IncremRefreshFailed(); } } diff --git a/src/client/libcurve_file.cpp b/src/client/libcurve_file.cpp index fbf5ffcea6..90f71cf14f 100644 --- a/src/client/libcurve_file.cpp +++ b/src/client/libcurve_file.cpp @@ -623,6 +623,8 @@ bool FileClient::StartDummyServer() { return true; } + // FIXME(wuhanqing): if curve-fuse and curve-client both want to start + // dummy server, curve-fuse mount will fail static std::once_flag flag; uint16_t dummyServerStartPort = clientconfig_.GetDummyserverStartPort(); std::call_once(flag, [&]() { diff --git a/src/client/mds_client.cpp b/src/client/mds_client.cpp index dc1f94ac1e..c87578ac1a 100644 --- a/src/client/mds_client.cpp +++ b/src/client/mds_client.cpp @@ -630,7 +630,7 @@ LIBCURVE_ERROR MDSClient::RefreshSession(const std::string &filename, << ", sessionid = " << sessionid << ", status code = " << StatusCode_Name(stcode); } else { - LOG_EVERY_SECOND(INFO) + LOG_EVERY_N(INFO, 100) << "RefreshSession returned: filename = " << filename << ", owner = " << userinfo.owner << ", sessionid = " << sessionid diff --git a/src/client/request_scheduler.h b/src/client/request_scheduler.h index fa87a8fa12..9732eccdc4 100644 --- a/src/client/request_scheduler.h +++ b/src/client/request_scheduler.h @@ -51,8 +51,8 @@ class RequestScheduler : public Uncopyable { RequestScheduler() : running_(false), stop_(true), - blockingQueue_(true), - client_() {} + client_(), + blockingQueue_(true) {} virtual ~RequestScheduler(); /** diff --git a/src/common/bitmap.cpp b/src/common/bitmap.cpp index 5f9141a280..99645bc991 100644 --- a/src/common/bitmap.cpp +++ b/src/common/bitmap.cpp @@ -87,6 +87,20 @@ Bitmap& Bitmap::operator = (const Bitmap& bitmap) { return *this; } +Bitmap::Bitmap(Bitmap&& other) noexcept + : bits_(other.bits_), bitmap_(other.bitmap_) { + other.bits_ = 0; + other.bitmap_ = nullptr; +} + +Bitmap& Bitmap::operator=(Bitmap&& other) noexcept { + using std::swap; + swap(bits_, other.bits_); + swap(bitmap_, other.bitmap_); + + return *this; +} + bool Bitmap::operator == (const Bitmap& bitmap) const { if (bits_ != bitmap.Size()) return false; @@ -107,6 +121,7 @@ void Bitmap::Set(uint32_t index) { } void Bitmap::Set(uint32_t startIndex, uint32_t endIndex) { + // TODO(wuhanqing): implement fast algorithm if one byte is all set for (uint32_t index = startIndex; index <= endIndex; ++index) { Set(index); } @@ -122,6 +137,7 @@ void Bitmap::Clear(uint32_t index) { } void Bitmap::Clear(uint32_t startIndex, uint32_t endIndex) { + // TODO(wuhanqing): implement fast algorithm if one byte is all clear for (uint32_t index = startIndex; index <= endIndex; ++index) { Clear(index); } diff --git a/src/common/bitmap.h b/src/common/bitmap.h index 26286b3145..7b4d917f10 100644 --- a/src/common/bitmap.h +++ b/src/common/bitmap.h @@ -62,8 +62,8 @@ class Bitmap { * @param bitmap: 外部提供的用于初始化的bitmap */ explicit Bitmap(uint32_t bits, const char* bitmap); - virtual ~Bitmap(); - Bitmap() = delete; + ~Bitmap(); + /** * 拷贝构造，使用深拷贝 * @param bitmap：从该对象拷贝内容 @@ -75,6 +75,10 @@ class Bitmap { * @reutrn：返回拷贝后对象引用 */ Bitmap& operator = (const Bitmap& bitmap); + + Bitmap(Bitmap&& other) noexcept; + Bitmap& operator=(Bitmap&& other) noexcept; + /** * 比较两个bitmap是否相同 * @param bitmap：待比较的bitmap diff --git a/src/common/encode.h b/src/common/encode.h index ae2116afa6..0879123f9f 100644 --- a/src/common/encode.h +++ b/src/common/encode.h @@ -23,7 +23,6 @@ #define SRC_COMMON_ENCODE_H_ #include -#include namespace curve { namespace common { @@ -40,6 +39,13 @@ static inline void EncodeBigEndian(char* buf, uint64_t value) { buf[7] = value & 0xff; } +inline void EncodeBigEndian_uint32(char* buf, uint32_t value) { + buf[0] = (value >> 24) & 0xff; + buf[1] = (value >> 16) & 0xff; + buf[2] = (value >> 8) & 0xff; + buf[3] = value & 0xff; +} + } // namespace common } // namespace curve diff --git a/src/common/fast_align.h b/src/common/fast_align.h index 5906a3fad6..c2091b8676 100644 --- a/src/common/fast_align.h +++ b/src/common/fast_align.h @@ -55,6 +55,10 @@ inline bool is_aligned(const void* ptr, std::size_t alignment) noexcept { return is_aligned((std::size_t)ptr, alignment); // NOLINT } +constexpr inline bool is_alignment(std::size_t value) noexcept { + return (value > 0) && ((value & (value - 1)) == 0); +} + } // namespace common } // namespace curve diff --git a/src/mds/nameserver2/allocstatistic/alloc_statistic.cpp b/src/mds/nameserver2/allocstatistic/alloc_statistic.cpp index 49ee146657..06f63bbe73 100644 --- a/src/mds/nameserver2/allocstatistic/alloc_statistic.cpp +++ b/src/mds/nameserver2/allocstatistic/alloc_statistic.cpp @@ -20,12 +20,19 @@ * Author: lixiaocui */ -#include -#include #include "src/mds/nameserver2/allocstatistic/alloc_statistic.h" + +#include + +#include +#include + +#include "proto/nameserver2.pb.h" +#include "src/common/concurrent/concurrent.h" +#include "src/common/namespace_define.h" +#include "src/common/timeutility.h" #include "src/mds/nameserver2/allocstatistic/alloc_statistic_helper.h" #include "src/mds/nameserver2/helper/namespace_helper.h" -#include "src/common/concurrent/concurrent.h" using ::curve::common::Thread; using ::curve::common::ReadLockGuard; diff --git a/src/mds/nameserver2/allocstatistic/alloc_statistic_helper.cpp b/src/mds/nameserver2/allocstatistic/alloc_statistic_helper.cpp index f735ac4139..89bec51c80 100644 --- a/src/mds/nameserver2/allocstatistic/alloc_statistic_helper.cpp +++ b/src/mds/nameserver2/allocstatistic/alloc_statistic_helper.cpp @@ -20,6 +20,7 @@ * Author: lixiaocui */ +#include #include #include #include "src/mds/nameserver2/allocstatistic/alloc_statistic_helper.h" diff --git a/test/integration/common/chunkservice_op.cpp b/test/integration/common/chunkservice_op.cpp index db4710edfa..6a197d9002 100644 --- a/test/integration/common/chunkservice_op.cpp +++ b/test/integration/common/chunkservice_op.cpp @@ -22,6 +22,7 @@ #include "test/integration/common/chunkservice_op.h" #include "proto/chunk.pb.h" +#include "include/curve_compiler_specific.h" namespace curve { namespace chunkserver { @@ -305,6 +306,7 @@ int ChunkServiceOp::GetChunkInfo(struct ChunkServiceOpConf *opConf, switch (response.chunksn().size()) { case 2: *snapSn = response.chunksn(1); + FALLTHROUGH_INTENDED; case 1: *curSn = response.chunksn(0); break;