From 2b1a5b60f1ccdc4db47d476a101cbf6f958a3e0a Mon Sep 17 00:00:00 2001 From: lijiachen Date: Sat, 25 Oct 2025 18:15:13 +0800 Subject: [PATCH 1/2] capacity check --- ucm/store/infra/status/status.h | 7 +- ucm/store/nfsstore/cc/api/nfsstore.cc | 3 +- ucm/store/nfsstore/cc/api/nfsstore.h | 4 +- .../nfsstore/cc/domain/space/space_layout.h | 1 + .../nfsstore/cc/domain/space/space_manager.cc | 28 +++- .../nfsstore/cc/domain/space/space_manager.h | 11 +- .../cc/domain/space/space_property.cc | 136 ++++++++++++++++++ .../nfsstore/cc/domain/space/space_property.h | 51 +++++++ .../cc/domain/space/space_shard_layout.cc | 15 +- .../cc/domain/space/space_shard_layout.h | 3 + ucm/store/nfsstore/cpy/nfsstore.py.cc | 1 + .../test/case/nfsstore/space_manager_test.cc | 10 ++ .../test/case/nfsstore/space_property_test.cc | 61 ++++++++ 13 files changed, 319 insertions(+), 12 deletions(-) create mode 100644 ucm/store/nfsstore/cc/domain/space/space_property.cc create mode 100644 ucm/store/nfsstore/cc/domain/space/space_property.h create mode 100644 ucm/store/test/case/nfsstore/space_property_test.cc diff --git a/ucm/store/infra/status/status.h b/ucm/store/infra/status/status.h index 8c373e12..809d2459 100644 --- a/ucm/store/infra/status/status.h +++ b/ucm/store/infra/status/status.h @@ -42,6 +42,7 @@ class Status { ESERIALIZE = UC_MAKE_STATUS_CODE(6), EDESERIALIZE = UC_MAKE_STATUS_CODE(7), EUNSUPPORTED = UC_MAKE_STATUS_CODE(8), + ENOSPACE = UC_MAKE_STATUS_CODE(9), #undef UC_MAKE_STATUS_CODE }; @@ -101,7 +102,11 @@ class Status { static Status s{Code::EUNSUPPORTED}; return s; } - + static Status& NoSpace() + { + static Status s{Code::ENOSPACE}; + return s; + } public: Status(const Status& status) { this->code_ = status.code_; } Status& operator=(const Status& status) diff --git a/ucm/store/nfsstore/cc/api/nfsstore.cc b/ucm/store/nfsstore/cc/api/nfsstore.cc index 013b5980..48746fcb 100644 --- a/ucm/store/nfsstore/cc/api/nfsstore.cc +++ b/ucm/store/nfsstore/cc/api/nfsstore.cc @@ -35,7 +35,7 @@ class NFSStoreImpl : public NFSStore { int32_t Setup(const Config& config) { auto status = this->spaceMgr_.Setup(config.storageBackends, config.kvcacheBlockSize, - config.tempDumpDirEnable); + config.tempDumpDirEnable, config.storageCapacity); if (status.Failure()) { UC_ERROR("Failed({}) to setup SpaceManager.", status); return status.Underlying(); @@ -120,6 +120,7 @@ class NFSStoreImpl : public NFSStore { UC_INFO("Set UC::TempDumpDirEnable to {}.", config.tempDumpDirEnable); UC_INFO("Set UC::HotnessInterval to {}.", config.hotnessInterval); UC_INFO("Set UC::HotnessEnable to {}.", config.hotnessEnable); + UC_INFO("Set UC::storageCapacity to {}.", config.storageCapacity); } private: diff --git a/ucm/store/nfsstore/cc/api/nfsstore.h b/ucm/store/nfsstore/cc/api/nfsstore.h index 8dc6854f..ce3229dd 100644 --- a/ucm/store/nfsstore/cc/api/nfsstore.h +++ b/ucm/store/nfsstore/cc/api/nfsstore.h @@ -43,13 +43,15 @@ class NFSStore : public CCStore { bool tempDumpDirEnable; bool hotnessEnable; size_t hotnessInterval; + size_t storageCapacity; Config(const std::vector& storageBackends, const size_t kvcacheBlockSize, const bool transferEnable) : storageBackends{storageBackends}, kvcacheBlockSize{kvcacheBlockSize}, transferEnable{transferEnable}, transferDeviceId{-1}, transferStreamNumber{32}, transferIoSize{262144}, transferBufferNumber{512}, transferTimeoutMs{30000}, - tempDumpDirEnable{false}, hotnessEnable{true}, hotnessInterval{60} + tempDumpDirEnable{false}, hotnessEnable{true}, hotnessInterval{60}, + storageCapacity{0} { } }; diff --git a/ucm/store/nfsstore/cc/domain/space/space_layout.h b/ucm/store/nfsstore/cc/domain/space/space_layout.h index 725d5ed5..65da50e2 100644 --- a/ucm/store/nfsstore/cc/domain/space/space_layout.h +++ b/ucm/store/nfsstore/cc/domain/space/space_layout.h @@ -36,6 +36,7 @@ class SpaceLayout { virtual Status Setup(const std::vector& storageBackends) = 0; virtual std::string DataFileParent(const std::string& blockId, bool activated) const = 0; virtual std::string DataFilePath(const std::string& blockId, bool activated) const = 0; + virtual std::string ClusterPropertyFilePath() const = 0; }; } // namespace UC diff --git a/ucm/store/nfsstore/cc/domain/space/space_manager.cc b/ucm/store/nfsstore/cc/domain/space/space_manager.cc index 09e11cb8..fa0d36f4 100644 --- a/ucm/store/nfsstore/cc/domain/space/space_manager.cc +++ b/ucm/store/nfsstore/cc/domain/space/space_manager.cc @@ -40,7 +40,7 @@ std::unique_ptr MakeSpaceLayout(const bool tempDumpDirEnable) } Status SpaceManager::Setup(const std::vector& storageBackends, const size_t blockSize, - const bool tempDumpDirEnable) + const bool tempDumpDirEnable, const size_t storageCapacity) { if (blockSize == 0) { UC_ERROR("Invalid block size({}).", blockSize); @@ -50,12 +50,17 @@ Status SpaceManager::Setup(const std::vector& storageBackends, cons if (!this->layout_) { return Status::OutOfMemory(); } auto status = this->layout_->Setup(storageBackends); if (status.Failure()) { return status; } + status = this->property_.Setup(this->layout_->ClusterPropertyFilePath()); + if (status.Failure()) { return status; } this->blockSize_ = blockSize; + this->capacity_ = storageCapacity; return Status::OK(); } -Status SpaceManager::NewBlock(const std::string& blockId) const +Status SpaceManager::NewBlock(const std::string& blockId) { + Status status = this->CapacityCheck(); + if (status.Failure()) { return status; } constexpr auto activated = true; auto parent = File::Make(this->layout_->DataFileParent(blockId, activated)); auto file = File::Make(this->layout_->DataFilePath(blockId, activated)); @@ -63,7 +68,7 @@ Status SpaceManager::NewBlock(const std::string& blockId) const UC_ERROR("Failed to new block({}).", blockId); return Status::OutOfMemory(); } - auto status = parent->MkDir(); + status = parent->MkDir(); if (status == Status::DuplicateKey()) { status = Status::OK(); } if (status.Failure()) { UC_ERROR("Failed({}) to new block({}).", status, blockId); @@ -88,10 +93,11 @@ Status SpaceManager::NewBlock(const std::string& blockId) const UC_ERROR("Failed({}) to new block({}).", status, blockId); return status; } + this->property_.IncreaseCapacity(this->blockSize_); return Status::OK(); } -Status SpaceManager::CommitBlock(const std::string& blockId, bool success) const +Status SpaceManager::CommitBlock(const std::string& blockId, bool success) { const auto activatedParent = this->layout_->DataFileParent(blockId, true); const auto activatedFile = this->layout_->DataFilePath(blockId, true); @@ -112,6 +118,7 @@ Status SpaceManager::CommitBlock(const std::string& blockId, bool success) const if (status.Failure()) { UC_ERROR("Failed({}) to {} block({}).", status, success ? "commit" : "cancel", blockId); } + this->property_.DecreaseCapacity(this->blockSize_); return status; } @@ -136,4 +143,17 @@ bool SpaceManager::LookupBlock(const std::string& blockId) const const SpaceLayout* SpaceManager::GetSpaceLayout() const { return this->layout_.get(); } +Status SpaceManager::CapacityCheck() const +{ + if (this->capacity_ == 0) { return Status::OK(); } + + const size_t used = this->property_.GetCapacity(); + if (used > this->capacity_ - this->blockSize_) { + UC_ERROR("Capacity is not enough, capacity: {}, current: {}, block size: {}.", + this->capacity_, used, this->blockSize_); + return Status::NoSpace(); + } + return Status::OK(); +} + } // namespace UC diff --git a/ucm/store/nfsstore/cc/domain/space/space_manager.h b/ucm/store/nfsstore/cc/domain/space/space_manager.h index ad74ec14..3688b34a 100644 --- a/ucm/store/nfsstore/cc/domain/space/space_manager.h +++ b/ucm/store/nfsstore/cc/domain/space/space_manager.h @@ -26,6 +26,7 @@ #include #include "space_layout.h" +#include "space_property.h" #include "status/status.h" namespace UC { @@ -33,15 +34,19 @@ namespace UC { class SpaceManager { public: Status Setup(const std::vector& storageBackends, const size_t blockSize, - const bool tempDumpDirEnable); - Status NewBlock(const std::string& blockId) const; - Status CommitBlock(const std::string& blockId, bool success = true) const; + const bool tempDumpDirEnable, const size_t storageCapacity = 0); + Status NewBlock(const std::string& blockId); + Status CommitBlock(const std::string& blockId, bool success = true); bool LookupBlock(const std::string& blockId) const; const SpaceLayout* GetSpaceLayout() const; +private: + Status CapacityCheck() const; private: std::unique_ptr layout_; + SpaceProperty property_; size_t blockSize_; + size_t capacity_; }; } // namespace UC diff --git a/ucm/store/nfsstore/cc/domain/space/space_property.cc b/ucm/store/nfsstore/cc/domain/space/space_property.cc new file mode 100644 index 00000000..2db7b71e --- /dev/null +++ b/ucm/store/nfsstore/cc/domain/space/space_property.cc @@ -0,0 +1,136 @@ +/** + * MIT License + * + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * */ +#include "space_property.h" +#include +#include +#include +#include "logger/logger.h" +#include "file/file.h" + +namespace UC { + +static constexpr uint32_t Magic = (('S' << 16) | ('p' << 8) | 1); +static constexpr size_t PropertySize = 256; + +struct Property { + std::atomic magic; + uint32_t padding; + std::atomic capacity; +}; +static_assert(sizeof(Property) <= PropertySize, "Property take too much space"); +static_assert(std::atomic::is_always_lock_free, "magic must be lock-free"); +static_assert(std::atomic::is_always_lock_free, "capacity must be lock-free"); + +inline auto PropertyPtr(void* addr) { return (Property*)addr; } + +SpaceProperty::~SpaceProperty() +{ + if (this->addr_) { + File::MUnmap(this->addr_, PropertySize); + } + this->addr_ = nullptr; +} + +Status SpaceProperty::Setup(const std::string& PropertyFilePath) +{ + auto file = File::Make(PropertyFilePath); + if (!file) { return Status::OutOfMemory(); } + auto flags = IFile::OpenFlag::CREATE | IFile::OpenFlag::EXCL | IFile::OpenFlag::READ_WRITE; + auto status = file->Open(flags); + if (status.Success()) { return this->InitShmProperty(file.get()); } + if (status == Status::DuplicateKey()) { return this->LoadShmProperty(file.get()); } + return status; +} + +void SpaceProperty::IncreaseCapacity(const size_t delta) +{ + PropertyPtr(this->addr_)->capacity += delta; +} + +void SpaceProperty::DecreaseCapacity(const size_t delta) +{ + auto property = PropertyPtr(this->addr_); + auto capacity = property->capacity.load(std::memory_order_acquire); + while (capacity > delta) { + if (property->capacity.compare_exchange_weak(capacity, capacity - delta, std::memory_order_acq_rel)) { + return; + } + capacity = property->capacity.load(std::memory_order_acquire); + } +} + +size_t SpaceProperty::GetCapacity() const +{ + return PropertyPtr(this->addr_)->capacity.load(std::memory_order_relaxed); +} + +Status SpaceProperty::InitShmProperty(IFile* shmPropertyFile) +{ + auto status = shmPropertyFile->Truncate(PropertySize); + if (status.Failure()) { return status; } + status = shmPropertyFile->MMap(this->addr_, PropertySize, true, true, true); + if (status.Failure()) { return status; } + std::fill_n((uint8_t*)this->addr_, PropertySize, 0); + auto property = PropertyPtr(this->addr_); + property->padding = 0; + property->capacity = 0; + property->magic = Magic; + return Status::OK(); +} + +Status SpaceProperty::LoadShmProperty(IFile* shmPropertyFile) +{ + auto status = shmPropertyFile->Open(IFile::OpenFlag::READ_WRITE); + if (status.Failure()) { return status; } + constexpr auto retryInterval = std::chrono::milliseconds(100); + constexpr auto maxTryTime = 100; + auto tryTime = 0; + IFile::FileStat stat; + do { + if (tryTime > maxTryTime) { + UC_ERROR("Shm file({}) not ready.", shmPropertyFile->Path()); + return Status::Retry(); + } + std::this_thread::sleep_for(retryInterval); + status = shmPropertyFile->Stat(stat); + if (status.Failure()) { return status; } + tryTime++; + } while (static_cast(stat.st_size) != PropertySize); + status = shmPropertyFile->MMap(this->addr_, PropertySize, true, true, true); + if (status.Failure()) { return status; } + auto property = PropertyPtr(this->addr_); + tryTime = 0; + do { + if (property->magic == Magic) { break; } + if (tryTime > maxTryTime) { + UC_ERROR("Shm file({}) not ready.", shmPropertyFile->Path()); + return Status::Retry(); + } + std::this_thread::sleep_for(retryInterval); + tryTime++; + } while (true); + return Status::OK(); +} + +} // namespace UC \ No newline at end of file diff --git a/ucm/store/nfsstore/cc/domain/space/space_property.h b/ucm/store/nfsstore/cc/domain/space/space_property.h new file mode 100644 index 00000000..b4e99908 --- /dev/null +++ b/ucm/store/nfsstore/cc/domain/space/space_property.h @@ -0,0 +1,51 @@ +/** + * MIT License + * + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * */ + +#ifndef UNIFIEDCACHE_SPACE_PROPERTY_H +#define UNIFIEDCACHE_SPACE_PROPERTY_H + +#include "file/ifile.h" +#include "status/status.h" + +namespace UC { + +class SpaceProperty { +public: + ~SpaceProperty(); + Status Setup(const std::string& propertyFilePath); + void IncreaseCapacity(const size_t delta); + void DecreaseCapacity(const size_t delta); + size_t GetCapacity() const; + +private: + Status InitShmProperty(IFile* shmPropertyFile); + Status LoadShmProperty(IFile* shmPropertyFile); + +private: + void* addr_{nullptr}; +}; + +} // namespace UC + +#endif \ No newline at end of file diff --git a/ucm/store/nfsstore/cc/domain/space/space_shard_layout.cc b/ucm/store/nfsstore/cc/domain/space/space_shard_layout.cc index 57f58010..a08b10af 100644 --- a/ucm/store/nfsstore/cc/domain/space/space_shard_layout.cc +++ b/ucm/store/nfsstore/cc/domain/space/space_shard_layout.cc @@ -111,10 +111,15 @@ std::string SpaceShardLayout::StorageBackend(const std::string& blockId) const return this->storageBackends_[hasher(blockId) % this->storageBackends_.size()]; } -std::vector SpaceShardLayout::RelativeRoots() const { return {this->DataFileRoot()}; } +std::vector SpaceShardLayout::RelativeRoots() const { + return { + this->DataFileRoot(), + this->ClusterFileRoot(), + }; +} std::string SpaceShardLayout::DataFileRoot() const { return "data"; } - +std::string SpaceShardLayout::ClusterFileRoot() const { return "cluster"; } void SpaceShardLayout::ShardBlockId(const std::string& blockId, uint64_t& front, uint64_t& back) const { @@ -123,4 +128,10 @@ void SpaceShardLayout::ShardBlockId(const std::string& blockId, uint64_t& front, back = id->back(); } +std::string SpaceShardLayout::StorageBackend() const { return this->storageBackends_.front(); } +std::string SpaceShardLayout::ClusterPropertyFilePath() const +{ + return fmt::format("{}{}/{}.bin", this->StorageBackend(), this->ClusterFileRoot(), "uc_property.bin"); +} + } // namespace UC diff --git a/ucm/store/nfsstore/cc/domain/space/space_shard_layout.h b/ucm/store/nfsstore/cc/domain/space/space_shard_layout.h index 024bec51..bddcc093 100644 --- a/ucm/store/nfsstore/cc/domain/space/space_shard_layout.h +++ b/ucm/store/nfsstore/cc/domain/space/space_shard_layout.h @@ -33,6 +33,7 @@ class SpaceShardLayout : public SpaceLayout { Status Setup(const std::vector& storageBackends) override; std::string DataFileParent(const std::string& blockId, bool activated) const override; std::string DataFilePath(const std::string& blockId, bool activated) const override; + std::string ClusterPropertyFilePath() const override; protected: virtual std::vector RelativeRoots() const; @@ -41,6 +42,8 @@ class SpaceShardLayout : public SpaceLayout { virtual Status AddSecondaryStorageBackend(const std::string& path); virtual std::string StorageBackend(const std::string& blockId) const; virtual std::string DataFileRoot() const; + virtual std::string ClusterFileRoot() const; + virtual std::string StorageBackend() const; virtual void ShardBlockId(const std::string& blockId, uint64_t& front, uint64_t& back) const; std::vector storageBackends_; }; diff --git a/ucm/store/nfsstore/cpy/nfsstore.py.cc b/ucm/store/nfsstore/cpy/nfsstore.py.cc index 7f148f1b..a585d8f3 100644 --- a/ucm/store/nfsstore/cpy/nfsstore.py.cc +++ b/ucm/store/nfsstore/cpy/nfsstore.py.cc @@ -125,6 +125,7 @@ PYBIND11_MODULE(ucmnfsstore, module) config.def_readwrite("tempDumpDirEnable", &UC::NFSStorePy::Config::tempDumpDirEnable); config.def_readwrite("hotnessEnable", &UC::NFSStorePy::Config::hotnessEnable); config.def_readwrite("hotnessInterval", &UC::NFSStorePy::Config::hotnessInterval); + config.def_readwrite("storageCapacity", &UC::NFSStorePy::Config::storageCapacity); store.def(py::init<>()); store.def("CCStoreImpl", &UC::NFSStorePy::CCStoreImpl); store.def("Setup", &UC::NFSStorePy::Setup); diff --git a/ucm/store/test/case/nfsstore/space_manager_test.cc b/ucm/store/test/case/nfsstore/space_manager_test.cc index 3d1d110d..af73b640 100644 --- a/ucm/store/test/case/nfsstore/space_manager_test.cc +++ b/ucm/store/test/case/nfsstore/space_manager_test.cc @@ -53,3 +53,13 @@ TEST_F(UCSpaceManagerTest, NewBlockTwiceWithTempDir) ASSERT_TRUE(spaceMgr.LookupBlock(block1)); ASSERT_EQ(spaceMgr.NewBlock(block1), UC::Status::DuplicateKey()); } + +TEST_F(UCSpaceManagerTest, CreateBlockWhenNoSpace) +{ + UC::SpaceManager spaceMgr; + size_t blockSize = 1024 * 1024; + size_t capacity = blockSize; + ASSERT_EQ(spaceMgr.Setup({this->Path()}, blockSize, false, capacity), UC::Status::OK()); + ASSERT_EQ(spaceMgr.NewBlock("block3"), UC::Status::OK()); + ASSERT_EQ(spaceMgr.NewBlock("block4"), UC::Status::NoSpace()); +} \ No newline at end of file diff --git a/ucm/store/test/case/nfsstore/space_property_test.cc b/ucm/store/test/case/nfsstore/space_property_test.cc new file mode 100644 index 00000000..5b5beb61 --- /dev/null +++ b/ucm/store/test/case/nfsstore/space_property_test.cc @@ -0,0 +1,61 @@ +/** + * MIT License + * + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * */ +#include "cmn/path_base.h" +#include "space/space_property.h" +#include "space/space_layout.h" +#include "space/space_shard_layout.h" +#include "space/space_manager.h" + +class UCSpacePropertyTest : public UC::PathBase {}; + +/* +* check the persistence of property +*/ +TEST_F(UCSpacePropertyTest, CapacityPersistence) +{ + size_t blocksize = 1024 * 1024; + UC::SpaceManager spaceMgr; + ASSERT_EQ(spaceMgr.Setup({this->Path()}, blocksize, false, blocksize * 5), UC::Status::OK()); + const UC::SpaceLayout* layout = spaceMgr.GetSpaceLayout(); + + const std::string path = layout->ClusterPropertyFilePath(); + + UC::SpaceProperty spaceProperty; + ASSERT_EQ(spaceProperty.Setup(path), UC::Status::OK()); + ASSERT_EQ(spaceProperty.GetCapacity(), 0); + + spaceProperty.IncreaseCapacity(blocksize * 2); + ASSERT_EQ(spaceProperty.GetCapacity(), blocksize*2); + + UC::SpaceProperty spaceProperty2; + ASSERT_EQ(spaceProperty2.Setup(path), UC::Status::OK()); + ASSERT_EQ(spaceProperty2.GetCapacity(), blocksize*2); + + spaceProperty2.DecreaseCapacity(blocksize); + ASSERT_EQ(spaceProperty2.GetCapacity(), blocksize); + + UC::SpaceProperty spaceProperty3; + ASSERT_EQ(spaceProperty3.Setup(path), UC::Status::OK()); + ASSERT_EQ(spaceProperty3.GetCapacity(), blocksize); + } \ No newline at end of file From 4fe78976f9d435ada2d2d391ceaa3146e4dd6b6a Mon Sep 17 00:00:00 2001 From: lijiachen Date: Tue, 28 Oct 2025 16:21:34 +0800 Subject: [PATCH 2/2] recycle --- ucm/store/infra/file/file.cc | 10 ++ ucm/store/infra/file/file.h | 1 + ucm/store/infra/template/topn_heap.h | 120 +++++++++++++++++ ucm/store/nfsstore/cc/api/nfsstore.cc | 5 +- ucm/store/nfsstore/cc/api/nfsstore.h | 4 +- .../nfsstore/cc/domain/space/space_layout.h | 6 + .../nfsstore/cc/domain/space/space_manager.cc | 17 ++- .../nfsstore/cc/domain/space/space_manager.h | 9 +- .../nfsstore/cc/domain/space/space_recycle.cc | 127 ++++++++++++++++++ .../nfsstore/cc/domain/space/space_recycle.h | 61 +++++++++ .../cc/domain/space/space_shard_layout.cc | 95 ++++++++++++- .../cc/domain/space/space_shard_layout.h | 5 + ucm/store/nfsstore/cpy/nfsstore.py.cc | 2 + .../test/case/nfsstore/space_manager_test.cc | 26 ++++ .../test/case/nfsstore/space_recycle_test.cc | 124 +++++++++++++++++ 15 files changed, 605 insertions(+), 7 deletions(-) create mode 100644 ucm/store/infra/template/topn_heap.h create mode 100644 ucm/store/nfsstore/cc/domain/space/space_recycle.cc create mode 100644 ucm/store/nfsstore/cc/domain/space/space_recycle.h create mode 100644 ucm/store/test/case/nfsstore/space_recycle_test.cc diff --git a/ucm/store/infra/file/file.cc b/ucm/store/infra/file/file.cc index 2ceec439..2dd9b75e 100644 --- a/ucm/store/infra/file/file.cc +++ b/ucm/store/infra/file/file.cc @@ -53,6 +53,16 @@ Status File::Access(const std::string& path, const int32_t mode) return FileImpl{path}.Access(mode); } +Status File::Stat(const std::string& path, IFile::FileStat& st) +{ + FileImpl file{path}; + auto status = file.Open(IFile::OpenFlag::READ_ONLY); + if (status.Failure()) { return status; } + status = file.Stat(st); + file.Close(); + return status; +} + Status File::Read(const std::string& path, const size_t offset, const size_t length, uintptr_t address, const bool directIo) { diff --git a/ucm/store/infra/file/file.h b/ucm/store/infra/file/file.h index 0e21da96..45d1d45b 100644 --- a/ucm/store/infra/file/file.h +++ b/ucm/store/infra/file/file.h @@ -36,6 +36,7 @@ class File { static Status RmDir(const std::string& path); static Status Rename(const std::string& path, const std::string& newName); static Status Access(const std::string& path, const int32_t mode); + static Status Stat(const std::string& path, IFile::FileStat& st); static Status Read(const std::string& path, const size_t offset, const size_t length, uintptr_t address, const bool directIo = false); static Status Write(const std::string& path, const size_t offset, const size_t length, diff --git a/ucm/store/infra/template/topn_heap.h b/ucm/store/infra/template/topn_heap.h new file mode 100644 index 00000000..98884c23 --- /dev/null +++ b/ucm/store/infra/template/topn_heap.h @@ -0,0 +1,120 @@ +/** + * MIT License + * + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * */ + +#ifndef UNIFIEDCACHE_TOP_N_HEAP_H +#define UNIFIEDCACHE_TOP_N_HEAP_H + +#include +#include + +namespace UC { + +template > +class TopNHeap { +public: + using ValueType = T; + using SizeType = uint32_t; + using ConstRef = const T&; + +private: + using IndexType = uint32_t; + std::vector val_{}; + std::vector idx_{}; + SizeType capacity_{0}; + SizeType size_{0}; + Compare cmp_{}; + +public: + explicit TopNHeap(const SizeType capacity) noexcept( + std::is_nothrow_default_constructible_v) + : capacity_{capacity} { + val_.resize(capacity); + idx_.resize(capacity); + } + TopNHeap(const TopNHeap&) = delete; + TopNHeap(const TopNHeap&&) = delete; + TopNHeap& operator=(const TopNHeap&) = delete; + TopNHeap& operator=(const TopNHeap&&) = delete; + ~TopNHeap() { Clear(); } + + SizeType Size() const noexcept { return size_; } + SizeType Capacity() const noexcept { return capacity_; } + bool Empty() const noexcept { return size_ == 0; } + + void Push(ConstRef value) noexcept { + if (size_ < capacity_) { + val_[size_] = value; + idx_[size_] = size_; + SiftUp(size_); + size_++; + return; + } + if (cmp_(val_[idx_.front()], value)) { + val_[idx_.front()] = value; + SiftDown(0); + } + } + ConstRef Top() const noexcept { return val_[idx_.front()]; } + void Pop() noexcept { + idx_[0] = idx_[--size_]; + if (size_) { SiftDown(0); } + } + void Clear() noexcept { size_ = 0; } +private: + static IndexType Parent(IndexType i) noexcept { return (i - 1) / 2; } + static IndexType Left(IndexType i) noexcept { return 2 * i + 1; } + static IndexType Right(IndexType i) noexcept { return 2 * i + 2; } + void SiftUp(IndexType i) noexcept { + auto pos = i; + while (pos > 0) { + auto p = Parent(pos); + if (!cmp_(val_[idx_[pos]], val_[idx_[p]])) { break; } + std::swap(idx_[pos], idx_[p]); + pos = p; + } + } + void SiftDown(IndexType i) noexcept { + auto pos = i; + for (;;) { + auto l = Left(pos); + auto r = Right(pos); + auto best = pos; + if (l < size_ && cmp_(val_[idx_[l]], val_[idx_[best]])) { best = l; } + if (r < size_ && cmp_(val_[idx_[r]], val_[idx_[best]])) { best = r; } + if (best == pos) { break; } + std::swap(idx_[pos], idx_[best]); + pos = best; + } + } +}; + +template > +class TopNFixedHeap : public TopNHeap { +public: + TopNFixedHeap() : TopNHeap{N} {} +}; + +} // namespace UC + +#endif \ No newline at end of file diff --git a/ucm/store/nfsstore/cc/api/nfsstore.cc b/ucm/store/nfsstore/cc/api/nfsstore.cc index 48746fcb..381369c5 100644 --- a/ucm/store/nfsstore/cc/api/nfsstore.cc +++ b/ucm/store/nfsstore/cc/api/nfsstore.cc @@ -35,7 +35,8 @@ class NFSStoreImpl : public NFSStore { int32_t Setup(const Config& config) { auto status = this->spaceMgr_.Setup(config.storageBackends, config.kvcacheBlockSize, - config.tempDumpDirEnable, config.storageCapacity); + config.tempDumpDirEnable, config.storageCapacity, + config.recycleEnable, config.recycleThresholdRatio); if (status.Failure()) { UC_ERROR("Failed({}) to setup SpaceManager.", status); return status.Underlying(); @@ -121,6 +122,8 @@ class NFSStoreImpl : public NFSStore { UC_INFO("Set UC::HotnessInterval to {}.", config.hotnessInterval); UC_INFO("Set UC::HotnessEnable to {}.", config.hotnessEnable); UC_INFO("Set UC::storageCapacity to {}.", config.storageCapacity); + UC_INFO("Set UC::RecycleEnable to {}.", config.recycleEnable); + UC_INFO("Set UC::RecycleThreshold to {}.", config.recycleThresholdRatio); } private: diff --git a/ucm/store/nfsstore/cc/api/nfsstore.h b/ucm/store/nfsstore/cc/api/nfsstore.h index ce3229dd..6ba14c63 100644 --- a/ucm/store/nfsstore/cc/api/nfsstore.h +++ b/ucm/store/nfsstore/cc/api/nfsstore.h @@ -44,6 +44,8 @@ class NFSStore : public CCStore { bool hotnessEnable; size_t hotnessInterval; size_t storageCapacity; + bool recycleEnable; + float recycleThresholdRatio; Config(const std::vector& storageBackends, const size_t kvcacheBlockSize, const bool transferEnable) @@ -51,7 +53,7 @@ class NFSStore : public CCStore { transferEnable{transferEnable}, transferDeviceId{-1}, transferStreamNumber{32}, transferIoSize{262144}, transferBufferNumber{512}, transferTimeoutMs{30000}, tempDumpDirEnable{false}, hotnessEnable{true}, hotnessInterval{60}, - storageCapacity{0} + storageCapacity{0}, recycleEnable{true}, recycleThresholdRatio{0.7f} { } }; diff --git a/ucm/store/nfsstore/cc/domain/space/space_layout.h b/ucm/store/nfsstore/cc/domain/space/space_layout.h index 65da50e2..3712abec 100644 --- a/ucm/store/nfsstore/cc/domain/space/space_layout.h +++ b/ucm/store/nfsstore/cc/domain/space/space_layout.h @@ -24,6 +24,7 @@ #ifndef UNIFIEDCACHE_SPACE_LAYOUT_H #define UNIFIEDCACHE_SPACE_LAYOUT_H +#include #include #include #include "status/status.h" @@ -31,12 +32,17 @@ namespace UC { class SpaceLayout { +public: + struct DataIterator; public: virtual ~SpaceLayout() = default; virtual Status Setup(const std::vector& storageBackends) = 0; virtual std::string DataFileParent(const std::string& blockId, bool activated) const = 0; virtual std::string DataFilePath(const std::string& blockId, bool activated) const = 0; virtual std::string ClusterPropertyFilePath() const = 0; + virtual std::shared_ptr CreateFilePathIterator() const = 0; + virtual std::string NextDataFilePath(std::shared_ptr iter) const = 0; + virtual bool IsActivatedFile(const std::string& filePath) const = 0; }; } // namespace UC diff --git a/ucm/store/nfsstore/cc/domain/space/space_manager.cc b/ucm/store/nfsstore/cc/domain/space/space_manager.cc index fa0d36f4..f088cc89 100644 --- a/ucm/store/nfsstore/cc/domain/space/space_manager.cc +++ b/ucm/store/nfsstore/cc/domain/space/space_manager.cc @@ -40,7 +40,8 @@ std::unique_ptr MakeSpaceLayout(const bool tempDumpDirEnable) } Status SpaceManager::Setup(const std::vector& storageBackends, const size_t blockSize, - const bool tempDumpDirEnable, const size_t storageCapacity) + const bool tempDumpDirEnable, const size_t storageCapacity, + const bool recycleEnable, const float recycleThresholdRatio) { if (blockSize == 0) { UC_ERROR("Invalid block size({}).", blockSize); @@ -51,9 +52,18 @@ Status SpaceManager::Setup(const std::vector& storageBackends, cons auto status = this->layout_->Setup(storageBackends); if (status.Failure()) { return status; } status = this->property_.Setup(this->layout_->ClusterPropertyFilePath()); + if (recycleEnable && storageCapacity > 0) { + auto totalBlocks = storageCapacity / blockSize; + status = this->recycle_.Setup(this->GetSpaceLayout(), totalBlocks, [this] { + this->property_.DecreaseCapacity(this->blockSize_); + }); + if (status.Failure()) { return status; } + } if (status.Failure()) { return status; } this->blockSize_ = blockSize; this->capacity_ = storageCapacity; + this->recycleEnable_ = recycleEnable; + this->capacityRecycleThreshold_ = static_cast(storageCapacity * recycleThresholdRatio); return Status::OK(); } @@ -143,11 +153,14 @@ bool SpaceManager::LookupBlock(const std::string& blockId) const const SpaceLayout* SpaceManager::GetSpaceLayout() const { return this->layout_.get(); } -Status SpaceManager::CapacityCheck() const +Status SpaceManager::CapacityCheck() { if (this->capacity_ == 0) { return Status::OK(); } const size_t used = this->property_.GetCapacity(); + if (this->recycleEnable_ && used >= this->capacityRecycleThreshold_) { + this->recycle_.Trigger(); + } if (used > this->capacity_ - this->blockSize_) { UC_ERROR("Capacity is not enough, capacity: {}, current: {}, block size: {}.", this->capacity_, used, this->blockSize_); diff --git a/ucm/store/nfsstore/cc/domain/space/space_manager.h b/ucm/store/nfsstore/cc/domain/space/space_manager.h index 3688b34a..e6690a95 100644 --- a/ucm/store/nfsstore/cc/domain/space/space_manager.h +++ b/ucm/store/nfsstore/cc/domain/space/space_manager.h @@ -28,25 +28,30 @@ #include "space_layout.h" #include "space_property.h" #include "status/status.h" +#include "space_recycle.h" namespace UC { class SpaceManager { public: Status Setup(const std::vector& storageBackends, const size_t blockSize, - const bool tempDumpDirEnable, const size_t storageCapacity = 0); + const bool tempDumpDirEnable, const size_t storageCapacity = 0, + const bool recycleEnable = false, const float recycleThresholdRatio = 0.7f); Status NewBlock(const std::string& blockId); Status CommitBlock(const std::string& blockId, bool success = true); bool LookupBlock(const std::string& blockId) const; const SpaceLayout* GetSpaceLayout() const; private: - Status CapacityCheck() const; + Status CapacityCheck(); private: std::unique_ptr layout_; SpaceProperty property_; + SpaceRecycle recycle_; size_t blockSize_; size_t capacity_; + bool recycleEnable_; + size_t capacityRecycleThreshold_; }; } // namespace UC diff --git a/ucm/store/nfsstore/cc/domain/space/space_recycle.cc b/ucm/store/nfsstore/cc/domain/space/space_recycle.cc new file mode 100644 index 00000000..194a6879 --- /dev/null +++ b/ucm/store/nfsstore/cc/domain/space/space_recycle.cc @@ -0,0 +1,127 @@ +/** + * MIT License + * + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * */ + +#include "space_recycle.h" +#include "logger/logger.h" +#include "file/file.h" +#include "template/topn_heap.h" + +namespace UC { + +constexpr float recyclePercent = 0.1f; /* recycle 10% of the capacity */ +constexpr uint32_t maxRecycleNum = 10240; /* max recycle num */ + +struct BlockInfo { + std::string path; + size_t timestamp; +}; + +struct CmpTimestamp { + bool operator()(const BlockInfo& lhs, const BlockInfo& rhs) const { + return lhs.timestamp > rhs.timestamp; + } +}; + +size_t GetFileTimestamp(const std::string& path) { + IFile::FileStat st; + if (File::Stat(path, st).Failure()) { return 0; } + return st.st_mtim.tv_sec; +} + +void RemoveBlockFile(const std::string& path) { + File::Remove(path); + auto pos = path.rfind('/'); + if (pos == std::string::npos) { return; } + auto parent = path.substr(0, pos); + File::RmDir(parent); +} + +void DoRecycle(const SpaceLayout* layout, const uint32_t recycleNum, + SpaceRecycle::RecycleOneBlockDone done) { + auto earliestHeap = std::make_unique>(recycleNum); + auto it = layout->CreateFilePathIterator(); + while (it) { + auto filePath = layout->NextDataFilePath(it); + if (filePath.empty()) { break; } + auto timestamp = GetFileTimestamp(filePath); + if (timestamp == 0) { continue; } + earliestHeap->Push({filePath, timestamp}); + } + while (!earliestHeap->Empty()) { + RemoveBlockFile(earliestHeap->Top().path); + if (done) { done(); } + earliestHeap->Pop(); + } +} +SpaceRecycle::~SpaceRecycle() { + { + std::lock_guard lock(this->mtx_); + this->stop_ = true; + this->cv_.notify_all(); + } + if (this->worker_.joinable()) { + this->worker_.join(); + } +} +Status SpaceRecycle::Setup(const SpaceLayout* layout, const size_t totalNumber, + RecycleOneBlockDone done) { + this->layout_ = layout; + this->recycleNum_ = totalNumber * recyclePercent; + this->recycleOneBlockDone_ = done; + if (this->recycleNum_ > maxRecycleNum) { + this->recycleNum_ = maxRecycleNum; + } + return Status::OK(); +} + +void SpaceRecycle::Trigger() +{ + if (!this->serviceRunning_) { + this->worker_ = std::thread(&SpaceRecycle::Recycler, this); + } + std::lock_guard lock(this->mtx_); + if (!this->recycling_) { + this->recycling_ = true; + this->cv_.notify_all(); + } +} + +void SpaceRecycle::Recycler() +{ + this->serviceRunning_ = true; + UC_INFO("Space Recycle service start successfully."); + while (true) { + { + std::unique_lock lock(this->mtx_); + this->cv_.wait(lock, [this] { return this->stop_ || this->recycling_; }); + if (this->stop_) { break; } + } + DoRecycle(this->layout_, this->recycleNum_, this->recycleOneBlockDone_); + { + std::lock_guard lock(this->mtx_); + this->recycling_ = false; + } + } +} +} // namespace UC diff --git a/ucm/store/nfsstore/cc/domain/space/space_recycle.h b/ucm/store/nfsstore/cc/domain/space/space_recycle.h new file mode 100644 index 00000000..31e89d31 --- /dev/null +++ b/ucm/store/nfsstore/cc/domain/space/space_recycle.h @@ -0,0 +1,61 @@ +/** + * MIT License + * + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * */ +#ifndef UNIFIEDCACHE_SPACE_RECYCLE_H +#define UNIFIEDCACHE_SPACE_RECYCLE_H + +#include +#include +#include +#include +#include +#include "space_layout.h" + +namespace UC { + +class SpaceRecycle { +public: + using RecycleOneBlockDone = std::function; + SpaceRecycle() = default; + SpaceRecycle(const SpaceRecycle&) = delete; + SpaceRecycle& operator=(const SpaceRecycle&) = delete; + ~SpaceRecycle(); + Status Setup(const SpaceLayout* layout, const size_t totalNumber, + RecycleOneBlockDone done); + void Trigger(); +private: + void Recycler(); +private: + bool stop_{false}; + bool recycling_{false}; + std::atomic_bool serviceRunning_{false}; + uint32_t recycleNum_{0}; + RecycleOneBlockDone recycleOneBlockDone_; + const SpaceLayout* layout_{nullptr}; + std::mutex mtx_; + std::condition_variable cv_; + std::thread worker_; +}; + +} // namespace UC +#endif \ No newline at end of file diff --git a/ucm/store/nfsstore/cc/domain/space/space_shard_layout.cc b/ucm/store/nfsstore/cc/domain/space/space_shard_layout.cc index a08b10af..012f0806 100644 --- a/ucm/store/nfsstore/cc/domain/space/space_shard_layout.cc +++ b/ucm/store/nfsstore/cc/domain/space/space_shard_layout.cc @@ -23,7 +23,9 @@ * */ #include "space_shard_layout.h" #include +#include #include +#include #include "file/file.h" #include "logger/logger.h" @@ -34,6 +36,69 @@ constexpr size_t nU64PerBlock = blockIdSize / sizeof(uint64_t); using BlockId = std::array; static_assert(sizeof(BlockId) == blockIdSize); +const std::string activatedFileSuffix = "act"; +const std::string archivedFileSuffix = "dat"; + +inline auto OpenDir(const std::string& path) +{ + auto dir = ::opendir(path.c_str()); + auto eno = errno; + if (!dir) { UC_ERROR("Failed({}) to open dir({}).", eno, path); } + return dir; +} + +// Define SpaceLayout::DataIterator as an empty base class +struct SpaceLayout::DataIterator { + virtual ~DataIterator() = default; +}; + +struct SpaceShardLayout::DataIterator : public SpaceLayout::DataIterator { + const SpaceLayout* layout{nullptr}; + std::string root; + std::string current; + std::stack> stk; + ~DataIterator() + { + while (!this->stk.empty()) { + ::closedir(this->stk.top().first); + this->stk.pop(); + } + } + Status Setup(const SpaceLayout* layout, const std::string& root) { + this->layout = layout; + this->root = root; + auto dir = OpenDir(root); + if (!dir) { return Status::OsApiError(); } + this->stk.emplace(dir, root); + return Status::OK(); + } + Status Next() { + this->current.clear(); + while (!this->stk.empty()) { + auto entry = ::readdir64(this->stk.top().first); + if (entry == nullptr) { + ::closedir(this->stk.top().first); + this->stk.pop(); + continue; + } + std::string name{entry->d_name}; + if (name.front() == '.') { continue; } + if (this->layout->IsActivatedFile(name)) { continue; } + const auto& dir = this->stk.top().second; + auto fullpath = this->stk.top().second + "/" + name; + if (dir == this->root) { + auto sub = OpenDir(fullpath); + if (!sub) { return Status::OsApiError(); } + this->stk.emplace(sub, fullpath); + continue; + } + this->current = std::move(fullpath); + return Status::OK(); + } + return Status::NotFound(); + } +}; + Status SpaceShardLayout::Setup(const std::vector& storageBackends) { if (storageBackends.empty()) { @@ -59,7 +124,7 @@ std::string SpaceShardLayout::DataFilePath(const std::string& blockId, bool acti uint64_t front, back; this->ShardBlockId(blockId, front, back); return fmt::format("{}{}/{:016x}/{:016x}.{}", this->StorageBackend(blockId), - this->DataFileRoot(), front, back, activated ? "act" : "dat"); + this->DataFileRoot(), front, back, activated ? activatedFileSuffix : archivedFileSuffix); } Status SpaceShardLayout::AddStorageBackend(const std::string& path) @@ -134,4 +199,32 @@ std::string SpaceShardLayout::ClusterPropertyFilePath() const return fmt::format("{}{}/{}.bin", this->StorageBackend(), this->ClusterFileRoot(), "uc_property.bin"); } +std::shared_ptr SpaceShardLayout::CreateFilePathIterator() const +{ + auto dataRoot = this->StorageBackend() + this->DataFileRoot(); + std::shared_ptr iter = nullptr; + try { + iter = std::make_shared(); + } catch (const std::exception& e) { + UC_ERROR("Failed to create data iterator: {}", e.what()); + return nullptr; + } + if (iter->Setup(this, dataRoot).Failure()) { + return nullptr; + } + return std::dynamic_pointer_cast(iter); +} + +std::string SpaceShardLayout::NextDataFilePath(std::shared_ptr iter) const +{ + auto shard_iter = std::dynamic_pointer_cast(iter); + if (!shard_iter) { return std::string{}; } + if (shard_iter->Next().Failure()) { return std::string{}; } + return shard_iter->current; +} + +bool SpaceShardLayout::IsActivatedFile(const std::string& filePath) const +{ + return std::equal(activatedFileSuffix.rbegin(), activatedFileSuffix.rend(), filePath.rbegin()); +} } // namespace UC diff --git a/ucm/store/nfsstore/cc/domain/space/space_shard_layout.h b/ucm/store/nfsstore/cc/domain/space/space_shard_layout.h index bddcc093..60dd4820 100644 --- a/ucm/store/nfsstore/cc/domain/space/space_shard_layout.h +++ b/ucm/store/nfsstore/cc/domain/space/space_shard_layout.h @@ -29,11 +29,16 @@ namespace UC { class SpaceShardLayout : public SpaceLayout { +public: + struct DataIterator; public: Status Setup(const std::vector& storageBackends) override; std::string DataFileParent(const std::string& blockId, bool activated) const override; std::string DataFilePath(const std::string& blockId, bool activated) const override; std::string ClusterPropertyFilePath() const override; + std::shared_ptr CreateFilePathIterator() const override; + std::string NextDataFilePath(std::shared_ptr iter) const override; + bool IsActivatedFile(const std::string& filePath) const override; protected: virtual std::vector RelativeRoots() const; diff --git a/ucm/store/nfsstore/cpy/nfsstore.py.cc b/ucm/store/nfsstore/cpy/nfsstore.py.cc index a585d8f3..1536d876 100644 --- a/ucm/store/nfsstore/cpy/nfsstore.py.cc +++ b/ucm/store/nfsstore/cpy/nfsstore.py.cc @@ -126,6 +126,8 @@ PYBIND11_MODULE(ucmnfsstore, module) config.def_readwrite("hotnessEnable", &UC::NFSStorePy::Config::hotnessEnable); config.def_readwrite("hotnessInterval", &UC::NFSStorePy::Config::hotnessInterval); config.def_readwrite("storageCapacity", &UC::NFSStorePy::Config::storageCapacity); + config.def_readwrite("recycleEnable", &UC::NFSStorePy::Config::recycleEnable); + config.def_readwrite("recycleThresholdRatio", &UC::NFSStorePy::Config::recycleThresholdRatio); store.def(py::init<>()); store.def("CCStoreImpl", &UC::NFSStorePy::CCStoreImpl); store.def("Setup", &UC::NFSStorePy::Setup); diff --git a/ucm/store/test/case/nfsstore/space_manager_test.cc b/ucm/store/test/case/nfsstore/space_manager_test.cc index af73b640..58d35b70 100644 --- a/ucm/store/test/case/nfsstore/space_manager_test.cc +++ b/ucm/store/test/case/nfsstore/space_manager_test.cc @@ -62,4 +62,30 @@ TEST_F(UCSpaceManagerTest, CreateBlockWhenNoSpace) ASSERT_EQ(spaceMgr.Setup({this->Path()}, blockSize, false, capacity), UC::Status::OK()); ASSERT_EQ(spaceMgr.NewBlock("block3"), UC::Status::OK()); ASSERT_EQ(spaceMgr.NewBlock("block4"), UC::Status::NoSpace()); +} + +TEST_F(UCSpaceManagerTest, IterAllBlockFile) +{ + constexpr size_t blockSize = 1024 * 1024; + constexpr size_t capacity = blockSize * 1024; + UC::SpaceManager spaceMgr; + ASSERT_EQ(spaceMgr.Setup({this->Path()}, blockSize, false, capacity), UC::Status::OK()); + const std::string block1 = "a1b2c3d4e5f6789012345678901234ab"; + const std::string block2 = "a2b2c3d4e5f6789012345678901234ab"; + const std::string block3 = "a3b2c3d4e5f6789012345678901234ab"; + ASSERT_EQ(spaceMgr.NewBlock(block1), UC::Status::OK()); + ASSERT_EQ(spaceMgr.NewBlock(block2), UC::Status::OK()); + ASSERT_EQ(spaceMgr.NewBlock(block3), UC::Status::OK()); + auto layout = spaceMgr.GetSpaceLayout(); + auto iter = layout->CreateFilePathIterator(); + size_t count = 0; + while (!layout->NextDataFilePath(iter).empty()) { count++; } + ASSERT_EQ(count, 0); + ASSERT_EQ(spaceMgr.CommitBlock(block1), UC::Status::OK()); + ASSERT_EQ(spaceMgr.CommitBlock(block2), UC::Status::OK()); + ASSERT_EQ(spaceMgr.CommitBlock(block3), UC::Status::OK()); + iter = layout->CreateFilePathIterator(); + count = 0; + while (!layout->NextDataFilePath(iter).empty()) { count++; } + ASSERT_EQ(count, 3); } \ No newline at end of file diff --git a/ucm/store/test/case/nfsstore/space_recycle_test.cc b/ucm/store/test/case/nfsstore/space_recycle_test.cc new file mode 100644 index 00000000..a9fb862f --- /dev/null +++ b/ucm/store/test/case/nfsstore/space_recycle_test.cc @@ -0,0 +1,124 @@ +/** + * MIT License + * + * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + * */ + +#include +#include "cmn/path_base.h" +#include "file/file.h" +#include "space/space_recycle.h" +#include "space/space_manager.h" +#include "thread/latch.h" + +namespace UC { + +void DoRecycle(const SpaceLayout* layout, const uint32_t recycleNum, + SpaceRecycle::RecycleOneBlockDone done); +} + +class UCSpaceRecycleTest : public UC::PathBase { +protected: + using OpenFlag = UC::IFile::OpenFlag; + using AccessMode = UC::IFile::AccessMode; + + void NewBlock(const UC::SpaceLayout* layout, const std::string& id) + { + std::string parent = layout->DataFileParent(id, false); + UC::File::MkDir(parent); + std::string path = layout->DataFilePath(id, false); + auto f = UC::File::Make(path); + f->Open(OpenFlag::CREATE | OpenFlag::READ_WRITE); + } + + bool ExistBlock(const UC::SpaceLayout* layout, const std::string& id) + { + std::string path = layout->DataFilePath(id, false); + return UC::File::Access(path, AccessMode::EXIST).Success(); + } + + void UpdateBlock(const UC::SpaceLayout* layout, const std::string& id) + { + struct utimbuf newTime; + auto tp = time(nullptr) + 3600; + newTime.modtime = tp; + newTime.actime = tp; + std::string path = layout->DataFilePath(id, false); + utime(path.c_str(), &newTime); + } +}; + +TEST_F(UCSpaceRecycleTest, TriggerRecycle) +{ + size_t blocksize = 1024 * 1024; + UC::SpaceManager spaceMgr; + ASSERT_EQ(spaceMgr.Setup({this->Path()}, blocksize, false, blocksize * 5), UC::Status::OK()); + const UC::SpaceLayout* layout = spaceMgr.GetSpaceLayout(); + std::string block1 = "a1b2c3d4e5f6789012345678901234ab"; + NewBlock(layout, block1); + ASSERT_TRUE(ExistBlock(layout, block1)); + + std::string block2 = "a2b2c3d4e5f6789012345678901234ab"; + NewBlock(layout, block2); + ASSERT_TRUE(ExistBlock(layout, block2)); + + UpdateBlock(layout, block1); + UC::SpaceRecycle recycle; + UC::Latch waiter{1}; + + ASSERT_TRUE(recycle.Setup(layout, 10, [&waiter] { waiter.Done([]{}); }).Success()); + recycle.Trigger(); + waiter.Wait(); + EXPECT_TRUE(ExistBlock(layout, block1)); + EXPECT_FALSE(ExistBlock(layout, block2)); +} + +TEST_F(UCSpaceRecycleTest, DoRecycle) +{ + size_t blocksize = 1024 * 1024; + UC::SpaceManager spaceMgr; + ASSERT_EQ(spaceMgr.Setup({this->Path()}, blocksize, false, blocksize * 5), UC::Status::OK()); + const UC::SpaceLayout* layout = spaceMgr.GetSpaceLayout(); + std::string recycleBlocks[] = { + "a1b2c3d4e5f6789012345678901234ab", + "a2b2c3d4e5f6789012345678901234ab", + "a3b2c3d4e5f6789012345678901234ab" + }; + std::string remainBlocks[] = { + "b1b2c3d4e5f6789012345678901234ab", + "b2b2c3d4e5f6789012345678901234ab", + "b3b2c3d4e5f6789012345678901234ab" + }; + for (auto &id: remainBlocks) + { + NewBlock(layout, id); + ASSERT_TRUE(ExistBlock(layout, id)); + } + for (auto &id: recycleBlocks) + { + NewBlock(layout, id); + ASSERT_TRUE(ExistBlock(layout, id)); + } + for (auto &id: remainBlocks) { UpdateBlock(layout, id); } + UC::DoRecycle(layout, 3, nullptr); + for (auto &id: remainBlocks) { EXPECT_TRUE(ExistBlock(layout, id)); } + for (auto &id: recycleBlocks) { EXPECT_FALSE(ExistBlock(layout, id)); } +} \ No newline at end of file