diff --git a/ucm/store/dramstore/CMakeLists.txt b/ucm/store/dramstore/CMakeLists.txt index 15295544..e69de29b 100644 --- a/ucm/store/dramstore/CMakeLists.txt +++ b/ucm/store/dramstore/CMakeLists.txt @@ -1,12 +0,0 @@ -file(GLOB_RECURSE UCMSTORE_DRAM_CC_SOURCE_FILES "./cc/*.cc") -add_library(dramstore STATIC ${UCMSTORE_DRAM_CC_SOURCE_FILES}) -target_include_directories(dramstore PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/cc/api - ${CMAKE_CURRENT_SOURCE_DIR}/cc/domain -) -target_link_libraries(dramstore PUBLIC storeinfra storedevice storetask) - -file(GLOB_RECURSE UCMSTORE_DRAM_CPY_SOURCE_FILES "./cpy/*.cc") -pybind11_add_module(ucmdramstore ${UCMSTORE_DRAM_CPY_SOURCE_FILES}) -target_link_libraries(ucmdramstore PRIVATE dramstore) -set_target_properties(ucmdramstore PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/ucm/store/dramstore/cc/api/dramstore.cc b/ucm/store/dramstore/cc/api/dramstore.cc deleted file mode 100644 index c59b7f2b..00000000 --- a/ucm/store/dramstore/cc/api/dramstore.cc +++ /dev/null @@ -1,104 +0,0 @@ -/** - * MIT License - * - * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * */ -#include "dramstore.h" -#include "logger/logger.h" -#include "status/status.h" -#include "trans/dram_trans_manager.h" -#include "memory/memory_pool.h" - -namespace UC { - -class DRAMStoreImpl : public DRAMStore { -public: - int32_t Setup(const Config& config) { - auto status = this->memPool_.Setup(config.deviceId, config.capacity, config.blockSize); - if (status.Failure()) { - UC_ERROR("Failed({}) to setup MemoryPool.", status); - return status.Underlying(); - } - status = this->transMgr_.Setup(config.deviceId, config.streamNumber, &this->memPool_, config.timeoutMs); - if (status.Failure()) { - UC_ERROR("Failed({}) to setup TsfTaskManager.", status); - return status.Underlying(); - } - return Status::OK().Underlying(); - } - int32_t Alloc(const std::string& block) override { return this->memPool_.NewBlock(block).Underlying(); } - bool Lookup(const std::string& block) override { return this->memPool_.LookupBlock(block); } - void Commit(const std::string& block, const bool success) override { this->memPool_.CommitBlock(block, success).Underlying(); } - std::list Alloc(const std::list& blocks) override - { - std::list results; - for (const auto &block : blocks) { - results.emplace_back(this->Alloc(block)); - } - return results; - } - std::list Lookup(const std::list& blocks) override - { - std::list founds; - for (const auto &block : blocks) { - founds.emplace_back(this->Lookup(block)); - } - return founds; - } - void Commit(const std::list& blocks, const bool success) override { - for (const auto &block : blocks) { - this->Commit(block, success); - } - } - size_t Submit(Task&& task) override { - auto taskId = Task::invalid; - auto status = this->transMgr_.Submit(std::move(task), taskId); - if (status.Failure()) { taskId = Task::invalid; } - return taskId; } - - int32_t Wait(const size_t task) override { - return this->transMgr_.Wait(task).Underlying(); - } - - int32_t Check(const size_t task, bool& finish) override { - return this->transMgr_.Check(task, finish).Underlying(); - } - - -private: - - DramTransManager transMgr_; - MemoryPool memPool_; - -}; - -int32_t DRAMStore::Setup(const Config& config) -{ - auto impl = new (std::nothrow) DRAMStoreImpl(); - if (!impl) { - UC_ERROR("Out of memory."); - return Status::OutOfMemory().Underlying(); - } - this->impl_ = impl; - return impl->Setup(config); -} - -} // namespace UC diff --git a/ucm/store/dramstore/cc/api/dramstore.h b/ucm/store/dramstore/cc/api/dramstore.h deleted file mode 100644 index 25d72612..00000000 --- a/ucm/store/dramstore/cc/api/dramstore.h +++ /dev/null @@ -1,83 +0,0 @@ -/** - * MIT License - * - * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * */ -#ifndef UNIFIEDCACHE_DRAMSTORE_H -#define UNIFIEDCACHE_DRAMSTORE_H - -#include "ucmstore.h" - -namespace UC { - -class DRAMStore : public CCStore<> { -public: - struct Config { - size_t capacity; - size_t blockSize; - int32_t deviceId; - size_t streamNumber; - size_t timeoutMs; - Config(const size_t capacity, const size_t blockSize, const int32_t deviceId, const size_t streamNumber, const size_t timeoutMs) - : capacity{capacity}, blockSize{blockSize}, deviceId{deviceId}, streamNumber{streamNumber}, timeoutMs{timeoutMs} - { - } - }; - -public: - DRAMStore() : impl_{nullptr} {} - ~DRAMStore() override - { - if (this->impl_) { delete this->impl_; } - } - int32_t Setup(const Config& config); - int32_t Alloc(const std::string& block) override { return this->impl_->Alloc(block); } - bool Lookup(const std::string& block) override { return this->impl_->Lookup(block); } - void Commit(const std::string& block, const bool success) override - { - this->impl_->Commit(block, success); - } - std::list Alloc(const std::list& blocks) override - { - return this->impl_->Alloc(blocks); - } - std::list Lookup(const std::list& blocks) override - { - return this->impl_->Lookup(blocks); - } - void Commit(const std::list& blocks, const bool success) override - { - this->impl_->Commit(blocks, success); - } - size_t Submit(Task&& task) override { return this->impl_->Submit(std::move(task)); } - int32_t Wait(const size_t task) override { return this->impl_->Wait(task); } - int32_t Check(const size_t task, bool& finish) override - { - return this->impl_->Check(task, finish); - } - -private: - DRAMStore* impl_; -}; - -} // namespace UC - -#endif diff --git a/ucm/store/dramstore/cc/domain/trans/dram_trans_manager.cc b/ucm/store/dramstore/cc/domain/trans/dram_trans_manager.cc deleted file mode 100644 index f9835612..00000000 --- a/ucm/store/dramstore/cc/domain/trans/dram_trans_manager.cc +++ /dev/null @@ -1,42 +0,0 @@ -/** - * MIT License - * - * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * */ - -#include "dram_trans_manager.h" - -namespace UC { - -Status DramTransManager::Setup(const int32_t deviceId, const size_t streamNumber, const MemoryPool* memPool, size_t timeoutMs) { - this->timeoutMs_ = timeoutMs; - auto status = Status::OK(); - for (size_t i = 0; i < streamNumber; i++) { - auto q = std::make_shared(); - status = - q->Setup(deviceId, &this->failureSet_, memPool, timeoutMs); - if (status.Failure()) { break; } - this->queues_.emplace_back(std::move(q)); - } - return status; -} - -} \ No newline at end of file diff --git a/ucm/store/dramstore/cc/domain/trans/dram_trans_manager.h b/ucm/store/dramstore/cc/domain/trans/dram_trans_manager.h deleted file mode 100644 index 7f9ef51b..00000000 --- a/ucm/store/dramstore/cc/domain/trans/dram_trans_manager.h +++ /dev/null @@ -1,39 +0,0 @@ -/** - * MIT License - * - * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * */ -#ifndef UNIFIEDCACHE_DRAM_TRANS_MANAGER_H -#define UNIFIEDCACHE_DRAM_TRANS_MANAGER_H - -#include "task_manager.h" -#include "dram_trans_queue.h" - -namespace UC { - -class DramTransManager : public TaskManager { -public: - Status Setup(const int32_t deviceId, const size_t streamNumber, const MemoryPool* memPool, size_t timeoutMs); -}; - -} // namespace UC - -#endif diff --git a/ucm/store/dramstore/cc/domain/trans/dram_trans_queue.cc b/ucm/store/dramstore/cc/domain/trans/dram_trans_queue.cc deleted file mode 100644 index cf7a3577..00000000 --- a/ucm/store/dramstore/cc/domain/trans/dram_trans_queue.cc +++ /dev/null @@ -1,126 +0,0 @@ -/** - * MIT License - * - * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * */ - -#include "dram_trans_queue.h" - -namespace UC { - -Status DramTransQueue::Setup(const int32_t deviceId, TaskSet* failureSet, - const MemoryPool* memPool, const size_t timeoutMs) { - this->deviceId_ = deviceId; - this->failureSet_ = failureSet; - this->memPool_ = memPool; - auto success = - this->backend_.SetWorkerInitFn([this](auto& device) { return this->Init(device); }) - .SetWorkerFn([this](auto& shards, const auto& device) { this->Work(shards, device); }) - .SetWorkerExitFn([this](auto& device) { this->Exit(device); }) - .Run(); - return success ? Status::OK() : Status::Error(); -} - -void DramTransQueue::Push(std::list& shards) noexcept { - this->backend_.Push(std::move(shards)); -} - -bool DramTransQueue::Init(Device& device) { - if (this->deviceId_ < 0) { return true; } - device = DeviceFactory::Make(this->deviceId_, 262144, 512); - if (!device) { - return false; - } - return device->Setup().Success(); -} - -void DramTransQueue::Exit(Device& device) { - device.reset(); -} - -void DramTransQueue::Work(std::list& shards, const Device& device) { - auto it = shards.begin(); - if (this->failureSet_->Contains(it->owner)) { - this->Done(shards, device, true); - } - auto status = Status::OK(); - if (it->type == Task::Type::DUMP) { - status = this->D2H(shards, device); - } else { - status = this->H2D(shards, device); - } - this->Done(shards, device, status.Success()); -} - -Status DramTransQueue::H2D(std::list& shards, const Device& device) { - size_t pool_offset = 0; - std::vector host_addrs(shards.size()); - std::vector device_addrs(shards.size()); - int shard_index = 0; - for (auto& shard : shards) { - bool found = this->memPool_->GetOffset(shard.block, &pool_offset); - if (!found) { - return Status::Error(); - } - auto host_addr = this->memPool_->GetStartAddr().get() + pool_offset + shard.offset; - auto device_addr = shard.address; - host_addrs[shard_index] = host_addr; - device_addrs[shard_index] = reinterpret_cast(device_addr); - shard_index++; - } - auto it = shards.begin(); - return device->H2DBatchSync(device_addrs.data(), const_cast(host_addrs.data()), shards.size(), it->length); -} - -Status DramTransQueue::D2H(std::list& shards, const Device& device) { - size_t pool_offset = 0; - std::vector host_addrs(shards.size()); - std::vector device_addrs(shards.size()); - int shard_index = 0; - for (auto& shard : shards) { - bool found = this->memPool_->GetOffset(shard.block, &pool_offset); - if (!found) { - return Status::Error(); - } - auto host_addr = this->memPool_->GetStartAddr().get() + pool_offset + shard.offset; - auto device_addr = shard.address; - host_addrs[shard_index] = host_addr; - device_addrs[shard_index] = reinterpret_cast(device_addr); - shard_index++; - } - auto it = shards.begin(); - return device->D2HBatchSync(host_addrs.data(), const_cast(device_addrs.data()), shards.size(), it->length); -} - -void DramTransQueue::Done(std::list& shards, const Device& device, const bool success) { - auto it = shards.begin(); - if (!success) { this->failureSet_->Insert(it->owner); } - for (auto& shard : shards) { - if (shard.done) { - if (device) { - if (device->Synchronized().Failure()) { this->failureSet_->Insert(shard.owner); } - } - shard.done(); - } - } -} - -} // namespace UC \ No newline at end of file diff --git a/ucm/store/dramstore/cc/domain/trans/dram_trans_queue.h b/ucm/store/dramstore/cc/domain/trans/dram_trans_queue.h deleted file mode 100644 index 72350709..00000000 --- a/ucm/store/dramstore/cc/domain/trans/dram_trans_queue.h +++ /dev/null @@ -1,61 +0,0 @@ -/** - * MIT License - * - * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * */ -#ifndef UNIFIEDCACHE_DRAM_TRANS_QUEUE_H -#define UNIFIEDCACHE_DRAM_TRANS_QUEUE_H - -#include "device/idevice.h" -#include "status/status.h" -#include "task_queue.h" -#include "task_set.h" -#include "thread/thread_pool.h" -#include "memory/memory_pool.h" - -namespace UC { - -class DramTransQueue : public TaskQueue { - using Device = std::unique_ptr; - int32_t deviceId_{-1}; - TaskSet* failureSet_{nullptr}; - const MemoryPool* memPool_{nullptr}; - ThreadPool, Device> backend_{}; - -public: - Status Setup(const int32_t deviceId, - TaskSet* failureSet, - const MemoryPool* memPool, - const size_t timeoutMs); - void Push(std::list& shards) noexcept override; - -private: - bool Init(Device& device); - void Exit(Device& device); - void Work(std::list& shards, const Device& device); - void Done(std::list& shards, const Device& device, const bool success); - Status H2D(std::list& shards, const Device& device); - Status D2H(std::list& shards, const Device& device); -}; - -} // namespace UC - -#endif diff --git a/ucm/store/dramstore/cpy/dramstore.py.cc b/ucm/store/dramstore/cpy/dramstore.py.cc deleted file mode 100644 index cb76d5d1..00000000 --- a/ucm/store/dramstore/cpy/dramstore.py.cc +++ /dev/null @@ -1,123 +0,0 @@ -/** - * MIT License - * - * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * */ -#include "dramstore.h" -#include - -namespace py = pybind11; - -namespace UC { - -class DRAMStorePy : public DRAMStore { -public: - void* CCStoreImpl() { return this; } - py::list AllocBatch(const py::list& blocks) - { - py::list results; - for (auto& block : blocks) { results.append(this->Alloc(block.cast())); } - return results; - } - py::list LookupBatch(const py::list& blocks) - { - py::list founds; - for (auto& block : blocks) { founds.append(this->Lookup(block.cast())); } - return founds; - } - void CommitBatch(const py::list& blocks, const bool success) - { - for (auto& block : blocks) { this->Commit(block.cast(), success); } - } - py::tuple CheckPy(const size_t task) - { - auto finish = false; - auto ret = this->Check(task, finish); - return py::make_tuple(ret, finish); - } - size_t Load(const py::list& blockIds, const py::list& offsets, const py::list& addresses, - const py::list& lengths) - { - return this->SubmitPy(blockIds, offsets, addresses, lengths, Task::Type::LOAD, - Task::Location::DEVICE, "DRAM::H2D"); - } - size_t Dump(const py::list& blockIds, const py::list& offsets, const py::list& addresses, - const py::list& lengths) - { - return this->SubmitPy(blockIds, offsets, addresses, lengths, Task::Type::DUMP, - Task::Location::DEVICE, "DRAM::D2H"); - } - -private: - size_t SubmitPy(const py::list& blockIds, const py::list& offsets, const py::list& addresses, - const py::list& lengths, Task::Type&& type, Task::Location&& location, - std::string&& brief) - { - Task task{std::move(type), std::move(location), std::move(brief)}; - auto blockId = blockIds.begin(); - auto offset = offsets.begin(); - auto address = addresses.begin(); - auto length = lengths.begin(); - while ((blockId != blockIds.end()) && (offset != offsets.end()) && - (address != addresses.end()) && (length != lengths.end())) { - task.Append(blockId->cast(), offset->cast(), - address->cast(), length->cast()); - blockId++; - offset++; - address++; - length++; - } - return this->Submit(std::move(task)); - } -}; - -} // namespace UC - -PYBIND11_MODULE(ucmdramstore, module) -{ - module.attr("project") = UCM_PROJECT_NAME; - module.attr("version") = UCM_PROJECT_VERSION; - module.attr("commit_id") = UCM_COMMIT_ID; - module.attr("build_type") = UCM_BUILD_TYPE; - auto store = py::class_(module, "DRAMStore"); - auto config = py::class_(store, "Config"); - config.def(py::init(), - py::arg("capacity"), py::arg("blockSize"), py::arg("deviceId"), py::arg("streamNumber"), py::arg("timeoutMs")); - config.def_readwrite("capacity", &UC::DRAMStorePy::Config::capacity); - config.def_readwrite("blockSize", &UC::DRAMStorePy::Config::blockSize); - config.def_readwrite("deviceId", &UC::DRAMStorePy::Config::deviceId); - config.def_readwrite("streamNumber", &UC::DRAMStorePy::Config::streamNumber); - config.def_readwrite("timeoutMs", &UC::DRAMStorePy::Config::timeoutMs); - store.def(py::init<>()); - store.def("CCStoreImpl", &UC::DRAMStorePy::CCStoreImpl); - store.def("Setup", &UC::DRAMStorePy::Setup); - store.def("Alloc", py::overload_cast(&UC::DRAMStorePy::Alloc)); - store.def("AllocBatch", &UC::DRAMStorePy::AllocBatch); - store.def("Lookup", py::overload_cast(&UC::DRAMStorePy::Lookup)); - store.def("LookupBatch", &UC::DRAMStorePy::LookupBatch); - store.def("Load", &UC::DRAMStorePy::Load); - store.def("Dump", &UC::DRAMStorePy::Dump); - store.def("Wait", &UC::DRAMStorePy::Wait); - store.def("Check", &UC::DRAMStorePy::Check); - store.def("Commit", - py::overload_cast(&UC::DRAMStorePy::Commit)); - store.def("CommitBatch", &UC::DRAMStorePy::CommitBatch); -} diff --git a/ucm/store/infra/memory/memory_pool.h b/ucm/store/infra/memory/memory_pool.h deleted file mode 100644 index 200d1286..00000000 --- a/ucm/store/infra/memory/memory_pool.h +++ /dev/null @@ -1,174 +0,0 @@ -/** - * MIT License - * - * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * */ -#ifndef UNIFIEDCACHE_MEMORY_POOL_H -#define UNIFIEDCACHE_MEMORY_POOL_H - -#include -#include -#include -#include -#include -#include -#include "status/status.h" -#include "device/idevice.h" -#include -#include -#include -#include "logger/logger.h" - -namespace UC { - -class MemoryPool { - - std::string DUMMY_SLOT_PREFIX{"__slot_"}; - using Device = std::unique_ptr; -public: - - Status Setup(int32_t deviceId, size_t capacity, size_t blockSize) { - capacity_ = capacity; - blockSize_ = blockSize; - device_ = DeviceFactory::Make(deviceId, blockSize, static_cast(capacity / blockSize)); - if (!device_) { - UC_ERROR("MemoryPool: failed to create device"); - return Status::Error(); - } - Status status = device_->Setup(); - if (!status.Success()) { - UC_ERROR("MemoryPool: failed to set up device"); - return Status::Error(); - } - pool_ = device_->GetBuffer(capacity_); - if (!pool_) { - UC_ERROR("MemoryPool: failed to get pool memory space"); - return Status::Error(); - } - - size_t slotNum = capacity_ / blockSize_; - for (size_t i = 0; i < slotNum; ++i) { - std::string dummy = DUMMY_SLOT_PREFIX + std::to_string(i); - size_t offset = i * blockSize_; - lruList_.push_front(dummy); - lruIndex_[dummy] = lruList_.begin(); - offsetMap_[dummy] = offset; - } - return Status::OK(); - - } - - Status NewBlock(const std::string& blockId) { - if (offsetMap_.count(blockId)) { - return Status::DuplicateKey(); - } - if (lruList_.empty()) { - // 所有空间里的块都正在写，那么就不能够分配 - return Status::Error(); - } - size_t offset = LRUEvictOne(); - offsetMap_[blockId] = offset; - return Status::OK(); - } - - bool LookupBlock(const std::string& blockId) const { - return availableBlocks_.count(blockId); - } - - bool GetOffset(const std::string& blockId, size_t* offset) const { - auto it = offsetMap_.find(blockId); - if (it == offsetMap_.end()) { - return false; - } - *offset = it->second; - return true; - } - - Status CommitBlock(const std::string& blockId, bool success) { - if (success) { - availableBlocks_.insert(blockId); - touchUnsafe(blockId); - } else { - resetSpaceOfBlock(blockId); - } - return Status::OK(); - } - - std::shared_ptr GetStartAddr() const { - return pool_; - } - -private: - std::shared_ptr pool_ = nullptr; - Device device_ = nullptr; - size_t capacity_; - size_t blockSize_; - - std::unordered_map offsetMap_; - std::set availableBlocks_; - - using ListType = std::list; - ListType lruList_; - std::unordered_map lruIndex_; - - void touchUnsafe(const std::string& blockId) { - auto it = lruIndex_.find(blockId); - if (it != lruIndex_.end()) { - lruList_.splice(lruList_.begin(), lruList_, it->second); - } - else { - lruList_.push_front(blockId); // 访问一次，该块就是最近使用了的，所以放到LRU队列的头部。这就是一般LRU的逻辑 - lruIndex_[blockId] = lruList_.begin(); - } - } - - size_t LRUEvictOne() { - const std::string& victim = lruList_.back(); - // 真实数据块，才从availableBlocks_中删掉 - if (victim.rfind(DUMMY_SLOT_PREFIX, 0) != 0) { - availableBlocks_.erase(victim); - } - size_t offset = offsetMap_[victim]; - offsetMap_.erase(victim); - lruIndex_.erase(victim); - lruList_.pop_back(); - return offset; - } - - void resetSpaceOfBlock(const std::string& blockId) { - auto it = offsetMap_.find(blockId); - size_t offset = it->second; - std::string dummy = DUMMY_SLOT_PREFIX + std::to_string(offset / blockSize_); - offsetMap_.erase(blockId); - - auto lit = lruIndex_.find(blockId); - if (lit != lruIndex_.end()) { - lruList_.erase(lit->second); - lruIndex_.erase(lit); - } - lruList_.push_back(dummy); // 将一个块commit false后，回收之前分配的内存，并且要将其放到LRU队列的尾部（下次可以写的时候，要马上就写。因为该块的优先级高于已经写了的块） - lruIndex_[dummy] = std::prev(lruList_.end()); - offsetMap_[dummy] = offset; - } -}; - -} // namespace UC -#endif \ No newline at end of file diff --git a/ucm/store/test/CMakeLists.txt b/ucm/store/test/CMakeLists.txt index 859c185c..0c4974ef 100644 --- a/ucm/store/test/CMakeLists.txt +++ b/ucm/store/test/CMakeLists.txt @@ -4,7 +4,7 @@ if(BUILD_UNIT_TESTS) add_executable(ucmstore.test ${UCMSTORE_TEST_SOURCE_FILES}) target_include_directories(ucmstore.test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/case) target_link_libraries(ucmstore.test PRIVATE - dramstore nfsstore localstore storeinfra storedevice + nfsstore localstore storeinfra storedevice gtest_main gtest mockcpp ) gtest_discover_tests(ucmstore.test) diff --git a/ucm/store/test/case/infra/mem_pool_test.cc b/ucm/store/test/case/infra/mem_pool_test.cc deleted file mode 100644 index f9ea0438..00000000 --- a/ucm/store/test/case/infra/mem_pool_test.cc +++ /dev/null @@ -1,169 +0,0 @@ -/** - * MIT License - * - * Copyright (c) 2025 Huawei Technologies Co., Ltd. All rights reserved. - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * */ - -#include "infra/memory/memory_pool.h" -#include - -class UCMemoryPoolTest : public ::testing::Test {}; - -TEST_F(UCMemoryPoolTest, NewBlockAllocateAndCommit) -{ - UC::MemoryPool memPool; // 初始化内存池 - ASSERT_EQ(memPool.Setup(-1, 10, 2), UC::Status::OK()); - const std::string block1 = "block1"; - size_t offset = 10; - ASSERT_FALSE(memPool.LookupBlock(block1)); - // ASSERT_EQ(memPool.GetOffset(block1), nullptr); - ASSERT_EQ(memPool.GetOffset(block1, &offset), false); - ASSERT_EQ(memPool.NewBlock(block1), UC::Status::OK()); - ASSERT_FALSE(memPool.LookupBlock(block1)); - // ASSERT_NE(memPool.GetOffset(block1), nullptr); - ASSERT_EQ(memPool.GetOffset(block1, &offset), true); - ASSERT_EQ(memPool.NewBlock(block1), UC::Status::DuplicateKey()); - ASSERT_EQ(memPool.CommitBlock(block1, true), UC::Status::OK()); - ASSERT_TRUE(memPool.LookupBlock(block1)); -} - -TEST_F(UCMemoryPoolTest, EvictOldBlock) -{ - UC::MemoryPool memPool; // 初始化内存池 - ASSERT_EQ(memPool.Setup(-1, 10, 5), UC::Status::OK()); - const std::string block1 = "block1"; - const std::string block2 = "block2"; - const std::string block3 = "block3"; - size_t offset = 10; - ASSERT_EQ(memPool.NewBlock(block1), UC::Status::OK()); - // ASSERT_NE(memPool.GetOffset(block1), nullptr); - ASSERT_EQ(memPool.GetOffset(block1, &offset), true); - ASSERT_EQ(memPool.NewBlock(block2), UC::Status::OK()); - // ASSERT_NE(memPool.GetOffset(block2), nullptr); - ASSERT_EQ(memPool.GetOffset(block2, &offset), true); - memPool.CommitBlock(block1, true); - memPool.CommitBlock(block2, true); - ASSERT_EQ(memPool.NewBlock(block3), UC::Status::OK()); - // ASSERT_NE(memPool.GetOffset(block3), nullptr); - ASSERT_EQ(memPool.GetOffset(block3, &offset), true); - // ASSERT_EQ(memPool.GetOffset(block1), nullptr); - ASSERT_EQ(memPool.GetOffset(block1, &offset), false); - // ASSERT_NE(memPool.GetOffset(block2), nullptr); - ASSERT_EQ(memPool.GetOffset(block2, &offset), true); - ASSERT_FALSE(memPool.LookupBlock(block1)); - ASSERT_TRUE(memPool.LookupBlock(block2)); - ASSERT_FALSE(memPool.LookupBlock(block3)); -} - -TEST_F(UCMemoryPoolTest, OldBlockCommitFalse) -{ - UC::MemoryPool memPool; // 初始化内存池 - ASSERT_EQ(memPool.Setup(-1, 32, 8), UC::Status::OK()); - const std::string block1 = "block1"; - const std::string block2 = "block2"; - const std::string block3 = "block3"; - const std::string block4 = "block4"; - const std::string block5 = "block5"; - size_t offset = 32; - ASSERT_EQ(memPool.NewBlock(block1), UC::Status::OK()); - // ASSERT_NE(memPool.GetOffset(block1), nullptr); - ASSERT_EQ(memPool.GetOffset(block1, &offset), true); - ASSERT_EQ(memPool.NewBlock(block2), UC::Status::OK()); - // ASSERT_NE(memPool.GetOffset(block2), nullptr); - ASSERT_EQ(memPool.GetOffset(block2, &offset), true); - ASSERT_EQ(memPool.NewBlock(block3), UC::Status::OK()); - // ASSERT_NE(memPool.GetOffset(block3), nullptr); - ASSERT_EQ(memPool.GetOffset(block3, &offset), true); - memPool.CommitBlock(block1, true); - memPool.CommitBlock(block2, false); - ASSERT_TRUE(memPool.LookupBlock(block1)); - ASSERT_FALSE(memPool.LookupBlock(block2)); - ASSERT_FALSE(memPool.LookupBlock(block3)); - ASSERT_EQ(memPool.NewBlock(block4), UC::Status::OK()); - // ASSERT_EQ(memPool.GetOffset(block4), 8); - ASSERT_EQ(memPool.GetOffset(block4, &offset), true); - ASSERT_EQ(offset, 8); - ASSERT_EQ(memPool.NewBlock(block5), UC::Status::OK()); - // ASSERT_EQ(memPool.GetOffset(block5), 24); - ASSERT_EQ(memPool.GetOffset(block5, &offset), true); - ASSERT_EQ(offset, 24); - memPool.CommitBlock(block3, true); - memPool.CommitBlock(block4, true); - memPool.CommitBlock(block5, true); - ASSERT_TRUE(memPool.LookupBlock(block1)); - ASSERT_FALSE(memPool.LookupBlock(block2)); - ASSERT_TRUE(memPool.LookupBlock(block3)); - ASSERT_TRUE(memPool.LookupBlock(block4)); - ASSERT_TRUE(memPool.LookupBlock(block5)); - - ASSERT_EQ(memPool.NewBlock(block1), UC::Status::DuplicateKey()); - ASSERT_EQ(memPool.NewBlock(block2), UC::Status::OK()); - // ASSERT_EQ(memPool.GetOffset(block2), 0); - ASSERT_EQ(memPool.GetOffset(block2, &offset), true); - ASSERT_EQ(offset, 0); - ASSERT_FALSE(memPool.LookupBlock(block1)); - ASSERT_FALSE(memPool.LookupBlock(block2)); - memPool.CommitBlock(block2, true); - ASSERT_TRUE(memPool.LookupBlock(block2)); -} - -TEST_F(UCMemoryPoolTest, NoCommittedBlock) -{ - UC::MemoryPool memPool; // 初始化内存池 - ASSERT_EQ(memPool.Setup(-1, 32, 8), UC::Status::OK()); - const std::string block1 = "block1"; - const std::string block2 = "block2"; - const std::string block3 = "block3"; - const std::string block4 = "block4"; - const std::string block5 = "block5"; - const std::string block6 = "block6"; - size_t offset = 32; - ASSERT_EQ(memPool.NewBlock(block1), UC::Status::OK()); - ASSERT_EQ(memPool.NewBlock(block2), UC::Status::OK()); - ASSERT_EQ(memPool.NewBlock(block3), UC::Status::OK()); - ASSERT_EQ(memPool.NewBlock(block4), UC::Status::OK()); - ASSERT_EQ(memPool.NewBlock(block5), UC::Status::Error()); - memPool.CommitBlock(block1, true); - ASSERT_TRUE(memPool.LookupBlock(block1)); - ASSERT_EQ(memPool.NewBlock(block5), UC::Status::OK()); - // ASSERT_EQ(memPool.GetOffset(block5), 0); - ASSERT_EQ(memPool.GetOffset(block5, &offset), true); - ASSERT_EQ(offset, 0); - ASSERT_FALSE(memPool.LookupBlock(block1)); - ASSERT_EQ(memPool.NewBlock(block6), UC::Status::Error()); - // ASSERT_EQ(memPool.GetOffset(block2), 8); - ASSERT_EQ(memPool.GetOffset(block2, &offset), true); - ASSERT_EQ(offset, 8); - memPool.CommitBlock(block2, false); - // ASSERT_EQ(memPool.GetOffset((block2)), nullptr); - ASSERT_EQ(memPool.GetOffset(block2, &offset), false); - ASSERT_FALSE(memPool.LookupBlock(block1)); - ASSERT_EQ(memPool.NewBlock(block6), UC::Status::OK()); - // ASSERT_EQ(memPool.GetOffset(block6), 8); - ASSERT_EQ(memPool.GetOffset(block6, &offset), true); - ASSERT_EQ(offset, 8); - ASSERT_FALSE(memPool.LookupBlock(block6)); - memPool.CommitBlock(block6, true); - ASSERT_TRUE(memPool.LookupBlock(block6)); - // ASSERT_EQ(memPool.GetOffset(block6), 8); - ASSERT_EQ(memPool.GetOffset(block6, &offset), true); - ASSERT_EQ(offset, 8); -} \ No newline at end of file