diff --git a/CMakeLists.txt b/CMakeLists.txt index d93bd1b8..ec57f499 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -36,7 +36,7 @@ add_definitions(-DBOOST_ERROR_CODE_HEADER_ONLY) # todo: remove -Wno-implicit-fallthrough once CI moves past gcc 7.4.0... set(warnings "-Wno-deprecated-declarations -Wall -Wextra -Werror -Wpedantic -Wno-implicit-fallthrough") if (APPLE) - set(warnings "${warnings} -Wno-extended-offsetof") + set(warnings "${warnings} -Wno-invalid-offsetof") else() # for beast and gcc release builds... if("${CMAKE_BUILD_TYPE}" STREQUAL "Release") @@ -98,6 +98,7 @@ add_subdirectory(chaos) add_subdirectory(crypto) add_subdirectory(monitor) add_subdirectory(mocks) +add_subdirectory(policy) include(cmake/static_analysis.cmake) diff --git a/crud/CMakeLists.txt b/crud/CMakeLists.txt index 31c79dbd..df6afbae 100644 --- a/crud/CMakeLists.txt +++ b/crud/CMakeLists.txt @@ -7,7 +7,7 @@ add_library(crud STATIC crud.hpp ) -target_link_libraries(crud proto) +target_link_libraries(crud proto policy) add_dependencies(crud boost jsoncpp openssl) target_include_directories(crud PRIVATE ${BLUZELLE_STD_INCLUDES}) add_subdirectory(test) diff --git a/crud/crud.cpp b/crud/crud.cpp index 2e94ec87..685d4e80 100644 --- a/crud/crud.cpp +++ b/crud/crud.cpp @@ -13,6 +13,8 @@ // along with this program. If not, see . #include +#include +#include #include #include #include @@ -210,6 +212,14 @@ crud::handle_create(const bzn::caller_id_t& caller_id, const database_msg& reque } else { + // bail on key value pairs that are too large right away! + if (this->max_database_size(perms) && request.create().key().length() + request.create().value().length() > this->max_database_size(perms)) + { + this->send_response(request, bzn::storage_result::value_too_large, database_response(), session); + + return; + } + if (this->expired(request.header().db_uuid(), request.create().key())) { this->send_response(request, bzn::storage_result::delete_pending, database_response(), session); @@ -219,16 +229,7 @@ crud::handle_create(const bzn::caller_id_t& caller_id, const database_msg& reque if (this->operation_exceeds_available_space(request, perms)) { - const auto key_value_size{request.create().key().length() + request.create().value().length()}; - - // Bail if the size of the key/value pair is larger than the database! If not, then check for a cache - // replacement policy - if (this->uses_random_eviction_policy(perms) && key_value_size < this->max_database_size(perms)) - { - // TODO: at present there is only one policy, later refactor this to use the strategy pattern - this->random_cache_replacement(request, key_value_size, this->max_database_size(perms)); - } - else + if (!this->do_eviction(request, this->max_database_size(perms))) { this->send_response(request, bzn::storage_result::db_full, database_response(), session); @@ -313,6 +314,14 @@ crud::handle_update(const bzn::caller_id_t& caller_id, const database_msg& reque } else { + // bail on key value pairs that are too large right away! + if (this->max_database_size(perms) && request.create().key().length() + request.create().value().length() > this->max_database_size(perms)) + { + this->send_response(request, bzn::storage_result::value_too_large, database_response(), session); + + return; + } + // expired? if (this->expired(request.header().db_uuid(), request.update().key())) { @@ -323,19 +332,8 @@ crud::handle_update(const bzn::caller_id_t& caller_id, const database_msg& reque if (this->operation_exceeds_available_space(request, perms)) { - // since this is an update, we do not change the key, so it's length can be ignored. - const auto key_value_size{request.update().value().length()}; - - // Bail if the size of the key/value pair is larger than the database! If not, then check for a cache - // replacement policy - if (this->uses_random_eviction_policy(perms) && key_value_size < this->max_database_size(perms)) - { - // TODO: at present there is only one policy, later refactor this to use the strategy pattern - // Since this is an update, we need to ensure that the key we are updating does not get randomly - // selected for eviction, so we need to tell the policy what key that is. - this->random_cache_replacement(request, key_value_size, this->max_database_size(perms), request.update().key()); - } - else + // let's try evicting some key/value pairs + if (!this->do_eviction(request, this->max_database_size(perms))) { this->send_response(request, bzn::storage_result::db_full, database_response(), session); @@ -995,10 +993,21 @@ crud::is_caller_a_writer(const bzn::caller_id_t& caller_id, const Json::Value& p } -bool -crud::uses_random_eviction_policy(const Json::Value& perms) const +std::shared_ptr +crud::get_eviction_policy(const Json::Value& perms) { - return perms[EVICTION_POLICY_KEY] == database_create_db::RANDOM; + // TODO: As we add more policies we may want to turn this into the strategy pattern and use + // a registry based approach here + if (perms[EVICTION_POLICY_KEY] == database_create_db::RANDOM) + { + return std::make_shared(this->storage); + } + else if (perms[EVICTION_POLICY_KEY] == database_create_db::VOLATILE_TTL) + { + return std::make_shared(this->storage); + } + + return nullptr; } @@ -1103,7 +1112,7 @@ crud::update_expiration_entry(const bzn::key_t& generated_key, uint64_t expire) if (result == bzn::storage_result::ok) { - LOG(debug) << "created ttl entry for: " << generated_key; + LOG(debug) << "created ttl entry [" << expires << "] for: " << generated_key; return; } @@ -1335,46 +1344,28 @@ crud::get_swarm_storage_usage() } -size_t -crud::evict_key(const bzn::uuid_t& db_uuid, const bzn::key_t& key) -{ - const auto pair_size = this->storage->get_key_size(db_uuid, key); - if (pair_size) - { - return this->storage->remove(db_uuid, key) == bzn::storage_result::ok ? *pair_size : 0; - } - return 0; -} - - -void -crud::random_cache_replacement(const database_msg& request, size_t key_value_size, size_t max_size, const bzn::key_t& key_exception) +bool +crud::do_eviction(const database_msg& request, size_t max_size) { - const auto [keys, size]{this->storage->get_size(request.header().db_uuid())}; - - uint64_t storage_to_free = key_value_size - (max_size - size); - - // We may need to remove one or more key/value pairs to make room for the new one - std::hash hasher; - size_t random_seed = hasher(request.header().request_hash()); - - boost::random::mt19937 mt(random_seed); - const boost::random::uniform_int_distribution<> dist(0, keys - 1); - - std::vector keys_to_evict; - - const auto available_keys = this->storage->get_keys(request.header().db_uuid()); - while (storage_to_free) + const auto PERMS{this->get_database_permissions(request.header().db_uuid()).second}; + if (auto eviction_policy = this->get_eviction_policy(PERMS)) { - const auto key_index = dist(mt); - const auto key_to_evict = this->storage->get_keys(request.header().db_uuid())[key_index]; - if ((key_to_evict != key_exception) && (keys_to_evict.end() == std::find(keys_to_evict.begin(), keys_to_evict.end(), key_to_evict))) + auto keys_to_evict {eviction_policy->keys_to_evict(request, max_size)}; + if (keys_to_evict.empty()) { - const size_t evicted_size = evict_key(request.header().db_uuid(), key_to_evict); - storage_to_free -= evicted_size < storage_to_free ? evicted_size : storage_to_free; - keys_to_evict.emplace_back(key_to_evict); + return false; } + + std::for_each( keys_to_evict.begin(), keys_to_evict.end(), + [&](const auto& key) + { + this->storage->remove(request.header().db_uuid(), key); + }); + + return true; } + + return false; } diff --git a/crud/crud.hpp b/crud/crud.hpp index e450f5d6..27df6092 100644 --- a/crud/crud.hpp +++ b/crud/crud.hpp @@ -21,19 +21,20 @@ #include #include #include +#include #include #include #include #include - namespace bzn { class crud final : public bzn::crud_base, public bzn::status_provider_base, public std::enable_shared_from_this { public: - crud(std::shared_ptr io_context, std::shared_ptr storage, std::shared_ptr subscription_manager, - std::shared_ptr node, bzn::key_t owner_public_key = ""); + crud(std::shared_ptr io_context, std::shared_ptr storage + , std::shared_ptr subscription_manager + , std::shared_ptr node, bzn::key_t owner_public_key = ""); void handle_request(const bzn::caller_id_t& caller_id, const database_msg& request, std::shared_ptr session) override; @@ -82,7 +83,6 @@ namespace bzn bool is_caller_a_writer(const bzn::caller_id_t& caller_id, const Json::Value& perms) const; void add_writers(const database_msg& request, Json::Value& perms); void remove_writers(const database_msg& request, Json::Value& perms); - bool uses_random_eviction_policy(const Json::Value& perms) const; uint64_t max_database_size(const Json::Value& perms) const; bool operation_exceeds_available_space(const database_msg& request, const Json::Value& perms); @@ -95,8 +95,8 @@ namespace bzn std::optional get_ttl(const bzn::uuid_t& uuid, const bzn::key_t& key) const; // cache replacement policy - size_t evict_key(const bzn::uuid_t& db_uuid, const bzn::key_t& key); - void random_cache_replacement(const database_msg& request, size_t key_value_size, size_t max_size, const bzn::key_t& key_exception = ""); + std::shared_ptr get_eviction_policy(const Json::Value& perms); + bool do_eviction(const database_msg& request, size_t max_size); std::shared_ptr storage; std::shared_ptr subscription_manager; diff --git a/crud/crud_base.hpp b/crud/crud_base.hpp index e5715470..5a9f7840 100644 --- a/crud/crud_base.hpp +++ b/crud/crud_base.hpp @@ -18,7 +18,6 @@ #include #include - namespace bzn { class pbft_base; diff --git a/crud/test/crud_test.cpp b/crud/test/crud_test.cpp index adf8c6fe..2cc74fae 100644 --- a/crud/test/crud_test.cpp +++ b/crud/test/crud_test.cpp @@ -167,21 +167,31 @@ namespace } database_msg - build_create_msg(const bzn::uuid_t &caller_uuid, const bzn::uuid_t &db_uuid, uint64_t nonce, const std::string& request_hash, const bzn::key_t &key, const bzn::value_t &value) + build_create_msg(const bzn::uuid_t &caller_uuid, const bzn::uuid_t &db_uuid, uint64_t nonce, const std::string& request_hash, const bzn::key_t &key, const bzn::value_t &value, uint64_t expire = 0) { database_msg msg {build_header_msg(caller_uuid, db_uuid, nonce, request_hash)}; msg.mutable_create()->set_key(key); msg.mutable_create()->set_value(value); + if (expire) + { + msg.mutable_create()->set_expire(expire); + } + return msg; } database_msg - build_update_msg(const bzn::uuid_t &caller_uuid, const bzn::uuid_t &db_uuid, uint64_t nonce, const std::string& request_hash, const bzn::key_t &key, const bzn::value_t &value) + build_update_msg(const bzn::uuid_t &caller_uuid, const bzn::uuid_t &db_uuid, uint64_t nonce, const std::string& request_hash, const bzn::key_t &key, const bzn::value_t &value, uint64_t expire = 0) { database_msg msg {build_header_msg(caller_uuid, db_uuid, nonce, request_hash)}; msg.mutable_header()->set_nonce(nonce); msg.mutable_update()->set_key(key); msg.mutable_update()->set_value(value); + if (expire) + { + msg.mutable_update()->set_expire(expire); + } + return msg; } @@ -257,9 +267,10 @@ namespace , const std::string& request_hash , const bzn::uuid_t& db , const std::string& key - , const std::string& value ) + , const std::string& value + , uint64_t expire = 0) { - database_msg msg{build_create_msg(caller, db, 123, request_hash, key, value)}; + database_msg msg{build_create_msg(caller, db, 123, request_hash, key, value, expire)}; // We are not testing create, so we can suppress the send_signed_message calls. EXPECT_CALL(*session, send_signed_message(_)); @@ -277,9 +288,10 @@ namespace , const std::string& request_hash , const bzn::uuid_t& db , const std::string& key - , const std::string& value ) + , const std::string& value + , uint64_t expire = 0) { - database_msg msg{build_update_msg(caller, db, 123, request_hash, key, value)}; + database_msg msg{build_update_msg(caller, db, 123, request_hash, key, value, expire)}; // We are not testing update, so we can suppress the send_signed_message calls. EXPECT_CALL(*session, send_signed_message(_)); @@ -391,7 +403,8 @@ namespace , const std::string request_hash , const bzn::uuid_t& db_uuid , size_t value_size - , size_t max_size) + , size_t max_size, + bool expires = false) { // We have a cache with random eviction and max size MAX_SIZE bytes, fill up the database to just under the // limit @@ -399,8 +412,14 @@ namespace const bzn::value_t VALUE{make_value(value_size)}; for (size_t index{0} ; index < ITEMS; ++index) { - create_key_value(crud, session, mock_node, caller_id, request_hash.empty() ? generate_random_hash() : request_hash, db_uuid, make_key(index), VALUE); + create_key_value( + crud, session, mock_node, caller_id + , request_hash.empty() ? generate_random_hash() : request_hash + , db_uuid, make_key(index), VALUE + , expires ? index * 1024 + 1024 : 0 + ); } + return ITEMS; } } @@ -2789,9 +2808,8 @@ TEST(crud, test_that_create_and_updates_which_exceed_db_limit_send_proper_respon msg.mutable_create()->set_key("key"); msg.mutable_create()->set_value("00000000"); // 1 too many (key+value size) - expect_signed_response(session, "uuid", uint64_t(123), database_response::kError, - bzn::storage_result_msg.at(bzn::storage_result::db_full)); - + expect_signed_response(session, "uuid", uint64_t(123), database_response::kError + , bzn::storage_result_msg.at(bzn::storage_result::value_too_large)); crud->handle_request("caller_id", msg, session); // create key... @@ -2817,6 +2835,7 @@ TEST(crud, test_that_create_and_updates_which_exceed_db_limit_send_proper_respon } +// TODO: RHN - Move the random eviction policy tests to the policy module TEST(crud, test_random_eviction_policy_randomly_removes_a_key_value_pair_for_create) { const size_t TEST_VALUE_SIZE{20}; @@ -2915,6 +2934,7 @@ TEST(crud, test_random_eviction_policy_with_large_value_requiring_many_evictions TEST(crud, test_random_eviction_policy_edge_case_of_create_with_value_larger_than_max_storage) { + // TODO: RHN - I think this test is not useful as it tests functionality that has nothing to do with eviction const uint64_t MAX_SIZE{8096}; const bzn::value_t TOO_LARGE_TEST_VALUE{make_value(MAX_SIZE)}; const bzn::uuid_t DB_UUID{"sut_uuid"}; @@ -2932,8 +2952,8 @@ TEST(crud, test_random_eviction_policy_edge_case_of_create_with_value_larger_tha database_msg msg{build_create_msg(CALLER_UUID, DB_UUID, NONCE, REQUEST_HASH, make_key(0), make_value(MAX_SIZE))}; - // In this case we expect a db_full error as the key/value pair is larger than the storage limit. - expect_signed_response(session, DB_UUID, NONCE, database_response::kError, bzn::storage_result_msg.at(bzn::storage_result::db_full)); + // In this case we expect a value_too_large error as the key/value pair is larger than the storage limit. + expect_signed_response(session, DB_UUID, NONCE, database_response::kError, bzn::storage_result_msg.at(bzn::storage_result::value_too_large)); EXPECT_CALL(*mock_node, send_signed_message(A(),_)); crud->handle_request(CALLER_UUID, msg, session); @@ -3040,31 +3060,104 @@ TEST(crud, test_random_eviction_policy_randomly_removes_many_key_value_pairs_for } -TEST(crud, test_random_eviction_policy_edge_case_of_update_with_value_larger_than_max_storage) +// These two eviction test should probably stay here as they are testing the eviction policies *and* crud +TEST(crud, test_that_two_cruds_evict_the_same_key_value_pairs_using_the_random_eviction_policy) { const size_t MAX_SIZE{8096}; - const bzn::key_t TEST_KEY{"test_key"}; - const bzn::value_t TOO_LARGE_TEST_VALUE{make_value(MAX_SIZE)}; const bzn::uuid_t DB_UUID{"sut_uuid"}; const bzn::uuid_t CALLER_UUID{"caller_id"}; + const size_t VALUE_SIZE{27}; + const std::string REQUEST_HASH{generate_random_hash()}; - std::shared_ptr session; - std::shared_ptr mock_node; + std::shared_ptr session_0; + std::shared_ptr mock_node_0; + auto crud_0{initialize_crud(session_0, mock_node_0, CALLER_UUID)}; - auto crud{initialize_crud(session, mock_node, CALLER_UUID)}; + std::shared_ptr session_1; + std::shared_ptr mock_node_1; + auto crud_1{initialize_crud(session_1, mock_node_1, CALLER_UUID)}; - remove_test_database(crud, session, mock_node, CALLER_UUID, DB_UUID); - create_test_database(crud, session, mock_node, CALLER_UUID, DB_UUID, MAX_SIZE, database_create_db_eviction_policy_type_RANDOM); + remove_test_database(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID); + create_test_database(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID, MAX_SIZE, database_create_db_eviction_policy_type_RANDOM); - create_key_value(crud, session, mock_node, CALLER_UUID, generate_random_hash(), DB_UUID, TEST_KEY, make_value(42)); + remove_test_database(crud_1, session_1, mock_node_1, CALLER_UUID, DB_UUID); + create_test_database(crud_1, session_1, mock_node_1, CALLER_UUID, DB_UUID, MAX_SIZE, database_create_db_eviction_policy_type_RANDOM); - update_key_value(crud, session, mock_node, CALLER_UUID, generate_random_hash(), DB_UUID, TEST_KEY, TOO_LARGE_TEST_VALUE); + // create a lot of keys to fill the database + const auto KEY_COUNT = fill_database(crud_0, session_0, mock_node_0, CALLER_UUID, REQUEST_HASH, DB_UUID, VALUE_SIZE, MAX_SIZE); + ASSERT_EQ(KEY_COUNT, fill_database(crud_1, session_1, mock_node_1, CALLER_UUID, REQUEST_HASH, DB_UUID, VALUE_SIZE, MAX_SIZE)); - remove_test_database(crud, session, mock_node, CALLER_UUID, DB_UUID); + const std::set pre_eviction_keys_0{get_database_keys(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID)}; + const std::set pre_eviction_keys_1{get_database_keys(crud_1, session_1, mock_node_1, CALLER_UUID, DB_UUID)}; + + EXPECT_EQ(pre_eviction_keys_0,pre_eviction_keys_1); + + // create a number of keys larger than the max size of the database + for(size_t index{KEY_COUNT}; index < KEY_COUNT + 100; ++index) + { + size_t db_size{0}; + std::tie(std::ignore, db_size) = get_database_size(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID); + const auto KEY = make_key(index); + const auto INNER_REQUEST_HASH{generate_random_hash()}; + + create_key_value(crud_0, session_0, mock_node_0, CALLER_UUID, INNER_REQUEST_HASH, DB_UUID, KEY, make_value(MAX_SIZE - db_size)); + create_key_value(crud_1, session_1, mock_node_1, CALLER_UUID, INNER_REQUEST_HASH, DB_UUID, KEY, make_value(MAX_SIZE - db_size)); + } + + const auto select_key_from_set = [](const auto& keys, uint64_t index) + { + std::set::const_iterator c_it(keys.begin()); + std::advance(c_it, index); + return *c_it; + }; + + // update a lot of keys with values that will exceed the database max storage + boost::random::mt19937 mt; + + // valgrind suppression needs this as there's different behaviour every run... + mt.seed(123); + + for(size_t i{0}; i < 20; ++i) + { + const std::set active_keys{get_database_keys(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID)}; + const boost::random::uniform_int_distribution<> dist(0, active_keys.size() - 1); + const bzn::key_t KEY{select_key_from_set(active_keys, dist(mt))}; + + size_t db_size{0}; + std::tie(std::ignore, db_size) = get_database_size(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID); + const auto VALUE{do_quickread(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID, KEY)}; + const auto NEW_VALUE_SIZE{MAX_SIZE - db_size + VALUE.length() + 1}; + + update_key_value(crud_0, session_0, mock_node_0, CALLER_UUID, REQUEST_HASH, DB_UUID, KEY, make_value(NEW_VALUE_SIZE)); + update_key_value(crud_1, session_1, mock_node_1, CALLER_UUID, REQUEST_HASH, DB_UUID, KEY, make_value(NEW_VALUE_SIZE)); + } + + // update a key with a value larger than the value of MAX_SIZE + { + const std::set active_keys{get_database_keys(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID)}; + const boost::random::uniform_int_distribution<> dist(0, active_keys.size() - 1); + const bzn::key_t KEY{select_key_from_set(active_keys, dist(mt))}; + update_key_value(crud_0, session_0, mock_node_0, CALLER_UUID, REQUEST_HASH, DB_UUID, KEY, make_value(MAX_SIZE)); + update_key_value(crud_1, session_1, mock_node_1, CALLER_UUID, REQUEST_HASH, DB_UUID, KEY, make_value(MAX_SIZE)); + } + + + // Are the databases in each crud the same? + const std::set post_eviction_keys_0{get_database_keys(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID)}; + const std::set post_eviction_keys_1{get_database_keys(crud_1, session_1, mock_node_1, CALLER_UUID, DB_UUID)}; + + std::set difference; + std::set_difference(pre_eviction_keys_0.begin(), pre_eviction_keys_0.end(), post_eviction_keys_0.begin(), post_eviction_keys_0.end(), std::inserter(difference, difference.begin())); + ASSERT_GT(difference.size(), size_t(0)); + + ASSERT_EQ(post_eviction_keys_0, post_eviction_keys_1); + + remove_test_database(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID); + remove_test_database(crud_1, session_1, mock_node_1, CALLER_UUID, DB_UUID); } -TEST(crud, test_that_two_cruds_evict_the_same_key_value_pairs) +TEST(crud, test_that_two_cruds_evict_the_same_key_value_pairs_using_the_volatile_ttl_eviction_policy) { const size_t MAX_SIZE{8096}; const bzn::uuid_t DB_UUID{"sut_uuid"}; @@ -3081,14 +3174,14 @@ TEST(crud, test_that_two_cruds_evict_the_same_key_value_pairs) auto crud_1{initialize_crud(session_1, mock_node_1, CALLER_UUID)}; remove_test_database(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID); - create_test_database(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID, MAX_SIZE, database_create_db_eviction_policy_type_RANDOM); + create_test_database(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID, MAX_SIZE, database_create_db_eviction_policy_type_VOLATILE_TTL); remove_test_database(crud_1, session_1, mock_node_1, CALLER_UUID, DB_UUID); - create_test_database(crud_1, session_1, mock_node_1, CALLER_UUID, DB_UUID, MAX_SIZE, database_create_db_eviction_policy_type_RANDOM); + create_test_database(crud_1, session_1, mock_node_1, CALLER_UUID, DB_UUID, MAX_SIZE, database_create_db_eviction_policy_type_VOLATILE_TTL); // create a lot of keys to fill the database - const auto KEY_COUNT = fill_database(crud_0, session_0, mock_node_0, CALLER_UUID, REQUEST_HASH, DB_UUID, VALUE_SIZE, MAX_SIZE); - ASSERT_EQ(KEY_COUNT, fill_database(crud_1, session_1, mock_node_1, CALLER_UUID, REQUEST_HASH, DB_UUID, VALUE_SIZE, MAX_SIZE)); + const auto KEY_COUNT = fill_database(crud_0, session_0, mock_node_0, CALLER_UUID, REQUEST_HASH, DB_UUID, VALUE_SIZE, MAX_SIZE, true); + ASSERT_EQ(KEY_COUNT, fill_database(crud_1, session_1, mock_node_1, CALLER_UUID, REQUEST_HASH, DB_UUID, VALUE_SIZE, MAX_SIZE, true)); const std::set pre_eviction_keys_0{get_database_keys(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID)}; const std::set pre_eviction_keys_1{get_database_keys(crud_1, session_1, mock_node_1, CALLER_UUID, DB_UUID)}; @@ -3103,8 +3196,11 @@ TEST(crud, test_that_two_cruds_evict_the_same_key_value_pairs) const auto KEY = make_key(index); const auto INNER_REQUEST_HASH{generate_random_hash()}; - create_key_value(crud_0, session_0, mock_node_0, CALLER_UUID, INNER_REQUEST_HASH, DB_UUID, KEY, make_value(MAX_SIZE - db_size)); - create_key_value(crud_1, session_1, mock_node_1, CALLER_UUID, INNER_REQUEST_HASH, DB_UUID, KEY, make_value(MAX_SIZE - db_size)); + const auto expire = 8 + index + 128; // Provide some expire values that are sortable + + const bzn::value_t VALUE{make_value(MAX_SIZE - db_size)}; + create_key_value(crud_0, session_0, mock_node_0, CALLER_UUID, INNER_REQUEST_HASH, DB_UUID, KEY, VALUE, expire); + create_key_value(crud_1, session_1, mock_node_1, CALLER_UUID, INNER_REQUEST_HASH, DB_UUID, KEY, VALUE, expire); } const auto select_key_from_set = [](const auto& keys, uint64_t index) @@ -3314,7 +3410,7 @@ TEST(crud, test_that_create_exceeding_max_swarm_storage_sends_proper_response) request.mutable_header()->set_db_uuid("uuid2"); request.mutable_header()->set_nonce(uint64_t(123)); request.mutable_update_db()->set_max_size(2048); - request.mutable_update_db()->set_eviction_policy(database_create_db::NONE); + request.mutable_update_db()->set_eviction_policy(database_create_db::VOLATILE_TTL); expect_signed_response(session, "uuid2", uint64_t(123), database_response::RESPONSE_NOT_SET); crud->handle_request("caller_id", request, session); diff --git a/policy/CMakeLists.txt b/policy/CMakeLists.txt new file mode 100644 index 00000000..6b2d1d1b --- /dev/null +++ b/policy/CMakeLists.txt @@ -0,0 +1,11 @@ +add_library(policy STATIC + eviction_base.hpp + random.cpp + volatile_ttl.cpp + ) + +add_dependencies(policy proto) + +target_include_directories(policy PRIVATE ${BLUZELLE_STD_INCLUDES}) + +add_subdirectory(test) diff --git a/policy/eviction_base.hpp b/policy/eviction_base.hpp new file mode 100644 index 00000000..ba814395 --- /dev/null +++ b/policy/eviction_base.hpp @@ -0,0 +1,36 @@ +// Copyright (C) 2018 Bluzelle +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License, version 3, +// as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#pragma once + +#include +#include +#include +#include + +namespace bzn::policy +{ + + class eviction_base + { + public: + eviction_base(std::shared_ptr storage) : storage{std::move(storage)} {} + + virtual ~eviction_base() = default; + + virtual std::set keys_to_evict(const database_msg& request, size_t max_size) = 0; + + std::shared_ptr storage; + }; +} // namespace bzn::crud::eviction diff --git a/policy/random.cpp b/policy/random.cpp new file mode 100644 index 00000000..0a409a3b --- /dev/null +++ b/policy/random.cpp @@ -0,0 +1,80 @@ +// Copyright (C) 2018 Bluzelle +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License, version 3, +// as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include +#include +#include +#include + +namespace bzn::policy +{ + std::set + random::keys_to_evict(const database_msg &request, size_t max_size) + { + const auto KEY_VALUE_SIZE{ + request.has_update() + ? request.update().value().size() + : request.create().key().size() + request.create().value().size() + }; + + const bzn::key_t IGNORE_KEY{ + request.has_update() + ? request.update().key() + : "" + }; + + const auto size{this->storage->get_size(request.header().db_uuid()).second}; + size_t storage_to_free{KEY_VALUE_SIZE - (max_size - size)}; + + // We may need to remove one or more key/value pairs to make room for the new one + std::hash hasher; + boost::random::mt19937 mt(hasher(request.header().request_hash())); + + std::vector keys_to_evict; + auto available_keys = this->storage->get_keys(request.header().db_uuid()); + while (storage_to_free && !available_keys.empty()) + { + // As we randomly select a key from the keys in the user's database, we *move* them from the vector of keys + // in the users' database to the vector of keys that need to be deleted. In the case of randomly selected + // the key that is being updated, we simnply remove that key from the vector of users' keys without putting + // it in the keys to delete. + const boost::random::uniform_int_distribution<> dist(0, available_keys.size() - 1); + const auto it = std::next(available_keys.begin(), dist(mt)); + if (*it != IGNORE_KEY) + { + const auto evicted_size = this->storage->get_key_size(request.header().db_uuid(), *it); + if (evicted_size.has_value()) + { + std::move(it, std::next(it,1), std::back_inserter(keys_to_evict)); + storage_to_free -= *evicted_size < storage_to_free ? *evicted_size : storage_to_free; + } + else + { + LOG(warning) << "While searching for keys to evict, the key " << *it << " was not found in the database " << request.header().db_uuid(); + } + } + available_keys.erase(it, std::next(it,1)); + } + + // Did we free enough storage? + if (!storage_to_free) + { + // It would have been nice to move the keys into the set directly, but std::move doesn't move from vector to set + + return std::set(keys_to_evict.begin(), keys_to_evict.end()); + } + + return std::set{}; + } +} // namespace bzn::crud::eviction diff --git a/policy/random.hpp b/policy/random.hpp new file mode 100644 index 00000000..788dc309 --- /dev/null +++ b/policy/random.hpp @@ -0,0 +1,30 @@ +// Copyright (C) 2018 Bluzelle +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License, version 3, +// as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#pragma once + +#include + +namespace bzn::policy +{ + class random : public eviction_base + { + public: + random(std::shared_ptr storage) : eviction_base{storage} + { + } + + std::set keys_to_evict(const database_msg& request, size_t max_size); + }; +} diff --git a/policy/test/CMakeLists.txt b/policy/test/CMakeLists.txt new file mode 100644 index 00000000..49d53bce --- /dev/null +++ b/policy/test/CMakeLists.txt @@ -0,0 +1,5 @@ +set(test_srcs eviction_test.cpp) + +set(test_libs policy proto storage ${Protobuf_LIBRARIES}) + +add_gmock_test(policy) diff --git a/policy/test/eviction_test.cpp b/policy/test/eviction_test.cpp new file mode 100644 index 00000000..ad86bbf1 --- /dev/null +++ b/policy/test/eviction_test.cpp @@ -0,0 +1,209 @@ +// Copyright (C) 2018 Bluzelle +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License, version 3, +// as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace ::testing; + +namespace +{ + const char* DB_UUID{"e6d16cab-cd0f-4af1-b45d-16de9bc2af5d"}; + //const char* CALLER_UUID{"9558fa26-ba03-4e24-98bc-1f2f562c4598"}; + // 2b3c2de4-5c14-4842-8284-b67eb09ec61d + // bf90f8d1-cb27-4217-a5c5-85c8e6565a25 + // 557bab19-8df2-4731-b74e-c2516e1f580f + + const std::string TTL_UUID{"TTL"}; + + database_msg + make_create_request(const bzn::uuid_t& db_uuid, const bzn::key_t key, const bzn::value_t& value) + { + database_msg request; + + auto header = new database_header(); + header->set_db_uuid(db_uuid); + + auto create = new database_create(); + create->set_key(key); + create->set_value(value); + + request.set_allocated_create(create); + request.set_allocated_header(header); + + return request; + } + + + database_msg + make_update_request(const bzn::uuid_t& db_uuid, const bzn::key_t key, const bzn::value_t& value) + { + database_msg request; + + auto header = new database_header(); + header->set_db_uuid(db_uuid); + + auto update = new database_update(); + update->set_key(key); + update->set_value(value); + + request.set_allocated_update(update); + request.set_allocated_header(header); + + return request; + } + + + // TODO: This is duplication of private code in storage, it may be a good idea to find a way + // to dry this up + bzn::key_t + generate_expire_key(const bzn::uuid_t& uuid, const bzn::key_t& key) + { + Json::Value value; + + value["uuid"] = uuid; + value["key"] = key; + + return value.toStyledString(); + } + + + size_t + insert_test_values(std::shared_ptr storage, const bzn::uuid_t& db_uuid, size_t number_of_items, size_t value_size=128) + { + for (size_t i{0}; icreate( db_uuid, key, std::string(value_size, 'B')); + } + + return storage->get_size(db_uuid).second; + } + + + void + insert_ttl_values(std::shared_ptr storage, const bzn::uuid_t& db_uuid, const std::vector keys) + { + size_t i{0}; + for (const auto& key : keys) + { + const auto expires = boost::lexical_cast( + std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count() + 1024 + i * 1024); + ++i; + storage->create(TTL_UUID, generate_expire_key(db_uuid,key), expires); + } + } +} + +namespace bzn +{ + TEST(policy_test, test_that_volatile_ttl_ignores_keys_with_no_ttl) + { + std::shared_ptr storage{std::make_shared()}; + const auto MAX_DB_SIZE{insert_test_values(storage, DB_UUID, 10) + 5}; + policy::volatile_ttl sut(storage); + + // performing a create on a db with no TTLs should return no keys to delete + { + const auto request{make_create_request(DB_UUID, "testvalue", std::string(256, 'C'))}; + const auto keys_to_delete{sut.keys_to_evict(request, MAX_DB_SIZE)}; + EXPECT_EQ(size_t(0), keys_to_delete.size()); + } + + // Update should behave the same way + { + const auto request{make_update_request(DB_UUID, "testvalue", std::string(256,'C'))}; + const auto keys_to_delete{sut.keys_to_evict(request, MAX_DB_SIZE)}; + EXPECT_EQ( size_t(0), keys_to_delete.size()); + } + } + + TEST(policy_test, test_that_volatile_ttl_returns_only_keys_with_ttl) + { + std::shared_ptr storage{std::make_shared()}; + + const size_t NUMBER_OF_ITEMS{10}; + const size_t VALUE_SIZE{128}; + const auto MAX_STORAGE{insert_test_values(storage, DB_UUID, NUMBER_OF_ITEMS, VALUE_SIZE)}; + + const bzn::key_t TEST_KEY{"key1"}; + std::vector keys{TEST_KEY}; + insert_ttl_values(storage, DB_UUID, keys); + + policy::volatile_ttl sut(storage); + + const bzn::value_t TEST_VALUE{std::string(64, 'B')}; + + // evict the one ttl key to make room for the created key value pair + { + auto request{make_create_request(DB_UUID, "KEY_CREATE", TEST_VALUE)}; + const auto keys_to_delete{sut.keys_to_evict(request, MAX_STORAGE)}; + EXPECT_EQ(size_t(1), keys_to_delete.size()); + EXPECT_EQ(TEST_KEY, *keys_to_delete.begin()); + } + + // evict the one ttl key to make room for the updated key value pair + { + auto request{make_update_request(DB_UUID, "KEY_UPDATE", TEST_VALUE)}; + const auto keys_to_delete{sut.keys_to_evict(request, MAX_STORAGE)}; + EXPECT_EQ(size_t(1), keys_to_delete.size()); + EXPECT_EQ(TEST_KEY, *keys_to_delete.begin()); + } + } + + TEST(policy_test, test_that_volatile_ttl_fails_if_there_are_not_enough_ttl_values_to_remove) + { + std::shared_ptr storage{std::make_shared()}; + + const size_t NUMBER_OF_ITEMS{10}; + + const size_t VALUE_SIZE{128}; + const auto MAX_STORAGE{insert_test_values(storage, DB_UUID, NUMBER_OF_ITEMS, VALUE_SIZE) + 5}; + + // set only one TTL value + const bzn::key_t TEST_KEY{"key1"}; + std::vector keys{TEST_KEY}; + insert_ttl_values(storage, DB_UUID, keys); + + policy::volatile_ttl sut(storage); + + // try creating a large value, this must fail as there is only one small key value pair with a TTL + // (return no keys to evict) + { + auto request{make_create_request(DB_UUID, "KEY_CREATE", std::string(2 * VALUE_SIZE, 'B'))}; + const auto keys_to_delete{sut.keys_to_evict(request, MAX_STORAGE)}; + EXPECT_EQ(size_t(0), keys_to_delete.size()); + } + + // try updating an existing value by doubling its size. This must fail (ibid) + { + auto request{make_update_request(DB_UUID, "KEY_CREATE", std::string(2 * VALUE_SIZE, 'B'))}; + const auto keys_to_delete{sut.keys_to_evict(request, MAX_STORAGE)}; + EXPECT_EQ(size_t(0), keys_to_delete.size()); + } + + // try updating the key with a TTL, this must fail (the only key with a ttl is the key we are updating.) + { + auto request{make_update_request(DB_UUID, TEST_KEY, std::string(2 * VALUE_SIZE, 'B'))}; + const auto keys_to_delete{sut.keys_to_evict(request, MAX_STORAGE)}; + EXPECT_EQ(size_t(0), keys_to_delete.size()); + } + } +} diff --git a/policy/volatile_ttl.cpp b/policy/volatile_ttl.cpp new file mode 100644 index 00000000..ce9e6368 --- /dev/null +++ b/policy/volatile_ttl.cpp @@ -0,0 +1,166 @@ +// Copyright (C) 2018 Bluzelle +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License, version 3, +// as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#include +#include + +namespace +{ + // TODO: This is a duplicate of code in crud, it would be a good idea to find a way to dry this up. + const std::string TTL_UUID{"TTL"}; +} + + +namespace bzn::policy +{ + struct ttl_item {bzn::key_t key; size_t value_size; size_t ttl;}; + + std::set + volatile_ttl::keys_to_evict(const database_msg &request, size_t max_db_size) + { + const auto db_uuid{request.header().db_uuid()}; + const auto key_to_ignore{request.has_update() ? request.update().key() : "" }; + + // get all the TTL's in storage + auto all_ttls{this->storage->get_keys(TTL_UUID)}; + + // get the TTL's associated with our database + // filter out the ones for this database, if update, ignore current key + auto our_ttls{this->filter_ttls(all_ttls, db_uuid, key_to_ignore)}; + if (our_ttls.empty()) + { + return {}; + } + + // Create a vector of [key, value size, ttl] + std::vector ttl_items{get_ttl_items(db_uuid, our_ttls)}; + + // 3: sort the keys in order of increasing ttl, secondarily sort on decreasing + // key value size + std::sort(ttl_items.begin(), ttl_items.end(), + [](const auto& a, const auto& b) + { + return a.value_size > b.value_size; + }); + + // 4: from the top of the list of keys pull out enough key value pairs + // to make room for the new pair or update, + const auto current_db_size{this->storage->get_size(request.header().db_uuid()).second}; + const auto current_free = (max_db_size - current_db_size) - (request.has_update() ? request.update().value().size() : 0); + const auto key_value_size { + request.has_update() + ? request.update().value().size() + : request.create().key().size() + request.create().value().size() + }; + + // how much room do we need to free? + // enough for the new key value pair: key_value_size + // How much room do we have? -> current_free + // How much do we need to free? -> required_size + std::set keys_to_evict; + auto required_size = key_value_size - current_free; + for (const auto ttl : ttl_items) + { + keys_to_evict.emplace(ttl.key); + required_size -= (ttl.value_size < required_size ? ttl.value_size : required_size); + if (!required_size) + { + break; + } + } + + // fail if we can't make enough room + if (required_size) + { + keys_to_evict.clear(); + } + + // 5: return the list of keys to delete + return keys_to_evict; + } + + + std::vector + volatile_ttl::get_ttl_items(const std::string &db_uuid, std::vector &our_ttls) const + { + std::vector ttl_items; + std::unique_ptr char_reader{ Json::CharReaderBuilder().newCharReader() }; + std::transform(our_ttls.begin(), our_ttls.end(), std::back_inserter(ttl_items), + [&](const auto& ttl)->ttl_item + { + Json::Value root; + std::string err; + if (char_reader->parse(ttl.c_str(), ttl.c_str() + ttl.size(), &root, &err)) + { + const auto key{root["key"].asString()}; + const auto expire_key{this->generate_expire_key(db_uuid, key)}; + const auto opt_expire{storage->read(TTL_UUID, expire_key)}; + const auto opt_value{storage->read(db_uuid, key)}; + + if (!opt_value.has_value()) + { + LOG (warning) << "item with key:[" << key << "] does not exist in db: [" << db_uuid << "]"; + + return ttl_item{"", 0 , 0}; + } + + if (!opt_expire.has_value()) + { + LOG (warning) << "TTL item with key:[" << expire_key << "] does not exist"; + + return ttl_item{"", 0, 0}; + } + + LOG (info) << "Found an item to evict: [" << key << "]" ; + + return ttl_item{ + key + , opt_value.value().size() + , boost::lexical_cast(opt_expire.value()) + }; + } + LOG(warning) << "Unable to parse TTL for Volatile TTL eviction: " << err; + + return ttl_item{"", 0 , 0}; + }); + + return ttl_items; + } + + + std::vector + volatile_ttl::filter_ttls(const std::vector& ttls, const bzn::uuid_t& db_uuid, const bzn::key_t& ignore_key) + { + std::vector filtered_ttls; + std::unique_ptr char_reader{ Json::CharReaderBuilder().newCharReader()}; + std::copy_if( + ttls.begin(), ttls.end(), std::back_inserter(filtered_ttls) + , [&char_reader, &db_uuid, &ignore_key](const auto& t) + { + Json::Value root; + std::string err; + if (char_reader->parse(t.c_str(), t.c_str() + t.size(), &root, &err)) + { + bool s{(root["key"] != ignore_key) && (root["uuid"].asString() == db_uuid)}; + + return s; + } + LOG(warning) << "Unable to parse TTL for Volatile TTL eviction: " << err; + + return false; + }); + + return filtered_ttls; + } +} // namespace bzn::crud::eviction diff --git a/policy/volatile_ttl.hpp b/policy/volatile_ttl.hpp new file mode 100644 index 00000000..79c798b2 --- /dev/null +++ b/policy/volatile_ttl.hpp @@ -0,0 +1,50 @@ +// Copyright (C) 2018 Bluzelle +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License, version 3, +// as published by the Free Software Foundation. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +#pragma once + +#include + +namespace bzn::policy +{ + struct ttl_item; + + class volatile_ttl : public eviction_base + { + public: + volatile_ttl(std::shared_ptr storage) : eviction_base(storage) + { + } + + std::set keys_to_evict(const database_msg& request, size_t max_size) override; + + private: + std::vector filter_ttls(const std::vector& ttls, const bzn::uuid_t& db_uuid, const bzn::key_t& ignore_key); + + std::vector get_ttl_items(const std::string& db_uuid, std::vector& our_ttls) const; + + + // TODO: This is a duplicate of a private crud method, it would be nice to find a way to dry this up, + // maybe move this function into a utility module + bzn::key_t generate_expire_key(const bzn::uuid_t& uuid, const bzn::key_t& key) const + { + Json::Value value; + + value["uuid"] = uuid; + value["key"] = key; + + return value.toStyledString(); + } + }; +} diff --git a/proto/database.proto b/proto/database.proto index 238dba2f..cf0fd4d5 100644 --- a/proto/database.proto +++ b/proto/database.proto @@ -69,6 +69,7 @@ message database_create_db { NONE = 0; RANDOM = 1; + VOLATILE_TTL = 2; } uint64 max_size = 1; diff --git a/utils/esr_peer_info.cpp b/utils/esr_peer_info.cpp index 3ecacb2a..44e70f83 100644 --- a/utils/esr_peer_info.cpp +++ b/utils/esr_peer_info.cpp @@ -41,8 +41,7 @@ namespace str_to_json(const std::string &json_str) { bzn::json_message json_msg; - Json::CharReaderBuilder builder; - Json::CharReader* reader = builder.newCharReader(); + std::unique_ptr reader{ Json::CharReaderBuilder().newCharReader() }; std::string errors; if(!reader->parse( json_str.c_str()