diff --git a/CMakeLists.txt b/CMakeLists.txt
index d93bd1b8..ec57f499 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -36,7 +36,7 @@ add_definitions(-DBOOST_ERROR_CODE_HEADER_ONLY)
# todo: remove -Wno-implicit-fallthrough once CI moves past gcc 7.4.0...
set(warnings "-Wno-deprecated-declarations -Wall -Wextra -Werror -Wpedantic -Wno-implicit-fallthrough")
if (APPLE)
- set(warnings "${warnings} -Wno-extended-offsetof")
+ set(warnings "${warnings} -Wno-invalid-offsetof")
else()
# for beast and gcc release builds...
if("${CMAKE_BUILD_TYPE}" STREQUAL "Release")
@@ -98,6 +98,7 @@ add_subdirectory(chaos)
add_subdirectory(crypto)
add_subdirectory(monitor)
add_subdirectory(mocks)
+add_subdirectory(policy)
include(cmake/static_analysis.cmake)
diff --git a/crud/CMakeLists.txt b/crud/CMakeLists.txt
index 31c79dbd..df6afbae 100644
--- a/crud/CMakeLists.txt
+++ b/crud/CMakeLists.txt
@@ -7,7 +7,7 @@ add_library(crud STATIC
crud.hpp
)
-target_link_libraries(crud proto)
+target_link_libraries(crud proto policy)
add_dependencies(crud boost jsoncpp openssl)
target_include_directories(crud PRIVATE ${BLUZELLE_STD_INCLUDES})
add_subdirectory(test)
diff --git a/crud/crud.cpp b/crud/crud.cpp
index 2e94ec87..685d4e80 100644
--- a/crud/crud.cpp
+++ b/crud/crud.cpp
@@ -13,6 +13,8 @@
// along with this program. If not, see .
#include
+#include
+#include
#include
#include
#include
@@ -210,6 +212,14 @@ crud::handle_create(const bzn::caller_id_t& caller_id, const database_msg& reque
}
else
{
+ // bail on key value pairs that are too large right away!
+ if (this->max_database_size(perms) && request.create().key().length() + request.create().value().length() > this->max_database_size(perms))
+ {
+ this->send_response(request, bzn::storage_result::value_too_large, database_response(), session);
+
+ return;
+ }
+
if (this->expired(request.header().db_uuid(), request.create().key()))
{
this->send_response(request, bzn::storage_result::delete_pending, database_response(), session);
@@ -219,16 +229,7 @@ crud::handle_create(const bzn::caller_id_t& caller_id, const database_msg& reque
if (this->operation_exceeds_available_space(request, perms))
{
- const auto key_value_size{request.create().key().length() + request.create().value().length()};
-
- // Bail if the size of the key/value pair is larger than the database! If not, then check for a cache
- // replacement policy
- if (this->uses_random_eviction_policy(perms) && key_value_size < this->max_database_size(perms))
- {
- // TODO: at present there is only one policy, later refactor this to use the strategy pattern
- this->random_cache_replacement(request, key_value_size, this->max_database_size(perms));
- }
- else
+ if (!this->do_eviction(request, this->max_database_size(perms)))
{
this->send_response(request, bzn::storage_result::db_full, database_response(), session);
@@ -313,6 +314,14 @@ crud::handle_update(const bzn::caller_id_t& caller_id, const database_msg& reque
}
else
{
+ // bail on key value pairs that are too large right away!
+ if (this->max_database_size(perms) && request.create().key().length() + request.create().value().length() > this->max_database_size(perms))
+ {
+ this->send_response(request, bzn::storage_result::value_too_large, database_response(), session);
+
+ return;
+ }
+
// expired?
if (this->expired(request.header().db_uuid(), request.update().key()))
{
@@ -323,19 +332,8 @@ crud::handle_update(const bzn::caller_id_t& caller_id, const database_msg& reque
if (this->operation_exceeds_available_space(request, perms))
{
- // since this is an update, we do not change the key, so it's length can be ignored.
- const auto key_value_size{request.update().value().length()};
-
- // Bail if the size of the key/value pair is larger than the database! If not, then check for a cache
- // replacement policy
- if (this->uses_random_eviction_policy(perms) && key_value_size < this->max_database_size(perms))
- {
- // TODO: at present there is only one policy, later refactor this to use the strategy pattern
- // Since this is an update, we need to ensure that the key we are updating does not get randomly
- // selected for eviction, so we need to tell the policy what key that is.
- this->random_cache_replacement(request, key_value_size, this->max_database_size(perms), request.update().key());
- }
- else
+ // let's try evicting some key/value pairs
+ if (!this->do_eviction(request, this->max_database_size(perms)))
{
this->send_response(request, bzn::storage_result::db_full, database_response(), session);
@@ -995,10 +993,21 @@ crud::is_caller_a_writer(const bzn::caller_id_t& caller_id, const Json::Value& p
}
-bool
-crud::uses_random_eviction_policy(const Json::Value& perms) const
+std::shared_ptr
+crud::get_eviction_policy(const Json::Value& perms)
{
- return perms[EVICTION_POLICY_KEY] == database_create_db::RANDOM;
+ // TODO: As we add more policies we may want to turn this into the strategy pattern and use
+ // a registry based approach here
+ if (perms[EVICTION_POLICY_KEY] == database_create_db::RANDOM)
+ {
+ return std::make_shared(this->storage);
+ }
+ else if (perms[EVICTION_POLICY_KEY] == database_create_db::VOLATILE_TTL)
+ {
+ return std::make_shared(this->storage);
+ }
+
+ return nullptr;
}
@@ -1103,7 +1112,7 @@ crud::update_expiration_entry(const bzn::key_t& generated_key, uint64_t expire)
if (result == bzn::storage_result::ok)
{
- LOG(debug) << "created ttl entry for: " << generated_key;
+ LOG(debug) << "created ttl entry [" << expires << "] for: " << generated_key;
return;
}
@@ -1335,46 +1344,28 @@ crud::get_swarm_storage_usage()
}
-size_t
-crud::evict_key(const bzn::uuid_t& db_uuid, const bzn::key_t& key)
-{
- const auto pair_size = this->storage->get_key_size(db_uuid, key);
- if (pair_size)
- {
- return this->storage->remove(db_uuid, key) == bzn::storage_result::ok ? *pair_size : 0;
- }
- return 0;
-}
-
-
-void
-crud::random_cache_replacement(const database_msg& request, size_t key_value_size, size_t max_size, const bzn::key_t& key_exception)
+bool
+crud::do_eviction(const database_msg& request, size_t max_size)
{
- const auto [keys, size]{this->storage->get_size(request.header().db_uuid())};
-
- uint64_t storage_to_free = key_value_size - (max_size - size);
-
- // We may need to remove one or more key/value pairs to make room for the new one
- std::hash hasher;
- size_t random_seed = hasher(request.header().request_hash());
-
- boost::random::mt19937 mt(random_seed);
- const boost::random::uniform_int_distribution<> dist(0, keys - 1);
-
- std::vector keys_to_evict;
-
- const auto available_keys = this->storage->get_keys(request.header().db_uuid());
- while (storage_to_free)
+ const auto PERMS{this->get_database_permissions(request.header().db_uuid()).second};
+ if (auto eviction_policy = this->get_eviction_policy(PERMS))
{
- const auto key_index = dist(mt);
- const auto key_to_evict = this->storage->get_keys(request.header().db_uuid())[key_index];
- if ((key_to_evict != key_exception) && (keys_to_evict.end() == std::find(keys_to_evict.begin(), keys_to_evict.end(), key_to_evict)))
+ auto keys_to_evict {eviction_policy->keys_to_evict(request, max_size)};
+ if (keys_to_evict.empty())
{
- const size_t evicted_size = evict_key(request.header().db_uuid(), key_to_evict);
- storage_to_free -= evicted_size < storage_to_free ? evicted_size : storage_to_free;
- keys_to_evict.emplace_back(key_to_evict);
+ return false;
}
+
+ std::for_each( keys_to_evict.begin(), keys_to_evict.end(),
+ [&](const auto& key)
+ {
+ this->storage->remove(request.header().db_uuid(), key);
+ });
+
+ return true;
}
+
+ return false;
}
diff --git a/crud/crud.hpp b/crud/crud.hpp
index e450f5d6..27df6092 100644
--- a/crud/crud.hpp
+++ b/crud/crud.hpp
@@ -21,19 +21,20 @@
#include
#include
#include
+#include
#include
#include
#include
#include
-
namespace bzn
{
class crud final : public bzn::crud_base, public bzn::status_provider_base, public std::enable_shared_from_this
{
public:
- crud(std::shared_ptr io_context, std::shared_ptr storage, std::shared_ptr subscription_manager,
- std::shared_ptr node, bzn::key_t owner_public_key = "");
+ crud(std::shared_ptr io_context, std::shared_ptr storage
+ , std::shared_ptr subscription_manager
+ , std::shared_ptr node, bzn::key_t owner_public_key = "");
void handle_request(const bzn::caller_id_t& caller_id, const database_msg& request, std::shared_ptr session) override;
@@ -82,7 +83,6 @@ namespace bzn
bool is_caller_a_writer(const bzn::caller_id_t& caller_id, const Json::Value& perms) const;
void add_writers(const database_msg& request, Json::Value& perms);
void remove_writers(const database_msg& request, Json::Value& perms);
- bool uses_random_eviction_policy(const Json::Value& perms) const;
uint64_t max_database_size(const Json::Value& perms) const;
bool operation_exceeds_available_space(const database_msg& request, const Json::Value& perms);
@@ -95,8 +95,8 @@ namespace bzn
std::optional get_ttl(const bzn::uuid_t& uuid, const bzn::key_t& key) const;
// cache replacement policy
- size_t evict_key(const bzn::uuid_t& db_uuid, const bzn::key_t& key);
- void random_cache_replacement(const database_msg& request, size_t key_value_size, size_t max_size, const bzn::key_t& key_exception = "");
+ std::shared_ptr get_eviction_policy(const Json::Value& perms);
+ bool do_eviction(const database_msg& request, size_t max_size);
std::shared_ptr storage;
std::shared_ptr subscription_manager;
diff --git a/crud/crud_base.hpp b/crud/crud_base.hpp
index e5715470..5a9f7840 100644
--- a/crud/crud_base.hpp
+++ b/crud/crud_base.hpp
@@ -18,7 +18,6 @@
#include
#include
-
namespace bzn
{
class pbft_base;
diff --git a/crud/test/crud_test.cpp b/crud/test/crud_test.cpp
index adf8c6fe..2cc74fae 100644
--- a/crud/test/crud_test.cpp
+++ b/crud/test/crud_test.cpp
@@ -167,21 +167,31 @@ namespace
}
database_msg
- build_create_msg(const bzn::uuid_t &caller_uuid, const bzn::uuid_t &db_uuid, uint64_t nonce, const std::string& request_hash, const bzn::key_t &key, const bzn::value_t &value)
+ build_create_msg(const bzn::uuid_t &caller_uuid, const bzn::uuid_t &db_uuid, uint64_t nonce, const std::string& request_hash, const bzn::key_t &key, const bzn::value_t &value, uint64_t expire = 0)
{
database_msg msg {build_header_msg(caller_uuid, db_uuid, nonce, request_hash)};
msg.mutable_create()->set_key(key);
msg.mutable_create()->set_value(value);
+ if (expire)
+ {
+ msg.mutable_create()->set_expire(expire);
+ }
+
return msg;
}
database_msg
- build_update_msg(const bzn::uuid_t &caller_uuid, const bzn::uuid_t &db_uuid, uint64_t nonce, const std::string& request_hash, const bzn::key_t &key, const bzn::value_t &value)
+ build_update_msg(const bzn::uuid_t &caller_uuid, const bzn::uuid_t &db_uuid, uint64_t nonce, const std::string& request_hash, const bzn::key_t &key, const bzn::value_t &value, uint64_t expire = 0)
{
database_msg msg {build_header_msg(caller_uuid, db_uuid, nonce, request_hash)};
msg.mutable_header()->set_nonce(nonce);
msg.mutable_update()->set_key(key);
msg.mutable_update()->set_value(value);
+ if (expire)
+ {
+ msg.mutable_update()->set_expire(expire);
+ }
+
return msg;
}
@@ -257,9 +267,10 @@ namespace
, const std::string& request_hash
, const bzn::uuid_t& db
, const std::string& key
- , const std::string& value )
+ , const std::string& value
+ , uint64_t expire = 0)
{
- database_msg msg{build_create_msg(caller, db, 123, request_hash, key, value)};
+ database_msg msg{build_create_msg(caller, db, 123, request_hash, key, value, expire)};
// We are not testing create, so we can suppress the send_signed_message calls.
EXPECT_CALL(*session, send_signed_message(_));
@@ -277,9 +288,10 @@ namespace
, const std::string& request_hash
, const bzn::uuid_t& db
, const std::string& key
- , const std::string& value )
+ , const std::string& value
+ , uint64_t expire = 0)
{
- database_msg msg{build_update_msg(caller, db, 123, request_hash, key, value)};
+ database_msg msg{build_update_msg(caller, db, 123, request_hash, key, value, expire)};
// We are not testing update, so we can suppress the send_signed_message calls.
EXPECT_CALL(*session, send_signed_message(_));
@@ -391,7 +403,8 @@ namespace
, const std::string request_hash
, const bzn::uuid_t& db_uuid
, size_t value_size
- , size_t max_size)
+ , size_t max_size,
+ bool expires = false)
{
// We have a cache with random eviction and max size MAX_SIZE bytes, fill up the database to just under the
// limit
@@ -399,8 +412,14 @@ namespace
const bzn::value_t VALUE{make_value(value_size)};
for (size_t index{0} ; index < ITEMS; ++index)
{
- create_key_value(crud, session, mock_node, caller_id, request_hash.empty() ? generate_random_hash() : request_hash, db_uuid, make_key(index), VALUE);
+ create_key_value(
+ crud, session, mock_node, caller_id
+ , request_hash.empty() ? generate_random_hash() : request_hash
+ , db_uuid, make_key(index), VALUE
+ , expires ? index * 1024 + 1024 : 0
+ );
}
+
return ITEMS;
}
}
@@ -2789,9 +2808,8 @@ TEST(crud, test_that_create_and_updates_which_exceed_db_limit_send_proper_respon
msg.mutable_create()->set_key("key");
msg.mutable_create()->set_value("00000000"); // 1 too many (key+value size)
- expect_signed_response(session, "uuid", uint64_t(123), database_response::kError,
- bzn::storage_result_msg.at(bzn::storage_result::db_full));
-
+ expect_signed_response(session, "uuid", uint64_t(123), database_response::kError
+ , bzn::storage_result_msg.at(bzn::storage_result::value_too_large));
crud->handle_request("caller_id", msg, session);
// create key...
@@ -2817,6 +2835,7 @@ TEST(crud, test_that_create_and_updates_which_exceed_db_limit_send_proper_respon
}
+// TODO: RHN - Move the random eviction policy tests to the policy module
TEST(crud, test_random_eviction_policy_randomly_removes_a_key_value_pair_for_create)
{
const size_t TEST_VALUE_SIZE{20};
@@ -2915,6 +2934,7 @@ TEST(crud, test_random_eviction_policy_with_large_value_requiring_many_evictions
TEST(crud, test_random_eviction_policy_edge_case_of_create_with_value_larger_than_max_storage)
{
+ // TODO: RHN - I think this test is not useful as it tests functionality that has nothing to do with eviction
const uint64_t MAX_SIZE{8096};
const bzn::value_t TOO_LARGE_TEST_VALUE{make_value(MAX_SIZE)};
const bzn::uuid_t DB_UUID{"sut_uuid"};
@@ -2932,8 +2952,8 @@ TEST(crud, test_random_eviction_policy_edge_case_of_create_with_value_larger_tha
database_msg msg{build_create_msg(CALLER_UUID, DB_UUID, NONCE, REQUEST_HASH, make_key(0), make_value(MAX_SIZE))};
- // In this case we expect a db_full error as the key/value pair is larger than the storage limit.
- expect_signed_response(session, DB_UUID, NONCE, database_response::kError, bzn::storage_result_msg.at(bzn::storage_result::db_full));
+ // In this case we expect a value_too_large error as the key/value pair is larger than the storage limit.
+ expect_signed_response(session, DB_UUID, NONCE, database_response::kError, bzn::storage_result_msg.at(bzn::storage_result::value_too_large));
EXPECT_CALL(*mock_node, send_signed_message(A(),_));
crud->handle_request(CALLER_UUID, msg, session);
@@ -3040,31 +3060,104 @@ TEST(crud, test_random_eviction_policy_randomly_removes_many_key_value_pairs_for
}
-TEST(crud, test_random_eviction_policy_edge_case_of_update_with_value_larger_than_max_storage)
+// These two eviction test should probably stay here as they are testing the eviction policies *and* crud
+TEST(crud, test_that_two_cruds_evict_the_same_key_value_pairs_using_the_random_eviction_policy)
{
const size_t MAX_SIZE{8096};
- const bzn::key_t TEST_KEY{"test_key"};
- const bzn::value_t TOO_LARGE_TEST_VALUE{make_value(MAX_SIZE)};
const bzn::uuid_t DB_UUID{"sut_uuid"};
const bzn::uuid_t CALLER_UUID{"caller_id"};
+ const size_t VALUE_SIZE{27};
+ const std::string REQUEST_HASH{generate_random_hash()};
- std::shared_ptr session;
- std::shared_ptr mock_node;
+ std::shared_ptr session_0;
+ std::shared_ptr mock_node_0;
+ auto crud_0{initialize_crud(session_0, mock_node_0, CALLER_UUID)};
- auto crud{initialize_crud(session, mock_node, CALLER_UUID)};
+ std::shared_ptr session_1;
+ std::shared_ptr mock_node_1;
+ auto crud_1{initialize_crud(session_1, mock_node_1, CALLER_UUID)};
- remove_test_database(crud, session, mock_node, CALLER_UUID, DB_UUID);
- create_test_database(crud, session, mock_node, CALLER_UUID, DB_UUID, MAX_SIZE, database_create_db_eviction_policy_type_RANDOM);
+ remove_test_database(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID);
+ create_test_database(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID, MAX_SIZE, database_create_db_eviction_policy_type_RANDOM);
- create_key_value(crud, session, mock_node, CALLER_UUID, generate_random_hash(), DB_UUID, TEST_KEY, make_value(42));
+ remove_test_database(crud_1, session_1, mock_node_1, CALLER_UUID, DB_UUID);
+ create_test_database(crud_1, session_1, mock_node_1, CALLER_UUID, DB_UUID, MAX_SIZE, database_create_db_eviction_policy_type_RANDOM);
- update_key_value(crud, session, mock_node, CALLER_UUID, generate_random_hash(), DB_UUID, TEST_KEY, TOO_LARGE_TEST_VALUE);
+ // create a lot of keys to fill the database
+ const auto KEY_COUNT = fill_database(crud_0, session_0, mock_node_0, CALLER_UUID, REQUEST_HASH, DB_UUID, VALUE_SIZE, MAX_SIZE);
+ ASSERT_EQ(KEY_COUNT, fill_database(crud_1, session_1, mock_node_1, CALLER_UUID, REQUEST_HASH, DB_UUID, VALUE_SIZE, MAX_SIZE));
- remove_test_database(crud, session, mock_node, CALLER_UUID, DB_UUID);
+ const std::set pre_eviction_keys_0{get_database_keys(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID)};
+ const std::set pre_eviction_keys_1{get_database_keys(crud_1, session_1, mock_node_1, CALLER_UUID, DB_UUID)};
+
+ EXPECT_EQ(pre_eviction_keys_0,pre_eviction_keys_1);
+
+ // create a number of keys larger than the max size of the database
+ for(size_t index{KEY_COUNT}; index < KEY_COUNT + 100; ++index)
+ {
+ size_t db_size{0};
+ std::tie(std::ignore, db_size) = get_database_size(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID);
+ const auto KEY = make_key(index);
+ const auto INNER_REQUEST_HASH{generate_random_hash()};
+
+ create_key_value(crud_0, session_0, mock_node_0, CALLER_UUID, INNER_REQUEST_HASH, DB_UUID, KEY, make_value(MAX_SIZE - db_size));
+ create_key_value(crud_1, session_1, mock_node_1, CALLER_UUID, INNER_REQUEST_HASH, DB_UUID, KEY, make_value(MAX_SIZE - db_size));
+ }
+
+ const auto select_key_from_set = [](const auto& keys, uint64_t index)
+ {
+ std::set::const_iterator c_it(keys.begin());
+ std::advance(c_it, index);
+ return *c_it;
+ };
+
+ // update a lot of keys with values that will exceed the database max storage
+ boost::random::mt19937 mt;
+
+ // valgrind suppression needs this as there's different behaviour every run...
+ mt.seed(123);
+
+ for(size_t i{0}; i < 20; ++i)
+ {
+ const std::set active_keys{get_database_keys(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID)};
+ const boost::random::uniform_int_distribution<> dist(0, active_keys.size() - 1);
+ const bzn::key_t KEY{select_key_from_set(active_keys, dist(mt))};
+
+ size_t db_size{0};
+ std::tie(std::ignore, db_size) = get_database_size(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID);
+ const auto VALUE{do_quickread(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID, KEY)};
+ const auto NEW_VALUE_SIZE{MAX_SIZE - db_size + VALUE.length() + 1};
+
+ update_key_value(crud_0, session_0, mock_node_0, CALLER_UUID, REQUEST_HASH, DB_UUID, KEY, make_value(NEW_VALUE_SIZE));
+ update_key_value(crud_1, session_1, mock_node_1, CALLER_UUID, REQUEST_HASH, DB_UUID, KEY, make_value(NEW_VALUE_SIZE));
+ }
+
+ // update a key with a value larger than the value of MAX_SIZE
+ {
+ const std::set active_keys{get_database_keys(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID)};
+ const boost::random::uniform_int_distribution<> dist(0, active_keys.size() - 1);
+ const bzn::key_t KEY{select_key_from_set(active_keys, dist(mt))};
+ update_key_value(crud_0, session_0, mock_node_0, CALLER_UUID, REQUEST_HASH, DB_UUID, KEY, make_value(MAX_SIZE));
+ update_key_value(crud_1, session_1, mock_node_1, CALLER_UUID, REQUEST_HASH, DB_UUID, KEY, make_value(MAX_SIZE));
+ }
+
+
+ // Are the databases in each crud the same?
+ const std::set post_eviction_keys_0{get_database_keys(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID)};
+ const std::set post_eviction_keys_1{get_database_keys(crud_1, session_1, mock_node_1, CALLER_UUID, DB_UUID)};
+
+ std::set difference;
+ std::set_difference(pre_eviction_keys_0.begin(), pre_eviction_keys_0.end(), post_eviction_keys_0.begin(), post_eviction_keys_0.end(), std::inserter(difference, difference.begin()));
+ ASSERT_GT(difference.size(), size_t(0));
+
+ ASSERT_EQ(post_eviction_keys_0, post_eviction_keys_1);
+
+ remove_test_database(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID);
+ remove_test_database(crud_1, session_1, mock_node_1, CALLER_UUID, DB_UUID);
}
-TEST(crud, test_that_two_cruds_evict_the_same_key_value_pairs)
+TEST(crud, test_that_two_cruds_evict_the_same_key_value_pairs_using_the_volatile_ttl_eviction_policy)
{
const size_t MAX_SIZE{8096};
const bzn::uuid_t DB_UUID{"sut_uuid"};
@@ -3081,14 +3174,14 @@ TEST(crud, test_that_two_cruds_evict_the_same_key_value_pairs)
auto crud_1{initialize_crud(session_1, mock_node_1, CALLER_UUID)};
remove_test_database(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID);
- create_test_database(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID, MAX_SIZE, database_create_db_eviction_policy_type_RANDOM);
+ create_test_database(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID, MAX_SIZE, database_create_db_eviction_policy_type_VOLATILE_TTL);
remove_test_database(crud_1, session_1, mock_node_1, CALLER_UUID, DB_UUID);
- create_test_database(crud_1, session_1, mock_node_1, CALLER_UUID, DB_UUID, MAX_SIZE, database_create_db_eviction_policy_type_RANDOM);
+ create_test_database(crud_1, session_1, mock_node_1, CALLER_UUID, DB_UUID, MAX_SIZE, database_create_db_eviction_policy_type_VOLATILE_TTL);
// create a lot of keys to fill the database
- const auto KEY_COUNT = fill_database(crud_0, session_0, mock_node_0, CALLER_UUID, REQUEST_HASH, DB_UUID, VALUE_SIZE, MAX_SIZE);
- ASSERT_EQ(KEY_COUNT, fill_database(crud_1, session_1, mock_node_1, CALLER_UUID, REQUEST_HASH, DB_UUID, VALUE_SIZE, MAX_SIZE));
+ const auto KEY_COUNT = fill_database(crud_0, session_0, mock_node_0, CALLER_UUID, REQUEST_HASH, DB_UUID, VALUE_SIZE, MAX_SIZE, true);
+ ASSERT_EQ(KEY_COUNT, fill_database(crud_1, session_1, mock_node_1, CALLER_UUID, REQUEST_HASH, DB_UUID, VALUE_SIZE, MAX_SIZE, true));
const std::set pre_eviction_keys_0{get_database_keys(crud_0, session_0, mock_node_0, CALLER_UUID, DB_UUID)};
const std::set pre_eviction_keys_1{get_database_keys(crud_1, session_1, mock_node_1, CALLER_UUID, DB_UUID)};
@@ -3103,8 +3196,11 @@ TEST(crud, test_that_two_cruds_evict_the_same_key_value_pairs)
const auto KEY = make_key(index);
const auto INNER_REQUEST_HASH{generate_random_hash()};
- create_key_value(crud_0, session_0, mock_node_0, CALLER_UUID, INNER_REQUEST_HASH, DB_UUID, KEY, make_value(MAX_SIZE - db_size));
- create_key_value(crud_1, session_1, mock_node_1, CALLER_UUID, INNER_REQUEST_HASH, DB_UUID, KEY, make_value(MAX_SIZE - db_size));
+ const auto expire = 8 + index + 128; // Provide some expire values that are sortable
+
+ const bzn::value_t VALUE{make_value(MAX_SIZE - db_size)};
+ create_key_value(crud_0, session_0, mock_node_0, CALLER_UUID, INNER_REQUEST_HASH, DB_UUID, KEY, VALUE, expire);
+ create_key_value(crud_1, session_1, mock_node_1, CALLER_UUID, INNER_REQUEST_HASH, DB_UUID, KEY, VALUE, expire);
}
const auto select_key_from_set = [](const auto& keys, uint64_t index)
@@ -3314,7 +3410,7 @@ TEST(crud, test_that_create_exceeding_max_swarm_storage_sends_proper_response)
request.mutable_header()->set_db_uuid("uuid2");
request.mutable_header()->set_nonce(uint64_t(123));
request.mutable_update_db()->set_max_size(2048);
- request.mutable_update_db()->set_eviction_policy(database_create_db::NONE);
+ request.mutable_update_db()->set_eviction_policy(database_create_db::VOLATILE_TTL);
expect_signed_response(session, "uuid2", uint64_t(123), database_response::RESPONSE_NOT_SET);
crud->handle_request("caller_id", request, session);
diff --git a/policy/CMakeLists.txt b/policy/CMakeLists.txt
new file mode 100644
index 00000000..6b2d1d1b
--- /dev/null
+++ b/policy/CMakeLists.txt
@@ -0,0 +1,11 @@
+add_library(policy STATIC
+ eviction_base.hpp
+ random.cpp
+ volatile_ttl.cpp
+ )
+
+add_dependencies(policy proto)
+
+target_include_directories(policy PRIVATE ${BLUZELLE_STD_INCLUDES})
+
+add_subdirectory(test)
diff --git a/policy/eviction_base.hpp b/policy/eviction_base.hpp
new file mode 100644
index 00000000..ba814395
--- /dev/null
+++ b/policy/eviction_base.hpp
@@ -0,0 +1,36 @@
+// Copyright (C) 2018 Bluzelle
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License, version 3,
+// as published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+#pragma once
+
+#include
+#include
+#include
+#include
+
+namespace bzn::policy
+{
+
+ class eviction_base
+ {
+ public:
+ eviction_base(std::shared_ptr storage) : storage{std::move(storage)} {}
+
+ virtual ~eviction_base() = default;
+
+ virtual std::set keys_to_evict(const database_msg& request, size_t max_size) = 0;
+
+ std::shared_ptr storage;
+ };
+} // namespace bzn::crud::eviction
diff --git a/policy/random.cpp b/policy/random.cpp
new file mode 100644
index 00000000..0a409a3b
--- /dev/null
+++ b/policy/random.cpp
@@ -0,0 +1,80 @@
+// Copyright (C) 2018 Bluzelle
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License, version 3,
+// as published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+#include
+#include
+#include
+#include
+
+namespace bzn::policy
+{
+ std::set
+ random::keys_to_evict(const database_msg &request, size_t max_size)
+ {
+ const auto KEY_VALUE_SIZE{
+ request.has_update()
+ ? request.update().value().size()
+ : request.create().key().size() + request.create().value().size()
+ };
+
+ const bzn::key_t IGNORE_KEY{
+ request.has_update()
+ ? request.update().key()
+ : ""
+ };
+
+ const auto size{this->storage->get_size(request.header().db_uuid()).second};
+ size_t storage_to_free{KEY_VALUE_SIZE - (max_size - size)};
+
+ // We may need to remove one or more key/value pairs to make room for the new one
+ std::hash hasher;
+ boost::random::mt19937 mt(hasher(request.header().request_hash()));
+
+ std::vector keys_to_evict;
+ auto available_keys = this->storage->get_keys(request.header().db_uuid());
+ while (storage_to_free && !available_keys.empty())
+ {
+ // As we randomly select a key from the keys in the user's database, we *move* them from the vector of keys
+ // in the users' database to the vector of keys that need to be deleted. In the case of randomly selected
+ // the key that is being updated, we simnply remove that key from the vector of users' keys without putting
+ // it in the keys to delete.
+ const boost::random::uniform_int_distribution<> dist(0, available_keys.size() - 1);
+ const auto it = std::next(available_keys.begin(), dist(mt));
+ if (*it != IGNORE_KEY)
+ {
+ const auto evicted_size = this->storage->get_key_size(request.header().db_uuid(), *it);
+ if (evicted_size.has_value())
+ {
+ std::move(it, std::next(it,1), std::back_inserter(keys_to_evict));
+ storage_to_free -= *evicted_size < storage_to_free ? *evicted_size : storage_to_free;
+ }
+ else
+ {
+ LOG(warning) << "While searching for keys to evict, the key " << *it << " was not found in the database " << request.header().db_uuid();
+ }
+ }
+ available_keys.erase(it, std::next(it,1));
+ }
+
+ // Did we free enough storage?
+ if (!storage_to_free)
+ {
+ // It would have been nice to move the keys into the set directly, but std::move doesn't move from vector to set
+
+ return std::set(keys_to_evict.begin(), keys_to_evict.end());
+ }
+
+ return std::set{};
+ }
+} // namespace bzn::crud::eviction
diff --git a/policy/random.hpp b/policy/random.hpp
new file mode 100644
index 00000000..788dc309
--- /dev/null
+++ b/policy/random.hpp
@@ -0,0 +1,30 @@
+// Copyright (C) 2018 Bluzelle
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License, version 3,
+// as published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+#pragma once
+
+#include
+
+namespace bzn::policy
+{
+ class random : public eviction_base
+ {
+ public:
+ random(std::shared_ptr storage) : eviction_base{storage}
+ {
+ }
+
+ std::set keys_to_evict(const database_msg& request, size_t max_size);
+ };
+}
diff --git a/policy/test/CMakeLists.txt b/policy/test/CMakeLists.txt
new file mode 100644
index 00000000..49d53bce
--- /dev/null
+++ b/policy/test/CMakeLists.txt
@@ -0,0 +1,5 @@
+set(test_srcs eviction_test.cpp)
+
+set(test_libs policy proto storage ${Protobuf_LIBRARIES})
+
+add_gmock_test(policy)
diff --git a/policy/test/eviction_test.cpp b/policy/test/eviction_test.cpp
new file mode 100644
index 00000000..ad86bbf1
--- /dev/null
+++ b/policy/test/eviction_test.cpp
@@ -0,0 +1,209 @@
+// Copyright (C) 2018 Bluzelle
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License, version 3,
+// as published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+#include
+
+using namespace ::testing;
+
+namespace
+{
+ const char* DB_UUID{"e6d16cab-cd0f-4af1-b45d-16de9bc2af5d"};
+ //const char* CALLER_UUID{"9558fa26-ba03-4e24-98bc-1f2f562c4598"};
+ // 2b3c2de4-5c14-4842-8284-b67eb09ec61d
+ // bf90f8d1-cb27-4217-a5c5-85c8e6565a25
+ // 557bab19-8df2-4731-b74e-c2516e1f580f
+
+ const std::string TTL_UUID{"TTL"};
+
+ database_msg
+ make_create_request(const bzn::uuid_t& db_uuid, const bzn::key_t key, const bzn::value_t& value)
+ {
+ database_msg request;
+
+ auto header = new database_header();
+ header->set_db_uuid(db_uuid);
+
+ auto create = new database_create();
+ create->set_key(key);
+ create->set_value(value);
+
+ request.set_allocated_create(create);
+ request.set_allocated_header(header);
+
+ return request;
+ }
+
+
+ database_msg
+ make_update_request(const bzn::uuid_t& db_uuid, const bzn::key_t key, const bzn::value_t& value)
+ {
+ database_msg request;
+
+ auto header = new database_header();
+ header->set_db_uuid(db_uuid);
+
+ auto update = new database_update();
+ update->set_key(key);
+ update->set_value(value);
+
+ request.set_allocated_update(update);
+ request.set_allocated_header(header);
+
+ return request;
+ }
+
+
+ // TODO: This is duplication of private code in storage, it may be a good idea to find a way
+ // to dry this up
+ bzn::key_t
+ generate_expire_key(const bzn::uuid_t& uuid, const bzn::key_t& key)
+ {
+ Json::Value value;
+
+ value["uuid"] = uuid;
+ value["key"] = key;
+
+ return value.toStyledString();
+ }
+
+
+ size_t
+ insert_test_values(std::shared_ptr storage, const bzn::uuid_t& db_uuid, size_t number_of_items, size_t value_size=128)
+ {
+ for (size_t i{0}; icreate( db_uuid, key, std::string(value_size, 'B'));
+ }
+
+ return storage->get_size(db_uuid).second;
+ }
+
+
+ void
+ insert_ttl_values(std::shared_ptr storage, const bzn::uuid_t& db_uuid, const std::vector keys)
+ {
+ size_t i{0};
+ for (const auto& key : keys)
+ {
+ const auto expires = boost::lexical_cast(
+ std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count() + 1024 + i * 1024);
+ ++i;
+ storage->create(TTL_UUID, generate_expire_key(db_uuid,key), expires);
+ }
+ }
+}
+
+namespace bzn
+{
+ TEST(policy_test, test_that_volatile_ttl_ignores_keys_with_no_ttl)
+ {
+ std::shared_ptr storage{std::make_shared()};
+ const auto MAX_DB_SIZE{insert_test_values(storage, DB_UUID, 10) + 5};
+ policy::volatile_ttl sut(storage);
+
+ // performing a create on a db with no TTLs should return no keys to delete
+ {
+ const auto request{make_create_request(DB_UUID, "testvalue", std::string(256, 'C'))};
+ const auto keys_to_delete{sut.keys_to_evict(request, MAX_DB_SIZE)};
+ EXPECT_EQ(size_t(0), keys_to_delete.size());
+ }
+
+ // Update should behave the same way
+ {
+ const auto request{make_update_request(DB_UUID, "testvalue", std::string(256,'C'))};
+ const auto keys_to_delete{sut.keys_to_evict(request, MAX_DB_SIZE)};
+ EXPECT_EQ( size_t(0), keys_to_delete.size());
+ }
+ }
+
+ TEST(policy_test, test_that_volatile_ttl_returns_only_keys_with_ttl)
+ {
+ std::shared_ptr storage{std::make_shared()};
+
+ const size_t NUMBER_OF_ITEMS{10};
+ const size_t VALUE_SIZE{128};
+ const auto MAX_STORAGE{insert_test_values(storage, DB_UUID, NUMBER_OF_ITEMS, VALUE_SIZE)};
+
+ const bzn::key_t TEST_KEY{"key1"};
+ std::vector keys{TEST_KEY};
+ insert_ttl_values(storage, DB_UUID, keys);
+
+ policy::volatile_ttl sut(storage);
+
+ const bzn::value_t TEST_VALUE{std::string(64, 'B')};
+
+ // evict the one ttl key to make room for the created key value pair
+ {
+ auto request{make_create_request(DB_UUID, "KEY_CREATE", TEST_VALUE)};
+ const auto keys_to_delete{sut.keys_to_evict(request, MAX_STORAGE)};
+ EXPECT_EQ(size_t(1), keys_to_delete.size());
+ EXPECT_EQ(TEST_KEY, *keys_to_delete.begin());
+ }
+
+ // evict the one ttl key to make room for the updated key value pair
+ {
+ auto request{make_update_request(DB_UUID, "KEY_UPDATE", TEST_VALUE)};
+ const auto keys_to_delete{sut.keys_to_evict(request, MAX_STORAGE)};
+ EXPECT_EQ(size_t(1), keys_to_delete.size());
+ EXPECT_EQ(TEST_KEY, *keys_to_delete.begin());
+ }
+ }
+
+ TEST(policy_test, test_that_volatile_ttl_fails_if_there_are_not_enough_ttl_values_to_remove)
+ {
+ std::shared_ptr storage{std::make_shared()};
+
+ const size_t NUMBER_OF_ITEMS{10};
+
+ const size_t VALUE_SIZE{128};
+ const auto MAX_STORAGE{insert_test_values(storage, DB_UUID, NUMBER_OF_ITEMS, VALUE_SIZE) + 5};
+
+ // set only one TTL value
+ const bzn::key_t TEST_KEY{"key1"};
+ std::vector keys{TEST_KEY};
+ insert_ttl_values(storage, DB_UUID, keys);
+
+ policy::volatile_ttl sut(storage);
+
+ // try creating a large value, this must fail as there is only one small key value pair with a TTL
+ // (return no keys to evict)
+ {
+ auto request{make_create_request(DB_UUID, "KEY_CREATE", std::string(2 * VALUE_SIZE, 'B'))};
+ const auto keys_to_delete{sut.keys_to_evict(request, MAX_STORAGE)};
+ EXPECT_EQ(size_t(0), keys_to_delete.size());
+ }
+
+ // try updating an existing value by doubling its size. This must fail (ibid)
+ {
+ auto request{make_update_request(DB_UUID, "KEY_CREATE", std::string(2 * VALUE_SIZE, 'B'))};
+ const auto keys_to_delete{sut.keys_to_evict(request, MAX_STORAGE)};
+ EXPECT_EQ(size_t(0), keys_to_delete.size());
+ }
+
+ // try updating the key with a TTL, this must fail (the only key with a ttl is the key we are updating.)
+ {
+ auto request{make_update_request(DB_UUID, TEST_KEY, std::string(2 * VALUE_SIZE, 'B'))};
+ const auto keys_to_delete{sut.keys_to_evict(request, MAX_STORAGE)};
+ EXPECT_EQ(size_t(0), keys_to_delete.size());
+ }
+ }
+}
diff --git a/policy/volatile_ttl.cpp b/policy/volatile_ttl.cpp
new file mode 100644
index 00000000..ce9e6368
--- /dev/null
+++ b/policy/volatile_ttl.cpp
@@ -0,0 +1,166 @@
+// Copyright (C) 2018 Bluzelle
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License, version 3,
+// as published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+#include
+#include
+
+namespace
+{
+ // TODO: This is a duplicate of code in crud, it would be a good idea to find a way to dry this up.
+ const std::string TTL_UUID{"TTL"};
+}
+
+
+namespace bzn::policy
+{
+ struct ttl_item {bzn::key_t key; size_t value_size; size_t ttl;};
+
+ std::set
+ volatile_ttl::keys_to_evict(const database_msg &request, size_t max_db_size)
+ {
+ const auto db_uuid{request.header().db_uuid()};
+ const auto key_to_ignore{request.has_update() ? request.update().key() : "" };
+
+ // get all the TTL's in storage
+ auto all_ttls{this->storage->get_keys(TTL_UUID)};
+
+ // get the TTL's associated with our database
+ // filter out the ones for this database, if update, ignore current key
+ auto our_ttls{this->filter_ttls(all_ttls, db_uuid, key_to_ignore)};
+ if (our_ttls.empty())
+ {
+ return {};
+ }
+
+ // Create a vector of [key, value size, ttl]
+ std::vector ttl_items{get_ttl_items(db_uuid, our_ttls)};
+
+ // 3: sort the keys in order of increasing ttl, secondarily sort on decreasing
+ // key value size
+ std::sort(ttl_items.begin(), ttl_items.end(),
+ [](const auto& a, const auto& b)
+ {
+ return a.value_size > b.value_size;
+ });
+
+ // 4: from the top of the list of keys pull out enough key value pairs
+ // to make room for the new pair or update,
+ const auto current_db_size{this->storage->get_size(request.header().db_uuid()).second};
+ const auto current_free = (max_db_size - current_db_size) - (request.has_update() ? request.update().value().size() : 0);
+ const auto key_value_size {
+ request.has_update()
+ ? request.update().value().size()
+ : request.create().key().size() + request.create().value().size()
+ };
+
+ // how much room do we need to free?
+ // enough for the new key value pair: key_value_size
+ // How much room do we have? -> current_free
+ // How much do we need to free? -> required_size
+ std::set keys_to_evict;
+ auto required_size = key_value_size - current_free;
+ for (const auto ttl : ttl_items)
+ {
+ keys_to_evict.emplace(ttl.key);
+ required_size -= (ttl.value_size < required_size ? ttl.value_size : required_size);
+ if (!required_size)
+ {
+ break;
+ }
+ }
+
+ // fail if we can't make enough room
+ if (required_size)
+ {
+ keys_to_evict.clear();
+ }
+
+ // 5: return the list of keys to delete
+ return keys_to_evict;
+ }
+
+
+ std::vector
+ volatile_ttl::get_ttl_items(const std::string &db_uuid, std::vector &our_ttls) const
+ {
+ std::vector ttl_items;
+ std::unique_ptr char_reader{ Json::CharReaderBuilder().newCharReader() };
+ std::transform(our_ttls.begin(), our_ttls.end(), std::back_inserter(ttl_items),
+ [&](const auto& ttl)->ttl_item
+ {
+ Json::Value root;
+ std::string err;
+ if (char_reader->parse(ttl.c_str(), ttl.c_str() + ttl.size(), &root, &err))
+ {
+ const auto key{root["key"].asString()};
+ const auto expire_key{this->generate_expire_key(db_uuid, key)};
+ const auto opt_expire{storage->read(TTL_UUID, expire_key)};
+ const auto opt_value{storage->read(db_uuid, key)};
+
+ if (!opt_value.has_value())
+ {
+ LOG (warning) << "item with key:[" << key << "] does not exist in db: [" << db_uuid << "]";
+
+ return ttl_item{"", 0 , 0};
+ }
+
+ if (!opt_expire.has_value())
+ {
+ LOG (warning) << "TTL item with key:[" << expire_key << "] does not exist";
+
+ return ttl_item{"", 0, 0};
+ }
+
+ LOG (info) << "Found an item to evict: [" << key << "]" ;
+
+ return ttl_item{
+ key
+ , opt_value.value().size()
+ , boost::lexical_cast(opt_expire.value())
+ };
+ }
+ LOG(warning) << "Unable to parse TTL for Volatile TTL eviction: " << err;
+
+ return ttl_item{"", 0 , 0};
+ });
+
+ return ttl_items;
+ }
+
+
+ std::vector
+ volatile_ttl::filter_ttls(const std::vector& ttls, const bzn::uuid_t& db_uuid, const bzn::key_t& ignore_key)
+ {
+ std::vector filtered_ttls;
+ std::unique_ptr char_reader{ Json::CharReaderBuilder().newCharReader()};
+ std::copy_if(
+ ttls.begin(), ttls.end(), std::back_inserter(filtered_ttls)
+ , [&char_reader, &db_uuid, &ignore_key](const auto& t)
+ {
+ Json::Value root;
+ std::string err;
+ if (char_reader->parse(t.c_str(), t.c_str() + t.size(), &root, &err))
+ {
+ bool s{(root["key"] != ignore_key) && (root["uuid"].asString() == db_uuid)};
+
+ return s;
+ }
+ LOG(warning) << "Unable to parse TTL for Volatile TTL eviction: " << err;
+
+ return false;
+ });
+
+ return filtered_ttls;
+ }
+} // namespace bzn::crud::eviction
diff --git a/policy/volatile_ttl.hpp b/policy/volatile_ttl.hpp
new file mode 100644
index 00000000..79c798b2
--- /dev/null
+++ b/policy/volatile_ttl.hpp
@@ -0,0 +1,50 @@
+// Copyright (C) 2018 Bluzelle
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License, version 3,
+// as published by the Free Software Foundation.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+#pragma once
+
+#include
+
+namespace bzn::policy
+{
+ struct ttl_item;
+
+ class volatile_ttl : public eviction_base
+ {
+ public:
+ volatile_ttl(std::shared_ptr storage) : eviction_base(storage)
+ {
+ }
+
+ std::set keys_to_evict(const database_msg& request, size_t max_size) override;
+
+ private:
+ std::vector filter_ttls(const std::vector& ttls, const bzn::uuid_t& db_uuid, const bzn::key_t& ignore_key);
+
+ std::vector get_ttl_items(const std::string& db_uuid, std::vector& our_ttls) const;
+
+
+ // TODO: This is a duplicate of a private crud method, it would be nice to find a way to dry this up,
+ // maybe move this function into a utility module
+ bzn::key_t generate_expire_key(const bzn::uuid_t& uuid, const bzn::key_t& key) const
+ {
+ Json::Value value;
+
+ value["uuid"] = uuid;
+ value["key"] = key;
+
+ return value.toStyledString();
+ }
+ };
+}
diff --git a/proto/database.proto b/proto/database.proto
index 238dba2f..cf0fd4d5 100644
--- a/proto/database.proto
+++ b/proto/database.proto
@@ -69,6 +69,7 @@ message database_create_db
{
NONE = 0;
RANDOM = 1;
+ VOLATILE_TTL = 2;
}
uint64 max_size = 1;
diff --git a/utils/esr_peer_info.cpp b/utils/esr_peer_info.cpp
index 3ecacb2a..44e70f83 100644
--- a/utils/esr_peer_info.cpp
+++ b/utils/esr_peer_info.cpp
@@ -41,8 +41,7 @@ namespace
str_to_json(const std::string &json_str)
{
bzn::json_message json_msg;
- Json::CharReaderBuilder builder;
- Json::CharReader* reader = builder.newCharReader();
+ std::unique_ptr reader{ Json::CharReaderBuilder().newCharReader() };
std::string errors;
if(!reader->parse(
json_str.c_str()