Skip to content
This repository has been archived by the owner on Mar 3, 2020. It is now read-only.

Commit

Permalink
KEP-1250 storage needs to track key/value size
Browse files Browse the repository at this point in the history
  • Loading branch information
@ebruck
ebruck committed Mar 21, 2019
1 parent 285cde3 commit 78c2c19
Show file tree
Hide file tree
Showing 5 changed files with 162 additions and 80 deletions.
34 changes: 17 additions & 17 deletions pbft/pbft_persistent_state.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ namespace bzn
// construct a persistent variable, retrieving its value from storage if it exists, otherwise
// initializing it with the provided default.
template <typename... K>
persistent(std::shared_ptr<bzn::storage_base> persist_storage, const T &default_value, const std::string& name, K... subkeys)
persistent(std::shared_ptr<bzn::storage_base> persist_storage, const T& default_value, const std::string& name, K... subkeys)
: storage(persist_storage), key(escape(name) + generate_key(subkeys...))
{
if (this->storage)
Expand Down Expand Up @@ -123,7 +123,7 @@ namespace bzn

// conversion constructor for comparison in maps etc. without placing in storage
// made explicit to avoid unintentionally assign a raw type to a persistent
explicit persistent(const T &value)
explicit persistent(const T& value)
: t(value)
{}

Expand Down Expand Up @@ -246,7 +246,7 @@ namespace bzn
// helper to initialize values in a key-value container
template<typename A, typename C>
static void
init_kv_container(std::shared_ptr<bzn::storage_base> storage, const std::string &basename, C &container)
init_kv_container(std::shared_ptr<bzn::storage_base> storage, const std::string& basename, C& container)
{
initialize<A>(storage, basename, [&container](auto value, auto key)
{
Expand All @@ -258,7 +258,7 @@ namespace bzn
// note - the outer container can be non-kv such as a set
template<typename A, typename B, typename C>
static void
init_kv_container2(std::shared_ptr<bzn::storage_base> storage, const std::string &basename, C &container)
init_kv_container2(std::shared_ptr<bzn::storage_base> storage, const std::string& basename, C& container)
{
initialize<A, B>(storage, basename, [&container](auto value, auto key1, auto key2)
{
Expand Down Expand Up @@ -334,24 +334,24 @@ namespace bzn
FRIEND_TEST(persistent_state_test, test_escaping);
};

template<> std::string persistent<std::string>::to_string(const std::string &value);
template<> std::string persistent<std::string>::from_string(const std::string &value);
template<> std::string persistent<std::string>::to_string(const std::string& value);
template<> std::string persistent<std::string>::from_string(const std::string& value);

template<> std::string persistent<bzn::log_key_t>::to_string(const bzn::log_key_t &key);
template<> bzn::log_key_t persistent<bzn::log_key_t>::from_string(const std::string &value);
template<> std::string persistent<bzn::log_key_t>::to_string(const bzn::log_key_t& key);
template<> bzn::log_key_t persistent<bzn::log_key_t>::from_string(const std::string& value);

template<> std::string persistent<bzn::operation_key_t>::to_string(const bzn::operation_key_t &key);
template<> bzn::operation_key_t persistent<bzn::operation_key_t>::from_string(const std::string &value);
template<> std::string persistent<bzn::operation_key_t>::to_string(const bzn::operation_key_t& key);
template<> bzn::operation_key_t persistent<bzn::operation_key_t>::from_string(const std::string& value);

template<> std::string persistent<bzn::checkpoint_t>::to_string(const bzn::checkpoint_t &cp);
template<> bzn::checkpoint_t persistent<bzn::checkpoint_t>::from_string(const std::string &value);
template<> std::string persistent<bzn::checkpoint_t>::to_string(const bzn::checkpoint_t& cp);
template<> bzn::checkpoint_t persistent<bzn::checkpoint_t>::from_string(const std::string& value);

template <> std::string persistent<uint64_t>::to_string(const uint64_t& val);
template <> uint64_t persistent<uint64_t>::from_string(const std::string& value);

template<> std::string persistent<bool>::to_string(const bool &val);
template<> bool persistent<bool>::from_string(const std::string &value);
template<> std::string persistent<bool>::to_string(const bool& val);
template<> bool persistent<bool>::from_string(const std::string& value);

template<> std::string persistent<bzn_envelope>::to_string(const bzn_envelope &val);
template<> bzn_envelope persistent<bzn_envelope>::from_string(const std::string &value);
}
template<> std::string persistent<bzn_envelope>::to_string(const bzn_envelope& val);
template<> bzn_envelope persistent<bzn_envelope>::from_string(const std::string& value);
}
66 changes: 32 additions & 34 deletions storage/mem_storage.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,18 +39,18 @@ mem_storage::create(const bzn::uuid_t& uuid, const std::string& key, const std::

if (auto search = this->kv_store.find(uuid); search != this->kv_store.end())
{
if (search->second.find(key)!= search->second.end() )
if (search->second.second.find(key)!= search->second.second.end() )
{
return bzn::storage_result::exists;
}
}

auto& inner_db = this->kv_store[uuid];

if (inner_db.find(key) == inner_db.end())
if (inner_db.second.find(key) == inner_db.second.end())
{
// todo: test if insert failed?
inner_db.insert(std::make_pair(key,value));
inner_db.first += value.size();
inner_db.second.insert(std::make_pair(key,value));
}
else
{
Expand All @@ -75,8 +75,8 @@ mem_storage::read(const bzn::uuid_t& uuid, const std::string& key)

// we have the db, let's see if the key exists
auto& inner_db = search->second;
auto inner_search = inner_db.find(key);
if (inner_search == inner_db.end())
auto inner_search = inner_db.second.find(key);
if (inner_search == inner_db.second.end())
{
return std::nullopt;
}
Expand All @@ -101,17 +101,19 @@ mem_storage::update(const bzn::uuid_t& uuid, const std::string& key, const std::
return bzn::storage_result::not_found;
}


// we have the db, let's see if the key exists
auto& inner_db = search->second;
auto inner_search = inner_db.find(key);
auto inner_search = inner_db.second.find(key);

if (inner_search == inner_db.end())
if (inner_search == inner_db.second.end())
{
return bzn::storage_result::not_found;
}

inner_db.first -= inner_search->second.size();
inner_search->second = value;
inner_db.first += inner_search->second.size();

return bzn::storage_result::ok;
}

Expand All @@ -128,14 +130,15 @@ mem_storage::remove(const bzn::uuid_t& uuid, const std::string& key)
return bzn::storage_result::not_found;
}

auto record = search->second.find(key);
auto record = search->second.second.find(key);

if (record == search->second.end())
if (record == search->second.second.end())
{
return bzn::storage_result::not_found;
}

search->second.erase(record);
search->second.first -= record->second.size();
search->second.second.erase(record);
return bzn::storage_result::ok;
}

Expand All @@ -153,7 +156,7 @@ mem_storage::get_keys(const bzn::uuid_t& uuid)
}

std::vector<std::string> keys;
for (const auto& p : inner_db->second)
for (const auto& p : inner_db->second.second)
{
keys.emplace_back(p.first);
}
Expand All @@ -170,7 +173,6 @@ mem_storage::has(const bzn::uuid_t& uuid, const std::string& key)
}


// todo: optimize! Track size as it grows and not iterate over every value!
std::pair<std::size_t, std::size_t>
mem_storage::get_size(const bzn::uuid_t& uuid)
{
Expand All @@ -184,16 +186,7 @@ mem_storage::get_size(const bzn::uuid_t& uuid)
return std::make_pair(0,0);
}

std::size_t size{};
std::size_t keys{};

for (const auto& record : it->second)
{
++keys;
size += record.second.size();
}

return std::make_pair(keys, size);
return std::make_pair(it->second.second.size(), it->second.first);
}


Expand Down Expand Up @@ -266,6 +259,7 @@ mem_storage::load_snapshot(const std::string& data)
return false;
}


void
mem_storage::remove_range(const bzn::uuid_t& uuid, const std::string& first, const std::string& last)
{
Expand All @@ -274,19 +268,21 @@ mem_storage::remove_range(const bzn::uuid_t& uuid, const std::string& first, con
auto inner_db = this->kv_store.find(uuid);
if (inner_db != this->kv_store.end())
{
auto match = inner_db->second.lower_bound(first);
auto end = inner_db->second.lower_bound(last);
auto match = inner_db->second.second.lower_bound(first);
auto end = inner_db->second.second.lower_bound(last);
while (match != end)
{
match = inner_db->second.erase(match);
inner_db->second.first -= match->second.size();
match = inner_db->second.second.erase(match);
}
}
}


void
mem_storage::do_if(const bzn::uuid_t& uuid, const std::string& first, const std::string& last
, std::optional<std::function<bool(const bzn::key_t&, const bzn::value_t&)>> predicate
, std::function<void(const bzn::key_t&, const bzn::value_t&)> action)
mem_storage::do_if(const bzn::uuid_t& uuid, const std::string& first, const std::string& last,
std::optional<std::function<bool(const bzn::key_t&, const bzn::value_t&)>> predicate,
std::function<void(const bzn::key_t&, const bzn::value_t&)> action)
{
if (!last.empty() && last <= first)
{
Expand All @@ -298,8 +294,8 @@ mem_storage::do_if(const bzn::uuid_t& uuid, const std::string& first, const std:
auto inner_db = this->kv_store.find(uuid);
if (inner_db != this->kv_store.end())
{
auto end_it = last.empty() ? inner_db->second.end() : inner_db->second.lower_bound(last);
for (auto it = inner_db->second.lower_bound(first); it != inner_db->second.end() && it != end_it; it++)
auto end_it = last.empty() ? inner_db->second.second.end() : inner_db->second.second.lower_bound(last);
for (auto it = inner_db->second.second.lower_bound(first); it != inner_db->second.second.end() && it != end_it; it++)
{
if (!predicate || (*predicate)(it->first, it->second))
{
Expand All @@ -309,9 +305,10 @@ mem_storage::do_if(const bzn::uuid_t& uuid, const std::string& first, const std:
}
}


std::vector<std::pair<bzn::key_t, bzn::value_t>>
mem_storage::read_if(const bzn::uuid_t& uuid, const std::string& first, const std::string& last
, std::optional<std::function<bool(const bzn::key_t&, const bzn::value_t&)>> predicate)
mem_storage::read_if(const bzn::uuid_t& uuid, const std::string& first, const std::string& last,
std::optional<std::function<bool(const bzn::key_t&, const bzn::value_t&)>> predicate)
{
std::vector<std::pair<bzn::key_t, bzn::value_t>> matches;

Expand All @@ -323,6 +320,7 @@ mem_storage::read_if(const bzn::uuid_t& uuid, const std::string& first, const st
return matches;
}


std::vector<bzn::key_t>
mem_storage::get_keys_if(const bzn::uuid_t& uuid, const std::string& first, const std::string& last
, std::optional<std::function<bool(const bzn::key_t&, const bzn::value_t&)>> predicate)
Expand Down
20 changes: 10 additions & 10 deletions storage/mem_storage.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,24 +50,24 @@ namespace bzn

void remove_range(const bzn::uuid_t& uuid, const std::string& first, const std::string& last) override;

std::vector<std::pair<bzn::key_t, bzn::value_t>> read_if(const bzn::uuid_t& uuid
, const std::string& first, const std::string& last
, std::optional<std::function<bool(const bzn::key_t&, const bzn::value_t&)>> predicate = std::nullopt) override;
std::vector<std::pair<bzn::key_t, bzn::value_t>> read_if(const bzn::uuid_t& uuid,
const std::string& first, const std::string& last,
std::optional<std::function<bool(const bzn::key_t&, const bzn::value_t&)>> predicate = std::nullopt) override;

std::vector<bzn::key_t> get_keys_if(const bzn::uuid_t& uuid
, const std::string& first, const std::string& last
, std::optional<std::function<bool(const bzn::key_t&, const bzn::value_t&)>> predicate = std::nullopt) override;
std::vector<bzn::key_t> get_keys_if(const bzn::uuid_t& uuid,
const std::string& first, const std::string& last,
std::optional<std::function<bool(const bzn::key_t&, const bzn::value_t&)>> predicate = std::nullopt) override;

private:
std::unordered_map<bzn::uuid_t, std::map<bzn::key_t, bzn::value_t>> kv_store;
std::unordered_map<bzn::uuid_t, std::pair<uint32_t, std::map<bzn::key_t, bzn::value_t>>> kv_store;

std::shared_mutex lock; // for multi-reader and single writer access

std::shared_ptr<std::string> latest_snapshot;

void do_if(const bzn::uuid_t& uuid, const std::string& first, const std::string& last
, std::optional<std::function<bool(const bzn::key_t&, const bzn::value_t&)>> predicate
, std::function<void(const bzn::key_t&, const bzn::value_t&)> action);
void do_if(const bzn::uuid_t& uuid, const std::string& first, const std::string& last,
std::optional<std::function<bool(const bzn::key_t&, const bzn::value_t&)>> predicate,
std::function<void(const bzn::key_t&, const bzn::value_t&)> action);
};

} // bzn

0 comments on commit 78c2c19

Please sign in to comment.