Skip to content

Commit

Permalink
mon/MonitorDBStore: improve get_chunk_tx limits
Browse files Browse the repository at this point in the history
The old version was horribly inefficient in that it would reencode the
transaction on every iteration.

Instead, estimate the size if we add an item and stop it if looks like it
will go over.  This isn't super precise, but it's close enough, since the
limits are approximate.

Drop the single-use helper since it only makes the code harder to
follow.

Signed-off-by: Sage Weil <sage@redhat.com>
(cherry picked from commit 83b2ada)
  • Loading branch information
liewegas authored and dvanders committed Mar 5, 2020
1 parent 36a211e commit 4c54cb8
Showing 1 changed file with 18 additions and 34 deletions.
52 changes: 18 additions & 34 deletions src/mon/MonitorDBStore.h
Original file line number Diff line number Diff line change
Expand Up @@ -415,38 +415,6 @@ class MonitorDBStore
StoreIteratorImpl() : done(false) { }
virtual ~StoreIteratorImpl() { }

bool add_chunk_entry(TransactionRef tx,
const string &prefix,
const string &key,
bufferlist &value,
uint64_t max) {
auto tmp(std::make_shared<Transaction>());
bufferlist tmp_bl;
tmp->put(prefix, key, value);
tmp->encode(tmp_bl);

bufferlist tx_bl;
tx->encode(tx_bl);

size_t len = tx_bl.length() + tmp_bl.length();

if (!tx->empty() && (len > max)) {
return false;
}

tx->append(tmp);
last_key.first = prefix;
last_key.second = key;

if (g_conf()->mon_sync_debug) {
encode(prefix, crc_bl);
encode(key, crc_bl);
encode(value, crc_bl);
}

return true;
}

virtual bool _is_valid() = 0;

public:
Expand Down Expand Up @@ -489,7 +457,7 @@ class MonitorDBStore
* differ from the one passed on to the function)
* @param last_key[out] Last key in the chunk
*/
void get_chunk_tx(TransactionRef tx, uint64_t max) override {
void get_chunk_tx(TransactionRef tx, uint64_t max_bytes) override {
ceph_assert(done == false);
ceph_assert(iter->valid() == true);

Expand All @@ -498,8 +466,24 @@ class MonitorDBStore
string key(iter->raw_key().second);
if (sync_prefixes.count(prefix)) {
bufferlist value = iter->value();
if (!add_chunk_entry(tx, prefix, key, value, max))
if (tx->empty() ||
(tx->get_bytes() + value.length() + key.size() +
prefix.size() < max_bytes)) {
// NOTE: putting every key in a separate transaction is
// questionable as far as efficiency goes
auto tmp(std::make_shared<Transaction>());
tmp->put(prefix, key, value);
tx->append(tmp);
if (g_conf()->mon_sync_debug) {
encode(prefix, crc_bl);
encode(key, crc_bl);
encode(value, crc_bl);
}
} else {
last_key.first = prefix;
last_key.second = key;
return;
}
}
iter->next();
}
Expand Down

0 comments on commit 4c54cb8

Please sign in to comment.