Skip to content

Commit

Permalink
os/bluestore: convert internal types to denc
Browse files Browse the repository at this point in the history 
Signed-off-by: Sage Weil <sage@redhat.com>
  • Loading branch information
@liewegas
liewegas committed Sep 19, 2016
1 parent afb15a3 commit d9dab11
Show file tree
Hide file tree
Showing 4 changed files with 336 additions and 371 deletions.
192 changes: 115 additions & 77 deletions src/os/bluestore/BlueStore.cc
View file
Expand Up @@ -1613,6 +1613,8 @@ bool BlueStore::ExtentMap::encode_some(uint32_t offset, uint32_t length,
uint32_t end = offset + length;

unsigned n = 0;
size_t bound = 0;
denc_varint(0, bound);
for (auto p = start;
p != extent_map.end() && p->logical_offset < end;
++p, ++n) {
Expand All @@ -1623,57 +1625,66 @@ bool BlueStore::ExtentMap::encode_some(uint32_t offset, uint32_t length,
<< std::dec << " hit new spanning blob " << *p << dendl;
return true;
}
}
small_encode_varint(n, bl);
if (pn) {
*pn = n;
denc_varint(0, bound); // blobid
denc_varint(0, bound); // logical_offset
denc_varint(0, bound); // len
denc_varint(0, bound); // blob_offset
p->blob->bound_encode(bound);
}

n = 0;
uint64_t pos = 0;
uint64_t prev_len = 0;
for (auto p = start;
p != extent_map.end() && p->logical_offset < end;
++p, ++n) {
unsigned blobid;
bool include_blob = false;
if (p->blob->id >= 0) {
blobid = p->blob->id << BLOBID_SHIFT_BITS;
blobid |= BLOBID_FLAG_SPANNING;
} else if (p->blob->last_encoded_id < 0) {
p->blob->last_encoded_id = n + 1; // so it is always non-zero
include_blob = true;
blobid = 0; // the decoder will infer the id from n
} else {
blobid = p->blob->last_encoded_id << BLOBID_SHIFT_BITS;
}
if (p->logical_offset == pos) {
blobid |= BLOBID_FLAG_CONTIGUOUS;
}
if (p->blob_offset == 0) {
blobid |= BLOBID_FLAG_ZEROOFFSET;
}
if (p->length == prev_len) {
blobid |= BLOBID_FLAG_SAMELENGTH;
} else {
prev_len = p->length;
}
small_encode_varint(blobid, bl);
if ((blobid & BLOBID_FLAG_CONTIGUOUS) == 0) {
small_encode_varint_lowz(p->logical_offset - pos, bl);
}
if ((blobid & BLOBID_FLAG_ZEROOFFSET) == 0) {
small_encode_varint_lowz(p->blob_offset, bl);
}
if ((blobid & BLOBID_FLAG_SAMELENGTH) == 0) {
small_encode_varint_lowz(p->length, bl);
{
auto app = bl.get_contiguous_appender(bound);
denc_varint(n, app);
if (pn) {
*pn = n;
}
pos = p->logical_offset + p->length;
if (include_blob) {
p->blob->encode(bl);

n = 0;
uint64_t pos = 0;
uint64_t prev_len = 0;
for (auto p = start;
p != extent_map.end() && p->logical_offset < end;
++p, ++n) {
unsigned blobid;
bool include_blob = false;
if (p->blob->id >= 0) {
blobid = p->blob->id << BLOBID_SHIFT_BITS;
blobid |= BLOBID_FLAG_SPANNING;
} else if (p->blob->last_encoded_id < 0) {
p->blob->last_encoded_id = n + 1; // so it is always non-zero
include_blob = true;
blobid = 0; // the decoder will infer the id from n
} else {
blobid = p->blob->last_encoded_id << BLOBID_SHIFT_BITS;
}
if (p->logical_offset == pos) {
blobid |= BLOBID_FLAG_CONTIGUOUS;
}
if (p->blob_offset == 0) {
blobid |= BLOBID_FLAG_ZEROOFFSET;
}
if (p->length == prev_len) {
blobid |= BLOBID_FLAG_SAMELENGTH;
} else {
prev_len = p->length;
}
denc_varint(blobid, app);
if ((blobid & BLOBID_FLAG_CONTIGUOUS) == 0) {
denc_varint_lowz(p->logical_offset - pos, app);
}
if ((blobid & BLOBID_FLAG_ZEROOFFSET) == 0) {
denc_varint_lowz(p->blob_offset, app);
}
if ((blobid & BLOBID_FLAG_SAMELENGTH) == 0) {
denc_varint_lowz(p->length, app);
}
pos = p->logical_offset + p->length;
if (include_blob) {
p->blob->encode(app);
}
}
}
/*
/*derr << __func__ << bl << dendl;
derr << __func__ << ":";
bl.hexdump(*_dout);
*_dout << dendl;
Expand All @@ -1683,34 +1694,37 @@ bool BlueStore::ExtentMap::encode_some(uint32_t offset, uint32_t length,

void BlueStore::ExtentMap::decode_some(bufferlist& bl)
{
/* derr << __func__ << ":";
/*
derr << __func__ << ":";
bl.hexdump(*_dout);
*_dout << dendl;
*/
bufferlist::iterator p = bl.begin();

assert(bl.get_num_buffers() <= 1);
auto p = bl.front().begin_deep();
uint32_t num;
small_decode_varint(num, p);
denc_varint(num, p);
vector<BlobRef> blobs(num);
uint64_t pos = 0;
uint64_t prev_len = 0;
unsigned n = 0;
while (!p.end()) {
Extent *le = new Extent();
uint64_t blobid;
small_decode_varint(blobid, p);
denc_varint(blobid, p);
if ((blobid & BLOBID_FLAG_CONTIGUOUS) == 0) {
uint64_t gap;
small_decode_varint_lowz(gap, p);
denc_varint_lowz(gap, p);
pos += gap;
}
le->logical_offset = pos;
if ((blobid & BLOBID_FLAG_ZEROOFFSET) == 0) {
small_decode_varint_lowz(le->blob_offset, p);
denc_varint_lowz(le->blob_offset, p);
} else {
le->blob_offset = 0;
}
if ((blobid & BLOBID_FLAG_SAMELENGTH) == 0) {
small_decode_varint_lowz(prev_len, p);
denc_varint_lowz(prev_len, p);
}
le->length = prev_len;
if (blobid & BLOBID_FLAG_SPANNING) {
Expand All @@ -1737,26 +1751,38 @@ void BlueStore::ExtentMap::decode_some(bufferlist& bl)
assert(n == num);
}

void BlueStore::ExtentMap::encode_spanning_blobs(bufferlist& bl)
void BlueStore::ExtentMap::bound_encode_spanning_blobs(size_t& p)
{
denc_varint((uint32_t)0, p);
size_t key_size = 0;
denc_varint((uint32_t)0, key_size);
p += spanning_blob_map.size() * key_size;
for (const auto& i : spanning_blob_map) {
i.bound_encode(p);
i.ref_map.bound_encode(p);
}
}

void BlueStore::ExtentMap::encode_spanning_blobs(
bufferlist::contiguous_appender& p)
{
unsigned n = spanning_blob_map.size();
small_encode_varint(n, bl);
denc_varint(spanning_blob_map.size(), p);
for (auto& b : spanning_blob_map) {
small_encode_varint(b.id, bl);
b.encode(bl);
b.ref_map.encode(bl);
denc_varint(b.id, p);
b.encode(p);
b.ref_map.encode(p);
}
}

void BlueStore::ExtentMap::decode_spanning_blobs(
Collection *c,
bufferlist::iterator& p)
bufferptr::iterator& p)
{
unsigned n;
small_decode_varint(n, p);
denc_varint(n, p);
while (n--) {
BlobRef b(new Blob());
small_decode_varint(b->id, p);
denc_varint(b->id, p);
spanning_blob_map.insert(*b);
b->get();
b->decode(p);
Expand Down Expand Up @@ -2133,13 +2159,13 @@ BlueStore::OnodeRef BlueStore::Collection::get_onode(
assert(r >=0 );
on = new Onode(&onode_map, this, oid, key);
on->exists = true;
bufferlist::iterator p = v.begin();
::decode(on->onode, p);
bufferptr::iterator p = v.front().begin();
on->onode.decode(p);

// initialize extent_map
on->extent_map.decode_spanning_blobs(this, p);
if (on->onode.extent_map_shards.empty()) {
::decode(on->extent_map.inline_bl, p);
denc(on->extent_map.inline_bl, p);
on->extent_map.decode_some(on->extent_map.inline_bl);
} else {
on->extent_map.init_shards(on, false, false);
Expand Down Expand Up @@ -4172,7 +4198,7 @@ int BlueStore::fsck()
bluestore_shared_blob_t shared_blob;
bufferlist bl = it->value();
bufferlist::iterator blp = bl.begin();
shared_blob.decode(blp);
::decode(shared_blob, blp);
dout(20) << __func__ << " " << *sbi.sb << " " << shared_blob << dendl;
if (shared_blob.ref_map != sbi.ref_map) {
derr << __func__ << " shared blob 0x" << std::hex << sbid << std::dec
Expand Down Expand Up @@ -5958,20 +5984,32 @@ void BlueStore::_txc_write_nodes(TransContext *txc, KeyValueDB::Transaction t)
logger->inc(l_bluestore_onode_reshard);
}

bufferlist bl;
::encode(o->onode, bl);
unsigned onode_part = bl.length();
o->extent_map.encode_spanning_blobs(bl);
unsigned blob_part = bl.length() - onode_part;
// bound encode
size_t bound = 0;
denc(o->onode, bound);
o->extent_map.bound_encode_spanning_blobs(bound);
if (o->onode.extent_map_shards.empty()) {
::encode(o->extent_map.inline_bl, bl);
denc(o->extent_map.inline_bl, bound);
}

// encode
bufferlist bl;
{
auto p = bl.get_contiguous_appender(bound);
denc(o->onode, p);
//unsigned onode_part = bl.length();
o->extent_map.encode_spanning_blobs(p);
//unsigned blob_part = bl.length() - onode_part;
if (o->onode.extent_map_shards.empty()) {
denc(o->extent_map.inline_bl, p);
}
//unsigned extent_part = bl.length() - onode_part - blob_part;
}
unsigned extent_part = bl.length() - onode_part - blob_part;

dout(20) << " onode " << o->oid << " is " << bl.length()
<< " (" << onode_part << " bytes onode + "
<< blob_part << " bytes spanning blobs + "
<< extent_part << " bytes inline extents)"
//<< " (" << onode_part << " bytes onode + "
//<< blob_part << " bytes spanning blobs + "
//<< extent_part << " bytes inline extents)"
<< dendl;
t->set(PREFIX_OBJ, o->key, bl);

Expand All @@ -5987,7 +6025,7 @@ void BlueStore::_txc_write_nodes(TransContext *txc, KeyValueDB::Transaction t)
t->rmkey(PREFIX_SHARED_BLOB, sb->key);
} else {
bufferlist bl;
sb->shared_blob.encode(bl);
::encode(sb->shared_blob, bl);
dout(20) << " shared_blob 0x" << std::hex << sb->sbid << std::dec
<< " is " << bl.length() << dendl;
t->set(PREFIX_SHARED_BLOB, sb->key, bl);
Expand Down
45 changes: 32 additions & 13 deletions src/os/bluestore/BlueStore.h
View file
Expand Up @@ -440,9 +440,6 @@ class BlueStore : public ObjectStore,
}
return blob;
}
size_t get_encoded_length() const {
return blob_bl.length();
}
bool is_dirty() const {
return dirty;
}
Expand All @@ -468,24 +465,45 @@ class BlueStore : public ObjectStore,
delete this;
}

void encode(bufferlist& bl) const {
//#define CACHE_BLOB_BL // not sure if this is a win yet or not... :/

#ifdef CACHE_BLOB_BL
void _encode() const {
if (dirty) {
// manage blob_bl memory carefully
blob_bl.clear();
blob_bl.reserve(blob.estimate_encoded_size());
::encode(blob, blob_bl);
dirty = false;
} else {
assert(blob_bl.length());
}
bl.append(blob_bl);
}
void decode(bufferlist::iterator& p) {
bufferlist::iterator s = p;
::decode(blob, p);
s.copy(p.get_off() - s.get_off(), blob_bl);
void bound_encode(size_t& p) const {
_encode();
p += blob_bl.length();
}
void encode(bufferlist::contiguous_appender& p) const {
_encode();
p.append(blob_bl);
}
void decode(bufferptr::iterator& p) {
const char *start = p.get_pos();
denc(blob, p);
const char *end = p.get_pos();
blob_bl.clear();
blob_bl.push_back(p.get_preceding_ptr(end - start));
dirty = false;
}
#else
void bound_encode(size_t& p) const {
denc(blob, p);
}
void encode(bufferlist::contiguous_appender& p) const {
denc(blob, p);
}
void decode(bufferptr::iterator& p) {
denc(blob, p);
}
#endif
};
typedef boost::intrusive_ptr<Blob> BlobRef;
typedef boost::intrusive::set<Blob> blob_map_t;
Expand Down Expand Up @@ -553,8 +571,9 @@ class BlueStore : public ObjectStore,
unsigned *pn);
void decode_some(bufferlist& bl);

void encode_spanning_blobs(bufferlist& bl);
void decode_spanning_blobs(Collection *c, bufferlist::iterator& p);
void bound_encode_spanning_blobs(size_t& p);
void encode_spanning_blobs(bufferlist::contiguous_appender& p);
void decode_spanning_blobs(Collection *c, bufferptr::iterator& p);

BlobRef get_spanning_blob(int id);

Expand Down

0 comments on commit d9dab11

Please sign in to comment.