Skip to content

Commit

Permalink
rbd-mirror: Add sparse read for sync image
Browse files Browse the repository at this point in the history 
Currently, the image sync do full read, and we shall add sparse read
to let the sync more efficiently.

Feature: http://tracker.ceph.com/issues/16780
Signed-off-by: tianqing <tianqing@unitedstack.com>
  • Loading branch information
tianqing committed Sep 26, 2016
1 parent 928c3c3 commit b5db8bc
Show file tree
Hide file tree
Showing 4 changed files with 46 additions and 23 deletions.
11 changes: 11 additions & 0 deletions src/test/librados_test_stub/MockTestMemIoCtxImpl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,16 @@ class MockTestMemIoCtxImpl : public TestMemIoCtxImpl {
return TestMemIoCtxImpl::notify(o, bl, timeout_ms, pbl);
}

MOCK_METHOD5(sparse_read, int(const std::string& oid,
uint64_t off,
size_t len,
std::map<uint64_t, uint64_t> *m,
bufferlist *bl));
int do_sparse_read(const std::string& oid, uint64_t off, size_t len,
std::map<uint64_t, uint64_t> *m, bufferlist *bl){
return TestMemIoCtxImpl::sparse_read(oid, off, len, m, bl);
}

MOCK_METHOD4(read, int(const std::string& oid,
size_t len,
uint64_t off,
Expand Down Expand Up @@ -142,6 +152,7 @@ class MockTestMemIoCtxImpl : public TestMemIoCtxImpl {
ON_CALL(*this, list_watchers(_, _)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_list_watchers));
ON_CALL(*this, notify(_, _, _, _)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_notify));
ON_CALL(*this, read(_, _, _, _)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_read));
ON_CALL(*this, sparse_read(_, _, _, _, _)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_sparse_read));
ON_CALL(*this, remove(_, _)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_remove));
ON_CALL(*this, selfmanaged_snap_create(_)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_selfmanaged_snap_create));
ON_CALL(*this, selfmanaged_snap_remove(_)).WillByDefault(Invoke(this, &MockTestMemIoCtxImpl::do_selfmanaged_snap_remove));
Expand Down
27 changes: 15 additions & 12 deletions src/test/rbd_mirror/image_sync/test_mock_ObjectCopyRequest.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include "test/rbd_mirror/test_mock_fixture.h"
#include "include/interval_set.h"
#include "include/rbd/librbd.hpp"
#include "include/rbd/object_map_types.h"
#include "librbd/AioImageRequestWQ.h"
#include "librbd/ImageCtx.h"
#include "librbd/ImageState.h"
Expand Down Expand Up @@ -108,20 +109,21 @@ class TestMockImageSyncObjectCopyRequest : public TestMockFixture {
0, on_finish);
}

void expect_read(librados::MockTestMemIoCtxImpl &mock_io_ctx, uint64_t offset,
uint64_t length, int r) {
auto &expect = EXPECT_CALL(mock_io_ctx, read(_, length, offset, _));
void expect_sparse_read(librados::MockTestMemIoCtxImpl &mock_io_ctx, uint64_t offset,
uint64_t length, int r) {

auto &expect = EXPECT_CALL(mock_io_ctx, sparse_read(_, offset, length, _, _));
if (r < 0) {
expect.WillOnce(Return(r));
} else {
expect.WillOnce(DoDefault());
}
}

void expect_read(librados::MockTestMemIoCtxImpl &mock_io_ctx,
void expect_sparse_read(librados::MockTestMemIoCtxImpl &mock_io_ctx,
const interval_set<uint64_t> &extents, int r) {
for (auto extent : extents) {
expect_read(mock_io_ctx, extent.first, extent.second, r);
expect_sparse_read(mock_io_ctx, extent.first, extent.second, r);
if (r < 0) {
break;
}
Expand Down Expand Up @@ -212,6 +214,7 @@ class TestMockImageSyncObjectCopyRequest : public TestMockFixture {
}
m_snap_map[remote_snap_id] = local_snap_ids;
m_local_snap_ids.push_back(local_snap_id);

return 0;
}

Expand Down Expand Up @@ -348,7 +351,7 @@ TEST_F(TestMockImageSyncObjectCopyRequest, Write) {

InSequence seq;
expect_list_snaps(mock_remote_image_ctx, mock_remote_io_ctx, 0);
expect_read(mock_remote_io_ctx, 0, one.range_end(), 0);
expect_sparse_read(mock_remote_io_ctx, 0, one.range_end(), 0);
expect_write(mock_local_io_ctx, 0, one.range_end(), 0);
expect_update_object_map(mock_local_image_ctx, mock_object_map,
m_local_snap_ids[0], OBJECT_EXISTS, 0);
Expand Down Expand Up @@ -381,7 +384,7 @@ TEST_F(TestMockImageSyncObjectCopyRequest, ReadError) {

InSequence seq;
expect_list_snaps(mock_remote_image_ctx, mock_remote_io_ctx, 0);
expect_read(mock_remote_io_ctx, 0, one.range_end(), -EINVAL);
expect_sparse_read(mock_remote_io_ctx, 0, one.range_end(), -EINVAL);

request->send();
ASSERT_EQ(-EINVAL, ctx.wait());
Expand Down Expand Up @@ -412,7 +415,7 @@ TEST_F(TestMockImageSyncObjectCopyRequest, WriteError) {

InSequence seq;
expect_list_snaps(mock_remote_image_ctx, mock_remote_io_ctx, 0);
expect_read(mock_remote_io_ctx, 0, one.range_end(), 0);
expect_sparse_read(mock_remote_io_ctx, 0, one.range_end(), 0);
expect_write(mock_local_io_ctx, 0, one.range_end(), -EINVAL);

request->send();
Expand Down Expand Up @@ -455,9 +458,9 @@ TEST_F(TestMockImageSyncObjectCopyRequest, WriteSnaps) {

InSequence seq;
expect_list_snaps(mock_remote_image_ctx, mock_remote_io_ctx, 0);
expect_read(mock_remote_io_ctx, 0, one.range_end(), 0);
expect_sparse_read(mock_remote_io_ctx, 0, one.range_end(), 0);
expect_write(mock_local_io_ctx, 0, one.range_end(), 0);
expect_read(mock_remote_io_ctx, two, 0);
expect_sparse_read(mock_remote_io_ctx, two, 0);
expect_write(mock_local_io_ctx, two, 0);
expect_update_object_map(mock_local_image_ctx, mock_object_map,
m_local_snap_ids[0], OBJECT_EXISTS, 0);
Expand Down Expand Up @@ -504,7 +507,7 @@ TEST_F(TestMockImageSyncObjectCopyRequest, Trim) {

InSequence seq;
expect_list_snaps(mock_remote_image_ctx, mock_remote_io_ctx, 0);
expect_read(mock_remote_io_ctx, 0, one.range_end(), 0);
expect_sparse_read(mock_remote_io_ctx, 0, one.range_end(), 0);
expect_write(mock_local_io_ctx, 0, one.range_end(), 0);
expect_truncate(mock_local_io_ctx, trim_offset, 0);
expect_update_object_map(mock_local_image_ctx, mock_object_map,
Expand Down Expand Up @@ -546,7 +549,7 @@ TEST_F(TestMockImageSyncObjectCopyRequest, Remove) {

InSequence seq;
expect_list_snaps(mock_remote_image_ctx, mock_remote_io_ctx, 0);
expect_read(mock_remote_io_ctx, 0, one.range_end(), 0);
expect_sparse_read(mock_remote_io_ctx, 0, one.range_end(), 0);
expect_write(mock_local_io_ctx, 0, one.range_end(), 0);
expect_remove(mock_local_io_ctx, 0);
expect_update_object_map(mock_local_image_ctx, mock_object_map,
Expand Down
29 changes: 19 additions & 10 deletions src/tools/rbd_mirror/image_sync/ObjectCopyRequest.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,11 +110,10 @@ void ObjectCopyRequest<I>::send_read_object() {
dout(20) << ": remote_snap_seq=" << remote_snap_seq << dendl;
read_required = true;
}

dout(20) << ": read op: " << std::get<1>(sync_op) << "~"
<< std::get<2>(sync_op) << dendl;
op.read(std::get<1>(sync_op), std::get<2>(sync_op),
&std::get<3>(sync_op), nullptr);
op.sparse_read(std::get<1>(sync_op), std::get<2>(sync_op), &std::get<4>(sync_op),
&std::get<3>(sync_op), nullptr);
break;
default:
break;
Expand Down Expand Up @@ -173,14 +172,19 @@ void ObjectCopyRequest<I>::send_write_object() {

auto &sync_ops = m_snap_sync_ops.begin()->second;
assert(!sync_ops.empty());

uint64_t buffer_offset;
librados::ObjectWriteOperation op;
for (auto &sync_op : sync_ops) {
switch (std::get<0>(sync_op)) {
case SYNC_OP_TYPE_WRITE:
dout(20) << ": write op: " << std::get<1>(sync_op) << "~"
<< std::get<3>(sync_op).length() << dendl;
op.write(std::get<1>(sync_op), std::get<3>(sync_op));
buffer_offset = 0;
for(auto it : std::get<4>(sync_op)){
bufferlist tmpbl;
tmpbl.substr_of(std::get<3>(sync_op), buffer_offset, it.second);
op.write(it.first, tmpbl);
buffer_offset += it.second;
dout(20) << ": write op: " << it.first<< "~" << it.second << dendl;
}
break;
case SYNC_OP_TYPE_TRUNC:
dout(20) << ": trunc op: " << std::get<1>(sync_op) << dendl;
Expand Down Expand Up @@ -228,6 +232,7 @@ void ObjectCopyRequest<I>::handle_write_object(int r) {

template <typename I>
void ObjectCopyRequest<I>::send_update_object_map() {

m_local_image_ctx->snap_lock.get_read();
if (!m_local_image_ctx->test_features(RBD_FEATURE_OBJECT_MAP,
m_local_image_ctx->snap_lock) ||
Expand Down Expand Up @@ -258,6 +263,7 @@ void ObjectCopyRequest<I>::send_update_object_map() {
Context *ctx = create_context_callback<
ObjectCopyRequest<I>, &ObjectCopyRequest<I>::handle_update_object_map>(
this);

m_local_image_ctx->object_map->aio_update(snap_object_state.first,
m_object_number,
m_object_number + 1,
Expand Down Expand Up @@ -333,20 +339,23 @@ void ObjectCopyRequest<I>::compute_diffs() {
m_snap_sync_ops[end_remote_snap_id].emplace_back(SYNC_OP_TYPE_WRITE,
it.get_start(),
it.get_len(),
bufferlist());
bufferlist(),
std::map<uint64_t, uint64_t>());
}
if (end_size < prev_end_size) {
dout(20) << ": trunc op: " << end_size << dendl;
m_snap_sync_ops[end_remote_snap_id].emplace_back(SYNC_OP_TYPE_TRUNC,
end_size, 0U,
bufferlist());
bufferlist(),
std::map<uint64_t, uint64_t>());
}
} else {
if (prev_exists) {
// object remove
dout(20) << ": remove op" << dendl;
m_snap_sync_ops[end_remote_snap_id].emplace_back(SYNC_OP_TYPE_REMOVE,
0U, 0U, bufferlist());
0U, 0U, bufferlist(),
std::map<uint64_t, uint64_t>());
}
}

Expand Down
2 changes: 1 addition & 1 deletion src/tools/rbd_mirror/image_sync/ObjectCopyRequest.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ class ObjectCopyRequest {
SYNC_OP_TYPE_REMOVE
};

typedef std::tuple<SyncOpType, uint64_t, uint64_t, bufferlist> SyncOp;
typedef std::tuple<SyncOpType, uint64_t, uint64_t, bufferlist, std::map<uint64_t, uint64_t> > SyncOp;
typedef std::list<SyncOp> SyncOps;
typedef std::map<librados::snap_t, SyncOps> SnapSyncOps;
typedef std::map<librados::snap_t, uint8_t> SnapObjectStates;
Expand Down

0 comments on commit b5db8bc

Please sign in to comment.