Skip to content

Commit

Permalink
rbd-mirror A/A: coordinate image syncs with leader
Browse files Browse the repository at this point in the history 
Fixes: http://tracker.ceph.com/issues/18789
Signed-off-by: Mykola Golub <mgolub@mirantis.com>
  • Loading branch information
Mykola Golub committed May 31, 2017
1 parent f06f07f commit 967ad82
Show file tree
Hide file tree
Showing 32 changed files with 1,629 additions and 793 deletions.
116 changes: 68 additions & 48 deletions src/test/rbd_mirror/image_replayer/test_mock_BootstrapRequest.cc
View file
Expand Up @@ -3,7 +3,7 @@

#include "test/rbd_mirror/test_mock_fixture.h"
#include "librbd/journal/TypeTraits.h"
#include "tools/rbd_mirror/ImageSyncThrottler.h"
#include "tools/rbd_mirror/InstanceWatcher.h"
#include "tools/rbd_mirror/Threads.h"
#include "tools/rbd_mirror/image_replayer/BootstrapRequest.h"
#include "tools/rbd_mirror/image_replayer/CloseImageRequest.h"
Expand Down Expand Up @@ -44,16 +44,42 @@ namespace mirror {
class ProgressContext;

template<>
struct ImageSyncThrottler<librbd::MockTestImageCtx> {
MOCK_METHOD10(start_sync, void(librbd::MockTestImageCtx *local_image_ctx,
librbd::MockTestImageCtx *remote_image_ctx,
SafeTimer *timer, Mutex *timer_lock,
const std::string &mirror_uuid,
::journal::MockJournaler *journaler,
librbd::journal::MirrorPeerClientMeta *client_meta,
ContextWQ *work_queue, Context *on_finish,
ProgressContext *progress_ctx));
MOCK_METHOD1(cancel_sync, void(const std::string& mirror_uuid));
struct ImageSync<librbd::MockTestImageCtx> {
static ImageSync* s_instance;
Context *on_finish = nullptr;

static ImageSync* create(
librbd::MockTestImageCtx *local_image_ctx,
librbd::MockTestImageCtx *remote_image_ctx,
SafeTimer *timer, Mutex *timer_lock, const std::string &mirror_uuid,
::journal::MockJournaler *journaler,
librbd::journal::MirrorPeerClientMeta *client_meta, ContextWQ *work_queue,
InstanceWatcher<librbd::MockTestImageCtx> *instance_watcher,
Context *on_finish, ProgressContext *progress_ctx) {
assert(s_instance != nullptr);
s_instance->on_finish = on_finish;
return s_instance;
}

ImageSync() {
assert(s_instance == nullptr);
s_instance = this;
}
~ImageSync() {
s_instance = nullptr;
}

MOCK_METHOD0(get, void());
MOCK_METHOD0(put, void());
MOCK_METHOD0(send, void());
MOCK_METHOD0(cancel, void());
};

ImageSync<librbd::MockTestImageCtx>*
ImageSync<librbd::MockTestImageCtx>::s_instance = nullptr;

template<>
struct InstanceWatcher<librbd::MockTestImageCtx> {
};

namespace image_replayer {
Expand Down Expand Up @@ -240,10 +266,11 @@ MATCHER_P(IsSameIoCtx, io_ctx, "") {

class TestMockImageReplayerBootstrapRequest : public TestMockFixture {
public:
typedef ImageSyncThrottlerRef<librbd::MockTestImageCtx> MockImageSyncThrottler;
typedef BootstrapRequest<librbd::MockTestImageCtx> MockBootstrapRequest;
typedef CloseImageRequest<librbd::MockTestImageCtx> MockCloseImageRequest;
typedef CreateImageRequest<librbd::MockTestImageCtx> MockCreateImageRequest;
typedef ImageSync<librbd::MockTestImageCtx> MockImageSync;
typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher;
typedef IsPrimaryRequest<librbd::MockTestImageCtx> MockIsPrimaryRequest;
typedef OpenImageRequest<librbd::MockTestImageCtx> MockOpenImageRequest;
typedef OpenLocalImageRequest<librbd::MockTestImageCtx> MockOpenLocalImageRequest;
Expand Down Expand Up @@ -380,14 +407,13 @@ class TestMockImageReplayerBootstrapRequest : public TestMockFixture {
}));
}

void expect_image_sync(MockImageSyncThrottler image_sync_throttler,
int r) {
EXPECT_CALL(*image_sync_throttler, start_sync(_, _, _, _,
StrEq("local mirror uuid"),
_, _, _, _, _))
.WillOnce(WithArg<8>(Invoke([this, r](Context *on_finish) {
m_threads->work_queue->queue(on_finish, r);
})));
void expect_image_sync(MockImageSync &mock_image_sync, int r) {
EXPECT_CALL(mock_image_sync, get());
EXPECT_CALL(mock_image_sync, send())
.WillOnce(Invoke([this, &mock_image_sync, r]() {
m_threads->work_queue->queue(mock_image_sync.on_finish, r);
}));
EXPECT_CALL(mock_image_sync, put());
}

bufferlist encode_tag_data(const librbd::journal::TagData &tag_data) {
Expand All @@ -396,7 +422,7 @@ class TestMockImageReplayerBootstrapRequest : public TestMockFixture {
return bl;
}

MockBootstrapRequest *create_request(MockImageSyncThrottler mock_image_sync_throttler,
MockBootstrapRequest *create_request(MockInstanceWatcher *mock_instance_watcher,
::journal::MockJournaler &mock_journaler,
const std::string &local_image_id,
const std::string &remote_image_id,
Expand All @@ -406,7 +432,7 @@ class TestMockImageReplayerBootstrapRequest : public TestMockFixture {
Context *on_finish) {
return new MockBootstrapRequest(m_local_io_ctx,
m_remote_io_ctx,
mock_image_sync_throttler,
mock_instance_watcher,
&m_local_test_image_ctx,
local_image_id,
remote_image_id,
Expand Down Expand Up @@ -472,10 +498,9 @@ TEST_F(TestMockImageReplayerBootstrapRequest, NonPrimaryRemoteSyncingState) {
expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0);

C_SaferCond ctx;
MockImageSyncThrottler mock_image_sync_throttler(
new ImageSyncThrottler<librbd::MockTestImageCtx>());
MockInstanceWatcher mock_instance_watcher;
MockBootstrapRequest *request = create_request(
mock_image_sync_throttler, mock_journaler, mock_local_image_ctx.id,
&mock_instance_watcher, mock_journaler, mock_local_image_ctx.id,
mock_remote_image_ctx.id, "global image id", "local mirror uuid",
"remote mirror uuid", &ctx);
request->send();
Expand Down Expand Up @@ -547,10 +572,9 @@ TEST_F(TestMockImageReplayerBootstrapRequest, RemoteDemotePromote) {
expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0);

C_SaferCond ctx;
MockImageSyncThrottler mock_image_sync_throttler(
new ImageSyncThrottler<librbd::MockTestImageCtx>());
MockInstanceWatcher mock_instance_watcher;
MockBootstrapRequest *request = create_request(
mock_image_sync_throttler, mock_journaler, mock_local_image_ctx.id,
&mock_instance_watcher, mock_journaler, mock_local_image_ctx.id,
mock_remote_image_ctx.id, "global image id", "local mirror uuid",
"remote mirror uuid", &ctx);
request->send();
Expand Down Expand Up @@ -632,10 +656,9 @@ TEST_F(TestMockImageReplayerBootstrapRequest, MultipleRemoteDemotePromotes) {
expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0);

C_SaferCond ctx;
MockImageSyncThrottler mock_image_sync_throttler(
new ImageSyncThrottler<librbd::MockTestImageCtx>());
MockInstanceWatcher mock_instance_watcher;
MockBootstrapRequest *request = create_request(
mock_image_sync_throttler, mock_journaler, mock_local_image_ctx.id,
&mock_instance_watcher, mock_journaler, mock_local_image_ctx.id,
mock_remote_image_ctx.id, "global image id", "local mirror uuid",
"remote mirror uuid", &ctx);
request->send();
Expand Down Expand Up @@ -705,10 +728,9 @@ TEST_F(TestMockImageReplayerBootstrapRequest, LocalDemoteRemotePromote) {
expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0);

C_SaferCond ctx;
MockImageSyncThrottler mock_image_sync_throttler(
new ImageSyncThrottler<librbd::MockTestImageCtx>());
MockInstanceWatcher mock_instance_watcher;
MockBootstrapRequest *request = create_request(
mock_image_sync_throttler, mock_journaler, mock_local_image_ctx.id,
&mock_instance_watcher, mock_journaler, mock_local_image_ctx.id,
mock_remote_image_ctx.id, "global image id", "local mirror uuid",
"remote mirror uuid", &ctx);
request->send();
Expand Down Expand Up @@ -777,10 +799,9 @@ TEST_F(TestMockImageReplayerBootstrapRequest, SplitBrainForcePromote) {
expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0);

C_SaferCond ctx;
MockImageSyncThrottler mock_image_sync_throttler(
new ImageSyncThrottler<librbd::MockTestImageCtx>());
MockInstanceWatcher mock_instance_watcher;
MockBootstrapRequest *request = create_request(
mock_image_sync_throttler, mock_journaler, mock_local_image_ctx.id,
&mock_instance_watcher, mock_journaler, mock_local_image_ctx.id,
mock_remote_image_ctx.id, "global image id", "local mirror uuid",
"remote mirror uuid", &ctx);
request->send();
Expand Down Expand Up @@ -837,10 +858,9 @@ TEST_F(TestMockImageReplayerBootstrapRequest, ResyncRequested) {
expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0);

C_SaferCond ctx;
MockImageSyncThrottler mock_image_sync_throttler(
new ImageSyncThrottler<librbd::MockTestImageCtx>());
MockInstanceWatcher mock_instance_watcher;
MockBootstrapRequest *request = create_request(
mock_image_sync_throttler, mock_journaler, mock_local_image_ctx.id,
&mock_instance_watcher, mock_journaler, mock_local_image_ctx.id,
mock_remote_image_ctx.id, "global image id", "local mirror uuid",
"remote mirror uuid", &ctx);
m_do_resync = false;
Expand Down Expand Up @@ -905,16 +925,16 @@ TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemote) {
expect_journaler_update_client(mock_journaler, client_data, 0);

// sync the remote image to the local image
MockImageSyncThrottler mock_image_sync_throttler(
new ImageSyncThrottler<librbd::MockTestImageCtx>());
expect_image_sync(mock_image_sync_throttler, 0);
MockImageSync mock_image_sync;
expect_image_sync(mock_image_sync, 0);

MockCloseImageRequest mock_close_image_request;
expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0);

C_SaferCond ctx;
MockInstanceWatcher mock_instance_watcher;
MockBootstrapRequest *request = create_request(
mock_image_sync_throttler, mock_journaler, "",
&mock_instance_watcher, mock_journaler, "",
mock_remote_image_ctx.id, "global image id", "local mirror uuid",
"remote mirror uuid", &ctx);
request->send();
Expand Down Expand Up @@ -981,16 +1001,16 @@ TEST_F(TestMockImageReplayerBootstrapRequest, PrimaryRemoteLocalDeleted) {
expect_journaler_update_client(mock_journaler, client_data, 0);

// sync the remote image to the local image
MockImageSyncThrottler mock_image_sync_throttler(
new ImageSyncThrottler<librbd::MockTestImageCtx>());
expect_image_sync(mock_image_sync_throttler, 0);
MockImageSync mock_image_sync;
expect_image_sync(mock_image_sync, 0);

MockCloseImageRequest mock_close_image_request;
expect_close_image(mock_close_image_request, mock_remote_image_ctx, 0);

C_SaferCond ctx;
MockInstanceWatcher mock_instance_watcher;
MockBootstrapRequest *request = create_request(
mock_image_sync_throttler, mock_journaler, "",
&mock_instance_watcher, mock_journaler, "",
mock_remote_image_ctx.id, "global image id", "local mirror uuid",
"remote mirror uuid", &ctx);
request->send();
Expand Down
20 changes: 13 additions & 7 deletions src/test/rbd_mirror/test_ImageReplayer.cc
View file
Expand Up @@ -36,7 +36,7 @@
#include "librbd/io/ReadResult.h"
#include "tools/rbd_mirror/types.h"
#include "tools/rbd_mirror/ImageReplayer.h"
#include "tools/rbd_mirror/ImageSyncThrottler.h"
#include "tools/rbd_mirror/InstanceWatcher.h"
#include "tools/rbd_mirror/Threads.h"
#include "tools/rbd_mirror/ImageDeleter.h"

Expand Down Expand Up @@ -118,14 +118,19 @@ class TestImageReplayer : public ::rbd::mirror::TestFixture {
m_image_deleter.reset(new rbd::mirror::ImageDeleter(m_threads->work_queue,
m_threads->timer,
&m_threads->timer_lock));
m_image_sync_throttler.reset(new rbd::mirror::ImageSyncThrottler<>());
m_instance_watcher = rbd::mirror::InstanceWatcher<>::create(
m_local_ioctx, m_threads->work_queue, nullptr);
m_instance_watcher->handle_acquire_leader();
}

~TestImageReplayer() override
~TestImageReplayer() override
{
unwatch();

m_instance_watcher->handle_release_leader();

delete m_replayer;
delete m_instance_watcher;
delete m_threads;

EXPECT_EQ(0, m_remote_cluster.pool_delete(m_remote_pool_name.c_str()));
Expand All @@ -134,9 +139,10 @@ class TestImageReplayer : public ::rbd::mirror::TestFixture {

template <typename ImageReplayerT = rbd::mirror::ImageReplayer<> >
void create_replayer() {
m_replayer = new ImageReplayerT(m_threads, m_image_deleter, m_image_sync_throttler,
rbd::mirror::RadosRef(new librados::Rados(m_local_ioctx)),
m_local_mirror_uuid, m_local_ioctx.get_id(), "global image id");
m_replayer = new ImageReplayerT(
m_threads, m_image_deleter, m_instance_watcher,
rbd::mirror::RadosRef(new librados::Rados(m_local_ioctx)),
m_local_mirror_uuid, m_local_ioctx.get_id(), "global image id");
m_replayer->add_remote_image(m_remote_mirror_uuid, m_remote_image_id,
m_remote_ioctx);
}
Expand Down Expand Up @@ -364,7 +370,7 @@ class TestImageReplayer : public ::rbd::mirror::TestFixture {
std::shared_ptr<rbd::mirror::ImageDeleter> m_image_deleter;
std::shared_ptr<librados::Rados> m_local_cluster;
librados::Rados m_remote_cluster;
std::shared_ptr<rbd::mirror::ImageSyncThrottler<>> m_image_sync_throttler;
rbd::mirror::InstanceWatcher<> *m_instance_watcher;
std::string m_local_mirror_uuid = "local mirror uuid";
std::string m_remote_mirror_uuid = "remote mirror uuid";
std::string m_local_pool_name, m_remote_pool_name;
Expand Down
12 changes: 11 additions & 1 deletion src/test/rbd_mirror/test_ImageSync.cc
View file
Expand Up @@ -15,6 +15,7 @@
#include "librbd/io/ReadResult.h"
#include "librbd/journal/Types.h"
#include "tools/rbd_mirror/ImageSync.h"
#include "tools/rbd_mirror/InstanceWatcher.h"
#include "tools/rbd_mirror/Threads.h"

void register_test_image_sync() {
Expand Down Expand Up @@ -55,6 +56,10 @@ class TestImageSync : public TestFixture {
create_and_open(m_local_io_ctx, &m_local_image_ctx);
create_and_open(m_remote_io_ctx, &m_remote_image_ctx);

m_instance_watcher = rbd::mirror::InstanceWatcher<>::create(
m_local_io_ctx, m_threads->work_queue, nullptr);
m_instance_watcher->handle_acquire_leader();

m_remote_journaler = new ::journal::Journaler(
m_threads->work_queue, m_threads->timer, &m_threads->timer_lock,
m_remote_io_ctx, m_remote_image_ctx->id, "mirror-uuid", {});
Expand All @@ -70,7 +75,11 @@ class TestImageSync : public TestFixture {

void TearDown() override {
TestFixture::TearDown();

m_instance_watcher->handle_release_leader();

delete m_remote_journaler;
delete m_instance_watcher;
}

void create_and_open(librados::IoCtx &io_ctx, librbd::ImageCtx **image_ctx) {
Expand All @@ -91,11 +100,12 @@ class TestImageSync : public TestFixture {
return new ImageSync<>(m_local_image_ctx, m_remote_image_ctx,
m_threads->timer, &m_threads->timer_lock,
"mirror-uuid", m_remote_journaler, &m_client_meta,
m_threads->work_queue, ctx);
m_threads->work_queue, m_instance_watcher, ctx);
}

librbd::ImageCtx *m_remote_image_ctx;
librbd::ImageCtx *m_local_image_ctx;
rbd::mirror::InstanceWatcher<> *m_instance_watcher;
::journal::Journaler *m_remote_journaler;
librbd::journal::MirrorPeerClientMeta m_client_meta;
};
Expand Down
3 changes: 3 additions & 0 deletions src/test/rbd_mirror/test_LeaderWatcher.cc
View file
Expand Up @@ -71,6 +71,9 @@ class TestLeaderWatcher : public ::rbd::mirror::TestFixture {
}
}

void update_leader_handler(const std::string &leader_instance_id) override {
}

private:
mutable Mutex m_test_lock;
int m_acquire_count = 0;
Expand Down

0 comments on commit 967ad82

Please sign in to comment.