rbd-mirror A/A: InstanceWatcher watch/notify stub for leader/follower…

… RPC

Fixes: http://tracker.ceph.com/issues/18783
Signed-off-by: Mykola Golub <mgolub@mirantis.com>

src/include/rbd_types.h

@@ -65,12 +65,12 @@
  */
 #define RBD_MIRRORING       "rbd_mirroring"
 
-
 /**
- * rbd_mirror_leader object is used for pool-level coordination
- * between rbd-mirror daemons.
+ * rbd_mirror_leader and rbd_mirror_instance.<instance id> objects are used
+ * for pool-level coordination between rbd-mirror daemons.
  */
-#define RBD_MIRROR_LEADER	"rbd_mirror_leader"
+#define RBD_MIRROR_LEADER               "rbd_mirror_leader"
+#define RBD_MIRROR_INSTANCE_PREFIX      "rbd_mirror_instance."
 
 #define RBD_MAX_OBJ_NAME_SIZE	96
 #define RBD_MAX_BLOCK_NAME_SIZE 24

src/test/rbd_mirror/CMakeLists.txt

@@ -4,6 +4,7 @@ set(rbd_mirror_test_srcs
   test_ImageReplayer.cc
   test_ImageDeleter.cc
   test_ImageSync.cc
+  test_InstanceWatcher.cc
   test_LeaderWatcher.cc
   test_fixture.cc
   )
@@ -17,6 +18,7 @@ add_executable(unittest_rbd_mirror
   test_mock_ImageReplayer.cc
   test_mock_ImageSync.cc
   test_mock_ImageSyncThrottler.cc
+  test_mock_InstanceWatcher.cc
   test_mock_LeaderWatcher.cc
   image_replayer/test_mock_BootstrapRequest.cc
   image_replayer/test_mock_CreateImageRequest.cc

src/test/rbd_mirror/test_InstanceWatcher.cc

@@ -0,0 +1,85 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "include/rados/librados.hpp"
+#include "include/stringify.h"
+#include "cls/rbd/cls_rbd_types.h"
+#include "cls/rbd/cls_rbd_client.h"
+#include "librbd/Utils.h"
+#include "librbd/internal.h"
+#include "test/rbd_mirror/test_fixture.h"
+#include "tools/rbd_mirror/InstanceWatcher.h"
+#include "tools/rbd_mirror/Threads.h"
+
+#include "test/librados/test.h"
+#include "gtest/gtest.h"
+
+using rbd::mirror::InstanceWatcher;
+
+void register_test_instance_watcher() {
+}
+
+class TestInstanceWatcher : public ::rbd::mirror::TestFixture {
+public:
+  std::string m_instance_id;
+  std::string m_oid;
+
+  virtual void SetUp() {
+    TestFixture::SetUp();
+    m_local_io_ctx.remove(RBD_MIRROR_LEADER);
+    EXPECT_EQ(0, m_local_io_ctx.create(RBD_MIRROR_LEADER, true));
+
+    m_instance_id = stringify(m_local_io_ctx.get_instance_id());
+    m_oid = RBD_MIRROR_INSTANCE_PREFIX + m_instance_id;
+  }
+
+  void get_instances(std::vector<std::string> *instance_ids) {
+    instance_ids->clear();
+    C_SaferCond on_get;
+    InstanceWatcher<>::get_instances(m_local_io_ctx, instance_ids, &on_get);
+    EXPECT_EQ(0, on_get.wait());
+  }
+};
+
+TEST_F(TestInstanceWatcher, InitShutdown)
+{
+  InstanceWatcher<> instance_watcher(m_local_io_ctx, m_threads->work_queue,
+                                     true);
+  std::vector<std::string> instance_ids;
+  get_instances(&instance_ids);
+  ASSERT_EQ(0U, instance_ids.size());
+
+  uint64_t size;
+  ASSERT_EQ(-ENOENT, m_local_io_ctx.stat(m_oid, &size, nullptr));
+
+  // Init
+  ASSERT_EQ(0, instance_watcher.init());
+
+  get_instances(&instance_ids);
+  ASSERT_EQ(1U, instance_ids.size());
+  ASSERT_EQ(m_instance_id, instance_ids[0]);
+
+  ASSERT_EQ(0, m_local_io_ctx.stat(m_oid, &size, nullptr));
+  std::list<obj_watch_t> watchers;
+  ASSERT_EQ(0, m_local_io_ctx.list_watchers(m_oid, &watchers));
+  ASSERT_EQ(1U, watchers.size());
+  ASSERT_EQ(m_instance_id, stringify(watchers.begin()->watcher_id));
+
+  InstanceWatcher<> slave_watcher(m_local_io_ctx, m_threads->work_queue, false);
+  ASSERT_EQ(0, slave_watcher.init());
+
+  get_instances(&instance_ids);
+  ASSERT_EQ(1U, instance_ids.size());
+
+  watchers.clear();
+  ASSERT_EQ(0, m_local_io_ctx.list_watchers(m_oid, &watchers));
+  ASSERT_EQ(2U, watchers.size());
+
+  // Shutdown
+  slave_watcher.shut_down();
+  instance_watcher.shut_down();
+
+  ASSERT_EQ(-ENOENT, m_local_io_ctx.stat(m_oid, &size, nullptr));
+  get_instances(&instance_ids);
+  ASSERT_EQ(0U, instance_ids.size());
+}

src/test/rbd_mirror/test_main.cc

@@ -9,20 +9,22 @@
 #include <string>
 
 extern void register_test_cluster_watcher();
+extern void register_test_image_sync();
+extern void register_test_instance_watcher();
+extern void register_test_leader_watcher();
 extern void register_test_pool_watcher();
 extern void register_test_rbd_mirror();
 extern void register_test_rbd_mirror_image_deleter();
-extern void register_test_image_sync();
-extern void register_test_leader_watcher();
 
 int main(int argc, char **argv)
 {
   register_test_cluster_watcher();
+  register_test_image_sync();
+  register_test_instance_watcher();
+  register_test_leader_watcher();
   register_test_pool_watcher();
   register_test_rbd_mirror();
   register_test_rbd_mirror_image_deleter();
-  register_test_image_sync();
-  register_test_leader_watcher();
 
   ::testing::InitGoogleTest(&argc, argv);
 

src/test/rbd_mirror/test_mock_InstanceWatcher.cc

@@ -0,0 +1,221 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "librbd/ManagedLock.h"
+#include "test/librbd/mock/MockImageCtx.h"
+#include "test/librados_test_stub/MockTestMemIoCtxImpl.h"
+#include "test/rbd_mirror/test_mock_fixture.h"
+#include "tools/rbd_mirror/InstanceWatcher.h"
+#include "tools/rbd_mirror/Threads.h"
+
+namespace librbd {
+
+namespace {
+
+struct MockTestImageCtx : public MockImageCtx {
+  MockTestImageCtx(librbd::ImageCtx &image_ctx)
+    : librbd::MockImageCtx(image_ctx) {
+  }
+};
+
+} // anonymous namespace
+
+template <>
+struct ManagedLock<MockTestImageCtx> {
+  static ManagedLock* s_instance;
+  librbd::MockTestImageCtx **image_ctx = nullptr;
+
+  static ManagedLock *create(librados::IoCtx& ioctx, ContextWQ *work_queue,
+			     const std::string& oid, librbd::Watcher *watcher,
+			     managed_lock::Mode  mode,
+			     bool blacklist_on_break_lock,
+			     uint32_t blacklist_expire_seconds) {
+    assert(s_instance != nullptr);
+    return s_instance;
+  }
+
+  ManagedLock() {
+    assert(s_instance == nullptr);
+    s_instance = this;
+  }
+
+  ~ManagedLock() {
+    assert(s_instance == this);
+    s_instance = nullptr;
+  }
+
+  MOCK_METHOD0(destroy, void());
+  MOCK_METHOD1(shut_down, void(Context *));
+  MOCK_METHOD1(acquire_lock, void(Context *));
+};
+
+ManagedLock<MockTestImageCtx> *ManagedLock<MockTestImageCtx>::s_instance = nullptr;
+
+} // namespace librbd
+
+// template definitions
+#include "tools/rbd_mirror/InstanceWatcher.cc"
+template class rbd::mirror::InstanceWatcher<librbd::MockTestImageCtx>;
+
+namespace rbd {
+namespace mirror {
+
+using ::testing::_;
+using ::testing::AtLeast;
+using ::testing::DoAll;
+using ::testing::InSequence;
+using ::testing::Invoke;
+using ::testing::Return;
+using ::testing::StrEq;
+
+class TestMockInstanceWatcher : public TestMockFixture {
+public:
+  typedef librbd::ManagedLock<librbd::MockTestImageCtx> MockManagedLock;
+  typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher;
+
+  std::string m_instance_id;
+  std::string m_oid;
+
+  virtual void SetUp() {
+    TestFixture::SetUp();
+    m_local_io_ctx.remove(RBD_MIRROR_LEADER);
+    EXPECT_EQ(0, m_local_io_ctx.create(RBD_MIRROR_LEADER, true));
+
+    m_instance_id = stringify(m_local_io_ctx.get_instance_id());
+    m_oid = RBD_MIRROR_INSTANCE_PREFIX + m_instance_id;
+  }
+
+  void expect_register_watch(librados::MockTestMemIoCtxImpl &mock_io_ctx) {
+    EXPECT_CALL(mock_io_ctx, aio_watch(m_oid, _, _, _));
+  }
+
+  void expect_unregister_watch(librados::MockTestMemIoCtxImpl &mock_io_ctx) {
+    EXPECT_CALL(mock_io_ctx, aio_unwatch(_, _));
+  }
+
+  void expect_register_instance(librados::MockTestMemIoCtxImpl &mock_io_ctx,
+                                int r) {
+    EXPECT_CALL(mock_io_ctx, exec(RBD_MIRROR_LEADER, _, StrEq("rbd"),
+                                  StrEq("mirror_instances_add"), _, _, _))
+      .WillOnce(Return(r));
+  }
+
+  void expect_unregister_instance(librados::MockTestMemIoCtxImpl &mock_io_ctx,
+                                  int r) {
+    EXPECT_CALL(mock_io_ctx, exec(RBD_MIRROR_LEADER, _, StrEq("rbd"),
+                                  StrEq("mirror_instances_remove"), _, _, _))
+      .WillOnce(Return(r));
+  }
+
+  void expect_acquire_lock(MockManagedLock &mock_managed_lock, int r) {
+    EXPECT_CALL(mock_managed_lock, acquire_lock(_))
+      .WillOnce(CompleteContext(r));
+  }
+
+  void expect_release_lock(MockManagedLock &mock_managed_lock, int r) {
+    EXPECT_CALL(mock_managed_lock, shut_down(_)).WillOnce(CompleteContext(r));
+  }
+
+  void expect_destroy_lock(MockManagedLock &mock_managed_lock) {
+    EXPECT_CALL(mock_managed_lock, destroy());
+  }
+
+};
+
+TEST_F(TestMockInstanceWatcher, InitShutdown) {
+  MockInstanceWatcher instance_watcher(m_local_io_ctx, m_threads->work_queue, true);
+
+  MockManagedLock mock_managed_lock;
+  librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx));
+  InSequence seq;
+
+  // Init
+  expect_register_instance(mock_io_ctx, 0);
+  expect_register_watch(mock_io_ctx);
+  expect_acquire_lock(mock_managed_lock, 0);
+  ASSERT_EQ(0, instance_watcher.init());
+
+  // Shutdown
+  expect_release_lock(mock_managed_lock, 0);
+  expect_destroy_lock(mock_managed_lock);
+  expect_unregister_watch(mock_io_ctx);
+  expect_unregister_instance(mock_io_ctx, 0);
+  instance_watcher.shut_down();
+}
+
+TEST_F(TestMockInstanceWatcher, InitError) {
+  MockInstanceWatcher instance_watcher(m_local_io_ctx, m_threads->work_queue,
+                                       true);
+  MockManagedLock mock_managed_lock;
+  librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx));
+  InSequence seq;
+
+  expect_register_instance(mock_io_ctx, 0);
+  expect_register_watch(mock_io_ctx);
+  expect_acquire_lock(mock_managed_lock, -EINVAL);
+  expect_destroy_lock(mock_managed_lock);
+  expect_unregister_watch(mock_io_ctx);
+  expect_unregister_instance(mock_io_ctx, 0);
+
+  ASSERT_EQ(-EINVAL, instance_watcher.init());
+}
+
+TEST_F(TestMockInstanceWatcher, ShutdownError) {
+  MockInstanceWatcher instance_watcher(m_local_io_ctx, m_threads->work_queue,
+                                       true);
+  MockManagedLock mock_managed_lock;
+  librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx));
+  InSequence seq;
+
+  // Init
+  expect_register_instance(mock_io_ctx, 0);
+  expect_register_watch(mock_io_ctx);
+  expect_acquire_lock(mock_managed_lock, 0);
+  ASSERT_EQ(0, instance_watcher.init());
+
+  // Shutdown
+  expect_release_lock(mock_managed_lock, -EINVAL);
+  expect_destroy_lock(mock_managed_lock);
+  expect_unregister_watch(mock_io_ctx);
+  expect_unregister_instance(mock_io_ctx, 0);
+  instance_watcher.shut_down();
+}
+
+TEST_F(TestMockInstanceWatcher, NotOwnerInitShutdown) {
+  MockInstanceWatcher instance_watcher(m_local_io_ctx, m_threads->work_queue,
+				       false);
+  MockManagedLock mock_managed_lock;
+  librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx));
+  InSequence seq;
+
+  // Init
+  expect_register_watch(mock_io_ctx);
+  ASSERT_EQ(0, instance_watcher.init());
+
+  // Shutdown
+  expect_release_lock(mock_managed_lock, 0);
+  expect_destroy_lock(mock_managed_lock);
+  expect_unregister_watch(mock_io_ctx);
+  instance_watcher.shut_down();
+}
+
+TEST_F(TestMockInstanceWatcher, NotOwnerShutdownError) {
+  MockInstanceWatcher instance_watcher(m_local_io_ctx, m_threads->work_queue,
+                                       false);
+  MockManagedLock mock_managed_lock;
+  librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx));
+  InSequence seq;
+
+  // Init
+  expect_register_watch(mock_io_ctx);
+  ASSERT_EQ(0, instance_watcher.init());
+
+  // Shutdown
+  expect_release_lock(mock_managed_lock, -EINVAL);
+  expect_destroy_lock(mock_managed_lock);
+  expect_unregister_watch(mock_io_ctx);
+  instance_watcher.shut_down();
+}
+
+} // namespace mirror
+} // namespace rbd

src/tools/rbd_mirror/CMakeLists.txt

@@ -3,10 +3,11 @@ add_library(rbd_mirror_types STATIC
 
 set(rbd_mirror_internal
   ClusterWatcher.cc
-  ImageReplayer.cc
   ImageDeleter.cc
+  ImageReplayer.cc
   ImageSync.cc
   ImageSyncThrottler.cc
+  InstanceWatcher.cc
   LeaderWatcher.cc
   Mirror.cc
   MirrorStatusWatcher.cc