rbd-mirror A/A: InstanceWatcher watch/notify stub for leader/follower…

… RPC Fixes: http://tracker.ceph.com/issues/18783 Signed-off-by: Mykola Golub <mgolub@mirantis.com>
ceph · Feb 15, 2017 · 04160a1 · 04160a1
1 parent 65a85b3
commit 04160a1
Show file tree

Hide file tree

Showing 10 changed files with 584 additions and 10 deletions.
diff --git a/src/include/rbd_types.h b/src/include/rbd_types.h
@@ -65,12 +65,12 @@
  */
 #define RBD_MIRRORING       "rbd_mirroring"
 
-
 /**
- * rbd_mirror_leader object is used for pool-level coordination
- * between rbd-mirror daemons.
+ * rbd_mirror_leader and rbd_mirror_instance.<instance id> objects are used
+ * for pool-level coordination between rbd-mirror daemons.
  */
-#define RBD_MIRROR_LEADER	"rbd_mirror_leader"
+#define RBD_MIRROR_LEADER               "rbd_mirror_leader"
+#define RBD_MIRROR_INSTANCE_PREFIX      "rbd_mirror_instance."
 
 #define RBD_MAX_OBJ_NAME_SIZE	96
 #define RBD_MAX_BLOCK_NAME_SIZE 24

diff --git a/src/test/rbd_mirror/CMakeLists.txt b/src/test/rbd_mirror/CMakeLists.txt
@@ -4,6 +4,7 @@ set(rbd_mirror_test_srcs
   test_ImageReplayer.cc
   test_ImageDeleter.cc
   test_ImageSync.cc
+  test_InstanceWatcher.cc
   test_LeaderWatcher.cc
   test_fixture.cc
   )
@@ -17,6 +18,7 @@ add_executable(unittest_rbd_mirror
   test_mock_ImageReplayer.cc
   test_mock_ImageSync.cc
   test_mock_ImageSyncThrottler.cc
+  test_mock_InstanceWatcher.cc
   test_mock_LeaderWatcher.cc
   image_replayer/test_mock_BootstrapRequest.cc
   image_replayer/test_mock_CreateImageRequest.cc

diff --git a/src/test/rbd_mirror/test_InstanceWatcher.cc b/src/test/rbd_mirror/test_InstanceWatcher.cc
@@ -0,0 +1,70 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "include/rados/librados.hpp"
+#include "include/stringify.h"
+#include "cls/rbd/cls_rbd_types.h"
+#include "cls/rbd/cls_rbd_client.h"
+#include "librbd/internal.h"
+#include "librbd/Utils.h"
+#include "test/rbd_mirror/test_fixture.h"
+#include "tools/rbd_mirror/InstanceWatcher.h"
+#include "tools/rbd_mirror/Threads.h"
+
+#include "test/librados/test.h"
+#include "gtest/gtest.h"
+
+using rbd::mirror::InstanceWatcher;
+
+void register_test_instance_watcher() {
+}
+
+class TestInstanceWatcher : public ::rbd::mirror::TestFixture {
+public:
+  std::string m_instance_id;
+  std::string m_oid;
+
+  virtual void SetUp() {
+    TestFixture::SetUp();
+    m_local_io_ctx.remove(RBD_MIRROR_LEADER);
+    EXPECT_EQ(0, m_local_io_ctx.create(RBD_MIRROR_LEADER, true));
+
+    m_instance_id = stringify(m_local_io_ctx.get_instance_id());
+    m_oid = RBD_MIRROR_INSTANCE_PREFIX + m_instance_id;
+  }
+};
+
+TEST_F(TestInstanceWatcher, InitShutdown)
+{
+  InstanceWatcher<> instance_watcher(m_local_io_ctx, m_threads->work_queue);
+
+  std::vector<std::string> instance_ids;
+  ASSERT_EQ(0, librbd::cls_client::mirror_instances_list(&m_local_io_ctx,
+                                                         &instance_ids));
+  ASSERT_EQ(0U, instance_ids.size());
+
+  uint64_t size;
+  ASSERT_EQ(-ENOENT, m_local_io_ctx.stat(m_oid, &size, nullptr));
+
+  // Init
+  ASSERT_EQ(0, instance_watcher.init());
+
+  ASSERT_EQ(0, librbd::cls_client::mirror_instances_list(&m_local_io_ctx,
+                                                         &instance_ids));
+  ASSERT_EQ(1U, instance_ids.size());
+  ASSERT_EQ(m_instance_id, instance_ids[0]);
+
+  ASSERT_EQ(0, m_local_io_ctx.stat(m_oid, &size, nullptr));
+  std::list<obj_watch_t> watchers;
+  ASSERT_EQ(0, m_local_io_ctx.list_watchers(m_oid, &watchers));
+  ASSERT_EQ(1U, watchers.size());
+  ASSERT_EQ(m_instance_id, stringify(watchers.begin()->watcher_id));
+
+  // Shutdown
+  instance_watcher.shut_down();
+
+  ASSERT_EQ(-ENOENT, m_local_io_ctx.stat(m_oid, &size, nullptr));
+  ASSERT_EQ(0, librbd::cls_client::mirror_instances_list(&m_local_io_ctx,
+                                                         &instance_ids));
+  ASSERT_EQ(0U, instance_ids.size());
+}
diff --git a/src/test/rbd_mirror/test_main.cc b/src/test/rbd_mirror/test_main.cc
@@ -9,20 +9,22 @@
 #include <string>
 
 extern void register_test_cluster_watcher();
+extern void register_test_image_sync();
+extern void register_test_instance_watcher();
+extern void register_test_leader_watcher();
 extern void register_test_pool_watcher();
 extern void register_test_rbd_mirror();
 extern void register_test_rbd_mirror_image_deleter();
-extern void register_test_image_sync();
-extern void register_test_leader_watcher();
 
 int main(int argc, char **argv)
 {
   register_test_cluster_watcher();
+  register_test_image_sync();
+  register_test_instance_watcher();
+  register_test_leader_watcher();
   register_test_pool_watcher();
   register_test_rbd_mirror();
   register_test_rbd_mirror_image_deleter();
-  register_test_image_sync();
-  register_test_leader_watcher();
 
   ::testing::InitGoogleTest(&argc, argv);
 

diff --git a/src/test/rbd_mirror/test_mock_InstanceWatcher.cc b/src/test/rbd_mirror/test_mock_InstanceWatcher.cc
@@ -0,0 +1,123 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "test/librbd/mock/MockImageCtx.h"
+#include "test/librados_test_stub/MockTestMemIoCtxImpl.h"
+#include "test/rbd_mirror/test_mock_fixture.h"
+#include "tools/rbd_mirror/InstanceWatcher.h"
+#include "tools/rbd_mirror/Threads.h"
+
+namespace librbd {
+
+namespace {
+
+struct MockTestImageCtx : public MockImageCtx {
+  MockTestImageCtx(librbd::ImageCtx &image_ctx)
+    : librbd::MockImageCtx(image_ctx) {
+  }
+};
+
+} // anonymous namespace
+
+} // namespace librbd
+
+// template definitions
+#include "tools/rbd_mirror/InstanceWatcher.cc"
+template class rbd::mirror::InstanceWatcher<librbd::MockTestImageCtx>;
+
+namespace rbd {
+namespace mirror {
+
+using ::testing::_;
+using ::testing::AtLeast;
+using ::testing::DoAll;
+using ::testing::InSequence;
+using ::testing::Invoke;
+using ::testing::Return;
+using ::testing::StrEq;
+
+class TestMockInstanceWatcher : public TestMockFixture {
+public:
+  typedef InstanceWatcher<librbd::MockTestImageCtx> MockInstanceWatcher;
+
+  std::string m_instance_id;
+  std::string m_oid;
+
+  virtual void SetUp() {
+    TestFixture::SetUp();
+    m_local_io_ctx.remove(RBD_MIRROR_LEADER);
+    EXPECT_EQ(0, m_local_io_ctx.create(RBD_MIRROR_LEADER, true));
+
+    m_instance_id = stringify(m_local_io_ctx.get_instance_id());
+    m_oid = RBD_MIRROR_INSTANCE_PREFIX + m_instance_id;
+  }
+
+  void expect_register_watch(librados::MockTestMemIoCtxImpl &mock_io_ctx) {
+    EXPECT_CALL(mock_io_ctx, aio_watch(m_oid, _, _, _));
+  }
+
+  void expect_unregister_watch(librados::MockTestMemIoCtxImpl &mock_io_ctx) {
+    EXPECT_CALL(mock_io_ctx, aio_unwatch(_, _));
+  }
+
+  void expect_register_instance(librados::MockTestMemIoCtxImpl &mock_io_ctx,
+                                int r) {
+    EXPECT_CALL(mock_io_ctx, exec(RBD_MIRROR_LEADER, _, StrEq("rbd"),
+                                  StrEq("mirror_instances_add"), _, _, _))
+      .WillOnce(Return(r));
+  }
+
+  void expect_unregister_instance(librados::MockTestMemIoCtxImpl &mock_io_ctx,
+                                  int r) {
+    EXPECT_CALL(mock_io_ctx, exec(RBD_MIRROR_LEADER, _, StrEq("rbd"),
+                                  StrEq("mirror_instances_remove"), _, _, _))
+      .WillOnce(Return(r));
+  }
+};
+
+TEST_F(TestMockInstanceWatcher, InitShutdown) {
+  MockInstanceWatcher instance_watcher(m_local_io_ctx, m_threads->work_queue);
+
+  librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx));
+  InSequence seq;
+
+  // Init
+  expect_register_instance(mock_io_ctx, 0);
+  expect_register_watch(mock_io_ctx);
+  ASSERT_EQ(0, instance_watcher.init());
+
+  // Shutdown
+  expect_unregister_watch(mock_io_ctx);
+  expect_unregister_instance(mock_io_ctx, 0);
+  instance_watcher.shut_down();
+}
+
+TEST_F(TestMockInstanceWatcher, InitError) {
+  MockInstanceWatcher instance_watcher(m_local_io_ctx, m_threads->work_queue);
+
+  librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx));
+  InSequence seq;
+
+  expect_register_instance(mock_io_ctx, -EINVAL);
+  ASSERT_EQ(-EINVAL, instance_watcher.init());
+}
+
+TEST_F(TestMockInstanceWatcher, ShutdownError) {
+  MockInstanceWatcher instance_watcher(m_local_io_ctx, m_threads->work_queue);
+
+  librados::MockTestMemIoCtxImpl &mock_io_ctx(get_mock_io_ctx(m_local_io_ctx));
+  InSequence seq;
+
+  // Init
+  expect_register_instance(mock_io_ctx, 0);
+  expect_register_watch(mock_io_ctx);
+  ASSERT_EQ(0, instance_watcher.init());
+
+  // Shutdown
+  expect_unregister_watch(mock_io_ctx);
+  expect_unregister_instance(mock_io_ctx, -EINVAL);
+  instance_watcher.shut_down();
+}
+
+} // namespace mirror
+} // namespace rbd
diff --git a/src/tools/rbd_mirror/CMakeLists.txt b/src/tools/rbd_mirror/CMakeLists.txt
@@ -3,10 +3,11 @@ add_library(rbd_mirror_types STATIC
 
 set(rbd_mirror_internal
   ClusterWatcher.cc
-  ImageReplayer.cc
   ImageDeleter.cc
+  ImageReplayer.cc
   ImageSync.cc
   ImageSyncThrottler.cc
+  InstanceWatcher.cc
   LeaderWatcher.cc
   Mirror.cc
   MirrorStatusWatcher.cc