rbd-mirror: option to automatically resync after journal client disco…

…nnect

Signed-off-by: Mykola Golub <mgolub@mirantis.com>

qa/workunits/rbd/rbd_mirror.sh

@@ -290,4 +290,20 @@ wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
 test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
 compare_images ${POOL} ${image}
 
+testlog " - rbd_mirroring_resync_after_disconnect config option"
+set_image_meta ${CLUSTER1} ${POOL} ${image} \
+	       conf_rbd_mirroring_resync_after_disconnect true
+disconnect_image ${CLUSTER2} ${POOL} ${image}
+wait_for_image_present ${CLUSTER1} ${POOL} ${image} 'deleted'
+wait_for_image_replay_started ${CLUSTER1} ${POOL} ${image}
+wait_for_replay_complete ${CLUSTER1} ${CLUSTER2} ${POOL} ${image}
+test -n "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
+compare_images ${POOL} ${image}
+set_image_meta ${CLUSTER1} ${POOL} ${image} \
+	       conf_rbd_mirroring_resync_after_disconnect false
+disconnect_image ${CLUSTER2} ${POOL} ${image}
+test -z "$(get_mirror_position ${CLUSTER2} ${POOL} ${image})"
+wait_for_image_replay_stopped ${CLUSTER1} ${POOL} ${image}
+test_status_in_pool_dir ${CLUSTER1} ${POOL} ${image} 'up+error' 'disconnected'
+
 echo OK

src/common/config_opts.h

@@ -1207,6 +1207,7 @@ OPTION(rbd_enable_alloc_hint, OPT_BOOL, true) // when writing a object, it will
 OPTION(rbd_tracing, OPT_BOOL, false) // true if LTTng-UST tracepoints should be enabled
 OPTION(rbd_validate_pool, OPT_BOOL, true) // true if empty pools should be validated for RBD compatibility
 OPTION(rbd_validate_names, OPT_BOOL, true) // true if image specs should be validated
+OPTION(rbd_mirroring_resync_after_disconnect, OPT_BOOL, false) // automatically start image resync after mirroring is disconnected due to being laggy
 
 /*
  * The following options change the behavior for librbd's image creation methods that

src/librbd/ImageCtx.cc

@@ -944,7 +944,8 @@ struct C_InvalidateCache : public Context {
         "rbd_journal_object_flush_age", false)(
         "rbd_journal_pool", false)(
         "rbd_journal_max_payload_bytes", false)(
-        "rbd_journal_max_concurrent_object_sets", false);
+        "rbd_journal_max_concurrent_object_sets", false)(
+        "rbd_mirroring_resync_after_disconnect", false);
 
     md_config_t local_config_t;
     std::map<std::string, bufferlist> res;
@@ -1001,6 +1002,7 @@ struct C_InvalidateCache : public Context {
     ASSIGN_OPTION(journal_pool);
     ASSIGN_OPTION(journal_max_payload_bytes);
     ASSIGN_OPTION(journal_max_concurrent_object_sets);
+    ASSIGN_OPTION(mirroring_resync_after_disconnect);
   }
 
   ExclusiveLock<ImageCtx> *ImageCtx::create_exclusive_lock() {

src/librbd/ImageCtx.h

@@ -187,6 +187,7 @@ namespace librbd {
     std::string journal_pool;
     uint32_t journal_max_payload_bytes;
     int journal_max_concurrent_object_sets;
+    bool mirroring_resync_after_disconnect;
 
     LibrbdAdminSocketHook *asok_hook;
 

src/test/librbd/mock/MockImageCtx.h

@@ -93,7 +93,9 @@ struct MockImageCtx {
       journal_pool(image_ctx.journal_pool),
       journal_max_payload_bytes(image_ctx.journal_max_payload_bytes),
       journal_max_concurrent_object_sets(
-          image_ctx.journal_max_concurrent_object_sets)
+          image_ctx.journal_max_concurrent_object_sets),
+      mirroring_resync_after_disconnect(
+          image_ctx.mirroring_resync_after_disconnect)
   {
     md_ctx.dup(image_ctx.md_ctx);
     data_ctx.dup(image_ctx.data_ctx);
@@ -260,6 +262,7 @@ struct MockImageCtx {
   std::string journal_pool;
   uint32_t journal_max_payload_bytes;
   int journal_max_concurrent_object_sets;
+  bool mirroring_resync_after_disconnect;
 };
 
 } // namespace librbd

src/test/rbd_mirror/test_ImageReplayer.cc

@@ -829,6 +829,9 @@ TEST_F(TestImageReplayer, Disconnect)
 {
   bootstrap();
 
+  // Make sure rbd_mirroring_resync_after_disconnect is not set
+  EXPECT_EQ(0, m_local_cluster->conf_set("rbd_mirroring_resync_after_disconnect", "false"));
+
   // Test start fails if disconnected
 
   librbd::ImageCtx *ictx;
@@ -889,4 +892,24 @@ TEST_F(TestImageReplayer, Disconnect)
   C_SaferCond cond4;
   m_replayer->start(&cond4);
   ASSERT_EQ(-ENOTCONN, cond4.wait());
+
+  // Test automatic resync if rbd_mirroring_resync_after_disconnect is set
+
+  EXPECT_EQ(0, m_local_cluster->conf_set("rbd_mirroring_resync_after_disconnect", "true"));
+
+  // Resync is flagged on first start attempt
+  C_SaferCond cond5;
+  m_replayer->start(&cond5);
+  ASSERT_EQ(-ENOTCONN, cond5.wait());
+  C_SaferCond delete_cond1;
+  m_image_deleter->wait_for_scheduled_deletion(
+    m_local_ioctx.get_id(), m_replayer->get_global_image_id(), &delete_cond1);
+  EXPECT_EQ(0, delete_cond1.wait());
+
+  C_SaferCond cond6;
+  m_replayer->start(&cond6);
+  ASSERT_EQ(0, cond6.wait());
+  wait_for_replay_complete();
+
+  stop();
 }

src/tools/rbd_mirror/ImageReplayer.cc

@@ -532,6 +532,10 @@ void ImageReplayer<I>::handle_init_remote_journaler(int r) {
 
   if (client.state != cls::journal::CLIENT_STATE_CONNECTED) {
     dout(5) << "client flagged disconnected, stopping image replay" << dendl;
+    if (m_local_image_ctx->mirroring_resync_after_disconnect) {
+      Mutex::Locker locker(m_lock);
+      m_stopping_for_resync = true;
+    }
     on_start_fail(-ENOTCONN, "disconnected");
     return;
   }