rbd-mirror: support deferred deletions of mirrored images

src/cls/rbd/cls_rbd_types.h

@@ -354,13 +354,22 @@ struct TrashImageSpec {
 
   TrashImageSpec() {}
   TrashImageSpec(TrashImageSource source, const std::string &name,
-                   utime_t deletion_time, utime_t deferment_end_time) :
-    source(source), name(name), deletion_time(deletion_time),
-    deferment_end_time(deferment_end_time) {}
+                 const utime_t& deletion_time,
+                 const utime_t& deferment_end_time)
+    : source(source), name(name), deletion_time(deletion_time),
+      deferment_end_time(deferment_end_time) {
+  }
 
   void encode(bufferlist &bl) const;
   void decode(bufferlist::iterator& it);
   void dump(Formatter *f) const;
+
+  inline bool operator==(const TrashImageSpec& rhs) const {
+    return (source == rhs.source &&
+            name == rhs.name &&
+            deletion_time == rhs.deletion_time &&
+            deferment_end_time == rhs.deferment_end_time);
+  }
 };
 WRITE_CLASS_ENCODER(TrashImageSpec);
 

src/common/options.cc

@@ -5873,6 +5873,10 @@ static std::vector<Option> get_rbd_options() {
     .set_default(false)
     .set_description("automatically start image resync after mirroring is disconnected due to being laggy"),
 
+    Option("rbd_mirroring_delete_delay", Option::TYPE_UINT, Option::LEVEL_ADVANCED)
+    .set_default(0)
+    .set_description("time-delay in seconds for rbd-mirror delete propagation"),
+
     Option("rbd_mirroring_replay_delay", Option::TYPE_INT, Option::LEVEL_ADVANCED)
     .set_default(0)
     .set_description("time-delay in seconds for rbd-mirror asynchronous replication"),

src/librbd/CMakeLists.txt

@@ -1,6 +1,7 @@
 add_library(rbd_types STATIC
   journal/Types.cc
   mirroring_watcher/Types.cc
+  trash_watcher/Types.cc
   watcher/Types.cc
   WatchNotifyTypes.cc)
 
@@ -20,6 +21,7 @@ set(librbd_internal_srcs
   MirroringWatcher.cc
   ObjectMap.cc
   Operations.cc
+  TrashWatcher.cc
   Utils.cc
   Watcher.cc
   api/DiffIterate.cc
@@ -63,6 +65,7 @@ set(librbd_internal_srcs
   journal/PromoteRequest.cc
   journal/RemoveRequest.cc
   journal/Replay.cc
+  journal/ResetRequest.cc
   journal/StandardPolicy.cc
   journal/Utils.cc
   managed_lock/AcquireRequest.cc
@@ -107,6 +110,7 @@ set(librbd_internal_srcs
   operation/SnapshotUnprotectRequest.cc
   operation/SnapshotLimitRequest.cc
   operation/TrimRequest.cc
+  trash/MoveRequest.cc
   watcher/Notifier.cc
   watcher/RewatchRequest.cc
   ${CMAKE_SOURCE_DIR}/src/common/ContextCompletion.cc)

src/librbd/ImageCtx.cc

@@ -1020,6 +1020,7 @@ struct C_InvalidateCache : public Context {
         "rbd_journal_max_payload_bytes", false)(
         "rbd_journal_max_concurrent_object_sets", false)(
         "rbd_mirroring_resync_after_disconnect", false)(
+        "rbd_mirroring_delete_delay", false)(
         "rbd_mirroring_replay_delay", false)(
         "rbd_skip_partial_discard", false)(
 	"rbd_qos_iops_limit", false);
@@ -1080,6 +1081,7 @@ struct C_InvalidateCache : public Context {
     ASSIGN_OPTION(journal_max_payload_bytes, uint64_t);
     ASSIGN_OPTION(journal_max_concurrent_object_sets, int64_t);
     ASSIGN_OPTION(mirroring_resync_after_disconnect, bool);
+    ASSIGN_OPTION(mirroring_delete_delay, uint64_t);
     ASSIGN_OPTION(mirroring_replay_delay, int64_t);
     ASSIGN_OPTION(skip_partial_discard, bool);
     ASSIGN_OPTION(blkin_trace_all, bool);

src/librbd/ImageCtx.h

@@ -196,6 +196,7 @@ namespace librbd {
     uint32_t journal_max_payload_bytes;
     int journal_max_concurrent_object_sets;
     bool mirroring_resync_after_disconnect;
+    uint64_t mirroring_delete_delay;
     int mirroring_replay_delay;
     bool skip_partial_discard;
     bool blkin_trace_all;

src/librbd/Journal.cc

@@ -20,6 +20,7 @@
 #include "librbd/journal/DemoteRequest.h"
 #include "librbd/journal/OpenRequest.h"
 #include "librbd/journal/RemoveRequest.h"
+#include "librbd/journal/ResetRequest.h"
 #include "librbd/journal/Replay.h"
 #include "librbd/journal/PromoteRequest.h"
 
@@ -405,54 +406,17 @@ int Journal<I>::reset(librados::IoCtx &io_ctx, const std::string &image_id) {
   CephContext *cct = reinterpret_cast<CephContext *>(io_ctx.cct());
   ldout(cct, 5) << __func__ << ": image=" << image_id << dendl;
 
-  Journaler journaler(io_ctx, image_id, IMAGE_CLIENT_ID, {});
+  ThreadPool *thread_pool;
+  ContextWQ *op_work_queue;
+  ImageCtx::get_thread_pool_instance(cct, &thread_pool, &op_work_queue);
 
   C_SaferCond cond;
-  journaler.init(&cond);
-  BOOST_SCOPE_EXIT_ALL(&journaler) {
-    journaler.shut_down();
-  };
-
-  int r = cond.wait();
-  if (r == -ENOENT) {
-    return 0;
-  } else if (r < 0) {
-    lderr(cct) << __func__ << ": "
-               << "failed to initialize journal: " << cpp_strerror(r) << dendl;
-    return r;
-  }
-
-  uint8_t order, splay_width;
-  int64_t pool_id;
-  journaler.get_metadata(&order, &splay_width, &pool_id);
-
-  std::string pool_name;
-  if (pool_id != -1) {
-    librados::Rados rados(io_ctx);
-    r = rados.pool_reverse_lookup(pool_id, &pool_name);
-    if (r < 0) {
-      lderr(cct) << __func__ << ": "
-                 << "failed to lookup data pool: " << cpp_strerror(r) << dendl;
-      return r;
-    }
-  }
-
-  C_SaferCond ctx1;
-  journaler.remove(true, &ctx1);
-  r = ctx1.wait();
-  if (r < 0) {
-    lderr(cct) << __func__ << ": "
-               << "failed to reset journal: " << cpp_strerror(r) << dendl;
-    return r;
-  }
+  auto req = journal::ResetRequest<I>::create(io_ctx, image_id, IMAGE_CLIENT_ID,
+                                              Journal<>::LOCAL_MIRROR_UUID,
+                                              op_work_queue, &cond);
+  req->send();
 
-  r = create(io_ctx, image_id, order, splay_width, pool_name);
-  if (r < 0) {
-    lderr(cct) << __func__ << ": "
-               << "failed to create journal: " << cpp_strerror(r) << dendl;
-    return r;
-  }
-  return 0;
+  return cond.wait();
 }
 
 template <typename I>

src/librbd/TrashWatcher.cc

@@ -0,0 +1,115 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "librbd/TrashWatcher.h"
+#include "include/rbd_types.h"
+#include "include/rados/librados.hpp"
+#include "common/errno.h"
+#include "librbd/Utils.h"
+#include "librbd/watcher/Utils.h"
+
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd::TrashWatcher: " << __func__ << ": "
+
+namespace librbd {
+
+using namespace trash_watcher;
+using namespace watcher;
+
+using librbd::util::create_rados_callback;
+
+namespace {
+
+static const uint64_t NOTIFY_TIMEOUT_MS = 5000;
+
+} // anonymous namespace
+
+template <typename I>
+TrashWatcher<I>::TrashWatcher(librados::IoCtx &io_ctx, ContextWQ *work_queue)
+  : Watcher(io_ctx, work_queue, RBD_TRASH) {
+}
+
+template <typename I>
+void TrashWatcher<I>::notify_image_added(
+    librados::IoCtx &io_ctx, const std::string& image_id,
+    const cls::rbd::TrashImageSpec& trash_image_spec, Context *on_finish) {
+  CephContext *cct = reinterpret_cast<CephContext*>(io_ctx.cct());
+  ldout(cct, 20) << dendl;
+
+  bufferlist bl;
+  ::encode(NotifyMessage{ImageAddedPayload{image_id, trash_image_spec}}, bl);
+
+  librados::AioCompletion *comp = create_rados_callback(on_finish);
+  int r = io_ctx.aio_notify(RBD_TRASH, comp, bl, NOTIFY_TIMEOUT_MS, nullptr);
+  assert(r == 0);
+  comp->release();
+}
+
+template <typename I>
+void TrashWatcher<I>::notify_image_removed(librados::IoCtx &io_ctx,
+                                           const std::string& image_id,
+                                           Context *on_finish) {
+  CephContext *cct = reinterpret_cast<CephContext*>(io_ctx.cct());
+  ldout(cct, 20) << dendl;
+
+  bufferlist bl;
+  ::encode(NotifyMessage{ImageRemovedPayload{image_id}}, bl);
+
+  librados::AioCompletion *comp = create_rados_callback(on_finish);
+  int r = io_ctx.aio_notify(RBD_TRASH, comp, bl, NOTIFY_TIMEOUT_MS, nullptr);
+  assert(r == 0);
+  comp->release();
+}
+
+template <typename I>
+void TrashWatcher<I>::handle_notify(uint64_t notify_id, uint64_t handle,
+                                    uint64_t notifier_id, bufferlist &bl) {
+  CephContext *cct = this->m_cct;
+  ldout(cct, 15) << "notify_id=" << notify_id << ", "
+                 << "handle=" << handle << dendl;
+
+
+  NotifyMessage notify_message;
+  try {
+    bufferlist::iterator iter = bl.begin();
+    ::decode(notify_message, iter);
+  } catch (const buffer::error &err) {
+    lderr(cct) << "error decoding image notification: " << err.what()
+               << dendl;
+    Context *ctx = new C_NotifyAck(this, notify_id, handle);
+    ctx->complete(0);
+    return;
+  }
+
+  apply_visitor(watcher::util::HandlePayloadVisitor<TrashWatcher<I>>(
+    this, notify_id, handle), notify_message.payload);
+}
+
+template <typename I>
+bool TrashWatcher<I>::handle_payload(const ImageAddedPayload &payload,
+                                     Context *on_notify_ack) {
+  CephContext *cct = this->m_cct;
+  ldout(cct, 20) << dendl;
+  handle_image_added(payload.image_id, payload.trash_image_spec);
+  return true;
+}
+
+template <typename I>
+bool TrashWatcher<I>::handle_payload(const ImageRemovedPayload &payload,
+                                     Context *on_notify_ack) {
+  CephContext *cct = this->m_cct;
+  ldout(cct, 20) << dendl;
+  handle_image_removed(payload.image_id);
+  return true;
+}
+
+template <typename I>
+bool TrashWatcher<I>::handle_payload(const UnknownPayload &payload,
+                                     Context *on_notify_ack) {
+  return true;
+}
+
+} // namespace librbd
+
+template class librbd::TrashWatcher<librbd::ImageCtx>;

src/librbd/TrashWatcher.h

@@ -0,0 +1,58 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_TRASH_WATCHER_H
+#define CEPH_LIBRBD_TRASH_WATCHER_H
+
+#include "include/int_types.h"
+#include "cls/rbd/cls_rbd_types.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/Watcher.h"
+#include "librbd/trash_watcher/Types.h"
+
+namespace librados { class IoCtx; }
+
+namespace librbd {
+
+namespace watcher {
+namespace util {
+template <typename> struct HandlePayloadVisitor;
+} // namespace util
+} // namespace watcher
+
+template <typename ImageCtxT = librbd::ImageCtx>
+class TrashWatcher : public Watcher {
+  friend struct watcher::util::HandlePayloadVisitor<TrashWatcher<ImageCtxT>>;
+public:
+  TrashWatcher(librados::IoCtx &io_ctx, ContextWQ *work_queue);
+
+  static void notify_image_added(librados::IoCtx &io_ctx,
+                                 const std::string& image_id,
+                                 const cls::rbd::TrashImageSpec& spec,
+                                 Context *on_finish);
+  static void notify_image_removed(librados::IoCtx &io_ctx,
+                                   const std::string& image_id,
+                                   Context *on_finish);
+
+protected:
+  virtual void handle_image_added(const std::string &image_id,
+                                  const cls::rbd::TrashImageSpec& spec) = 0;
+  virtual void handle_image_removed(const std::string &image_id) = 0;
+
+private:
+  void handle_notify(uint64_t notify_id, uint64_t handle,
+                     uint64_t notifier_id, bufferlist &bl) override;
+
+  bool handle_payload(const trash_watcher::ImageAddedPayload &payload,
+                      Context *on_notify_ack);
+  bool handle_payload(const trash_watcher::ImageRemovedPayload &payload,
+                      Context *on_notify_ack);
+  bool handle_payload(const trash_watcher::UnknownPayload &payload,
+                      Context *on_notify_ack);
+};
+
+} // namespace librbd
+
+extern template class librbd::TrashWatcher<librbd::ImageCtx>;
+
+#endif // CEPH_LIBRBD_TRASH_WATCHER_H