librbd: separate locker query into standalone state machine

Signed-off-by: Jason Dillaman <dillaman@redhat.com>

src/librbd/CMakeLists.txt

@@ -31,6 +31,7 @@ set(librbd_internal_srcs
   cache/PassthroughImageCache.cc
   exclusive_lock/AcquireRequest.cc
   exclusive_lock/AutomaticPolicy.cc
+  exclusive_lock/GetLockerRequest.cc
   exclusive_lock/ReacquireRequest.cc
   exclusive_lock/ReleaseRequest.cc
   exclusive_lock/StandardPolicy.cc

src/librbd/exclusive_lock/AcquireRequest.cc

@@ -15,6 +15,7 @@
 #include "librbd/Journal.h"
 #include "librbd/ObjectMap.h"
 #include "librbd/Utils.h"
+#include "librbd/exclusive_lock/GetLockerRequest.h"
 #include "librbd/image/RefreshRequest.h"
 #include "librbd/journal/Policy.h"
 
@@ -154,7 +155,7 @@ Context *AcquireRequest<I>::handle_lock(int *ret_val) {
     return m_on_finish;
   }
 
-  send_get_lockers();
+  send_get_locker();
   return nullptr;
 }
 
@@ -393,83 +394,34 @@ Context *AcquireRequest<I>::handle_unlock(int *ret_val) {
 }
 
 template <typename I>
-void AcquireRequest<I>::send_get_lockers() {
+void AcquireRequest<I>::send_get_locker() {
   CephContext *cct = m_image_ctx.cct;
   ldout(cct, 10) << __func__ << dendl;
 
-  librados::ObjectReadOperation op;
-  rados::cls::lock::get_lock_info_start(&op, RBD_LOCK_NAME);
-
-  using klass = AcquireRequest<I>;
-  librados::AioCompletion *rados_completion =
-    create_rados_ack_callback<klass, &klass::handle_get_lockers>(this);
-  m_out_bl.clear();
-  int r = m_image_ctx.md_ctx.aio_operate(m_image_ctx.header_oid,
-                                         rados_completion, &op, &m_out_bl);
-  assert(r == 0);
-  rados_completion->release();
+  Context *ctx = create_context_callback<
+    AcquireRequest<I>, &AcquireRequest<I>::handle_get_locker>(this);
+  auto req = GetLockerRequest<I>::create(m_image_ctx, &m_locker, ctx);
+  req->send();
 }
 
 template <typename I>
-Context *AcquireRequest<I>::handle_get_lockers(int *ret_val) {
+Context *AcquireRequest<I>::handle_get_locker(int *ret_val) {
   CephContext *cct = m_image_ctx.cct;
   ldout(cct, 10) << __func__ << ": r=" << *ret_val << dendl;
 
-  std::map<rados::cls::lock::locker_id_t,
-           rados::cls::lock::locker_info_t> lockers;
-  ClsLockType lock_type = LOCK_NONE;
-  std::string lock_tag;
-  if (*ret_val == 0) {
-    bufferlist::iterator it = m_out_bl.begin();
-    *ret_val = rados::cls::lock::get_lock_info_finish(&it, &lockers,
-                                                      &lock_type, &lock_tag);
-  }
-
-  if (*ret_val < 0) {
-    lderr(cct) << "failed to retrieve lockers: " << cpp_strerror(*ret_val)
-               << dendl;
-    return m_on_finish;
-  }
-
-  if (lockers.empty()) {
+  if (*ret_val == -ENOENT) {
     ldout(cct, 20) << "no lockers detected" << dendl;
     send_lock();
     return nullptr;
-  }
-
-  if (lock_tag != ExclusiveLock<>::WATCHER_LOCK_TAG) {
-    ldout(cct, 5) <<"locked by external mechanism: tag=" << lock_tag << dendl;
-    *ret_val = -EBUSY;
+  } else if (*ret_val == -EBUSY) {
+    ldout(cct, 5) << "incompatible lock detected" << dendl;
     return m_on_finish;
-  }
-
-  if (lock_type == LOCK_SHARED) {
-    ldout(cct, 5) << "shared lock type detected" << dendl;
-    *ret_val = -EBUSY;
-    return m_on_finish;
-  }
-
-  std::map<rados::cls::lock::locker_id_t,
-           rados::cls::lock::locker_info_t>::iterator iter = lockers.begin();
-  if (!ExclusiveLock<>::decode_lock_cookie(iter->first.cookie,
-                                            &m_locker_handle)) {
-    ldout(cct, 5) << "locked by external mechanism: "
-                  << "cookie=" << iter->first.cookie << dendl;
-    *ret_val = -EBUSY;
+  } else if (*ret_val < 0) {
+    lderr(cct) << "failed to retrieve lockers: " << cpp_strerror(*ret_val)
+               << dendl;
     return m_on_finish;
   }
 
-  m_locker_entity = iter->first.locker;
-  m_locker_cookie = iter->first.cookie;
-  m_locker_address = stringify(iter->second.addr);
-  if (m_locker_cookie.empty() || m_locker_address.empty()) {
-    ldout(cct, 20) << "no valid lockers detected" << dendl;
-    send_lock();
-    return nullptr;
-  }
-
-  ldout(cct, 10) << "retrieved exclusive locker: "
-                 << m_locker_entity << "@" << m_locker_address << dendl;
   send_get_watchers();
   return nullptr;
 }
@@ -507,9 +459,9 @@ Context *AcquireRequest<I>::handle_get_watchers(int *ret_val) {
   }
 
   for (auto &watcher : m_watchers) {
-    if ((strncmp(m_locker_address.c_str(),
+    if ((strncmp(m_locker.address.c_str(),
                  watcher.addr, sizeof(watcher.addr)) == 0) &&
-        (m_locker_handle == watcher.cookie)) {
+        (m_locker.handle == watcher.cookie)) {
       ldout(cct, 10) << "lock owner is still alive" << dendl;
 
       *ret_val = -EAGAIN;
@@ -536,7 +488,7 @@ void AcquireRequest<I>::send_blacklist() {
   Context *ctx = create_context_callback<klass, &klass::handle_blacklist>(
     this);
   m_image_ctx.op_work_queue->queue(new C_BlacklistClient<I>(m_image_ctx,
-                                                            m_locker_address,
+                                                            m_locker.address,
                                                             ctx), 0);
 }
 
@@ -560,8 +512,8 @@ void AcquireRequest<I>::send_break_lock() {
   ldout(cct, 10) << __func__ << dendl;
 
   librados::ObjectWriteOperation op;
-  rados::cls::lock::break_lock(&op, RBD_LOCK_NAME, m_locker_cookie,
-                               m_locker_entity);
+  rados::cls::lock::break_lock(&op, RBD_LOCK_NAME, m_locker.cookie,
+                               m_locker.entity);
 
   using klass = AcquireRequest<I>;
   librados::AioCompletion *rados_completion =

src/librbd/exclusive_lock/AcquireRequest.h

@@ -7,7 +7,7 @@
 #include "include/int_types.h"
 #include "include/buffer.h"
 #include "librbd/ImageCtx.h"
-#include "msg/msg_types.h"
+#include "librbd/exclusive_lock/Types.h"
 #include <string>
 
 class Context;
@@ -94,10 +94,7 @@ class AcquireRequest {
   decltype(m_image_ctx.object_map) m_object_map;
   decltype(m_image_ctx.journal) m_journal;
 
-  entity_name_t m_locker_entity;
-  std::string m_locker_cookie;
-  std::string m_locker_address;
-  uint64_t m_locker_handle;
+  Locker m_locker;
 
   int m_error_result;
   bool m_prepare_lock_completed = false;
@@ -132,8 +129,8 @@ class AcquireRequest {
   void send_unlock();
   Context *handle_unlock(int *ret_val);
 
-  void send_get_lockers();
-  Context *handle_get_lockers(int *ret_val);
+  void send_get_locker();
+  Context *handle_get_locker(int *ret_val);
 
   void send_get_watchers();
   Context *handle_get_watchers(int *ret_val);

src/librbd/exclusive_lock/GetLockerRequest.cc

@@ -0,0 +1,124 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#include "librbd/exclusive_lock/GetLockerRequest.h"
+#include "cls/lock/cls_lock_client.h"
+#include "cls/lock/cls_lock_types.h"
+#include "common/dout.h"
+#include "common/errno.h"
+#include "include/stringify.h"
+#include "librbd/ExclusiveLock.h"
+#include "librbd/ImageCtx.h"
+#include "librbd/Utils.h"
+#include "librbd/exclusive_lock/Types.h"
+
+#define dout_subsys ceph_subsys_rbd
+#undef dout_prefix
+#define dout_prefix *_dout << "librbd::exclusive_lock::GetLockerRequest: " \
+                           << this << " " << __func__ << ": "
+
+namespace librbd {
+namespace exclusive_lock {
+
+using util::create_rados_ack_callback;
+
+template <typename I>
+void GetLockerRequest<I>::send() {
+  send_get_lockers();
+}
+
+template <typename I>
+void GetLockerRequest<I>::send_get_lockers() {
+  CephContext *cct = m_image_ctx.cct;
+  ldout(cct, 10) << dendl;
+
+  librados::ObjectReadOperation op;
+  rados::cls::lock::get_lock_info_start(&op, RBD_LOCK_NAME);
+
+  using klass = GetLockerRequest<I>;
+  librados::AioCompletion *rados_completion =
+    create_rados_ack_callback<klass, &klass::handle_get_lockers>(this);
+  m_out_bl.clear();
+  int r = m_image_ctx.md_ctx.aio_operate(m_image_ctx.header_oid,
+                                         rados_completion, &op, &m_out_bl);
+  assert(r == 0);
+  rados_completion->release();
+}
+
+template <typename I>
+void GetLockerRequest<I>::handle_get_lockers(int r) {
+  CephContext *cct = m_image_ctx.cct;
+  ldout(cct, 10) << "r=" << r << dendl;
+
+  std::map<rados::cls::lock::locker_id_t,
+           rados::cls::lock::locker_info_t> lockers;
+  ClsLockType lock_type = LOCK_NONE;
+  std::string lock_tag;
+  if (r == 0) {
+    bufferlist::iterator it = m_out_bl.begin();
+    r = rados::cls::lock::get_lock_info_finish(&it, &lockers,
+                                                      &lock_type, &lock_tag);
+  }
+
+  if (r < 0) {
+    lderr(cct) << "failed to retrieve lockers: " << cpp_strerror(r) << dendl;
+    finish(r);
+    return;
+  }
+
+  if (lockers.empty()) {
+    ldout(cct, 20) << "no lockers detected" << dendl;
+    finish(-ENOENT);
+    return;
+  }
+
+  if (lock_tag != ExclusiveLock<>::WATCHER_LOCK_TAG) {
+    ldout(cct, 5) <<"locked by external mechanism: tag=" << lock_tag << dendl;
+    finish(-EBUSY);
+    return;
+  }
+
+  if (lock_type == LOCK_SHARED) {
+    ldout(cct, 5) << "shared lock type detected" << dendl;
+    finish(-EBUSY);
+    return;
+  }
+
+  std::map<rados::cls::lock::locker_id_t,
+           rados::cls::lock::locker_info_t>::iterator iter = lockers.begin();
+  if (!ExclusiveLock<>::decode_lock_cookie(iter->first.cookie,
+                                            &m_locker->handle)) {
+    ldout(cct, 5) << "locked by external mechanism: "
+                  << "cookie=" << iter->first.cookie << dendl;
+    finish(-EBUSY);
+    return;
+  }
+
+  m_locker->entity = iter->first.locker;
+  m_locker->cookie = iter->first.cookie;
+  m_locker->address = stringify(iter->second.addr);
+  if (m_locker->cookie.empty() || m_locker->address.empty()) {
+    ldout(cct, 20) << "no valid lockers detected" << dendl;
+    finish(-ENOENT);
+    return;
+  }
+
+  ldout(cct, 10) << "retrieved exclusive locker: "
+                 << m_locker->entity << "@" << m_locker->address << dendl;
+  finish(0);
+}
+
+template <typename I>
+void GetLockerRequest<I>::finish(int r) {
+  CephContext *cct = m_image_ctx.cct;
+  ldout(cct, 10) << "r=" << r << dendl;
+
+  m_on_finish->complete(r);
+  delete this;
+}
+
+} // namespace exclusive_lock
+} // namespace librbd
+
+template class librbd::exclusive_lock::GetLockerRequest<librbd::ImageCtx>;
+

src/librbd/exclusive_lock/GetLockerRequest.h

@@ -0,0 +1,53 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_EXCLUSIVE_LOCK_GET_LOCKER_REQUEST_H
+#define CEPH_LIBRBD_EXCLUSIVE_LOCK_GET_LOCKER_REQUEST_H
+
+#include "include/int_types.h"
+#include "include/buffer.h"
+
+class Context;
+
+namespace librbd {
+
+struct ImageCtx;
+
+namespace exclusive_lock {
+
+struct Locker;
+
+template <typename ImageCtxT = ImageCtx>
+class GetLockerRequest {
+public:
+  static GetLockerRequest* create(ImageCtxT &image_ctx, Locker *locker,
+                                  Context *on_finish) {
+    return new GetLockerRequest(image_ctx, locker, on_finish);
+  }
+
+  void send();
+
+private:
+  ImageCtxT &m_image_ctx;
+  Locker *m_locker;
+  Context *m_on_finish;
+
+  bufferlist m_out_bl;
+
+  GetLockerRequest(ImageCtxT &image_ctx, Locker *locker, Context *on_finish)
+    : m_image_ctx(image_ctx), m_locker(locker),  m_on_finish(on_finish) {
+  }
+
+  void send_get_lockers();
+  void handle_get_lockers(int r);
+
+  void finish(int r);
+
+};
+
+} // namespace exclusive_lock
+} // namespace librbd
+
+extern template class librbd::exclusive_lock::GetLockerRequest<librbd::ImageCtx>;
+
+#endif // CEPH_LIBRBD_EXCLUSIVE_LOCK_GET_LOCKER_REQUEST_H

src/librbd/exclusive_lock/Types.h

@@ -0,0 +1,23 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+
+#ifndef CEPH_LIBRBD_EXCLUSIVE_LOCK_TYPES_H
+#define CEPH_LIBRBD_EXCLUSIVE_LOCK_TYPES_H
+
+#include "msg/msg_types.h"
+#include <string>
+
+namespace librbd {
+namespace exclusive_lock {
+
+struct Locker {
+  entity_name_t entity;
+  std::string cookie;
+  std::string address;
+  uint64_t handle;
+};
+
+} // namespace exclusive_lock
+} // namespace librbd
+
+#endif // CEPH_LIBRBD_EXCLUSIVE_LOCK_TYPES_H

src/test/librbd/CMakeLists.txt

@@ -33,6 +33,7 @@ set(unittest_librbd_srcs
   test_mock_ObjectMap.cc
   test_mock_ObjectWatcher.cc
   exclusive_lock/test_mock_AcquireRequest.cc
+  exclusive_lock/test_mock_GetLockerRequest.cc
   exclusive_lock/test_mock_ReacquireRequest.cc
   exclusive_lock/test_mock_ReleaseRequest.cc
   image/test_mock_RefreshRequest.cc