From 7af3e86c2e4992db35637864b83832535c94d0e6 Mon Sep 17 00:00:00 2001
From: Brad Hubbard <bhubbard@redhat.com>
Date: Mon, 24 Apr 2017 14:10:47 +1000
Subject: [PATCH] osd: Implement asynchronous scrub sleep

Rather than blocking the main op queue just do an async sleep.

Fixes: http://tracker.ceph.com/issues/19497

Signed-off-by: Brad Hubbard <bhubbard@redhat.com>
---
 src/osd/PG.cc | 38 ++++++++++++++++++++++++++++----------
 src/osd/PG.h  | 12 ++++++++++++
 2 files changed, 40 insertions(+), 10 deletions(-)

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
index ad241dcbae0f4..bcc93d687b6f5 100644
--- a/src/osd/PG.cc
+++ b/src/osd/PG.cc
@@ -324,7 +324,9 @@ PG::PG(OSDService *o, OSDMapRef curmap,
   peer_features(CEPH_FEATURES_SUPPORTED_DEFAULT),
   acting_features(CEPH_FEATURES_SUPPORTED_DEFAULT),
   upacting_features(CEPH_FEATURES_SUPPORTED_DEFAULT),
-  last_epoch(0)
+  last_epoch(0),
+  scrub_sleep_lock("PG::scrub_sleep_lock"),
+  scrub_sleep_timer(o->cct, scrub_sleep_lock, false /* relax locking */)
 {
 #ifdef PG_DEBUG_REFS
   osd->add_pgid(p, this);
@@ -335,6 +337,8 @@ PG::PG(OSDService *o, OSDMapRef curmap,
 PG::~PG()
 {
   pgstate_history.set_pg_in_destructor();
+  Mutex::Locker l(scrub_sleep_lock);
+  scrub_sleep_timer.shutdown();
 #ifdef PG_DEBUG_REFS
   osd->remove_pgid(info.pgid, this);
 #endif
@@ -2821,6 +2825,8 @@ void PG::init(
   dirty_info = true;
   dirty_big_info = true;
   write_if_dirty(*t);
+
+  scrub_sleep_timer.init();
 }
 
 #pragma GCC diagnostic ignored "-Wpragmas"
@@ -4208,22 +4214,34 @@ void PG::scrub(epoch_t queued, ThreadPool::TPHandle &handle)
 {
   if (cct->_conf->osd_scrub_sleep > 0 &&
       (scrubber.state == PG::Scrubber::NEW_CHUNK ||
-       scrubber.state == PG::Scrubber::INACTIVE)) {
+       scrubber.state == PG::Scrubber::INACTIVE) && scrubber.needs_sleep) {
+    ceph_assert(!scrubber.sleeping);
     dout(20) << __func__ << " state is INACTIVE|NEW_CHUNK, sleeping" << dendl;
-    unlock();
-    utime_t t;
-    t.set_from_double(cct->_conf->osd_scrub_sleep);
-    handle.suspend_tp_timeout();
-    t.sleep();
-    handle.reset_tp_timeout();
-    lock();
-    dout(20) << __func__ << " slept for " << t << dendl;
+    // Do an async sleep so we don't block the op queue
+    auto scrub_requeue_callback = new FunctionContext([this](int r) {
+      lock();
+      scrubber.sleeping = false;
+      scrubber.needs_sleep = false;
+      dout(20) << __func__ << " slept for "
+               << ceph_clock_now() - scrubber.sleep_start
+               << ", re-queuing scrub" << dendl;
+      scrub_queued = false;
+      requeue_scrub();
+      scrubber.sleep_start = utime_t();
+      unlock();
+    });
+    Mutex::Locker l(scrub_sleep_lock);
+    scrub_sleep_timer.add_event_after(cct->_conf->osd_scrub_sleep, scrub_requeue_callback);
+    scrubber.sleeping = true;
+    scrubber.sleep_start = ceph_clock_now();
+    return;
   }
   if (pg_has_reset_since(queued)) {
     return;
   }
   assert(scrub_queued);
   scrub_queued = false;
+  scrubber.needs_sleep = true;
 
   if (!is_primary() || !is_active() || !is_clean() || !is_scrubbing()) {
     dout(10) << "scrub -- not primary or active or not clean" << dendl;
diff --git a/src/osd/PG.h b/src/osd/PG.h
index c7bd076bd3c28..eb44e8c98ae44 100644
--- a/src/osd/PG.h
+++ b/src/osd/PG.h
@@ -36,6 +36,7 @@
 #include "include/xlist.h"
 #include "SnapMapper.h"
 #include "Session.h"
+#include "common/Timer.h"
 
 #include "PGLog.h"
 #include "OSDMap.h"
@@ -1202,6 +1203,11 @@ class PG : public DoutPrefixProvider {
     OpRequestRef active_rep_scrub;
     utime_t scrub_reg_stamp;  // stamp we registered for
 
+    // For async sleep
+    bool sleeping = false;
+    bool needs_sleep = true;
+    utime_t sleep_start;
+
     // flags to indicate explicitly requested scrubs (by admin)
     bool must_scrub, must_deep_scrub, must_repair;
 
@@ -1316,6 +1322,9 @@ class PG : public DoutPrefixProvider {
       authoritative.clear();
       num_digest_updates_pending = 0;
       cleaned_meta_map = ScrubMap();
+      sleeping = false;
+      needs_sleep = true;
+      sleep_start = utime_t();
     }
 
     void create_results(const hobject_t& obj);
@@ -2189,6 +2198,9 @@ class PG : public DoutPrefixProvider {
 
   epoch_t last_epoch;
 
+  Mutex scrub_sleep_lock;
+  SafeTimer scrub_sleep_timer;
+
  public:
   const spg_t&      get_pgid() const { return pg_id; }