PrimaryLogPG: reimplement osd_snap_trim_sleep within the state machine

Rather than blocking the main op queue, just pause for that amount of
time between state machine cycles.

Also, add osd_snap_trim_sleep to a few of the thrasher yamls.

Signed-off-by: Samuel Just <sjust@redhat.com>

qa/suites/rados/thrash/thrashers/default.yaml

@@ -10,6 +10,7 @@ tasks:
         osd scrub min interval: 60
         osd scrub max interval: 120
         osd max backfills: 3
+        osd snap trim sleep: 2
 - thrashosds:
     timeout: 1200
     chance_pgnum_grow: 1

qa/suites/rados/thrash/thrashers/pggrow.yaml

@@ -10,6 +10,7 @@ tasks:
         osd scrub max interval: 120
         filestore odsync write: true
         osd max backfills: 2
+        osd snap trim sleep: .5
 - thrashosds:
     timeout: 1200
     chance_pgnum_grow: 2

src/common/config_opts.h

@@ -757,7 +757,7 @@ OPTION(osd_op_thread_suicide_timeout, OPT_INT, 150)
 OPTION(osd_recovery_thread_timeout, OPT_INT, 30)
 OPTION(osd_recovery_thread_suicide_timeout, OPT_INT, 300)
 OPTION(osd_recovery_sleep, OPT_FLOAT, 0)         // seconds to sleep between recovery ops
-OPTION(osd_snap_trim_sleep, OPT_FLOAT, 0)
+OPTION(osd_snap_trim_sleep, OPT_DOUBLE, 0)
 OPTION(osd_scrub_invalid_stats, OPT_BOOL, true)
 OPTION(osd_remove_thread_timeout, OPT_INT, 60*60)
 OPTION(osd_remove_thread_suicide_timeout, OPT_INT, 10*60*60)

src/osd/OSD.cc

@@ -261,6 +261,9 @@ OSDService::OSDService(OSD *osd) :
   remote_reserver(&reserver_finisher, cct->_conf->osd_max_backfills,
 		  cct->_conf->osd_min_recovery_priority),
   pg_temp_lock("OSDService::pg_temp_lock"),
+  snap_sleep_lock("OSDService::snap_sleep_lock"),
+  snap_sleep_timer(
+    osd->client_messenger->cct, snap_sleep_lock, false /* relax locking */),
   snap_reserver(&reserver_finisher,
 		cct->_conf->osd_max_trimming_pgs),
   recovery_lock("OSDService::recovery_lock"),
@@ -493,6 +496,12 @@ void OSDService::shutdown()
     Mutex::Locker l(backfill_request_lock);
     backfill_request_timer.shutdown();
   }
+
+  {
+    Mutex::Locker l(snap_sleep_lock);
+    snap_sleep_timer.shutdown();
+  }
+
   osdmap = OSDMapRef();
   next_osdmap = OSDMapRef();
 }
@@ -504,6 +513,7 @@ void OSDService::init()
   objecter->set_client_incarnation(0);
   watch_timer.init();
   agent_timer.init();
+  snap_sleep_timer.init();
 
   agent_thread.create("osd_srv_agent");
 

src/osd/OSD.h

@@ -912,6 +912,9 @@ class OSDService {
 
   void queue_for_peering(PG *pg);
 
+  Mutex snap_sleep_lock;
+  SafeTimer snap_sleep_timer;
+
   AsyncReserver<spg_t> snap_reserver;
   void queue_for_snap_trim(PG *pg);
 

src/osd/PrimaryLogPG.cc

@@ -3734,14 +3734,6 @@ void PrimaryLogPG::snap_trimmer(epoch_t queued)
   if (deleting || pg_has_reset_since(queued)) {
     return;
   }
-  if (cct->_conf->osd_snap_trim_sleep > 0) {
-    unlock();
-    utime_t t;
-    t.set_from_double(cct->_conf->osd_snap_trim_sleep);
-    t.sleep();
-    lock();
-    dout(20) << __func__ << " slept for " << t << dendl;
-  }
 
   assert(is_primary());
 

src/osd/PrimaryLogPG.h

@@ -1451,6 +1451,9 @@ class PrimaryLogPG : public PG, public PGBackend::Listener {
   struct SnapTrimReserved : boost::statechart::event< SnapTrimReserved > {
     SnapTrimReserved() : boost::statechart::event< SnapTrimReserved >() {}
   };
+  struct SnapTrimTimerReady : boost::statechart::event< SnapTrimTimerReady > {
+    SnapTrimTimerReady() : boost::statechart::event< SnapTrimTimerReady >() {}
+  };
 
   struct NotTrimming;
   struct SnapTrimmer : public boost::statechart::state_machine< SnapTrimmer, NotTrimming > {
@@ -1493,6 +1496,57 @@ class PrimaryLogPG : public PG, public PGBackend::Listener {
   };
 
   /* SnapTrimmerStates */
+  struct WaitTrimTimer : boost::statechart::state< WaitTrimTimer, Trimming >, NamedState {
+    typedef boost::mpl::list <
+      boost::statechart::custom_reaction< SnapTrimTimerReady >
+      > reactions;
+    Context *wakeup = nullptr;
+    explicit WaitTrimTimer(my_context ctx)
+      : my_base(ctx),
+	NamedState(context< SnapTrimmer >().pg->cct, "Trimming/WaitTrimTimer") {
+      context< SnapTrimmer >().log_enter(state_name);
+      assert(context<Trimming>().in_flight.empty());
+      struct OnTimer : Context {
+	PrimaryLogPGRef pg;
+	epoch_t epoch;
+	OnTimer(PrimaryLogPGRef pg, epoch_t epoch) : pg(pg), epoch(epoch) {}
+	void finish(int) override {
+	  pg->lock();
+	  if (!pg->pg_has_reset_since(epoch))
+	    pg->snap_trimmer_machine.process_event(SnapTrimTimerReady());
+	  pg->unlock();
+	}
+      };
+      auto *pg = context< SnapTrimmer >().pg;
+      if (pg->cct->_conf->osd_snap_trim_sleep > 0) {
+	wakeup = new OnTimer{pg, pg->get_osdmap()->get_epoch()};
+	Mutex::Locker l(pg->osd->snap_sleep_lock);
+	pg->osd->snap_sleep_timer.add_event_after(
+	  pg->cct->_conf->osd_snap_trim_sleep, wakeup);
+      } else {
+	post_event(SnapTrimTimerReady());
+      }
+    }
+    void exit() {
+      context< SnapTrimmer >().log_exit(state_name, enter_time);
+      auto *pg = context< SnapTrimmer >().pg;
+      if (wakeup) {
+	Mutex::Locker l(pg->osd->snap_sleep_lock);
+	pg->osd->snap_sleep_timer.cancel_event(wakeup);
+	wakeup = nullptr;
+      }
+    }
+    boost::statechart::result react(const SnapTrimTimerReady &) {
+      wakeup = nullptr;
+      if (!context< SnapTrimmer >().can_trim()) {
+	post_event(KickTrim());
+	return transit< NotTrimming >();
+      } else {
+	return transit< AwaitAsyncWork >();
+      }
+    }
+  };
+
   struct WaitRWLock : boost::statechart::state< WaitRWLock, Trimming >, NamedState {
     typedef boost::mpl::list <
       boost::statechart::custom_reaction< TrimWriteUnblocked >
@@ -1534,7 +1588,7 @@ class PrimaryLogPG : public PG, public PGBackend::Listener {
 	post_event(KickTrim());
 	return transit< NotTrimming >();
       } else {
-	return transit< AwaitAsyncWork >();
+	return transit< WaitTrimTimer >();
       }
     }
   };