osd: use distinct wait list for scrub

We've traditionally (ab)used waiting_for_active for ops blocked
by scrub.  In order to preserve the op requeue ordering for
backoffs relating to unreadable and degraded, however, we need
to order the scrub waits different.  The first step is to have it
on its own list.

Signed-off-by: Sage Weil <sage@redhat.com>

src/osd/PG.cc

@@ -2215,6 +2215,7 @@ void PG::split_ops(PG *child, unsigned split_bits) {
   assert(waiting_for_degraded_object.empty());
   assert(waiting_for_ondisk.empty());
   assert(waiting_for_active.empty());
+  assert(waiting_for_scrub.empty());
 
   osd->dequeue_pg(this, &waiting_for_peered);
 
@@ -4311,7 +4312,7 @@ void PG::chunky_scrub(ThreadPool::TPHandle &handle)
 	scrubber.run_callbacks();
 
         // requeue the writes from the chunk that just finished
-        requeue_ops(waiting_for_active);
+        requeue_ops(waiting_for_scrub);
 
 	scrubber.state = PG::Scrubber::WAIT_DIGEST_UPDATES;
 
@@ -4361,7 +4362,7 @@ void PG::scrub_clear_state()
   if (scrubber.active)
     osd->dec_scrubs_active();
 
-  requeue_ops(waiting_for_active);
+  requeue_ops(waiting_for_scrub);
 
   if (scrubber.queue_snap_trim) {
     dout(10) << "scrub finished, requeuing snap_trimmer" << dendl;

src/osd/PG.h

@@ -831,6 +831,7 @@ class PG : protected DoutPrefixProvider {
 
   // ops waiting on active (require peered as well)
   list<OpRequestRef>            waiting_for_active;
+  list<OpRequestRef>            waiting_for_scrub;
 
   list<OpRequestRef>            waiting_for_cache_not_full;
   list<OpRequestRef>            waiting_for_all_missing;

src/osd/PrimaryLogPG.cc

@@ -1858,7 +1858,7 @@ void PrimaryLogPG::do_op(OpRequestRef& op)
   if (write_ordered &&
       scrubber.write_blocked_by_scrub(head, get_sort_bitwise())) {
     dout(20) << __func__ << ": waiting for scrub" << dendl;
-    waiting_for_active.push_back(op);
+    waiting_for_scrub.push_back(op);
     op->mark_delayed("waiting for scrub");
     return;
   }
@@ -2880,10 +2880,10 @@ void PrimaryLogPG::promote_object(ObjectContextRef obc,
     dout(10) << __func__ << " " << hoid
 	     << " blocked by scrub" << dendl;
     if (op) {
-      waiting_for_active.push_back(op);
+      waiting_for_scrub.push_back(op);
       op->mark_delayed("waiting for scrub");
       dout(10) << __func__ << " " << hoid
-	       << " placing op in waiting_for_active" << dendl;
+	       << " placing op in waiting_for_scrub" << dendl;
     } else {
       dout(10) << __func__ << " " << hoid
 	       << " no op, dropping on the floor" << dendl;
@@ -10005,10 +10005,11 @@ void PrimaryLogPG::on_change(ObjectStore::Transaction *t)
   // requeue everything in the reverse order they should be
   // reexamined.
   requeue_ops(waiting_for_peered);
+  requeue_ops(waiting_for_active);
 
   clear_scrub_reserved();
 
-  // requeues waiting_for_active
+  // requeues waiting_for_scrub
   scrub_clear_state();
 
   cancel_copy_ops(is_primary());

src/osd/PrimaryLogPG.h

@@ -824,8 +824,8 @@ class PrimaryLogPG : public PG, public PGBackend::Listener {
 	if (scrubber.write_blocked_by_scrub(
 	      p.first.get_head(),
 	      get_sort_bitwise())) {
-	  waiting_for_active.splice(
-	    waiting_for_active.begin(),
+	  waiting_for_scrub.splice(
+	    waiting_for_scrub.begin(),
 	    p.second,
 	    p.second.begin(),
 	    p.second.end());