osd: use the sum of max allowed backfill and recovery as reserver's m…

…ax_allowed

also, it would be easier to trigger the preemption of recovery by
decreasing the reserver's max_allowed.

Fixes: http://tracker.ceph.com/issues/22043
Signed-off-by: Kefu Chai <kchai@redhat.com>

qa/suites/rados/singleton/all/recovery-preemption.yaml

@@ -19,6 +19,7 @@ tasks:
         osd recovery sleep: .1
         osd min pg log entries: 100
         osd max pg log entries: 1000
+        osd recovery max active: 20
     log-whitelist:
       - \(POOL_APP_NOT_ENABLED\)
       - \(OSDMAP_FLAGS\)
@@ -43,7 +44,7 @@ tasks:
       - rados -p foo bench 3 write -b 4096 --no-cleanup
       - ceph osd unset noup
       - sleep 10
-      - for f in 0 1 2 3 ; do sudo ceph daemon osd.$f config set osd_recovery_sleep 0 ; sudo ceph daemon osd.$f config set osd_recovery_max_active 20 ; done
+      - for f in 0 1 2 3 ; do sudo ceph daemon osd.$f config set osd_recovery_sleep 0 ; sudo ceph daemon osd.$f config set osd_recovery_max_active 3 ; done
 - ceph.healthy:
 - exec:
     osd.0:

src/osd/OSD.cc

@@ -246,9 +246,13 @@ OSDService::OSDService(OSD *osd) :
   recovery_sleep_lock("OSDService::recovery_sleep_lock"),
   recovery_sleep_timer(cct, recovery_sleep_lock, false),
   reserver_finisher(cct),
-  local_reserver(cct, &reserver_finisher, cct->_conf->osd_max_backfills,
+  local_reserver(cct, &reserver_finisher,
+		 (cct->_conf->osd_max_backfills +
+		  cct->_conf->osd_recovery_max_active),
 		 cct->_conf->osd_min_recovery_priority),
-  remote_reserver(cct, &reserver_finisher, cct->_conf->osd_max_backfills,
+  remote_reserver(cct, &reserver_finisher,
+		  (cct->_conf->osd_max_backfills +
+		   cct->_conf->osd_recovery_max_active),
 		  cct->_conf->osd_min_recovery_priority),
   pg_temp_lock("OSDService::pg_temp_lock"),
   snap_sleep_lock("OSDService::snap_sleep_lock"),
@@ -9305,9 +9309,12 @@ const char** OSD::get_tracked_conf_keys() const
 void OSD::handle_conf_change(const struct md_config_t *conf,
 			     const std::set <std::string> &changed)
 {
-  if (changed.count("osd_max_backfills")) {
-    service.local_reserver.set_max(cct->_conf->osd_max_backfills);
-    service.remote_reserver.set_max(cct->_conf->osd_max_backfills);
+  if (changed.count("osd_max_backfills") ||
+      changed.count("osd_recovery_max_active")) {
+    auto max_allowed = (cct->_conf->osd_max_backfills +
+			cct->_conf->osd_recovery_max_active);
+    service.local_reserver.set_max(max_allowed);
+    service.remote_reserver.set_max(max_allowed);
   }
   if (changed.count("osd_min_recovery_priority")) {
     service.local_reserver.set_min_priority(cct->_conf->osd_min_recovery_priority);

src/osd/OSD.h

@@ -831,7 +831,7 @@ class OSDService {
     return (ceph_tid_t)last_tid++;
   }
 
-  // -- backfill_reservation --
+  // -- recovery and backfill reservation --
   Finisher reserver_finisher;
   AsyncReserver<spg_t> local_reserver;
   AsyncReserver<spg_t> remote_reserver;