From fbafc4313bc231397b562cb2176d1507276aa9a3 Mon Sep 17 00:00:00 2001
From: Kefu Chai <kchai@redhat.com>
Date: Mon, 20 Nov 2017 19:40:53 +0800
Subject: [PATCH] osd: use the sum of max allowed backfill and recovery as
 reserver's max_allowed

also, it would be easier to trigger the preemption of recovery by
decreasing the reserver's max_allowed.

Fixes: http://tracker.ceph.com/issues/22043
Signed-off-by: Kefu Chai <kchai@redhat.com>
---
 .../singleton/all/recovery-preemption.yaml      |  3 ++-
 src/osd/OSD.cc                                  | 17 ++++++++++++-----
 src/osd/OSD.h                                   |  2 +-
 3 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/qa/suites/rados/singleton/all/recovery-preemption.yaml b/qa/suites/rados/singleton/all/recovery-preemption.yaml
index 6b4bb04af272ac..d6f6590cad16be 100644
--- a/qa/suites/rados/singleton/all/recovery-preemption.yaml
+++ b/qa/suites/rados/singleton/all/recovery-preemption.yaml
@@ -19,6 +19,7 @@ tasks:
         osd recovery sleep: .1
         osd min pg log entries: 100
         osd max pg log entries: 1000
+        osd recovery max active: 20
     log-whitelist:
       - \(POOL_APP_NOT_ENABLED\)
       - \(OSDMAP_FLAGS\)
@@ -43,7 +44,7 @@ tasks:
       - rados -p foo bench 3 write -b 4096 --no-cleanup
       - ceph osd unset noup
       - sleep 10
-      - for f in 0 1 2 3 ; do sudo ceph daemon osd.$f config set osd_recovery_sleep 0 ; sudo ceph daemon osd.$f config set osd_recovery_max_active 20 ; done
+      - for f in 0 1 2 3 ; do sudo ceph daemon osd.$f config set osd_recovery_sleep 0 ; sudo ceph daemon osd.$f config set osd_recovery_max_active 3 ; done
 - ceph.healthy:
 - exec:
     osd.0:
diff --git a/src/osd/OSD.cc b/src/osd/OSD.cc
index 0c72845e44acb1..35a3ec347f06ff 100644
--- a/src/osd/OSD.cc
+++ b/src/osd/OSD.cc
@@ -246,9 +246,13 @@ OSDService::OSDService(OSD *osd) :
   recovery_sleep_lock("OSDService::recovery_sleep_lock"),
   recovery_sleep_timer(cct, recovery_sleep_lock, false),
   reserver_finisher(cct),
-  local_reserver(cct, &reserver_finisher, cct->_conf->osd_max_backfills,
+  local_reserver(cct, &reserver_finisher,
+		 (cct->_conf->osd_max_backfills +
+		  cct->_conf->osd_recovery_max_active),
 		 cct->_conf->osd_min_recovery_priority),
-  remote_reserver(cct, &reserver_finisher, cct->_conf->osd_max_backfills,
+  remote_reserver(cct, &reserver_finisher,
+		  (cct->_conf->osd_max_backfills +
+		   cct->_conf->osd_recovery_max_active),
 		  cct->_conf->osd_min_recovery_priority),
   pg_temp_lock("OSDService::pg_temp_lock"),
   snap_sleep_lock("OSDService::snap_sleep_lock"),
@@ -9305,9 +9309,12 @@ const char** OSD::get_tracked_conf_keys() const
 void OSD::handle_conf_change(const struct md_config_t *conf,
 			     const std::set <std::string> &changed)
 {
-  if (changed.count("osd_max_backfills")) {
-    service.local_reserver.set_max(cct->_conf->osd_max_backfills);
-    service.remote_reserver.set_max(cct->_conf->osd_max_backfills);
+  if (changed.count("osd_max_backfills") ||
+      changed.count("osd_recovery_max_active")) {
+    auto max_allowed = (cct->_conf->osd_max_backfills +
+			cct->_conf->osd_recovery_max_active);
+    service.local_reserver.set_max(max_allowed);
+    service.remote_reserver.set_max(max_allowed);
   }
   if (changed.count("osd_min_recovery_priority")) {
     service.local_reserver.set_min_priority(cct->_conf->osd_min_recovery_priority);
diff --git a/src/osd/OSD.h b/src/osd/OSD.h
index 9e3d4a0c84e1b6..8e088f44fa5777 100644
--- a/src/osd/OSD.h
+++ b/src/osd/OSD.h
@@ -831,7 +831,7 @@ class OSDService {
     return (ceph_tid_t)last_tid++;
   }
 
-  // -- backfill_reservation --
+  // -- recovery and backfill reservation --
   Finisher reserver_finisher;
   AsyncReserver<spg_t> local_reserver;
   AsyncReserver<spg_t> remote_reserver;