mon/PGMap: factor mon_osd_full_ratio into MAX AVAIL calc

If we only fill OSDs to 95%, we should factor that into
the MAX AVAIL calculation for the pool.

Fixes: http://tracker.ceph.com/issues/18522
Signed-off-by: Sage Weil <sage@redhat.com>

doc/rados/operations/monitoring.rst

@@ -122,6 +122,8 @@ on the number of replicas, clones and snapshots.
 - **USED:** The notional amount of data stored in kilobytes, unless the number 
   appends **M** for megabytes or **G** for gigabytes.
 - **%USED:** The notional percentage of storage used per pool.
+- **MAX AVAIL:** An estimate of the notional amount of data that can be written
+  to this pool.
 - **Objects:** The notional number of objects stored per pool.
 
 .. note:: The numbers in the **POOLS** section are notional. They are not 
@@ -130,6 +132,11 @@ on the number of replicas, clones and snapshots.
    **RAW USED** and **%RAW USED** amounts in the **GLOBAL** section of the 
    output.
 
+.. note:: The **MAX AVAIL** value is a complicated function of the
+   replication or erasure code used, the CRUSH rule that maps storage
+   to devices, the utilization of those devices, and the configured
+   mon_osd_full_ratio.
+
 
 Checking a Cluster's Status
 ===========================

src/mon/PGMap.cc

@@ -1834,15 +1834,17 @@ int64_t PGMap::get_rule_avail(const OSDMap& osdmap, int ruleno) const
 {
   map<int,float> wm;
   int r = osdmap.crush->get_rule_weight_osd_map(ruleno, &wm);
-  if (r < 0)
+  if (r < 0) {
     return r;
-
-  if(wm.empty())
+  }
+  if (wm.empty()) {
     return 0;
+  }
 
   int64_t min = -1;
   for (map<int,float>::iterator p = wm.begin(); p != wm.end(); ++p) {
-    ceph::unordered_map<int32_t,osd_stat_t>::const_iterator osd_info = osd_stat.find(p->first);
+    ceph::unordered_map<int32_t,osd_stat_t>::const_iterator osd_info =
+      osd_stat.find(p->first);
     if (osd_info != osd_stat.end()) {
       if (osd_info->second.kb == 0 || p->second == 0) {
 	// osd must be out, hence its stats have been zeroed
@@ -1852,10 +1854,14 @@ int64_t PGMap::get_rule_avail(const OSDMap& osdmap, int ruleno) const
 	// calculate proj below.
 	continue;
       }
-      int64_t proj = (int64_t)((double)((osd_info->second).kb_avail * 1024ull) /
-                     (double)p->second);
-      if (min < 0 || proj < min)
+      double unusable = (double)osd_info->second.kb *
+	(1.0 - g_conf->mon_osd_full_ratio);
+      double avail = MAX(0.0, (double)osd_info->second.kb_avail - unusable);
+      avail *= 1024.0;
+      int64_t proj = (int64_t)(avail / (double)p->second);
+      if (min < 0 || proj < min) {
 	min = proj;
+      }
     } else {
       dout(0) << "Cannot get stat of OSD " << p->first << dendl;
     }