os/bluestore: warn on no per-pool omap

Signed-off-by: Sage Weil <sage@redhat.com>

doc/rados/operations/health-checks.rst

@@ -356,6 +356,27 @@ This warning can be disabled with::
 
   ceph config set global bluestore_warn_on_legacy_statfs false
 
+BLUESTORE_NO_PER_POOL_OMAP
+__________________________
+
+Starting with the Octopus release, BlueStore tracks omap space utilization
+by pool, and one or more OSDs have volumes that were created prior to
+Octopus.  If all OSDs are not running BlueStore with the new tracking
+enabled, the cluster will report and approximate value for per-pool omap usage
+based on the most recent deep-scrub.
+
+The old OSDs can be updated to track by pool by stopping each OSD,
+running a repair operation, and the restarting it.  For example, if
+``osd.123`` needed to be updated,::
+
+  systemctl stop ceph-osd@123
+  ceph-bluestore-tool repair --path /var/lib/ceph/osd/ceph-123
+  systemctl start ceph-osd@123
+
+This warning can be disabled with::
+
+  ceph config set global bluestore_warn_on_no_per_pool_omap false
+
 
 BLUESTORE_DISK_SIZE_MISMATCH
 ____________________________

qa/suites/fs/upgrade/featureful_client/old_client/tasks/2-upgrade.yaml

@@ -13,6 +13,7 @@ overrides:
     conf:
       global:
         bluestore warn on legacy statfs: false
+        bluestore warn on no per pool omap: false
       mon:
         mon warn on osd down out interval zero: false
 

qa/suites/fs/upgrade/featureful_client/upgraded_client/tasks/2-upgrade.yaml

@@ -13,6 +13,7 @@ overrides:
     conf:
       global:
         bluestore warn on legacy statfs: false
+        bluestore warn on no per pool omap: false
       mon:
         mon warn on osd down out interval zero: false
 

qa/suites/upgrade/mimic-x-singleton/0-cluster/start.yaml

@@ -13,6 +13,7 @@ overrides:
         ms dump corrupt message level: 0
         ms bind msgr2: false
         bluestore warn on legacy statfs: false
+        bluestore warn on no per pool omap: false
       mds:
         debug ms: 1
         debug mds: 20

qa/suites/upgrade/mimic-x/parallel/1-ceph-install/mimic.yaml

@@ -34,6 +34,7 @@ tasks:
       global:
         mon warn on pool no app: false
         bluestore_warn_on_legacy_statfs: false
+        bluestore warn on no per pool omap: false
 - exec:
     osd.0:
       - ceph osd require-osd-release mimic

qa/suites/upgrade/mimic-x/stress-split/1-ceph-install/mimic.yaml

@@ -16,6 +16,7 @@ tasks:
     conf:
       global:
         bluestore_warn_on_legacy_statfs: false
+        bluestore warn on no per pool omap: false
 - exec:
     osd.0:
       - ceph osd require-osd-release mimic

qa/suites/upgrade/nautilus-x-singleton/0-cluster/start.yaml

@@ -12,6 +12,7 @@ overrides:
       global:
         ms dump corrupt message level: 0
         ms bind msgr2: false
+        bluestore warn on no per pool omap: false
       mds:
         debug ms: 1
         debug mds: 20

qa/suites/upgrade/nautilus-x/parallel/1-ceph-install/nautilus.yaml

@@ -26,6 +26,7 @@ tasks:
       global:
         mon warn on pool no app: false
         bluestore_warn_on_legacy_statfs: false
+        bluestore warn on no per pool omap: false
 - exec:
     osd.0:
       - ceph osd set-require-min-compat-client nautilus

qa/suites/upgrade/nautilus-x/stress-split/1-ceph-install/nautilus.yaml

@@ -8,6 +8,7 @@ tasks:
     conf:
       global:
         bluestore_warn_on_legacy_statfs: false
+        bluestore warn on no per pool omap: false
 - exec:
     osd.0:
       - ceph osd require-osd-release nautilus

src/common/legacy_config_opts.h

@@ -1072,6 +1072,7 @@ OPTION(bluestore_no_per_pool_stats_tolerance, OPT_STR)
 OPTION(bluestore_warn_on_bluefs_spillover, OPT_BOOL)
 OPTION(bluestore_warn_on_legacy_statfs, OPT_BOOL)
 OPTION(bluestore_fsck_error_on_no_per_pool_omap, OPT_BOOL)
+OPTION(bluestore_warn_on_no_per_pool_omap, OPT_BOOL)
 OPTION(bluestore_log_op_age, OPT_DOUBLE)
 OPTION(bluestore_log_omap_iterator_age, OPT_DOUBLE)
 OPTION(bluestore_log_collection_list_age, OPT_DOUBLE)

src/common/options.cc

@@ -4868,6 +4868,10 @@ std::vector<Option> get_global_options() {
     .set_default(false)
     .set_description("Make fsck error (instead of warn) when objects without per-pool omap are found"),
 
+    Option("bluestore_warn_on_no_per_pool_omap", Option::TYPE_BOOL, Option::LEVEL_ADVANCED)
+    .set_default(true)
+    .set_description("Enable health indication on lack of per-pool omap"),
+
     Option("bluestore_log_op_age", Option::TYPE_FLOAT, Option::LEVEL_ADVANCED)
     .set_default(5)
     .set_description("log operation if it's slower than this age (seconds)"),

src/mon/PGMap.cc

@@ -3068,6 +3068,8 @@ void PGMap::get_health_checks(
 	summary = "Legacy BlueStore stats reporting detected";
       } else if (asum.first == "BLUESTORE_DISK_SIZE_MISMATCH") {
 	summary = "BlueStore has dangerous mismatch between block device and free list sizes";
+      } else if (asum.first == "BLUESTORE_NO_PER_POOL_OMAP") {
+	summary = "Legacy BlueStore does not track omap usage by pool";
       }
       summary += " on ";
       summary += stringify(asum.second.first);

src/os/bluestore/BlueStore.cc

@@ -4135,6 +4135,7 @@ const char **BlueStore::get_tracked_conf_keys() const
     "bluestore_cache_autotune_interval",
     "bluestore_no_per_pool_stats_tolerance",
     "bluestore_warn_on_legacy_statfs",
+    "bluestore_warn_on_no_per_pool_omap",
     NULL
   };
   return KEYS;
@@ -4147,6 +4148,9 @@ void BlueStore::handle_conf_change(const ConfigProxy& conf,
       changed.count("bluestore_warn_on_legacy_statfs")) {
     _check_legacy_statfs_alert();
   }
+  if (changed.count("bluestore_warn_on_no_per_pool_omap")) {
+    _check_no_per_pool_omap_alert();
+  }
 
   if (changed.count("bluestore_csum_type")) {
     _set_csum();
@@ -8453,6 +8457,18 @@ void BlueStore::_check_legacy_statfs_alert()
   legacy_statfs_alert = s;
 }
 
+void BlueStore::_check_no_per_pool_omap_alert()
+{
+  string s;
+  if (!per_pool_omap &&
+      cct->_conf->bluestore_warn_on_no_per_pool_omap) {
+    s = "legacy (not per-pool) omap detected, "
+      "suggest to run store repair to measure per-pool omap usage";
+  }
+  std::lock_guard l(qlock);
+  no_per_pool_omap_alert = s;
+}
+
 // ---------------
 // cache
 
@@ -9979,6 +9995,7 @@ int BlueStore::_open_super_meta()
     } else {
       dout(10) << __func__ << " per_pool_omap not present" << dendl;
     }
+    _check_no_per_pool_omap_alert();
   }
 
   _open_statfs();
@@ -14255,6 +14272,11 @@ void BlueStore::_log_alerts(osd_alert_list_t& alerts)
       "BLUEFS_SPILLOVER",
       spillover_alert);
   }
+  if (!no_per_pool_omap_alert.empty()) {
+    alerts.emplace(
+      "BLUESTORE_NO_PER_POOL_OMAP",
+      no_per_pool_omap_alert);
+  }
   string s0(failed_cmode);
 
   if (!failed_compressors.empty()) {

src/os/bluestore/BlueStore.h

@@ -2604,6 +2604,7 @@ class BlueStore : public ObjectStore,
   set<string> failed_compressors;
   string spillover_alert;
   string legacy_statfs_alert;
+  string no_per_pool_omap_alert;
   string disk_size_mismatch_alert;
 
   void _log_alerts(osd_alert_list_t& alerts);
@@ -2632,6 +2633,7 @@ class BlueStore : public ObjectStore,
   }
 
   void _check_legacy_statfs_alert();
+  void _check_no_per_pool_omap_alert();
   void _set_disk_size_mismatch_alert(const string& s) {
     std::lock_guard l(qlock);
     disk_size_mismatch_alert = s;