Merge pull request #41919 from agayev/zoned-more-cleaning-support

os/bluestore: More support for cleaning zones.

Reviewed-by: Igor Fedotov <ifedotov@suse.com>

src/blk/BlockDevice.h

@@ -199,6 +199,9 @@ class BlockDevice {
     ceph_assert(is_smr());
     return conventional_region_size;
   }
+  virtual void reset_zones(const std::set<uint64_t>& zones) {
+    ceph_assert(is_smr());
+  }
 
   virtual void aio_submit(IOContext *ioc) = 0;
 

src/blk/zoned/HMSMRDevice.cc

@@ -412,6 +412,14 @@ void HMSMRDevice::_detect_vdo()
   return;
 }
 
+void HMSMRDevice::reset_zones(const std::set<uint64_t>& zones) {
+  for (auto zone_num : zones) {
+    if (zbd_reset_zones(zbd_fd, zone_num * zone_size, zone_size) != 0) {
+      derr << __func__ << " resetting zone failed for zone " << zone_num << dendl;
+    }
+  }
+}
+
 bool HMSMRDevice::get_thin_utilization(uint64_t *total, uint64_t *avail) const
 {
   if (vdo_fd < 0) {

src/blk/zoned/HMSMRDevice.h

@@ -41,6 +41,7 @@ class HMSMRDevice final : public BlockDevice {
   string vdo_name;
 
   std::string devname;  ///< kernel dev name (/sys/block/$devname), if any
+  int zbd_fd = -1;	///< fd for the zoned block device
 
   ceph::mutex debug_lock = ceph::make_mutex("HMSMRDevice::debug_lock");
   interval_set<uint64_t> debug_inflight;
@@ -135,6 +136,8 @@ class HMSMRDevice final : public BlockDevice {
 
   bool is_smr() const final { return true; }
 
+  void reset_zones(const std::set<uint64_t>& zones) override;
+
   bool get_thin_utilization(uint64_t *total, uint64_t *avail) const final;
 
   int read(uint64_t off, uint64_t len, bufferlist *pbl,

src/os/bluestore/BlueStore.cc

@@ -12376,6 +12376,19 @@ void BlueStore::_kv_finalize_thread()
 void BlueStore::_zoned_cleaner_start() {
   dout(10) << __func__ << dendl;
 
+  auto f = dynamic_cast<ZonedFreelistManager*>(fm);
+  ceph_assert(f);
+
+  auto zones_to_clean = f->get_cleaning_in_progress_zones(db);
+  if (!zones_to_clean.empty()) {
+    dout(10) << __func__ << " resuming cleaning after unclean shutdown." << dendl;
+    for (auto zone_num : zones_to_clean) {
+      _zoned_clean_zone(zone_num);
+    }
+    bdev->reset_zones(zones_to_clean);
+    f->mark_zones_to_clean_free(zones_to_clean, db);
+  }
+
   zoned_cleaner_thread.create("bstore_zcleaner");
 }
 
@@ -12418,10 +12431,12 @@ void BlueStore::_zoned_cleaner_thread() {
       dout(20) << __func__ << " wake" << dendl;
     } else {
       l.unlock();
+      f->mark_zones_to_clean_in_progress(*zones_to_clean, db);
       for (auto zone_num : *zones_to_clean) {
 	_zoned_clean_zone(zone_num);
       }
-      f->mark_zones_to_clean_free(zones_to_clean, db);
+      bdev->reset_zones(*zones_to_clean);
+      f->mark_zones_to_clean_free(*zones_to_clean, db);
       a->mark_zones_to_clean_free();
       l.lock();
     }

src/os/bluestore/ZonedFreelistManager.cc

@@ -314,16 +314,43 @@ int ZonedFreelistManager::_read_cfg(cfg_reader_t cfg_reader) {
   return 0;
 }
 
+std::set<uint64_t> ZonedFreelistManager::get_cleaning_in_progress_zones(
+    KeyValueDB *kvdb) const {
+  bufferlist bl;
+  std::set<uint64_t> zones_to_clean;
+  if (kvdb->get(meta_prefix, CLEANING_IN_PROGRESS_KEY, &bl) == 0) {
+    decode(zones_to_clean, bl);
+  }
+  return zones_to_clean;
+}
+
 void ZonedFreelistManager::mark_zones_to_clean_free(
-    const std::set<uint64_t> *zones_to_clean, KeyValueDB *kvdb) {
+    const std::set<uint64_t>& zones_to_clean, KeyValueDB *kvdb) {
   dout(10) << __func__ << dendl;
 
   KeyValueDB::Transaction txn = kvdb->get_transaction();
-  for (auto zone_num : *zones_to_clean) {
+  for (auto zone_num : zones_to_clean) {
     ldout(cct, 10) << __func__ << " zone " << zone_num << " is now clean in DB" << dendl;
 
     zone_state_t zone_state;
     write_zone_state_to_db(zone_num, zone_state, txn);
   }
+  txn->rmkey(meta_prefix, CLEANING_IN_PROGRESS_KEY);
+  kvdb->submit_transaction_sync(txn);
+}
+
+// Marks the zones currently being cleaned in the db. Should be called before
+// starting the cleaning. If we crash mid-cleaning, the recovery code will check
+// if there is a key CLEANING_IN_PROGRESS_KEY in the meta_prefix namespace, and
+// if so, will read the zones and resume cleaning.
+void ZonedFreelistManager::mark_zones_to_clean_in_progress(
+    const std::set<uint64_t>& zones_to_clean, KeyValueDB *kvdb) {
+  dout(10) << __func__ << dendl;
+
+  bufferlist bl;
+  encode(zones_to_clean, bl);
+
+  KeyValueDB::Transaction txn = kvdb->get_transaction();
+  txn->set(meta_prefix, CLEANING_IN_PROGRESS_KEY, bl);
   kvdb->submit_transaction_sync(txn);
 }

src/os/bluestore/ZonedFreelistManager.h

@@ -2,9 +2,7 @@
 // vim: ts=8 sw=2 smarttab
 
 //
-// A freelist manager for zoned devices.  This iteration just keeps the write
-// pointer per zone.  Following iterations will add enough information to enable
-// cleaning of zones.
+// A freelist manager for zoned devices.
 //
 // Copyright (C) 2020 Abutalib Aghayev
 //
@@ -24,6 +22,8 @@
 
 using cfg_reader_t = std::function<int(const std::string&, std::string*)>;
 
+const string CLEANING_IN_PROGRESS_KEY = "cleaning_in_progress";
+
 class ZonedFreelistManager : public FreelistManager {
   std::string meta_prefix;    ///< device size, zone size, etc.
   std::string info_prefix;    ///< per zone write pointer, dead bytes
@@ -102,8 +102,11 @@ class ZonedFreelistManager : public FreelistManager {
 		std::vector<std::pair<string, string>>*) const override;
 
   std::vector<zone_state_t> get_zone_states(KeyValueDB *kvdb) const;
-  void mark_zones_to_clean_free(const std::set<uint64_t> *zones_to_clean, 
+  std::set<uint64_t> get_cleaning_in_progress_zones(KeyValueDB *kvdb) const;
+  void mark_zones_to_clean_free(const std::set<uint64_t>& zones_to_clean, 
 				KeyValueDB *kvdb);
+  void mark_zones_to_clean_in_progress(const std::set<uint64_t>& zones_to_clean,
+				       KeyValueDB *kvdb);
 };
 
 #endif