mon: trim the creating_pgs after updating it with pgmap

src/mon/CreatingPGs.h

@@ -11,6 +11,31 @@ struct creating_pgs_t {
   epoch_t last_scan_epoch = 0;
   std::map<pg_t, std::pair<epoch_t, utime_t> > pgs;
   std::set<int64_t> created_pools;
+
+  unsigned create_pool(int64_t poolid, uint32_t pg_num,
+		       epoch_t created, utime_t modified) {
+    if (created_pools.count(poolid)) {
+      return 0;
+    }
+    const unsigned total = pgs.size();
+    for (ps_t ps = 0; ps < pg_num; ps++) {
+      const pg_t pgid{ps, static_cast<uint64_t>(poolid)};
+      if (pgs.count(pgid)) {
+	continue;
+      }
+      pgs.emplace(pgid, make_pair(created, modified));
+    }
+    return pgs.size() - total;
+  }
+
+  unsigned remove_pool(int64_t removed_pool) {
+    const unsigned total = pgs.size();
+    auto first = pgs.lower_bound(pg_t{0, (uint64_t)removed_pool});
+    auto last = pgs.lower_bound(pg_t{0, (uint64_t)removed_pool + 1});
+    pgs.erase(first, last);
+    created_pools.erase(removed_pool);
+    return total - pgs.size();
+  }
   void encode(bufferlist& bl) const {
     ENCODE_START(1, 1, bl);
     ::encode(last_scan_epoch, bl);

src/mon/OSDMonitor.cc

@@ -919,42 +919,48 @@ OSDMonitor::update_pending_pgs(const OSDMap::Incremental& inc)
   if (pending_creatings.last_scan_epoch > inc.epoch) {
     return pending_creatings;
   }
-  for (auto& pg : pending_created_pgs) {
-    pending_creatings.created_pools.insert(pg.pool());
-    pending_creatings.pgs.erase(pg);
-  }
-  pending_created_pgs.clear();
-  // PAXOS_PGMAP is less than PAXOS_OSDMAP, so PGMonitor::update_from_paxos()
-  // should have prepared the latest pgmap if any
   const auto& pgm = mon->pgmon()->pg_map;
   if (pgm.last_pg_scan >= creating_pgs.last_scan_epoch) {
     // TODO: please stop updating pgmap with pgstats once the upgrade is completed
+    const unsigned total = pending_creatings.pgs.size();
     for (auto& pgid : pgm.creating_pgs) {
       auto st = pgm.pg_stat.find(pgid);
       assert(st != pgm.pg_stat.end());
       auto created = make_pair(st->second.created, st->second.last_scrub_stamp);
       // no need to add the pg, if it already exists in creating_pgs
       pending_creatings.pgs.emplace(pgid, created);
     }
+    dout(7) << __func__ << total - pending_creatings.pgs.size()
+	    << " pgs added from pgmap" << dendl;
+  }
+  unsigned added = 0;
+  added += scan_for_creating_pgs(osdmap.get_pools(),
+				 inc.old_pools,
+				 inc.modified,
+				 &pending_creatings);
+  added += scan_for_creating_pgs(inc.new_pools,
+				 inc.old_pools,
+				 inc.modified,
+				 &pending_creatings);
+  dout(10) << __func__ << added << " pg added for new pools" << dendl;
+  for (auto deleted_pool : inc.old_pools) {
+    auto removed = pending_creatings.remove_pool(deleted_pool);
+    dout(10) << __func__ << removed
+	     << " pg removed because containing pool deleted: "
+	     << deleted_pool << dendl;
+  }
+  // pgmon updates its creating_pgs in check_osd_map() which is called by
+  // on_active() and check_osd_map() could be delayed if lease expires, so its
+  // creating_pgs could be stale in comparison with the one of osdmon. let's
+  // trim them here. otherwise, they will be added back after being erased.
+  unsigned removed = 0;
+  for (auto& pg : pending_created_pgs) {
+    pending_creatings.created_pools.insert(pg.pool());
+    removed += pending_creatings.pgs.erase(pg);
   }
-  for (auto old_pool : inc.old_pools) {
-    pending_creatings.created_pools.erase(old_pool);
-    const auto removed_pool = (uint64_t)old_pool;
-    auto first =
-      pending_creatings.pgs.lower_bound(pg_t{0, removed_pool});
-    auto last =
-      pending_creatings.pgs.lower_bound(pg_t{0, removed_pool + 1});
-    pending_creatings.pgs.erase(first, last);
-    last_epoch_clean.remove_pool(removed_pool);
-  }
-  scan_for_creating_pgs(osdmap.get_pools(),
-			inc.old_pools,
-			inc.modified,
-			&pending_creatings);
-  scan_for_creating_pgs(inc.new_pools,
-			inc.old_pools,
-			inc.modified,
-			&pending_creatings);
+  pending_created_pgs.clear();
+  dout(10) << __func__ << removed << " pgs removed because they're created"
+	   << dendl;
   pending_creatings.last_scan_epoch = osdmap.get_epoch();
   return pending_creatings;
 }
@@ -3159,12 +3165,13 @@ void OSDMonitor::check_pg_creates_sub(Subscription *sub)
   }
 }
 
-void OSDMonitor::scan_for_creating_pgs(
+unsigned OSDMonitor::scan_for_creating_pgs(
   const mempool::osdmap::map<int64_t,pg_pool_t>& pools,
   const mempool::osdmap::set<int64_t>& removed_pools,
   utime_t modified,
   creating_pgs_t* creating_pgs) const
 {
+  unsigned total = 0;
   for (auto& p : pools) {
     int64_t poolid = p.first;
     const pg_pool_t& pool = p.second;
@@ -3185,24 +3192,13 @@ void OSDMonitor::scan_for_creating_pgs(
 	       << " " << pool << dendl;
       continue;
     }
-    dout(10) << __func__ << " scanning pool " << poolid
+    unsigned added = creating_pgs->create_pool(poolid, pool.get_pg_num(),
+					       created, modified);
+    dout(10) << __func__ << added << " pgs added for pool "<< poolid
 	     << " " << pool << dendl;
-    if (creating_pgs->created_pools.count(poolid)) {
-      // split pgs are skipped by OSD, so drop it early.
-      continue;
-    }
-    // first pgs in this pool
-    for (ps_t ps = 0; ps < pool.get_pg_num(); ps++) {
-      const pg_t pgid{ps, static_cast<uint64_t>(poolid)};
-      if (creating_pgs->pgs.count(pgid)) {
-	dout(20) << __func__ << " already have " << pgid << dendl;
-	continue;
-      }
-      creating_pgs->pgs.emplace(pgid, make_pair(created, modified));
-      dout(10) << __func__ << " adding " << pgid
-	       << " at " << osdmap.get_epoch() << dendl;
-    }
+    total += added;
   }
+  return total;
 }
 
 void OSDMonitor::update_creating_pgs()

src/mon/OSDMonitor.h

@@ -453,7 +453,7 @@ class OSDMonitor : public PaxosService {
 
   creating_pgs_t update_pending_pgs(const OSDMap::Incremental& inc);
   void trim_creating_pgs(creating_pgs_t *creating_pgs, const PGMap& pgm);
-  void scan_for_creating_pgs(
+  unsigned scan_for_creating_pgs(
     const mempool::osdmap::map<int64_t,pg_pool_t>& pools,
     const mempool::osdmap::set<int64_t>& removed_pools,
     utime_t modified,