osd/PG: force rebuild of missing set on jewel upgrade

Previously we were detecting the need to rebuild missing based on
whether the "divergent_priors" omap key was present.  Unfortunately,
jewel does not always set this, so it is not a reliable indicator.
(It only gets set if you actually have a divergent prior at some
point in the PG's life time on that OSD.)

Fix by using the info_struct_v on the PG to detect whether we need
to do the conversion.  We didn't bump the value when we adding
the missing persistence, but the fastinfo was also added during
the same period between jewel and kraken, so it will work just as
well.

Fixes: http://tracker.ceph.com/issues/20958
Signed-off-by: Sage Weil <sage@redhat.com>

src/osd/PG.cc

@@ -3315,19 +3315,37 @@ void PG::read_state(ObjectStore *store, bufferlist &bl)
 
   last_written_info = info;
 
+  // if we are upgrading from jewel, we need to force rebuild of
+  // missing set.  v9 was fastinfo, added v11.0.2-331-g1d5dc29a13
+  // (before kraken).  persisted missing set was circa
+  // v11.0.0-866-gb0e239da95 (a bit earlier, also before kraken).
+  // v8 was pre-jewel (per-pg meta object).
+  bool force_rebuild_missing = info_struct_v < 9;
+  if (force_rebuild_missing) {
+    dout(10) << __func__ << " detected upgrade from jewel, force_rebuild_missing"
+	     << dendl;
+  }
+
   ostringstream oss;
   pg_log.read_log_and_missing(
     store,
     coll,
     info_struct_v < 8 ? coll_t::meta() : coll,
     ghobject_t(info_struct_v < 8 ? OSD::make_pg_log_oid(pg_id) : pgmeta_oid),
     info,
+    force_rebuild_missing,
     oss,
     cct->_conf->osd_ignore_stale_divergent_priors,
     cct->_conf->osd_debug_verify_missing_on_start);
   if (oss.tellp())
     osd->clog->error() << oss.rdbuf();
 
+  if (force_rebuild_missing) {
+    dout(10) << __func__ << " forced rebuild of missing got "
+	     << pg_log.get_missing()
+	     << dendl;
+  }
+
   // log any weirdness
   log_weirdness();
 }

src/osd/PGLog.h

@@ -1238,13 +1238,14 @@ struct PGLog : DoutPrefixProvider {
     coll_t log_coll,
     ghobject_t log_oid,
     const pg_info_t &info,
+    bool force_rebuild_missing,
     ostringstream &oss,
     bool tolerate_divergent_missing_log,
     bool debug_verify_stored_missing = false
     ) {
     return read_log_and_missing(
       store, pg_coll, log_coll, log_oid, info,
-      log, missing, oss,
+      log, missing, force_rebuild_missing, oss,
       tolerate_divergent_missing_log,
       &clear_divergent_priors,
       this,
@@ -1261,6 +1262,7 @@ struct PGLog : DoutPrefixProvider {
     const pg_info_t &info,
     IndexedLog &log,
     missing_type &missing,
+    bool force_rebuild_missing,
     ostringstream &oss,
     bool tolerate_divergent_missing_log,
     bool *clear_divergent_priors = nullptr,
@@ -1282,7 +1284,6 @@ struct PGLog : DoutPrefixProvider {
     eversion_t on_disk_rollback_info_trimmed_to = eversion_t();
     ObjectMap::ObjectMapIterator p = store->get_omap_iterator(log_coll, log_oid);
     map<eversion_t, hobject_t> divergent_priors;
-    bool has_divergent_priors = false;
     missing.may_include_deletes = false;
     list<pg_log_entry_t> entries;
     list<pg_log_dup_t> dups;
@@ -1297,7 +1298,7 @@ struct PGLog : DoutPrefixProvider {
 	  ::decode(divergent_priors, bp);
 	  ldpp_dout(dpp, 20) << "read_log_and_missing " << divergent_priors.size()
 			     << " divergent_priors" << dendl;
-	  has_divergent_priors = true;
+	  assert(force_rebuild_missing);
 	  debug_verify_stored_missing = false;
 	} else if (p->key() == "can_rollback_to") {
 	  ::decode(on_disk_can_rollback_to, bp);
@@ -1344,7 +1345,7 @@ struct PGLog : DoutPrefixProvider {
       std::move(entries),
       std::move(dups));
 
-    if (has_divergent_priors || debug_verify_stored_missing) {
+    if (force_rebuild_missing || debug_verify_stored_missing) {
       // build missing
       if (debug_verify_stored_missing || info.last_complete < info.last_update) {
 	ldpp_dout(dpp, 10)
@@ -1437,7 +1438,7 @@ struct PGLog : DoutPrefixProvider {
 	    }
 	  }
 	} else {
-	  assert(has_divergent_priors);
+	  assert(force_rebuild_missing);
 	  for (map<eversion_t, hobject_t>::reverse_iterator i =
 		 divergent_priors.rbegin();
 	       i != divergent_priors.rend();
@@ -1491,7 +1492,7 @@ struct PGLog : DoutPrefixProvider {
       }
     }
 
-    if (!has_divergent_priors) {
+    if (!force_rebuild_missing) {
       if (clear_divergent_priors)
 	(*clear_divergent_priors) = false;
       missing.flush();

src/tools/ceph_objectstore_tool.cc

@@ -328,7 +328,9 @@ int get_log(ObjectStore *fs, __u8 struct_ver,
     PGLog::read_log_and_missing(fs, coll,
 		    struct_ver >= 8 ? coll : coll_t::meta(),
 		    struct_ver >= 8 ? pgid.make_pgmeta_oid() : log_oid,
-		    info, log, missing, oss,
+		    info, log, missing,
+				struct_ver < 9,
+				oss,
 		    g_ceph_context->_conf->osd_ignore_stale_divergent_priors);
     if (debug && oss.str().size())
       cerr << oss.str() << std::endl;