osd: process _scan_snaps() with all snapshots with head

Test with more than 1 osd with scrub errors Fixes: http://tracker.ceph.com/issues/22881 Signed-off-by: David Zafman <dzafman@redhat.com>
ceph · Apr 20, 2018 · d165b1b · d165b1b
1 parent 5c4f8c7
commit d165b1b
Show file tree

Hide file tree

Showing 2 changed files with 101 additions and 57 deletions.
diff --git a/qa/standalone/scrub/osd-scrub-snaps.sh b/qa/standalone/scrub/osd-scrub-snaps.sh
@@ -37,19 +37,24 @@ function run() {
     done
 }
 
-function TEST_scrub_snaps() {
+function scrub_snaps() {
     local dir=$1
+    shift
+    local OSDS=$1
+    shift
+    local scrub_errors=$1
+    shift
+
     local poolname=test
     local OBJS=15
-    local OSDS=1
 
     TESTDATA="testdata.$$"
 
     run_mon $dir a --osd_pool_default_size=$OSDS || return 1
     run_mgr $dir x || return 1
     for osd in $(seq 0 $(expr $OSDS - 1))
     do
-      run_osd $dir $osd || return 1
+      run_osd $dir $osd "$@" || return 1
     done
 
     # Create a pool with a single pg
@@ -107,60 +112,63 @@ function TEST_scrub_snaps() {
 
     # Don't need to use ceph_objectstore_tool() function because osd stopped
 
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj1)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" --force remove
-
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --op list obj5 | grep \"snapid\":2)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" remove
-
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --op list obj5 | grep \"snapid\":1)"
-    OBJ5SAVE="$JSON"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" remove
-
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --op list obj5 | grep \"snapid\":4)"
-    dd if=/dev/urandom of=$TESTDATA bs=256 count=18
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" set-bytes $TESTDATA
-
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj3)"
-    dd if=/dev/urandom of=$TESTDATA bs=256 count=15
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" set-bytes $TESTDATA
-
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --op list obj4 | grep \"snapid\":7)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" remove
-
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj2)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" rm-attr snapset
-
-    # Create a clone which isn't in snapset and doesn't have object info
-    JSON="$(echo "$OBJ5SAVE" | sed s/snapid\":1/snapid\":7/)"
-    dd if=/dev/urandom of=$TESTDATA bs=256 count=7
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" set-bytes $TESTDATA
-
-    rm -f $TESTDATA
-
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj6)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj7)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset corrupt
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj8)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset seq
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj9)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset clone_size
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj10)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset clone_overlap
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj11)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset clones
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj12)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset head
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj13)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset snaps
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj14)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" clear-snapset size
-
     echo "garbage" > $dir/bad
-    JSON="$(ceph-objectstore-tool --data-path $dir/${primary} --head --op list obj15)"
-    ceph-objectstore-tool --data-path $dir/${primary} "$JSON" set-attr snapset $dir/bad
+    for osd in $(seq 0 $(expr $OSDS - 1))
+    do
+      JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj1)"
+      ceph-objectstore-tool --data-path $dir/${osd} "$JSON" --force remove
+
+      JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --op list obj5 | grep \"snapid\":2)"
+      ceph-objectstore-tool --data-path $dir/${osd} "$JSON" remove
+
+      JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --op list obj5 | grep \"snapid\":1)"
+      OBJ5SAVE="$JSON"
+      ceph-objectstore-tool --data-path $dir/${osd} "$JSON" remove
+
+      JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --op list obj5 | grep \"snapid\":4)"
+      dd if=/dev/urandom of=$TESTDATA bs=256 count=18
+      ceph-objectstore-tool --data-path $dir/${osd} "$JSON" set-bytes $TESTDATA
+
+      JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj3)"
+      dd if=/dev/urandom of=$TESTDATA bs=256 count=15
+      ceph-objectstore-tool --data-path $dir/${osd} "$JSON" set-bytes $TESTDATA
+
+      JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --op list obj4 | grep \"snapid\":7)"
+      ceph-objectstore-tool --data-path $dir/${osd} "$JSON" remove
+
+      JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj2)"
+      ceph-objectstore-tool --data-path $dir/${osd} "$JSON" rm-attr snapset
+
+      # Create a clone which isn't in snapset and doesn't have object info
+      JSON="$(echo "$OBJ5SAVE" | sed s/snapid\":1/snapid\":7/)"
+      dd if=/dev/urandom of=$TESTDATA bs=256 count=7
+      ceph-objectstore-tool --data-path $dir/${osd} "$JSON" set-bytes $TESTDATA
+
+      JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj6)"
+      ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset
+      JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj7)"
+      ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset corrupt
+      JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj8)"
+      ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset seq
+      JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj9)"
+      ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset clone_size
+      JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj10)"
+      ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset clone_overlap
+      JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj11)"
+      ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset clones
+      JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj12)"
+      ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset head
+      JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj13)"
+      ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset snaps
+      JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj14)"
+      ceph-objectstore-tool --data-path $dir/${osd} "$JSON" clear-snapset size
+
+      JSON="$(ceph-objectstore-tool --data-path $dir/${osd} --head --op list obj15)"
+      ceph-objectstore-tool --data-path $dir/${osd} "$JSON" set-attr snapset $dir/bad
+    done
+
     rm -f $dir/bad
+    rm -f $TESTDATA
 
     for osd in $(seq 0 $(expr $OSDS - 1))
     do
@@ -676,7 +684,7 @@ EOF
     err_strings[17]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj9:1 is missing in clone_size"
     err_strings[18]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj11:1 is an unexpected clone"
     err_strings[19]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj14:1 size 1032 != clone_size 1033"
-    err_strings[20]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub 20 errors"
+    err_strings[20]="log_channel[(]cluster[)] log [[]ERR[]] : [0-9]*[.]0 scrub ${scrub_errors} errors"
     err_strings[21]="log_channel[(]cluster[)] log [[]ERR[]] : scrub [0-9]*[.]0 .*:::obj15:head can't decode 'snapset' attr buffer"
 
     for err_string in "${err_strings[@]}"
@@ -698,6 +706,14 @@ EOF
     return 0
 }
 
+function TEST_scrub_snaps() {
+  scrub_snaps $dir 1 20
+}
+
+function TEST_scrub_snaps_multi() {
+  scrub_snaps $dir 2 30 --osd_scrub_chunk_min=3 --osd_scrub_chunk_max=3
+}
+
 main osd-scrub-snaps "$@"
 
 # Local Variables:

diff --git a/src/osd/PG.cc b/src/osd/PG.cc
@@ -4366,12 +4366,17 @@ void PG::_scan_snaps(ScrubMap &smap)
 {
   hobject_t head;
   SnapSet snapset;
+
+  dout(20) << __func__ << " start" << dendl;
+
   for (map<hobject_t, ScrubMap::object>::reverse_iterator i = smap.objects.rbegin();
        i != smap.objects.rend();
        ++i) {
     const hobject_t &hoid = i->first;
     ScrubMap::object &o = i->second;
 
+    dout(20) << __func__ << " " << hoid << dendl;
+
     assert(!hoid.is_snapdir());
     if (hoid.is_head()) {
       // parse the SnapSet
@@ -4555,8 +4560,29 @@ int PG::build_scrub_map_chunk(
   // finish
   dout(20) << __func__ << " finishing" << dendl;
   assert(pos.done());
-  _scan_snaps(map);
   _repair_oinfo_oid(map);
+  if (!is_primary()) {
+    ScrubMap for_meta_scrub;
+    scrubber.cleaned_meta_map.insert(map);
+    if (end.is_max() || scrubber.cleaned_meta_map.objects.empty()) {
+      scrubber.cleaned_meta_map.swap(for_meta_scrub);
+    } else {
+      auto iter = scrubber.cleaned_meta_map.objects.end();
+      --iter; // not empty, see if clause
+      auto begin = scrubber.cleaned_meta_map.objects.begin();
+      while (iter != begin) {
+        auto next = iter--;
+        if (next->first.get_head() != iter->first.get_head()) {
+	  ++iter;
+	  break;
+        }
+      }
+      for_meta_scrub.objects.insert(begin, iter);
+      scrubber.cleaned_meta_map.objects.erase(begin, iter);
+    }
+
+    _scan_snaps(for_meta_scrub);
+  }
 
   dout(20) << __func__ << " done, got " << map.objects.size() << " items"
 	   << dendl;
@@ -5337,6 +5363,8 @@ void PG::scrub_compare_maps()
 
   // ok, do the pg-type specific scrubbing
   scrub_snapshot_metadata(for_meta_scrub, missing_digest);
+  // Called here on the primary can use an authoritative map if it isn't the primary
+  _scan_snaps(for_meta_scrub);
   if (!scrubber.store->empty()) {
     if (state_test(PG_STATE_REPAIR)) {
       dout(10) << __func__ << ": discarding scrub results" << dendl;