MB-100 Fix failing MOI test TestIdxCorruptPartitionedIndex

Change-Id: I90b06e97ec43a497db0a89e9bb14cbc23906d8b1
couchbase · Nov 12, 2021 · 5e41a44 · 5e41a44
1 parent 99326e2
commit 5e41a44
Show file tree

Hide file tree

Showing 3 changed files with 42 additions and 20 deletions.
diff --git a/secondary/tests/framework/common/memdb_slice_mock.go b/secondary/tests/framework/common/memdb_slice_mock.go
@@ -4,10 +4,12 @@ import (
 	"container/list"
 	"encoding/json"
 	"io/ioutil"
+	"log"
 	"os"
 	"path/filepath"
 	"sort"
 	"strings"
+	"time"
 )
 
 const tmpDirName = ".tmp" // Same as indexer.tmpDirName
@@ -50,32 +52,49 @@ func NewSnapshotInfoContainer(infos []*MemdbSnapshotInfo) *snapshotInfoContainer
 	return sc
 }
 
-func GetMemDBSnapshots(slicePath string) ([]*MemdbSnapshotInfo, error) {
+func GetMemDBSnapshots(slicePath string, retry bool) ([]*MemdbSnapshotInfo, error) {
 	var files []string
 	pattern := "*/manifest.json"
-	all, _ := filepath.Glob(filepath.Join(slicePath, pattern))
-	for _, f := range all {
-		if !strings.Contains(f, tmpDirName) {
-			files = append(files, f)
-		}
-	}
-	sort.Strings(files)
 
 	var infos []*MemdbSnapshotInfo
-	for i := len(files) - 1; i >= 0; i-- {
-		f := files[i]
-		info := &MemdbSnapshotInfo{DataPath: filepath.Dir(f)}
-		fd, err := os.Open(f)
-		if err == nil {
-			defer fd.Close()
-			bs, err := ioutil.ReadAll(fd)
+
+	for i := 0; i < 100; i++ {
+		all, err := filepath.Glob(filepath.Join(slicePath, pattern))
+		if err != nil {
+			log.Printf("Error in filepath.Glob %v\n", err)
+			continue
+		}
+
+		for _, f := range all {
+			if !strings.Contains(f, tmpDirName) {
+				files = append(files, f)
+			}
+		}
+		sort.Strings(files)
+
+		for i := len(files) - 1; i >= 0; i-- {
+			f := files[i]
+			info := &MemdbSnapshotInfo{DataPath: filepath.Dir(f)}
+			fd, err := os.Open(f)
 			if err == nil {
-				err = json.Unmarshal(bs, info)
+				defer fd.Close()
+				bs, err := ioutil.ReadAll(fd)
 				if err == nil {
-					infos = append(infos, info)
+					err = json.Unmarshal(bs, info)
+					if err == nil {
+						infos = append(infos, info)
+					}
 				}
 			}
 		}
+
+		if len(infos) != 0 || !retry {
+			break
+		}
+
+		time.Sleep(100 * time.Millisecond)
+		log.Printf("GetMemDBSnapshots: retrying %v\n", i+1)
 	}
+
 	return infos, nil
 }
diff --git a/secondary/tests/framework/secondaryindex/secondaryindexmanagement.go b/secondary/tests/framework/secondaryindex/secondaryindexmanagement.go
@@ -661,7 +661,7 @@ func corruptMOIIndex(indexName, bucketName, dirPath string, partnId c.PartitionI
 
 	log.Printf("Corrupting index %v slicePath %v", indexName, slicePath)
 
-	infos, err := tc.GetMemDBSnapshots(slicePath)
+	infos, err := tc.GetMemDBSnapshots(slicePath, true)
 	if err != nil {
 		return err
 	}
@@ -673,6 +673,9 @@ func corruptMOIIndex(indexName, bucketName, dirPath string, partnId c.PartitionI
 		return errors.New("Latest Snapshot not found")
 	}
 
+	// Allow indexer to persist the checksums.json before overwriting it.
+	time.Sleep(5 * time.Second)
+
 	for _, snapInfo := range allSnapshots {
 		fmt.Println("snapshot datapath = ", snapInfo.DataPath)
 		datadir := filepath.Join(snapInfo.DataPath, "data")
@@ -734,7 +737,7 @@ func CorruptMOIIndexLatestSnapshot(indexName, bucketName, dirPath, indexUsing st
 
 	log.Printf("Corrupting index %v slicePath %v", indexName, slicePath)
 
-	infos, err := tc.GetMemDBSnapshots(slicePath)
+	infos, err := tc.GetMemDBSnapshots(slicePath, true)
 	if err != nil {
 		return err
 	}

diff --git a/secondary/tests/functionaltests/set99_storage_stats_test.go b/secondary/tests/functionaltests/set99_storage_stats_test.go
@@ -382,7 +382,7 @@ func TestIdxCorruptMOITwoSnapsBothCorrupt(t *testing.T) {
 	}
 
 	// Step 3: Corrupt all snapshots
-	infos, err := tc.GetMemDBSnapshots(slicePath)
+	infos, err := tc.GetMemDBSnapshots(slicePath, false)
 	FailTestIfError(err, "Error in GetMemDBSnapshots", t)
 
 	snapInfoContainer := tc.NewSnapshotInfoContainer(infos)