block: expose GatherFileStats and use it

Signed-off-by: Giedrius Statkevičius <giedriuswork@gmail.com>
stolostron · Dec 9, 2020 · 259c70b · 259c70b
1 parent 4ca451f
commit 259c70b
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 24 deletions.
diff --git a/pkg/block/block.go b/pkg/block/block.go
@@ -121,7 +121,7 @@ func Upload(ctx context.Context, logger log.Logger, bkt objstore.Bucket, bdir st
 		return errors.New("empty external labels are not allowed for Thanos block.")
 	}
 
-	meta.Thanos.Files, err = gatherFileStats(bdir, hf)
+	meta.Thanos.Files, err = GatherFileStats(bdir, hf)
 	if err != nil {
 		return errors.Wrap(err, "gather meta file stats")
 	}
@@ -279,8 +279,9 @@ func GetSegmentFiles(blockDir string) []string {
 	return result
 }
 
+// GatherFileStats returns an array with data about all files in a given block dir.
 // TODO(bwplotka): Gather stats when dirctly uploading files.
-func gatherFileStats(blockDir string, hf metadata.HashFunc) (res []metadata.File, _ error) {
+func GatherFileStats(blockDir string, hf metadata.HashFunc) (res []metadata.File, _ error) {
 	files, err := ioutil.ReadDir(filepath.Join(blockDir, ChunksDirname))
 	if err != nil {
 		return nil, errors.Wrapf(err, "read dir %v", filepath.Join(blockDir, ChunksDirname))
@@ -290,7 +291,7 @@ func gatherFileStats(blockDir string, hf metadata.HashFunc) (res []metadata.File
 			RelPath:   filepath.Join(ChunksDirname, f.Name()),
 			SizeBytes: f.Size(),
 		}
-		if hf != metadata.NoneFunc {
+		if hf != metadata.NoneFunc && !f.IsDir() {
 			h, err := metadata.CalculateHash(filepath.Join(blockDir, ChunksDirname, f.Name()), hf)
 			if err != nil {
 				return nil, errors.Wrapf(err, "calculate hash %v", filepath.Join(ChunksDirname, f.Name()))

diff --git a/pkg/testutil/e2eutil/prometheus.go b/pkg/testutil/e2eutil/prometheus.go
@@ -33,6 +33,7 @@ import (
 	"github.com/prometheus/prometheus/tsdb/index"
 	"golang.org/x/sync/errgroup"
 
+	"github.com/thanos-io/thanos/pkg/block"
 	"github.com/thanos-io/thanos/pkg/block/metadata"
 	"github.com/thanos-io/thanos/pkg/runutil"
 	"github.com/thanos-io/thanos/pkg/testutil"
@@ -482,29 +483,10 @@ func createBlock(
 
 	files := []metadata.File{}
 	if addHashes {
-		paths := []string{}
-		err := filepath.Walk(blockDir, func(path string, info os.FileInfo, err error) error {
-			if info.IsDir() {
-				return filepath.SkipDir
-			}
-			paths = append(paths, path)
-			return nil
-		})
+		files, err = block.GatherFileStats(blockDir, metadata.SHA256Func)
 		if err != nil {
-			return id, errors.Wrapf(err, "walking %s", dir)
+			return id, errors.Wrapf(err, "gathering %s file stats", blockDir)
 		}
-
-		for _, p := range paths {
-			pHash, err := metadata.CalculateHash(blockDir+p, metadata.SHA256Func)
-			if err != nil {
-				return id, errors.Wrapf(err, "calculating hash of %s", blockDir+p)
-			}
-			files = append(files, metadata.File{
-				RelPath: p,
-				Hash:    &pHash,
-			})
-		}
-
 	}
 
 	if _, err = metadata.InjectThanos(log.NewNopLogger(), blockDir, metadata.Thanos{