diff --git a/pkg/block/block.go b/pkg/block/block.go index 15a6e5c85f..3c7fe4d601 100644 --- a/pkg/block/block.go +++ b/pkg/block/block.go @@ -121,7 +121,7 @@ func Upload(ctx context.Context, logger log.Logger, bkt objstore.Bucket, bdir st return errors.New("empty external labels are not allowed for Thanos block.") } - meta.Thanos.Files, err = gatherFileStats(bdir, hf) + meta.Thanos.Files, err = GatherFileStats(bdir, hf) if err != nil { return errors.Wrap(err, "gather meta file stats") } @@ -279,8 +279,9 @@ func GetSegmentFiles(blockDir string) []string { return result } +// GatherFileStats returns an array with data about all files in a given block dir. // TODO(bwplotka): Gather stats when dirctly uploading files. -func gatherFileStats(blockDir string, hf metadata.HashFunc) (res []metadata.File, _ error) { +func GatherFileStats(blockDir string, hf metadata.HashFunc) (res []metadata.File, _ error) { files, err := ioutil.ReadDir(filepath.Join(blockDir, ChunksDirname)) if err != nil { return nil, errors.Wrapf(err, "read dir %v", filepath.Join(blockDir, ChunksDirname)) @@ -290,7 +291,7 @@ func gatherFileStats(blockDir string, hf metadata.HashFunc) (res []metadata.File RelPath: filepath.Join(ChunksDirname, f.Name()), SizeBytes: f.Size(), } - if hf != metadata.NoneFunc { + if hf != metadata.NoneFunc && !f.IsDir() { h, err := metadata.CalculateHash(filepath.Join(blockDir, ChunksDirname, f.Name()), hf) if err != nil { return nil, errors.Wrapf(err, "calculate hash %v", filepath.Join(ChunksDirname, f.Name())) diff --git a/pkg/testutil/e2eutil/prometheus.go b/pkg/testutil/e2eutil/prometheus.go index f5cf3ddb35..6da926b1f7 100644 --- a/pkg/testutil/e2eutil/prometheus.go +++ b/pkg/testutil/e2eutil/prometheus.go @@ -33,6 +33,7 @@ import ( "github.com/prometheus/prometheus/tsdb/index" "golang.org/x/sync/errgroup" + "github.com/thanos-io/thanos/pkg/block" "github.com/thanos-io/thanos/pkg/block/metadata" "github.com/thanos-io/thanos/pkg/runutil" "github.com/thanos-io/thanos/pkg/testutil" @@ -482,29 +483,10 @@ func createBlock( files := []metadata.File{} if addHashes { - paths := []string{} - err := filepath.Walk(blockDir, func(path string, info os.FileInfo, err error) error { - if info.IsDir() { - return filepath.SkipDir - } - paths = append(paths, path) - return nil - }) + files, err = block.GatherFileStats(blockDir, metadata.SHA256Func) if err != nil { - return id, errors.Wrapf(err, "walking %s", dir) + return id, errors.Wrapf(err, "gathering %s file stats", blockDir) } - - for _, p := range paths { - pHash, err := metadata.CalculateHash(blockDir+p, metadata.SHA256Func) - if err != nil { - return id, errors.Wrapf(err, "calculating hash of %s", blockDir+p) - } - files = append(files, metadata.File{ - RelPath: p, - Hash: &pHash, - }) - } - } if _, err = metadata.InjectThanos(log.NewNopLogger(), blockDir, metadata.Thanos{