indexheader: don't fail the start of the Snapshotter

Signed-off-by: Vladimir Varankin <vladimir.varankin@grafana.com>

pkg/storegateway/bucket.go

@@ -361,14 +361,10 @@ func (s *BucketStore) syncBlocks(ctx context.Context, initialSync bool) error {
 	}
 
 	// Start snapshotter in the end of the sync, but do that only once per BucketStore's lifetime.
-	// We do that here so the snapshotter watched after blocks from both initial sync and those discovered later.
-	var err error
+	// We do that here, so the snapshotter watched after blocks from both initial sync and those discovered later.
 	s.snapshotterStartOnce.Do(func() {
-		err = s.snapshotter.Start(ctx, s.indexReaderPool)
+		s.snapshotter.Start(ctx, s.indexReaderPool)
 	})
-	if err != nil {
-		return errors.Wrap(err, "start index headers snapshotter")
-	}
 
 	return nil
 }

pkg/storegateway/indexheader/snapshotter.go

@@ -6,7 +6,6 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
-	"fmt"
 	"os"
 	"path/filepath"
 	"time"
@@ -46,13 +45,15 @@ type blocksLoader interface {
 	LoadedBlocks() map[ulid.ULID]int64
 }
 
-func (s *Snapshotter) Start(ctx context.Context, bl blocksLoader) error {
-	err := s.PersistLoadedBlocks(bl)
-	if err != nil {
-		return fmt.Errorf("persist initial list of lazy-loaded index headers: %w", err)
-	}
-
+// Start spawns a background job that periodically persists the list of lazy-loaded index headers.
+func (s *Snapshotter) Start(ctx context.Context, bl blocksLoader) {
 	go func() {
+		err := s.PersistLoadedBlocks(bl)
+		if err != nil {
+			// Note, the decision here is to only log the error but not failing the job. We may reconsider that later.
+			level.Warn(s.logger).Log("msg", "failed to persist initial list of lazy-loaded index headers", "err", err)
+		}
+
 		tick := time.NewTicker(time.Minute)
 		defer tick.Stop()
 
@@ -69,8 +70,6 @@ func (s *Snapshotter) Start(ctx context.Context, bl blocksLoader) error {
 			}
 		}
 	}()
-
-	return nil
 }
 
 func (s *Snapshotter) Stop() {