Receive: Add experimental snapshot on shutdown (#5836)

* Add experimental snapshot on shutdown to receive

Signed-off-by: Matej Gera <matejgera@gmail.com>

* Add CHANGELOG

Signed-off-by: Matej Gera <matejgera@gmail.com>

Signed-off-by: Matej Gera <matejgera@gmail.com>

CHANGELOG.md

@@ -16,6 +16,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re
 
 - [#5814](https://github.com/thanos-io/thanos/pull/5814) Store: Add metric `thanos_bucket_store_postings_size_bytes` that shows the distribution of how many postings (in bytes) were needed for each Series() call in Thanos Store. Useful for determining limits.
 - [#5801](https://github.com/thanos-io/thanos/pull/5801) Store: add a new limiter `--store.grpc.downloaded-bytes-limit` that limits the number of bytes downloaded in each Series/LabelNames/LabelValues call. Use `thanos_bucket_store_postings_size_bytes` for determining the limits.
+- [#5836](https://github.com/thanos-io/thanos/pull/5836) Receive: Add hidden flag `tsdb.memory-snapshot-on-shutdown` to enable experimental TSDB feature to snapshot on shutdown. This is intended to speed up receiver restart.
 
 ### Changed
 

cmd/thanos/receive.go

@@ -77,14 +77,15 @@ func registerReceive(app *extkingpin.App) {
 		}
 
 		tsdbOpts := &tsdb.Options{
-			MinBlockDuration:         int64(time.Duration(*conf.tsdbMinBlockDuration) / time.Millisecond),
-			MaxBlockDuration:         int64(time.Duration(*conf.tsdbMaxBlockDuration) / time.Millisecond),
-			RetentionDuration:        int64(time.Duration(*conf.retention) / time.Millisecond),
-			NoLockfile:               conf.noLockFile,
-			WALCompression:           conf.walCompression,
-			MaxExemplars:             conf.tsdbMaxExemplars,
-			EnableExemplarStorage:    true,
-			HeadChunksWriteQueueSize: int(conf.tsdbWriteQueueSize),
+			MinBlockDuration:               int64(time.Duration(*conf.tsdbMinBlockDuration) / time.Millisecond),
+			MaxBlockDuration:               int64(time.Duration(*conf.tsdbMaxBlockDuration) / time.Millisecond),
+			RetentionDuration:              int64(time.Duration(*conf.retention) / time.Millisecond),
+			NoLockfile:                     conf.noLockFile,
+			WALCompression:                 conf.walCompression,
+			MaxExemplars:                   conf.tsdbMaxExemplars,
+			EnableExemplarStorage:          true,
+			HeadChunksWriteQueueSize:       int(conf.tsdbWriteQueueSize),
+			EnableMemorySnapshotOnShutdown: conf.tsdbMemorySnapshotOnShutdown,
 		}
 
 		// Are we running in IngestorOnly, RouterOnly or RouterIngestor mode?
@@ -772,11 +773,12 @@ type receiveConfig struct {
 	forwardTimeout    *model.Duration
 	compression       string
 
-	tsdbMinBlockDuration       *model.Duration
-	tsdbMaxBlockDuration       *model.Duration
-	tsdbAllowOverlappingBlocks bool
-	tsdbMaxExemplars           int64
-	tsdbWriteQueueSize         int64
+	tsdbMinBlockDuration         *model.Duration
+	tsdbMaxBlockDuration         *model.Duration
+	tsdbAllowOverlappingBlocks   bool
+	tsdbMaxExemplars             int64
+	tsdbWriteQueueSize           int64
+	tsdbMemorySnapshotOnShutdown bool
 
 	walCompression bool
 	noLockFile     bool
@@ -876,6 +878,10 @@ func (rc *receiveConfig) registerFlag(cmd extkingpin.FlagClause) {
 			"A queue size of zero (default) disables this feature entirely.").
 		Default("0").Hidden().Int64Var(&rc.tsdbWriteQueueSize)
 
+	cmd.Flag("tsdb.memory-snapshot-on-shutdown",
+		"[EXPERIMENTAL] Enables feature to snapshot in-memory chunks on shutdown for faster restarts.").
+		Default("false").Hidden().BoolVar(&rc.tsdbMemorySnapshotOnShutdown)
+
 	cmd.Flag("hash-func", "Specify which hash function to use when calculating the hashes of produced files. If no function has been specified, it does not happen. This permits avoiding downloading some files twice albeit at some performance cost. Possible values are: \"\", \"SHA256\".").
 		Default("").EnumVar(&rc.hashFunc, "SHA256", "")
 

pkg/receive/multitsdb.go

@@ -59,7 +59,7 @@ type MultiTSDB struct {
 }
 
 // NewMultiTSDB creates new MultiTSDB.
-// NOTE: Passed labels has to be sorted by name.
+// NOTE: Passed labels must be sorted lexicographically (alphabetically).
 func NewMultiTSDB(
 	dataDir string,
 	l log.Logger,