carbonserver: introduce file list cache v2

This commit introduces flc v2, the main purpose is to have better quota support during restart.
Mainly for saving physical and logical size info in the flc.

Users can transparently switch from v1 to v2 and vice versa.

More context on quota friendly: with flc v1, when go-carbon is restarted, it would create a trie index
tree using the logical size as physical size for metrics and this would potentially temporary high
physical size accounting and leads to unnecessary throttling if physical size quota is enabled.

With flc v2, go-carbon would keep physical, logical, data points, and first_seen_at timestamp in the
cache file and use that for rebuilding the trie index after restart. This would avoids temporary incorrect
physical size accounting issues with flcv1.

README.md

@@ -377,7 +377,26 @@ trie-index = false
 # most of the indexing time if it contains a high number of metrics (10
 # - 40 millions). go-carbon only reads the cached file list once after
 # reboot and the cache result is updated after every scan. (EXPERIMENTAL)
-file-list-cache = ""
+#
+# file-list-cache = "/var/lib/carbon/carbonserver-file-list-cache.bin"
+
+# Supported FLC (file list cache) version values are: 2, 1 (default value is 1
+# for backward compatibility).
+#
+# File list cache v2 is is more advanced of file list cache for go-carbon, with
+# good quota support during restarts (by keeping physical, logical sizes, metric
+# creation time in the cache). For more details, see carbonserver/flc.go.
+#
+# V2 file is no longer a plain text file compressed with gzip, but a semi-binary
+# file. For reading flc v2 cache file, use go-carbon with flag -print-file-list-cache:
+#
+#     go-carbon -print-file-list-cache /var/lib/carbon/carbonserver-file-list-cache.bin
+#
+# For upgrade or downgrade to v2 or v1, you can just update the config,
+# go-carbon would transparently detect the existing cache file on restart. No
+# manual deletion needed.
+#
+# file-list-cache-version = 2
 
 # Enable concurrently building index without maintaining a new copy
 # index structure. More memory efficient.

carbon/app.go

@@ -502,6 +502,7 @@ func (app *App) Start(version string) (err error) {
 		carbonserver.SetTrieIndex(conf.Carbonserver.TrieIndex)
 		carbonserver.SetConcurrentIndex(conf.Carbonserver.ConcurrentIndex)
 		carbonserver.SetFileListCache(conf.Carbonserver.FileListCache)
+		carbonserver.SetFileListCacheVersion(conf.Carbonserver.FileListCacheVersion)
 		carbonserver.SetInternalStatsDir(conf.Carbonserver.InternalStatsDir)
 		carbonserver.SetPercentiles(conf.Carbonserver.Percentiles)
 		// carbonserver.SetQueryTimeout(conf.Carbonserver.QueryTimeout.Value())

carbon/config.go

@@ -126,10 +126,12 @@ type carbonserverConfig struct {
 	MaxMetricsGlobbed  int `toml:"max-metrics-globbed"`
 	MaxMetricsRendered int `toml:"max-metrics-rendered"`
 
-	TrieIndex       bool   `toml:"trie-index"`
-	ConcurrentIndex bool   `toml:"concurrent-index"`
-	RealtimeIndex   int    `toml:"realtime-index"`
-	FileListCache   string `toml:"file-list-cache"`
+	TrieIndex       bool `toml:"trie-index"`
+	ConcurrentIndex bool `toml:"concurrent-index"`
+	RealtimeIndex   int  `toml:"realtime-index"`
+
+	FileListCache        string `toml:"file-list-cache"`
+	FileListCacheVersion int    `toml:"file-list-cache-version"`
 
 	QuotaUsageReportFrequency *Duration `toml:"quota-usage-report-frequency"`
 
@@ -259,13 +261,14 @@ func NewConfig() *Config {
 			WriteTimeout: &Duration{
 				Duration: 60 * time.Second,
 			},
-			QueryCacheEnabled:  true,
-			QueryCacheSizeMB:   0,
-			FindCacheEnabled:   true,
-			TrigramIndex:       true,
-			CacheScan:          false,
-			MaxMetricsGlobbed:  10_000_000,
-			MaxMetricsRendered: 1_000_000,
+			QueryCacheEnabled:    true,
+			QueryCacheSizeMB:     0,
+			FindCacheEnabled:     true,
+			TrigramIndex:         true,
+			CacheScan:            false,
+			MaxMetricsGlobbed:    10_000_000,
+			MaxMetricsRendered:   1_000_000,
+			FileListCacheVersion: 1, // see carbonserver/flc.go
 		},
 		Carbonlink: carbonlinkConfig{
 			Listen:  "127.0.0.1:7002",