Retain Grafana Cloud/GEM schema fields

CHANGELOG.md

@@ -1,3 +1,6 @@
+# unreleased 
+* retain Grafana Cloud/GEM fields when parsing storage-schemas.conf #502
+
 # v1.2.1: patch maintenance release. December 22, 2022
 
 * build using go 1.18 #497

persister/whisper_schema.go

@@ -21,6 +21,11 @@ type Schema struct {
 	RetentionStr string
 	Retentions   whisper.Retentions
 	Priority     int64
+
+	// Optional fields that are only used by Grafana Cloud Graphite and Grafana Enterprise Metrics
+	// Documentation https://grafana.com/docs/enterprise-metrics/latest/graphite/schemas/#storage-schemas
+	IntervalsStr       string
+	RelativeToQueryStr string
 }
 
 // WhisperSchemas contains schema settings
@@ -53,6 +58,12 @@ func (s WhisperSchemas) String() string {
 		buf.WriteString(fmt.Sprintf("pattern = %s\n", schema.Pattern.String()))
 		buf.WriteString(fmt.Sprintf("retentions = %s\n", schema.RetentionStr))
 		buf.WriteString(fmt.Sprintf("priority = %d\n", schema.Priority))
+		if schema.IntervalsStr != "" {
+			buf.WriteString(fmt.Sprintf("intervals = %s\n", schema.IntervalsStr))
+		}
+		if schema.RelativeToQueryStr != "" {
+			buf.WriteString(fmt.Sprintf("relativeToQuery = %s\n", schema.RelativeToQueryStr))
+		}
 	}
 	return buf.String()
 }
@@ -127,6 +138,12 @@ func ReadWhisperSchemas(filename string) (WhisperSchemas, error) {
 		}
 		schema.Priority = int64(p)<<32 - int64(i) // to sort records with same priority by position in file
 
+		schema.IntervalsStr = section["intervals"]
+		// The CRNG ini parser is case-insensitive, which is why we look up "relativetoquery" instead of "relativeToQuery".
+		// The GrafanaNet/GEM schemas parser is case-sensitive.
+		// In future CRNG releases, we might change this parser to be case-sensitive as well.
+		schema.RelativeToQueryStr = section["relativetoquery"]
+
 		schemas = append(schemas, schema)
 	}
 

persister/whisper_schema_test.go

@@ -5,9 +5,10 @@ import (
 	"os"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/grafana/carbon-relay-ng/go-whisper"
 	"github.com/grafana/carbon-relay-ng/pkg/test"
-	"github.com/stretchr/testify/assert"
 )
 
 func assertRetentionsEq(t *testing.T, ret whisper.Retentions, s string) {
@@ -71,9 +72,11 @@ func parseSchemas(t *testing.T, content string) (WhisperSchemas, error) {
 }
 
 type testcase struct {
-	name       string
-	pattern    string
-	retentions string
+	name            string
+	pattern         string
+	retentions      string
+	intervals       string
+	relativeToQuery string
 }
 
 func assertSchemas(t *testing.T, content string, expected []testcase, msg ...interface{}) WhisperSchemas {
@@ -87,6 +90,8 @@ func assertSchemas(t *testing.T, content string, expected []testcase, msg ...int
 		assert.Equal(t, expected[i].name, schemas[i].Name)
 		assert.Equal(t, expected[i].pattern, schemas[i].Pattern.String())
 		assertRetentionsEq(t, schemas[i].Retentions, expected[i].retentions)
+		assert.Equal(t, expected[i].intervals, schemas[i].IntervalsStr)
+		assert.Equal(t, expected[i].relativeToQuery, schemas[i].RelativeToQueryStr)
 	}
 
 	return schemas
@@ -103,8 +108,8 @@ pattern = .*
 retentions = 1m:30d,1h:5y
 	`,
 		[]testcase{
-			{"carbon", "^carbon\\.", "60s:90d"},
-			{"default", ".*", "1m:30d,1h:5y"},
+			{name: "carbon", pattern: "^carbon\\.", retentions: "60s:90d"},
+			{name: "default", pattern: ".*", retentions: "1m:30d,1h:5y"},
 		},
 	)
 }
@@ -117,7 +122,7 @@ pattern = ^carbon\.
 retentions = 60s:90d
 	`,
 		[]testcase{
-			{"carbon", "^carbon\\.", "60s:90d"},
+			{name: "carbon", pattern: "^carbon\\.", retentions: "60s:90d"},
 		},
 	)
 }
@@ -151,11 +156,11 @@ retentions = 1s:7d
 priority = 10
 	`,
 		[]testcase{
-			{"gitlab", "^gitlab\\.", "1s:7d"},
-			{"collector", "^.*\\.collector\\.", "5s:5m,5m:30d"},
-			{"jira", "^server\\.", "1s:7d"},
-			{"carbon", "^carbon\\.", "1m:90d"},
-			{"db", "^db\\.", "1m:30d,1h:5y"},
+			{name: "gitlab", pattern: "^gitlab\\.", retentions: "1s:7d"},
+			{name: "collector", pattern: "^.*\\.collector\\.", retentions: "5s:5m,5m:30d"},
+			{name: "jira", pattern: "^server\\.", retentions: "1s:7d"},
+			{name: "carbon", pattern: "^carbon\\.", retentions: "1m:90d"},
+			{name: "db", pattern: "^db\\.", retentions: "1m:30d,1h:5y"},
 		},
 	)
 
@@ -173,6 +178,32 @@ priority = 10
 	assert.False(ok)
 }
 
+func TestParseSchemasGrafanaNet(t *testing.T) {
+	assertSchemas(t, `
+[carbon]
+pattern = ^carbon\.
+retentions = 60s:90d
+relativeToQuery = true
+
+[a]
+pattern = ^a\.
+retentions = 60s:90d
+intervals = 1594166400:30min,1625702400:15s
+
+[default]
+pattern = .*
+retentions = 1m:30d,1h:5y
+intervals = 1594166400:30min,1625702400:1s
+relativeToQuery = false
+	`,
+		[]testcase{
+			{name: "carbon", pattern: "^carbon\\.", retentions: "60s:90d", relativeToQuery: "true"},
+			{name: "a", pattern: "^a\\.", retentions: "60s:90d", intervals: "1594166400:30min,1625702400:15s"},
+			{name: "default", pattern: ".*", retentions: "1m:30d,1h:5y", relativeToQuery: "false", intervals: "1594166400:30min,1625702400:1s"},
+		},
+	)
+}
+
 func TestSchemasNotFound(t *testing.T) {
 	// create and remove file
 	assert := assert.New(t)
@@ -269,6 +300,7 @@ priority =
 `, nil, "Empty priority")
 }
 
+// TODO: add intervals and relative to query
 // TestSchemasString tests that routes.String() matches what we expect
 func TestSchemasString(t *testing.T) {
 	input := `# This is a wild comment
@@ -285,7 +317,10 @@ priority = 6
 [hello]
 pattern = .*
 retentions = 10m:90d,10m:365y
-priority = 1`
+priority = 1
+relativeToQuery = true
+intervals = 1594166400:30min,1625702400:15s
+`
 
 	exp := `[fancy-patt-with-legacy-retentions]
 pattern = ^patt2.*$
@@ -295,6 +330,8 @@ priority = 25769803775
 pattern = .*
 retentions = 10m:90d,10m:365y
 priority = 4294967294
+intervals = 1594166400:30min,1625702400:15s
+relativeToQuery = true
 [first]
 pattern = ^carbon\.
 retentions = 600s:90d,5m:365y