Merge pull request #5625 from nicolastakashi/main

[BUGFIX]: prometheus agent wal compression support
prometheus-operator · Jun 2, 2023 · 54ed924 · 54ed924
2 parents 0c0b9ef + 31e31df
commit 54ed924
Show file tree

Hide file tree

Showing 13 changed files with 104 additions and 32 deletions.
diff --git a/Documentation/api.md b/Documentation/api.md
@@ -2202,8 +2202,9 @@ bool
 </em>
 </td>
 <td>
-<p>Enable compression of the write-ahead log using Snappy. This flag is
-only available in versions of Prometheus &gt;= 2.11.0.</p>
+<p>Configures compression of the write-ahead log using Snappy.
+This flag is only available in versions of Prometheus &gt;= 2.11.0.
+WAL compression is enabled by default for Prometheus &gt;= 2.20.0</p>
 </td>
 </tr>
 <tr>
@@ -6108,8 +6109,9 @@ bool
 </em>
 </td>
 <td>
-<p>Enable compression of the write-ahead log using Snappy. This flag is
-only available in versions of Prometheus &gt;= 2.11.0.</p>
+<p>Configures compression of the write-ahead log using Snappy.
+This flag is only available in versions of Prometheus &gt;= 2.11.0.
+WAL compression is enabled by default for Prometheus &gt;= 2.20.0</p>
 </td>
 </tr>
 <tr>
@@ -9425,8 +9427,9 @@ bool
 </em>
 </td>
 <td>
-<p>Enable compression of the write-ahead log using Snappy. This flag is
-only available in versions of Prometheus &gt;= 2.11.0.</p>
+<p>Configures compression of the write-ahead log using Snappy.
+This flag is only available in versions of Prometheus &gt;= 2.11.0.
+WAL compression is enabled by default for Prometheus &gt;= 2.20.0</p>
 </td>
 </tr>
 <tr>
@@ -14393,8 +14396,9 @@ bool
 </em>
 </td>
 <td>
-<p>Enable compression of the write-ahead log using Snappy. This flag is
-only available in versions of Prometheus &gt;= 2.11.0.</p>
+<p>Configures compression of the write-ahead log using Snappy.
+This flag is only available in versions of Prometheus &gt;= 2.11.0.
+WAL compression is enabled by default for Prometheus &gt;= 2.20.0</p>
 </td>
 </tr>
 <tr>
@@ -16994,8 +16998,9 @@ bool
 </em>
 </td>
 <td>
-<p>Enable compression of the write-ahead log using Snappy. This flag is
-only available in versions of Prometheus &gt;= 2.11.0.</p>
+<p>Configures compression of the write-ahead log using Snappy.
+This flag is only available in versions of Prometheus &gt;= 2.11.0.
+WAL compression is enabled by default for Prometheus &gt;= 2.20.0</p>
 </td>
 </tr>
 <tr>

diff --git a/bundle.yaml b/bundle.yaml
@@ -20835,8 +20835,9 @@ spec:
                   type: object
                 type: array
               walCompression:
-                description: Enable compression of the write-ahead log using Snappy.
+                description: Configures compression of the write-ahead log using Snappy.
                   This flag is only available in versions of Prometheus >= 2.11.0.
+                  WAL compression is enabled by default for Prometheus >= 2.20.0
                 type: boolean
               web:
                 description: Defines the web command line flags when starting Prometheus.
@@ -30389,8 +30390,9 @@ spec:
                   type: object
                 type: array
               walCompression:
-                description: Enable compression of the write-ahead log using Snappy.
+                description: Configures compression of the write-ahead log using Snappy.
                   This flag is only available in versions of Prometheus >= 2.11.0.
+                  WAL compression is enabled by default for Prometheus >= 2.20.0
                 type: boolean
               web:
                 description: Defines the web command line flags when starting Prometheus.

diff --git a/example/prometheus-operator-crd-full/monitoring.coreos.com_prometheusagents.yaml b/example/prometheus-operator-crd-full/monitoring.coreos.com_prometheusagents.yaml
@@ -7705,8 +7705,9 @@ spec:
                   type: object
                 type: array
               walCompression:
-                description: Enable compression of the write-ahead log using Snappy.
+                description: Configures compression of the write-ahead log using Snappy.
                   This flag is only available in versions of Prometheus >= 2.11.0.
+                  WAL compression is enabled by default for Prometheus >= 2.20.0
                 type: boolean
               web:
                 description: Defines the web command line flags when starting Prometheus.

diff --git a/example/prometheus-operator-crd-full/monitoring.coreos.com_prometheuses.yaml b/example/prometheus-operator-crd-full/monitoring.coreos.com_prometheuses.yaml
@@ -9216,8 +9216,9 @@ spec:
                   type: object
                 type: array
               walCompression:
-                description: Enable compression of the write-ahead log using Snappy.
+                description: Configures compression of the write-ahead log using Snappy.
                   This flag is only available in versions of Prometheus >= 2.11.0.
+                  WAL compression is enabled by default for Prometheus >= 2.20.0
                 type: boolean
               web:
                 description: Defines the web command line flags when starting Prometheus.

diff --git a/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml b/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml
@@ -7705,8 +7705,9 @@ spec:
                   type: object
                 type: array
               walCompression:
-                description: Enable compression of the write-ahead log using Snappy.
+                description: Configures compression of the write-ahead log using Snappy.
                   This flag is only available in versions of Prometheus >= 2.11.0.
+                  WAL compression is enabled by default for Prometheus >= 2.20.0
                 type: boolean
               web:
                 description: Defines the web command line flags when starting Prometheus.

diff --git a/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml b/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml
@@ -9216,8 +9216,9 @@ spec:
                   type: object
                 type: array
               walCompression:
-                description: Enable compression of the write-ahead log using Snappy.
+                description: Configures compression of the write-ahead log using Snappy.
                   This flag is only available in versions of Prometheus >= 2.11.0.
+                  WAL compression is enabled by default for Prometheus >= 2.20.0
                 type: boolean
               web:
                 description: Defines the web command line flags when starting Prometheus.

diff --git a/jsonnet/prometheus-operator/prometheusagents-crd.json b/jsonnet/prometheus-operator/prometheusagents-crd.json
@@ -6828,7 +6828,7 @@
                     "type": "array"
                   },
                   "walCompression": {
-                    "description": "Enable compression of the write-ahead log using Snappy. This flag is only available in versions of Prometheus >= 2.11.0.",
+                    "description": "Configures compression of the write-ahead log using Snappy. This flag is only available in versions of Prometheus >= 2.11.0. WAL compression is enabled by default for Prometheus >= 2.20.0",
                     "type": "boolean"
                   },
                   "web": {

diff --git a/jsonnet/prometheus-operator/prometheuses-crd.json b/jsonnet/prometheus-operator/prometheuses-crd.json
@@ -8402,7 +8402,7 @@
                     "type": "array"
                   },
                   "walCompression": {
-                    "description": "Enable compression of the write-ahead log using Snappy. This flag is only available in versions of Prometheus >= 2.11.0.",
+                    "description": "Configures compression of the write-ahead log using Snappy. This flag is only available in versions of Prometheus >= 2.11.0. WAL compression is enabled by default for Prometheus >= 2.20.0",
                     "type": "boolean"
                   },
                   "web": {

diff --git a/pkg/apis/monitoring/v1/prometheus_types.go b/pkg/apis/monitoring/v1/prometheus_types.go
@@ -359,8 +359,9 @@ type CommonPrometheusFields struct {
 	// operator itself) or when providing an invalid argument the reconciliation will
 	// fail and an error will be logged.
 	AdditionalArgs []Argument `json:"additionalArgs,omitempty"`
-	// Enable compression of the write-ahead log using Snappy. This flag is
-	// only available in versions of Prometheus >= 2.11.0.
+	// Configures compression of the write-ahead log using Snappy.
+	// This flag is only available in versions of Prometheus >= 2.11.0.
+	// WAL compression is enabled by default for Prometheus >= 2.20.0
 	WALCompression *bool `json:"walCompression,omitempty"`
 	// List of references to PodMonitor, ServiceMonitor, Probe and PrometheusRule objects
 	// to be excluded from enforcing a namespace label of origin.

diff --git a/pkg/prometheus/agent/statefulset.go b/pkg/prometheus/agent/statefulset.go
@@ -201,7 +201,7 @@ func makeStatefulSetSpec(
 
 	cpf.EnableFeatures = append(cpf.EnableFeatures, "agent")
 	promArgs := prompkg.BuildCommonPrometheusArgs(cpf, cg, webRoutePrefix)
-	promArgs = appendAgentArgs(promArgs, cg)
+	promArgs = appendAgentArgs(promArgs, cg, cpf.WALCompression)
 
 	var ports []v1.ContainerPort
 	if !cpf.ListenLocal {
@@ -473,11 +473,21 @@ func makeStatefulSetService(p *monitoringv1alpha1.PrometheusAgent, config operat
 }
 
 // appendAgentArgs appends arguments that are only valid for the Prometheus agent.
-func appendAgentArgs(promArgs []monitoringv1.Argument, cg *prompkg.ConfigGenerator) []monitoringv1.Argument {
+func appendAgentArgs(
+	promArgs []monitoringv1.Argument,
+	cg *prompkg.ConfigGenerator,
+	walCompression *bool) []monitoringv1.Argument {
 
 	promArgs = append(promArgs,
 		monitoringv1.Argument{Name: "storage.agent.path", Value: prompkg.StorageDir},
 	)
 
+	if walCompression != nil {
+		arg := monitoringv1.Argument{Name: "no-storage.agent.wal-compression"}
+		if *walCompression {
+			arg.Name = "storage.agent.wal-compression"
+		}
+		promArgs = cg.AppendCommandlineArgument(promArgs, arg)
+	}
 	return promArgs
 }
diff --git a/pkg/prometheus/agent/statefulset_test.go b/pkg/prometheus/agent/statefulset_test.go
@@ -138,6 +138,54 @@ func TestListenTLS(t *testing.T) {
 	}
 }
 
+func TestWALCompression(t *testing.T) {
+	var (
+		tr = true
+		fa = false
+	)
+	tests := []struct {
+		version       string
+		enabled       *bool
+		expectedArg   string
+		shouldContain bool
+	}{
+		// Nil should not have either flag.
+		{"v2.30.0", &fa, "--storage.agent.wal-compression", false},
+		{"v2.32.0", nil, "--storage.agent.wal-compression", false},
+		{"v2.32.0", &fa, "--no-storage.agent.wal-compression", true},
+		{"v2.32.0", &tr, "--storage.agent.wal-compression", true},
+	}
+
+	for _, test := range tests {
+		sset, err := makeStatefulSetFromPrometheus(monitoringv1alpha1.PrometheusAgent{
+			Spec: monitoringv1alpha1.PrometheusAgentSpec{
+				CommonPrometheusFields: monitoringv1.CommonPrometheusFields{
+					Version:        test.version,
+					WALCompression: test.enabled,
+				},
+			},
+		})
+		require.NoError(t, err)
+
+		promArgs := sset.Spec.Template.Spec.Containers[0].Args
+		found := false
+		for _, flag := range promArgs {
+			if flag == test.expectedArg {
+				found = true
+				break
+			}
+		}
+
+		if found != test.shouldContain {
+			if test.shouldContain {
+				t.Fatalf("expected Prometheus args to contain %v, but got %v", test.expectedArg, promArgs)
+			} else {
+				t.Fatalf("expected Prometheus args to NOT contain %v, but got %v", test.expectedArg, promArgs)
+			}
+		}
+	}
+}
+
 func newLogger() log.Logger {
 	return level.NewFilter(log.NewLogfmtLogger(os.Stderr), level.AllowWarn())
 }

diff --git a/pkg/prometheus/server/statefulset.go b/pkg/prometheus/server/statefulset.go
@@ -272,7 +272,7 @@ func makeStatefulSetSpec(
 		webRoutePrefix = cpf.RoutePrefix
 	}
 	promArgs := prompkg.BuildCommonPrometheusArgs(cpf, cg, webRoutePrefix)
-	promArgs = appendServerArgs(promArgs, cg, retention, retentionSize, rules, query, allowOverlappingBlocks, enableAdminAPI)
+	promArgs = appendServerArgs(promArgs, cg, retention, retentionSize, rules, query, allowOverlappingBlocks, enableAdminAPI, cpf.WALCompression)
 
 	var ports []v1.ContainerPort
 	if !cpf.ListenLocal {
@@ -541,7 +541,9 @@ func appendServerArgs(
 	retentionSize monitoringv1.ByteSize,
 	rules monitoringv1.Rules,
 	query *monitoringv1.QuerySpec,
-	allowOverlappingBlocks, enableAdminAPI bool) []monitoringv1.Argument {
+	allowOverlappingBlocks,
+	enableAdminAPI bool,
+	walCompression *bool) []monitoringv1.Argument {
 	var (
 		retentionTimeFlagName  = "storage.tsdb.retention.time"
 		retentionTimeFlagValue = string(retention)
@@ -602,6 +604,14 @@ func appendServerArgs(
 	if allowOverlappingBlocks {
 		promArgs = cg.WithMinimumVersion("2.11.0").WithMaximumVersion("2.39.0").AppendCommandlineArgument(promArgs, monitoringv1.Argument{Name: "storage.tsdb.allow-overlapping-blocks"})
 	}
+
+	if walCompression != nil {
+		arg := monitoringv1.Argument{Name: "no-storage.tsdb.wal-compression"}
+		if *walCompression {
+			arg.Name = "storage.tsdb.wal-compression"
+		}
+		promArgs = cg.WithMinimumVersion("2.11.0").AppendCommandlineArgument(promArgs, arg)
+	}
 	return promArgs
 }
 

diff --git a/pkg/prometheus/statefulset.go b/pkg/prometheus/statefulset.go
@@ -224,14 +224,6 @@ func BuildCommonPrometheusArgs(cpf monitoringv1.CommonPrometheusFields, cg *Conf
 		promArgs = cg.WithMinimumVersion("2.6.0").AppendCommandlineArgument(promArgs, monitoringv1.Argument{Name: "log.format", Value: cpf.LogFormat})
 	}
 
-	if cpf.WALCompression != nil {
-		arg := monitoringv1.Argument{Name: "no-storage.tsdb.wal-compression"}
-		if *cpf.WALCompression {
-			arg.Name = "storage.tsdb.wal-compression"
-		}
-		promArgs = cg.WithMinimumVersion("2.11.0").AppendCommandlineArgument(promArgs, arg)
-	}
-
 	if cpf.ListenLocal {
 		promArgs = append(promArgs, monitoringv1.Argument{Name: "web.listen-address", Value: "127.0.0.1:9090"})
 	}