pkg/manifests: Lower remote write concurrency and increase batch

The previous configuration had very high concurrency at a small batch
size, this change reverses this, and has a large batch at lower
concurrency and also increased the backoff times for retries as well as
decreased the overall of retries attempted (from up to 50 retries to 5).

pkg/manifests/manifests.go

@@ -1129,7 +1129,15 @@ func (f *Factory) PrometheusK8s(host string, grpcTLS *v1.Secret, trustedCABundle
 			URL:         f.config.TelemeterClientConfig.TelemeterServerURL,
 			BearerToken: base64.StdEncoding.EncodeToString(compositeToken),
 			QueueConfig: &monv1.QueueConfig{
-				MaxShards: 5000,
+				Capacity: 10000,
+				// Default: 100
+				MaxSamplesPerSend: 1000,
+				// Default: 1000
+				MaxShards: 100,
+				// Default: 30ms
+				MinBackoff: "500ms",
+				// Default: 100ms
+				MaxBackoff: "2s",
 			},
 			WriteRelabelConfigs: []monv1.RelabelConfig{
 				*selectorRelabelConfig,