Bug 1860889: decrease CMO log verbosity from 3 to 2

With `-v=3`, the logs are spammed by client-go telling that client
requests have been throttled. Looking at the Kubernetes code, we won't
lose anything useful by lowering the verbosity level.

The change also increases the QPS and burst settings (respectively 5->20
and 10->40) because the default values aren't suited for CMO that runs
lots of tasks in parallel. With QPS=100 and Burst=200, the throttling
logs disappeared but we felt that these values were too high for the API
server.

Signed-off-by: Simon Pasquier <spasquie@redhat.com>

OWNERS

@@ -1,15 +1,13 @@
 component: "Monitoring"
 
 reviewers:
-- brancz
 - s-urbaniak
 - paulfantom
 - LiliC
 - pgier
 - simonpasquier
 
 approvers:
-- brancz
 - bparees
 - s-urbaniak
 - paulfantom
@@ -18,5 +16,6 @@ approvers:
 - simonpasquier
 
 emeritus_approvers:
+- brancz
 - squat
 - metalmatze

cmd/operator/main.go

@@ -155,6 +155,10 @@ func Main() int {
 		return 1
 	}
 
+	// CMO runs many tasks in parallel and the default values for rate limiting are too low.
+	config.QPS = 20
+	config.Burst = 40
+
 	userWorkloadConfigMapName := "user-workload-monitoring-config"
 	o, err := cmo.New(config, *releaseVersion, *namespace, *namespaceUserWorkload, *namespaceSelector, *configMapName, userWorkloadConfigMapName, *remoteWrite, images.asMap(), telemetryConfig.Matches)
 	if err != nil {

manifests/0000_50_cluster_monitoring_operator_05-deployment.yaml

@@ -71,7 +71,7 @@ spec:
         - "-configmap=cluster-monitoring-config"
         - "-release-version=$(RELEASE_VERSION)"
         - "-logtostderr=true"
-        - "-v=3"
+        - "-v=2"
         - "-images=prometheus-operator=quay.io/openshift/origin-prometheus-operator:latest"
         - "-images=prometheus-config-reloader=quay.io/openshift/origin-prometheus-config-reloader:latest"
         - "-images=configmap-reloader=quay.io/openshift/origin-configmap-reloader:latest"

pkg/client/client.go

@@ -716,7 +716,7 @@ func (c *Client) WaitForDeploymentRollout(dep *appsv1.Deployment) error {
 		}
 		if d.Status.UnavailableReplicas != 0 {
 			lastErr = errors.Errorf("got %d unavailable replicas",
-				d.Status.UpdatedReplicas)
+				d.Status.UnavailableReplicas)
 			return false, nil
 		}
 		return true, nil

pkg/tasks/tasks.go

@@ -41,9 +41,9 @@ func (tl *TaskRunner) RunAll() (string, error) {
 		i := i
 
 		g.Go(func() error {
-			klog.V(3).Infof("running task %d of %d: %v", i+1, len(tl.tasks), ts.Name)
+			klog.V(2).Infof("running task %d of %d: %v", i+1, len(tl.tasks), ts.Name)
 			err := tl.ExecuteTask(ts)
-			klog.V(3).Infof("ran task %d of %d: %v", i+1, len(tl.tasks), ts.Name)
+			klog.V(2).Infof("ran task %d of %d: %v", i+1, len(tl.tasks), ts.Name)
 			if err != nil {
 				return taskErr{error: errors.Wrapf(err, "running task %v failed", ts.Name), name: ts.Name}
 			}