Limit docker and k8s error logging

Prevent noisy logs if we're failing on the same thing over and over
(such as permissions or different API versions)

checks/containers.go

@@ -14,6 +14,8 @@ const (
 	kubernetesMetaTTL = 3 * time.Minute
 )
 
+var lastKubeErr string
+
 func GetKubernetesMeta() *agentpayload.KubeMetadataPayload {
 	if os.Getenv("KUBERNETES_SERVICE_HOST") == "" {
 		// If this is not defined then we're not running in a k8s cluster.
@@ -26,13 +28,17 @@ func GetKubernetesMeta() *agentpayload.KubeMetadataPayload {
 	if !ok {
 		payload, err := kubernetes.GetPayload()
 		if err != nil {
-			// Swallowing this error for now with an error as it shouldn't block collection.
-			log.Errorf("Unable to get kubernetes metadata: %s", err)
+			if err.Error() != lastKubeErr {
+				// Swallowing this error for now with an error as it shouldn't block collection.
+				log.Errorf("Unable to get kubernetes metadata: %s", err)
+				// Only log the same error once to prevent noisy logs.
+				lastKubeErr = err.Error()
+			}
 			return nil
 		}
 		kubeMeta = payload.(*agentpayload.KubeMetadataPayload)
 		cache.SetWithTTL(cacheKey, kubeMeta, kubernetesMetaTTL)
-	} else {
+	} else if payload != nil {
 		kubeMeta = payload.(*agentpayload.KubeMetadataPayload)
 	}
 	return kubeMeta

checks/process.go

@@ -19,6 +19,8 @@ const (
 	cpuDelta = 1 * time.Second
 )
 
+var lastDockerErr string
+
 func CollectProcesses(cfg *config.AgentConfig, groupID int32) ([]model.MessageBody, error) {
 	start := time.Now()
 	var err error
@@ -32,8 +34,11 @@ func CollectProcesses(cfg *config.AgentConfig, groupID int32) ([]model.MessageBo
 		pids = append(pids, fp.Pid)
 	}
 	containerByPID, err := docker.ContainersByPID(pids)
-	if err != nil && err != docker.ErrDockerNotAvailable {
+	if err != nil && err != docker.ErrDockerNotAvailable && err.Error() != lastDockerErr {
+		// Limit docker error logging to once per Agent run to prevent noise when permissions
+		// aren't correct.
 		log.Warnf("unable to get docker stats: %s", err)
+		lastDockerErr = err.Error()
 	}
 
 	info, err := collectSystemInfo(cfg)