Log partial K8s metrics API errors (#196)

Partial errors are now logged to help with debugging.
Partial errors cause full error if requireKubernetesMetrics is true.
Partial errors do not stop autoscaling if requireKubernetesMetrics is false.
Update to k8shorizmetrics/v3.

CHANGELOG.md

@@ -5,6 +5,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Changed
+- Updated to `k8shorizmetrics/v3`.
+- Partial K8s metrics server errors are logged to help with debugging.
 
 ## [v2.10.0] - 2023-12-24
 ### Added

go.mod

@@ -6,52 +6,52 @@ require (
 	github.com/go-chi/chi v4.1.2+incompatible
 	github.com/golang/glog v1.2.0
 	github.com/google/go-cmp v0.6.0
-	github.com/jthomperoo/k8shorizmetrics/v2 v2.0.2
-	k8s.io/api v0.29.0
-	k8s.io/apimachinery v0.29.0
-	k8s.io/client-go v0.29.0
+	github.com/jthomperoo/k8shorizmetrics/v3 v3.0.0
+	k8s.io/api v0.29.3
+	k8s.io/apimachinery v0.29.3
+	k8s.io/client-go v0.29.3
 )
 
 require (
 	github.com/BurntSushi/toml v1.3.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
-	github.com/evanphx/json-patch v5.7.0+incompatible // indirect
+	github.com/emicklei/go-restful/v3 v3.12.0 // indirect
+	github.com/evanphx/json-patch v5.9.0+incompatible // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
-	github.com/go-openapi/jsonpointer v0.20.2 // indirect
-	github.com/go-openapi/jsonreference v0.20.4 // indirect
-	github.com/go-openapi/swag v0.22.6 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.21.0 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/golang/protobuf v1.5.4 // indirect
 	github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/uuid v1.5.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	golang.org/x/exp/typeparams v0.0.0-20231219180239-dc181d75b848 // indirect
-	golang.org/x/mod v0.14.0 // indirect
-	golang.org/x/net v0.19.0 // indirect
-	golang.org/x/oauth2 v0.15.0 // indirect
-	golang.org/x/sys v0.15.0 // indirect
-	golang.org/x/term v0.15.0 // indirect
+	golang.org/x/exp/typeparams v0.0.0-20240318143956-a85f2c67cd81 // indirect
+	golang.org/x/mod v0.16.0 // indirect
+	golang.org/x/net v0.22.0 // indirect
+	golang.org/x/oauth2 v0.18.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/term v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.5.0 // indirect
-	golang.org/x/tools v0.16.1 // indirect
+	golang.org/x/tools v0.19.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
-	google.golang.org/protobuf v1.32.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	honnef.co/go/tools v0.4.6 // indirect
-	k8s.io/klog/v2 v2.110.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20231214164306-ab13479f8bf8 // indirect
-	k8s.io/metrics v0.29.0 // indirect
-	k8s.io/utils v0.0.0-20231127182322-b307cd553661 // indirect
+	honnef.co/go/tools v0.4.7 // indirect
+	k8s.io/klog/v2 v2.120.1 // indirect
+	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
+	k8s.io/metrics v0.29.3 // indirect
+	k8s.io/utils v0.0.0-20240310230437-4693a0247e57 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect

internal/fake/gather.go

@@ -17,7 +17,7 @@ limitations under the License.
 package fake
 
 import (
-	"github.com/jthomperoo/k8shorizmetrics/v2/metrics"
+	"github.com/jthomperoo/k8shorizmetrics/v3/metrics"
 	autoscalingv2 "k8s.io/api/autoscaling/v2"
 	"k8s.io/apimachinery/pkg/labels"
 )

internal/metricget/metricget.go

@@ -21,14 +21,16 @@ package metricget
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 
 	"github.com/golang/glog"
 	"github.com/jthomperoo/custom-pod-autoscaler/v2/config"
 	"github.com/jthomperoo/custom-pod-autoscaler/v2/internal/execute"
 	"github.com/jthomperoo/custom-pod-autoscaler/v2/k8smetric"
 	"github.com/jthomperoo/custom-pod-autoscaler/v2/metric"
-	"github.com/jthomperoo/k8shorizmetrics/v2/metrics"
+	"github.com/jthomperoo/k8shorizmetrics/v3"
+	"github.com/jthomperoo/k8shorizmetrics/v3/metrics"
 	autoscalingv2 "k8s.io/api/autoscaling/v2"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
@@ -60,9 +62,25 @@ func (m *Gatherer) GetMetrics(info metric.Info, podSelector labels.Selector, cur
 		gatheredMetrics, err := m.K8sMetricGatherer.Gather(
 			convertCPAMetricSpecsToK8sMetricSpecs(m.Config.KubernetesMetricSpecs), m.Config.Namespace, podSelector)
 		if err != nil {
-			if m.Config.RequireKubernetesMetrics {
-				return nil, fmt.Errorf("failed to get required Kubernetes metrics: %w", err)
+			gatherErr := &k8shorizmetrics.GathererMultiMetricError{}
+			if errors.As(err, &gatherErr) {
+				for i, subErr := range gatherErr.Errors {
+					glog.Errorf("Error when retrieving K8s metrics (%d/%d): %+v", i+1, len(gatherErr.Errors), subErr)
+				}
+
+				if m.Config.RequireKubernetesMetrics {
+					if gatherErr.Partial {
+						return nil, fmt.Errorf("failed to get some (%d/%d) required Kubernetes metrics: %w", len(gatherErr.Errors), len(m.Config.KubernetesMetricSpecs), err)
+					}
+
+					return nil, fmt.Errorf("failed to get all required Kubernetes metrics: %w", err)
+				}
+			} else {
+				if m.Config.RequireKubernetesMetrics {
+					return nil, fmt.Errorf("failed to get required Kubernetes metrics: %w", err)
+				}
 			}
+
 			glog.Errorf("Failed to retrieve K8s metrics, not required so continuing: %+v", err)
 		} else {
 			glog.V(3).Infof("Successfully retrieved K8s metrics: %+v", gatheredMetrics)
@@ -221,5 +239,6 @@ func convertCPAMetricSpecsToK8sMetricSpecs(specs []config.K8sMetricSpec) []autos
 	for _, spec := range specs {
 		k8sSpecs = append(k8sSpecs, (autoscalingv2.MetricSpec)(spec))
 	}
+	glog.V(1).Infof("specs: %+v", k8sSpecs)
 	return k8sSpecs
 }