lib/promscrape/discovery/kubernetes: use v1 API instead of v1beta1 AP…

…I for `role: ingress` and `role: endpointslices`

This should fix service discovery for these roles in Kubernetes v1.22 and newer versions.
See https://kubernetes.io/docs/reference/using-api/deprecation-guide/#ingress-v122

The corresponding change in Prometheus - prometheus/prometheus#9205

docs/CHANGELOG.md

@@ -8,6 +8,7 @@ sort: 15
 
 * FEATURE: vmagent: add ability to read scrape configs from multiple files specified in `scrape_config_files` section. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1559).
 * FEATURE: vmagent: reduce memory usage and CPU usage when Prometheus staleness tracking is enabled for metrics exported from the deleted or disappeared scrape targets.
+* FEATURE: vmagent: discover `role: ingress` and `role: endpointslice` in [kubernetes_sd_config](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#kubernetes_sd_config) via v1 API instead of v1beta1 API if Kubernetes supports it. This fixes service discovery in Kubernetes v1.22 and newer versions. See [these docs](https://kubernetes.io/docs/reference/using-api/deprecation-guide/#ingress-v122).
 * FEATURE: take into account failed queries in `vm_request_duration_seconds` summary at `/metrics`. Previously only successful queries were taken into account. This could result in skewed summary. See [this pull request](https://github.com/VictoriaMetrics/VictoriaMetrics/pull/1537).
 * FEATURE: vmalert: add `-disableAlertgroupLabel` command-line flag for disabling the label with alert group name. This may be needed for proper deduplication in Alertmanager. See [this issue](https://github.com/VictoriaMetrics/VictoriaMetrics/issues/1532).
 * FEATURE: update Go builder from v1.16.7 to v1.17.0. This improves data ingestion and query performance by up to 5% according to benchmarks. See [the release post for Go1.17](https://go.dev/blog/go1.17).

lib/promscrape/discovery/kubernetes/api_watcher.go

@@ -13,6 +13,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/VictoriaMetrics/VictoriaMetrics/lib/logger"
@@ -159,6 +160,14 @@ func (aw *apiWatcher) getScrapeWorkObjects() []interface{} {
 // groupWatcher watches for Kubernetes objects on the given apiServer with the given namespaces,
 // selectors using the given client.
 type groupWatcher struct {
+	// Old Kubernetes doesn't support /apis/networking.k8s.io/v1/, so /apis/networking.k8s.io/v1beta1/ must be used instead.
+	// This flag is used for automatic substitution of v1 API path with v1beta1 API path during requests to apiServer.
+	useNetworkingV1Beta1 uint32
+
+	// Old Kubernetes doesn't support /apis/discovery.k8s.io/v1/, so discovery.k8s.io/v1beta1/ must be used instead.
+	// This flag is used for automatic substitution of v1 API path with v1beta1 API path during requests to apiServer.
+	useDiscoveryV1Beta1 uint32
+
 	apiServer     string
 	namespaces    []string
 	selectors     []Selector
@@ -294,14 +303,36 @@ func (gw *groupWatcher) startWatchersForRole(role string, aw *apiWatcher) {
 
 // doRequest performs http request to the given requestURL.
 func (gw *groupWatcher) doRequest(requestURL string) (*http.Response, error) {
+	if strings.Contains(requestURL, "/apis/networking.k8s.io/v1/") && atomic.LoadUint32(&gw.useNetworkingV1Beta1) == 1 {
+		// Update networking URL for old Kubernetes API, which supports only v1beta1 path.
+		requestURL = strings.Replace(requestURL, "/apis/networking.k8s.io/v1/", "/apis/networking.k8s.io/v1beta1/", 1)
+	}
+	if strings.Contains(requestURL, "/apis/discovery.k8s.io/v1/") && atomic.LoadUint32(&gw.useDiscoveryV1Beta1) == 1 {
+		// Update discovery URL for old Kuberentes API, which supports only v1beta1 path.
+		requestURL = strings.Replace(requestURL, "/apis/discovery.k8s.io/v1/", "/apis/discovery.k8s.io/v1beta1/", 1)
+	}
 	req, err := http.NewRequest("GET", requestURL, nil)
 	if err != nil {
 		logger.Fatalf("cannot create a request for %q: %s", requestURL, err)
 	}
 	if ah := gw.getAuthHeader(); ah != "" {
 		req.Header.Set("Authorization", ah)
 	}
-	return gw.client.Do(req)
+	resp, err := gw.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode == http.StatusNotFound {
+		if strings.Contains(requestURL, "/apis/networking.k8s.io/v1/") && atomic.LoadUint32(&gw.useNetworkingV1Beta1) == 0 {
+			atomic.StoreUint32(&gw.useNetworkingV1Beta1, 1)
+			return gw.doRequest(requestURL)
+		}
+		if strings.Contains(requestURL, "/apis/discovery.k8s.io/v1/") && atomic.LoadUint32(&gw.useDiscoveryV1Beta1) == 0 {
+			atomic.StoreUint32(&gw.useDiscoveryV1Beta1, 1)
+			return gw.doRequest(requestURL)
+		}
+	}
+	return resp, nil
 }
 
 func (gw *groupWatcher) registerPendingAPIWatchers() {
@@ -682,10 +713,10 @@ func getAPIPath(objectType, namespace, query string) string {
 		suffix += "?" + query
 	}
 	if objectType == "ingresses" {
-		return "/apis/networking.k8s.io/v1beta1/" + suffix
+		return "/apis/networking.k8s.io/v1/" + suffix
 	}
 	if objectType == "endpointslices" {
-		return "/apis/discovery.k8s.io/v1beta1/" + suffix
+		return "/apis/discovery.k8s.io/v1/" + suffix
 	}
 	return "/api/v1/" + suffix
 }

lib/promscrape/discovery/kubernetes/api_watcher_test.go

@@ -129,20 +129,20 @@ func TestGetAPIPathsWithNamespaces(t *testing.T) {
 	})
 
 	// role=endpointslices
-	f("endpointslices", nil, nil, []string{"/apis/discovery.k8s.io/v1beta1/endpointslices"})
+	f("endpointslices", nil, nil, []string{"/apis/discovery.k8s.io/v1/endpointslices"})
 	f("endpointslices", []string{"x", "y"}, []Selector{
 		{
 			Role:  "endpointslices",
 			Field: "field",
 			Label: "label",
 		},
 	}, []string{
-		"/apis/discovery.k8s.io/v1beta1/namespaces/x/endpointslices?labelSelector=label&fieldSelector=field",
-		"/apis/discovery.k8s.io/v1beta1/namespaces/y/endpointslices?labelSelector=label&fieldSelector=field",
+		"/apis/discovery.k8s.io/v1/namespaces/x/endpointslices?labelSelector=label&fieldSelector=field",
+		"/apis/discovery.k8s.io/v1/namespaces/y/endpointslices?labelSelector=label&fieldSelector=field",
 	})
 
 	// role=ingress
-	f("ingress", nil, nil, []string{"/apis/networking.k8s.io/v1beta1/ingresses"})
+	f("ingress", nil, nil, []string{"/apis/networking.k8s.io/v1/ingresses"})
 	f("ingress", []string{"x", "y"}, []Selector{
 		{
 			Role:  "node",
@@ -161,8 +161,8 @@ func TestGetAPIPathsWithNamespaces(t *testing.T) {
 			Label: "baaa",
 		},
 	}, []string{
-		"/apis/networking.k8s.io/v1beta1/namespaces/x/ingresses?labelSelector=cde%2Cbaaa&fieldSelector=abc",
-		"/apis/networking.k8s.io/v1beta1/namespaces/y/ingresses?labelSelector=cde%2Cbaaa&fieldSelector=abc",
+		"/apis/networking.k8s.io/v1/namespaces/x/ingresses?labelSelector=cde%2Cbaaa&fieldSelector=abc",
+		"/apis/networking.k8s.io/v1/namespaces/y/ingresses?labelSelector=cde%2Cbaaa&fieldSelector=abc",
 	})
 }
 
@@ -577,9 +577,9 @@ func TestGetScrapeWorkObjects(t *testing.T) {
 			initAPIObjectsByRole: map[string][]byte{
 				"ingress": []byte(`{
   "kind": "IngressList",
-  "apiVersion": "extensions/v1beta1",
+  "apiVersion": "extensions/v1",
   "metadata": {
-    "selfLink": "/apis/extensions/v1beta1/ingresses",
+    "selfLink": "/apis/extensions/v1/ingresses",
     "resourceVersion": "351452"
   },
   "items": [
@@ -678,9 +678,9 @@ func TestGetScrapeWorkObjects(t *testing.T) {
 			initAPIObjectsByRole: map[string][]byte{
 				"endpointslices": []byte(`{
   "kind": "EndpointSliceList",
-  "apiVersion": "discovery.k8s.io/v1beta1",
+  "apiVersion": "discovery.k8s.io/v1",
   "metadata": {
-    "selfLink": "/apis/discovery.k8s.io/v1beta1/endpointslices",
+    "selfLink": "/apis/discovery.k8s.io/v1/endpointslices",
     "resourceVersion": "1177"
   },
   "items": [

lib/promscrape/discovery/kubernetes/endpoints.go

@@ -79,7 +79,7 @@ type ObjectReference struct {
 
 // EndpointPort implements k8s endpoint port.
 //
-// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointport-v1beta1-discovery-k8s-io
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#endpointport-v1-discovery-k8s-io
 type EndpointPort struct {
 	AppProtocol string
 	Name        string

lib/promscrape/discovery/kubernetes/endpointslices.go

@@ -146,16 +146,17 @@ func getEndpointSliceLabels(eps *EndpointSlice, addr string, ea Endpoint, epp En
 	return m
 }
 
-// EndpointSliceList - implements kubernetes endpoint slice list object,
-// that groups service endpoints slices.
-// https://v1-17.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointslice-v1beta1-discovery-k8s-io
+// EndpointSliceList - implements kubernetes endpoint slice list object, that groups service endpoints slices.
+//
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#endpointslicelist-v1-discovery-k8s-io
 type EndpointSliceList struct {
 	Metadata ListMeta
 	Items    []*EndpointSlice
 }
 
 // EndpointSlice - implements kubernetes endpoint slice.
-// https://v1-17.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointslice-v1beta1-discovery-k8s-io
+//
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#endpointslice-v1-discovery-k8s-io
 type EndpointSlice struct {
 	Metadata    ObjectMeta
 	Endpoints   []Endpoint
@@ -164,7 +165,8 @@ type EndpointSlice struct {
 }
 
 // Endpoint implements kubernetes object endpoint for endpoint slice.
-// https://v1-17.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpoint-v1beta1-discovery-k8s-io
+//
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#endpoint-v1-discovery-k8s-io
 type Endpoint struct {
 	Addresses  []string
 	Conditions EndpointConditions
@@ -174,7 +176,8 @@ type Endpoint struct {
 }
 
 // EndpointConditions implements kubernetes endpoint condition.
-// https://v1-17.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#endpointconditions-v1beta1-discovery-k8s-io
+//
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#endpointconditions-v1-discovery-k8s-io
 type EndpointConditions struct {
 	Ready bool
 }

lib/promscrape/discovery/kubernetes/endpointslices_test.go

@@ -32,17 +32,17 @@ func TestParseEndpointSliceListFail(t *testing.T) {
 func TestParseEndpointSliceListSuccess(t *testing.T) {
 	data := `{
   "kind": "EndpointSliceList",
-  "apiVersion": "discovery.k8s.io/v1beta1",
+  "apiVersion": "discovery.k8s.io/v1",
   "metadata": {
-    "selfLink": "/apis/discovery.k8s.io/v1beta1/endpointslices",
+    "selfLink": "/apis/discovery.k8s.io/v1/endpointslices",
     "resourceVersion": "1177"
   },
   "items": [
     {
       "metadata": {
         "name": "kubernetes",
         "namespace": "default",
-        "selfLink": "/apis/discovery.k8s.io/v1beta1/namespaces/default/endpointslices/kubernetes",
+        "selfLink": "/apis/discovery.k8s.io/v1/namespaces/default/endpointslices/kubernetes",
         "uid": "a60d9173-5fe4-4bc3-87a6-269daee71f8a",
         "resourceVersion": "159",
         "generation": 1,
@@ -54,7 +54,7 @@ func TestParseEndpointSliceListSuccess(t *testing.T) {
           {
             "manager": "kube-apiserver",
             "operation": "Update",
-            "apiVersion": "discovery.k8s.io/v1beta1",
+            "apiVersion": "discovery.k8s.io/v1",
             "time": "2020-09-07T14:27:22Z",
             "fieldsType": "FieldsV1",
             "fieldsV1": {"f:addressType":{},"f:endpoints":{},"f:metadata":{"f:labels":{".":{},"f:kubernetes.io/service-name":{}}},"f:ports":{}}
@@ -85,7 +85,7 @@ func TestParseEndpointSliceListSuccess(t *testing.T) {
         "name": "kube-dns-22mvb",
         "generateName": "kube-dns-",
         "namespace": "kube-system",
-        "selfLink": "/apis/discovery.k8s.io/v1beta1/namespaces/kube-system/endpointslices/kube-dns-22mvb",
+        "selfLink": "/apis/discovery.k8s.io/v1/namespaces/kube-system/endpointslices/kube-dns-22mvb",
         "uid": "7c95c854-f34c-48e1-86f5-bb8269113c11",
         "resourceVersion": "604",
         "generation": 5,
@@ -111,7 +111,7 @@ func TestParseEndpointSliceListSuccess(t *testing.T) {
           {
             "manager": "kube-controller-manager",
             "operation": "Update",
-            "apiVersion": "discovery.k8s.io/v1beta1",
+            "apiVersion": "discovery.k8s.io/v1",
             "time": "2020-09-07T14:28:35Z",
             "fieldsType": "FieldsV1",
             "fieldsV1": {"f:addressType":{},"f:endpoints":{},"f:metadata":{"f:annotations":{".":{},"f:endpoints.kubernetes.io/last-change-trigger-time":{}},"f:generateName":{},"f:labels":{".":{},"f:endpointslice.kubernetes.io/managed-by":{},"f:kubernetes.io/service-name":{}},"f:ownerReferences":{".":{},"k:{\"uid\":\"509e80d8-6d05-487b-bfff-74f5768f1024\"}":{".":{},"f:apiVersion":{},"f:blockOwnerDeletion":{},"f:controller":{},"f:kind":{},"f:name":{},"f:uid":{}}}},"f:ports":{}}

lib/promscrape/discovery/kubernetes/ingress.go

@@ -33,53 +33,53 @@ func parseIngress(data []byte) (object, error) {
 
 // IngressList represents ingress list in k8s.
 //
-// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#ingresslist-v1beta1-extensions
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#ingresslist-v1-networking-k8s-io
 type IngressList struct {
 	Metadata ListMeta
 	Items    []*Ingress
 }
 
 // Ingress represents ingress in k8s.
 //
-// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#ingress-v1beta1-extensions
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#ingress-v1-networking-k8s-io
 type Ingress struct {
 	Metadata ObjectMeta
 	Spec     IngressSpec
 }
 
 // IngressSpec represents ingress spec in k8s.
 //
-// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#ingressspec-v1beta1-extensions
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#ingressspec-v1-networking-k8s-io
 type IngressSpec struct {
 	TLS   []IngressTLS `json:"tls"`
 	Rules []IngressRule
 }
 
 // IngressTLS represents ingress TLS spec in k8s.
 //
-// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#ingresstls-v1beta1-extensions
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#ingresstls-v1-networking-k8s-io
 type IngressTLS struct {
 	Hosts []string
 }
 
 // IngressRule represents ingress rule in k8s.
 //
-// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#ingressrule-v1beta1-extensions
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#ingressrule-v1-networking-k8s-io
 type IngressRule struct {
 	Host string
 	HTTP HTTPIngressRuleValue `json:"http"`
 }
 
 // HTTPIngressRuleValue represents HTTP ingress rule value in k8s.
 //
-// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#httpingressrulevalue-v1beta1-extensions
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#httpingressrulevalue-v1-networking-k8s-io
 type HTTPIngressRuleValue struct {
 	Paths []HTTPIngressPath
 }
 
 // HTTPIngressPath represents HTTP ingress path in k8s.
 //
-// See https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.17/#httpingresspath-v1beta1-extensions
+// See https://v1-21.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.21/#httpingresspath-v1-networking-k8s-io
 type HTTPIngressPath struct {
 	Path string
 }

lib/promscrape/discovery/kubernetes/ingress_test.go

@@ -30,23 +30,23 @@ func TestParseIngressListSuccess(t *testing.T) {
 	data := `
 {
   "kind": "IngressList",
-  "apiVersion": "extensions/v1beta1",
+  "apiVersion": "extensions/v1",
   "metadata": {
-    "selfLink": "/apis/extensions/v1beta1/ingresses",
+    "selfLink": "/apis/extensions/v1/ingresses",
     "resourceVersion": "351452"
   },
   "items": [
     {
       "metadata": {
         "name": "test-ingress",
         "namespace": "default",
-        "selfLink": "/apis/extensions/v1beta1/namespaces/default/ingresses/test-ingress",
+        "selfLink": "/apis/extensions/v1/namespaces/default/ingresses/test-ingress",
         "uid": "6d3f38f9-de89-4bc9-b273-c8faf74e8a27",
         "resourceVersion": "351445",
         "generation": 1,
         "creationTimestamp": "2020-04-13T16:43:52Z",
         "annotations": {
-          "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"networking.k8s.io/v1beta1\",\"kind\":\"Ingress\",\"metadata\":{\"annotations\":{},\"name\":\"test-ingress\",\"namespace\":\"default\"},\"spec\":{\"backend\":{\"serviceName\":\"testsvc\",\"servicePort\":80}}}\n"
+          "kubectl.kubernetes.io/last-applied-configuration": "{\"apiVersion\":\"networking.k8s.io/v1\",\"kind\":\"Ingress\",\"metadata\":{\"annotations\":{},\"name\":\"test-ingress\",\"namespace\":\"default\"},\"spec\":{\"backend\":{\"serviceName\":\"testsvc\",\"servicePort\":80}}}\n"
         }
       },
       "spec": {
@@ -85,7 +85,7 @@ func TestParseIngressListSuccess(t *testing.T) {
 	expectedLabelss := [][]prompbmarshal.Label{
 		discoveryutils.GetSortedLabels(map[string]string{
 			"__address__": "foobar",
-			"__meta_kubernetes_ingress_annotation_kubectl_kubernetes_io_last_applied_configuration":        `{"apiVersion":"networking.k8s.io/v1beta1","kind":"Ingress","metadata":{"annotations":{},"name":"test-ingress","namespace":"default"},"spec":{"backend":{"serviceName":"testsvc","servicePort":80}}}` + "\n",
+			"__meta_kubernetes_ingress_annotation_kubectl_kubernetes_io_last_applied_configuration":        `{"apiVersion":"networking.k8s.io/v1","kind":"Ingress","metadata":{"annotations":{},"name":"test-ingress","namespace":"default"},"spec":{"backend":{"serviceName":"testsvc","servicePort":80}}}` + "\n",
 			"__meta_kubernetes_ingress_annotationpresent_kubectl_kubernetes_io_last_applied_configuration": "true",
 			"__meta_kubernetes_ingress_host":   "foobar",
 			"__meta_kubernetes_ingress_name":   "test-ingress",