Metrics (4/4)

related to #124 This PR contains several specific metrics ```gaolang Health records *prometheus.GaugeVec StatusPerIngressHosts *prometheus.GaugeVec StatusFailover *prometheus.GaugeVec StatusRoundRobin *prometheus.GaugeVec StatusGeoIP *prometheus.GaugeVec ZoneUpdateTotal *prometheus.CounterVec ErrorTotal *prometheus.CounterVec ReconciliationTotal prometheus.Counter ``` Signed-off-by: kuritka <kuritka@gmail.com>
k8gb-io · Aug 19, 2021 · bc86c95 · bc86c95
1 parent 3153172
commit bc86c95
Show file tree

Hide file tree

Showing 9 changed files with 366 additions and 25 deletions.
diff --git a/Makefile b/Makefile
@@ -289,6 +289,7 @@ manager: lint
 mocks:
 	go install github.com/golang/mock/mockgen@v1.5.0
 	mockgen -source=controllers/providers/assistant/assistant.go -destination=controllers/providers/assistant/assistant_mock.go -package=assistant
+	mockgen -source=controllers/providers/dns/dns.go -destination=controllers/providers/dns/dns_mock.go -package=dns
 	$(call golic)
 
 # remove clusters and redeploy

diff --git a/controllers/dnsupdate.go b/controllers/dnsupdate.go
@@ -34,6 +34,7 @@ func sortTargets(targets []string) []string {
 	})
 	return targets
 }
+
 func (r *GslbReconciler) gslbDNSEndpoint(gslb *k8gbv1beta1.Gslb) (*externaldns.DNSEndpoint, error) {
 	var gslbHosts []*externaldns.Endpoint
 	var ttl = externaldns.TTL(gslb.Spec.Strategy.DNSTtlSeconds)
@@ -102,6 +103,7 @@ func (r *GslbReconciler) gslbDNSEndpoint(gslb *k8gbv1beta1.Gslb) (*externaldns.D
 			log.Info().Msgf("No external targets have been found for host %s", host)
 		}
 
+		r.updateRuntimeStatus(gslb, isPrimary, health, finalTargets)
 		log.Info().Msgf("Final target list for %s Gslb: %v", gslb.Name, finalTargets)
 
 		if len(finalTargets) > 0 {
@@ -137,3 +139,14 @@ func (r *GslbReconciler) gslbDNSEndpoint(gslb *k8gbv1beta1.Gslb) (*externaldns.D
 	}
 	return dnsEndpoint, err
 }
+
+func (r *GslbReconciler) updateRuntimeStatus(gslb *k8gbv1beta1.Gslb, isPrimary bool, isHealthy k8gbv1beta1.HealthStatus, finalTargets []string) {
+	switch gslb.Spec.Strategy.Type {
+	case roundRobinStrategy:
+		m.UpdateRoundrobinStatus(gslb, isPrimary, isHealthy, finalTargets)
+	case geoStrategy:
+		m.UpdateGeoIPStatus(gslb, isPrimary, isHealthy, finalTargets)
+	case failoverStrategy:
+		m.UpdateFailoverStatus(gslb, isPrimary, isHealthy, finalTargets)
+	}
+}
diff --git a/controllers/gslb_controller.go b/controllers/gslb_controller.go
@@ -84,11 +84,13 @@ func (r *GslbReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.
 			// Return and don't requeue
 			return result.Stop()
 		}
+		m.ErrorIncrement(gslb)
 		return result.RequeueError(fmt.Errorf("error reading the object (%s)", err))
 	}
 
 	err = r.DepResolver.ResolveGslbSpec(ctx, gslb, r.Client)
 	if err != nil {
+		m.ErrorIncrement(gslb)
 		return result.RequeueError(fmt.Errorf("resolving spec (%s)", err))
 	}
 	log.Debug().
@@ -127,42 +129,49 @@ func (r *GslbReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.
 	// Add finalizer for this CR
 	if !contains(gslb.GetFinalizers(), gslbFinalizer) {
 		if err := r.addFinalizer(gslb); err != nil {
+			m.ErrorIncrement(gslb)
 			return result.RequeueError(err)
 		}
 	}
 
 	// == Ingress ==========
 	ingress, err := r.gslbIngress(gslb)
 	if err != nil {
+		m.ErrorIncrement(gslb)
 		return result.RequeueError(err)
 	}
 
 	err = r.saveIngress(gslb, ingress)
 	if err != nil {
+		m.ErrorIncrement(gslb)
 		return result.RequeueError(err)
 	}
 
 	// == external-dns dnsendpoints CRs ==
 	dnsEndpoint, err := r.gslbDNSEndpoint(gslb)
 	if err != nil {
+		m.ErrorIncrement(gslb)
 		return result.RequeueError(err)
 	}
 
 	err = r.DNSProvider.SaveDNSEndpoint(gslb, dnsEndpoint)
 	if err != nil {
+		m.ErrorIncrement(gslb)
 		return result.RequeueError(err)
 	}
 
 	// == handle delegated zone in Edge DNS
 	err = r.DNSProvider.CreateZoneDelegationForExternalDNS(gslb)
 	if err != nil {
 		log.Err(err).Msg("Unable to create zone delegation")
+		m.ErrorIncrement(gslb)
 		return result.Requeue()
 	}
 
 	// == Status =
 	err = r.updateGslbStatus(gslb)
 	if err != nil {
+		m.ErrorIncrement(gslb)
 		return result.RequeueError(err)
 	}
 

diff --git a/controllers/gslb_controller_test.go b/controllers/gslb_controller_test.go
@@ -27,6 +27,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/golang/mock/gomock"
+
 	str "github.com/AbsaOSS/gopkg/strings"
 	k8gbv1beta1 "github.com/AbsaOSS/k8gb/api/v1beta1"
 	"github.com/AbsaOSS/k8gb/controllers/depresolver"
@@ -150,7 +152,7 @@ func TestIngressHostsPerStatusMetric(t *testing.T) {
 	settings := provideSettings(t, predefinedConfig)
 	expectedHostsMetricCount := 3
 	// act
-	ingressHostsPerStatusMetric := metrics.Metrics().Get("k8gb_gslb_ingress_hosts_per_status").AsGaugeVec()
+	ingressHostsPerStatusMetric := metrics.Metrics().Get("k8gb_gslb_status_per_ingress_hosts").AsGaugeVec()
 	err := settings.client.Get(context.TODO(), settings.request.NamespacedName, settings.gslb)
 	actualHostsMetricCount := testutil.CollectAndCount(ingressHostsPerStatusMetric)
 	// assert
@@ -172,7 +174,7 @@ func TestIngressHostsPerStatusMetricReflectionForHealthyStatus(t *testing.T) {
 			reconcileAndUpdateGslb(t, settings)
 			// act
 			err := settings.client.Get(context.TODO(), settings.request.NamespacedName, settings.gslb)
-			ingressHostsPerStatusMetric := metrics.Metrics().Get("k8gb_gslb_ingress_hosts_per_status").AsGaugeVec()
+			ingressHostsPerStatusMetric := metrics.Metrics().Get("k8gb_gslb_status_per_ingress_hosts").AsGaugeVec()
 			healthyHosts := ingressHostsPerStatusMetric.With(prometheus.Labels{"namespace": settings.gslb.Namespace,
 				"name": settings.gslb.Name, "status": k8gbv1beta1.Healthy.String()})
 			actualHostsMetric := testutil.ToFloat64(healthyHosts)
@@ -190,7 +192,7 @@ func TestIngressHostsPerStatusMetricReflectionForUnhealthyStatus(t *testing.T) {
 	err := settings.client.Get(context.TODO(), settings.request.NamespacedName, settings.gslb)
 	expectedHostsMetricCount := 0.
 	// act
-	ingressHostsPerStatusMetric := metrics.Metrics().Get("k8gb_gslb_ingress_hosts_per_status").AsGaugeVec()
+	ingressHostsPerStatusMetric := metrics.Metrics().Get("k8gb_gslb_status_per_ingress_hosts").AsGaugeVec()
 	unhealthyHosts := ingressHostsPerStatusMetric.With(prometheus.Labels{"namespace": settings.gslb.Namespace,
 		"name": settings.gslb.Name, "status": k8gbv1beta1.Unhealthy.String()})
 	actualHostsMetricCount := testutil.ToFloat64(unhealthyHosts)
@@ -228,7 +230,7 @@ func TestIngressHostsPerStatusMetricReflectionForNotFoundStatus(t *testing.T) {
 	// act
 	err := settings.client.Get(context.TODO(), settings.request.NamespacedName, settings.gslb)
 	require.NoError(t, err, "Failed to get expected gslb")
-	ingressHostsPerStatusMetric := metrics.Metrics().Get("k8gb_gslb_ingress_hosts_per_status").AsGaugeVec()
+	ingressHostsPerStatusMetric := metrics.Metrics().Get("k8gb_gslb_status_per_ingress_hosts").AsGaugeVec()
 	unknownHosts, err := ingressHostsPerStatusMetric.GetMetricWith(
 		prometheus.Labels{"namespace": settings.gslb.Namespace, "name": settings.gslb.Name, "status": k8gbv1beta1.NotFound.String()})
 	require.NoError(t, err, "Failed to get ingress metrics")
@@ -270,7 +272,7 @@ func TestHealthyRecordMetric(t *testing.T) {
 func TestMetricLinterCheck(t *testing.T) {
 	// arrange
 	healthyRecordsMetric := metrics.Metrics().Get("k8gb_gslb_healthy_records").AsGaugeVec()
-	ingressHostsPerStatusMetric := metrics.Metrics().Get("k8gb_gslb_ingress_hosts_per_status").AsGaugeVec()
+	ingressHostsPerStatusMetric := metrics.Metrics().Get("k8gb_gslb_status_per_ingress_hosts").AsGaugeVec()
 	reconciliationTotal := metrics.Metrics().Get("k8gb_gslb_reconciliation_total").AsCounter()
 	for name, scenario := range map[string]prometheus.Collector{
 		"healthy_records":          healthyRecordsMetric,
@@ -298,6 +300,33 @@ func TestGslbReconciliationTotalIncrement(t *testing.T) {
 	assert.Equal(t, cnt+1, cnt2)
 }
 
+func TestGslbErrorsIncrement(t *testing.T) {
+	// arrange
+	const key = "k8gb_gslb_error_total"
+	ctrl := gomock.NewController(t)
+	defer ctrl.Finish()
+	settings := provideSettings(t, predefinedConfig)
+	var label = prometheus.Labels{"namespace": settings.gslb.Namespace, "name": settings.gslb.Name}
+	m := dns.NewMockProvider(ctrl)
+	cnt := testutil.ToFloat64(metrics.Metrics().Get(key).AsCounterVec().With(label))
+	m.EXPECT().GslbIngressExposedIPs(gomock.Any()).Return([]string{}, nil).Times(1)
+	m.EXPECT().SaveDNSEndpoint(gomock.Any(), gomock.Any()).Return(fmt.Errorf("save DNS error")).Times(1)
+	m.EXPECT().GetExternalTargets(gomock.Any()).Return([]string{}).AnyTimes()
+	m.EXPECT().CreateZoneDelegationForExternalDNS(gomock.Any()).Return(nil).AnyTimes()
+	settings.reconciler.DNSProvider = m
+	// act
+	_, err := settings.reconciler.Reconcile(context.TODO(), settings.request)
+	require.Error(t, err)
+	// let's break it on different place
+	m.EXPECT().SaveDNSEndpoint(gomock.Any(), gomock.Any()).Return(nil).AnyTimes()
+	m.EXPECT().GslbIngressExposedIPs(gomock.Any()).Return([]string{}, fmt.Errorf("exposed IP's error")).AnyTimes()
+	_, err = settings.reconciler.Reconcile(context.TODO(), settings.request)
+	cnt2 := testutil.ToFloat64(metrics.Metrics().Get(key).AsCounterVec().With(label))
+	// assert
+	assert.Error(t, err)
+	assert.Equal(t, cnt+2, cnt2)
+}
+
 func TestGslbCreatesDNSEndpointCRForHealthyIngressHosts(t *testing.T) {
 	// arrange
 	serviceName := "frontend-podinfo"

diff --git a/controllers/providers/dns/dns_mock.go b/controllers/providers/dns/dns_mock.go