Merge pull request #148 from 3scale-ops/metrics

Metrics

examples/local/envoyconfig.yaml

@@ -44,7 +44,7 @@ spec:
       - name: http
         value: |
           name: http
-          address: { socket_address: { address: 0.0.0.0, port_value: 8080 }}
+          address: { socket_address: { address: 0.0.0.0, port_value: 8888 }}
           filter_chains:
             - filters:
                 - name: envoy.filters.network.http_connection_manager

pkg/discoveryservice/xds_server.go

@@ -29,6 +29,7 @@ import (
 	server_v3 "github.com/envoyproxy/go-control-plane/pkg/server/v3"
 	"github.com/go-logr/logr"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
 
 	envoy_service_discovery_v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3"
 
@@ -68,6 +69,10 @@ func NewXdsServer(ctx context.Context, xDSPort uint, tlsConfig *tls.Config, logg
 
 	discoveryStatsV3 := stats.New()
 
+	// register the custom metrics collector with the global
+	// prometheus registry
+	metrics.Registry.MustRegister(discoveryStatsV3)
+
 	snapshotCacheV3 := cache_v3.NewSnapshotCache(
 		true,
 		cache_v3.IDHash{},

pkg/discoveryservice/xdss/stats/collector.go

@@ -0,0 +1,87 @@
+package stats
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+// ensure Stats implements the prometheus Collector interface
+var _ prometheus.Collector = Stats{}
+
+// Descriptors used to create the metrics
+var (
+	requestCountDesc = prometheus.NewDesc(
+		"marin3r_xdss_discovery_requests_total",
+		"Number of discovery requests",
+		[]string{"node_id", "resource_type", "pod_name"}, nil,
+	)
+	ackCountDesc = prometheus.NewDesc(
+		"marin3r_xdss_discovery_ack_total",
+		"Number of discovery ACK responses",
+		[]string{"node_id", "resource_type", "pod_name"}, nil,
+	)
+	nackCountDesc = prometheus.NewDesc(
+		"marin3r_xdss_discovery_nack_total",
+		"Number of discovery NACK responses",
+		[]string{"node_id", "resource_type", "pod_name"}, nil,
+	)
+	infoDesc = prometheus.NewDesc(
+		"marin3r_xdss_discovery_info",
+		"Information about the version a certain resource type is at",
+		[]string{"node_id", "resource_type", "pod_name", "version"}, nil,
+	)
+)
+
+// Describe is implemented with DescribeByCollect. That's possible because the
+// Collect method will always return the same 4 metrics with the same 4
+// descriptors.
+func (xmc Stats) Describe(ch chan<- *prometheus.Desc) {
+	prometheus.DescribeByCollect(xmc, ch)
+}
+
+// Collect dumps all the keys in the stats cache. Then it
+// creates constant metrics for each modeID/resourceType/pod on the fly based on the
+// dumped returned data.
+func (s Stats) Collect(ch chan<- prometheus.Metric) {
+
+	items := s.DumpAll()
+	for k, v := range items {
+		key := NewKeyFromString(k)
+
+		switch metric := key.StatName + "/" + key.Version; metric {
+
+		case "request_counter/*":
+			ch <- prometheus.MustNewConstMetric(
+				requestCountDesc,
+				prometheus.CounterValue,
+				float64(v.Object.(int64)),
+				key.NodeID, key.ResourceType, key.PodID,
+			)
+
+		case "ack_counter/*":
+			ch <- prometheus.MustNewConstMetric(
+				ackCountDesc,
+				prometheus.CounterValue,
+				float64(v.Object.(int64)),
+				key.NodeID, key.ResourceType, key.PodID,
+			)
+
+		case "ack_counter/" + key.Version:
+			ch <- prometheus.MustNewConstMetric(
+				infoDesc,
+				prometheus.UntypedValue,
+				float64(0),
+				key.NodeID, key.ResourceType, key.PodID, key.Version,
+			)
+
+		case "nack_counter/*":
+			ch <- prometheus.MustNewConstMetric(
+				nackCountDesc,
+				prometheus.CounterValue,
+				float64(v.Object.(int64)),
+				key.NodeID, key.ResourceType, key.PodID,
+			)
+
+		}
+
+	}
+}

pkg/discoveryservice/xdss/stats/kv.go

@@ -18,16 +18,16 @@ type Key struct {
 	ResourceType string
 	Version      string
 	PodID        string
-	Key          string
+	StatName     string
 }
 
-func NewKey(nodeID, rType, version, podID, key string) *Key {
+func NewKey(nodeID, rType, version, podID, statName string) *Key {
 	return &Key{
 		NodeID:       nodeID,
 		ResourceType: rType,
 		Version:      version,
 		PodID:        podID,
-		Key:          key,
+		StatName:     statName,
 	}
 }
 
@@ -38,16 +38,16 @@ func NewKeyFromString(key string) *Key {
 		ResourceType: values[1],
 		Version:      values[2],
 		PodID:        values[3],
-		Key:          values[4],
+		StatName:     strings.Join(values[4:], ":"),
 	}
 }
 
 func (k *Key) String() string {
-	return strings.Join([]string{k.NodeID, k.ResourceType, k.Version, k.PodID, k.Key}, ":")
+	return strings.Join([]string{k.NodeID, k.ResourceType, k.Version, k.PodID, k.StatName}, ":")
 }
 
-func (s *Stats) GetString(nodeID, rtype, version, podID, key string) (string, error) {
-	k := NewKey(nodeID, rtype, version, podID, key).String()
+func (s *Stats) GetString(nodeID, rtype, version, podID, statName string) (string, error) {
+	k := NewKey(nodeID, rtype, version, podID, statName).String()
 	if v, ok := s.store.Get(k); ok {
 		if value, ok := v.(string); !ok {
 			return "", fmt.Errorf("value of key '%s' is not a string", k)
@@ -60,16 +60,16 @@ func (s *Stats) GetString(nodeID, rtype, version, podID, key string) (string, er
 	}
 }
 
-func (s *Stats) SetString(nodeID, rType, version, podID, key, value string) {
-	s.store.SetDefault(NewKey(nodeID, rType, version, podID, key).String(), value)
+func (s *Stats) SetString(nodeID, rType, version, podID, statName, value string) {
+	s.store.SetDefault(NewKey(nodeID, rType, version, podID, statName).String(), value)
 }
 
-func (s *Stats) SetStringWithExpiration(nodeID, rType, version, podID, key, value string, expiration time.Duration) {
-	s.store.Set(NewKey(nodeID, rType, version, podID, key).String(), value, expiration)
+func (s *Stats) SetStringWithExpiration(nodeID, rType, version, podID, statName, value string, expiration time.Duration) {
+	s.store.Set(NewKey(nodeID, rType, version, podID, statName).String(), value, expiration)
 }
 
-func (s *Stats) GetCounter(nodeID, rtype, version, podID, key string) (int64, error) {
-	k := NewKey(nodeID, rtype, version, podID, key).String()
+func (s *Stats) GetCounter(nodeID, rtype, version, podID, statName string) (int64, error) {
+	k := NewKey(nodeID, rtype, version, podID, statName).String()
 	if v, ok := s.store.Get(k); ok {
 		if value, ok := v.(int64); !ok {
 			return 0, fmt.Errorf("value of key '%s' is not an int", k)
@@ -82,25 +82,26 @@ func (s *Stats) GetCounter(nodeID, rtype, version, podID, key string) (int64, er
 	}
 }
 
-func (s *Stats) IncrementCounter(nodeID, rType, version, podID, key string, increment int64) {
-	if _, err := s.store.IncrementInt64(NewKey(nodeID, rType, version, podID, key).String(), increment); err != nil {
-		// The key does not exist yet in the kv store
-		s.store.SetDefault(NewKey(nodeID, rType, version, podID, key).String(), increment)
-	}
+// IncrementCounter increments the counter if it already exists or creates it if it doesn't. IncrementCount
+// removes any expiration that the cache item might had previously.
+func (s *Stats) IncrementCounter(nodeID, rType, version, podID, statName string, increment int64) {
+	// GetCounter returns 0 when an error happens so we don't need to check for errors
+	counter, _ := s.GetCounter(nodeID, rType, version, podID, statName)
+	s.store.SetDefault(NewKey(nodeID, rType, version, podID, statName).String(), counter+increment)
 }
 
-func (s *Stats) DecrementCounter(nodeID, rType, version, podID, key string, decrement int64) {
-	if _, err := s.store.DecrementInt64(NewKey(nodeID, rType, version, podID, key).String(), decrement); err != nil {
-		// The key does not exist yet in the kv store
-		s.store.SetDefault(NewKey(nodeID, rType, version, podID, key).String(), 0)
-	}
+// ExpireCounter adds expiration to a counter.
+func (s *Stats) ExpireCounter(nodeID, rType, version, podID, statName string, expiration time.Duration) {
+	counter, _ := s.GetCounter(nodeID, rType, version, podID, statName)
+	s.store.Set(NewKey(nodeID, rType, version, podID, statName).String(), counter, expiration)
 }
 
 func (s *Stats) FilterKeys(filters ...string) map[string]kv.Item {
 	all := s.store.Items()
 	selected := map[string]kv.Item{}
+	var isSelected bool
 	for key, value := range all {
-		isSelected := true
+		isSelected = true
 		for _, filter := range filters {
 			if !strings.Contains(key, filter) {
 				isSelected = false
@@ -119,3 +120,7 @@ func (s *Stats) DeleteKeysByFilter(filters ...string) {
 		s.store.Delete(k)
 	}
 }
+
+func (s *Stats) DumpAll() map[string]kv.Item {
+	return s.store.Items()
+}