daemon: Add assertions for endpoint state metrics

This commit adds new assertions on endpoint state metrics to the
existing test cases. This commit helps validate that when an endpoint
fails during creation, it will be deleted, and the bookkeeping of the
endpoint state metrics is correct.

Additionally, this commits increases the code coverage around the
creation of endpoints when they fail.

Signed-off-by: Chris Tarazi <chris@isovalent.com>

daemon/cmd/daemon_test.go

@@ -25,6 +25,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/cilium/cilium/api/v1/models"
 	"github.com/cilium/cilium/pkg/common"
 	"github.com/cilium/cilium/pkg/datapath"
 	fakedatapath "github.com/cilium/cilium/pkg/datapath/fake"
@@ -35,10 +36,12 @@ import (
 	"github.com/cilium/cilium/pkg/kvstore"
 	"github.com/cilium/cilium/pkg/labelsfilter"
 	"github.com/cilium/cilium/pkg/lock"
+	"github.com/cilium/cilium/pkg/metrics"
 	monitorAPI "github.com/cilium/cilium/pkg/monitor/api"
 	"github.com/cilium/cilium/pkg/option"
 	"github.com/cilium/cilium/pkg/policy"
 
+	"github.com/prometheus/client_golang/prometheus"
 	. "gopkg.in/check.v1"
 )
 
@@ -59,6 +62,9 @@ type DaemonSuite struct {
 	OnQueueEndpointBuild  func(ctx context.Context, epID uint64) (func(), error)
 	OnGetCompilationLock  func() *lock.RWMutex
 	OnSendNotification    func(typ monitorAPI.AgentNotification, text string) error
+
+	// Metrics
+	collectors []prometheus.Collector
 }
 
 func setupTestDirectories() {
@@ -105,6 +111,19 @@ type dummyEpSyncher struct{}
 func (epSync *dummyEpSyncher) RunK8sCiliumEndpointSync(e *endpoint.Endpoint, conf endpoint.EndpointStatusConfiguration) {
 }
 
+func (ds *DaemonSuite) SetUpSuite(c *C) {
+	// Register metrics once before running the suite
+	_, ds.collectors = metrics.CreateConfiguration([]string{"cilium_endpoint_state"})
+	metrics.MustRegister(ds.collectors...)
+}
+
+func (ds *DaemonSuite) TearDownSuite(c *C) {
+	// Unregister the metrics after the suite has finished
+	for _, c := range ds.collectors {
+		metrics.Unregister(c)
+	}
+}
+
 func (ds *DaemonSuite) SetUpTest(c *C) {
 
 	setupTestDirectories()
@@ -124,6 +143,14 @@ func (ds *DaemonSuite) SetUpTest(c *C) {
 	ds.OnGetCompilationLock = nil
 	ds.OnSendNotification = nil
 	ds.d.endpointManager = endpointmanager.NewEndpointManager(&dummyEpSyncher{})
+
+	// Reset the most common endpoint states before each test.
+	for _, s := range []string{
+		string(models.EndpointStateReady),
+		string(models.EndpointStateWaitingForIdentity),
+		string(models.EndpointStateWaitingToRegenerate)} {
+		metrics.EndpointStateCount.WithLabelValues(s).Set(0.0)
+	}
 }
 
 func (ds *DaemonSuite) TearDownTest(c *C) {

daemon/cmd/endpoint_test.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"net"
+	"runtime"
 	"time"
 
 	"github.com/cilium/cilium/api/v1/models"
@@ -27,6 +28,8 @@ import (
 	endpointid "github.com/cilium/cilium/pkg/endpoint/id"
 	"github.com/cilium/cilium/pkg/identity"
 	"github.com/cilium/cilium/pkg/labels"
+	"github.com/cilium/cilium/pkg/metrics"
+
 	. "gopkg.in/check.v1"
 )
 
@@ -47,27 +50,59 @@ func getEPTemplate(c *C, d *Daemon) *models.EndpointChangeRequest {
 }
 
 func (ds *DaemonSuite) TestEndpointAddReservedLabel(c *C) {
+	assertOnMetric(c, string(models.EndpointStateWaitingForIdentity), 0)
+
 	epTemplate := getEPTemplate(c, ds.d)
 	epTemplate.Labels = []string{"reserved:world"}
 	_, code, err := ds.d.createEndpoint(context.TODO(), epTemplate)
 	c.Assert(err, Not(IsNil))
 	c.Assert(code, Equals, apiEndpoint.PutEndpointIDInvalidCode)
+
+	// Endpoint was created with invalid data; should transition from
+	// WaitForIdentity -> Invalid.
+	assertOnMetric(c, string(models.EndpointStateWaitingForIdentity), 0)
+	assertOnMetric(c, string(models.EndpointStateInvalid), 0)
+
+	// Endpoint is created with inital label as well as disallowed
+	// reserved:world label.
+	epTemplate.Labels = append(epTemplate.Labels, "reserved:init")
+	_, code, err = ds.d.createEndpoint(context.TODO(), epTemplate)
+	c.Assert(err, ErrorMatches, "not allowed to add reserved labels:.+")
+	c.Assert(code, Equals, apiEndpoint.PutEndpointIDInvalidCode)
+
+	// Endpoint was created with invalid data; should transition from
+	// WaitForIdentity -> Invalid.
+	assertOnMetric(c, string(models.EndpointStateWaitingForIdentity), 0)
+	assertOnMetric(c, string(models.EndpointStateInvalid), 0)
 }
 
 func (ds *DaemonSuite) TestEndpointAddInvalidLabel(c *C) {
+	assertOnMetric(c, string(models.EndpointStateWaitingForIdentity), 0)
+
 	epTemplate := getEPTemplate(c, ds.d)
 	epTemplate.Labels = []string{"reserved:foo"}
 	_, code, err := ds.d.createEndpoint(context.TODO(), epTemplate)
 	c.Assert(err, Not(IsNil))
 	c.Assert(code, Equals, apiEndpoint.PutEndpointIDInvalidCode)
+
+	// Endpoint was created with invalid data; should transition from
+	// WaitForIdentity -> Invalid.
+	assertOnMetric(c, string(models.EndpointStateWaitingForIdentity), 0)
+	assertOnMetric(c, string(models.EndpointStateInvalid), 0)
 }
 
 func (ds *DaemonSuite) TestEndpointAddNoLabels(c *C) {
+	assertOnMetric(c, string(models.EndpointStateWaitingForIdentity), 0)
+
 	// Create the endpoint without any labels.
 	epTemplate := getEPTemplate(c, ds.d)
 	_, _, err := ds.d.createEndpoint(context.TODO(), epTemplate)
 	c.Assert(err, IsNil)
 
+	// Endpoint enters WaitingToRegenerate as it has its labels updated during
+	// creation.
+	assertOnMetric(c, string(models.EndpointStateWaitingToRegenerate), 1)
+
 	expectedLabels := labels.Labels{
 		labels.IDNameInit: labels.NewLabel(labels.IDNameInit, "", labels.LabelSourceReserved),
 	}
@@ -79,6 +114,10 @@ func (ds *DaemonSuite) TestEndpointAddNoLabels(c *C) {
 	secID := ep.WaitForIdentity(3 * time.Second)
 	c.Assert(secID, Not(IsNil))
 	c.Assert(secID.ID, Equals, identity.ReservedIdentityInit)
+
+	// Endpoint should transition from WaitingToRegenerate -> Ready.
+	assertOnMetric(c, string(models.EndpointStateWaitingToRegenerate), 0)
+	assertOnMetric(c, string(models.EndpointStateReady), 1)
 }
 
 func (ds *DaemonSuite) TestUpdateSecLabels(c *C) {
@@ -87,3 +126,28 @@ func (ds *DaemonSuite) TestUpdateSecLabels(c *C) {
 	c.Assert(err, Not(IsNil))
 	c.Assert(code, Equals, apiEndpoint.PatchEndpointIDLabelsUpdateFailedCode)
 }
+
+func (ds *DaemonSuite) TestUpdateLabelsFailed(c *C) {
+	cancelledContext, cancelFunc := context.WithTimeout(context.Background(), 1*time.Second)
+	cancelFunc() // Cancel immediatly to trigger the codepath to test.
+
+	// Create the endpoint without any labels.
+	epTemplate := getEPTemplate(c, ds.d)
+	_, _, err := ds.d.createEndpoint(cancelledContext, epTemplate)
+	c.Assert(err, ErrorMatches, "request cancelled while resolving identity")
+
+	assertOnMetric(c, string(models.EndpointStateReady), 0)
+}
+
+func getMetricValue(state string) int64 {
+	return int64(metrics.GetGaugeValue(metrics.EndpointStateCount.WithLabelValues(state)))
+}
+
+func assertOnMetric(c *C, state string, expected int64) {
+	obtained := getMetricValue(state)
+	if obtained != expected {
+		_, _, line, _ := runtime.Caller(1)
+		c.Errorf("Metrics assertion failed on line %d for Endpoint state %s: obtained %d, expected %d",
+			line, state, obtained, expected)
+	}
+}