Skip to content

Commit

Permalink
Add routegroup and fabricgateway metrics
Browse files Browse the repository at this point in the history 
Signed-off-by: Alexander Yastrebov <alexander.yastrebov@zalando.de>
  • Loading branch information
@AlexanderYastrebov
AlexanderYastrebov committed Apr 8, 2022
1 parent d8addc9 commit d50142d
Show file tree
Hide file tree
Showing 5 changed files with 85 additions and 49 deletions.
22 changes: 21 additions & 1 deletion controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,8 @@ var (
var metrics = struct {
lastSyncTimestamp prometheus.Gauge
ingressesTotal prometheus.Gauge
routegroupsTotal prometheus.Gauge
fabricgatewaysTotal prometheus.Gauge
stacksTotal prometheus.Gauge
ownedAutoscalingGroupsTotal prometheus.Gauge
targetedAutoscalingGroupsTotal prometheus.Gauge
Expand All @@ -116,7 +118,23 @@ var metrics = struct {
Namespace: "kube_ingress_aws",
Subsystem: "controller",
Name: "ingresses_total",
Help: "Number of managed Kubernetes Ingresses and Route Groups",
Help: "Number of managed Kubernetes Ingresses",
},
),
routegroupsTotal: prometheus.NewGauge(
prometheus.GaugeOpts{
Namespace: "kube_ingress_aws",
Subsystem: "controller",
Name: "routegroups_total",
Help: "Number of managed Route Groups",
},
),
fabricgatewaysTotal: prometheus.NewGauge(
prometheus.GaugeOpts{
Namespace: "kube_ingress_aws",
Subsystem: "controller",
Name: "fabricgateways_total",
Help: "Number of managed Fabric Gateways",
},
),
stacksTotal: prometheus.NewGauge(
Expand Down Expand Up @@ -180,6 +198,8 @@ var metrics = struct {
func init() {
prometheus.MustRegister(metrics.lastSyncTimestamp)
prometheus.MustRegister(metrics.ingressesTotal)
prometheus.MustRegister(metrics.routegroupsTotal)
prometheus.MustRegister(metrics.fabricgatewaysTotal)
prometheus.MustRegister(metrics.stacksTotal)
prometheus.MustRegister(metrics.ownedAutoscalingGroupsTotal)
prometheus.MustRegister(metrics.targetedAutoscalingGroupsTotal)
Expand Down
55 changes: 21 additions & 34 deletions kubernetes/adapter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,27 +26,14 @@ type Adapter struct {
fabricSupport bool
}

type ingressType int
type IngressType string

const (
ingressTypeIngress ingressType = iota + 1
ingressTypeRouteGroup
ingressTypeFabric
TypeIngress IngressType = "ingress"
TypeRouteGroup IngressType = "routegroup"
TypeFabricGateway IngressType = "fabricgateway"
)

func (t ingressType) String() string {
switch t {
case ingressTypeIngress:
return "ingress"
case ingressTypeRouteGroup:
return "routegroup"
case ingressTypeFabric:
return "fabric"
default:
return "unknown"
}
}

const (
DefaultClusterLocalDomain = ".cluster.local"
loadBalancerTypeNLB = "nlb"
Expand Down Expand Up @@ -85,12 +72,13 @@ var (
// Ingress is the ingress-controller's business object. It is used to
// store Kubernetes ingress and routegroup resources.
type Ingress struct {
ResourceType IngressType
Namespace string
Name string
Shared bool
HTTP2 bool
ClusterLocal bool
CertificateARN string
Namespace string
Name string
Hostname string
Scheme string
SecurityGroup string
Expand All @@ -99,12 +87,11 @@ type Ingress struct {
LoadBalancerType string
WAFWebACLID string
Hostnames []string
resourceType ingressType
}

// String returns a string representation of the Ingress instance containing the type, namespace and the resource name.
func (i *Ingress) String() string {
return fmt.Sprintf("%s %s/%s", i.resourceType, i.Namespace, i.Name)
return fmt.Sprintf("%s %s/%s", i.ResourceType, i.Namespace, i.Name)
}

// ConfigMap is the ingress-controller's representation of a Kubernetes
Expand Down Expand Up @@ -159,7 +146,7 @@ func (a *Adapter) newIngressFromKube(kubeIngress *ingress) (*Ingress, error) {
}
}

return a.newIngress(ingressTypeIngress, kubeIngress.Metadata, host, hostnames)
return a.newIngress(TypeIngress, kubeIngress.Metadata, host, hostnames)
}

func (a *Adapter) newIngressFromRouteGroup(rg *routegroup) (*Ingress, error) {
Expand All @@ -178,7 +165,7 @@ func (a *Adapter) newIngressFromRouteGroup(rg *routegroup) (*Ingress, error) {
}
}

return a.newIngress(ingressTypeRouteGroup, rg.Metadata, host, hostnames)
return a.newIngress(TypeRouteGroup, rg.Metadata, host, hostnames)
}

func (a *Adapter) newIngressFromFabric(fg *fabric) (*Ingress, error) {
Expand Down Expand Up @@ -206,10 +193,10 @@ func (a *Adapter) newIngressFromFabric(fg *fabric) (*Ingress, error) {
}
}

return a.newIngress(ingressTypeFabric, fg.Metadata, host, hostnames)
return a.newIngress(TypeFabricGateway, fg.Metadata, host, hostnames)
}

func (a *Adapter) newIngress(typ ingressType, metadata kubeItemMetadata, host string, hostnames []string) (*Ingress, error) {
func (a *Adapter) newIngress(typ IngressType, metadata kubeItemMetadata, host string, hostnames []string) (*Ingress, error) {
annotations := metadata.Annotations

var scheme string
Expand Down Expand Up @@ -280,7 +267,7 @@ func (a *Adapter) newIngress(typ ingressType, metadata kubeItemMetadata, host st
}

return &Ingress{
resourceType: typ,
ResourceType: typ,
Namespace: metadata.Namespace,
Name: metadata.Name,
Hostname: host,
Expand Down Expand Up @@ -364,7 +351,7 @@ func (a *Adapter) ListIngress() ([]*Ingress, error) {
ret = append(ret, ing)
} else {
log.WithFields(log.Fields{
"type": ingressTypeIngress.String(),
"type": TypeIngress,
"ns": ingress.Metadata.Namespace,
"name": ingress.Metadata.Name,
}).Errorf("%v", err)
Expand Down Expand Up @@ -393,7 +380,7 @@ func (a *Adapter) ListRoutegroups() ([]*Ingress, error) {
ret = append(ret, ing)
} else {
log.WithFields(log.Fields{
"type": ingressTypeRouteGroup.String(),
"type": TypeRouteGroup,
"ns": rg.Metadata.Namespace,
"name": rg.Metadata.Name,
}).Errorf("%v", err)
Expand Down Expand Up @@ -422,7 +409,7 @@ func (a *Adapter) ListFabricgateways() ([]*Ingress, error) {
ret = append(ret, ing)
} else {
log.WithFields(log.Fields{
"type": ingressTypeFabric.String(),
"type": TypeFabricGateway,
"ns": fg.Metadata.Namespace,
"name": fg.Metadata.Name,
}).Errorf("%v", err)
Expand Down Expand Up @@ -476,15 +463,15 @@ func (a *Adapter) UpdateIngressLoadBalancer(ingress *Ingress, loadBalancerDNSNam
return ErrUpdateNotNeeded
}

switch ingress.resourceType {
case ingressTypeFabric:
switch ingress.ResourceType {
case TypeFabricGateway:
return updateFabricgatewayLoadBalancer(a.kubeClient, ingress.Namespace, ingress.Name, loadBalancerDNSName)
case ingressTypeRouteGroup:
case TypeRouteGroup:
return updateRoutegroupLoadBalancer(a.kubeClient, ingress.Namespace, ingress.Name, loadBalancerDNSName)
case ingressTypeIngress:
case TypeIngress:
return a.ingressClient.updateIngressLoadBalancer(a.kubeClient, ingress.Namespace, ingress.Name, loadBalancerDNSName)
}
return fmt.Errorf("Unknown resourceType '%s', failed to update Kubernetes resource", ingress.resourceType)
return fmt.Errorf("Unknown resourceType '%s', failed to update Kubernetes resource", ingress.ResourceType)
}

// GetConfigMap retrieves the ConfigMap with name from namespace.
Expand Down
26 changes: 13 additions & 13 deletions kubernetes/adapter_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func TestNewIngressFromKube(tt *testing.T) {
SSLPolicy: testSSLPolicy,
IPAddressType: testIPAddressTypeDefault,
LoadBalancerType: aws.LoadBalancerTypeApplication,
resourceType: ingressTypeIngress,
ResourceType: TypeIngress,
WAFWebACLID: testWAFWebACLID,
},
kubeIngress: &ingress{
Expand Down Expand Up @@ -102,7 +102,7 @@ func TestNewIngressFromKube(tt *testing.T) {
SSLPolicy: testSSLPolicy,
IPAddressType: testIPAddressTypeDefault,
LoadBalancerType: aws.LoadBalancerTypeApplication,
resourceType: ingressTypeIngress,
ResourceType: TypeIngress,
WAFWebACLID: testWAFWebACLID,
},
kubeIngress: &ingress{
Expand Down Expand Up @@ -154,7 +154,7 @@ func TestNewIngressFromKube(tt *testing.T) {
SSLPolicy: testSSLPolicy,
IPAddressType: testIPAddressTypeDefault,
LoadBalancerType: aws.LoadBalancerTypeApplication,
resourceType: ingressTypeIngress,
ResourceType: TypeIngress,
WAFWebACLID: testWAFWebACLID,
},
kubeIngress: &ingress{
Expand Down Expand Up @@ -199,7 +199,7 @@ func TestNewIngressFromKube(tt *testing.T) {
SSLPolicy: testSSLPolicy,
IPAddressType: testIPAddressTypeDualStack,
LoadBalancerType: aws.LoadBalancerTypeApplication,
resourceType: ingressTypeIngress,
ResourceType: TypeIngress,
WAFWebACLID: testWAFWebACLID,
},
kubeIngress: &ingress{
Expand Down Expand Up @@ -232,7 +232,7 @@ func TestNewIngressFromKube(tt *testing.T) {
msg: "test default NLB without annotations",
defaultLoadBalancerType: aws.LoadBalancerTypeNetwork,
ingress: &Ingress{
resourceType: ingressTypeIngress,
ResourceType: TypeIngress,
Namespace: "default",
Name: "foo",
Hostname: "bar",
Expand Down Expand Up @@ -263,7 +263,7 @@ func TestNewIngressFromKube(tt *testing.T) {
msg: "test default NLB with security group fallbacks to ALB",
defaultLoadBalancerType: aws.LoadBalancerTypeNetwork,
ingress: &Ingress{
resourceType: ingressTypeIngress,
ResourceType: TypeIngress,
Namespace: "default",
Name: "foo",
Hostname: "bar",
Expand Down Expand Up @@ -297,7 +297,7 @@ func TestNewIngressFromKube(tt *testing.T) {
msg: "test default NLB with internal annotations fallbacks to ALB",
defaultLoadBalancerType: aws.LoadBalancerTypeNetwork,
ingress: &Ingress{
resourceType: ingressTypeIngress,
ResourceType: TypeIngress,
Namespace: "default",
Name: "foo",
Hostname: "bar",
Expand Down Expand Up @@ -331,7 +331,7 @@ func TestNewIngressFromKube(tt *testing.T) {
msg: "test default ALB with lb type annotation nlb and internal annotation uses NLB",
defaultLoadBalancerType: aws.LoadBalancerTypeApplication,
ingress: &Ingress{
resourceType: ingressTypeIngress,
ResourceType: TypeIngress,
Namespace: "default",
Name: "foo",
Hostname: "bar",
Expand Down Expand Up @@ -366,7 +366,7 @@ func TestNewIngressFromKube(tt *testing.T) {
msg: "test default NLB with WAF fallbacks to ALB",
defaultLoadBalancerType: aws.LoadBalancerTypeNetwork,
ingress: &Ingress{
resourceType: ingressTypeIngress,
ResourceType: TypeIngress,
Namespace: "default",
Name: "foo",
Hostname: "bar",
Expand Down Expand Up @@ -455,7 +455,7 @@ func TestNewIngressFromKube(tt *testing.T) {
} else {
assert.NoError(t, err)
assert.Equal(t, tc.ingress, got, "mapping from kubernetes ingress to adapter failed")
assert.Equal(t, got.String(), fmt.Sprintf("%s %s/%s", tc.ingress.resourceType, tc.ingress.Namespace, tc.ingress.Name), "wrong value from String()")
assert.Equal(t, got.String(), fmt.Sprintf("%s %s/%s", tc.ingress.ResourceType, tc.ingress.Namespace, tc.ingress.Name), "wrong value from String()")
}
})
}
Expand Down Expand Up @@ -548,7 +548,7 @@ func TestAdapterUpdateIngressLoadBalancer(t *testing.T) {
Name: "foo",
Hostname: "bar",
CertificateARN: "zbr",
resourceType: ingressTypeIngress,
ResourceType: TypeIngress,
}
if err := a.UpdateIngressLoadBalancer(ing, "bar"); err != ErrUpdateNotNeeded {
t.Error("expected ErrUpdateNotNeeded")
Expand Down Expand Up @@ -577,7 +577,7 @@ func TestUpdateRouteGroupLoadBalancer(t *testing.T) {
Name: "foo",
Hostname: "bar",
CertificateARN: "zbr",
resourceType: ingressTypeRouteGroup,
ResourceType: TypeRouteGroup,
}
if err := a.UpdateIngressLoadBalancer(ing, "bar"); err != ErrUpdateNotNeeded {
t.Error("expected ErrUpdateNotNeeded")
Expand Down Expand Up @@ -606,7 +606,7 @@ func TestUpdateFabricGatewayLoadBalancer(t *testing.T) {
Name: "foo",
Hostname: "bar",
CertificateARN: "zbr",
resourceType: ingressTypeFabric,
ResourceType: TypeFabricGateway,
}
if err := a.UpdateIngressLoadBalancer(ing, "bar"); err != ErrUpdateNotNeeded {
t.Error("expected ErrUpdateNotNeeded")
Expand Down
14 changes: 13 additions & 1 deletion worker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -305,7 +305,11 @@ func doWork(
log.Infof("Found %d certificate(s)", len(certificateSummaries))
log.Infof("Found %d cloudwatch alarm configuration(s)", len(cwAlarms))

metrics.ingressesTotal.Set(float64(len(ingresses)))
counts := countByIngressType(ingresses)

metrics.ingressesTotal.Set(float64(counts[kubernetes.TypeIngress]))
metrics.routegroupsTotal.Set(float64(counts[kubernetes.TypeRouteGroup]))
metrics.fabricgatewaysTotal.Set(float64(counts[kubernetes.TypeFabricGateway]))
metrics.stacksTotal.Set(float64(len(stacks)))
metrics.ownedAutoscalingGroupsTotal.Set(float64(len(awsAdapter.OwnedAutoScalingGroups)))
metrics.targetedAutoscalingGroupsTotal.Set(float64(len(awsAdapter.TargetedAutoScalingGroups)))
Expand Down Expand Up @@ -336,6 +340,14 @@ func doWork(
return
}

func countByIngressType(ingresses []*kubernetes.Ingress) map[kubernetes.IngressType]int {
counts := make(map[kubernetes.IngressType]int)
for _, ing := range ingresses {
counts[ing.ResourceType]++
}
return counts
}

func sortStacks(stacks []*aws.Stack) {
sort.Slice(stacks, func(i, j int) bool {
if len(stacks[i].CertificateARNs) == len(stacks[j].CertificateARNs) {
Expand Down
17 changes: 17 additions & 0 deletions worker_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1038,3 +1038,20 @@ func Test_cniEventHandler(t *testing.T) {
}, wait.ForeverTestTimeout, time.Millisecond*100)
})
}

func TestCountByIngressType(t *testing.T) {
ingresses := []*kubernetes.Ingress{
&kubernetes.Ingress{ResourceType: kubernetes.TypeIngress},
&kubernetes.Ingress{ResourceType: kubernetes.TypeIngress},
&kubernetes.Ingress{ResourceType: kubernetes.TypeIngress},
&kubernetes.Ingress{ResourceType: kubernetes.TypeRouteGroup},
&kubernetes.Ingress{ResourceType: kubernetes.TypeRouteGroup},
&kubernetes.Ingress{ResourceType: kubernetes.TypeFabricGateway},
}

counts := countByIngressType(ingresses)

assert.Equal(t, 3, counts[kubernetes.TypeIngress])
assert.Equal(t, 2, counts[kubernetes.TypeRouteGroup])
assert.Equal(t, 1, counts[kubernetes.TypeFabricGateway])
}

0 comments on commit d50142d

Please sign in to comment.