Skip to content

Commit

Permalink
feat: add metrics instrumentation for status updater (#3347)
Browse files Browse the repository at this point in the history
* add metrics instrumentation for status updater

Signed-off-by: shawnh2 <shawnhxh@outlook.com>

* remove status update not found metrics

Signed-off-by: shawnh2 <shawnhxh@outlook.com>

* retrieve kind information from type instead of objectKind

Signed-off-by: shawnh2 <shawnhxh@outlook.com>

* cleanup comments

Signed-off-by: shawnh2 <shawnhxh@outlook.com>

---------

Signed-off-by: shawnh2 <shawnhxh@outlook.com>
  • Loading branch information
shawnh2 committed May 14, 2024
1 parent 9e64358 commit 675da4d
Show file tree
Hide file tree
Showing 4 changed files with 106 additions and 7 deletions.
2 changes: 1 addition & 1 deletion internal/metrics/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

package metrics

// Options encode changes to the options passed to a Metric at creation time.
// MetricOption encode changes to the options passed to a Metric at creation time.
type MetricOption func(*MetricOptions)

type MetricOptions struct {
Expand Down
5 changes: 2 additions & 3 deletions internal/metrics/register.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,15 @@ import (
"net/http"
"time"

"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
otelprom "go.opentelemetry.io/otel/exporters/prometheus"
"go.opentelemetry.io/otel/sdk/metric"
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics"

"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"

"github.com/envoyproxy/gateway/api/v1alpha1"
"github.com/envoyproxy/gateway/internal/envoygateway/config"
)
Expand Down
19 changes: 19 additions & 0 deletions internal/provider/kubernetes/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
// Copyright Envoy Gateway Authors
// SPDX-License-Identifier: Apache-2.0
// The full text of the Apache license is available in the LICENSE file at
// the root of the repo.

package kubernetes

import "github.com/envoyproxy/gateway/internal/metrics"

var (
statusUpdateTotal = metrics.NewCounter("status_update_total", "Total number of status updates by object kind.")
statusUpdateFailed = metrics.NewCounter("status_update_failed_total", "Number of status updates that failed by object kind.")
statusUpdateConflict = metrics.NewCounter("status_update_conflict_total", "Number of status update conflicts encountered by object kind.")
statusUpdateSuccess = metrics.NewCounter("status_update_success_total", "Number of status updates that succeeded by object kind.")
statusUpdateNoop = metrics.NewCounter("status_update_noop_total", "Number of status updates that are no-ops by object kind. This is a subset of successful status updates.")
statusUpdateDurationSeconds = metrics.NewHistogram("status_update_duration_seconds", "How long a status update takes to finish.", []float64{0.001, 0.01, 0.1, 1, 5, 10})

kindLabel = metrics.NewLabel("kind")
)
87 changes: 84 additions & 3 deletions internal/provider/kubernetes/status_updater.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ package kubernetes

import (
"context"
"time"

"github.com/go-logr/logr"
"github.com/google/go-cmp/cmp"
Expand All @@ -21,6 +22,7 @@ import (
gwapiv1a3 "sigs.k8s.io/gateway-api/apis/v1alpha3"

egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
"github.com/envoyproxy/gateway/internal/gatewayapi"
)

// Update contains an all the information needed to update an object's status.
Expand Down Expand Up @@ -66,9 +68,26 @@ func NewUpdateHandler(log logr.Logger, client client.Client) *UpdateHandler {
}

func (u *UpdateHandler) apply(update Update) {
if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
obj := update.Resource
var (
startTime = time.Now()
obj = update.Resource
objKind = kindOf(obj)
)

statusUpdateTotal.With(kindLabel.Value(objKind)).Increment()

defer func() {
updateDuration := time.Since(startTime)
statusUpdateDurationSeconds.With(kindLabel.Value(objKind)).Record(updateDuration.Seconds())
}()

if err := retry.OnError(retry.DefaultBackoff, func(err error) bool {
if kerrors.IsConflict(err) {
statusUpdateConflict.With(kindLabel.Value(objKind)).Increment()
return true
}
return false
}, func() error {
// Get the resource.
if err := u.client.Get(context.Background(), update.NamespacedName, obj); err != nil {
if kerrors.IsNotFound(err) {
Expand All @@ -83,6 +102,8 @@ func (u *UpdateHandler) apply(update Update) {
u.log.WithName(update.NamespacedName.Name).
WithName(update.NamespacedName.Namespace).
Info("status unchanged, bypassing update")

statusUpdateNoop.With(kindLabel.Value(objKind)).Increment()
return nil
}

Expand All @@ -92,6 +113,10 @@ func (u *UpdateHandler) apply(update Update) {
}); err != nil {
u.log.Error(err, "unable to update status", "name", update.NamespacedName.Name,
"namespace", update.NamespacedName.Namespace)

statusUpdateFailed.With(kindLabel.Value(objKind)).Increment()
} else {
statusUpdateSuccess.With(kindLabel.Value(objKind)).Increment()
}
}

Expand Down Expand Up @@ -162,8 +187,10 @@ func (u *UpdateWriter) Send(update Update) {
// GRPCRoute
// EnvoyPatchPolicy
// ClientTrafficPolicy
// BackendTrafficPolicy
// SecurityPolicy
// BackendTLSPolicy
// EnvoyExtensionPolicy
func isStatusEqual(objA, objB interface{}) bool {
opts := cmpopts.IgnoreFields(metav1.Condition{}, "LastTransitionTime")
switch a := objA.(type) {
Expand Down Expand Up @@ -233,7 +260,7 @@ func isStatusEqual(objA, objB interface{}) bool {
return true
}
}
case gwapiv1a3.BackendTLSPolicy:
case *gwapiv1a3.BackendTLSPolicy:
if b, ok := objB.(*gwapiv1a3.BackendTLSPolicy); ok {
if cmp.Equal(a.Status, b.Status, opts) {
return true
Expand All @@ -248,3 +275,57 @@ func isStatusEqual(objA, objB interface{}) bool {
}
return false
}

// kindOf returns the known kind string for the given Kubernetes object,
// returns Unknown for the unsupported object.
//
// Supported objects:
//
// GatewayClasses
// Gateway
// HTTPRoute
// TLSRoute
// TCPRoute
// UDPRoute
// GRPCRoute
// EnvoyPatchPolicy
// ClientTrafficPolicy
// BackendTrafficPolicy
// SecurityPolicy
// BackendTLSPolicy
// EnvoyExtensionPolicy
func kindOf(obj interface{}) string {
var kind string
switch obj.(type) {
case *gwapiv1.GatewayClass:
kind = gatewayapi.KindGatewayClass
case *gwapiv1.Gateway:
kind = gatewayapi.KindGateway
case *gwapiv1.HTTPRoute:
kind = gatewayapi.KindHTTPRoute
case *gwapiv1a2.TLSRoute:
kind = gatewayapi.KindTLSRoute
case *gwapiv1a2.TCPRoute:
kind = gatewayapi.KindTCPRoute
case *gwapiv1a2.UDPRoute:
kind = gatewayapi.KindUDPRoute
case *gwapiv1.GRPCRoute:
kind = gatewayapi.KindGRPCRoute
case *egv1a1.EnvoyPatchPolicy:
kind = gatewayapi.KindEnvoyPatchPolicy
case *egv1a1.ClientTrafficPolicy:
kind = gatewayapi.KindClientTrafficPolicy
case *egv1a1.BackendTrafficPolicy:
kind = gatewayapi.KindBackendTrafficPolicy
case *egv1a1.SecurityPolicy:
kind = gatewayapi.KindSecurityPolicy
case *egv1a1.EnvoyExtensionPolicy:
kind = gatewayapi.KindEnvoyExtensionPolicy
case *gwapiv1a3.BackendTLSPolicy:
kind = gatewayapi.KindBackendTLSPolicy
default:
kind = "Unknown"
}

return kind
}

0 comments on commit 675da4d

Please sign in to comment.