Skip to content

Commit

Permalink
add metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
BEvgeniyS committed Jan 30, 2023
1 parent 82b8f89 commit 8d12572
Show file tree
Hide file tree
Showing 7 changed files with 303 additions and 15 deletions.
27 changes: 23 additions & 4 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"time"

"github.com/estahn/k8s-image-swapper/pkg/config"
"github.com/estahn/k8s-image-swapper/pkg/metrics"
"github.com/estahn/k8s-image-swapper/pkg/registry"
"github.com/estahn/k8s-image-swapper/pkg/secrets"
"github.com/estahn/k8s-image-swapper/pkg/types"
Expand All @@ -40,6 +41,8 @@ import (
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
kwhhttp "github.com/slok/kubewebhook/v2/pkg/http"
kwhprometheus "github.com/slok/kubewebhook/v2/pkg/metrics/prometheus"
kwhwebhook "github.com/slok/kubewebhook/v2/pkg/webhook"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"k8s.io/client-go/kubernetes"
Expand All @@ -59,8 +62,6 @@ A mutating webhook for Kubernetes, pointing the images to a new location.`,
// Uncomment the following line if your bare application
// has an action associated with it:
Run: func(cmd *cobra.Command, args []string) {
//promReg := prometheus.NewRegistry()
//metricsRec := metrics.NewPrometheus(promReg)
log.Trace().Interface("config", cfg).Msg("config")

rClient, err := registry.NewECRClient(cfg.Target.AWS.Region, cfg.Target.AWS.EcrDomain(), cfg.Target.AWS.AccountID, cfg.Target.AWS.Role, cfg.Target.AWS.ECROptions.AccessPolicy, cfg.Target.AWS.ECROptions.LifecyclePolicy)
Expand All @@ -73,11 +74,13 @@ A mutating webhook for Kubernetes, pointing the images to a new location.`,

imageSwapPolicy, err := types.ParseImageSwapPolicy(cfg.ImageSwapPolicy)
if err != nil {
metrics.IncrementError("ParseImageSwapPolicyFail")
log.Err(err).Str("policy", cfg.ImageSwapPolicy).Msg("parsing image swap policy failed")
}

imageCopyPolicy, err := types.ParseImageCopyPolicy(cfg.ImageCopyPolicy)
if err != nil {
metrics.IncrementError("ImageCopyPolicyFail")
log.Err(err).Str("policy", cfg.ImageCopyPolicy).Msg("parsing image copy policy failed")
}

Expand All @@ -98,19 +101,32 @@ A mutating webhook for Kubernetes, pointing the images to a new location.`,
)
if err != nil {
log.Err(err).Msg("error creating webhook")
metrics.IncrementError("NewImageSwapperWebhookWithOptsFail")
os.Exit(1)
}

// Configure builtin webhook metrics
recorderConfig := kwhprometheus.RecorderConfig{
Registry: metrics.PromReg,
}

promRecorder, err := kwhprometheus.NewRecorder(recorderConfig)
if err != nil {
log.Err(err).Msg("could not create prometheus metrics recorder")
}

// Get the handler for our webhook.
whHandler, err := kwhhttp.HandlerFor(kwhhttp.HandlerConfig{Webhook: wh})
whHandler, err := kwhhttp.HandlerFor(kwhhttp.HandlerConfig{
Webhook: kwhwebhook.NewMeasuredWebhook(promRecorder, wh),
})
if err != nil {
log.Err(err).Msg("error creating webhook handler")
os.Exit(1)
}

handler := http.NewServeMux()
handler.Handle("/webhook", whHandler)
handler.Handle("/metrics", promhttp.Handler())
handler.Handle("/metrics", promhttp.HandlerFor(metrics.PromReg, promhttp.HandlerOpts{}))
handler.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
_, err := w.Write([]byte(`<html>
<head><title>k8s-image-webhook</title></head>
Expand Down Expand Up @@ -233,6 +249,7 @@ func initConfig() {

if err := viper.Unmarshal(&cfg); err != nil {
log.Err(err).Msg("failed to unmarshal the config file")
metrics.IncrementError("ConfigUnmarshalFail")
}

//validate := validator.New()
Expand Down Expand Up @@ -267,12 +284,14 @@ func setupImagePullSecretsProvider() secrets.ImagePullSecretsProvider {
config, err := rest.InClusterConfig()
if err != nil {
log.Warn().Err(err).Msg("failed to configure Kubernetes client, will continue without reading secrets")
metrics.IncrementError("setupImagePullSecretsProviderFail")
return secrets.NewDummyImagePullSecretsProvider()
}

clientset, err := kubernetes.NewForConfig(config)
if err != nil {
log.Warn().Err(err).Msg("failed to configure Kubernetes client, will continue without reading secrets")
metrics.IncrementError("setupImagePullSecretsProviderFail")
return secrets.NewDummyImagePullSecretsProvider()
}

Expand Down
19 changes: 19 additions & 0 deletions docs/metrics.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
## Metrics

The following metrics are available to be scraped by Prometheus on traffic port and `/metrics` path

| Metric Name | Meaning |
| ----------- | ----------- |
go_* | Standard Go instrumentation metrics
process_* | Standard Go instrumentation metrics
promhttp_metric_handler_requests_in_flight | Amount of parallel requests in flight
promhttp_metric_handler_requests_total | Total count of metric requests (scrapes)
kubewebhook_mutating_webhook_review_duration_seconds | Webhook duration, in buckets for percentiles
kubewebhook_webhook_review_warnings_total | Webhook warnings
k8s_image_swapper_ecr_errors | Number of errors related to ecr provider
k8s_image_swapper_main_errors | Number of errors
k8s_image_swapper_cache_hits | Number of registry cache hits
k8s_image_swapper_cache_misses | Number of registry cache misses
k8s_image_swapper_cache_filtered | Number of registry cache filtered out
k8s_image_swapper_cache_images_copied | Number of images copied
k8s_image_swapper_cache_repos_created | Number of repositories created
210 changes: 210 additions & 0 deletions pkg/metrics/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
/*
Copyright © 2020 Enrico Stahn <enrico.stahn@gmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

package metrics

import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
)

var (
commonRepoLabels = []string{"resource_namespace", "registry", "repo"}

errors = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "k8s_image_swapper",
Subsystem: "main",
Name: "errors",
Help: "Number of errors",
},
[]string{"error_type"},
)
ecrErrors = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "k8s_image_swapper",
Subsystem: "ecr",
Name: "errors",
Help: "Number of ecr errors",
},
append(commonRepoLabels, "error_type"),
)

cacheHits = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "k8s_image_swapper",
Subsystem: "cache",
Name: "hits",
Help: "Number of registry cache hits",
},
commonRepoLabels,
)

cacheMisses = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "k8s_image_swapper",
Subsystem: "cache",
Name: "misses",
Help: "Number of registry cache misses",
},
commonRepoLabels,
)
cacheFiltered = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "k8s_image_swapper",
Subsystem: "cache",
Name: "filtered",
Help: "Number of registry cache filtered out",
},
commonRepoLabels,
)

imageCopyDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: "k8s_image_swapper",
Subsystem: "cache",
Name: "image_copy_duration_seconds",
Help: "Image copy duration distribution in seconds",
Buckets: prometheus.ExponentialBuckets(4, 2, 10),
},
commonRepoLabels,
)

reposCreateRequests = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "k8s_image_swapper",
Subsystem: "cache",
Name: "repos_create_requests",
Help: "Number of repository create requests",
},
commonRepoLabels,
)
reposCreated = prometheus.NewCounterVec(
prometheus.CounterOpts{
Subsystem: "cache",
Name: "repos_created",
Help: "Number of repositories created",
},
[]string{"registry", "repo"},
)
)

var PromReg *prometheus.Registry

func init() {
PromReg = prometheus.NewRegistry()
PromReg.MustRegister(collectors.NewGoCollector())
PromReg.MustRegister(errors)
PromReg.MustRegister(ecrErrors)
PromReg.MustRegister(cacheHits)
PromReg.MustRegister(cacheMisses)
PromReg.MustRegister(cacheFiltered)
PromReg.MustRegister(imageCopyDuration)
PromReg.MustRegister(reposCreateRequests)
PromReg.MustRegister(reposCreated)
}

// Increments the counter of errors
func IncrementError(errType string) {
errors.With(
prometheus.Labels{
"error_type": errType,
},
).Inc()
}

// Increments the counter of ecr errors
func IncrementEcrError(resource_namespace string, registry string, repo string, errType string) {
ecrErrors.With(
prometheus.Labels{
"resource_namespace": resource_namespace,
"registry": registry,
"repo": repo,
"error_type": errType,
},
).Inc()
}

// Increments the counter of registry cache hits
func IncrementCacheHits(resource_namespace string, registry string, repo string) {
cacheHits.With(
prometheus.Labels{
"resource_namespace": resource_namespace,
"registry": registry,
"repo": repo,
},
).Inc()
}

// Increments the counter of registry cache misses
func IncrementCacheMisses(resource_namespace string, registry string, repo string) {
cacheMisses.With(
prometheus.Labels{
"resource_namespace": resource_namespace,
"registry": registry,
"repo": repo,
},
).Inc()
}

// Increments the counter of registry cache ignored/filtered out
func IncrementCacheFiltered(resource_namespace string, registry string, repo string) {
cacheFiltered.With(
prometheus.Labels{
"resource_namespace": resource_namespace,
"registry": registry,
"repo": repo,
},
).Inc()
}

// Sets the duration of image copy operation
func SetImageCopyDuration(resource_namespace string, registry string, repo string, duration float64) {
imageCopyDuration.With(
prometheus.Labels{
"resource_namespace": resource_namespace,
"registry": registry,
"repo": repo,
},
).Observe(duration)
}

// Increments the counter of repo create requests
func IncrementReposCreateRequests(resource_namespace string, registry string, repo string) {
reposCreateRequests.With(
prometheus.Labels{
"resource_namespace": resource_namespace,
"registry": registry,
"repo": repo,
},
).Inc()
}

// Increments the counter of repos created
func IncrementReposCreated(registry string, repo string) {
reposCreated.With(
prometheus.Labels{
"registry": registry,
"repo": repo,
},
).Inc()
}
10 changes: 8 additions & 2 deletions pkg/registry/ecr.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"encoding/base64"
"net/http"
"os/exec"
"strings"
"time"

"github.com/aws/aws-sdk-go/aws"
Expand All @@ -15,6 +16,7 @@ import (
"github.com/aws/aws-sdk-go/service/ecr/ecriface"
"github.com/dgraph-io/ristretto"
"github.com/estahn/k8s-image-swapper/pkg/config"
"github.com/estahn/k8s-image-swapper/pkg/metrics"
"github.com/go-co-op/gocron"
"github.com/rs/zerolog/log"
)
Expand Down Expand Up @@ -74,7 +76,7 @@ func (e *ECRClient) CreateRepository(ctx context.Context, name string) error {
})

if err != nil {
log.Err(err).Msg(err.Error())
log.Err(err).Str("repo", name).Str("accessPolicy", e.accessPolicy).Msg("error setting access policy on repo")
return err
}
}
Expand All @@ -88,13 +90,17 @@ func (e *ECRClient) CreateRepository(ctx context.Context, name string) error {
})

if err != nil {
log.Err(err).Msg(err.Error())
log.Err(err).Str("repo", name).Str("lifecyclePolicy", e.lifecyclePolicy).Msg("error setting lifecycle policy on repo")
return err
}
}

e.cache.Set(name, "", 1)

registry := strings.SplitN(name, "/", 2)[0]
repo := strings.SplitN(name, "/", 2)[1]
metrics.IncrementReposCreated(registry, repo)

return nil
}

Expand Down
3 changes: 3 additions & 0 deletions pkg/secrets/kubernetes.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"io/ioutil"
"os"

"github.com/estahn/k8s-image-swapper/pkg/metrics"
jsonpatch "github.com/evanphx/json-patch"
"github.com/rs/zerolog/log"
v1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -72,6 +73,7 @@ func (p *KubernetesImagePullSecretsProvider) GetImagePullSecrets(ctx context.Con
ServiceAccounts(pod.Namespace).
Get(ctx, pod.Spec.ServiceAccountName, metav1.GetOptions{})
if err != nil {
metrics.IncrementError("kubernetesClientFail")
log.Ctx(ctx).Warn().Msg("error fetching referenced service account, continue without service account imagePullSecrets")
}

Expand All @@ -88,6 +90,7 @@ func (p *KubernetesImagePullSecretsProvider) GetImagePullSecrets(ctx context.Con

secret, err := p.kubernetesClient.CoreV1().Secrets(pod.Namespace).Get(ctx, imagePullSecret.Name, metav1.GetOptions{})
if err != nil {
metrics.IncrementError("kubernetesClientFail")
log.Ctx(ctx).Err(err).Msg("error fetching secret, continue without imagePullSecrets")
}

Expand Down
Loading

0 comments on commit 8d12572

Please sign in to comment.