Skip to content

Commit

Permalink
Merge remote-tracking branch upstream/main into add-metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
BEvgeniyS committed Jan 25, 2023
1 parent f0bf37e commit 2adeb76
Show file tree
Hide file tree
Showing 12 changed files with 322 additions and 32 deletions.
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -17,3 +17,5 @@
.idea/
coverage.txt
k8s-image-swapper
__debug_bin
.vscode/launch.json
4 changes: 2 additions & 2 deletions .k8s-image-swapper.yml
Original file line number Diff line number Diff line change
Expand Up @@ -39,9 +39,9 @@ source:
target:
type: aws
aws:
accountId: 123456789
accountId: 123456789101
region: ap-southeast-2
role: arn:aws:iam::123456789012:role/roleName
# role: arn:aws:iam::123456789012:role/roleName
ecrOptions:
tags:
- key: CreatedBy
Expand Down
27 changes: 23 additions & 4 deletions cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (
"time"

"github.com/estahn/k8s-image-swapper/pkg/config"
"github.com/estahn/k8s-image-swapper/pkg/metrics"
"github.com/estahn/k8s-image-swapper/pkg/registry"
"github.com/estahn/k8s-image-swapper/pkg/secrets"
"github.com/estahn/k8s-image-swapper/pkg/types"
Expand All @@ -40,6 +41,8 @@ import (
"github.com/rs/zerolog"
"github.com/rs/zerolog/log"
kwhhttp "github.com/slok/kubewebhook/v2/pkg/http"
kwhprometheus "github.com/slok/kubewebhook/v2/pkg/metrics/prometheus"
kwhwebhook "github.com/slok/kubewebhook/v2/pkg/webhook"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"k8s.io/client-go/kubernetes"
Expand All @@ -59,8 +62,6 @@ A mutating webhook for Kubernetes, pointing the images to a new location.`,
// Uncomment the following line if your bare application
// has an action associated with it:
Run: func(cmd *cobra.Command, args []string) {
//promReg := prometheus.NewRegistry()
//metricsRec := metrics.NewPrometheus(promReg)
log.Trace().Interface("config", cfg).Msg("config")

rClient, err := registry.NewECRClient(cfg.Target.AWS.Region, cfg.Target.AWS.EcrDomain(), cfg.Target.AWS.AccountID, cfg.Target.AWS.Role, cfg.Target.AWS.ECROptions.AccessPolicy, cfg.Target.AWS.ECROptions.LifecyclePolicy)
Expand All @@ -73,11 +74,13 @@ A mutating webhook for Kubernetes, pointing the images to a new location.`,

imageSwapPolicy, err := types.ParseImageSwapPolicy(cfg.ImageSwapPolicy)
if err != nil {
metrics.IncrementError("ParseImageSwapPolicyFail")
log.Err(err).Str("policy", cfg.ImageSwapPolicy).Msg("parsing image swap policy failed")
}

imageCopyPolicy, err := types.ParseImageCopyPolicy(cfg.ImageCopyPolicy)
if err != nil {
metrics.IncrementError("imageCopyPolicyFail")
log.Err(err).Str("policy", cfg.ImageCopyPolicy).Msg("parsing image copy policy failed")
}

Expand All @@ -92,19 +95,32 @@ A mutating webhook for Kubernetes, pointing the images to a new location.`,
)
if err != nil {
log.Err(err).Msg("error creating webhook")
metrics.IncrementError("NewImageSwapperWebhookWithOptsFail")
os.Exit(1)
}

// Configure builtin webhook metrics
recorderConfig := kwhprometheus.RecorderConfig{
Registry: metrics.PromReg,
}

promRecorder, err := kwhprometheus.NewRecorder(recorderConfig)
if err != nil {
log.Err(err).Msg("could not create prometheus metrics recorder")
}

// Get the handler for our webhook.
whHandler, err := kwhhttp.HandlerFor(kwhhttp.HandlerConfig{Webhook: wh})
whHandler, err := kwhhttp.HandlerFor(kwhhttp.HandlerConfig{
Webhook: kwhwebhook.NewMeasuredWebhook(promRecorder, wh),
})
if err != nil {
log.Err(err).Msg("error creating webhook handler")
os.Exit(1)
}

handler := http.NewServeMux()
handler.Handle("/webhook", whHandler)
handler.Handle("/metrics", promhttp.Handler())
handler.Handle("/metrics", promhttp.HandlerFor(metrics.PromReg, promhttp.HandlerOpts{}))
handler.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
_, err := w.Write([]byte(`<html>
<head><title>k8s-image-webhook</title></head>
Expand Down Expand Up @@ -227,6 +243,7 @@ func initConfig() {

if err := viper.Unmarshal(&cfg); err != nil {
log.Err(err).Msg("failed to unmarshal the config file")
metrics.IncrementError("ConfigUnmarshalFail")
}

//validate := validator.New()
Expand Down Expand Up @@ -261,12 +278,14 @@ func setupImagePullSecretsProvider() secrets.ImagePullSecretsProvider {
config, err := rest.InClusterConfig()
if err != nil {
log.Warn().Err(err).Msg("failed to configure Kubernetes client, will continue without reading secrets")
metrics.IncrementError("setupImagePullSecretsProviderFail")
return secrets.NewDummyImagePullSecretsProvider()
}

clientset, err := kubernetes.NewForConfig(config)
if err != nil {
log.Warn().Err(err).Msg("failed to configure Kubernetes client, will continue without reading secrets")
metrics.IncrementError("setupImagePullSecretsProviderFail")
return secrets.NewDummyImagePullSecretsProvider()
}

Expand Down
4 changes: 2 additions & 2 deletions docs/getting-started.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,10 +45,10 @@ target:
role: arn:aws:iam::123456789012:role/roleName
```
!!! note
Make sure that target role has proper trust permissions that allow to assume it cross-account
Make sure that target role has proper trust permissions that allow to assume it cross-account

!!! note
In order te be able to pull images from outside accounts, you will have to apply proper access policy
In order te be able to pull images from outside accounts, you will have to apply proper access policy


#### Access policy
Expand Down
20 changes: 20 additions & 0 deletions docs/metrics.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
## Metrics

The following metrics are available to be scraped by Prometheus on traffic port and `/metrics` path

| Metric Name | Meaning |
| ----------- | ----------- |
go_* | Standard Go instrumentation metrics
process_* | Standard Go instrumentation metrics
promhttp_metric_handler_requests_in_flight | Amount of parallel requests in flight
promhttp_metric_handler_requests_total | Total count of metric requests (scrapes)
kubewebhook_mutating_webhook_review_duration_seconds | Webhook duration, in buckets for percentiles
kubewebhook_webhook_review_warnings_total | Webhook warnings
k8s_image_swapper_ecr_errors | Number of errors related to ecr provider
k8s_image_swapper_main_errors | Number of errors
k8s_image_swapper_cache_hits | Number of registry cache hits
k8s_image_swapper_cache_misses | Number of registry cache misses
k8s_image_swapper_cache_filtered | Number of registry cache filtered out
k8s_image_swapper_cache_image_copy_duration_seconds | Copy duration distribution and count
k8s_image_swapper_cache_repos_create_requests | Number of repositories creation requests
k8s_image_swapper_cache_repos_created | Number of repositories created
1 change: 1 addition & 0 deletions mkdocs.yml
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ nav:
- Getting started: getting-started.md
- Configuration: configuration.md
- FAQ: faq.md
- Metrics: metrics.md
# - Operations:
# - Production considerations: foo
# - FAQ: faq.md
Expand Down
210 changes: 210 additions & 0 deletions pkg/metrics/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
/*
Copyright © 2020 Enrico Stahn <enrico.stahn@gmail.com>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
*/

package metrics

import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/collectors"
)

var (
commonRepoLabels = []string{"resource_namespace", "registry", "repo"}

errors = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "k8s_image_swapper",
Subsystem: "main",
Name: "errors",
Help: "Number of errors",
},
[]string{"error_type"},
)
ecrErrors = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "k8s_image_swapper",
Subsystem: "ecr",
Name: "errors",
Help: "Number of ecr errors",
},
append(commonRepoLabels, "error_type"),
)

cacheHits = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "k8s_image_swapper",
Subsystem: "cache",
Name: "hits",
Help: "Number of registry cache hits",
},
commonRepoLabels,
)

cacheMisses = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "k8s_image_swapper",
Subsystem: "cache",
Name: "misses",
Help: "Number of registry cache misses",
},
commonRepoLabels,
)
cacheFiltered = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "k8s_image_swapper",
Subsystem: "cache",
Name: "filtered",
Help: "Number of registry cache filtered out",
},
commonRepoLabels,
)

imageCopyDuration = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: "k8s_image_swapper",
Subsystem: "cache",
Name: "image_copy_duration_seconds",
Help: "Image copy duration distribution in seconds",
Buckets: prometheus.ExponentialBuckets(4, 2, 10),
},
commonRepoLabels,
)

reposCreateRequests = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "k8s_image_swapper",
Subsystem: "cache",
Name: "repos_create_requests",
Help: "Number of repository create requests",
},
commonRepoLabels,
)
reposCreated = prometheus.NewCounterVec(
prometheus.CounterOpts{
Subsystem: "cache",
Name: "repos_created",
Help: "Number of repositories created",
},
[]string{"registry", "repo"},
)
)

var PromReg *prometheus.Registry

func init() {
PromReg = prometheus.NewRegistry()
PromReg.MustRegister(collectors.NewGoCollector())
PromReg.MustRegister(errors)
PromReg.MustRegister(ecrErrors)
PromReg.MustRegister(cacheHits)
PromReg.MustRegister(cacheMisses)
PromReg.MustRegister(cacheFiltered)
PromReg.MustRegister(imageCopyDuration)
PromReg.MustRegister(reposCreateRequests)
PromReg.MustRegister(reposCreated)
}

// Increments the counter of errors
func IncrementError(errType string) {
errors.With(
prometheus.Labels{
"error_type": errType,
},
).Inc()
}

// Increments the counter of ecr errors
func IncrementEcrError(resource_namespace string, registry string, repo string, errType string) {
ecrErrors.With(
prometheus.Labels{
"resource_namespace": resource_namespace,
"registry": registry,
"repo": repo,
"error_type": errType,
},
).Inc()
}

// Increments the counter of registry cache hits
func IncrementCacheHits(resource_namespace string, registry string, repo string) {
cacheHits.With(
prometheus.Labels{
"resource_namespace": resource_namespace,
"registry": registry,
"repo": repo,
},
).Inc()
}

// Increments the counter of registry cache misses
func IncrementCacheMisses(resource_namespace string, registry string, repo string) {
cacheMisses.With(
prometheus.Labels{
"resource_namespace": resource_namespace,
"registry": registry,
"repo": repo,
},
).Inc()
}

// Increments the counter of registry cache ignored/filtered out
func IncrementCacheFiltered(resource_namespace string, registry string, repo string) {
cacheFiltered.With(
prometheus.Labels{
"resource_namespace": resource_namespace,
"registry": registry,
"repo": repo,
},
).Inc()
}

// Sets the duration of image copy operation
func SetImageCopyDuration(resource_namespace string, registry string, repo string, duration float64) {
imageCopyDuration.With(
prometheus.Labels{
"resource_namespace": resource_namespace,
"registry": registry,
"repo": repo,
},
).Observe(duration)
}

// Increments the counter of repo create requests
func IncrementReposCreateRequests(resource_namespace string, registry string, repo string) {
reposCreateRequests.With(
prometheus.Labels{
"resource_namespace": resource_namespace,
"registry": registry,
"repo": repo,
},
).Inc()
}

// Increments the counter of repos created
func IncrementReposCreated(registry string, repo string) {
reposCreated.With(
prometheus.Labels{
"registry": registry,
"repo": repo,
},
).Inc()
}
1 change: 1 addition & 0 deletions pkg/registry/client.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ package registry
type Client interface {
CreateRepository(string) error
RepositoryExists() bool
RepositoryInCache(string) bool
CopyImage() error
PullImage() error
PutImage() error
Expand Down
Loading

0 comments on commit 2adeb76

Please sign in to comment.