From 374a8245785b6d43c67c9b3eb514719e544e6154 Mon Sep 17 00:00:00 2001 From: Joe Lanford Date: Wed, 10 Jun 2020 11:15:53 -0400 Subject: [PATCH] pkg/metrics,pkg/reconciler: support legacy operator cr metrics --- go.mod | 1 + go.sum | 1 + main.go | 2 - pkg/reconciler/internal/metrics/legacy.go | 80 ++++++++++ pkg/reconciler/internal/metrics/metrics.go | 174 +++++++++++++++++++++ pkg/reconciler/reconciler.go | 17 ++ 6 files changed, 273 insertions(+), 2 deletions(-) create mode 100644 pkg/reconciler/internal/metrics/legacy.go create mode 100644 pkg/reconciler/internal/metrics/metrics.go diff --git a/go.mod b/go.mod index c16de0da..9f5f6e5f 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/go-logr/logr v0.1.0 github.com/onsi/ginkgo v1.12.0 github.com/onsi/gomega v1.9.0 + github.com/prometheus/client_golang v1.0.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.5.1 go.uber.org/zap v1.13.0 diff --git a/go.sum b/go.sum index e2b2f647..7fc8c554 100644 --- a/go.sum +++ b/go.sum @@ -67,6 +67,7 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver v3.5.0+incompatible h1:CGxCgetQ64DKk7rdZ++Vfnb1+ogGNnB17OJKJXD2Cfs= github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= diff --git a/main.go b/main.go index 9c5f368d..444fe9e9 100644 --- a/main.go +++ b/main.go @@ -172,8 +172,6 @@ func main() { setupLog.Info("configured watch", "gvk", w.GroupVersionKind, "chartPath", w.ChartPath, "maxConcurrentReconciles", maxConcurrentReconciles, "reconcilePeriod", reconcilePeriod) } - // TODO(joelanford): kube-state-metrics? - setupLog.Info("starting manager") if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { setupLog.Error(err, "problem running manager") diff --git a/pkg/reconciler/internal/metrics/legacy.go b/pkg/reconciler/internal/metrics/legacy.go new file mode 100644 index 00000000..91ed2ac8 --- /dev/null +++ b/pkg/reconciler/internal/metrics/legacy.go @@ -0,0 +1,80 @@ +/* +Copyright 2020 The Operator-SDK Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "fmt" + "strings" + + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/event" +) + +type LegacyInfoGauge struct { + *prometheus.GaugeVec +} + +func legacyInfoGaugeName(kind string) string { + return fmt.Sprintf("%s_info", strings.ToLower(kind)) +} + +func NewLegacyInfoGauge(kind string) *LegacyInfoGauge { + return &LegacyInfoGauge{ + prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: legacyInfoGaugeName(kind), + Help: fmt.Sprintf("Information about the %s custom resource", kind), + }, []string{"namespace", "name"}), + } +} + +func (vec *LegacyInfoGauge) Create(e event.CreateEvent) { + name := e.Meta.GetName() + namespace := e.Meta.GetNamespace() + vec.set(name, namespace) +} + +func (vec *LegacyInfoGauge) Update(e event.UpdateEvent) { + name := e.MetaNew.GetName() + namespace := e.MetaNew.GetNamespace() + vec.set(name, namespace) +} + +func (vec *LegacyInfoGauge) Delete(e event.DeleteEvent) { + vec.GaugeVec.Delete(map[string]string{ + "name": e.Meta.GetName(), + "namespace": e.Meta.GetNamespace(), + }) +} + +func (vec *LegacyInfoGauge) set(name, namespace string) { + labels := map[string]string{ + "name": name, + "namespace": namespace, + } + m, err := vec.GaugeVec.GetMetricWith(labels) + if err != nil { + panic(err) + } + m.Set(1) +} + +func NewLegacyRegistry(kind string) RegistererGathererPredicater { + crInfo := NewLegacyInfoGauge(kind) + r := NewRegistry() + r.MustRegister(crInfo) + return r +} diff --git a/pkg/reconciler/internal/metrics/metrics.go b/pkg/reconciler/internal/metrics/metrics.go new file mode 100644 index 00000000..76c4bcf4 --- /dev/null +++ b/pkg/reconciler/internal/metrics/metrics.go @@ -0,0 +1,174 @@ +/* +Copyright 2020 The Operator-SDK Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "context" + "net" + "net/http" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "sigs.k8s.io/controller-runtime/pkg/event" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +var ( + log = logf.Log.WithName("kube-state-metrics") +) + +type CreateEventHandler interface { + Create(e event.CreateEvent) +} + +type UpdateEventHandler interface { + Update(e event.UpdateEvent) +} + +type DeleteEventHandler interface { + Delete(e event.DeleteEvent) +} + +type GenericEventHandler interface { + Generic(e event.GenericEvent) +} + +type RegistererGathererPredicater interface { + prometheus.Registerer + prometheus.Gatherer + Predicate() predicate.Predicate +} + +type Registry struct { + *prometheus.Registry + metrics []prometheus.Collector +} + +func NewRegistry() RegistererGathererPredicater { + return &Registry{ + Registry: prometheus.NewRegistry(), + } +} + +func (r *Registry) Register(c prometheus.Collector) error { + if err := r.Registry.Register(c); err != nil { + return err + } + r.metrics = append(r.metrics, c) + return nil +} + +func (r *Registry) MustRegister(cs ...prometheus.Collector) { + for _, c := range cs { + if err := r.Register(c); err != nil { + panic(err) + } + } +} + +func (r *Registry) Predicate() predicate.Predicate { + createHandlers := []CreateEventHandler{} + updateHandlers := []UpdateEventHandler{} + deleteHandlers := []DeleteEventHandler{} + genericHandlers := []GenericEventHandler{} + + for _, m := range r.metrics { + if m, ok := m.(CreateEventHandler); ok { + createHandlers = append(createHandlers, m) + } + if m, ok := m.(UpdateEventHandler); ok { + updateHandlers = append(updateHandlers, m) + } + if m, ok := m.(DeleteEventHandler); ok { + deleteHandlers = append(deleteHandlers, m) + } + if m, ok := m.(GenericEventHandler); ok { + genericHandlers = append(genericHandlers, m) + } + } + + return predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { + for _, m := range createHandlers { + m.Create(e) + } + return true + }, + UpdateFunc: func(e event.UpdateEvent) bool { + for _, m := range updateHandlers { + m.Update(e) + } + return true + }, + DeleteFunc: func(e event.DeleteEvent) bool { + for _, m := range deleteHandlers { + m.Delete(e) + } + return true + }, + GenericFunc: func(e event.GenericEvent) bool { + for _, m := range genericHandlers { + m.Generic(e) + } + return true + }, + } +} + +type Server struct { + Gatherer prometheus.Gatherer + ListenAddress string +} + +const metricsPath = "/metrics" + +func (s *Server) Start(stop <-chan struct{}) error { + log.Info("metrics server is starting to listen", "addr", s.ListenAddress) + l, err := net.Listen("tcp", s.ListenAddress) + if err != nil { + return err + } + + handler := promhttp.HandlerFor(s.Gatherer, promhttp.HandlerOpts{ + ErrorHandling: promhttp.HTTPErrorOnError, + }) + mux := http.NewServeMux() + mux.Handle(metricsPath, handler) + + server := http.Server{ + Handler: mux, + } + + errChan := make(chan error) + go func() { + log.Info("starting metrics server", "path", metricsPath) + if err := server.Serve(l); err != nil && err != http.ErrServerClosed { + errChan <- err + } + }() + + select { + case err := <-errChan: + return err + case <-stop: + if err := server.Shutdown(context.Background()); err != nil { + return err + } + } + return nil +} diff --git a/pkg/reconciler/reconciler.go b/pkg/reconciler/reconciler.go index df0a0be5..e125b9a4 100644 --- a/pkg/reconciler/reconciler.go +++ b/pkg/reconciler/reconciler.go @@ -41,6 +41,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" "sigs.k8s.io/controller-runtime/pkg/source" "github.com/joelanford/helm-operator/pkg/annotation" @@ -49,6 +50,7 @@ import ( "github.com/joelanford/helm-operator/pkg/internal/sdk/controllerutil" "github.com/joelanford/helm-operator/pkg/reconciler/internal/conditions" internalhook "github.com/joelanford/helm-operator/pkg/reconciler/internal/hook" + "github.com/joelanford/helm-operator/pkg/reconciler/internal/metrics" "github.com/joelanford/helm-operator/pkg/reconciler/internal/updater" "github.com/joelanford/helm-operator/pkg/reconciler/internal/values" ) @@ -63,6 +65,7 @@ type Reconciler struct { eventRecorder record.EventRecorder preHooks []hook.PreHook postHooks []hook.PostHook + metricsRegistry metrics.RegistererGathererPredicater log logr.Logger gvk *schema.GroupVersionKind @@ -130,6 +133,13 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error { return err } + if r.metricsRegistry != nil { + mgr.Add(&metrics.Server{ + Gatherer: r.metricsRegistry, + ListenAddress: "0.0.0.0:8686", + }) + } + if err := r.setupWatches(mgr, c); err != nil { return err } @@ -729,6 +739,7 @@ func (r *Reconciler) addDefaults(mgr ctrl.Manager, controllerName string) { if r.valueMapper == nil { r.valueMapper = values.DefaultMapper } + r.metricsRegistry = metrics.NewLegacyRegistry(r.gvk.Kind) } func (r *Reconciler) setupScheme(mgr ctrl.Manager) { @@ -737,11 +748,17 @@ func (r *Reconciler) setupScheme(mgr ctrl.Manager) { } func (r *Reconciler) setupWatches(mgr ctrl.Manager, c controller.Controller) error { + var predicates []predicate.Predicate + if r.metricsRegistry != nil { + predicates = append(predicates, r.metricsRegistry.Predicate()) + } + obj := &unstructured.Unstructured{} obj.SetGroupVersionKind(*r.gvk) if err := c.Watch( &source.Kind{Type: obj}, &handler.EnqueueRequestForObject{}, + predicates..., ); err != nil { return err }