Skip to content

Commit

Permalink
Refactor metrics using operator-observability package
Browse files Browse the repository at this point in the history
Signed-off-by: João Vilaça <jvilaca@redhat.com>
  • Loading branch information
machadovilaca committed Oct 30, 2023
1 parent 344d4ba commit 785c6ec
Show file tree
Hide file tree
Showing 48 changed files with 7,252 additions and 353 deletions.
14 changes: 12 additions & 2 deletions cmd/manager/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"os"
"runtime"

"github.com/machadovilaca/operator-observability/pkg/operatormetrics"
osv1 "github.com/openshift/api/operator/v1"
"github.com/spf13/pflag"
apiruntime "k8s.io/apimachinery/pkg/runtime"
Expand All @@ -15,11 +16,12 @@ import (
"sigs.k8s.io/controller-runtime/pkg/client/config"
"sigs.k8s.io/controller-runtime/pkg/manager"
"sigs.k8s.io/controller-runtime/pkg/manager/signals"
controllerruntimemetrics "sigs.k8s.io/controller-runtime/pkg/metrics"

cnaov1 "github.com/kubevirt/cluster-network-addons-operator/pkg/apis/networkaddonsoperator/v1"
cnaov1alpha1 "github.com/kubevirt/cluster-network-addons-operator/pkg/apis/networkaddonsoperator/v1alpha1"
"github.com/kubevirt/cluster-network-addons-operator/pkg/controller"
"github.com/kubevirt/cluster-network-addons-operator/pkg/monitoring"
"github.com/kubevirt/cluster-network-addons-operator/pkg/monitoring/metrics"
"github.com/kubevirt/cluster-network-addons-operator/pkg/util/k8s"
)

Expand Down Expand Up @@ -65,14 +67,22 @@ func main() {
mgr, err := manager.New(cfg, manager.Options{
Scheme: scheme,
Namespace: namespace,
MetricsBindAddress: monitoring.GetMetricsAddress(),
MetricsBindAddress: controllerruntimemetrics.DefaultBindAddress,
MapperProvider: k8s.NewDynamicRESTMapper,
})
if err != nil {
log.Printf("failed to instantiate new operator manager: %v", err)
os.Exit(1)
}

// Setup Monitoring
operatormetrics.Register = controllerruntimemetrics.Registry.Register
err = metrics.SetupMetrics()
if err != nil {
log.Printf("failed to setup metrics: %v", err)
os.Exit(1)
}

log.Print("registering Components")

if err := osv1.Install(mgr.GetScheme()); err != nil {
Expand Down
17 changes: 11 additions & 6 deletions docs/metrics.md
Original file line number Diff line number Diff line change
@@ -1,20 +1,25 @@
# Cluster Network Addons Operator metrics
> This file is auto generated by metricsdocs, run 'make generate-doc' in order to update it.
# Cluster Network Addons Operator Metrics

This document aims to help users that are not familiar with metrics exposed by the Cluster Network Addons Operator.
All metrics documented here are auto-generated by the utility tool 'tools/metricsdocs' and reflects exactly what is being exposed.
## Cluster Network Addons Operator Metrics List
### kubevirt_cnao_cr_kubemacpool_aggregated
Total count of KubeMacPool manager pods deployed by CNAO CR. Type: Gauge.

### kubevirt_cnao_cr_kubemacpool_deployed
KubeMacpool is deployed by CNAO CR. Type: Gauge.

### kubevirt_cnao_cr_ready
CNAO CR Ready. Type: Gauge.

### kubevirt_cnao_kubemacpool_duplicate_macs
Total count of duplicate KubeMacPool MAC addresses. Type: Gauge.

### kubevirt_cnao_kubemacpool_manager_up
Total count of running KubeMacPool manager pods. Type: Gauge.

### kubevirt_cnao_operator_up
Total count of running CNAO operators. Type: Gauge.

## Developing new metrics
After developing new metrics or changing old ones, please run 'make generate-doc' to regenerate this document.

All metrics documented here are auto-generated and reflect exactly what is being
exposed. After developing new metrics or changing old ones please regenerate
this document.
6 changes: 4 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ require (
github.com/go-git/go-git/v5 v5.7.0
github.com/gobwas/glob v0.2.3
github.com/google/go-github/v32 v32.1.0
github.com/kubevirt/monitoring/pkg/metrics/parser v0.0.0-20230627123556-81a891d4462a
github.com/kubevirt/monitoring/pkg/metrics/parser v0.0.0-20231024120544-6a3ba1a680b4
github.com/machadovilaca/operator-observability v0.0.7
github.com/onsi/ginkgo/v2 v2.11.0
github.com/onsi/gomega v1.27.10
github.com/openshift/api v0.0.0
Expand All @@ -23,7 +24,6 @@ require (
github.com/operator-framework/operator-sdk v1.12.0
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.14.0
github.com/prometheus/client_model v0.4.0
github.com/prometheus/common v0.37.0
github.com/spf13/pflag v1.0.5
github.com/thanhpk/randstr v1.0.4
Expand Down Expand Up @@ -182,6 +182,8 @@ require (
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
github.com/pjbgf/sha1cd v0.3.0 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.64.1 // indirect
github.com/prometheus/client_model v0.4.0 // indirect
github.com/prometheus/procfs v0.11.1 // indirect
github.com/rivo/uniseg v0.2.0 // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
Expand Down
8 changes: 6 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1045,8 +1045,8 @@ github.com/krolaw/dhcp4 v0.0.0-20180925202202-7cead472c414 h1:6wnYc2S/lVM7BvR32B
github.com/kshvakov/clickhouse v1.3.5/go.mod h1:DMzX7FxRymoNkVgizH0DWAL8Cur7wHLgx3MUnGwJqpE=
github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0 h1:nHHjmvjitIiyPlUHk/ofpgvBcNcawJLtf4PYHORLjAA=
github.com/kubernetes-csi/external-snapshotter/client/v4 v4.2.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys=
github.com/kubevirt/monitoring/pkg/metrics/parser v0.0.0-20230627123556-81a891d4462a h1:cdX+oxWw1lJDS3EchP+7Oz1XbErk4r7ffVJu1b1MKgI=
github.com/kubevirt/monitoring/pkg/metrics/parser v0.0.0-20230627123556-81a891d4462a/go.mod h1:qGj2agzgwQ27nYhP3xhLs+IBzE5+ALNUg8bDfMcwPqo=
github.com/kubevirt/monitoring/pkg/metrics/parser v0.0.0-20231024120544-6a3ba1a680b4 h1:cV6chMpDgV6JPtyfX5cNK2PMlhWqb+t1tjj7Jsiz9tM=
github.com/kubevirt/monitoring/pkg/metrics/parser v0.0.0-20231024120544-6a3ba1a680b4/go.mod h1:qGj2agzgwQ27nYhP3xhLs+IBzE5+ALNUg8bDfMcwPqo=
github.com/kylelemons/go-gypsy v0.0.0-20160905020020-08cad365cd28/go.mod h1:T/T7jsxVqf9k/zYOqbgNAsANsjxTd1Yq3htjDhQ1H0c=
github.com/kylelemons/godebug v0.0.0-20160406211939-eadb3ce320cb/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
Expand All @@ -1071,6 +1071,8 @@ github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0U
github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
github.com/lovoo/gcloud-opentracing v0.3.0/go.mod h1:ZFqk2y38kMDDikZPAK7ynTTGuyt17nSPdS3K5e+ZTBY=
github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
github.com/machadovilaca/operator-observability v0.0.7 h1:4eYTuxBJQupw+i/Kw3yv8KQnViEh4CiYSol//A0K0Fs=
github.com/machadovilaca/operator-observability v0.0.7/go.mod h1:B1d+efJGa36WWcLYH2ZhqbjiH/OGv/b7nhe9Vb0WL+8=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls=
Expand Down Expand Up @@ -1354,6 +1356,8 @@ github.com/povsister/scp v0.0.0-20210427074412-33febfd9f13e h1:VtsDti2SgX7M7jy0Q
github.com/poy/onpar v0.0.0-20190519213022-ee068f8ea4d1 h1:oL4IBbcqwhhNWh31bjOX8C/OCy0zs9906d/VUru+bqg=
github.com/poy/onpar v0.0.0-20190519213022-ee068f8ea4d1/go.mod h1:nSbFQvMj97ZyhFRSJYtut+msi4sOY6zJDGCdSc+/rZU=
github.com/pquerna/cachecontrol v0.1.0/go.mod h1:NrUG3Z7Rdu85UNR3vm7SOsl1nFIeSiQnrHV5K9mBcUI=
github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.64.1 h1:bvntWler8vOjDJtxBwGDakGNC6srSZmgawGM9Jf7HC8=
github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.64.1/go.mod h1:cfNgxpCPGyIydmt3HcwDqKDt0nYdlGRhzftl+DZH7WA=
github.com/prometheus/alertmanager v0.18.0/go.mod h1:WcxHBl40VSPuOaqWae6l6HpnEOVRIycEJ7i9iYkadEE=
github.com/prometheus/alertmanager v0.20.0/go.mod h1:9g2i48FAyZW6BtbsnvHtMHQXl2aVtrORKwKVCQ+nbrg=
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ import (
"github.com/kubevirt/cluster-network-addons-operator/pkg/apply"
"github.com/kubevirt/cluster-network-addons-operator/pkg/controller/statusmanager"
"github.com/kubevirt/cluster-network-addons-operator/pkg/eventemitter"
"github.com/kubevirt/cluster-network-addons-operator/pkg/monitoring"
"github.com/kubevirt/cluster-network-addons-operator/pkg/monitoring/metrics"
"github.com/kubevirt/cluster-network-addons-operator/pkg/names"
"github.com/kubevirt/cluster-network-addons-operator/pkg/network"
"github.com/kubevirt/cluster-network-addons-operator/pkg/util/k8s"
Expand Down Expand Up @@ -221,7 +221,7 @@ func (r *ReconcileNetworkAddonsConfig) Reconcile(ctx context.Context, request re
r.stopTrackingObjects()

if r.clusterInfo.MonitoringAvailable {
monitoring.ResetMonitoredComponents()
metrics.ResetMonitoredComponents()
}

// Owned objects are automatically garbage collected. Return and don't requeue
Expand Down Expand Up @@ -313,7 +313,7 @@ func (r *ReconcileNetworkAddonsConfig) Reconcile(ctx context.Context, request re
r.statusManager.SetFromPods()

if r.clusterInfo.MonitoringAvailable {
monitoring.TrackMonitoredComponents(&networkAddonsConfig.Spec, r.statusManager)
metrics.TrackMonitoredComponents(&networkAddonsConfig.Spec, r.statusManager)
}

// Kubernetes sometimes fails to apply objects while we remove and recreate
Expand Down
65 changes: 0 additions & 65 deletions pkg/monitoring/metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,9 @@ import (
"strconv"

"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"sigs.k8s.io/controller-runtime/pkg/metrics"

cnao "github.com/kubevirt/cluster-network-addons-operator/pkg/apis/networkaddonsoperator/shared"
"github.com/kubevirt/cluster-network-addons-operator/pkg/controller/statusmanager"
"github.com/kubevirt/cluster-network-addons-operator/pkg/render"
)

Expand All @@ -24,68 +21,6 @@ const (
runbookURLTemplateEnv = "RUNBOOK_URL_TEMPLATE"
)

type MetricsOpts struct {
Name string
Help string
Type string
}

type MetricsKey string

const (
ReadyGauge MetricsKey = "readyGauge"
KMPDeployGauge MetricsKey = "kmpDeployedGauge"
)

var MetricsOptsList = map[MetricsKey]MetricsOpts{
ReadyGauge: {
Name: "kubevirt_cnao_cr_ready",
Help: "CNAO CR Ready",
Type: "Gauge",
},
KMPDeployGauge: {
Name: "kubevirt_cnao_cr_kubemacpool_deployed",
Help: "KubeMacpool is deployed by CNAO CR",
Type: "Gauge",
},
}

var (
readyGauge = prometheus.NewGauge(
prometheus.GaugeOpts{
Name: MetricsOptsList[ReadyGauge].Name,
Help: MetricsOptsList[ReadyGauge].Help,
})
kmpDeployedGauge = prometheus.NewGauge(
prometheus.GaugeOpts{
Name: MetricsOptsList[KMPDeployGauge].Name,
Help: MetricsOptsList[KMPDeployGauge].Help,
})
)

func init() {
metrics.Registry.MustRegister(readyGauge, kmpDeployedGauge)
}

func setGaugeParam(setTrueFlag bool, gaugeParam *prometheus.Gauge) {
if setTrueFlag {
(*gaugeParam).Set(1)
} else {
(*gaugeParam).Set(0)
}
}

func ResetMonitoredComponents() {
setGaugeParam(false, &readyGauge)
setGaugeParam(false, &kmpDeployedGauge)
}

func TrackMonitoredComponents(conf *cnao.NetworkAddonsConfigSpec, statusManager *statusmanager.StatusManager) {
isKubemacpoolDeployed := conf.KubeMacPool != nil
setGaugeParam(isKubemacpoolDeployed, &kmpDeployedGauge)
setGaugeParam(statusManager.IsStatusAvailable(), &readyGauge)
}

func RenderMonitoring(manifestDir string, monitoringAvailable bool) ([]*unstructured.Unstructured, error) {
if !monitoringAvailable {
return nil, nil
Expand Down
17 changes: 17 additions & 0 deletions pkg/monitoring/metrics/metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package metrics

import "github.com/machadovilaca/operator-observability/pkg/operatormetrics"

var (
metrics = [][]operatormetrics.Metric{
operatorMetrics,
}
)

func SetupMetrics() error {
return operatormetrics.RegisterMetrics(metrics...)
}

func ListMetrics() []operatormetrics.Metric {
return operatormetrics.ListMetrics()
}
48 changes: 48 additions & 0 deletions pkg/monitoring/metrics/operator_metrics.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
package metrics

import (
"github.com/machadovilaca/operator-observability/pkg/operatormetrics"

cnao "github.com/kubevirt/cluster-network-addons-operator/pkg/apis/networkaddonsoperator/shared"
"github.com/kubevirt/cluster-network-addons-operator/pkg/controller/statusmanager"
)

var (
operatorMetrics = []operatormetrics.Metric{
cnaoCrReady,
kmpDeployed,
}

cnaoCrReady = operatormetrics.NewGauge(
operatormetrics.MetricOpts{
Name: "kubevirt_cnao_cr_ready",
Help: "CNAO CR Ready",
},
)

kmpDeployed = operatormetrics.NewGauge(
operatormetrics.MetricOpts{
Name: "kubevirt_cnao_cr_kubemacpool_deployed",
Help: "KubeMacpool is deployed by CNAO CR",
},
)
)

func ResetMonitoredComponents() {
setGaugeParam(false, cnaoCrReady)
setGaugeParam(false, kmpDeployed)
}

func TrackMonitoredComponents(conf *cnao.NetworkAddonsConfigSpec, statusManager *statusmanager.StatusManager) {
isKubemacpoolDeployed := conf.KubeMacPool != nil
setGaugeParam(isKubemacpoolDeployed, kmpDeployed)
setGaugeParam(statusManager.IsStatusAvailable(), cnaoCrReady)
}

func setGaugeParam(setTrueFlag bool, gaugeParam *operatormetrics.Gauge) {
if setTrueFlag {
gaugeParam.Set(1)
} else {
gaugeParam.Set(0)
}
}
Loading

0 comments on commit 785c6ec

Please sign in to comment.