metrics: add /metrics endpoint and console_helm_install_count metric

Works with openshift/console-operator#601 to add a console_helm_install_count counter vector metric. This change will increment the counter each time a user installs a Helm chart in the console. Closes: https://issues.redhat.com/browse/HELM-235 Signed-off-by: Allen Bai <abai@redhat.com> helm/metrics: update according to code reviews Signed-off-by: Allen Bai <abai@redhat.com> helm/metrics: add metrics for helm release upgrades and uninstalls Signed-off-by: Allen Bai <abai@redhat.com>
zonggen · Oct 13, 2021 · 50835fc · 50835fc
1 parent 3c3c955
commit 50835fc
Show file tree

Hide file tree

Showing 6 changed files with 308 additions and 1 deletion.
diff --git a/pkg/helm/actions/install_chart.go b/pkg/helm/actions/install_chart.go
@@ -1,6 +1,7 @@
 package actions
 
 import (
+	"github.com/openshift/console/pkg/helm/metrics"
 	"helm.sh/helm/v3/pkg/action"
 	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/chart/loader"
@@ -40,5 +41,10 @@ func InstallChart(ns, name, url string, vals map[string]interface{}, conf *actio
 	if err != nil {
 		return nil, err
 	}
+
+	if ch.Metadata.Name != "" && ch.Metadata.Version != "" {
+		metrics.HandleconsoleHelmInstallsTotal(ch.Metadata.Name, ch.Metadata.Version)
+	}
+
 	return release, nil
 }
diff --git a/pkg/helm/actions/uninstall_release.go b/pkg/helm/actions/uninstall_release.go
@@ -3,6 +3,7 @@ package actions
 import (
 	"strings"
 
+	"github.com/openshift/console/pkg/helm/metrics"
 	"helm.sh/helm/v3/pkg/action"
 	"helm.sh/helm/v3/pkg/release"
 )
@@ -16,5 +17,11 @@ func UninstallRelease(name string, conf *action.Configuration) (*release.Uninsta
 		}
 		return nil, err
 	}
+
+	ch := resp.Release.Chart
+	if ch.Metadata.Name != "" && ch.Metadata.Version != "" {
+		metrics.HandleconsoleHelmUninstallsTotal(ch.Metadata.Name, ch.Metadata.Version)
+	}
+
 	return resp, nil
 }
diff --git a/pkg/helm/actions/upgrade_release.go b/pkg/helm/actions/upgrade_release.go
@@ -3,6 +3,7 @@ package actions
 import (
 	"strings"
 
+	"github.com/openshift/console/pkg/helm/metrics"
 	"helm.sh/helm/v3/pkg/action"
 	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/chart/loader"
@@ -61,5 +62,14 @@ func UpgradeRelease(ns, name, url string, vals map[string]interface{}, conf *act
 		ch.Metadata.Annotations["chart_url"] = url
 	}
 
-	return client.Run(name, ch, vals)
+	rel, err = client.Run(name, ch, vals)
+	if err != nil {
+		return nil, err
+	}
+
+	if ch.Metadata.Name != "" && ch.Metadata.Version != "" {
+		metrics.HandleconsoleHelmUpgradesTotal(ch.Metadata.Name, ch.Metadata.Version)
+	}
+
+	return rel, nil
 }
diff --git a/pkg/helm/metrics/metrics.go b/pkg/helm/metrics/metrics.go
@@ -0,0 +1,75 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"k8s.io/klog/v2"
+)
+
+const (
+	consoleHelmInstallsTotalMetric   = "console_helm_installs_total"
+	consoleHelmUpgradesTotalMetric   = "console_helm_upgrades_total"
+	consoleHelmUninstallsTotalMetric = "console_helm_uninstalls_total"
+
+	consoleHelmChartNameLabel    = "console_helm_chart_name"
+	consoleHelmChartVersionLabel = "console_helm_chart_version"
+)
+
+var (
+	consoleHelmInstallsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: consoleHelmInstallsTotalMetric,
+			Help: "Number of Helm installations from console by chart name and version.",
+		},
+		[]string{consoleHelmChartNameLabel, consoleHelmChartVersionLabel},
+	)
+	consoleHelmUpgradesTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: consoleHelmUpgradesTotalMetric,
+			Help: "Number of Helm release upgrades from console by chart name and version.",
+		},
+		[]string{consoleHelmChartNameLabel, consoleHelmChartVersionLabel},
+	)
+	consoleHelmUninstallsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: consoleHelmUninstallsTotalMetric,
+			Help: "Number of Helm release uninstallations from console by chart name and version.",
+		},
+		[]string{consoleHelmChartNameLabel, consoleHelmChartVersionLabel},
+	)
+)
+
+func init() {
+	prometheus.MustRegister(consoleHelmInstallsTotal)
+	prometheus.MustRegister(consoleHelmUpgradesTotal)
+	prometheus.MustRegister(consoleHelmUninstallsTotal)
+}
+
+func HandleconsoleHelmInstallsTotal(chartName, chartVersion string) {
+	klog.V(4).Infof("metric %s: %s %s", consoleHelmInstallsTotalMetric, chartName, chartVersion)
+	counter, err := consoleHelmInstallsTotal.GetMetricWithLabelValues(chartName, chartVersion)
+	if err != nil {
+		klog.Errorf("Recovering from metric function - %v", err)
+		return
+	}
+	counter.Add(1)
+}
+
+func HandleconsoleHelmUpgradesTotal(chartName, chartVersion string) {
+	klog.V(4).Infof("metric %s: %s %s", consoleHelmUpgradesTotalMetric, chartName, chartVersion)
+	counter, err := consoleHelmUpgradesTotal.GetMetricWithLabelValues(chartName, chartVersion)
+	if err != nil {
+		klog.Errorf("Recovering from metric function - %v", err)
+		return
+	}
+	counter.Add(1)
+}
+
+func HandleconsoleHelmUninstallsTotal(chartName, chartVersion string) {
+	klog.V(4).Infof("metric %s: %s %s", consoleHelmUninstallsTotalMetric, chartName, chartVersion)
+	counter, err := consoleHelmUninstallsTotal.GetMetricWithLabelValues(chartName, chartVersion)
+	if err != nil {
+		klog.Errorf("Recovering from metric function - %v", err)
+		return
+	}
+	counter.Add(1)
+}
diff --git a/pkg/helm/metrics/metrics_test.go b/pkg/helm/metrics/metrics_test.go
@@ -0,0 +1,191 @@
+package metrics
+
+import (
+	"bufio"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+func TestMetricsNoRelease(t *testing.T) {
+	consoleHelmInstallsTotal.Reset()
+	consoleHelmUpgradesTotal.Reset()
+	consoleHelmUninstallsTotal.Reset()
+	ts := httptest.NewServer(promhttp.Handler())
+	defer ts.Close()
+
+	count := countMetric(t, ts, consoleHelmInstallsTotalMetric)
+	if count > 0 {
+		t.Errorf("%s should not be available", consoleHelmInstallsTotalMetric)
+	}
+
+	count = countMetric(t, ts, consoleHelmUpgradesTotalMetric)
+	if count > 0 {
+		t.Errorf("%s should not be available", consoleHelmUpgradesTotalMetric)
+	}
+
+	count = countMetric(t, ts, consoleHelmUninstallsTotalMetric)
+	if count > 0 {
+		t.Errorf("%s should not be available", consoleHelmUninstallsTotalMetric)
+	}
+}
+
+func TestMetricsSingleRelease(t *testing.T) {
+	consoleHelmInstallsTotal.Reset()
+	consoleHelmUpgradesTotal.Reset()
+	consoleHelmUninstallsTotal.Reset()
+	ts := httptest.NewServer(promhttp.Handler())
+	defer ts.Close()
+
+	chartName, chartVersion := "test-chart", "0.0.1"
+	chartNameLabel, chartVersionLabel := fmt.Sprintf("%s=\"%v\"", consoleHelmChartNameLabel, chartName), fmt.Sprintf("%s=\"%v\"", consoleHelmChartVersionLabel, chartVersion)
+	HandleconsoleHelmInstallsTotal(chartName, chartVersion)
+	HandleconsoleHelmUpgradesTotal(chartName, chartVersion)
+	HandleconsoleHelmUninstallsTotal(chartName, chartVersion)
+
+	count := countMetric(t, ts, consoleHelmInstallsTotalMetric, chartNameLabel, chartVersionLabel)
+	if count != 1 {
+		t.Errorf("%s with labels %s, %s should be 1: %v", consoleHelmInstallsTotalMetric, chartNameLabel, chartVersionLabel, count)
+	}
+
+	count = countMetric(t, ts, consoleHelmUpgradesTotalMetric, chartNameLabel, chartVersionLabel)
+	if count != 1 {
+		t.Errorf("%s with labels %s, %s should be 1: %v", consoleHelmUpgradesTotalMetric, chartNameLabel, chartVersionLabel, count)
+	}
+
+	count = countMetric(t, ts, consoleHelmUninstallsTotalMetric, chartNameLabel, chartVersionLabel)
+	if count != 1 {
+		t.Errorf("%s with labels %s, %s should be 1: %v", consoleHelmUninstallsTotalMetric, chartNameLabel, chartVersionLabel, count)
+	}
+}
+
+func TestMetricsMultipleReleases(t *testing.T) {
+	consoleHelmInstallsTotal.Reset()
+	consoleHelmUpgradesTotal.Reset()
+	consoleHelmUninstallsTotal.Reset()
+	ts := httptest.NewServer(promhttp.Handler())
+	defer ts.Close()
+
+	chartName, chartVersion := "test-chart", "0.0.1"
+	chartNameLabel, chartVersionLabel := fmt.Sprintf("%s=\"%v\"", consoleHelmChartNameLabel, chartName), fmt.Sprintf("%s=\"%v\"", consoleHelmChartVersionLabel, chartVersion)
+	HandleconsoleHelmInstallsTotal(chartName, chartVersion)
+	HandleconsoleHelmInstallsTotal(chartName, chartVersion)
+
+	HandleconsoleHelmUpgradesTotal(chartName, chartVersion)
+	HandleconsoleHelmUpgradesTotal(chartName, chartVersion)
+
+	HandleconsoleHelmUninstallsTotal(chartName, chartVersion)
+	HandleconsoleHelmUninstallsTotal(chartName, chartVersion)
+
+	count := countMetric(t, ts, consoleHelmInstallsTotalMetric, chartNameLabel, chartVersionLabel)
+	if count != 2 {
+		t.Errorf("%s with labels %s, %s should be 2: %v", consoleHelmInstallsTotalMetric, chartNameLabel, chartVersionLabel, count)
+	}
+
+	count = countMetric(t, ts, consoleHelmUpgradesTotalMetric, chartNameLabel, chartVersionLabel)
+	if count != 2 {
+		t.Errorf("%s with labels %s, %s should be 2: %v", consoleHelmUpgradesTotalMetric, chartNameLabel, chartVersionLabel, count)
+	}
+
+	count = countMetric(t, ts, consoleHelmUninstallsTotalMetric, chartNameLabel, chartVersionLabel)
+	if count != 2 {
+		t.Errorf("%s with labels %s, %s should be 2: %v", consoleHelmUninstallsTotalMetric, chartNameLabel, chartVersionLabel, count)
+	}
+
+	chartName, chartVersion = "test-chart-2", "0.0.2"
+	chartNameLabel, chartVersionLabel = fmt.Sprintf("%s=\"%v\"", consoleHelmChartNameLabel, chartName), fmt.Sprintf("%s=\"%v\"", consoleHelmChartVersionLabel, chartVersion)
+	HandleconsoleHelmInstallsTotal(chartName, chartVersion)
+	HandleconsoleHelmUpgradesTotal(chartName, chartVersion)
+	HandleconsoleHelmUninstallsTotal(chartName, chartVersion)
+
+	count = countMetric(t, ts, consoleHelmInstallsTotalMetric, chartNameLabel, chartVersionLabel)
+	if count != 1 {
+		t.Errorf("%s with labels %s, %s should be 1: %v", consoleHelmInstallsTotalMetric, chartNameLabel, chartVersionLabel, count)
+	}
+
+	count = countMetric(t, ts, consoleHelmUpgradesTotalMetric, chartNameLabel, chartVersionLabel)
+	if count != 1 {
+		t.Errorf("%s with labels %s, %s should be 1: %v", consoleHelmUpgradesTotalMetric, chartNameLabel, chartVersionLabel, count)
+	}
+
+	count = countMetric(t, ts, consoleHelmUninstallsTotalMetric, chartNameLabel, chartVersionLabel)
+	if count != 1 {
+		t.Errorf("%s with labels %s, %s should be 1: %v", consoleHelmUninstallsTotalMetric, chartNameLabel, chartVersionLabel, count)
+	}
+
+	// Metrics without specific labels
+	count = countMetric(t, ts, consoleHelmInstallsTotalMetric)
+	if count != 3 {
+		t.Errorf("%s without labels should be 3: %v", consoleHelmInstallsTotalMetric, count)
+	}
+
+	count = countMetric(t, ts, consoleHelmUpgradesTotalMetric)
+	if count != 3 {
+		t.Errorf("%s without labels should be 3: %v", consoleHelmUpgradesTotalMetric, count)
+	}
+
+	count = countMetric(t, ts, consoleHelmUninstallsTotalMetric)
+	if count != 3 {
+		t.Errorf("%s without labels should be 3: %v", consoleHelmUninstallsTotalMetric, count)
+	}
+}
+
+func getMetrics(t *testing.T, ts *httptest.Server) *http.Response {
+	res, err := http.Get(ts.URL + "/metrics")
+	if err != nil {
+		t.Errorf("http error: %s", err)
+	}
+
+	if res.StatusCode != 200 {
+		t.Errorf("http error: %d %s", res.StatusCode, http.StatusText(res.StatusCode))
+	}
+	return res
+}
+
+func countMetric(t *testing.T, ts *httptest.Server, metric string, labels ...string) int {
+	res := getMetrics(t, ts)
+	defer res.Body.Close()
+
+	bytes, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		t.Fatalf("read error: %s", err)
+	}
+
+	return countMetricWithLabels(t, string(bytes), metric, labels...)
+}
+
+func countMetricWithLabels(t *testing.T, response, metric string, labels ...string) (count int) {
+	scanner := bufio.NewScanner(strings.NewReader(response))
+	for scanner.Scan() {
+		text := scanner.Text()
+		// skip comments
+		if strings.HasPrefix(text, "#") {
+			continue
+		}
+		if strings.Contains(text, metric) {
+			t.Logf("found %s\n", scanner.Text())
+			curr_count, _ := strconv.Atoi(text[len(text)-1:])
+			// no specific labels, count all
+			if len(labels) == 0 {
+				count += curr_count
+			}
+			// return metric value with specified labels
+			for i, label := range labels {
+				if !strings.Contains(text, label) {
+					break
+				}
+				if i == len(labels)-1 {
+					// return directly since metrics are aggregated
+					return curr_count
+				}
+			}
+		}
+	}
+	return count
+}
diff --git a/pkg/server/server.go b/pkg/server/server.go
@@ -17,6 +17,7 @@ import (
 	"time"
 
 	"github.com/coreos/pkg/health"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 
 	"github.com/openshift/console/pkg/auth"
 	"github.com/openshift/console/pkg/graphql/resolver"
@@ -470,6 +471,23 @@ func (s *Server) HTTPHandler() http.Handler {
 	})
 
 	// Helm Endpoints
+	metricsHandler := func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			// Requests from prometheus-k8s have the access token in headers instead of cookies.
+			// This allows metric requests with proper tokens in either headers or cookies.
+			if r.URL.Path == "/metrics" {
+				openshiftSessionCookieName := "openshift-session-token"
+				openshiftSessionCookieValue := r.Header.Get("Authorization")
+				r.AddCookie(&http.Cookie{Name: openshiftSessionCookieName, Value: openshiftSessionCookieValue})
+			}
+			next.ServeHTTP(w, r)
+		})
+	}
+
+	handle("/metrics", metricsHandler(authHandler(func(w http.ResponseWriter, r *http.Request) {
+		promhttp.Handler().ServeHTTP(w, r)
+	})))
+
 	handle("/api/helm/template", authHandlerWithUser(helmHandlers.HandleHelmRenderManifests))
 	handle("/api/helm/releases", authHandlerWithUser(helmHandlers.HandleHelmList))
 	handle("/api/helm/chart", authHandlerWithUser(helmHandlers.HandleChartGet))