diff --git a/pkg/helm/actions/install_chart.go b/pkg/helm/actions/install_chart.go index ee8bca36edaa..6edd037e7538 100644 --- a/pkg/helm/actions/install_chart.go +++ b/pkg/helm/actions/install_chart.go @@ -1,6 +1,7 @@ package actions import ( + "github.com/openshift/console/pkg/helm/metrics" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" @@ -40,5 +41,10 @@ func InstallChart(ns, name, url string, vals map[string]interface{}, conf *actio if err != nil { return nil, err } + + if ch.Metadata.Name != "" && ch.Metadata.Version != "" { + metrics.HandleconsoleHelmInstallsTotal(ch.Metadata.Name, ch.Metadata.Version) + } + return release, nil } diff --git a/pkg/helm/actions/uninstall_release.go b/pkg/helm/actions/uninstall_release.go index 141f53d1ba9a..705f5f43076e 100644 --- a/pkg/helm/actions/uninstall_release.go +++ b/pkg/helm/actions/uninstall_release.go @@ -3,6 +3,7 @@ package actions import ( "strings" + "github.com/openshift/console/pkg/helm/metrics" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/release" ) @@ -16,5 +17,11 @@ func UninstallRelease(name string, conf *action.Configuration) (*release.Uninsta } return nil, err } + + ch := resp.Release.Chart + if ch.Metadata.Name != "" && ch.Metadata.Version != "" { + metrics.HandleconsoleHelmUninstallsTotal(ch.Metadata.Name, ch.Metadata.Version) + } + return resp, nil } diff --git a/pkg/helm/actions/upgrade_release.go b/pkg/helm/actions/upgrade_release.go index 2c0a2028a32a..b973f41874c6 100644 --- a/pkg/helm/actions/upgrade_release.go +++ b/pkg/helm/actions/upgrade_release.go @@ -3,6 +3,7 @@ package actions import ( "strings" + "github.com/openshift/console/pkg/helm/metrics" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" @@ -61,5 +62,14 @@ func UpgradeRelease(ns, name, url string, vals map[string]interface{}, conf *act ch.Metadata.Annotations["chart_url"] = url } - return client.Run(name, ch, vals) + rel, err = client.Run(name, ch, vals) + if err != nil { + return nil, err + } + + if ch.Metadata.Name != "" && ch.Metadata.Version != "" { + metrics.HandleconsoleHelmUpgradesTotal(ch.Metadata.Name, ch.Metadata.Version) + } + + return rel, nil } diff --git a/pkg/helm/metrics/metrics.go b/pkg/helm/metrics/metrics.go new file mode 100644 index 000000000000..adab0f341697 --- /dev/null +++ b/pkg/helm/metrics/metrics.go @@ -0,0 +1,75 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "k8s.io/klog/v2" +) + +const ( + consoleHelmInstallsTotalMetric = "console_helm_installs_total" + consoleHelmUpgradesTotalMetric = "console_helm_upgrades_total" + consoleHelmUninstallsTotalMetric = "console_helm_uninstalls_total" + + consoleHelmChartNameLabel = "console_helm_chart_name" + consoleHelmChartVersionLabel = "console_helm_chart_version" +) + +var ( + consoleHelmInstallsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: consoleHelmInstallsTotalMetric, + Help: "Number of Helm installations from console by chart name and version.", + }, + []string{consoleHelmChartNameLabel, consoleHelmChartVersionLabel}, + ) + consoleHelmUpgradesTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: consoleHelmUpgradesTotalMetric, + Help: "Number of Helm release upgrades from console by chart name and version.", + }, + []string{consoleHelmChartNameLabel, consoleHelmChartVersionLabel}, + ) + consoleHelmUninstallsTotal = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: consoleHelmUninstallsTotalMetric, + Help: "Number of Helm release uninstallations from console by chart name and version.", + }, + []string{consoleHelmChartNameLabel, consoleHelmChartVersionLabel}, + ) +) + +func init() { + prometheus.MustRegister(consoleHelmInstallsTotal) + prometheus.MustRegister(consoleHelmUpgradesTotal) + prometheus.MustRegister(consoleHelmUninstallsTotal) +} + +func HandleconsoleHelmInstallsTotal(chartName, chartVersion string) { + klog.V(4).Infof("metric %s: %s %s", consoleHelmInstallsTotalMetric, chartName, chartVersion) + counter, err := consoleHelmInstallsTotal.GetMetricWithLabelValues(chartName, chartVersion) + if err != nil { + klog.Errorf("Recovering from metric function - %v", err) + return + } + counter.Add(1) +} + +func HandleconsoleHelmUpgradesTotal(chartName, chartVersion string) { + klog.V(4).Infof("metric %s: %s %s", consoleHelmUpgradesTotalMetric, chartName, chartVersion) + counter, err := consoleHelmUpgradesTotal.GetMetricWithLabelValues(chartName, chartVersion) + if err != nil { + klog.Errorf("Recovering from metric function - %v", err) + return + } + counter.Add(1) +} + +func HandleconsoleHelmUninstallsTotal(chartName, chartVersion string) { + klog.V(4).Infof("metric %s: %s %s", consoleHelmUninstallsTotalMetric, chartName, chartVersion) + counter, err := consoleHelmUninstallsTotal.GetMetricWithLabelValues(chartName, chartVersion) + if err != nil { + klog.Errorf("Recovering from metric function - %v", err) + return + } + counter.Add(1) +} diff --git a/pkg/helm/metrics/metrics_test.go b/pkg/helm/metrics/metrics_test.go new file mode 100644 index 000000000000..451d0e17855a --- /dev/null +++ b/pkg/helm/metrics/metrics_test.go @@ -0,0 +1,191 @@ +package metrics + +import ( + "bufio" + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +func TestMetricsNoRelease(t *testing.T) { + consoleHelmInstallsTotal.Reset() + consoleHelmUpgradesTotal.Reset() + consoleHelmUninstallsTotal.Reset() + ts := httptest.NewServer(promhttp.Handler()) + defer ts.Close() + + count := countMetric(t, ts, consoleHelmInstallsTotalMetric) + if count > 0 { + t.Errorf("%s should not be available", consoleHelmInstallsTotalMetric) + } + + count = countMetric(t, ts, consoleHelmUpgradesTotalMetric) + if count > 0 { + t.Errorf("%s should not be available", consoleHelmUpgradesTotalMetric) + } + + count = countMetric(t, ts, consoleHelmUninstallsTotalMetric) + if count > 0 { + t.Errorf("%s should not be available", consoleHelmUninstallsTotalMetric) + } +} + +func TestMetricsSingleRelease(t *testing.T) { + consoleHelmInstallsTotal.Reset() + consoleHelmUpgradesTotal.Reset() + consoleHelmUninstallsTotal.Reset() + ts := httptest.NewServer(promhttp.Handler()) + defer ts.Close() + + chartName, chartVersion := "test-chart", "0.0.1" + chartNameLabel, chartVersionLabel := fmt.Sprintf("%s=\"%v\"", consoleHelmChartNameLabel, chartName), fmt.Sprintf("%s=\"%v\"", consoleHelmChartVersionLabel, chartVersion) + HandleconsoleHelmInstallsTotal(chartName, chartVersion) + HandleconsoleHelmUpgradesTotal(chartName, chartVersion) + HandleconsoleHelmUninstallsTotal(chartName, chartVersion) + + count := countMetric(t, ts, consoleHelmInstallsTotalMetric, chartNameLabel, chartVersionLabel) + if count != 1 { + t.Errorf("%s with labels %s, %s should be 1: %v", consoleHelmInstallsTotalMetric, chartNameLabel, chartVersionLabel, count) + } + + count = countMetric(t, ts, consoleHelmUpgradesTotalMetric, chartNameLabel, chartVersionLabel) + if count != 1 { + t.Errorf("%s with labels %s, %s should be 1: %v", consoleHelmUpgradesTotalMetric, chartNameLabel, chartVersionLabel, count) + } + + count = countMetric(t, ts, consoleHelmUninstallsTotalMetric, chartNameLabel, chartVersionLabel) + if count != 1 { + t.Errorf("%s with labels %s, %s should be 1: %v", consoleHelmUninstallsTotalMetric, chartNameLabel, chartVersionLabel, count) + } +} + +func TestMetricsMultipleReleases(t *testing.T) { + consoleHelmInstallsTotal.Reset() + consoleHelmUpgradesTotal.Reset() + consoleHelmUninstallsTotal.Reset() + ts := httptest.NewServer(promhttp.Handler()) + defer ts.Close() + + chartName, chartVersion := "test-chart", "0.0.1" + chartNameLabel, chartVersionLabel := fmt.Sprintf("%s=\"%v\"", consoleHelmChartNameLabel, chartName), fmt.Sprintf("%s=\"%v\"", consoleHelmChartVersionLabel, chartVersion) + HandleconsoleHelmInstallsTotal(chartName, chartVersion) + HandleconsoleHelmInstallsTotal(chartName, chartVersion) + + HandleconsoleHelmUpgradesTotal(chartName, chartVersion) + HandleconsoleHelmUpgradesTotal(chartName, chartVersion) + + HandleconsoleHelmUninstallsTotal(chartName, chartVersion) + HandleconsoleHelmUninstallsTotal(chartName, chartVersion) + + count := countMetric(t, ts, consoleHelmInstallsTotalMetric, chartNameLabel, chartVersionLabel) + if count != 2 { + t.Errorf("%s with labels %s, %s should be 2: %v", consoleHelmInstallsTotalMetric, chartNameLabel, chartVersionLabel, count) + } + + count = countMetric(t, ts, consoleHelmUpgradesTotalMetric, chartNameLabel, chartVersionLabel) + if count != 2 { + t.Errorf("%s with labels %s, %s should be 2: %v", consoleHelmUpgradesTotalMetric, chartNameLabel, chartVersionLabel, count) + } + + count = countMetric(t, ts, consoleHelmUninstallsTotalMetric, chartNameLabel, chartVersionLabel) + if count != 2 { + t.Errorf("%s with labels %s, %s should be 2: %v", consoleHelmUninstallsTotalMetric, chartNameLabel, chartVersionLabel, count) + } + + chartName, chartVersion = "test-chart-2", "0.0.2" + chartNameLabel, chartVersionLabel = fmt.Sprintf("%s=\"%v\"", consoleHelmChartNameLabel, chartName), fmt.Sprintf("%s=\"%v\"", consoleHelmChartVersionLabel, chartVersion) + HandleconsoleHelmInstallsTotal(chartName, chartVersion) + HandleconsoleHelmUpgradesTotal(chartName, chartVersion) + HandleconsoleHelmUninstallsTotal(chartName, chartVersion) + + count = countMetric(t, ts, consoleHelmInstallsTotalMetric, chartNameLabel, chartVersionLabel) + if count != 1 { + t.Errorf("%s with labels %s, %s should be 1: %v", consoleHelmInstallsTotalMetric, chartNameLabel, chartVersionLabel, count) + } + + count = countMetric(t, ts, consoleHelmUpgradesTotalMetric, chartNameLabel, chartVersionLabel) + if count != 1 { + t.Errorf("%s with labels %s, %s should be 1: %v", consoleHelmUpgradesTotalMetric, chartNameLabel, chartVersionLabel, count) + } + + count = countMetric(t, ts, consoleHelmUninstallsTotalMetric, chartNameLabel, chartVersionLabel) + if count != 1 { + t.Errorf("%s with labels %s, %s should be 1: %v", consoleHelmUninstallsTotalMetric, chartNameLabel, chartVersionLabel, count) + } + + // Metrics without specific labels + count = countMetric(t, ts, consoleHelmInstallsTotalMetric) + if count != 3 { + t.Errorf("%s without labels should be 3: %v", consoleHelmInstallsTotalMetric, count) + } + + count = countMetric(t, ts, consoleHelmUpgradesTotalMetric) + if count != 3 { + t.Errorf("%s without labels should be 3: %v", consoleHelmUpgradesTotalMetric, count) + } + + count = countMetric(t, ts, consoleHelmUninstallsTotalMetric) + if count != 3 { + t.Errorf("%s without labels should be 3: %v", consoleHelmUninstallsTotalMetric, count) + } +} + +func getMetrics(t *testing.T, ts *httptest.Server) *http.Response { + res, err := http.Get(ts.URL + "/metrics") + if err != nil { + t.Errorf("http error: %s", err) + } + + if res.StatusCode != 200 { + t.Errorf("http error: %d %s", res.StatusCode, http.StatusText(res.StatusCode)) + } + return res +} + +func countMetric(t *testing.T, ts *httptest.Server, metric string, labels ...string) int { + res := getMetrics(t, ts) + defer res.Body.Close() + + bytes, err := ioutil.ReadAll(res.Body) + if err != nil { + t.Fatalf("read error: %s", err) + } + + return countMetricWithLabels(t, string(bytes), metric, labels...) +} + +func countMetricWithLabels(t *testing.T, response, metric string, labels ...string) (count int) { + scanner := bufio.NewScanner(strings.NewReader(response)) + for scanner.Scan() { + text := scanner.Text() + // skip comments + if strings.HasPrefix(text, "#") { + continue + } + if strings.Contains(text, metric) { + t.Logf("found %s\n", scanner.Text()) + curr_count, _ := strconv.Atoi(text[len(text)-1:]) + // no specific labels, count all + if len(labels) == 0 { + count += curr_count + } + // return metric value with specified labels + for i, label := range labels { + if !strings.Contains(text, label) { + break + } + if i == len(labels)-1 { + // return directly since metrics are aggregated + return curr_count + } + } + } + } + return count +} diff --git a/pkg/server/server.go b/pkg/server/server.go index 7bae7aba4755..e9f12e2367ca 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -17,6 +17,7 @@ import ( "time" "github.com/coreos/pkg/health" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/openshift/console/pkg/auth" "github.com/openshift/console/pkg/graphql/resolver" @@ -470,6 +471,23 @@ func (s *Server) HTTPHandler() http.Handler { }) // Helm Endpoints + metricsHandler := func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Requests from prometheus-k8s have the access token in headers instead of cookies. + // This allows metric requests with proper tokens in either headers or cookies. + if r.URL.Path == "/metrics" { + openshiftSessionCookieName := "openshift-session-token" + openshiftSessionCookieValue := r.Header.Get("Authorization") + r.AddCookie(&http.Cookie{Name: openshiftSessionCookieName, Value: openshiftSessionCookieValue}) + } + next.ServeHTTP(w, r) + }) + } + + handle("/metrics", metricsHandler(authHandler(func(w http.ResponseWriter, r *http.Request) { + promhttp.Handler().ServeHTTP(w, r) + }))) + handle("/api/helm/template", authHandlerWithUser(helmHandlers.HandleHelmRenderManifests)) handle("/api/helm/releases", authHandlerWithUser(helmHandlers.HandleHelmList)) handle("/api/helm/chart", authHandlerWithUser(helmHandlers.HandleChartGet))