Improve duration metrics

Fixes #406.

docs/references/monitoring.md

@@ -9,11 +9,30 @@ in Prometheus format.
 | Metric | Description
 |--------|---
 | `release_count` | Count of releases managed by the operator. |
-| `release_duration_seconds` | Release synchronization duration in seconds. This duration includes one or many `release_phase_durations`. |
-| `release_phase_duration_seconds` | Release phase synchronization duration in seconds. |
+| `release_action_duration_seconds` | Duration of release sync actions in seconds. See [release actions](#release-actions). |
 | `release_phase_info` | The (negative) integer equaling the current phase of a release. Negative values are failed phases, `0` equals to unknown. See [release phases](#release-phases).
 | `release_queue_length_count` | Count of release jobs waiting in the queue to be processed. |
 
+### Release actions
+
+`release_action_duration_seconds` supports the following labels and label values.
+
+| Label          | Label Value |
+|----------------|---
+| `namespace`    | `metadata.namespace` of `HelmRelease`
+| `release_name` | `metadata.name` of `HelmRelease`
+| `success`      | Whether the action was successful (`true` or `false`)
+| `action`       | The release action, see below.
+
+| Action            | Description
+|-------------------|---
+| `sync`            | One entire release sync attempt, as configured to occur once every [--charts-sync-interval](operator.md#reconciliation-configuration)
+| `install`         | Installation attempt
+| `upgrade`         | Upgrade attempt
+| `rollback`        | Rollback attempt
+| `uninstall`       | Uninstallation attempt
+| `dry-run-compare` | Dry run compare attempt to [determine whether to upgrade](../helmrelease-guide/reconciliation-and-upgrades.md#what-triggers-an-upgrade)
+| `annotate`        | [Annotation](../helmrelease-guide/reconciliation-and-upgrades.md#the-antecedent-annotation) attempt
 
 ### Release phases
 

pkg/release/metrics.go

@@ -16,22 +16,34 @@ const (
 )
 
 var (
-	durationBuckets = []float64{1, 5, 10, 30, 60, 120, 180, 300}
+	durationBuckets = []float64{1, 5, 10, 30, 60, 120, 180, 300, 600, 1800}
+	// Deprecated
 	releaseDuration = prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
 		Namespace: "flux",
 		Subsystem: "helm_operator",
 		Name:      "release_duration_seconds",
 		Help:      "Release synchronization duration in seconds.",
 		Buckets:   durationBuckets,
 	}, []string{LabelSuccess, LabelNamespace, LabelReleaseName})
-	releaseActionDuration = prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
+	// Deprecated
+	releasePhaseDuration = prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
 		Namespace:   "flux",
 		Subsystem:   "helm_operator",
 		Name:        "release_phase_duration_seconds",
 		Help:        "Release synchronization phase duration in seconds.",
 		ConstLabels: nil,
 		Buckets:     durationBuckets,
 	}, []string{LabelAction, LabelSuccess, LabelNamespace, LabelReleaseName})
+	releaseActionDuration = prometheus.NewHistogramFrom(stdprometheus.HistogramOpts{
+		Namespace:   "flux",
+		Subsystem:   "helm_operator",
+		Name:        "release_action_duration_seconds",
+		Help:        "Release synchronization action duration in seconds.",
+		ConstLabels: nil,
+		Buckets:     durationBuckets,
+	}, []string{LabelAction, LabelSuccess, LabelNamespace, LabelReleaseName})
+	releaseActionHistograms = []*prometheus.Histogram{releaseActionDuration, releasePhaseDuration}
+	syncAction              = "sync"
 )
 
 func ObserveRelease(start time.Time, success bool, namespace, releaseName string) {
@@ -40,13 +52,21 @@ func ObserveRelease(start time.Time, success bool, namespace, releaseName string
 		LabelNamespace, namespace,
 		LabelReleaseName, releaseName,
 	).Observe(time.Since(start).Seconds())
-}
-
-func ObserveReleaseAction(start time.Time, action action, success bool, namespace, releaseName string) {
 	releaseActionDuration.With(
-		LabelAction, string(action),
+		LabelAction, syncAction,
 		LabelSuccess, fmt.Sprint(success),
 		LabelNamespace, namespace,
 		LabelReleaseName, releaseName,
 	).Observe(time.Since(start).Seconds())
 }
+
+func ObserveReleaseAction(start time.Time, action action, success bool, namespace, releaseName string) {
+	for _, metric := range releaseActionHistograms {
+		metric.With(
+			LabelAction, string(action),
+			LabelSuccess, fmt.Sprint(success),
+			LabelNamespace, namespace,
+			LabelReleaseName, releaseName,
+		).Observe(time.Since(start).Seconds())
+	}
+}