Switch to counters for the job scheduling duration

Using gauges has the drawback that the last emitted value will persist even if the job is removed and no new metrics are emitted for this particular job.

metric/emitter/prometheus.go

@@ -28,9 +28,10 @@ type PrometheusEmitter struct {
 
 	httpRequestsDuration *prometheus.HistogramVec
 
-	schedulingFullDuration    *prometheus.GaugeVec
-	schedulingLoadingDuration *prometheus.GaugeVec
-	schedulingJobDuration     *prometheus.GaugeVec
+	schedulingFullDuration        *prometheus.GaugeVec
+	schedulingLoadingDuration     *prometheus.GaugeVec
+	schedulingJobDurationTotalVec *prometheus.CounterVec
+	schedulingJobTotalVec         *prometheus.CounterVec
 
 	dbQueriesTotal prometheus.Counter
 	dbConnections  *prometheus.GaugeVec
@@ -183,16 +184,27 @@ func (config *PrometheusConfig) NewEmitter() (metric.Emitter, error) {
 	)
 	prometheus.MustRegister(schedulingLoadingDuration)
 
-	schedulingJobDuration := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
+	schedulingJobDurationTotalVec := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
 			Namespace: "concourse",
 			Subsystem: "scheduling",
-			Name:      "job_duration_seconds",
-			Help:      "Last time taken to calculate the set of valid input versions for a pipeline.",
+			Name:      "job_duration_seconds_total",
+			Help:      "Total amount of time spend calculating the set of valid input versions for a job.",
 		},
 		[]string{"pipeline", "job"},
 	)
-	prometheus.MustRegister(schedulingJobDuration)
+	schedulingJobTotalVec := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "concourse",
+			Subsystem: "scheduling",
+			Name:      "job_total",
+			Help:      "Number of times the job was scheduled",
+		},
+		[]string{"pipeline", "job"},
+	)
+
+	prometheus.MustRegister(schedulingJobDurationTotalVec)
+	prometheus.MustRegister(schedulingJobTotalVec)
 
 	dbQueriesTotal := prometheus.NewCounter(prometheus.CounterOpts{
 		Namespace: "concourse",
@@ -235,9 +247,10 @@ func (config *PrometheusConfig) NewEmitter() (metric.Emitter, error) {
 
 		httpRequestsDuration: httpRequestsDuration,
 
-		schedulingFullDuration:    schedulingFullDuration,
-		schedulingLoadingDuration: schedulingLoadingDuration,
-		schedulingJobDuration:     schedulingJobDuration,
+		schedulingFullDuration:        schedulingFullDuration,
+		schedulingLoadingDuration:     schedulingLoadingDuration,
+		schedulingJobDurationTotalVec: schedulingJobDurationTotalVec,
+		schedulingJobTotalVec:         schedulingJobTotalVec,
 
 		dbQueriesTotal: dbQueriesTotal,
 		dbConnections:  dbConnections,
@@ -392,8 +405,10 @@ func (emitter *PrometheusEmitter) schedulingMetrics(logger lager.Logger, event m
 		if !exists {
 			logger.Error("failed-to-find-job-in-event", fmt.Errorf("expected job to exist in event.Attributes"))
 		}
-		// concourse_scheduling_job_duration_seconds
-		emitter.schedulingJobDuration.WithLabelValues(pipeline, job).Set(duration / 1000)
+		// concourse_scheduling_job_duration_seconds_total
+		emitter.schedulingJobDurationTotalVec.WithLabelValues(pipeline, job).Add(duration / 1000)
+		// concourse_scheduling_job_total
+		emitter.schedulingJobTotalVec.WithLabelValues(pipeline, job).Inc()
 	default:
 	}
 }