Fixed the logic to correctly update the 'time-since-last-run' metrics…

… for both pipelines and pipelines jobs
mvisonneau · May 28, 2020 · 5a91ba2 · 5a91ba2
1 parent e4b9009
commit 5a91ba2
Show file tree

Hide file tree

Showing 2 changed files with 65 additions and 25 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,10 @@ and this project adheres to [0ver](https://0ver.org) (more or less).
 
 ## [Unreleased]
 
+### Changed
+
+- Fixed a bug where `gitlab_ci_pipeline_time_since_last_run_seconds` and `gitlab_ci_pipeline_time_since_last_job_run_seconds` would not get updated after beingfetched for the first time on each pipelines (#106)
+
 ## [0.3.2] - 2019-05-27
 
 ### Changed

diff --git a/lib/exporter/client.go b/lib/exporter/client.go
@@ -41,13 +41,14 @@ const (
 type ProjectRef struct {
 	*schemas.Project
 
-	Kind                        ProjectRefKind
-	ID                          int
-	PathWithNamespace           string
-	Topics                      string
-	Ref                         string
-	MostRecentPipeline          *gitlab.Pipeline
-	MostRecentPipelineVariables string
+	Kind                          ProjectRefKind
+	ID                            int
+	PathWithNamespace             string
+	Topics                        string
+	Ref                           string
+	MostRecentPipeline            *gitlab.Pipeline
+	MostRecentPipelineVariables   string
+	PreviouslyEmittedPipelineJobs map[string]int
 }
 
 // ProjectsRefs allows us to keep track of all the ProjectRef
@@ -139,6 +140,11 @@ func (c *Client) pollPipelineJobs(pr *ProjectRef) error {
 	var resp *gitlab.Response
 	var err error
 
+	// Initialize the variable
+	if pr.PreviouslyEmittedPipelineJobs == nil {
+		pr.PreviouslyEmittedPipelineJobs = map[string]int{}
+	}
+
 	options := &gitlab.ListJobsOptions{
 		ListOptions: gitlab.ListOptions{
 			PerPage: 20,
@@ -154,11 +160,41 @@ func (c *Client) pollPipelineJobs(pr *ProjectRef) error {
 		}
 
 		// otherwise proceed
-		log.Infof("Found %d jobs for pipeline %d", len(jobs), pr.MostRecentPipeline.ID)
+		log.WithFields(
+			log.Fields{
+				"project-id":  pr.ID,
+				"pipeline-id": pr.MostRecentPipeline.ID,
+				"jobs-count":  len(jobs),
+			},
+		).Info("found pipeline jobs")
+
 		for _, job := range jobs {
 			jobValues := append(pr.defaultLabelsValues(), job.Stage, job.Name)
 
-			log.Debugf("Job %s for pipeline %d", job.Name, pr.MostRecentPipeline.ID)
+			// In case a job gets restarted, it will have an ID greated than the previous one(s)
+			// jobs in new pipelines should get greated IDs too
+			if previousJobID, ok := pr.PreviouslyEmittedPipelineJobs[job.Name]; ok {
+				if previousJobID == job.ID {
+					timeSinceLastJobRun.WithLabelValues(jobValues...).Set(time.Since(*job.CreatedAt).Round(time.Second).Seconds())
+				}
+
+				if previousJobID >= job.ID {
+					continue
+				}
+			}
+
+			log.WithFields(
+				log.Fields{
+					"project-id":  pr.ID,
+					"pipeline-id": pr.MostRecentPipeline.ID,
+					"job-name":    job.Name,
+					"job-id":      job.ID,
+				},
+			).Debug("processing pipeline job metrics")
+
+			pr.PreviouslyEmittedPipelineJobs[job.Name] = job.ID
+
+			timeSinceLastJobRun.WithLabelValues(jobValues...).Set(time.Since(*job.CreatedAt).Round(time.Second).Seconds())
 			lastRunJobDuration.WithLabelValues(jobValues...).Set(job.Duration)
 
 			emitStatusMetric(
@@ -232,6 +268,7 @@ func (c *Client) pollProjectRefMostRecentPipeline(pr *ProjectRef) error {
 		return fmt.Errorf("could not read content of last pipeline %s:%s", pr.PathWithNamespace, pr.Ref)
 	}
 
+	defaultLabelValues := pr.defaultLabelsValues()
 	if pr.MostRecentPipeline == nil || !reflect.DeepEqual(pipeline, pr.MostRecentPipeline) {
 		pr.MostRecentPipeline = pipeline
 
@@ -246,7 +283,6 @@ func (c *Client) pollProjectRefMostRecentPipeline(pr *ProjectRef) error {
 			pr.MostRecentPipelineVariables = ""
 		}
 
-		defaultLabelValues := pr.defaultLabelsValues()
 		if pipeline.Status == "running" {
 			runCount.WithLabelValues(defaultLabelValues...).Inc()
 		}
@@ -269,25 +305,25 @@ func (c *Client) pollProjectRefMostRecentPipeline(pr *ProjectRef) error {
 			pipeline.Status,
 			pr.OutputSparseStatusMetrics(c.Config),
 		)
+	}
 
-		if pr.FetchPipelineJobMetrics(c.Config) {
-			if err := c.pollPipelineJobs(pr); err != nil {
-				log.WithFields(
-					log.Fields{
-						"project-path-with-namespace": pr.PathWithNamespace,
-						"project-id":                  pr.ID,
-						"project-ref":                 pr.Ref,
-						"project-ref-kind":            pr.Kind,
-						"pipeline-id":                 pipeline.ID,
-						"error":                       err.Error(),
-					},
-				).Error("polling pipeline jobs metrics")
-			}
+	if pr.FetchPipelineJobMetrics(c.Config) {
+		if err := c.pollPipelineJobs(pr); err != nil {
+			log.WithFields(
+				log.Fields{
+					"project-path-with-namespace": pr.PathWithNamespace,
+					"project-id":                  pr.ID,
+					"project-ref":                 pr.Ref,
+					"project-ref-kind":            pr.Kind,
+					"pipeline-id":                 pipeline.ID,
+					"error":                       err.Error(),
+				},
+			).Error("polling pipeline jobs metrics")
 		}
-
-		timeSinceLastRun.WithLabelValues(defaultLabelValues...).Set(time.Since(*pipeline.CreatedAt).Round(time.Second).Seconds())
 	}
 
+	timeSinceLastRun.WithLabelValues(defaultLabelValues...).Set(time.Since(*pipeline.CreatedAt).Round(time.Second).Seconds())
+
 	return nil
 }