Skip to content

Commit

Permalink
Add a branch label to workflow metrics (#164)
Browse files Browse the repository at this point in the history 
Signed-off-by: Rauf Guliyev <rguliyev@yugabyte.com>
  • Loading branch information
@rguliyev
rguliyev authored Feb 18, 2024
1 parent b0d602a commit 02cb8bc
Show file tree
Hide file tree
Showing 4 changed files with 82 additions and 46 deletions.
40 changes: 20 additions & 20 deletions internal/server/metrics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,36 +8,36 @@ var (
Help: "Time that a workflow job took to reach a given state.",
Buckets: prometheus.ExponentialBuckets(1, 1.4, 30),
},
[]string{"org", "repo", "state", "runner_group", "workflow_name", "job_name"},
[]string{"org", "repo", "branch", "state", "runner_group", "workflow_name", "job_name"},
)

workflowJobDurationCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "workflow_job_duration_seconds_total",
Help: "The total duration of jobs.",
},
[]string{"org", "repo", "status", "conclusion", "runner_group", "workflow_name", "job_name"},
[]string{"org", "repo", "branch", "status", "conclusion", "runner_group", "workflow_name", "job_name"},
)

workflowJobStatusCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "workflow_job_status_count",
Help: "Count of workflow job events.",
},
[]string{"org", "repo", "status", "conclusion", "runner_group", "workflow_name", "job_name"},
[]string{"org", "repo", "branch", "status", "conclusion", "runner_group", "workflow_name", "job_name"},
)

workflowRunHistogramVec = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Name: "workflow_execution_time_seconds",
Help: "Time that a workflow took to run.",
Buckets: prometheus.ExponentialBuckets(1, 1.4, 30),
},
[]string{"org", "repo", "workflow_name", "conclusion"},
[]string{"org", "repo", "branch", "workflow_name", "conclusion"},
)

workflowRunStatusCounter = prometheus.NewCounterVec(prometheus.CounterOpts{
Name: "workflow_status_count",
Help: "Count of the occurrences of different workflow states.",
},
[]string{"org", "repo", "status", "conclusion", "workflow_name"},
[]string{"org", "repo", "branch", "status", "conclusion", "workflow_name"},
)

totalMinutesUsedActions = prometheus.NewGaugeVec(prometheus.GaugeOpts{
Expand Down Expand Up @@ -83,36 +83,36 @@ func init() {
}

type WorkflowObserver interface {
ObserveWorkflowJobDuration(org, repo, state, runnerGroup, workflowName, jobName string, seconds float64)
CountWorkflowJobStatus(org, repo, status, conclusion, runnerGroup, workflowName, jobName string)
CountWorkflowJobDuration(org, repo, status, conclusion, runnerGroup, workflowName, jobName string, seconds float64)
ObserveWorkflowJobDuration(org, repo, branch, state, runnerGroup, workflowName, jobName string, seconds float64)
CountWorkflowJobStatus(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName string)
CountWorkflowJobDuration(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName string, seconds float64)

ObserveWorkflowRunDuration(org, repo, workflow, conclusion string, seconds float64)
CountWorkflowRunStatus(org, repo, status, conclusion, workflow string)
ObserveWorkflowRunDuration(org, repo, branch, workflow, conclusion string, seconds float64)
CountWorkflowRunStatus(org, repo, branch, status, conclusion, workflow string)
}

var _ WorkflowObserver = (*PrometheusObserver)(nil)

type PrometheusObserver struct{}

func (o *PrometheusObserver) ObserveWorkflowJobDuration(org, repo, state, runnerGroup, workflowName, jobName string, seconds float64) {
workflowJobHistogramVec.WithLabelValues(org, repo, state, runnerGroup, workflowName, jobName).
func (o *PrometheusObserver) ObserveWorkflowJobDuration(org, repo, branch, state, runnerGroup, workflowName, jobName string, seconds float64) {
workflowJobHistogramVec.WithLabelValues(org, repo, branch, state, runnerGroup, workflowName, jobName).
Observe(seconds)
}

func (o *PrometheusObserver) CountWorkflowJobStatus(org, repo, status, conclusion, runnerGroup, workflowName, jobName string) {
workflowJobStatusCounter.WithLabelValues(org, repo, status, conclusion, runnerGroup, workflowName, jobName).Inc()
func (o *PrometheusObserver) CountWorkflowJobStatus(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName string) {
workflowJobStatusCounter.WithLabelValues(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName).Inc()
}

func (o *PrometheusObserver) CountWorkflowJobDuration(org, repo, status, conclusion, runnerGroup, workflowName, jobName string, seconds float64) {
workflowJobDurationCounter.WithLabelValues(org, repo, status, conclusion, runnerGroup, workflowName, jobName).Add(seconds)
func (o *PrometheusObserver) CountWorkflowJobDuration(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName string, seconds float64) {
workflowJobDurationCounter.WithLabelValues(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName).Add(seconds)
}

func (o *PrometheusObserver) ObserveWorkflowRunDuration(org, repo, workflowName, conclusion string, seconds float64) {
workflowRunHistogramVec.WithLabelValues(org, repo, workflowName, conclusion).
func (o *PrometheusObserver) ObserveWorkflowRunDuration(org, repo, branch, workflowName, conclusion string, seconds float64) {
workflowRunHistogramVec.WithLabelValues(org, repo, branch, workflowName, conclusion).
Observe(seconds)
}

func (o *PrometheusObserver) CountWorkflowRunStatus(org, repo, status, conclusion, workflowName string) {
workflowRunStatusCounter.WithLabelValues(org, repo, status, conclusion, workflowName).Inc()
func (o *PrometheusObserver) CountWorkflowRunStatus(org, repo, branch, status, conclusion, workflowName string) {
workflowRunStatusCounter.WithLabelValues(org, repo, branch, status, conclusion, workflowName).Inc()
}
6 changes: 4 additions & 2 deletions internal/server/server_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ func Test_Server_MetricsRouteAfterWorkflowJob(t *testing.T) {
}()

repo := "some-repo"
branch := "some-branch"
org := "someone"
expectedDuration := 10.0
jobStartedAt := time.Unix(1650308740, 0)
Expand All @@ -104,6 +105,7 @@ func Test_Server_MetricsRouteAfterWorkflowJob(t *testing.T) {
},
},
WorkflowJob: &github.WorkflowJob{
HeadBranch: &branch,
Status: github.String("completed"),
Conclusion: github.String("success"),
StartedAt: &github.Timestamp{Time: jobStartedAt},
Expand All @@ -129,6 +131,6 @@ func Test_Server_MetricsRouteAfterWorkflowJob(t *testing.T) {

payload, err := io.ReadAll(metricsRes.Body)
require.NoError(t, err)
assert.Contains(t, string(payload), `workflow_job_duration_seconds_bucket{job_name="Test",org="someone",repo="some-repo",runner_group="runner-group",state="in_progress",workflow_name="Build and test",le="10.541350399999995"} 1`)
assert.Contains(t, string(payload), `workflow_job_duration_seconds_total{conclusion="success",job_name="Test",org="someone",repo="some-repo",runner_group="runner-group",status="completed",workflow_name="Build and test"} 10`)
assert.Contains(t, string(payload), `workflow_job_duration_seconds_bucket{branch="some-branch",job_name="Test",org="someone",repo="some-repo",runner_group="runner-group",state="in_progress",workflow_name="Build and test",le="10.541350399999995"} 1`)
assert.Contains(t, string(payload), `workflow_job_duration_seconds_total{branch="some-branch",conclusion="success",job_name="Test",org="someone",repo="some-repo",runner_group="runner-group",status="completed",workflow_name="Build and test"} 10`)
}
17 changes: 10 additions & 7 deletions internal/server/workflow_metrics_exporter.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,14 +77,15 @@ func (c *WorkflowMetricsExporter) HandleGHWebHook(w http.ResponseWriter, r *http
_ = level.Info(c.Logger).Log("msg", "got workflow_job event",
"org", event.GetRepo().GetOwner().GetLogin(),
"repo", event.GetRepo().GetName(),
"branch", event.GetWorkflowJob().GetHeadBranch(),
"runId", event.GetWorkflowJob().GetRunID(),
"action", event.GetAction(),
"workflow_name", event.GetWorkflowJob().GetWorkflowName(),
"job_name", event.GetWorkflowJob().GetName())
go c.CollectWorkflowJobEvent(event)
case "workflow_run":
event := model.WorkflowRunEventFromJSON(io.NopCloser(bytes.NewBuffer(buf)))
_ = level.Info(c.Logger).Log("msg", "got workflow_run event", "org", event.GetRepo().GetOwner().GetLogin(), "repo", event.GetRepo().GetName(), "workflow_name", event.GetWorkflow().GetName(), "runNumber", event.GetWorkflowRun().GetRunNumber(), "action", event.GetAction())
_ = level.Info(c.Logger).Log("msg", "got workflow_run event", "org", event.GetRepo().GetOwner().GetLogin(), "repo", event.GetRepo().GetName(), "branch", event.GetWorkflowRun().GetHeadBranch(), "workflow_name", event.GetWorkflow().GetName(), "runNumber", event.GetWorkflowRun().GetRunNumber(), "action", event.GetAction())
go c.CollectWorkflowRunEvent(event)
default:
_ = level.Info(c.Logger).Log("msg", "not implemented", "eventType", eventType)
Expand All @@ -99,6 +100,7 @@ func (c *WorkflowMetricsExporter) HandleGHWebHook(w http.ResponseWriter, r *http
func (c *WorkflowMetricsExporter) CollectWorkflowJobEvent(event *github.WorkflowJobEvent) {
repo := event.GetRepo().GetName()
org := event.GetRepo().GetOwner().GetLogin()
branch := event.WorkflowJob.GetHeadBranch()
action := event.GetAction()

workflowJob := event.GetWorkflowJob()
Expand All @@ -120,34 +122,35 @@ func (c *WorkflowMetricsExporter) CollectWorkflowJobEvent(event *github.Workflow

firstStep := workflowJob.Steps[0]
queuedSeconds := firstStep.StartedAt.Time.Sub(workflowJob.GetStartedAt().Time).Seconds()
c.PrometheusObserver.ObserveWorkflowJobDuration(org, repo, "queued", runnerGroup, workflowName, jobName, math.Max(0, queuedSeconds))
c.PrometheusObserver.ObserveWorkflowJobDuration(org, repo, branch, "queued", runnerGroup, workflowName, jobName, math.Max(0, queuedSeconds))
case "completed":
if workflowJob.StartedAt == nil || workflowJob.CompletedAt == nil {
_ = level.Debug(c.Logger).Log("msg", "unable to calculate job duration of completed event steps are missing timestamps")
break
}

jobSeconds := math.Max(0, workflowJob.GetCompletedAt().Time.Sub(workflowJob.GetStartedAt().Time).Seconds())
c.PrometheusObserver.ObserveWorkflowJobDuration(org, repo, "in_progress", runnerGroup, workflowName, jobName, jobSeconds)
c.PrometheusObserver.CountWorkflowJobDuration(org, repo, status, conclusion, runnerGroup, workflowName, jobName, jobSeconds)
c.PrometheusObserver.ObserveWorkflowJobDuration(org, repo, branch, "in_progress", runnerGroup, workflowName, jobName, jobSeconds)
c.PrometheusObserver.CountWorkflowJobDuration(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName, jobSeconds)
}

c.PrometheusObserver.CountWorkflowJobStatus(org, repo, status, conclusion, runnerGroup, workflowName, jobName)
c.PrometheusObserver.CountWorkflowJobStatus(org, repo, branch, status, conclusion, runnerGroup, workflowName, jobName)
}

func (c *WorkflowMetricsExporter) CollectWorkflowRunEvent(event *github.WorkflowRunEvent) {
repo := event.GetRepo().GetName()
org := event.GetRepo().GetOwner().GetLogin()
branch := event.GetWorkflowRun().GetHeadBranch()
workflowName := event.GetWorkflow().GetName()
conclusion := event.GetWorkflowRun().GetConclusion()

if event.GetAction() == "completed" {
seconds := event.GetWorkflowRun().UpdatedAt.Time.Sub(event.GetWorkflowRun().RunStartedAt.Time).Seconds()
c.PrometheusObserver.ObserveWorkflowRunDuration(org, repo, workflowName, conclusion, seconds)
c.PrometheusObserver.ObserveWorkflowRunDuration(org, repo, branch, workflowName, conclusion, seconds)
}

status := event.GetWorkflowRun().GetStatus()
c.PrometheusObserver.CountWorkflowRunStatus(org, repo, status, conclusion, workflowName)
c.PrometheusObserver.CountWorkflowRunStatus(org, repo, branch, status, conclusion, workflowName)
}

// validateSignature validate the incoming github event.
Expand Down
Loading

0 comments on commit 02cb8bc

Please sign in to comment.