From 6378a587bc6900b2074f35205039eec453fd8051 Mon Sep 17 00:00:00 2001 From: Simon Behar Date: Tue, 7 Jul 2020 11:26:54 -0700 Subject: [PATCH] fix: Fix concurrency issues with metrics (#3401) --- workflow/metrics/metrics.go | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/workflow/metrics/metrics.go b/workflow/metrics/metrics.go index a61ec250c4ea..819c3ce8b534 100644 --- a/workflow/metrics/metrics.go +++ b/workflow/metrics/metrics.go @@ -2,6 +2,7 @@ package metrics import ( "fmt" + "sync" "time" "github.com/prometheus/client_golang/prometheus" @@ -34,6 +35,8 @@ type metric struct { } type Metrics struct { + // Ensures mutual exclusion in workflows map + workflowsMutex sync.Mutex metricsConfig ServerConfig telemetryConfig ServerConfig @@ -89,6 +92,9 @@ func (m *Metrics) allMetrics() []prometheus.Metric { } func (m *Metrics) WorkflowAdded(key string, phase v1alpha1.NodePhase) { + m.workflowsMutex.Lock() + defer m.workflowsMutex.Unlock() + if m.workflows[key] { return } @@ -99,7 +105,10 @@ func (m *Metrics) WorkflowAdded(key string, phase v1alpha1.NodePhase) { } func (m *Metrics) WorkflowUpdated(key string, fromPhase, toPhase v1alpha1.NodePhase) { - if fromPhase == toPhase || !m.workflows[key] { + m.workflowsMutex.Lock() + hasKey := m.workflows[key] + m.workflowsMutex.Unlock() + if fromPhase == toPhase || !hasKey { return } m.WorkflowDeleted(key, fromPhase) @@ -107,6 +116,9 @@ func (m *Metrics) WorkflowUpdated(key string, fromPhase, toPhase v1alpha1.NodePh } func (m *Metrics) WorkflowDeleted(key string, phase v1alpha1.NodePhase) { + m.workflowsMutex.Lock() + defer m.workflowsMutex.Unlock() + if !m.workflows[key] { return }