From 2b26283627e54d89b28fb9d5c841b6ae535f88b2 Mon Sep 17 00:00:00 2001 From: Steve Simpson Date: Fri, 9 Jul 2021 12:44:11 +0200 Subject: [PATCH] Fix StateReplicationWithSharding_InitialSyncFromPeers flaking. The test is writing a single silence and checking a metric which indicates whether replicating the silence has been attempted yet. This is so we can check later on that no replication activity occurs. The assertions later on in the test are passing, but the first one is not, indicating that the replication doesn't trigger early enough. This makes sense because the replication is not synchronous with the writing of the silence. Signed-off-by: Steve Simpson --- pkg/alertmanager/multitenant_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/alertmanager/multitenant_test.go b/pkg/alertmanager/multitenant_test.go index dc60301eb8e..cb78ae1369a 100644 --- a/pkg/alertmanager/multitenant_test.go +++ b/pkg/alertmanager/multitenant_test.go @@ -1820,7 +1820,14 @@ func TestAlertmanager_StateReplicationWithSharding_InitialSyncFromPeers(t *testi { metrics := registries.BuildMetricFamiliesPerUser() assert.Equal(t, float64(1), metrics.GetSumOfGauges("cortex_alertmanager_silences")) - assert.Equal(t, float64(1), metrics.GetSumOfCounters("cortex_alertmanager_state_replication_total")) + } + // 2.c. Wait for the silence replication to be attempted; note this is asynchronous. + { + test.Poll(t, 5*time.Second, float64(1), func() interface{} { + metrics := registries.BuildMetricFamiliesPerUser() + return metrics.GetSumOfCounters("cortex_alertmanager_state_replication_total") + }) + metrics := registries.BuildMetricFamiliesPerUser() assert.Equal(t, float64(0), metrics.GetSumOfCounters("cortex_alertmanager_state_replication_failed_total")) }