Add test for alertmanager targets update.

Signed-off-by: Julien Pivotto <roidelapluie@o11y.eu>

notifier/notifier_test.go

@@ -571,3 +571,109 @@ func makeInputTargetGroup() *targetgroup.Group {
 func TestLabelsToOpenAPILabelSet(t *testing.T) {
 	require.Equal(t, models.LabelSet{"aaa": "111", "bbb": "222"}, labelsToOpenAPILabelSet(labels.Labels{{Name: "aaa", Value: "111"}, {Name: "bbb", Value: "222"}}))
 }
+
+// TestHangingNotifier validates that targets updates happen even when there are
+// queued alerts.
+// Note: When targets are not updated in time, this test is flaky. Run go test
+// with -count=10 to reproduce.
+func TestHangingNotifier(t *testing.T) {
+	var (
+		done    = make(chan struct{})
+		changed = make(chan struct{})
+		syncCh  = make(chan map[string][]*targetgroup.Group)
+	)
+
+	defer func() {
+		close(done)
+	}()
+
+	// Setting up a bad server. This server hangs for 2 seconds.
+	badServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		select {
+		case <-done:
+		case <-time.After(2 * time.Second):
+		}
+	}))
+	badURL, err := url.Parse(badServer.URL)
+	require.NoError(t, err)
+	badAddress := badURL.Host // Used for __name__ label in targets.
+
+	// Setting up a bad server. This server returns fast, signaling requests on
+	// by closing the changed channel.
+	goodServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		close(changed)
+	}))
+	goodURL, err := url.Parse(goodServer.URL)
+	require.NoError(t, err)
+	goodAddress := goodURL.Host // Used for __name__ label in targets.
+
+	h := NewManager(
+		&Options{
+			QueueCapacity: 20 * maxBatchSize,
+		},
+		nil,
+	)
+
+	h.alertmanagers = make(map[string]*alertmanagerSet)
+
+	am1Cfg := config.DefaultAlertmanagerConfig
+	am1Cfg.Timeout = model.Duration(2 * time.Second)
+
+	h.alertmanagers["config-0"] = &alertmanagerSet{
+		ams:     []alertmanager{},
+		cfg:     &am1Cfg,
+		metrics: h.metrics,
+	}
+	go h.Run(syncCh)
+	defer h.Stop()
+
+	var alerts []*Alert
+	for i := range make([]struct{}, 20*maxBatchSize) {
+		alerts = append(alerts, &Alert{
+			Labels: labels.FromStrings("alertname", fmt.Sprintf("%d", i)),
+		})
+	}
+
+	// Injecting the bad server URL.
+	syncCh <- map[string][]*targetgroup.Group{
+		"config-0": {
+			{
+				Targets: []model.LabelSet{
+					{
+						model.AddressLabel: model.LabelValue(badAddress),
+					},
+				},
+			},
+		},
+	}
+
+	// Queing alerts.
+	h.Send(alerts...)
+
+	// Updating with a functionning alertmanager target.
+	go func() {
+		select {
+		case syncCh <- map[string][]*targetgroup.Group{
+			"config-0": {
+				{
+					Targets: []model.LabelSet{
+						{
+							model.AddressLabel: model.LabelValue(goodAddress),
+						},
+					},
+				},
+			},
+		}:
+		case <-done:
+		}
+	}()
+
+	select {
+	case <-time.After(3 * time.Second):
+		t.Fatalf("Timeout after 3 seconds, targets not synced in time.")
+	case <-changed:
+		// The good server has been hit in less than 3 seconds, therefore
+		// targets have been updated before a second call could be made to the
+		// bad server.
+	}
+}