ruler: Fix thanos-io#2204 bug where alert queue is unpoppable causing…

… full queue and dropped alerts (thanos-io#2238)

* Add test for alert queue Pop after multiple Push

Signed-off-by: Robin Clarke-Williams <43950815+robincw-gr@users.noreply.github.com>

* Fix alert queue bug by resignal after Pop (thanos-io#2204)

Signed-off-by: Robin Clarke-Williams <43950815+robincw-gr@users.noreply.github.com>

* Fix alert queue test and simplify

Signed-off-by: Robin Clarke-Williams <43950815+robincw-gr@users.noreply.github.com>

* Update CHANGELOG.md

Signed-off-by: Robin Clarke-Williams <43950815+robincw-gr@users.noreply.github.com>

* Link to thanos-io/thanos PR in CHANGELOG.md

Signed-off-by: Robin Clarke-Williams <43950815+robincw-gr@users.noreply.github.com>

CHANGELOG.md

@@ -11,6 +11,10 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 
 ## Unreleased
 
+### Fixed
+
+- [#2238](https://github.com/thanos-io/thanos/pull/2238) Ruler: Fixed Issue #2204 bug in alert queue signalling filled up queue and alerts were dropped
+
 ## [v0.11.0](https://github.com/thanos-io/thanos/releases/tag/v0.11.0-rc.1) - 2020.03.02
 
 ### Fixed

pkg/alert/alert.go

@@ -193,6 +193,12 @@ func (q *Queue) Pop(termc <-chan struct{}) []*Alert {
 
 	q.popped.Add(float64(n))
 
+	if len(q.queue) > 0 {
+		select {
+		case q.morec <- struct{}{}:
+		default:
+		}
+	}
 	return as[:n]
 }
 

pkg/alert/alert_test.go

@@ -19,6 +19,31 @@ import (
 	"github.com/thanos-io/thanos/pkg/testutil"
 )
 
+func TestQueue_Pop_all_Pushed(t *testing.T) {
+	qcapacity := 10
+	batchsize := 1
+	pushes := 3
+
+	q := NewQueue(
+		nil, nil, qcapacity, batchsize, nil, nil,
+	)
+	for i := 0; i < pushes; i++ {
+		q.Push([]*Alert{
+			{},
+			{},
+		})
+	}
+
+	timeoutc := make(chan struct{}, 1)
+	time.AfterFunc(time.Second, func() { timeoutc <- struct{}{} })
+	popped := 0
+	for p := q.Pop(timeoutc); p != nil; p = q.Pop(timeoutc) {
+		popped += len(p)
+	}
+
+	testutil.Equals(t, pushes*2, popped)
+}
+
 func TestQueue_Push_Relabelled(t *testing.T) {
 	q := NewQueue(
 		nil, nil, 10, 10,