Skip to content

Commit

Permalink
improve cluster kill instance test
Browse files Browse the repository at this point in the history
  • Loading branch information
cenkalti committed Oct 2, 2020
1 parent d86fe9a commit 6c3b181
Showing 1 changed file with 10 additions and 11 deletions.
21 changes: 10 additions & 11 deletions cluster_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ func TestCluster(t *testing.T) {

func TestClusterSuccession(t *testing.T) {
const numInstances = 3
const testTimeout = 3 * time.Second
instances := make([]*Dalga, numInstances)
ctxes := make([]context.Context, numInstances)
cancels := make([]func(), numInstances)
Expand All @@ -132,15 +133,13 @@ func TestClusterSuccession(t *testing.T) {
instance := r.Header.Get("dalga-instance")
killStart <- instance
killDone <- instance
w.WriteHeader(200)
}

mux := http.NewServeMux()
mux.HandleFunc("/kill", kill)
srv := httptest.NewServer(mux)
defer srv.Close()

var client *Client
for i := 0; i < numInstances; i++ {
config := DefaultConfig
config.MySQL.SkipLocked = false
Expand All @@ -149,14 +148,14 @@ func TestClusterSuccession(t *testing.T) {
config.Jobs.ScanFrequency = 100 * time.Millisecond
config.Endpoint.BaseURL = "http://" + srv.Listener.Addr().String() + "/"
config.Listen.Port = 34200 + i
config.Listen.ShutdownTimeout = 2 * time.Second
config.Endpoint.Timeout = time.Second

d, lis, cleanup := newDalga(t, config)
d, _, cleanup := newDalga(t, config)
instances[i] = d
if i == 0 {
client = NewClient("http://" + lis.Addr())
}
defer cleanup()
}
client := NewClient("http://" + instances[0].listener.Addr().String())

for i, inst := range instances {
go inst.Run(ctxes[i])
Expand All @@ -181,7 +180,7 @@ func TestClusterSuccession(t *testing.T) {
var doomedInstance string
select {
case doomedInstance = <-killStart:
case <-time.After(time.Second * 2):
case <-time.After(testTimeout):
t.Fatal("Didn't get the kill start")
}

Expand All @@ -205,20 +204,20 @@ func TestClusterSuccession(t *testing.T) {
}
select {
case <-done:
case <-time.After(time.Second * 2):
case <-time.After(testTimeout):
t.Fatal("Didn't get the done signal from the dead Dalga")
}
select {
case <-killDone:
case <-time.After(time.Second * 2):
case <-time.After(testTimeout):
t.Fatal("Didn't finish the kill")
}

// Now, another instance will pick up that job and try again.
var replacement string
select {
case replacement = <-killStart:
case <-time.After(time.Second * 2):
case <-time.After(testTimeout):
t.Fatal("Job wasn't picked up by a replacement instance")
}
if replacement == doomedInstance {
Expand All @@ -229,7 +228,7 @@ func TestClusterSuccession(t *testing.T) {
if finished != replacement {
t.Fatalf("Something else is very wrong, finished is '%s' but should be '%s'", finished, replacement)
}
case <-time.After(time.Second * 2):
case <-time.After(testTimeout):
t.Fatal("Job was never finished")
}
})
Expand Down

0 comments on commit 6c3b181

Please sign in to comment.