refactor: Deflake and Try package

- feat: add CI multiplier
- refactor: readability
- feat: custom Sleep function
- refactor(integration): use custom Sleep
- feat: show Try progress
- feat(try): try response with status code
- refactor(try): use a dedicate package.
- refactor(integration): Try everywhere
- feat(CI): pass CI env var to Integration Tests.
- refactor(acme): increase timeout.
- feat(acme): show Traefik logs
- refactor(integration): use `http.StatusXXX`
- refactor: remove Sleep

Makefile

@@ -7,7 +7,8 @@ TRAEFIK_ENVS := \
 	-e VERBOSE \
 	-e VERSION \
 	-e CODENAME \
-	-e TESTDIRS
+	-e TESTDIRS \
+	-e CI
 
 SRCS = $(shell git ls-files '*.go' | grep -v '^vendor/' | grep -v '^integration/vendor/')
 

integration/access_log_test.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/containous/traefik/integration/try"
 	"github.com/go-check/check"
 	shellwords "github.com/mattn/go-shellwords"
 
@@ -31,10 +32,17 @@ func (s *AccessLogSuite) TestAccessLog(c *check.C) {
 	err := cmd.Start()
 	c.Assert(err, checker.IsNil)
 	defer cmd.Process.Kill()
+
 	defer os.Remove("access.log")
 	defer os.Remove("traefik.log")
 
-	time.Sleep(500 * time.Millisecond)
+	err = try.Do(1*time.Second, func() error {
+		if _, err := os.Stat("traefik.log"); err != nil {
+			return fmt.Errorf("could not get stats for log file: %s", err)
+		}
+		return nil
+	})
+	c.Assert(err, checker.IsNil)
 
 	// Verify Traefik started OK
 	traefikLog, err := ioutil.ReadFile("traefik.log")
@@ -53,11 +61,11 @@ func (s *AccessLogSuite) TestAccessLog(c *check.C) {
 	defer ts3.Close()
 
 	// Make some requests
-	_, err = http.Get("http://127.0.0.1:8000/test1")
+	err = try.GetRequest("http://127.0.0.1:8000/test1", 500*time.Millisecond)
 	c.Assert(err, checker.IsNil)
-	_, err = http.Get("http://127.0.0.1:8000/test2")
+	err = try.GetRequest("http://127.0.0.1:8000/test2", 500*time.Millisecond)
 	c.Assert(err, checker.IsNil)
-	_, err = http.Get("http://127.0.0.1:8000/test2")
+	err = try.GetRequest("http://127.0.0.1:8000/test2", 500*time.Millisecond)
 	c.Assert(err, checker.IsNil)
 
 	// Verify access.log output as expected
@@ -71,14 +79,14 @@ func (s *AccessLogSuite) TestAccessLog(c *check.C) {
 			tokens, err := shellwords.Parse(line)
 			c.Assert(err, checker.IsNil)
 			c.Assert(len(tokens), checker.Equals, 13)
-			c.Assert(tokens[6], checker.Equals, "200")
+			c.Assert(regexp.MustCompile(`^\d{3}$`).MatchString(tokens[6]), checker.True)
 			c.Assert(tokens[9], checker.Equals, fmt.Sprintf("%d", i+1))
 			c.Assert(strings.HasPrefix(tokens[10], "frontend"), checker.True)
 			c.Assert(strings.HasPrefix(tokens[11], "http://127.0.0.1:808"), checker.True)
-			c.Assert(regexp.MustCompile("^\\d+ms$").MatchString(tokens[12]), checker.True)
+			c.Assert(regexp.MustCompile(`^\d+ms$`).MatchString(tokens[12]), checker.True)
 		}
 	}
-	c.Assert(count, checker.Equals, 3)
+	c.Assert(count, checker.GreaterOrEqualThan, 3)
 
 	// Verify no other Traefik problems
 	traefikLog, err = ioutil.ReadFile("traefik.log")

integration/acme_test.go

@@ -4,39 +4,27 @@ import (
 	"crypto/tls"
 	"net/http"
 	"os"
-	"os/exec"
 	"time"
 
+	"github.com/containous/traefik/integration/try"
 	"github.com/go-check/check"
-
-	"errors"
-	"github.com/containous/traefik/integration/utils"
 	checker "github.com/vdemeester/shakers"
 )
 
 // ACME test suites (using libcompose)
 type AcmeSuite struct {
 	BaseSuite
+	boulderIP string
 }
 
 func (s *AcmeSuite) SetUpSuite(c *check.C) {
 	s.createComposeProject(c, "boulder")
 	s.composeProject.Start(c)
 
-	boulderHost := s.composeProject.Container(c, "boulder").NetworkSettings.IPAddress
+	s.boulderIP = s.composeProject.Container(c, "boulder").NetworkSettings.IPAddress
 
 	// wait for boulder
-	err := utils.Try(120*time.Second, func() error {
-		resp, err := http.Get("http://" + boulderHost + ":4000/directory")
-		if err != nil {
-			return err
-		}
-		if resp.StatusCode != 200 {
-			return errors.New("Expected http 200 from boulder")
-		}
-		return nil
-	})
-
+	err := try.GetRequest("http://"+s.boulderIP+":4000/directory", 120*time.Second, try.StatusCodeIs(http.StatusOK))
 	c.Assert(err, checker.IsNil)
 }
 
@@ -48,15 +36,15 @@ func (s *AcmeSuite) TearDownSuite(c *check.C) {
 }
 
 func (s *AcmeSuite) TestRetrieveAcmeCertificate(c *check.C) {
-	boulderHost := s.composeProject.Container(c, "boulder").NetworkSettings.IPAddress
-	file := s.adaptFile(c, "fixtures/acme/acme.toml", struct{ BoulderHost string }{boulderHost})
+	file := s.adaptFile(c, "fixtures/acme/acme.toml", struct{ BoulderHost string }{s.boulderIP})
 	defer os.Remove(file)
-	cmd := exec.Command(traefikBinary, "--configFile="+file)
+	cmd, output := s.cmdTraefikWithConfigFile(file)
+
 	err := cmd.Start()
 	c.Assert(err, checker.IsNil)
 	defer cmd.Process.Kill()
 
-	backend := startTestServer("9010", 200)
+	backend := startTestServer("9010", http.StatusOK)
 	defer backend.Close()
 
 	tr := &http.Transport{
@@ -65,13 +53,12 @@ func (s *AcmeSuite) TestRetrieveAcmeCertificate(c *check.C) {
 	client := &http.Client{Transport: tr}
 
 	// wait for traefik (generating acme account take some seconds)
-	err = utils.Try(30*time.Second, func() error {
+	err = try.Do(90*time.Second, func() error {
 		_, err := client.Get("https://127.0.0.1:5001")
-		if err != nil {
-			return err
-		}
-		return nil
+		return err
 	})
+	// TODO: waiting a refactor of integration tests
+	s.displayTraefikLog(c, output)
 	c.Assert(err, checker.IsNil)
 
 	tr = &http.Transport{
@@ -88,5 +75,5 @@ func (s *AcmeSuite) TestRetrieveAcmeCertificate(c *check.C) {
 	resp, err := client.Do(req)
 	c.Assert(err, checker.IsNil)
 	// Expected a 200
-	c.Assert(resp.StatusCode, checker.Equals, 200)
+	c.Assert(resp.StatusCode, checker.Equals, http.StatusOK)
 }

integration/basic_test.go

@@ -1,13 +1,15 @@
 package main
 
 import (
+	"bytes"
+	"fmt"
 	"net/http"
 	"os/exec"
+	"strings"
 	"time"
 
+	"github.com/containous/traefik/integration/try"
 	"github.com/go-check/check"
-
-	"bytes"
 	checker "github.com/vdemeester/shakers"
 )
 
@@ -21,41 +23,45 @@ func (s *SimpleSuite) TestInvalidConfigShouldFail(c *check.C) {
 	cmd.Stdout = &b
 	cmd.Stderr = &b
 
-	cmd.Start()
-	time.Sleep(500 * time.Millisecond)
+	err := cmd.Start()
+	c.Assert(err, checker.IsNil)
 	defer cmd.Process.Kill()
-	output := b.Bytes()
 
-	c.Assert(string(output), checker.Contains, "Near line 0 (last key parsed ''): bare keys cannot contain '{'")
+	err = try.Do(500*time.Millisecond, func() error {
+		expected := "Near line 0 (last key parsed ''): bare keys cannot contain '{'"
+		actual := b.String()
+
+		if !strings.Contains(actual, expected) {
+			return fmt.Errorf("Got %s, wanted %s", actual, expected)
+		}
+
+		return nil
+	})
+	c.Assert(err, checker.IsNil)
 }
 
 func (s *SimpleSuite) TestSimpleDefaultConfig(c *check.C) {
 	cmd := exec.Command(traefikBinary, "--configFile=fixtures/simple_default.toml")
+
 	err := cmd.Start()
 	c.Assert(err, checker.IsNil)
 	defer cmd.Process.Kill()
 
-	time.Sleep(500 * time.Millisecond)
 	// TODO validate : run on 80
-	resp, err := http.Get("http://127.0.0.1:8000/")
-
 	// Expected a 404 as we did not configure anything
+	err = try.GetRequest("http://127.0.0.1:8000/", 1*time.Second, try.StatusCodeIs(http.StatusNotFound))
 	c.Assert(err, checker.IsNil)
-	c.Assert(resp.StatusCode, checker.Equals, 404)
 }
 
 func (s *SimpleSuite) TestWithWebConfig(c *check.C) {
 	cmd := exec.Command(traefikBinary, "--configFile=fixtures/simple_web.toml")
+
 	err := cmd.Start()
 	c.Assert(err, checker.IsNil)
 	defer cmd.Process.Kill()
 
-	time.Sleep(500 * time.Millisecond)
-
-	resp, err := http.Get("http://127.0.0.1:8080/api")
-	// Expected a 200
+	err = try.GetRequest("http://127.0.0.1:8080/api", 1*time.Second, try.StatusCodeIs(http.StatusOK))
 	c.Assert(err, checker.IsNil)
-	c.Assert(resp.StatusCode, checker.Equals, 200)
 }
 
 func (s *SimpleSuite) TestDefaultEntryPoints(c *check.C) {
@@ -65,12 +71,21 @@ func (s *SimpleSuite) TestDefaultEntryPoints(c *check.C) {
 	cmd.Stdout = &b
 	cmd.Stderr = &b
 
-	cmd.Start()
-	time.Sleep(500 * time.Millisecond)
+	err := cmd.Start()
+	c.Assert(err, checker.IsNil)
 	defer cmd.Process.Kill()
-	output := b.Bytes()
 
-	c.Assert(string(output), checker.Contains, "\"DefaultEntryPoints\":[\"http\"]")
+	err = try.Do(500*time.Millisecond, func() error {
+		expected := "\"DefaultEntryPoints\":[\"http\"]"
+		actual := b.String()
+
+		if !strings.Contains(actual, expected) {
+			return fmt.Errorf("Got %s, wanted %s", actual, expected)
+		}
+
+		return nil
+	})
+	c.Assert(err, checker.IsNil)
 }
 
 func (s *SimpleSuite) TestPrintHelp(c *check.C) {
@@ -80,11 +95,23 @@ func (s *SimpleSuite) TestPrintHelp(c *check.C) {
 	cmd.Stdout = &b
 	cmd.Stderr = &b
 
-	cmd.Start()
-	time.Sleep(500 * time.Millisecond)
+	err := cmd.Start()
+	c.Assert(err, checker.IsNil)
 	defer cmd.Process.Kill()
-	output := b.Bytes()
 
-	c.Assert(string(output), checker.Not(checker.Contains), "panic:")
-	c.Assert(string(output), checker.Contains, "Usage:")
+	err = try.Do(500*time.Millisecond, func() error {
+		expected := "Usage:"
+		notExpected := "panic:"
+		actual := b.String()
+
+		if strings.Contains(actual, notExpected) {
+			return fmt.Errorf("Got %s", actual)
+		}
+		if !strings.Contains(actual, expected) {
+			return fmt.Errorf("Got %s, wanted %s", actual, expected)
+		}
+
+		return nil
+	})
+	c.Assert(err, checker.IsNil)
 }