Merge pull request #14885 from serathius/linearizability-recreate-clu…

…ster

Linearizability recreate cluster

.github/workflows/linearizability.yaml

@@ -13,7 +13,7 @@ jobs:
         make build
         mkdir -p /tmp/linearizability
         cat server/etcdserver/raft.fail.go
-        EXPECT_DEBUG=true GO_TEST_FLAGS=-v RESULTS_DIR=/tmp/linearizability make test-linearizability
+        EXPECT_DEBUG=true GO_TEST_FLAGS='-v --count 60 --failfast --run TestLinearizability' RESULTS_DIR=/tmp/linearizability make test-linearizability
     - uses: actions/upload-artifact@v2
       if: always()
       with:

tests/linearizability/linearizability_test.go

@@ -34,8 +34,6 @@ const (
 	minimalQPS = 100.0
 	// maximalQPS limits number of requests send to etcd to avoid linearizability analysis taking too long.
 	maximalQPS = 200.0
-	// failpointTriggersCount
-	failpointTriggersCount = 60
 	// waitBetweenFailpointTriggers
 	waitBetweenFailpointTriggers = time.Second
 )
@@ -77,7 +75,8 @@ func TestLinearizability(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			failpoint := FailpointConfig{
 				failpoint:           tc.failpoint,
-				count:               failpointTriggersCount,
+				count:               1,
+				retries:             3,
 				waitBetweenTriggers: waitBetweenFailpointTriggers,
 			}
 			traffic := trafficConfig{
@@ -117,26 +116,27 @@ func triggerFailpoints(ctx context.Context, t *testing.T, clus *e2e.EtcdProcessC
 	var err error
 	successes := 0
 	failures := 0
-	time.Sleep(config.waitBetweenTriggers)
-	for successes < config.count && failures < config.count {
+	for successes < config.count && failures < config.retries {
+		time.Sleep(config.waitBetweenTriggers)
 		err = config.failpoint.Trigger(t, ctx, clus)
 		if err != nil {
 			t.Logf("Failed to trigger failpoint %q, err: %v\n", config.failpoint.Name(), err)
 			failures++
 			continue
 		}
 		successes++
-		time.Sleep(config.waitBetweenTriggers)
 	}
-	if successes < config.count || failures >= config.count {
+	if successes < config.count || failures >= config.retries {
 		return fmt.Errorf("failed to trigger failpoints enough times, err: %v", err)
 	}
+	time.Sleep(config.waitBetweenTriggers)
 	return nil
 }
 
 type FailpointConfig struct {
 	failpoint           Failpoint
 	count               int
+	retries             int
 	waitBetweenTriggers time.Duration
 }