From cde6cd006d9542fe3922bbc7c9b122930d265717 Mon Sep 17 00:00:00 2001 From: Siyuan Zhang Date: Tue, 30 Apr 2024 13:07:02 -0700 Subject: [PATCH 1/3] e2e: add flag to pass specific binary path for last release. Signed-off-by: Siyuan Zhang --- tests/framework/e2e/flags.go | 5 +++++ tests/robustness/makefile.mk | 16 ++++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/tests/framework/e2e/flags.go b/tests/framework/e2e/flags.go index 23bdde6590c..593117ff346 100644 --- a/tests/framework/e2e/flags.go +++ b/tests/framework/e2e/flags.go @@ -69,6 +69,8 @@ func InitFlags() { certDirDef := FixturesDir binDir := flag.String("bin-dir", binDirDef, "The directory for store etcd and etcdctl binaries.") + binLastRelease := flag.String("bin-last-release", "", "The path for the last release etcd binary.") + flag.StringVar(&CertDir, "cert-dir", certDirDef, "The directory for store certificate files.") flag.Parse() @@ -79,6 +81,9 @@ func InitFlags() { Etcdutl: *binDir + "/etcdutl", LazyFS: *binDir + "/lazyfs", } + if *binLastRelease != "" { + BinPath.EtcdLastRelease = *binLastRelease + } CertPath = CertDir + "/server.crt" PrivateKeyPath = CertDir + "/server.key.insecure" CaPath = CertDir + "/ca.crt" diff --git a/tests/robustness/makefile.mk b/tests/robustness/makefile.mk index 2af7f7a3965..85e91cb48a0 100644 --- a/tests/robustness/makefile.mk +++ b/tests/robustness/makefile.mk @@ -5,9 +5,13 @@ test-robustness-reports: # Test previous release branches +.PHONY: test-robustness-release-3.6 +test-robustness-release-3.6: /tmp/etcd-release-3.6-failpoints/bin /tmp/etcd-release-3.5-failpoints/bin + GO_TEST_FLAGS="$${GO_TEST_FLAGS} --bin-dir=/tmp/etcd-release-3.6-failpoints/bin --bin-last-release=/tmp/etcd-release-3.5-failpoints/bin/etcd" make test-robustness + .PHONY: test-robustness-release-3.5 -test-robustness-release-3.5: /tmp/etcd-release-3.5-failpoints/bin - GO_TEST_FLAGS="$${GO_TEST_FLAGS} --bin-dir=/tmp/etcd-release-3.5-failpoints/bin" make test-robustness +test-robustness-release-3.5: /tmp/etcd-release-3.5-failpoints/bin /tmp/etcd-release-3.4-failpoints/bin + GO_TEST_FLAGS="$${GO_TEST_FLAGS} --bin-dir=/tmp/etcd-release-3.5-failpoints/bin --bin-last-release=/tmp/etcd-release-3.4-failpoints/bin/etcd" make test-robustness .PHONY: test-robustness-release-3.4 test-robustness-release-3.4: /tmp/etcd-release-3.4-failpoints/bin @@ -72,6 +76,14 @@ $(GOPATH)/bin/gofail: tools/mod/go.mod tools/mod/go.sum make gofail-enable; \ make build; +/tmp/etcd-release-3.6-failpoints/bin: $(GOPATH)/bin/gofail + rm -rf /tmp/etcd-release-3.6-failpoints/ + mkdir -p /tmp/etcd-release-3.6-failpoints/ + cd /tmp/etcd-release-3.6-failpoints/; \ + git clone --depth 1 --branch main https://github.com/etcd-io/etcd.git .; \ + make gofail-enable; \ + make build; + /tmp/etcd-v3.5.2-failpoints/bin: /tmp/etcd-v3.5.4-failpoints/bin: /tmp/etcd-v3.5.5-failpoints/bin: From b54d7552a7223746f340eb57731b48ecad2f0597 Mon Sep 17 00:00:00 2001 From: Siyuan Zhang Date: Wed, 15 May 2024 22:22:06 +0000 Subject: [PATCH 2/3] robustness: add mix version option in exploratoryScenarios. Signed-off-by: Siyuan Zhang --- tests/e2e/etcd_mix_versions_test.go | 13 +++--- tests/framework/e2e/cluster.go | 42 +++++++++---------- tests/framework/e2e/cluster_test.go | 7 ---- tests/framework/e2e/etcd_process.go | 15 +++++++ tests/robustness/failpoint/cluster.go | 5 ++- tests/robustness/failpoint/network.go | 2 +- .../options/server_config_options.go | 4 ++ tests/robustness/scenarios.go | 24 +++++------ 8 files changed, 63 insertions(+), 49 deletions(-) diff --git a/tests/e2e/etcd_mix_versions_test.go b/tests/e2e/etcd_mix_versions_test.go index aed2563898e..f4c297b2f40 100644 --- a/tests/e2e/etcd_mix_versions_test.go +++ b/tests/e2e/etcd_mix_versions_test.go @@ -130,12 +130,13 @@ func mixVersionsSnapshotTestByMockPartition(t *testing.T, cfg *e2e.EtcdProcessCl t.Skipf("%q does not exist", e2e.BinPath.EtcdLastRelease) } + clusterOptions := []e2e.EPClusterOption{e2e.WithConfig(cfg), e2e.WithSnapshotCount(10)} + // TODO: remove version check after 3.5.14 release. + if cfg.Version == e2e.CurrentVersion { + clusterOptions = append(clusterOptions, e2e.WithSnapshotCatchUpEntries(10)) + } t.Logf("Create an etcd cluster with %d member", cfg.ClusterSize) - epc, err := e2e.NewEtcdProcessCluster(context.TODO(), t, - e2e.WithConfig(cfg), - e2e.WithSnapshotCount(10), - e2e.WithSnapshotCatchUpEntries(10), - ) + epc, err := e2e.NewEtcdProcessCluster(context.TODO(), t, clusterOptions...) require.NoError(t, err, "failed to start etcd cluster: %v", err) defer func() { derr := epc.Close() @@ -161,7 +162,7 @@ func mixVersionsSnapshotTestByMockPartition(t *testing.T, cfg *e2e.EtcdProcessCl assertKVHash(t, epc) leaderEPC = epc.Procs[epc.WaitLeader(t)] - if leaderEPC.Config().ExecPath == e2e.BinPath.Etcd { + if cfg.Version == e2e.CurrentVersion { t.Log("Verify logs to check snapshot be sent from leader to follower") e2e.AssertProcessLogs(t, leaderEPC, "sent database snapshot") } diff --git a/tests/framework/e2e/cluster.go b/tests/framework/e2e/cluster.go index 8f3a102c305..e9f4b6388fe 100644 --- a/tests/framework/e2e/cluster.go +++ b/tests/framework/e2e/cluster.go @@ -570,27 +570,6 @@ func (cfg *EtcdProcessClusterConfig) EtcdServerProcessConfig(tb testing.TB, i in args = append(args, "--discovery="+cfg.Discovery) } - defaultValues := values(*embed.NewConfig()) - overrideValues := values(cfg.ServerConfig) - for flag, value := range overrideValues { - if defaultValue := defaultValues[flag]; value == "" || value == defaultValue { - continue - } - if flag == "experimental-snapshot-catchup-entries" && !(cfg.Version == CurrentVersion || (cfg.Version == MinorityLastVersion && i <= cfg.ClusterSize/2) || (cfg.Version == QuorumLastVersion && i > cfg.ClusterSize/2)) { - continue - } - args = append(args, fmt.Sprintf("--%s=%s", flag, value)) - } - envVars := map[string]string{} - for key, value := range cfg.EnvVars { - envVars[key] = value - } - var gofailPort int - if cfg.GoFailEnabled { - gofailPort = (i+1)*10000 + 2381 - envVars["GOFAIL_HTTP"] = fmt.Sprintf("127.0.0.1:%d", gofailPort) - } - var execPath string switch cfg.Version { case CurrentVersion: @@ -613,6 +592,27 @@ func (cfg *EtcdProcessClusterConfig) EtcdServerProcessConfig(tb testing.TB, i in panic(fmt.Sprintf("Unknown cluster version %v", cfg.Version)) } + defaultValues := values(*embed.NewConfig()) + overrideValues := values(cfg.ServerConfig) + for flag, value := range overrideValues { + if defaultValue := defaultValues[flag]; value == "" || value == defaultValue { + continue + } + if flag == "experimental-snapshot-catchup-entries" && !CouldSetSnapshotCatchupEntries(execPath) { + continue + } + args = append(args, fmt.Sprintf("--%s=%s", flag, value)) + } + envVars := map[string]string{} + for key, value := range cfg.EnvVars { + envVars[key] = value + } + var gofailPort int + if cfg.GoFailEnabled { + gofailPort = (i+1)*10000 + 2381 + envVars["GOFAIL_HTTP"] = fmt.Sprintf("127.0.0.1:%d", gofailPort) + } + return &EtcdServerProcessConfig{ lg: cfg.Logger, ExecPath: execPath, diff --git a/tests/framework/e2e/cluster_test.go b/tests/framework/e2e/cluster_test.go index fea89c4e594..a711cdef180 100644 --- a/tests/framework/e2e/cluster_test.go +++ b/tests/framework/e2e/cluster_test.go @@ -74,13 +74,6 @@ func TestEtcdServerProcessConfig(t *testing.T) { "--experimental-snapshot-catchup-entries=100", }, }, - { - name: "CatchUpEntriesLastVersion", - config: NewConfig(WithSnapshotCatchUpEntries(100), WithVersion(LastVersion)), - expectArgsNotContain: []string{ - "--experimental-snapshot-catchup-entries=100", - }, - }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { diff --git a/tests/framework/e2e/etcd_process.go b/tests/framework/e2e/etcd_process.go index f9d2089a3e3..170828a4b6a 100644 --- a/tests/framework/e2e/etcd_process.go +++ b/tests/framework/e2e/etcd_process.go @@ -508,3 +508,18 @@ func GetVersionFromBinary(binaryPath string) (*semver.Version, error) { return nil, fmt.Errorf("could not find version in binary output of %s, lines outputted were %v", binaryPath, lines) } + +func CouldSetSnapshotCatchupEntries(execPath string) bool { + if !fileutil.Exist(execPath) { + // default to true if the binary does not exist, because binary might not exist for unit test, + // which does not really matter because "experimental-snapshot-catchup-entries" can be set for v3.6 and v3.5. + return true + } + v, err := GetVersionFromBinary(execPath) + if err != nil { + panic(err) + } + // snapshot-catchup-entries flag was backported in https://github.com/etcd-io/etcd/pull/17808 + v3_5_13 := semver.Version{Major: 3, Minor: 5, Patch: 13} + return v.Compare(v3_5_13) >= 0 +} diff --git a/tests/robustness/failpoint/cluster.go b/tests/robustness/failpoint/cluster.go index 2a68fcb73ea..502b866236b 100644 --- a/tests/robustness/failpoint/cluster.go +++ b/tests/robustness/failpoint/cluster.go @@ -135,8 +135,9 @@ func (f memberReplace) Name() string { return "MemberReplace" } -func (f memberReplace) Available(config e2e.EtcdProcessClusterConfig, _ e2e.EtcdProcess) bool { - return config.ClusterSize > 1 +func (f memberReplace) Available(config e2e.EtcdProcessClusterConfig, member e2e.EtcdProcess) bool { + // a lower etcd version may not be able to join a cluster with higher cluster version. + return config.ClusterSize > 1 && member.Config().ExecPath != e2e.BinPath.EtcdLastRelease } func getID(ctx context.Context, cc clientv3.Cluster, name string) (id uint64, found bool, err error) { diff --git a/tests/robustness/failpoint/network.go b/tests/robustness/failpoint/network.go index d202454bcdf..b355b5182bc 100644 --- a/tests/robustness/failpoint/network.go +++ b/tests/robustness/failpoint/network.go @@ -59,7 +59,7 @@ func (tb triggerBlackhole) Trigger(ctx context.Context, t *testing.T, member e2e func (tb triggerBlackhole) Available(config e2e.EtcdProcessClusterConfig, process e2e.EtcdProcess) bool { // Avoid triggering failpoint if waiting for failpoint would take too long to fit into timeout. // Number of required entries for snapshot depends on etcd configuration. - if tb.waitTillSnapshot && entriesToGuaranteeSnapshot(config) > 200 { + if tb.waitTillSnapshot && (entriesToGuaranteeSnapshot(config) > 200 || !e2e.CouldSetSnapshotCatchupEntries(process.Config().ExecPath)) { return false } return config.ClusterSize > 1 && process.PeerProxy() != nil diff --git a/tests/robustness/options/server_config_options.go b/tests/robustness/options/server_config_options.go index 5018d1bf90a..4471a869fd0 100644 --- a/tests/robustness/options/server_config_options.go +++ b/tests/robustness/options/server_config_options.go @@ -49,3 +49,7 @@ func WithExperimentalWatchProgressNotifyInterval(input ...time.Duration) e2e.EPC c.ServerConfig.ExperimentalWatchProgressNotifyInterval = input[internalRand.Intn(len(input))] } } + +func WithVersion(input ...e2e.ClusterVersion) e2e.EPClusterOption { + return func(c *e2e.EtcdProcessClusterConfig) { c.Version = input[internalRand.Intn(len(input))] } +} diff --git a/tests/robustness/scenarios.go b/tests/robustness/scenarios.go index 7cd121b6e39..a4451c5f85a 100644 --- a/tests/robustness/scenarios.go +++ b/tests/robustness/scenarios.go @@ -19,9 +19,8 @@ import ( "testing" "time" - "github.com/coreos/go-semver/semver" - "go.etcd.io/etcd/api/v3/version" + "go.etcd.io/etcd/client/pkg/v3/fileutil" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/robustness/failpoint" "go.etcd.io/etcd/tests/v3/robustness/options" @@ -61,11 +60,7 @@ type testScenario struct { watch watchConfig } -func exploratoryScenarios(t *testing.T) []testScenario { - v, err := e2e.GetVersionFromBinary(e2e.BinPath.Etcd) - if err != nil { - t.Fatalf("Failed checking etcd version binary, binary: %q, err: %v", e2e.BinPath.Etcd, err) - } +func exploratoryScenarios(_ *testing.T) []testScenario { enableLazyFS := e2e.BinPath.LazyFSAvailable() randomizableOptions := []e2e.EPClusterOption{ options.WithClusterOptionGroups( @@ -74,6 +69,10 @@ func exploratoryScenarios(t *testing.T) []testScenario { options.ClusterOptions{options.WithTickMs(100), options.WithElectionMs(2000)}), } + // 66% current version, 33% MinorityLastVersion and QuorumLastVersion + mixedVersionOption := options.WithVersion(e2e.CurrentVersion, e2e.CurrentVersion, e2e.CurrentVersion, + e2e.CurrentVersion, e2e.MinorityLastVersion, e2e.QuorumLastVersion) + baseOptions := []e2e.EPClusterOption{ options.WithSnapshotCount(50, 100, 1000), options.WithSubsetOptions(randomizableOptions...), @@ -81,9 +80,8 @@ func exploratoryScenarios(t *testing.T) []testScenario { e2e.WithCompactionBatchLimit(100), e2e.WithWatchProcessNotifyInterval(100 * time.Millisecond), } - // snapshot-catchup-entries flag was backported in https://github.com/etcd-io/etcd/pull/17808 - v3_5_13 := semver.Version{Major: 3, Minor: 5, Patch: 13} - if v.Compare(v3_5_13) >= 0 { + + if e2e.CouldSetSnapshotCatchupEntries(e2e.BinPath.Etcd) { baseOptions = append(baseOptions, e2e.WithSnapshotCatchUpEntries(100)) } scenarios := []testScenario{} @@ -109,6 +107,9 @@ func exploratoryScenarios(t *testing.T) []testScenario { clusterOfSize3Options := baseOptions clusterOfSize3Options = append(clusterOfSize3Options, e2e.WithIsPeerTLS(true)) clusterOfSize3Options = append(clusterOfSize3Options, e2e.WithPeerProxy(true)) + if fileutil.Exist(e2e.BinPath.EtcdLastRelease) { + clusterOfSize3Options = append(clusterOfSize3Options, mixedVersionOption) + } scenarios = append(scenarios, testScenario{ name: name, traffic: tp.Traffic, @@ -172,8 +173,7 @@ func regressionScenarios(t *testing.T) []testScenario { e2e.WithPeerProxy(true), e2e.WithIsPeerTLS(true), } - v3_5_13 := semver.Version{Major: 3, Minor: 5, Patch: 13} - if v.Compare(v3_5_13) >= 0 { + if e2e.CouldSetSnapshotCatchupEntries(e2e.BinPath.Etcd) { opts = append(opts, e2e.WithSnapshotCatchUpEntries(100)) } scenarios = append(scenarios, testScenario{ From 0f94c2ca4fd4fb8f1e650b592c7f5b61a414449d Mon Sep 17 00:00:00 2001 From: Siyuan Zhang Date: Thu, 16 May 2024 22:14:39 +0000 Subject: [PATCH 3/3] robustness: add mix version scenario with fixed leader. Signed-off-by: Siyuan Zhang --- tests/framework/e2e/cluster.go | 63 ++++++++++++++++--- tests/framework/e2e/cluster_test.go | 32 ++++++++++ tests/framework/e2e/etcd_process.go | 21 ++++--- tests/framework/e2e/etcdctl.go | 7 +++ tests/robustness/failpoint/cluster.go | 2 +- .../options/server_config_options.go | 4 ++ tests/robustness/scenarios.go | 20 +++++- 7 files changed, 131 insertions(+), 18 deletions(-) diff --git a/tests/framework/e2e/cluster.go b/tests/framework/e2e/cluster.go index e9f4b6388fe..cb8b35d7fd8 100644 --- a/tests/framework/e2e/cluster.go +++ b/tests/framework/e2e/cluster.go @@ -151,8 +151,11 @@ type EtcdProcessClusterConfig struct { // Cluster setup config - ClusterSize int - RollingStart bool + ClusterSize int + // InitialLeaderIndex makes sure the leader is the ith proc + // when the cluster starts if it is specified (>=0). + InitialLeaderIndex int + RollingStart bool // BaseDataDirPath specifies the data-dir for the members. If test cases // do not specify `BaseDataDirPath`, then e2e framework creates a // temporary directory for each member; otherwise, it creates a @@ -180,10 +183,10 @@ type EtcdProcessClusterConfig struct { func DefaultConfig() *EtcdProcessClusterConfig { cfg := &EtcdProcessClusterConfig{ - ClusterSize: 3, - CN: true, - - ServerConfig: *embed.NewConfig(), + ClusterSize: 3, + CN: true, + InitialLeaderIndex: -1, + ServerConfig: *embed.NewConfig(), } cfg.ServerConfig.InitialClusterToken = "new" return cfg @@ -207,6 +210,10 @@ func WithVersion(version ClusterVersion) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.Version = version } } +func WithInitialLeaderIndex(i int) EPClusterOption { + return func(c *EtcdProcessClusterConfig) { c.InitialLeaderIndex = i } +} + func WithDataDirPath(path string) EPClusterOption { return func(c *EtcdProcessClusterConfig) { c.BaseDataDirPath = path } } @@ -398,6 +405,16 @@ func InitEtcdProcessCluster(t testing.TB, cfg *EtcdProcessClusterConfig) (*EtcdP cfg.ServerConfig.SnapshotCount = etcdserver.DefaultSnapshotCount } + // validate SnapshotCatchUpEntries could be set for at least one member + if cfg.ServerConfig.SnapshotCatchUpEntries != etcdserver.DefaultSnapshotCatchUpEntries { + if !CouldSetSnapshotCatchupEntries(BinPath.Etcd) { + return nil, fmt.Errorf("cannot set SnapshotCatchUpEntries for current etcd version: %s", BinPath.Etcd) + } + if cfg.Version == LastVersion && !CouldSetSnapshotCatchupEntries(BinPath.EtcdLastRelease) { + return nil, fmt.Errorf("cannot set SnapshotCatchUpEntries for last etcd version: %s", BinPath.EtcdLastRelease) + } + } + etcdCfgs := cfg.EtcdAllServerProcessConfigs(t) epc := &EtcdProcessCluster{ Cfg: cfg, @@ -437,7 +454,11 @@ func StartEtcdProcessCluster(ctx context.Context, t testing.TB, epc *EtcdProcess t.Skip("please run 'make gofail-enable && make build' before running the test") } } - + if cfg.InitialLeaderIndex >= 0 { + if err := epc.MoveLeader(ctx, t, cfg.InitialLeaderIndex); err != nil { + return nil, fmt.Errorf("failed to move leader: %v", err) + } + } return epc, nil } @@ -1050,3 +1071,31 @@ func (epc *EtcdProcessCluster) WaitMembersForLeader(ctx context.Context, t testi t.Fatal("impossible path of execution") return -1 } + +// MoveLeader moves the leader to the ith process. +func (epc *EtcdProcessCluster) MoveLeader(ctx context.Context, t testing.TB, i int) error { + if i < 0 || i >= len(epc.Procs) { + return fmt.Errorf("invalid index: %d, must between 0 and %d", i, len(epc.Procs)-1) + } + t.Logf("moving leader to Procs[%d]", i) + oldLeader := epc.WaitMembersForLeader(ctx, t, epc.Procs) + if oldLeader == i { + t.Logf("Procs[%d] is already the leader", i) + return nil + } + resp, err := epc.Procs[i].Etcdctl().Status(ctx) + if err != nil { + return err + } + memberID := resp[0].Header.MemberId + err = epc.Procs[oldLeader].Etcdctl().MoveLeader(ctx, memberID) + if err != nil { + return err + } + newLeader := epc.WaitMembersForLeader(ctx, t, epc.Procs) + if newLeader != i { + t.Fatalf("expect new leader to be Procs[%d] but got Procs[%d]", i, newLeader) + } + t.Logf("moved leader from Procs[%d] to Procs[%d]", oldLeader, i) + return nil +} diff --git a/tests/framework/e2e/cluster_test.go b/tests/framework/e2e/cluster_test.go index a711cdef180..bf6c9361677 100644 --- a/tests/framework/e2e/cluster_test.go +++ b/tests/framework/e2e/cluster_test.go @@ -15,17 +15,22 @@ package e2e import ( + "fmt" "testing" + "github.com/coreos/go-semver/semver" "github.com/stretchr/testify/assert" ) func TestEtcdServerProcessConfig(t *testing.T) { + v3_5_12 := semver.Version{Major: 3, Minor: 5, Patch: 12} + v3_5_13 := semver.Version{Major: 3, Minor: 5, Patch: 13} tcs := []struct { name string config *EtcdProcessClusterConfig expectArgsNotContain []string expectArgsContain []string + mockBinaryVersion *semver.Version }{ { name: "Default", @@ -73,10 +78,37 @@ func TestEtcdServerProcessConfig(t *testing.T) { expectArgsContain: []string{ "--experimental-snapshot-catchup-entries=100", }, + mockBinaryVersion: &v3_5_13, + }, + { + name: "CatchUpEntriesNoVersion", + config: NewConfig(WithSnapshotCatchUpEntries(100), WithVersion(LastVersion)), + expectArgsNotContain: []string{ + "--experimental-snapshot-catchup-entries=100", + }, + }, + { + name: "CatchUpEntriesOldVersion", + config: NewConfig(WithSnapshotCatchUpEntries(100), WithVersion(LastVersion)), + expectArgsNotContain: []string{ + "--experimental-snapshot-catchup-entries=100", + }, + mockBinaryVersion: &v3_5_12, }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { + var mockGetVersionFromBinary func(binaryPath string) (*semver.Version, error) + if tc.mockBinaryVersion == nil { + mockGetVersionFromBinary = func(binaryPath string) (*semver.Version, error) { + return nil, fmt.Errorf("could not get binary version") + } + } else { + mockGetVersionFromBinary = func(binaryPath string) (*semver.Version, error) { + return tc.mockBinaryVersion, nil + } + } + setGetVersionFromBinary(t, mockGetVersionFromBinary) args := tc.config.EtcdServerProcessConfig(t, 0).Args if len(tc.expectArgsContain) != 0 { assert.Subset(t, args, tc.expectArgsContain) diff --git a/tests/framework/e2e/etcd_process.go b/tests/framework/e2e/etcd_process.go index 170828a4b6a..fe2a1765e02 100644 --- a/tests/framework/e2e/etcd_process.go +++ b/tests/framework/e2e/etcd_process.go @@ -485,7 +485,10 @@ func parseFailpointsBody(body io.Reader) (map[string]string, error) { return failpoints, nil } -func GetVersionFromBinary(binaryPath string) (*semver.Version, error) { +var GetVersionFromBinary = func(binaryPath string) (*semver.Version, error) { + if !fileutil.Exist(binaryPath) { + return nil, fmt.Errorf("binary path does not exist: %s", binaryPath) + } lines, err := RunUtilCompletion([]string{binaryPath, "--version"}, nil) if err != nil { return nil, fmt.Errorf("could not find binary version from %s, err: %w", binaryPath, err) @@ -509,15 +512,19 @@ func GetVersionFromBinary(binaryPath string) (*semver.Version, error) { return nil, fmt.Errorf("could not find version in binary output of %s, lines outputted were %v", binaryPath, lines) } +// setGetVersionFromBinary changes the GetVersionFromBinary function to a mock in testing. +func setGetVersionFromBinary(tb testing.TB, f func(binaryPath string) (*semver.Version, error)) { + origGetVersionFromBinary := GetVersionFromBinary + GetVersionFromBinary = f + tb.Cleanup(func() { + GetVersionFromBinary = origGetVersionFromBinary + }) +} + func CouldSetSnapshotCatchupEntries(execPath string) bool { - if !fileutil.Exist(execPath) { - // default to true if the binary does not exist, because binary might not exist for unit test, - // which does not really matter because "experimental-snapshot-catchup-entries" can be set for v3.6 and v3.5. - return true - } v, err := GetVersionFromBinary(execPath) if err != nil { - panic(err) + return false } // snapshot-catchup-entries flag was backported in https://github.com/etcd-io/etcd/pull/17808 v3_5_13 := semver.Version{Major: 3, Minor: 5, Patch: 13} diff --git a/tests/framework/e2e/etcdctl.go b/tests/framework/e2e/etcdctl.go index 0d6bb3a5dcd..81d57c088d5 100644 --- a/tests/framework/e2e/etcdctl.go +++ b/tests/framework/e2e/etcdctl.go @@ -312,6 +312,13 @@ func (ctl *EtcdctlV3) MemberPromote(ctx context.Context, id uint64) (*clientv3.M return &resp, err } +// MoveLeader requests current leader to transfer its leadership to the transferee. +// Request must be made to the leader. +func (ctl *EtcdctlV3) MoveLeader(ctx context.Context, transfereeID uint64) error { + _, err := SpawnWithExpectLines(ctx, ctl.cmdArgs("move-leader", fmt.Sprintf("%x", transfereeID)), nil, expect.ExpectedResponse{Value: "Leadership transferred"}) + return err +} + func (ctl *EtcdctlV3) cmdArgs(args ...string) []string { cmdArgs := []string{BinPath.Etcdctl} for k, v := range ctl.flags() { diff --git a/tests/robustness/failpoint/cluster.go b/tests/robustness/failpoint/cluster.go index 502b866236b..1cc2116e9da 100644 --- a/tests/robustness/failpoint/cluster.go +++ b/tests/robustness/failpoint/cluster.go @@ -137,7 +137,7 @@ func (f memberReplace) Name() string { func (f memberReplace) Available(config e2e.EtcdProcessClusterConfig, member e2e.EtcdProcess) bool { // a lower etcd version may not be able to join a cluster with higher cluster version. - return config.ClusterSize > 1 && member.Config().ExecPath != e2e.BinPath.EtcdLastRelease + return config.ClusterSize > 1 && (config.Version == e2e.QuorumLastVersion || member.Config().ExecPath == e2e.BinPath.Etcd) } func getID(ctx context.Context, cc clientv3.Cluster, name string) (id uint64, found bool, err error) { diff --git a/tests/robustness/options/server_config_options.go b/tests/robustness/options/server_config_options.go index 4471a869fd0..e955784bf62 100644 --- a/tests/robustness/options/server_config_options.go +++ b/tests/robustness/options/server_config_options.go @@ -53,3 +53,7 @@ func WithExperimentalWatchProgressNotifyInterval(input ...time.Duration) e2e.EPC func WithVersion(input ...e2e.ClusterVersion) e2e.EPClusterOption { return func(c *e2e.EtcdProcessClusterConfig) { c.Version = input[internalRand.Intn(len(input))] } } + +func WithInitialLeaderIndex(input ...int) e2e.EPClusterOption { + return func(c *e2e.EtcdProcessClusterConfig) { c.InitialLeaderIndex = input[internalRand.Intn(len(input))] } +} diff --git a/tests/robustness/scenarios.go b/tests/robustness/scenarios.go index a4451c5f85a..cca05dea49c 100644 --- a/tests/robustness/scenarios.go +++ b/tests/robustness/scenarios.go @@ -69,9 +69,23 @@ func exploratoryScenarios(_ *testing.T) []testScenario { options.ClusterOptions{options.WithTickMs(100), options.WithElectionMs(2000)}), } - // 66% current version, 33% MinorityLastVersion and QuorumLastVersion - mixedVersionOption := options.WithVersion(e2e.CurrentVersion, e2e.CurrentVersion, e2e.CurrentVersion, - e2e.CurrentVersion, e2e.MinorityLastVersion, e2e.QuorumLastVersion) + mixedVersionOption := options.WithClusterOptionGroups( + // 60% with all members of current version + options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)}, + options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)}, + options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)}, + options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)}, + options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)}, + options.ClusterOptions{options.WithVersion(e2e.CurrentVersion)}, + // 10% with 2 members of current version, 1 member last version, leader is current version + options.ClusterOptions{options.WithVersion(e2e.MinorityLastVersion), options.WithInitialLeaderIndex(0)}, + // 10% with 2 members of current version, 1 member last version, leader is last version + options.ClusterOptions{options.WithVersion(e2e.MinorityLastVersion), options.WithInitialLeaderIndex(2)}, + // 10% with 2 members of last version, 1 member current version, leader is last version + options.ClusterOptions{options.WithVersion(e2e.QuorumLastVersion), options.WithInitialLeaderIndex(0)}, + // 10% with 2 members of last version, 1 member current version, leader is current version + options.ClusterOptions{options.WithVersion(e2e.QuorumLastVersion), options.WithInitialLeaderIndex(2)}, + ) baseOptions := []e2e.EPClusterOption{ options.WithSnapshotCount(50, 100, 1000),