From 4d360294c0955a343f87d57e4636c7245ae6b12d Mon Sep 17 00:00:00 2001 From: Wojciech Tyczynski Date: Tue, 28 Feb 2017 17:43:56 +0100 Subject: [PATCH] Etcd upgrade test --- hack/verify-flags/known-flags.txt | 2 ++ test/e2e/cluster_upgrade.go | 18 ++++++++++++++++++ test/e2e/framework/nodes_util.go | 21 +++++++++++++++++++++ test/e2e/framework/test_context.go | 4 ++++ test/e2e/framework/util.go | 8 ++++++++ 5 files changed, 53 insertions(+) diff --git a/hack/verify-flags/known-flags.txt b/hack/verify-flags/known-flags.txt index 9eb83c5898e0..8a898da69a0f 100644 --- a/hack/verify-flags/known-flags.txt +++ b/hack/verify-flags/known-flags.txt @@ -178,6 +178,8 @@ etcd-quorum-read etcd-server etcd-servers etcd-servers-overrides +etcd-upgrade-storage +etcd-upgrade-version event-burst event-qps event-ttl diff --git a/test/e2e/cluster_upgrade.go b/test/e2e/cluster_upgrade.go index 8fd43d5b51a3..ec5a854c57f8 100644 --- a/test/e2e/cluster_upgrade.go +++ b/test/e2e/cluster_upgrade.go @@ -118,6 +118,24 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() { }) }) +var _ = framework.KubeDescribe("etcd uUpgrade [Feature:EtcdUpgrade]", func() { + f := framework.NewDefaultFramework("etcd-upgrade") + + framework.KubeDescribe("etcd upgrade", func() { + It("should maintain a functioning cluster", func() { + cm := chaosmonkey.New(func() { + framework.ExpectNoError(framework.EtcdUpgrade(framework.TestContext.EtcdUpgradeStorage, framework.TestContext.EtcdUpgradeVersion)) + // TODO(mml): verify etcd version + }) + cm.Register(func(sem *chaosmonkey.Semaphore) { + // Close over f. + testServiceRemainsUp(f, sem) + }) + cm.Do() + }) + }) +}) + // realVersion turns a version constant s into a version string deployable on // GKE. See hack/get-build.sh for more information. func realVersion(s string) (string, error) { diff --git a/test/e2e/framework/nodes_util.go b/test/e2e/framework/nodes_util.go index 7ff5e96ef06b..4bb05695ba37 100644 --- a/test/e2e/framework/nodes_util.go +++ b/test/e2e/framework/nodes_util.go @@ -18,6 +18,7 @@ package framework import ( "fmt" + "os" "path" "strings" "time" @@ -28,6 +29,15 @@ import ( "k8s.io/kubernetes/pkg/util/wait" ) +func EtcdUpgrade(targetStorage, targetVersion string) error { + switch TestContext.Provider { + case "gce": + return etcdUpgradeGCE(targetStorage, targetVersion) + default: + return fmt.Errorf("EtcdUpgrade() is not implemented for provider %s", TestContext.Provider) + } +} + // The following upgrade functions are passed into the framework below and used // to do the actual upgrades. var MasterUpgrade = func(v string) error { @@ -41,6 +51,17 @@ var MasterUpgrade = func(v string) error { } } +func etcdUpgradeGCE(targetStorage, targetVersion string) error { + env := append( + os.Environ(), + "TEST_ETCD_VERSION="+targetVersion, + "STORAGE_BACKEND="+targetStorage, + "TEST_ETCD_IMAGE=3.0.17") + + _, _, err := RunCmdEnv(env, path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh"), "-l", "-M") + return err +} + func masterUpgradeGCE(rawV string) error { v := "v" + rawV _, _, err := RunCmd(path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh"), "-M", v) diff --git a/test/e2e/framework/test_context.go b/test/e2e/framework/test_context.go index 22a5a576fd9a..bb97b333aa09 100644 --- a/test/e2e/framework/test_context.go +++ b/test/e2e/framework/test_context.go @@ -49,6 +49,8 @@ type TestContextType struct { // Timeout for waiting for system pods to be running SystemPodsStartupTimeout time.Duration UpgradeTarget string + EtcdUpgradeStorage string + EtcdUpgradeVersion string UpgradeImage string PrometheusPushGateway string ContainerRuntime string @@ -188,6 +190,8 @@ func RegisterClusterFlags() { flag.IntVar(&TestContext.MinStartupPods, "minStartupPods", 0, "The number of pods which we need to see in 'Running' state with a 'Ready' condition of true, before we try running tests. This is useful in any cluster which needs some base pod-based services running before it can be used.") flag.DurationVar(&TestContext.SystemPodsStartupTimeout, "system-pods-startup-timeout", 10*time.Minute, "Timeout for waiting for all system pods to be running before starting tests.") flag.StringVar(&TestContext.UpgradeTarget, "upgrade-target", "ci/latest", "Version to upgrade to (e.g. 'release/stable', 'release/latest', 'ci/latest', '0.19.1', '0.19.1-669-gabac8c8') if doing an upgrade test.") + flag.StringVar(&TestContext.EtcdUpgradeStorage, "etcd-upgrade-storage", "", "The storage version to upgrade to (either 'etcdv2' or 'etcdv3') if doing an etcd upgrade test.") + flag.StringVar(&TestContext.EtcdUpgradeVersion, "etcd-upgrade-version", "", "The etcd binary version to upgrade to (e.g., '3.0.14', '2.3.7') if doing an etcd upgrade test.") flag.StringVar(&TestContext.UpgradeImage, "upgrade-image", "", "Image to upgrade to (e.g. 'container_vm' or 'gci') if doing an upgrade test.") flag.StringVar(&TestContext.PrometheusPushGateway, "prom-push-gateway", "", "The URL to prometheus gateway, so that metrics can be pushed during e2es and scraped by prometheus. Typically something like 127.0.0.1:9091.") flag.BoolVar(&TestContext.CleanStart, "clean-start", false, "If true, purge all namespaces except default and system before running tests. This serves to Cleanup test namespaces from failed/interrupted e2e runs in a long-lived cluster.") diff --git a/test/e2e/framework/util.go b/test/e2e/framework/util.go index 62b8918f363d..7ec5c96e3424 100644 --- a/test/e2e/framework/util.go +++ b/test/e2e/framework/util.go @@ -4639,6 +4639,13 @@ func GetPodsInNamespace(c clientset.Interface, ns string, ignoreLabels map[strin // RunCmd runs cmd using args and returns its stdout and stderr. It also outputs // cmd's stdout and stderr to their respective OS streams. func RunCmd(command string, args ...string) (string, string, error) { + return RunCmdEnv(nil, command, args...) +} + +// RunCmdEnv runs cmd with the provided environment and args and +// returns its stdout and stderr. It also outputs cmd's stdout and +// stderr to their respective OS streams. +func RunCmdEnv(env []string, command string, args ...string) (string, string, error) { Logf("Running %s %v", command, args) var bout, berr bytes.Buffer cmd := exec.Command(command, args...) @@ -4649,6 +4656,7 @@ func RunCmd(command string, args ...string) (string, string, error) { // newlines. cmd.Stdout = io.MultiWriter(os.Stdout, &bout) cmd.Stderr = io.MultiWriter(os.Stderr, &berr) + cmd.Env = env err := cmd.Run() stdout, stderr := bout.String(), berr.String() if err != nil {