Write reports for each upgrade test

hack/.linted_packages

@@ -456,6 +456,7 @@ staging/src/k8s.io/metrics/pkg/apis/custom_metrics/install
 staging/src/k8s.io/metrics/pkg/apis/metrics/install
 staging/src/k8s.io/sample-apiserver
 staging/src/k8s.io/sample-apiserver/pkg/apis/wardle/install
+test/e2e/framework/ginkgowrapper
 test/e2e/perftype
 test/e2e_node/runner/local
 test/images/clusterapi-tester
@@ -493,3 +494,4 @@ test/integration/thirdparty
 test/integration/ttlcontroller
 test/soak/cauldron
 test/soak/serve_hostnames
+test/utils/junit

test/e2e/BUILD

@@ -164,13 +164,15 @@ go_library(
         "//test/e2e/chaosmonkey:go_default_library",
         "//test/e2e/common:go_default_library",
         "//test/e2e/framework:go_default_library",
+        "//test/e2e/framework/ginkgowrapper:go_default_library",
         "//test/e2e/generated:go_default_library",
         "//test/e2e/perf:go_default_library",
         "//test/e2e/scheduling:go_default_library",
         "//test/e2e/upgrades:go_default_library",
         "//test/e2e_federation:go_default_library",
         "//test/images/net/nat:go_default_library",
         "//test/utils:go_default_library",
+        "//test/utils/junit:go_default_library",
         "//vendor/github.com/davecgh/go-spew/spew:go_default_library",
         "//vendor/github.com/elazarl/goproxy:go_default_library",
         "//vendor/github.com/ghodss/yaml:go_default_library",

test/e2e/cluster_upgrade.go

@@ -17,11 +17,20 @@ limitations under the License.
 package e2e
 
 import (
+	"encoding/xml"
+	"fmt"
+	"os"
+	"path/filepath"
+	"sync"
+	"time"
+
 	"k8s.io/client-go/discovery"
 	"k8s.io/kubernetes/pkg/util/version"
 	"k8s.io/kubernetes/test/e2e/chaosmonkey"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/ginkgowrapper"
 	"k8s.io/kubernetes/test/e2e/upgrades"
+	"k8s.io/kubernetes/test/utils/junit"
 
 	. "github.com/onsi/ginkgo"
 )
@@ -45,32 +54,24 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
 
 	// Create the frameworks here because we can only create them
 	// in a "Describe".
-	testFrameworks := map[string]*framework.Framework{}
-	for _, t := range upgradeTests {
-		testFrameworks[t.Name()] = framework.NewDefaultFramework(t.Name())
-	}
-
+	testFrameworks := createUpgradeFrameworks()
 	framework.KubeDescribe("master upgrade", func() {
 		It("should maintain a functioning cluster [Feature:MasterUpgrade]", func() {
 			upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget)
 			framework.ExpectNoError(err)
 
-			cm := chaosmonkey.New(func() {
+			testSuite := &junit.TestSuite{Name: "Master upgrade"}
+			masterUpgradeTest := &junit.TestCase{Name: "master-upgrade", Classname: "upgrade_tests"}
+			testSuite.TestCases = append(testSuite.TestCases, masterUpgradeTest)
+
+			upgradeFunc := func() {
+				start := time.Now()
+				defer finalizeUpgradeTest(start, masterUpgradeTest)
 				target := upgCtx.Versions[1].Version.String()
 				framework.ExpectNoError(framework.MasterUpgrade(target))
 				framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, target))
-			})
-			for _, t := range upgradeTests {
-				cma := chaosMonkeyAdapter{
-					test:        t,
-					framework:   testFrameworks[t.Name()],
-					upgradeType: upgrades.MasterUpgrade,
-					upgCtx:      *upgCtx,
-				}
-				cm.Register(cma.Test)
 			}
-
-			cm.Do()
+			runUpgradeSuite(f, testFrameworks, testSuite, upgCtx, upgrades.MasterUpgrade, upgradeFunc)
 		})
 	})
 
@@ -79,21 +80,17 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
 			upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget)
 			framework.ExpectNoError(err)
 
-			cm := chaosmonkey.New(func() {
+			testSuite := &junit.TestSuite{Name: "Node upgrade"}
+			nodeUpgradeTest := &junit.TestCase{Name: "node-upgrade", Classname: "upgrade_tests"}
+
+			upgradeFunc := func() {
+				start := time.Now()
+				defer finalizeUpgradeTest(start, nodeUpgradeTest)
 				target := upgCtx.Versions[1].Version.String()
 				framework.ExpectNoError(framework.NodeUpgrade(f, target, framework.TestContext.UpgradeImage))
 				framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, target))
-			})
-			for _, t := range upgradeTests {
-				cma := chaosMonkeyAdapter{
-					test:        t,
-					framework:   testFrameworks[t.Name()],
-					upgradeType: upgrades.NodeUpgrade,
-					upgCtx:      *upgCtx,
-				}
-				cm.Register(cma.Test)
 			}
-			cm.Do()
+			runUpgradeSuite(f, testFrameworks, testSuite, upgCtx, upgrades.NodeUpgrade, upgradeFunc)
 		})
 	})
 
@@ -102,23 +99,19 @@ var _ = framework.KubeDescribe("Upgrade [Feature:Upgrade]", func() {
 			upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget)
 			framework.ExpectNoError(err)
 
-			cm := chaosmonkey.New(func() {
+			testSuite := &junit.TestSuite{Name: "Cluster upgrade"}
+			clusterUpgradeTest := &junit.TestCase{Name: "cluster-upgrade", Classname: "upgrade_tests"}
+			testSuite.TestCases = append(testSuite.TestCases, clusterUpgradeTest)
+			upgradeFunc := func() {
+				start := time.Now()
+				defer finalizeUpgradeTest(start, clusterUpgradeTest)
 				target := upgCtx.Versions[1].Version.String()
 				framework.ExpectNoError(framework.MasterUpgrade(target))
 				framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, target))
 				framework.ExpectNoError(framework.NodeUpgrade(f, target, framework.TestContext.UpgradeImage))
 				framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, target))
-			})
-			for _, t := range upgradeTests {
-				cma := chaosMonkeyAdapter{
-					test:        t,
-					framework:   testFrameworks[t.Name()],
-					upgradeType: upgrades.ClusterUpgrade,
-					upgCtx:      *upgCtx,
-				}
-				cm.Register(cma.Test)
 			}
-			cm.Do()
+			runUpgradeSuite(f, testFrameworks, testSuite, upgCtx, upgrades.ClusterUpgrade, upgradeFunc)
 		})
 	})
 })
@@ -138,24 +131,21 @@ var _ = framework.KubeDescribe("Downgrade [Feature:Downgrade]", func() {
 			upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget)
 			framework.ExpectNoError(err)
 
-			cm := chaosmonkey.New(func() {
+			testSuite := &junit.TestSuite{Name: "Cluster downgrade"}
+			clusterDowngradeTest := &junit.TestCase{Name: "cluster-downgrade", Classname: "upgrade_tests"}
+			testSuite.TestCases = append(testSuite.TestCases, clusterDowngradeTest)
+
+			upgradeFunc := func() {
+				start := time.Now()
+				defer finalizeUpgradeTest(start, clusterDowngradeTest)
 				// Yes this really is a downgrade. And nodes must downgrade first.
 				target := upgCtx.Versions[1].Version.String()
 				framework.ExpectNoError(framework.NodeUpgrade(f, target, framework.TestContext.UpgradeImage))
 				framework.ExpectNoError(framework.CheckNodesVersions(f.ClientSet, target))
 				framework.ExpectNoError(framework.MasterUpgrade(target))
 				framework.ExpectNoError(framework.CheckMasterVersion(f.ClientSet, target))
-			})
-			for _, t := range upgradeTests {
-				cma := chaosMonkeyAdapter{
-					test:        t,
-					framework:   testFrameworks[t.Name()],
-					upgradeType: upgrades.ClusterUpgrade,
-					upgCtx:      *upgCtx,
-				}
-				cm.Register(cma.Test)
 			}
-			cm.Do()
+			runUpgradeSuite(f, testFrameworks, testSuite, upgCtx, upgrades.ClusterUpgrade, upgradeFunc)
 		})
 	})
 })
@@ -165,55 +155,138 @@ var _ = framework.KubeDescribe("etcd Upgrade [Feature:EtcdUpgrade]", func() {
 
 	// Create the frameworks here because we can only create them
 	// in a "Describe".
-	testFrameworks := map[string]*framework.Framework{}
-	for _, t := range upgradeTests {
-		testFrameworks[t.Name()] = framework.NewDefaultFramework(t.Name())
-	}
-
+	testFrameworks := createUpgradeFrameworks()
 	framework.KubeDescribe("etcd upgrade", func() {
 		It("should maintain a functioning cluster", func() {
 			upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), "")
 			framework.ExpectNoError(err)
 
-			cm := chaosmonkey.New(func() {
+			testSuite := &junit.TestSuite{Name: "Etcd upgrade"}
+			etcdTest := &junit.TestCase{Name: "etcd-upgrade", Classname: "upgrade_tests"}
+			testSuite.TestCases = append(testSuite.TestCases, etcdTest)
+
+			upgradeFunc := func() {
+				start := time.Now()
+				defer finalizeUpgradeTest(start, etcdTest)
 				framework.ExpectNoError(framework.EtcdUpgrade(framework.TestContext.EtcdUpgradeStorage, framework.TestContext.EtcdUpgradeVersion))
-				// TODO(mml): verify the etcd version
-			})
-			for _, t := range upgradeTests {
-				cma := chaosMonkeyAdapter{
-					test:        t,
-					framework:   testFrameworks[t.Name()],
-					upgradeType: upgrades.EtcdUpgrade,
-					upgCtx:      *upgCtx,
-				}
-				cm.Register(cma.Test)
 			}
-
-			cm.Do()
+			runUpgradeSuite(f, testFrameworks, testSuite, upgCtx, upgrades.EtcdUpgrade, upgradeFunc)
 		})
 	})
 })
 
 type chaosMonkeyAdapter struct {
 	test        upgrades.Test
+	testReport  *junit.TestCase
 	framework   *framework.Framework
 	upgradeType upgrades.UpgradeType
 	upgCtx      upgrades.UpgradeContext
 }
 
 func (cma *chaosMonkeyAdapter) Test(sem *chaosmonkey.Semaphore) {
+	start := time.Now()
+	var once sync.Once
+	ready := func() {
+		once.Do(func() {
+			sem.Ready()
+		})
+	}
+	defer finalizeUpgradeTest(start, cma.testReport)
+	defer ready()
 	if skippable, ok := cma.test.(upgrades.Skippable); ok && skippable.Skip(cma.upgCtx) {
 		By("skipping test " + cma.test.Name())
-		sem.Ready()
+		cma.testReport.Skipped = "skipping test " + cma.test.Name()
 		return
 	}
 
 	defer cma.test.Teardown(cma.framework)
 	cma.test.Setup(cma.framework)
-	sem.Ready()
+	ready()
 	cma.test.Test(cma.framework, sem.StopCh, cma.upgradeType)
 }
 
+func finalizeUpgradeTest(start time.Time, tc *junit.TestCase) {
+	tc.Time = time.Since(start).Seconds()
+	r := recover()
+	if r == nil {
+		return
+	}
+
+	switch r := r.(type) {
+	case ginkgowrapper.FailurePanic:
+		tc.Failures = []*junit.Failure{
+			{
+				Message: r.Message,
+				Type:    "Failure",
+				Value:   fmt.Sprintf("%s\n\n%s", r.Message, r.FullStackTrace),
+			},
+		}
+	case ginkgowrapper.SkipPanic:
+		tc.Skipped = fmt.Sprintf("%s:%d %q", r.Filename, r.Line, r.Message)
+	default:
+		tc.Errors = []*junit.Error{
+			{
+				Message: fmt.Sprintf("%v", r),
+				Type:    "Panic",
+				Value:   fmt.Sprintf("%v", r),
+			},
+		}
+	}
+}
+
+func createUpgradeFrameworks() map[string]*framework.Framework {
+	testFrameworks := map[string]*framework.Framework{}
+	for _, t := range upgradeTests {
+		testFrameworks[t.Name()] = framework.NewDefaultFramework(t.Name())
+	}
+	return testFrameworks
+}
+
+func runUpgradeSuite(
+	f *framework.Framework,
+	testFrameworks map[string]*framework.Framework,
+	testSuite *junit.TestSuite,
+	upgCtx *upgrades.UpgradeContext,
+	upgradeType upgrades.UpgradeType,
+	upgradeFunc func(),
+) {
+	upgCtx, err := getUpgradeContext(f.ClientSet.Discovery(), framework.TestContext.UpgradeTarget)
+	framework.ExpectNoError(err)
+
+	cm := chaosmonkey.New(upgradeFunc)
+	for _, t := range upgradeTests {
+		testCase := &junit.TestCase{
+			Name:      t.Name(),
+			Classname: "upgrade_tests",
+		}
+		testSuite.TestCases = append(testSuite.TestCases, testCase)
+		cma := chaosMonkeyAdapter{
+			test:        t,
+			testReport:  testCase,
+			framework:   testFrameworks[t.Name()],
+			upgradeType: upgradeType,
+			upgCtx:      *upgCtx,
+		}
+		cm.Register(cma.Test)
+	}
+
+	start := time.Now()
+	defer func() {
+		testSuite.Update()
+		testSuite.Time = time.Since(start).Seconds()
+		if framework.TestContext.ReportDir != "" {
+			fname := filepath.Join(framework.TestContext.ReportDir, fmt.Sprintf("junit_%supgrades.xml", framework.TestContext.ReportPrefix))
+			f, err := os.Create(fname)
+			if err != nil {
+				return
+			}
+			defer f.Close()
+			xml.NewEncoder(f).Encode(testSuite)
+		}
+	}()
+	cm.Do()
+}
+
 func getUpgradeContext(c discovery.DiscoveryInterface, upgradeTarget string) (*upgrades.UpgradeContext, error) {
 	current, err := c.ServerVersion()
 	if err != nil {

test/e2e/common/container_probe.go

@@ -237,7 +237,7 @@ var _ = framework.KubeDescribe("Probing container", func() {
 
 	It("should be restarted with a docker exec liveness probe with timeout [Conformance]", func() {
 		// TODO: enable this test once the default exec handler supports timeout.
-		Skip("The default exec handler, dockertools.NativeExecHandler, does not support timeouts due to a limitation in the Docker Remote API")
+		framework.Skipf("The default exec handler, dockertools.NativeExecHandler, does not support timeouts due to a limitation in the Docker Remote API")
 		runLivenessTest(f, &v1.Pod{
 			ObjectMeta: metav1.ObjectMeta{
 				Name:   "liveness-exec",

test/e2e/common/pods.go

@@ -196,7 +196,7 @@ var _ = framework.KubeDescribe("Pods", func() {
 				framework.Failf("Failed to observe pod creation: %v", event)
 			}
 		case <-time.After(framework.PodStartTimeout):
-			Fail("Timeout while waiting for pod creation")
+			framework.Failf("Timeout while waiting for pod creation")
 		}
 
 		// We need to wait for the pod to be running, otherwise the deletion
@@ -245,14 +245,14 @@ var _ = framework.KubeDescribe("Pods", func() {
 					deleted = true
 				case watch.Error:
 					framework.Logf("received a watch error: %v", event.Object)
-					Fail("watch closed with error")
+					framework.Failf("watch closed with error")
 				}
 			case <-timer:
-				Fail("timed out waiting for pod deletion")
+				framework.Failf("timed out waiting for pod deletion")
 			}
 		}
 		if !deleted {
-			Fail("Failed to observe pod deletion")
+			framework.Failf("Failed to observe pod deletion")
 		}
 
 		Expect(lastPod.DeletionTimestamp).ToNot(BeNil())

test/e2e/e2e.go

@@ -44,6 +44,7 @@ import (
 	"k8s.io/kubernetes/pkg/util/logs"
 	commontest "k8s.io/kubernetes/test/e2e/common"
 	"k8s.io/kubernetes/test/e2e/framework"
+	"k8s.io/kubernetes/test/e2e/framework/ginkgowrapper"
 	"k8s.io/kubernetes/test/e2e/generated"
 	federationtest "k8s.io/kubernetes/test/e2e_federation"
 	testutils "k8s.io/kubernetes/test/utils"
@@ -318,7 +319,7 @@ func RunE2ETests(t *testing.T) {
 	logs.InitLogs()
 	defer logs.FlushLogs()
 
-	gomega.RegisterFailHandler(ginkgo.Fail)
+	gomega.RegisterFailHandler(ginkgowrapper.Fail)
 	// Disable skipped tests unless they are explicitly requested.
 	if config.GinkgoConfig.FocusString == "" && config.GinkgoConfig.SkipString == "" {
 		config.GinkgoConfig.SkipString = `\[Flaky\]|\[Feature:.+\]`