test: Generate test name updates rather than calculate dynamically

cmd/openshift-tests/e2e.go

@@ -9,6 +9,7 @@ import (
 	"github.com/openshift/origin/pkg/test/ginkgo"
 
 	_ "github.com/openshift/origin/test/extended"
+	_ "github.com/openshift/origin/test/extended/util/annotate/generated"
 )
 
 // staticSuites are all known test suites this binary should run

cmd/openshift-tests/openshift-tests.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"encoding/json"
 	"flag"
 	goflag "flag"
 	"fmt"
@@ -10,27 +9,17 @@ import (
 	"os"
 	"time"
 
-	"github.com/onsi/ginkgo"
-	"github.com/onsi/gomega"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 
 	utilflag "k8s.io/component-base/cli/flag"
 	"k8s.io/component-base/logs"
-	"k8s.io/klog"
 	"k8s.io/kubectl/pkg/util/templates"
-	reale2e "k8s.io/kubernetes/test/e2e"
-	e2e "k8s.io/kubernetes/test/e2e/framework"
 
 	"github.com/openshift/library-go/pkg/serviceability"
 	"github.com/openshift/origin/pkg/monitor"
 	testginkgo "github.com/openshift/origin/pkg/test/ginkgo"
 	exutil "github.com/openshift/origin/test/extended/util"
-	exutilcloud "github.com/openshift/origin/test/extended/util/cloud"
-
-	// these are loading important global flags that we need to get and set
-	_ "k8s.io/kubernetes/test/e2e"
-	_ "k8s.io/kubernetes/test/e2e/lifecycle"
 )
 
 func main() {
@@ -122,13 +111,11 @@ func newRunCommand() *cobra.Command {
 		SilenceErrors: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return mirrorToFile(opt, func() error {
-				if err := initProvider(opt.Provider, opt.DryRun); err != nil {
+				matchFn, err := initializeTestFramework(exutil.TestContext, opt.Provider, opt.DryRun)
+				if err != nil {
 					return err
 				}
-
-				e2e.AfterReadingAllFlags(exutil.TestContext)
-				e2e.TestContext.DumpLogsOnFailure = true
-				exutil.TestContext.DumpLogsOnFailure = true
+				opt.MatchFn = matchFn
 				return opt.Run(args)
 			})
 		},
@@ -177,7 +164,7 @@ func newRunUpgradeCommand() *cobra.Command {
 							upgradeOpt.Suite = suite.Name
 							upgradeOpt.JUnitDir = opt.JUnitDir
 							value := upgradeOpt.ToEnv()
-							if err := initUpgrade(value); err != nil {
+							if _, err := initUpgrade(value); err != nil {
 								return err
 							}
 							opt.SuiteOptions = value
@@ -186,12 +173,11 @@ func newRunUpgradeCommand() *cobra.Command {
 					}
 				}
 
-				if err := initProvider(opt.Provider, opt.DryRun); err != nil {
+				matchFn, err := initializeTestFramework(exutil.TestContext, opt.Provider, opt.DryRun)
+				if err != nil {
 					return err
 				}
-				e2e.AfterReadingAllFlags(exutil.TestContext)
-				e2e.TestContext.DumpLogsOnFailure = true
-				exutil.TestContext.DumpLogsOnFailure = true
+				opt.MatchFn = matchFn
 				return opt.Run(args)
 			})
 		},
@@ -221,15 +207,15 @@ func newRunTestCommand() *cobra.Command {
 		SilenceUsage:  true,
 		SilenceErrors: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			if err := initProvider(os.Getenv("TEST_PROVIDER"), testOpt.DryRun); err != nil {
+			upgradeOpts, err := initUpgrade(os.Getenv("TEST_SUITE_OPTIONS"))
+			if err != nil {
 				return err
 			}
-			if err := initUpgrade(os.Getenv("TEST_SUITE_OPTIONS")); err != nil {
+			if _, err := initializeTestFramework(exutil.TestContext, os.Getenv("TEST_PROVIDER"), testOpt.DryRun); err != nil {
 				return err
 			}
-			e2e.AfterReadingAllFlags(exutil.TestContext)
-			e2e.TestContext.DumpLogsOnFailure = true
-			exutil.TestContext.DumpLogsOnFailure = true
+			exutil.TestContext.ReportDir = upgradeOpts.JUnitDir
+
 			return testOpt.Run(args)
 		},
 	}
@@ -241,23 +227,28 @@ func newRunTestCommand() *cobra.Command {
 // any error returned from fn. The function returns fn() or any error encountered while
 // attempting to open the file.
 func mirrorToFile(opt *testginkgo.Options, fn func() error) error {
+	if opt.Out == nil {
+		opt.Out = os.Stdout
+	}
+	if opt.ErrOut == nil {
+		opt.ErrOut = os.Stderr
+	}
 	if len(opt.OutFile) == 0 {
-		opt.Out, opt.ErrOut = os.Stdout, os.Stderr
 		return fn()
 	}
 
 	f, err := os.OpenFile(opt.OutFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0640)
 	if err != nil {
 		return err
 	}
-	opt.Out = io.MultiWriter(os.Stdout, f)
-	opt.ErrOut = io.MultiWriter(os.Stderr, f)
+	opt.Out = io.MultiWriter(opt.Out, f)
+	opt.ErrOut = io.MultiWriter(opt.ErrOut, f)
 	exitErr := fn()
 	if exitErr != nil {
 		fmt.Fprintf(f, "error: %s", exitErr)
 	}
 	if err := f.Close(); err != nil {
-		fmt.Fprintf(os.Stderr, "error: Unable to close output file\n")
+		fmt.Fprintf(opt.ErrOut, "error: Unable to close output file\n")
 	}
 	return exitErr
 }
@@ -274,68 +265,3 @@ func bindOptions(opt *testginkgo.Options, flags *pflag.FlagSet) {
 	flags.DurationVar(&opt.Timeout, "timeout", opt.Timeout, "Set the maximum time a test can run before being aborted. This is read from the suite by default, but will be 10 minutes otherwise.")
 	flags.BoolVar(&opt.IncludeSuccessOutput, "include-success", opt.IncludeSuccessOutput, "Print output from successful tests.")
 }
-
-func initProvider(provider string, dryRun bool) error {
-	// record the exit error to the output file
-	if err := decodeProviderTo(provider, exutil.TestContext, dryRun); err != nil {
-		return err
-	}
-	exutil.TestContext.AllowedNotReadyNodes = 100
-	exutil.TestContext.MaxNodesToGather = 0
-	reale2e.SetViperConfig(os.Getenv("VIPERCONFIG"))
-
-	if err := initCSITests(dryRun); err != nil {
-		return err
-	}
-
-	exutil.AnnotateTestSuite()
-	err := exutil.InitTest(dryRun)
-	gomega.RegisterFailHandler(ginkgo.Fail)
-
-	// TODO: infer SSH keys from the cluster
-	return err
-}
-
-func decodeProviderTo(provider string, testContext *e2e.TestContextType, dryRun bool) error {
-	switch provider {
-	case "":
-		if _, ok := os.LookupEnv("KUBE_SSH_USER"); ok {
-			if _, ok := os.LookupEnv("LOCAL_SSH_KEY"); ok {
-				testContext.Provider = "local"
-				break
-			}
-		}
-		if dryRun {
-			break
-		}
-		fallthrough
-
-	case "azure", "aws", "gce", "vsphere":
-		provider, cfg, err := exutilcloud.LoadConfig()
-		if err != nil {
-			return err
-		}
-		if cfg != nil {
-			testContext.Provider = provider
-			testContext.CloudConfig = *cfg
-		}
-
-	default:
-		var providerInfo struct{ Type string }
-		if err := json.Unmarshal([]byte(provider), &providerInfo); err != nil {
-			return fmt.Errorf("provider must be a JSON object with the 'type' key at a minimum: %v", err)
-		}
-		if len(providerInfo.Type) == 0 {
-			return fmt.Errorf("provider must be a JSON object with the 'type' key")
-		}
-		testContext.Provider = providerInfo.Type
-		if err := json.Unmarshal([]byte(provider), &testContext.CloudConfig); err != nil {
-			return fmt.Errorf("provider must decode into the cloud config object: %v", err)
-		}
-	}
-	if len(testContext.Provider) == 0 {
-		testContext.Provider = "skeleton"
-	}
-	klog.V(2).Infof("Provider %s: %#v", testContext.Provider, testContext.CloudConfig)
-	return nil
-}

cmd/openshift-tests/provider.go

@@ -0,0 +1,110 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+
+	reale2e "k8s.io/kubernetes/test/e2e"
+	e2e "k8s.io/kubernetes/test/e2e/framework"
+
+	exutil "github.com/openshift/origin/test/extended/util"
+	exutilcloud "github.com/openshift/origin/test/extended/util/cloud"
+
+	// these are loading important global flags that we need to get and set
+	_ "k8s.io/kubernetes/test/e2e"
+	_ "k8s.io/kubernetes/test/e2e/lifecycle"
+)
+
+type TestNameMatchesFunc func(name string) bool
+
+func initializeTestFramework(context *e2e.TestContextType, provider string, dryRun bool) (TestNameMatchesFunc, error) {
+	config, err := decodeProvider(provider, dryRun)
+	if err != nil {
+		return nil, err
+	}
+
+	// update context with loaded config
+	context.Provider = config.ProviderName
+	context.CloudConfig = e2e.CloudConfig{
+		ProjectID:   config.ProjectID,
+		Region:      config.Region,
+		Zone:        config.Zone,
+		NumNodes:    config.NumNodes,
+		MultiMaster: config.MultiMaster,
+		MultiZone:   config.MultiZone,
+		ConfigFile:  config.CloudConfigFile,
+	}
+	context.AllowedNotReadyNodes = 100
+	context.MaxNodesToGather = 0
+	reale2e.SetViperConfig(os.Getenv("VIPERCONFIG"))
+
+	if err := initCSITests(dryRun); err != nil {
+		return nil, err
+	}
+	if err := exutil.InitTest(dryRun); err != nil {
+		return nil, err
+	}
+	gomega.RegisterFailHandler(ginkgo.Fail)
+
+	e2e.AfterReadingAllFlags(context)
+	context.DumpLogsOnFailure = true
+
+	// given the configuration we have loaded, skip tests that our provider should exclude
+	// or our network plugin should exclude
+	skipProvider := fmt.Sprintf("[Skipped:%s", config.ProviderName)
+	skipNetworkPlugin := fmt.Sprintf("[Skipped:Network/%s", config.NetworkPlugin)
+	skipFn := func(name string) bool {
+		return !strings.Contains(name, skipProvider) && !strings.Contains(name, skipNetworkPlugin)
+	}
+	return skipFn, nil
+}
+
+func decodeProvider(provider string, dryRun bool) (*exutilcloud.ClusterConfiguration, error) {
+	switch provider {
+	case "":
+		if _, ok := os.LookupEnv("KUBE_SSH_USER"); ok {
+			if _, ok := os.LookupEnv("LOCAL_SSH_KEY"); ok {
+				return &exutilcloud.ClusterConfiguration{ProviderName: "local"}, nil
+			}
+		}
+		if dryRun {
+			return &exutilcloud.ClusterConfiguration{ProviderName: "skeleton"}, nil
+		}
+		fallthrough
+
+	case "azure", "aws", "gce", "vsphere":
+		clientConfig, err := e2e.LoadConfig(true)
+		if err != nil {
+			return nil, err
+		}
+		config, err := exutilcloud.LoadConfig(clientConfig)
+		if err != nil {
+			return nil, err
+		}
+		if len(config.ProviderName) == 0 {
+			config.ProviderName = "skeleton"
+		}
+		return config, nil
+
+	default:
+		var providerInfo struct{ Type string }
+		if err := json.Unmarshal([]byte(provider), &providerInfo); err != nil {
+			return nil, fmt.Errorf("provider must be a JSON object with the 'type' key at a minimum: %v", err)
+		}
+		if len(providerInfo.Type) == 0 {
+			return nil, fmt.Errorf("provider must be a JSON object with the 'type' key")
+		}
+		config := &exutilcloud.ClusterConfiguration{
+			ProviderName: providerInfo.Type,
+		}
+		if err := json.Unmarshal([]byte(provider), config); err != nil {
+			return nil, fmt.Errorf("provider must decode into the cloud config object: %v", err)
+		}
+		return config, nil
+	}
+}

cmd/openshift-tests/upgrade.go

@@ -9,12 +9,10 @@ import (
 	"github.com/spf13/pflag"
 
 	"k8s.io/kubectl/pkg/util/templates"
-	"k8s.io/kubernetes/test/e2e/cloud/gcp"
 	"k8s.io/kubernetes/test/e2e/upgrades"
 
 	"github.com/openshift/origin/pkg/test/ginkgo"
 	"github.com/openshift/origin/test/e2e/upgrade"
-	exutil "github.com/openshift/origin/test/extended/util"
 )
 
 // upgradeSuites are all known upgade test suites this binary should run
@@ -84,30 +82,30 @@ func (o *UpgradeOptions) ToEnv() string {
 	return string(out)
 }
 
-func initUpgrade(value string) error {
+func initUpgrade(value string) (*UpgradeOptions, error) {
+	var opt UpgradeOptions
 	if len(value) == 0 {
-		return nil
+		return &opt, nil
 	}
-	var opt UpgradeOptions
 	if err := json.Unmarshal([]byte(value), &opt); err != nil {
-		return err
+		return nil, err
 	}
 	for _, suite := range upgradeSuites {
 		if suite.Name == opt.Suite {
-			gcp.SetUpgradeTarget("")
-			gcp.SetUpgradeImage(opt.ToImage)
-			exutil.TestContext.ReportDir = opt.JUnitDir
 			o, err := opt.OptionsMap()
 			if err != nil {
-				return err
+				return nil, err
 			}
 			if suite.Init != nil {
-				return suite.Init(o)
+				if err := suite.Init(o); err != nil {
+					return nil, err
+				}
 			}
-			return nil
+			upgrade.SetToImage(opt.ToImage)
+			return &opt, nil
 		}
 	}
-	return fmt.Errorf("unrecognized upgrade info")
+	return nil, fmt.Errorf("unrecognized upgrade info")
 }
 
 func filterUpgrade(tests []upgrades.Test, match func(string) bool) error {

hack/update-generated-bindata.sh

@@ -2,6 +2,8 @@
 STARTTIME=$(date +%s)
 source "$(dirname "${BASH_SOURCE}")/lib/init.sh"
 
+go generate ./test/extended
+
 os::build::setup_env
 
 OUTPUT_PARENT=${OUTPUT_ROOT:-$OS_ROOT}

hack/verify-generated-bindata.sh

@@ -13,6 +13,7 @@ os::test::junit::declare_suite_start "verify/bindata"
 os::cmd::expect_success "${OS_ROOT}/hack/update-generated-bindata.sh"
 os::cmd::expect_success "git diff --quiet vendor/k8s.io/kubernetes/staging/src/k8s.io/kubectl/pkg/generated/bindata.go"
 os::cmd::expect_success "git diff --quiet vendor/k8s.io/kubernetes/test/e2e/generated/bindata.go"
+os::cmd::expect_success "git diff --quiet ${OS_ROOT}/test/extended/util/annotate/generated/"
 os::cmd::expect_success "git diff --quiet ${OS_ROOT}/test/extended/testdata/bindata.go"
 
 os::test::junit::declare_suite_end