Skip to content

Commit

Permalink
test: Remove dependency on e2e in cloud config lookup
Browse files Browse the repository at this point in the history
Also move skip logic into provider initialization, and buffer a
small amount of suite output. GCP tests are excluded from core
infra, they are not used.
  • Loading branch information
smarterclayton committed Jan 23, 2020
1 parent ce13500 commit 2c65427
Show file tree
Hide file tree
Showing 9 changed files with 241 additions and 231 deletions.
121 changes: 27 additions & 94 deletions cmd/openshift-tests/openshift-tests.go
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
package main

import (
"encoding/json"
"bufio"
"flag"
goflag "flag"
"fmt"
Expand All @@ -10,27 +10,17 @@ import (
"os"
"time"

"github.com/onsi/ginkgo"
"github.com/onsi/gomega"
"github.com/spf13/cobra"
"github.com/spf13/pflag"

utilflag "k8s.io/component-base/cli/flag"
"k8s.io/component-base/logs"
"k8s.io/klog"
"k8s.io/kubectl/pkg/util/templates"
reale2e "k8s.io/kubernetes/test/e2e"
e2e "k8s.io/kubernetes/test/e2e/framework"

"github.com/openshift/library-go/pkg/serviceability"
"github.com/openshift/origin/pkg/monitor"
testginkgo "github.com/openshift/origin/pkg/test/ginkgo"
exutil "github.com/openshift/origin/test/extended/util"
exutilcloud "github.com/openshift/origin/test/extended/util/cloud"

// these are loading important global flags that we need to get and set
_ "k8s.io/kubernetes/test/e2e"
_ "k8s.io/kubernetes/test/e2e/lifecycle"
)

func main() {
Expand Down Expand Up @@ -121,14 +111,17 @@ func newRunCommand() *cobra.Command {
SilenceUsage: true,
SilenceErrors: true,
RunE: func(cmd *cobra.Command, args []string) error {
// output from suite execution is often written in batches and can safely be buffered
w := bufio.NewWriterSize(os.Stdout, 4096)
opt.Out = w
defer w.Flush()

return mirrorToFile(opt, func() error {
if err := initProvider(opt.Provider, opt.DryRun); err != nil {
matchFn, err := initializeTestFramework(exutil.TestContext, opt.Provider, opt.DryRun)
if err != nil {
return err
}

e2e.AfterReadingAllFlags(exutil.TestContext)
e2e.TestContext.DumpLogsOnFailure = true
exutil.TestContext.DumpLogsOnFailure = true
opt.MatchFn = matchFn
return opt.Run(args)
})
},
Expand Down Expand Up @@ -177,7 +170,7 @@ func newRunUpgradeCommand() *cobra.Command {
upgradeOpt.Suite = suite.Name
upgradeOpt.JUnitDir = opt.JUnitDir
value := upgradeOpt.ToEnv()
if err := initUpgrade(value); err != nil {
if _, err := initUpgrade(value); err != nil {
return err
}
opt.SuiteOptions = value
Expand All @@ -186,12 +179,11 @@ func newRunUpgradeCommand() *cobra.Command {
}
}

if err := initProvider(opt.Provider, opt.DryRun); err != nil {
matchFn, err := initializeTestFramework(exutil.TestContext, opt.Provider, opt.DryRun)
if err != nil {
return err
}
e2e.AfterReadingAllFlags(exutil.TestContext)
e2e.TestContext.DumpLogsOnFailure = true
exutil.TestContext.DumpLogsOnFailure = true
opt.MatchFn = matchFn
return opt.Run(args)
})
},
Expand Down Expand Up @@ -221,15 +213,15 @@ func newRunTestCommand() *cobra.Command {
SilenceUsage: true,
SilenceErrors: true,
RunE: func(cmd *cobra.Command, args []string) error {
if err := initProvider(os.Getenv("TEST_PROVIDER"), testOpt.DryRun); err != nil {
upgradeOpts, err := initUpgrade(os.Getenv("TEST_SUITE_OPTIONS"))
if err != nil {
return err
}
if err := initUpgrade(os.Getenv("TEST_SUITE_OPTIONS")); err != nil {
if _, err := initializeTestFramework(exutil.TestContext, os.Getenv("TEST_PROVIDER"), testOpt.DryRun); err != nil {
return err
}
e2e.AfterReadingAllFlags(exutil.TestContext)
e2e.TestContext.DumpLogsOnFailure = true
exutil.TestContext.DumpLogsOnFailure = true
exutil.TestContext.ReportDir = upgradeOpts.JUnitDir

return testOpt.Run(args)
},
}
Expand All @@ -241,23 +233,28 @@ func newRunTestCommand() *cobra.Command {
// any error returned from fn. The function returns fn() or any error encountered while
// attempting to open the file.
func mirrorToFile(opt *testginkgo.Options, fn func() error) error {
if opt.Out == nil {
opt.Out = os.Stdout
}
if opt.ErrOut == nil {
opt.ErrOut = os.Stderr
}
if len(opt.OutFile) == 0 {
opt.Out, opt.ErrOut = os.Stdout, os.Stderr
return fn()
}

f, err := os.OpenFile(opt.OutFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0640)
if err != nil {
return err
}
opt.Out = io.MultiWriter(os.Stdout, f)
opt.ErrOut = io.MultiWriter(os.Stderr, f)
opt.Out = io.MultiWriter(opt.Out, f)
opt.ErrOut = io.MultiWriter(opt.ErrOut, f)
exitErr := fn()
if exitErr != nil {
fmt.Fprintf(f, "error: %s", exitErr)
}
if err := f.Close(); err != nil {
fmt.Fprintf(os.Stderr, "error: Unable to close output file\n")
fmt.Fprintf(opt.ErrOut, "error: Unable to close output file\n")
}
return exitErr
}
Expand All @@ -274,67 +271,3 @@ func bindOptions(opt *testginkgo.Options, flags *pflag.FlagSet) {
flags.DurationVar(&opt.Timeout, "timeout", opt.Timeout, "Set the maximum time a test can run before being aborted. This is read from the suite by default, but will be 10 minutes otherwise.")
flags.BoolVar(&opt.IncludeSuccessOutput, "include-success", opt.IncludeSuccessOutput, "Print output from successful tests.")
}

func initProvider(provider string, dryRun bool) error {
// record the exit error to the output file
if err := decodeProviderTo(provider, exutil.TestContext, dryRun); err != nil {
return err
}
exutil.TestContext.AllowedNotReadyNodes = 100
exutil.TestContext.MaxNodesToGather = 0
reale2e.SetViperConfig(os.Getenv("VIPERCONFIG"))

if err := initCSITests(dryRun); err != nil {
return err
}

err := exutil.InitTest(dryRun)
gomega.RegisterFailHandler(ginkgo.Fail)

// TODO: infer SSH keys from the cluster
return err
}

func decodeProviderTo(provider string, testContext *e2e.TestContextType, dryRun bool) error {
switch provider {
case "":
if _, ok := os.LookupEnv("KUBE_SSH_USER"); ok {
if _, ok := os.LookupEnv("LOCAL_SSH_KEY"); ok {
testContext.Provider = "local"
break
}
}
if dryRun {
break
}
fallthrough

case "azure", "aws", "gce", "vsphere":
provider, cfg, err := exutilcloud.LoadConfig()
if err != nil {
return err
}
if cfg != nil {
testContext.Provider = provider
testContext.CloudConfig = *cfg
}

default:
var providerInfo struct{ Type string }
if err := json.Unmarshal([]byte(provider), &providerInfo); err != nil {
return fmt.Errorf("provider must be a JSON object with the 'type' key at a minimum: %v", err)
}
if len(providerInfo.Type) == 0 {
return fmt.Errorf("provider must be a JSON object with the 'type' key")
}
testContext.Provider = providerInfo.Type
if err := json.Unmarshal([]byte(provider), &testContext.CloudConfig); err != nil {
return fmt.Errorf("provider must decode into the cloud config object: %v", err)
}
}
if len(testContext.Provider) == 0 {
testContext.Provider = "skeleton"
}
klog.V(2).Infof("Provider %s: %#v", testContext.Provider, testContext.CloudConfig)
return nil
}
110 changes: 110 additions & 0 deletions cmd/openshift-tests/provider.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
package main

import (
"encoding/json"
"fmt"
"os"
"strings"

"github.com/onsi/ginkgo"
"github.com/onsi/gomega"

reale2e "k8s.io/kubernetes/test/e2e"
e2e "k8s.io/kubernetes/test/e2e/framework"

exutil "github.com/openshift/origin/test/extended/util"
exutilcloud "github.com/openshift/origin/test/extended/util/cloud"

// these are loading important global flags that we need to get and set
_ "k8s.io/kubernetes/test/e2e"
_ "k8s.io/kubernetes/test/e2e/lifecycle"
)

type TestNameMatchesFunc func(name string) bool

func initializeTestFramework(context *e2e.TestContextType, provider string, dryRun bool) (TestNameMatchesFunc, error) {
config, err := decodeProvider(provider, dryRun)
if err != nil {
return nil, err
}

// update context with loaded config
context.Provider = config.ProviderName
context.CloudConfig = e2e.CloudConfig{
ProjectID: config.ProjectID,
Region: config.Region,
Zone: config.Zone,
NumNodes: config.NumNodes,
MultiMaster: config.MultiMaster,
MultiZone: config.MultiZone,
ConfigFile: config.CloudConfigFile,
}
context.AllowedNotReadyNodes = 100
context.MaxNodesToGather = 0
reale2e.SetViperConfig(os.Getenv("VIPERCONFIG"))

if err := initCSITests(dryRun); err != nil {
return nil, err
}
if err := exutil.InitTest(dryRun); err != nil {
return nil, err
}
gomega.RegisterFailHandler(ginkgo.Fail)

e2e.AfterReadingAllFlags(context)
context.DumpLogsOnFailure = true

// given the configuration we have loaded, skip tests that our provider should exclude
// or our network plugin should exclude
skipProvider := fmt.Sprintf("[Skipped:%s", config.ProviderName)
skipNetworkPlugin := fmt.Sprintf("[Skipped:Network/%s", config.NetworkPlugin)
skipFn := func(name string) bool {
return !strings.Contains(name, skipProvider) && !strings.Contains(name, skipNetworkPlugin)
}
return skipFn, nil
}

func decodeProvider(provider string, dryRun bool) (*exutilcloud.ClusterConfiguration, error) {
switch provider {
case "":
if _, ok := os.LookupEnv("KUBE_SSH_USER"); ok {
if _, ok := os.LookupEnv("LOCAL_SSH_KEY"); ok {
return &exutilcloud.ClusterConfiguration{ProviderName: "local"}, nil
}
}
if dryRun {
return &exutilcloud.ClusterConfiguration{ProviderName: "skeleton"}, nil
}
fallthrough

case "azure", "aws", "gce", "vsphere":
clientConfig, err := e2e.LoadConfig(true)
if err != nil {
return nil, err
}
config, err := exutilcloud.LoadConfig(clientConfig)
if err != nil {
return nil, err
}
if len(config.ProviderName) == 0 {
config.ProviderName = "skeleton"
}
return config, nil

default:
var providerInfo struct{ Type string }
if err := json.Unmarshal([]byte(provider), &providerInfo); err != nil {
return nil, fmt.Errorf("provider must be a JSON object with the 'type' key at a minimum: %v", err)
}
if len(providerInfo.Type) == 0 {
return nil, fmt.Errorf("provider must be a JSON object with the 'type' key")
}
config := &exutilcloud.ClusterConfiguration{
ProviderName: providerInfo.Type,
}
if err := json.Unmarshal([]byte(provider), config); err != nil {
return nil, fmt.Errorf("provider must decode into the cloud config object: %v", err)
}
return config, nil
}
}
24 changes: 11 additions & 13 deletions cmd/openshift-tests/upgrade.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,10 @@ import (
"github.com/spf13/pflag"

"k8s.io/kubectl/pkg/util/templates"
"k8s.io/kubernetes/test/e2e/cloud/gcp"
"k8s.io/kubernetes/test/e2e/upgrades"

"github.com/openshift/origin/pkg/test/ginkgo"
"github.com/openshift/origin/test/e2e/upgrade"
exutil "github.com/openshift/origin/test/extended/util"
)

// upgradeSuites are all known upgade test suites this binary should run
Expand Down Expand Up @@ -84,30 +82,30 @@ func (o *UpgradeOptions) ToEnv() string {
return string(out)
}

func initUpgrade(value string) error {
func initUpgrade(value string) (*UpgradeOptions, error) {
var opt UpgradeOptions
if len(value) == 0 {
return nil
return &opt, nil
}
var opt UpgradeOptions
if err := json.Unmarshal([]byte(value), &opt); err != nil {
return err
return nil, err
}
for _, suite := range upgradeSuites {
if suite.Name == opt.Suite {
gcp.SetUpgradeTarget("")
gcp.SetUpgradeImage(opt.ToImage)
exutil.TestContext.ReportDir = opt.JUnitDir
o, err := opt.OptionsMap()
if err != nil {
return err
return nil, err
}
if suite.Init != nil {
return suite.Init(o)
if err := suite.Init(o); err != nil {
return nil, err
}
}
return nil
upgrade.SetToImage(opt.ToImage)
return &opt, nil
}
}
return fmt.Errorf("unrecognized upgrade info")
return nil, fmt.Errorf("unrecognized upgrade info")
}

func filterUpgrade(tests []upgrades.Test, match func(string) bool) error {
Expand Down

0 comments on commit 2c65427

Please sign in to comment.