Skip to content

Commit

Permalink
Merge pull request #68483 from pohly/e2e-refactor-pr
Browse files Browse the repository at this point in the history 
e2e refactor
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot committed Oct 19, 2018
2 parents 8e31d05 + 97101a6 commit 8b36038
Show file tree
Hide file tree
Showing 43 changed files with 2,099 additions and 1,598 deletions.
5 changes: 5 additions & 0 deletions hack/.golint_failures
View file
Original file line number Diff line number Diff line change
Expand Up @@ -732,7 +732,12 @@ test/e2e/autoscaling
test/e2e/chaosmonkey
test/e2e/common
test/e2e/framework
test/e2e/framework/ingress
test/e2e/framework/metrics
test/e2e/framework/providers/aws
test/e2e/framework/providers/azure
test/e2e/framework/providers/gce
test/e2e/framework/providers/kubemark
test/e2e/framework/timer
test/e2e/instrumentation
test/e2e/instrumentation/logging
Expand Down
6 changes: 4 additions & 2 deletions test/e2e/BUILD
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,6 @@ go_library(
importpath = "k8s.io/kubernetes/test/e2e",
deps = [
"//pkg/api/v1/pod:go_default_library",
"//pkg/cloudprovider/providers/azure:go_default_library",
"//pkg/cloudprovider/providers/gce:go_default_library",
"//pkg/version:go_default_library",
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/api/rbac/v1beta1:go_default_library",
Expand All @@ -63,6 +61,10 @@ go_library(
"//test/e2e/framework:go_default_library",
"//test/e2e/framework/ginkgowrapper:go_default_library",
"//test/e2e/framework/metrics:go_default_library",
"//test/e2e/framework/providers/aws:go_default_library",
"//test/e2e/framework/providers/azure:go_default_library",
"//test/e2e/framework/providers/gce:go_default_library",
"//test/e2e/framework/providers/kubemark:go_default_library",
"//test/e2e/framework/testfiles:go_default_library",
"//test/e2e/manifest:go_default_library",
"//test/utils:go_default_library",
Expand Down
92 changes: 6 additions & 86 deletions test/e2e/e2e.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,6 @@ import (
runtimeutils "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/util/logs"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/kubernetes/pkg/cloudprovider/providers/azure"
gcecloud "k8s.io/kubernetes/pkg/cloudprovider/providers/gce"
"k8s.io/kubernetes/pkg/version"
commontest "k8s.io/kubernetes/test/e2e/common"
"k8s.io/kubernetes/test/e2e/framework"
Expand All @@ -46,86 +44,18 @@ import (

// ensure auth plugins are loaded
_ "k8s.io/client-go/plugin/pkg/client/auth"

// ensure that cloud providers are loaded
_ "k8s.io/kubernetes/test/e2e/framework/providers/aws"
_ "k8s.io/kubernetes/test/e2e/framework/providers/azure"
_ "k8s.io/kubernetes/test/e2e/framework/providers/gce"
_ "k8s.io/kubernetes/test/e2e/framework/providers/kubemark"
)

var (
cloudConfig = &framework.TestContext.CloudConfig
)

// setupProviderConfig validates and sets up cloudConfig based on framework.TestContext.Provider.
func setupProviderConfig() error {
switch framework.TestContext.Provider {
case "":
glog.Info("The --provider flag is not set. Treating as a conformance test. Some tests may not be run.")

case "gce", "gke":
framework.Logf("Fetching cloud provider for %q\r", framework.TestContext.Provider)
zone := framework.TestContext.CloudConfig.Zone
region := framework.TestContext.CloudConfig.Region

var err error
if region == "" {
region, err = gcecloud.GetGCERegion(zone)
if err != nil {
return fmt.Errorf("error parsing GCE/GKE region from zone %q: %v", zone, err)
}
}
managedZones := []string{} // Manage all zones in the region
if !framework.TestContext.CloudConfig.MultiZone {
managedZones = []string{zone}
}

gceCloud, err := gcecloud.CreateGCECloud(&gcecloud.CloudConfig{
ApiEndpoint: framework.TestContext.CloudConfig.ApiEndpoint,
ProjectID: framework.TestContext.CloudConfig.ProjectID,
Region: region,
Zone: zone,
ManagedZones: managedZones,
NetworkName: "", // TODO: Change this to use framework.TestContext.CloudConfig.Network?
SubnetworkName: "",
NodeTags: nil,
NodeInstancePrefix: "",
TokenSource: nil,
UseMetadataServer: false,
AlphaFeatureGate: gcecloud.NewAlphaFeatureGate([]string{}),
})

if err != nil {
return fmt.Errorf("Error building GCE/GKE provider: %v", err)
}

cloudConfig.Provider = gceCloud

// Arbitrarily pick one of the zones we have nodes in
if cloudConfig.Zone == "" && framework.TestContext.CloudConfig.MultiZone {
zones, err := gceCloud.GetAllZonesFromCloudProvider()
if err != nil {
return err
}

cloudConfig.Zone, _ = zones.PopAny()
}

case "aws":
if cloudConfig.Zone == "" {
return fmt.Errorf("gce-zone must be specified for AWS")
}
case "azure":
if cloudConfig.ConfigFile == "" {
return fmt.Errorf("config-file must be specified for Azure")
}
config, err := os.Open(cloudConfig.ConfigFile)
if err != nil {
framework.Logf("Couldn't open cloud provider configuration %s: %#v",
cloudConfig.ConfigFile, err)
}
defer config.Close()
cloudConfig.Provider, err = azure.NewCloud(config)
}

return nil
}

// There are certain operations we only want to run once per overall test invocation
// (such as deleting old namespaces, or verifying that all system pods are running.
// Because of the way Ginkgo runs tests in parallel, we must use SynchronizedBeforeSuite
Expand All @@ -137,10 +67,6 @@ func setupProviderConfig() error {
var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {
// Run only on Ginkgo node 1

if err := setupProviderConfig(); err != nil {
framework.Failf("Failed to setup provider config: %v", err)
}

switch framework.TestContext.Provider {
case "gce", "gke":
framework.LogClusterImageSources()
Expand Down Expand Up @@ -214,12 +140,6 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte {

}, func(data []byte) {
// Run on all Ginkgo nodes

if cloudConfig.Provider == nil {
if err := setupProviderConfig(); err != nil {
framework.Failf("Failed to setup provider config: %v", err)
}
}
})

// Similar to SynchronizedBeforeSuite, we want to run some operations only once (such as collecting cluster logs).
Expand Down
34 changes: 7 additions & 27 deletions test/e2e/framework/BUILD
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
package(default_visibility = ["//visibility:public"])

load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)
load("@io_bazel_rules_go//go:def.bzl", "go_library")

go_library(
name = "go_default_library",
Expand All @@ -14,13 +10,11 @@ go_library(
"crd_util.go",
"deployment_util.go",
"exec_util.go",
"firewall_util.go",
"flake_reporting_util.go",
"framework.go",
"get-kubemark-resource-usage.go",
"google_compute.go",
"gpu_util.go",
"ingress_utils.go",
"jobs_util.go",
"kubelet_stats.go",
"log_size_monitoring.go",
Expand All @@ -30,6 +24,7 @@ go_library(
"perf_util.go",
"pods.go",
"profile_gatherer.go",
"provider.go",
"psp_util.go",
"pv_util.go",
"rc_util.go",
Expand All @@ -53,9 +48,6 @@ go_library(
"//pkg/apis/extensions:go_default_library",
"//pkg/client/clientset_generated/internalclientset:go_default_library",
"//pkg/client/conditions:go_default_library",
"//pkg/cloudprovider/providers/aws:go_default_library",
"//pkg/cloudprovider/providers/azure:go_default_library",
"//pkg/cloudprovider/providers/gce:go_default_library",
"//pkg/controller:go_default_library",
"//pkg/controller/deployment/util:go_default_library",
"//pkg/controller/job:go_default_library",
Expand All @@ -70,7 +62,6 @@ go_library(
"//pkg/kubelet/metrics:go_default_library",
"//pkg/kubelet/sysctl:go_default_library",
"//pkg/kubelet/util/format:go_default_library",
"//pkg/kubemark:go_default_library",
"//pkg/master/ports:go_default_library",
"//pkg/scheduler/algorithm/predicates:go_default_library",
"//pkg/scheduler/cache:go_default_library",
Expand Down Expand Up @@ -117,7 +108,6 @@ go_library(
"//staging/src/k8s.io/client-go/discovery:go_default_library",
"//staging/src/k8s.io/client-go/discovery/cached:go_default_library",
"//staging/src/k8s.io/client-go/dynamic:go_default_library",
"//staging/src/k8s.io/client-go/informers:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/apps/v1:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes/typed/authorization/v1beta1:go_default_library",
Expand All @@ -131,21 +121,14 @@ go_library(
"//staging/src/k8s.io/client-go/tools/remotecommand:go_default_library",
"//staging/src/k8s.io/client-go/tools/watch:go_default_library",
"//staging/src/k8s.io/client-go/util/retry:go_default_library",
"//staging/src/k8s.io/cloud-provider:go_default_library",
"//staging/src/k8s.io/csi-api/pkg/client/clientset/versioned:go_default_library",
"//staging/src/k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset:go_default_library",
"//test/e2e/framework/ginkgowrapper:go_default_library",
"//test/e2e/framework/metrics:go_default_library",
"//test/e2e/framework/testfiles:go_default_library",
"//test/e2e/manifest:go_default_library",
"//test/e2e/perftype:go_default_library",
"//test/utils:go_default_library",
"//test/utils/image:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/aws/awserr:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/aws/session:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/service/autoscaling:go_default_library",
"//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/onsi/ginkgo:go_default_library",
"//vendor/github.com/onsi/ginkgo/config:go_default_library",
Expand All @@ -155,8 +138,6 @@ go_library(
"//vendor/github.com/prometheus/common/model:go_default_library",
"//vendor/golang.org/x/crypto/ssh:go_default_library",
"//vendor/golang.org/x/net/websocket:go_default_library",
"//vendor/google.golang.org/api/compute/v1:go_default_library",
"//vendor/google.golang.org/api/googleapi:go_default_library",
"//vendor/k8s.io/utils/exec:go_default_library",
],
)
Expand All @@ -174,16 +155,15 @@ filegroup(
":package-srcs",
"//test/e2e/framework/config:all-srcs",
"//test/e2e/framework/ginkgowrapper:all-srcs",
"//test/e2e/framework/ingress:all-srcs",
"//test/e2e/framework/metrics:all-srcs",
"//test/e2e/framework/providers/aws:all-srcs",
"//test/e2e/framework/providers/azure:all-srcs",
"//test/e2e/framework/providers/gce:all-srcs",
"//test/e2e/framework/providers/kubemark:all-srcs",
"//test/e2e/framework/testfiles:all-srcs",
"//test/e2e/framework/timer:all-srcs",
"//test/e2e/framework/viperconfig:all-srcs",
],
tags = ["automanaged"],
)

go_test(
name = "go_default_test",
srcs = ["firewall_util_test.go"],
embed = [":go_default_library"],
)
34 changes: 7 additions & 27 deletions test/e2e/framework/framework.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,11 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

// Package framework contains provider-independent helper code for
// building and running E2E tests with Ginkgo. The actual Ginkgo test
// suites gets assembled by combining this framework, the optional
// provider support code and specific tests via a separate .go file
// like Kubernetes' test/e2e.go.
package framework

import (
Expand All @@ -36,17 +41,14 @@ import (
"k8s.io/client-go/discovery"
cacheddiscovery "k8s.io/client-go/discovery/cached"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/informers"
clientset "k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/restmapper"
scaleclient "k8s.io/client-go/scale"
"k8s.io/client-go/tools/clientcmd"
csi "k8s.io/csi-api/pkg/client/clientset/versioned"
aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
"k8s.io/kubernetes/pkg/api/legacyscheme"
"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
"k8s.io/kubernetes/pkg/kubemark"
"k8s.io/kubernetes/test/e2e/framework/metrics"
testutils "k8s.io/kubernetes/test/utils"

Expand Down Expand Up @@ -108,8 +110,6 @@ type Framework struct {
// or stdout if ReportDir is not set once test ends.
TestSummaries []TestDataSummary

kubemarkControllerCloseChannel chan struct{}

// Place to keep ClusterAutoscaler metrics from before test in order to compute delta.
clusterAutoscalerMetricsBeforeTest metrics.MetricsCollection
}
Expand Down Expand Up @@ -210,25 +210,7 @@ func (f *Framework) BeforeEach() {
resolver := scaleclient.NewDiscoveryScaleKindResolver(cachedDiscoClient)
f.ScalesGetter = scaleclient.New(restClient, restMapper, dynamic.LegacyAPIPathResolverFunc, resolver)

if ProviderIs("kubemark") && TestContext.KubemarkExternalKubeConfig != "" && TestContext.CloudConfig.KubemarkController == nil {
externalConfig, err := clientcmd.BuildConfigFromFlags("", TestContext.KubemarkExternalKubeConfig)
externalConfig.QPS = f.Options.ClientQPS
externalConfig.Burst = f.Options.ClientBurst
Expect(err).NotTo(HaveOccurred())
externalClient, err := clientset.NewForConfig(externalConfig)
Expect(err).NotTo(HaveOccurred())
f.KubemarkExternalClusterClientSet = externalClient
f.kubemarkControllerCloseChannel = make(chan struct{})
externalInformerFactory := informers.NewSharedInformerFactory(externalClient, 0)
kubemarkInformerFactory := informers.NewSharedInformerFactory(f.ClientSet, 0)
kubemarkNodeInformer := kubemarkInformerFactory.Core().V1().Nodes()
go kubemarkNodeInformer.Informer().Run(f.kubemarkControllerCloseChannel)
TestContext.CloudConfig.KubemarkController, err = kubemark.NewKubemarkController(f.KubemarkExternalClusterClientSet, externalInformerFactory, f.ClientSet, kubemarkNodeInformer)
Expect(err).NotTo(HaveOccurred())
externalInformerFactory.Start(f.kubemarkControllerCloseChannel)
Expect(TestContext.CloudConfig.KubemarkController.WaitForCacheSync(f.kubemarkControllerCloseChannel)).To(BeTrue())
go TestContext.CloudConfig.KubemarkController.Run(f.kubemarkControllerCloseChannel)
}
TestContext.CloudConfig.Provider.FrameworkBeforeEach(f)
}

if !f.SkipNamespaceCreation {
Expand Down Expand Up @@ -393,9 +375,7 @@ func (f *Framework) AfterEach() {
}
}

if TestContext.CloudConfig.KubemarkController != nil {
close(f.kubemarkControllerCloseChannel)
}
TestContext.CloudConfig.Provider.FrameworkAfterEach(f)

// Report any flakes that were observed in the e2e test and reset.
if f.flakeReport != nil && f.flakeReport.GetFlakeCount() > 0 {
Expand Down
42 changes: 42 additions & 0 deletions test/e2e/framework/ingress/BUILD
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")

go_library(
name = "go_default_library",
srcs = ["ingress_utils.go"],
importpath = "k8s.io/kubernetes/test/e2e/framework/ingress",
visibility = ["//visibility:public"],
deps = [
"//staging/src/k8s.io/api/core/v1:go_default_library",
"//staging/src/k8s.io/api/extensions/v1beta1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
"//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//staging/src/k8s.io/client-go/kubernetes:go_default_library",
"//test/e2e/framework:go_default_library",
"//test/e2e/framework/testfiles:go_default_library",
"//test/e2e/manifest:go_default_library",
"//test/utils:go_default_library",
"//vendor/github.com/golang/glog:go_default_library",
"//vendor/github.com/onsi/ginkgo:go_default_library",
"//vendor/github.com/onsi/gomega:go_default_library",
"//vendor/google.golang.org/api/compute/v1:go_default_library",
],
)

filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)

filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
Loading

0 comments on commit 8b36038

Please sign in to comment.