Introduce IBMCloud provider, skip/fix tests

test/extended/apiserver/root_403.go

@@ -59,6 +59,9 @@ var _ = g.Describe("[Feature:APIServer]", func() {
 })
 
 func anonymousHttpTransport(restConfig *rest.Config) (*http.Transport, error) {
+	if len(restConfig.TLSClientConfig.CAData) == 0 {
+		return &http.Transport{}, nil
+	}
 	pool := x509.NewCertPool()
 	if ok := pool.AppendCertsFromPEM(restConfig.TLSClientConfig.CAData); !ok {
 		return nil, errors.New("failed to add server CA certificates to client pool")

test/extended/authentication/front_proxy.go

@@ -12,10 +12,12 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/client-go/rest"
+	e2e "k8s.io/kubernetes/test/e2e/framework"
 
 	userv1 "github.com/openshift/api/user/v1"
 	"github.com/openshift/origin/test/extended/scheme"
 	exutil "github.com/openshift/origin/test/extended/util"
+	"github.com/openshift/origin/test/extended/util/ibmcloud"
 )
 
 var _ = g.Describe("[Feature:Authentication] ", func() {
@@ -24,6 +26,10 @@ var _ = g.Describe("[Feature:Authentication] ", func() {
 
 	g.Describe("TestFrontProxy", func() {
 		g.It(fmt.Sprintf("should succeed"), func() {
+			if e2e.TestContext.Provider == ibmcloud.ProviderName {
+				e2e.Skipf("IBM ROKS clusters do not have an aggregator-client secret in the cluster. Because the control plane lives outside the cluster, the aggregator-client secret is not needed in the cluster.")
+			}
+
 			frontProxySecret, err := oc.AdminKubeClient().CoreV1().Secrets("openshift-kube-apiserver").Get("aggregator-client", metav1.GetOptions{})
 			o.Expect(err).NotTo(o.HaveOccurred())
 

test/extended/bootstrap_user/bootstrap_user_login.go

@@ -19,6 +19,7 @@ import (
 	"github.com/openshift/library-go/pkg/operator/events"
 	"github.com/openshift/library-go/pkg/operator/resource/resourceapply"
 	exutil "github.com/openshift/origin/test/extended/util"
+	"github.com/openshift/origin/test/extended/util/ibmcloud"
 )
 
 var _ = g.Describe("The bootstrap user", func() {
@@ -29,6 +30,9 @@ var _ = g.Describe("The bootstrap user", func() {
 	oc := exutil.NewCLI("bootstrap-login", exutil.KubeConfigPath())
 
 	g.It("should successfully login with password decoded from kubeadmin secret", func() {
+		if e2e.TestContext.Provider == ibmcloud.ProviderName {
+			e2e.Skipf("IBM ROKS clusters do not respond to the kube-system/kubeadmin secret's presence for authentication.")
+		}
 		var originalPasswordHash []byte
 		secretExists := true
 		recorder := events.NewInMemoryRecorder("")

test/extended/csrapprover/csrapprover.go

@@ -19,19 +19,26 @@ import (
 	certv1beta1 "k8s.io/api/certificates/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
+	e2e "k8s.io/kubernetes/test/e2e/framework"
 
 	kubeclient "k8s.io/client-go/kubernetes"
 	certclientv1beta1 "k8s.io/client-go/kubernetes/typed/certificates/v1beta1"
 	restclient "k8s.io/client-go/rest"
 
 	exutil "github.com/openshift/origin/test/extended/util"
+	"github.com/openshift/origin/test/extended/util/ibmcloud"
 )
 
 var _ = g.Describe("node client cert requests armoring:", func() {
 	oc := exutil.NewCLI("cluster-client-cert", exutil.KubeConfigPath())
 	defer g.GinkgoRecover()
 
 	g.It("deny pod's access to /config/master API endpoint", func() {
+
+		if e2e.TestContext.Provider == ibmcloud.ProviderName {
+			e2e.Skipf("IBM ROKS clusters do not expose machine configuration externally because they don't use RHCOS workers.")
+		}
+
 		// the /config/master API port+endpoint is only visible from inside the cluster
 		// (-> we need to create a pod to try to reach it)  and contains the token
 		// of the node-bootstrapper SA, so no random pods should be able to see it
@@ -59,6 +66,10 @@ var _ = g.Describe("node client cert requests armoring:", func() {
 	})
 
 	g.It("node-approver SA token compromised, don't approve random CSRs with client auth", func() {
+
+		if e2e.TestContext.Provider == ibmcloud.ProviderName {
+			e2e.Skipf("IBM ROKS clusters do not handle node bootstrapping in the cluster. The openshift-machine-config-operator/node-bootstrapper service account does not exist")
+		}
 		// we somehow were able to get the node-approver token, make sure we can't
 		// create node certs with client auth with it
 		priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)

test/extended/etcd/etcd_test_runner.go

@@ -15,8 +15,10 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	restclient "k8s.io/client-go/rest"
+	e2e "k8s.io/kubernetes/test/e2e/framework"
 
 	exutil "github.com/openshift/origin/test/extended/util"
+	"github.com/openshift/origin/test/extended/util/ibmcloud"
 )
 
 var _ = g.Describe("[Serial] API data in etcd", func() {
@@ -25,6 +27,10 @@ var _ = g.Describe("[Serial] API data in etcd", func() {
 	oc := exutil.NewCLI("etcd-storage-path", exutil.KubeConfigPath())
 
 	_ = g.It("should be stored at the correct location and version for all resources", func() {
+		if e2e.TestContext.Provider == ibmcloud.ProviderName {
+			e2e.Skipf("IBM ROKS clusters run etcd outside of the cluster. Etcd cannot be accessed directly from within the cluster")
+		}
+
 		ctx, cancel := context.WithCancel(context.Background())
 		cmd := exec.CommandContext(ctx, "oc", "port-forward", "service/etcd", ":2379", "-n", "openshift-etcd", "--config", exutil.KubeConfigPath())
 

test/extended/images/imagestream.go

@@ -2,6 +2,8 @@ package images
 
 import (
 	"encoding/json"
+	"fmt"
+	"math/rand"
 	"os"
 
 	"k8s.io/apimachinery/pkg/runtime"
@@ -64,10 +66,10 @@ func TestImageStreamMappingCreate(t g.GinkgoTInterface, oc *exutil.CLI) {
 	// verify we can tag a second time with the same data, and nothing changes
 	_, err = clusterAdminImageClient.ImageStreamMappings(oc.Namespace()).Create(mapping)
 	o.Expect(err).NotTo(o.HaveOccurred())
-
 	g.By("creating an image directly")
+	name := fmt.Sprintf("image-%d", rand.Intn(10000))
 	image := &imagev1.Image{
-		ObjectMeta: metav1.ObjectMeta{Name: "image2"},
+		ObjectMeta: metav1.ObjectMeta{Name: name},
 		DockerImageMetadata: runtime.RawExtension{
 			Object: &docker10.DockerImage{
 				Config: &docker10.DockerConfig{
@@ -77,14 +79,14 @@ func TestImageStreamMappingCreate(t g.GinkgoTInterface, oc *exutil.CLI) {
 		},
 	}
 	if _, err := clusterAdminImageClient.Images().Create(image); err == nil {
-		t.Error("unexpected non-error")
+		t.Fatalf("unexpected non-error")
 	}
 	defer clusterAdminImageClient.Images().Delete(image.Name, nil)
 	image.DockerImageReference = "some/other/name" // can reuse references across multiple images
 	actual, err := clusterAdminImageClient.Images().Create(image)
 	o.Expect(err).NotTo(o.HaveOccurred())
 	if actual == nil || actual.Name != image.Name {
-		t.Errorf("unexpected object: %#v", actual)
+		t.Fatalf("unexpected object: %#v", actual)
 	}
 
 	// verify that image stream mappings cannot mutate / overwrite the image (images are immutable)
@@ -107,19 +109,19 @@ func TestImageStreamMappingCreate(t g.GinkgoTInterface, oc *exutil.CLI) {
 
 	o.Expect(err).NotTo(o.HaveOccurred())
 	if updated.Spec.Tags != nil && len(updated.Spec.Tags) > 0 {
-		t.Errorf("unexpected object: %#v", updated.Spec.Tags)
+		t.Fatalf("unexpected object: %#v", updated.Spec.Tags)
 	}
 
 	fromTag, err := clusterAdminImageClient.ImageStreamTags(oc.Namespace()).Get(stream.Name+":newer", metav1.GetOptions{})
 	o.Expect(err).NotTo(o.HaveOccurred())
 	if fromTag.Name != "test:newer" || fromTag.Image.UID == "" || fromTag.Image.DockerImageReference != "some/other/name" {
-		t.Errorf("unexpected object: %#v", fromTag)
+		t.Fatalf("unexpected object: %#v", fromTag)
 	}
 
 	fromTag, err = clusterAdminImageClient.ImageStreamTags(oc.Namespace()).Get(stream.Name+":newest", metav1.GetOptions{})
 	o.Expect(err).NotTo(o.HaveOccurred())
 	if fromTag.Name != "test:newest" || fromTag.Image.UID == "" || fromTag.Image.DockerImageReference != "different" {
-		t.Errorf("unexpected object: %#v", fromTag)
+		t.Fatalf("unexpected object: %#v", fromTag)
 	}
 
 	// verify that image stream mappings can use the same image for different tags
@@ -135,7 +137,7 @@ func TestImageStreamMappingCreate(t g.GinkgoTInterface, oc *exutil.CLI) {
 	updated, err = clusterAdminImageClient.ImageStreams(oc.Namespace()).Get(stream.Name, metav1.GetOptions{})
 	o.Expect(err).NotTo(o.HaveOccurred())
 	if updated.Spec.Tags != nil && len(updated.Spec.Tags) > 0 {
-		t.Errorf("unexpected object: %#v", updated.Spec.Tags)
+		t.Fatalf("unexpected object: %#v", updated.Spec.Tags)
 	}
 
 	// expect not found error for non-existent imagestream tag
@@ -146,18 +148,18 @@ func TestImageStreamMappingCreate(t g.GinkgoTInterface, oc *exutil.CLI) {
 	fromTag, err = clusterAdminImageClient.ImageStreamTags(oc.Namespace()).Get(stream.Name+":newer", metav1.GetOptions{})
 	o.Expect(err).NotTo(o.HaveOccurred())
 	if fromTag.Name != "test:newer" || fromTag.Image.UID == "" || fromTag.Image.DockerImageReference != "some/other/name" {
-		t.Errorf("unexpected object: %#v", fromTag)
+		t.Fatalf("unexpected object: %#v", fromTag)
 	}
 
 	fromTag, err = clusterAdminImageClient.ImageStreamTags(oc.Namespace()).Get(stream.Name+":newest", metav1.GetOptions{})
 	o.Expect(err).NotTo(o.HaveOccurred())
 	if fromTag.Name != "test:newest" || fromTag.Image.UID == "" || fromTag.Image.DockerImageReference != "different" {
-		t.Errorf("unexpected object: %#v", fromTag)
+		t.Fatalf("unexpected object: %#v", fromTag)
 	}
 	fromTag, err = clusterAdminImageClient.ImageStreamTags(oc.Namespace()).Get(stream.Name+":anothertag", metav1.GetOptions{})
 	o.Expect(err).NotTo(o.HaveOccurred())
 	if fromTag.Name != "test:anothertag" || fromTag.Image.UID == "" || fromTag.Image.DockerImageReference != "some/other/name" {
-		t.Errorf("unexpected object: %#v", fromTag)
+		t.Fatalf("unexpected object: %#v", fromTag)
 	}
 
 	// try an update with an incorrect resource version - needs to have conflict error
@@ -187,7 +189,7 @@ func TestImageStreamMappingCreate(t g.GinkgoTInterface, oc *exutil.CLI) {
 	o.Expect(err).NotTo(o.HaveOccurred())
 
 	if fromTag.Name != "test:brandnew" || fromTag.Image.UID == "" || fromTag.Tag.From.Name != "newest" {
-		t.Errorf("unexpected object: %#v", fromTag)
+		t.Fatalf("unexpected object: %#v", fromTag)
 	}
 }
 
@@ -198,7 +200,7 @@ func TestImageStreamWithoutDockerImageConfig(t g.GinkgoTInterface, oc *exutil.CL
 	expected, err := clusterAdminImageClient.ImageStreams(oc.Namespace()).Create(stream)
 	o.Expect(err).NotTo(o.HaveOccurred())
 	if expected.Name == "" {
-		t.Errorf("Unexpected empty image Name %v", expected)
+		t.Fatalf("Unexpected empty image Name %v", expected)
 	}
 
 	imageConfig := docker10.DockerConfig{

test/extended/oauth/oauthcertfallback.go

@@ -17,6 +17,7 @@ import (
 	userv1client "github.com/openshift/client-go/user/clientset/versioned"
 	"github.com/openshift/library-go/pkg/crypto"
 	exutil "github.com/openshift/origin/test/extended/util"
+	"github.com/openshift/origin/test/extended/util/ibmcloud"
 )
 
 const (
@@ -36,6 +37,9 @@ var _ = g.Describe("[Feature:OAuthServer] OAuth server", func() {
 		if len(os.Getenv("TEST_UNSUPPORTED_ALLOW_VERSION_SKEW")) > 0 {
 			e2e.Skipf("Authenticator order changed in 4.4 to match kubernetes which precludes running this test against a skewed cluster.")
 		}
+		if e2e.TestContext.Provider == ibmcloud.ProviderName {
+			e2e.Skipf("IBM ROKS clusters do not contain a kube-control-plane-signer secret inside the cluster. The secret lives outside the cluster with the rest of the control plane.")
+		}
 		var (
 			// We have to generate this dynamically in order to have an invalid cert signed by a signer with the same name as the valid CA
 			invalidCert = restclient.TLSClientConfig{}

test/extended/oauth/requestheaders.go

@@ -32,6 +32,7 @@ import (
 	clusteroperatorhelpers "github.com/openshift/library-go/pkg/config/clusteroperator/v1helpers"
 
 	exutil "github.com/openshift/origin/test/extended/util"
+	"github.com/openshift/origin/test/extended/util/ibmcloud"
 )
 
 func init() {
@@ -60,6 +61,11 @@ var _ = g.Describe("[Serial] [Feature:OAuthServer] [RequestHeaders] [IdP]", func
 	var oc = exutil.NewCLI("request-headers", exutil.KubeConfigPath())
 
 	g.It("test RequestHeaders IdP", func() {
+
+		if e2e.TestContext.Provider == ibmcloud.ProviderName {
+			e2e.Skipf("IBM ROKS clusters do not allow customization of the Identity Providers for the cluster.")
+		}
+
 		caCert, caKey := createClientCA(oc.AdminKubeClient().CoreV1())
 		defer oc.AdminKubeClient().CoreV1().ConfigMaps("openshift-config").Delete(clientCAName, &metav1.DeleteOptions{})
 

test/extended/oauth/well_known.go

@@ -11,10 +11,12 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/rest"
+	e2e "k8s.io/kubernetes/test/e2e/framework"
 
 	"github.com/openshift/library-go/pkg/oauth/oauthdiscovery"
 
 	exutil "github.com/openshift/origin/test/extended/util"
+	"github.com/openshift/origin/test/extended/util/ibmcloud"
 )
 
 var _ = g.Describe("[Feature:OAuthServer] well-known endpoint", func() {
@@ -32,14 +34,19 @@ var _ = g.Describe("[Feature:OAuthServer] well-known endpoint", func() {
 		metadata := &oauthdiscovery.OauthAuthorizationServerMetadata{}
 		err = json.Unmarshal([]byte(metadataJSON), metadata)
 		o.Expect(err).NotTo(o.HaveOccurred())
+
+		// If not running on an IBM ROKS cluster,
 		// compare to openshift-authentication route
-		route, err := oc.AdminRouteClient().RouteV1().Routes(oauthNamespace).Get(oauthRoute, metav1.GetOptions{})
-		o.Expect(err).NotTo(o.HaveOccurred())
-		u, err := url.Parse("https://" + route.Spec.Host)
-		o.Expect(err).NotTo(o.HaveOccurred())
-		u.Path = path.Join(u.Path, "oauth/authorize")
-		authEndpointFromRoute := u.String()
-		o.Expect(metadata.AuthorizationEndpoint).To(o.Equal(authEndpointFromRoute))
+		// (On a ROKS cluster the openshift-authentication route does not live in the cluster)
+		if e2e.TestContext.Provider != ibmcloud.ProviderName {
+			route, err := oc.AdminRouteClient().RouteV1().Routes(oauthNamespace).Get(oauthRoute, metav1.GetOptions{})
+			o.Expect(err).NotTo(o.HaveOccurred())
+			u, err := url.Parse("https://" + route.Spec.Host)
+			o.Expect(err).NotTo(o.HaveOccurred())
+			u.Path = path.Join(u.Path, "oauth/authorize")
+			authEndpointFromRoute := u.String()
+			o.Expect(metadata.AuthorizationEndpoint).To(o.Equal(authEndpointFromRoute))
+		}
 		tlsClientConfig, err := rest.TLSConfigFor(oc.AdminConfig())
 		o.Expect(err).NotTo(o.HaveOccurred())
 

test/extended/prometheus/prometheus.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/openshift/origin/test/extended/networking"
 	exutil "github.com/openshift/origin/test/extended/util"
+	"github.com/openshift/origin/test/extended/util/ibmcloud"
 )
 
 const waitForPrometheusStartSeconds = 240
@@ -129,20 +130,26 @@ var _ = g.Describe("[Feature:Prometheus][Conformance] Prometheus", func() {
 				o.Expect(err).NotTo(o.HaveOccurred())
 
 				g.By("verifying all expected jobs have a working target")
-				lastErrs = all(
-					// The OpenShift control plane
-					targets.Expect(labels{"job": "api"}, "up", "^https://.*/metrics$"),
-					targets.Expect(labels{"job": "controller-manager"}, "up", "^https://.*/metrics$"),
-
-					// The kube control plane
-					targets.Expect(labels{"job": "etcd"}, "up", "^https://.*/metrics$"),
-					targets.Expect(labels{"job": "apiserver"}, "up", "^https://.*/metrics$"),
-					targets.Expect(labels{"job": "kube-controller-manager"}, "up", "^https://.*/metrics$"),
-					targets.Expect(labels{"job": "scheduler"}, "up", "^https://.*/metrics$"),
-					targets.Expect(labels{"job": "kube-state-metrics"}, "up", "^https://.*/metrics$"),
 
+				if e2e.TestContext.Provider != ibmcloud.ProviderName {
+					lastErrs = all(
+						// The OpenShift control plane
+						targets.Expect(labels{"job": "api"}, "up", "^https://.*/metrics$"),
+						targets.Expect(labels{"job": "controller-manager"}, "up", "^https://.*/metrics$"),
+
+						// The kube control plane
+						targets.Expect(labels{"job": "etcd"}, "up", "^https://.*/metrics$"),
+						targets.Expect(labels{"job": "apiserver"}, "up", "^https://.*/metrics$"),
+						targets.Expect(labels{"job": "kube-controller-manager"}, "up", "^https://.*/metrics$"),
+						targets.Expect(labels{"job": "scheduler"}, "up", "^https://.*/metrics$"),
+						targets.Expect(labels{"job": "kube-state-metrics"}, "up", "^https://.*/metrics$"),
+
+						// Cluster version operator
+						targets.Expect(labels{"job": "cluster-version-operator"}, "up", "^http://.*/metrics$"),
+					)
+				}
+				lastErrs = append(lastErrs, all(
 					// TODO: should probably be https
-					targets.Expect(labels{"job": "cluster-version-operator"}, "up", "^http://.*/metrics$"),
 					targets.Expect(labels{"job": "prometheus-k8s", "namespace": "openshift-monitoring", "pod": "prometheus-k8s-0"}, "up", "^https://.*/metrics$"),
 					targets.Expect(labels{"job": "kubelet"}, "up", "^https://.*/metrics$"),
 					targets.Expect(labels{"job": "kubelet"}, "up", "^https://.*/metrics/cadvisor$"),
@@ -151,7 +158,7 @@ var _ = g.Describe("[Feature:Prometheus][Conformance] Prometheus", func() {
 					targets.Expect(labels{"job": "alertmanager-main"}, "up", "^https://.*/metrics$"),
 					targets.Expect(labels{"job": "crio"}, "up", "^http://.*/metrics$"),
 					targets.Expect(labels{"job": "telemeter-client"}, "up", "^https://.*/metrics$"),
-				)
+				)...)
 				if len(lastErrs) > 0 {
 					e2e.Logf("missing some targets: %v", lastErrs)
 					return false, nil

test/extended/prometheus/prometheus_builds.go

@@ -15,6 +15,7 @@ import (
 
 	buildv1 "github.com/openshift/api/build/v1"
 	exutil "github.com/openshift/origin/test/extended/util"
+	"github.com/openshift/origin/test/extended/util/ibmcloud"
 )
 
 var _ = g.Describe("[Feature:Prometheus][Feature:Builds] Prometheus", func() {
@@ -41,6 +42,11 @@ var _ = g.Describe("[Feature:Prometheus][Feature:Builds] Prometheus", func() {
 
 	g.Describe("when installed on the cluster", func() {
 		g.It("should start and expose a secured proxy and verify build metrics", func() {
+
+			if e2e.TestContext.Provider == ibmcloud.ProviderName {
+				e2e.Skipf("Prometheus in IBM ROKS clusters does not collect metrics from the OpenShift Controller Manager. The openshift_build_total metric expected by this test is reported by the OCM")
+			}
+
 			oc.SetupProject()
 			ns := oc.Namespace()
 			appTemplate := exutil.FixturePath("testdata", "builds", "build-pruning", "successful-build-config.yaml")