Merge pull request #18 from gabemontero/add-e2e-guts

BUILD-191: add e2e tests

Makefile

@@ -25,9 +25,24 @@ test: ## Run unit tests. Example: make test
 	go test ./cmd/... ./pkg/...
 .PHONY: test
 
+test-e2e:
+	./deploy/deploy.sh
+	KUBERNETES_CONFIG=${KUBECONFIG} go test -count 1 -tags normal -timeout 30m -v ./test/e2e/...
+.PHONY: test-e2e
+
+test-e2e-slow:
+	./deploy/deploy.sh
+	KUBERNETES_CONFIG=${KUBECONFIG} go test -count 1 -tags slow -timeout 30m -v ./test/e2e/...
+.PHONY: test-e2e
+
+test-e2e-disruptive:
+	./deploy/deploy.sh
+	KUBERNETES_CONFIG=${KUBECONFIG} go test -count 1 -tags disruptive -timeout 30m -v ./test/e2e/...
+.PHONY: test-e2e
+
 verify: ## Run verifications. Example: make verify
-	go vet ./cmd/... ./pkg/...
-	gofmt -w ./cmd/ ./pkg/
+	go vet ./cmd/... ./pkg/... ./test/...
+	gofmt -w ./cmd/ ./pkg/ ./test/
 .PHONY: verify
 
 build: ## Build the executable. Example: make build

cmd/main.go

@@ -12,6 +12,7 @@ import (
 
 	"k8s.io/klog/v2"
 
+	"github.com/openshift/csi-driver-projected-resource/pkg/client"
 	"github.com/openshift/csi-driver-projected-resource/pkg/controller"
 	"github.com/openshift/csi-driver-projected-resource/pkg/hostpath"
 )
@@ -61,7 +62,7 @@ func init() {
 
 	rootCmd.Flags().AddGoFlagSet(flag.CommandLine)
 	rootCmd.Flags().StringVar(&endPoint, "endpoint", "unix://tmp/csi.sock", "CSI endpoint")
-	rootCmd.Flags().StringVar(&driverName, "drivername", "csi-driver-projected-resource.openshift.io", "name of the driver")
+	rootCmd.Flags().StringVar(&driverName, "drivername", client.DriverName, "name of the driver")
 	rootCmd.Flags().StringVar(&nodeID, "nodeid", "", "node id")
 	rootCmd.Flags().Int64Var(&maxVolumesPerNode, "maxvolumespernode", 0, "limit of volumes per node")
 	rootCmd.Flags().StringVar(&shareRelistInterval, "share-relist-interval", "",

go.mod

@@ -5,14 +5,15 @@ go 1.14
 require (
 	github.com/container-storage-interface/spec v1.3.0
 	github.com/kubernetes-csi/csi-lib-utils v0.7.0
-	github.com/spf13/cobra v1.0.0
+	github.com/spf13/cobra v1.1.1
 	github.com/spf13/pflag v1.0.5
-	golang.org/x/net v0.0.0-20200707034311-ab3426394381
+	golang.org/x/net v0.0.0-20201110031124-69a78807bb2b
 	google.golang.org/grpc v1.31.0
-	k8s.io/api v0.19.0
-	k8s.io/apimachinery v0.19.0
-	k8s.io/client-go v0.19.0
-	k8s.io/code-generator v0.19.0
-	k8s.io/klog/v2 v2.2.0
-	k8s.io/utils v0.0.0-20200731180307-f00132d28269
+	k8s.io/api v0.20.1
+	k8s.io/apimachinery v0.20.1
+	k8s.io/client-go v0.20.1
+	k8s.io/code-generator v0.20.1
+	k8s.io/klog/v2 v2.4.0
+	k8s.io/kubectl v0.20.1
+	k8s.io/utils v0.0.0-20201110183641-67b214c5f920
 )

pkg/client/client.go

@@ -22,6 +22,11 @@ import (
 	"k8s.io/client-go/tools/record"
 )
 
+const (
+	DefaultNamespace = "csi-driver-projected-resource"
+	DriverName       = "csi-driver-projected-resource.openshift.io"
+)
+
 var (
 	kubeClient kubernetes.Interface
 	recorder   record.EventRecorder
@@ -79,8 +84,8 @@ func initClient() error {
 
 	}
 	eventBroadcaster := record.NewBroadcaster()
-	eventBroadcaster.StartRecordingToSink(&ktypedclient.EventSinkImpl{Interface: kubeClient.CoreV1().Events("csi-driver-projected-resource")})
-	recorder = eventBroadcaster.NewRecorder(runtime.NewScheme(), corev1.EventSource{Component: "csi-driver-projected-resource"})
+	eventBroadcaster.StartRecordingToSink(&ktypedclient.EventSinkImpl{Interface: kubeClient.CoreV1().Events(DefaultNamespace)})
+	recorder = eventBroadcaster.NewRecorder(runtime.NewScheme(), corev1.EventSource{Component: DefaultNamespace})
 	return nil
 }
 

test/e2e/disruptive_test.go

@@ -0,0 +1,25 @@
+// +build disruptive
+
+package e2e
+
+import (
+	"github.com/openshift/csi-driver-projected-resource/test/framework"
+	"testing"
+	"time"
+)
+
+func TestBasicThenDriverRestartThenChangeShare(t *testing.T) {
+	prep(t)
+	testNS := framework.CreateTestNamespace(t)
+	defer framework.CleanupTestNamespace(testNS, t)
+	basicShareSetupAndVerification(testNS, t)
+
+	t.Logf("initiating csi driver restart")
+	framework.RestartDaemonSet(t)
+	t.Logf("csi driver restart complete, check test pod")
+	framework.ExecPod(testNS, "openshift-config:openshift-install", false, 30*time.Second, t)
+
+	t.Logf("now changing share")
+	framework.ChangeShare(testNS, t)
+	framework.ExecPod(testNS, "openshift-config:pull-secret", false, 30*time.Second, t)
+}

test/e2e/main_test.go

@@ -0,0 +1,10 @@
+package e2e
+
+import (
+	"os"
+	"testing"
+)
+
+func TestMain(m *testing.M) {
+	os.Exit(m.Run())
+}

test/e2e/normal_test.go

@@ -0,0 +1,42 @@
+// +build normal
+
+package e2e
+
+import (
+	"github.com/openshift/csi-driver-projected-resource/test/framework"
+	"testing"
+	"time"
+)
+
+func TestNoRBAC(t *testing.T) {
+	prep(t)
+	testNS := framework.CreateTestNamespace(t)
+	defer framework.CleanupTestNamespace(testNS, t)
+	framework.CreateShare(testNS, t)
+	framework.CreateTestPod(testNS, false, t)
+}
+
+func TestNoShare(t *testing.T) {
+	prep(t)
+	testNS := framework.CreateTestNamespace(t)
+	defer framework.CleanupTestNamespace(testNS, t)
+	framework.CreateShareRelatedRBAC(testNS, t)
+	framework.CreateTestPod(testNS, false, t)
+}
+
+func TestBasicThenNoShareThenShare(t *testing.T) {
+	prep(t)
+	testNS := framework.CreateTestNamespace(t)
+	defer framework.CleanupTestNamespace(testNS, t)
+	basicShareSetupAndVerification(testNS, t)
+
+	t.Logf("deleting share for %s", testNS)
+
+	framework.DeleteShare(testNS, t)
+	framework.ExecPod(testNS, "openshift-config:openshift-install", true, 30*time.Second, t)
+
+	t.Logf("adding share back for %s", testNS)
+
+	framework.CreateShare(testNS, t)
+	framework.ExecPod(testNS, "openshift-config:openshift-install", false, 30*time.Second, t)
+}

test/e2e/slow_test.go

@@ -0,0 +1,25 @@
+// +build slow
+
+package e2e
+
+import (
+	"github.com/openshift/csi-driver-projected-resource/test/framework"
+	"testing"
+	"time"
+)
+
+// this requires up to a 20 minute delay for 2 separate relist
+func TestBasicThenNoRBACThenRBAC(t *testing.T) {
+	prep(t)
+	testNS := framework.CreateTestNamespace(t)
+	defer framework.CleanupTestNamespace(testNS, t)
+	basicShareSetupAndVerification(testNS, t)
+
+	framework.DeleteShareRelatedRBAC(testNS, t)
+	t.Logf("wait up to 10 minutes for examining pod %s since the controller does not currently watch all clusterroles and clusterrolebindings and reverse engineer which ones satisfied the SAR calls, so we wait for relist on shares", testNS)
+	framework.ExecPod(testNS, "openshift-config:openshift-install", true, 10*time.Minute, t)
+
+	framework.CreateShareRelatedRBAC(testNS, t)
+	t.Logf("wait up to 10 minutes for examining pod %s since the controller does not currently watch all clusterroles and clusterrolebindings and reverse engineer which ones satisfied the SAR calls, so we wait for relist on shares", testNS)
+	framework.ExecPod(testNS, "openshift-config:openshift-install", false, 10*time.Minute, t)
+}

test/e2e/util_test.go

@@ -0,0 +1,25 @@
+package e2e
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/openshift/csi-driver-projected-resource/test/framework"
+)
+
+func prep(t *testing.T) {
+	framework.SetupClients(t)
+	err := framework.WaitForDaemonSet(true)
+	if err != nil {
+		framework.LogAndDebugTestError(fmt.Sprintf("csi driver daemon not up: %s", err.Error()), t)
+	}
+}
+
+func basicShareSetupAndVerification(name string, t *testing.T) {
+	framework.CreateShareRelatedRBAC(name, t)
+	framework.CreateShare(name, t)
+	framework.CreateTestPod(name, true, t)
+	framework.ExecPod(name, "openshift-config:openshift-install", false, 30*time.Second, t)
+
+}

test/framework/client.go

@@ -0,0 +1,78 @@
+package framework
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"testing"
+
+	kubeset "k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
+	rbacv1client "k8s.io/client-go/kubernetes/typed/rbac/v1"
+	"k8s.io/client-go/rest"
+
+	"github.com/openshift/csi-driver-projected-resource/pkg/client"
+	shareset "github.com/openshift/csi-driver-projected-resource/pkg/generated/clientset/versioned"
+)
+
+var (
+	kubeConfig               *rest.Config
+	kubeClient               *kubeset.Clientset
+	podClient                corev1client.PodInterface
+	restClient               *rest.RESTClient
+	namespaceClient          corev1client.NamespaceInterface
+	clusterRoleClient        rbacv1client.ClusterRoleInterface
+	clusterRoleBindingClient rbacv1client.ClusterRoleBindingInterface
+	shareClient              shareset.Interface
+)
+
+func SetupClients(t *testing.T) {
+	var err error
+	if kubeConfig == nil {
+		kubeConfig, err = client.GetConfig()
+		if err != nil {
+			t.Fatalf("%#v", err)
+		}
+	}
+	if kubeClient == nil {
+		kubeClient, err = kubeset.NewForConfig(kubeConfig)
+		if err != nil {
+			t.Fatalf("%#v", err)
+		}
+	}
+	if restClient == nil {
+		restClient, err = rest.RESTClientFor(setRESTConfigDefaults(*kubeConfig))
+		if err != nil {
+			t.Fatalf("%#v", err)
+		}
+	}
+	if namespaceClient == nil {
+		namespaceClient = kubeClient.CoreV1().Namespaces()
+	}
+	if podClient == nil {
+		podClient = kubeClient.CoreV1().Pods(client.DefaultNamespace)
+	}
+	if clusterRoleClient == nil {
+		clusterRoleClient = kubeClient.RbacV1().ClusterRoles()
+	}
+	if clusterRoleBindingClient == nil {
+		clusterRoleBindingClient = kubeClient.RbacV1().ClusterRoleBindings()
+	}
+	shareClient, err = shareset.NewForConfig(kubeConfig)
+	if err != nil {
+		t.Fatalf("%#v", err)
+	}
+}
+
+func setRESTConfigDefaults(config rest.Config) *rest.Config {
+	if config.GroupVersion == nil {
+		config.GroupVersion = &schema.GroupVersion{Group: "", Version: "v1"}
+	}
+	if config.NegotiatedSerializer == nil {
+		config.NegotiatedSerializer = scheme.Codecs
+	}
+	if len(config.UserAgent) == 0 {
+		config.UserAgent = rest.DefaultKubernetesUserAgent()
+	}
+	config.APIPath = "/api"
+	return &config
+}

test/framework/daemonset.go

@@ -0,0 +1,95 @@
+package framework
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	"github.com/openshift/csi-driver-projected-resource/pkg/client"
+)
+
+func WaitForDaemonSet(up bool) error {
+	dsClient := kubeClient.AppsV1().DaemonSets(client.DefaultNamespace)
+	err := wait.PollImmediate(1*time.Second, 10*time.Minute, func() (bool, error) {
+		_, err := dsClient.Get(context.TODO(), "csi-hostpathplugin", metav1.GetOptions{})
+		if err != nil {
+			fmt.Printf("error waiting for driver daemonset to exist: %v\n", err)
+			return false, nil
+		}
+		fmt.Println("found operator deployment")
+		return true, nil
+	})
+	podClient = kubeClient.CoreV1().Pods(client.DefaultNamespace)
+	err = wait.PollImmediate(10*time.Second, 2*time.Minute, func() (bool, error) {
+		podList, err := podClient.List(context.TODO(), metav1.ListOptions{})
+		if err != nil {
+			fmt.Printf("error listing pods: %s\n", err.Error())
+			return false, nil
+		}
+
+		if up {
+			if podList.Items == nil || len(podList.Items) != 3 {
+				fmt.Printf("number of pods not yet at 3\n")
+				return false, nil
+			}
+			for _, pod := range podList.Items {
+				if pod.Status.Phase != corev1.PodRunning || pod.DeletionTimestamp != nil {
+					fmt.Printf("pod %s in phase %s with deletion timestamp %v\n", pod.Name, pod.Status.Phase, pod.DeletionTimestamp)
+					return false, nil
+				}
+			}
+			fmt.Printf("all 3 daemonset pods are running\n")
+		} else {
+			if podList.Items == nil || len(podList.Items) == 0 {
+				fmt.Printf("pod list emtpy so daemonset is down\n")
+				return true, nil
+			}
+			for _, pod := range podList.Items {
+				fmt.Printf("pod %s has status %s and delete timestamp %v\n", pod.Name, pod.Status.Phase, pod.DeletionTimestamp)
+				if pod.DeletionTimestamp == nil {
+					fmt.Printf("pod %s still does not have a deletion timestamp\n", pod.Name)
+					return false, nil
+				}
+			}
+			fmt.Printf("all daemonset pods are either gone or have deletion timestamp\n")
+		}
+		return true, nil
+	})
+	return err
+}
+
+func RestartDaemonSet(t *testing.T) {
+	err := wait.PollImmediate(1*time.Second, 5*time.Second, func() (bool, error) {
+		podList, err := podClient.List(context.TODO(), metav1.ListOptions{})
+		if err != nil {
+			fmt.Printf("unexpected error list driver pods: %s", err.Error())
+			return false, nil
+		}
+		for _, pod := range podList.Items {
+			err = podClient.Delete(context.TODO(), pod.Name, metav1.DeleteOptions{})
+			if err != nil {
+				fmt.Printf("unexpected error deleting pod %s: %s", pod.Name, err.Error())
+			}
+		}
+		return true, nil
+	})
+	if err != nil {
+		LogAndDebugTestError(fmt.Sprintf("could not delete driver pods in time: %s", err.Error()), t)
+	}
+
+	err = WaitForDaemonSet(false)
+	if err != nil {
+		LogAndDebugTestError("csi driver pod deletion not recognized in time", t)
+	}
+	t.Logf("csi driver pods deletion confirmed, now waiting on pod recreate")
+	// k8s will recreate pods after delete automatically
+	err = WaitForDaemonSet(true)
+	if err != nil {
+		LogAndDebugTestError("csi driver restart not recognized in time", t)
+	}
+}