Skip to content

Commit

Permalink
Use CSI controller cluster kubeconfig if provided and csi pod log col…
Browse files Browse the repository at this point in the history 
…lection improvements
  • Loading branch information
@sashrith
sashrith committed Jun 18, 2022
1 parent 76b983e commit db6ce48
Show file tree
Hide file tree
Showing 5 changed files with 83 additions and 24 deletions.
3 changes: 3 additions & 0 deletions tests/e2e/docs/vanilla_cluster_setup.md
View file
Expand Up @@ -81,6 +81,9 @@ list of datastore URLs where you want to deploy file share volumes. Retrieve thi
# To run common e2e tests (block & file), need to set the following env variable to identify the file volume setup
export ACCESS_MODE="RWX"

# If CSI controller is running on a different k8s cluster, we need to pass the kubeconfig for that cluster using
export CONTROLLER_CLUSTER_KUBECONFIG="/path/to/remote/kubeconfig"

### To run full sync test, need do extra following steps

#### Setting SSH keys for VC with your local machine to run full sync test
Expand Down
1 change: 1 addition & 0 deletions tests/e2e/e2e_common.go
View file
Expand Up @@ -33,6 +33,7 @@ const (
busyBoxImageOnGcr = "gcr.io/google_containers/busybox:1.27"
nginxImage = "k8s.gcr.io/nginx-slim:0.8"
configSecret = "vsphere-config-secret"
contollerClusterKubeConfig = "CONTROLLER_CLUSTER_KUBECONFIG"
crdCNSNodeVMAttachment = "cnsnodevmattachments"
crdCNSVolumeMetadatas = "cnsvolumemetadatas"
crdCNSFileAccessConfig = "cnsfileaccessconfigs"
Expand Down
18 changes: 13 additions & 5 deletions tests/e2e/fullsync_test_for_block_volume.go
View file
Expand Up @@ -725,6 +725,14 @@ var _ bool = ginkgo.Describe("full-sync-test", func() {
pandoraSyncWaitTime = defaultPandoraSyncWaitTime
}

c := client
controllerClusterConfig := os.Getenv(contollerClusterKubeConfig)
if controllerClusterConfig != "" {
framework.Logf("Creating client for remote kubeconfig")
remoteC, err := createKubernetesClientFromConfig(controllerClusterConfig)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
c = remoteC
}
datastoreURL = GetAndExpectStringEnvVar(envSharedDatastoreURL)
finder := find.NewFinder(e2eVSphere.Client.Client, false)

Expand All @@ -745,14 +753,14 @@ var _ bool = ginkgo.Describe("full-sync-test", func() {
time.Sleep(time.Duration(pandoraSyncWaitTime) * time.Second)

ginkgo.By("Get controller replica count before scaling down")
deployment, err := client.AppsV1().Deployments(csiControllerNamespace).Get(ctx,
deployment, err := c.AppsV1().Deployments(csiControllerNamespace).Get(ctx,
vSphereCSIControllerPodNamePrefix, metav1.GetOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
replica_before_scale_down := *deployment.Spec.Replicas
framework.Logf("Replica before scale down %d", replica_before_scale_down)

ginkgo.By("Scaling down the csi driver to zero replica")
deployment = updateDeploymentReplica(client, 0, vSphereCSIControllerPodNamePrefix, csiControllerNamespace)
deployment = updateDeploymentReplica(c, 0, vSphereCSIControllerPodNamePrefix, csiControllerNamespace)
ginkgo.By(fmt.Sprintf("Successfully scaled down the csi driver deployment:%s to zero replicas", deployment.Name))

ginkgo.By(fmt.Sprintf("Creating the PV with the fcdID %s", fcdID))
Expand All @@ -762,10 +770,10 @@ var _ bool = ginkgo.Describe("full-sync-test", func() {
pv, err = client.CoreV1().PersistentVolumes().Create(ctx, pv, metav1.CreateOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())

ginkgo.By("Scaling up the csi driver to one replica")
deployment = updateDeploymentReplica(client, replica_before_scale_down,
ginkgo.By("Scaling up the csi driver")
deployment = updateDeploymentReplica(c, replica_before_scale_down,
vSphereCSIControllerPodNamePrefix, csiControllerNamespace)
ginkgo.By(fmt.Sprintf("Successfully scaled up the csi driver deployment:%s to one replica", deployment.Name))
ginkgo.By(fmt.Sprintf("Successfully scaled up the csi driver deployment:%s", deployment.Name))

ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow full sync finish", fullSyncWaitTime))
time.Sleep(time.Duration(fullSyncWaitTime) * time.Second)
Expand Down
82 changes: 65 additions & 17 deletions tests/e2e/improved_csi_idempotency.go
View file
Expand Up @@ -47,6 +47,7 @@ var _ = ginkgo.Describe("[csi-block-vanilla] [csi-file-vanilla] "+
const defaultVolumeOpsScaleWCP = 29
var (
client clientset.Interface
remoteC clientset.Interface
fullSyncWaitTime int
namespace string
scParameters map[string]string
Expand Down Expand Up @@ -104,15 +105,32 @@ var _ = ginkgo.Describe("[csi-block-vanilla] [csi-file-vanilla] "+
}

// Get CSI Controller's replica count from the setup
deployment, err = client.AppsV1().Deployments(csiSystemNamespace).Get(ctx,
vSphereCSIControllerPodNamePrefix, metav1.GetOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
csiReplicaCount = *deployment.Spec.Replicas
controllerClusterConfig := os.Getenv(contollerClusterKubeConfig)
if controllerClusterConfig != "" {
framework.Logf("Creating client for remote kubeconfig")
remoteC, err = createKubernetesClientFromConfig(controllerClusterConfig)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
deployment, err = remoteC.AppsV1().Deployments(csiSystemNamespace).Get(ctx,
vSphereCSIControllerPodNamePrefix, metav1.GetOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
csiReplicaCount = *deployment.Spec.Replicas

} else {
deployment, err = client.AppsV1().Deployments(csiSystemNamespace).Get(ctx,
vSphereCSIControllerPodNamePrefix, metav1.GetOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
csiReplicaCount = *deployment.Spec.Replicas
}
})

ginkgo.AfterEach(func() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
controllerClusterConfig := os.Getenv(contollerClusterKubeConfig)
c := client
if controllerClusterConfig != "" {
c = remoteC
}
if supervisorCluster {
deleteResourceQuota(client, namespace)
}
Expand All @@ -124,7 +142,7 @@ var _ = ginkgo.Describe("[csi-block-vanilla] [csi-file-vanilla] "+
if serviceName == "CSI" {
framework.Logf("Starting CSI driver")
ignoreLabels := make(map[string]string)
err := updateDeploymentReplicawithWait(client, csiReplicaCount, vSphereCSIControllerPodNamePrefix,
err := updateDeploymentReplicawithWait(c, csiReplicaCount, vSphereCSIControllerPodNamePrefix,
csiSystemNamespace)
gomega.Expect(err).NotTo(gomega.HaveOccurred())

Expand Down Expand Up @@ -152,7 +170,7 @@ var _ = ginkgo.Describe("[csi-block-vanilla] [csi-file-vanilla] "+
}

ginkgo.By(fmt.Sprintf("Resetting provisioner time interval to %s sec", defaultProvisionerTimeInSec))
updateCSIDeploymentProvisionerTimeout(client, csiSystemNamespace, defaultProvisionerTimeInSec)
updateCSIDeploymentProvisionerTimeout(c, csiSystemNamespace, defaultProvisionerTimeInSec)
})

/*
Expand Down Expand Up @@ -306,6 +324,8 @@ func createVolumesByReducingProvisionerTime(namespace string, client clientset.I
var err error
pvclaims = make([]*v1.PersistentVolumeClaim, volumeOpsScale)

c := client

// Decide which test setup is available to run
if vanillaCluster {
ginkgo.By("CNS_TEST: Running for vanilla k8s setup")
Expand All @@ -322,6 +342,14 @@ func createVolumesByReducingProvisionerTime(namespace string, client clientset.I
curtimestring := strconv.FormatInt(curtime, 10)
scName := "idempotency" + curtimestring + val
storageclass, err = createStorageClass(client, scParameters, nil, "", "", false, scName)

controllerClusterConfig := os.Getenv(contollerClusterKubeConfig)
if controllerClusterConfig != "" {
framework.Logf("Creating client for remote kubeconfig")
remoteC, err := createKubernetesClientFromConfig(controllerClusterConfig)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
c = remoteC
}
} else if supervisorCluster {
ginkgo.By("CNS_TEST: Running for WCP setup")
thickProvPolicy := os.Getenv(envStoragePolicyNameWithThickProvision)
Expand Down Expand Up @@ -357,15 +385,15 @@ func createVolumesByReducingProvisionerTime(namespace string, client clientset.I
}()

// TODO: Stop printing csi logs on the console
collectPodLogs(ctx, client, csiSystemNamespace)
collectPodLogs(ctx, c, csiSystemNamespace)

// This assumes the tkg-controller-manager's auto sync is disabled
ginkgo.By(fmt.Sprintf("Reducing Provisioner time interval to %s Sec for the test...", customProvisionerTimeout))
updateCSIDeploymentProvisionerTimeout(client, csiSystemNamespace, customProvisionerTimeout)
updateCSIDeploymentProvisionerTimeout(c, csiSystemNamespace, customProvisionerTimeout)

defer func() {
ginkgo.By(fmt.Sprintf("Resetting provisioner time interval to %s sec", defaultProvisionerTimeInSec))
updateCSIDeploymentProvisionerTimeout(client, csiSystemNamespace, defaultProvisionerTimeInSec)
updateCSIDeploymentProvisionerTimeout(c, csiSystemNamespace, defaultProvisionerTimeInSec)
}()

ginkgo.By("Creating PVCs using the Storage Class")
Expand Down Expand Up @@ -421,6 +449,8 @@ func createVolumeWithServiceDown(serviceName string, namespace string, client cl
var fullSyncWaitTime int
pvclaims = make([]*v1.PersistentVolumeClaim, volumeOpsScale)

c := client

// Decide which test setup is available to run
if vanillaCluster {
ginkgo.By("CNS_TEST: Running for vanilla k8s setup")
Expand All @@ -437,6 +467,14 @@ func createVolumeWithServiceDown(serviceName string, namespace string, client cl
curtimestring := strconv.FormatInt(curtime, 10)
scName := "idempotency" + curtimestring + val
storageclass, err = createStorageClass(client, scParameters, nil, "", "", false, scName)

controllerClusterConfig := os.Getenv(contollerClusterKubeConfig)
if controllerClusterConfig != "" {
framework.Logf("Creating client for remote kubeconfig")
remoteC, err := createKubernetesClientFromConfig(controllerClusterConfig)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
c = remoteC
}
} else if supervisorCluster {
ginkgo.By("CNS_TEST: Running for WCP setup")
thickProvPolicy := os.Getenv(envStoragePolicyNameWithThickProvision)
Expand Down Expand Up @@ -488,24 +526,24 @@ func createVolumeWithServiceDown(serviceName string, namespace string, client cl

if serviceName == "CSI" {
// Get CSI Controller's replica count from the setup
deployment, err := client.AppsV1().Deployments(csiSystemNamespace).Get(ctx,
deployment, err := c.AppsV1().Deployments(csiSystemNamespace).Get(ctx,
vSphereCSIControllerPodNamePrefix, metav1.GetOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
csiReplicaCount := *deployment.Spec.Replicas

ginkgo.By("Stopping CSI driver")
isServiceStopped, err = stopCSIPods(ctx, client)
isServiceStopped, err = stopCSIPods(ctx, c)
gomega.Expect(err).NotTo(gomega.HaveOccurred())

defer func() {
if isServiceStopped {
framework.Logf("Starting CSI driver")
isServiceStopped, err = startCSIPods(ctx, client, csiReplicaCount)
isServiceStopped, err = startCSIPods(ctx, c, csiReplicaCount)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
}
}()
framework.Logf("Starting CSI driver")
isServiceStopped, err = startCSIPods(ctx, client, csiReplicaCount)
isServiceStopped, err = startCSIPods(ctx, c, csiReplicaCount)
gomega.Expect(err).NotTo(gomega.HaveOccurred())

ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow full sync finish", fullSyncWaitTime))
Expand Down Expand Up @@ -617,6 +655,8 @@ func extendVolumeWithServiceDown(serviceName string, namespace string, client cl
var fullSyncWaitTime int
pvclaims = make([]*v1.PersistentVolumeClaim, volumeOpsScale)

c := client

// Decide which test setup is available to run
if vanillaCluster {
ginkgo.By("CNS_TEST: Running for vanilla k8s setup")
Expand All @@ -633,6 +673,14 @@ func extendVolumeWithServiceDown(serviceName string, namespace string, client cl
curtimestring := strconv.FormatInt(curtime, 10)
scName := "idempotency" + curtimestring + val
storageclass, err = createStorageClass(client, scParameters, nil, "", "", true, scName)

controllerClusterConfig := os.Getenv(contollerClusterKubeConfig)
if controllerClusterConfig != "" {
framework.Logf("Creating client for remote kubeconfig")
remoteC, err := createKubernetesClientFromConfig(controllerClusterConfig)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
c = remoteC
}
} else if supervisorCluster {
ginkgo.By("CNS_TEST: Running for WCP setup")
thickProvPolicy := os.Getenv(envStoragePolicyNameWithThickProvision)
Expand Down Expand Up @@ -734,25 +782,25 @@ func extendVolumeWithServiceDown(serviceName string, namespace string, client cl

if serviceName == "CSI" {
// Get CSI Controller's replica count from the setup
deployment, err := client.AppsV1().Deployments(csiSystemNamespace).Get(ctx,
deployment, err := c.AppsV1().Deployments(csiSystemNamespace).Get(ctx,
vSphereCSIControllerPodNamePrefix, metav1.GetOptions{})
gomega.Expect(err).NotTo(gomega.HaveOccurred())
csiReplicaCount := *deployment.Spec.Replicas

ginkgo.By("Stopping CSI driver")
isServiceStopped, err = stopCSIPods(ctx, client)
isServiceStopped, err = stopCSIPods(ctx, c)
gomega.Expect(err).NotTo(gomega.HaveOccurred())

defer func() {
if isServiceStopped {
framework.Logf("Starting CSI driver")
isServiceStopped, err = startCSIPods(ctx, client, csiReplicaCount)
isServiceStopped, err = startCSIPods(ctx, c, csiReplicaCount)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
}
}()

framework.Logf("Starting CSI driver")
isServiceStopped, err = startCSIPods(ctx, client, csiReplicaCount)
isServiceStopped, err = startCSIPods(ctx, c, csiReplicaCount)
gomega.Expect(err).NotTo(gomega.HaveOccurred())

ginkgo.By(fmt.Sprintf("Sleeping for %v seconds to allow full sync finish", fullSyncWaitTime))
Expand Down
3 changes: 1 addition & 2 deletions tests/e2e/util.go
View file
Expand Up @@ -4344,8 +4344,7 @@ func collectPodLogs(ctx context.Context, client clientset.Interface, namespace s

//Collect Pod logs
for _, cont := range pod.Spec.Containers {
cmd := []string{"logs", "--namespace=" + namespace, pod.Name, "-c", cont.Name}
output, err := framework.RunKubectl(namespace, cmd...)
output, err := fpod.GetPodLogs(client, pod.Namespace, pod.Name, cont.Name)
gomega.Expect(err).NotTo(gomega.HaveOccurred())
framework.Logf("Writing the logs into the file %v", "logs/"+pod.Name+cont.Name+filename)
err = writeToFile("logs/"+pod.Name+cont.Name+filename, output)
Expand Down

0 comments on commit db6ce48

Please sign in to comment.