ci: add cluster upgrade test

Signed-off-by: Anish Ramasekar <anish.ramasekar@gmail.com>
Azure · Mar 18, 2021 · 2b15c31 · 2b15c31
1 parent 3f1a6c0
commit 2b15c31
Show file tree

Hide file tree

Showing 9 changed files with 85 additions and 17 deletions.
diff --git a/.pipelines/e2e-job-azure.yaml b/.pipelines/e2e-job-azure.yaml
@@ -16,7 +16,8 @@ jobs:
     parameters:
       clusterTypes:
       - "aks"
-      - "aks-engine"
+      # - "aks-engine"
       osTypes:
       - "linux"
-      - "windows"
+      # - "windows"
+      testClusterUpgrade: true
diff --git a/.pipelines/e2e-tests-template.yaml b/.pipelines/e2e-tests-template.yaml
@@ -3,13 +3,16 @@ parameters:
     type: object
   - name: osTypes
     type: object
+  - name: testClusterUpgrade
+    type: boolean
+    default: false
 
 jobs:
   - ${{ each clusterType in parameters.clusterTypes }}:
     - ${{ each osType in parameters.osTypes }}:
       - job:
         displayName: ${{ format('{0}/{1}', clusterType, osType) }}
-        timeoutInMinutes: 45
+        timeoutInMinutes: 60
         cancelTimeoutInMinutes: 5
         workspace:
           clean: all
@@ -42,8 +45,13 @@ jobs:
           displayName: 'Set up workspace and install dependencies'
           
         - template: templates/build-images.yaml
-        - template: templates/aks-engine-setup.yaml
-        - template: templates/aks-setup.yaml
+
+        - ${{ if eq(clusterType, 'aks') }}:
+          - template: templates/aks-setup.yaml
+
+        - ${{ if not(eq(clusterType, 'aks')) }}:
+          - template: templates/aks-engine-setup.yaml
+
         - template: templates/assign-user-identity.yaml
           parameters:
             identitySubscriptionID: $(SUBSCRIPTION_ID)
@@ -63,8 +71,31 @@ jobs:
             SECRET_NAME: $(SECRET_NAME)
             SUBSCRIPTION_ID: $(SUBSCRIPTION_ID)
             TENANT_ID: $(TENANT_ID)
+            IS_UPGRADE_TEST: ${{ parameters.testClusterUpgrade }}
             ${{ if eq(osType, 'windows') }}:
               TEST_WINDOWS: true
 
+        - ${{ if parameters.testClusterUpgrade }}:
+          - template: templates/aks-upgrade.yaml
+
+          - script: |
+              export REGISTRY="${REGISTRY:-$(REGISTRY_NAME).azurecr.io/k8s/csi/secrets-store}"
+
+              make e2e-test
+            displayName: "Run e2e tests"
+            env:
+              AZURE_CLIENT_ID: $(AZURE_CLIENT_ID)
+              AZURE_CLIENT_SECRET: $(AZURE_CLIENT_SECRET)
+              KEY_NAME: $(KEY_NAME)
+              KEY_VERSION: $(KEY_VERSION)
+              KEYVAULT_NAME: $(KEYVAULT_NAME)
+              RESOURCE_GROUP: $(RESOURCE_GROUP)
+              SECRET_NAME: $(SECRET_NAME)
+              SUBSCRIPTION_ID: $(SUBSCRIPTION_ID)
+              TENANT_ID: $(TENANT_ID)
+              IS_UPGRADE_TEST: ${{ parameters.testClusterUpgrade }}
+              ${{ if eq(osType, 'windows') }}:
+                TEST_WINDOWS: true
+
         - template: templates/teardown.yaml
         - template: templates/cleanup-images.yaml
diff --git a/.pipelines/templates/aks-engine-setup.yaml b/.pipelines/templates/aks-engine-setup.yaml
@@ -2,20 +2,20 @@ steps:
   - script: |
       echo "##vso[task.setvariable variable=AZURE_CLUSTER_NAME]sscd-e2e-$(openssl rand -hex 6)"
       echo ${AZURE_CLUSTER_NAME}
-    displayName: "[AKS Engine] Set cluster name"
+    displayName: "Set cluster name"
     condition: and(succeeded(), eq(variables['CLUSTER_TYPE'], 'aks-engine'))
 
   - script: |
       echo "##vso[task.setvariable variable=AZURE_ADMIN_USERNAME]$USER"
       echo -e 'y\n' | ssh-keygen -f ~/.ssh/${AZURE_CLUSTER_NAME} -t rsa -N ''
       echo "##vso[task.setvariable variable=AZURE_SSH_KEY]$(cat ~/.ssh/${AZURE_CLUSTER_NAME}.pub)"
       echo "##vso[task.setvariable variable=LOCAL_SSH_KEY]$HOME/.ssh/${AZURE_CLUSTER_NAME}"
-    displayName: "[AKS Engine] Generate SSH key"
+    displayName: "Generate SSH key"
     condition: and(succeeded(), eq(variables['CLUSTER_TYPE'], 'aks-engine'))
 
   - script: |
       envsubst < test/e2e/cluster_configs/$(CLUSTER_CONFIG).json > cluster.json
-    displayName: "[AKS Engine] Build AKS-Engine API Model"
+    displayName: "Build AKS-Engine API Model"
     condition: and(succeeded(), eq(variables['CLUSTER_TYPE'], 'aks-engine'))
 
   - script: |
@@ -39,5 +39,5 @@ steps:
       kubectl wait pod -n kube-system --for=condition=Ready --all
       kubectl get nodes -owide
       kubectl cluster-info
-    displayName: "[AKS Engine] Deploy Azure cluster"
+    displayName: "Deploy Kubernetes cluster"
     condition: and(succeeded(), eq(variables['CLUSTER_TYPE'], 'aks-engine'))
diff --git a/.pipelines/templates/aks-setup.yaml b/.pipelines/templates/aks-setup.yaml
@@ -2,7 +2,7 @@ steps:
   - script: |
       echo "##vso[task.setvariable variable=AZURE_CLUSTER_NAME]sscd-e2e-$(openssl rand -hex 6)"
       echo ${AZURE_CLUSTER_NAME}
-    displayName: "[AKS] Set cluster name"
+    displayName: "Set cluster name"
     condition: and(succeeded(), eq(variables['CLUSTER_TYPE'], 'aks'))
 
   - script: |
@@ -40,10 +40,11 @@ steps:
             --node-count 1
       fi
 
+      # store kubeconfig
       az aks get-credentials -n ${AZURE_CLUSTER_NAME} -g ${AZURE_CLUSTER_NAME}
 
       # set CLUSTER_RESOURCE_GROUP for e2e test config
-      export CLUSTER_RESOURCE_GROUP="MC_${AZURE_CLUSTER_NAME}_${AZURE_CLUSTER_NAME}_$(AZURE_LOCATION)"
+      CLUSTER_RESOURCE_GROUP="$(az aks show -g ${AZURE_CLUSTER_NAME} -n ${AZURE_CLUSTER_NAME} --query nodeResourceGroup -otsv)"
       echo "##vso[task.setvariable variable=CLUSTER_RESOURCE_GROUP]${CLUSTER_RESOURCE_GROUP}"
 
       # Sleep for 120 seconds to wait for nodes and pods to become ready
@@ -52,5 +53,5 @@ steps:
       kubectl wait pod -n kube-system --for=condition=Ready --all
       kubectl get nodes -owide
       kubectl cluster-info
-    displayName: "[AKS] Deploy Azure cluster"
+    displayName: "Deploy Kubernetes cluster"
     condition: and(succeeded(), eq(variables['CLUSTER_TYPE'], 'aks'))
diff --git a/.pipelines/templates/aks-upgrade.yaml b/.pipelines/templates/aks-upgrade.yaml
@@ -0,0 +1,18 @@
+steps:
+  - script: |
+      AKS_UPGRADE_VERSION=$(az aks get-upgrades -g ${AZURE_CLUSTER_NAME} -n ${AZURE_CLUSTER_NAME} --query 'max(controlPlaneProfile.upgrades[].kubernetesVersion)' -otsv)
+      echo "##vso[task.setvariable variable=AKS_UPGRADE_VERSION]${AKS_UPGRADE_VERSION}"
+    displayName: "[AKS] Get upgrade version"
+
+  - script: |
+      echo "Upgrading to Kubernetes ${AKS_UPGRADE_VERSION}"
+      az aks upgrade -g ${AZURE_CLUSTER_NAME} -n ${AZURE_CLUSTER_NAME} -k ${AKS_UPGRADE_VERSION} --yes
+
+      # Sleep for 120 seconds to wait for nodes and pods to become ready
+      sleep 2m
+      kubectl wait --for=condition=ready node --all
+      kubectl wait pod -n kube-system --for=condition=Ready --all
+      kubectl get nodes -owide
+      kubectl cluster-info
+    displayName: "Upgrade kubernetes version"
+    condition: and(succeeded(), eq(variables['CLUSTER_TYPE'], 'aks'))
diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go
@@ -46,10 +46,12 @@ var _ = BeforeSuite(func() {
 	kvClient = keyvault.NewClient(config)
 
 	if !config.IsSoakTest {
-		By("Installing Secrets Store CSI Driver and Azure Key Vault Provider via Helm")
-		helm.Install(helm.InstallInput{
-			Config: config,
-		})
+		if !(config.IsUpgradeTest && helm.ReleaseExists()) {
+			By("Installing Secrets Store CSI Driver and Azure Key Vault Provider via Helm")
+			helm.Install(helm.InstallInput{
+				Config: config,
+			})
+		}
 	}
 
 	kubeClient = clusterProxy.GetClient()
@@ -58,7 +60,7 @@ var _ = BeforeSuite(func() {
 
 var _ = AfterSuite(func() {
 	defer func() {
-		if !config.IsSoakTest {
+		if !config.IsSoakTest && !config.IsUpgradeTest {
 			By("Uninstalling Secrets Store CSI Driver and Azure Key Vault Provider via Helm")
 			helm.Uninstall()
 		}

diff --git a/test/e2e/framework/config.go b/test/e2e/framework/config.go
@@ -25,6 +25,7 @@ type Config struct {
 	PodIdentityUserMSIName            string `envconfig:"POD_IDENTITY_USER_MSI_NAME"`
 	PodIdentityUserAssignedIdentityID string `envconfig:"POD_IDENTITY_USER_ASSIGN_IDENTITY_ID"`
 	ResourceGroup                     string `envconfig:"RESOURCE_GROUP"`
+	IsUpgradeTest                     bool   `envconfig:"IS_UPGRADE_TEST"`
 }
 
 func (c *Config) DeepCopy() *Config {
@@ -46,6 +47,7 @@ func (c *Config) DeepCopy() *Config {
 	copy.PodIdentityUserMSIName = c.PodIdentityUserMSIName
 	copy.PodIdentityUserAssignedIdentityID = c.PodIdentityUserAssignedIdentityID
 	copy.ResourceGroup = c.ResourceGroup
+	copy.IsUpgradeTest = c.IsUpgradeTest
 
 	return copy
 }

diff --git a/test/e2e/framework/exec/kubectl.go b/test/e2e/framework/exec/kubectl.go
@@ -40,6 +40,7 @@ func KubectlExec(kubeconfigPath, podName, namespace string, args []string) (stri
 		"exec",
 		fmt.Sprintf("--kubeconfig=%s", kubeconfigPath),
 		fmt.Sprintf("--namespace=%s", namespace),
+		fmt.Sprintf("--request-timeout=5s"),
 		podName,
 		"--",
 	}, args...)

diff --git a/test/e2e/framework/helm/helm.go b/test/e2e/framework/helm/helm.go
@@ -78,6 +78,18 @@ func Uninstall() {
 	_ = helm(args)
 }
 
+// ReleaseExists checks if csi release exists
+func ReleaseExists() bool {
+	args := []string{
+		"status",
+		chartName,
+	}
+
+	err := helm(args)
+	// chart not found error
+	return err == nil
+}
+
 func generateValueArgs(config *framework.Config) []string {
 	args := []string{
 		fmt.Sprintf("--set=image.repository=%s/%s", config.Registry, config.ImageName),