feat: runs e2e test on 3 aks versions (#503)

* feat: runs e2e test with 3 aks versions * fix: updates shell script * fix: corrects filename * test: file path * test: file path via system variable * test: system var * fix: argument * test: debug * test: upgrade loop * fix: pipeline syntax * fix: piepline syntax * test: upgrade * fix: az pipeline * test: az pipeline * fix: template * fix: test template * fix: Job name * fix: removes condition * fix: runs az login * fix: removes version template * fix: removes commented code and debug * fix: sets ver to latest for non cluster upgrade run * fix: updates timeout to 90m * chore: removes cluster upgrade test from azure job Signed-off-by: Nilekh Chaudhari <1626598+nilekhc@users.noreply.github.com> * chore: removes white space * fix: updates query to get latest non-preview ver * fix: updates query * fix: updates upgrade logic * fix: updates cluster and rg name * chore: creates e2e test template * fix: updates template param usage * fix: sets helm chart dir if passed in param * fix: updates helm chart dir * fix: removes helm chart dir param * fix: removes helm chart dir env var * test: helm char dir * fix: bool condition * test: debug * fix: updates template path * fix: bool check * fix: test bool * fix: condition * fix: updates test run params * fix: removes cluster upgrade flag * test: whole pipeline * chore: typo * chore: updates job name * chore: typo * fix: sets localtion var
Azure · May 13, 2021 · 512316a · 512316a
1 parent 40bb678
commit 512316a
Show file tree

Hide file tree

Showing 4 changed files with 122 additions and 66 deletions.
diff --git a/.pipelines/templates/aks-setup.yaml b/.pipelines/templates/aks-setup.yaml
@@ -1,16 +1,42 @@
+parameters:
+  - name: testClusterUpgrade
+    type: boolean
+    default: false
+
 steps:
   - script: |
       echo "##vso[task.setvariable variable=AZURE_CLUSTER_NAME]sscd-e2e-$(openssl rand -hex 6)"
       echo ${AZURE_CLUSTER_NAME}
     displayName: "Set cluster name"
     condition: and(succeeded(), eq(variables['CLUSTER_TYPE'], 'aks'))
+    
+  - script: | 
+      #Run test with latest on preview aks version available
+      aksVersion=$(az aks get-versions -l $(AZURE_LOCATION) --query "max(orchestrators[?isPreview==null].orchestratorVersion)" -otsv)
+      echo "AKS Install version - $aksVersion"
+
+      echo "##vso[task.setvariable variable=AKS_INSTALL_VERSION]$aksVersion"
+    displayName: "Determine aks install version"
+    condition: and(succeeded(), eq(variables['CLUSTER_TYPE'], 'aks'))
+
+  - ${{ if eq(parameters.testClusterUpgrade, true) }}:
+    #Overrride AKS_INSTALL_VERSION if testing with k8s upgrade
+    #If we are running test with cluster upgrade, start with minimum possible non preview version. 
+    - script: |
+        aksVersion=$(az aks get-versions -l $(AZURE_LOCATION) --query "min(orchestrators[?isPreview==null].orchestratorVersion)" -otsv)
+        echo "AKS Install version - $aksVersion"
+
+        echo "##vso[task.setvariable variable=AKS_INSTALL_VERSION]$aksVersion"
+      displayName: "Determine aks install version"
+      condition: and(succeeded(), eq(variables['CLUSTER_TYPE'], 'aks'))
 
   - script: |
       az group create -n ${AZURE_CLUSTER_NAME} -l $(AZURE_LOCATION)
 
       az aks create \
         -g ${AZURE_CLUSTER_NAME} \
         -n ${AZURE_CLUSTER_NAME} \
+        --kubernetes-version $(AKS_INSTALL_VERSION) \
         --node-count $(AGENT_COUNT) \
         --nodepool-name nodepool1 \
         --node-vm-size Standard_DS2_v2 \

diff --git a/.pipelines/templates/aks-upgrade.yaml b/.pipelines/templates/aks-upgrade.yaml
@@ -1,12 +1,14 @@
 steps:
   - script: |
-      AKS_UPGRADE_VERSION=$(az aks get-upgrades -g ${AZURE_CLUSTER_NAME} -n ${AZURE_CLUSTER_NAME} --query 'max(controlPlaneProfile.upgrades[].kubernetesVersion)' -otsv)
-      echo "##vso[task.setvariable variable=AKS_UPGRADE_VERSION]${AKS_UPGRADE_VERSION}"
-    displayName: "Get upgrade version"
+      #Get minor version of currently installed aks version. major.minor.patch -> major.minor
+      minorVersion=$(az aks get-upgrades -n ${AZURE_CLUSTER_NAME} -g ${AZURE_CLUSTER_NAME} --query "controlPlaneProfile.kubernetesVersion" | jq 'split(".") | .[:2] | join(".")')
+      echo "Minor version is - $minorVersion"
 
-  - script: |
-      echo "Upgrading to Kubernetes ${AKS_UPGRADE_VERSION}"
-      az aks upgrade -g ${AZURE_CLUSTER_NAME} -n ${AZURE_CLUSTER_NAME} -k ${AKS_UPGRADE_VERSION} --yes
+      #Get maximum non preview upgrade version from upgrade profiles that does not start with minorVersion(major.minor)
+      upgradeVersion=$(az aks get-upgrades -n ${AZURE_CLUSTER_NAME} -g ${AZURE_CLUSTER_NAME} --query "max(controlPlaneProfile.upgrades[?isPreview==null && !(starts_with(kubernetesVersion, '$minorVersion'))].kubernetesVersion)" -otsv)
+
+      echo "Upgrading to Kubernetes $upgradeVersion"
+      az aks upgrade -g ${AZURE_CLUSTER_NAME} -n ${AZURE_CLUSTER_NAME} -k $upgradeVersion --yes
 
       # Sleep for 120 seconds to wait for nodes and pods to become ready
       sleep 2m

diff --git a/.pipelines/templates/e2e-test-azure.yaml b/.pipelines/templates/e2e-test-azure.yaml
@@ -12,7 +12,7 @@ jobs:
     - ${{ each osType in parameters.osTypes }}:
       - job:
         displayName: ${{ format('{0}/{1}', clusterType, osType) }}
-        timeoutInMinutes: 60
+        timeoutInMinutes: 90
         cancelTimeoutInMinutes: 5
         workspace:
           clean: all
@@ -40,6 +40,8 @@ jobs:
 
         - ${{ if eq(clusterType, 'aks') }}:
           - template: aks-setup.yaml
+            parameters:
+              testClusterUpgrade: ${{ parameters.testClusterUpgrade }}
 
         - ${{ if eq(clusterType, 'aks-engine') }}:
           - template: aks-engine-setup.yaml
@@ -48,71 +50,44 @@ jobs:
           parameters:
             identitySubscriptionID: $(SUBSCRIPTION_ID)
 
-        - script: |
-            #This gets latest released chart(tgz) version from charts directory
-            export HELM_CHART_DIR=$(ls -h charts/*.tgz | sort --version-sort --field-separator=- --key=6 | tail -n 1)
-
-            make e2e-test
-          displayName: "Run e2e tests with Released Version"
-          env:
-            AZURE_CLIENT_ID: $(AZURE_CLIENT_ID)
-            AZURE_CLIENT_SECRET: $(AZURE_CLIENT_SECRET)
-            KEY_NAME: $(KEY_NAME)
-            KEY_VERSION: $(KEY_VERSION)
-            KEYVAULT_NAME: $(KEYVAULT_NAME)
-            RESOURCE_GROUP: $(RESOURCE_GROUP)
-            SECRET_NAME: $(SECRET_NAME)
-            SUBSCRIPTION_ID: $(SUBSCRIPTION_ID)
-            TENANT_ID: $(TENANT_ID)
-            IS_UPGRADE_TEST: ${{ parameters.testClusterUpgrade }}
-            CONFIG_IMAGE_VERSION: ""
-            IS_BACKWARD_COMPATIBILITY_TEST: true
-            ${{ if eq(osType, 'windows') }}:
-              TEST_WINDOWS: true
-
-        - script: |
-            export REGISTRY="${REGISTRY:-$(REGISTRY_NAME).azurecr.io/k8s/csi/secrets-store}"
+        # Run e2e tests with Released Version
+        - template: e2e-test.yaml
+          parameters:
+            testName: "Run e2e test with Released Version"
+            osType: ${{ osType }}
+            testClusterUpgrade: ${{ parameters.testClusterUpgrade }}
+            resetImageVersion: true
+            isBackwardCompatibilityTest: true
+            testReleasedVersion: true
 
-            make e2e-test
-          displayName: "Run e2e tests with New Version"
-          env:
-            AZURE_CLIENT_ID: $(AZURE_CLIENT_ID)
-            AZURE_CLIENT_SECRET: $(AZURE_CLIENT_SECRET)
-            KEY_NAME: $(KEY_NAME)
-            KEY_VERSION: $(KEY_VERSION)
-            KEYVAULT_NAME: $(KEYVAULT_NAME)
-            RESOURCE_GROUP: $(RESOURCE_GROUP)
-            SECRET_NAME: $(SECRET_NAME)
-            SUBSCRIPTION_ID: $(SUBSCRIPTION_ID)
-            TENANT_ID: $(TENANT_ID)
-            IS_UPGRADE_TEST: ${{ parameters.testClusterUpgrade }}
-            HELM_CHART_DIR: "manifest_staging/charts/csi-secrets-store-provider-azure"
-            IS_BACKWARD_COMPATIBILITY_TEST: true
-            ${{ if eq(osType, 'windows') }}:
-              TEST_WINDOWS: true
+        # Run e2e tests with New Version
+        - template: e2e-test.yaml
+          parameters:
+            testName: "Run e2e test with New Version"
+            osType: ${{ osType }}
+            testClusterUpgrade: ${{ parameters.testClusterUpgrade }}
+            isBackwardCompatibilityTest: true
+            setRegistry: true
 
         - ${{ if and(parameters.testClusterUpgrade, eq(clusterType, 'aks')) }}:
           - template: aks-upgrade.yaml
 
-          - script: |
-              export REGISTRY="${REGISTRY:-$(REGISTRY_NAME).azurecr.io/k8s/csi/secrets-store}"
+          # Run e2e tests after cluster upgrade
+          - template: e2e-test.yaml
+            parameters:
+              testName: "Run e2e test with New Version"
+              osType: ${{ osType }}
+              testClusterUpgrade: ${{ parameters.testClusterUpgrade }}
+              setRegistry: true
+
+          - template: aks-upgrade.yaml
 
-              make e2e-test
-            displayName: "Run e2e tests after cluster upgrade with New Version"
-            env:
-              AZURE_CLIENT_ID: $(AZURE_CLIENT_ID)
-              AZURE_CLIENT_SECRET: $(AZURE_CLIENT_SECRET)
-              KEY_NAME: $(KEY_NAME)
-              KEY_VERSION: $(KEY_VERSION)
-              KEYVAULT_NAME: $(KEYVAULT_NAME)
-              RESOURCE_GROUP: $(RESOURCE_GROUP)
-              SECRET_NAME: $(SECRET_NAME)
-              SUBSCRIPTION_ID: $(SUBSCRIPTION_ID)
-              TENANT_ID: $(TENANT_ID)
-              IS_UPGRADE_TEST: ${{ parameters.testClusterUpgrade }}
-              IS_CLUSTER_UPGRADED: true
-              ${{ if eq(osType, 'windows') }}:
-                TEST_WINDOWS: true
+          - template: e2e-test.yaml
+            parameters:
+              testName: "Run e2e test with New Version"
+              osType: ${{ osType }}
+              testClusterUpgrade: ${{ parameters.testClusterUpgrade }}
+              setRegistry: true
 
         - template: teardown.yaml
         - template: cleanup-images.yaml
diff --git a/.pipelines/templates/e2e-test.yaml b/.pipelines/templates/e2e-test.yaml
@@ -0,0 +1,53 @@
+parameters:
+  - name: testName
+    type: string
+  - name: osType
+    type: string
+  - name: testReleasedVersion
+    type: boolean
+    default: false    
+  - name: resetImageVersion
+    type: boolean
+    default: false
+  - name: testClusterUpgrade
+    type: boolean
+    default: false
+  - name: isBackwardCompatibilityTest
+    type: boolean
+    default: false
+  - name: setRegistry
+    type: boolean
+    default: false
+
+steps:
+  - script: |
+      if [ "${{ parameters.setRegistry }}" == True ] ; then
+        export REGISTRY="${REGISTRY:-$(REGISTRY_NAME).azurecr.io/k8s/csi/secrets-store}"
+      fi
+
+      if [ "${{ parameters.testReleasedVersion }}" == True ] ; then
+        #This gets latest released chart(tgz) version from charts directory
+        export HELM_CHART_DIR=$(ls -h charts/*.tgz | sort --version-sort --field-separator=- --key=6 | tail -n 1)
+      else
+        export HELM_CHART_DIR=manifest_staging/charts/csi-secrets-store-provider-azure
+      fi
+
+      make e2e-test
+    displayName: "${{ parameters.testName }}"
+    env:
+      AZURE_CLIENT_ID: $(AZURE_CLIENT_ID)
+      AZURE_CLIENT_SECRET: $(AZURE_CLIENT_SECRET)
+      KEY_NAME: $(KEY_NAME)
+      KEY_VERSION: $(KEY_VERSION)
+      KEYVAULT_NAME: $(KEYVAULT_NAME)
+      RESOURCE_GROUP: $(RESOURCE_GROUP)
+      SECRET_NAME: $(SECRET_NAME)
+      SUBSCRIPTION_ID: $(SUBSCRIPTION_ID)
+      TENANT_ID: $(TENANT_ID)
+      IS_UPGRADE_TEST: ${{ parameters.testClusterUpgrade }}
+      ${{ if parameters.resetImageVersion }}:
+        CONFIG_IMAGE_VERSION: ""
+      ${{ if parameters.isBackwardCompatibilityTest }}:
+        IS_BACKWARD_COMPATIBILITY_TEST: ${{ parameters.isBackwardCompatibilityTest }}
+      ${{ if eq(parameters.osType, 'windows') }}:
+        TEST_WINDOWS: true