ci: [CNI] Adding aks cluster creation steps for k8s e2e test (#2052)

* ci: [CNI] Adding aks cluster creation steps for k8s e2e test

* Add  validate step to the pipeline

* Adding the telemetry config to the cluster

(cherry picked from commit 846e508)

.pipelines/cni/singletenancy/windows-cni-load-test-template.yaml

@@ -0,0 +1,134 @@
+parameters:
+  dependsOn: ""
+  name: ""
+  clusterType: ""
+  clusterName: ""
+  nodeCount: ""
+  vmSize: ""
+  windowsVMSize: ""
+  os: ""
+  cni: ""
+
+stages:
+  - stage: createAKSclusterWindows
+    dependsOn: ${{ parameters.dependsOn }}
+    displayName: "Windows AKS Cluster ${{ parameters.cni }}"
+    jobs:
+      - job: create_aks_cluster_with_${{ parameters.name }}
+        steps:
+          - template: ../load-test-templates/create-cluster-template.yaml
+            parameters:
+              clusterType: ${{ parameters.clusterType }}
+              clusterName: ${{ parameters.clusterName }}
+              nodeCount: ${{ parameters.nodeCount }}
+              vmSize: ${{ parameters.vmSize }}
+              windowsVMSize: ${{ parameters.windowsVMSize }}
+  - stage: build_images
+    dependsOn: ${{ parameters.dependsOn }}
+    displayName: "Build CNI Images"
+    jobs:
+      - job: build_cni_images
+        pool:
+          name: "$(BUILD_POOL_NAME_LINUX_AMD64)"
+        strategy:
+          matrix:
+            windows_cniv1_amd64:
+              os: windows
+              name: cni-plugin
+              arch: amd64
+              os_version: ltsc2022
+        steps:
+          - template: ../../containers/container-template.yaml
+            parameters:
+              arch: $(arch)
+              name: $(name)
+              os: $(os)
+              os_version: $(os_version)
+  - stage:  update_cni
+    dependsOn: 
+      - createAKSclusterWindows
+      - build_images
+    displayName: "Update CNI on Cluster"
+    jobs:
+      - job: deploy_pods
+        strategy:
+          matrix:
+            windows_cniv1_amd64:
+              os: windows
+              arch: amd64
+              os_version: ltsc2022
+        steps:
+          - task: AzureCLI@1
+            inputs:
+              azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
+              scriptLocation: "inlineScript"
+              scriptType: "bash"
+              addSpnToEnvironment: true
+              inlineScript: |
+                set -ex
+                export CNI_IMAGE=$(make cni-plugin-image-name-and-tag OS=$(os) ARCH=$(arch) OS_VERSION=$(os_version))
+                az extension add --name aks-preview
+                clusterName=${{ parameters.clusterName }}-$(make revision)
+                make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${clusterName}
+                make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION)
+                envsubst < ./hack/manifests/windows-update.yaml | kubectl apply -f -
+            name: "UploadCNI"
+            displayName: "Upload CNI"
+          - script: |
+              set -ex
+              kubectl rollout status daemonset/azure-cni-windows -n kube-system
+              kubectl get pods -A
+            name: "WaitForCNI"
+            displayName: "Wait For CNI"
+  - stage: pod_deployment_windows
+    dependsOn: update_cni
+    displayName: "Pod Deployment"
+    jobs:
+      - job: deploy_pods
+        steps:
+          - template: ../load-test-templates/pod-deployment-template.yaml
+            parameters:
+              clusterName: ${{ parameters.clusterName }}
+              scaleup: ${WINDOWS_SCALEUP}
+              os: ${{ parameters.os }}
+              cni: ${{ parameters.cni }}
+              iterations: ${WINDOWS_ITERATIONS}
+  - stage: validate_state_windows
+    dependsOn: pod_deployment_windows
+    displayName: "Validate State"
+    jobs:
+      - job: validate_state
+        steps:
+          - template: ../load-test-templates/validate-state-template.yaml
+            parameters:
+              clusterName: ${{ parameters.clusterName }}
+              os: ${{ parameters.os }}
+              cni: ${{ parameters.cni }}
+  - stage: delete_resources
+    displayName: "Delete Resources"
+    dependsOn:
+      - validate_state_windows
+    jobs:
+      - job: delete_resources
+        steps:
+          - task: AzureCLI@1
+            inputs:
+              azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
+              scriptLocation: "inlineScript"
+              scriptType: "bash"
+              addSpnToEnvironment: true
+              inlineScript: |
+                set -ex
+                if [ "$(DELETE_RESOURCES)" ]
+                then
+                  echo "Deleting Cluster and resource group"
+                  make -C ./hack/aks set-kubeconf AZCLI=az CLUSTER=${{ parameters.clusterName }}-$(make revision)
+                  make -C ./hack/aks azcfg AZCLI=az REGION=$(LOCATION)
+                  make -C ./hack/aks down AZCLI=az REGION=$(LOCATION) SUB=$(SUBSCRIPTION_ID) CLUSTER=${{ parameters.clusterName }}-$(make revision)
+                  echo "Cluster and resources down"
+                else
+                  echo "Deletion of resources is False"
+                fi
+            name: "CleanUpCluster"
+            displayName: "Cleanup cluster"
+    condition: always()

.pipelines/pipeline.yaml

@@ -37,6 +37,7 @@ stages:
               echo "##vso[task.setvariable variable=Tag;isOutput=true]$(make version)"
               echo "##vso[task.setvariable variable=cniVersion;isOutput=true]$(make cni-version)"
               echo "##vso[task.setvariable variable=npmVersion;isOutput=true]$(make npm-version)"
+              echo "##vso[task.setvariable variable=dropgzVersion;isOutput=true]$(make cni-dropgz-version)"
               cat /etc/os-release
               uname -a
               sudo chown -R $(whoami):$(whoami) .
@@ -167,6 +168,16 @@ stages:
               arch: amd64
               name: cni-dropgz-test
               os: linux
+            cni_plugin_windows2022_amd64:
+              os: windows
+              name: cni-plugin
+              arch: amd64
+              os_version: ltsc2022
+            cni_plugin_windows2019_amd64:
+              os: windows
+              name: cni-plugin
+              arch: amd64
+              os_version: ltsc2019
             cns_linux_amd64:
               arch: amd64
               name: cns
@@ -344,6 +355,48 @@ stages:
       clusterName: "swifte2e"
       osSku: "Ubuntu"
 
+  - template: singletenancy/aks/e2e-job-template.yaml
+    parameters:
+      name: "aks_ubuntu_18_04_linux_e2e"
+      displayName: AKS Ubuntu 18.04
+      arch: 'amd64'
+      os: 'linux'
+      clusterType: linux-cniv1-up
+      clusterName: 'ubuntu18e2e'
+      k8sVersion: 1.24.9
+
+  - template: singletenancy/aks/e2e-job-template.yaml
+    parameters:
+      name: "aks_ubuntu_22_linux_e2e"
+      displayName: AKS Ubuntu 22
+      arch: 'amd64'
+      os: 'linux'
+      clusterType: linux-cniv1-up
+      clusterName: 'ubuntu22e2e'
+      k8sVersion: 1.25
+
+  - template: singletenancy/aks/e2e-job-template.yaml
+    parameters:
+      name: "aks_windows_19_03_e2e"
+      displayName: AKS Windows 1903
+      arch: amd64
+      os: windows
+      clusterType: windows-cniv1-up
+      clusterName: 'win19e2e'
+      windowsOsSku: 'Windows2019'
+      os_version: 'ltsc2019'
+
+  - template: singletenancy/aks/e2e-job-template.yaml
+    parameters:
+      name: "aks_windows_22_e2e"
+      displayName: AKS Windows 2022
+      arch: amd64
+      os: windows
+      clusterType: windows-cniv1-up
+      clusterName: 'win22e2e'
+      windowsOsSku: 'Windows2022'
+      os_version: 'ltsc2022'
+
   - template: singletenancy/aks-engine/e2e-job-template.yaml
     parameters:
       name: "ubuntu_18_04_linux_e2e"
@@ -389,6 +442,10 @@ stages:
     dependsOn:
       - "aks_swift_e2e"
       - "cilium_e2e"
+      - "aks_ubuntu_18_04_linux_e2e"
+      - "aks_windows_19_03_e2e"
+      - "aks_ubuntu_22_linux_e2e"
+      - "aks_windows_22_e2e"
       - "ubuntu_18_04_linux_e2e"
       - "windows_19_03_e2e"
     jobs:

.pipelines/singletenancy/aks/e2e-job-template.yaml

@@ -0,0 +1,44 @@
+parameters:
+  name: ""
+  displayName: ""
+  arch: ""  
+  os: ""
+  clusterType: ""
+  clusterName: ""
+  k8sVersion: ""
+  windowsOsSku: ""
+  os_version: ""
+
+stages:
+  - stage: ${{ parameters.name }}
+    displayName: E2E - ${{ parameters.displayName }}
+    dependsOn: 
+    - setup
+    - publish
+    jobs:
+      - job: ${{ parameters.name }}
+        displayName: Singletenancy AKS - (${{ parameters.name }})
+        pool:
+          name: $(BUILD_POOL_NAME_DEFAULT)
+          demands: 
+          - agent.os -equals Linux
+          - Role -equals $(CUSTOM_E2E_ROLE)
+        variables:
+          GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
+          GOBIN: "$(GOPATH)/bin" # Go binaries path
+          modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
+          dropgzVersion: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.dropgzVersion'] ]
+          cniVersion: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.cniVersion'] ]
+        steps:
+          - template: e2e-step-template.yaml
+            parameters:
+              name: ${{ parameters.name }}
+              clusterType: ${{ parameters.clusterType }}
+              clusterName: ${{ parameters.clusterName }}
+              arch: ${{ parameters.arch }}
+              os: ${{ parameters.os }}
+              k8sVersion: ${{ parameters.k8sVersion }}
+              windowsOsSku: ${{ parameters.windowsOsSku }}
+              os_version: ${{ parameters.os_version }}
+              version: $(dropgzVersion)
+              cniVersion: $(cniVersion)

.pipelines/singletenancy/aks/e2e-step-template.yaml

@@ -0,0 +1,95 @@
+parameters:
+  name: ""
+  clusterType: ""
+  clusterName: ""
+  nodeCount: ""
+  vmSize: ""
+  windowsVMSize: ""
+  k8sVersion: ""
+  version: ""
+  os: ""
+  windowsOsSku: ""
+  cniVersion: ""
+  os_version: ""
+
+steps:
+  - bash: |
+      go version
+      go env
+      mkdir -p '$(GOBIN)'
+      mkdir -p '$(GOPATH)/pkg'
+      mkdir -p '$(modulePath)'
+      echo '##vso[task.prependpath]$(GOBIN)'
+      echo '##vso[task.prependpath]$(GOROOT)/bin'
+    name: "GoEnv"
+    displayName: "Set up the Go environment"
+  - task: AzureCLI@1
+    inputs:
+      azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      scriptType: "bash"
+      addSpnToEnvironment: true
+      inlineScript: |
+        mkdir -p ~/.kube/
+        echo "Create AKS cluster"
+        echo "parameters ${{ parameters.windowsOsSku }}"
+        make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)
+        make -C ./hack/aks ${{ parameters.clusterType }} AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision) K8S_VER=${{ parameters.k8sVersion }} WINDOWS_OS_SKU=${{ parameters.windowsOsSku }} WINDOWS_USERNAME=${WINDOWS_USERNAME} WINDOWS_PASSWORD=${WINDOWS_PASSWORD}
+        echo "Cluster successfully created"
+    displayName: Create test cluster
+  - script: |
+      echo "Upload CNI"
+      if [ "${{parameters.os}}" == "windows" ]; then
+        export DROP_GZ_URL=$( make cni-dropgz-test-image-name-and-tag OS='linux' ARCH=${{ parameters.arch }} CNI_DROPGZ_VERSION=${{ parameters.version }})
+        envsubst < ./test/integration/manifests/cni/cni-installer-v1.yaml | kubectl apply -f -
+        kubectl rollout status daemonset/azure-cni -n kube-system
+        export CNI_IMAGE=$(make cni-plugin-image-name-and-tag OS=${{ parameters.os }} ARCH=${{ parameters.arch }} OS_VERSION=${{ parameters.os_version }} CNI_VERSION=${{ parameters.cniVersion }})
+        envsubst < ./hack/manifests/windows-update.yaml | kubectl apply -f -
+        kubectl rollout status daemonset/azure-cni-windows -n kube-system
+      else
+        export DROP_GZ_URL=$( make cni-dropgz-test-image-name-and-tag OS=${{ parameters.os }} ARCH=${{ parameters.arch }} CNI_DROPGZ_VERSION=${{ parameters.version }})
+        envsubst < ./test/integration/manifests/cni/cni-installer-v1.yaml | kubectl apply -f -
+        kubectl rollout status daemonset/azure-cni -n kube-system
+      fi
+    name: "UploadCni"
+    displayName: "Upload CNI"
+  - task: AzureCLI@1
+    inputs:
+      azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      scriptType: "bash"
+      addSpnToEnvironment: true
+      inlineScript: |
+        clusterName=${{ parameters.clusterName }}-$(make revision)
+        echo "Restarting nodes"
+        for val in $(az vmss list -g MC_${clusterName}_${clusterName}_$(REGION_AKS_CLUSTER_TEST) --query "[].name" -o tsv); do 
+          make -C ./hack/aks restart-vmss AZCLI=az CLUSTER=${clusterName} REGION=$(REGION_AKS_CLUSTER_TEST) VMSS_NAME=${val} 
+        done
+    displayName: "Restart Nodes"
+  - script: |
+      kubectl get pods -A -o wide
+      echo "Deploying test pods"
+      cd test/integration/load
+      go test -count 1 -timeout 30m -tags load -run ^TestLoad$ -tags=load -iterations=2 -scaleup=40 -os=${{ parameters.os }}
+      cd ../../..
+      # Remove this once we have cniv1 support for validating the test cluster
+      echo "Validate State skipped for linux cniv1 for now"
+      if [ "${{parameters.os}}" == "windows" ]; then
+        make test-validate-state OS=${{ parameters.os }}
+      fi
+    displayName: "Validate State"
+    retryCountOnTaskFailure: 3
+  - task: AzureCLI@1
+    inputs:
+      azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      scriptType: "bash"
+      addSpnToEnvironment: true
+      inlineScript: |
+        echo "Skipping Deleting cluster"
+        make -C ./hack/aks azcfg AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST)
+        make -C ./hack/aks down AZCLI=az REGION=$(REGION_AKS_CLUSTER_TEST) SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) CLUSTER=${{ parameters.clusterName }}-$(make revision)
+        echo "Cluster and resources down"
+    displayName: "Delete test cluster"
+    condition: always()
+