Persist cluster configs in AZP

.pipelines/Dockerfile

@@ -1,5 +1,5 @@
 FROM ubuntu:16.04
-RUN apt-get update && apt-get install -y software-properties-common sudo wget apt-transport-https curl lsb-release gnupg
+RUN apt-get update && apt-get install -y software-properties-common sudo wget apt-transport-https curl lsb-release gnupg jq
 RUN wget -q https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
 RUN sudo dpkg -i packages-microsoft-prod.deb
 RUN add-apt-repository ppa:longsleep/golang-backports 

.pipelines/e2e-job-template.yaml

@@ -1,7 +1,7 @@
 parameters:
   name: ""
   pipelineBuildImage: "containernetworking/pipeline-ci:1.0.4"
-  clusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json"
+  clusterDefinition: ""
   clusterDefinitionCniTypeKey: ""
   clusterDefinitionCniBuildOS: ""
   clusterDefinitionCniBuildExt: ""
@@ -18,13 +18,15 @@ jobs:
       GOPATH: "$(System.DefaultWorkingDirectory)/gopath"
       GOBIN: "$(GOPATH)/bin" # Go binaries path
       modulePath: "$(GOPATH)/src/github.com/Azure/aks-engine"
+      acnPath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
       Tag: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.Tag'] ]
       CommitHash: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.CommitHash'] ]
 
     steps:
       - template: e2e-step-template.yaml
         parameters:
-          clusterDefinitionUrl: ${{ parameters.clusterDefinitionUrl }}
+          name: ${{ parameters.name }}
+          clusterDefinition: ${{ parameters.clusterDefinition }}
           clusterDefinitionCniTypeKey: ${{ parameters.clusterDefinitionCniTypeKey }}
           clusterDefinitionCniBuildOS: ${{ parameters.clusterDefinitionCniBuildOS }}
           clusterDefinitionCniBuildExt: ${{ parameters.clusterDefinitionCniBuildExt }}

.pipelines/e2e-step-template.yaml

@@ -1,6 +1,6 @@
 parameters:
   name: ""
-  clusterDefinitionUrl: ""
+  clusterDefinition: ""
   clusterDefinitionCniTypeKey: ""
   clusterDefinitionCniBuildOS: ""
   clusterDefinitionCniBuildExt: ""
@@ -19,16 +19,24 @@ steps:
     name: "GoEnv"
     displayName: "Set up the Go environment"
 
+  - task: DownloadPipelineArtifact@2
+    inputs:
+      buildType: current
+      artifactName: clusterdefinitions
+      targetPath: "$(modulePath)"
+      #buildType: 'current' # Options: current, specific
+      #project: # Required when buildType == Specific
+
   - bash: |
-      rm -f clusterDefinition.json*
-      wget '${{ parameters.clusterDefinitionUrl }}' -O clusterDefinition.json
+      ls -lah
       export CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-${{ parameters.clusterDefinitionCniBuildOS }}-amd64-$(Tag)${{ parameters.clusterDefinitionCniBuildExt }}'"'
       export CNI_TYPE=${{ parameters.clusterDefinitionCniTypeKey }}
       echo CNI type is $CNI_TYPE
-      sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":$CNI_URL|g" clusterDefinition.json
-      sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" clusterDefinition.json
+      sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":$CNI_URL|g" '${{ parameters.clusterDefinition }}'
+      sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" '${{ parameters.clusterDefinition }}'
       echo "Running E2E tests against a cluster built with the following API model:"
-      cat ./clusterDefinition.json
+      cat '${{ parameters.clusterDefinition }}'
+      cp ${{ parameters.clusterDefinition }} clusterDefinition.json
       curl -L https://dl.k8s.io/v1.16.0/kubernetes-client-linux-amd64.tar.gz | tar xvzf -
       sudo cp kubernetes/client/bin/kubectl /usr/local/bin/kubectl
       sudo cp kubernetes/client/bin/kubectl /usr/local/bin/k
@@ -38,7 +46,7 @@ steps:
     workingDirectory: "$(modulePath)"
 
   - bash: |
-      export CLUSTER_DEFINITION=./clusterDefinition.json 
+      export CLUSTER_DEFINITION=./'${{ parameters.clusterDefinition }}'
       export ORCHESTRATOR=kubernetes 
       export CREATE_VNET=false 
       export TIMEOUT=10m 
@@ -50,6 +58,16 @@ steps:
       export REGIONS=$(AKS_ENGINE_REGION) 
       export IS_JENKINS=false 
       make test-kubernetes
+      mkdir -p $(Build.ArtifactStagingDirectory)/kube-${{ parameters.name }}
+      cp -r _output/k*/kubeconfig/kubeconfig.$REGIONS.json $(Build.ArtifactStagingDirectory)/kube-${{ parameters.name }}
+      cp -r _output/kubernetes-*-ssh $(Build.ArtifactStagingDirectory)/kube-${{ parameters.name }}
     name: DeployAKSEngine
     displayName: Deploy AKS-Engine
     workingDirectory: "$(modulePath)"
+
+  - task: PublishBuildArtifacts@1
+    inputs:
+      artifactName: "kube-${{ parameters.name }}"
+      pathtoPublish: "$(Build.ArtifactStagingDirectory)/kube-${{ parameters.name }}"
+    displayName: "Save cluster configs"
+    condition: always()

.pipelines/pipeline.yaml

@@ -80,6 +80,7 @@ stages:
             displayName: "Install Go dependencies"
 
           - script: |
+              echo Building in $(pwd)
               echo Build tag is $(EnvironmentalVariables.Tag)
               export GOOS=linux
               make all-binaries VERSION=$(EnvironmentalVariables.Tag)
@@ -103,14 +104,15 @@ stages:
                 } | { read xs; exit $xs; } 
               } 4>&1
             workingDirectory: "$(modulePath)"
-            failOnStderr: true
             name: "Test"
             displayName: "Run Tests"
 
           - bash: |
               bash <(curl -s https://codecov.io/bash)
               gocov convert coverage.out > coverage.json
               gocov-xml < coverage.json > coverage.xml
+              echo listing cluster definitions
+              ls $(modulePath)/test/e2e/kubernetes/*
             workingDirectory: "$(modulePath)"
             name: "Coverage"
             displayName: "Generate Coverage Reports"
@@ -140,6 +142,9 @@ stages:
               pathtoPublish: "$(Build.ArtifactStagingDirectory)"
             condition: succeeded()
 
+          - publish: $(modulePath)/test/e2e/kubernetes/
+            artifact: clusterdefinitions
+
           - task: AzureCLI@1
             inputs:
               azureSubscription: $(ARTIFACT_SUBSCRIPTION)
@@ -153,22 +158,41 @@ stages:
       - template: e2e-job-template.yaml
         parameters:
           name: "ubuntu_16_04_linux_e2e"
-          pipelineBuildImage: "containernetworking/pipeline-ci:1.0.4"
-          clusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json"
+          pipelineBuildImage: "containernetworking/pipeline-ci:1.0.5"
+          clusterDefinition: "cniLinux1604.json"
+          clusterDefinitionCniTypeKey: "azureCNIURLLinux"
+          clusterDefinitionCniBuildOS: "linux"
+          clusterDefinitionCniBuildExt: ".tgz"
+
+      - template: e2e-job-template.yaml
+        parameters:
+          name: "ubuntu_18_04_linux_e2e"
+          pipelineBuildImage: "containernetworking/pipeline-ci:1.0.5"
+          clusterDefinition: "cniLinux1804.json"
           clusterDefinitionCniTypeKey: "azureCNIURLLinux"
           clusterDefinitionCniBuildOS: "linux"
           clusterDefinitionCniBuildExt: ".tgz"
 
       - template: e2e-job-template.yaml
         parameters:
-          name: "windows_e2e"
-          pipelineBuildImage: "containernetworking/pipeline-ci:1.0.4"
-          clusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniWindows.json"
+          name: "windows_18_09_e2e"
+          pipelineBuildImage: "containernetworking/pipeline-ci:1.0.5"
+          clusterDefinition: "cniWindows1809.json"
+          clusterDefinitionCniTypeKey: "azureCNIURLWindows"
+          clusterDefinitionCniBuildOS: "windows"
+          clusterDefinitionCniBuildExt: ".zip"
+
+      - template: e2e-job-template.yaml
+        parameters:
+          name: "windows_19_03_e2e"
+          pipelineBuildImage: "containernetworking/pipeline-ci:1.0.5"
+          clusterDefinition: "cniWindows1903.json"
           clusterDefinitionCniTypeKey: "azureCNIURLWindows"
           clusterDefinitionCniBuildOS: "windows"
           clusterDefinitionCniBuildExt: ".zip"
 
   - stage: cleanup
+    condition: always()
     jobs:
       - job: delete_remote_artifacts
         pool:
@@ -178,7 +202,7 @@ stages:
           image: containernetworking/pipeline-ci:1.0.4
         variables:
           Tag: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.Tag'] ]
-
+        condition: always()
         steps:
           - checkout: none
           - task: AzureCLI@1
@@ -190,3 +214,4 @@ stages:
                 az storage container delete -n acn-$CommitHash --account-name $(STORAGE_ACCOUNT_NAME)
               workingDirectory: "$(modulePath)"
             displayName: Cleanup remote Azure storage container
+            condition: always()

test/e2e/kubernetes/cniLinux1604.json

@@ -0,0 +1,44 @@
+{
+   "apiVersion": "vlabs",
+   "properties": {
+      "orchestratorProfile": {
+         "orchestratorType": "Kubernetes",
+         "orchestratorRelease": "1.14",
+         "kubernetesConfig": {
+            "networkPlugin": "azure",
+            "networkPolicy": "azure",
+            "azureCNIVersion": "",
+            "azureCNIURLLinux": ""
+         }
+      },
+      "masterProfile": {
+         "count": 1,
+         "dnsPrefix": "cniLinux",
+         "vmSize": "Standard_D2_v2"
+      },
+      "agentPoolProfiles": [
+         {
+            "name": "agentpool1",
+            "count": 3,
+            "vmSize": "Standard_D2_v2",
+            "availabilityProfile": "AvailabilitySet",
+            "osType": "Linux",
+            "distro": "aks-ubuntu-16.04"
+         }
+      ],
+      "linuxProfile": {
+         "adminUsername": "azureuser",
+         "ssh": {
+            "publicKeys": [
+               {
+                  "keyData": ""
+               }
+            ]
+         }
+      },
+      "servicePrincipalProfile": {
+         "clientId": "",
+         "secret": ""
+      }
+   }
+}

test/e2e/kubernetes/cniLinux1804.json

@@ -0,0 +1,44 @@
+{
+   "apiVersion": "vlabs",
+   "properties": {
+      "orchestratorProfile": {
+         "orchestratorType": "Kubernetes",
+         "orchestratorRelease": "1.14",
+         "kubernetesConfig": {
+            "networkPlugin": "azure",
+            "networkPolicy": "azure",
+            "azureCNIVersion": "",
+            "azureCNIURLLinux": ""
+         }
+      },
+      "masterProfile": {
+         "count": 1,
+         "dnsPrefix": "cniLinux",
+         "vmSize": "Standard_D2_v2"
+      },
+      "agentPoolProfiles": [
+         {
+            "name": "agentpool1",
+            "count": 3,
+            "vmSize": "Standard_D2_v2",
+            "availabilityProfile": "AvailabilitySet",
+            "osType": "Linux",
+            "distro": "aks-ubuntu-18.04"
+         }
+      ],
+      "linuxProfile": {
+         "adminUsername": "azureuser",
+         "ssh": {
+            "publicKeys": [
+               {
+                  "keyData": ""
+               }
+            ]
+         }
+      },
+      "servicePrincipalProfile": {
+         "clientId": "",
+         "secret": ""
+      }
+   }
+}