E2E CI

.pipelines/Dockerfile

@@ -1,9 +1,10 @@
 FROM ubuntu:16.04
-RUN apt-get update && apt-get install -y software-properties-common sudo wget apt-transport-https curl
+RUN apt-get update && apt-get install -y software-properties-common sudo wget apt-transport-https curl lsb-release gnupg
 RUN wget -q https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
 RUN sudo dpkg -i packages-microsoft-prod.deb
-RUN add-apt-repository ppa:longsleep/golang-backports && apt-get update
-RUN apt-get install -y git golang-go=2:1.13~1longsleep1+xenial iptables ipset iproute2 ebtables python-pip gcc zip dotnet-sdk-2.2
+RUN add-apt-repository ppa:longsleep/golang-backports 
+RUN curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+RUN apt-get install -y git golang-go=2:1.13~1longsleep1+xenial iptables ipset iproute2 ebtables python-pip gcc zip dotnet-sdk-2.2 azure-cli
 RUN sudo pip install coverage
 RUN if [ -f Gopkg.toml ]; then curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh ; fi 
 RUN go get github.com/docker/libnetwork/driverapi

.pipelines/e2e-job-template.yaml

@@ -0,0 +1,30 @@
+parameters:
+  name: ""
+  pipelineBuildImage: "containernetworking/pipeline-ci:1.0.4"
+  clusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json"
+  clusterDefinitionCniTypeKey: ""
+  clusterDefinitionCniBuildOS: ""
+  clusterDefinitionCniBuildExt: ""
+
+jobs:
+  - job: ${{ parameters.name }}
+    dependsOn: unit_tests
+    pool:
+      name: Networking-ContainerNetworking
+      demands: agent.os -equals Linux
+    container:
+      image: ${{ parameters.pipelineBuildImage }}
+    variables:
+      GOPATH: "$(System.DefaultWorkingDirectory)/gopath"
+      GOBIN: "$(GOPATH)/bin" # Go binaries path
+      modulePath: "$(GOPATH)/src/github.com/Azure/aks-engine"
+      Tag: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.Tag'] ]
+      CommitHash: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.CommitHash'] ]
+
+    steps:
+      - template: e2e-step-template.yaml
+        parameters:
+          clusterDefinitionUrl: ${{ parameters.clusterDefinitionUrl }}
+          clusterDefinitionCniTypeKey: ${{ parameters.clusterDefinitionCniTypeKey }}
+          clusterDefinitionCniBuildOS: ${{ parameters.clusterDefinitionCniBuildOS }}
+          clusterDefinitionCniBuildExt: ${{ parameters.clusterDefinitionCniBuildExt }}

.pipelines/e2e-step-template.yaml

@@ -0,0 +1,55 @@
+parameters:
+  name: ""
+  clusterDefinitionUrl: ""
+  clusterDefinitionCniTypeKey: ""
+  clusterDefinitionCniBuildOS: ""
+  clusterDefinitionCniBuildExt: ""
+
+steps:
+  - checkout: none
+  - bash: |
+      go version
+      go env
+      go get -v github.com/Azure/aks-engine
+      mkdir -p '$(GOBIN)'
+      mkdir -p '$(GOPATH)/pkg'
+      mkdir -p '$(modulePath)'
+      echo '##vso[task.prependpath]$(GOBIN)'
+      echo '##vso[task.prependpath]$(GOROOT)/bin'
+    name: "GoEnv"
+    displayName: "Set up the Go environment"
+
+  - bash: |
+      rm -f clusterDefinition.json*
+      wget '${{ parameters.clusterDefinitionUrl }}' -O clusterDefinition.json
+      export CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-${{ parameters.clusterDefinitionCniBuildOS }}-amd64-$(Tag)${{ parameters.clusterDefinitionCniBuildExt }}'"'
+      export CNI_TYPE=${{ parameters.clusterDefinitionCniTypeKey }}
+      echo CNI type is $CNI_TYPE
+      sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":$CNI_URL|g" clusterDefinition.json
+      sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" clusterDefinition.json
+      echo "Running E2E tests against a cluster built with the following API model:"
+      cat ./clusterDefinition.json
+      curl -L https://dl.k8s.io/v1.16.0/kubernetes-client-linux-amd64.tar.gz | tar xvzf -
+      sudo cp kubernetes/client/bin/kubectl /usr/local/bin/kubectl
+      sudo cp kubernetes/client/bin/kubectl /usr/local/bin/k
+      make bootstrap
+      make build-binary
+    displayName: Build AKS-Engine
+    workingDirectory: "$(modulePath)"
+
+  - bash: |
+      export CLUSTER_DEFINITION=./clusterDefinition.json 
+      export ORCHESTRATOR=kubernetes 
+      export CREATE_VNET=false 
+      export TIMEOUT=10m 
+      export CLIENT_ID=$(AKS_ENGINE_CLIENT_ID) 
+      export CLIENT_SECRET=$(AKS_ENGINE_CLIENT_SECRET) 
+      export TENANT_ID=$(AKS_ENGINE_TENANT_ID) 
+      export SUBSCRIPTION_ID=$(AKS_ENGINE_SUBSCRIPTION_ID) 
+      export CLEANUP_ON_EXIT=true 
+      export REGIONS=$(AKS_ENGINE_REGION) 
+      export IS_JENKINS=false 
+      make test-kubernetes
+    name: DeployAKSEngine
+    displayName: Deploy AKS-Engine
+    workingDirectory: "$(modulePath)"

.pipelines/pipeline.yaml

@@ -0,0 +1,191 @@
+pr:
+  branches:
+    include:
+      - master
+
+trigger:
+  branches:
+    include:
+      - master
+
+stages:
+  - stage: build_and_test
+    jobs:
+      - job: unit_tests
+        pool:
+          name: Networking-ContainerNetworking
+          demands: agent.os -equals Linux
+
+        container:
+          image: containernetworking/pipeline-ci:1.0.4
+          options: "--privileged"
+
+        # Go setup for the vmImage:
+        # https://github.com/Microsoft/azure-pipelines-image-generation/blob/master/images/linux/scripts/installers/go.sh
+        variables:
+          GOBIN: "$(GOPATH)/bin" # Go binaries path
+          GOPATH: "$(System.DefaultWorkingDirectory)/gopath" # Go workspace path
+          modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" # $(build.repository.name)' # Path to the module's code
+
+        steps:
+          - bash: |
+              echo $UID
+              sudo rm /run/docker/plugins/test.sock || true
+              sudo ip link del dev dummy || true
+            displayName: "Set up OS environment"
+
+          - bash: |
+              go version
+              go env
+              mkdir -p '$(GOBIN)'
+              mkdir -p '$(GOPATH)/pkg'
+              mkdir -p '$(modulePath)'
+              shopt -s extglob
+              shopt -s dotglob
+              mv !(gopath) '$(modulePath)'
+              echo '##vso[task.prependpath]$(GOBIN)'
+              echo '##vso[task.prependpath]$(GOROOT)/bin'
+            name: "GoEnv"
+            displayName: "Set up the Go environment"
+
+          - script: |
+              echo "##vso[task.setvariable variable=CommitHash;isOutput=true]$(git rev-parse HEAD)"
+              BRANCH=$(git rev-parse --abbrev-ref HEAD)
+              if [[ "$BRANCH" == "master" ]]; then
+                echo "##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --abbrev=0)"
+                echo "Set tag to $(git describe --tags --abbrev=0)"
+              else
+                echo "##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --always --dirty)"
+                echo "Set tag to $(git describe --tags --always --dirty)"
+              fi
+            workingDirectory: "$(modulePath)"
+            name: "EnvironmentalVariables"
+            displayName: "Set build environmental variables"
+            condition: always()
+
+          - script: |
+              go get -v -t -d ./...
+              if [ -f Gopkg.toml ]; then
+                  curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+              fi
+              go get github.com/docker/libnetwork/driverapi
+              go get github.com/gorilla/mux
+              go get github.com/jstemmer/go-junit-report
+              go get github.com/axw/gocov/gocov
+              go get github.com/AlekSi/gocov-xml
+              go get -u gopkg.in/matm/v1/gocov-html
+            workingDirectory: "$(modulePath)"
+            name: "GoDependencies"
+            displayName: "Install Go dependencies"
+
+          - script: |
+              echo Build tag is $(EnvironmentalVariables.Tag)
+              export GOOS=linux
+              make all-binaries VERSION=$(EnvironmentalVariables.Tag)
+              export GOOS=windows
+              make all-binaries VERSION=$(EnvironmentalVariables.Tag)
+              cd output 
+              sudo find . -mindepth 2 -type f -regextype posix-extended ! -iregex '.*\.(zip|tgz)$' -delete
+              sudo find . -mindepth 2 -type f -print -exec mv {} . \;
+              sudo rm -R -- */
+            workingDirectory: "$(modulePath)"
+            name: "Build"
+            displayName: "Build"
+
+          - bash: |
+              # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout
+              { { { { 
+                    sudo -E env "PATH=$PATH" make test-all; 
+                    echo $? >&3; 
+                    } | tee >(go-junit-report > report.xml) >&4; 
+                  } 3>&1; 
+                } | { read xs; exit $xs; } 
+              } 4>&1
+            workingDirectory: "$(modulePath)"
+            failOnStderr: true
+            name: "Test"
+            displayName: "Run Tests"
+
+          - bash: |
+              bash <(curl -s https://codecov.io/bash)
+              gocov convert coverage.out > coverage.json
+              gocov-xml < coverage.json > coverage.xml
+            workingDirectory: "$(modulePath)"
+            name: "Coverage"
+            displayName: "Generate Coverage Reports"
+            condition: always()
+
+          - task: PublishTestResults@2
+            inputs:
+              testRunner: JUnit
+              testResultsFiles: $(System.DefaultWorkingDirectory)/**/report.xml
+            condition: always()
+
+          - task: PublishCodeCoverageResults@1
+            inputs:
+              codeCoverageTool: Cobertura
+              summaryFileLocation: $(System.DefaultWorkingDirectory)/**/coverage.xml
+            condition: always()
+
+          - task: CopyFiles@2
+            inputs:
+              sourceFolder: "$(modulePath)/output"
+              targetFolder: $(Build.ArtifactStagingDirectory)
+            condition: succeeded()
+
+          - task: PublishBuildArtifacts@1
+            inputs:
+              artifactName: "output"
+              pathtoPublish: "$(Build.ArtifactStagingDirectory)"
+            condition: succeeded()
+
+          - task: AzureCLI@1
+            inputs:
+              azureSubscription: $(ARTIFACT_SUBSCRIPTION)
+              scriptLocation: "inlineScript"
+              inlineScript: |
+                az storage container create -n acn-$(EnvironmentalVariables.CommitHash) --account-name $(STORAGE_ACCOUNT_NAME) --public-access container
+                az storage blob upload-batch -d acn-$(EnvironmentalVariables.CommitHash) -s ./output/  --account-name $(STORAGE_ACCOUNT_NAME)
+              workingDirectory: "$(modulePath)"
+            displayName: Create artifact storage container
+
+      - template: e2e-job-template.yaml
+        parameters:
+          name: "ubuntu_16_04_linux_e2e"
+          pipelineBuildImage: "containernetworking/pipeline-ci:1.0.4"
+          clusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json"
+          clusterDefinitionCniTypeKey: "azureCNIURLLinux"
+          clusterDefinitionCniBuildOS: "linux"
+          clusterDefinitionCniBuildExt: ".tgz"
+
+      - template: e2e-job-template.yaml
+        parameters:
+          name: "windows_e2e"
+          pipelineBuildImage: "containernetworking/pipeline-ci:1.0.4"
+          clusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniWindows.json"
+          clusterDefinitionCniTypeKey: "azureCNIURLWindows"
+          clusterDefinitionCniBuildOS: "windows"
+          clusterDefinitionCniBuildExt: ".zip"
+
+  - stage: cleanup
+    jobs:
+      - job: delete_remote_artifacts
+        pool:
+          name: Networking-ContainerNetworking
+          demands: agent.os -equals Linux
+        container:
+          image: containernetworking/pipeline-ci:1.0.4
+        variables:
+          Tag: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.Tag'] ]
+
+        steps:
+          - checkout: none
+          - task: AzureCLI@1
+            inputs:
+              azureSubscription: $(ARTIFACT_SUBSCRIPTION)
+              scriptLocation: "inlineScript"
+              inlineScript: |
+                export CommitHash=(git rev-parse HEAD)
+                az storage container delete -n acn-$CommitHash --account-name $(STORAGE_ACCOUNT_NAME)
+              workingDirectory: "$(modulePath)"
+            displayName: Cleanup remote Azure storage container