From 9156565707e6f134067b0584da2b24790be22d4d Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@microsoft.com>
Date: Tue, 20 Apr 2021 14:24:10 -0700
Subject: [PATCH 01/15] codecov

---
 .pipelines/pipeline.yaml                      | 257 +++++++++---------
 .../aks-engine/e2e-step-template.yaml         |  60 ++--
 .../aks-swift/e2e-step-template.yaml          |   3 +-
 3 files changed, 177 insertions(+), 143 deletions(-)

diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
index d18ce9b3aa..571174dc14 100644
--- a/.pipelines/pipeline.yaml
+++ b/.pipelines/pipeline.yaml
@@ -10,112 +10,112 @@ trigger:
 
 stages:
   - stage: build_and_test
-    displayName: Build and Unit Test
+    displayName: ACN
     jobs:
-      - job: unit_tests
-        displayName: CNI, NPM, CNS, CNM
+      - job: setup
+        displayName: Setup 
         pool:
           name: "$(BUILD_POOL_NAME_DEFAULT)"
-
-        # Go setup for the vmImage:
-        # https://github.com/Microsoft/azure-pipelines-image-generation/blob/master/images/linux/scripts/installers/go.sh
-        variables:
-          GOBIN: "$(GOPATH)/bin" # Go binaries path
-          GOPATH: "$(System.DefaultWorkingDirectory)/gopath" # Go workspace path
-          modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" # $(build.repository.name)' # Path to the module's code
-
         steps:
-
-          - bash: |
-              go version 
-              mkdir -p '$(GOBIN)'
-              mkdir -p '$(GOPATH)/pkg'
-              mkdir -p '$(modulePath)'
-              shopt -s extglob
-              shopt -s dotglob
-              mv !(gopath) '$(modulePath)'
-              echo '##vso[task.prependpath]$(GOBIN)'
-              echo '##vso[task.prependpath]$(GOROOT)/bin'
-              go env
-            name: "GoEnv"
-            displayName: "Set up the Go environment"
-
           - script: |
               BUILD_NUMBER=$(Build.BuildNumber)
               echo "##vso[task.setvariable variable=StorageID;isOutput=true]$(echo ${BUILD_NUMBER//./-})"
-              echo "##vso[task.setvariable variable=CommitHash;isOutput=true]$(git rev-parse HEAD)"
               echo "##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --always --dirty)"
-              echo "##vso[task.setvariable variable=ImageTag;isOutput=true]$(git describe --tags --always --dirty)-test"
               sudo chown -R $(whoami):$(whoami) .
-            workingDirectory: "$(modulePath)"
             name: "EnvironmentalVariables"
             displayName: "Set environmental variables"
             condition: always()
 
-          - script: |
-              go get github.com/docker/libnetwork/driverapi
-              go get github.com/gorilla/mux
-              go get github.com/jstemmer/go-junit-report
-              go get github.com/axw/gocov/gocov
-              go get github.com/AlekSi/gocov-xml
-              go get github.com/matm/gocov-html
-              go get github.com/rakyll/gotest
-            workingDirectory: "$(modulePath)"
-            name: "GoDependencies"
-            displayName: "Install Go dependencies"
+      - job: build
+        displayName: Build Binaries
+        dependsOn:
+          - "setup"
+        variables:
+          TAG: $[ dependencies.setup.outputs['EnvironmentalVariables.Tag'] ]
+          STORAGE_ID: $[ dependencies.setup.outputs['EnvironmentalVariables.StorageID'] ]
+        pool:
+          name: "$(BUILD_POOL_NAME_DEFAULT)"
+        steps:
 
           - script: |
-              GOOS=windows make all-binaries VERSION=$(EnvironmentalVariables.Tag)
-            workingDirectory: "$(modulePath)"
+              GOOS=windows make all-binaries VERSION=$(TAG)
             name: "BuildWindows"
             displayName: "Build Windows"
 
           - script: |
-              GOOS=linux make all-binaries VERSION=$(EnvironmentalVariables.Tag)
-            workingDirectory: "$(modulePath)"
+              GOOS=linux make all-binaries VERSION=$(TAG)
             name: "BuildLinux"
             displayName: "Build Linux"
 
-          - bash: |
-              # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout
-              { { { { 
-                    sudo -E env "PATH=$PATH" make test-all; 
-                    echo $? >&3; 
-                    } | tee >(go-junit-report > report.xml) >&4; 
-                  } 3>&1; 
-                } | { read xs; exit $xs; } 
-              } 4>&1
-            workingDirectory: "$(modulePath)"
-            name: "Test"
-            displayName: "Run Tests"
+          - script: |
+              mkdir -p ./output/bins
+              cd ./output/linux_amd64
+              sudo find . -mindepth 2 -type f -regextype posix-extended ! -iregex '.*\.(zip|tgz)$' -delete
+              sudo find . -mindepth 2 -type f -print -exec mv {} ../bins \;
+              sudo rm -R -- */ && cd ..
+              cd ./windows_amd64
+              sudo find . -mindepth 2 -type f -regextype posix-extended ! -iregex '.*\.(zip|tgz)$' -delete
+              sudo find . -mindepth 2 -type f -print -exec mv {} ../bins \;
+              sudo rm -R -- */ && cd ..
+              rmdir ./linux_amd64 && rmdir ./windows_amd64
+            name: "PrepareArtifacts"
+            displayName: "Prepare Artifacts"
+
+          - task: CopyFiles@2
+            inputs:
+              sourceFolder: "output"
+              targetFolder: $(Build.ArtifactStagingDirectory)
+            condition: succeeded()
+
+          - task: PublishBuildArtifacts@1
+            inputs:
+              artifactName: "output"
+              pathtoPublish: "$(Build.ArtifactStagingDirectory)"
+            condition: succeeded()
 
+          - task: AzureCLI@1
+            inputs:
+              azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+              scriptLocation: "inlineScript"
+              inlineScript: |
+                echo Creating storage container with name acn-$(STORAGE_ID) and account name $(STORAGE_ACCOUNT_NAME)
+                az storage container create -n acn-$(STORAGE_ID) --account-name $(STORAGE_ACCOUNT_NAME) --public-access container
+                az storage blob upload-batch -d acn-$(STORAGE_ID) -s ./output/bins/  --account-name $(STORAGE_ACCOUNT_NAME)
+            displayName: Create artifact storage container
+            condition: succeeded()
+
+      - job: build_images
+        displayName: Build Images
+        dependsOn:
+          - "setup"
+        variables:
+          TAG: $[ dependencies.setup.outputs['EnvironmentalVariables.Tag'] ]
+        pool:
+          name: "$(BUILD_POOL_NAME_DEFAULT)"
+        steps:
           - script: |
-              sudo make tools-images VERSION=$(EnvironmentalVariables.Tag)
-              sudo make all-images VERSION=$(EnvironmentalVariables.Tag)
-            workingDirectory: "$(modulePath)"
+              echo Tag is $(TAG)
+              sudo make tools-images VERSION=$(TAG)
+              sudo make all-images VERSION=$(TAG)
             name: "BuildImages"
             displayName: "Build Images"
 
-          - bash: |
-              export CODECOV_TOKEN=$(CODECOV_TOKEN)
-              bash <(curl -s https://codecov.io/bash)
-              gocov convert coverage.out > coverage.json
-              gocov-xml < coverage.json > coverage.xml
-            workingDirectory: "$(modulePath)"
-            name: "Coverage"
-            displayName: "Generate Coverage Reports"
-            condition: always()
+          - task: Docker@2
+            displayName: Docker Login
+            inputs:
+              containerRegistry: $(ACR_SERVICE_CONNECTION)
+              command: 'login'
+              addPipelineData: false
 
           - script: |
-              echo '$(DOCKER_PASSWORD)' | docker login $IMAGE_REGISTRY -u '$(DOCKER_USERNAME)' --password-stdin
-              docker tag $IMAGE_REGISTRY/azure-npm:$(EnvironmentalVariables.Tag) $IMAGE_REGISTRY/azure-npm:$(EnvironmentalVariables.Tag)-test
-              docker push $IMAGE_REGISTRY/azure-npm:$(EnvironmentalVariables.Tag)-test
+              docker tag $IMAGE_REGISTRY/azure-npm:$(TAG) $IMAGE_REGISTRY/azure-npm:$(TAG)-test
+              docker push $IMAGE_REGISTRY/azure-npm:$(TAG)-test
 
-              docker tag $IMAGE_REGISTRY/azure-cns:$(EnvironmentalVariables.Tag) $IMAGE_REGISTRY/azure-cns:$(EnvironmentalVariables.Tag)-test
-              docker push $IMAGE_REGISTRY/azure-cns:$(EnvironmentalVariables.Tag)-test
+              docker tag $IMAGE_REGISTRY/azure-cns:$(TAG) $IMAGE_REGISTRY/azure-cns:$(TAG)-test
+              docker push $IMAGE_REGISTRY/azure-cns:$(TAG)-test
 
-              docker tag $IMAGE_REGISTRY/azure-cni-manager:$(EnvironmentalVariables.Tag) $IMAGE_REGISTRY/azure-cni-manager:$(EnvironmentalVariables.Tag)-test
-              docker push $IMAGE_REGISTRY/azure-cni-manager:$(EnvironmentalVariables.Tag)-test
+              docker tag $IMAGE_REGISTRY/azure-cni-manager:$(TAG) $IMAGE_REGISTRY/azure-cni-manager:$(TAG)-test
+              docker push $IMAGE_REGISTRY/azure-cni-manager:$(TAG)-test
 
               function auto-retry()
               {
@@ -127,45 +127,22 @@ stages:
                   done
               }
 
-              auto-retry docker pull $IMAGE_REGISTRY/azure-npm:$(EnvironmentalVariables.Tag)-test
-              auto-retry docker pull $IMAGE_REGISTRY/azure-cns:$(EnvironmentalVariables.Tag)-test
-              auto-retry docker pull $IMAGE_REGISTRY/azure-cni-manager:$(EnvironmentalVariables.Tag)-test
-            workingDirectory: "$(modulePath)"
+              auto-retry docker pull $IMAGE_REGISTRY/azure-npm:$(TAG)-test
+              auto-retry docker pull $IMAGE_REGISTRY/azure-cns:$(TAG)-test
+              auto-retry docker pull $IMAGE_REGISTRY/azure-cni-manager:$(TAG)-test
             name: "mcrreplication"
             displayName: "Push NPM Image and Wait for Repository"
 
-          - script: |
-              mkdir -p ./output/bins
-              cd ./output/linux_amd64
-              sudo find . -mindepth 2 -type f -regextype posix-extended ! -iregex '.*\.(zip|tgz)$' -delete
-              sudo find . -mindepth 2 -type f -print -exec mv {} ../bins \;
-              sudo rm -R -- */ && cd ..
-              cd ./windows_amd64
-              sudo find . -mindepth 2 -type f -regextype posix-extended ! -iregex '.*\.(zip|tgz)$' -delete
-              sudo find . -mindepth 2 -type f -print -exec mv {} ../bins \;
-              sudo rm -R -- */ && cd ..
-              rmdir ./linux_amd64 && rmdir ./windows_amd64
-            workingDirectory: "$(modulePath)"
-            name: "PrepareArtifacts"
-            displayName: "Prepare Artifacts"
-
-          - task: PublishTestResults@2
-            inputs:
-              testRunner: JUnit
-              testResultsFiles: $(modulePath)/report.xml
-            displayName: "Publish Test Results"
-            condition: always()
-
-          - task: PublishCodeCoverageResults@1
+          - task: Docker@2
+            displayName: Docker Logout
             inputs:
-              codeCoverageTool: Cobertura
-              summaryFileLocation: $(modulePath)/coverage.xml
-            displayName: "Publish Code Coverage Results"
-            condition: always()
+              containerRegistry: $(ACR_SERVICE_CONNECTION)
+              command: 'logout'
+              addPipelineData: false
 
           - task: CopyFiles@2
             inputs:
-              sourceFolder: "$(modulePath)/output"
+              sourceFolder: "output"
               targetFolder: $(Build.ArtifactStagingDirectory)
             condition: succeeded()
 
@@ -175,20 +152,59 @@ stages:
               pathtoPublish: "$(Build.ArtifactStagingDirectory)"
             condition: succeeded()
 
-          - publish: $(modulePath)/test/apimodels/
-            artifact: clusterdefinitions
+      - job: test
+        displayName: Run Tests
+        dependsOn:
+          - "setup"
+        variables:
+          STORAGE_ID: $[ dependencies.setup.outputs['EnvironmentalVariables.StorageID'] ]
+        pool:
+          name: "$(BUILD_POOL_NAME_DEFAULT)"
+        steps:
 
-          - task: AzureCLI@1
+          - script: |
+              go env
+              export PATH=$HOME/go/bin:$PATH
+              go get github.com/docker/libnetwork/driverapi
+              go get github.com/gorilla/mux
+              go get github.com/jstemmer/go-junit-report
+              go get github.com/axw/gocov/gocov
+              go get github.com/AlekSi/gocov-xml
+              go get github.com/matm/gocov-html
+              go get github.com/rakyll/gotest
+
+              # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout
+              { { { { 
+                    sudo -E env "PATH=$PATH" make test-all; 
+                    echo $? >&3; 
+                    } | tee >(go-junit-report > report.xml) >&4; 
+                  } 3>&1; 
+                } | { read xs; exit $xs; } 
+              } 4>&1
+            name: "Test"
+            displayName: "Run Tests"
+
+          - bash: |
+              export PATH=$HOME/go/bin:$PATH
+              gocov convert coverage.out > coverage.json
+              gocov-xml < coverage.json > coverage.xml
+            name: "Coverage"
+            displayName: "Generate Coverage Reports"
+            condition: always()
+
+          - task: PublishTestResults@2
             inputs:
-              azureSubscription: $(ARTIFACT_SUBSCRIPTION)
-              scriptLocation: "inlineScript"
-              inlineScript: |
-                echo Creating storage container with name acn-$(EnvironmentalVariables.StorageID) and account name $(STORAGE_ACCOUNT_NAME)
-                az storage container create -n acn-$(EnvironmentalVariables.StorageID) --account-name $(STORAGE_ACCOUNT_NAME) --public-access container
-                az storage blob upload-batch -d acn-$(EnvironmentalVariables.StorageID) -s ./output/bins/  --account-name $(STORAGE_ACCOUNT_NAME)
-              workingDirectory: "$(modulePath)"
-            displayName: Create artifact storage container
-            condition: succeeded()
+              testRunner: JUnit
+              testResultsFiles: report.xml
+            displayName: "Publish Test Results"
+            condition: always()
+
+          - task: PublishCodeCoverageResults@1
+            inputs:
+              codeCoverageTool: Cobertura
+              summaryFileLocation: coverage.xml
+            displayName: "Publish Code Coverage Results"
+            condition: always()
 
   - template: singletenancy/aks-swift/e2e-job-template.yaml
     parameters:
@@ -243,7 +259,7 @@ stages:
           - checkout: none
           - task: AzureCLI@1
             inputs:
-              azureSubscription: $(ARTIFACT_SUBSCRIPTION)
+              azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
               scriptLocation: "inlineScript"
               inlineScript: |
                 BUILD_NUMBER=$(Build.BuildNumber)
@@ -252,5 +268,4 @@ stages:
                 az storage container delete -n acn-$BUILD_NUMBER --account-name $(STORAGE_ACCOUNT_NAME)
                 echo Pruning old docker images...
                 sudo docker system prune -f
-              workingDirectory: "$(modulePath)"
             displayName: Cleanup remote Azure storage container
diff --git a/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml b/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
index 0cccab1573..aed529ff3a 100644
--- a/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
@@ -72,26 +72,46 @@ steps:
     displayName: Configure AKS-Engine
     workingDirectory: "$(modulePath)"
 
-  - bash: |
-      export PATH=$PATH:'$(GOPATH)'
-      export CLUSTER_DEFINITION=./'${{ parameters.clusterDefinition }}'
-      export ORCHESTRATOR=kubernetes 
-      export CREATE_VNET=false 
-      export TIMEOUT=20m 
-      export CLIENT_ID=$(AKS_ENGINE_CLIENT_ID) 
-      export CLIENT_SECRET=$(AKS_ENGINE_CLIENT_SECRET) 
-      export TENANT_ID=$(AKS_ENGINE_TENANT_ID) 
-      export SUBSCRIPTION_ID=$(AKS_ENGINE_SUBSCRIPTION_ID) 
-      export CLEANUP_ON_EXIT=true
-      export CLEANUP_IF_FAIL=false
-      export REGIONS=$(AKS_ENGINE_REGION) 
-      export IS_JENKINS=false 
-      export DEBUG_CRASHING_PODS=true
-      export AZURE_CORE_ONLY_SHOW_ERRORS=True
-      make test-kubernetes
-    name: DeployAKSEngine
-    displayName: Run AKS-Engine E2E Tests
-    workingDirectory: "$(modulePath)"
+
+
+  - task: AzureCLI@1
+    inputs:
+      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      inlineScript: |
+
+        if [[ -z "${CLIENT_ID}" ]]; then
+          AZSP=$(az ad sp create-for-rbac --name pipeline-aks-engine-$(Build.BuildId) --role="Contributor" --scopes="/subscriptions/${SUBSCRIPTION_ID}")
+          CLIENT_ID=$(echo $AZSP | jq -r '.appId')
+          CLIENT_SECRET=$(echo $AZSP | jq -r '.password')
+          echo Sleeping 40 seconds
+          sleep 40
+        fi
+        export PATH=$PATH:'$(GOPATH)'
+        export CLUSTER_DEFINITION=./'${{ parameters.clusterDefinition }}'
+        export ORCHESTRATOR=kubernetes 
+        export CREATE_VNET=false 
+        export TIMEOUT=20m 
+        export TENANT_ID=$(AKS_ENGINE_TENANT_ID) 
+        export SUBSCRIPTION_ID=$(AKS_ENGINE_SUBSCRIPTION_ID) 
+        export CLEANUP_ON_EXIT=true
+        export CLEANUP_IF_FAIL=false
+        export REGIONS=$(AKS_ENGINE_REGION) 
+        export IS_JENKINS=false 
+        export DEBUG_CRASHING_PODS=true
+        export AZURE_CORE_ONLY_SHOW_ERRORS=True
+        make test-kubernetes
+    displayName: Run AKS-Engine E2E Tests 
+
+
+  - task: AzureCLI@1
+    inputs:
+      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+      scriptLocation: "inlineScript"
+      inlineScript: |
+        az ad sp delete pipeline-aks-engine-$(Build.BuildId)
+    displayName: Run AKS-Engine E2E Tests 
+    condition: always()
 
   - task: CopyFiles@2
     inputs:
diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 91833c51b0..2a28443e0b 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -26,7 +26,7 @@ steps:
 
   - task: AzureCLI@1
     inputs:
-      azureSubscription: $(ARTIFACT_SUBSCRIPTION)
+      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
       scriptLocation: "inlineScript"
       addSpnToEnvironment: true
       inlineScript: |
@@ -62,7 +62,6 @@ steps:
       export KUBECONFIG=~/.kube/dnc-underlay
       kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c dnc --prefix > dnc-logs.txt
       kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c requestcontroller --prefix > dnc-rc-logs.txt
-
       sudo cp *.txt $ARTIFACT_DIR
     name: "GetLogs"
     displayName: "Get logs"

From f5b59c033b68784de1d85cb1264d0fa635361d18 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@microsoft.com>
Date: Wed, 21 Apr 2021 11:02:45 -0700
Subject: [PATCH 02/15] whitespace

---
 .pipelines/npm/npm-conformance-tests.yaml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.pipelines/npm/npm-conformance-tests.yaml b/.pipelines/npm/npm-conformance-tests.yaml
index 5e18acaac8..7fdd289dd9 100644
--- a/.pipelines/npm/npm-conformance-tests.yaml
+++ b/.pipelines/npm/npm-conformance-tests.yaml
@@ -164,19 +164,13 @@ jobs:
         echo "##vso[task.setvariable variable=FQDN]$FQDN"
 
   - bash: |
-      ls -la
-      ls
-      ls -la $(Pipeline.Workspace)
-      ls -la $(Pipeline.Workspace)/Test
       echo $FQDN
-      ls ~
       chmod +x $(Pipeline.Workspace)/Test/e2e.test
       KUBERNETES_SERVICE_HOST="$FQDN" KUBERNETES_SERVICE_PORT=443 $(Pipeline.Workspace)/Test/e2e.test --provider=local --ginkgo.focus="NetworkPolicy" --ginkgo.skip="SCTP" --kubeconfig=./kubeconfig
     displayName: "Run Test Suite"
   - bash: |
       curl -LO https://dl.k8s.io/release/v1.20.0/bin/linux/amd64/kubectl
       chmod +x kubectl
-      ls -al
       npmPodList=`kubectl get pods -n kube-system | grep npm | awk '{print $1}'`
       mkdir -p $(System.DefaultWorkingDirectory)/npmLogs
       for npm in $npmPodList; do ./kubectl logs -n kube-system $npm --kubeconfig=./kubeconfig > $(System.DefaultWorkingDirectory)/npmLogs/$npm ;done

From ab6378c499b5d5bb88187ab712612c5342e4bd12 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@microsoft.com>
Date: Wed, 21 Apr 2021 11:12:49 -0700
Subject: [PATCH 03/15] conformance

---
 .pipelines/npm/npm-conformance-tests.yaml     | 21 +++++++++++++++----
 .../aks-engine/e2e-step-template.yaml         |  1 -
 2 files changed, 17 insertions(+), 5 deletions(-)

diff --git a/.pipelines/npm/npm-conformance-tests.yaml b/.pipelines/npm/npm-conformance-tests.yaml
index 7fdd289dd9..501d350e6b 100644
--- a/.pipelines/npm/npm-conformance-tests.yaml
+++ b/.pipelines/npm/npm-conformance-tests.yaml
@@ -64,15 +64,28 @@ jobs:
 
   - checkout: self
 
+  - task: Docker@2
+    displayName: Docker Login
+    inputs:
+      containerRegistry: $(ACR_SERVICE_CONNECTION)
+      command: 'login'
+      addPipelineData: false
+
   - script: |
       go env
       echo Tag: $(TAG)
       echo ResourceGroup: $(RESOURCE_GROUP)
       VERSION=$(TAG) make azure-npm-image
-      echo '$(DOCKER_PASSWORD)' | docker login $IMAGE_REGISTRY -u '$(DOCKER_USERNAME)' --password-stdin
       docker push $IMAGE_REGISTRY/azure-npm:$(TAG)
     displayName: 'Build and Push NPM Image'
 
+  - task: Docker@2
+    displayName: Docker Logout
+    inputs:
+      containerRegistry: $(ACR_SERVICE_CONNECTION)
+      command: 'logout'
+      addPipelineData: false
+
   - script: git clone https://github.com/vakalapa/kubernetes.git
     displayName: 'Clone Kubernetes Repo'
   - bash: |
@@ -107,7 +120,7 @@ jobs:
   - task: AzureCLI@2
     displayName: "Deploy"
     inputs:
-      azureSubscription: $(BUILD_VALIDATIONS_SUBSCRIPTION)
+      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
       scriptType: 'bash'
       scriptLocation: 'inlineScript'
       inlineScript: |
@@ -140,7 +153,7 @@ jobs:
   - task: AzureCLI@2
     displayName: "Deploy NPM to Test Cluster"
     inputs:
-      azureSubscription: $(BUILD_VALIDATIONS_SUBSCRIPTION)
+      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
       scriptType: 'bash'
       scriptLocation: 'inlineScript'
       inlineScript: |
@@ -200,7 +213,7 @@ jobs:
     - task: AzureCLI@2
       displayName: "Delete Test Cluster Resource Group"
       inputs:
-        azureSubscription: $(BUILD_VALIDATIONS_SUBSCRIPTION)
+        azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
         scriptType: 'bash'
         scriptLocation: 'inlineScript'
         inlineScript: |
diff --git a/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml b/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
index aed529ff3a..7a006fe804 100644
--- a/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
@@ -103,7 +103,6 @@ steps:
         make test-kubernetes
     displayName: Run AKS-Engine E2E Tests 
 
-
   - task: AzureCLI@1
     inputs:
       azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)

From f09fee3a778e542d780263a8f81482c22bac088e Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@microsoft.com>
Date: Wed, 21 Apr 2021 11:39:18 -0700
Subject: [PATCH 04/15] module path

---
 .pipelines/pipeline.yaml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
index 571174dc14..e47f5b1c26 100644
--- a/.pipelines/pipeline.yaml
+++ b/.pipelines/pipeline.yaml
@@ -84,6 +84,9 @@ stages:
             displayName: Create artifact storage container
             condition: succeeded()
 
+          - publish: ./test/apimodels/
+            artifact: clusterdefinitions
+
       - job: build_images
         displayName: Build Images
         dependsOn:

From d83db79794cf76141fb9a9b50bd53953de79b7d3 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@microsoft.com>
Date: Wed, 21 Apr 2021 12:25:10 -0700
Subject: [PATCH 05/15] remove images

---
 .pipelines/npm/npm-conformance-tests.yaml             | 11 +----------
 .../singletenancy/aks-engine/e2e-step-template.yaml   |  4 ++--
 .../singletenancy/aks-swift/e2e-job-template.yaml     |  3 ---
 3 files changed, 3 insertions(+), 15 deletions(-)

diff --git a/.pipelines/npm/npm-conformance-tests.yaml b/.pipelines/npm/npm-conformance-tests.yaml
index 501d350e6b..e570b7a4c6 100644
--- a/.pipelines/npm/npm-conformance-tests.yaml
+++ b/.pipelines/npm/npm-conformance-tests.yaml
@@ -17,8 +17,7 @@ jobs:
     demands: 
     - agent.os -equals Linux
     - Role -equals Build
-  container:
-    image: "$(BUILD_IMAGE)" # build image set as variable in pipeline runtime for flexibility
+
   steps:
 
   - checkout: self
@@ -48,8 +47,6 @@ jobs:
     demands: 
     - agent.os -equals Linux
     - Role -equals Build
-  container:
-    image: "$(BUILD_IMAGE)" # build image set as variable in pipeline runtime for flexibility
   dependsOn: [setup]
   variables:
     GOBIN: "$(GOPATH)/bin" # Go binaries path
@@ -104,8 +101,6 @@ jobs:
     demands: 
     - agent.os -equals Linux
     - Role -equals Build
-  container:
-    image: "$(BUILD_IMAGE)" # build image set as variable in pipeline runtime for flexibility
   dependsOn: [setup]
   variables:
     RESOURCE_GROUP: $[ dependencies.setup.outputs['EnvironmentalVariables.RESOURCE_GROUP'] ]
@@ -140,8 +135,6 @@ jobs:
     demands: 
     - agent.os -equals Linux
     - Role -equals Build
-  container:
-    image: "$(BUILD_IMAGE)" # build image set as variable in pipeline runtime for flexibility
   dependsOn: [Create_cluster, Build_test, setup]
   variables:
     RESOURCE_GROUP: $[ dependencies.setup.outputs['EnvironmentalVariables.RESOURCE_GROUP'] ]
@@ -201,8 +194,6 @@ jobs:
     demands: 
     - agent.os -equals Linux
     - Role -equals Build
-  container:
-    image: "$(BUILD_IMAGE)" # build image set as variable in pipeline runtime for flexibility
   condition: always()
   dependsOn: [Run_test, setup]
   variables:
diff --git a/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml b/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
index 7a006fe804..f1934dae7f 100644
--- a/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
@@ -108,8 +108,8 @@ steps:
       azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
       scriptLocation: "inlineScript"
       inlineScript: |
-        az ad sp delete pipeline-aks-engine-$(Build.BuildId)
-    displayName: Run AKS-Engine E2E Tests 
+        az ad sp delete --id pipeline-aks-engine-$(Build.BuildId)
+    displayName: Cleanup 
     condition: always()
 
   - task: CopyFiles@2
diff --git a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
index bc2a4da97a..69013ed76c 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
@@ -16,9 +16,6 @@ stages:
           demands: 
           - agent.os -equals Linux
           - Role -equals $(CUSTOM_E2E_ROLE)
-        container:
-          image: "$(BUILD_IMAGE)" # build image set as variable in pipeline runtime for flexibilit
-
         variables:
           GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
           GOBIN: "$(GOPATH)/bin" # Go binaries path

From fb80ae606fd2d4977bf22b3752f45a1deb1a19c2 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@microsoft.com>
Date: Wed, 21 Apr 2021 12:43:20 -0700
Subject: [PATCH 06/15] use spn

---
 .../aks-engine/e2e-step-template.yaml         | 27 ++++++-------------
 1 file changed, 8 insertions(+), 19 deletions(-)

diff --git a/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml b/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
index f1934dae7f..8721dc888c 100644
--- a/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
@@ -72,21 +72,15 @@ steps:
     displayName: Configure AKS-Engine
     workingDirectory: "$(modulePath)"
 
-
-
   - task: AzureCLI@1
     inputs:
       azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
       scriptLocation: "inlineScript"
+      addSpnToEnvironment: true
+      workingDirectory: "$(modulePath)"
       inlineScript: |
-
-        if [[ -z "${CLIENT_ID}" ]]; then
-          AZSP=$(az ad sp create-for-rbac --name pipeline-aks-engine-$(Build.BuildId) --role="Contributor" --scopes="/subscriptions/${SUBSCRIPTION_ID}")
-          CLIENT_ID=$(echo $AZSP | jq -r '.appId')
-          CLIENT_SECRET=$(echo $AZSP | jq -r '.password')
-          echo Sleeping 40 seconds
-          sleep 40
-        fi
+        export CLIENT_ID=$servicePrincipalId
+        export CLIENT_SECRET=$servicePrincipalKey 
         export PATH=$PATH:'$(GOPATH)'
         export CLUSTER_DEFINITION=./'${{ parameters.clusterDefinition }}'
         export ORCHESTRATOR=kubernetes 
@@ -101,16 +95,10 @@ steps:
         export DEBUG_CRASHING_PODS=true
         export AZURE_CORE_ONLY_SHOW_ERRORS=True
         make test-kubernetes
-    displayName: Run AKS-Engine E2E Tests 
+    name: DeployAKSEngine
+    displayName: Run AKS-Engine E2E Tests
+      
 
-  - task: AzureCLI@1
-    inputs:
-      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
-      scriptLocation: "inlineScript"
-      inlineScript: |
-        az ad sp delete --id pipeline-aks-engine-$(Build.BuildId)
-    displayName: Cleanup 
-    condition: always()
 
   - task: CopyFiles@2
     inputs:
@@ -123,3 +111,4 @@ steps:
       artifactName: ${{ parameters.name }}
       pathtoPublish: "$(Build.ArtifactStagingDirectory)/${{ parameters.name }}"
     condition: always()
+    
\ No newline at end of file

From eb073f46101663e0e26486c709fbdf562a04faa0 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@microsoft.com>
Date: Wed, 21 Apr 2021 16:24:34 -0700
Subject: [PATCH 07/15] echo

---
 .pipelines/singletenancy/aks-engine/e2e-step-template.yaml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml b/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
index 8721dc888c..a6625b4e3d 100644
--- a/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
@@ -62,6 +62,8 @@ steps:
       export CNI_URL=https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(StorageID)/azure-vnet-cni-${{ parameters.clusterDefinitionCniBuildOS }}-amd64-$(Tag)${{ parameters.clusterDefinitionCniBuildExt }}
       export CNI_TYPE=${{ parameters.clusterDefinitionCniTypeKey }}
       echo CNI type is $CNI_TYPE
+      echo Config:
+      cat '${{ parameters.clusterDefinition }}'
       cat '${{ parameters.clusterDefinition }}' | jq --arg cnikey $CNI_TYPE --arg cniurl $CNI_URL '.properties.orchestratorProfile.kubernetesConfig[$cnikey]= $cniurl' > '${{ parameters.clusterDefinition }}'.tmp	
       cat '${{ parameters.clusterDefinition }}'.tmp | jq --arg tag $(Tag) '.properties.orchestratorProfile.kubernetesConfig.azureCNIVersion = $tag' > '${{ parameters.clusterDefinition }}'
       cat '${{ parameters.clusterDefinition }}' | jq --arg npmimage $IMAGE_REGISTRY/azure-npm:$(Tag)-test '.properties.orchestratorProfile.kubernetesConfig.addons[0].containers[0].image = $npmimage' > '${{ parameters.clusterDefinition }}'.tmp
@@ -94,6 +96,8 @@ steps:
         export IS_JENKINS=false 
         export DEBUG_CRASHING_PODS=true
         export AZURE_CORE_ONLY_SHOW_ERRORS=True
+        echo Cluster Def $CLUSTER_DEFINITION
+        cat $CLUSTER_DEFINITION
         make test-kubernetes
     name: DeployAKSEngine
     displayName: Run AKS-Engine E2E Tests

From bda61d3b7b694c4d652f9eadf7ed1409f061157a Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@microsoft.com>
Date: Wed, 21 Apr 2021 17:46:26 -0700
Subject: [PATCH 08/15] build image

---
 .pipelines/singletenancy/aks-swift/e2e-job-template.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
index 69013ed76c..b787fa6791 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
@@ -16,6 +16,8 @@ stages:
           demands: 
           - agent.os -equals Linux
           - Role -equals $(CUSTOM_E2E_ROLE)
+        container:
+          image: "$(BUILD_IMAGE)" # build image set as variable in pipeline runtime for flexibilit
         variables:
           GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
           GOBIN: "$(GOPATH)/bin" # Go binaries path

From 243c32747bc9e8b37ea745a662b844b81e3f684c Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@microsoft.com>
Date: Wed, 21 Apr 2021 18:09:29 -0700
Subject: [PATCH 09/15] cluster

---
 .../singletenancy/aks-engine/e2e-step-template.yaml  | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml b/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
index a6625b4e3d..85cf54adfe 100644
--- a/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
@@ -69,8 +69,13 @@ steps:
       cat '${{ parameters.clusterDefinition }}' | jq --arg npmimage $IMAGE_REGISTRY/azure-npm:$(Tag)-test '.properties.orchestratorProfile.kubernetesConfig.addons[0].containers[0].image = $npmimage' > '${{ parameters.clusterDefinition }}'.tmp
       mv '${{ parameters.clusterDefinition }}'.tmp '${{ parameters.clusterDefinition }}'
       echo "Running E2E tests against a cluster built with the following API model:"
-      cat '${{ parameters.clusterDefinition }}'
+      
       cp ${{ parameters.clusterDefinition }} clusterDefinition.json
+      echo "-----"
+      pwd
+      ls
+      cat '${{ parameters.clusterDefinition }}'
+      echo "-----"
     displayName: Configure AKS-Engine
     workingDirectory: "$(modulePath)"
 
@@ -81,6 +86,11 @@ steps:
       addSpnToEnvironment: true
       workingDirectory: "$(modulePath)"
       inlineScript: |
+        echo "-----"
+        pwd
+        ls
+        cat clusterDefinition.json
+        echo "-----"
         export CLIENT_ID=$servicePrincipalId
         export CLIENT_SECRET=$servicePrincipalKey 
         export PATH=$PATH:'$(GOPATH)'

From 69ce7a8a487f435f07c18c8a22fa9b46bc2c8576 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@microsoft.com>
Date: Wed, 21 Apr 2021 18:24:17 -0700
Subject: [PATCH 10/15] cluster def

---
 .../singletenancy/aks-engine/e2e-step-template.yaml   | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml b/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
index 85cf54adfe..494f6df1e3 100644
--- a/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
@@ -62,11 +62,16 @@ steps:
       export CNI_URL=https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(StorageID)/azure-vnet-cni-${{ parameters.clusterDefinitionCniBuildOS }}-amd64-$(Tag)${{ parameters.clusterDefinitionCniBuildExt }}
       export CNI_TYPE=${{ parameters.clusterDefinitionCniTypeKey }}
       echo CNI type is $CNI_TYPE
-      echo Config:
+      echo Config: '${{ parameters.clusterDefinition }}'
+      echo "-----"
       cat '${{ parameters.clusterDefinition }}'
+      echo "2 -----"
       cat '${{ parameters.clusterDefinition }}' | jq --arg cnikey $CNI_TYPE --arg cniurl $CNI_URL '.properties.orchestratorProfile.kubernetesConfig[$cnikey]= $cniurl' > '${{ parameters.clusterDefinition }}'.tmp	
+      echo "3 -----"
       cat '${{ parameters.clusterDefinition }}'.tmp | jq --arg tag $(Tag) '.properties.orchestratorProfile.kubernetesConfig.azureCNIVersion = $tag' > '${{ parameters.clusterDefinition }}'
+      echo "4 -----"
       cat '${{ parameters.clusterDefinition }}' | jq --arg npmimage $IMAGE_REGISTRY/azure-npm:$(Tag)-test '.properties.orchestratorProfile.kubernetesConfig.addons[0].containers[0].image = $npmimage' > '${{ parameters.clusterDefinition }}'.tmp
+      echo "5 -----"
       mv '${{ parameters.clusterDefinition }}'.tmp '${{ parameters.clusterDefinition }}'
       echo "Running E2E tests against a cluster built with the following API model:"
       
@@ -74,7 +79,7 @@ steps:
       echo "-----"
       pwd
       ls
-      cat '${{ parameters.clusterDefinition }}'
+      cat clusterDefinition.json
       echo "-----"
     displayName: Configure AKS-Engine
     workingDirectory: "$(modulePath)"
@@ -94,7 +99,7 @@ steps:
         export CLIENT_ID=$servicePrincipalId
         export CLIENT_SECRET=$servicePrincipalKey 
         export PATH=$PATH:'$(GOPATH)'
-        export CLUSTER_DEFINITION=./'${{ parameters.clusterDefinition }}'
+        export CLUSTER_DEFINITION=./clusterDefinition.json
         export ORCHESTRATOR=kubernetes 
         export CREATE_VNET=false 
         export TIMEOUT=20m 

From 9fd26a028044326b828a0d46db0b75bb1bc00d2d Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@microsoft.com>
Date: Wed, 21 Apr 2021 18:49:44 -0700
Subject: [PATCH 11/15] envs

---
 .../aks-engine/e2e-job-template.yaml          |  6 ++---
 .../aks-engine/e2e-step-template.yaml         | 22 ++++---------------
 2 files changed, 7 insertions(+), 21 deletions(-)

diff --git a/.pipelines/singletenancy/aks-engine/e2e-job-template.yaml b/.pipelines/singletenancy/aks-engine/e2e-job-template.yaml
index 49dd01440b..ab0c62dbf7 100644
--- a/.pipelines/singletenancy/aks-engine/e2e-job-template.yaml
+++ b/.pipelines/singletenancy/aks-engine/e2e-job-template.yaml
@@ -27,9 +27,9 @@ stages:
           GOBIN: "$(GOPATH)/bin" # Go binaries path
           modulePath: "$(GOPATH)/src/github.com/Azure/aks-engine"
           acnPath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
-          Tag: $[ stagedependencies.build_and_test.unit_tests.outputs['EnvironmentalVariables.Tag'] ]
-          CommitHash: $[ stagedependencies.build_and_test.unit_tests.outputs['EnvironmentalVariables.CommitHash'] ]
-          StorageID: $[ stagedependencies.build_and_test.unit_tests.outputs['EnvironmentalVariables.StorageID'] ]
+          Tag: $[ stagedependencies.build_and_test.setup.outputs['EnvironmentalVariables.Tag'] ]
+          CommitHash: $[ stagedependencies.build_and_test.setup.outputs['EnvironmentalVariables.CommitHash'] ]
+          StorageID: $[ stagedependencies.build_and_test.setup.outputs['EnvironmentalVariables.StorageID'] ]
         steps:
           - template: e2e-step-template.yaml
             parameters:
diff --git a/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml b/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
index 494f6df1e3..a8e33eed11 100644
--- a/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-engine/e2e-step-template.yaml
@@ -62,40 +62,26 @@ steps:
       export CNI_URL=https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(StorageID)/azure-vnet-cni-${{ parameters.clusterDefinitionCniBuildOS }}-amd64-$(Tag)${{ parameters.clusterDefinitionCniBuildExt }}
       export CNI_TYPE=${{ parameters.clusterDefinitionCniTypeKey }}
       echo CNI type is $CNI_TYPE
+      echo Tag is $(Tag)
+      echo CNI_URL is $CNI_URL
       echo Config: '${{ parameters.clusterDefinition }}'
-      echo "-----"
       cat '${{ parameters.clusterDefinition }}'
-      echo "2 -----"
       cat '${{ parameters.clusterDefinition }}' | jq --arg cnikey $CNI_TYPE --arg cniurl $CNI_URL '.properties.orchestratorProfile.kubernetesConfig[$cnikey]= $cniurl' > '${{ parameters.clusterDefinition }}'.tmp	
-      echo "3 -----"
       cat '${{ parameters.clusterDefinition }}'.tmp | jq --arg tag $(Tag) '.properties.orchestratorProfile.kubernetesConfig.azureCNIVersion = $tag' > '${{ parameters.clusterDefinition }}'
-      echo "4 -----"
       cat '${{ parameters.clusterDefinition }}' | jq --arg npmimage $IMAGE_REGISTRY/azure-npm:$(Tag)-test '.properties.orchestratorProfile.kubernetesConfig.addons[0].containers[0].image = $npmimage' > '${{ parameters.clusterDefinition }}'.tmp
-      echo "5 -----"
       mv '${{ parameters.clusterDefinition }}'.tmp '${{ parameters.clusterDefinition }}'
-      echo "Running E2E tests against a cluster built with the following API model:"
-      
+      echo "Running E2E tests against a cluster built with the following API model:" 
       cp ${{ parameters.clusterDefinition }} clusterDefinition.json
-      echo "-----"
-      pwd
-      ls
-      cat clusterDefinition.json
-      echo "-----"
     displayName: Configure AKS-Engine
     workingDirectory: "$(modulePath)"
 
   - task: AzureCLI@1
     inputs:
-      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+      azureSubscription: $(AKS_ENGINE_SERVICE_CONNECTION)
       scriptLocation: "inlineScript"
       addSpnToEnvironment: true
       workingDirectory: "$(modulePath)"
       inlineScript: |
-        echo "-----"
-        pwd
-        ls
-        cat clusterDefinition.json
-        echo "-----"
         export CLIENT_ID=$servicePrincipalId
         export CLIENT_SECRET=$servicePrincipalKey 
         export PATH=$PATH:'$(GOPATH)'

From d20bee746092509aaa6a53c0036dc7b20aee0514 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@microsoft.com>
Date: Wed, 21 Apr 2021 22:24:56 -0700
Subject: [PATCH 12/15] images

---
 .pipelines/npm/npm-conformance-tests.yaml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/.pipelines/npm/npm-conformance-tests.yaml b/.pipelines/npm/npm-conformance-tests.yaml
index e570b7a4c6..5c566a1636 100644
--- a/.pipelines/npm/npm-conformance-tests.yaml
+++ b/.pipelines/npm/npm-conformance-tests.yaml
@@ -17,7 +17,8 @@ jobs:
     demands: 
     - agent.os -equals Linux
     - Role -equals Build
-
+  container:
+    image: "$(BUILD_IMAGE)"
   steps:
 
   - checkout: self
@@ -47,6 +48,8 @@ jobs:
     demands: 
     - agent.os -equals Linux
     - Role -equals Build
+  container:
+    image: "$(BUILD_IMAGE)"
   dependsOn: [setup]
   variables:
     GOBIN: "$(GOPATH)/bin" # Go binaries path
@@ -101,6 +104,8 @@ jobs:
     demands: 
     - agent.os -equals Linux
     - Role -equals Build
+  container:
+    image: "$(BUILD_IMAGE)"
   dependsOn: [setup]
   variables:
     RESOURCE_GROUP: $[ dependencies.setup.outputs['EnvironmentalVariables.RESOURCE_GROUP'] ]
@@ -135,6 +140,8 @@ jobs:
     demands: 
     - agent.os -equals Linux
     - Role -equals Build
+  container:
+    image: "$(BUILD_IMAGE)"
   dependsOn: [Create_cluster, Build_test, setup]
   variables:
     RESOURCE_GROUP: $[ dependencies.setup.outputs['EnvironmentalVariables.RESOURCE_GROUP'] ]
@@ -194,6 +201,8 @@ jobs:
     demands: 
     - agent.os -equals Linux
     - Role -equals Build
+  container:
+    image: "$(BUILD_IMAGE)"
   condition: always()
   dependsOn: [Run_test, setup]
   variables:

From 442d83854f1fc9cb008cb733f313bd6760560e2e Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@microsoft.com>
Date: Thu, 22 Apr 2021 09:40:16 -0700
Subject: [PATCH 13/15] swift outputs

---
 .pipelines/singletenancy/aks-swift/e2e-job-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
index b787fa6791..631ca1f5b2 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
@@ -21,7 +21,7 @@ stages:
         variables:
           GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
           GOBIN: "$(GOPATH)/bin" # Go binaries path
-          ImageTag: $[ stagedependencies.build_and_test.unit_tests.outputs['EnvironmentalVariables.ImageTag'] ]
+          ImageTag: $[ stagedependencies.build_and_test.setup.outputs['EnvironmentalVariables.ImageTag'] ]
           modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
         steps:
           - template: e2e-step-template.yaml

From cdb9255ea918eea4e10496d6fecda01e8d35f849 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@microsoft.com>
Date: Thu, 22 Apr 2021 10:19:12 -0700
Subject: [PATCH 14/15] remove circleci and codecov configs

---
 .circleci/config.yml | 50 --------------------------------------------
 codecov.yml          | 35 -------------------------------
 2 files changed, 85 deletions(-)
 delete mode 100644 .circleci/config.yml
 delete mode 100644 codecov.yml

diff --git a/.circleci/config.yml b/.circleci/config.yml
deleted file mode 100644
index de6b141455..0000000000
--- a/.circleci/config.yml
+++ /dev/null
@@ -1,50 +0,0 @@
-version: 2
-jobs:
-  setup-and-test:
-    # docker:
-    # - image: golang:1.12
-    machine:
-      image: ubuntu-1604:201903-01
-    steps:
-      - checkout
-      - run:
-          name: Setup-and-test
-          command: |
-            sudo -E env "PATH=$PATH" apt-get update
-            sudo -E env "PATH=$PATH" apt-get install -y ebtables
-            sudo -E env "PATH=$PATH" apt-get install -y ipset
-            sudo -E env "PATH=$PATH" apt-get install -y python3-dev gcc
-            sudo -E env "PATH=$PATH" pip install coverage
-            mkdir -p /home/circleci/go1-12
-            mkdir --parents /home/circleci/.goproject/src/github.com/Azure/azure-container-networking
-            wget https://storage.googleapis.com/golang/go1.12.6.linux-amd64.tar.gz
-            tar -C /home/circleci/go1-12 -xvf go1.12.6.linux-amd64.tar.gz
-            rm go1.12.6.linux-amd64.tar.gz
-            mv * /home/circleci/.goproject/src/github.com/Azure/azure-container-networking
-            cd /home/circleci/.goproject/src/github.com/Azure/azure-container-networking
-            export GOROOT='/home/circleci/go1-12/go'
-            export GOPATH='/home/circleci/.goproject'
-            export PATH=$GOROOT/bin:$PATH
-            go get ./...
-            go get github.com/docker/libnetwork/driverapi
-            go get github.com/gorilla/mux
-            go get golang.org/x/tools/cmd/cover
-            sudo -E env "PATH=$PATH" go test ./ipam/ -coverprofile coverage-ipam.out
-            sudo -E env "PATH=$PATH" go test ./log/ -coverprofile coverage-log.out
-            sudo -E env "PATH=$PATH" go test ./netlink/ -coverprofile coverage-netlink.out
-            sudo -E env "PATH=$PATH" go test ./store/ -coverprofile coverage-store.out
-            sudo -E env "PATH=$PATH" go test ./telemetry/ -coverprofile coverage-telemetry.out
-            sudo -E env "PATH=$PATH" go test ./network/ovssnat/ -coverprofile coverage-ovssnat.out
-            sudo -E env "PATH=$PATH" go test ./cni/ipam/ -coverprofile coverage-ipam.out
-            sudo -E env "PATH=$PATH" go test ./cnm/network/ -coverprofile coverage-network.out
-            sudo -E env "PATH=$PATH" go test ./cns/ipamclient/ -coverprofile coverage-ipamclient.out
-            sudo -E env "PATH=$PATH" go test ./npm/iptm/ -coverprofile coverage-iptm.out
-            sudo -E env "PATH=$PATH" go test ./npm/ipsm/ -coverprofile coverage-ipsm.out
-            sudo -E env "PATH=$PATH" go test ./npm/ -coverprofile coverage-npm.out
-            #sudo -E env "PATH=$PATH" go test ./cns/restserver/ -coverprofile coverage-restserver.out
-            bash <(curl -s https://codecov.io/bash)
-workflows:
-  version: 2
-  run-tests:
-    jobs:
-      - setup-and-test
diff --git a/codecov.yml b/codecov.yml
deleted file mode 100644
index 297e220eeb..0000000000
--- a/codecov.yml
+++ /dev/null
@@ -1,35 +0,0 @@
-codecov:
-  notify:
-    require_ci_to_pass: no
-
-coverage:
-  precision: 2
-  round: down
-  range: "50...100"
-  status:
-    project:
-      default:
-        threshold: 0.5
-    patch:
-      default:
-        threshold: 0.5
-    changes: no
-
-parsers:
-  gcov:
-    branch_detection:
-      conditional: yes
-      loop: yes
-      method: no
-      macro: no
-
-comment:
-  layout: "header, diff"
-  behavior: default
-  require_changes: no
-
-ignore:
-  - "vendor/"
-
-github_checks:
-  annotations: false
\ No newline at end of file

From 1557c058c04c3dcf7e5d66000b1c26178a4a9b1d Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@microsoft.com>
Date: Thu, 22 Apr 2021 11:30:43 -0700
Subject: [PATCH 15/15] imagetag

---
 .pipelines/pipeline.yaml       | 1 +
 test/integration/setup_test.go | 6 ++++++
 2 files changed, 7 insertions(+)

diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
index e47f5b1c26..ee479f3029 100644
--- a/.pipelines/pipeline.yaml
+++ b/.pipelines/pipeline.yaml
@@ -21,6 +21,7 @@ stages:
               BUILD_NUMBER=$(Build.BuildNumber)
               echo "##vso[task.setvariable variable=StorageID;isOutput=true]$(echo ${BUILD_NUMBER//./-})"
               echo "##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --always --dirty)"
+              echo "##vso[task.setvariable variable=ImageTag;isOutput=true]$(git describe --tags --always --dirty)-test"
               sudo chown -R $(whoami):$(whoami) .
             name: "EnvironmentalVariables"
             displayName: "Set environmental variables"
diff --git a/test/integration/setup_test.go b/test/integration/setup_test.go
index a48b17a25d..6a4125ddd5 100644
--- a/test/integration/setup_test.go
+++ b/test/integration/setup_test.go
@@ -81,6 +81,7 @@ func TestMain(m *testing.M) {
 		if installCNI, err := strconv.ParseBool(installopt); err == nil && installCNI == true {
 			if cnicleanup, err = installCNIManagerDaemonset(ctx, clientset, os.Getenv(envImageTag), logDir); err != nil {
 				log.Print(err)
+				exitCode = 2
 				return
 			}
 		}
@@ -92,6 +93,7 @@ func TestMain(m *testing.M) {
 		// create dirty cns ds
 		if installCNS, err := strconv.ParseBool(installopt); err == nil && installCNS == true {
 			if cnscleanup, err = installCNSDaemonset(ctx, clientset, os.Getenv(envImageTag), logDir); err != nil {
+				exitCode = 2
 				return
 			}
 		}
@@ -164,6 +166,10 @@ func installCNIManagerDaemonset(ctx context.Context, clientset *kubernetes.Clien
 		cni v1.DaemonSet
 	)
 
+	if imageTag == "" {
+		return nil, fmt.Errorf("No image tag specified, set using the VERSION environmental variable")
+	}
+
 	if cni, err = mustParseDaemonSet(cniDaemonSetPath); err != nil {
 		return nil, err
 	}