From 8a515b2eb29f3ed0f009b78ec7b623151f35a567 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Thu, 12 Sep 2019 17:23:02 -0700
Subject: [PATCH 01/83] Initial Azure Pipelines config (#314)

* initial azure-pipeline config

* remove cns restserver from ci

* publish coverage report to pipelines

* use default go version on runner
---
 .codecov/codecov.yml    |   3 ++
 .gitignore              |   3 ++
 .pipelines/pipeline.yml | 102 ++++++++++++++++++++++++++++++++++++++++
 Makefile                |  18 ++++++-
 4 files changed, 125 insertions(+), 1 deletion(-)
 create mode 100644 .codecov/codecov.yml
 create mode 100644 .pipelines/pipeline.yml

diff --git a/.codecov/codecov.yml b/.codecov/codecov.yml
new file mode 100644
index 0000000000..e2a23c23d8
--- /dev/null
+++ b/.codecov/codecov.yml
@@ -0,0 +1,3 @@
+codecov:
+  notify:
+    require_ci_to_pass: no
diff --git a/.gitignore b/.gitignore
index 331501803f..c9274d2f13 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,6 @@ ipam-*.xml
 
 # Environment
 .vscode/*
+
+# Coverage 
+*.out
\ No newline at end of file
diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
new file mode 100644
index 0000000000..2820f83d00
--- /dev/null
+++ b/.pipelines/pipeline.yml
@@ -0,0 +1,102 @@
+pr:
+- master
+
+trigger:
+  branches:
+    include:
+    - master
+
+jobs:
+- job: Test
+  pool:
+    vmImage: 'Ubuntu-16.04' 
+
+  # Go setup for the vmImage:
+  # https://github.com/Microsoft/azure-pipelines-image-generation/blob/master/images/linux/scripts/installers/go.sh
+  variables:
+    GOBIN:  '$(GOPATH)/bin' # Go binaries path
+    GOPATH: '$(System.DefaultWorkingDirectory)/gopath' # Go workspace path
+    modulePath: '$(GOPATH)/src/github.com/Azure/azure-container-networking' # $(build.repository.name)' # Path to the module's code
+
+  steps:
+  - script: |
+      echo $UID
+      sudo apt-get update
+      sudo apt-get install -y ebtables ipset python3-dev gcc zip
+      sudo pip install coverage
+    displayName: 'Install OS dependencies'
+
+  - script: |
+      go version
+      go env
+      mkdir -p '$(GOBIN)'
+      mkdir -p '$(GOPATH)/pkg'
+      mkdir -p '$(modulePath)'
+      shopt -s extglob
+      shopt -s dotglob
+      mv !(gopath) '$(modulePath)'
+      echo '##vso[task.prependpath]$(GOBIN)'
+      echo '##vso[task.prependpath]$(GOROOT)/bin'
+    displayName: 'Set up the Go workspace'
+
+  - script: |
+      go get -v -t -d ./...
+      if [ -f Gopkg.toml ]; then
+          curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+          dep ensure
+      fi
+      go get github.com/docker/libnetwork/driverapi
+      go get github.com/gorilla/mux
+      go get github.com/jstemmer/go-junit-report
+      go get github.com/axw/gocov/gocov
+      go get github.com/AlekSi/gocov-xml
+      go get -u gopkg.in/matm/v1/gocov-html
+    workingDirectory: '$(modulePath)'
+    displayName: 'Install Go dependencies'
+
+  - script: |
+      set -o pipefail
+      set -e
+      sudo -E env "PATH=$PATH" make test-all 2>&1 | tee >(go-junit-report > report.xml)
+      bash <(curl -s https://codecov.io/bash)
+      gocov convert coverage.out > coverage.json
+      gocov-xml < coverage.json > coverage.xml
+    workingDirectory: '$(modulePath)'
+    displayName: 'RunTest'
+
+  - task: PublishTestResults@2
+    inputs:
+      testRunner: JUnit
+      testResultsFiles: $(System.DefaultWorkingDirectory)/**/report.xml
+    condition: always()
+
+  - task: PublishCodeCoverageResults@1
+    inputs:
+      codeCoverageTool: Cobertura 
+      summaryFileLocation: $(System.DefaultWorkingDirectory)/**/coverage.xml
+    condition: always()
+
+  - script: |
+      export GOOS=linux
+      sudo -E env "PATH=$PATH" make all-binaries
+      export GOOS=windows
+      sudo -E env "PATH=$PATH" make all-binaries
+      cd output 
+      sudo find . -mindepth 2 -type f -print -exec mv {} . \;
+      sudo rm -R -- */
+      sudo find . -type f -regextype posix-extended ! -iregex '.*\.(zip|tgz)$' -delete
+    workingDirectory: '$(modulePath)'
+    displayName: 'Build'
+    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
+
+  - task: CopyFiles@2
+    inputs:
+      sourceFolder: '$(modulePath)/output'
+      targetFolder: $(Build.ArtifactStagingDirectory)
+    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
+
+  - task: PublishBuildArtifacts@1
+    inputs:
+      artifactName: 'output'
+      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
+    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
diff --git a/Makefile b/Makefile
index af3b7b4a84..d04844f00d 100644
--- a/Makefile
+++ b/Makefile
@@ -301,4 +301,20 @@ ifeq ($(GOOS),linux)
 	cp telemetry/azure-vnet-telemetry.config $(NPM_BUILD_DIR)/azure-vnet-telemetry.config
 	cd $(NPM_BUILD_DIR) && $(ARCHIVE_CMD) $(NPM_ARCHIVE_NAME) azure-npm$(EXE_EXT) azure-vnet-telemetry$(EXE_EXT) azure-vnet-telemetry.config
 	chown $(BUILD_USER):$(BUILD_USER) $(NPM_BUILD_DIR)/$(NPM_ARCHIVE_NAME)
-endif
\ No newline at end of file
+endif
+
+# run all tests
+.PHONY: test-all
+test-all:
+	go test -v -covermode count -coverprofile=coverage.out \
+        ./ipam/ \
+        ./log/ \
+        ./netlink/ \
+        ./store/ \
+        ./telemetry/ \
+        ./cnm/network/ \
+        ./cni/ipam/ \
+        ./cns/ipamclient/ \
+        ./npm/iptm/ \
+        ./npm/ipsm/ \
+        ./npm
\ No newline at end of file

From fb13488cb187cadbd34d261491abfc7384b9fa3a Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Fri, 13 Sep 2019 11:18:57 -0700
Subject: [PATCH 02/83] emit correct exit status code during test

---
 .pipelines/pipeline.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 2820f83d00..d2d7f5cd15 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -57,7 +57,8 @@ jobs:
   - script: |
       set -o pipefail
       set -e
-      sudo -E env "PATH=$PATH" make test-all 2>&1 | tee >(go-junit-report > report.xml)
+      # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to a subshell stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top shell stdout
+      ((((sudo -E env "PATH=$PATH" make test-all; echo $? >&3) | tee >(go-junit-report > report.xml) >&4) 3>&1) | (read xs; exit $xs)) 4>&1
       bash <(curl -s https://codecov.io/bash)
       gocov convert coverage.out > coverage.json
       gocov-xml < coverage.json > coverage.xml

From 01905a67c4ca648a3eb633ab80fa9a5654978157 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Fri, 13 Sep 2019 11:48:27 -0700
Subject: [PATCH 03/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 55 ++++++++++++++++++++++++-----------------
 1 file changed, 32 insertions(+), 23 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index d2d7f5cd15..c8e0e91ec5 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -19,14 +19,14 @@ jobs:
     modulePath: '$(GOPATH)/src/github.com/Azure/azure-container-networking' # $(build.repository.name)' # Path to the module's code
 
   steps:
-  - script: |
+  - bash: |
       echo $UID
       sudo apt-get update
       sudo apt-get install -y ebtables ipset python3-dev gcc zip
       sudo pip install coverage
     displayName: 'Install OS dependencies'
 
-  - script: |
+  - bash: |
       go version
       go env
       mkdir -p '$(GOBIN)'
@@ -39,7 +39,7 @@ jobs:
       echo '##vso[task.prependpath]$(GOROOT)/bin'
     displayName: 'Set up the Go workspace'
 
-  - script: |
+  - bash: |
       go get -v -t -d ./...
       if [ -f Gopkg.toml ]; then
           curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
@@ -54,17 +54,39 @@ jobs:
     workingDirectory: '$(modulePath)'
     displayName: 'Install Go dependencies'
 
-  - script: |
-      set -o pipefail
-      set -e
-      # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to a subshell stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top shell stdout
-      ((((sudo -E env "PATH=$PATH" make test-all; echo $? >&3) | tee >(go-junit-report > report.xml) >&4) 3>&1) | (read xs; exit $xs)) 4>&1
+  - bash: |
+      export GOOS=linux
+      sudo -E env "PATH=$PATH" make all-binaries
+      export GOOS=windows
+      sudo -E env "PATH=$PATH" make all-binaries
+      cd output 
+      sudo find . -mindepth 2 -type f -print -exec mv {} . \;
+      sudo rm -R -- */
+      sudo find . -type f -regextype posix-extended ! -iregex '.*\.(zip|tgz)$' -delete
+    workingDirectory: '$(modulePath)'
+    displayName: 'Build'
+
+  - bash: |
+      # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout
+      { { { { 
+            sudo -E env "PATH=$PATH" make test-all; 
+            echo $? >&3; 
+            } | tee > go-junit-report > report.xml >&4; 
+          } 3>&1; 
+        } | { read xs; exit $xs; } 
+      } 4>&1
+    workingDirectory: '$(modulePath)'
+    failOnStderr: true
+    displayName: 'Run Tests'
+
+  - bash: | 
       bash <(curl -s https://codecov.io/bash)
       gocov convert coverage.out > coverage.json
       gocov-xml < coverage.json > coverage.xml
     workingDirectory: '$(modulePath)'
-    displayName: 'RunTest'
-
+    displayName: 'Generate Coverage Reports'
+    condition: always()
+    
   - task: PublishTestResults@2
     inputs:
       testRunner: JUnit
@@ -77,19 +99,6 @@ jobs:
       summaryFileLocation: $(System.DefaultWorkingDirectory)/**/coverage.xml
     condition: always()
 
-  - script: |
-      export GOOS=linux
-      sudo -E env "PATH=$PATH" make all-binaries
-      export GOOS=windows
-      sudo -E env "PATH=$PATH" make all-binaries
-      cd output 
-      sudo find . -mindepth 2 -type f -print -exec mv {} . \;
-      sudo rm -R -- */
-      sudo find . -type f -regextype posix-extended ! -iregex '.*\.(zip|tgz)$' -delete
-    workingDirectory: '$(modulePath)'
-    displayName: 'Build'
-    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
-
   - task: CopyFiles@2
     inputs:
       sourceFolder: '$(modulePath)/output'

From 56cf5db03cfe82437daf1d49da342084160b4b3a Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Tue, 17 Sep 2019 13:53:52 -0700
Subject: [PATCH 04/83] better logging lines in npm

---
 .pipelines/pipeline.yml |  2 +-
 npm/ipsm/ipsm.go        |  4 ++--
 npm/ipsm/ipsm_test.go   | 17 ++++++++++++++++-
 npm/iptm/iptm.go        |  5 +++--
 npm/iptm/iptm_test.go   |  5 ++++-
 npm/namespace_test.go   |  5 ++++-
 npm/nwpolicy_test.go    | 27 +++++++++++++++++++++------
 7 files changed, 51 insertions(+), 14 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index c8e0e91ec5..bc348c5aec 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -71,7 +71,7 @@ jobs:
       { { { { 
             sudo -E env "PATH=$PATH" make test-all; 
             echo $? >&3; 
-            } | tee > go-junit-report > report.xml >&4; 
+            } | tee >(go-junit-report > report.xml) >&4; 
           } 3>&1; 
         } | { read xs; exit $xs; } 
       } 4>&1
diff --git a/npm/ipsm/ipsm.go b/npm/ipsm/ipsm.go
index 79c08fc853..0aab88fc46 100644
--- a/npm/ipsm/ipsm.go
+++ b/npm/ipsm/ipsm.go
@@ -350,8 +350,8 @@ func (ipsMgr *IpsetManager) Run(entry *ipsEntry) (int, error) {
 	_, err := exec.Command(cmdName, cmdArgs...).Output()
 	if msg, failed := err.(*exec.ExitError); failed {
 		errCode := msg.Sys().(syscall.WaitStatus).ExitStatus()
-		if errCode > 1 {
-			log.Errorf("Error: There was an error running command: %s %s Arguments:%v", err, cmdName, cmdArgs)
+		if errCode > 0 {
+			log.Errorf("Error: There was an error running command: [%s %v] Stderr: [%v, %s]", cmdName, strings.Join(cmdArgs, " "), err, strings.TrimSuffix(string(msg.Stderr), "\n"))
 		}
 
 		return errCode, err
diff --git a/npm/ipsm/ipsm_test.go b/npm/ipsm/ipsm_test.go
index 818941d4dc..5735ae1a1f 100644
--- a/npm/ipsm/ipsm_test.go
+++ b/npm/ipsm/ipsm_test.go
@@ -4,6 +4,7 @@ package ipsm
 
 import (
 	"testing"
+	"os"
 
 	"github.com/Azure/azure-container-networking/npm/util"
 )
@@ -76,6 +77,10 @@ func TestAddToList(t *testing.T) {
 		}
 	}()
 
+	if err := ipsMgr.CreateSet("test-set"); err != nil {
+		t.Errorf("TestAddToList failed @ ipsMgr.CreateSet")
+	}
+
 	if err := ipsMgr.AddToList("test-list", "test-set"); err != nil {
 		t.Errorf("TestAddToList failed @ ipsMgr.AddToList")
 	}
@@ -93,6 +98,10 @@ func TestDeleteFromList(t *testing.T) {
 		}
 	}()
 
+	if err := ipsMgr.CreateSet("test-set"); err != nil {
+		t.Errorf("TestDeleteFromList failed @ ipsMgr.CreateSet")
+	}
+
 	if err := ipsMgr.AddToList("test-list", "test-set"); err != nil {
 		t.Errorf("TestDeleteFromList failed @ ipsMgr.AddToList")
 	}
@@ -100,6 +109,10 @@ func TestDeleteFromList(t *testing.T) {
 	if err := ipsMgr.DeleteFromList("test-list", "test-set"); err != nil {
 		t.Errorf("TestDeleteFromList failed @ ipsMgr.DeleteFromList")
 	}
+
+	if err := ipsMgr.DeleteSet("test-set"); err != nil {
+		t.Errorf("TestDeleteSet failed @ ipsMgr.DeleteSet")
+	}
 }
 
 func TestCreateSet(t *testing.T) {
@@ -246,7 +259,9 @@ func TestMain(m *testing.M) {
 	ipsMgr := NewIpsetManager()
 	ipsMgr.Save(util.IpsetConfigFile)
 
-	m.Run()
+	exitCode := m.Run()
 
 	ipsMgr.Restore(util.IpsetConfigFile)
+	
+	os.Exit(exitCode)
 }
diff --git a/npm/iptm/iptm.go b/npm/iptm/iptm.go
index 5d2e0d74a2..ea4705ee5c 100644
--- a/npm/iptm/iptm.go
+++ b/npm/iptm/iptm.go
@@ -9,6 +9,7 @@ package iptm
 import (
 	"os"
 	"os/exec"
+	"strings"
 	"syscall"
 	"time"
 
@@ -365,8 +366,8 @@ func (iptMgr *IptablesManager) Run(entry *IptEntry) (int, error) {
 
 	if msg, failed := err.(*exec.ExitError); failed {
 		errCode := msg.Sys().(syscall.WaitStatus).ExitStatus()
-		if errCode > 1 {
-			log.Errorf("Error: There was an error running command: %s %s Arguments:%v", err, cmdName, cmdArgs)
+		if errCode > 0 {
+			log.Errorf("Error: There was an error running command: [%s %v] Stderr: [%v, %s]", cmdName, strings.Join(cmdArgs, " "), err, strings.TrimSuffix(string(msg.Stderr), "\n"))
 		}
 
 		return errCode, err
diff --git a/npm/iptm/iptm_test.go b/npm/iptm/iptm_test.go
index ba483a9bf9..04a454d6a5 100644
--- a/npm/iptm/iptm_test.go
+++ b/npm/iptm/iptm_test.go
@@ -2,6 +2,7 @@ package iptm
 
 import (
 	"testing"
+	"os"
 
 	"github.com/Azure/azure-container-networking/npm/util"
 )
@@ -204,7 +205,9 @@ func TestMain(m *testing.M) {
 	iptMgr := NewIptablesManager()
 	iptMgr.Save(util.IptablesConfigFile)
 
-	m.Run()
+	exitCode := m.Run()
 
 	iptMgr.Restore(util.IptablesConfigFile)
+
+	os.Exit(exitCode)
 }
diff --git a/npm/namespace_test.go b/npm/namespace_test.go
index a446ad70d8..74814fc9e3 100644
--- a/npm/namespace_test.go
+++ b/npm/namespace_test.go
@@ -4,6 +4,7 @@ package npm
 
 import (
 	"testing"
+	"os"
 
 	"github.com/Azure/azure-container-networking/npm/iptm"
 	"github.com/Azure/azure-container-networking/telemetry"
@@ -190,8 +191,10 @@ func TestMain(m *testing.M) {
 	ipsMgr := ipsm.NewIpsetManager()
 	ipsMgr.Save(util.IpsetConfigFile)
 
-	m.Run()
+	exitCode := m.Run()
 
 	iptMgr.Restore(util.IptablesConfigFile)
 	ipsMgr.Restore(util.IpsetConfigFile)
+
+	os.Exit(exitCode)
 }
diff --git a/npm/nwpolicy_test.go b/npm/nwpolicy_test.go
index e6f48ce72d..352cd5b5b6 100644
--- a/npm/nwpolicy_test.go
+++ b/npm/nwpolicy_test.go
@@ -42,6 +42,11 @@ func TestAddNetworkPolicy(t *testing.T) {
 		t.Errorf("TestAddNetworkPolicy failed @ ipsMgr.Save")
 	}
 
+	// Create ns-kube-system set
+	if err := ipsMgr.CreateSet("ns-" + util.KubeSystemFlag); err != nil {
+		t.Errorf("TestAddNetworkPolicy failed @ ipsMgr.CreateSet, adding kube-system set%+v", err)
+	}
+
 	defer func() {
 		if err := iptMgr.Restore(util.IptablesTestConfigFile); err != nil {
 			t.Errorf("TestAddNetworkPolicy failed @ iptMgr.Restore")
@@ -140,24 +145,29 @@ func TestUpdateNetworkPolicy(t *testing.T) {
 
 	iptMgr := iptm.NewIptablesManager()
 	if err := iptMgr.Save(util.IptablesTestConfigFile); err != nil {
-		t.Errorf("UpdateAddNetworkPolicy failed @ iptMgr.Save")
+		t.Errorf("TestUpdateNetworkPolicy failed @ iptMgr.Save")
 	}
 
 	ipsMgr := ipsm.NewIpsetManager()
 	if err := ipsMgr.Save(util.IpsetTestConfigFile); err != nil {
-		t.Errorf("UpdateAddNetworkPolicy failed @ ipsMgr.Save")
+		t.Errorf("TestUpdateNetworkPolicy failed @ ipsMgr.Save")
 	}
 
 	defer func() {
 		if err := iptMgr.Restore(util.IptablesTestConfigFile); err != nil {
-			t.Errorf("UpdateAddNetworkPolicy failed @ iptMgr.Restore")
+			t.Errorf("TestUpdateNetworkPolicy failed @ iptMgr.Restore")
 		}
 
 		if err := ipsMgr.Restore(util.IpsetTestConfigFile); err != nil {
-			t.Errorf("UpdateAddNetworkPolicy failed @ ipsMgr.Restore")
+			t.Errorf("TestUpdateNetworkPolicy failed @ ipsMgr.Restore")
 		}
 	}()
 
+	// Create ns-kube-system set
+	if err := ipsMgr.CreateSet("ns-" + util.KubeSystemFlag); err != nil {
+		t.Errorf("TestUpdateNetworkPolicy failed @ ipsMgr.CreateSet, adding kube-system set%+v", err)
+	}
+
 	nsObj := &corev1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "test-nwpolicy",
@@ -168,7 +178,7 @@ func TestUpdateNetworkPolicy(t *testing.T) {
 	}
 
 	if err := npMgr.AddNamespace(nsObj); err != nil {
-		t.Errorf("TestAddNetworkPolicy @ npMgr.AddNamespace")
+		t.Errorf("TestUpdateNetworkPolicy @ npMgr.AddNamespace")
 	}
 
 	tcp, udp := corev1.ProtocolTCP, corev1.ProtocolUDP
@@ -265,6 +275,11 @@ func TestDeleteNetworkPolicy(t *testing.T) {
 		}
 	}()
 
+	// Create ns-kube-system set
+	if err := ipsMgr.CreateSet("ns-" + util.KubeSystemFlag); err != nil {
+		t.Errorf("TestDeleteNetworkPolicy failed @ ipsMgr.CreateSet, adding kube-system set%+v", err)
+	}
+
 	nsObj := &corev1.Namespace{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: "test-nwpolicy",
@@ -308,6 +323,6 @@ func TestDeleteNetworkPolicy(t *testing.T) {
 	}
 
 	if err := npMgr.DeleteNetworkPolicy(allow); err != nil {
-		t.Errorf("TestAddNetworkPolicy failed @ DeleteNetworkPolicy")
+		t.Errorf("TestDeleteNetworkPolicy failed @ DeleteNetworkPolicy")
 	}
 }

From d1b2a59e2bbda2d18c0ef73fbb70c23c4f71a243 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 18 Sep 2019 17:58:34 -0700
Subject: [PATCH 05/83] publish build artifacts on succeed

---
 .pipelines/pipeline.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index bc348c5aec..f70bf8c1ed 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -103,10 +103,10 @@ jobs:
     inputs:
       sourceFolder: '$(modulePath)/output'
       targetFolder: $(Build.ArtifactStagingDirectory)
-    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
+    condition: and(succeeded())
 
   - task: PublishBuildArtifacts@1
     inputs:
       artifactName: 'output'
       pathtoPublish: '$(Build.ArtifactStagingDirectory)'
-    condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
+    condition: and(succeeded())

From 3bb17bb706ee7ea5c0c8f93f530642de5f0b4ca7 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 18 Sep 2019 18:05:00 -0700
Subject: [PATCH 06/83] remove codecov yaml

---
 .codecov/codecov.yml    | 3 ---
 .pipelines/pipeline.yml | 4 ++--
 2 files changed, 2 insertions(+), 5 deletions(-)
 delete mode 100644 .codecov/codecov.yml

diff --git a/.codecov/codecov.yml b/.codecov/codecov.yml
deleted file mode 100644
index e2a23c23d8..0000000000
--- a/.codecov/codecov.yml
+++ /dev/null
@@ -1,3 +0,0 @@
-codecov:
-  notify:
-    require_ci_to_pass: no
diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index f70bf8c1ed..9c4d77aafb 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -103,10 +103,10 @@ jobs:
     inputs:
       sourceFolder: '$(modulePath)/output'
       targetFolder: $(Build.ArtifactStagingDirectory)
-    condition: and(succeeded())
+    condition: succeeded()
 
   - task: PublishBuildArtifacts@1
     inputs:
       artifactName: 'output'
       pathtoPublish: '$(Build.ArtifactStagingDirectory)'
-    condition: and(succeeded())
+    condition: succeeded()

From e85aec36521b4efcae05aa9fc0f3a2cb585be33f Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 18 Sep 2019 18:24:01 -0700
Subject: [PATCH 07/83] update readme with azure pipelines badge

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 9634a1d896..6c51c86e91 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
 # Microsoft Azure Container Networking
 
-[![CircleCI](https://circleci.com/gh/Azure/azure-container-networking/tree/master.svg?style=svg)](https://circleci.com/gh/Azure/azure-container-networking/tree/master) [![Go Report Card](https://goreportcard.com/badge/github.com/Azure/azure-container-networking)](https://goreportcard.com/report/github.com/Azure/azure-container-networking) ![GitHub release](https://img.shields.io/github/release/Azure/azure-container-networking.svg)
+[![Build Status](https://msazure.visualstudio.com/One/_apis/build/status/Custom/Networking/ContainerNetworking/Azure.azure-container-networking?branchName=master)](https://msazure.visualstudio.com/One/_build/latest?definitionId=95007&branchName=master) [![Go Report Card](https://goreportcard.com/badge/github.com/Azure/azure-container-networking)](https://goreportcard.com/report/github.com/Azure/azure-container-networking) ![GitHub release](https://img.shields.io/github/release/Azure/azure-container-networking.svg)
 [![codecov](https://codecov.io/gh/Azure/azure-container-networking/branch/master/graph/badge.svg)](https://codecov.io/gh/Azure/azure-container-networking)
 
 ## Overview

From 74f72d8987649232d28f4a25234f9d8455d8df17 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <Mathew.Merrick@microsoft.com>
Date: Thu, 19 Sep 2019 12:26:53 -0700
Subject: [PATCH 08/83] update published artifacts

---
 .pipelines/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 9c4d77aafb..5e39856478 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -60,9 +60,9 @@ jobs:
       export GOOS=windows
       sudo -E env "PATH=$PATH" make all-binaries
       cd output 
+      sudo find . -mindepth 2 -type f -regextype posix-extended ! -iregex '.*\.(zip|tgz)$' -delete
       sudo find . -mindepth 2 -type f -print -exec mv {} . \;
       sudo rm -R -- */
-      sudo find . -type f -regextype posix-extended ! -iregex '.*\.(zip|tgz)$' -delete
     workingDirectory: '$(modulePath)'
     displayName: 'Build'
 

From be54e9ffa9e2aae95e1831402d624a81d9ef6f58 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Thu, 19 Sep 2019 14:26:56 -0700
Subject: [PATCH 09/83] example policy has ingress and egress

---
 npm/translatePolicy_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/npm/translatePolicy_test.go b/npm/translatePolicy_test.go
index 200b81546a..7039fe51bb 100644
--- a/npm/translatePolicy_test.go
+++ b/npm/translatePolicy_test.go
@@ -2807,7 +2807,7 @@ func TestTranslatePolicy(t *testing.T) {
 		},
 	}
 	expectedIptEntries = append(expectedIptEntries, nonKubeSystemEntries...)
-	expectedIptEntries = append(expectedIptEntries, getDefaultDropEntries("testnamespace", targetSelector, false, true)...)
+	expectedIptEntries = append(expectedIptEntries, getDefaultDropEntries("testnamespace", targetSelector, true, true)...)
 	if !reflect.DeepEqual(iptEntries, expectedIptEntries) {
 		t.Errorf("translatedPolicy failed @ k8s-example-policy policy comparison")
 		marshalledIptEntries, _ := json.Marshal(iptEntries)

From 290c620704d536110ad0900aa5b8ac1a9b1660ab Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Thu, 19 Sep 2019 16:32:44 -0700
Subject: [PATCH 10/83] specify runner pool

---
 .pipelines/pipeline.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 5e39856478..5af470023c 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -7,9 +7,10 @@ trigger:
     - master
 
 jobs:
-- job: Test
+- job: Build and run Unit Tests
   pool:
-    vmImage: 'Ubuntu-16.04' 
+    name: Networking-ContainerNetworking
+    demands: agent.os -equals Linux
 
   # Go setup for the vmImage:
   # https://github.com/Microsoft/azure-pipelines-image-generation/blob/master/images/linux/scripts/installers/go.sh

From 0229c609111d7587a2a7097c03e041f219d21739 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Thu, 19 Sep 2019 16:34:24 -0700
Subject: [PATCH 11/83] job name

---
 .pipelines/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 5af470023c..09471180b6 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -7,7 +7,7 @@ trigger:
     - master
 
 jobs:
-- job: Build and run Unit Tests
+- job: UnitTest
   pool:
     name: Networking-ContainerNetworking
     demands: agent.os -equals Linux

From 10ebb6163ab5b6f7d7711f2c4457650ef5a69b85 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Thu, 19 Sep 2019 17:29:48 -0700
Subject: [PATCH 12/83] cnm fixes

---
 network/network_linux.go | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/network/network_linux.go b/network/network_linux.go
index b2966790de..380f9bc6f4 100644
--- a/network/network_linux.go
+++ b/network/network_linux.go
@@ -269,14 +269,19 @@ func (nm *networkManager) applyIPConfig(extIf *externalInterface, targetIf *net.
 }
 
 func applyDnsConfig(extIf *externalInterface, ifName string) error {
-	cmd := fmt.Sprintf("systemd-resolve --interface=%s --set-dns=%s", ifName, extIf.DNSInfo.Servers[0])
-	_, err := platform.ExecuteCommand(cmd)
-	if err != nil {
-		return err
+	var err error
+
+	if(extIf != nil && len(extIf.DNSInfo.Servers) > 0) {
+		cmd := fmt.Sprintf("systemd-resolve --interface=%s --set-dns=%s", ifName, extIf.DNSInfo.Servers[0])
+		_, err = platform.ExecuteCommand(cmd)
+		if err != nil {
+			return err
+		}
+
+		cmd = fmt.Sprintf("systemd-resolve --interface=%s --set-domain=%s", ifName, extIf.DNSInfo.Suffix)
+		_, err = platform.ExecuteCommand(cmd)
 	}
 
-	cmd = fmt.Sprintf("systemd-resolve --interface=%s --set-domain=%s", ifName, extIf.DNSInfo.Suffix)
-	_, err = platform.ExecuteCommand(cmd)
 	return err
 }
 

From 6321f88d70e8c63420865b92f6821e871eb47b26 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Thu, 19 Sep 2019 17:40:06 -0700
Subject: [PATCH 13/83] remove test.sock

---
 .pipelines/pipeline.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 09471180b6..7f2ebc118f 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -36,6 +36,7 @@ jobs:
       shopt -s extglob
       shopt -s dotglob
       mv !(gopath) '$(modulePath)'
+      sudo rm /run/docker/plugins/test.sock
       echo '##vso[task.prependpath]$(GOBIN)'
       echo '##vso[task.prependpath]$(GOROOT)/bin'
     displayName: 'Set up the Go workspace'

From f6cc628d17c4a83f43a78287c1827e8670ff332f Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Thu, 19 Sep 2019 17:43:25 -0700
Subject: [PATCH 14/83] make as non-root

---
 .pipelines/pipeline.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 7f2ebc118f..a3465e5802 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -58,9 +58,9 @@ jobs:
 
   - bash: |
       export GOOS=linux
-      sudo -E env "PATH=$PATH" make all-binaries
+      make all-binaries
       export GOOS=windows
-      sudo -E env "PATH=$PATH" make all-binaries
+      make all-binaries
       cd output 
       sudo find . -mindepth 2 -type f -regextype posix-extended ! -iregex '.*\.(zip|tgz)$' -delete
       sudo find . -mindepth 2 -type f -print -exec mv {} . \;

From 652a8b1c7c67f7ab9842816261bba975a987e649 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Thu, 19 Sep 2019 17:59:14 -0700
Subject: [PATCH 15/83] test cleanup

---
 .pipelines/pipeline.yml | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index a3465e5802..acc1fb4f9e 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -36,7 +36,6 @@ jobs:
       shopt -s extglob
       shopt -s dotglob
       mv !(gopath) '$(modulePath)'
-      sudo rm /run/docker/plugins/test.sock
       echo '##vso[task.prependpath]$(GOBIN)'
       echo '##vso[task.prependpath]$(GOROOT)/bin'
     displayName: 'Set up the Go workspace'
@@ -88,6 +87,13 @@ jobs:
     workingDirectory: '$(modulePath)'
     displayName: 'Generate Coverage Reports'
     condition: always()
+
+  - bash: | 
+      sudo rm /run/docker/plugins/test.sock
+      sudo ip link del dev dummy
+    workingDirectory: '$(modulePath)'
+    displayName: 'Test cleanup'
+    condition: always()
     
   - task: PublishTestResults@2
     inputs:

From 075033d6d71fe5bc82a2ea56156b1cd0c2694b02 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Thu, 19 Sep 2019 18:20:46 -0700
Subject: [PATCH 16/83] remove test cleanup

---
 .pipelines/pipeline.yml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index acc1fb4f9e..e0b0bf647f 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -87,13 +87,6 @@ jobs:
     workingDirectory: '$(modulePath)'
     displayName: 'Generate Coverage Reports'
     condition: always()
-
-  - bash: | 
-      sudo rm /run/docker/plugins/test.sock
-      sudo ip link del dev dummy
-    workingDirectory: '$(modulePath)'
-    displayName: 'Test cleanup'
-    condition: always()
     
   - task: PublishTestResults@2
     inputs:

From 5545c2b6b3896a078d0f4838ef48cad28b9b13a7 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Fri, 20 Sep 2019 10:45:43 -0700
Subject: [PATCH 17/83] add test setup

---
 .pipelines/pipeline.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index e0b0bf647f..67182a949c 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -55,6 +55,13 @@ jobs:
     workingDirectory: '$(modulePath)'
     displayName: 'Install Go dependencies'
 
+  - bash: | 
+      sudo rm /run/docker/plugins/test.sock || true
+      sudo ip link del dev dummy || true
+    workingDirectory: '$(modulePath)'
+    displayName: 'Workspace setup'
+    condition: always()
+
   - bash: |
       export GOOS=linux
       make all-binaries

From 482710bc57ec216ad0529f519500d1df9b9ea6e8 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Fri, 20 Sep 2019 11:24:37 -0700
Subject: [PATCH 18/83] address comments

---
 .pipelines/pipeline.yml  | 203 +++++++++++++++++++--------------------
 network/network_linux.go |   2 +-
 2 files changed, 102 insertions(+), 103 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 67182a949c..496f294802 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -1,120 +1,119 @@
 pr:
-- master
+  - master
 
 trigger:
   branches:
     include:
-    - master
+      - master
 
 jobs:
-- job: UnitTest
-  pool:
-    name: Networking-ContainerNetworking
-    demands: agent.os -equals Linux
+  - job: UnitTest
+    pool:
+      name: Networking-ContainerNetworking
+      demands: agent.os -equals Linux
 
-  # Go setup for the vmImage:
-  # https://github.com/Microsoft/azure-pipelines-image-generation/blob/master/images/linux/scripts/installers/go.sh
-  variables:
-    GOBIN:  '$(GOPATH)/bin' # Go binaries path
-    GOPATH: '$(System.DefaultWorkingDirectory)/gopath' # Go workspace path
-    modulePath: '$(GOPATH)/src/github.com/Azure/azure-container-networking' # $(build.repository.name)' # Path to the module's code
+    # Go setup for the vmImage:
+    # https://github.com/Microsoft/azure-pipelines-image-generation/blob/master/images/linux/scripts/installers/go.sh
+    variables:
+      GOBIN: "$(GOPATH)/bin" # Go binaries path
+      GOPATH: "$(System.DefaultWorkingDirectory)/gopath" # Go workspace path
+      modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" # $(build.repository.name)' # Path to the module's code
 
-  steps:
-  - bash: |
-      echo $UID
-      sudo apt-get update
-      sudo apt-get install -y ebtables ipset python3-dev gcc zip
-      sudo pip install coverage
-    displayName: 'Install OS dependencies'
+    steps:
+      - bash: |
+          echo $UID
+          sudo apt-get update
+          sudo apt-get install -y ebtables ipset python3-dev gcc zip
+          sudo pip install coverage
+        displayName: "Install OS dependencies"
 
-  - bash: |
-      go version
-      go env
-      mkdir -p '$(GOBIN)'
-      mkdir -p '$(GOPATH)/pkg'
-      mkdir -p '$(modulePath)'
-      shopt -s extglob
-      shopt -s dotglob
-      mv !(gopath) '$(modulePath)'
-      echo '##vso[task.prependpath]$(GOBIN)'
-      echo '##vso[task.prependpath]$(GOROOT)/bin'
-    displayName: 'Set up the Go workspace'
+      - bash: |
+          go version
+          go env
+          mkdir -p '$(GOBIN)'
+          mkdir -p '$(GOPATH)/pkg'
+          mkdir -p '$(modulePath)'
+          shopt -s extglob
+          shopt -s dotglob
+          mv !(gopath) '$(modulePath)'
+          echo '##vso[task.prependpath]$(GOBIN)'
+          echo '##vso[task.prependpath]$(GOROOT)/bin'
+        displayName: "Set up the Go workspace"
 
-  - bash: |
-      go get -v -t -d ./...
-      if [ -f Gopkg.toml ]; then
-          curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-          dep ensure
-      fi
-      go get github.com/docker/libnetwork/driverapi
-      go get github.com/gorilla/mux
-      go get github.com/jstemmer/go-junit-report
-      go get github.com/axw/gocov/gocov
-      go get github.com/AlekSi/gocov-xml
-      go get -u gopkg.in/matm/v1/gocov-html
-    workingDirectory: '$(modulePath)'
-    displayName: 'Install Go dependencies'
+      - bash: |
+          go get -v -t -d ./...
+          if [ -f Gopkg.toml ]; then
+              curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+          fi
+          go get github.com/docker/libnetwork/driverapi
+          go get github.com/gorilla/mux
+          go get github.com/jstemmer/go-junit-report
+          go get github.com/axw/gocov/gocov
+          go get github.com/AlekSi/gocov-xml
+          go get -u gopkg.in/matm/v1/gocov-html
+        workingDirectory: "$(modulePath)"
+        displayName: "Install Go dependencies"
 
-  - bash: | 
-      sudo rm /run/docker/plugins/test.sock || true
-      sudo ip link del dev dummy || true
-    workingDirectory: '$(modulePath)'
-    displayName: 'Workspace setup'
-    condition: always()
+      - bash: |
+          sudo rm /run/docker/plugins/test.sock || true
+          sudo ip link del dev dummy || true
+        workingDirectory: "$(modulePath)"
+        displayName: "Workspace setup"
+        condition: always()
 
-  - bash: |
-      export GOOS=linux
-      make all-binaries
-      export GOOS=windows
-      make all-binaries
-      cd output 
-      sudo find . -mindepth 2 -type f -regextype posix-extended ! -iregex '.*\.(zip|tgz)$' -delete
-      sudo find . -mindepth 2 -type f -print -exec mv {} . \;
-      sudo rm -R -- */
-    workingDirectory: '$(modulePath)'
-    displayName: 'Build'
+      - bash: |
+          export GOOS=linux
+          make all-binaries
+          export GOOS=windows
+          make all-binaries
+          cd output 
+          sudo find . -mindepth 2 -type f -regextype posix-extended ! -iregex '.*\.(zip|tgz)$' -delete
+          sudo find . -mindepth 2 -type f -print -exec mv {} . \;
+          sudo rm -R -- */
+        workingDirectory: "$(modulePath)"
+        displayName: "Build"
 
-  - bash: |
-      # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout
-      { { { { 
-            sudo -E env "PATH=$PATH" make test-all; 
-            echo $? >&3; 
-            } | tee >(go-junit-report > report.xml) >&4; 
-          } 3>&1; 
-        } | { read xs; exit $xs; } 
-      } 4>&1
-    workingDirectory: '$(modulePath)'
-    failOnStderr: true
-    displayName: 'Run Tests'
+      - bash: |
+          # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout
+          { { { { 
+                sudo -E env "PATH=$PATH" make test-all; 
+                echo $? >&3; 
+                } | tee >(go-junit-report > report.xml) >&4; 
+              } 3>&1; 
+            } | { read xs; exit $xs; } 
+          } 4>&1
+        workingDirectory: "$(modulePath)"
+        failOnStderr: true
+        displayName: "Run Tests"
 
-  - bash: | 
-      bash <(curl -s https://codecov.io/bash)
-      gocov convert coverage.out > coverage.json
-      gocov-xml < coverage.json > coverage.xml
-    workingDirectory: '$(modulePath)'
-    displayName: 'Generate Coverage Reports'
-    condition: always()
-    
-  - task: PublishTestResults@2
-    inputs:
-      testRunner: JUnit
-      testResultsFiles: $(System.DefaultWorkingDirectory)/**/report.xml
-    condition: always()
+      - bash: |
+          bash <(curl -s https://codecov.io/bash)
+          gocov convert coverage.out > coverage.json
+          gocov-xml < coverage.json > coverage.xml
+        workingDirectory: "$(modulePath)"
+        displayName: "Generate Coverage Reports"
+        condition: always()
 
-  - task: PublishCodeCoverageResults@1
-    inputs:
-      codeCoverageTool: Cobertura 
-      summaryFileLocation: $(System.DefaultWorkingDirectory)/**/coverage.xml
-    condition: always()
+      - task: PublishTestResults@2
+        inputs:
+          testRunner: JUnit
+          testResultsFiles: $(System.DefaultWorkingDirectory)/**/report.xml
+        condition: always()
 
-  - task: CopyFiles@2
-    inputs:
-      sourceFolder: '$(modulePath)/output'
-      targetFolder: $(Build.ArtifactStagingDirectory)
-    condition: succeeded()
+      - task: PublishCodeCoverageResults@1
+        inputs:
+          codeCoverageTool: Cobertura
+          summaryFileLocation: $(System.DefaultWorkingDirectory)/**/coverage.xml
+        condition: always()
 
-  - task: PublishBuildArtifacts@1
-    inputs:
-      artifactName: 'output'
-      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
-    condition: succeeded()
+      - task: CopyFiles@2
+        inputs:
+          sourceFolder: "$(modulePath)/output"
+          targetFolder: $(Build.ArtifactStagingDirectory)
+        condition: succeeded()
+
+      - task: PublishBuildArtifacts@1
+        inputs:
+          artifactName: "output"
+          pathtoPublish: "$(Build.ArtifactStagingDirectory)"
+        condition: succeeded()
diff --git a/network/network_linux.go b/network/network_linux.go
index 380f9bc6f4..195fcccc86 100644
--- a/network/network_linux.go
+++ b/network/network_linux.go
@@ -271,7 +271,7 @@ func (nm *networkManager) applyIPConfig(extIf *externalInterface, targetIf *net.
 func applyDnsConfig(extIf *externalInterface, ifName string) error {
 	var err error
 
-	if(extIf != nil && len(extIf.DNSInfo.Servers) > 0) {
+	if extIf != nil && len(extIf.DNSInfo.Servers) > 0 {
 		cmd := fmt.Sprintf("systemd-resolve --interface=%s --set-dns=%s", ifName, extIf.DNSInfo.Servers[0])
 		_, err = platform.ExecuteCommand(cmd)
 		if err != nil {

From b9ab4e0413a8476d5eec5b9831e56e6241d4c951 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Fri, 20 Sep 2019 11:44:16 -0700
Subject: [PATCH 19/83] run in container on custom agent test

---
 .pipelines/pipeline.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 496f294802..d08851b5f2 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -12,6 +12,8 @@ jobs:
       name: Networking-ContainerNetworking
       demands: agent.os -equals Linux
 
+    container: golang:1.12
+
     # Go setup for the vmImage:
     # https://github.com/Microsoft/azure-pipelines-image-generation/blob/master/images/linux/scripts/installers/go.sh
     variables:

From eb818683e4a9964030dcc9c52b8991e46388161d Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Fri, 20 Sep 2019 14:52:13 -0700
Subject: [PATCH 20/83] custom container image env

---
 .pipelines/Dockerfile   | 12 ++++++++++++
 .pipelines/pipeline.yml | 25 +++----------------------
 2 files changed, 15 insertions(+), 22 deletions(-)
 create mode 100644 .pipelines/Dockerfile

diff --git a/.pipelines/Dockerfile b/.pipelines/Dockerfile
new file mode 100644
index 0000000000..600a775b5f
--- /dev/null
+++ b/.pipelines/Dockerfile
@@ -0,0 +1,12 @@
+FROM ubuntu:16.04
+RUN apt-get update && apt-get install -y software-properties-common sudo 
+RUN add-apt-repository ppa:longsleep/golang-backports && apt-get update
+RUN apt-get install -y git golang-go=2:1.13~1longsleep1+xenial iptables ipset iproute2 ebtables
+RUN go get -v -t -d ./... 
+RUN if [ -f Gopkg.toml ]; then curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh ; fi 
+RUN go get github.com/docker/libnetwork/driverapi
+RUN go get github.com/gorilla/mux
+RUN go get github.com/jstemmer/go-junit-report
+RUN go get github.com/axw/gocov/gocov
+RUN go get github.com/AlekSi/gocov-xml
+RUN go get -u gopkg.in/matm/v1/gocov-html
diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index d08851b5f2..0fdf0d3d07 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -12,7 +12,9 @@ jobs:
       name: Networking-ContainerNetworking
       demands: agent.os -equals Linux
 
-    container: golang:1.12
+    container:
+      image: containernetworking/pipeline-ci:1.0
+      options: "--privileged"
 
     # Go setup for the vmImage:
     # https://github.com/Microsoft/azure-pipelines-image-generation/blob/master/images/linux/scripts/installers/go.sh
@@ -22,13 +24,6 @@ jobs:
       modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" # $(build.repository.name)' # Path to the module's code
 
     steps:
-      - bash: |
-          echo $UID
-          sudo apt-get update
-          sudo apt-get install -y ebtables ipset python3-dev gcc zip
-          sudo pip install coverage
-        displayName: "Install OS dependencies"
-
       - bash: |
           go version
           go env
@@ -42,20 +37,6 @@ jobs:
           echo '##vso[task.prependpath]$(GOROOT)/bin'
         displayName: "Set up the Go workspace"
 
-      - bash: |
-          go get -v -t -d ./...
-          if [ -f Gopkg.toml ]; then
-              curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-          fi
-          go get github.com/docker/libnetwork/driverapi
-          go get github.com/gorilla/mux
-          go get github.com/jstemmer/go-junit-report
-          go get github.com/axw/gocov/gocov
-          go get github.com/AlekSi/gocov-xml
-          go get -u gopkg.in/matm/v1/gocov-html
-        workingDirectory: "$(modulePath)"
-        displayName: "Install Go dependencies"
-
       - bash: |
           sudo rm /run/docker/plugins/test.sock || true
           sudo ip link del dev dummy || true

From e0f49ae9a2289df901afe26363ca96b8eb3c4a66 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Fri, 20 Sep 2019 15:02:13 -0700
Subject: [PATCH 21/83] update ci image

---
 .pipelines/pipeline.yml | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 0fdf0d3d07..465f422297 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -13,7 +13,7 @@ jobs:
       demands: agent.os -equals Linux
 
     container:
-      image: containernetworking/pipeline-ci:1.0
+      image: containernetworking/pipeline-ci:1.0.1
       options: "--privileged"
 
     # Go setup for the vmImage:
@@ -24,6 +24,12 @@ jobs:
       modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" # $(build.repository.name)' # Path to the module's code
 
     steps:
+      - bash: |
+          echo $UID
+          sudo apt-get install -y ebtables ipset python3-dev gcc zip iptables ipset
+          sudo pip install coverage
+        displayName: "Install OS dependencies"
+
       - bash: |
           go version
           go env
@@ -37,6 +43,20 @@ jobs:
           echo '##vso[task.prependpath]$(GOROOT)/bin'
         displayName: "Set up the Go workspace"
 
+      - bash: |
+          go get -v -t -d ./...
+          if [ -f Gopkg.toml ]; then
+              curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+          fi
+          go get github.com/docker/libnetwork/driverapi
+          go get github.com/gorilla/mux
+          go get github.com/jstemmer/go-junit-report
+          go get github.com/axw/gocov/gocov
+          go get github.com/AlekSi/gocov-xml
+          go get -u gopkg.in/matm/v1/gocov-html
+        workingDirectory: "$(modulePath)"
+        displayName: "Install Go dependencies"
+
       - bash: |
           sudo rm /run/docker/plugins/test.sock || true
           sudo ip link del dev dummy || true

From 2ba855b30293d435ccbb9cea0d015debde243a7d Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Fri, 20 Sep 2019 15:28:01 -0700
Subject: [PATCH 22/83] update ci image 2

---
 .pipelines/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 465f422297..948b5eba14 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -13,7 +13,7 @@ jobs:
       demands: agent.os -equals Linux
 
     container:
-      image: containernetworking/pipeline-ci:1.0.1
+      image: containernetworking/pipeline-ci:1.0.2
       options: "--privileged"
 
     # Go setup for the vmImage:

From a16543cf3d3e9399b029a57719de6522a961f19f Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Fri, 20 Sep 2019 15:59:24 -0700
Subject: [PATCH 23/83] update ci image 3

---
 .pipelines/Dockerfile   | 10 +++++++---
 .pipelines/pipeline.yml |  2 +-
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/.pipelines/Dockerfile b/.pipelines/Dockerfile
index 600a775b5f..f515ab9a45 100644
--- a/.pipelines/Dockerfile
+++ b/.pipelines/Dockerfile
@@ -1,8 +1,10 @@
 FROM ubuntu:16.04
-RUN apt-get update && apt-get install -y software-properties-common sudo 
+RUN apt-get update && apt-get install -y software-properties-common sudo wget apt-transport-https curl
+RUN wget -q https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
+RUN sudo dpkg -i packages-microsoft-prod.deb
 RUN add-apt-repository ppa:longsleep/golang-backports && apt-get update
-RUN apt-get install -y git golang-go=2:1.13~1longsleep1+xenial iptables ipset iproute2 ebtables
-RUN go get -v -t -d ./... 
+RUN apt-get install -y git golang-go=2:1.13~1longsleep1+xenial iptables ipset iproute2 ebtables python-pip gcc zip dotnet-sdk-2.2
+RUN sudo pip install coverage
 RUN if [ -f Gopkg.toml ]; then curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh ; fi 
 RUN go get github.com/docker/libnetwork/driverapi
 RUN go get github.com/gorilla/mux
@@ -10,3 +12,5 @@ RUN go get github.com/jstemmer/go-junit-report
 RUN go get github.com/axw/gocov/gocov
 RUN go get github.com/AlekSi/gocov-xml
 RUN go get -u gopkg.in/matm/v1/gocov-html
+ENV PATH="/root/go/bin:${PATH}"
+
diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 948b5eba14..c4d07f51b3 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -13,7 +13,7 @@ jobs:
       demands: agent.os -equals Linux
 
     container:
-      image: containernetworking/pipeline-ci:1.0.2
+      image: containernetworking/pipeline-ci:1.0.3
       options: "--privileged"
 
     # Go setup for the vmImage:

From 1fcc07644c121a4b608135999c1b0867909f6130 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Mon, 23 Sep 2019 14:37:39 -0700
Subject: [PATCH 24/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 40 +++++++++++++++++++++++++++++-----------
 1 file changed, 29 insertions(+), 11 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index c4d07f51b3..6a46e3209f 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -7,7 +7,7 @@ trigger:
       - master
 
 jobs:
-  - job: UnitTest
+  - job: Unit_Tests
     pool:
       name: Networking-ContainerNetworking
       demands: agent.os -equals Linux
@@ -24,11 +24,25 @@ jobs:
       modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" # $(build.repository.name)' # Path to the module's code
 
     steps:
+
+      - bash: |
+          BRANCH=$(git rev-parse --abbrev-ref HEAD)
+          if [[ "$BRANCH" == "master" ]]; then
+            echo "##vso[task.setvariable variable=myOutputVar=Tag;]$(git describe --tags --abbrev=0)"
+          else
+            echo "##vso[task.setvariable variable=myOutputVar=Tag;]$(git describe --tags --always --dirty)"
+          fi
+        workingDirectory: "$(modulePath)"
+        displayName: "EnvironmentalVariables"
+        condition: always()
+
       - bash: |
           echo $UID
           sudo apt-get install -y ebtables ipset python3-dev gcc zip iptables ipset
           sudo pip install coverage
-        displayName: "Install OS dependencies"
+          sudo rm /run/docker/plugins/test.sock || true
+          sudo ip link del dev dummy || true
+        displayName: "Set up OS environment"
 
       - bash: |
           go version
@@ -57,18 +71,11 @@ jobs:
         workingDirectory: "$(modulePath)"
         displayName: "Install Go dependencies"
 
-      - bash: |
-          sudo rm /run/docker/plugins/test.sock || true
-          sudo ip link del dev dummy || true
-        workingDirectory: "$(modulePath)"
-        displayName: "Workspace setup"
-        condition: always()
-
       - bash: |
           export GOOS=linux
-          make all-binaries
+          make all-binaries VERSION=$(EnvironmentalVariables.Tag)
           export GOOS=windows
-          make all-binaries
+          make all-binaries VERSION=$(EnvironmentalVariables.Tag)
           cd output 
           sudo find . -mindepth 2 -type f -regextype posix-extended ! -iregex '.*\.(zip|tgz)$' -delete
           sudo find . -mindepth 2 -type f -print -exec mv {} . \;
@@ -120,3 +127,14 @@ jobs:
           artifactName: "output"
           pathtoPublish: "$(Build.ArtifactStagingDirectory)"
         condition: succeeded()
+
+  - job: E2E_Tests
+    dependsOn: Unit_Tests
+    pool:
+      name: Networking-ContainerNetworking
+      demands: agent.os -equals Linux
+    variables:
+      Tag: $[ dependencies.Unit_Tests.outputs['VERSION=$(EnvironmentalVariables.Tag)'] ] 
+    steps:
+    - script: echo $(Tag)
+      name: echovar
\ No newline at end of file

From ae082b5faea2844049ea17e3e26d1da6130ffd0f Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Mon, 23 Sep 2019 14:45:04 -0700
Subject: [PATCH 25/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 6a46e3209f..56f5369c2f 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -25,17 +25,6 @@ jobs:
 
     steps:
 
-      - bash: |
-          BRANCH=$(git rev-parse --abbrev-ref HEAD)
-          if [[ "$BRANCH" == "master" ]]; then
-            echo "##vso[task.setvariable variable=myOutputVar=Tag;]$(git describe --tags --abbrev=0)"
-          else
-            echo "##vso[task.setvariable variable=myOutputVar=Tag;]$(git describe --tags --always --dirty)"
-          fi
-        workingDirectory: "$(modulePath)"
-        displayName: "EnvironmentalVariables"
-        condition: always()
-
       - bash: |
           echo $UID
           sudo apt-get install -y ebtables ipset python3-dev gcc zip iptables ipset
@@ -55,7 +44,18 @@ jobs:
           mv !(gopath) '$(modulePath)'
           echo '##vso[task.prependpath]$(GOBIN)'
           echo '##vso[task.prependpath]$(GOROOT)/bin'
-        displayName: "Set up the Go workspace"
+        displayName: "Set up the Go environment"
+
+      - bash: |
+          BRANCH=$(git rev-parse --abbrev-ref HEAD)
+          if [[ "$BRANCH" == "master" ]]; then
+            echo "##vso[task.setvariable variable=myOutputVar=Tag;]$(git describe --tags --abbrev=0)"
+          else
+            echo "##vso[task.setvariable variable=myOutputVar=Tag;]$(git describe --tags --always --dirty)"
+          fi
+        workingDirectory: "$(modulePath)"
+        displayName: "EnvironmentalVariables"
+        condition: always()
 
       - bash: |
           go get -v -t -d ./...

From c396f71af3b57042321170d17131aa50d20e9b1c Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Mon, 23 Sep 2019 14:53:35 -0700
Subject: [PATCH 26/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 56f5369c2f..50472255ee 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -49,9 +49,11 @@ jobs:
       - bash: |
           BRANCH=$(git rev-parse --abbrev-ref HEAD)
           if [[ "$BRANCH" == "master" ]]; then
-            echo "##vso[task.setvariable variable=myOutputVar=Tag;]$(git describe --tags --abbrev=0)"
+            echo "##vso[task.setvariable variable=Tag;]$(git describe --tags --abbrev=0)"
+            echo "Set tag to $(git describe --tags --abbrev=0)"
           else
-            echo "##vso[task.setvariable variable=myOutputVar=Tag;]$(git describe --tags --always --dirty)"
+            echo "##vso[task.setvariable variable=Tag;]$(git describe --tags --always --dirty)"
+            echo "Set tag to $(git describe --tags --always --dirty)"
           fi
         workingDirectory: "$(modulePath)"
         displayName: "EnvironmentalVariables"

From 4e4e8e10fd167cdfba5dbe67d1dd6454bfd39e9f Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Mon, 23 Sep 2019 15:03:27 -0700
Subject: [PATCH 27/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 50472255ee..9d09908c39 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -49,10 +49,10 @@ jobs:
       - bash: |
           BRANCH=$(git rev-parse --abbrev-ref HEAD)
           if [[ "$BRANCH" == "master" ]]; then
-            echo "##vso[task.setvariable variable=Tag;]$(git describe --tags --abbrev=0)"
+            echo "##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --abbrev=0)"
             echo "Set tag to $(git describe --tags --abbrev=0)"
           else
-            echo "##vso[task.setvariable variable=Tag;]$(git describe --tags --always --dirty)"
+            echo "##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --always --dirty)"
             echo "Set tag to $(git describe --tags --always --dirty)"
           fi
         workingDirectory: "$(modulePath)"

From ef80800929e4aa91c3661f93f053310e95f496d7 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Mon, 23 Sep 2019 15:11:38 -0700
Subject: [PATCH 28/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 9d09908c39..e11839e9b1 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -74,6 +74,7 @@ jobs:
         displayName: "Install Go dependencies"
 
       - bash: |
+          echo "Using tag $(EnvironmentalVariables.Tag)"
           export GOOS=linux
           make all-binaries VERSION=$(EnvironmentalVariables.Tag)
           export GOOS=windows

From b0db2dfaf236fe3add1715e133b58a75e937aa1a Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Mon, 23 Sep 2019 15:12:31 -0700
Subject: [PATCH 29/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index e11839e9b1..4bc9e72628 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -137,7 +137,7 @@ jobs:
       name: Networking-ContainerNetworking
       demands: agent.os -equals Linux
     variables:
-      Tag: $[ dependencies.Unit_Tests.outputs['VERSION=$(EnvironmentalVariables.Tag)'] ] 
+      Tag: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.Tag'] ] 
     steps:
     - script: echo $(Tag)
       name: echovar
\ No newline at end of file

From c8a8aa52598d0b0ae90be1456eac0340480a2236 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Mon, 23 Sep 2019 15:29:29 -0700
Subject: [PATCH 30/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 4bc9e72628..470af894e9 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -24,7 +24,6 @@ jobs:
       modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" # $(build.repository.name)' # Path to the module's code
 
     steps:
-
       - bash: |
           echo $UID
           sudo apt-get install -y ebtables ipset python3-dev gcc zip iptables ipset
@@ -49,10 +48,10 @@ jobs:
       - bash: |
           BRANCH=$(git rev-parse --abbrev-ref HEAD)
           if [[ "$BRANCH" == "master" ]]; then
-            echo "##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --abbrev=0)"
+            echo '##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --abbrev=0)'
             echo "Set tag to $(git describe --tags --abbrev=0)"
           else
-            echo "##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --always --dirty)"
+            echo '##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --always --dirty)'
             echo "Set tag to $(git describe --tags --always --dirty)"
           fi
         workingDirectory: "$(modulePath)"
@@ -74,7 +73,7 @@ jobs:
         displayName: "Install Go dependencies"
 
       - bash: |
-          echo "Using tag $(EnvironmentalVariables.Tag)"
+          echo $(EnvironmentalVariables.Tag)
           export GOOS=linux
           make all-binaries VERSION=$(EnvironmentalVariables.Tag)
           export GOOS=windows

From a51cbc7844941284d075f50f49a4faa038bcb54f Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Mon, 23 Sep 2019 15:36:28 -0700
Subject: [PATCH 31/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 470af894e9..8e64e7a846 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -45,7 +45,7 @@ jobs:
           echo '##vso[task.prependpath]$(GOROOT)/bin'
         displayName: "Set up the Go environment"
 
-      - bash: |
+      - script: |
           BRANCH=$(git rev-parse --abbrev-ref HEAD)
           if [[ "$BRANCH" == "master" ]]; then
             echo '##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --abbrev=0)'
@@ -58,7 +58,7 @@ jobs:
         displayName: "EnvironmentalVariables"
         condition: always()
 
-      - bash: |
+      - script: |
           go get -v -t -d ./...
           if [ -f Gopkg.toml ]; then
               curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
@@ -72,8 +72,8 @@ jobs:
         workingDirectory: "$(modulePath)"
         displayName: "Install Go dependencies"
 
-      - bash: |
-          echo $(EnvironmentalVariables.Tag)
+      - script: |
+          echo Build tag is $(EnvironmentalVariables.Tag)
           export GOOS=linux
           make all-binaries VERSION=$(EnvironmentalVariables.Tag)
           export GOOS=windows

From 2f7af2c03796de7477a4c2cbc05ff942ff5c6860 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Mon, 23 Sep 2019 15:39:28 -0700
Subject: [PATCH 32/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 8e64e7a846..a2fb3741c0 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -48,10 +48,10 @@ jobs:
       - script: |
           BRANCH=$(git rev-parse --abbrev-ref HEAD)
           if [[ "$BRANCH" == "master" ]]; then
-            echo '##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --abbrev=0)'
+            echo '##vso[task.setvariable variable=Tag;isOutput=true]testmaster'
             echo "Set tag to $(git describe --tags --abbrev=0)"
           else
-            echo '##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --always --dirty)'
+            echo '##vso[task.setvariable variable=Tag;isOutput=true]test'
             echo "Set tag to $(git describe --tags --always --dirty)"
           fi
         workingDirectory: "$(modulePath)"

From 55b1451f87b8bb7da83567ce0651e52f3bc1b60d Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Mon, 23 Sep 2019 15:43:10 -0700
Subject: [PATCH 33/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 12 +++++-------
 1 file changed, 5 insertions(+), 7 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index a2fb3741c0..6e23fd7a7f 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -26,8 +26,6 @@ jobs:
     steps:
       - bash: |
           echo $UID
-          sudo apt-get install -y ebtables ipset python3-dev gcc zip iptables ipset
-          sudo pip install coverage
           sudo rm /run/docker/plugins/test.sock || true
           sudo ip link del dev dummy || true
         displayName: "Set up OS environment"
@@ -48,10 +46,10 @@ jobs:
       - script: |
           BRANCH=$(git rev-parse --abbrev-ref HEAD)
           if [[ "$BRANCH" == "master" ]]; then
-            echo '##vso[task.setvariable variable=Tag;isOutput=true]testmaster'
+            echo '##vso[task.setvariable variable=TAG;isOutput=true]testmaster'
             echo "Set tag to $(git describe --tags --abbrev=0)"
           else
-            echo '##vso[task.setvariable variable=Tag;isOutput=true]test'
+            echo '##vso[task.setvariable variable=TAG;isOutput=true]test'
             echo "Set tag to $(git describe --tags --always --dirty)"
           fi
         workingDirectory: "$(modulePath)"
@@ -73,11 +71,11 @@ jobs:
         displayName: "Install Go dependencies"
 
       - script: |
-          echo Build tag is $(EnvironmentalVariables.Tag)
+          echo Build tag is $TAG
           export GOOS=linux
-          make all-binaries VERSION=$(EnvironmentalVariables.Tag)
+          make all-binaries VERSION=$TAG
           export GOOS=windows
-          make all-binaries VERSION=$(EnvironmentalVariables.Tag)
+          make all-binaries VERSION=$TAG
           cd output 
           sudo find . -mindepth 2 -type f -regextype posix-extended ! -iregex '.*\.(zip|tgz)$' -delete
           sudo find . -mindepth 2 -type f -print -exec mv {} . \;

From 7d806f3b2bafe62d9366e0e6e45af77a5c7d9854 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Mon, 23 Sep 2019 15:49:25 -0700
Subject: [PATCH 34/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 6e23fd7a7f..e46e7026f1 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -83,6 +83,11 @@ jobs:
         workingDirectory: "$(modulePath)"
         displayName: "Build"
 
+      - script: echo "##vso[task.setvariable variable=myOutputVar;isOutput=true]this is the $(configuration) value"
+        name: setvarStep
+      - script: echo $(setvarStep.myOutputVar)
+        name: echovar
+
       - bash: |
           # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout
           { { { { 
@@ -135,6 +140,9 @@ jobs:
       demands: agent.os -equals Linux
     variables:
       Tag: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.Tag'] ] 
+      myVarFromJobADebug: $[ dependencies.A.outputs['debugJob.setvarStep.myOutputVar'] ]
     steps:
     - script: echo $(Tag)
+      name: echovar1
+    - script: echo $(myVarFromJobADebug)
       name: echovar
\ No newline at end of file

From 40ef4227cb65984e6df214ca7dbe1cea5f7761f3 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Mon, 23 Sep 2019 15:56:06 -0700
Subject: [PATCH 35/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index e46e7026f1..023d9a4edc 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -46,7 +46,7 @@ jobs:
       - script: |
           BRANCH=$(git rev-parse --abbrev-ref HEAD)
           if [[ "$BRANCH" == "master" ]]; then
-            echo '##vso[task.setvariable variable=TAG;isOutput=true]testmaster'
+            echo "##vso[task.setvariable variable=myOutputVar;isOutput=true]this is the $BRANCH value"
             echo "Set tag to $(git describe --tags --abbrev=0)"
           else
             echo '##vso[task.setvariable variable=TAG;isOutput=true]test'
@@ -56,6 +56,9 @@ jobs:
         displayName: "EnvironmentalVariables"
         condition: always()
 
+      - script: echo $(EnvironmentalVariables.myOutputVar)
+        name: echovar
+
       - script: |
           go get -v -t -d ./...
           if [ -f Gopkg.toml ]; then
@@ -83,10 +86,6 @@ jobs:
         workingDirectory: "$(modulePath)"
         displayName: "Build"
 
-      - script: echo "##vso[task.setvariable variable=myOutputVar;isOutput=true]this is the $(configuration) value"
-        name: setvarStep
-      - script: echo $(setvarStep.myOutputVar)
-        name: echovar
 
       - bash: |
           # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout

From 655a06141f3fd07c7e9b8a23b78e57c2811e1d5e Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Mon, 23 Sep 2019 15:58:36 -0700
Subject: [PATCH 36/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 023d9a4edc..8d0187f370 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -46,14 +46,14 @@ jobs:
       - script: |
           BRANCH=$(git rev-parse --abbrev-ref HEAD)
           if [[ "$BRANCH" == "master" ]]; then
-            echo "##vso[task.setvariable variable=myOutputVar;isOutput=true]this is the $BRANCH value"
+            echo "##vso[task.setvariable variable=myOutputVar;isOutput=true]this is the $(git describe --tags --abbrev=0) value"
             echo "Set tag to $(git describe --tags --abbrev=0)"
           else
-            echo '##vso[task.setvariable variable=TAG;isOutput=true]test'
+            echo "##vso[task.setvariable variable=TAG;isOutput=true]$(git describe --tags --always --dirty)"
             echo "Set tag to $(git describe --tags --always --dirty)"
           fi
         workingDirectory: "$(modulePath)"
-        displayName: "EnvironmentalVariables"
+        name: "EnvironmentalVariables"
         condition: always()
 
       - script: echo $(EnvironmentalVariables.myOutputVar)
@@ -86,7 +86,6 @@ jobs:
         workingDirectory: "$(modulePath)"
         displayName: "Build"
 
-
       - bash: |
           # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout
           { { { { 

From eded7a058544535a93150a943c94336551d16f78 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Mon, 23 Sep 2019 16:00:19 -0700
Subject: [PATCH 37/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 8d0187f370..37903c2436 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -46,17 +46,17 @@ jobs:
       - script: |
           BRANCH=$(git rev-parse --abbrev-ref HEAD)
           if [[ "$BRANCH" == "master" ]]; then
-            echo "##vso[task.setvariable variable=myOutputVar;isOutput=true]this is the $(git describe --tags --abbrev=0) value"
+            echo "##vso[task.setvariable variable=Tag;isOutput=true]this is the $(git describe --tags --abbrev=0) value"
             echo "Set tag to $(git describe --tags --abbrev=0)"
           else
-            echo "##vso[task.setvariable variable=TAG;isOutput=true]$(git describe --tags --always --dirty)"
+            echo "##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --always --dirty)"
             echo "Set tag to $(git describe --tags --always --dirty)"
           fi
         workingDirectory: "$(modulePath)"
         name: "EnvironmentalVariables"
         condition: always()
 
-      - script: echo $(EnvironmentalVariables.myOutputVar)
+      - script: echo $(EnvironmentalVariables.Tag)
         name: echovar
 
       - script: |

From 71e0965bef41faf0140e4bf580237825250fa63d Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Mon, 23 Sep 2019 16:07:21 -0700
Subject: [PATCH 38/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 17 +++++++++++++++--
 1 file changed, 15 insertions(+), 2 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 37903c2436..71099b6427 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -8,6 +8,7 @@ trigger:
 
 jobs:
   - job: Unit_Tests
+    displayName: Unit Tests
     pool:
       name: Networking-ContainerNetworking
       demands: agent.os -equals Linux
@@ -46,7 +47,7 @@ jobs:
       - script: |
           BRANCH=$(git rev-parse --abbrev-ref HEAD)
           if [[ "$BRANCH" == "master" ]]; then
-            echo "##vso[task.setvariable variable=Tag;isOutput=true]this is the $(git describe --tags --abbrev=0) value"
+            echo "##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --abbrev=0)"
             echo "Set tag to $(git describe --tags --abbrev=0)"
           else
             echo "##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --always --dirty)"
@@ -71,6 +72,7 @@ jobs:
           go get github.com/AlekSi/gocov-xml
           go get -u gopkg.in/matm/v1/gocov-html
         workingDirectory: "$(modulePath)"
+        name: "GoDependencies"
         displayName: "Install Go dependencies"
 
       - script: |
@@ -84,6 +86,7 @@ jobs:
           sudo find . -mindepth 2 -type f -print -exec mv {} . \;
           sudo rm -R -- */
         workingDirectory: "$(modulePath)"
+        name: "Build"
         displayName: "Build"
 
       - bash: |
@@ -97,6 +100,7 @@ jobs:
           } 4>&1
         workingDirectory: "$(modulePath)"
         failOnStderr: true
+        name: "Test"
         displayName: "Run Tests"
 
       - bash: |
@@ -104,6 +108,7 @@ jobs:
           gocov convert coverage.out > coverage.json
           gocov-xml < coverage.json > coverage.xml
         workingDirectory: "$(modulePath)"
+        name: "Test"
         displayName: "Generate Coverage Reports"
         condition: always()
 
@@ -131,14 +136,22 @@ jobs:
           pathtoPublish: "$(Build.ArtifactStagingDirectory)"
         condition: succeeded()
 
+      - task: AzureFileCopy@3
+        inputs:
+          SourcePath: '$(modulePath)/output'
+          azureSubscription: $(ARTIFACT_SUBSCRIPTION)
+          Destination: 'AzureBlob'
+          storage: $(ARTIFACT_STORAGE)
+          ContainerName: $(EnvironmentalVariables.Tag)
+
   - job: E2E_Tests
+    displayName: AKS-Engine E2E Tests
     dependsOn: Unit_Tests
     pool:
       name: Networking-ContainerNetworking
       demands: agent.os -equals Linux
     variables:
       Tag: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.Tag'] ] 
-      myVarFromJobADebug: $[ dependencies.A.outputs['debugJob.setvarStep.myOutputVar'] ]
     steps:
     - script: echo $(Tag)
       name: echovar1

From 99c261b6ee071f08e437e2e058a8477971d2538d Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Mon, 23 Sep 2019 16:09:39 -0700
Subject: [PATCH 39/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 71099b6427..74b1731290 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -42,6 +42,7 @@ jobs:
           mv !(gopath) '$(modulePath)'
           echo '##vso[task.prependpath]$(GOBIN)'
           echo '##vso[task.prependpath]$(GOROOT)/bin'
+        name: "GoEnv"
         displayName: "Set up the Go environment"
 
       - script: |
@@ -55,11 +56,9 @@ jobs:
           fi
         workingDirectory: "$(modulePath)"
         name: "EnvironmentalVariables"
+        displayName: "Set build environmental variables"
         condition: always()
 
-      - script: echo $(EnvironmentalVariables.Tag)
-        name: echovar
-
       - script: |
           go get -v -t -d ./...
           if [ -f Gopkg.toml ]; then
@@ -108,7 +107,7 @@ jobs:
           gocov convert coverage.out > coverage.json
           gocov-xml < coverage.json > coverage.xml
         workingDirectory: "$(modulePath)"
-        name: "Test"
+        name: "Coverage"
         displayName: "Generate Coverage Reports"
         condition: always()
 

From 76c6180b4dc63c9823dc854fe9fed74a816d96cf Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Mon, 23 Sep 2019 16:42:32 -0700
Subject: [PATCH 40/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 23 +++++++++++++++--------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 74b1731290..0792d10de0 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -75,11 +75,11 @@ jobs:
         displayName: "Install Go dependencies"
 
       - script: |
-          echo Build tag is $TAG
+          echo Build tag is $(EnvironmentalVariables.Tag)
           export GOOS=linux
-          make all-binaries VERSION=$TAG
+          make all-binaries VERSION=$(EnvironmentalVariables.Tag)
           export GOOS=windows
-          make all-binaries VERSION=$TAG
+          make all-binaries VERSION=$(EnvironmentalVariables.Tag)
           cd output 
           sudo find . -mindepth 2 -type f -regextype posix-extended ! -iregex '.*\.(zip|tgz)$' -delete
           sudo find . -mindepth 2 -type f -print -exec mv {} . \;
@@ -135,13 +135,20 @@ jobs:
           pathtoPublish: "$(Build.ArtifactStagingDirectory)"
         condition: succeeded()
 
-      - task: AzureFileCopy@3
+      - task: AzureCLI@1
         inputs:
-          SourcePath: '$(modulePath)/output'
           azureSubscription: $(ARTIFACT_SUBSCRIPTION)
-          Destination: 'AzureBlob'
-          storage: $(ARTIFACT_STORAGE)
-          ContainerName: $(EnvironmentalVariables.Tag)
+          scriptLocation: 'inlineScript'
+          inlineScript: 'az storage container create -n acn-$(EnvironmentalVariables.Tag) --account-name $(STORAGE_ACCOUNT_NAME) --public-access container'
+          workingDirectory: '$(modulePath)'
+        displayName: Create artifact storage container
+
+      - task: AzureCLI@1
+        inputs:
+          azureSubscription: $(ARTIFACT_SUBSCRIPTION)
+          scriptLocation: 'inlineScript'
+          inlineScript: 'az storage blob upload-batch -d acn-$(EnvironmentalVariables.Tag) -s output/ --pattern * --account-name $(STORAGE_ACCOUNT_NAME)'
+          workingDirectory: '$(modulePath)'
 
   - job: E2E_Tests
     displayName: AKS-Engine E2E Tests

From 7643a1b3b4d8e56cd53386eba3ffe12162a5056f Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Tue, 24 Sep 2019 10:18:33 -0700
Subject: [PATCH 41/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 0792d10de0..79e16bc50a 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -14,7 +14,7 @@ jobs:
       demands: agent.os -equals Linux
 
     container:
-      image: containernetworking/pipeline-ci:1.0.3
+      image: containernetworking/pipeline-ci:1.0.4
       options: "--privileged"
 
     # Go setup for the vmImage:

From 4e26d9a9af69ca11cc28012dc4df66047ce16c1b Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Tue, 24 Sep 2019 10:38:21 -0700
Subject: [PATCH 42/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 79e16bc50a..21f57e8a3c 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -46,6 +46,7 @@ jobs:
         displayName: "Set up the Go environment"
 
       - script: |
+          echo "##vso[task.setvariable variable=CommitHash;isOutput=true]$(git rev-parse HEAD)"
           BRANCH=$(git rev-parse --abbrev-ref HEAD)
           if [[ "$BRANCH" == "master" ]]; then
             echo "##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --abbrev=0)"
@@ -139,7 +140,7 @@ jobs:
         inputs:
           azureSubscription: $(ARTIFACT_SUBSCRIPTION)
           scriptLocation: 'inlineScript'
-          inlineScript: 'az storage container create -n acn-$(EnvironmentalVariables.Tag) --account-name $(STORAGE_ACCOUNT_NAME) --public-access container'
+          inlineScript: 'az storage container create -n acn-$(EnvironmentalVariables.CommitHash) --account-name $(STORAGE_ACCOUNT_NAME) --public-access container'
           workingDirectory: '$(modulePath)'
         displayName: Create artifact storage container
 
@@ -147,7 +148,7 @@ jobs:
         inputs:
           azureSubscription: $(ARTIFACT_SUBSCRIPTION)
           scriptLocation: 'inlineScript'
-          inlineScript: 'az storage blob upload-batch -d acn-$(EnvironmentalVariables.Tag) -s output/ --pattern * --account-name $(STORAGE_ACCOUNT_NAME)'
+          inlineScript: 'az storage blob upload-batch -d acn-$(EnvironmentalVariables.CommitHash) -s output/ --pattern * --account-name $(STORAGE_ACCOUNT_NAME)'
           workingDirectory: '$(modulePath)'
 
   - job: E2E_Tests

From a8daea330a7258948397ca8e6e50fb9cf004af90 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Tue, 24 Sep 2019 10:53:36 -0700
Subject: [PATCH 43/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 21f57e8a3c..1b3a7a3866 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -148,7 +148,7 @@ jobs:
         inputs:
           azureSubscription: $(ARTIFACT_SUBSCRIPTION)
           scriptLocation: 'inlineScript'
-          inlineScript: 'az storage blob upload-batch -d acn-$(EnvironmentalVariables.CommitHash) -s output/ --pattern * --account-name $(STORAGE_ACCOUNT_NAME)'
+          inlineScript: 'az storage blob upload-batch -d acn-$(EnvironmentalVariables.CommitHash) -s ./output/  --account-name $(STORAGE_ACCOUNT_NAME)'
           workingDirectory: '$(modulePath)'
 
   - job: E2E_Tests

From a719766b9d6ce157c726dbe5995aa41d1d8d61ef Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Tue, 24 Sep 2019 11:17:02 -0700
Subject: [PATCH 44/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 340 +++++++++++++++++++++-------------------
 1 file changed, 182 insertions(+), 158 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 1b3a7a3866..4f1212bab7 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -6,161 +6,185 @@ trigger:
     include:
       - master
 
-jobs:
-  - job: Unit_Tests
-    displayName: Unit Tests
-    pool:
-      name: Networking-ContainerNetworking
-      demands: agent.os -equals Linux
-
-    container:
-      image: containernetworking/pipeline-ci:1.0.4
-      options: "--privileged"
-
-    # Go setup for the vmImage:
-    # https://github.com/Microsoft/azure-pipelines-image-generation/blob/master/images/linux/scripts/installers/go.sh
-    variables:
-      GOBIN: "$(GOPATH)/bin" # Go binaries path
-      GOPATH: "$(System.DefaultWorkingDirectory)/gopath" # Go workspace path
-      modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" # $(build.repository.name)' # Path to the module's code
-
-    steps:
-      - bash: |
-          echo $UID
-          sudo rm /run/docker/plugins/test.sock || true
-          sudo ip link del dev dummy || true
-        displayName: "Set up OS environment"
-
-      - bash: |
-          go version
-          go env
-          mkdir -p '$(GOBIN)'
-          mkdir -p '$(GOPATH)/pkg'
-          mkdir -p '$(modulePath)'
-          shopt -s extglob
-          shopt -s dotglob
-          mv !(gopath) '$(modulePath)'
-          echo '##vso[task.prependpath]$(GOBIN)'
-          echo '##vso[task.prependpath]$(GOROOT)/bin'
-        name: "GoEnv"
-        displayName: "Set up the Go environment"
-
-      - script: |
-          echo "##vso[task.setvariable variable=CommitHash;isOutput=true]$(git rev-parse HEAD)"
-          BRANCH=$(git rev-parse --abbrev-ref HEAD)
-          if [[ "$BRANCH" == "master" ]]; then
-            echo "##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --abbrev=0)"
-            echo "Set tag to $(git describe --tags --abbrev=0)"
-          else
-            echo "##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --always --dirty)"
-            echo "Set tag to $(git describe --tags --always --dirty)"
-          fi
-        workingDirectory: "$(modulePath)"
-        name: "EnvironmentalVariables"
-        displayName: "Set build environmental variables"
-        condition: always()
-
-      - script: |
-          go get -v -t -d ./...
-          if [ -f Gopkg.toml ]; then
-              curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-          fi
-          go get github.com/docker/libnetwork/driverapi
-          go get github.com/gorilla/mux
-          go get github.com/jstemmer/go-junit-report
-          go get github.com/axw/gocov/gocov
-          go get github.com/AlekSi/gocov-xml
-          go get -u gopkg.in/matm/v1/gocov-html
-        workingDirectory: "$(modulePath)"
-        name: "GoDependencies"
-        displayName: "Install Go dependencies"
-
-      - script: |
-          echo Build tag is $(EnvironmentalVariables.Tag)
-          export GOOS=linux
-          make all-binaries VERSION=$(EnvironmentalVariables.Tag)
-          export GOOS=windows
-          make all-binaries VERSION=$(EnvironmentalVariables.Tag)
-          cd output 
-          sudo find . -mindepth 2 -type f -regextype posix-extended ! -iregex '.*\.(zip|tgz)$' -delete
-          sudo find . -mindepth 2 -type f -print -exec mv {} . \;
-          sudo rm -R -- */
-        workingDirectory: "$(modulePath)"
-        name: "Build"
-        displayName: "Build"
-
-      - bash: |
-          # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout
-          { { { { 
-                sudo -E env "PATH=$PATH" make test-all; 
-                echo $? >&3; 
-                } | tee >(go-junit-report > report.xml) >&4; 
-              } 3>&1; 
-            } | { read xs; exit $xs; } 
-          } 4>&1
-        workingDirectory: "$(modulePath)"
-        failOnStderr: true
-        name: "Test"
-        displayName: "Run Tests"
-
-      - bash: |
-          bash <(curl -s https://codecov.io/bash)
-          gocov convert coverage.out > coverage.json
-          gocov-xml < coverage.json > coverage.xml
-        workingDirectory: "$(modulePath)"
-        name: "Coverage"
-        displayName: "Generate Coverage Reports"
-        condition: always()
-
-      - task: PublishTestResults@2
-        inputs:
-          testRunner: JUnit
-          testResultsFiles: $(System.DefaultWorkingDirectory)/**/report.xml
-        condition: always()
-
-      - task: PublishCodeCoverageResults@1
-        inputs:
-          codeCoverageTool: Cobertura
-          summaryFileLocation: $(System.DefaultWorkingDirectory)/**/coverage.xml
-        condition: always()
-
-      - task: CopyFiles@2
-        inputs:
-          sourceFolder: "$(modulePath)/output"
-          targetFolder: $(Build.ArtifactStagingDirectory)
-        condition: succeeded()
-
-      - task: PublishBuildArtifacts@1
-        inputs:
-          artifactName: "output"
-          pathtoPublish: "$(Build.ArtifactStagingDirectory)"
-        condition: succeeded()
-
-      - task: AzureCLI@1
-        inputs:
-          azureSubscription: $(ARTIFACT_SUBSCRIPTION)
-          scriptLocation: 'inlineScript'
-          inlineScript: 'az storage container create -n acn-$(EnvironmentalVariables.CommitHash) --account-name $(STORAGE_ACCOUNT_NAME) --public-access container'
-          workingDirectory: '$(modulePath)'
-        displayName: Create artifact storage container
-
-      - task: AzureCLI@1
-        inputs:
-          azureSubscription: $(ARTIFACT_SUBSCRIPTION)
-          scriptLocation: 'inlineScript'
-          inlineScript: 'az storage blob upload-batch -d acn-$(EnvironmentalVariables.CommitHash) -s ./output/  --account-name $(STORAGE_ACCOUNT_NAME)'
-          workingDirectory: '$(modulePath)'
-
-  - job: E2E_Tests
-    displayName: AKS-Engine E2E Tests
-    dependsOn: Unit_Tests
-    pool:
-      name: Networking-ContainerNetworking
-      demands: agent.os -equals Linux
-    variables:
-      Tag: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.Tag'] ] 
-    steps:
-    - script: echo $(Tag)
-      name: echovar1
-    - script: echo $(myVarFromJobADebug)
-      name: echovar
\ No newline at end of file
+stages:
+  - stage: Tests
+    displayName: Build and Run Tests
+    jobs:
+      - job: Unit_Tests
+        displayName: Unit Tests
+        pool:
+          name: Networking-ContainerNetworking
+          demands: agent.os -equals Linux
+
+        container:
+          image: containernetworking/pipeline-ci:1.0.4
+          options: "--privileged"
+
+        # Go setup for the vmImage:
+        # https://github.com/Microsoft/azure-pipelines-image-generation/blob/master/images/linux/scripts/installers/go.sh
+        variables:
+          GOBIN: "$(GOPATH)/bin" # Go binaries path
+          GOPATH: "$(System.DefaultWorkingDirectory)/gopath" # Go workspace path
+          modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" # $(build.repository.name)' # Path to the module's code
+
+        steps:
+          - bash: |
+              echo $UID
+              sudo rm /run/docker/plugins/test.sock || true
+              sudo ip link del dev dummy || true
+            displayName: "Set up OS environment"
+
+          - bash: |
+              go version
+              go env
+              mkdir -p '$(GOBIN)'
+              mkdir -p '$(GOPATH)/pkg'
+              mkdir -p '$(modulePath)'
+              shopt -s extglob
+              shopt -s dotglob
+              mv !(gopath) '$(modulePath)'
+              echo '##vso[task.prependpath]$(GOBIN)'
+              echo '##vso[task.prependpath]$(GOROOT)/bin'
+            name: "GoEnv"
+            displayName: "Set up the Go environment"
+
+          - script: |
+              echo "##vso[task.setvariable variable=CommitHash;isOutput=true]$(git rev-parse HEAD)"
+              BRANCH=$(git rev-parse --abbrev-ref HEAD)
+              if [[ "$BRANCH" == "master" ]]; then
+                echo "##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --abbrev=0)"
+                echo "Set tag to $(git describe --tags --abbrev=0)"
+              else
+                echo "##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --always --dirty)"
+                echo "Set tag to $(git describe --tags --always --dirty)"
+              fi
+            workingDirectory: "$(modulePath)"
+            name: "EnvironmentalVariables"
+            displayName: "Set build environmental variables"
+            condition: always()
+
+          - script: |
+              go get -v -t -d ./...
+              if [ -f Gopkg.toml ]; then
+                  curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+              fi
+              go get github.com/docker/libnetwork/driverapi
+              go get github.com/gorilla/mux
+              go get github.com/jstemmer/go-junit-report
+              go get github.com/axw/gocov/gocov
+              go get github.com/AlekSi/gocov-xml
+              go get -u gopkg.in/matm/v1/gocov-html
+            workingDirectory: "$(modulePath)"
+            name: "GoDependencies"
+            displayName: "Install Go dependencies"
+
+          - script: |
+              echo Build tag is $(EnvironmentalVariables.Tag)
+              export GOOS=linux
+              make all-binaries VERSION=$(EnvironmentalVariables.Tag)
+              export GOOS=windows
+              make all-binaries VERSION=$(EnvironmentalVariables.Tag)
+              cd output 
+              sudo find . -mindepth 2 -type f -regextype posix-extended ! -iregex '.*\.(zip|tgz)$' -delete
+              sudo find . -mindepth 2 -type f -print -exec mv {} . \;
+              sudo rm -R -- */
+            workingDirectory: "$(modulePath)"
+            name: "Build"
+            displayName: "Build"
+
+          - bash: |
+              # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout
+              { { { { 
+                    sudo -E env "PATH=$PATH" make test-all; 
+                    echo $? >&3; 
+                    } | tee >(go-junit-report > report.xml) >&4; 
+                  } 3>&1; 
+                } | { read xs; exit $xs; } 
+              } 4>&1
+            workingDirectory: "$(modulePath)"
+            failOnStderr: true
+            name: "Test"
+            displayName: "Run Tests"
+
+          - bash: |
+              bash <(curl -s https://codecov.io/bash)
+              gocov convert coverage.out > coverage.json
+              gocov-xml < coverage.json > coverage.xml
+            workingDirectory: "$(modulePath)"
+            name: "Coverage"
+            displayName: "Generate Coverage Reports"
+            condition: always()
+
+          - task: PublishTestResults@2
+            inputs:
+              testRunner: JUnit
+              testResultsFiles: $(System.DefaultWorkingDirectory)/**/report.xml
+            condition: always()
+
+          - task: PublishCodeCoverageResults@1
+            inputs:
+              codeCoverageTool: Cobertura
+              summaryFileLocation: $(System.DefaultWorkingDirectory)/**/coverage.xml
+            condition: always()
+
+          - task: CopyFiles@2
+            inputs:
+              sourceFolder: "$(modulePath)/output"
+              targetFolder: $(Build.ArtifactStagingDirectory)
+            condition: succeeded()
+
+          - task: PublishBuildArtifacts@1
+            inputs:
+              artifactName: "output"
+              pathtoPublish: "$(Build.ArtifactStagingDirectory)"
+            condition: succeeded()
+
+          - task: AzureCLI@1
+            inputs:
+              azureSubscription: $(ARTIFACT_SUBSCRIPTION)
+              scriptLocation: 'inlineScript'
+              inlineScript: |
+                'az storage container create -n acn-$(EnvironmentalVariables.CommitHash) --account-name $(STORAGE_ACCOUNT_NAME) --public-access container'
+                'az storage blob upload-batch -d acn-$(EnvironmentalVariables.CommitHash) -s ./output/  --account-name $(STORAGE_ACCOUNT_NAME)'
+              workingDirectory: '$(modulePath)'
+            displayName: Create artifact storage container
+
+
+      - job: E2E_Tests
+        displayName: AKS-Engine E2E Tests
+        dependsOn: Unit_Tests
+        pool:
+          name: Networking-ContainerNetworking
+          demands: agent.os -equals Linux
+        variables:
+          Tag: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.Tag'] ] 
+        steps:
+        - script: echo $(Tag)
+          name: echovar1
+        - script: echo $(myVarFromJobADebug)
+          name: echovar
+
+
+      - job: Cleanup
+        displayName: Cleanup remote artifacts
+        dependsOn: 
+          - Unit_Tests
+          - E2E_Tests
+        pool:
+          name: Networking-ContainerNetworking
+          demands: agent.os -equals Linux
+        variables:
+          Tag: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.Tag'] ] 
+          CommitHash: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.CommitHash'] ] 
+        container:
+          image: containernetworking/pipeline-ci:1.0.4
+
+        steps:
+          - task: AzureCLI@1
+            inputs:
+              azureSubscription: $(ARTIFACT_SUBSCRIPTION)
+              scriptLocation: 'inlineScript'
+              inlineScript: |
+                'az storage container delete -n acn-$(EnvironmentalVariables.CommitHash) --account-name $(STORAGE_ACCOUNT_NAME)''
+              workingDirectory: '$(modulePath)'
+            displayName: Cleanup remote Azure storage container

From 389ea4249793a797555eec046fc76f69f7ee5439 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Tue, 24 Sep 2019 11:23:15 -0700
Subject: [PATCH 45/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 4f1212bab7..b58e36ef40 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -143,12 +143,18 @@ stages:
             inputs:
               azureSubscription: $(ARTIFACT_SUBSCRIPTION)
               scriptLocation: 'inlineScript'
-              inlineScript: |
-                'az storage container create -n acn-$(EnvironmentalVariables.CommitHash) --account-name $(STORAGE_ACCOUNT_NAME) --public-access container'
-                'az storage blob upload-batch -d acn-$(EnvironmentalVariables.CommitHash) -s ./output/  --account-name $(STORAGE_ACCOUNT_NAME)'
+              inlineScript: 'az storage container create -n acn-$(EnvironmentalVariables.CommitHash) --account-name $(STORAGE_ACCOUNT_NAME) --public-access container'
+
               workingDirectory: '$(modulePath)'
             displayName: Create artifact storage container
 
+          - task: AzureCLI@1
+            inputs:
+              azureSubscription: $(ARTIFACT_SUBSCRIPTION)
+              scriptLocation: 'inlineScript'
+              inlineScript: 'az storage blob upload-batch -d acn-$(EnvironmentalVariables.CommitHash) -s ./output/  --account-name $(STORAGE_ACCOUNT_NAME)'
+              workingDirectory: '$(modulePath)'
+            displayName: Upload artifacts to Azure storage container
 
       - job: E2E_Tests
         displayName: AKS-Engine E2E Tests

From f550677c8c8df6fd35bd3c7e3acce91c8f1d449c Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Tue, 24 Sep 2019 11:32:34 -0700
Subject: [PATCH 46/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index b58e36ef40..1fb0037971 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -143,19 +143,12 @@ stages:
             inputs:
               azureSubscription: $(ARTIFACT_SUBSCRIPTION)
               scriptLocation: 'inlineScript'
-              inlineScript: 'az storage container create -n acn-$(EnvironmentalVariables.CommitHash) --account-name $(STORAGE_ACCOUNT_NAME) --public-access container'
-
+              inlineScript: |
+                az storage container create -n acn-$(EnvironmentalVariables.CommitHash) --account-name $(STORAGE_ACCOUNT_NAME) --public-access container
+                az storage blob upload-batch -d acn-$(EnvironmentalVariables.CommitHash) -s ./output/  --account-name $(STORAGE_ACCOUNT_NAME)
               workingDirectory: '$(modulePath)'
             displayName: Create artifact storage container
 
-          - task: AzureCLI@1
-            inputs:
-              azureSubscription: $(ARTIFACT_SUBSCRIPTION)
-              scriptLocation: 'inlineScript'
-              inlineScript: 'az storage blob upload-batch -d acn-$(EnvironmentalVariables.CommitHash) -s ./output/  --account-name $(STORAGE_ACCOUNT_NAME)'
-              workingDirectory: '$(modulePath)'
-            displayName: Upload artifacts to Azure storage container
-
       - job: E2E_Tests
         displayName: AKS-Engine E2E Tests
         dependsOn: Unit_Tests
@@ -191,6 +184,6 @@ stages:
               azureSubscription: $(ARTIFACT_SUBSCRIPTION)
               scriptLocation: 'inlineScript'
               inlineScript: |
-                'az storage container delete -n acn-$(EnvironmentalVariables.CommitHash) --account-name $(STORAGE_ACCOUNT_NAME)''
+                az storage container delete -n acn-$(CommitHash) --account-name $(STORAGE_ACCOUNT_NAME)
               workingDirectory: '$(modulePath)'
             displayName: Cleanup remote Azure storage container

From 8beb2dc47719194559405f4ab7df6e1901f598ba Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Tue, 24 Sep 2019 11:50:56 -0700
Subject: [PATCH 47/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 1fb0037971..1d4db99935 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -157,12 +157,37 @@ stages:
           demands: agent.os -equals Linux
         variables:
           Tag: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.Tag'] ] 
+          CommitHash: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.CommitHash'] ] 
         steps:
         - script: echo $(Tag)
           name: echovar1
+
         - script: echo $(myVarFromJobADebug)
           name: echovar
 
+        - bash: |
+            curl -o get-akse.sh https://raw.githubusercontent.com/Azure/aks-engine/master/scripts/get-akse.sh
+            chmod 700 get-akse.sh
+            ./get-akse.sh
+          name: AKSEngine
+          displayName: Install AKS-Engine
+
+        - bash: |
+            rm -f cniLinux.json*
+            wget https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json
+            export LINUX_CNI_URL='"'https://acnjenkinsstorage.blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-linux-amd64-$(Tag).tgz'"'
+            sed -i "s|\"azureCNIURLLinux\":\".*\"|\"azureCNIURLLinux\":"$LINUX_CNI_URL"|g" cniLinux.json
+            sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$GIT_TAG\"|g" cniLinux.json
+            echo "Running E2E tests against a cluster built with the following API model:"
+            cat ./cniLinux.json
+          name: ConfigureAKSEngine
+          displayName: Configure AKS-Engine
+
+        - bash: |
+            sudo docker run --rm -v /var/lib/jenkins/workspace/k8s-linux-github:/go/src/github.com/Azure/aks-engine -w /go/src/github.com/Azure/aks-engine -e CLUSTER_DEFINITION=./cniLinux.json -e ORCHESTRATOR=kubernetes -e CREATE_VNET=false -e TIMEOUT=10m -e CLIENT_ID=$(AKS_ENGINE_CLIENT_ID) -e CLIENT_SECRET=$(AKS_ENGINE_CLIENT_SECRET) -e TENANT_ID=$(AKS_ENGINE_TENANT_ID) -e SUBSCRIPTION_ID=$(AKS_ENGINE_SUBSCRIPTION_ID) -e CLEANUP_ON_EXIT=true -e REGIONS=$(AKS_ENGINE_REGION) -e IS_JENKINS=false quay.io/deis/go-dev:latest make test-kubernetes
+          name: DeployAKSEngine
+          displayName: Deploy AKS Engine
+
 
       - job: Cleanup
         displayName: Cleanup remote artifacts

From 9909c1ad9886024e4cb27df06341046eeb3cce22 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Tue, 24 Sep 2019 12:08:34 -0700
Subject: [PATCH 48/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 1d4db99935..561e3a7416 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -184,6 +184,8 @@ stages:
           displayName: Configure AKS-Engine
 
         - bash: |
+            git clone git clone https://github.com/Azure/aks-engine
+            cd aks-engine
             sudo docker run --rm -v /var/lib/jenkins/workspace/k8s-linux-github:/go/src/github.com/Azure/aks-engine -w /go/src/github.com/Azure/aks-engine -e CLUSTER_DEFINITION=./cniLinux.json -e ORCHESTRATOR=kubernetes -e CREATE_VNET=false -e TIMEOUT=10m -e CLIENT_ID=$(AKS_ENGINE_CLIENT_ID) -e CLIENT_SECRET=$(AKS_ENGINE_CLIENT_SECRET) -e TENANT_ID=$(AKS_ENGINE_TENANT_ID) -e SUBSCRIPTION_ID=$(AKS_ENGINE_SUBSCRIPTION_ID) -e CLEANUP_ON_EXIT=true -e REGIONS=$(AKS_ENGINE_REGION) -e IS_JENKINS=false quay.io/deis/go-dev:latest make test-kubernetes
           name: DeployAKSEngine
           displayName: Deploy AKS Engine
@@ -202,7 +204,7 @@ stages:
           CommitHash: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.CommitHash'] ] 
         container:
           image: containernetworking/pipeline-ci:1.0.4
-
+        condition: succeeded('Unit_Tests')
         steps:
           - task: AzureCLI@1
             inputs:

From f776505124d8486acc4945924d849536ba5e1a85 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Tue, 24 Sep 2019 12:15:49 -0700
Subject: [PATCH 49/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 561e3a7416..6da88085e0 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -184,7 +184,7 @@ stages:
           displayName: Configure AKS-Engine
 
         - bash: |
-            git clone git clone https://github.com/Azure/aks-engine
+            git clone https://github.com/Azure/aks-engine
             cd aks-engine
             sudo docker run --rm -v /var/lib/jenkins/workspace/k8s-linux-github:/go/src/github.com/Azure/aks-engine -w /go/src/github.com/Azure/aks-engine -e CLUSTER_DEFINITION=./cniLinux.json -e ORCHESTRATOR=kubernetes -e CREATE_VNET=false -e TIMEOUT=10m -e CLIENT_ID=$(AKS_ENGINE_CLIENT_ID) -e CLIENT_SECRET=$(AKS_ENGINE_CLIENT_SECRET) -e TENANT_ID=$(AKS_ENGINE_TENANT_ID) -e SUBSCRIPTION_ID=$(AKS_ENGINE_SUBSCRIPTION_ID) -e CLEANUP_ON_EXIT=true -e REGIONS=$(AKS_ENGINE_REGION) -e IS_JENKINS=false quay.io/deis/go-dev:latest make test-kubernetes
           name: DeployAKSEngine

From 7d8a98530e6eda137619250a30acdb2ab4f5d601 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Tue, 24 Sep 2019 12:43:34 -0700
Subject: [PATCH 50/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 6da88085e0..d095e39c44 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -186,7 +186,8 @@ stages:
         - bash: |
             git clone https://github.com/Azure/aks-engine
             cd aks-engine
-            sudo docker run --rm -v /var/lib/jenkins/workspace/k8s-linux-github:/go/src/github.com/Azure/aks-engine -w /go/src/github.com/Azure/aks-engine -e CLUSTER_DEFINITION=./cniLinux.json -e ORCHESTRATOR=kubernetes -e CREATE_VNET=false -e TIMEOUT=10m -e CLIENT_ID=$(AKS_ENGINE_CLIENT_ID) -e CLIENT_SECRET=$(AKS_ENGINE_CLIENT_SECRET) -e TENANT_ID=$(AKS_ENGINE_TENANT_ID) -e SUBSCRIPTION_ID=$(AKS_ENGINE_SUBSCRIPTION_ID) -e CLEANUP_ON_EXIT=true -e REGIONS=$(AKS_ENGINE_REGION) -e IS_JENKINS=false quay.io/deis/go-dev:latest make test-kubernetes
+            make bootstrap
+            CLUSTER_DEFINITION=./cniLinux.json ORCHESTRATOR=kubernetes CREATE_VNET=false TIMEOUT=10m CLIENT_ID=$(AKS_ENGINE_CLIENT_ID) CLIENT_SECRET=$(AKS_ENGINE_CLIENT_SECRET) TENANT_ID=$(AKS_ENGINE_TENANT_ID) SUBSCRIPTION_ID=$(AKS_ENGINE_SUBSCRIPTION_ID) CLEANUP_ON_EXIT=true REGIONS=$(AKS_ENGINE_REGION) IS_JENKINS=false make test-kubernetes
           name: DeployAKSEngine
           displayName: Deploy AKS Engine
 

From da8dcd9a52d3d30c6431603bbda2627d5cddb1fd Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Tue, 24 Sep 2019 12:56:07 -0700
Subject: [PATCH 51/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index d095e39c44..94ea386523 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -173,6 +173,8 @@ stages:
           displayName: Install AKS-Engine
 
         - bash: |
+            git clone https://github.com/Azure/aks-engine
+            cd aks-engine
             rm -f cniLinux.json*
             wget https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json
             export LINUX_CNI_URL='"'https://acnjenkinsstorage.blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-linux-amd64-$(Tag).tgz'"'
@@ -180,18 +182,11 @@ stages:
             sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$GIT_TAG\"|g" cniLinux.json
             echo "Running E2E tests against a cluster built with the following API model:"
             cat ./cniLinux.json
-          name: ConfigureAKSEngine
-          displayName: Configure AKS-Engine
-
-        - bash: |
-            git clone https://github.com/Azure/aks-engine
-            cd aks-engine
             make bootstrap
             CLUSTER_DEFINITION=./cniLinux.json ORCHESTRATOR=kubernetes CREATE_VNET=false TIMEOUT=10m CLIENT_ID=$(AKS_ENGINE_CLIENT_ID) CLIENT_SECRET=$(AKS_ENGINE_CLIENT_SECRET) TENANT_ID=$(AKS_ENGINE_TENANT_ID) SUBSCRIPTION_ID=$(AKS_ENGINE_SUBSCRIPTION_ID) CLEANUP_ON_EXIT=true REGIONS=$(AKS_ENGINE_REGION) IS_JENKINS=false make test-kubernetes
           name: DeployAKSEngine
           displayName: Deploy AKS Engine
 
-
       - job: Cleanup
         displayName: Cleanup remote artifacts
         dependsOn: 

From 8ea0c3d640e3866b94cb89a65d4f6ed596d342cf Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Tue, 24 Sep 2019 13:40:25 -0700
Subject: [PATCH 52/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 94ea386523..2bfa4a865b 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -155,6 +155,8 @@ stages:
         pool:
           name: Networking-ContainerNetworking
           demands: agent.os -equals Linux
+        container:
+          image: containernetworking/pipeline-ci:1.0.4
         variables:
           Tag: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.Tag'] ] 
           CommitHash: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.CommitHash'] ] 
@@ -195,11 +197,11 @@ stages:
         pool:
           name: Networking-ContainerNetworking
           demands: agent.os -equals Linux
+        container:
+          image: containernetworking/pipeline-ci:1.0.4
         variables:
           Tag: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.Tag'] ] 
           CommitHash: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.CommitHash'] ] 
-        container:
-          image: containernetworking/pipeline-ci:1.0.4
         condition: succeeded('Unit_Tests')
         steps:
           - task: AzureCLI@1

From a02fe30756a1a68cefa17d199fdbabec614a1201 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Tue, 24 Sep 2019 13:56:43 -0700
Subject: [PATCH 53/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 2bfa4a865b..ab650ed4e7 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -158,6 +158,8 @@ stages:
         container:
           image: containernetworking/pipeline-ci:1.0.4
         variables:
+          GOPATH: "$(System.DefaultWorkingDirectory)/gopath"
+          AKS_Engine: github.com/Azure/aks-engine
           Tag: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.Tag'] ] 
           CommitHash: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.CommitHash'] ] 
         steps:
@@ -175,8 +177,9 @@ stages:
           displayName: Install AKS-Engine
 
         - bash: |
-            git clone https://github.com/Azure/aks-engine
-            cd aks-engine
+            go get $(AKS_ENGINE)
+            cd $(GOPATH)/src/$(AKS_ENGINE)
+            go get ./...
             rm -f cniLinux.json*
             wget https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json
             export LINUX_CNI_URL='"'https://acnjenkinsstorage.blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-linux-amd64-$(Tag).tgz'"'

From ff531b5630f611eb9af72a10cb6f89c6a4aa9617 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Tue, 24 Sep 2019 14:17:49 -0700
Subject: [PATCH 54/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 65 +++++++++++++++++++++++------------------
 1 file changed, 36 insertions(+), 29 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index ab650ed4e7..744b87c5d9 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -159,38 +159,45 @@ stages:
           image: containernetworking/pipeline-ci:1.0.4
         variables:
           GOPATH: "$(System.DefaultWorkingDirectory)/gopath"
-          AKS_Engine: github.com/Azure/aks-engine
+          GOBIN: "$(GOPATH)/bin" # Go binaries path
+          AKS_Engine: "github.com/Azure/aks-engine"
+          modulePath: "$(GOPATH)/src/github.com/Azure/aks-engine"
           Tag: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.Tag'] ] 
           CommitHash: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.CommitHash'] ] 
+
         steps:
-        - script: echo $(Tag)
-          name: echovar1
-
-        - script: echo $(myVarFromJobADebug)
-          name: echovar
-
-        - bash: |
-            curl -o get-akse.sh https://raw.githubusercontent.com/Azure/aks-engine/master/scripts/get-akse.sh
-            chmod 700 get-akse.sh
-            ./get-akse.sh
-          name: AKSEngine
-          displayName: Install AKS-Engine
-
-        - bash: |
-            go get $(AKS_ENGINE)
-            cd $(GOPATH)/src/$(AKS_ENGINE)
-            go get ./...
-            rm -f cniLinux.json*
-            wget https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json
-            export LINUX_CNI_URL='"'https://acnjenkinsstorage.blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-linux-amd64-$(Tag).tgz'"'
-            sed -i "s|\"azureCNIURLLinux\":\".*\"|\"azureCNIURLLinux\":"$LINUX_CNI_URL"|g" cniLinux.json
-            sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$GIT_TAG\"|g" cniLinux.json
-            echo "Running E2E tests against a cluster built with the following API model:"
-            cat ./cniLinux.json
-            make bootstrap
-            CLUSTER_DEFINITION=./cniLinux.json ORCHESTRATOR=kubernetes CREATE_VNET=false TIMEOUT=10m CLIENT_ID=$(AKS_ENGINE_CLIENT_ID) CLIENT_SECRET=$(AKS_ENGINE_CLIENT_SECRET) TENANT_ID=$(AKS_ENGINE_TENANT_ID) SUBSCRIPTION_ID=$(AKS_ENGINE_SUBSCRIPTION_ID) CLEANUP_ON_EXIT=true REGIONS=$(AKS_ENGINE_REGION) IS_JENKINS=false make test-kubernetes
-          name: DeployAKSEngine
-          displayName: Deploy AKS Engine
+          - bash: |
+              curl -o get-akse.sh https://raw.githubusercontent.com/Azure/aks-engine/master/scripts/get-akse.sh
+              chmod 700 get-akse.sh
+              ./get-akse.sh
+            name: AKSEngine
+            displayName: Install AKS-Engine
+
+          - bash: |
+              go version
+              go env
+              mkdir -p '$(GOBIN)'
+              mkdir -p '$(GOPATH)/pkg'
+              mkdir -p '$(modulePath)'
+              echo '##vso[task.prependpath]$(GOBIN)'
+              echo '##vso[task.prependpath]$(GOROOT)/bin'
+            name: "GoEnv"
+            displayName: "Set up the Go environment"
+
+          - bash: |
+              go get $(AKS_ENGINE)
+              rm -f cniLinux.json*
+              wget https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json
+              export LINUX_CNI_URL='"'https://acnjenkinsstorage.blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-linux-amd64-$(Tag).tgz'"'
+              sed -i "s|\"azureCNIURLLinux\":\".*\"|\"azureCNIURLLinux\":"$LINUX_CNI_URL"|g" cniLinux.json
+              sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$GIT_TAG\"|g" cniLinux.json
+              echo "Running E2E tests against a cluster built with the following API model:"
+              cat ./cniLinux.json
+              make bootstrap
+              CLUSTER_DEFINITION=./cniLinux.json ORCHESTRATOR=kubernetes CREATE_VNET=false TIMEOUT=10m CLIENT_ID=$(AKS_ENGINE_CLIENT_ID) CLIENT_SECRET=$(AKS_ENGINE_CLIENT_SECRET) TENANT_ID=$(AKS_ENGINE_TENANT_ID) SUBSCRIPTION_ID=$(AKS_ENGINE_SUBSCRIPTION_ID) CLEANUP_ON_EXIT=true REGIONS=$(AKS_ENGINE_REGION) IS_JENKINS=false make test-kubernetes
+            name: DeployAKSEngine
+            displayName: Deploy AKS Engine
+            workingDirectory: "$(modulePath)"
 
       - job: Cleanup
         displayName: Cleanup remote artifacts

From e1e67e8d796fbd28e4e7267ccd232276d75ddd02 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Tue, 24 Sep 2019 14:26:30 -0700
Subject: [PATCH 55/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 744b87c5d9..1df5e6aa05 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -161,7 +161,7 @@ stages:
           GOPATH: "$(System.DefaultWorkingDirectory)/gopath"
           GOBIN: "$(GOPATH)/bin" # Go binaries path
           AKS_Engine: "github.com/Azure/aks-engine"
-          modulePath: "$(GOPATH)/src/github.com/Azure/aks-engine"
+          modulePath: "$(GOPATH)/src/$(AKS_Engine)"
           Tag: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.Tag'] ] 
           CommitHash: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.CommitHash'] ] 
 
@@ -185,7 +185,8 @@ stages:
             displayName: "Set up the Go environment"
 
           - bash: |
-              go get $(AKS_ENGINE)
+              pwd
+              go get -v $(AKS_ENGINE)
               rm -f cniLinux.json*
               wget https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json
               export LINUX_CNI_URL='"'https://acnjenkinsstorage.blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-linux-amd64-$(Tag).tgz'"'

From 1bd12e3d549549f2e039b8bcbde5a210d904bf2e Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Tue, 24 Sep 2019 14:34:12 -0700
Subject: [PATCH 56/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 1df5e6aa05..e56b39f8b8 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -185,15 +185,17 @@ stages:
             displayName: "Set up the Go environment"
 
           - bash: |
-              pwd
+              
               go get -v $(AKS_ENGINE)
               rm -f cniLinux.json*
               wget https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json
-              export LINUX_CNI_URL='"'https://acnjenkinsstorage.blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-linux-amd64-$(Tag).tgz'"'
+              export LINUX_CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-linux-amd64-$(Tag).tgz'"'
               sed -i "s|\"azureCNIURLLinux\":\".*\"|\"azureCNIURLLinux\":"$LINUX_CNI_URL"|g" cniLinux.json
               sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$GIT_TAG\"|g" cniLinux.json
               echo "Running E2E tests against a cluster built with the following API model:"
               cat ./cniLinux.json
+              pwd
+              ls
               make bootstrap
               CLUSTER_DEFINITION=./cniLinux.json ORCHESTRATOR=kubernetes CREATE_VNET=false TIMEOUT=10m CLIENT_ID=$(AKS_ENGINE_CLIENT_ID) CLIENT_SECRET=$(AKS_ENGINE_CLIENT_SECRET) TENANT_ID=$(AKS_ENGINE_TENANT_ID) SUBSCRIPTION_ID=$(AKS_ENGINE_SUBSCRIPTION_ID) CLEANUP_ON_EXIT=true REGIONS=$(AKS_ENGINE_REGION) IS_JENKINS=false make test-kubernetes
             name: DeployAKSEngine

From bd757a014c9623b38b76bb85531cb6ec4cede00e Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Tue, 24 Sep 2019 14:42:14 -0700
Subject: [PATCH 57/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index e56b39f8b8..935b24eaa7 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -176,6 +176,7 @@ stages:
           - bash: |
               go version
               go env
+              go get -v $(AKS_ENGINE)
               mkdir -p '$(GOBIN)'
               mkdir -p '$(GOPATH)/pkg'
               mkdir -p '$(modulePath)'
@@ -185,8 +186,6 @@ stages:
             displayName: "Set up the Go environment"
 
           - bash: |
-              
-              go get -v $(AKS_ENGINE)
               rm -f cniLinux.json*
               wget https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json
               export LINUX_CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-linux-amd64-$(Tag).tgz'"'

From 3e77ca8f27b34c8f48a73de6faa0276edbe6f9c1 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Tue, 24 Sep 2019 14:53:44 -0700
Subject: [PATCH 58/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 935b24eaa7..1dccb6d421 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -166,13 +166,6 @@ stages:
           CommitHash: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.CommitHash'] ] 
 
         steps:
-          - bash: |
-              curl -o get-akse.sh https://raw.githubusercontent.com/Azure/aks-engine/master/scripts/get-akse.sh
-              chmod 700 get-akse.sh
-              ./get-akse.sh
-            name: AKSEngine
-            displayName: Install AKS-Engine
-
           - bash: |
               go version
               go env
@@ -196,6 +189,7 @@ stages:
               pwd
               ls
               make bootstrap
+              make build
               CLUSTER_DEFINITION=./cniLinux.json ORCHESTRATOR=kubernetes CREATE_VNET=false TIMEOUT=10m CLIENT_ID=$(AKS_ENGINE_CLIENT_ID) CLIENT_SECRET=$(AKS_ENGINE_CLIENT_SECRET) TENANT_ID=$(AKS_ENGINE_TENANT_ID) SUBSCRIPTION_ID=$(AKS_ENGINE_SUBSCRIPTION_ID) CLEANUP_ON_EXIT=true REGIONS=$(AKS_ENGINE_REGION) IS_JENKINS=false make test-kubernetes
             name: DeployAKSEngine
             displayName: Deploy AKS Engine

From 8d5f8177eea11f4054b7c4115cdcd356f58b067a Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Tue, 24 Sep 2019 15:13:15 -0700
Subject: [PATCH 59/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 1dccb6d421..5cc1ad89a3 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -187,7 +187,8 @@ stages:
               echo "Running E2E tests against a cluster built with the following API model:"
               cat ./cniLinux.json
               pwd
-              ls
+              ls -lah
+              alias k=kubectl
               make bootstrap
               make build
               CLUSTER_DEFINITION=./cniLinux.json ORCHESTRATOR=kubernetes CREATE_VNET=false TIMEOUT=10m CLIENT_ID=$(AKS_ENGINE_CLIENT_ID) CLIENT_SECRET=$(AKS_ENGINE_CLIENT_SECRET) TENANT_ID=$(AKS_ENGINE_TENANT_ID) SUBSCRIPTION_ID=$(AKS_ENGINE_SUBSCRIPTION_ID) CLEANUP_ON_EXIT=true REGIONS=$(AKS_ENGINE_REGION) IS_JENKINS=false make test-kubernetes

From a2bff13553e88e629b352c8f8cc5b6a3c3e1e063 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Tue, 24 Sep 2019 16:47:58 -0700
Subject: [PATCH 60/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 5cc1ad89a3..4c90d47e87 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -188,6 +188,8 @@ stages:
               cat ./cniLinux.json
               pwd
               ls -lah
+              curl -L https://dl.k8s.io/v1.16.0/kubernetes-client-linux-amd64.tar.gz | tar xvzf -
+              sudo cp kubernetes/client/bin/kubectl /usr/local/bin/
               alias k=kubectl
               make bootstrap
               make build

From 78bdeb7695bd704b2c861db6f228245b24a7ceba Mon Sep 17 00:00:00 2001
From: Mathew Merrick <matmerr@users.noreply.github.com>
Date: Tue, 24 Sep 2019 17:02:01 -0700
Subject: [PATCH 61/83] Update pipeline.yml for Azure Pipelines

---
 .pipelines/pipeline.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 4c90d47e87..45e50e5576 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -189,8 +189,8 @@ stages:
               pwd
               ls -lah
               curl -L https://dl.k8s.io/v1.16.0/kubernetes-client-linux-amd64.tar.gz | tar xvzf -
-              sudo cp kubernetes/client/bin/kubectl /usr/local/bin/
-              alias k=kubectl
+              sudo cp kubernetes/client/bin/kubectl /usr/local/bin/kubectl
+              sudo cp kubernetes/client/bin/kubectl /usr/local/bin/k
               make bootstrap
               make build
               CLUSTER_DEFINITION=./cniLinux.json ORCHESTRATOR=kubernetes CREATE_VNET=false TIMEOUT=10m CLIENT_ID=$(AKS_ENGINE_CLIENT_ID) CLIENT_SECRET=$(AKS_ENGINE_CLIENT_SECRET) TENANT_ID=$(AKS_ENGINE_TENANT_ID) SUBSCRIPTION_ID=$(AKS_ENGINE_SUBSCRIPTION_ID) CLEANUP_ON_EXIT=true REGIONS=$(AKS_ENGINE_REGION) IS_JENKINS=false make test-kubernetes

From 9695477e3f7066742ddaf0d30d8cc626a6381cd5 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 25 Sep 2019 12:28:50 -0700
Subject: [PATCH 62/83] pipeline templating

---
 .pipelines/Dockerfile             |   7 +-
 .pipelines/e2e-job-template.yaml  |  24 ++++
 .pipelines/e2e-step-template.yaml |  49 +++++++
 .pipelines/pipeline.old.yml       | 221 ++++++++++++++++++++++++++++++
 .pipelines/pipeline.yml           |  67 ++-------
 5 files changed, 310 insertions(+), 58 deletions(-)
 create mode 100644 .pipelines/e2e-job-template.yaml
 create mode 100644 .pipelines/e2e-step-template.yaml
 create mode 100644 .pipelines/pipeline.old.yml

diff --git a/.pipelines/Dockerfile b/.pipelines/Dockerfile
index f515ab9a45..65815fc384 100644
--- a/.pipelines/Dockerfile
+++ b/.pipelines/Dockerfile
@@ -1,9 +1,10 @@
 FROM ubuntu:16.04
-RUN apt-get update && apt-get install -y software-properties-common sudo wget apt-transport-https curl
+RUN apt-get update && apt-get install -y software-properties-common sudo wget apt-transport-https curl lsb-release gnupg
 RUN wget -q https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb
 RUN sudo dpkg -i packages-microsoft-prod.deb
-RUN add-apt-repository ppa:longsleep/golang-backports && apt-get update
-RUN apt-get install -y git golang-go=2:1.13~1longsleep1+xenial iptables ipset iproute2 ebtables python-pip gcc zip dotnet-sdk-2.2
+RUN add-apt-repository ppa:longsleep/golang-backports 
+RUN curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+RUN apt-get install -y git golang-go=2:1.13~1longsleep1+xenial iptables ipset iproute2 ebtables python-pip gcc zip dotnet-sdk-2.2 azure-cli
 RUN sudo pip install coverage
 RUN if [ -f Gopkg.toml ]; then curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh ; fi 
 RUN go get github.com/docker/libnetwork/driverapi
diff --git a/.pipelines/e2e-job-template.yaml b/.pipelines/e2e-job-template.yaml
new file mode 100644
index 0000000000..cef93b0ed8
--- /dev/null
+++ b/.pipelines/e2e-job-template.yaml
@@ -0,0 +1,24 @@
+parameters:
+  name: ""
+  pipelineBuildImage: "containernetworking/pipeline-ci:1.0.4"
+  testClusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json"
+
+jobs:
+  - job: ${{ parameters.name }}
+    dependsOn: Unit_Tests
+    pool:
+      name: Networking-ContainerNetworking
+      demands: agent.os -equals Linux
+    container:
+      image: ${{ parameters.pipelineBuildImage }}
+    variables:
+      GOPATH: "$(System.DefaultWorkingDirectory)/gopath"
+      GOBIN: "$(GOPATH)/bin" # Go binaries path
+      modulePath: "$(GOPATH)/src/github.com/Azure/aks-engine"
+      Tag: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.Tag'] ]
+      CommitHash: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.CommitHash'] ]
+
+    steps:
+      - template: e2e-step-template.yaml
+        parameters:
+          testClusterDefinitionUrl: ${{ parameters.testClusterDefinitionUrl }}
diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
new file mode 100644
index 0000000000..380311ae34
--- /dev/null
+++ b/.pipelines/e2e-step-template.yaml
@@ -0,0 +1,49 @@
+parameters:
+  name: ""
+  testClusterDefinitionUrl: ""
+
+steps:
+  - bash: |
+      go version
+      go env
+      go get -v $(AKS_ENGINE)
+      mkdir -p '$(GOBIN)'
+      mkdir -p '$(GOPATH)/pkg'
+      mkdir -p '$(modulePath)'
+      echo '##vso[task.prependpath]$(GOBIN)'
+      echo '##vso[task.prependpath]$(GOROOT)/bin'
+    name: "GoEnv"
+    displayName: "Set up the Go environment"
+
+  - bash: |
+      rm -f clusterDefinition.json*
+      wget '${{ parameters.testClusterDefinitionUrl }}' -O clusterDefinition.json
+      export LINUX_CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-linux-amd64-$(Tag).tgz'"'
+      sed -i "s|\"azureCNIURLLinux\":\".*\"|\"azureCNIURLLinux\":"$LINUX_CNI_URL"|g" clusterDefinition.json
+      sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" clusterDefinition.json
+      echo "Running E2E tests against a cluster built with the following API model:"
+      cat ./clusterDefinition.json
+      curl -L https://dl.k8s.io/v1.16.0/kubernetes-client-linux-amd64.tar.gz | tar xvzf -
+      sudo cp kubernetes/client/bin/kubectl /usr/local/bin/kubectl
+      sudo cp kubernetes/client/bin/kubectl /usr/local/bin/k
+      make bootstrap
+      make build-binary
+    displayName: Build AKS-Engine
+    workingDirectory: "$(modulePath)"
+
+  - bash: |
+      export CLUSTER_DEFINITION=./clusterDefinition.json 
+      export ORCHESTRATOR=kubernetes 
+      export CREATE_VNET=false 
+      export TIMEOUT=10m 
+      export CLIENT_ID=$(AKS_ENGINE_CLIENT_ID) 
+      export CLIENT_SECRET=$(AKS_ENGINE_CLIENT_SECRET) 
+      export TENANT_ID=$(AKS_ENGINE_TENANT_ID) 
+      export SUBSCRIPTION_ID=$(AKS_ENGINE_SUBSCRIPTION_ID) 
+      export CLEANUP_ON_EXIT=true 
+      export REGIONS=$(AKS_ENGINE_REGION) 
+      export IS_JENKINS=false 
+      make test-kubernetes
+    name: DeployAKSEngine
+    displayName: Deploy AKS-Engine
+    workingDirectory: "$(modulePath)"
diff --git a/.pipelines/pipeline.old.yml b/.pipelines/pipeline.old.yml
new file mode 100644
index 0000000000..fd48c052c0
--- /dev/null
+++ b/.pipelines/pipeline.old.yml
@@ -0,0 +1,221 @@
+pr:
+  - master
+
+trigger:
+  branches:
+    include:
+      - master
+
+stages:
+  - stage: Tests
+    displayName: Build and Run Tests
+    jobs:
+      - job: Unit_Tests
+        displayName: Unit Tests
+        pool:
+          name: Networking-ContainerNetworking
+          demands: agent.os -equals Linux
+
+        container:
+          image: containernetworking/pipeline-ci:1.0.4
+          options: "--privileged"
+
+        # Go setup for the vmImage:
+        # https://github.com/Microsoft/azure-pipelines-image-generation/blob/master/images/linux/scripts/installers/go.sh
+        variables:
+          GOBIN: "$(GOPATH)/bin" # Go binaries path
+          GOPATH: "$(System.DefaultWorkingDirectory)/gopath" # Go workspace path
+          modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" # $(build.repository.name)' # Path to the module's code
+
+        steps:
+          - bash: |
+              echo $UID
+              sudo rm /run/docker/plugins/test.sock || true
+              sudo ip link del dev dummy || true
+            displayName: "Set up OS environment"
+
+          - bash: |
+              go version
+              go env
+              mkdir -p '$(GOBIN)'
+              mkdir -p '$(GOPATH)/pkg'
+              mkdir -p '$(modulePath)'
+              shopt -s extglob
+              shopt -s dotglob
+              mv !(gopath) '$(modulePath)'
+              echo '##vso[task.prependpath]$(GOBIN)'
+              echo '##vso[task.prependpath]$(GOROOT)/bin'
+            name: "GoEnv"
+            displayName: "Set up the Go environment"
+
+          - script: |
+              echo "##vso[task.setvariable variable=CommitHash;isOutput=true]$(git rev-parse HEAD)"
+              BRANCH=$(git rev-parse --abbrev-ref HEAD)
+              if [[ "$BRANCH" == "master" ]]; then
+                echo "##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --abbrev=0)"
+                echo "Set tag to $(git describe --tags --abbrev=0)"
+              else
+                echo "##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --always --dirty)"
+                echo "Set tag to $(git describe --tags --always --dirty)"
+              fi
+            workingDirectory: "$(modulePath)"
+            name: "EnvironmentalVariables"
+            displayName: "Set build environmental variables"
+            condition: always()
+
+          - script: |
+              go get -v -t -d ./...
+              if [ -f Gopkg.toml ]; then
+                  curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
+              fi
+              go get github.com/docker/libnetwork/driverapi
+              go get github.com/gorilla/mux
+              go get github.com/jstemmer/go-junit-report
+              go get github.com/axw/gocov/gocov
+              go get github.com/AlekSi/gocov-xml
+              go get -u gopkg.in/matm/v1/gocov-html
+            workingDirectory: "$(modulePath)"
+            name: "GoDependencies"
+            displayName: "Install Go dependencies"
+
+          - script: |
+              echo Build tag is $(EnvironmentalVariables.Tag)
+              export GOOS=linux
+              make all-binaries VERSION=$(EnvironmentalVariables.Tag)
+              export GOOS=windows
+              make all-binaries VERSION=$(EnvironmentalVariables.Tag)
+              cd output 
+              sudo find . -mindepth 2 -type f -regextype posix-extended ! -iregex '.*\.(zip|tgz)$' -delete
+              sudo find . -mindepth 2 -type f -print -exec mv {} . \;
+              sudo rm -R -- */
+            workingDirectory: "$(modulePath)"
+            name: "Build"
+            displayName: "Build"
+
+          - bash: |
+              # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout
+              { { { { 
+                    sudo -E env "PATH=$PATH" make test-all; 
+                    echo $? >&3; 
+                    } | tee >(go-junit-report > report.xml) >&4; 
+                  } 3>&1; 
+                } | { read xs; exit $xs; } 
+              } 4>&1
+            workingDirectory: "$(modulePath)"
+            failOnStderr: true
+            name: "Test"
+            displayName: "Run Tests"
+
+          - bash: |
+              bash <(curl -s https://codecov.io/bash)
+              gocov convert coverage.out > coverage.json
+              gocov-xml < coverage.json > coverage.xml
+            workingDirectory: "$(modulePath)"
+            name: "Coverage"
+            displayName: "Generate Coverage Reports"
+            condition: always()
+
+          - task: PublishTestResults@2
+            inputs:
+              testRunner: JUnit
+              testResultsFiles: $(System.DefaultWorkingDirectory)/**/report.xml
+            condition: always()
+
+          - task: PublishCodeCoverageResults@1
+            inputs:
+              codeCoverageTool: Cobertura
+              summaryFileLocation: $(System.DefaultWorkingDirectory)/**/coverage.xml
+            condition: always()
+
+          - task: CopyFiles@2
+            inputs:
+              sourceFolder: "$(modulePath)/output"
+              targetFolder: $(Build.ArtifactStagingDirectory)
+            condition: succeeded()
+
+          - task: PublishBuildArtifacts@1
+            inputs:
+              artifactName: "output"
+              pathtoPublish: "$(Build.ArtifactStagingDirectory)"
+            condition: succeeded()
+
+          - task: AzureCLI@1
+            inputs:
+              azureSubscription: $(ARTIFACT_SUBSCRIPTION)
+              scriptLocation: "inlineScript"
+              inlineScript: |
+                az storage container create -n acn-$(EnvironmentalVariables.CommitHash) --account-name $(STORAGE_ACCOUNT_NAME) --public-access container
+                az storage blob upload-batch -d acn-$(EnvironmentalVariables.CommitHash) -s ./output/  --account-name $(STORAGE_ACCOUNT_NAME)
+              workingDirectory: "$(modulePath)"
+            displayName: Create artifact storage container
+
+      - job: E2E_Tests
+        displayName: AKS-Engine E2E Tests
+        dependsOn: Unit_Tests
+        pool:
+          name: Networking-ContainerNetworking
+          demands: agent.os -equals Linux
+        container:
+          image: containernetworking/pipeline-ci:1.0.4
+        variables:
+          GOPATH: "$(System.DefaultWorkingDirectory)/gopath"
+          GOBIN: "$(GOPATH)/bin" # Go binaries path
+          AKS_Engine: "github.com/Azure/aks-engine"
+          modulePath: "$(GOPATH)/src/$(AKS_Engine)"
+          Tag: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.Tag'] ]
+          CommitHash: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.CommitHash'] ]
+
+        steps:
+          - bash: |
+              go version
+              go env
+              go get -v $(AKS_ENGINE)
+              mkdir -p '$(GOBIN)'
+              mkdir -p '$(GOPATH)/pkg'
+              mkdir -p '$(modulePath)'
+              echo '##vso[task.prependpath]$(GOBIN)'
+              echo '##vso[task.prependpath]$(GOROOT)/bin'
+            name: "GoEnv"
+            displayName: "Set up the Go environment"
+
+          - bash: |
+              rm -f cniLinux.json*
+              wget https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json
+              export LINUX_CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-linux-amd64-$(Tag).tgz'"'
+              sed -i "s|\"azureCNIURLLinux\":\".*\"|\"azureCNIURLLinux\":"$LINUX_CNI_URL"|g" cniLinux.json
+              sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$GIT_TAG\"|g" cniLinux.json
+              echo "Running E2E tests against a cluster built with the following API model:"
+              cat ./cniLinux.json
+              curl -L https://dl.k8s.io/v1.16.0/kubernetes-client-linux-amd64.tar.gz | tar xvzf -
+              sudo cp kubernetes/client/bin/kubectl /usr/local/bin/kubectl
+              sudo cp kubernetes/client/bin/kubectl /usr/local/bin/k
+              make bootstrap
+              make build-binary
+              CLUSTER_DEFINITION=./cniLinux.json ORCHESTRATOR=kubernetes CREATE_VNET=false TIMEOUT=10m CLIENT_ID=$(AKS_ENGINE_CLIENT_ID) CLIENT_SECRET=$(AKS_ENGINE_CLIENT_SECRET) TENANT_ID=$(AKS_ENGINE_TENANT_ID) SUBSCRIPTION_ID=$(AKS_ENGINE_SUBSCRIPTION_ID) CLEANUP_ON_EXIT=true REGIONS=$(AKS_ENGINE_REGION) IS_JENKINS=false make test-kubernetes
+            name: DeployAKSEngine
+            displayName: Deploy AKS Engine
+            workingDirectory: "$(modulePath)"
+
+      - job: Cleanup
+        displayName: Cleanup remote artifacts
+        dependsOn:
+          - Unit_Tests
+          - E2E_Tests
+        pool:
+          name: Networking-ContainerNetworking
+          demands: agent.os -equals Linux
+        container:
+          image: containernetworking/pipeline-ci:1.0.4
+        variables:
+          Tag: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.Tag'] ]
+          CommitHash: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.CommitHash'] ]
+        condition: succeeded('Unit_Tests')
+        steps:
+          - task: AzureCLI@1
+            inputs:
+              azureSubscription: $(ARTIFACT_SUBSCRIPTION)
+              scriptLocation: "inlineScript"
+              inlineScript: |
+                az storage container delete -n acn-$(CommitHash) --account-name $(STORAGE_ACCOUNT_NAME)
+              workingDirectory: "$(modulePath)"
+            displayName: Cleanup remote Azure storage container
diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 45e50e5576..17e36b2a8f 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -142,65 +142,22 @@ stages:
           - task: AzureCLI@1
             inputs:
               azureSubscription: $(ARTIFACT_SUBSCRIPTION)
-              scriptLocation: 'inlineScript'
+              scriptLocation: "inlineScript"
               inlineScript: |
                 az storage container create -n acn-$(EnvironmentalVariables.CommitHash) --account-name $(STORAGE_ACCOUNT_NAME) --public-access container
                 az storage blob upload-batch -d acn-$(EnvironmentalVariables.CommitHash) -s ./output/  --account-name $(STORAGE_ACCOUNT_NAME)
-              workingDirectory: '$(modulePath)'
+              workingDirectory: "$(modulePath)"
             displayName: Create artifact storage container
 
-      - job: E2E_Tests
-        displayName: AKS-Engine E2E Tests
-        dependsOn: Unit_Tests
-        pool:
-          name: Networking-ContainerNetworking
-          demands: agent.os -equals Linux
-        container:
-          image: containernetworking/pipeline-ci:1.0.4
-        variables:
-          GOPATH: "$(System.DefaultWorkingDirectory)/gopath"
-          GOBIN: "$(GOPATH)/bin" # Go binaries path
-          AKS_Engine: "github.com/Azure/aks-engine"
-          modulePath: "$(GOPATH)/src/$(AKS_Engine)"
-          Tag: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.Tag'] ] 
-          CommitHash: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.CommitHash'] ] 
-
-        steps:
-          - bash: |
-              go version
-              go env
-              go get -v $(AKS_ENGINE)
-              mkdir -p '$(GOBIN)'
-              mkdir -p '$(GOPATH)/pkg'
-              mkdir -p '$(modulePath)'
-              echo '##vso[task.prependpath]$(GOBIN)'
-              echo '##vso[task.prependpath]$(GOROOT)/bin'
-            name: "GoEnv"
-            displayName: "Set up the Go environment"
-
-          - bash: |
-              rm -f cniLinux.json*
-              wget https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json
-              export LINUX_CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-linux-amd64-$(Tag).tgz'"'
-              sed -i "s|\"azureCNIURLLinux\":\".*\"|\"azureCNIURLLinux\":"$LINUX_CNI_URL"|g" cniLinux.json
-              sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$GIT_TAG\"|g" cniLinux.json
-              echo "Running E2E tests against a cluster built with the following API model:"
-              cat ./cniLinux.json
-              pwd
-              ls -lah
-              curl -L https://dl.k8s.io/v1.16.0/kubernetes-client-linux-amd64.tar.gz | tar xvzf -
-              sudo cp kubernetes/client/bin/kubectl /usr/local/bin/kubectl
-              sudo cp kubernetes/client/bin/kubectl /usr/local/bin/k
-              make bootstrap
-              make build
-              CLUSTER_DEFINITION=./cniLinux.json ORCHESTRATOR=kubernetes CREATE_VNET=false TIMEOUT=10m CLIENT_ID=$(AKS_ENGINE_CLIENT_ID) CLIENT_SECRET=$(AKS_ENGINE_CLIENT_SECRET) TENANT_ID=$(AKS_ENGINE_TENANT_ID) SUBSCRIPTION_ID=$(AKS_ENGINE_SUBSCRIPTION_ID) CLEANUP_ON_EXIT=true REGIONS=$(AKS_ENGINE_REGION) IS_JENKINS=false make test-kubernetes
-            name: DeployAKSEngine
-            displayName: Deploy AKS Engine
-            workingDirectory: "$(modulePath)"
+      - template: e2e-job-template.yaml
+        parameters:
+          name: "ubuntu_16_04_linux_e2e"
+          pipelineBuildImage: "containernetworking/pipeline-ci:1.0.4"
+          testClusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json"
 
       - job: Cleanup
         displayName: Cleanup remote artifacts
-        dependsOn: 
+        dependsOn:
           - Unit_Tests
           - E2E_Tests
         pool:
@@ -209,15 +166,15 @@ stages:
         container:
           image: containernetworking/pipeline-ci:1.0.4
         variables:
-          Tag: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.Tag'] ] 
-          CommitHash: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.CommitHash'] ] 
+          Tag: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.Tag'] ]
+          CommitHash: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.CommitHash'] ]
         condition: succeeded('Unit_Tests')
         steps:
           - task: AzureCLI@1
             inputs:
               azureSubscription: $(ARTIFACT_SUBSCRIPTION)
-              scriptLocation: 'inlineScript'
+              scriptLocation: "inlineScript"
               inlineScript: |
                 az storage container delete -n acn-$(CommitHash) --account-name $(STORAGE_ACCOUNT_NAME)
-              workingDirectory: '$(modulePath)'
+              workingDirectory: "$(modulePath)"
             displayName: Cleanup remote Azure storage container

From 4f198dac4269453642241c971793a30ef0259c74 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 25 Sep 2019 12:34:07 -0700
Subject: [PATCH 63/83] remove cleanup depends on

---
 .pipelines/pipeline.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 17e36b2a8f..b5b5afb028 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -157,9 +157,6 @@ stages:
 
       - job: Cleanup
         displayName: Cleanup remote artifacts
-        dependsOn:
-          - Unit_Tests
-          - E2E_Tests
         pool:
           name: Networking-ContainerNetworking
           demands: agent.os -equals Linux

From 15e31c486a1e4bfd0c64d9d5b3afcacb632db95c Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 25 Sep 2019 12:45:37 -0700
Subject: [PATCH 64/83] aks-engine

---
 .pipelines/e2e-step-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index 380311ae34..b1d76c09d8 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -6,7 +6,7 @@ steps:
   - bash: |
       go version
       go env
-      go get -v $(AKS_ENGINE)
+      go get -v github.com/Azure/aks-engine
       mkdir -p '$(GOBIN)'
       mkdir -p '$(GOPATH)/pkg'
       mkdir -p '$(modulePath)'

From 7bb6fb17a595bb1f5eafb344f2507cbe8c04d307 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 25 Sep 2019 13:04:03 -0700
Subject: [PATCH 65/83] parallel

---
 .pipelines/e2e-job-template.yaml  |  6 +++---
 .pipelines/e2e-step-template.yaml |  1 +
 .pipelines/pipeline.yml           | 24 ++++++++++++++++++------
 3 files changed, 22 insertions(+), 9 deletions(-)

diff --git a/.pipelines/e2e-job-template.yaml b/.pipelines/e2e-job-template.yaml
index cef93b0ed8..8202d12b55 100644
--- a/.pipelines/e2e-job-template.yaml
+++ b/.pipelines/e2e-job-template.yaml
@@ -5,7 +5,7 @@ parameters:
 
 jobs:
   - job: ${{ parameters.name }}
-    dependsOn: Unit_Tests
+    dependsOn: unit_tests
     pool:
       name: Networking-ContainerNetworking
       demands: agent.os -equals Linux
@@ -15,8 +15,8 @@ jobs:
       GOPATH: "$(System.DefaultWorkingDirectory)/gopath"
       GOBIN: "$(GOPATH)/bin" # Go binaries path
       modulePath: "$(GOPATH)/src/github.com/Azure/aks-engine"
-      Tag: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.Tag'] ]
-      CommitHash: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.CommitHash'] ]
+      Tag: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.Tag'] ]
+      CommitHash: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.CommitHash'] ]
 
     steps:
       - template: e2e-step-template.yaml
diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index b1d76c09d8..b225465314 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -3,6 +3,7 @@ parameters:
   testClusterDefinitionUrl: ""
 
 steps:
+  - checkout: none
   - bash: |
       go version
       go env
diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index b5b5afb028..48b567f646 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -7,15 +7,17 @@ trigger:
       - master
 
 stages:
-  - stage: Tests
-    displayName: Build and Run Tests
+  - stage: ACN Tests
     jobs:
-      - job: Unit_Tests
+      - job: unit_tests
         displayName: Unit Tests
         pool:
           name: Networking-ContainerNetworking
           demands: agent.os -equals Linux
 
+        strategy:
+          parallel: 4
+
         container:
           image: containernetworking/pipeline-ci:1.0.4
           options: "--privileged"
@@ -155,6 +157,12 @@ stages:
           pipelineBuildImage: "containernetworking/pipeline-ci:1.0.4"
           testClusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json"
 
+      - template: e2e-job-template.yaml
+        parameters:
+          name: "windows_e2e"
+          pipelineBuildImage: "containernetworking/pipeline-ci:1.0.4"
+          testClusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniWindows.json"
+
       - job: Cleanup
         displayName: Cleanup remote artifacts
         pool:
@@ -163,9 +171,13 @@ stages:
         container:
           image: containernetworking/pipeline-ci:1.0.4
         variables:
-          Tag: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.Tag'] ]
-          CommitHash: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.CommitHash'] ]
-        condition: succeeded('Unit_Tests')
+          Tag: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.Tag'] ]
+          CommitHash: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.CommitHash'] ]
+        dependsOn:
+          - unit_tests
+          - ubuntu_16_04_linux_e2e
+          - windows_e2e
+        condition: succeeded('unit_tests')
         steps:
           - task: AzureCLI@1
             inputs:

From 44c7576a2cc3dc38dec7997669f0e980a0de92f1 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 25 Sep 2019 13:05:44 -0700
Subject: [PATCH 66/83] parallel 2

---
 .pipelines/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 48b567f646..2fabcbb9d8 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -7,7 +7,7 @@ trigger:
       - master
 
 stages:
-  - stage: ACN Tests
+  - stage: ACN_Tests
     jobs:
       - job: unit_tests
         displayName: Unit Tests

From 5ae9f87b1c42f211c0eb0dacce5c1d8de41de0a8 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 25 Sep 2019 14:12:41 -0700
Subject: [PATCH 67/83] pipeline 3

---
 .pipelines/pipeline.yml | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 2fabcbb9d8..cdb2a718c0 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -10,14 +10,10 @@ stages:
   - stage: ACN_Tests
     jobs:
       - job: unit_tests
-        displayName: Unit Tests
         pool:
           name: Networking-ContainerNetworking
           demands: agent.os -equals Linux
 
-        strategy:
-          parallel: 4
-
         container:
           image: containernetworking/pipeline-ci:1.0.4
           options: "--privileged"
@@ -163,7 +159,11 @@ stages:
           pipelineBuildImage: "containernetworking/pipeline-ci:1.0.4"
           testClusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniWindows.json"
 
-      - job: Cleanup
+stages:
+  - stage: Cleanup
+    dependsOn: ACN_Tests
+    jobs:
+      - job: Remote Artifacts
         displayName: Cleanup remote artifacts
         pool:
           name: Networking-ContainerNetworking
@@ -173,10 +173,6 @@ stages:
         variables:
           Tag: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.Tag'] ]
           CommitHash: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.CommitHash'] ]
-        dependsOn:
-          - unit_tests
-          - ubuntu_16_04_linux_e2e
-          - windows_e2e
         condition: succeeded('unit_tests')
         steps:
           - task: AzureCLI@1

From 6381dcbe9a1d23ddbc7b84628b5a9905ff3710b1 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 25 Sep 2019 14:13:37 -0700
Subject: [PATCH 68/83] pipeline 3

---
 .pipelines/pipeline.yml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index cdb2a718c0..148e1eb88a 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -159,7 +159,6 @@ stages:
           pipelineBuildImage: "containernetworking/pipeline-ci:1.0.4"
           testClusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniWindows.json"
 
-stages:
   - stage: Cleanup
     dependsOn: ACN_Tests
     jobs:

From f12796aeb2f66c5a165ced0d5564c673af6f53d2 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 25 Sep 2019 14:14:28 -0700
Subject: [PATCH 69/83] pipeline 4

---
 .pipelines/pipeline.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yml
index 148e1eb88a..b47ed4abee 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yml
@@ -162,7 +162,7 @@ stages:
   - stage: Cleanup
     dependsOn: ACN_Tests
     jobs:
-      - job: Remote Artifacts
+      - job: RemoteArtifacts
         displayName: Cleanup remote artifacts
         pool:
           name: Networking-ContainerNetworking

From b26b81b55e594801f31ff80e8ea772f47214c8ed Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 25 Sep 2019 14:54:59 -0700
Subject: [PATCH 70/83] pipeline 5

---
 .pipelines/{pipeline.yml => pipeline.yaml} | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
 rename .pipelines/{pipeline.yml => pipeline.yaml} (99%)

diff --git a/.pipelines/pipeline.yml b/.pipelines/pipeline.yaml
similarity index 99%
rename from .pipelines/pipeline.yml
rename to .pipelines/pipeline.yaml
index b47ed4abee..c832fb587e 100644
--- a/.pipelines/pipeline.yml
+++ b/.pipelines/pipeline.yaml
@@ -163,7 +163,7 @@ stages:
     dependsOn: ACN_Tests
     jobs:
       - job: RemoteArtifacts
-        displayName: Cleanup remote artifacts
+        displayName: Delete remote artifacts
         pool:
           name: Networking-ContainerNetworking
           demands: agent.os -equals Linux

From 5c829d37d8108eff2d4eb07bb94c396d72964451 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 25 Sep 2019 15:58:05 -0700
Subject: [PATCH 71/83] pipeline 6

---
 .pipelines/e2e-job-template.yaml  | 6 ++++--
 .pipelines/e2e-step-template.yaml | 9 +++++----
 .pipelines/pipeline.yaml          | 6 ++++--
 3 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/.pipelines/e2e-job-template.yaml b/.pipelines/e2e-job-template.yaml
index 8202d12b55..47071b3343 100644
--- a/.pipelines/e2e-job-template.yaml
+++ b/.pipelines/e2e-job-template.yaml
@@ -1,7 +1,8 @@
 parameters:
   name: ""
   pipelineBuildImage: "containernetworking/pipeline-ci:1.0.4"
-  testClusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json"
+  clusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json"
+  clusterDefinitionCniType: ""
 
 jobs:
   - job: ${{ parameters.name }}
@@ -21,4 +22,5 @@ jobs:
     steps:
       - template: e2e-step-template.yaml
         parameters:
-          testClusterDefinitionUrl: ${{ parameters.testClusterDefinitionUrl }}
+          clusterDefinitionUrl: ${{ parameters.testClusterDefinitionUrl }}
+          clusterDefinitionCniType: ${{ parameters.cniUrlConfigKeyName }}
diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index b225465314..07c2f6538b 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -1,6 +1,7 @@
 parameters:
   name: ""
-  testClusterDefinitionUrl: ""
+  clusterDefinitionUrl: ""
+  clusterDefinitionCniType: ""
 
 steps:
   - checkout: none
@@ -18,9 +19,9 @@ steps:
 
   - bash: |
       rm -f clusterDefinition.json*
-      wget '${{ parameters.testClusterDefinitionUrl }}' -O clusterDefinition.json
-      export LINUX_CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-linux-amd64-$(Tag).tgz'"'
-      sed -i "s|\"azureCNIURLLinux\":\".*\"|\"azureCNIURLLinux\":"$LINUX_CNI_URL"|g" clusterDefinition.json
+      wget '${{ parameters.clusterDefinitionUrl }}' -O clusterDefinition.json
+      export CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-linux-amd64-$(Tag).tgz'"'
+      sed -i "s|\"${{ parameters.clusterDefinitionCniType }}\":\".*\"|\"${{ parameters.clusterDefinitionCniType }}\":"$CNI_URL"|g" clusterDefinition.json
       sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" clusterDefinition.json
       echo "Running E2E tests against a cluster built with the following API model:"
       cat ./clusterDefinition.json
diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
index c832fb587e..0a9c55d16a 100644
--- a/.pipelines/pipeline.yaml
+++ b/.pipelines/pipeline.yaml
@@ -151,13 +151,15 @@ stages:
         parameters:
           name: "ubuntu_16_04_linux_e2e"
           pipelineBuildImage: "containernetworking/pipeline-ci:1.0.4"
-          testClusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json"
+          clusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json"
+          clusterDefinitionCniType: "azureCNIURLLinux"
 
       - template: e2e-job-template.yaml
         parameters:
           name: "windows_e2e"
           pipelineBuildImage: "containernetworking/pipeline-ci:1.0.4"
-          testClusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniWindows.json"
+          clusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniWindows.json"
+          clusterDefinitionCniType: "azureCNIURLWindows"
 
   - stage: Cleanup
     dependsOn: ACN_Tests

From 6396d7b2130ffd1f63ec4b77b95d423a3db35ac0 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 25 Sep 2019 16:19:17 -0700
Subject: [PATCH 72/83] pipeline 7

---
 .pipelines/e2e-step-template.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index 07c2f6538b..5bd13f5e56 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -21,7 +21,8 @@ steps:
       rm -f clusterDefinition.json*
       wget '${{ parameters.clusterDefinitionUrl }}' -O clusterDefinition.json
       export CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-linux-amd64-$(Tag).tgz'"'
-      sed -i "s|\"${{ parameters.clusterDefinitionCniType }}\":\".*\"|\"${{ parameters.clusterDefinitionCniType }}\":"$CNI_URL"|g" clusterDefinition.json
+      export CNI_TYPE=${{ parameters.clusterDefinitionCniType }}
+      sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":\"$CNI_URL\"|g" clusterDefinition.json
       sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" clusterDefinition.json
       echo "Running E2E tests against a cluster built with the following API model:"
       cat ./clusterDefinition.json

From f1cf96150279f6ce790cb1e6c51a9e0345e739b1 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 25 Sep 2019 16:30:31 -0700
Subject: [PATCH 73/83] pipeline 8

---
 .pipelines/e2e-step-template.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index 5bd13f5e56..c5af75c0c4 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -22,6 +22,7 @@ steps:
       wget '${{ parameters.clusterDefinitionUrl }}' -O clusterDefinition.json
       export CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-linux-amd64-$(Tag).tgz'"'
       export CNI_TYPE=${{ parameters.clusterDefinitionCniType }}
+      echo CNI type is $CNI_TYPE
       sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":\"$CNI_URL\"|g" clusterDefinition.json
       sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" clusterDefinition.json
       echo "Running E2E tests against a cluster built with the following API model:"

From 5b0d56578c0ea11bc5efc37100b21ef3cc506fc6 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 25 Sep 2019 16:57:02 -0700
Subject: [PATCH 74/83] pipeline 9

---
 .pipelines/e2e-job-template.yaml  | 8 ++++++--
 .pipelines/e2e-step-template.yaml | 8 +++++---
 .pipelines/pipeline.yaml          | 4 ++++
 3 files changed, 15 insertions(+), 5 deletions(-)

diff --git a/.pipelines/e2e-job-template.yaml b/.pipelines/e2e-job-template.yaml
index 47071b3343..1e487e6c73 100644
--- a/.pipelines/e2e-job-template.yaml
+++ b/.pipelines/e2e-job-template.yaml
@@ -2,7 +2,9 @@ parameters:
   name: ""
   pipelineBuildImage: "containernetworking/pipeline-ci:1.0.4"
   clusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json"
-  clusterDefinitionCniType: ""
+  clusterDefinitionCniTypeKey: ""
+  clusterDefinitionCniBuildOS: ""
+  clusterDefinitionCniBuildExt: ""
 
 jobs:
   - job: ${{ parameters.name }}
@@ -23,4 +25,6 @@ jobs:
       - template: e2e-step-template.yaml
         parameters:
           clusterDefinitionUrl: ${{ parameters.testClusterDefinitionUrl }}
-          clusterDefinitionCniType: ${{ parameters.cniUrlConfigKeyName }}
+          clusterDefinitionCniTypeKey: ${{ parameters.clusterDefinitionCniTypeKey }}
+          clusterDefinitionCniBuildOS: ${{ parameters.clusterDefinitionCniBuildOS }}
+          clusterDefinitionCniBuildExt: ${{ parameters.clusterDefinitionCniBuildExt }}
diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index c5af75c0c4..8ba7ad1ad4 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -1,7 +1,9 @@
 parameters:
   name: ""
   clusterDefinitionUrl: ""
-  clusterDefinitionCniType: ""
+  clusterDefinitionCniTypeKey: ""
+  clusterDefinitionCniBuildOS: ""
+  clusterDefinitionCniBuildExt: ""
 
 steps:
   - checkout: none
@@ -20,8 +22,8 @@ steps:
   - bash: |
       rm -f clusterDefinition.json*
       wget '${{ parameters.clusterDefinitionUrl }}' -O clusterDefinition.json
-      export CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-linux-amd64-$(Tag).tgz'"'
-      export CNI_TYPE=${{ parameters.clusterDefinitionCniType }}
+      export CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-${{ parameters.clusterDefinitionCniBuildOS }}-amd64-$(Tag).${{ parameters.clusterDefinitionCniBuildExt }}'"'
+      export CNI_TYPE=${{ parameters.clusterDefinitionCniTypeKey }}
       echo CNI type is $CNI_TYPE
       sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":\"$CNI_URL\"|g" clusterDefinition.json
       sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" clusterDefinition.json
diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
index 0a9c55d16a..0745425b17 100644
--- a/.pipelines/pipeline.yaml
+++ b/.pipelines/pipeline.yaml
@@ -153,6 +153,8 @@ stages:
           pipelineBuildImage: "containernetworking/pipeline-ci:1.0.4"
           clusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json"
           clusterDefinitionCniType: "azureCNIURLLinux"
+          clusterDefinitionCniBuildOS: "linux"
+          clusterDefinitionCniBuildExt: ".tgz"
 
       - template: e2e-job-template.yaml
         parameters:
@@ -160,6 +162,8 @@ stages:
           pipelineBuildImage: "containernetworking/pipeline-ci:1.0.4"
           clusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniWindows.json"
           clusterDefinitionCniType: "azureCNIURLWindows"
+          clusterDefinitionCniBuildOS: "windows"
+          clusterDefinitionCniBuildExt: ".zip"
 
   - stage: Cleanup
     dependsOn: ACN_Tests

From 007688158ebe4c892754a096d477067c5554d227 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 25 Sep 2019 17:20:14 -0700
Subject: [PATCH 75/83] pipeline 10

---
 .pipelines/e2e-step-template.yaml | 2 +-
 .pipelines/pipeline.yaml          | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index 8ba7ad1ad4..59294fe4a5 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -22,7 +22,7 @@ steps:
   - bash: |
       rm -f clusterDefinition.json*
       wget '${{ parameters.clusterDefinitionUrl }}' -O clusterDefinition.json
-      export CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-${{ parameters.clusterDefinitionCniBuildOS }}-amd64-$(Tag).${{ parameters.clusterDefinitionCniBuildExt }}'"'
+      export CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-${{ parameters.clusterDefinitionCniBuildOS }}-amd64-$(Tag)${{ parameters.clusterDefinitionCniBuildExt }}'"'
       export CNI_TYPE=${{ parameters.clusterDefinitionCniTypeKey }}
       echo CNI type is $CNI_TYPE
       sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":\"$CNI_URL\"|g" clusterDefinition.json
diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
index 0745425b17..1259546c25 100644
--- a/.pipelines/pipeline.yaml
+++ b/.pipelines/pipeline.yaml
@@ -152,7 +152,7 @@ stages:
           name: "ubuntu_16_04_linux_e2e"
           pipelineBuildImage: "containernetworking/pipeline-ci:1.0.4"
           clusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json"
-          clusterDefinitionCniType: "azureCNIURLLinux"
+          clusterDefinitionCniTypeKey: "azureCNIURLLinux"
           clusterDefinitionCniBuildOS: "linux"
           clusterDefinitionCniBuildExt: ".tgz"
 
@@ -161,7 +161,7 @@ stages:
           name: "windows_e2e"
           pipelineBuildImage: "containernetworking/pipeline-ci:1.0.4"
           clusterDefinitionUrl: "https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniWindows.json"
-          clusterDefinitionCniType: "azureCNIURLWindows"
+          clusterDefinitionCniTypeKey: "azureCNIURLWindows"
           clusterDefinitionCniBuildOS: "windows"
           clusterDefinitionCniBuildExt: ".zip"
 

From c80de537e818d6e8d1985f2353a0a948718c9f05 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 25 Sep 2019 17:29:59 -0700
Subject: [PATCH 76/83] pipeline 11

---
 .pipelines/e2e-job-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/e2e-job-template.yaml b/.pipelines/e2e-job-template.yaml
index 1e487e6c73..10edd32538 100644
--- a/.pipelines/e2e-job-template.yaml
+++ b/.pipelines/e2e-job-template.yaml
@@ -24,7 +24,7 @@ jobs:
     steps:
       - template: e2e-step-template.yaml
         parameters:
-          clusterDefinitionUrl: ${{ parameters.testClusterDefinitionUrl }}
+          clusterDefinitionUrl: ${{ parameters.clusterDefinitionUrl }}
           clusterDefinitionCniTypeKey: ${{ parameters.clusterDefinitionCniTypeKey }}
           clusterDefinitionCniBuildOS: ${{ parameters.clusterDefinitionCniBuildOS }}
           clusterDefinitionCniBuildExt: ${{ parameters.clusterDefinitionCniBuildExt }}

From bc49121e866d7a8e95c86d6b5196127a2050eceb Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 25 Sep 2019 18:12:01 -0700
Subject: [PATCH 77/83] pipeline 12

---
 .pipelines/e2e-step-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index 59294fe4a5..4b109901a0 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -25,7 +25,7 @@ steps:
       export CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-${{ parameters.clusterDefinitionCniBuildOS }}-amd64-$(Tag)${{ parameters.clusterDefinitionCniBuildExt }}'"'
       export CNI_TYPE=${{ parameters.clusterDefinitionCniTypeKey }}
       echo CNI type is $CNI_TYPE
-      sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":\"$CNI_URL\"|g" clusterDefinition.json
+      sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":$CNI_URL|g" clusterDefinition.json
       sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" clusterDefinition.json
       echo "Running E2E tests against a cluster built with the following API model:"
       cat ./clusterDefinition.json

From a615de3d38e1f704b07a387f0d8b3aa3a3031fc3 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 25 Sep 2019 19:25:14 -0700
Subject: [PATCH 78/83] pipeline 13

---
 .pipelines/pipeline.yaml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
index 1259546c25..6da859de87 100644
--- a/.pipelines/pipeline.yaml
+++ b/.pipelines/pipeline.yaml
@@ -165,11 +165,12 @@ stages:
           clusterDefinitionCniBuildOS: "windows"
           clusterDefinitionCniBuildExt: ".zip"
 
-  - stage: Cleanup
-    dependsOn: ACN_Tests
-    jobs:
       - job: RemoteArtifacts
         displayName: Delete remote artifacts
+        dependsOn:
+          - unit_tests
+          - ubuntu_16_04_linux_e2e
+          - windows_e2e
         pool:
           name: Networking-ContainerNetworking
           demands: agent.os -equals Linux

From be9ae8b83b5edc6b6dd123138ca58a26aef8301d Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Thu, 26 Sep 2019 11:12:53 -0700
Subject: [PATCH 79/83] pipeline 14

---
 .pipelines/pipeline.old.yml | 221 ------------------------------------
 .pipelines/pipeline.yaml    |   1 +
 2 files changed, 1 insertion(+), 221 deletions(-)
 delete mode 100644 .pipelines/pipeline.old.yml

diff --git a/.pipelines/pipeline.old.yml b/.pipelines/pipeline.old.yml
deleted file mode 100644
index fd48c052c0..0000000000
--- a/.pipelines/pipeline.old.yml
+++ /dev/null
@@ -1,221 +0,0 @@
-pr:
-  - master
-
-trigger:
-  branches:
-    include:
-      - master
-
-stages:
-  - stage: Tests
-    displayName: Build and Run Tests
-    jobs:
-      - job: Unit_Tests
-        displayName: Unit Tests
-        pool:
-          name: Networking-ContainerNetworking
-          demands: agent.os -equals Linux
-
-        container:
-          image: containernetworking/pipeline-ci:1.0.4
-          options: "--privileged"
-
-        # Go setup for the vmImage:
-        # https://github.com/Microsoft/azure-pipelines-image-generation/blob/master/images/linux/scripts/installers/go.sh
-        variables:
-          GOBIN: "$(GOPATH)/bin" # Go binaries path
-          GOPATH: "$(System.DefaultWorkingDirectory)/gopath" # Go workspace path
-          modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking" # $(build.repository.name)' # Path to the module's code
-
-        steps:
-          - bash: |
-              echo $UID
-              sudo rm /run/docker/plugins/test.sock || true
-              sudo ip link del dev dummy || true
-            displayName: "Set up OS environment"
-
-          - bash: |
-              go version
-              go env
-              mkdir -p '$(GOBIN)'
-              mkdir -p '$(GOPATH)/pkg'
-              mkdir -p '$(modulePath)'
-              shopt -s extglob
-              shopt -s dotglob
-              mv !(gopath) '$(modulePath)'
-              echo '##vso[task.prependpath]$(GOBIN)'
-              echo '##vso[task.prependpath]$(GOROOT)/bin'
-            name: "GoEnv"
-            displayName: "Set up the Go environment"
-
-          - script: |
-              echo "##vso[task.setvariable variable=CommitHash;isOutput=true]$(git rev-parse HEAD)"
-              BRANCH=$(git rev-parse --abbrev-ref HEAD)
-              if [[ "$BRANCH" == "master" ]]; then
-                echo "##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --abbrev=0)"
-                echo "Set tag to $(git describe --tags --abbrev=0)"
-              else
-                echo "##vso[task.setvariable variable=Tag;isOutput=true]$(git describe --tags --always --dirty)"
-                echo "Set tag to $(git describe --tags --always --dirty)"
-              fi
-            workingDirectory: "$(modulePath)"
-            name: "EnvironmentalVariables"
-            displayName: "Set build environmental variables"
-            condition: always()
-
-          - script: |
-              go get -v -t -d ./...
-              if [ -f Gopkg.toml ]; then
-                  curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-              fi
-              go get github.com/docker/libnetwork/driverapi
-              go get github.com/gorilla/mux
-              go get github.com/jstemmer/go-junit-report
-              go get github.com/axw/gocov/gocov
-              go get github.com/AlekSi/gocov-xml
-              go get -u gopkg.in/matm/v1/gocov-html
-            workingDirectory: "$(modulePath)"
-            name: "GoDependencies"
-            displayName: "Install Go dependencies"
-
-          - script: |
-              echo Build tag is $(EnvironmentalVariables.Tag)
-              export GOOS=linux
-              make all-binaries VERSION=$(EnvironmentalVariables.Tag)
-              export GOOS=windows
-              make all-binaries VERSION=$(EnvironmentalVariables.Tag)
-              cd output 
-              sudo find . -mindepth 2 -type f -regextype posix-extended ! -iregex '.*\.(zip|tgz)$' -delete
-              sudo find . -mindepth 2 -type f -print -exec mv {} . \;
-              sudo rm -R -- */
-            workingDirectory: "$(modulePath)"
-            name: "Build"
-            displayName: "Build"
-
-          - bash: |
-              # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout
-              { { { { 
-                    sudo -E env "PATH=$PATH" make test-all; 
-                    echo $? >&3; 
-                    } | tee >(go-junit-report > report.xml) >&4; 
-                  } 3>&1; 
-                } | { read xs; exit $xs; } 
-              } 4>&1
-            workingDirectory: "$(modulePath)"
-            failOnStderr: true
-            name: "Test"
-            displayName: "Run Tests"
-
-          - bash: |
-              bash <(curl -s https://codecov.io/bash)
-              gocov convert coverage.out > coverage.json
-              gocov-xml < coverage.json > coverage.xml
-            workingDirectory: "$(modulePath)"
-            name: "Coverage"
-            displayName: "Generate Coverage Reports"
-            condition: always()
-
-          - task: PublishTestResults@2
-            inputs:
-              testRunner: JUnit
-              testResultsFiles: $(System.DefaultWorkingDirectory)/**/report.xml
-            condition: always()
-
-          - task: PublishCodeCoverageResults@1
-            inputs:
-              codeCoverageTool: Cobertura
-              summaryFileLocation: $(System.DefaultWorkingDirectory)/**/coverage.xml
-            condition: always()
-
-          - task: CopyFiles@2
-            inputs:
-              sourceFolder: "$(modulePath)/output"
-              targetFolder: $(Build.ArtifactStagingDirectory)
-            condition: succeeded()
-
-          - task: PublishBuildArtifacts@1
-            inputs:
-              artifactName: "output"
-              pathtoPublish: "$(Build.ArtifactStagingDirectory)"
-            condition: succeeded()
-
-          - task: AzureCLI@1
-            inputs:
-              azureSubscription: $(ARTIFACT_SUBSCRIPTION)
-              scriptLocation: "inlineScript"
-              inlineScript: |
-                az storage container create -n acn-$(EnvironmentalVariables.CommitHash) --account-name $(STORAGE_ACCOUNT_NAME) --public-access container
-                az storage blob upload-batch -d acn-$(EnvironmentalVariables.CommitHash) -s ./output/  --account-name $(STORAGE_ACCOUNT_NAME)
-              workingDirectory: "$(modulePath)"
-            displayName: Create artifact storage container
-
-      - job: E2E_Tests
-        displayName: AKS-Engine E2E Tests
-        dependsOn: Unit_Tests
-        pool:
-          name: Networking-ContainerNetworking
-          demands: agent.os -equals Linux
-        container:
-          image: containernetworking/pipeline-ci:1.0.4
-        variables:
-          GOPATH: "$(System.DefaultWorkingDirectory)/gopath"
-          GOBIN: "$(GOPATH)/bin" # Go binaries path
-          AKS_Engine: "github.com/Azure/aks-engine"
-          modulePath: "$(GOPATH)/src/$(AKS_Engine)"
-          Tag: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.Tag'] ]
-          CommitHash: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.CommitHash'] ]
-
-        steps:
-          - bash: |
-              go version
-              go env
-              go get -v $(AKS_ENGINE)
-              mkdir -p '$(GOBIN)'
-              mkdir -p '$(GOPATH)/pkg'
-              mkdir -p '$(modulePath)'
-              echo '##vso[task.prependpath]$(GOBIN)'
-              echo '##vso[task.prependpath]$(GOROOT)/bin'
-            name: "GoEnv"
-            displayName: "Set up the Go environment"
-
-          - bash: |
-              rm -f cniLinux.json*
-              wget https://raw.githubusercontent.com/Azure/azure-container-networking/master/test/e2e/kubernetes/cniLinux.json
-              export LINUX_CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-linux-amd64-$(Tag).tgz'"'
-              sed -i "s|\"azureCNIURLLinux\":\".*\"|\"azureCNIURLLinux\":"$LINUX_CNI_URL"|g" cniLinux.json
-              sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$GIT_TAG\"|g" cniLinux.json
-              echo "Running E2E tests against a cluster built with the following API model:"
-              cat ./cniLinux.json
-              curl -L https://dl.k8s.io/v1.16.0/kubernetes-client-linux-amd64.tar.gz | tar xvzf -
-              sudo cp kubernetes/client/bin/kubectl /usr/local/bin/kubectl
-              sudo cp kubernetes/client/bin/kubectl /usr/local/bin/k
-              make bootstrap
-              make build-binary
-              CLUSTER_DEFINITION=./cniLinux.json ORCHESTRATOR=kubernetes CREATE_VNET=false TIMEOUT=10m CLIENT_ID=$(AKS_ENGINE_CLIENT_ID) CLIENT_SECRET=$(AKS_ENGINE_CLIENT_SECRET) TENANT_ID=$(AKS_ENGINE_TENANT_ID) SUBSCRIPTION_ID=$(AKS_ENGINE_SUBSCRIPTION_ID) CLEANUP_ON_EXIT=true REGIONS=$(AKS_ENGINE_REGION) IS_JENKINS=false make test-kubernetes
-            name: DeployAKSEngine
-            displayName: Deploy AKS Engine
-            workingDirectory: "$(modulePath)"
-
-      - job: Cleanup
-        displayName: Cleanup remote artifacts
-        dependsOn:
-          - Unit_Tests
-          - E2E_Tests
-        pool:
-          name: Networking-ContainerNetworking
-          demands: agent.os -equals Linux
-        container:
-          image: containernetworking/pipeline-ci:1.0.4
-        variables:
-          Tag: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.Tag'] ]
-          CommitHash: $[ dependencies.Unit_Tests.outputs['EnvironmentalVariables.CommitHash'] ]
-        condition: succeeded('Unit_Tests')
-        steps:
-          - task: AzureCLI@1
-            inputs:
-              azureSubscription: $(ARTIFACT_SUBSCRIPTION)
-              scriptLocation: "inlineScript"
-              inlineScript: |
-                az storage container delete -n acn-$(CommitHash) --account-name $(STORAGE_ACCOUNT_NAME)
-              workingDirectory: "$(modulePath)"
-            displayName: Cleanup remote Azure storage container
diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
index 6da859de87..807e1c293a 100644
--- a/.pipelines/pipeline.yaml
+++ b/.pipelines/pipeline.yaml
@@ -181,6 +181,7 @@ stages:
           CommitHash: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.CommitHash'] ]
         condition: succeeded('unit_tests')
         steps:
+          - checkout: none
           - task: AzureCLI@1
             inputs:
               azureSubscription: $(ARTIFACT_SUBSCRIPTION)

From 407afe8e900f3e70d1c915db86757da111309ee2 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Thu, 26 Sep 2019 11:18:37 -0700
Subject: [PATCH 80/83] pipeline 15

---
 .pipelines/pipeline.yaml | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
index 807e1c293a..0494f951e2 100644
--- a/.pipelines/pipeline.yaml
+++ b/.pipelines/pipeline.yaml
@@ -7,7 +7,7 @@ trigger:
       - master
 
 stages:
-  - stage: ACN_Tests
+  - stage: build_and_test
     jobs:
       - job: unit_tests
         pool:
@@ -165,12 +165,9 @@ stages:
           clusterDefinitionCniBuildOS: "windows"
           clusterDefinitionCniBuildExt: ".zip"
 
-      - job: RemoteArtifacts
-        displayName: Delete remote artifacts
-        dependsOn:
-          - unit_tests
-          - ubuntu_16_04_linux_e2e
-          - windows_e2e
+  - stage: cleanup
+    jobs:
+      - job: delete_remote_artifacts
         pool:
           name: Networking-ContainerNetworking
           demands: agent.os -equals Linux

From 38809afdf05559ceaa00e5537e196e7e146b30c9 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Thu, 26 Sep 2019 12:06:24 -0700
Subject: [PATCH 81/83] pipeline 16

---
 .pipelines/pipeline.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
index 0494f951e2..1ae9fb9cef 100644
--- a/.pipelines/pipeline.yaml
+++ b/.pipelines/pipeline.yaml
@@ -176,7 +176,6 @@ stages:
         variables:
           Tag: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.Tag'] ]
           CommitHash: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.CommitHash'] ]
-        condition: succeeded('unit_tests')
         steps:
           - checkout: none
           - task: AzureCLI@1

From f167ab849fcca1dc7985f193cf92ef49e117df0d Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Thu, 26 Sep 2019 14:19:19 -0700
Subject: [PATCH 82/83] pipeline 17

---
 .pipelines/pipeline.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
index 1ae9fb9cef..796c5d25ad 100644
--- a/.pipelines/pipeline.yaml
+++ b/.pipelines/pipeline.yaml
@@ -175,7 +175,7 @@ stages:
           image: containernetworking/pipeline-ci:1.0.4
         variables:
           Tag: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.Tag'] ]
-          CommitHash: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.CommitHash'] ]
+
         steps:
           - checkout: none
           - task: AzureCLI@1
@@ -183,6 +183,7 @@ stages:
               azureSubscription: $(ARTIFACT_SUBSCRIPTION)
               scriptLocation: "inlineScript"
               inlineScript: |
-                az storage container delete -n acn-$(CommitHash) --account-name $(STORAGE_ACCOUNT_NAME)
+                export CommitHash=(git rev-parse HEAD)
+                az storage container delete -n acn-$CommitHash --account-name $(STORAGE_ACCOUNT_NAME)
               workingDirectory: "$(modulePath)"
             displayName: Cleanup remote Azure storage container

From c914aa1fa9bf8755167cf1504f5ba23eb9cb04b8 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Thu, 26 Sep 2019 17:03:54 -0700
Subject: [PATCH 83/83] pipeline 18

---
 .pipelines/pipeline.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
index 796c5d25ad..0d5f428156 100644
--- a/.pipelines/pipeline.yaml
+++ b/.pipelines/pipeline.yaml
@@ -1,5 +1,7 @@
 pr:
-  - master
+  branches:
+    include:
+      - master
 
 trigger:
   branches: