From e6d640e98859ea5f51c5a0471c212f56a8d7a705 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Thu, 7 Nov 2019 11:53:24 -0800
Subject: [PATCH 01/31] allow all egress and ingress without drop

---
 npm/nwpolicy.go             | 10 +++---
 npm/translatePolicy.go      | 22 +++++++++----
 npm/translatePolicy_test.go | 62 +++++++++++++++++++++++++++++++++++--
 3 files changed, 80 insertions(+), 14 deletions(-)

diff --git a/npm/nwpolicy.go b/npm/nwpolicy.go
index b2f8aa912e..ae3f3bf630 100644
--- a/npm/nwpolicy.go
+++ b/npm/nwpolicy.go
@@ -3,8 +3,8 @@
 package npm
 
 import (
-	"github.com/Azure/azure-container-networking/npm/iptm"
 	"github.com/Azure/azure-container-networking/log"
+	"github.com/Azure/azure-container-networking/npm/iptm"
 	"github.com/Azure/azure-container-networking/npm/util"
 	networkingv1 "k8s.io/api/networking/v1"
 )
@@ -33,7 +33,7 @@ func (npMgr *NetworkPolicyManager) AddNetworkPolicy(npObj *networkingv1.NetworkP
 		ns  *namespace
 	)
 
-	npNs, npName := "ns-" + npObj.ObjectMeta.Namespace, npObj.ObjectMeta.Name
+	npNs, npName := "ns-"+npObj.ObjectMeta.Namespace, npObj.ObjectMeta.Name
 	log.Printf("NETWORK POLICY CREATING: %v", npObj)
 
 	var exists bool
@@ -148,7 +148,7 @@ func (npMgr *NetworkPolicyManager) DeleteNetworkPolicy(npObj *networkingv1.Netwo
 		ns  *namespace
 	)
 
-	npNs, npName := "ns-" + npObj.ObjectMeta.Namespace, npObj.ObjectMeta.Name
+	npNs, npName := "ns-"+npObj.ObjectMeta.Namespace, npObj.ObjectMeta.Name
 	log.Printf("NETWORK POLICY DELETING: %v", npObj)
 
 	var exists bool
@@ -178,14 +178,14 @@ func (npMgr *NetworkPolicyManager) DeleteNetworkPolicy(npObj *networkingv1.Netwo
 		if err != nil {
 			log.Printf("Error deducting policy %s from %s", npName, oldPolicy.ObjectMeta.Name)
 		}
-		
+
 		if deductedPolicy == nil {
 			delete(ns.processedNpMap, hashedSelector)
 		} else {
 			ns.processedNpMap[hashedSelector] = deductedPolicy
 		}
 	}
-	
+
 	if npMgr.canCleanUpNpmChains() {
 		if err = iptMgr.UninitNpmChains(); err != nil {
 			log.Errorf("Error: failed to uninitialize azure-npm chains.")
diff --git a/npm/translatePolicy.go b/npm/translatePolicy.go
index d8a86e84c4..d19fd5baf2 100644
--- a/npm/translatePolicy.go
+++ b/npm/translatePolicy.go
@@ -964,9 +964,9 @@ func getAllowKubeSystemEntries(ns string, targetSelector metav1.LabelSelector) [
 // 3. iptables entries generated from the input network policy object.
 func translatePolicy(npObj *networkingv1.NetworkPolicy) ([]string, []string, []*iptm.IptEntry) {
 	var (
-		resultSets          []string
-		resultLists         []string
-		entries             []*iptm.IptEntry
+		resultSets            []string
+		resultLists           []string
+		entries               []*iptm.IptEntry
 		hasIngress, hasEgress bool
 	)
 
@@ -1010,7 +1010,12 @@ func translatePolicy(npObj *networkingv1.NetworkPolicy) ([]string, []string, []*
 			resultSets = append(resultSets, ingressSets...)
 			resultLists = append(resultLists, ingressLists...)
 			entries = append(entries, ingressEntries...)
-			hasIngress = true
+
+			if npObj.Spec.Ingress != nil && len(npObj.Spec.Ingress) == 1 && len(npObj.Spec.Ingress[0].Ports) == 0 && len(npObj.Spec.Ingress[0].From) == 0 {
+				hasIngress = false
+			} else {
+				hasIngress = true
+			}
 		}
 
 		if ptype == networkingv1.PolicyTypeEgress {
@@ -1018,12 +1023,17 @@ func translatePolicy(npObj *networkingv1.NetworkPolicy) ([]string, []string, []*
 			resultSets = append(resultSets, egressSets...)
 			resultLists = append(resultLists, egressLists...)
 			entries = append(entries, egressEntries...)
-			hasEgress = true
+
+			if npObj.Spec.Egress != nil && len(npObj.Spec.Egress) == 1 && len(npObj.Spec.Egress[0].Ports) == 0 && len(npObj.Spec.Egress[0].To) == 0 {
+				hasEgress = false
+			} else {
+				hasEgress = true
+			}
 		}
 	}
 
 	entries = append(entries, getDefaultDropEntries(npNs, npObj.Spec.PodSelector, hasIngress, hasEgress)...)
-
+	log.Printf("NEWPOLICY +%v", npObj)
 	resultSets, resultLists = util.UniqueStrSlice(resultSets), util.UniqueStrSlice(resultLists)
 
 	return resultSets, resultLists, entries
diff --git a/npm/translatePolicy_test.go b/npm/translatePolicy_test.go
index 71f2bf14a9..59b72e1bcb 100644
--- a/npm/translatePolicy_test.go
+++ b/npm/translatePolicy_test.go
@@ -2325,7 +2325,6 @@ func TestTranslatePolicy(t *testing.T) {
 			PolicyTypes: []networkingv1.PolicyType{
 				networkingv1.PolicyTypeEgress,
 			},
-			Egress: []networkingv1.NetworkPolicyEgressRule{},
 		},
 	}
 
@@ -2361,6 +2360,63 @@ func TestTranslatePolicy(t *testing.T) {
 		t.Errorf("expectedIptEntries: %s", marshalledExpectedIptEntries)
 	}
 
+	///////////////////////////////////////////////////////////////////////////
+
+	targetSelector = metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			"app": "backend",
+		},
+	}
+	allowAllEgress := &networkingv1.NetworkPolicy{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "ALLOW-all-FROM-app:backend-policy",
+			Namespace: "testnamespace",
+		},
+		Spec: networkingv1.NetworkPolicySpec{
+			PodSelector: targetSelector,
+			PolicyTypes: []networkingv1.PolicyType{
+				networkingv1.PolicyTypeEgress,
+			},
+			Egress: []networkingv1.NetworkPolicyEgressRule{},
+		},
+	}
+
+	sets, lists, iptEntries = translatePolicy(allowAllEgress)
+
+	expectedSets = []string{
+		"app:backend",
+	}
+	if !reflect.DeepEqual(sets, expectedSets) {
+		t.Errorf("translatedPolicy failed @ ALLOW-all-FROM-app:backend-policy sets comparison")
+		t.Errorf("sets: %v", sets)
+		t.Errorf("expectedSets: %v", expectedSets)
+	}
+
+	expectedLists = []string{}
+	if !reflect.DeepEqual(lists, expectedLists) {
+		t.Errorf("translatedPolicy failed @ ALLOW-all-FROM-app:backend-policy lists comparison")
+		t.Errorf("lists: %v", lists)
+		t.Errorf("expectedLists: %v", expectedLists)
+	}
+
+	expectedIptEntries = []*iptm.IptEntry{}
+	expectedIptEntries = append(
+		expectedIptEntries,
+		getAllowKubeSystemEntries("testnamespace", targetSelector)...,
+	)
+
+	// has egress, but empty map means allow all
+	expectedIptEntries = append(expectedIptEntries, getDefaultDropEntries("testnamespace", targetSelector, false, false)...)
+	if !reflect.DeepEqual(iptEntries, expectedIptEntries) {
+		t.Errorf("translatedPolicy failed @ ALLOW-all-FROM-app:backend-policy policy comparison")
+		marshalledIptEntries, _ := json.Marshal(iptEntries)
+		marshalledExpectedIptEntries, _ := json.Marshal(expectedIptEntries)
+		t.Errorf("iptEntries: %s", marshalledIptEntries)
+		t.Errorf("expectedIptEntries: %s", marshalledExpectedIptEntries)
+	}
+
+	///////////////////////////////////////////////////////////////////////////
+
 	targetSelector = metav1.LabelSelector{}
 	denyAllFromNsUnsafePolicy := &networkingv1.NetworkPolicy{
 		ObjectMeta: metav1.ObjectMeta{
@@ -2835,8 +2891,8 @@ func TestAllowPrecedenceOverDeny(t *testing.T) {
 	}
 	denyAllPolicy := &networkingv1.NetworkPolicy{
 		ObjectMeta: metav1.ObjectMeta{
-			Name: 			"default-deny",
-			Namespace:      "default",
+			Name:      "default-deny",
+			Namespace: "default",
 		},
 		Spec: networkingv1.NetworkPolicySpec{
 			PodSelector: targetSelector,

From 70005c8987fcf3fb332e1fb6ba819e18daeabb61 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Thu, 7 Nov 2019 12:11:10 -0800
Subject: [PATCH 02/31] remove comment breaks

---
 npm/translatePolicy_test.go | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/npm/translatePolicy_test.go b/npm/translatePolicy_test.go
index 59b72e1bcb..36002a8340 100644
--- a/npm/translatePolicy_test.go
+++ b/npm/translatePolicy_test.go
@@ -2360,8 +2360,6 @@ func TestTranslatePolicy(t *testing.T) {
 		t.Errorf("expectedIptEntries: %s", marshalledExpectedIptEntries)
 	}
 
-	///////////////////////////////////////////////////////////////////////////
-
 	targetSelector = metav1.LabelSelector{
 		MatchLabels: map[string]string{
 			"app": "backend",
@@ -2415,8 +2413,6 @@ func TestTranslatePolicy(t *testing.T) {
 		t.Errorf("expectedIptEntries: %s", marshalledExpectedIptEntries)
 	}
 
-	///////////////////////////////////////////////////////////////////////////
-
 	targetSelector = metav1.LabelSelector{}
 	denyAllFromNsUnsafePolicy := &networkingv1.NetworkPolicy{
 		ObjectMeta: metav1.ObjectMeta{

From 39cd0807c2c64b5ebc6b440e4cdae72edc28ca2d Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Thu, 7 Nov 2019 13:56:52 -0800
Subject: [PATCH 03/31] update test

---
 npm/translatePolicy_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/npm/translatePolicy_test.go b/npm/translatePolicy_test.go
index 36002a8340..6de0d031da 100644
--- a/npm/translatePolicy_test.go
+++ b/npm/translatePolicy_test.go
@@ -2375,7 +2375,7 @@ func TestTranslatePolicy(t *testing.T) {
 			PolicyTypes: []networkingv1.PolicyType{
 				networkingv1.PolicyTypeEgress,
 			},
-			Egress: []networkingv1.NetworkPolicyEgressRule{},
+			Egress: []networkingv1.NetworkPolicyEgressRule{networkingv1.NetworkPolicyEgressRule{}},
 		},
 	}
 

From 55b7e9b1b31a5f2fddb899dfe0afc3082248626f Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Fri, 8 Nov 2019 14:41:02 -0800
Subject: [PATCH 04/31] remove sleep and socket cleanup

---
 cni/telemetry/service/telemetrymain.go | 4 ++++
 npm/plugin/main.go                     | 2 --
 npm/pod.go                             | 7 ++-----
 3 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/cni/telemetry/service/telemetrymain.go b/cni/telemetry/service/telemetrymain.go
index 7d93bf12ad..07cc5e08a3 100644
--- a/cni/telemetry/service/telemetrymain.go
+++ b/cni/telemetry/service/telemetrymain.go
@@ -123,6 +123,10 @@ func main() {
 
 	log.Logf("read config returned %+v", config)
 
+	// Cleaning up orphan socket if present
+	tbtemp := telemetry.NewTelemetryBuffer("")
+	tbtemp.Cleanup(telemetry.FdName)
+
 	for {
 		tb = telemetry.NewTelemetryBuffer("")
 
diff --git a/npm/plugin/main.go b/npm/plugin/main.go
index 501b271354..1c6f8c0ebe 100644
--- a/npm/plugin/main.go
+++ b/npm/plugin/main.go
@@ -62,8 +62,6 @@ func main() {
 
 	go npMgr.SendNpmTelemetry()
 
-	time.Sleep(time.Second * waitForTelemetryInSeconds)
-
 	if err = npMgr.Start(wait.NeverStop); err != nil {
 		log.Logf("npm failed with error %v.", err)
 		panic(err.Error)
diff --git a/npm/pod.go b/npm/pod.go
index 427e01d44a..5e30c34abc 100644
--- a/npm/pod.go
+++ b/npm/pod.go
@@ -10,10 +10,7 @@ import (
 )
 
 func isValidPod(podObj *corev1.Pod) bool {
-	return podObj.Status.Phase != corev1.PodPhase(util.KubePodStatusFailedFlag) &&
-		podObj.Status.Phase != corev1.PodPhase(util.KubePodStatusSucceededFlag) &&
-		podObj.Status.Phase != corev1.PodPhase(util.KubePodStatusUnknownFlag) &&
-		len(podObj.Status.PodIP) > 0
+	return len(podObj.Status.PodIP) > 0
 }
 
 func isSystemPod(podObj *corev1.Pod) bool {
@@ -142,7 +139,7 @@ func (npMgr *NetworkPolicyManager) DeletePod(podObj *corev1.Pod) error {
 		if err = ipsMgr.DeleteFromSet(podLabelKey, podIP); err != nil {
 			log.Errorf("Error: failed to delete pod from label ipset.")
 			return err
-		}		
+		}
 
 		label := podLabelKey + ":" + podLabelVal
 		log.Printf("Deleting pod %s from ipset %s", podIP, label)

From 6aba156fc4a084660172f8e0100cea98c41b6177 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Fri, 8 Nov 2019 16:37:54 -0800
Subject: [PATCH 05/31] address feedback

---
 npm/translatePolicy.go      | 12 +++++++++---
 npm/translatePolicy_test.go |  4 ++++
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/npm/translatePolicy.go b/npm/translatePolicy.go
index d19fd5baf2..45ec6bd5fa 100644
--- a/npm/translatePolicy.go
+++ b/npm/translatePolicy.go
@@ -1011,7 +1011,10 @@ func translatePolicy(npObj *networkingv1.NetworkPolicy) ([]string, []string, []*
 			resultLists = append(resultLists, ingressLists...)
 			entries = append(entries, ingressEntries...)
 
-			if npObj.Spec.Ingress != nil && len(npObj.Spec.Ingress) == 1 && len(npObj.Spec.Ingress[0].Ports) == 0 && len(npObj.Spec.Ingress[0].From) == 0 {
+			if npObj.Spec.Ingress != nil &&
+				len(npObj.Spec.Ingress) == 1 &&
+				len(npObj.Spec.Ingress[0].Ports) == 0 &&
+				len(npObj.Spec.Ingress[0].From) == 0 {
 				hasIngress = false
 			} else {
 				hasIngress = true
@@ -1024,7 +1027,10 @@ func translatePolicy(npObj *networkingv1.NetworkPolicy) ([]string, []string, []*
 			resultLists = append(resultLists, egressLists...)
 			entries = append(entries, egressEntries...)
 
-			if npObj.Spec.Egress != nil && len(npObj.Spec.Egress) == 1 && len(npObj.Spec.Egress[0].Ports) == 0 && len(npObj.Spec.Egress[0].To) == 0 {
+			if npObj.Spec.Egress != nil &&
+				len(npObj.Spec.Egress) == 1 &&
+				len(npObj.Spec.Egress[0].Ports) == 0 &&
+				len(npObj.Spec.Egress[0].To) == 0 {
 				hasEgress = false
 			} else {
 				hasEgress = true
@@ -1033,7 +1039,7 @@ func translatePolicy(npObj *networkingv1.NetworkPolicy) ([]string, []string, []*
 	}
 
 	entries = append(entries, getDefaultDropEntries(npNs, npObj.Spec.PodSelector, hasIngress, hasEgress)...)
-	log.Printf("NEWPOLICY +%v", npObj)
+	log.Printf("New Policy%+v", npObj)
 	resultSets, resultLists = util.UniqueStrSlice(resultSets), util.UniqueStrSlice(resultLists)
 
 	return resultSets, resultLists, entries
diff --git a/npm/translatePolicy_test.go b/npm/translatePolicy_test.go
index 6de0d031da..3461df000d 100644
--- a/npm/translatePolicy_test.go
+++ b/npm/translatePolicy_test.go
@@ -2365,6 +2365,10 @@ func TestTranslatePolicy(t *testing.T) {
 			"app": "backend",
 		},
 	}
+
+	//////
+	/// This policy tests the case where pods should have unlimited egress traffic
+	//////
 	allowAllEgress := &networkingv1.NetworkPolicy{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "ALLOW-all-FROM-app:backend-policy",

From ed6e4baf561d36eeb30046f6acb5670fe28dbd96 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Fri, 8 Nov 2019 17:31:28 -0800
Subject: [PATCH 06/31] all namespaces

---
 npm/translatePolicy_test.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/npm/translatePolicy_test.go b/npm/translatePolicy_test.go
index 3461df000d..9e0c0b1a7f 100644
--- a/npm/translatePolicy_test.go
+++ b/npm/translatePolicy_test.go
@@ -2394,7 +2394,9 @@ func TestTranslatePolicy(t *testing.T) {
 		t.Errorf("expectedSets: %v", expectedSets)
 	}
 
-	expectedLists = []string{}
+	expectedLists = []string{
+		util.KubeAllNamespacesFlag,
+	}
 	if !reflect.DeepEqual(lists, expectedLists) {
 		t.Errorf("translatedPolicy failed @ ALLOW-all-FROM-app:backend-policy lists comparison")
 		t.Errorf("lists: %v", lists)

From 62a261a88c3c1ed818aa1851ecbfa2b31ebfe0d8 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Mon, 11 Nov 2019 19:09:55 -0800
Subject: [PATCH 07/31] fix tests

---
 npm/translatePolicy.go      | 2 +-
 npm/translatePolicy_test.go | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/npm/translatePolicy.go b/npm/translatePolicy.go
index 45ec6bd5fa..84c7a4308f 100644
--- a/npm/translatePolicy.go
+++ b/npm/translatePolicy.go
@@ -1039,7 +1039,7 @@ func translatePolicy(npObj *networkingv1.NetworkPolicy) ([]string, []string, []*
 	}
 
 	entries = append(entries, getDefaultDropEntries(npNs, npObj.Spec.PodSelector, hasIngress, hasEgress)...)
-	log.Printf("New Policy%+v", npObj)
+	log.Printf("Translating Policy: %+v", npObj)
 	resultSets, resultLists = util.UniqueStrSlice(resultSets), util.UniqueStrSlice(resultLists)
 
 	return resultSets, resultLists, entries
diff --git a/npm/translatePolicy_test.go b/npm/translatePolicy_test.go
index 9e0c0b1a7f..f2d2da41d0 100644
--- a/npm/translatePolicy_test.go
+++ b/npm/translatePolicy_test.go
@@ -1271,7 +1271,7 @@ func TestTranslatePolicy(t *testing.T) {
 		},
 	}
 	expectedIptEntries = append(expectedIptEntries, nonKubeSystemEntries...)
-	expectedIptEntries = append(expectedIptEntries, getDefaultDropEntries("testnamespace", targetSelector, true, false)...)
+	expectedIptEntries = append(expectedIptEntries, getDefaultDropEntries("testnamespace", targetSelector, false, false)...)
 	if !reflect.DeepEqual(iptEntries, expectedIptEntries) {
 		t.Errorf("translatedPolicy failed @ ALLOW-all-TO-app:frontend-FROM-all-namespaces-policy policy comparison")
 		marshalledIptEntries, _ := json.Marshal(iptEntries)
@@ -2018,7 +2018,7 @@ func TestTranslatePolicy(t *testing.T) {
 	}
 
 	expectedIptEntries = append(expectedIptEntries, nonKubeSystemEntries...)
-	expectedIptEntries = append(expectedIptEntries, getDefaultDropEntries("dangerous", targetSelector, true, false)...)
+	expectedIptEntries = append(expectedIptEntries, getDefaultDropEntries("dangerous", targetSelector, false, false)...)
 	if !reflect.DeepEqual(iptEntries, expectedIptEntries) {
 		t.Errorf("translatedPolicy failed @ ALLOW-ALL-TO-app:backdoor-policy policy comparison")
 		marshalledIptEntries, _ := json.Marshal(iptEntries)

From a91f3462c4a4f89d05e1547ba823225db374d581 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Tue, 12 Nov 2019 15:13:20 -0800
Subject: [PATCH 08/31] update npm test

---
 npm/translatePolicy_test.go | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/npm/translatePolicy_test.go b/npm/translatePolicy_test.go
index f2d2da41d0..c8aa4f5e93 100644
--- a/npm/translatePolicy_test.go
+++ b/npm/translatePolicy_test.go
@@ -2409,6 +2409,31 @@ func TestTranslatePolicy(t *testing.T) {
 		getAllowKubeSystemEntries("testnamespace", targetSelector)...,
 	)
 
+	nonKubeSystemEntries = []*iptm.IptEntry{
+		&iptm.IptEntry{
+			Chain: util.IptablesAzureEgressPortChain,
+			Specs: []string{
+				util.IptablesModuleFlag,
+				util.IptablesSetModuleFlag,
+				util.IptablesMatchSetFlag,
+				util.GetHashedName("app:backend"),
+				util.IptablesSrcFlag,
+				util.IptablesModuleFlag,
+				util.IptablesSetModuleFlag,
+				util.IptablesMatchSetFlag,
+				util.GetHashedName(util.KubeAllNamespacesFlag),
+				util.IptablesDstFlag,
+				util.IptablesJumpFlag,
+				util.IptablesAccept,
+				util.IptablesModuleFlag,
+				util.IptablesCommentModuleFlag,
+				util.IptablesCommentFlag,
+				"ALLOW-ALL-FROM-app:backend-TO-" +
+					util.KubeAllNamespacesFlag,
+			},
+		},
+	}
+	expectedIptEntries = append(expectedIptEntries, nonKubeSystemEntries...)
 	// has egress, but empty map means allow all
 	expectedIptEntries = append(expectedIptEntries, getDefaultDropEntries("testnamespace", targetSelector, false, false)...)
 	if !reflect.DeepEqual(iptEntries, expectedIptEntries) {

From ea5217e94d7bf2165a1e4f0d439ddf39fc0d0a87 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Tue, 12 Nov 2019 16:45:55 -0800
Subject: [PATCH 09/31] aks-engine

---
 .pipelines/e2e-step-template.yaml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index 4b109901a0..7d670a2784 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -13,7 +13,7 @@ steps:
       go get -v github.com/Azure/aks-engine
       mkdir -p '$(GOBIN)'
       mkdir -p '$(GOPATH)/pkg'
-      mkdir -p '$(modulePath)'
+      shopt -s extglob
       echo '##vso[task.prependpath]$(GOBIN)'
       echo '##vso[task.prependpath]$(GOROOT)/bin'
     name: "GoEnv"
@@ -35,7 +35,6 @@ steps:
       make bootstrap
       make build-binary
     displayName: Build AKS-Engine
-    workingDirectory: "$(modulePath)"
 
   - bash: |
       export CLUSTER_DEFINITION=./clusterDefinition.json 
@@ -50,6 +49,6 @@ steps:
       export REGIONS=$(AKS_ENGINE_REGION) 
       export IS_JENKINS=false 
       make test-kubernetes
+      rm -rf
     name: DeployAKSEngine
     displayName: Deploy AKS-Engine
-    workingDirectory: "$(modulePath)"

From abda25cd70bbbf7ddb8b31a0f89ab060f4da9012 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Tue, 12 Nov 2019 16:48:29 -0800
Subject: [PATCH 10/31] aks-engine

---
 .pipelines/e2e-step-template.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index 7d670a2784..902a9261a7 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -13,7 +13,7 @@ steps:
       go get -v github.com/Azure/aks-engine
       mkdir -p '$(GOBIN)'
       mkdir -p '$(GOPATH)/pkg'
-      shopt -s extglob
+      mkdir -p '$(modulePath)'
       echo '##vso[task.prependpath]$(GOBIN)'
       echo '##vso[task.prependpath]$(GOROOT)/bin'
     name: "GoEnv"
@@ -35,6 +35,7 @@ steps:
       make bootstrap
       make build-binary
     displayName: Build AKS-Engine
+    workingDirectory: "$(modulePath)"
 
   - bash: |
       export CLUSTER_DEFINITION=./clusterDefinition.json 
@@ -49,6 +50,7 @@ steps:
       export REGIONS=$(AKS_ENGINE_REGION) 
       export IS_JENKINS=false 
       make test-kubernetes
-      rm -rf
+      rm -rf *
     name: DeployAKSEngine
     displayName: Deploy AKS-Engine
+    workingDirectory: "$(modulePath)"

From ecb1b597fa6be583b365b221a4a2bf1d0b726378 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Tue, 12 Nov 2019 17:31:25 -0800
Subject: [PATCH 11/31] pipeline

---
 .pipelines/e2e-job-template.yaml  | 9 ++++++---
 .pipelines/e2e-step-template.yaml | 7 +++++--
 2 files changed, 11 insertions(+), 5 deletions(-)

diff --git a/.pipelines/e2e-job-template.yaml b/.pipelines/e2e-job-template.yaml
index 456dfdfaa1..160d219c89 100644
--- a/.pipelines/e2e-job-template.yaml
+++ b/.pipelines/e2e-job-template.yaml
@@ -15,13 +15,16 @@ jobs:
     container:
       image: ${{ parameters.pipelineBuildImage }}
     variables:
-      GOPATH: "$(System.DefaultWorkingDirectory)/gopath"
-      GOBIN: "$(GOPATH)/bin" # Go binaries path
       modulePath: "$(GOPATH)/src/github.com/Azure/aks-engine"
       acnPath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
       Tag: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.Tag'] ]
       CommitHash: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.CommitHash'] ]
-
+      CLEANUP_ON_EXIT: true
+      CLEANUP_IF_FAIL: true
+      GOBIN: "$(GOPATH)/bin" # Go binaries path
+      GOROOT: "/usr/local/go" # Go installation path
+      GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
+      AKS_ENGINE_VERSION: v0.43.1
     steps:
       - template: e2e-step-template.yaml
         parameters:
diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index 7b949b1789..23ede1e568 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -10,12 +10,15 @@ steps:
   - bash: |
       go version
       go env
-      go get -v github.com/Azure/aks-engine
+      #go get -v github.com/Azure/aks-engine
       mkdir -p '$(GOBIN)'
       mkdir -p '$(GOPATH)/pkg'
-      mkdir -p '$(modulePath)'
       echo '##vso[task.prependpath]$(GOBIN)'
       echo '##vso[task.prependpath]$(GOROOT)/bin'
+      mkdir -P $(dirname '$(modulePath)')
+      cd $(dirname '$(modulePath)')
+      wget https://github.com/Azure/aks-engine/archive/$(AKS_ENGINE_VERSION).tar.gz
+      tar -zxvf *.tar.gz && mv aks-engine* aks-engine
     name: "GoEnv"
     displayName: "Set up the Go environment"
 

From 0e1a853f0e7c03334c48c53b5b47778dd8f0f2ac Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Tue, 12 Nov 2019 17:46:48 -0800
Subject: [PATCH 12/31] pipeline

---
 .pipelines/e2e-step-template.yaml | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index 23ede1e568..bbba0d0470 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -18,7 +18,9 @@ steps:
       mkdir -P $(dirname '$(modulePath)')
       cd $(dirname '$(modulePath)')
       wget https://github.com/Azure/aks-engine/archive/$(AKS_ENGINE_VERSION).tar.gz
-      tar -zxvf *.tar.gz && mv aks-engine* aks-engine
+      tar -zxf *.tar.gz && mv aks-engine* aks-engine
+      cd aks-engine
+      ls -lah
     name: "GoEnv"
     displayName: "Set up the Go environment"
 

From 3a751870fbba289a625ac363e546a194d156baf2 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Tue, 12 Nov 2019 17:54:02 -0800
Subject: [PATCH 13/31] pipeline

---
 .pipelines/e2e-job-template.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.pipelines/e2e-job-template.yaml b/.pipelines/e2e-job-template.yaml
index 160d219c89..cb5875c48d 100644
--- a/.pipelines/e2e-job-template.yaml
+++ b/.pipelines/e2e-job-template.yaml
@@ -15,15 +15,15 @@ jobs:
     container:
       image: ${{ parameters.pipelineBuildImage }}
     variables:
+      GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
+      GOBIN: "$(GOPATH)/bin" # Go binaries path
+      GOROOT: "/usr/local/go" # Go installation path
       modulePath: "$(GOPATH)/src/github.com/Azure/aks-engine"
       acnPath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
       Tag: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.Tag'] ]
       CommitHash: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.CommitHash'] ]
       CLEANUP_ON_EXIT: true
       CLEANUP_IF_FAIL: true
-      GOBIN: "$(GOPATH)/bin" # Go binaries path
-      GOROOT: "/usr/local/go" # Go installation path
-      GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
       AKS_ENGINE_VERSION: v0.43.1
     steps:
       - template: e2e-step-template.yaml

From c187f0ede6bf84f5d756104e690fa7618f2faf21 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Tue, 12 Nov 2019 18:00:03 -0800
Subject: [PATCH 14/31] pipeline

---
 .pipelines/e2e-step-template.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index bbba0d0470..530f3fa724 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -34,6 +34,8 @@ steps:
 
   - bash: |
       ls -lah
+      pwd
+      echo "$(modulePath)"
       export CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-${{ parameters.clusterDefinitionCniBuildOS }}-amd64-$(Tag)${{ parameters.clusterDefinitionCniBuildExt }}'"'
       export CNI_TYPE=${{ parameters.clusterDefinitionCniTypeKey }}
       echo CNI type is $CNI_TYPE

From be3e2440726719a6eae6023f5ff95bc9942ba0ab Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Tue, 12 Nov 2019 18:20:05 -0800
Subject: [PATCH 15/31] remove comment breaks

---
 .pipelines/e2e-job-template.yaml  | 1 -
 .pipelines/e2e-step-template.yaml | 3 ++-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pipelines/e2e-job-template.yaml b/.pipelines/e2e-job-template.yaml
index cb5875c48d..e5cecd41a2 100644
--- a/.pipelines/e2e-job-template.yaml
+++ b/.pipelines/e2e-job-template.yaml
@@ -17,7 +17,6 @@ jobs:
     variables:
       GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
       GOBIN: "$(GOPATH)/bin" # Go binaries path
-      GOROOT: "/usr/local/go" # Go installation path
       modulePath: "$(GOPATH)/src/github.com/Azure/aks-engine"
       acnPath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
       Tag: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.Tag'] ]
diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index 530f3fa724..04a17446c6 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -15,12 +15,13 @@ steps:
       mkdir -p '$(GOPATH)/pkg'
       echo '##vso[task.prependpath]$(GOBIN)'
       echo '##vso[task.prependpath]$(GOROOT)/bin'
-      mkdir -P $(dirname '$(modulePath)')
+      mkdir -p $(dirname '$(modulePath)')
       cd $(dirname '$(modulePath)')
       wget https://github.com/Azure/aks-engine/archive/$(AKS_ENGINE_VERSION).tar.gz
       tar -zxf *.tar.gz && mv aks-engine* aks-engine
       cd aks-engine
       ls -lah
+      pwd
     name: "GoEnv"
     displayName: "Set up the Go environment"
 

From 07899662b5005bfcf2d03544afe376f3e5d4b3d1 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Tue, 12 Nov 2019 18:38:53 -0800
Subject: [PATCH 16/31] remove comment breaks

---
 .pipelines/e2e-step-template.yaml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index 04a17446c6..07351badf5 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -20,6 +20,10 @@ steps:
       wget https://github.com/Azure/aks-engine/archive/$(AKS_ENGINE_VERSION).tar.gz
       tar -zxf *.tar.gz && mv aks-engine* aks-engine
       cd aks-engine
+      wget https://github.com/Azure/aks-engine/releases/download/v0.43.1/aks-engine-$(AKS_ENGINE_VERSION)-linux-amd64.tar.gz
+      tar -zxf *.tar.gz
+      mkdir ./bin
+      mv aks-engine-$(AKS_ENGINE_VERSION)-linux-amd64/aks-engine ./bin
       ls -lah
       pwd
     name: "GoEnv"
@@ -49,7 +53,7 @@ steps:
       sudo cp kubernetes/client/bin/kubectl /usr/local/bin/kubectl
       sudo cp kubernetes/client/bin/kubectl /usr/local/bin/k
       make bootstrap
-      make build-binary
+      #make build-binary
     displayName: Build AKS-Engine
     workingDirectory: "$(modulePath)"
 

From 6cb5276c91c83e1dfcf7ee3171cf8ba42f6bdfb1 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Tue, 12 Nov 2019 19:14:59 -0800
Subject: [PATCH 17/31] remove comment breaks

---
 .pipelines/e2e-step-template.yaml | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index 07351badf5..04a17446c6 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -20,10 +20,6 @@ steps:
       wget https://github.com/Azure/aks-engine/archive/$(AKS_ENGINE_VERSION).tar.gz
       tar -zxf *.tar.gz && mv aks-engine* aks-engine
       cd aks-engine
-      wget https://github.com/Azure/aks-engine/releases/download/v0.43.1/aks-engine-$(AKS_ENGINE_VERSION)-linux-amd64.tar.gz
-      tar -zxf *.tar.gz
-      mkdir ./bin
-      mv aks-engine-$(AKS_ENGINE_VERSION)-linux-amd64/aks-engine ./bin
       ls -lah
       pwd
     name: "GoEnv"
@@ -53,7 +49,7 @@ steps:
       sudo cp kubernetes/client/bin/kubectl /usr/local/bin/kubectl
       sudo cp kubernetes/client/bin/kubectl /usr/local/bin/k
       make bootstrap
-      #make build-binary
+      make build-binary
     displayName: Build AKS-Engine
     workingDirectory: "$(modulePath)"
 

From 5c8c501ac37c918af4a87f8e503431be02592b14 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Tue, 12 Nov 2019 19:19:13 -0800
Subject: [PATCH 18/31] remove comment breaks

---
 .pipelines/e2e-step-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index 04a17446c6..e276387c3a 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -69,7 +69,7 @@ steps:
       mkdir -p $(Build.ArtifactStagingDirectory)/kube-${{ parameters.name }}
       cp -r _output/k*/kubeconfig/kubeconfig.$REGIONS.json $(Build.ArtifactStagingDirectory)/kube-${{ parameters.name }}
       cp -r _output/kubernetes-*-ssh $(Build.ArtifactStagingDirectory)/kube-${{ parameters.name }}
-      rm -rf *
+      cd ~ && sudo rm -rf $(GOPATH)
       name: DeployAKSEngine
     displayName: Deploy AKS-Engine
     workingDirectory: "$(modulePath)"

From 08a9522bc9af10acf5ae63a22e8784cd8b9512f2 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Tue, 12 Nov 2019 20:06:29 -0800
Subject: [PATCH 19/31] pipeline

---
 .pipelines/e2e-job-template.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.pipelines/e2e-job-template.yaml b/.pipelines/e2e-job-template.yaml
index e5cecd41a2..e0fe990fa4 100644
--- a/.pipelines/e2e-job-template.yaml
+++ b/.pipelines/e2e-job-template.yaml
@@ -9,6 +9,7 @@ parameters:
 jobs:
   - job: ${{ parameters.name }}
     dependsOn: unit_tests
+    timeoutInMinutes: 90
     pool:
       name: Networking-ContainerNetworking
       demands: agent.os -equals Linux

From 7d37fde9c5f72e9863b36d562260901fe8662dfe Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 13 Nov 2019 11:59:52 -0800
Subject: [PATCH 20/31] pipeline

---
 .pipelines/e2e-step-template.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index e276387c3a..1cbfad11dd 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -39,6 +39,7 @@ steps:
       echo "$(modulePath)"
       export CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-${{ parameters.clusterDefinitionCniBuildOS }}-amd64-$(Tag)${{ parameters.clusterDefinitionCniBuildExt }}'"'
       export CNI_TYPE=${{ parameters.clusterDefinitionCniTypeKey }}
+      echo CNI_URL is $CNI_URL
       echo CNI type is $CNI_TYPE
       sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":$CNI_URL|g" '${{ parameters.clusterDefinition }}'
       sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" '${{ parameters.clusterDefinition }}'

From ceda740adcb337b39d45cfeedb483431a4b41cdf Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 13 Nov 2019 12:21:39 -0800
Subject: [PATCH 21/31] pipeline

---
 .pipelines/e2e-step-template.yaml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index 1cbfad11dd..3510bf9cf3 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -41,8 +41,10 @@ steps:
       export CNI_TYPE=${{ parameters.clusterDefinitionCniTypeKey }}
       echo CNI_URL is $CNI_URL
       echo CNI type is $CNI_TYPE
-      sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":$CNI_URL|g" '${{ parameters.clusterDefinition }}'
-      sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" '${{ parameters.clusterDefinition }}'
+      #sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":$CNI_URL|g" '${{ parameters.clusterDefinition }}'
+      cat '${{ parameters.clusterDefinition }}' | jq '.properties.orchestratorProfile.kubernetesConfig.$CNI_TYPE = \"$CNI_URL\"' > '${{ parameters.clusterDefinition }}'.tmp
+      #sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" '${{ parameters.clusterDefinition }}'
+      cat '${{ parameters.clusterDefinition }}'.tmp | jq '.properties.orchestratorProfile.kubernetesConfig.azureCNIVersion = \"$(Tag)\"' > '${{ parameters.clusterDefinition }}'
       echo "Running E2E tests against a cluster built with the following API model:"
       cat '${{ parameters.clusterDefinition }}'
       cp ${{ parameters.clusterDefinition }} clusterDefinition.json

From bd7d28a93c66715fd62c1daf2db88b1c4ecd2849 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 13 Nov 2019 14:58:57 -0800
Subject: [PATCH 22/31] pipeline

---
 .pipelines/e2e-step-template.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index 3510bf9cf3..a00de46dfe 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -42,9 +42,9 @@ steps:
       echo CNI_URL is $CNI_URL
       echo CNI type is $CNI_TYPE
       #sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":$CNI_URL|g" '${{ parameters.clusterDefinition }}'
-      cat '${{ parameters.clusterDefinition }}' | jq '.properties.orchestratorProfile.kubernetesConfig.$CNI_TYPE = \"$CNI_URL\"' > '${{ parameters.clusterDefinition }}'.tmp
+      cat cniLinux1604.json | jq --arg cnikey $CNI_TYPE --arg cniurl $CNI_URL'.properties.orchestratorProfile.kubernetesConfig[$cnikey]= $cniurl' > '${{ parameters.clusterDefinition }}'.tmp
       #sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" '${{ parameters.clusterDefinition }}'
-      cat '${{ parameters.clusterDefinition }}'.tmp | jq '.properties.orchestratorProfile.kubernetesConfig.azureCNIVersion = \"$(Tag)\"' > '${{ parameters.clusterDefinition }}'
+      cat '${{ parameters.clusterDefinition }}'.tmp | jq --arg tag $(Tag) '.properties.orchestratorProfile.kubernetesConfig.azureCNIVersion = $tag' > '${{ parameters.clusterDefinition }}'
       echo "Running E2E tests against a cluster built with the following API model:"
       cat '${{ parameters.clusterDefinition }}'
       cp ${{ parameters.clusterDefinition }} clusterDefinition.json

From 4680ff6b9e3a06b55b1f93f549255fbe2bf25d49 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 13 Nov 2019 15:06:37 -0800
Subject: [PATCH 23/31] pipeline

---
 .pipelines/e2e-step-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index a00de46dfe..7cf50a2c30 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -42,7 +42,7 @@ steps:
       echo CNI_URL is $CNI_URL
       echo CNI type is $CNI_TYPE
       #sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":$CNI_URL|g" '${{ parameters.clusterDefinition }}'
-      cat cniLinux1604.json | jq --arg cnikey $CNI_TYPE --arg cniurl $CNI_URL'.properties.orchestratorProfile.kubernetesConfig[$cnikey]= $cniurl' > '${{ parameters.clusterDefinition }}'.tmp
+      cat '${{ parameters.clusterDefinition }}' | jq --arg cnikey $CNI_TYPE --arg cniurl $CNI_URL '.properties.orchestratorProfile.kubernetesConfig[$cnikey]= $cniurl' > '${{ parameters.clusterDefinition }}'.tmp
       #sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" '${{ parameters.clusterDefinition }}'
       cat '${{ parameters.clusterDefinition }}'.tmp | jq --arg tag $(Tag) '.properties.orchestratorProfile.kubernetesConfig.azureCNIVersion = $tag' > '${{ parameters.clusterDefinition }}'
       echo "Running E2E tests against a cluster built with the following API model:"

From 4fc5d62e2f611d99e8b14c7cc37525a885d7de34 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 13 Nov 2019 15:12:44 -0800
Subject: [PATCH 24/31] pipeline

---
 .pipelines/e2e-step-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index 7cf50a2c30..7d91bc0329 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -37,7 +37,7 @@ steps:
       ls -lah
       pwd
       echo "$(modulePath)"
-      export CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-${{ parameters.clusterDefinitionCniBuildOS }}-amd64-$(Tag)${{ parameters.clusterDefinitionCniBuildExt }}'"'
+      export CNI_URL=https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-${{ parameters.clusterDefinitionCniBuildOS }}-amd64-$(Tag)${{ parameters.clusterDefinitionCniBuildExt }}
       export CNI_TYPE=${{ parameters.clusterDefinitionCniTypeKey }}
       echo CNI_URL is $CNI_URL
       echo CNI type is $CNI_TYPE

From 2e183561078aa5f49d23e4041771992e49311eab Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 13 Nov 2019 16:18:59 -0800
Subject: [PATCH 25/31] pipeline

---
 .pipelines/e2e-job-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/e2e-job-template.yaml b/.pipelines/e2e-job-template.yaml
index e0fe990fa4..2994d9972b 100644
--- a/.pipelines/e2e-job-template.yaml
+++ b/.pipelines/e2e-job-template.yaml
@@ -24,7 +24,7 @@ jobs:
       CommitHash: $[ dependencies.unit_tests.outputs['EnvironmentalVariables.CommitHash'] ]
       CLEANUP_ON_EXIT: true
       CLEANUP_IF_FAIL: true
-      AKS_ENGINE_VERSION: v0.43.1
+      AKS_ENGINE_VERSION: v0.41.3
     steps:
       - template: e2e-step-template.yaml
         parameters:

From ef02bb79e0bdb1927392f33359838ea535209781 Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 13 Nov 2019 17:18:22 -0800
Subject: [PATCH 26/31] pipeline

---
 .pipelines/e2e-step-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index 7d91bc0329..a374f40ec1 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -68,7 +68,7 @@ steps:
       export CLEANUP_ON_EXIT=true 
       export REGIONS=$(AKS_ENGINE_REGION) 
       export IS_JENKINS=false 
-      make test-kubernetes
+      sudo make test-kubernetes
       mkdir -p $(Build.ArtifactStagingDirectory)/kube-${{ parameters.name }}
       cp -r _output/k*/kubeconfig/kubeconfig.$REGIONS.json $(Build.ArtifactStagingDirectory)/kube-${{ parameters.name }}
       cp -r _output/kubernetes-*-ssh $(Build.ArtifactStagingDirectory)/kube-${{ parameters.name }}

From 3dc8d394aa91dde012f46f6737749025fde8325c Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 13 Nov 2019 19:03:21 -0800
Subject: [PATCH 27/31] pipeline

---
 .pipelines/e2e-step-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index a374f40ec1..7d91bc0329 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -68,7 +68,7 @@ steps:
       export CLEANUP_ON_EXIT=true 
       export REGIONS=$(AKS_ENGINE_REGION) 
       export IS_JENKINS=false 
-      sudo make test-kubernetes
+      make test-kubernetes
       mkdir -p $(Build.ArtifactStagingDirectory)/kube-${{ parameters.name }}
       cp -r _output/k*/kubeconfig/kubeconfig.$REGIONS.json $(Build.ArtifactStagingDirectory)/kube-${{ parameters.name }}
       cp -r _output/kubernetes-*-ssh $(Build.ArtifactStagingDirectory)/kube-${{ parameters.name }}

From 5c253ad7cecfa57beaef70b7a1338134cdf5e03f Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Wed, 13 Nov 2019 20:07:34 -0800
Subject: [PATCH 28/31] pipeline

---
 .pipelines/e2e-step-template.yaml | 28 ++++++++++------------------
 1 file changed, 10 insertions(+), 18 deletions(-)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index 7d91bc0329..30d9eac624 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -10,18 +10,12 @@ steps:
   - bash: |
       go version
       go env
-      #go get -v github.com/Azure/aks-engine
+      go get -v github.com/Azure/aks-engine
       mkdir -p '$(GOBIN)'
       mkdir -p '$(GOPATH)/pkg'
+      mkdir -p '$(modulePath)'
       echo '##vso[task.prependpath]$(GOBIN)'
       echo '##vso[task.prependpath]$(GOROOT)/bin'
-      mkdir -p $(dirname '$(modulePath)')
-      cd $(dirname '$(modulePath)')
-      wget https://github.com/Azure/aks-engine/archive/$(AKS_ENGINE_VERSION).tar.gz
-      tar -zxf *.tar.gz && mv aks-engine* aks-engine
-      cd aks-engine
-      ls -lah
-      pwd
     name: "GoEnv"
     displayName: "Set up the Go environment"
 
@@ -35,16 +29,11 @@ steps:
 
   - bash: |
       ls -lah
-      pwd
-      echo "$(modulePath)"
-      export CNI_URL=https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-${{ parameters.clusterDefinitionCniBuildOS }}-amd64-$(Tag)${{ parameters.clusterDefinitionCniBuildExt }}
+      export CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-${{ parameters.clusterDefinitionCniBuildOS }}-amd64-$(Tag)${{ parameters.clusterDefinitionCniBuildExt }}'"'
       export CNI_TYPE=${{ parameters.clusterDefinitionCniTypeKey }}
-      echo CNI_URL is $CNI_URL
       echo CNI type is $CNI_TYPE
-      #sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":$CNI_URL|g" '${{ parameters.clusterDefinition }}'
-      cat '${{ parameters.clusterDefinition }}' | jq --arg cnikey $CNI_TYPE --arg cniurl $CNI_URL '.properties.orchestratorProfile.kubernetesConfig[$cnikey]= $cniurl' > '${{ parameters.clusterDefinition }}'.tmp
-      #sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" '${{ parameters.clusterDefinition }}'
-      cat '${{ parameters.clusterDefinition }}'.tmp | jq --arg tag $(Tag) '.properties.orchestratorProfile.kubernetesConfig.azureCNIVersion = $tag' > '${{ parameters.clusterDefinition }}'
+      sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":$CNI_URL|g" '${{ parameters.clusterDefinition }}'
+      sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" '${{ parameters.clusterDefinition }}'
       echo "Running E2E tests against a cluster built with the following API model:"
       cat '${{ parameters.clusterDefinition }}'
       cp ${{ parameters.clusterDefinition }} clusterDefinition.json
@@ -72,8 +61,11 @@ steps:
       mkdir -p $(Build.ArtifactStagingDirectory)/kube-${{ parameters.name }}
       cp -r _output/k*/kubeconfig/kubeconfig.$REGIONS.json $(Build.ArtifactStagingDirectory)/kube-${{ parameters.name }}
       cp -r _output/kubernetes-*-ssh $(Build.ArtifactStagingDirectory)/kube-${{ parameters.name }}
-      cd ~ && sudo rm -rf $(GOPATH)
-      name: DeployAKSEngine
+      echo "Deleting work directory"
+      sudo rm -rf ./
+      echo "Deleting file in GOPATH"
+      sudo rm -rf '$(GOPATH)'
+    name: DeployAKSEngine
     displayName: Deploy AKS-Engine
     workingDirectory: "$(modulePath)"
 

From 92bea6475ed34fcc662ac16dfc3d3f54ce739b7e Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Thu, 14 Nov 2019 03:15:34 -0800
Subject: [PATCH 29/31] pipeline

---
 .pipelines/e2e-step-template.yaml | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index 30d9eac624..31f064ac21 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -32,8 +32,12 @@ steps:
       export CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-${{ parameters.clusterDefinitionCniBuildOS }}-amd64-$(Tag)${{ parameters.clusterDefinitionCniBuildExt }}'"'
       export CNI_TYPE=${{ parameters.clusterDefinitionCniTypeKey }}
       echo CNI type is $CNI_TYPE
-      sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":$CNI_URL|g" '${{ parameters.clusterDefinition }}'
+      #sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":$CNI_URL|g" '${{ parameters.clusterDefinition }}'	      
+      # sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":$CNI_URL|g" '${{ parameters.clusterDefinition }}'
+      cat '${{ parameters.clusterDefinition }}' | jq --arg cnikey $CNI_TYPE --arg cniurl $CNI_URL '.properties.orchestratorProfile.kubernetesConfig[$cnikey]= $cniurl' > '${{ parameters.clusterDefinition }}'.tmp	      #
       sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" '${{ parameters.clusterDefinition }}'
+      #sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" '${{ parameters.clusterDefinition }}'	
+      cat '${{ parameters.clusterDefinition }}'.tmp | jq --arg tag $(Tag) '.properties.orchestratorProfile.kubernetesConfig.azureCNIVersion = $tag' > '${{ parameters.clusterDefinition }}'
       echo "Running E2E tests against a cluster built with the following API model:"
       cat '${{ parameters.clusterDefinition }}'
       cp ${{ parameters.clusterDefinition }} clusterDefinition.json

From 1886d44cfb63b3a6fb9803464f9cb890eef8cccd Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Thu, 14 Nov 2019 03:16:01 -0800
Subject: [PATCH 30/31] pipeline

---
 .pipelines/e2e-step-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index 31f064ac21..1541437b25 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -35,7 +35,7 @@ steps:
       #sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":$CNI_URL|g" '${{ parameters.clusterDefinition }}'	      
       # sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":$CNI_URL|g" '${{ parameters.clusterDefinition }}'
       cat '${{ parameters.clusterDefinition }}' | jq --arg cnikey $CNI_TYPE --arg cniurl $CNI_URL '.properties.orchestratorProfile.kubernetesConfig[$cnikey]= $cniurl' > '${{ parameters.clusterDefinition }}'.tmp	      #
-      sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" '${{ parameters.clusterDefinition }}'
+      # sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" '${{ parameters.clusterDefinition }}'
       #sed -i "s|\"azureCNIVersion\":\".*\"|\"azureCNIVersion\":\"$(Tag)\"|g" '${{ parameters.clusterDefinition }}'	
       cat '${{ parameters.clusterDefinition }}'.tmp | jq --arg tag $(Tag) '.properties.orchestratorProfile.kubernetesConfig.azureCNIVersion = $tag' > '${{ parameters.clusterDefinition }}'
       echo "Running E2E tests against a cluster built with the following API model:"

From 692306297c6618b1ce6c9bf313161fc0fad7145c Mon Sep 17 00:00:00 2001
From: Mathew Merrick <mathew.merrick@microsoft.com>
Date: Thu, 14 Nov 2019 03:30:07 -0800
Subject: [PATCH 31/31] pipeline

---
 .pipelines/e2e-step-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/e2e-step-template.yaml b/.pipelines/e2e-step-template.yaml
index 1541437b25..904a9100b5 100644
--- a/.pipelines/e2e-step-template.yaml
+++ b/.pipelines/e2e-step-template.yaml
@@ -29,7 +29,7 @@ steps:
 
   - bash: |
       ls -lah
-      export CNI_URL='"'https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-${{ parameters.clusterDefinitionCniBuildOS }}-amd64-$(Tag)${{ parameters.clusterDefinitionCniBuildExt }}'"'
+      export CNI_URL=https://$(ARTIFACT_STORAGE).blob.core.windows.net/acn-$(CommitHash)/azure-vnet-cni-${{ parameters.clusterDefinitionCniBuildOS }}-amd64-$(Tag)${{ parameters.clusterDefinitionCniBuildExt }}
       export CNI_TYPE=${{ parameters.clusterDefinitionCniTypeKey }}
       echo CNI type is $CNI_TYPE
       #sed -i "s|\"$CNI_TYPE\":\".*\"|\"$CNI_TYPE\":$CNI_URL|g" '${{ parameters.clusterDefinition }}'