From bef4c568286fc023eb5e4332c4159815ee1fe4aa Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Wed, 3 Mar 2021 15:14:34 -0800
Subject: [PATCH 01/36] done

---
 .../aks-swift/e2e-step-template.yaml          | 71 ++++++++++++++++++-
 1 file changed, 68 insertions(+), 3 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 576a3e6f55..ac1532c365 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -31,16 +31,81 @@ steps:
       addSpnToEnvironment: true
       inlineScript: |
         mkdir -p ~/.kube/
-        az keyvault secret show --name $(KV_AKS_SWIFT_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//'
-        az keyvault secret show --name $(KV_AKS_SWIFT_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//' > ~/.kube/config
+        echo "DNC Underlay config"
+        az keyvault secret show --name $(KV_DNC_UNDERLAY_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//'
+        az keyvault secret show --name $(KV_DNC_UNDERLAY_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//' > ~/.kube/dnc-underlay
+        echo "Dirty ACN cx cluster config"
+        az keyvault secret show --name $(KV_AKS_SWIFT_CX_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//'
+        az keyvault secret show --name $(KV_AKS_SWIFT_CX_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//' > ~/.kube/dirty-acn-cx-cluster
     displayName: Pull Kubeconfig from Keyvault
     condition: succeeded()
 
+  - script: |
+      set -e
+      echo "installing kubectl"
+      curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+      sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
+      echo "Installing dirty CNS version by overriding kube-addon-manager's ConfigMap on the underlay cluster for the Stable DNC namespace $(STABLE_DNC_NAMESPACE)"
+      export KUBECONFIG=~/.kube/dnc-underlay
+      kubectl get cm -n $STABLE_DNC_NAMESPACE kube-addons -ojson > configmap.json
+      
+      echo "Replacing configmap image value for CNS"
+      # This line is gross I know, unfortunatly the addon configmap squishes each deployment onto one big line so I had to do a regex search and replace, no json or yaml editing available
+      sed -i 's/[[:alnum:]\.\:\/\-]*azure-cns:[[:alnum:]\.\:\/\-]*/'"$IMAGE_REGISTRY"'\/azure-cns:'"$(EnvironmentalVariables.Tag)"'-test\\/' configmap.json
+      kubectl replace -f configmap.json
+
+      # Kill a kube addon manager pod to reboot the system
+      echo "killing addonmanager pod to reboot CNS on cx cluster"
+      POD_TO_DELETE=`kubectl get po -n $STABLE_DNC_NAMESPACE -l k8s-app=kube-addon-manager | grep -o -m 1 'kube-addon-manager\S*'`
+      kubectl delete pod -n $STABLE_DNC_NAMESPACE $POD_TO_DELETE
+
+      # Wait until pod is back up
+      export KUBECONFIG=~/.kube/dirty-acn-cx-cluster
+      echo "Waiting for CNS dameonset to be ready..."
+
+      NUM_NODES=`kubectl get no | wc -l`
+      NUM_NODES=$[$NUM_NODES-1]
+
+      export i="1"
+      export attempts="100"
+      while [[ $(kubectl get ds -n kube-system azure-cns -ojson | jq '.status.numberReady') != $NUM_NODES ]]; do
+        sleep 3;
+        i=$[$i+1]
+        if [[ $i -gt $attempts ]]
+        then
+          echo "CNS daemonset never got ready, exiting"
+          exit 1
+        fi
+      done
+
+      echo "Daemonset is ready"
+      kubectl get ds -n kube-system azure-cns
+      kubectl get po -owide -n kube-system -l k8s-app=azure-cns
+
+    name: "installcns"
+    displayName: "Install CNS"
+    condition: succeeded()
+
   - script: |
       ls -lah
       pwd
-      sudo -E env "PATH=$PATH" VERSION=$(ImageTag) INSTALL_CNI=true INSTALL_CNS=true make test-integration
+      cp ~/.kube/dirty-acn-cx-cluster ~/.kube/config
+      export KUBECONFIG=~/.kube/config
+      sudo -E env "PATH=$PATH" VERSION=$(ImageTag) INSTALL_CNI=true INSTALL_CNS=false make test-integration
     name: "aksswifte2e"
     displayName: "Run AKS Swift E2E"
 
+  - script: |
+      export KUBECONFIG=~/.kube/dnc-underlay
+      kubectl get cm -n $STABLE_DNC_NAMESPACE kube-addons -ojson > configmap.json
+      
+      echo "Replacing configmap image value for CNS"
+      # This line is gross I know, unfortunatly the addon configmap squishes each deployment onto one big line so I had to do a regex search and replace, no json or yaml editing available
+      sed -i 's/[[:alnum:]\.\:\/\-]*azure-cns:[[:alnum:]\.\:\/\-]*/'"$STABLE_CNS_IMAGE_FULL"'\\/' configmap.json
+      kubectl replace -f configmap.json
+
+    name: "cleanup"
+    displayName: "Restore to stable CNS"
+    condition: always()
+
 

From 2154d87ce2f7813bd739cfc41f017306ff6ccd26 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Wed, 3 Mar 2021 15:15:38 -0800
Subject: [PATCH 02/36] less comments

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index ac1532c365..3177f22fcf 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -54,12 +54,10 @@ steps:
       sed -i 's/[[:alnum:]\.\:\/\-]*azure-cns:[[:alnum:]\.\:\/\-]*/'"$IMAGE_REGISTRY"'\/azure-cns:'"$(EnvironmentalVariables.Tag)"'-test\\/' configmap.json
       kubectl replace -f configmap.json
 
-      # Kill a kube addon manager pod to reboot the system
       echo "killing addonmanager pod to reboot CNS on cx cluster"
       POD_TO_DELETE=`kubectl get po -n $STABLE_DNC_NAMESPACE -l k8s-app=kube-addon-manager | grep -o -m 1 'kube-addon-manager\S*'`
       kubectl delete pod -n $STABLE_DNC_NAMESPACE $POD_TO_DELETE
 
-      # Wait until pod is back up
       export KUBECONFIG=~/.kube/dirty-acn-cx-cluster
       echo "Waiting for CNS dameonset to be ready..."
 

From f8861ecd023cac3001949952c35e7779cc395fc0 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Wed, 3 Mar 2021 18:07:15 -0800
Subject: [PATCH 03/36] fix

---
 .pipelines/pipeline.yaml                                  | 1 +
 .pipelines/singletenancy/aks-swift/e2e-job-template.yaml  | 2 ++
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 5 ++++-
 3 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
index 1ad4c40b30..c12d50c22c 100644
--- a/.pipelines/pipeline.yaml
+++ b/.pipelines/pipeline.yaml
@@ -205,6 +205,7 @@ stages:
       name: "aks_on_swift_e2e"
       displayName: AKS on Swift
       pipelineBuildImage: "$(BUILD_IMAGE)"
+      cnsImageTag: "$(EnvironmentalVariables.Tag)"
 
   - template: singletenancy/aks-engine/e2e-job-template.yaml
     parameters:
diff --git a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
index 77d45d254e..c9fa737030 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
@@ -2,6 +2,7 @@ parameters:
   name: ""
   displayName: ""
   pipelineBuildImage: "$(BUILD_IMAGE)"
+  cnsImageTag: ""
 
 stages:
   - stage: ${{ parameters.name }}
@@ -27,3 +28,4 @@ stages:
           - template: e2e-step-template.yaml
             parameters:
               name: ${{ parameters.name }}
+              cnsImageTag: ${{ parameters.cnsImageTag }}
diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 3177f22fcf..820987a982 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -1,5 +1,6 @@
 parameters:
   name: ""
+  cnsImageTag: ""
 
 steps:
   - bash: |
@@ -42,6 +43,7 @@ steps:
 
   - script: |
       set -e
+      printenv
       echo "installing kubectl"
       curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
       sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
@@ -51,7 +53,7 @@ steps:
       
       echo "Replacing configmap image value for CNS"
       # This line is gross I know, unfortunatly the addon configmap squishes each deployment onto one big line so I had to do a regex search and replace, no json or yaml editing available
-      sed -i 's/[[:alnum:]\.\:\/\-]*azure-cns:[[:alnum:]\.\:\/\-]*/'"$IMAGE_REGISTRY"'\/azure-cns:'"$(EnvironmentalVariables.Tag)"'-test\\/' configmap.json
+      sed -i 's/[[:alnum:]\.\:\/\-]*azure-cns:[[:alnum:]\.\:\/\-]*/'"$IMAGE_REGISTRY"'\/azure-cns:'"${{parameters.cnsImageTag}}"'-test\\/' configmap.json
       kubectl replace -f configmap.json
 
       echo "killing addonmanager pod to reboot CNS on cx cluster"
@@ -99,6 +101,7 @@ steps:
       
       echo "Replacing configmap image value for CNS"
       # This line is gross I know, unfortunatly the addon configmap squishes each deployment onto one big line so I had to do a regex search and replace, no json or yaml editing available
+      echo $STABLE_CNS_IMAGE_FULL
       sed -i 's/[[:alnum:]\.\:\/\-]*azure-cns:[[:alnum:]\.\:\/\-]*/'"$STABLE_CNS_IMAGE_FULL"'\\/' configmap.json
       kubectl replace -f configmap.json
 

From 961f06150b1da235f6fdba72a3fc403f969b296d Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Wed, 3 Mar 2021 18:08:56 -0800
Subject: [PATCH 04/36] test

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 820987a982..ddfdc0ef3e 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -50,10 +50,15 @@ steps:
       echo "Installing dirty CNS version by overriding kube-addon-manager's ConfigMap on the underlay cluster for the Stable DNC namespace $(STABLE_DNC_NAMESPACE)"
       export KUBECONFIG=~/.kube/dnc-underlay
       kubectl get cm -n $STABLE_DNC_NAMESPACE kube-addons -ojson > configmap.json
+      echo "config map before"
+      cat configmap.json | grep azure-cns.yaml
       
       echo "Replacing configmap image value for CNS"
       # This line is gross I know, unfortunatly the addon configmap squishes each deployment onto one big line so I had to do a regex search and replace, no json or yaml editing available
       sed -i 's/[[:alnum:]\.\:\/\-]*azure-cns:[[:alnum:]\.\:\/\-]*/'"$IMAGE_REGISTRY"'\/azure-cns:'"${{parameters.cnsImageTag}}"'-test\\/' configmap.json
+
+      echo "config map after"
+      cat configmap.json | grep azure-cns.yaml
       kubectl replace -f configmap.json
 
       echo "killing addonmanager pod to reboot CNS on cx cluster"

From afcdff943dee0909fa48482b728abb1132d9105f Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Wed, 3 Mar 2021 18:27:36 -0800
Subject: [PATCH 05/36] done

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index ddfdc0ef3e..90ad069c14 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -106,6 +106,8 @@ steps:
       
       echo "Replacing configmap image value for CNS"
       # This line is gross I know, unfortunatly the addon configmap squishes each deployment onto one big line so I had to do a regex search and replace, no json or yaml editing available
+      export STABLE_CNS_IMAGE_FULL=`echo $STABLE_CNS_IMAGE_FULL | sed 's/\//\\\//g'`
+      echo "Sed friendly cns image path"
       echo $STABLE_CNS_IMAGE_FULL
       sed -i 's/[[:alnum:]\.\:\/\-]*azure-cns:[[:alnum:]\.\:\/\-]*/'"$STABLE_CNS_IMAGE_FULL"'\\/' configmap.json
       kubectl replace -f configmap.json

From 5a861b0a65a58a8a9178853b9894b2fd78690f67 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Thu, 4 Mar 2021 09:20:29 -0800
Subject: [PATCH 06/36] why

---
 .../singletenancy/aks-swift/e2e-step-template.yaml     | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 90ad069c14..5fef3b6a8f 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -55,7 +55,7 @@ steps:
       
       echo "Replacing configmap image value for CNS"
       # This line is gross I know, unfortunatly the addon configmap squishes each deployment onto one big line so I had to do a regex search and replace, no json or yaml editing available
-      sed -i 's/[[:alnum:]\.\:\/\-]*azure-cns:[[:alnum:]\.\:\/\-]*/'"$IMAGE_REGISTRY"'\/azure-cns:'"${{parameters.cnsImageTag}}"'-test\\/' configmap.json
+      sed -i 's/[[:alnum:]\.\:\/\-]*azure-cns:[[:alnum:]\.\:\/\-]*/'"$IMAGE_REGISTRY"'\/azure-cns:'"$IMAGETAG"'-test\\/' configmap.json
 
       echo "config map after"
       cat configmap.json | grep azure-cns.yaml
@@ -101,15 +101,15 @@ steps:
     displayName: "Run AKS Swift E2E"
 
   - script: |
+      printenv
       export KUBECONFIG=~/.kube/dnc-underlay
       kubectl get cm -n $STABLE_DNC_NAMESPACE kube-addons -ojson > configmap.json
-      
       echo "Replacing configmap image value for CNS"
       # This line is gross I know, unfortunatly the addon configmap squishes each deployment onto one big line so I had to do a regex search and replace, no json or yaml editing available
-      export STABLE_CNS_IMAGE_FULL=`echo $STABLE_CNS_IMAGE_FULL | sed 's/\//\\\//g'`
+      export STABLE_CNS_IMAGE_FULL_SED=`echo $STABLE_CNS_IMAGE_FULL | sed 's/\//\\\//g'`
       echo "Sed friendly cns image path"
-      echo $STABLE_CNS_IMAGE_FULL
-      sed -i 's/[[:alnum:]\.\:\/\-]*azure-cns:[[:alnum:]\.\:\/\-]*/'"$STABLE_CNS_IMAGE_FULL"'\\/' configmap.json
+      echo $STABLE_CNS_IMAGE_FULL_SED
+      sed -i 's/[[:alnum:]\.\:\/\-]*azure-cns:[[:alnum:]\.\:\/\-]*/'"$STABLE_CNS_IMAGE_FULL_SED"'\\/' configmap.json
       kubectl replace -f configmap.json
 
     name: "cleanup"

From 42494aabfbf49e459d8aebf393ff999263ed84bf Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Thu, 4 Mar 2021 10:38:48 -0800
Subject: [PATCH 07/36] testing

---
 .pipelines/pipeline.yaml                                  | 1 -
 .pipelines/singletenancy/aks-swift/e2e-job-template.yaml  | 1 -
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 7 +++----
 3 files changed, 3 insertions(+), 6 deletions(-)

diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
index c12d50c22c..1ad4c40b30 100644
--- a/.pipelines/pipeline.yaml
+++ b/.pipelines/pipeline.yaml
@@ -205,7 +205,6 @@ stages:
       name: "aks_on_swift_e2e"
       displayName: AKS on Swift
       pipelineBuildImage: "$(BUILD_IMAGE)"
-      cnsImageTag: "$(EnvironmentalVariables.Tag)"
 
   - template: singletenancy/aks-engine/e2e-job-template.yaml
     parameters:
diff --git a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
index c9fa737030..de0d280c76 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
@@ -28,4 +28,3 @@ stages:
           - template: e2e-step-template.yaml
             parameters:
               name: ${{ parameters.name }}
-              cnsImageTag: ${{ parameters.cnsImageTag }}
diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 5fef3b6a8f..ad23a87216 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -1,6 +1,5 @@
 parameters:
   name: ""
-  cnsImageTag: ""
 
 steps:
   - bash: |
@@ -49,13 +48,15 @@ steps:
       sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
       echo "Installing dirty CNS version by overriding kube-addon-manager's ConfigMap on the underlay cluster for the Stable DNC namespace $(STABLE_DNC_NAMESPACE)"
       export KUBECONFIG=~/.kube/dnc-underlay
+      kubectl cluster-info
+      kubectl get po -n $STABLE_DNC_NAMESPACE -owide
       kubectl get cm -n $STABLE_DNC_NAMESPACE kube-addons -ojson > configmap.json
       echo "config map before"
       cat configmap.json | grep azure-cns.yaml
       
       echo "Replacing configmap image value for CNS"
       # This line is gross I know, unfortunatly the addon configmap squishes each deployment onto one big line so I had to do a regex search and replace, no json or yaml editing available
-      sed -i 's/[[:alnum:]\.\:\/\-]*azure-cns:[[:alnum:]\.\:\/\-]*/'"$IMAGE_REGISTRY"'\/azure-cns:'"$IMAGETAG"'-test\\/' configmap.json
+      sed -i 's/[[:alnum:]\.\:\/\-]*azure-cns:[[:alnum:]\.\:\/\-]*/'"$IMAGE_REGISTRY"'\/azure-cns:'"$IMAGETAG"'\\/' configmap.json
 
       echo "config map after"
       cat configmap.json | grep azure-cns.yaml
@@ -115,5 +116,3 @@ steps:
     name: "cleanup"
     displayName: "Restore to stable CNS"
     condition: always()
-
-

From dc14e17546ce5edea125ff3a57fd0dd9fac6f346 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Thu, 4 Mar 2021 17:46:12 -0800
Subject: [PATCH 08/36] fix

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index ad23a87216..c5649ad4fb 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -107,7 +107,8 @@ steps:
       kubectl get cm -n $STABLE_DNC_NAMESPACE kube-addons -ojson > configmap.json
       echo "Replacing configmap image value for CNS"
       # This line is gross I know, unfortunatly the addon configmap squishes each deployment onto one big line so I had to do a regex search and replace, no json or yaml editing available
-      export STABLE_CNS_IMAGE_FULL_SED=`echo $STABLE_CNS_IMAGE_FULL | sed 's/\//\\\//g'`
+      STABLE_CNS_IMAGE_FULL_SED=$(echo $STABLE_CNS_IMAGE_FULL | sed 's/\//\\\//g')
+      echo $STABLE_CNS_IMAGE_FULL
       echo "Sed friendly cns image path"
       echo $STABLE_CNS_IMAGE_FULL_SED
       sed -i 's/[[:alnum:]\.\:\/\-]*azure-cns:[[:alnum:]\.\:\/\-]*/'"$STABLE_CNS_IMAGE_FULL_SED"'\\/' configmap.json

From c71714b96d9340d2e4e0b88498a738ab04871846 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Thu, 4 Mar 2021 17:52:36 -0800
Subject: [PATCH 09/36] logs

---
 .../aks-swift/e2e-step-template.yaml          | 30 +++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index c5649ad4fb..fecaffe212 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -93,10 +93,21 @@ steps:
     condition: succeeded()
 
   - script: |
-      ls -lah
-      pwd
+      set -e
+      echo "Versions about to run"
+      export KUBECONFIG=~/.kube/dnc-underlay
+      kubectl get po -n $STABLE_DNC_NAMESPACE -l app=dnc -oyaml | grep -B 4 -A 4 -m 2 image:
+      
       cp ~/.kube/dirty-acn-cx-cluster ~/.kube/config
       export KUBECONFIG=~/.kube/config
+      kubectl get po -n kube-system -l k8s-app=azure-cns -oyaml | grep -m 2 -B 4 -A 4 image:
+
+      # Vendoring doesn't include tests so we have to git clone the ACN github
+      git clone https://github.com/Azure/azure-container-networking.git
+      cd azure-container-networking/
+      
+      kubectl cluster-info
+      kubectl get po -owide -A
       sudo -E env "PATH=$PATH" VERSION=$(ImageTag) INSTALL_CNI=true INSTALL_CNS=false make test-integration
     name: "aksswifte2e"
     displayName: "Run AKS Swift E2E"
@@ -117,3 +128,18 @@ steps:
     name: "cleanup"
     displayName: "Restore to stable CNS"
     condition: always()
+
+  - script: |
+      echo "Logs will be available as a build artifact"
+      cd ./output 
+      export KUBECONFIG=~/.kube/dnc-underlay
+      kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c dnc --prefix > dnc-logs.txt
+      kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c requestcontroller --prefix > dnc-rc-logs.txt
+
+      export KUBECONFIG=~/.kube/dirty-acn-cx-cluster
+      kubectl logs -n kube-system -l k8s-app=azure-cns --tail -1 --prefix > cns-logs.txt
+
+    workingDirectory: "$(modulePath)"
+    name: "GetLogs"
+    displayName: "Get logs"
+    condition: always()

From fff6594e0c32b4c9e588b40eb7f922bf7a68d2af Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Thu, 4 Mar 2021 19:58:06 -0800
Subject: [PATCH 10/36] refresh dnc

---
 .../aks-swift/e2e-step-template.yaml          | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index fecaffe212..5d2a14c152 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -92,6 +92,31 @@ steps:
     displayName: "Install CNS"
     condition: succeeded()
 
+  - script: |
+      echo "Refreshing dnc pod"
+      export KUBECONFIG=~/.kube/dnc-underlay
+      kubectl get po -n $STABLE_DNC_NAMESPACE -owide
+      kubectl delete po -n $STABLE_DNC_NAMESPACE -l app=dnc
+      echo "Waiting for new dnc pod to be ready..."
+      export i="1"
+      export attempts="100"
+      while [[ $(kubectl get pods -n $STABLE_DNC_NAMESPACE -l app=dnc -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do
+        sleep 3;
+        i=$[$i+1]
+        if [[ $i -gt $attempts ]]
+        then
+          echo "Pod never showed up, exiting"
+          exit 1
+        fi
+      done
+
+      echo "Pod is up and running"
+      kubectl get po -n $STABLE_DNC_NAMESPACE -owide
+      kubectl get po -n $STABLE_DNC_NAMESPACE -l app=dnc -oyaml | grep -B 4 -A 4 -m 2 image:
+    name: "refreshdnc"
+    displayName: "Refresh DNC"
+    condition: succeeded()
+
   - script: |
       set -e
       echo "Versions about to run"

From ea24e2eae575f26ad6f82c41ec0319299331f784 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Thu, 4 Mar 2021 20:02:14 -0800
Subject: [PATCH 11/36] Wait for cns to come back up after restore

---
 .../aks-swift/e2e-step-template.yaml          | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 5d2a14c152..b830134521 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -149,6 +149,29 @@ steps:
       echo $STABLE_CNS_IMAGE_FULL_SED
       sed -i 's/[[:alnum:]\.\:\/\-]*azure-cns:[[:alnum:]\.\:\/\-]*/'"$STABLE_CNS_IMAGE_FULL_SED"'\\/' configmap.json
       kubectl replace -f configmap.json
+      sleep 15
+
+      export KUBECONFIG=~/.kube/dirty-acn-cx-cluster
+      echo "Waiting for CNS dameonset to be ready..."
+
+      NUM_NODES=`kubectl get no | wc -l`
+      NUM_NODES=$[$NUM_NODES-1]
+
+      export i="1"
+      export attempts="100"
+      while [[ $(kubectl get ds -n kube-system azure-cns -ojson | jq '.status.numberReady') != $NUM_NODES ]]; do
+        sleep 3;
+        i=$[$i+1]
+        if [[ $i -gt $attempts ]]
+        then
+          echo "CNS daemonset never got ready, exiting"
+          exit 1
+        fi
+      done
+
+      echo "Daemonset is ready"
+      kubectl get ds -n kube-system azure-cns
+      kubectl get po -owide -n kube-system -l k8s-app=azure-cns
 
     name: "cleanup"
     displayName: "Restore to stable CNS"

From 38d3fcd66549500733da35c8138c1d73699862d1 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Mon, 29 Mar 2021 11:04:29 -0700
Subject: [PATCH 12/36] logs

---
 .../singletenancy/aks-swift/e2e-step-template.yaml     | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index b830134521..d76f887f32 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -179,7 +179,9 @@ steps:
 
   - script: |
       echo "Logs will be available as a build artifact"
-      cd ./output 
+      mkdir aks-swift-output
+      cd aks-swift-output
+      pwd
       export KUBECONFIG=~/.kube/dnc-underlay
       kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c dnc --prefix > dnc-logs.txt
       kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c requestcontroller --prefix > dnc-rc-logs.txt
@@ -191,3 +193,9 @@ steps:
     name: "GetLogs"
     displayName: "Get logs"
     condition: always()
+
+  - task: CopyFiles@2
+    inputs:
+      sourceFolder: "$(modulePath)/aks-swift-output"
+      targetFolder: $(Build.ArtifactStagingDirectory)/aks-swift-output
+    condition: always()

From 8ae53c5e3c7fb4c91fa9b9d6297c8270313cc574 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Mon, 29 Mar 2021 12:37:36 -0700
Subject: [PATCH 13/36] logs again

---
 .../singletenancy/aks-swift/e2e-job-template.yaml |  1 +
 .../aks-swift/e2e-step-template.yaml              | 15 +++++++++------
 2 files changed, 10 insertions(+), 6 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
index de0d280c76..b62926b18c 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
@@ -24,6 +24,7 @@ stages:
           GOPATH: "$(Agent.TempDirectory)/go" # Go workspace path
           GOBIN: "$(GOPATH)/bin" # Go binaries path
           ImageTag: $[ stagedependencies.build_and_test.unit_tests.outputs['EnvironmentalVariables.ImageTag'] ]
+          modulePath: "$(GOPATH)/src/github.com/Azure/azure-container-networking"
         steps:
           - template: e2e-step-template.yaml
             parameters:
diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index d76f887f32..513d151348 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -121,20 +121,17 @@ steps:
       set -e
       echo "Versions about to run"
       export KUBECONFIG=~/.kube/dnc-underlay
-      kubectl get po -n $STABLE_DNC_NAMESPACE -l app=dnc -oyaml | grep -B 4 -A 4 -m 2 image:
+      kubectl get po -n $STABLE_DNC_NAMESPACE -l app=dnc -oyaml | grep -B 4 -A 4 -m 4 image:
       
       cp ~/.kube/dirty-acn-cx-cluster ~/.kube/config
       export KUBECONFIG=~/.kube/config
       kubectl get po -n kube-system -l k8s-app=azure-cns -oyaml | grep -m 2 -B 4 -A 4 image:
-
-      # Vendoring doesn't include tests so we have to git clone the ACN github
-      git clone https://github.com/Azure/azure-container-networking.git
-      cd azure-container-networking/
       
       kubectl cluster-info
       kubectl get po -owide -A
       sudo -E env "PATH=$PATH" VERSION=$(ImageTag) INSTALL_CNI=true INSTALL_CNS=false make test-integration
     name: "aksswifte2e"
+    workingDirectory: "$(modulePath)"
     displayName: "Run AKS Swift E2E"
 
   - script: |
@@ -188,7 +185,7 @@ steps:
 
       export KUBECONFIG=~/.kube/dirty-acn-cx-cluster
       kubectl logs -n kube-system -l k8s-app=azure-cns --tail -1 --prefix > cns-logs.txt
-
+      ls
     workingDirectory: "$(modulePath)"
     name: "GetLogs"
     displayName: "Get logs"
@@ -199,3 +196,9 @@ steps:
       sourceFolder: "$(modulePath)/aks-swift-output"
       targetFolder: $(Build.ArtifactStagingDirectory)/aks-swift-output
     condition: always()
+  
+  - task: PublishBuildArtifacts@1
+    inputs:
+      artifactName: aks-swift-output
+      pathtoPublish: "$(Build.ArtifactStagingDirectory)/aks-swift-output"
+    condition: always()

From 67212a9d5c5a147ddf2c54db1bec708872589676 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Mon, 29 Mar 2021 13:14:14 -0700
Subject: [PATCH 14/36] without module working dir

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 1 -
 1 file changed, 1 deletion(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 513d151348..62b043fee2 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -131,7 +131,6 @@ steps:
       kubectl get po -owide -A
       sudo -E env "PATH=$PATH" VERSION=$(ImageTag) INSTALL_CNI=true INSTALL_CNS=false make test-integration
     name: "aksswifte2e"
-    workingDirectory: "$(modulePath)"
     displayName: "Run AKS Swift E2E"
 
   - script: |

From 88d9a9bbbf6d325d351ee39254a9d39433165869 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Mon, 29 Mar 2021 14:06:50 -0700
Subject: [PATCH 15/36] sleep

---
 .pipelines/singletenancy/aks-swift/e2e-job-template.yaml  | 1 -
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 2 ++
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
index b62926b18c..7974863f98 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
@@ -2,7 +2,6 @@ parameters:
   name: ""
   displayName: ""
   pipelineBuildImage: "$(BUILD_IMAGE)"
-  cnsImageTag: ""
 
 stages:
   - stage: ${{ parameters.name }}
diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 62b043fee2..d14a073f2b 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -98,6 +98,7 @@ steps:
       kubectl get po -n $STABLE_DNC_NAMESPACE -owide
       kubectl delete po -n $STABLE_DNC_NAMESPACE -l app=dnc
       echo "Waiting for new dnc pod to be ready..."
+      sleep 15
       export i="1"
       export attempts="100"
       while [[ $(kubectl get pods -n $STABLE_DNC_NAMESPACE -l app=dnc -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do
@@ -149,6 +150,7 @@ steps:
 
       export KUBECONFIG=~/.kube/dirty-acn-cx-cluster
       echo "Waiting for CNS dameonset to be ready..."
+      sleep 30
 
       NUM_NODES=`kubectl get no | wc -l`
       NUM_NODES=$[$NUM_NODES-1]

From f48ad2a1b51bb147c41c4cf34613d3dd8d0d920e Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Tue, 30 Mar 2021 10:32:00 -0700
Subject: [PATCH 16/36] simpler execution, install cns via integration flag

---
 .../aks-swift/e2e-step-template.yaml          | 136 +-----------------
 1 file changed, 5 insertions(+), 131 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index d14a073f2b..e80852c0cc 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -36,145 +36,19 @@ steps:
         az keyvault secret show --name $(KV_DNC_UNDERLAY_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//' > ~/.kube/dnc-underlay
         echo "Dirty ACN cx cluster config"
         az keyvault secret show --name $(KV_AKS_SWIFT_CX_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//'
-        az keyvault secret show --name $(KV_AKS_SWIFT_CX_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//' > ~/.kube/dirty-acn-cx-cluster
+        az keyvault secret show --name $(KV_AKS_SWIFT_CX_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//' > ~/.kube/config
     displayName: Pull Kubeconfig from Keyvault
     condition: succeeded()
 
   - script: |
-      set -e
-      printenv
-      echo "installing kubectl"
-      curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
-      sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
-      echo "Installing dirty CNS version by overriding kube-addon-manager's ConfigMap on the underlay cluster for the Stable DNC namespace $(STABLE_DNC_NAMESPACE)"
-      export KUBECONFIG=~/.kube/dnc-underlay
-      kubectl cluster-info
-      kubectl get po -n $STABLE_DNC_NAMESPACE -owide
-      kubectl get cm -n $STABLE_DNC_NAMESPACE kube-addons -ojson > configmap.json
-      echo "config map before"
-      cat configmap.json | grep azure-cns.yaml
-      
-      echo "Replacing configmap image value for CNS"
-      # This line is gross I know, unfortunatly the addon configmap squishes each deployment onto one big line so I had to do a regex search and replace, no json or yaml editing available
-      sed -i 's/[[:alnum:]\.\:\/\-]*azure-cns:[[:alnum:]\.\:\/\-]*/'"$IMAGE_REGISTRY"'\/azure-cns:'"$IMAGETAG"'\\/' configmap.json
-
-      echo "config map after"
-      cat configmap.json | grep azure-cns.yaml
-      kubectl replace -f configmap.json
-
-      echo "killing addonmanager pod to reboot CNS on cx cluster"
-      POD_TO_DELETE=`kubectl get po -n $STABLE_DNC_NAMESPACE -l k8s-app=kube-addon-manager | grep -o -m 1 'kube-addon-manager\S*'`
-      kubectl delete pod -n $STABLE_DNC_NAMESPACE $POD_TO_DELETE
-
-      export KUBECONFIG=~/.kube/dirty-acn-cx-cluster
-      echo "Waiting for CNS dameonset to be ready..."
-
-      NUM_NODES=`kubectl get no | wc -l`
-      NUM_NODES=$[$NUM_NODES-1]
-
-      export i="1"
-      export attempts="100"
-      while [[ $(kubectl get ds -n kube-system azure-cns -ojson | jq '.status.numberReady') != $NUM_NODES ]]; do
-        sleep 3;
-        i=$[$i+1]
-        if [[ $i -gt $attempts ]]
-        then
-          echo "CNS daemonset never got ready, exiting"
-          exit 1
-        fi
-      done
-
-      echo "Daemonset is ready"
-      kubectl get ds -n kube-system azure-cns
-      kubectl get po -owide -n kube-system -l k8s-app=azure-cns
-
-    name: "installcns"
-    displayName: "Install CNS"
-    condition: succeeded()
-
-  - script: |
-      echo "Refreshing dnc pod"
-      export KUBECONFIG=~/.kube/dnc-underlay
-      kubectl get po -n $STABLE_DNC_NAMESPACE -owide
-      kubectl delete po -n $STABLE_DNC_NAMESPACE -l app=dnc
-      echo "Waiting for new dnc pod to be ready..."
-      sleep 15
-      export i="1"
-      export attempts="100"
-      while [[ $(kubectl get pods -n $STABLE_DNC_NAMESPACE -l app=dnc -o 'jsonpath={..status.conditions[?(@.type=="Ready")].status}') != "True" ]]; do
-        sleep 3;
-        i=$[$i+1]
-        if [[ $i -gt $attempts ]]
-        then
-          echo "Pod never showed up, exiting"
-          exit 1
-        fi
-      done
-
-      echo "Pod is up and running"
-      kubectl get po -n $STABLE_DNC_NAMESPACE -owide
-      kubectl get po -n $STABLE_DNC_NAMESPACE -l app=dnc -oyaml | grep -B 4 -A 4 -m 2 image:
-    name: "refreshdnc"
-    displayName: "Refresh DNC"
-    condition: succeeded()
-
-  - script: |
-      set -e
-      echo "Versions about to run"
-      export KUBECONFIG=~/.kube/dnc-underlay
-      kubectl get po -n $STABLE_DNC_NAMESPACE -l app=dnc -oyaml | grep -B 4 -A 4 -m 4 image:
-      
-      cp ~/.kube/dirty-acn-cx-cluster ~/.kube/config
-      export KUBECONFIG=~/.kube/config
-      kubectl get po -n kube-system -l k8s-app=azure-cns -oyaml | grep -m 2 -B 4 -A 4 image:
-      
+      ls -lah
+      pwd
       kubectl cluster-info
       kubectl get po -owide -A
-      sudo -E env "PATH=$PATH" VERSION=$(ImageTag) INSTALL_CNI=true INSTALL_CNS=false make test-integration
+      sudo -E env "PATH=$PATH" VERSION=$(ImageTag) INSTALL_CNI=true INSTALL_CNS=true make test-integration
     name: "aksswifte2e"
     displayName: "Run AKS Swift E2E"
 
-  - script: |
-      printenv
-      export KUBECONFIG=~/.kube/dnc-underlay
-      kubectl get cm -n $STABLE_DNC_NAMESPACE kube-addons -ojson > configmap.json
-      echo "Replacing configmap image value for CNS"
-      # This line is gross I know, unfortunatly the addon configmap squishes each deployment onto one big line so I had to do a regex search and replace, no json or yaml editing available
-      STABLE_CNS_IMAGE_FULL_SED=$(echo $STABLE_CNS_IMAGE_FULL | sed 's/\//\\\//g')
-      echo $STABLE_CNS_IMAGE_FULL
-      echo "Sed friendly cns image path"
-      echo $STABLE_CNS_IMAGE_FULL_SED
-      sed -i 's/[[:alnum:]\.\:\/\-]*azure-cns:[[:alnum:]\.\:\/\-]*/'"$STABLE_CNS_IMAGE_FULL_SED"'\\/' configmap.json
-      kubectl replace -f configmap.json
-      sleep 15
-
-      export KUBECONFIG=~/.kube/dirty-acn-cx-cluster
-      echo "Waiting for CNS dameonset to be ready..."
-      sleep 30
-
-      NUM_NODES=`kubectl get no | wc -l`
-      NUM_NODES=$[$NUM_NODES-1]
-
-      export i="1"
-      export attempts="100"
-      while [[ $(kubectl get ds -n kube-system azure-cns -ojson | jq '.status.numberReady') != $NUM_NODES ]]; do
-        sleep 3;
-        i=$[$i+1]
-        if [[ $i -gt $attempts ]]
-        then
-          echo "CNS daemonset never got ready, exiting"
-          exit 1
-        fi
-      done
-
-      echo "Daemonset is ready"
-      kubectl get ds -n kube-system azure-cns
-      kubectl get po -owide -n kube-system -l k8s-app=azure-cns
-
-    name: "cleanup"
-    displayName: "Restore to stable CNS"
-    condition: always()
-
   - script: |
       echo "Logs will be available as a build artifact"
       mkdir aks-swift-output
@@ -184,7 +58,7 @@ steps:
       kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c dnc --prefix > dnc-logs.txt
       kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c requestcontroller --prefix > dnc-rc-logs.txt
 
-      export KUBECONFIG=~/.kube/dirty-acn-cx-cluster
+      export KUBECONFIG=~/.kube/config
       kubectl logs -n kube-system -l k8s-app=azure-cns --tail -1 --prefix > cns-logs.txt
       ls
     workingDirectory: "$(modulePath)"

From fb53514e52cf36138debeebefeb9086bb78daef9 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Tue, 30 Mar 2021 11:55:34 -0700
Subject: [PATCH 17/36] cns not installing?

---
 .../singletenancy/aks-swift/e2e-step-template.yaml       | 3 +++
 test/integration/setup_test.go                           | 9 +++++++--
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index e80852c0cc..a7b7ee5600 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -43,6 +43,9 @@ steps:
   - script: |
       ls -lah
       pwd
+      echo "installing kubectl"
+      curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+      sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
       kubectl cluster-info
       kubectl get po -owide -A
       sudo -E env "PATH=$PATH" VERSION=$(ImageTag) INSTALL_CNI=true INSTALL_CNS=true make test-integration
diff --git a/test/integration/setup_test.go b/test/integration/setup_test.go
index 1d118eca10..67c1c2337d 100644
--- a/test/integration/setup_test.go
+++ b/test/integration/setup_test.go
@@ -84,11 +84,14 @@ func TestMain(m *testing.M) {
 	} else {
 		log.Printf("Env %v not set to true, skipping", envInstallCNI)
 	}
-
+	fmt.Printf("Before install cns")
 	if installopt := os.Getenv(envInstallCNS); installopt != "" {
+		fmt.Printf("In if CNS")
 		// create dirty cns ds
 		if installCNS, err := strconv.ParseBool(installopt); err != nil && installCNS == true {
+			fmt.Printf("Thing was bool and true")
 			if cnscleanup, err = installCNSDaemonset(ctx, clientset, os.Getenv(envImageTag)); err != nil {
+				fmt.Printf("no error")
 				return
 			}
 		}
@@ -106,11 +109,13 @@ func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, i
 	)
 
 	if imageTag == "" {
+		fmt.Print("No image")
 		return nil, fmt.Errorf("Azure CNS image tag not set")
 	}
 
 	// setup daemonset
 	if cns, err = mustParseDaemonSet(cnsDaemonSetPath); err != nil {
+		fmt.Print("Must parse failed")
 		return nil, err
 	}
 
@@ -119,7 +124,7 @@ func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, i
 	cnsDaemonsetClient := clientset.AppsV1().DaemonSets(cns.Namespace)
 
 	log.Printf("Installing CNS with image %s", cns.Spec.Template.Spec.Containers[0].Image)
-
+	fmt.Printf("Installing CNS")
 	// setup the CNS configmap
 	if err := mustSetupConfigMap(ctx, clientset, cnsConfigMapPath); err != nil {
 		return nil, err

From e62ad62c5f3bf89d00d1b71f9a76c993d07e9b64 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Tue, 30 Mar 2021 12:24:57 -0700
Subject: [PATCH 18/36] debug

---
 test/integration/setup_test.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/test/integration/setup_test.go b/test/integration/setup_test.go
index 67c1c2337d..ca4e557f2f 100644
--- a/test/integration/setup_test.go
+++ b/test/integration/setup_test.go
@@ -96,6 +96,11 @@ func TestMain(m *testing.M) {
 			}
 		}
 	} else {
+		fmt.Printf("Env %v not set to true, skipping", envInstallCNS)
+		fmt.Printf("installopt %v", installopt)
+		fmt.Printf("InstallCNS %v", installCNS)
+		fmt.Printf("error %v", err)
+		fmt.Printf("envImageTag %v", envImageTag)
 		log.Printf("Env %v not set to true, skipping", envInstallCNS)
 	}
 

From a2b8d3328d904bcc0352179477e8d34e2309fa73 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Tue, 30 Mar 2021 13:09:13 -0700
Subject: [PATCH 19/36] debug

---
 test/integration/setup_test.go | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/test/integration/setup_test.go b/test/integration/setup_test.go
index ca4e557f2f..c2e0ebaf5c 100644
--- a/test/integration/setup_test.go
+++ b/test/integration/setup_test.go
@@ -94,13 +94,14 @@ func TestMain(m *testing.M) {
 				fmt.Printf("no error")
 				return
 			}
+		} else {
+			fmt.Printf("installopt %v", installopt)
+			fmt.Printf("InstallCNS %v", installCNS)
+			fmt.Printf("error %v", err)
+			fmt.Printf("envImageTag %v", os.Getenv(envImageTag))
 		}
 	} else {
 		fmt.Printf("Env %v not set to true, skipping", envInstallCNS)
-		fmt.Printf("installopt %v", installopt)
-		fmt.Printf("InstallCNS %v", installCNS)
-		fmt.Printf("error %v", err)
-		fmt.Printf("envImageTag %v", envImageTag)
 		log.Printf("Env %v not set to true, skipping", envInstallCNS)
 	}
 

From 072d560e166443d619f9d97dbc1214f6da6638b4 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Tue, 30 Mar 2021 14:48:23 -0700
Subject: [PATCH 20/36] reverse nil check on error, we want to proceed if there
 was no error

---
 test/integration/setup_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/integration/setup_test.go b/test/integration/setup_test.go
index c2e0ebaf5c..8272174e04 100644
--- a/test/integration/setup_test.go
+++ b/test/integration/setup_test.go
@@ -75,7 +75,7 @@ func TestMain(m *testing.M) {
 	ctx := context.Background()
 	if installopt := os.Getenv(envInstallCNI); installopt != "" {
 		// create dirty cni-manager ds
-		if installCNI, err := strconv.ParseBool(installopt); err != nil && installCNI == true {
+		if installCNI, err := strconv.ParseBool(installopt); err == nil && installCNI == true {
 			if cnicleanup, err = installCNIManagerDaemonset(ctx, clientset, os.Getenv(envImageTag)); err != nil {
 				log.Print(err)
 				return
@@ -88,7 +88,7 @@ func TestMain(m *testing.M) {
 	if installopt := os.Getenv(envInstallCNS); installopt != "" {
 		fmt.Printf("In if CNS")
 		// create dirty cns ds
-		if installCNS, err := strconv.ParseBool(installopt); err != nil && installCNS == true {
+		if installCNS, err := strconv.ParseBool(installopt); err == nil && installCNS == true {
 			fmt.Printf("Thing was bool and true")
 			if cnscleanup, err = installCNSDaemonset(ctx, clientset, os.Getenv(envImageTag)); err != nil {
 				fmt.Printf("no error")

From f3daa0d9e8bd1d7fb221b74f718dbae4316c4b4a Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Tue, 30 Mar 2021 14:50:54 -0700
Subject: [PATCH 21/36] remove debug statements

---
 test/integration/setup_test.go | 15 +--------------
 1 file changed, 1 insertion(+), 14 deletions(-)

diff --git a/test/integration/setup_test.go b/test/integration/setup_test.go
index 8272174e04..3669e6219e 100644
--- a/test/integration/setup_test.go
+++ b/test/integration/setup_test.go
@@ -83,25 +83,15 @@ func TestMain(m *testing.M) {
 		}
 	} else {
 		log.Printf("Env %v not set to true, skipping", envInstallCNI)
-	}
-	fmt.Printf("Before install cns")
+  }
 	if installopt := os.Getenv(envInstallCNS); installopt != "" {
-		fmt.Printf("In if CNS")
 		// create dirty cns ds
 		if installCNS, err := strconv.ParseBool(installopt); err == nil && installCNS == true {
-			fmt.Printf("Thing was bool and true")
 			if cnscleanup, err = installCNSDaemonset(ctx, clientset, os.Getenv(envImageTag)); err != nil {
-				fmt.Printf("no error")
 				return
 			}
-		} else {
-			fmt.Printf("installopt %v", installopt)
-			fmt.Printf("InstallCNS %v", installCNS)
-			fmt.Printf("error %v", err)
-			fmt.Printf("envImageTag %v", os.Getenv(envImageTag))
 		}
 	} else {
-		fmt.Printf("Env %v not set to true, skipping", envInstallCNS)
 		log.Printf("Env %v not set to true, skipping", envInstallCNS)
 	}
 
@@ -115,13 +105,11 @@ func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, i
 	)
 
 	if imageTag == "" {
-		fmt.Print("No image")
 		return nil, fmt.Errorf("Azure CNS image tag not set")
 	}
 
 	// setup daemonset
 	if cns, err = mustParseDaemonSet(cnsDaemonSetPath); err != nil {
-		fmt.Print("Must parse failed")
 		return nil, err
 	}
 
@@ -130,7 +118,6 @@ func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, i
 	cnsDaemonsetClient := clientset.AppsV1().DaemonSets(cns.Namespace)
 
 	log.Printf("Installing CNS with image %s", cns.Spec.Template.Spec.Containers[0].Image)
-	fmt.Printf("Installing CNS")
 	// setup the CNS configmap
 	if err := mustSetupConfigMap(ctx, clientset, cnsConfigMapPath); err != nil {
 		return nil, err

From 34a785653a625d074b2f7126275af64671ba34bb Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Tue, 30 Mar 2021 15:19:05 -0700
Subject: [PATCH 22/36] Image pull failed, making it acnpublic instead

---
 test/integration/manifests/cni/manager.yaml   | 2 +-
 test/integration/manifests/cns/daemonset.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/integration/manifests/cni/manager.yaml b/test/integration/manifests/cni/manager.yaml
index 979e64cb09..ecaadda6d6 100644
--- a/test/integration/manifests/cni/manager.yaml
+++ b/test/integration/manifests/cni/manager.yaml
@@ -19,7 +19,7 @@ spec:
       hostNetwork: true
       containers:
         - name: azure-cni-installer
-          image: mcr.microsoft.com/containernetworking/azure-cni-manager:v1.2.0-2-g0671b63 
+          image: acnpublic.azurecr.io/azure-cni-manager:v1.2.0-2-g0671b63 
           imagePullPolicy: Always
           env:
             - name: AZURE_CNI_OS
diff --git a/test/integration/manifests/cns/daemonset.yaml b/test/integration/manifests/cns/daemonset.yaml
index b4d1de61bc..ed990549cd 100644
--- a/test/integration/manifests/cns/daemonset.yaml
+++ b/test/integration/manifests/cns/daemonset.yaml
@@ -41,7 +41,7 @@ spec:
           effect: NoSchedule
       containers:
         - name: cns-container
-          image: mcr.microsoft.com/containernetworking/azure-cns:v1.2.0
+          image: acnpublic.azurecr.io/azure-cns:v1.2.8
           imagePullPolicy: IfNotPresent
           args: [ "-c", "tcp://$(CNSIpAddress):$(CNSPort)", "-t", "$(CNSLogTarget)"]
           volumeMounts:

From 35b89fcdd301a7f6e4090ad2cc500ac999914cb8 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Tue, 30 Mar 2021 16:11:16 -0700
Subject: [PATCH 23/36] fixing cni-manager dockerfile

---
 tools/acncli/Dockerfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tools/acncli/Dockerfile b/tools/acncli/Dockerfile
index 0adbe4c315..8d71a2a175 100644
--- a/tools/acncli/Dockerfile
+++ b/tools/acncli/Dockerfile
@@ -19,4 +19,4 @@ ENV AZURE_CNI_TENANCY=singletenancy
 ENV AZURE_CNI_IPAM=azure-cns
 ENV AZURE_CNI_MODE=transparent
 
-CMD ["./acn", "manager", "-f"]
+CMD ["./acn", "cni", "manager", "--follow", "--mode", "transparent"]

From 775066403a8624e965a493350a6088ffb049dec8 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Tue, 30 Mar 2021 16:38:01 -0700
Subject: [PATCH 24/36] more retry time

---
 test/integration/utils_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/integration/utils_test.go b/test/integration/utils_test.go
index b928dd0502..1012a5fc2a 100644
--- a/test/integration/utils_test.go
+++ b/test/integration/utils_test.go
@@ -220,6 +220,6 @@ func waitForPodsRunning(ctx context.Context, clientset *kubernetes.Clientset, na
 		return nil
 	}
 
-	retrier := retry.Retrier{Attempts: 10, Delay: 2 * time.Second}
+	retrier := retry.Retrier{Attempts: 10, Delay: 6 * time.Second}
 	return retrier.Do(ctx, checkPodIPsFn)
 }

From 25537f25376927a767d958b93e2840fb76cfca0d Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Wed, 31 Mar 2021 12:32:27 -0700
Subject: [PATCH 25/36] cni-manager debugging

---
 Makefile                                    | 2 +-
 test/integration/manifests/cni/manager.yaml | 4 +++-
 tools/acncli/Dockerfile                     | 2 +-
 tools/acncli/api/constants.go               | 2 +-
 4 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/Makefile b/Makefile
index 86098b9cd2..9a3fb920f9 100644
--- a/Makefile
+++ b/Makefile
@@ -266,7 +266,7 @@ tools: acncli
 
 .PHONY: tools-images
 tools-images: 
-	docker build -f ./tools/acncli/Dockerfile --build-arg VERSION=$(VERSION) -t $(AZURE_CNI_IMAGE):$(VERSION) .
+	docker build --no-cache -f ./tools/acncli/Dockerfile --build-arg VERSION=$(VERSION) -t $(AZURE_CNI_IMAGE):$(VERSION) .
 
 # Build the Azure CNM plugin image, installable with "docker plugin install".
 .PHONY: azure-vnet-plugin-image
diff --git a/test/integration/manifests/cni/manager.yaml b/test/integration/manifests/cni/manager.yaml
index ecaadda6d6..5c9fc19e00 100644
--- a/test/integration/manifests/cni/manager.yaml
+++ b/test/integration/manifests/cni/manager.yaml
@@ -19,7 +19,9 @@ spec:
       hostNetwork: true
       containers:
         - name: azure-cni-installer
-          image: acnpublic.azurecr.io/azure-cni-manager:v1.2.0-2-g0671b63 
+          image: acnpublic.azurecr.io/azure-cni-manager:v1.2.8-32-g77506640
+          command: ["./acn"]
+          args: ["cni", "manager", "--follow", "--mode", "transparent", "--ipam", "azure-cns"]
           imagePullPolicy: Always
           env:
             - name: AZURE_CNI_OS
diff --git a/tools/acncli/Dockerfile b/tools/acncli/Dockerfile
index 8d71a2a175..c18523403d 100644
--- a/tools/acncli/Dockerfile
+++ b/tools/acncli/Dockerfile
@@ -19,4 +19,4 @@ ENV AZURE_CNI_TENANCY=singletenancy
 ENV AZURE_CNI_IPAM=azure-cns
 ENV AZURE_CNI_MODE=transparent
 
-CMD ["./acn", "cni", "manager", "--follow", "--mode", "transparent"]
+ENTRYPOINT ["./acn", "cni", "manager", "--follow", "--mode", "transparent"]
diff --git a/tools/acncli/api/constants.go b/tools/acncli/api/constants.go
index 7c535a1a7f..18a7e64fe7 100644
--- a/tools/acncli/api/constants.go
+++ b/tools/acncli/api/constants.go
@@ -82,7 +82,7 @@ var (
 		FlagTenancy:                  Singletenancy,
 		FlagIPAM:                     AzureVNETIPAM,
 		FlagExempt:                   AzureTelemetryBin + "," + AzureTelemetryConfig,
-		FlagMode:                     Bridge,
+		FlagMode:                     Transparent,
 		FlagTarget:                   Local,
 		FlagBinDirectory:             DefaultBinDirLinux,
 		FlagConflistDirectory:        DefaultConflistDirLinux,

From 07e8ff0508ebcf9f4af9e5e31bc4c0bd016b2182 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Thu, 1 Apr 2021 11:31:28 -0700
Subject: [PATCH 26/36] add cni-manager logs to artifacts

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index a7b7ee5600..1f5c09aaf8 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -57,12 +57,12 @@ steps:
       mkdir aks-swift-output
       cd aks-swift-output
       pwd
+      export KUBECONFIG=~/.kube/config
+      kubectl logs -n kube-system -l k8s-app=azure-cns --tail -1 --prefix > cns-logs.txt
+      kubectl logs -n kube-system -l acn=azure-cni-manager --tail -1 --prefix > cni-manager-logs.txt
       export KUBECONFIG=~/.kube/dnc-underlay
       kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c dnc --prefix > dnc-logs.txt
       kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c requestcontroller --prefix > dnc-rc-logs.txt
-
-      export KUBECONFIG=~/.kube/config
-      kubectl logs -n kube-system -l k8s-app=azure-cns --tail -1 --prefix > cns-logs.txt
       ls
     workingDirectory: "$(modulePath)"
     name: "GetLogs"

From 202ade3c181c4c62d3d5ac7d89c7da26e0e402f0 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Thu, 1 Apr 2021 15:25:19 -0700
Subject: [PATCH 27/36] get cni-manager logs and cns logs before deleting
 daemonsets

---
 .gitignore                                    |  5 +-
 .../aks-swift/e2e-step-template.yaml          |  5 +-
 test/integration/setup_test.go                | 19 +++++--
 test/integration/utils_test.go                | 56 +++++++++++++++++++
 4 files changed, 75 insertions(+), 10 deletions(-)

diff --git a/.gitignore b/.gitignore
index 1a96645186..96c9505544 100644
--- a/.gitignore
+++ b/.gitignore
@@ -13,4 +13,7 @@ ipam-*.xml
 *.out
 
 # Goland
-.idea/*
\ No newline at end of file
+.idea/*
+
+# Logs
+logs/*
\ No newline at end of file
diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 1f5c09aaf8..88d5f798f2 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -54,12 +54,11 @@ steps:
 
   - script: |
       echo "Logs will be available as a build artifact"
+      rm -rf aks-swift-output
       mkdir aks-swift-output
+      cp test/integration/logs/* aks-swift-output/
       cd aks-swift-output
       pwd
-      export KUBECONFIG=~/.kube/config
-      kubectl logs -n kube-system -l k8s-app=azure-cns --tail -1 --prefix > cns-logs.txt
-      kubectl logs -n kube-system -l acn=azure-cni-manager --tail -1 --prefix > cni-manager-logs.txt
       export KUBECONFIG=~/.kube/dnc-underlay
       kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c dnc --prefix > dnc-logs.txt
       kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c requestcontroller --prefix > dnc-rc-logs.txt
diff --git a/test/integration/setup_test.go b/test/integration/setup_test.go
index 3669e6219e..68b3d04df1 100644
--- a/test/integration/setup_test.go
+++ b/test/integration/setup_test.go
@@ -1,5 +1,3 @@
-// +build integration
-
 package k8s
 
 import (
@@ -36,6 +34,9 @@ const (
 	cnsRoleBindingPath        = cnsManifestFolder + "/rolebinding.yaml"
 	cnsServiceAccountPath     = cnsManifestFolder + "/serviceaccount.yaml"
 	cnsLabelSelector          = "k8s-app=azure-cns"
+
+  // relative log directory
+  logDir = "logs/"
 )
 
 func TestMain(m *testing.M) {
@@ -76,7 +77,7 @@ func TestMain(m *testing.M) {
 	if installopt := os.Getenv(envInstallCNI); installopt != "" {
 		// create dirty cni-manager ds
 		if installCNI, err := strconv.ParseBool(installopt); err == nil && installCNI == true {
-			if cnicleanup, err = installCNIManagerDaemonset(ctx, clientset, os.Getenv(envImageTag)); err != nil {
+			if cnicleanup, err = installCNIManagerDaemonset(ctx, clientset, os.Getenv(envImageTag), logDir); err != nil {
 				log.Print(err)
 				return
 			}
@@ -87,7 +88,7 @@ func TestMain(m *testing.M) {
 	if installopt := os.Getenv(envInstallCNS); installopt != "" {
 		// create dirty cns ds
 		if installCNS, err := strconv.ParseBool(installopt); err == nil && installCNS == true {
-			if cnscleanup, err = installCNSDaemonset(ctx, clientset, os.Getenv(envImageTag)); err != nil {
+			if cnscleanup, err = installCNSDaemonset(ctx, clientset, os.Getenv(envImageTag), logDir); err != nil {
 				return
 			}
 		}
@@ -98,7 +99,7 @@ func TestMain(m *testing.M) {
 	exitCode = m.Run()
 }
 
-func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, imageTag string) (func() error, error) {
+func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, imageTag, logDir string) (func() error, error) {
 	var (
 		err error
 		cns v1.DaemonSet
@@ -142,6 +143,9 @@ func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, i
 	}
 
 	cleanupds := func() error {
+    if err := exportLogsByLabelSelector(ctx, clientset, cns.Namespace, cnsLabelSelector, logDir); err != nil {
+      return err
+    }
 		if err := mustDeleteDaemonset(ctx, cnsDaemonsetClient, cns); err != nil {
 			return err
 		}
@@ -151,7 +155,7 @@ func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, i
 	return cleanupds, nil
 }
 
-func installCNIManagerDaemonset(ctx context.Context, clientset *kubernetes.Clientset, imageTag string) (func() error, error) {
+func installCNIManagerDaemonset(ctx context.Context, clientset *kubernetes.Clientset, imageTag, logDir string) (func() error, error) {
 	var (
 		err error
 		cni v1.DaemonSet
@@ -177,6 +181,9 @@ func installCNIManagerDaemonset(ctx context.Context, clientset *kubernetes.Clien
 	}
 
 	cleanupds := func() error {
+    if err := exportLogsByLabelSelector(ctx, clientset, cni.Namespace, cniLabelSelector, logDir); err != nil {
+      return err
+    }
 		if err := mustDeleteDaemonset(ctx, cniDaemonsetClient, cni); err != nil {
 			return err
 		}
diff --git a/test/integration/utils_test.go b/test/integration/utils_test.go
index 1012a5fc2a..4788ab3b0d 100644
--- a/test/integration/utils_test.go
+++ b/test/integration/utils_test.go
@@ -3,8 +3,10 @@
 package k8s
 
 import (
+	"bytes"
 	"context"
 	"errors"
+	"io"
 	"log"
 	"strings"
 	"time"
@@ -223,3 +225,57 @@ func waitForPodsRunning(ctx context.Context, clientset *kubernetes.Clientset, na
 	retrier := retry.Retrier{Attempts: 10, Delay: 6 * time.Second}
 	return retrier.Do(ctx, checkPodIPsFn)
 }
+
+
+func exportLogsByLabelSelector(ctx context.Context, clientset *kubernetes.Clientset, namespace, labelselector, logDir string) error {
+  podsClient := clientset.CoreV1().Pods(namespace)
+  podLogOpts := corev1.PodLogOptions{}
+  logExtension := ".log"
+  podList, err := podsClient.List(ctx, metav1.ListOptions{LabelSelector: labelselector})
+  if err != nil {
+    return err
+  }
+
+  for _, pod := range podList.Items {
+    req := podsClient.GetLogs(pod.Name, &podLogOpts)
+    podLogs, err := req.Stream(ctx)
+    if err != nil {
+        return err
+    }
+    defer podLogs.Close()
+
+    buf := new(bytes.Buffer)
+    _, err = io.Copy(buf, podLogs)
+    if err != nil {
+        return err
+    }
+    str := buf.String()
+    err = writeToFile(logDir, pod.Name + logExtension, str)
+    if err != nil {
+      return err
+    }
+  }
+  return nil
+}
+
+func writeToFile(dir, fileName, str string) error {
+  if _, err := os.Stat(dir); os.IsNotExist(err) { 
+    // your dir does not exist
+    os.MkdirAll(dir, 0700)
+}
+  // open output file
+  f, err := os.Create(dir + fileName)
+  if err != nil {
+    return err
+  }
+  // close fo on exit and check for its returned error
+  defer func() {
+    if err := f.Close(); err != nil {
+      panic(err)
+    }
+  }()
+  
+  // If write went ok then err is nil
+  _, err = f.WriteString(str)
+  return err
+}

From f78e1bec6a4493cf0a508eaa82125078915ebd0d Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Thu, 1 Apr 2021 16:57:58 -0700
Subject: [PATCH 28/36] ignore log files in git

---
 .gitignore | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index 96c9505544..ac260ece45 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,4 +16,4 @@ ipam-*.xml
 .idea/*
 
 # Logs
-logs/*
\ No newline at end of file
+*.log
\ No newline at end of file

From 69973c1823e4a27ceb9578d486389b14da69e54e Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Thu, 1 Apr 2021 17:03:52 -0700
Subject: [PATCH 29/36] fixed spacing

---
 test/integration/setup_test.go | 19 +++----
 test/integration/utils_test.go | 94 +++++++++++++++++-----------------
 2 files changed, 57 insertions(+), 56 deletions(-)

diff --git a/test/integration/setup_test.go b/test/integration/setup_test.go
index 68b3d04df1..ce16411074 100644
--- a/test/integration/setup_test.go
+++ b/test/integration/setup_test.go
@@ -35,8 +35,8 @@ const (
 	cnsServiceAccountPath     = cnsManifestFolder + "/serviceaccount.yaml"
 	cnsLabelSelector          = "k8s-app=azure-cns"
 
-  // relative log directory
-  logDir = "logs/"
+	// relative log directory
+	logDir = "logs/"
 )
 
 func TestMain(m *testing.M) {
@@ -84,7 +84,8 @@ func TestMain(m *testing.M) {
 		}
 	} else {
 		log.Printf("Env %v not set to true, skipping", envInstallCNI)
-  }
+	}
+
 	if installopt := os.Getenv(envInstallCNS); installopt != "" {
 		// create dirty cns ds
 		if installCNS, err := strconv.ParseBool(installopt); err == nil && installCNS == true {
@@ -143,9 +144,9 @@ func installCNSDaemonset(ctx context.Context, clientset *kubernetes.Clientset, i
 	}
 
 	cleanupds := func() error {
-    if err := exportLogsByLabelSelector(ctx, clientset, cns.Namespace, cnsLabelSelector, logDir); err != nil {
-      return err
-    }
+		if err := exportLogsByLabelSelector(ctx, clientset, cns.Namespace, cnsLabelSelector, logDir); err != nil {
+			return err
+		}
 		if err := mustDeleteDaemonset(ctx, cnsDaemonsetClient, cns); err != nil {
 			return err
 		}
@@ -181,9 +182,9 @@ func installCNIManagerDaemonset(ctx context.Context, clientset *kubernetes.Clien
 	}
 
 	cleanupds := func() error {
-    if err := exportLogsByLabelSelector(ctx, clientset, cni.Namespace, cniLabelSelector, logDir); err != nil {
-      return err
-    }
+		if err := exportLogsByLabelSelector(ctx, clientset, cni.Namespace, cniLabelSelector, logDir); err != nil {
+			return err
+		}
 		if err := mustDeleteDaemonset(ctx, cniDaemonsetClient, cni); err != nil {
 			return err
 		}
diff --git a/test/integration/utils_test.go b/test/integration/utils_test.go
index 4788ab3b0d..748c1b1d08 100644
--- a/test/integration/utils_test.go
+++ b/test/integration/utils_test.go
@@ -228,54 +228,54 @@ func waitForPodsRunning(ctx context.Context, clientset *kubernetes.Clientset, na
 
 
 func exportLogsByLabelSelector(ctx context.Context, clientset *kubernetes.Clientset, namespace, labelselector, logDir string) error {
-  podsClient := clientset.CoreV1().Pods(namespace)
-  podLogOpts := corev1.PodLogOptions{}
-  logExtension := ".log"
-  podList, err := podsClient.List(ctx, metav1.ListOptions{LabelSelector: labelselector})
-  if err != nil {
-    return err
-  }
-
-  for _, pod := range podList.Items {
-    req := podsClient.GetLogs(pod.Name, &podLogOpts)
-    podLogs, err := req.Stream(ctx)
-    if err != nil {
-        return err
-    }
-    defer podLogs.Close()
-
-    buf := new(bytes.Buffer)
-    _, err = io.Copy(buf, podLogs)
-    if err != nil {
-        return err
-    }
-    str := buf.String()
-    err = writeToFile(logDir, pod.Name + logExtension, str)
-    if err != nil {
-      return err
-    }
-  }
-  return nil
+	podsClient := clientset.CoreV1().Pods(namespace)
+	podLogOpts := corev1.PodLogOptions{}
+	logExtension := ".log"
+	podList, err := podsClient.List(ctx, metav1.ListOptions{LabelSelector: labelselector})
+	if err != nil {
+		return err
+	}
+
+	for _, pod := range podList.Items {
+		req := podsClient.GetLogs(pod.Name, &podLogOpts)
+		podLogs, err := req.Stream(ctx)
+		if err != nil {
+			return err
+		}
+		defer podLogs.Close()
+
+		buf := new(bytes.Buffer)
+		_, err = io.Copy(buf, podLogs)
+		if err != nil {
+			return err
+		}
+		str := buf.String()
+		err = writeToFile(logDir, pod.Name + logExtension, str)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 func writeToFile(dir, fileName, str string) error {
-  if _, err := os.Stat(dir); os.IsNotExist(err) { 
-    // your dir does not exist
-    os.MkdirAll(dir, 0700)
-}
-  // open output file
-  f, err := os.Create(dir + fileName)
-  if err != nil {
-    return err
-  }
-  // close fo on exit and check for its returned error
-  defer func() {
-    if err := f.Close(); err != nil {
-      panic(err)
-    }
-  }()
-  
-  // If write went ok then err is nil
-  _, err = f.WriteString(str)
-  return err
+	if _, err := os.Stat(dir); os.IsNotExist(err) { 
+		// your dir does not exist
+		os.MkdirAll(dir, 0700)
+	}
+	// open output file
+	f, err := os.Create(dir + fileName)
+	if err != nil {
+		return err
+	}
+	// close fo on exit and check for its returned error
+	defer func() {
+		if err := f.Close(); err != nil {
+			panic(err)
+		}
+	}()
+
+	// If write went ok then err is nil
+	_, err = f.WriteString(str)
+	return err
 }

From 985fda0ca8c08ad648e283e2cdd5a21a05cbcba3 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Thu, 1 Apr 2021 17:08:07 -0700
Subject: [PATCH 30/36] adding back build tag

---
 test/integration/setup_test.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/test/integration/setup_test.go b/test/integration/setup_test.go
index ce16411074..a48b17a25d 100644
--- a/test/integration/setup_test.go
+++ b/test/integration/setup_test.go
@@ -1,3 +1,5 @@
+// +build integration
+
 package k8s
 
 import (

From eb2a5e90cb5547dfa545a91783862f8d72ca7797 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Fri, 2 Apr 2021 10:01:40 -0700
Subject: [PATCH 31/36] read write on logs dir

---
 test/integration/utils_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/integration/utils_test.go b/test/integration/utils_test.go
index 748c1b1d08..b1b07308c8 100644
--- a/test/integration/utils_test.go
+++ b/test/integration/utils_test.go
@@ -261,7 +261,7 @@ func exportLogsByLabelSelector(ctx context.Context, clientset *kubernetes.Client
 func writeToFile(dir, fileName, str string) error {
 	if _, err := os.Stat(dir); os.IsNotExist(err) { 
 		// your dir does not exist
-		os.MkdirAll(dir, 0700)
+		os.MkdirAll(dir, 0644)
 	}
 	// open output file
 	f, err := os.Create(dir + fileName)

From 3051a0bb411fb646ac2ed5c6cbbaa6db794ca8a6 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Fri, 2 Apr 2021 10:38:05 -0700
Subject: [PATCH 32/36] tree

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 88d5f798f2..efb02775b5 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -55,6 +55,7 @@ steps:
   - script: |
       echo "Logs will be available as a build artifact"
       rm -rf aks-swift-output
+      tree
       mkdir aks-swift-output
       cp test/integration/logs/* aks-swift-output/
       cd aks-swift-output

From 4489abdee41a6f67a36c5270c201738600dc1f37 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Fri, 2 Apr 2021 10:39:27 -0700
Subject: [PATCH 33/36] perms

---
 test/integration/utils_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/integration/utils_test.go b/test/integration/utils_test.go
index b1b07308c8..c6a33505c2 100644
--- a/test/integration/utils_test.go
+++ b/test/integration/utils_test.go
@@ -261,7 +261,7 @@ func exportLogsByLabelSelector(ctx context.Context, clientset *kubernetes.Client
 func writeToFile(dir, fileName, str string) error {
 	if _, err := os.Stat(dir); os.IsNotExist(err) { 
 		// your dir does not exist
-		os.MkdirAll(dir, 0644)
+		os.MkdirAll(dir, 0666)
 	}
 	// open output file
 	f, err := os.Create(dir + fileName)

From 00a6ad9928c7b31b2dbaa18b90ed1f1de555988b Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Fri, 2 Apr 2021 11:39:50 -0700
Subject: [PATCH 34/36] debugging logs

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index efb02775b5..e1f4924fcb 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -55,16 +55,17 @@ steps:
   - script: |
       echo "Logs will be available as a build artifact"
       rm -rf aks-swift-output
-      tree
+      ls -la
+      sudo ls test/integration
+      sudo ls test/integration/logs
       mkdir aks-swift-output
-      cp test/integration/logs/* aks-swift-output/
+      sudo cp test/integration/logs/* aks-swift-output/
       cd aks-swift-output
       pwd
       export KUBECONFIG=~/.kube/dnc-underlay
       kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c dnc --prefix > dnc-logs.txt
       kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c requestcontroller --prefix > dnc-rc-logs.txt
       ls
-    workingDirectory: "$(modulePath)"
     name: "GetLogs"
     displayName: "Get logs"
     condition: always()

From 99f936f6c2789a131e35398858307aa806ed5a64 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Fri, 2 Apr 2021 12:36:09 -0700
Subject: [PATCH 35/36] logs

---
 .../aks-swift/e2e-step-template.yaml          | 22 ++++++-------------
 1 file changed, 7 insertions(+), 15 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index e1f4924fcb..368f9e9754 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -54,28 +54,20 @@ steps:
 
   - script: |
       echo "Logs will be available as a build artifact"
-      rm -rf aks-swift-output
-      ls -la
-      sudo ls test/integration
-      sudo ls test/integration/logs
-      mkdir aks-swift-output
-      sudo cp test/integration/logs/* aks-swift-output/
-      cd aks-swift-output
-      pwd
+      ARTIFACT_DIR=$(Build.ArtifactsStagingDirectory)/aks-swift-output/
+      echo $ARTIFACT_DIR
+      sudo rm -rf $ARTIFACT_DIR
+      sudo mkdir $ARTIFACT_DIR
+      sudo cp test/integration/logs/* $ARTIFACT_DIR
       export KUBECONFIG=~/.kube/dnc-underlay
       kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c dnc --prefix > dnc-logs.txt
       kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c requestcontroller --prefix > dnc-rc-logs.txt
-      ls
+
+      sudo cp *.txt $ARTIFACT_DIR
     name: "GetLogs"
     displayName: "Get logs"
     condition: always()
 
-  - task: CopyFiles@2
-    inputs:
-      sourceFolder: "$(modulePath)/aks-swift-output"
-      targetFolder: $(Build.ArtifactStagingDirectory)/aks-swift-output
-    condition: always()
-  
   - task: PublishBuildArtifacts@1
     inputs:
       artifactName: aks-swift-output

From 13acd19b0cc26095386c8488ca05e0a2d4837419 Mon Sep 17 00:00:00 2001
From: Paul Johnston <johnstonpaul801@gmail.com>
Date: Fri, 2 Apr 2021 12:45:23 -0700
Subject: [PATCH 36/36] misspelled

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 368f9e9754..c0479925af 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -54,7 +54,7 @@ steps:
 
   - script: |
       echo "Logs will be available as a build artifact"
-      ARTIFACT_DIR=$(Build.ArtifactsStagingDirectory)/aks-swift-output/
+      ARTIFACT_DIR=$(Build.ArtifactStagingDirectory)/aks-swift-output/
       echo $ARTIFACT_DIR
       sudo rm -rf $ARTIFACT_DIR
       sudo mkdir $ARTIFACT_DIR