From 1e4fde46263c2fa26dd5776920d255048f512eac Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Mon, 18 Jul 2022 16:31:23 -0700
Subject: [PATCH 01/51] byo cluster pipeline update

---
 .../aks-swift/e2e-step-template.yaml          | 22 +++++++++----------
 cns/azure-cns.yaml                            |  2 +-
 test/integration/manifests/cni/manager.yaml   |  5 ++++-
 3 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 931a588a28..ef84f9b917 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -31,13 +31,11 @@ steps:
       addSpnToEnvironment: true
       inlineScript: |
         mkdir -p ~/.kube/
-        echo "DNC Underlay config"
-        az keyvault secret show --name $(KV_DNC_UNDERLAY_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//'
-        az keyvault secret show --name $(KV_DNC_UNDERLAY_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//' > ~/.kube/dnc-underlay
-        echo "Dirty ACN ccp config"
-        az keyvault secret show --name $(KV_AKS_SWIFT_CX_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//'
-        az keyvault secret show --name $(KV_AKS_SWIFT_CX_SECRET_NAME) --vault-name $(KV_NAME) --query value | sed 's/\\n/\n/g' | sed -e 's/^"//' -e 's/"$//' > ~/.kube/config
-    displayName: Pull Kubeconfig from Keyvault
+        echo "Create AKS cluster"
+        cd ./hack/swift
+        sudo make byocni-up
+        cd ../..
+    displayName: Create test cluster
     condition: succeeded()
 
   - script: |
@@ -60,10 +58,6 @@ steps:
       sudo rm -rf $ARTIFACT_DIR
       sudo mkdir $ARTIFACT_DIR
       sudo cp test/integration/logs/* $ARTIFACT_DIR
-      export KUBECONFIG=~/.kube/dnc-underlay
-      kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c dnc --prefix > dnc-logs.txt
-      kubectl logs -n $STABLE_DNC_NAMESPACE -l app=dnc --tail -1 -c requestcontroller --prefix > dnc-rc-logs.txt
-      sudo cp *.txt $ARTIFACT_DIR
     name: "GetLogs"
     displayName: "Get logs"
     condition: always()
@@ -82,3 +76,9 @@ steps:
     name: "Cleanupartifactdir"
     displayName: "Cleanup artifact dir"
     condition: always()
+
+  - script: |
+      make down
+    name: "Cleanupcluster"
+    displayName: "Cleanup cluster"
+    condition: always()
diff --git a/cns/azure-cns.yaml b/cns/azure-cns.yaml
index 9bb98b713e..dd4c93ec56 100644
--- a/cns/azure-cns.yaml
+++ b/cns/azure-cns.yaml
@@ -96,7 +96,7 @@ spec:
           effect: NoSchedule
       containers:
         - name: cns-container
-          image: mcr.microsoft.com/containernetworking/azure-cns:v1.4.7
+          image: mcr.microsoft.com/containernetworking/azure-cns:v1.4.29
           imagePullPolicy: IfNotPresent
           args: [ "-c", "tcp://$(CNSIpAddress):$(CNSPort)", "-t", "$(CNSLogTarget)"]
           volumeMounts:
diff --git a/test/integration/manifests/cni/manager.yaml b/test/integration/manifests/cni/manager.yaml
index 5db6dddf9d..9aea779af0 100644
--- a/test/integration/manifests/cni/manager.yaml
+++ b/test/integration/manifests/cni/manager.yaml
@@ -16,10 +16,13 @@ spec:
         - effect: NoSchedule
           key: node-role.kubernetes.io/master
           operator: Exists
+        - effect: NoSchedule
+          key: node.kubernetes.io/not-ready
+          operator: Exists
       hostNetwork: true
       containers:
         - name: azure-cni-installer
-          image: acnpublic.azurecr.io/acncli:v1.2.8-32-g77506640
+          image: acnpublic.azurecr.io/acncli:v1.4.28-42-gd086cf1b
           command: ["./acn"]
           args: ["cni", "manager", "--follow", "--mode", "transparent", "--ipam", "azure-cns"]
           imagePullPolicy: Always

From 9cd6d9bd2a7ac9fe110d32f5ebf7a6ec26f57b25 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Tue, 19 Jul 2022 08:58:27 -0700
Subject: [PATCH 02/51] adding sub var

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index ef84f9b917..13444b666f 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -33,8 +33,9 @@ steps:
         mkdir -p ~/.kube/
         echo "Create AKS cluster"
         cd ./hack/swift
-        sudo make byocni-up
+        sudo make byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_RUNNERS)
         cd ../..
+        echo "Cluster successfully created"
     displayName: Create test cluster
     condition: succeeded()
 
@@ -78,7 +79,10 @@ steps:
     condition: always()
 
   - script: |
-      make down
+      cd ./hack/swift
+      echo "Deleting cluster"
+      sudo make down SUB=$(SUB_AZURE_NETWORK_AGENT_RUNNERS)
+      cd ../..
     name: "Cleanupcluster"
     displayName: "Cleanup cluster"
     condition: always()

From 1b0713b0b8e14263e6b0a87781efdf8ec929138e Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Tue, 19 Jul 2022 11:34:54 -0700
Subject: [PATCH 03/51] use build validations sub

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 13444b666f..84b251a3ea 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -33,7 +33,7 @@ steps:
         mkdir -p ~/.kube/
         echo "Create AKS cluster"
         cd ./hack/swift
-        sudo make byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_RUNNERS)
+        sudo make byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS)
         cd ../..
         echo "Cluster successfully created"
     displayName: Create test cluster
@@ -81,7 +81,7 @@ steps:
   - script: |
       cd ./hack/swift
       echo "Deleting cluster"
-      sudo make down SUB=$(SUB_AZURE_NETWORK_AGENT_RUNNERS)
+      sudo make down SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS)
       cd ../..
     name: "Cleanupcluster"
     displayName: "Cleanup cluster"

From 7b26b8d6060a58b466e115de3b5611db93d8048e Mon Sep 17 00:00:00 2001
From: Evan Baker <rbtr@users.noreply.github.com>
Date: Tue, 19 Jul 2022 20:27:00 +0000
Subject: [PATCH 04/51] test

Signed-off-by: GitHub <noreply@github.com>
---
 .pipelines/pipeline.yaml                      | 494 +++++++++---------
 .../aks-swift/e2e-job-template.yaml           |   2 +-
 hack/swift/Makefile                           |  19 +-
 3 files changed, 258 insertions(+), 257 deletions(-)

diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
index 356ae1c3c3..a1c97699a5 100644
--- a/.pipelines/pipeline.yaml
+++ b/.pipelines/pipeline.yaml
@@ -34,214 +34,214 @@ stages:
             displayName: "Set environmental variables"
             condition: always()
 
-  - stage: test
-    displayName: Test ACN
-    dependsOn:
-      - setup
-    jobs:
-      - job: test
-        displayName: Run Tests
-        variables:
-          STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ]
-        pool:
-          name: "$(BUILD_POOL_NAME_DEFAULT)"
-        steps:
-          - script: |
-              make tools
-              # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout
-              { { { {
-                    sudo -E env "PATH=$PATH" make test-all;
-                    echo $? >&3;
-                    } | tee >(build/tools/bin/go-junit-report > report.xml) >&4;
-                  } 3>&1;
-                } | { read xs; exit $xs; }
-              } 4>&1
-            retryCountOnTaskFailure: 3
-            name: "Test"
-            displayName: "Run Tests"
+  # - stage: test
+  #   displayName: Test ACN
+  #   dependsOn:
+  #     - setup
+  #   jobs:
+  #     - job: test
+  #       displayName: Run Tests
+  #       variables:
+  #         STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ]
+  #       pool:
+  #         name: "$(BUILD_POOL_NAME_DEFAULT)"
+  #       steps:
+  #         - script: |
+  #             make tools
+  #             # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout
+  #             { { { {
+  #                   sudo -E env "PATH=$PATH" make test-all;
+  #                   echo $? >&3;
+  #                   } | tee >(build/tools/bin/go-junit-report > report.xml) >&4;
+  #                 } 3>&1;
+  #               } | { read xs; exit $xs; }
+  #             } 4>&1
+  #           retryCountOnTaskFailure: 3
+  #           name: "Test"
+  #           displayName: "Run Tests"
 
-          - bash: |
-              build/tools/bin/gocov convert coverage.out > coverage.json
-              build/tools/bin/gocov-xml < coverage.json > coverage.xml
-            name: "Coverage"
-            displayName: "Generate Coverage Reports"
-            condition: always()
+  #         - bash: |
+  #             build/tools/bin/gocov convert coverage.out > coverage.json
+  #             build/tools/bin/gocov-xml < coverage.json > coverage.xml
+  #           name: "Coverage"
+  #           displayName: "Generate Coverage Reports"
+  #           condition: always()
 
-          - task: PublishTestResults@2
-            inputs:
-              testRunner: JUnit
-              testResultsFiles: report.xml
-            displayName: "Publish Test Results"
-            condition: always()
+  #         - task: PublishTestResults@2
+  #           inputs:
+  #             testRunner: JUnit
+  #             testResultsFiles: report.xml
+  #           displayName: "Publish Test Results"
+  #           condition: always()
 
-          - task: PublishCodeCoverageResults@1
-            inputs:
-              codeCoverageTool: Cobertura
-              summaryFileLocation: coverage.xml
-            displayName: "Publish Code Coverage Results"
-            condition: always()
+  #         - task: PublishCodeCoverageResults@1
+  #           inputs:
+  #             codeCoverageTool: Cobertura
+  #             summaryFileLocation: coverage.xml
+  #           displayName: "Publish Code Coverage Results"
+  #           condition: always()
 
-  - stage: binaries
-    displayName: Build Binaries
-    dependsOn:
-      - setup
-      - test
-    jobs:
-      - job: build
-        displayName: Build Binaries
-        variables:
-          TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ]
-          STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ]
-        pool:
-          name: "$(BUILD_POOL_NAME_DEFAULT)"
-        steps:
-          - script: |
-              make all-binaries-platforms VERSION=$(TAG)
-            name: "BuildAllPlatformBinaries"
-            displayName: "Build all platform binaries"
+  # - stage: binaries
+  #   displayName: Build Binaries
+  #   dependsOn:
+  #     - setup
+  #     - test
+  #   jobs:
+  #     - job: build
+  #       displayName: Build Binaries
+  #       variables:
+  #         TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ]
+  #         STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ]
+  #       pool:
+  #         name: "$(BUILD_POOL_NAME_DEFAULT)"
+  #       steps:
+  #         - script: |
+  #             make all-binaries-platforms VERSION=$(TAG)
+  #           name: "BuildAllPlatformBinaries"
+  #           displayName: "Build all platform binaries"
 
-          - script: |
-              mkdir -p ./output/bins
-              cd ./output
-              find . -name '*.tgz' -print -exec mv -t ./bins/ {} +
-              find . -name '*.zip' -print -exec mv -t ./bins/ {} +
-              shopt -s extglob
-              rm -rf !("bins")
-            name: "PrepareArtifacts"
-            displayName: "Prepare Artifacts"
+  #         - script: |
+  #             mkdir -p ./output/bins
+  #             cd ./output
+  #             find . -name '*.tgz' -print -exec mv -t ./bins/ {} +
+  #             find . -name '*.zip' -print -exec mv -t ./bins/ {} +
+  #             shopt -s extglob
+  #             rm -rf !("bins")
+  #           name: "PrepareArtifacts"
+  #           displayName: "Prepare Artifacts"
 
-          - task: CopyFiles@2
-            inputs:
-              sourceFolder: "output"
-              targetFolder: $(Build.ArtifactStagingDirectory)
-            condition: succeeded()
+  #         - task: CopyFiles@2
+  #           inputs:
+  #             sourceFolder: "output"
+  #             targetFolder: $(Build.ArtifactStagingDirectory)
+  #           condition: succeeded()
 
-          - task: PublishBuildArtifacts@1
-            inputs:
-              artifactName: "output"
-              pathtoPublish: "$(Build.ArtifactStagingDirectory)"
-            condition: succeeded()
+  #         - task: PublishBuildArtifacts@1
+  #           inputs:
+  #             artifactName: "output"
+  #             pathtoPublish: "$(Build.ArtifactStagingDirectory)"
+  #           condition: succeeded()
 
-          - task: AzureCLI@1
-            inputs:
-              azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
-              scriptLocation: "inlineScript"
-              inlineScript: |
-                echo Creating storage container with name acn-$(STORAGE_ID) and account name $(STORAGE_ACCOUNT_NAME)
-                az storage container create -n acn-$(STORAGE_ID) --account-name $(STORAGE_ACCOUNT_NAME) --public-access container
-                az storage blob upload-batch -d acn-$(STORAGE_ID) -s ./output/bins/  --account-name $(STORAGE_ACCOUNT_NAME)
-            displayName: Create artifact storage container
-            condition: succeeded()
+  #         - task: AzureCLI@1
+  #           inputs:
+  #             azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+  #             scriptLocation: "inlineScript"
+  #             inlineScript: |
+  #               echo Creating storage container with name acn-$(STORAGE_ID) and account name $(STORAGE_ACCOUNT_NAME)
+  #               az storage container create -n acn-$(STORAGE_ID) --account-name $(STORAGE_ACCOUNT_NAME) --public-access container
+  #               az storage blob upload-batch -d acn-$(STORAGE_ID) -s ./output/bins/  --account-name $(STORAGE_ACCOUNT_NAME)
+  #           displayName: Create artifact storage container
+  #           condition: succeeded()
 
-          - publish: ./test/apimodels/
-            artifact: clusterdefinitions
+  #         - publish: ./test/apimodels/
+  #           artifact: clusterdefinitions
 
-  - stage: containerize
-    displayName: Build Images
-    dependsOn:
-      - setup
-      - test
-    jobs:
-      - job: containerize
-        displayName: Build Images
-        variables:
-          TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ]
-        pool:
-          name: "$(BUILD_POOL_NAME_DEFAULT)"
-        strategy:
-          matrix:
-            acncli_linux_amd64:
-              arch: amd64
-              os: linux
-              name: acncli
-            acncli_linux_arm64:
-              arch: arm64
-              os: linux
-              name: acncli
-            cni_dropgz_linux_amd64:
-              arch: amd64
-              os: linux
-              name: cni-dropgz
-            cni_dropgz_linux_arm64:
-              arch: arm64
-              os: linux
-              name: cni-dropgz
-            cns_linux_amd64:
-              arch: amd64
-              os: linux
-              name: cns
-            cns_linux_arm64:
-              arch: arm64
-              os: linux
-              name: cns
-            npm_linux_amd64:
-              arch: amd64
-              os: linux
-              name: npm
-            npm_linux_arm64:
-              arch: arm64
-              os: linux
-              name: npm
-        steps:
-          - template: containers/container-template.yaml
-            parameters:
-              name: $(name)
-              os: $(os)
-              arch: $(arch)
-              tag: $(TAG)
-      - job: containerize_windows
-        displayName: Build Images
-        variables:
-          TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ]
-        pool:
-          name: "$(BUILD_POOL_NAME_DEFAULT_WINDOWS)"
-        strategy:
-          matrix:
-            cns_windows_amd64:
-              arch: amd64
-              name: cns
-            npm_windows_amd64:
-              arch: amd64
-              name: npm
-        steps:
-          - template: containers/container-template-windows.yaml
-            parameters:
-              name: $(name)
-              arch: $(arch)
-              tag: $(TAG)
+  # - stage: containerize
+  #   displayName: Build Images
+  #   dependsOn:
+  #     - setup
+  #     - test
+  #   jobs:
+  #     - job: containerize
+  #       displayName: Build Images
+  #       variables:
+  #         TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ]
+  #       pool:
+  #         name: "$(BUILD_POOL_NAME_DEFAULT)"
+  #       strategy:
+  #         matrix:
+  #           acncli_linux_amd64:
+  #             arch: amd64
+  #             os: linux
+  #             name: acncli
+  #           acncli_linux_arm64:
+  #             arch: arm64
+  #             os: linux
+  #             name: acncli
+  #           cni_dropgz_linux_amd64:
+  #             arch: amd64
+  #             os: linux
+  #             name: cni-dropgz
+  #           cni_dropgz_linux_arm64:
+  #             arch: arm64
+  #             os: linux
+  #             name: cni-dropgz
+  #           cns_linux_amd64:
+  #             arch: amd64
+  #             os: linux
+  #             name: cns
+  #           cns_linux_arm64:
+  #             arch: arm64
+  #             os: linux
+  #             name: cns
+  #           npm_linux_amd64:
+  #             arch: amd64
+  #             os: linux
+  #             name: npm
+  #           npm_linux_arm64:
+  #             arch: arm64
+  #             os: linux
+  #             name: npm
+  #       steps:
+  #         - template: containers/container-template.yaml
+  #           parameters:
+  #             name: $(name)
+  #             os: $(os)
+  #             arch: $(arch)
+  #             tag: $(TAG)
+  #     - job: containerize_windows
+  #       displayName: Build Images
+  #       variables:
+  #         TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ]
+  #       pool:
+  #         name: "$(BUILD_POOL_NAME_DEFAULT_WINDOWS)"
+  #       strategy:
+  #         matrix:
+  #           cns_windows_amd64:
+  #             arch: amd64
+  #             name: cns
+  #           npm_windows_amd64:
+  #             arch: amd64
+  #             name: npm
+  #       steps:
+  #         - template: containers/container-template-windows.yaml
+  #           parameters:
+  #             name: $(name)
+  #             arch: $(arch)
+  #             tag: $(TAG)
 
-  - stage: publish
-    displayName: Publish Multiarch Manifests
-    dependsOn:
-      - setup
-      - containerize
-    jobs:
-      - job: manifest
-        displayName: Compile Manifests
-        variables:
-          TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ]
-        pool:
-          name: "$(BUILD_POOL_NAME_DEFAULT)"
-        strategy:
-          matrix:
-            acncli:
-              name: acncli 
-              platforms: linux/amd64 linux/arm64
-            cni_dropgz:
-              name: cni-dropgz
-              platforms: linux/amd64 linux/arm64
-            cns:
-              name: cns
-              platforms: linux/amd64 linux/arm64 windows/amd64
-            npm:
-              name: npm
-              platforms: linux/amd64 linux/arm64 windows/amd64
-        steps:
-          - template: containers/manifest-template.yaml
-            parameters:
-              name: $(name)
-              platforms: $(platforms)
-              tag: $(TAG)
+  # - stage: publish
+  #   displayName: Publish Multiarch Manifests
+  #   dependsOn:
+  #     - setup
+  #     - containerize
+  #   jobs:
+  #     - job: manifest
+  #       displayName: Compile Manifests
+  #       variables:
+  #         TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ]
+  #       pool:
+  #         name: "$(BUILD_POOL_NAME_DEFAULT)"
+  #       strategy:
+  #         matrix:
+  #           acncli:
+  #             name: acncli 
+  #             platforms: linux/amd64 linux/arm64
+  #           cni_dropgz:
+  #             name: cni-dropgz
+  #             platforms: linux/amd64 linux/arm64
+  #           cns:
+  #             name: cns
+  #             platforms: linux/amd64 linux/arm64 windows/amd64
+  #           npm:
+  #             name: npm
+  #             platforms: linux/amd64 linux/arm64 windows/amd64
+  #       steps:
+  #         - template: containers/manifest-template.yaml
+  #           parameters:
+  #             name: $(name)
+  #             platforms: $(platforms)
+  #             tag: $(TAG)
 
   - template: singletenancy/aks-swift/e2e-job-template.yaml
     parameters:
@@ -249,55 +249,55 @@ stages:
       displayName: AKS Swift
       pipelineBuildImage: "$(BUILD_IMAGE)"
 
-  - template: singletenancy/aks-engine/e2e-job-template.yaml
-    parameters:
-      name: "ubuntu_18_04_linux_e2e"
-      displayName: Ubuntu 18.04
-      pipelineBuildImage: "$(BUILD_IMAGE)"
-      clusterDefinition: "cniLinux1804.json"
-      clusterDefinitionCniTypeKey: "azureCNIURLLinux"
-      clusterDefinitionCniBuildOS: "linux"
-      clusterDefinitionCniBuildExt: ".tgz"
+  # - template: singletenancy/aks-engine/e2e-job-template.yaml
+  #   parameters:
+  #     name: "ubuntu_18_04_linux_e2e"
+  #     displayName: Ubuntu 18.04
+  #     pipelineBuildImage: "$(BUILD_IMAGE)"
+  #     clusterDefinition: "cniLinux1804.json"
+  #     clusterDefinitionCniTypeKey: "azureCNIURLLinux"
+  #     clusterDefinitionCniBuildOS: "linux"
+  #     clusterDefinitionCniBuildExt: ".tgz"
 
-  - template: singletenancy/aks-engine/e2e-job-template.yaml
-    parameters:
-      name: "windows_19_03_e2e"
-      displayName: "Windows 1903"
-      pipelineBuildImage: "$(BUILD_IMAGE)"
-      clusterDefinition: "cniWindows1903.json"
-      clusterDefinitionCniTypeKey: "azureCNIURLWindows"
-      clusterDefinitionCniBuildOS: "windows"
-      clusterDefinitionCniBuildExt: ".zip"
+  # - template: singletenancy/aks-engine/e2e-job-template.yaml
+  #   parameters:
+  #     name: "windows_19_03_e2e"
+  #     displayName: "Windows 1903"
+  #     pipelineBuildImage: "$(BUILD_IMAGE)"
+  #     clusterDefinition: "cniWindows1903.json"
+  #     clusterDefinitionCniTypeKey: "azureCNIURLWindows"
+  #     clusterDefinitionCniBuildOS: "windows"
+  #     clusterDefinitionCniBuildExt: ".zip"
 
-  - template: singletenancy/aks-engine/e2e-job-template.yaml
-    parameters:
-      name: "windows_20_04_e2e"
-      displayName: "Windows 2004"
-      pipelineBuildImage: "$(BUILD_IMAGE)"
-      clusterDefinition: "cniWindows2004.json"
-      clusterDefinitionCniTypeKey: "azureCNIURLWindows"
-      clusterDefinitionCniBuildOS: "windows"
-      clusterDefinitionCniBuildExt: ".zip"
+  # - template: singletenancy/aks-engine/e2e-job-template.yaml
+  #   parameters:
+  #     name: "windows_20_04_e2e"
+  #     displayName: "Windows 2004"
+  #     pipelineBuildImage: "$(BUILD_IMAGE)"
+  #     clusterDefinition: "cniWindows2004.json"
+  #     clusterDefinitionCniTypeKey: "azureCNIURLWindows"
+  #     clusterDefinitionCniBuildOS: "windows"
+  #     clusterDefinitionCniBuildExt: ".zip"
 
-  - template: singletenancy/aks-engine/e2e-job-template.yaml
-    parameters:
-      name: "windows_20_22_e2e"
-      displayName: "Windows 2022"
-      pipelineBuildImage: "$(BUILD_IMAGE)"
-      clusterDefinition: "cniWindows2022.json"
-      clusterDefinitionCniTypeKey: "azureCNIURLWindows"
-      clusterDefinitionCniBuildOS: "windows"
-      clusterDefinitionCniBuildExt: ".zip"
+  # - template: singletenancy/aks-engine/e2e-job-template.yaml
+  #   parameters:
+  #     name: "windows_20_22_e2e"
+  #     displayName: "Windows 2022"
+  #     pipelineBuildImage: "$(BUILD_IMAGE)"
+  #     clusterDefinition: "cniWindows2022.json"
+  #     clusterDefinitionCniTypeKey: "azureCNIURLWindows"
+  #     clusterDefinitionCniBuildOS: "windows"
+  #     clusterDefinitionCniBuildExt: ".zip"
 
-  - template: singletenancy/aks-engine/e2e-dualstack-job-template.yaml
-    parameters:
-      name: "ubuntu_18_04_linux_dualstack_e2e"
-      displayName: "Ubuntu 18.04 Dualstack"
-      pipelineBuildImage: "$(BUILD_IMAGE)"
-      clusterDefinition: "cniLinuxDualstack1804.json"
-      clusterDefinitionCniTypeKey: "azureCNIURLLinux"
-      clusterDefinitionCniBuildOS: "linux"
-      clusterDefinitionCniBuildExt: ".tgz"
+  # - template: singletenancy/aks-engine/e2e-dualstack-job-template.yaml
+  #   parameters:
+  #     name: "ubuntu_18_04_linux_dualstack_e2e"
+  #     displayName: "Ubuntu 18.04 Dualstack"
+  #     pipelineBuildImage: "$(BUILD_IMAGE)"
+  #     clusterDefinition: "cniLinuxDualstack1804.json"
+  #     clusterDefinitionCniTypeKey: "azureCNIURLLinux"
+  #     clusterDefinitionCniBuildOS: "linux"
+  #     clusterDefinitionCniBuildExt: ".tgz"
 
   # - template: singletenancy/aks-engine/e2e-dualstack-job-template.yaml
   #   parameters:
@@ -313,11 +313,11 @@ stages:
     displayName: Cleanup
     dependsOn:
       - "aks_swift_e2e"
-      - "ubuntu_18_04_linux_e2e"
-      - "windows_19_03_e2e"
-      - "windows_20_04_e2e"
-      - "windows_20_22_e2e"
-      - "ubuntu_18_04_linux_dualstack_e2e"
+      # - "ubuntu_18_04_linux_e2e"
+      # - "windows_19_03_e2e"
+      # - "windows_20_04_e2e"
+      # - "windows_20_22_e2e"
+      # - "ubuntu_18_04_linux_dualstack_e2e"
       # - "windows_20_04_dualstack_e2e"
     jobs:
       - job: delete_remote_artifacts
diff --git a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
index 79057bb764..d9f80dc396 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
@@ -9,7 +9,7 @@ stages:
     lockBehavior: sequential
     dependsOn: 
     - setup
-    - publish
+    # - publish
     jobs:
       - job: ${{ parameters.name }}
         displayName: Singletenancy AKS Swift Suite - (${{ parameters.name }})
diff --git a/hack/swift/Makefile b/hack/swift/Makefile
index 24da0ac3be..a082ede588 100644
--- a/hack/swift/Makefile
+++ b/hack/swift/Makefile
@@ -25,10 +25,7 @@ help:  ## Display this help
 
 azlogin:
 	@$(AZCLI) login
-
-azcfg:
 	@$(AZCLI) account set -s $(SUB)
-	@$(AZCLI) group create --location $(REGION) --name $(GROUP)
 	@$(AZCLI) extension add --name aks-preview --yes
 
 set-kubeconf: ## Adds the kubeconf for $CLUSTER
@@ -52,17 +49,21 @@ vars: ## Show the env vars configured for the swift command
 	@echo VNET=$(VNET)
 	@echo CLUSTER=$(CLUSTER)
 
-swift-net-up: azcfg ## Create required swift vnet/subnets
+rg-up: ## Create resource group $GROUP in $SUB/$REGION
+	@$(AZCLI) group create --location $(REGION) --name $(GROUP)
+
+rg-down: ## Delete the $GROUP in $SUB/$REGION
+	$(AZCLI) group delete -g $(GROUP) --yes
+
+net-up: ## Create required swift vnet/subnets
 	$(AZCLI) network vnet create -g $(GROUP) -l $(REGION) --name $(VNET) --address-prefixes 10.0.0.0/8 -o none
 	$(AZCLI) network vnet subnet create -g $(GROUP) --vnet-name $(VNET) --name nodenet --address-prefixes 10.240.0.0/16 -o none
 	$(AZCLI) network vnet subnet create -g $(GROUP) --vnet-name $(VNET) --name podnet --address-prefixes 10.241.0.0/16 -o none
 
-rg-down: azcfg ## Delete the $GROUP in $SUB/$REGION
-	$(AZCLI) group delete -g $(GROUP) --yes
 
 ##@ AKS Clusters
 
-swift-up: azcfg swift-net-up ## Brings up a swift cluster $name in $SUB/$REGION
+swift-up: rg-up net-up ## Brings up a swift cluster $name in $SUB/$REGION
 	$(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \
 		--max-pods 250 \
 		--node-count 3 \
@@ -72,7 +73,7 @@ swift-up: azcfg swift-net-up ## Brings up a swift cluster $name in $SUB/$REGION
 		--yes
 	@$(MAKE) set-kubeconf
 
-byocni-up: azcfg swift-net-up ## Brings up a swift cluster $name in $SUB/$REGION
+byocni-up: rg-up net-up ## Brings up a swift cluster $name in $SUB/$REGION
 	$(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \
 		--max-pods 250 \
 		--node-count 3 \
@@ -82,7 +83,7 @@ byocni-up: azcfg swift-net-up ## Brings up a swift cluster $name in $SUB/$REGION
 		--yes
 	@$(MAKE) set-kubeconf
 
-down: azcfg ## Deletes the swift resources $SUB/$REGION
+down: ## Deletes the swift resources $SUB/$REGION
 	$(AZCLI) aks delete -g $(GROUP) -n $(CLUSTER) --yes
 	@$(MAKE) unset-kubeconf
 	@$(MAKE) swift-rg-down

From b568d74d65587bfd7e0171dcdda7f47cbd75dc44 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Tue, 19 Jul 2022 14:12:25 -0700
Subject: [PATCH 05/51] use native azcli

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 84b251a3ea..00232a1e7b 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -33,7 +33,7 @@ steps:
         mkdir -p ~/.kube/
         echo "Create AKS cluster"
         cd ./hack/swift
-        sudo make byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS)
+        sudo make byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) AZCLI=az
         cd ../..
         echo "Cluster successfully created"
     displayName: Create test cluster
@@ -81,7 +81,7 @@ steps:
   - script: |
       cd ./hack/swift
       echo "Deleting cluster"
-      sudo make down SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS)
+      sudo make down SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) AZCLI=az
       cd ../..
     name: "Cleanupcluster"
     displayName: "Cleanup cluster"

From 0da809988f7a3d3024c1d0e58cab69ad4980fa43 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Tue, 19 Jul 2022 14:22:14 -0700
Subject: [PATCH 06/51] test /usr/bin/az

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 00232a1e7b..ab6df2a459 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -33,7 +33,7 @@ steps:
         mkdir -p ~/.kube/
         echo "Create AKS cluster"
         cd ./hack/swift
-        sudo make byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) AZCLI=az
+        sudo make byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) AZCLI=/usr/bin/az
         cd ../..
         echo "Cluster successfully created"
     displayName: Create test cluster
@@ -81,7 +81,7 @@ steps:
   - script: |
       cd ./hack/swift
       echo "Deleting cluster"
-      sudo make down SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) AZCLI=az
+      sudo make down SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) AZCLI=/usr/bin/az
       cd ../..
     name: "Cleanupcluster"
     displayName: "Cleanup cluster"

From cc3f504e32350b922f2ddb2c7166210a74488367 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Tue, 19 Jul 2022 14:53:37 -0700
Subject: [PATCH 07/51] login and set sub

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index ab6df2a459..c46556d43f 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -33,7 +33,8 @@ steps:
         mkdir -p ~/.kube/
         echo "Create AKS cluster"
         cd ./hack/swift
-        sudo make byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) AZCLI=/usr/bin/az
+        sudo make azlogin SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS)
+        sudo make byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) 
         cd ../..
         echo "Cluster successfully created"
     displayName: Create test cluster
@@ -81,7 +82,7 @@ steps:
   - script: |
       cd ./hack/swift
       echo "Deleting cluster"
-      sudo make down SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) AZCLI=/usr/bin/az
+      sudo make down SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS)
       cd ../..
     name: "Cleanupcluster"
     displayName: "Cleanup cluster"

From 4bdb0ac1e62f46137d470888ebff3d73a3f5ff3d Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Tue, 19 Jul 2022 16:28:10 -0700
Subject: [PATCH 08/51] testing

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index c46556d43f..758457154c 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -33,8 +33,7 @@ steps:
         mkdir -p ~/.kube/
         echo "Create AKS cluster"
         cd ./hack/swift
-        sudo make azlogin SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS)
-        sudo make byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) 
+        make byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) AZCLI=az
         cd ../..
         echo "Cluster successfully created"
     displayName: Create test cluster
@@ -82,7 +81,7 @@ steps:
   - script: |
       cd ./hack/swift
       echo "Deleting cluster"
-      sudo make down SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS)
+      sudo make down SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) AZCLI=az
       cd ../..
     name: "Cleanupcluster"
     displayName: "Cleanup cluster"

From 7ae90f42a78ad8185d312a6b355ba4d1171330ff Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Tue, 19 Jul 2022 17:13:20 -0700
Subject: [PATCH 09/51] try upgrade cli

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 758457154c..af5671ebe9 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -32,6 +32,7 @@ steps:
       inlineScript: |
         mkdir -p ~/.kube/
         echo "Create AKS cluster"
+        az upgrade
         cd ./hack/swift
         make byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) AZCLI=az
         cd ../..
@@ -81,7 +82,7 @@ steps:
   - script: |
       cd ./hack/swift
       echo "Deleting cluster"
-      sudo make down SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) AZCLI=az
+      make down SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) AZCLI=az
       cd ../..
     name: "Cleanupcluster"
     displayName: "Cleanup cluster"

From b6307ea9ac4e6897775f31cf07823dccefd8179c Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Tue, 19 Jul 2022 17:17:57 -0700
Subject: [PATCH 10/51] try upgrade azcli 2

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index af5671ebe9..95ec5fdd2b 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -32,7 +32,7 @@ steps:
       inlineScript: |
         mkdir -p ~/.kube/
         echo "Create AKS cluster"
-        az upgrade
+        az upgrade --yes
         cd ./hack/swift
         make byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) AZCLI=az
         cd ../..

From 77fb911c38a81b4b861526e833291c5a3fca3f37 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Wed, 20 Jul 2022 09:10:15 -0700
Subject: [PATCH 11/51] testing

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 3 ++-
 hack/swift/Makefile                                       | 5 ++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 95ec5fdd2b..ea3b84f1a8 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -32,8 +32,9 @@ steps:
       inlineScript: |
         mkdir -p ~/.kube/
         echo "Create AKS cluster"
-        az upgrade --yes
+        az upgrade --all --yes
         cd ./hack/swift
+        make azcfg AZCLI=az
         make byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) AZCLI=az
         cd ../..
         echo "Cluster successfully created"
diff --git a/hack/swift/Makefile b/hack/swift/Makefile
index a082ede588..57ca83943d 100644
--- a/hack/swift/Makefile
+++ b/hack/swift/Makefile
@@ -26,6 +26,8 @@ help:  ## Display this help
 azlogin:
 	@$(AZCLI) login
 	@$(AZCLI) account set -s $(SUB)
+
+azcfg:
 	@$(AZCLI) extension add --name aks-preview --yes
 
 set-kubeconf: ## Adds the kubeconf for $CLUSTER
@@ -77,9 +79,10 @@ byocni-up: rg-up net-up ## Brings up a swift cluster $name in $SUB/$REGION
 	$(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \
 		--max-pods 250 \
 		--node-count 3 \
-		--network-plugin none \
+		--network-plugin None \
 		--vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \
 		--pod-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/podnet \
+		--generate-ssh-keys \
 		--yes
 	@$(MAKE) set-kubeconf
 

From c4efd9fa8849f1dcf0ab82e6dd63cb623a9bc5bb Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Wed, 20 Jul 2022 09:20:48 -0700
Subject: [PATCH 12/51] testing

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 3 +--
 hack/swift/Makefile                                       | 2 +-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index ea3b84f1a8..f024c729b1 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -34,8 +34,7 @@ steps:
         echo "Create AKS cluster"
         az upgrade --all --yes
         cd ./hack/swift
-        make azcfg AZCLI=az
-        make byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) AZCLI=az
+        make byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS)
         cd ../..
         echo "Cluster successfully created"
     displayName: Create test cluster
diff --git a/hack/swift/Makefile b/hack/swift/Makefile
index 57ca83943d..898b2c59b7 100644
--- a/hack/swift/Makefile
+++ b/hack/swift/Makefile
@@ -79,7 +79,7 @@ byocni-up: rg-up net-up ## Brings up a swift cluster $name in $SUB/$REGION
 	$(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \
 		--max-pods 250 \
 		--node-count 3 \
-		--network-plugin None \
+		--network-plugin none \
 		--vnet-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/nodenet \
 		--pod-subnet-id /subscriptions/$(SUB)/resourceGroups/$(GROUP)/providers/Microsoft.Network/virtualNetworks/$(VNET)/subnets/podnet \
 		--generate-ssh-keys \

From e0f02d4033b0af0e74a2854c4a66e652ccad75c1 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Wed, 20 Jul 2022 11:10:02 -0700
Subject: [PATCH 13/51] test svc conn

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index f024c729b1..43f2ef132b 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -26,7 +26,7 @@ steps:
 
   - task: AzureCLI@1
     inputs:
-      azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+      azureSubscription: $(NETWORK_AGENT_TEST_SERVICE_CONNECTION)
       scriptLocation: "inlineScript"
       addSpnToEnvironment: true
       inlineScript: |
@@ -34,7 +34,8 @@ steps:
         echo "Create AKS cluster"
         az upgrade --all --yes
         cd ./hack/swift
-        make byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS)
+        make azcfg AZCLI=az
+        make byocni-up SUB=$(9b8218f9-902a-4d20-a65c-e98acec5362f) AZCLI=az
         cd ../..
         echo "Cluster successfully created"
     displayName: Create test cluster
@@ -82,7 +83,7 @@ steps:
   - script: |
       cd ./hack/swift
       echo "Deleting cluster"
-      make down SUB=$(SUB_AZURE_NETWORK_AGENT_BUILD_VALIDATIONS) AZCLI=az
+      make down SUB=$(9b8218f9-902a-4d20-a65c-e98acec5362f) AZCLI=az
       cd ../..
     name: "Cleanupcluster"
     displayName: "Cleanup cluster"

From 4f93caca737f298bf1720b2429083abc27082af4 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Wed, 20 Jul 2022 11:15:25 -0700
Subject: [PATCH 14/51] test svc conn

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 43f2ef132b..4d96e17482 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -26,7 +26,7 @@ steps:
 
   - task: AzureCLI@1
     inputs:
-      azureSubscription: $(NETWORK_AGENT_TEST_SERVICE_CONNECTION)
+      azureSubscription: NETWORK_AGENT_TEST_SERVICE_CONNECTION
       scriptLocation: "inlineScript"
       addSpnToEnvironment: true
       inlineScript: |

From 78b24683f3f2b754cb3d7951128760be81d0cdf3 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Wed, 20 Jul 2022 11:39:53 -0700
Subject: [PATCH 15/51] test svc conn

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 4d96e17482..a19d64a819 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -26,7 +26,7 @@ steps:
 
   - task: AzureCLI@1
     inputs:
-      azureSubscription: NETWORK_AGENT_TEST_SERVICE_CONNECTION
+      azureSubscription: "NETWORK_AGENT_TEST_SERVICE_CONNECTION"
       scriptLocation: "inlineScript"
       addSpnToEnvironment: true
       inlineScript: |

From a2ca982e5bb7a6070e64d02efb89231ede46755b Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Wed, 20 Jul 2022 12:06:22 -0700
Subject: [PATCH 16/51] test svc conn

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index a19d64a819..b71fd05cfb 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -26,7 +26,7 @@ steps:
 
   - task: AzureCLI@1
     inputs:
-      azureSubscription: "NETWORK_AGENT_TEST_SERVICE_CONNECTION"
+      azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
       scriptLocation: "inlineScript"
       addSpnToEnvironment: true
       inlineScript: |

From 545968dcff08a3e47a91bbb9d11147a56fb75118 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Wed, 20 Jul 2022 12:08:05 -0700
Subject: [PATCH 17/51] test svc conn

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index b71fd05cfb..2ad4143cc8 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -26,7 +26,7 @@ steps:
 
   - task: AzureCLI@1
     inputs:
-      azureSubscription: $(TEST_SUB_SERVICE_CONNECTION)
+      azureSubscription: "Azure Container Networking - Test"
       scriptLocation: "inlineScript"
       addSpnToEnvironment: true
       inlineScript: |

From b52073e899cee8e1ac7132644a6338844fa6d833 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Wed, 20 Jul 2022 12:15:52 -0700
Subject: [PATCH 18/51] test svc conn

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 2ad4143cc8..5e0993ae05 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -35,7 +35,7 @@ steps:
         az upgrade --all --yes
         cd ./hack/swift
         make azcfg AZCLI=az
-        make byocni-up SUB=$(9b8218f9-902a-4d20-a65c-e98acec5362f) AZCLI=az
+        make byocni-up SUB=9b8218f9-902a-4d20-a65c-e98acec5362f AZCLI=az
         cd ../..
         echo "Cluster successfully created"
     displayName: Create test cluster
@@ -83,7 +83,7 @@ steps:
   - script: |
       cd ./hack/swift
       echo "Deleting cluster"
-      make down SUB=$(9b8218f9-902a-4d20-a65c-e98acec5362f) AZCLI=az
+      make down SUB=9b8218f9-902a-4d20-a65c-e98acec5362f AZCLI=az
       cd ../..
     name: "Cleanupcluster"
     displayName: "Cleanup cluster"

From 4deb45b560f99244c8e4f844250a7367f84825d3 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Wed, 20 Jul 2022 14:01:49 -0700
Subject: [PATCH 19/51] testing

---
 .../singletenancy/aks-swift/e2e-step-template.yaml       | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 5e0993ae05..e998bcfc19 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -35,7 +35,7 @@ steps:
         az upgrade --all --yes
         cd ./hack/swift
         make azcfg AZCLI=az
-        make byocni-up SUB=9b8218f9-902a-4d20-a65c-e98acec5362f AZCLI=az
+        make byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az
         cd ../..
         echo "Cluster successfully created"
     displayName: Create test cluster
@@ -49,7 +49,9 @@ steps:
       sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
       kubectl cluster-info
       kubectl get po -owide -A
-      sudo -E env "PATH=$PATH" VERSION=$(Tag) INSTALL_CNI=true INSTALL_CNS=true make test-integration
+      kubectl apply -f ./test/integration/manifests/cni/manager.yaml
+      kubectl apply -f ./cns/azure-cns.yaml
+      sudo make test-integration
     retryCountOnTaskFailure: 3
     name: "aksswifte2e"
     displayName: "Run AKS Swift E2E"
@@ -83,7 +85,8 @@ steps:
   - script: |
       cd ./hack/swift
       echo "Deleting cluster"
-      make down SUB=9b8218f9-902a-4d20-a65c-e98acec5362f AZCLI=az
+      make azcfg AZCLI=az
+      make down SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az
       cd ../..
     name: "Cleanupcluster"
     displayName: "Cleanup cluster"

From 709d80eb7642da02c545707d7efa0fc64457f739 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Wed, 20 Jul 2022 14:41:40 -0700
Subject: [PATCH 20/51] testing

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index e998bcfc19..85dd251821 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -35,7 +35,7 @@ steps:
         az upgrade --all --yes
         cd ./hack/swift
         make azcfg AZCLI=az
-        make byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az
+        make byocni-up SUB=9b8218f9-902a-4d20-a65c-e98acec5362f AZCLI=az
         cd ../..
         echo "Cluster successfully created"
     displayName: Create test cluster
@@ -86,7 +86,7 @@ steps:
       cd ./hack/swift
       echo "Deleting cluster"
       make azcfg AZCLI=az
-      make down SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az
+      make down SUB=9b8218f9-902a-4d20-a65c-e98acec5362f AZCLI=az
       cd ../..
     name: "Cleanupcluster"
     displayName: "Cleanup cluster"

From 719f3b4923b677b5bb0fd3bb2620eaf1cec68bb4 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Wed, 20 Jul 2022 15:39:33 -0700
Subject: [PATCH 21/51] testing

---
 .../aks-swift/e2e-step-template.yaml          | 19 ++++++++++++-------
 1 file changed, 12 insertions(+), 7 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 85dd251821..a776d6316e 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -35,7 +35,7 @@ steps:
         az upgrade --all --yes
         cd ./hack/swift
         make azcfg AZCLI=az
-        make byocni-up SUB=9b8218f9-902a-4d20-a65c-e98acec5362f AZCLI=az
+        make byocni-up SUB=9b8218f9-902a-4d20-a65c-e98acec5362f AZCLI=az CLUSTER=$(Tag)
         cd ../..
         echo "Cluster successfully created"
     displayName: Create test cluster
@@ -82,12 +82,17 @@ steps:
     displayName: "Cleanup artifact dir"
     condition: always()
 
-  - script: |
-      cd ./hack/swift
-      echo "Deleting cluster"
-      make azcfg AZCLI=az
-      make down SUB=9b8218f9-902a-4d20-a65c-e98acec5362f AZCLI=az
-      cd ../..
+  - task: AzureCLI@1
+    inputs:
+      azureSubscription: "Azure Container Networking - Test"
+      scriptLocation: "inlineScript"
+      addSpnToEnvironment: true
+      inlineScript: |
+        cd ./hack/swift
+        echo "Deleting cluster"
+        make azcfg AZCLI=az
+        make down SUB=9b8218f9-902a-4d20-a65c-e98acec5362f AZCLI=az CLUSTER=$(TAG)
+        cd ../..
     name: "Cleanupcluster"
     displayName: "Cleanup cluster"
     condition: always()

From 7aefdd2e2942d4add8fe5256a57e712b6a6df63e Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Wed, 20 Jul 2022 15:59:38 -0700
Subject: [PATCH 22/51] testing

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 5 ++---
 Makefile                                                  | 6 +++++-
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index a776d6316e..733616f07a 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -32,10 +32,9 @@ steps:
       inlineScript: |
         mkdir -p ~/.kube/
         echo "Create AKS cluster"
-        az upgrade --all --yes
         cd ./hack/swift
         make azcfg AZCLI=az
-        make byocni-up SUB=9b8218f9-902a-4d20-a65c-e98acec5362f AZCLI=az CLUSTER=$(Tag)
+        make byocni-up SUB=9b8218f9-902a-4d20-a65c-e98acec5362f AZCLI=az CLUSTER=swifte2e-$(make revision)
         cd ../..
         echo "Cluster successfully created"
     displayName: Create test cluster
@@ -91,7 +90,7 @@ steps:
         cd ./hack/swift
         echo "Deleting cluster"
         make azcfg AZCLI=az
-        make down SUB=9b8218f9-902a-4d20-a65c-e98acec5362f AZCLI=az CLUSTER=$(TAG)
+        make down SUB=9b8218f9-902a-4d20-a65c-e98acec5362f AZCLI=az CLUSTER=swifte2e-$(make revision)
         cd ../..
     name: "Cleanupcluster"
     displayName: "Cleanup cluster"
diff --git a/Makefile b/Makefile
index 19f809131d..bdb2da489b 100644
--- a/Makefile
+++ b/Makefile
@@ -89,7 +89,8 @@ CNS_IMAGE_INFO_FILE = azure-cns-$(VERSION).txt
 CNM_PLUGIN_IMAGE ?= microsoft/azure-vnet-plugin
 CNM_PLUGIN_ROOTFS = azure-vnet-plugin-rootfs
 
-VERSION ?= $(shell git describe --exclude "zapai*" --tags --always --dirty)
+REVISION ?= $(shell git rev-parse --short HEAD)
+VERSION  ?= $(shell git describe --exclude "zapai*" --tags --always --dirty)
 
 # Default target
 all-binaries-platforms: ## Make all platform binaries
@@ -558,6 +559,9 @@ install-hooks: $(REPO_ROOT)/.git/hooks/pre-push ## installs git hooks
 
 setup: tools install-hooks ## performs common required repo setup
 
+revision: ## print the current git revision
+	@echo $(REVISION)
+	
 version: ## prints the version
 	@echo $(VERSION)
 

From 9e37a97798e8c5ac34735b2797bb999c596197c1 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Wed, 20 Jul 2022 16:44:47 -0700
Subject: [PATCH 23/51] testing

---
 .../singletenancy/aks-swift/e2e-step-template.yaml | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 733616f07a..328f37c1c7 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -32,10 +32,8 @@ steps:
       inlineScript: |
         mkdir -p ~/.kube/
         echo "Create AKS cluster"
-        cd ./hack/swift
-        make azcfg AZCLI=az
-        make byocni-up SUB=9b8218f9-902a-4d20-a65c-e98acec5362f AZCLI=az CLUSTER=swifte2e-$(make revision)
-        cd ../..
+        make -C ./hack/swift azcfg AZCLI=az
+        make -C ./hack/swift byocni-up SUB=9b8218f9-902a-4d20-a65c-e98acec5362f AZCLI=az CLUSTER=swifte2e-$(make revision)
         echo "Cluster successfully created"
     displayName: Create test cluster
     condition: succeeded()
@@ -81,17 +79,15 @@ steps:
     displayName: "Cleanup artifact dir"
     condition: always()
 
-  - task: AzureCLI@1
+  - task: AzureCLI@2
     inputs:
       azureSubscription: "Azure Container Networking - Test"
       scriptLocation: "inlineScript"
       addSpnToEnvironment: true
       inlineScript: |
-        cd ./hack/swift
         echo "Deleting cluster"
-        make azcfg AZCLI=az
-        make down SUB=9b8218f9-902a-4d20-a65c-e98acec5362f AZCLI=az CLUSTER=swifte2e-$(make revision)
-        cd ../..
+        make -C ./hack/swift azcfg AZCLI=az
+        make -C ./hack/swift down SUB=9b8218f9-902a-4d20-a65c-e98acec5362f AZCLI=az CLUSTER=swifte2e-$(make revision)
     name: "Cleanupcluster"
     displayName: "Cleanup cluster"
     condition: always()

From ee6668d5cf86149f5b5513837aa5d202a05bd81c Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Wed, 20 Jul 2022 16:52:00 -0700
Subject: [PATCH 24/51] testing

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 328f37c1c7..dc9e3ba378 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -48,6 +48,7 @@ steps:
       kubectl get po -owide -A
       kubectl apply -f ./test/integration/manifests/cni/manager.yaml
       kubectl apply -f ./cns/azure-cns.yaml
+      kubectl get po -owide -A
       sudo make test-integration
     retryCountOnTaskFailure: 3
     name: "aksswifte2e"

From f3985fde7bf3fc0ca8d56904f2274fafbe848386 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Wed, 20 Jul 2022 17:01:36 -0700
Subject: [PATCH 25/51] testing

---
 .pipelines/pipeline.yaml | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
index 2d04f6932f..0a6aaafec7 100644
--- a/.pipelines/pipeline.yaml
+++ b/.pipelines/pipeline.yaml
@@ -313,20 +313,12 @@ stages:
     displayName: Cleanup
     dependsOn:
       - "aks_swift_e2e"
-<<<<<<< HEAD
       # - "ubuntu_18_04_linux_e2e"
       # - "windows_19_03_e2e"
       # - "windows_20_04_e2e"
-      # - "windows_20_22_e2e"
+      # # - "windows_20_22_e2e"
       # - "ubuntu_18_04_linux_dualstack_e2e"
-=======
-      - "ubuntu_18_04_linux_e2e"
-      - "windows_19_03_e2e"
-      - "windows_20_04_e2e"
-      # - "windows_20_22_e2e"
-      - "ubuntu_18_04_linux_dualstack_e2e"
->>>>>>> 76e5f20dbfd1bb3e6d8e098a96d9db812fee21da
-      # - "windows_20_04_dualstack_e2e"
+      # # - "windows_20_04_dualstack_e2e"
     jobs:
       - job: delete_remote_artifacts
         displayName: Delete remote artifacts

From 6ede83b8f5ce3ca3d7262e60e389f54906a23964 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Thu, 21 Jul 2022 08:35:23 -0700
Subject: [PATCH 26/51] testing

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index dc9e3ba378..72ee07367b 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -84,6 +84,7 @@ steps:
     inputs:
       azureSubscription: "Azure Container Networking - Test"
       scriptLocation: "inlineScript"
+      scriptType: "bash"
       addSpnToEnvironment: true
       inlineScript: |
         echo "Deleting cluster"

From 847e926bd31ca4b9619d644dd94420088ee414d4 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Thu, 21 Jul 2022 11:14:38 -0700
Subject: [PATCH 27/51] testing

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 2 +-
 hack/swift/Makefile                                       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 72ee07367b..814c6f906c 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -49,7 +49,7 @@ steps:
       kubectl apply -f ./test/integration/manifests/cni/manager.yaml
       kubectl apply -f ./cns/azure-cns.yaml
       kubectl get po -owide -A
-      sudo make test-integration
+      sudo -E env "PATH=$PATH" make test-integration
     retryCountOnTaskFailure: 3
     name: "aksswifte2e"
     displayName: "Run AKS Swift E2E"
diff --git a/hack/swift/Makefile b/hack/swift/Makefile
index 898b2c59b7..864b955495 100644
--- a/hack/swift/Makefile
+++ b/hack/swift/Makefile
@@ -5,7 +5,7 @@ KUBECFG = $(HOME)/.kube
 SSH     = $(HOME)/.ssh
 AZCFG   = $(HOME)/.azure
 AZIMG   = mcr.microsoft.com/azure-cli
-AZCLI   = docker run -v $(AZCFG):/root/.azure -v $(KUBECFG):/root/.kube -v $(SSH):/root/.ssh $(AZIMG) az
+AZCLI   ?= docker run -v $(AZCFG):/root/.azure -v $(KUBECFG):/root/.kube -v $(SSH):/root/.ssh $(AZIMG) az
 
 # overrideable variables
 USER       ?= $(whoami)

From b1614035002f0af16f6ce7c34b0110cb7105bb41 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Thu, 21 Jul 2022 11:16:42 -0700
Subject: [PATCH 28/51] testing

---
 hack/swift/Makefile | 1 +
 1 file changed, 1 insertion(+)

diff --git a/hack/swift/Makefile b/hack/swift/Makefile
index 864b955495..b8330ca914 100644
--- a/hack/swift/Makefile
+++ b/hack/swift/Makefile
@@ -77,6 +77,7 @@ swift-up: rg-up net-up ## Brings up a swift cluster $name in $SUB/$REGION
 
 byocni-up: rg-up net-up ## Brings up a swift cluster $name in $SUB/$REGION
 	$(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \
+		--debug
 		--max-pods 250 \
 		--node-count 3 \
 		--network-plugin none \

From dbffdea0ddc1323ad6a57ed7c257744e6ee199ef Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Thu, 21 Jul 2022 11:56:19 -0700
Subject: [PATCH 29/51] testing

---
 hack/swift/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/swift/Makefile b/hack/swift/Makefile
index b8330ca914..1347c29e16 100644
--- a/hack/swift/Makefile
+++ b/hack/swift/Makefile
@@ -77,7 +77,7 @@ swift-up: rg-up net-up ## Brings up a swift cluster $name in $SUB/$REGION
 
 byocni-up: rg-up net-up ## Brings up a swift cluster $name in $SUB/$REGION
 	$(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \
-		--debug
+		--debug \
 		--max-pods 250 \
 		--node-count 3 \
 		--network-plugin none \

From b049b06ab4f5367654d95bfd685e456a75d65000 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Thu, 21 Jul 2022 13:56:56 -0700
Subject: [PATCH 30/51] testing

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 814c6f906c..68dd3296a4 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -28,12 +28,13 @@ steps:
     inputs:
       azureSubscription: "Azure Container Networking - Test"
       scriptLocation: "inlineScript"
+      scriptType: "bash"
       addSpnToEnvironment: true
       inlineScript: |
         mkdir -p ~/.kube/
         echo "Create AKS cluster"
         make -C ./hack/swift azcfg AZCLI=az
-        make -C ./hack/swift byocni-up SUB=9b8218f9-902a-4d20-a65c-e98acec5362f AZCLI=az CLUSTER=swifte2e-$(make revision)
+        make -C ./hack/swift byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=swifte2e-$(make revision)
         echo "Cluster successfully created"
     displayName: Create test cluster
     condition: succeeded()
@@ -89,7 +90,7 @@ steps:
       inlineScript: |
         echo "Deleting cluster"
         make -C ./hack/swift azcfg AZCLI=az
-        make -C ./hack/swift down SUB=9b8218f9-902a-4d20-a65c-e98acec5362f AZCLI=az CLUSTER=swifte2e-$(make revision)
+        make -C ./hack/swift down SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=swifte2e-$(make revision)
     name: "Cleanupcluster"
     displayName: "Cleanup cluster"
     condition: always()

From c591cf2e162513ea0ee7b880d59974a674c10064 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Thu, 21 Jul 2022 15:30:15 -0700
Subject: [PATCH 31/51] testing

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 68dd3296a4..47a2b44ce3 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -47,10 +47,7 @@ steps:
       sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
       kubectl cluster-info
       kubectl get po -owide -A
-      kubectl apply -f ./test/integration/manifests/cni/manager.yaml
-      kubectl apply -f ./cns/azure-cns.yaml
-      kubectl get po -owide -A
-      sudo -E env "PATH=$PATH" make test-integration
+      sudo -E env "PATH=$PATH" VERSION=$(Tag) INSTALL_CNI=true INSTALL_CNS=true make test-integration
     retryCountOnTaskFailure: 3
     name: "aksswifte2e"
     displayName: "Run AKS Swift E2E"

From 05ef974c5891224f1d6b018a44d79a19907c9b2a Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Thu, 21 Jul 2022 16:14:26 -0700
Subject: [PATCH 32/51] testing

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 47a2b44ce3..7be34a96b8 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -35,6 +35,8 @@ steps:
         echo "Create AKS cluster"
         make -C ./hack/swift azcfg AZCLI=az
         make -C ./hack/swift byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=swifte2e-$(make revision)
+        kubectl cluster-info
+        kubectl get po -owide -A
         echo "Cluster successfully created"
     displayName: Create test cluster
     condition: succeeded()

From 4c562fa3dffce23fc7c1abc4f485919f2c6a25b5 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Thu, 21 Jul 2022 16:19:02 -0700
Subject: [PATCH 33/51] testing

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 7be34a96b8..47a2b44ce3 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -35,8 +35,6 @@ steps:
         echo "Create AKS cluster"
         make -C ./hack/swift azcfg AZCLI=az
         make -C ./hack/swift byocni-up SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=swifte2e-$(make revision)
-        kubectl cluster-info
-        kubectl get po -owide -A
         echo "Cluster successfully created"
     displayName: Create test cluster
     condition: succeeded()

From 98a072bc28f5468379e0691a508e87f8f0709052 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Thu, 21 Jul 2022 17:15:25 -0700
Subject: [PATCH 34/51] fix rg-down

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 5 ++++-
 hack/swift/Makefile                                       | 2 +-
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 47a2b44ce3..5f271792dc 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -47,7 +47,10 @@ steps:
       sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
       kubectl cluster-info
       kubectl get po -owide -A
-      sudo -E env "PATH=$PATH" VERSION=$(Tag) INSTALL_CNI=true INSTALL_CNS=true make test-integration
+      kubectl apply -f ./test/integration/manifests/cni/manager.yaml
+      kubectl apply -f ./cns/azure-cns.yaml
+      kubectl get po -owide -A
+      sudo make test-integration
     retryCountOnTaskFailure: 3
     name: "aksswifte2e"
     displayName: "Run AKS Swift E2E"
diff --git a/hack/swift/Makefile b/hack/swift/Makefile
index 1347c29e16..c74455b0a8 100644
--- a/hack/swift/Makefile
+++ b/hack/swift/Makefile
@@ -90,4 +90,4 @@ byocni-up: rg-up net-up ## Brings up a swift cluster $name in $SUB/$REGION
 down: ## Deletes the swift resources $SUB/$REGION
 	$(AZCLI) aks delete -g $(GROUP) -n $(CLUSTER) --yes
 	@$(MAKE) unset-kubeconf
-	@$(MAKE) swift-rg-down
+	@$(MAKE) rg-down

From 07d4e5de2337421fe972eb5f9fa06fc2775fda19 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Mon, 25 Jul 2022 08:57:10 -0700
Subject: [PATCH 35/51] testing

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 5f271792dc..9b3dffbb9f 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -61,7 +61,7 @@ steps:
       echo $ARTIFACT_DIR
       sudo rm -rf $ARTIFACT_DIR
       sudo mkdir $ARTIFACT_DIR
-      sudo cp test/integration/logs/* $ARTIFACT_DIR
+      sudo cp test/integration/logs $ARTIFACT_DIR
     name: "GetLogs"
     displayName: "Get logs"
     condition: always()

From 1339d9aef22923709b6f7462259177943442a392 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Mon, 25 Jul 2022 09:55:06 -0700
Subject: [PATCH 36/51] testing

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 9b3dffbb9f..b8875b14a8 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -61,7 +61,7 @@ steps:
       echo $ARTIFACT_DIR
       sudo rm -rf $ARTIFACT_DIR
       sudo mkdir $ARTIFACT_DIR
-      sudo cp test/integration/logs $ARTIFACT_DIR
+      sudo cp ./test/integration/logs/* $ARTIFACT_DIR
     name: "GetLogs"
     displayName: "Get logs"
     condition: always()

From c3bcc588b89fd770fb8b2da327ae115da8e25364 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Mon, 25 Jul 2022 10:53:15 -0700
Subject: [PATCH 37/51] testing

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index b8875b14a8..1a3d489994 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -51,6 +51,9 @@ steps:
       kubectl apply -f ./cns/azure-cns.yaml
       kubectl get po -owide -A
       sudo make test-integration
+      sudo mkdir integration-logs
+      sudo ls
+      sudo cp test/integration/logs/* integration-logs
     retryCountOnTaskFailure: 3
     name: "aksswifte2e"
     displayName: "Run AKS Swift E2E"
@@ -61,7 +64,7 @@ steps:
       echo $ARTIFACT_DIR
       sudo rm -rf $ARTIFACT_DIR
       sudo mkdir $ARTIFACT_DIR
-      sudo cp ./test/integration/logs/* $ARTIFACT_DIR
+      sudo cp test/integration/logs/* $ARTIFACT_DIR
     name: "GetLogs"
     displayName: "Get logs"
     condition: always()

From 975a5409b84b59f314593cb9f281e373932ca072 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Mon, 25 Jul 2022 11:45:49 -0700
Subject: [PATCH 38/51] testing

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 1a3d489994..c64e1bc8c1 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -51,9 +51,6 @@ steps:
       kubectl apply -f ./cns/azure-cns.yaml
       kubectl get po -owide -A
       sudo make test-integration
-      sudo mkdir integration-logs
-      sudo ls
-      sudo cp test/integration/logs/* integration-logs
     retryCountOnTaskFailure: 3
     name: "aksswifte2e"
     displayName: "Run AKS Swift E2E"
@@ -64,7 +61,8 @@ steps:
       echo $ARTIFACT_DIR
       sudo rm -rf $ARTIFACT_DIR
       sudo mkdir $ARTIFACT_DIR
-      sudo cp test/integration/logs/* $ARTIFACT_DIR
+      sudo cp test/integration $ARTIFACT_DIR
+      sudo ls test/integration/logs
     name: "GetLogs"
     displayName: "Get logs"
     condition: always()

From b2fff9d7cde1accc671539d5bdb082908b096960 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Mon, 25 Jul 2022 12:29:17 -0700
Subject: [PATCH 39/51] testing

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index c64e1bc8c1..5109553d6e 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -61,8 +61,7 @@ steps:
       echo $ARTIFACT_DIR
       sudo rm -rf $ARTIFACT_DIR
       sudo mkdir $ARTIFACT_DIR
-      sudo cp test/integration $ARTIFACT_DIR
-      sudo ls test/integration/logs
+      sudo cp -r ./test/integration/... $ARTIFACT_DIR
     name: "GetLogs"
     displayName: "Get logs"
     condition: always()

From 7511179fa308fac242f6fbf5e8a9b755eddf7073 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Mon, 25 Jul 2022 13:02:59 -0700
Subject: [PATCH 40/51] testing

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 5109553d6e..c4f463b135 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -61,7 +61,7 @@ steps:
       echo $ARTIFACT_DIR
       sudo rm -rf $ARTIFACT_DIR
       sudo mkdir $ARTIFACT_DIR
-      sudo cp -r ./test/integration/... $ARTIFACT_DIR
+      sudo cp -r test/integration $ARTIFACT_DIR
     name: "GetLogs"
     displayName: "Get logs"
     condition: always()

From 90db37c5a0a57c0096c284a8e9661afbfce5888d Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Mon, 25 Jul 2022 15:48:57 -0700
Subject: [PATCH 41/51] testing

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index c4f463b135..7323fe97ea 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -47,10 +47,7 @@ steps:
       sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
       kubectl cluster-info
       kubectl get po -owide -A
-      kubectl apply -f ./test/integration/manifests/cni/manager.yaml
-      kubectl apply -f ./cns/azure-cns.yaml
-      kubectl get po -owide -A
-      sudo make test-integration
+      sudo -E env "PATH=$PATH" VERSION=$(Tag) INSTALL_CNI=true INSTALL_CNS=true make test-integration
     retryCountOnTaskFailure: 3
     name: "aksswifte2e"
     displayName: "Run AKS Swift E2E"
@@ -61,7 +58,7 @@ steps:
       echo $ARTIFACT_DIR
       sudo rm -rf $ARTIFACT_DIR
       sudo mkdir $ARTIFACT_DIR
-      sudo cp -r test/integration $ARTIFACT_DIR
+      sudo cp -r test/integration/logs/* $ARTIFACT_DIR
     name: "GetLogs"
     displayName: "Get logs"
     condition: always()

From 774600cb011693667f754a361d189c18259dd368 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Mon, 25 Jul 2022 16:25:08 -0700
Subject: [PATCH 42/51] update ds

---
 test/integration/manifests/cns/daemonset.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/integration/manifests/cns/daemonset.yaml b/test/integration/manifests/cns/daemonset.yaml
index 5c426e8175..f6b31d7b0b 100644
--- a/test/integration/manifests/cns/daemonset.yaml
+++ b/test/integration/manifests/cns/daemonset.yaml
@@ -41,7 +41,7 @@ spec:
           effect: NoSchedule
       containers:
         - name: cns-container
-          image: acnpublic.azurecr.io/azure-cns:v1.2.8
+          image: acnpublic.azurecr.io/azure-cns:v1.4.29
           imagePullPolicy: IfNotPresent
           args: [ "-c", "tcp://$(CNSIpAddress):$(CNSPort)", "-t", "$(CNSLogTarget)"]
           volumeMounts:

From 0f3cd4a9f763f009fc61057f0ecf1529ad210de5 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Mon, 25 Jul 2022 17:53:08 -0700
Subject: [PATCH 43/51] testing

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 7323fe97ea..c4f463b135 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -47,7 +47,10 @@ steps:
       sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
       kubectl cluster-info
       kubectl get po -owide -A
-      sudo -E env "PATH=$PATH" VERSION=$(Tag) INSTALL_CNI=true INSTALL_CNS=true make test-integration
+      kubectl apply -f ./test/integration/manifests/cni/manager.yaml
+      kubectl apply -f ./cns/azure-cns.yaml
+      kubectl get po -owide -A
+      sudo make test-integration
     retryCountOnTaskFailure: 3
     name: "aksswifte2e"
     displayName: "Run AKS Swift E2E"
@@ -58,7 +61,7 @@ steps:
       echo $ARTIFACT_DIR
       sudo rm -rf $ARTIFACT_DIR
       sudo mkdir $ARTIFACT_DIR
-      sudo cp -r test/integration/logs/* $ARTIFACT_DIR
+      sudo cp -r test/integration $ARTIFACT_DIR
     name: "GetLogs"
     displayName: "Get logs"
     condition: always()

From 0b49dcb91b2d212c6c72d9eceeb48943a8f397be Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Tue, 26 Jul 2022 11:35:08 -0700
Subject: [PATCH 44/51] build imgs

---
 .pipelines/pipeline.yaml                      | 394 +++++++++---------
 .../aks-swift/e2e-step-template.yaml          |   7 +-
 2 files changed, 199 insertions(+), 202 deletions(-)

diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
index 0a6aaafec7..47039b14b5 100644
--- a/.pipelines/pipeline.yaml
+++ b/.pipelines/pipeline.yaml
@@ -34,214 +34,214 @@ stages:
             displayName: "Set environmental variables"
             condition: always()
 
-  # - stage: test
-  #   displayName: Test ACN
-  #   dependsOn:
-  #     - setup
-  #   jobs:
-  #     - job: test
-  #       displayName: Run Tests
-  #       variables:
-  #         STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ]
-  #       pool:
-  #         name: "$(BUILD_POOL_NAME_DEFAULT)"
-  #       steps:
-  #         - script: |
-  #             make tools
-  #             # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout
-  #             { { { {
-  #                   sudo -E env "PATH=$PATH" make test-all;
-  #                   echo $? >&3;
-  #                   } | tee >(build/tools/bin/go-junit-report > report.xml) >&4;
-  #                 } 3>&1;
-  #               } | { read xs; exit $xs; }
-  #             } 4>&1
-  #           retryCountOnTaskFailure: 3
-  #           name: "Test"
-  #           displayName: "Run Tests"
+  - stage: test
+    displayName: Test ACN
+    dependsOn:
+      - setup
+    jobs:
+      - job: test
+        displayName: Run Tests
+        variables:
+          STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ]
+        pool:
+          name: "$(BUILD_POOL_NAME_DEFAULT)"
+        steps:
+          - script: |
+              make tools
+              # run test, echo exit status code to fd 3, pipe output from test to tee, which splits output to stdout and go-junit-report (which converts test output to report.xml), stdout from tee is redirected to fd 4. Take output written to fd 3 (which is the exit code of test), redirect to stdout, pipe to read from stdout then exit with that status code. Read all output from fd 4 (output from tee) and write to top stdout
+              { { { {
+                    sudo -E env "PATH=$PATH" make test-all;
+                    echo $? >&3;
+                    } | tee >(build/tools/bin/go-junit-report > report.xml) >&4;
+                  } 3>&1;
+                } | { read xs; exit $xs; }
+              } 4>&1
+            retryCountOnTaskFailure: 3
+            name: "Test"
+            displayName: "Run Tests"
 
-  #         - bash: |
-  #             build/tools/bin/gocov convert coverage.out > coverage.json
-  #             build/tools/bin/gocov-xml < coverage.json > coverage.xml
-  #           name: "Coverage"
-  #           displayName: "Generate Coverage Reports"
-  #           condition: always()
+          - bash: |
+              build/tools/bin/gocov convert coverage.out > coverage.json
+              build/tools/bin/gocov-xml < coverage.json > coverage.xml
+            name: "Coverage"
+            displayName: "Generate Coverage Reports"
+            condition: always()
 
-  #         - task: PublishTestResults@2
-  #           inputs:
-  #             testRunner: JUnit
-  #             testResultsFiles: report.xml
-  #           displayName: "Publish Test Results"
-  #           condition: always()
+          - task: PublishTestResults@2
+            inputs:
+              testRunner: JUnit
+              testResultsFiles: report.xml
+            displayName: "Publish Test Results"
+            condition: always()
 
-  #         - task: PublishCodeCoverageResults@1
-  #           inputs:
-  #             codeCoverageTool: Cobertura
-  #             summaryFileLocation: coverage.xml
-  #           displayName: "Publish Code Coverage Results"
-  #           condition: always()
+          - task: PublishCodeCoverageResults@1
+            inputs:
+              codeCoverageTool: Cobertura
+              summaryFileLocation: coverage.xml
+            displayName: "Publish Code Coverage Results"
+            condition: always()
 
-  # - stage: binaries
-  #   displayName: Build Binaries
-  #   dependsOn:
-  #     - setup
-  #     - test
-  #   jobs:
-  #     - job: build
-  #       displayName: Build Binaries
-  #       variables:
-  #         TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ]
-  #         STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ]
-  #       pool:
-  #         name: "$(BUILD_POOL_NAME_DEFAULT)"
-  #       steps:
-  #         - script: |
-  #             make all-binaries-platforms VERSION=$(TAG)
-  #           name: "BuildAllPlatformBinaries"
-  #           displayName: "Build all platform binaries"
+  - stage: binaries
+    displayName: Build Binaries
+    dependsOn:
+      - setup
+      - test
+    jobs:
+      - job: build
+        displayName: Build Binaries
+        variables:
+          TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ]
+          STORAGE_ID: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.StorageID'] ]
+        pool:
+          name: "$(BUILD_POOL_NAME_DEFAULT)"
+        steps:
+          - script: |
+              make all-binaries-platforms VERSION=$(TAG)
+            name: "BuildAllPlatformBinaries"
+            displayName: "Build all platform binaries"
 
-  #         - script: |
-  #             mkdir -p ./output/bins
-  #             cd ./output
-  #             find . -name '*.tgz' -print -exec mv -t ./bins/ {} +
-  #             find . -name '*.zip' -print -exec mv -t ./bins/ {} +
-  #             shopt -s extglob
-  #             rm -rf !("bins")
-  #           name: "PrepareArtifacts"
-  #           displayName: "Prepare Artifacts"
+          - script: |
+              mkdir -p ./output/bins
+              cd ./output
+              find . -name '*.tgz' -print -exec mv -t ./bins/ {} +
+              find . -name '*.zip' -print -exec mv -t ./bins/ {} +
+              shopt -s extglob
+              rm -rf !("bins")
+            name: "PrepareArtifacts"
+            displayName: "Prepare Artifacts"
 
-  #         - task: CopyFiles@2
-  #           inputs:
-  #             sourceFolder: "output"
-  #             targetFolder: $(Build.ArtifactStagingDirectory)
-  #           condition: succeeded()
+          - task: CopyFiles@2
+            inputs:
+              sourceFolder: "output"
+              targetFolder: $(Build.ArtifactStagingDirectory)
+            condition: succeeded()
 
-  #         - task: PublishBuildArtifacts@1
-  #           inputs:
-  #             artifactName: "output"
-  #             pathtoPublish: "$(Build.ArtifactStagingDirectory)"
-  #           condition: succeeded()
+          - task: PublishBuildArtifacts@1
+            inputs:
+              artifactName: "output"
+              pathtoPublish: "$(Build.ArtifactStagingDirectory)"
+            condition: succeeded()
 
-  #         - task: AzureCLI@1
-  #           inputs:
-  #             azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
-  #             scriptLocation: "inlineScript"
-  #             inlineScript: |
-  #               echo Creating storage container with name acn-$(STORAGE_ID) and account name $(STORAGE_ACCOUNT_NAME)
-  #               az storage container create -n acn-$(STORAGE_ID) --account-name $(STORAGE_ACCOUNT_NAME) --public-access container
-  #               az storage blob upload-batch -d acn-$(STORAGE_ID) -s ./output/bins/  --account-name $(STORAGE_ACCOUNT_NAME)
-  #           displayName: Create artifact storage container
-  #           condition: succeeded()
+          - task: AzureCLI@1
+            inputs:
+              azureSubscription: $(BUILD_VALIDATIONS_SERVICE_CONNECTION)
+              scriptLocation: "inlineScript"
+              inlineScript: |
+                echo Creating storage container with name acn-$(STORAGE_ID) and account name $(STORAGE_ACCOUNT_NAME)
+                az storage container create -n acn-$(STORAGE_ID) --account-name $(STORAGE_ACCOUNT_NAME) --public-access container
+                az storage blob upload-batch -d acn-$(STORAGE_ID) -s ./output/bins/  --account-name $(STORAGE_ACCOUNT_NAME)
+            displayName: Create artifact storage container
+            condition: succeeded()
 
-  #         - publish: ./test/apimodels/
-  #           artifact: clusterdefinitions
+          - publish: ./test/apimodels/
+            artifact: clusterdefinitions
 
-  # - stage: containerize
-  #   displayName: Build Images
-  #   dependsOn:
-  #     - setup
-  #     - test
-  #   jobs:
-  #     - job: containerize
-  #       displayName: Build Images
-  #       variables:
-  #         TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ]
-  #       pool:
-  #         name: "$(BUILD_POOL_NAME_DEFAULT)"
-  #       strategy:
-  #         matrix:
-  #           acncli_linux_amd64:
-  #             arch: amd64
-  #             os: linux
-  #             name: acncli
-  #           acncli_linux_arm64:
-  #             arch: arm64
-  #             os: linux
-  #             name: acncli
-  #           cni_dropgz_linux_amd64:
-  #             arch: amd64
-  #             os: linux
-  #             name: cni-dropgz
-  #           cni_dropgz_linux_arm64:
-  #             arch: arm64
-  #             os: linux
-  #             name: cni-dropgz
-  #           cns_linux_amd64:
-  #             arch: amd64
-  #             os: linux
-  #             name: cns
-  #           cns_linux_arm64:
-  #             arch: arm64
-  #             os: linux
-  #             name: cns
-  #           npm_linux_amd64:
-  #             arch: amd64
-  #             os: linux
-  #             name: npm
-  #           npm_linux_arm64:
-  #             arch: arm64
-  #             os: linux
-  #             name: npm
-  #       steps:
-  #         - template: containers/container-template.yaml
-  #           parameters:
-  #             name: $(name)
-  #             os: $(os)
-  #             arch: $(arch)
-  #             tag: $(TAG)
-  #     - job: containerize_windows
-  #       displayName: Build Images
-  #       variables:
-  #         TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ]
-  #       pool:
-  #         name: "$(BUILD_POOL_NAME_DEFAULT_WINDOWS)"
-  #       strategy:
-  #         matrix:
-  #           cns_windows_amd64:
-  #             arch: amd64
-  #             name: cns
-  #           npm_windows_amd64:
-  #             arch: amd64
-  #             name: npm
-  #       steps:
-  #         - template: containers/container-template-windows.yaml
-  #           parameters:
-  #             name: $(name)
-  #             arch: $(arch)
-  #             tag: $(TAG)
+  - stage: containerize
+    displayName: Build Images
+    dependsOn:
+      - setup
+      - test
+    jobs:
+      - job: containerize
+        displayName: Build Images
+        variables:
+          TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ]
+        pool:
+          name: "$(BUILD_POOL_NAME_DEFAULT)"
+        strategy:
+          matrix:
+            acncli_linux_amd64:
+              arch: amd64
+              os: linux
+              name: acncli
+            acncli_linux_arm64:
+              arch: arm64
+              os: linux
+              name: acncli
+            cni_dropgz_linux_amd64:
+              arch: amd64
+              os: linux
+              name: cni-dropgz
+            cni_dropgz_linux_arm64:
+              arch: arm64
+              os: linux
+              name: cni-dropgz
+            cns_linux_amd64:
+              arch: amd64
+              os: linux
+              name: cns
+            cns_linux_arm64:
+              arch: arm64
+              os: linux
+              name: cns
+            npm_linux_amd64:
+              arch: amd64
+              os: linux
+              name: npm
+            npm_linux_arm64:
+              arch: arm64
+              os: linux
+              name: npm
+        steps:
+          - template: containers/container-template.yaml
+            parameters:
+              name: $(name)
+              os: $(os)
+              arch: $(arch)
+              tag: $(TAG)
+      - job: containerize_windows
+        displayName: Build Images
+        variables:
+          TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ]
+        pool:
+          name: "$(BUILD_POOL_NAME_DEFAULT_WINDOWS)"
+        strategy:
+          matrix:
+            cns_windows_amd64:
+              arch: amd64
+              name: cns
+            npm_windows_amd64:
+              arch: amd64
+              name: npm
+        steps:
+          - template: containers/container-template-windows.yaml
+            parameters:
+              name: $(name)
+              arch: $(arch)
+              tag: $(TAG)
 
-  # - stage: publish
-  #   displayName: Publish Multiarch Manifests
-  #   dependsOn:
-  #     - setup
-  #     - containerize
-  #   jobs:
-  #     - job: manifest
-  #       displayName: Compile Manifests
-  #       variables:
-  #         TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ]
-  #       pool:
-  #         name: "$(BUILD_POOL_NAME_DEFAULT)"
-  #       strategy:
-  #         matrix:
-  #           acncli:
-  #             name: acncli 
-  #             platforms: linux/amd64 linux/arm64
-  #           cni_dropgz:
-  #             name: cni-dropgz
-  #             platforms: linux/amd64 linux/arm64
-  #           cns:
-  #             name: cns
-  #             platforms: linux/amd64 linux/arm64 windows/amd64
-  #           npm:
-  #             name: npm
-  #             platforms: linux/amd64 linux/arm64 windows/amd64
-  #       steps:
-  #         - template: containers/manifest-template.yaml
-  #           parameters:
-  #             name: $(name)
-  #             platforms: $(platforms)
-  #             tag: $(TAG)
+  - stage: publish
+    displayName: Publish Multiarch Manifests
+    dependsOn:
+      - setup
+      - containerize
+    jobs:
+      - job: manifest
+        displayName: Compile Manifests
+        variables:
+          TAG: $[ stagedependencies.setup.env.outputs['EnvironmentalVariables.Tag'] ]
+        pool:
+          name: "$(BUILD_POOL_NAME_DEFAULT)"
+        strategy:
+          matrix:
+            acncli:
+              name: acncli 
+              platforms: linux/amd64 linux/arm64
+            cni_dropgz:
+              name: cni-dropgz
+              platforms: linux/amd64 linux/arm64
+            cns:
+              name: cns
+              platforms: linux/amd64 linux/arm64 windows/amd64
+            npm:
+              name: npm
+              platforms: linux/amd64 linux/arm64 windows/amd64
+        steps:
+          - template: containers/manifest-template.yaml
+            parameters:
+              name: $(name)
+              platforms: $(platforms)
+              tag: $(TAG)
 
   - template: singletenancy/aks-swift/e2e-job-template.yaml
     parameters:
diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index c4f463b135..47a2b44ce3 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -47,10 +47,7 @@ steps:
       sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl
       kubectl cluster-info
       kubectl get po -owide -A
-      kubectl apply -f ./test/integration/manifests/cni/manager.yaml
-      kubectl apply -f ./cns/azure-cns.yaml
-      kubectl get po -owide -A
-      sudo make test-integration
+      sudo -E env "PATH=$PATH" VERSION=$(Tag) INSTALL_CNI=true INSTALL_CNS=true make test-integration
     retryCountOnTaskFailure: 3
     name: "aksswifte2e"
     displayName: "Run AKS Swift E2E"
@@ -61,7 +58,7 @@ steps:
       echo $ARTIFACT_DIR
       sudo rm -rf $ARTIFACT_DIR
       sudo mkdir $ARTIFACT_DIR
-      sudo cp -r test/integration $ARTIFACT_DIR
+      sudo cp test/integration/logs/* $ARTIFACT_DIR
     name: "GetLogs"
     displayName: "Get logs"
     condition: always()

From eaa3589dccb5141806d9a6b5251c50d0012d9740 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Tue, 26 Jul 2022 13:02:56 -0700
Subject: [PATCH 45/51] test

---
 .pipelines/singletenancy/aks-swift/e2e-job-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
index d9f80dc396..79057bb764 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
@@ -9,7 +9,7 @@ stages:
     lockBehavior: sequential
     dependsOn: 
     - setup
-    # - publish
+    - publish
     jobs:
       - job: ${{ parameters.name }}
         displayName: Singletenancy AKS Swift Suite - (${{ parameters.name }})

From df86fc12da195cfd0ca0944973b5e2f0164f1bfb Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Wed, 27 Jul 2022 09:02:47 -0700
Subject: [PATCH 46/51] uncomment pipeline

---
 .pipelines/pipeline.yaml | 120 +++++++++++++++++++--------------------
 1 file changed, 60 insertions(+), 60 deletions(-)

diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
index 47039b14b5..f7521c307c 100644
--- a/.pipelines/pipeline.yaml
+++ b/.pipelines/pipeline.yaml
@@ -249,76 +249,76 @@ stages:
       displayName: AKS Swift
       pipelineBuildImage: "$(BUILD_IMAGE)"
 
-  # - template: singletenancy/aks-engine/e2e-job-template.yaml
-  #   parameters:
-  #     name: "ubuntu_18_04_linux_e2e"
-  #     displayName: Ubuntu 18.04
-  #     pipelineBuildImage: "$(BUILD_IMAGE)"
-  #     clusterDefinition: "cniLinux1804.json"
-  #     clusterDefinitionCniTypeKey: "azureCNIURLLinux"
-  #     clusterDefinitionCniBuildOS: "linux"
-  #     clusterDefinitionCniBuildExt: ".tgz"
+  - template: singletenancy/aks-engine/e2e-job-template.yaml
+    parameters:
+      name: "ubuntu_18_04_linux_e2e"
+      displayName: Ubuntu 18.04
+      pipelineBuildImage: "$(BUILD_IMAGE)"
+      clusterDefinition: "cniLinux1804.json"
+      clusterDefinitionCniTypeKey: "azureCNIURLLinux"
+      clusterDefinitionCniBuildOS: "linux"
+      clusterDefinitionCniBuildExt: ".tgz"
 
-  # - template: singletenancy/aks-engine/e2e-job-template.yaml
-  #   parameters:
-  #     name: "windows_19_03_e2e"
-  #     displayName: "Windows 1903"
-  #     pipelineBuildImage: "$(BUILD_IMAGE)"
-  #     clusterDefinition: "cniWindows1903.json"
-  #     clusterDefinitionCniTypeKey: "azureCNIURLWindows"
-  #     clusterDefinitionCniBuildOS: "windows"
-  #     clusterDefinitionCniBuildExt: ".zip"
+  - template: singletenancy/aks-engine/e2e-job-template.yaml
+    parameters:
+      name: "windows_19_03_e2e"
+      displayName: "Windows 1903"
+      pipelineBuildImage: "$(BUILD_IMAGE)"
+      clusterDefinition: "cniWindows1903.json"
+      clusterDefinitionCniTypeKey: "azureCNIURLWindows"
+      clusterDefinitionCniBuildOS: "windows"
+      clusterDefinitionCniBuildExt: ".zip"
 
-  # - template: singletenancy/aks-engine/e2e-job-template.yaml
-  #   parameters:
-  #     name: "windows_20_04_e2e"
-  #     displayName: "Windows 2004"
-  #     pipelineBuildImage: "$(BUILD_IMAGE)"
-  #     clusterDefinition: "cniWindows2004.json"
-  #     clusterDefinitionCniTypeKey: "azureCNIURLWindows"
-  #     clusterDefinitionCniBuildOS: "windows"
-  #     clusterDefinitionCniBuildExt: ".zip"
+  - template: singletenancy/aks-engine/e2e-job-template.yaml
+    parameters:
+      name: "windows_20_04_e2e"
+      displayName: "Windows 2004"
+      pipelineBuildImage: "$(BUILD_IMAGE)"
+      clusterDefinition: "cniWindows2004.json"
+      clusterDefinitionCniTypeKey: "azureCNIURLWindows"
+      clusterDefinitionCniBuildOS: "windows"
+      clusterDefinitionCniBuildExt: ".zip"
 
-  # - template: singletenancy/aks-engine/e2e-job-template.yaml
-  #   parameters:
-  #     name: "windows_20_22_e2e"
-  #     displayName: "Windows 2022"
-  #     pipelineBuildImage: "$(BUILD_IMAGE)"
-  #     clusterDefinition: "cniWindows2022.json"
-  #     clusterDefinitionCniTypeKey: "azureCNIURLWindows"
-  #     clusterDefinitionCniBuildOS: "windows"
-  #     clusterDefinitionCniBuildExt: ".zip"
+  - template: singletenancy/aks-engine/e2e-job-template.yaml
+    parameters:
+      name: "windows_20_22_e2e"
+      displayName: "Windows 2022"
+      pipelineBuildImage: "$(BUILD_IMAGE)"
+      clusterDefinition: "cniWindows2022.json"
+      clusterDefinitionCniTypeKey: "azureCNIURLWindows"
+      clusterDefinitionCniBuildOS: "windows"
+      clusterDefinitionCniBuildExt: ".zip"
 
-  # - template: singletenancy/aks-engine/e2e-dualstack-job-template.yaml
-  #   parameters:
-  #     name: "ubuntu_18_04_linux_dualstack_e2e"
-  #     displayName: "Ubuntu 18.04 Dualstack"
-  #     pipelineBuildImage: "$(BUILD_IMAGE)"
-  #     clusterDefinition: "cniLinuxDualstack1804.json"
-  #     clusterDefinitionCniTypeKey: "azureCNIURLLinux"
-  #     clusterDefinitionCniBuildOS: "linux"
-  #     clusterDefinitionCniBuildExt: ".tgz"
+  - template: singletenancy/aks-engine/e2e-dualstack-job-template.yaml
+    parameters:
+      name: "ubuntu_18_04_linux_dualstack_e2e"
+      displayName: "Ubuntu 18.04 Dualstack"
+      pipelineBuildImage: "$(BUILD_IMAGE)"
+      clusterDefinition: "cniLinuxDualstack1804.json"
+      clusterDefinitionCniTypeKey: "azureCNIURLLinux"
+      clusterDefinitionCniBuildOS: "linux"
+      clusterDefinitionCniBuildExt: ".tgz"
 
-  # - template: singletenancy/aks-engine/e2e-dualstack-job-template.yaml
-  #   parameters:
-  #     name: "windows_20_04_dualstack_e2e"
-  #     displayName: "Windows 20.04 Dualstack"
-  #     pipelineBuildImage: "$(BUILD_IMAGE)"
-  #     clusterDefinition: "cniWindowsDualstack2004.json"
-  #     clusterDefinitionCniTypeKey: "azureCNIURLWindows"
-  #     clusterDefinitionCniBuildOS: "windows"
-  #     clusterDefinitionCniBuildExt: ".zip"
+  - template: singletenancy/aks-engine/e2e-dualstack-job-template.yaml
+    parameters:
+      name: "windows_20_04_dualstack_e2e"
+      displayName: "Windows 20.04 Dualstack"
+      pipelineBuildImage: "$(BUILD_IMAGE)"
+      clusterDefinition: "cniWindowsDualstack2004.json"
+      clusterDefinitionCniTypeKey: "azureCNIURLWindows"
+      clusterDefinitionCniBuildOS: "windows"
+      clusterDefinitionCniBuildExt: ".zip"
 
   - stage: cleanup
     displayName: Cleanup
     dependsOn:
       - "aks_swift_e2e"
-      # - "ubuntu_18_04_linux_e2e"
-      # - "windows_19_03_e2e"
-      # - "windows_20_04_e2e"
-      # # - "windows_20_22_e2e"
-      # - "ubuntu_18_04_linux_dualstack_e2e"
-      # # - "windows_20_04_dualstack_e2e"
+      - "ubuntu_18_04_linux_e2e"
+      - "windows_19_03_e2e"
+      - "windows_20_04_e2e"
+      # - "windows_20_22_e2e"
+      - "ubuntu_18_04_linux_dualstack_e2e"
+      # - "windows_20_04_dualstack_e2e"
     jobs:
       - job: delete_remote_artifacts
         displayName: Delete remote artifacts

From f13ea20ab8ec65a992280df49c683f2fea4b908f Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Wed, 27 Jul 2022 09:04:38 -0700
Subject: [PATCH 47/51] remove debug

---
 hack/swift/Makefile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/hack/swift/Makefile b/hack/swift/Makefile
index c74455b0a8..f599e3f1b7 100644
--- a/hack/swift/Makefile
+++ b/hack/swift/Makefile
@@ -77,7 +77,6 @@ swift-up: rg-up net-up ## Brings up a swift cluster $name in $SUB/$REGION
 
 byocni-up: rg-up net-up ## Brings up a swift cluster $name in $SUB/$REGION
 	$(AZCLI) aks create -n $(CLUSTER) -g $(GROUP) -l $(REGION) \
-		--debug \
 		--max-pods 250 \
 		--node-count 3 \
 		--network-plugin none \

From 06065a5675ed2a79858e5889733fd5c083da968f Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Wed, 27 Jul 2022 09:09:19 -0700
Subject: [PATCH 48/51] pipeline update

---
 .pipelines/pipeline.yaml | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/.pipelines/pipeline.yaml b/.pipelines/pipeline.yaml
index f7521c307c..444f5a658c 100644
--- a/.pipelines/pipeline.yaml
+++ b/.pipelines/pipeline.yaml
@@ -279,15 +279,15 @@ stages:
       clusterDefinitionCniBuildOS: "windows"
       clusterDefinitionCniBuildExt: ".zip"
 
-  - template: singletenancy/aks-engine/e2e-job-template.yaml
-    parameters:
-      name: "windows_20_22_e2e"
-      displayName: "Windows 2022"
-      pipelineBuildImage: "$(BUILD_IMAGE)"
-      clusterDefinition: "cniWindows2022.json"
-      clusterDefinitionCniTypeKey: "azureCNIURLWindows"
-      clusterDefinitionCniBuildOS: "windows"
-      clusterDefinitionCniBuildExt: ".zip"
+  # - template: singletenancy/aks-engine/e2e-job-template.yaml
+  #   parameters:
+  #     name: "windows_20_22_e2e"
+  #     displayName: "Windows 2022"
+  #     pipelineBuildImage: "$(BUILD_IMAGE)"
+  #     clusterDefinition: "cniWindows2022.json"
+  #     clusterDefinitionCniTypeKey: "azureCNIURLWindows"
+  #     clusterDefinitionCniBuildOS: "windows"
+  #     clusterDefinitionCniBuildExt: ".zip"
 
   - template: singletenancy/aks-engine/e2e-dualstack-job-template.yaml
     parameters:
@@ -299,15 +299,15 @@ stages:
       clusterDefinitionCniBuildOS: "linux"
       clusterDefinitionCniBuildExt: ".tgz"
 
-  - template: singletenancy/aks-engine/e2e-dualstack-job-template.yaml
-    parameters:
-      name: "windows_20_04_dualstack_e2e"
-      displayName: "Windows 20.04 Dualstack"
-      pipelineBuildImage: "$(BUILD_IMAGE)"
-      clusterDefinition: "cniWindowsDualstack2004.json"
-      clusterDefinitionCniTypeKey: "azureCNIURLWindows"
-      clusterDefinitionCniBuildOS: "windows"
-      clusterDefinitionCniBuildExt: ".zip"
+  # - template: singletenancy/aks-engine/e2e-dualstack-job-template.yaml
+  #   parameters:
+  #     name: "windows_20_04_dualstack_e2e"
+  #     displayName: "Windows 20.04 Dualstack"
+  #     pipelineBuildImage: "$(BUILD_IMAGE)"
+  #     clusterDefinition: "cniWindowsDualstack2004.json"
+  #     clusterDefinitionCniTypeKey: "azureCNIURLWindows"
+  #     clusterDefinitionCniBuildOS: "windows"
+  #     clusterDefinitionCniBuildExt: ".zip"
 
   - stage: cleanup
     displayName: Cleanup

From 4be9395a4baa042e1729d2066891727c303aacef Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Wed, 27 Jul 2022 09:54:13 -0700
Subject: [PATCH 49/51] review fixes

---
 .pipelines/singletenancy/aks-swift/e2e-job-template.yaml  | 1 -
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 2 +-
 cns/azure-cns.yaml                                        | 2 +-
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
index 79057bb764..3fbaeb59af 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-job-template.yaml
@@ -6,7 +6,6 @@ parameters:
 stages:
   - stage: ${{ parameters.name }}
     displayName: E2E - ${{ parameters.displayName }}
-    lockBehavior: sequential
     dependsOn: 
     - setup
     - publish
diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 47a2b44ce3..0ff6d7caba 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -26,7 +26,7 @@ steps:
 
   - task: AzureCLI@1
     inputs:
-      azureSubscription: "Azure Container Networking - Test"
+      azureSubscription: $(SUB_AZURE_NETWORK_AGENT_TEST)
       scriptLocation: "inlineScript"
       scriptType: "bash"
       addSpnToEnvironment: true
diff --git a/cns/azure-cns.yaml b/cns/azure-cns.yaml
index dd4c93ec56..9bb98b713e 100644
--- a/cns/azure-cns.yaml
+++ b/cns/azure-cns.yaml
@@ -96,7 +96,7 @@ spec:
           effect: NoSchedule
       containers:
         - name: cns-container
-          image: mcr.microsoft.com/containernetworking/azure-cns:v1.4.29
+          image: mcr.microsoft.com/containernetworking/azure-cns:v1.4.7
           imagePullPolicy: IfNotPresent
           args: [ "-c", "tcp://$(CNSIpAddress):$(CNSPort)", "-t", "$(CNSLogTarget)"]
           volumeMounts:

From 0ff81865d9354d88b58c13149dc471798cce9b94 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Wed, 27 Jul 2022 10:01:56 -0700
Subject: [PATCH 50/51] update svc conn

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 0ff6d7caba..9baf22f439 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -26,7 +26,7 @@ steps:
 
   - task: AzureCLI@1
     inputs:
-      azureSubscription: $(SUB_AZURE_NETWORK_AGENT_TEST)
+      azureSubscription: $(AZURE_TEST_AGENT_SERVICE_CONNECTION)
       scriptLocation: "inlineScript"
       scriptType: "bash"
       addSpnToEnvironment: true

From cf61295df448dc5096205eb3efdb737eec791d68 Mon Sep 17 00:00:00 2001
From: Camryn Lee <camrynlee@microsoft.com>
Date: Wed, 27 Jul 2022 10:30:40 -0700
Subject: [PATCH 51/51] testing

---
 .pipelines/singletenancy/aks-swift/e2e-step-template.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
index 9baf22f439..202b9e8b82 100644
--- a/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
+++ b/.pipelines/singletenancy/aks-swift/e2e-step-template.yaml
@@ -88,6 +88,7 @@ steps:
         echo "Deleting cluster"
         make -C ./hack/swift azcfg AZCLI=az
         make -C ./hack/swift down SUB=$(SUB_AZURE_NETWORK_AGENT_TEST) AZCLI=az CLUSTER=swifte2e-$(make revision)
+        echo "Cluster and resources down"
     name: "Cleanupcluster"
     displayName: "Cleanup cluster"
     condition: always()