From 4d1705f431482fd664d6ee8c75e77b3d4efef40c Mon Sep 17 00:00:00 2001
From: Fuming Zhang <fumingzhang@microsoft.com>
Date: Tue, 22 Jun 2021 17:09:16 +0800
Subject: [PATCH 01/10] update

---
 .../azcli_aks_live_test/{ => configs}/ext_matrix_default.json     | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename src/aks-preview/azcli_aks_live_test/{ => configs}/ext_matrix_default.json (100%)

diff --git a/src/aks-preview/azcli_aks_live_test/ext_matrix_default.json b/src/aks-preview/azcli_aks_live_test/configs/ext_matrix_default.json
similarity index 100%
rename from src/aks-preview/azcli_aks_live_test/ext_matrix_default.json
rename to src/aks-preview/azcli_aks_live_test/configs/ext_matrix_default.json

From 5422ed295102098bff370df41caa2e01d4c8b6e5 Mon Sep 17 00:00:00 2001
From: Fuming Zhang <fumingzhang@microsoft.com>
Date: Tue, 22 Jun 2021 18:54:56 +0800
Subject: [PATCH 02/10] update pipeline

---
 .../{ => scripts}/clean_up.sh                 |   0
 .../{ => scripts}/clone_repo.sh               |  17 +--
 .../{ => scripts}/prepare_image.sh            |   4 -
 .../azcli_aks_live_test/scripts/setup_venv.sh | 107 ++++++++++++++++++
 .../{ => scripts}/start_container.sh          |   3 +-
 .../{ => scripts}/test_cli_live.sh            |   0
 .../{ => scripts}/test_cli_unit.sh            |   0
 .../{ => scripts}/test_ext_live.sh            |   8 +-
 .../{ => scripts}/test_ext_unit.sh            |   0
 .../{ => scripts}/transcribe_env.sh           |   0
 .../azcli_aks_live_test/setup_venv.sh         |  46 --------
 .../vsts-azcli-aks-live-test.yaml             |  25 ++--
 .../vsts-azcli-aks-unit-test.yaml             |  21 +++-
 13 files changed, 150 insertions(+), 81 deletions(-)
 rename src/aks-preview/azcli_aks_live_test/{ => scripts}/clean_up.sh (100%)
 rename src/aks-preview/azcli_aks_live_test/{ => scripts}/clone_repo.sh (66%)
 rename src/aks-preview/azcli_aks_live_test/{ => scripts}/prepare_image.sh (96%)
 create mode 100755 src/aks-preview/azcli_aks_live_test/scripts/setup_venv.sh
 rename src/aks-preview/azcli_aks_live_test/{ => scripts}/start_container.sh (89%)
 rename src/aks-preview/azcli_aks_live_test/{ => scripts}/test_cli_live.sh (100%)
 rename src/aks-preview/azcli_aks_live_test/{ => scripts}/test_cli_unit.sh (100%)
 rename src/aks-preview/azcli_aks_live_test/{ => scripts}/test_ext_live.sh (90%)
 rename src/aks-preview/azcli_aks_live_test/{ => scripts}/test_ext_unit.sh (100%)
 rename src/aks-preview/azcli_aks_live_test/{ => scripts}/transcribe_env.sh (100%)
 delete mode 100755 src/aks-preview/azcli_aks_live_test/setup_venv.sh

diff --git a/src/aks-preview/azcli_aks_live_test/clean_up.sh b/src/aks-preview/azcli_aks_live_test/scripts/clean_up.sh
similarity index 100%
rename from src/aks-preview/azcli_aks_live_test/clean_up.sh
rename to src/aks-preview/azcli_aks_live_test/scripts/clean_up.sh
diff --git a/src/aks-preview/azcli_aks_live_test/clone_repo.sh b/src/aks-preview/azcli_aks_live_test/scripts/clone_repo.sh
similarity index 66%
rename from src/aks-preview/azcli_aks_live_test/clone_repo.sh
rename to src/aks-preview/azcli_aks_live_test/scripts/clone_repo.sh
index 8eca3be875b..e932487bf7e 100755
--- a/src/aks-preview/azcli_aks_live_test/clone_repo.sh
+++ b/src/aks-preview/azcli_aks_live_test/scripts/clone_repo.sh
@@ -11,14 +11,10 @@ set -o xtrace
 [[ -z "${CLI_BRANCH}" ]] && (echo "CLI_BRANCH is empty"; exit 1)
 [[ -z "${EXT_REPO}" ]] && (echo "EXT_REPO is empty"; exit 1)
 [[ -z "${EXT_BRANCH}" ]] && (echo "EXT_BRANCH is empty"; exit 1)
-[[ -z "${MANUAL_EXT}" ]] && (echo "MANUAL_EXT is empty"; exit 1)
-
-# dir
-pwd
-ls -alh
+[[ -z "${BUILD_REASON}" ]] && (echo "BUILD_REASON is empty"; exit 1)
+[[ -z "${LIVE_TEST_BASE_DIR}" ]] && (echo "LIVE_TEST_BASE_DIR is empty"; exit 1)
 
 # clone azure-cli (default is the official repo)
-# git clone https://github.com/Azure/azure-cli.git
 git clone ${CLI_REPO}
 
 # ckeckout to a specific azure-cli branch (default is the dev branch)
@@ -27,9 +23,9 @@ git branch -a
 git checkout ${CLI_BRANCH}
 popd
 
-# clone azure-cli-extensions when manually specify the extension repo
-if [[ ${MANUAL_EXT} == true ]]; then
-    echo "Manually specify the extension repo, delete the current 'azure-cli-extensions' directory!"
+# clone azure-cli-extensions when manually trigger the pipeline
+if [[ ${BUILD_REASON} == "Manual" ]]; then
+    echo "Manually trigger the pipeline, delete the current 'azure-cli-extensions' directory!"
     rm -rf azure-cli-extensions/
     git clone ${EXT_REPO}
     pushd azure-cli-extensions/
@@ -45,6 +41,5 @@ git log -10
 popd
 
 # copy live test related files to the same level as the checkout directory ($(Agent.BuildDirectory)/s)
-cp -rT azure-cli-extensions/src/aks-preview/azcli_aks_live_test/ ./
-cp -r azure-cli-extensions/src/aks-preview/az_aks_tool/ ./
+cp -rT ${LIVE_TEST_BASE_DIR} ./
 ls -alh
diff --git a/src/aks-preview/azcli_aks_live_test/prepare_image.sh b/src/aks-preview/azcli_aks_live_test/scripts/prepare_image.sh
similarity index 96%
rename from src/aks-preview/azcli_aks_live_test/prepare_image.sh
rename to src/aks-preview/azcli_aks_live_test/scripts/prepare_image.sh
index 1f603ea5759..6843d74722f 100755
--- a/src/aks-preview/azcli_aks_live_test/prepare_image.sh
+++ b/src/aks-preview/azcli_aks_live_test/scripts/prepare_image.sh
@@ -11,10 +11,6 @@ set -o xtrace
 [[ -z "${IMAGE_NAME}" ]] && (echo "IMAGE_NAME is empty"; exit 1)
 [[ -z "${IMAGE_TAG}" ]] && (echo "IMAGE_TAG is empty"; exit 1)
 
-# dir
-pwd
-ls -alh
-
 # prepare docker image
 echo "Pulling test image from '${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}'..."
 docker pull ${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}
diff --git a/src/aks-preview/azcli_aks_live_test/scripts/setup_venv.sh b/src/aks-preview/azcli_aks_live_test/scripts/setup_venv.sh
new file mode 100755
index 00000000000..fa7fceee81a
--- /dev/null
+++ b/src/aks-preview/azcli_aks_live_test/scripts/setup_venv.sh
@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+
+# check var
+[[ -z "${PYTHON_VERSION}" ]] && (echo "PYTHON_VERSION is empty"; exit 1)
+
+setupVenv(){
+    # delete existing venv
+    deactivate || true
+    rm -rf azEnv || true
+
+    # create new venv
+    python${PYTHON_VERSION} -m venv azEnv
+    source azEnv/bin/activate
+    python -m pip install -U pip
+}
+
+installBuildTools(){
+    pip install tox
+    pip install -U build
+}
+
+setupAZ(){
+    cli_repo=${1:-"."}
+    ext_repo=${2:-""}
+
+    # install azdev, used later to install azcli and extension
+    pip install azdev==0.1.32
+
+    # pre-install-az: check existing az
+    which az || az version || az extension list || true
+
+    # install-az: from cloned repos with azdev
+    if [[ -z ${ext_repo} ]]; then
+        azdev setup -c ${cli_repo}
+    else
+        azdev setup -c ${cli_repo} -r ${ext_repo}
+    fi
+
+    # post-install-az: check installation result
+    which az && az version
+}
+
+installTestPackages(){
+    # install pytest plugins
+    pip install pytest-json-report pytest-rerunfailures --upgrade
+
+    # install coverage for measuring code coverage
+    pip install coverage
+}
+
+installAZAKSTOOLFromLocal(){
+    wheel_file=${1}
+    pip install ${wheel_file}
+}
+
+installAZAKSTOOL(){
+    wheel_file="az_aks_tool-latest-py3-none-any.whl"
+    wheel_url="https://akspreview.blob.core.windows.net/azakstool/${wheel_file}"
+    curl -sLO ${wheel_url}
+    installAZAKSTOOLFromLocal ${wheel_file}
+}
+
+removeKustoPTHFile(){
+    pushd azEnv/lib/python${PYTHON_VERSION}/site-packages
+    rm azure_kusto_data*nspkg.pth
+    rm azure_kusto_ingest*nspkg.pth
+    popd
+}
+
+if [[ -n ${1} ]]; then
+    # bash options
+    set -o errexit
+    set -o nounset
+    set -o pipefail
+    set -o xtrace
+
+    # create new venv if second arg is not "n"
+    new_venv=${2:-"y"}
+    if [[ ! ${new_venv} == "n" ]]; then
+        echo "Create new venv!"
+        setupVenv
+    fi
+
+    if [[ ${1} == "build" ]]; then
+        echo "Start to build az-aks-tool!"
+        installBuildTools
+    elif [[ ${1} == "setuptool" ]]; then
+        echo "Start to setup az-aks-tool!"
+        local_setup=${3:-"n"}
+        if [[ ${local_setup} == "y" ]]; then
+            wheel_file=${4}
+            installAZAKSTOOLFromLocal ${wheel_file}
+        else
+            installAZAKSTOOL
+        fi
+        removeKustoPTHFile
+    elif [[ ${1} == "setupaz" ]]; then
+        echo "Start to setup azure-cli!"
+        cli_repo=${3:-"azure-cli/"}
+        ext_repo=${4:-""}
+        setupAZ ${cli_repo} ${ext_repo}
+        installTestPackages
+    else
+        echo "Unknown arg '${1}'!"
+    fi
+    echo "All Done!"
+fi
diff --git a/src/aks-preview/azcli_aks_live_test/start_container.sh b/src/aks-preview/azcli_aks_live_test/scripts/start_container.sh
similarity index 89%
rename from src/aks-preview/azcli_aks_live_test/start_container.sh
rename to src/aks-preview/azcli_aks_live_test/scripts/start_container.sh
index 0edadb91c9d..ed4818d6edc 100755
--- a/src/aks-preview/azcli_aks_live_test/start_container.sh
+++ b/src/aks-preview/azcli_aks_live_test/scripts/start_container.sh
@@ -10,7 +10,6 @@ set -o xtrace
 # take the first arg as container name
 container_name=${1:-"azcli-aks-live-test-container"}
 [[ -z "${MAPPED_AZCLI_ALT_CLIENT_SECRET}" ]] && (echo "MAPPED_AZCLI_ALT_CLIENT_SECRET is empty"; exit 1)
-[[ -z "${MAPPED_AZCLI_ALT_CLIENT_SECRET}" ]] && (echo "MAPPED_AZCLI_ALT_CLIENT_SECRET is empty"; exit 1)
 [[ -z "${IMAGE_NAME}" ]] && (echo "IMAGE_NAME is empty"; exit 1)
 [[ -z "${IMAGE_TAG}" ]] && (echo "IMAGE_TAG is empty"; exit 1)
 
@@ -19,7 +18,7 @@ pwd
 ls -alh
 
 # transcribe environment variables into file 'env.list'
-./transcribe_env.sh
+./scripts/transcribe_env.sh
 
 # start container in backgroud with tty
 # mount current directory ($(Agent.BuildDirectory)/s) to /opt in container
diff --git a/src/aks-preview/azcli_aks_live_test/test_cli_live.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_cli_live.sh
similarity index 100%
rename from src/aks-preview/azcli_aks_live_test/test_cli_live.sh
rename to src/aks-preview/azcli_aks_live_test/scripts/test_cli_live.sh
diff --git a/src/aks-preview/azcli_aks_live_test/test_cli_unit.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_cli_unit.sh
similarity index 100%
rename from src/aks-preview/azcli_aks_live_test/test_cli_unit.sh
rename to src/aks-preview/azcli_aks_live_test/scripts/test_cli_unit.sh
diff --git a/src/aks-preview/azcli_aks_live_test/test_ext_live.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh
similarity index 90%
rename from src/aks-preview/azcli_aks_live_test/test_ext_live.sh
rename to src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh
index 0f7129a3363..62f0f5ff0d9 100755
--- a/src/aks-preview/azcli_aks_live_test/test_ext_live.sh
+++ b/src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh
@@ -45,7 +45,7 @@ source azEnv/bin/activate
 az aks fake-command --debug || true
 
 # prepare run flags
-run_flags="-e -em ext_matrix_default.json --no-exitfirst --report-path ./reports --reruns 3 --capture=sys"
+run_flags="-e --no-exitfirst --report-path ./reports --reruns 3 --capture=sys"
 # parallel
 if [ ${PARALLELISM} -ge 2 ]; then
     run_flags+=" -j ${PARALLELISM}"
@@ -58,7 +58,7 @@ if [[ -n ${TEST_CASES} ]]; then
 fi
 # ext matrix
 if [[ -n ${EXT_TEST_MATRIX} ]]; then
-    run_flags+=" -em ${EXT_TEST_MATRIX}"
+    run_flags+=" -em ./configs/${EXT_TEST_MATRIX}"
 fi
 # ext extra filter
 if [[ -n ${EXT_TEST_FILTER} ]]; then
@@ -75,7 +75,7 @@ if [[ ${TEST_MODE} == "record" || ${TEST_MODE} == "all" ]]; then
     run_flags+=" --json-report-file=ext_report.json"
     run_flags+=" --xml-file=ext_result.xml"
     echo "run flags: ${run_flags}"
-    echo "${run_flags}" | xargs python -u az_aks_tool/main.py
+    azaks ${run_flags}
 fi
 
 # live test
@@ -87,5 +87,5 @@ if [[ ${TEST_MODE} == "live" || ${TEST_MODE} == "all" ]]; then
     run_flags+=" -l --json-report-file=ext_live_report.json"
     run_flags+=" --xml-file=ext_live_result.xml"
     echo "run flags: ${run_flags}"
-    echo "${run_flags}" | xargs python -u az_aks_tool/main.py 
+    azaks ${run_flags}
 fi
diff --git a/src/aks-preview/azcli_aks_live_test/test_ext_unit.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_ext_unit.sh
similarity index 100%
rename from src/aks-preview/azcli_aks_live_test/test_ext_unit.sh
rename to src/aks-preview/azcli_aks_live_test/scripts/test_ext_unit.sh
diff --git a/src/aks-preview/azcli_aks_live_test/transcribe_env.sh b/src/aks-preview/azcli_aks_live_test/scripts/transcribe_env.sh
similarity index 100%
rename from src/aks-preview/azcli_aks_live_test/transcribe_env.sh
rename to src/aks-preview/azcli_aks_live_test/scripts/transcribe_env.sh
diff --git a/src/aks-preview/azcli_aks_live_test/setup_venv.sh b/src/aks-preview/azcli_aks_live_test/setup_venv.sh
deleted file mode 100755
index e63b00264f8..00000000000
--- a/src/aks-preview/azcli_aks_live_test/setup_venv.sh
+++ /dev/null
@@ -1,46 +0,0 @@
-#!/usr/bin/env bash
-
-# bash options
-set -o errexit
-set -o nounset
-set -o pipefail
-set -o xtrace
-
-# check var
-PYTHON_VERSION=${PYTHON_VERSION:-"3.8"}
-
-# dir
-pwd
-ls -alh
-
-# delete existing venv
-rm -rf azEnv || true
-
-# install python packages
-python${PYTHON_VERSION} -m venv azEnv
-source azEnv/bin/activate
-python -m pip install -U pip
-# install azdev, used later to install azcli and extension
-pip install azdev==0.1.32
-# install pytest plugins
-pip install pytest-json-report pytest-rerunfailures --upgrade
-# pip install pytest-html --upgrade
-# module for measuring code coverage
-pip install coverage
-
-# pre-install: check existing az 
-which az || az version || az extension list || true
-
-# install az from cloned repos with azdev
-azdev setup -c azure-cli/ -r azure-cli-extensions/
-deactivate
-
-# post-install: reactivate venv to check installation result
-source azEnv/bin/activate
-which az && az version
-
-# mkdir to store reports
-mkdir -p reports/
-
-# export PYTHONPATH
-export PYTHONPATH=$(pwd)
diff --git a/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml
index cf082cd587a..6b90a7a1526 100644
--- a/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml
+++ b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml
@@ -10,6 +10,13 @@ pr:
     include:
     - src/aks-preview/
 
+variables:
+  - group: azcli-aks-tool
+  - name: ContainerName
+    value: "azcli-aks-live-test"
+  - name: LIVE_TEST_BASE_DIR
+    value: "azure-cli-extensions/src/aks-preview/azcli_aks_live_test"
+
 jobs:
 - job: LiveTest
   pool:
@@ -27,15 +34,15 @@ jobs:
         ls -alh
       displayName: "Move All Checkout Files to the Newly Created 'azure-cli-extensions' Directory"
     - bash: |
-        ./azure-cli-extensions/src/aks-preview/azcli_aks_live_test/clone_repo.sh
+        $(LIVE_TEST_BASE_DIR)/clone_repo.sh
       condition: succeeded()
-      displayName: "Clone GitHub Repo and Move Live Test Related Files"
+      displayName: "Clone GitHub Repo and Move Test Related Files"
     - bash: |
-        ./prepare_image.sh
+        ./scripts/prepare_image.sh
       condition: succeeded()
-      displayName: "Prepare Live Test Image"
+      displayName: "Prepare Test Image"
     - bash: |
-        ./start_container.sh
+        ./scripts/start_container.sh $(ContainerName)
       env:
         MAPPED_AZCLI_ALT_CLIENT_SECRET: $(AZCLI_ALT_CLIENT_SECRET)
         BUILD_REASON: $(Build.Reason)
@@ -43,20 +50,22 @@ jobs:
       condition: succeeded()
       displayName: "Start Container"
     - bash: |
-        docker exec "azcli-aks-live-test-container" /opt/setup_venv.sh
+        docker exec $(ContainerName) /opt/scripts/setup_venv.sh setuptool y
+        docker exec $(ContainerName) /opt/scripts/setup_venv.sh setupaz n azure-cli/ azure-cli-extensions/
       condition: succeeded()
       displayName: "Set up Virtual Environment"
     - bash: |
-        docker exec "azcli-aks-live-test-container" /opt/test_cli_live.sh
+        docker exec $(ContainerName) /opt/scripts/test_cli_live.sh
       condition: and(succeeded(), in(variables['COVERAGE'], 'cli', 'all'))
       displayName: Perform Live Test for CLI
     - bash: |
-        docker exec "azcli-aks-live-test-container" /opt/test_ext_live.sh
+        docker exec $(ContainerName) /opt/scripts/test_ext_live.sh
       condition: and(succeededOrFailed(), in(variables['COVERAGE'], 'ext', 'all'))
       displayName: Perform Live Test for EXT
     - task: CopyFiles@2
       inputs:
         contents: 'reports/**'
+        flattenFolders: true
         targetFolder: $(Build.ArtifactStagingDirectory)
       condition: succeededOrFailed()
     - task: PublishBuildArtifacts@1
diff --git a/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-unit-test.yaml b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-unit-test.yaml
index 5cd8c268ce8..66a0fb8d953 100644
--- a/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-unit-test.yaml
+++ b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-unit-test.yaml
@@ -10,6 +10,13 @@ pr:
     include:
     - src/aks-preview/
 
+variables:
+  - group: azcli-aks-tool
+  - name: ContainerName
+    value: "azcli-aks-unit-test"
+  - name: LIVE_TEST_BASE_DIR
+    value: "azure-cli-extensions/src/aks-preview/azcli_aks_live_test"
+
 jobs:
 - job: UnitTest
   pool:
@@ -27,15 +34,15 @@ jobs:
         ls -alh
       displayName: "Move All Checkout Files to the Newly Created 'azure-cli-extensions' Directory"
     - bash: |
-        ./azure-cli-extensions/src/aks-preview/azcli_aks_live_test/clone_repo.sh
+        $(LIVE_TEST_BASE_DIR)/clone_repo.sh
       condition: succeeded()
       displayName: "Clone GitHub Repo and Move Test Related Files"
     - bash: |
-        ./prepare_image.sh
+        ./scripts/prepare_image.sh
       condition: succeeded()
       displayName: "Prepare Test Image"
     - bash: |
-        ./start_container.sh "azcli-aks-unit-test-container"
+        ./scripts/start_container.sh $(ContainerName)
       env:
         MAPPED_AZCLI_ALT_CLIENT_SECRET: $(AZCLI_ALT_CLIENT_SECRET)
         BUILD_REASON: $(Build.Reason)
@@ -43,20 +50,22 @@ jobs:
       condition: succeeded()
       displayName: "Start Container"
     - bash: |
-        docker exec "azcli-aks-unit-test-container" /opt/setup_venv.sh
+        docker exec $(ContainerName) /opt/scripts/setup_venv.sh setuptool y
+        docker exec $(ContainerName) /opt/scripts/setup_venv.sh setupaz n azure-cli/ azure-cli-extensions/
       condition: succeeded()
       displayName: "Set up Virtual Environment"
     - bash: |
-        docker exec "azcli-aks-unit-test-container" /opt/test_cli_unit.sh
+        docker exec $(ContainerName) /opt/scripts/test_cli_unit.sh
       condition: and(succeeded(), in(variables['COVERAGE'], 'cli', 'all'))
       displayName: Perform Unit Test for CLI
     - bash: |
-        docker exec "azcli-aks-unit-test-container" /opt/test_ext_unit.sh
+        docker exec $(ContainerName) /opt/scripts/test_ext_unit.sh
       condition: and(succeededOrFailed(), in(variables['COVERAGE'], 'ext', 'all'))
       displayName: Perform Unit Test for EXT
     - task: CopyFiles@2
       inputs:
         contents: 'reports/**'
+        flattenFolders: true
         targetFolder: $(Build.ArtifactStagingDirectory)
       condition: succeededOrFailed()
     - task: PublishBuildArtifacts@1

From c2bb3e31aec1886fd539b822608a6aed97176375 Mon Sep 17 00:00:00 2001
From: Fuming Zhang <fumingzhang@microsoft.com>
Date: Wed, 23 Jun 2021 15:01:23 +0800
Subject: [PATCH 03/10] update pipeline

---
 .../azcli_aks_live_test/scripts/transcribe_env.sh         | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/aks-preview/azcli_aks_live_test/scripts/transcribe_env.sh b/src/aks-preview/azcli_aks_live_test/scripts/transcribe_env.sh
index ea56ba31f61..8e18c380cfd 100755
--- a/src/aks-preview/azcli_aks_live_test/scripts/transcribe_env.sh
+++ b/src/aks-preview/azcli_aks_live_test/scripts/transcribe_env.sh
@@ -26,6 +26,9 @@ set -o xtrace
 [[ -z "${TEST_MODE}" ]] && (echo "TEST_MODE is empty"; exit 1)
 [[ -z "${PARALLELISM}" ]] && (echo "PARALLELISM is empty"; exit 1)
 [[ -z "${TEST_CASES}" ]] && (echo "TEST_CASES is empty")
+[[ -z "${CLI_TEST_MATRIX}" ]] && (echo "CLI_TEST_MATRIX is empty")
+[[ -z "${CLI_TEST_FILTER}" ]] && (echo "CLI_TEST_FILTER is empty")
+[[ -z "${CLI_TEST_COVERAGE}" ]] && (echo "CLI_TEST_COVERAGE is empty")
 [[ -z "${EXT_TEST_MATRIX}" ]] && (echo "EXT_TEST_MATRIX is empty")
 [[ -z "${EXT_TEST_FILTER}" ]] && (echo "EXT_TEST_FILTER is empty")
 [[ -z "${EXT_TEST_COVERAGE}" ]] && (echo "EXT_TEST_COVERAGE is empty")
@@ -33,7 +36,6 @@ set -o xtrace
 [[ -z "${CLI_BRANCH}" ]] && (echo "CLI_BRANCH is empty"; exit 1)
 [[ -z "${EXT_REPO}" ]] && (echo "EXT_REPO is empty"; exit 1)
 [[ -z "${EXT_BRANCH}" ]] && (echo "EXT_BRANCH is empty"; exit 1)
-[[ -z "${MANUAL_EXT}" ]] && (echo "MANUAL_EXT is empty"; exit 1)
 
 # clear
 cat /dev/null > env.list
@@ -64,6 +66,9 @@ echo "COVERAGE=${COVERAGE}" >> env.list
 echo "TEST_MODE=${TEST_MODE}" >> env.list
 echo "PARALLELISM=${PARALLELISM}" >> env.list
 echo "TEST_CASES=${TEST_CASES}" >> env.list
+echo "CLI_TEST_MATRIX=${CLI_TEST_MATRIX}" >> env.list
+echo "CLI_TEST_FILTER=${CLI_TEST_FILTER}" >> env.list
+echo "CLI_TEST_COVERAGE=${CLI_TEST_COVERAGE}" >> env.list
 echo "EXT_TEST_MATRIX=${EXT_TEST_MATRIX}" >> env.list
 echo "EXT_TEST_FILTER=${EXT_TEST_FILTER}" >> env.list
 echo "EXT_TEST_COVERAGE=${EXT_TEST_COVERAGE}" >> env.list
@@ -73,4 +78,3 @@ echo "CLI_REPO=${CLI_REPO}" >> env.list
 echo "CLI_BRANCH=${CLI_BRANCH}" >> env.list
 echo "EXT_REPO=${EXT_REPO}" >> env.list
 echo "EXT_BRANCH=${EXT_BRANCH}" >> env.list
-echo "MANUAL_EXT=${MANUAL_EXT}" >> env.list

From 2f553fc8eba83d7509f899fb66be0d5d2f1d9b26 Mon Sep 17 00:00:00 2001
From: Fuming Zhang <fumingzhang@microsoft.com>
Date: Thu, 24 Jun 2021 10:59:27 +0800
Subject: [PATCH 04/10] update pipeline

---
 .../azcli_aks_live_test/HISTORY.md            |  9 +++
 .../azcli_aks_live_test/HISTORY.rst           |  9 ---
 src/aks-preview/azcli_aks_live_test/README.md | 10 +++-
 .../azcli_aks_live_test/scripts/setup_venv.sh | 47 ++++++++++++++-
 .../scripts/test_ext_live.sh                  | 24 +-------
 .../scripts/test_ext_unit.sh                  | 58 +++++--------------
 .../vsts-azcli-aks-live-test.yaml             |  6 +-
 .../vsts-azcli-aks-unit-test.yaml             |  6 +-
 8 files changed, 82 insertions(+), 87 deletions(-)
 create mode 100644 src/aks-preview/azcli_aks_live_test/HISTORY.md
 delete mode 100644 src/aks-preview/azcli_aks_live_test/HISTORY.rst

diff --git a/src/aks-preview/azcli_aks_live_test/HISTORY.md b/src/aks-preview/azcli_aks_live_test/HISTORY.md
new file mode 100644
index 00000000000..2bb7ac8874d
--- /dev/null
+++ b/src/aks-preview/azcli_aks_live_test/HISTORY.md
@@ -0,0 +1,9 @@
+# History
+
+## 0.1.0 (4/23/2021)
+
+* Add live test pipeline for aks commands (in aks-preview)
+
+## 0.2.0 (6/24/2021)
+
+* Remove the direct (source code) dependency on az-aks-tool, install and use the pre-compiled wheel file instead
diff --git a/src/aks-preview/azcli_aks_live_test/HISTORY.rst b/src/aks-preview/azcli_aks_live_test/HISTORY.rst
deleted file mode 100644
index 93372ba3c64..00000000000
--- a/src/aks-preview/azcli_aks_live_test/HISTORY.rst
+++ /dev/null
@@ -1,9 +0,0 @@
-.. :changelog:
-
-Release History
-===============
-
-0.1.0 (4/23/2021)
-++++++
-
-* Add live test pipeline for aks commands
diff --git a/src/aks-preview/azcli_aks_live_test/README.md b/src/aks-preview/azcli_aks_live_test/README.md
index 206a5eae203..9a372abb586 100644
--- a/src/aks-preview/azcli_aks_live_test/README.md
+++ b/src/aks-preview/azcli_aks_live_test/README.md
@@ -1,9 +1,13 @@
 # Azure CLI AKS Live Test Pipeline & Azure CLI AKS Unit Test Pipeline
 
-These pipelines are used to test newly added aks commands in module aks-preview (azure-cli-extensions) / acs (azure-cli, not covered by default).
+These pipelines are used to test newly added aks commands in module aks-preview (azure-cli-extensions) / acs (azure-cli, not covered by default). For more details, you may refer to this [wiki](https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/CLI-AKS-Live-Unit-Test-Pipeline).
 
 ## How to use
 
-**By default**, these pipelines will be **triggered** when submitting a **PR** to the master branch of the official repo which involves modifying the files under src/aks-preview. Then they will test the aks-preview command group in azure-cli-extensions. For live test pipeline, the test will be performed in record mode first, and then in live mode. Due to some specific [reasons](https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/157433/Live-Test-Failures-in-aks-preview-(with-bare-sub)), some test cases would fail. These test cases have been filtered out by file 'ext_matrix_default.json'. For unit test pipeline, the test will be perfomed with 'unittest' and 'pytest' modules. A code coverage report will be generated after the unit tests. You can find test reports and coverage report from pipeline artifacts.
+These pipelines (live and unit test pipelines) will be **triggered** when submitting a **PR** to the master branch of the official repo which involves modifying the files under *src/aks-preview*.
 
-You can also trigger this pipeline **manually**. For more details, you may refer to this [wiki](https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/Azure-CLI-AKS-Live-Test-Pipeline).
+By default, for **live test pipeline**, the test will be performed in **record mode first**, and **then in live mode**. The test mainly uses test cases located in ```src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py```. Due to some specific reasons (more details in another [wiki](https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/157433/Live-Test-Failures-in-aks-preview-(with-bare-sub))), some test cases would fail. These test cases have been filtered out by file 'ext_matrix_default.json' (more details in [section Filter](#Filter)). For **unit test pipeline**, the test will be perfomed with 'unittest' and 'pytest' modules. A code coverage report will be generated after the unit tests. For both of the test pipelines, you can find test reports and coverage report from pipeline artifacts. 
+
+If the newly added commands and test cases use the **features** that are being previewed, that is, some feature under container service needs to be manually registered before using the command, then such cases will not be able to execute/pass the test temporarily, since the subscription used for testing does not (and does not intend to) enable these additional features. In the future, we will use customer header to pass these features in test cases, but for now you can just bypass these cases. For now,  you can follow the instructions in [section Bypass Test Case](#bypass-test-case) to **bypass such test cases**. 
+
+You can also trigger this pipeline **manually** and adjust variables such as test coverage, test filter, test location, etc. as needed. For more details, you may refer to the following sections.
\ No newline at end of file
diff --git a/src/aks-preview/azcli_aks_live_test/scripts/setup_venv.sh b/src/aks-preview/azcli_aks_live_test/scripts/setup_venv.sh
index fa7fceee81a..f4e61c1c9ff 100755
--- a/src/aks-preview/azcli_aks_live_test/scripts/setup_venv.sh
+++ b/src/aks-preview/azcli_aks_live_test/scripts/setup_venv.sh
@@ -14,11 +14,13 @@ setupVenv(){
     python -m pip install -U pip
 }
 
+# need to be executed in a venv
 installBuildTools(){
-    pip install tox
+    pip install tox coverage
     pip install -U build
 }
 
+# need to be executed in a venv
 setupAZ(){
     cli_repo=${1:-"."}
     ext_repo=${2:-""}
@@ -40,6 +42,7 @@ setupAZ(){
     which az && az version
 }
 
+# need to be executed in a venv
 installTestPackages(){
     # install pytest plugins
     pip install pytest-json-report pytest-rerunfailures --upgrade
@@ -48,11 +51,13 @@ installTestPackages(){
     pip install coverage
 }
 
+# need to be executed in a venv
 installAZAKSTOOLFromLocal(){
     wheel_file=${1}
     pip install ${wheel_file}
 }
 
+# need to be executed in a venv
 installAZAKSTOOL(){
     wheel_file="az_aks_tool-latest-py3-none-any.whl"
     wheel_url="https://akspreview.blob.core.windows.net/azakstool/${wheel_file}"
@@ -60,6 +65,7 @@ installAZAKSTOOL(){
     installAZAKSTOOLFromLocal ${wheel_file}
 }
 
+# need to be executed in a venv with kusto related modules installed
 removeKustoPTHFile(){
     pushd azEnv/lib/python${PYTHON_VERSION}/site-packages
     rm azure_kusto_data*nspkg.pth
@@ -67,6 +73,37 @@ removeKustoPTHFile(){
     popd
 }
 
+# need to be executed in a venv after 'setupAZ'
+igniteAKSPreview(){
+    # use a fake command to force trigger the command index update of azure-cli, in order to load aks-preview commands
+    # otherwise, cold boot execution of azdev test / pytest would only use commands in the acs module
+    az aks fake-command --debug || true
+}
+
+# need to be executed in a venv
+removeAKSPreview(){
+    # remove extension
+    echo "Remove existing aks-preview extension (if any)"
+    if az extension remove --name aks-preview || azdev extension remove aks-preview; then
+        deactivate
+        source azEnv/bin/activate
+    fi
+}
+
+# need to be executed in a venv after 'setupAZ'
+setupAKSPreview(){
+    # remove extension
+    removeAKSPreview
+
+    # install latest extension
+    echo "Install the latest aks-preview extension and re-activate the virtualenv"
+    azdev extension add aks-preview
+    az extension list
+    azdev extension list | grep "aks-preview" -C 5
+    deactivate
+    source azEnv/bin/activate
+}
+
 if [[ -n ${1} ]]; then
     # bash options
     set -o errexit
@@ -84,7 +121,7 @@ if [[ -n ${1} ]]; then
     if [[ ${1} == "build" ]]; then
         echo "Start to build az-aks-tool!"
         installBuildTools
-    elif [[ ${1} == "setuptool" ]]; then
+    elif [[ ${1} == "setup-tool" ]]; then
         echo "Start to setup az-aks-tool!"
         local_setup=${3:-"n"}
         if [[ ${local_setup} == "y" ]]; then
@@ -94,12 +131,16 @@ if [[ -n ${1} ]]; then
             installAZAKSTOOL
         fi
         removeKustoPTHFile
-    elif [[ ${1} == "setupaz" ]]; then
+    elif [[ ${1} == "setup-az" ]]; then
         echo "Start to setup azure-cli!"
         cli_repo=${3:-"azure-cli/"}
         ext_repo=${4:-""}
         setupAZ ${cli_repo} ${ext_repo}
         installTestPackages
+    elif [[ ${1} == "setup-akspreview" ]]; then
+        echo "Start to setup aks-preview!"
+        setupAKSPreview
+        igniteAKSPreview
     else
         echo "Unknown arg '${1}'!"
     fi
diff --git a/src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh
index 62f0f5ff0d9..16f07b4977e 100755
--- a/src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh
+++ b/src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh
@@ -18,31 +18,11 @@ set -o xtrace
 [[ -z "${EXT_TEST_FILTER}" ]] && (echo "EXT_TEST_FILTER is empty")
 [[ -z "${EXT_TEST_COVERAGE}" ]] && (echo "EXT_TEST_COVERAGE is empty")
 
-# dir
-pwd
-ls -alh
-
 # activate virtualenv
 source azEnv/bin/activate
 
-# remove extension
-echo "Remove existing aks-preview extension (if any)"
-if az extension remove --name aks-preview || azdev extension remove aks-preview; then
-    deactivate
-    source azEnv/bin/activate
-fi
-
-# install latest extension
-echo "Install the latest aks-preview extension and re-activate the virtualenv"
-azdev extension add aks-preview
-az extension list
-azdev extension list | grep "aks-preview" -C 5
-deactivate
-source azEnv/bin/activate
-
-# use a fake command to force trigger the command index update of azure-cli, in order to load aks-preview commands
-# otherwise, cold boot execution of azdev test / pytest would only use commands in the acs module
-az aks fake-command --debug || true
+# setup aks-preview
+./scripts/setup_venv.sh setup-akspreview
 
 # prepare run flags
 run_flags="-e --no-exitfirst --report-path ./reports --reruns 3 --capture=sys"
diff --git a/src/aks-preview/azcli_aks_live_test/scripts/test_ext_unit.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_ext_unit.sh
index db13e91fe5e..e1daa0236fa 100755
--- a/src/aks-preview/azcli_aks_live_test/scripts/test_ext_unit.sh
+++ b/src/aks-preview/azcli_aks_live_test/scripts/test_ext_unit.sh
@@ -6,63 +6,33 @@ set -o nounset
 set -o pipefail
 set -o xtrace
 
-# dir
-pwd
-ls -alh
+# const
+aks_preview_base_dir="azure-cli-extensions/src/aks-preview/azext_aks_preview"
 
 # activate virtualenv
 source azEnv/bin/activate
 
-# remove extension
-echo "Remove existing aks-preview extension (if any)"
-if az extension remove --name aks-preview || azdev extension remove aks-preview; then
-    deactivate
-    source azEnv/bin/activate
-fi
-
-# install latest extension
-echo "Install the latest aks-preview extension and re-activate the virtualenv"
-azdev extension add aks-preview
-az extension list
-azdev extension list | grep "aks-preview" -C 5
-deactivate
-source azEnv/bin/activate
-
-# use a fake command to force trigger the command index update of azure-cli, in order to load aks-preview commands
-# otherwise, cold boot execution of azdev test / pytest would only use commands in the acs module
-az aks fake-command --debug || true
+# setup aks-preview
+./scripts/setup_venv.sh setup-akspreview
 
 # unit test & coverage report
-# az_aks_tool
-az_aks_tool_unit_test_result=""
-pushd azure-cli-extensions/src/aks-preview/az_aks_tool/
-# clean existing coverage report
-(coverage combine || true) && (coverage erase || true)
-if ! coverage run --source=. --omit=*/tests/* -p -m unittest discover; then
-    az_aks_tool_unit_test_result="error"
-fi
-# currently no test written in pytest format under 'az_aks_tool/'
-# coverage run --source=. --omit=*/tests/* -p -m pytest
-coverage combine && coverage json -o coverage_az_aks_tool.json
-coverage report -m
-popd
-cp azure-cli-extensions/src/aks-preview/az_aks_tool/coverage_az_aks_tool.json reports/
-
-# azext_aks_preview
-azext_aks_preview_unit_test_result=""
-pushd azure-cli-extensions/src/aks-preview/azext_aks_preview
+azext_aks_preview_unit_test_failed=""
+pushd ${aks_preview_base_dir}
 # clean existing coverage report
 (coverage combine || true) && (coverage erase || true)
-# currently test using module 'unittest' is the same as module 'pytest', and test using 'pytest' is just recording test
-if ! coverage run --source=. --omit=*/vendored_sdks/*,*/tests/* -p -m unittest discover || ! coverage run --source=. --omit=*/vendored_sdks/*,*/tests/* -p -m pytest; then
-    azext_aks_preview_unit_test_result="error"
+# perform unit test with module 'unittest'
+# since recording test (performed in test_ext_live.sh) is based on module 'pytest', so skip here
+# coverage run --source=. --omit=*/vendored_sdks/*,*/tests/* -p -m pytest
+if ! coverage run --source=. --omit=*/vendored_sdks/*,*/tests/* -p -m unittest discover; then
+    azext_aks_preview_unit_test_failed="true"
 fi
+# generate & copy coverage report
 coverage combine && coverage json -o coverage_azext_aks_preview.json
 coverage report -m
 popd
-cp azure-cli-extensions/src/aks-preview/azext_aks_preview/coverage_azext_aks_preview.json reports/
+cp ${aks_preview_base_dir}/coverage_azext_aks_preview.json reports/
 
-if [[ ${az_aks_tool_unit_test_result} == "error" || ${azext_aks_preview_unit_test_result} == "error" ]]; then
+if [[ ${azext_aks_preview_unit_test_failed} == "true" ]]; then
     echo "Unit test failed!"
     exit 1
 fi
diff --git a/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml
index 6b90a7a1526..065b9b542de 100644
--- a/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml
+++ b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml
@@ -34,7 +34,7 @@ jobs:
         ls -alh
       displayName: "Move All Checkout Files to the Newly Created 'azure-cli-extensions' Directory"
     - bash: |
-        $(LIVE_TEST_BASE_DIR)/clone_repo.sh
+        $(LIVE_TEST_BASE_DIR)/scripts/clone_repo.sh
       condition: succeeded()
       displayName: "Clone GitHub Repo and Move Test Related Files"
     - bash: |
@@ -50,8 +50,8 @@ jobs:
       condition: succeeded()
       displayName: "Start Container"
     - bash: |
-        docker exec $(ContainerName) /opt/scripts/setup_venv.sh setuptool y
-        docker exec $(ContainerName) /opt/scripts/setup_venv.sh setupaz n azure-cli/ azure-cli-extensions/
+        docker exec $(ContainerName) /opt/scripts/setup_venv.sh setup-tool y
+        docker exec $(ContainerName) /opt/scripts/setup_venv.sh setup-az n azure-cli/ azure-cli-extensions/
       condition: succeeded()
       displayName: "Set up Virtual Environment"
     - bash: |
diff --git a/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-unit-test.yaml b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-unit-test.yaml
index 66a0fb8d953..a1baf6631c6 100644
--- a/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-unit-test.yaml
+++ b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-unit-test.yaml
@@ -34,7 +34,7 @@ jobs:
         ls -alh
       displayName: "Move All Checkout Files to the Newly Created 'azure-cli-extensions' Directory"
     - bash: |
-        $(LIVE_TEST_BASE_DIR)/clone_repo.sh
+        $(LIVE_TEST_BASE_DIR)/scripts/clone_repo.sh
       condition: succeeded()
       displayName: "Clone GitHub Repo and Move Test Related Files"
     - bash: |
@@ -50,8 +50,8 @@ jobs:
       condition: succeeded()
       displayName: "Start Container"
     - bash: |
-        docker exec $(ContainerName) /opt/scripts/setup_venv.sh setuptool y
-        docker exec $(ContainerName) /opt/scripts/setup_venv.sh setupaz n azure-cli/ azure-cli-extensions/
+        docker exec $(ContainerName) /opt/scripts/setup_venv.sh setup-tool y
+        docker exec $(ContainerName) /opt/scripts/setup_venv.sh setup-az n azure-cli/ azure-cli-extensions/
       condition: succeeded()
       displayName: "Set up Virtual Environment"
     - bash: |

From 90b45b189fcbe6aefe62bfef37470c5fced653c9 Mon Sep 17 00:00:00 2001
From: Fuming Zhang <fumingzhang@microsoft.com>
Date: Thu, 24 Jun 2021 11:02:02 +0800
Subject: [PATCH 05/10] remove az_aks_tool folder

---
 src/aks-preview/az_aks_tool/.gitignore        |   1 -
 src/aks-preview/az_aks_tool/cli.py            |  42 ---
 src/aks-preview/az_aks_tool/const.py          |  17 -
 src/aks-preview/az_aks_tool/ext.py            |  46 ---
 src/aks-preview/az_aks_tool/filter.py         | 120 --------
 src/aks-preview/az_aks_tool/index.py          | 290 ------------------
 src/aks-preview/az_aks_tool/log.py            |  40 ---
 src/aks-preview/az_aks_tool/main.py           | 151 ---------
 src/aks-preview/az_aks_tool/run.py            | 199 ------------
 src/aks-preview/az_aks_tool/tests/__init__.py |   0
 src/aks-preview/az_aks_tool/tests/test_cli.py |  22 --
 src/aks-preview/az_aks_tool/tests/test_ext.py |  22 --
 .../az_aks_tool/tests/test_filter.py          | 121 --------
 .../az_aks_tool/tests/test_index.py           |  34 --
 src/aks-preview/az_aks_tool/tests/test_log.py |  34 --
 .../az_aks_tool/tests/test_main.py            |  21 --
 src/aks-preview/az_aks_tool/tests/test_run.py |  33 --
 .../az_aks_tool/tests/test_utils.py           |  54 ----
 .../az_aks_tool/tests/testdata.json           |  18 --
 src/aks-preview/az_aks_tool/utils.py          |  92 ------
 20 files changed, 1357 deletions(-)
 delete mode 100644 src/aks-preview/az_aks_tool/.gitignore
 delete mode 100644 src/aks-preview/az_aks_tool/cli.py
 delete mode 100644 src/aks-preview/az_aks_tool/const.py
 delete mode 100644 src/aks-preview/az_aks_tool/ext.py
 delete mode 100644 src/aks-preview/az_aks_tool/filter.py
 delete mode 100644 src/aks-preview/az_aks_tool/index.py
 delete mode 100644 src/aks-preview/az_aks_tool/log.py
 delete mode 100644 src/aks-preview/az_aks_tool/main.py
 delete mode 100644 src/aks-preview/az_aks_tool/run.py
 delete mode 100644 src/aks-preview/az_aks_tool/tests/__init__.py
 delete mode 100644 src/aks-preview/az_aks_tool/tests/test_cli.py
 delete mode 100644 src/aks-preview/az_aks_tool/tests/test_ext.py
 delete mode 100644 src/aks-preview/az_aks_tool/tests/test_filter.py
 delete mode 100644 src/aks-preview/az_aks_tool/tests/test_index.py
 delete mode 100644 src/aks-preview/az_aks_tool/tests/test_log.py
 delete mode 100644 src/aks-preview/az_aks_tool/tests/test_main.py
 delete mode 100644 src/aks-preview/az_aks_tool/tests/test_run.py
 delete mode 100644 src/aks-preview/az_aks_tool/tests/test_utils.py
 delete mode 100644 src/aks-preview/az_aks_tool/tests/testdata.json
 delete mode 100644 src/aks-preview/az_aks_tool/utils.py

diff --git a/src/aks-preview/az_aks_tool/.gitignore b/src/aks-preview/az_aks_tool/.gitignore
deleted file mode 100644
index bee8a64b79a..00000000000
--- a/src/aks-preview/az_aks_tool/.gitignore
+++ /dev/null
@@ -1 +0,0 @@
-__pycache__
diff --git a/src/aks-preview/az_aks_tool/cli.py b/src/aks-preview/az_aks_tool/cli.py
deleted file mode 100644
index ed21d2ce0a2..00000000000
--- a/src/aks-preview/az_aks_tool/cli.py
+++ /dev/null
@@ -1,42 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------
-
-import glob
-import os
-import logging
-
-import az_aks_tool.const as const
-import az_aks_tool.index as index
-logger = logging.getLogger(__name__)
-
-
-def get_cli_mod_data(mod_name=const.ACS_MOD_NAME, profile="latest"):
-    profile_split = profile.split('-')
-    profile_namespace = '_'.join([profile_split[-1]] + profile_split[:-1])
-
-    # key value pairs of all modules(in azcli & extention) and its absolute path, used later to find test indexes
-    path_table = index.get_path_table()
-    command_modules = path_table["mod"]
-    inverse_name_table = index.get_name_index(invert=True)
-
-    # construct 'import_name' & mod_data', used later to find test indexes
-    acs_mod_path = command_modules[mod_name]
-    mod_data = {
-        "alt_name": "{}{}".format(const.COMMAND_MODULE_PREFIX, mod_name),
-        "filepath": os.path.join(acs_mod_path, "tests", profile_namespace),
-        "base_path": "azure.cli.command_modules.{}.tests.{}".format(mod_name, profile_namespace),
-        "files": {}
-    }
-
-    cli_test = index.discover_module_tests(mod_name, mod_data)
-    return cli_test
-
-
-def get_cli_test_index(module_data=None, mod_name=const.ACS_MOD_NAME, profile="latest"):
-    if mod_name in module_data:
-        mod_data = module_data[mod_name]
-    else:
-        mod_data = get_cli_mod_data(mod_name=mod_name, profile=profile)    
-    return mod_data["files"]
diff --git a/src/aks-preview/az_aks_tool/const.py b/src/aks-preview/az_aks_tool/const.py
deleted file mode 100644
index 8285bbb80c3..00000000000
--- a/src/aks-preview/az_aks_tool/const.py
+++ /dev/null
@@ -1,17 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------
-
-import sys
-
-IS_WINDOWS = sys.platform.lower() in ['windows', 'win32']
-
-CLI_REPO_NAME = "azure-cli"
-EXT_REPO_NAME = 'azure-cli-extensions'
-COMMAND_MODULE_PREFIX = 'azure-cli-'
-EXTENSION_PREFIX = 'azext_'
-ACS_MOD_NAME = "acs"
-AKS_PREVIEW_MOD_NAME = EXTENSION_PREFIX + "aks_preview"  # azext_aks_preview
-
-ENV_VAR_TEST_LIVE = 'AZURE_TEST_RUN_LIVE'               # denotes that tests should be run live instead of played back
diff --git a/src/aks-preview/az_aks_tool/ext.py b/src/aks-preview/az_aks_tool/ext.py
deleted file mode 100644
index 8fa39c1e242..00000000000
--- a/src/aks-preview/az_aks_tool/ext.py
+++ /dev/null
@@ -1,46 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------
-
-import glob
-import os
-import logging
-
-import az_aks_tool.const as const
-import az_aks_tool.index as index
-logger = logging.getLogger(__name__)
-
-
-def get_ext_mod_data(mod_name=const.AKS_PREVIEW_MOD_NAME, profile="latest"):
-    profile_split = profile.split('-')
-    profile_namespace = '_'.join([profile_split[-1]] + profile_split[:-1])
-
-    # key value pairs of all modules(in azcli & extention) and its absolute path, used later to find test indexes
-    path_table = index.get_path_table()
-    extensions = path_table["ext"]
-    inverse_name_table = index.get_name_index(invert=True)
-
-    # construct 'import_name' & mod_data', used later to find test indexes
-    aks_preview_mod_path = extensions[mod_name]
-    glob_pattern = os.path.normcase(
-        os.path.join("{}*".format(const.EXTENSION_PREFIX)))
-    file_path = glob.glob(os.path.join(aks_preview_mod_path, glob_pattern))[0]
-    import_name = os.path.basename(file_path)
-    mod_data = {
-        "alt_name": inverse_name_table[mod_name],
-        "filepath": os.path.join(file_path, "tests", profile_namespace),
-        "base_path": "{}.tests.{}".format(import_name, profile_namespace),
-        "files": {}
-    }
-
-    ext_test = index.discover_module_tests(import_name, mod_data)
-    return ext_test
-
-
-def get_ext_test_index(module_data=None, mod_name=const.AKS_PREVIEW_MOD_NAME, profile="latest"):
-    if mod_name in module_data:
-        mod_data = module_data[mod_name]
-    else:
-        mod_data = get_ext_mod_data(mod_name=mod_name, profile=profile)
-    return mod_data["files"]
diff --git a/src/aks-preview/az_aks_tool/filter.py b/src/aks-preview/az_aks_tool/filter.py
deleted file mode 100644
index 718a34d4058..00000000000
--- a/src/aks-preview/az_aks_tool/filter.py
+++ /dev/null
@@ -1,120 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------
-
-import logging
-from collections import Iterable
-logger = logging.getLogger(__name__)
-
-
-def extract_file_class_pairs(tag_list):
-    pairs = []
-    for k in tag_list:
-        tags = k.split(".")
-        if len(tags) == 2:
-            pairs.append((tags[0], tags[1]))
-    return pairs
-
-
-def filter_valid_file_class_pairs(pairs, test_index):
-    valid_pairs = []
-    for pair in pairs:
-        if pair[0] in test_index and pair[1] in test_index[pair[0]]:
-            valid_pairs.append(pair)
-            logger.debug("Valid file & class pair: '{}'".format(pair))
-        else:
-            logger.debug("Invalid file & class pair: '{}'".format(pair))
-    return valid_pairs
-
-
-def get_all_values_from_nested_dict(d):
-    for v in d.values():
-        if isinstance(v, dict):
-            yield from get_all_values_from_nested_dict(v)
-        else:
-            yield v
-
-
-def flatten_nested_list(lis):
-    for item in lis:
-        if isinstance(item, Iterable) and not isinstance(item, str):
-            for x in flatten_nested_list(item):
-                yield x
-        else:
-            yield item
-
-
-def filter_valid_test_cases(test_cases, test_index):
-    valid_test_cases = []
-    nested_test_cases = list(get_all_values_from_nested_dict(test_index))
-    falttened_test_cases = list(flatten_nested_list(nested_test_cases))
-    for test_case in test_cases:
-        if test_case in falttened_test_cases:
-            valid_test_cases.append(test_case)
-            logger.debug("Valid test case: '{}'".format(test_case))
-        else:
-            logger.debug("Invalid test case: '{}'".format(test_case))
-    return valid_test_cases
-
-
-def get_test_cases(test_index, matrix, extra_coverage=None):
-    test_cases = []
-    coverage = matrix.get("coverage", {})
-    # default coverage
-    for fileName, className in coverage.items():
-        for c in className:
-            test_cases.extend(test_index[fileName][c])
-    # custom extra coverage
-    if extra_coverage:
-        # method 1: fileName.className
-        file_class_pairs = extract_file_class_pairs(extra_coverage)
-        valid_file_class_pairs = filter_valid_file_class_pairs(
-            file_class_pairs, test_index)
-        for valid_pair in valid_file_class_pairs:
-            test_cases.extend(
-                test_index[valid_pair[0]][valid_pair[1]])
-        # method 2: test cases
-        test_cases.extend(filter_valid_test_cases(
-            extra_coverage, test_index))
-    return list(set(test_cases))
-
-
-def get_exclude_test_cases(test_index, matrix, extra_filter=None):
-    exclude_test_cases = []
-    exclude = matrix.get("exclude", {})
-    # default exclude
-    if not extra_filter or "default" in extra_filter:
-        matrix_test_cases = []
-        matrix_file_class_pairs = []
-        for k, v in exclude.items():
-            # method 1: reason -> test cases
-            matrix_test_cases.extend(v)
-            # method 2: fileName -> className
-            matrix_file_class_pairs.extend((k, x) for x in v)
-        # method 1: reason -> test cases
-        exclude_test_cases.extend(
-            filter_valid_test_cases(matrix_test_cases, test_index))
-        # method 2: fileName -> className
-        valid_matrix_file_class_pairs = filter_valid_file_class_pairs(
-            matrix_file_class_pairs, test_index)
-        for valid_matrix_pair in valid_matrix_file_class_pairs:
-            exclude_test_cases.extend(
-                test_index[valid_matrix_pair[0]][valid_matrix_pair[1]])
-    # custom extra_filter
-    if extra_filter:
-        # method 1: matrix exclude key
-        for k, v in exclude.items():
-            if k in extra_filter:
-                exclude_test_cases.extend(v)
-        # method 2: fileName.className
-        file_class_pairs = extract_file_class_pairs(extra_filter)
-        valid_file_class_pairs = filter_valid_file_class_pairs(
-            file_class_pairs, test_index)
-        for valid_pair in valid_file_class_pairs:
-            exclude_test_cases.extend(
-                test_index[valid_pair[0]][valid_pair[1]])
-        # method 3: test cases
-        exclude_test_cases.extend(
-            filter_valid_test_cases(extra_filter, test_index))
-    return list(set(exclude_test_cases))
diff --git a/src/aks-preview/az_aks_tool/index.py b/src/aks-preview/az_aks_tool/index.py
deleted file mode 100644
index 11d7e405179..00000000000
--- a/src/aks-preview/az_aks_tool/index.py
+++ /dev/null
@@ -1,290 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------
-
-import glob
-import logging
-import os
-from importlib import import_module
-
-import az_aks_tool.utils as utils
-import az_aks_tool.const as const
-logger = logging.getLogger(__name__)
-
-
-def get_repo_path(repo_name, root_path=None):
-    # find cache from environment variable
-    repo_path = os.environ.get("{}_PATH".format(repo_name), "")
-    if os.path.isdir(repo_path):
-        logger.info("Find cached '{}' repo path: '{}'".format(repo_name, repo_path))
-        return repo_path
-
-    # search from root_path
-    candidate_root_paths = [os.getcwd(), os.path.expanduser("~")]
-    valid_repo_paths = []
-    repo_path = ""
-    if root_path is None or not os.path.isdir(root_path):
-        logger.warning("Invalid root path '{}'!".format(root_path))
-    else:
-        candidate_root_paths = [root_path] + candidate_root_paths
-    logger.info("Setting root path from '{}'".format(candidate_root_paths))
-    for candidate_root_path in candidate_root_paths:
-        root_path = candidate_root_path
-        logger.info("Searching from root path: '{}'".format(root_path))
-        for path, _, _ in os.walk(root_path):
-            pattern = os.path.join(path, repo_name)
-            valid_repo_paths.extend(glob.glob(pattern))
-        if len(valid_repo_paths) >= 1:
-            repo_path = valid_repo_paths[0]
-            if len(valid_repo_paths) >= 2:
-                logger.warning("Find {} '{}' repo paths: {}".format(len(valid_repo_paths), repo_name, valid_repo_paths))
-            logger.info("Set '{}' repo path as '{}'".format(repo_name, repo_path))
-            os.environ["{}_PATH".format(repo_name)] = str(repo_path)
-            return repo_path
-        else:
-            logger.warning("Could not find valid path to repo '{}' from '{}'".format(repo_name, root_path))
-    return repo_path
-
-def find_files(root_paths, file_pattern):
-    """ Returns the paths to all files that match a given pattern.
-
-    :returns: Paths ([str]) to files matching the given pattern.
-    """
-    if isinstance(root_paths, str):
-        root_paths = [root_paths]
-    paths = []
-    for root_path in root_paths:
-        for path, _, _ in os.walk(root_path):
-            pattern = os.path.join(path, file_pattern)
-            paths.extend(glob.glob(pattern))
-    return paths
-
-def get_name_index(invert=False, include_whl_extensions=False):
-    """ Returns a dictionary containing the long and short names of modules and extensions is {SHORT:LONG} format or
-        {LONG:SHORT} format when invert=True. """
-    from azure.cli.core.extension import EXTENSIONS_DIR  # pylint: disable=import-error
-
-    table = {}
-    cli_repo_path = get_repo_path(const.CLI_REPO_NAME)
-    ext_repo_paths = get_repo_path(const.EXT_REPO_NAME)
-
-    # unified azure-cli package (2.0.68 and later)
-    paths = os.path.normcase(
-        os.path.join(
-            cli_repo_path, 'src', 'azure-cli', 'azure', 'cli', 'command_modules', '*', '__init__.py'
-        )
-    )
-    modules_paths = glob.glob(paths)
-    core_paths = glob.glob(os.path.normcase(os.path.join(cli_repo_path, 'src', '*', 'setup.py')))
-    ext_paths = [x for x in find_files(ext_repo_paths, '*.*-info') if 'site-packages' not in x]
-    whl_ext_paths = []
-    if include_whl_extensions:
-        whl_ext_paths = [x for x in find_files(EXTENSIONS_DIR, '*.*-info') if 'site-packages' not in x]
-
-    def _update_table(paths, key):
-        folder = None
-        long_name = None
-        short_name = None
-        for path in paths:
-            folder = os.path.dirname(path)
-            base_name = os.path.basename(folder)
-            # determine long-names
-            if key == 'ext':
-                short_name = base_name
-                for item in os.listdir(folder):
-                    if item.startswith(const.EXTENSION_PREFIX):
-                        long_name = item
-                        break
-            elif base_name.startswith(const.COMMAND_MODULE_PREFIX):
-                long_name = base_name
-                short_name = base_name.replace(const.COMMAND_MODULE_PREFIX, '') or '__main__'
-            else:
-                short_name = base_name
-                long_name = '{}{}'.format(const.COMMAND_MODULE_PREFIX, base_name)
-            if not invert:
-                table[short_name] = long_name
-            else:
-                table[long_name] = short_name
-
-    _update_table(modules_paths, 'mod')
-    _update_table(core_paths, 'core')
-    _update_table(ext_paths, 'ext')
-    _update_table(whl_ext_paths, 'ext')
-
-    return table
-
-# pylint: disable=too-many-statements
-def get_path_table(include_only=None, include_whl_extensions=False):
-    """ Returns a table containing the long and short names of different modules and extensions and the path to them.
-        The structure looks like:
-    {
-        'core': {
-            NAME: PATH,
-            ...
-        },
-        'mod': {
-            NAME: PATH,
-            ...
-        },
-        'ext': {
-            NAME: PATH,
-            ...
-        }
-    }
-    """
-    from azure.cli.core.extension import EXTENSIONS_DIR  # pylint: disable=import-error
-
-    # determine whether the call will filter or return all
-    if isinstance(include_only, str):
-        include_only = [include_only]
-    get_all = not include_only
-
-    table = {}
-    cli_repo_path = get_repo_path(const.CLI_REPO_NAME)
-    ext_repo_paths = get_repo_path(const.EXT_REPO_NAME)
-
-    paths = os.path.normcase(
-        os.path.join(
-            cli_repo_path, 'src', 'azure-cli', 'azure', 'cli', 'command_modules', '*', '__init__.py'
-        )
-    )
-    modules_paths = glob.glob(paths)
-    core_paths = glob.glob(os.path.normcase(os.path.join(cli_repo_path, 'src', '*', 'setup.py')))
-    ext_paths = [x for x in find_files(ext_repo_paths, '*.*-info') if 'site-packages' not in x]
-    whl_ext_paths = [x for x in find_files(EXTENSIONS_DIR, '*.*-info') if 'site-packages' not in x]
-
-    def _update_table(package_paths, key):
-        if key not in table:
-            table[key] = {}
-
-        for path in package_paths:
-            folder = os.path.dirname(path)
-            base_name = os.path.basename(folder)
-
-            if key == 'ext':
-                short_name = base_name
-                long_name = next((item for item in os.listdir(folder) if item.startswith(const.EXTENSION_PREFIX)), None)
-            else:
-                short_name = base_name
-                long_name = '{}{}'.format(const.COMMAND_MODULE_PREFIX, base_name)
-
-            if get_all:
-                table[key][long_name if key == 'ext' else short_name] = folder
-            elif not include_only:
-                return  # nothing left to filter
-            else:
-                # check and update filter
-                if short_name in include_only:
-                    include_only.remove(short_name)
-                    table[key][short_name] = folder
-                if long_name in include_only:
-                    # long name takes precedence to ensure path doesn't appear twice
-                    include_only.remove(long_name)
-                    table[key].pop(short_name, None)
-                    table[key][long_name] = folder
-
-    _update_table(modules_paths, 'mod')
-    _update_table(core_paths, 'core')
-    _update_table(ext_paths, 'ext')
-    if include_whl_extensions:
-        _update_table(whl_ext_paths, 'ext')
-
-    if include_only:
-        whl_extensions = [mod for whl_ext_path in whl_ext_paths for mod in include_only if mod in whl_ext_path]
-        if whl_extensions:
-            err = 'extension(s): [ {} ] installed from a wheel may need --include-whl-extensions option'.format(
-                ', '.join(whl_extensions))
-            raise Exception(err)
-
-        raise Exception('unrecognized modules: [ {} ]'.format(', '.join(include_only)))
-
-    return table
-
-def discover_module_tests(mod_name, mod_data):
-
-    # get the list of test files in each module
-    total_tests = 0
-    total_files = 0
-    logger.info('Mod: %s', mod_name)
-    try:
-        contents = os.listdir(mod_data['filepath'])
-        test_files = {
-            x[:-len('.py')]: {} for x in contents if x.startswith('test_') and x.endswith('.py')
-        }
-        total_files = len(test_files)
-    except FileNotFoundError:
-        logger.info('  No test files found.')
-        return None
-
-    for file_name in test_files:
-        mod_data['files'][file_name] = {}
-        test_file_path = mod_data['base_path'] + '.' + file_name
-        try:
-            module = import_module(test_file_path)
-        except ImportError as ex:
-            logger.info('    %s', ex)
-            continue
-        module_dict = module.__dict__
-        possible_test_classes = {x: y for x, y in module_dict.items() if not x.startswith('_')}
-        for class_name, class_def in possible_test_classes.items():
-            try:
-                class_dict = class_def.__dict__
-            except AttributeError:
-                # skip non-class symbols in files like constants, imported methods, etc.
-                continue
-            if class_dict.get('__module__') == test_file_path:
-                tests = [x for x in class_def.__dict__ if x.startswith('test_')]
-                if tests:
-                    mod_data['files'][file_name][class_name] = tests
-                total_tests += len(tests)
-    logger.info('  %s tests found in %s files.', total_tests, total_files)
-    return mod_data
-
-
-def build_test_index(module_data):
-    test_index = {}
-    conflicted_keys = []
-
-    def add_to_index(key, path):
-        key = key or mod_name
-        if key in test_index:
-            if key not in conflicted_keys:
-                conflicted_keys.append(key)
-            mod1 = utils.extract_module_name(path)
-            mod2 = utils.extract_module_name(test_index[key])
-            if mod1 != mod2:
-                # resolve conflicted keys by prefixing with the module name and a dot (.)
-                logger.warning("'%s' exists in both '%s' and '%s'. Resolve using `%s.%s` or `%s.%s`",
-                               key, mod1, mod2, mod1, key, mod2, key)
-                test_index['{}.{}'.format(mod1, key)] = path
-                test_index['{}.{}'.format(mod2, key)] = test_index[key]
-            else:
-                logger.error("'%s' exists twice in the '%s' module", key, mod1)
-        else:
-            test_index[key] = path
-
-    # build the index
-    for mod_name, mod_data in module_data.items():
-        # don't add empty mods to the index
-        if not mod_data:
-            continue
-
-        mod_path = mod_data['filepath']
-        for file_name, file_data in mod_data['files'].items():
-            file_path = os.path.join(mod_path, file_name) + '.py'
-            for class_name, test_list in file_data.items():
-                for test_name in test_list:
-                    test_path = '{}::{}::{}'.format(file_path, class_name, test_name)
-                    add_to_index(test_name, test_path)
-                class_path = '{}::{}'.format(file_path, class_name)
-                add_to_index(class_name, class_path)
-            add_to_index(file_name, file_path)
-        add_to_index(mod_name, mod_path)
-        add_to_index(mod_data['alt_name'], mod_path)
-
-    # remove the conflicted keys since they would arbitrarily point to a random implementation
-    for key in conflicted_keys:
-        del test_index[key]
-
-    return test_index
diff --git a/src/aks-preview/az_aks_tool/log.py b/src/aks-preview/az_aks_tool/log.py
deleted file mode 100644
index a5d27925983..00000000000
--- a/src/aks-preview/az_aks_tool/log.py
+++ /dev/null
@@ -1,40 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------
-
-import logging
-import sys
-
-
-def parse_module_name(levels=1):
-    module_name = None
-    module_levels = __name__.split(".")
-    if len(module_levels) < levels:
-        print("Failed to parse {}-level module name from '{}'".format(levels, __name__))
-    else:
-        module_name = ".".join(module_levels[:levels])
-    return module_name
-
-
-def setup_logging(root_logger_name=None, log_path="az_aks_tool.log"):
-    if root_logger_name == "" or root_logger_name.isspace():
-        root_logger_name = parse_module_name()
-    logger = logging.getLogger(root_logger_name)
-    logger.setLevel(level=logging.DEBUG)
-
-    # Formatter
-    formatter = logging.Formatter(
-        '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
-
-    # FileHandler
-    file_handler = logging.FileHandler(filename=log_path, mode="w")
-    file_handler.setFormatter(formatter)
-    file_handler.setLevel(level=logging.DEBUG)
-    logger.addHandler(file_handler)
-
-    # StreamHandler
-    stream_handler = logging.StreamHandler(sys.stdout)
-    stream_handler.setFormatter(formatter)
-    stream_handler.setLevel(level=logging.INFO)
-    logger.addHandler(stream_handler)
diff --git a/src/aks-preview/az_aks_tool/main.py b/src/aks-preview/az_aks_tool/main.py
deleted file mode 100644
index 57b8f77bd75..00000000000
--- a/src/aks-preview/az_aks_tool/main.py
+++ /dev/null
@@ -1,151 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------
-
-import argparse
-import os
-import sys
-import logging
-
-import az_aks_tool.const as const
-import az_aks_tool.log as log
-import az_aks_tool.utils as utils
-import az_aks_tool.cli as cli
-import az_aks_tool.ext as ext
-import az_aks_tool.index as index
-import az_aks_tool.run as run
-
-
-def init_argparse(args):
-    parser = argparse.ArgumentParser()
-    parser.add_argument("-c", "--cli", action="store_true", default=False,
-                        help="enbale cli test")
-    parser.add_argument("-e", "--ext", action="store_true", default=False,
-                        help="enbale ext test")
-    parser.add_argument("-a", "--all", action="store_true", default=False,
-                        help="enbale all tests (cli & ext)")
-    parser.add_argument("-t", "--tests", nargs='+', help="test case names")
-    parser.add_argument("-cm", "--cli-matrix",  type=str,
-                        help="full path to cli test matrix")
-    parser.add_argument("-cc", "--cli-coverage", nargs="+",
-                        help="cli test extra coverage")
-    parser.add_argument("-cf", "--cli-filter", nargs="+",
-                        help="cli test filter")
-    parser.add_argument("-em", "--ext-matrix",  type=str,
-                        help="full path to extension test matrix")
-    parser.add_argument("-ec", "--ext-coverage", nargs="+",
-                        help="extension test extra coverage")
-    parser.add_argument("-ef", "--ext-filter", nargs="+",
-                        help="extension test filter")
-    parser.add_argument("-s", "--series", action="store_true",
-                        default=False, help="series test")
-    parser.add_argument("-l", "--live", action="store_true",
-                        default=False, help="live test")
-    parser.add_argument("-ne", "--no-exitfirst", action="store_true",
-                        default=False, help="no exit first")
-    parser.add_argument("-j", "--parallelism", type=str,
-                        default="8", help="test parallelism")
-    parser.add_argument("--reruns", type=str,
-                        default="3", help="rerun times")
-    parser.add_argument("--capture", type=str,
-                        default="sys", help="test capture")
-    parser.add_argument("-p", "--report-path", type=str,
-                        required=True, help="report path")
-    parser.add_argument("-f", "--json-report-file", type=str,
-                        default="azcli_aks_runner_report.json", help="json report filename")
-    parser.add_argument("--xml-file", type=str,
-                        default="azcli_aks_runner.xml", help="junit/xml report filename")
-    parser.add_argument("--log-file", type=str,
-                        default="az_aks_tool.log", help="log filename")
-    args = parser.parse_args(args)
-    return args
-
-
-def main():
-    # parse args
-    print("raw args: {}".format(sys.argv))
-    args = init_argparse(sys.argv[1:])
-
-    # check directory
-    utils.create_directory(args.report_path)
-
-    # setup logger
-    root_module_name = log.parse_module_name(levels=1)
-    log.setup_logging(root_module_name, os.path.join(
-        args.report_path, args.log_file))
-    logger = logging.getLogger("{}.{}".format(root_module_name, __name__))
-
-    # # check test cases
-    test_cases = args.tests
-    ext_matrix_file_path = args.ext_matrix
-    cli_matrix_file_path = args.cli_matrix
-
-    # prepare pytest args
-    pytest_args = []
-    if not args.series and args.parallelism:
-        pytest_args.append("-n {}".format(args.parallelism))
-    pytest_args.append("--json-report")
-    pytest_args.append("--reruns {}".format(args.reruns))
-    pytest_args.append("--capture {}".format(args.capture))
-    pytest_args = [" ".join(pytest_args)]
-    logger.info("pytest_args: {}".format(pytest_args))
-
-    # check mode & collect module data
-    enable_cli = False
-    enable_ext = False
-    module_data = {}
-    if args.cli or args.all:
-        enable_cli = True
-        module_data[const.ACS_MOD_NAME] = cli.get_cli_mod_data()
-        cli_test_index = cli.get_cli_test_index(module_data)
-
-    if args.ext or args.all:
-        enable_ext = True
-        module_data[const.AKS_PREVIEW_MOD_NAME] = ext.get_ext_mod_data()
-        ext_test_index = ext.get_ext_test_index(module_data)
-
-    # build test index
-    if enable_cli or enable_ext:
-        logger.info("Building test index...")
-        test_index = index.build_test_index(module_data)
-    else:
-        logger.error(
-            "Both modes 'cli' and 'ext' are not enabled! No test will be performed!")
-        logger.error(
-            "Please provide at least one of the following parameters (-a, -c, -e) to enable the test!")
-
-    # cli matrix test
-    if enable_cli:
-        cli_qualified_test_cases = utils.get_fully_qualified_test_cases(
-            cli_test_index, cli_matrix_file_path, const.ACS_MOD_NAME, args.cli_coverage, args.cli_filter)
-        logger.info("Perform following cli tests: {}".format(
-            cli_qualified_test_cases))
-        exit_code = run.run_tests(cli_qualified_test_cases, test_index, mode="cli", base_path=args.report_path, xml_file=args.xml_file, json_file=args.json_report_file, in_series=args.series,
-                                  run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args)
-        if exit_code != 0:
-            sys.exit("CLI test failed with exit code: {}".format(exit_code))
-
-    # ext matrix test
-    if enable_ext:
-        ext_qualified_test_cases = utils.get_fully_qualified_test_cases(
-            ext_test_index, ext_matrix_file_path, const.AKS_PREVIEW_MOD_NAME, args.ext_coverage, args.ext_filter)
-        logger.info("Perform following ext tests: {}".format(
-            ext_qualified_test_cases))
-        exit_code = run.run_tests(ext_qualified_test_cases, test_index, mode="ext", base_path=args.report_path, xml_file=args.xml_file, json_file=args.json_report_file, in_series=args.series,
-                                  run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args)
-        if exit_code != 0:
-            sys.exit("EXT test failed with exit code: {}".format(exit_code))
-
-    # raw tests
-    if test_cases:
-        logger.info("Get {} cases!".format(len(test_cases)))
-        logger.info("Perform following raw tets: {}".format(test_cases))
-        exit_code = run.run_tests(test_cases, test_index, mode="raw", base_path=args.report_path, xml_file=args.xml_file, json_file=args.json_report_file, in_series=args.series,
-                                  run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args)
-        if exit_code != 0:
-            sys.exit("Raw test failed with exit code: {}".format(exit_code))
-
-
-if __name__ == "__main__":
-    main()
diff --git a/src/aks-preview/az_aks_tool/run.py b/src/aks-preview/az_aks_tool/run.py
deleted file mode 100644
index 13c2f2ae426..00000000000
--- a/src/aks-preview/az_aks_tool/run.py
+++ /dev/null
@@ -1,199 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------
-
-import logging
-import os
-import subprocess
-import sys
-import traceback
-from knack.util import CommandResultItem
-
-from az_aks_tool.const import IS_WINDOWS, ENV_VAR_TEST_LIVE
-from az_aks_tool.utils import heading
-logger = logging.getLogger(__name__)
-
-
-class ProfileContext:
-    def __init__(self, profile_name=None):
-        self.target_profile = profile_name
-
-        self.origin_profile = current_profile()
-
-    def __enter__(self):
-        if self.target_profile is None or self.target_profile == self.origin_profile:
-            logger.info('The tests are set to run against current profile "{}"'.format(
-                self.origin_profile))
-        else:
-            result = cmd('az cloud update --profile {}'.format(self.target_profile),
-                         'Switching to target profile "{}"...'.format(self.target_profile))
-            if result.exit_code != 0:
-                raise Exception(result.error.output.decode('utf-8'))
-
-    def __exit__(self, exc_type, exc_val, exc_tb):
-        if self.target_profile is not None and self.target_profile != self.origin_profile:
-            logger.info('Switching back to origin profile "{}"...'.format(
-                self.origin_profile))
-            call('az cloud update --profile {}'.format(self.origin_profile))
-
-        if exc_tb:
-            traceback.print_exception(exc_type, exc_val, exc_tb)
-
-
-def current_profile():
-    return cmd('az cloud show --query profile -otsv', show_stderr=False).result
-
-
-class CommandError(Exception):
-
-    def __init__(self, output, exit_code, command):
-        message = "Command `{}` failed with exit code {}:\n{}".format(
-            command, exit_code, output)
-        self.exit_code = exit_code
-        self.output = output
-        self.command = command
-        super().__init__(message)
-
-
-def call(command, **kwargs):
-    """ Run an arbitrary command but don't buffer the output.
-
-    :param command: The entire command line to run.
-    :param kwargs: Any kwargs supported by subprocess.Popen
-    :returns: (int) process exit code.
-    """
-    return subprocess.call(
-        command,
-        shell=True,
-        **kwargs)
-
-
-def cmd(command, message=False, show_stderr=True, raise_error=False, **kwargs):
-    """ Run an arbitrary command.
-
-    :param command: The entire command line to run.
-    :param message: A custom message to display, or True (bool) to use a default.
-    :param show_stderr: On error, display the contents of STDERR.
-    :param raise_error: On error, raise CommandError.
-    :param kwargs: Any kwargs supported by subprocess.Popen
-    :returns: CommandResultItem object.
-    """
-
-    # use default message if custom not provided
-    if message is True:
-        message = 'Running: {}\n'.format(command)
-
-    if message:
-        logger.info(message)
-
-    logger.info("Running: %s", command)
-    try:
-        output = subprocess.check_output(
-            command.split(),
-            stderr=subprocess.STDOUT if show_stderr else None,
-            shell=IS_WINDOWS,
-            **kwargs).decode('utf-8').strip()
-        logger.debug(output)
-        return CommandResultItem(output, exit_code=0, error=None)
-    except subprocess.CalledProcessError as err:
-        if raise_error:
-            raise CommandError(err.output.decode(), err.returncode, command)
-        return CommandResultItem(err.output, exit_code=err.returncode, error=err)
-
-
-def get_test_runner(parallel, log_path, last_failed, no_exit_first, mark):
-    """Create a pytest execution method"""
-    def _run(test_paths, pytest_args):
-
-        if os.name == 'posix':
-            arguments = ['-x', '-v', '--boxed', '-p no:warnings',
-                         '--log-level=WARN', '--junit-xml', log_path]
-        else:
-            arguments = ['-x', '-v', '-p no:warnings',
-                         '--log-level=WARN', '--junit-xml', log_path]
-
-        if no_exit_first:
-            arguments.remove('-x')
-
-        if mark:
-            arguments.append('-m "{}"'.format(mark))
-
-        arguments.extend(test_paths)
-        if parallel:
-            arguments += ['-n', 'auto']
-        if last_failed:
-            arguments.append('--lf')
-        if pytest_args:
-            arguments += pytest_args
-        cmd = 'python -m pytest {}'.format(' '.join(arguments))
-        logger.info('Running: %s', cmd)
-        return call(cmd)
-
-    return _run
-
-
-def run_tests(tests, test_index, mode, base_path, xml_file, json_file, in_series=False,
-              run_live=False, profile=None, last_failed=False, no_exit_first=False, mark=None, pytest_args=None):
-
-    heading('Run Tests')
-
-    # process file path
-    if not xml_file.startswith(mode):
-        xml_file = "{}_{}".format(mode, xml_file)
-    if not json_file.startswith(mode):
-        json_file = "{}_{}".format(mode, json_file)
-    xml_path = os.path.realpath(os.path.join(base_path, xml_file))
-    json_path = os.path.realpath(os.path.join(base_path, json_file))
-    pytest_args.append("--json-report-file {}".format(json_path))
-    logger.info("junit/xml report file full path: {}".format(xml_path))
-    logger.info("json report file full path: {}".format(json_path))
-
-    # process environment variables
-    if run_live:
-        logger.warning('RUNNING TESTS LIVE')
-        os.environ[ENV_VAR_TEST_LIVE] = 'True'
-
-    def _find_test(index, name):
-        name_comps = name.split('.')
-        num_comps = len(name_comps)
-        key_error = KeyError()
-
-        for i in range(num_comps):
-            check_name = '.'.join(name_comps[(-1 - i):])
-            try:
-                match = index[check_name]
-                if check_name != name:
-                    logger.info(
-                        "Test found using just '%s'. The rest of the name was ignored.", check_name)
-                return match
-            except KeyError as ex:
-                key_error = ex
-                continue
-        raise key_error
-
-    # lookup test paths from index
-    test_paths = []
-    for t in tests:
-        try:
-            test_path = os.path.normpath(_find_test(test_index, t))
-            test_paths.append(test_path)
-        except KeyError:
-            logger.warning("'%s' not found.", t)
-            continue
-
-    # Tests have been collected. Now run them.
-    exit_code = 0
-    if not test_paths:
-        logger.warning('No tests selected to run.')
-        return exit_code
-
-    with ProfileContext(profile):
-        runner = get_test_runner(parallel=not in_series,
-                                 log_path=xml_path,
-                                 last_failed=last_failed,
-                                 no_exit_first=no_exit_first,
-                                 mark=mark)
-        exit_code = runner(test_paths=test_paths, pytest_args=pytest_args)
-
-    return 0 if not exit_code else 1
diff --git a/src/aks-preview/az_aks_tool/tests/__init__.py b/src/aks-preview/az_aks_tool/tests/__init__.py
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/src/aks-preview/az_aks_tool/tests/test_cli.py b/src/aks-preview/az_aks_tool/tests/test_cli.py
deleted file mode 100644
index 5b9000b7af6..00000000000
--- a/src/aks-preview/az_aks_tool/tests/test_cli.py
+++ /dev/null
@@ -1,22 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------
-
-import os
-import unittest
-
-import az_aks_tool.utils as utils
-import az_aks_tool.cli as cli
-
-THIS_FILE = os.path.abspath(__file__)
-THIS_DIR = os.path.dirname(THIS_FILE)
-PARENT_DIR = os.path.dirname(THIS_DIR)
-
-class CliTestCase(unittest.TestCase):
-
-    def test_get_cli_mod_data(self):
-        pass
-
-    def test_get_cli_test_index(self):
-        pass
diff --git a/src/aks-preview/az_aks_tool/tests/test_ext.py b/src/aks-preview/az_aks_tool/tests/test_ext.py
deleted file mode 100644
index ec194f16641..00000000000
--- a/src/aks-preview/az_aks_tool/tests/test_ext.py
+++ /dev/null
@@ -1,22 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------
-
-import os
-import unittest
-
-import az_aks_tool.utils as utils
-import az_aks_tool.ext as ext
-
-THIS_FILE = os.path.abspath(__file__)
-THIS_DIR = os.path.dirname(THIS_FILE)
-PARENT_DIR = os.path.dirname(THIS_DIR)
-
-class ExtTestCase(unittest.TestCase):
-
-    def test_get_ext_mod_data(self):
-        pass
-
-    def test_get_ext_test_index(self):
-        pass
diff --git a/src/aks-preview/az_aks_tool/tests/test_filter.py b/src/aks-preview/az_aks_tool/tests/test_filter.py
deleted file mode 100644
index d9da9e6a624..00000000000
--- a/src/aks-preview/az_aks_tool/tests/test_filter.py
+++ /dev/null
@@ -1,121 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------
-
-import os
-import unittest
-
-import az_aks_tool.utils as utils
-import az_aks_tool.filter as custom_filter
-
-THIS_FILE = os.path.abspath(__file__)
-THIS_DIR = os.path.dirname(THIS_FILE)
-PARENT_DIR = os.path.dirname(THIS_DIR)
-
-
-class FilterTestCase(unittest.TestCase):
-
-    def test_extract_file_class_pairs(self):
-        test_cases = ["abc", "a.b", "a.b.c"]
-        s = custom_filter.extract_file_class_pairs(test_cases)
-        self.assertEqual(s, [("a", "b")])
-
-    def test_filter_valid_file_class_pairs(self):
-        pairs = [("a", "b"), ("c", "d"), ("e", "f")]
-        test_index = {
-            "a": {
-                "b": ["aaa", "bbb"]
-            },
-            "x": {
-                "y": ["xxx", "yyy"]
-            }
-        }
-        s = custom_filter.filter_valid_file_class_pairs(pairs, test_index)
-        self.assertEqual(s, [("a", "b")])
-
-    def test_get_all_values_from_nested_dict(self):
-        d = {
-            "a": {
-                "b": ["aaa", "bbb"]
-            },
-            "x": ["xxx"]
-        }
-        s = list(custom_filter.get_all_values_from_nested_dict(d))
-        self.assertEqual(s, [["aaa", "bbb"], ["xxx"]])
-
-    def test_flatten_nested_list(self):
-        lis = [["aaa", "bbb"], ["xxx"]]
-        s = list(custom_filter.flatten_nested_list(lis))
-        self.assertEqual(s, ["aaa", "bbb", "xxx"])
-
-    def test_filter_valid_test_cases(self):
-        test_cases = ["abc", "a.b", "a.b.c", "aaa", "yyy"]
-        test_index = {
-            "a": {
-                "b": ["aaa", "bbb"]
-            },
-            "x": {
-                "y": ["xxx", "yyy"]
-            }
-        }
-        s = custom_filter.filter_valid_test_cases(test_cases, test_index)
-        self.assertEqual(s, ["aaa", "yyy"])
-
-    def test_get_test_cases(self):
-        test_index = {
-            "test_validators": {
-                "TestValidateIPRanges": ["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces", "test_disable_authorized_ip_ranges", "test_local_ip_address", "test_invalid_ip", "test_IPv6"],
-                "TestClusterAutoscalerParamsValidators": ["test_empty_key_empty_value", "test_non_empty_key_empty_value", "test_two_empty_keys_empty_value", "test_one_empty_key_in_pair_one_non_empty", "test_invalid_key", "test_valid_parameters"],
-                "TestSubnetId": ["test_invalid_subnet_id", "test_valid_vnet_subnet_id", "test_none_vnet_subnet_id", "test_empty_vnet_subnet_id"]
-            }
-        }
-        matrix = utils.get_test_matrix(
-            os.path.join(THIS_DIR, "testdata.json"))
-        base = test_index["test_validators"]
-        s1 = sorted(custom_filter.get_test_cases(test_index, matrix, []))
-        t1 = sorted((base["TestValidateIPRanges"] +
-                     base["TestClusterAutoscalerParamsValidators"]))
-        s2 = sorted(custom_filter.get_test_cases(test_index, matrix, [
-            "test_valid_vnet_subnet_id", "abc"]))
-        t2 = sorted((base["TestValidateIPRanges"] + base["TestClusterAutoscalerParamsValidators"] +
-                     ["test_valid_vnet_subnet_id"]))
-        s3 = sorted(custom_filter.get_test_cases(test_index, matrix, [
-            "test_validators.TestSubnetId"]))
-        t3 = sorted((base["TestValidateIPRanges"] +
-                     base["TestClusterAutoscalerParamsValidators"] + base["TestSubnetId"]))
-        self.assertEqual(s1, t1)
-        self.assertEqual(s2, t2)
-        self.assertEqual(s3, t3)
-
-    def test_get_exclude_test_cases(self):
-        test_index = {
-            "test_validators": {
-                "TestValidateIPRanges": ["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces", "test_disable_authorized_ip_ranges", "test_local_ip_address", "test_invalid_ip", "test_IPv6"],
-                "TestClusterAutoscalerParamsValidators": ["test_empty_key_empty_value", "test_non_empty_key_empty_value", "test_two_empty_keys_empty_value", "test_one_empty_key_in_pair_one_non_empty", "test_invalid_key", "test_valid_parameters"],
-                "TestSubnetId": ["test_invalid_subnet_id", "test_valid_vnet_subnet_id", "test_none_vnet_subnet_id", "test_empty_vnet_subnet_id"]
-            }
-        }
-        matrix = utils.get_test_matrix(
-            os.path.join(THIS_DIR, "testdata.json"))
-        base = test_index["test_validators"]
-        s1 = sorted(custom_filter.get_exclude_test_cases(
-            test_index, matrix, []))
-        t1 = sorted(["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces",
-                    "test_disable_authorized_ip_ranges"] + base["TestClusterAutoscalerParamsValidators"])
-        s2 = sorted(custom_filter.get_exclude_test_cases(test_index,
-                    matrix, ["iprange"]))
-        t2 = sorted(["test_simultaneous_allow_and_disallow_with_spaces",
-                    "test_simultaneous_enable_and_disable_with_spaces", "test_disable_authorized_ip_ranges"])
-        s3 = sorted(custom_filter.get_exclude_test_cases(test_index,
-                    matrix, ["default", "test_invalid_subnet_id"]))
-        t3 = sorted(["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces",
-                    "test_disable_authorized_ip_ranges"] + base["TestClusterAutoscalerParamsValidators"] + ["test_invalid_subnet_id"])
-        s4 = sorted(custom_filter.get_exclude_test_cases(test_index, matrix, [
-            "test_valid_vnet_subnet_id", "abc", "test_validators.TestClusterAutoscalerParamsValidators"]))
-        t4 = sorted(
-            (base["TestClusterAutoscalerParamsValidators"] + ["test_valid_vnet_subnet_id"]))
-        self.assertEqual(s1, t1)
-        self.assertEqual(s2, t2)
-        self.assertEqual(s3, t3)
-        self.assertEqual(s4, t4)
diff --git a/src/aks-preview/az_aks_tool/tests/test_index.py b/src/aks-preview/az_aks_tool/tests/test_index.py
deleted file mode 100644
index 68b5816db2f..00000000000
--- a/src/aks-preview/az_aks_tool/tests/test_index.py
+++ /dev/null
@@ -1,34 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------
-
-import os
-import unittest
-
-import az_aks_tool.utils as utils
-import az_aks_tool.index as index
-
-THIS_FILE = os.path.abspath(__file__)
-THIS_DIR = os.path.dirname(THIS_FILE)
-PARENT_DIR = os.path.dirname(THIS_DIR)
-
-class IndexTestCase(unittest.TestCase):
-
-    def test_get_repo_path(self):
-        pass
-
-    def test_find_files(self):
-        pass
-
-    def test_get_name_index(self):
-        pass
-
-    def test_get_path_table(self):
-        pass
-
-    def test_discover_module_tests(self):
-        pass
-
-    def build_test_index(self):
-        pass
diff --git a/src/aks-preview/az_aks_tool/tests/test_log.py b/src/aks-preview/az_aks_tool/tests/test_log.py
deleted file mode 100644
index 4caafc341ba..00000000000
--- a/src/aks-preview/az_aks_tool/tests/test_log.py
+++ /dev/null
@@ -1,34 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------
-
-import logging
-import os
-import unittest
-
-import az_aks_tool.log as log
-
-THIS_FILE = os.path.abspath(__file__)
-THIS_DIR = os.path.dirname(THIS_FILE)
-PARENT_DIR = os.path.dirname(THIS_DIR)
-
-
-class LogTestCase(unittest.TestCase):
-
-    def test_parse_module_name(self):
-        root_module_name = log.parse_module_name(levels=1)
-        error_module_name = log.parse_module_name(levels=5)
-        self.assertEqual(root_module_name, "az_aks_tool")
-        self.assertEqual(error_module_name, None)
-
-    def test_setup_logging(self):
-        log.setup_logging("unittest", "unittest_log.log")
-        logger = logging.getLogger("unittest.test_setup_logging")
-        logger.debug("test setup logging")
-        logger.info("test setup logging")
-        logger.warning("test setup logging")
-        f = open("unittest_log.log", "r")
-        raw_logs = f.readlines()
-        f.close()
-        self.assertEqual(len(raw_logs), 3)
diff --git a/src/aks-preview/az_aks_tool/tests/test_main.py b/src/aks-preview/az_aks_tool/tests/test_main.py
deleted file mode 100644
index 36f4becd0f4..00000000000
--- a/src/aks-preview/az_aks_tool/tests/test_main.py
+++ /dev/null
@@ -1,21 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------
-
-import logging
-import os
-import unittest
-
-import az_aks_tool.main as main
-
-THIS_FILE = os.path.abspath(__file__)
-THIS_DIR = os.path.dirname(THIS_FILE)
-PARENT_DIR = os.path.dirname(THIS_DIR)
-
-
-class MainTestCase(unittest.TestCase):
-
-    def test_init_argparse(self):
-        args = main.init_argparse(["-p", "./"])
-        self.assertEqual(args.report_path, "./")
diff --git a/src/aks-preview/az_aks_tool/tests/test_run.py b/src/aks-preview/az_aks_tool/tests/test_run.py
deleted file mode 100644
index d69375d491d..00000000000
--- a/src/aks-preview/az_aks_tool/tests/test_run.py
+++ /dev/null
@@ -1,33 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------
-
-import os
-import unittest
-
-import az_aks_tool.utils as utils
-import az_aks_tool.cli as cli
-
-THIS_FILE = os.path.abspath(__file__)
-THIS_DIR = os.path.dirname(THIS_FILE)
-PARENT_DIR = os.path.dirname(THIS_DIR)
-
-class RunTestCase(unittest.TestCase):
-
-    def test_current_profile(self):
-        pass
-
-    def test_call(self):
-        pass
-
-    def test_cmd(self):
-        pass
-
-    def test_get_test_runner(self):
-        pass
-
-    def run_tests(self):
-        pass
-    
-
diff --git a/src/aks-preview/az_aks_tool/tests/test_utils.py b/src/aks-preview/az_aks_tool/tests/test_utils.py
deleted file mode 100644
index 6bef0b49a0f..00000000000
--- a/src/aks-preview/az_aks_tool/tests/test_utils.py
+++ /dev/null
@@ -1,54 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------
-
-import os
-import unittest
-
-import az_aks_tool.utils as utils
-
-THIS_FILE = os.path.abspath(__file__)
-THIS_DIR = os.path.dirname(THIS_FILE)
-PARENT_DIR = os.path.dirname(THIS_DIR)
-
-
-class UtilsTestCase(unittest.TestCase):
-
-    def test_check_file_existence(self):
-        s1 = utils.check_file_existence(None)
-        s2 = utils.check_file_existence(THIS_DIR)
-        s3 = utils.check_file_existence(THIS_FILE)
-        s4 = utils.check_file_existence(
-            os.path.join(THIS_DIR, "testdata.json"))
-        self.assertEqual(s1, False)
-        self.assertEqual(s2, False)
-        self.assertEqual(s3, True)
-        self.assertEqual(s4, True)
-
-    def test_get_test_matrix(self):
-        s = utils.get_test_matrix(os.path.join(THIS_DIR, "testdata.json"))
-        self.assertEqual(len(s), 2)     # number of keys
-
-    def test_get_filted_test_cases(self):
-        test_cases = ["a", "b", "c", "d"]
-        s1 = utils.get_filted_test_cases(test_cases, [])
-        t1 = ["a", "b", "c", "d"]
-        s2 = utils.get_filted_test_cases(test_cases, ["a", "x"])
-        t2 = ["b", "c", "d"]
-        self.assertEqual(s1, t1)
-        self.assertEqual(s2, t2)
-    
-    def test_add_qualified_prefix(self):
-        test_cases = ["a", "b", "c"]
-        s = utils.add_qualified_prefix(test_cases, "p")
-        self.assertEqual(s, ["p.a", "p.b", "p.c"])
-
-    def test_get_fully_qualified_test_cases(self):
-        pass
-
-    def test_heading(self):
-        pass
-
-    def test_extract_module_name(self):
-        pass
diff --git a/src/aks-preview/az_aks_tool/tests/testdata.json b/src/aks-preview/az_aks_tool/tests/testdata.json
deleted file mode 100644
index 221e6291e3e..00000000000
--- a/src/aks-preview/az_aks_tool/tests/testdata.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-    "coverage": {
-        "test_validators": [
-            "TestValidateIPRanges",
-            "TestClusterAutoscalerParamsValidators"
-        ]
-    },
-    "exclude": {
-        "iprange": [
-            "test_simultaneous_allow_and_disallow_with_spaces",
-            "test_simultaneous_enable_and_disable_with_spaces",
-            "test_disable_authorized_ip_ranges"
-        ],
-        "test_validators": [
-            "TestClusterAutoscalerParamsValidators"
-        ]
-    }
-}
\ No newline at end of file
diff --git a/src/aks-preview/az_aks_tool/utils.py b/src/aks-preview/az_aks_tool/utils.py
deleted file mode 100644
index 6607d026a1b..00000000000
--- a/src/aks-preview/az_aks_tool/utils.py
+++ /dev/null
@@ -1,92 +0,0 @@
-# --------------------------------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License. See License.txt in the project root for license information.
-# --------------------------------------------------------------------------------------------
-
-import json
-import os
-import sys
-import re
-import logging
-import pathlib
-
-import az_aks_tool.filter as custom_filter
-logger = logging.getLogger(__name__)
-
-
-def create_directory(dir_path):
-    if dir_path:
-        if not os.path.isdir(dir_path):
-            print("Directory '{}' not exist, creating...".format(dir_path))
-            pathlib.Path(dir_path).mkdir(parents=True, exist_ok=True)
-    else:
-        print("Invalid dir path: '{}'".format(dir_path))
-
-
-def check_file_existence(file_path):
-    if file_path is not None and os.path.isfile(file_path):
-        return True
-    return False
-
-
-def get_test_matrix(matrix_file_path):
-    test_matrix = {}
-    if check_file_existence(matrix_file_path):
-        json_file = open(matrix_file_path, 'r')
-        test_matrix = json.load(json_file)
-        json_file.close()
-    else:
-        logger.warning("Matrix file '{}' not exists!".format(matrix_file_path))
-    return test_matrix
-
-
-def get_filted_test_cases(test_cases, exclude_test_cases):
-    filtered_test_cases = [
-        x for x in test_cases if x not in exclude_test_cases]
-    logger.info("Find {} cases, exclude {} cases, finally get {} cases!".format(
-        len(test_cases), len(exclude_test_cases), len(filtered_test_cases)))
-    return filtered_test_cases
-
-
-def add_qualified_prefix(test_cases, prefix):
-    decorated_test_cases = ["{}.{}".format(prefix, x) for x in test_cases]
-    return decorated_test_cases
-
-
-def get_fully_qualified_test_cases(test_index, matrix_file_path, mod_name, extra_coverage=None, extra_filter=None):
-    qualified_test_cases = []
-    matrix = get_test_matrix(matrix_file_path)
-    test_cases = custom_filter.get_test_cases(
-        test_index, matrix, extra_coverage)
-    exclude_test_cases = custom_filter.get_exclude_test_cases(test_index,
-                                                              matrix, extra_filter)
-    filtered_test_cases = get_filted_test_cases(
-        test_cases, exclude_test_cases)
-    # add prefix
-    qualified_test_cases = add_qualified_prefix(
-        filtered_test_cases, mod_name)
-    return qualified_test_cases
-
-
-def heading(txt):
-    """ Create standard heading to stderr """
-    line_len = len(txt) + 4
-    print('\n' + '=' * line_len, file=sys.stderr)
-    print('| {} |'.format(txt), file=sys.stderr)
-    print('=' * line_len + '\n', file=sys.stderr)
-
-
-def extract_module_name(path):
-    _CORE_NAME_REGEX = re.compile(
-        r'azure-cli-(?P<name>[^/\\]+)[/\\]azure[/\\]cli')
-    _MOD_NAME_REGEX = re.compile(
-        r'azure-cli[/\\]azure[/\\]cli[/\\]command_modules[/\\](?P<name>[^/\\]+)')
-    _EXT_NAME_REGEX = re.compile(r'.*(?P<name>azext_[^/\\]+).*')
-
-    for expression in [_MOD_NAME_REGEX, _CORE_NAME_REGEX, _EXT_NAME_REGEX]:
-        match = re.search(expression, path)
-        if not match:
-            continue
-        return match.groupdict().get('name')
-    raise Exception(
-        'unexpected error: unable to extract name from path: {}'.format(path))

From 0179ee0485698cd44b28a8b5c341e3665d9b6980 Mon Sep 17 00:00:00 2001
From: Fuming Zhang <fumingzhang@microsoft.com>
Date: Thu, 24 Jun 2021 11:19:10 +0800
Subject: [PATCH 06/10] update .gitignore

---
 src/aks-preview/azcli_aks_live_test/.gitignore       |  3 +--
 .../configs/cli_matrix_default.json                  | 12 ++++++++++++
 2 files changed, 13 insertions(+), 2 deletions(-)
 create mode 100644 src/aks-preview/azcli_aks_live_test/configs/cli_matrix_default.json

diff --git a/src/aks-preview/azcli_aks_live_test/.gitignore b/src/aks-preview/azcli_aks_live_test/.gitignore
index 819270b0650..444e542f053 100644
--- a/src/aks-preview/azcli_aks_live_test/.gitignore
+++ b/src/aks-preview/azcli_aks_live_test/.gitignore
@@ -1,5 +1,4 @@
 env.list
 *.json
 *.xml
-!ext_matrix_default.json
-!testdata.json
+!configs/*
diff --git a/src/aks-preview/azcli_aks_live_test/configs/cli_matrix_default.json b/src/aks-preview/azcli_aks_live_test/configs/cli_matrix_default.json
new file mode 100644
index 00000000000..33c36b35b85
--- /dev/null
+++ b/src/aks-preview/azcli_aks_live_test/configs/cli_matrix_default.json
@@ -0,0 +1,12 @@
+{
+    "coverage": {
+        "test_aks_commands": [
+            "AzureKubernetesServiceScenarioTest"
+        ]
+    },
+    "exclude": {
+        "need additional feature": [
+            "test_managed_aad_enable_azure_rbac"
+        ]
+    }
+}
\ No newline at end of file

From a1f360abff7ad676d56543a9d3d3e6efe09f19a7 Mon Sep 17 00:00:00 2001
From: Fuming Zhang <fumingzhang@microsoft.com>
Date: Thu, 24 Jun 2021 12:08:28 +0800
Subject: [PATCH 07/10] * fix missing tool * update cli scripts

---
 .../azcli_aks_live_test/scripts/setup_venv.sh |  7 +++
 .../scripts/test_cli_live.sh                  | 47 +++++++++++++------
 .../scripts/test_cli_unit.sh                  | 34 ++++++++++----
 .../scripts/test_ext_live.sh                  |  4 --
 .../scripts/test_raw_cases.sh                 |  7 +++
 5 files changed, 72 insertions(+), 27 deletions(-)
 create mode 100644 src/aks-preview/azcli_aks_live_test/scripts/test_raw_cases.sh

diff --git a/src/aks-preview/azcli_aks_live_test/scripts/setup_venv.sh b/src/aks-preview/azcli_aks_live_test/scripts/setup_venv.sh
index f4e61c1c9ff..ba0448ce643 100755
--- a/src/aks-preview/azcli_aks_live_test/scripts/setup_venv.sh
+++ b/src/aks-preview/azcli_aks_live_test/scripts/setup_venv.sh
@@ -3,6 +3,10 @@
 # check var
 [[ -z "${PYTHON_VERSION}" ]] && (echo "PYTHON_VERSION is empty"; exit 1)
 
+patchImageTools(){
+    apt install -y curl
+}
+
 setupVenv(){
     # delete existing venv
     deactivate || true
@@ -111,6 +115,9 @@ if [[ -n ${1} ]]; then
     set -o pipefail
     set -o xtrace
 
+    # install missing tools in the image
+    patchImageTools
+
     # create new venv if second arg is not "n"
     new_venv=${2:-"y"}
     if [[ ! ${new_venv} == "n" ]]; then
diff --git a/src/aks-preview/azcli_aks_live_test/scripts/test_cli_live.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_cli_live.sh
index 412e3c6702a..ebd54f1a0c5 100755
--- a/src/aks-preview/azcli_aks_live_test/scripts/test_cli_live.sh
+++ b/src/aks-preview/azcli_aks_live_test/scripts/test_cli_live.sh
@@ -14,32 +14,51 @@ set -o xtrace
 [[ -z "${TEST_MODE}" ]] && (echo "TEST_MODE is empty"; exit 1)
 [[ -z "${PARALLELISM}" ]] && (echo "PARALLELISM is empty"; exit 1)
 
-# dir
-pwd
-ls -alh
-
 # activate virtualenv
 source azEnv/bin/activate
 
-# remove extension
-echo "Remove existing aks-preview extension (if any)"
-if az extension remove --name aks-preview || azdev extension remove aks-preview; then
-    deactivate
-    source azEnv/bin/activate
+# remove aks-preview
+source ./scripts/setup_venv.sh
+removeAKSPreview
+
+# prepare run flags
+run_flags="-c --no-exitfirst --report-path ./reports --reruns 3 --capture=sys"
+# parallel
+if [ ${PARALLELISM} -ge 2 ]; then
+    run_flags+=" -j ${PARALLELISM}"
+else
+    run_flags+=" -s"
+fi
+# cli matrix
+if [[ -n ${CLI_TEST_MATRIX} ]]; then
+    run_flags+=" -cm ./configs/${CLI_TEST_MATRIX}"
+fi
+# cli extra filter
+if [[ -n ${CLI_TEST_FILTER} ]]; then
+    run_flags+=" -cf ${CLI_TEST_FILTER}"
+fi
+# cli extra coverage
+if [[ -n ${CLI_TEST_COVERAGE} ]]; then
+    run_flags+=" -cc ${CLI_TEST_COVERAGE}"
 fi
 
-# test cli
+# recording test
 if [[ ${TEST_MODE} == "record" || ${TEST_MODE} == "all" ]]; then
     echo "Test in record mode!"
-    azdev test acs --no-exitfirst --xml-path cli_result.xml --discover -a "-n ${PARALLELISM} --json-report --json-report-file=cli_report.json --reruns 3 --capture=sys"
-    cp *cli_report.json *cli_result.xml reports/
+    run_flags+=" --json-report-file=cli_report.json"
+    run_flags+=" --xml-file=cli_result.xml"
+    echo "run flags: ${run_flags}"
+    azaks ${run_flags}
 fi
 
+# live test
 if [[ ${TEST_MODE} == "live" || ${TEST_MODE} == "all" ]]; then
     echo "Test in live mode!"
     az login --service-principal -u ${AZCLI_ALT_CLIENT_ID} -p ${AZCLI_ALT_CLIENT_SECRET} -t ${TENANT_ID}
     az account set -s ${AZCLI_ALT_SUBSCRIPTION_ID}
     az account show
-    azdev test acs --live --no-exitfirst --xml-path cli_live_result.xml --discover -a "-n ${PARALLELISM} --json-report --json-report-file=cli_live_report.json --reruns 3 --capture=sys"
-    cp *cli_live_report.json *cli_live_result.xml reports/
+    run_flags+=" -l --json-report-file=cli_live_report.json"
+    run_flags+=" --xml-file=cli_live_result.xml"
+    echo "run flags: ${run_flags}"
+    azaks ${run_flags}
 fi
diff --git a/src/aks-preview/azcli_aks_live_test/scripts/test_cli_unit.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_cli_unit.sh
index 1c70f2f4c6b..3bad35afe1a 100755
--- a/src/aks-preview/azcli_aks_live_test/scripts/test_cli_unit.sh
+++ b/src/aks-preview/azcli_aks_live_test/scripts/test_cli_unit.sh
@@ -6,18 +6,34 @@ set -o nounset
 set -o pipefail
 set -o xtrace
 
-# dir
-pwd
-ls -alh
+# const
+acs_base_dir="azure-cli/src/azure-cli/azure/cli/command_modules/acs"
 
 # activate virtualenv
 source azEnv/bin/activate
 
-# remove extension
-echo "Remove existing aks-preview extension (if any)"
-if az extension remove --name aks-preview || azdev extension remove aks-preview; then
-    deactivate
-    source azEnv/bin/activate
+# remove aks-preview
+source ./scripts/setup_venv.sh
+removeAKSPreview
+
+# unit test & coverage report
+acs_unit_test_failed=""
+pushd ${acs_base_dir}
+# clean existing coverage report
+(coverage combine || true) && (coverage erase || true)
+# perform unit test with module 'unittest'
+# since recording test (performed in test_cli_live.sh) is based on module 'pytest', so skip here
+# coverage run --source=. --omit=*/tests/* -p -m pytest
+if ! coverage run --source=. --omit=*/tests/* -p -m unittest discover; then
+    acs_unit_test_failed="true"
 fi
+# generate & copy coverage report
+coverage combine && coverage json -o coverage_acs.json
+coverage report -m
+popd
+cp ${acs_base_dir}/coverage_acs.json reports/
 
-echo "Implementing, pass for now!"
+if [[ ${acs_unit_test_failed} == "true" ]]; then
+    echo "Unit test failed!"
+    exit 1
+fi
diff --git a/src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh
index 16f07b4977e..9a8bf0cef4f 100755
--- a/src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh
+++ b/src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh
@@ -32,10 +32,6 @@ if [ ${PARALLELISM} -ge 2 ]; then
 else
     run_flags+=" -s"
 fi
-# test cases
-if [[ -n ${TEST_CASES} ]]; then
-    run_flags+=" -t ${TEST_CASES}"
-fi
 # ext matrix
 if [[ -n ${EXT_TEST_MATRIX} ]]; then
     run_flags+=" -em ./configs/${EXT_TEST_MATRIX}"
diff --git a/src/aks-preview/azcli_aks_live_test/scripts/test_raw_cases.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_raw_cases.sh
new file mode 100644
index 00000000000..8f132ac633b
--- /dev/null
+++ b/src/aks-preview/azcli_aks_live_test/scripts/test_raw_cases.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+# bash options
+set -o errexit
+set -o nounset
+set -o pipefail
+set -o xtrace

From f9f3ab0e8ce3ae11afdacd66b144512eea98a70b Mon Sep 17 00:00:00 2001
From: Fuming Zhang <fumingzhang@microsoft.com>
Date: Thu, 24 Jun 2021 12:22:23 +0800
Subject: [PATCH 08/10] update create venv

---
 src/aks-preview/azcli_aks_live_test/scripts/setup_venv.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/aks-preview/azcli_aks_live_test/scripts/setup_venv.sh b/src/aks-preview/azcli_aks_live_test/scripts/setup_venv.sh
index ba0448ce643..34b867b010e 100755
--- a/src/aks-preview/azcli_aks_live_test/scripts/setup_venv.sh
+++ b/src/aks-preview/azcli_aks_live_test/scripts/setup_venv.sh
@@ -119,10 +119,12 @@ if [[ -n ${1} ]]; then
     patchImageTools
 
     # create new venv if second arg is not "n"
-    new_venv=${2:-"y"}
-    if [[ ! ${new_venv} == "n" ]]; then
+    new_venv=${2:-"n"}
+    if [[ ${new_venv} == "y" ]]; then
         echo "Create new venv!"
         setupVenv
+    else
+        source azEnv/bin/activate
     fi
 
     if [[ ${1} == "build" ]]; then

From 25bcead2b6b8b394280ef54d86d74af361ee4b0b Mon Sep 17 00:00:00 2001
From: Fuming Zhang <fumingzhang@microsoft.com>
Date: Thu, 24 Jun 2021 12:50:25 +0800
Subject: [PATCH 09/10] update live test var check

---
 src/aks-preview/azcli_aks_live_test/scripts/test_cli_live.sh | 3 +++
 src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh | 1 -
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/aks-preview/azcli_aks_live_test/scripts/test_cli_live.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_cli_live.sh
index ebd54f1a0c5..751c916bf59 100755
--- a/src/aks-preview/azcli_aks_live_test/scripts/test_cli_live.sh
+++ b/src/aks-preview/azcli_aks_live_test/scripts/test_cli_live.sh
@@ -13,6 +13,9 @@ set -o xtrace
 [[ -z "${AZCLI_ALT_CLIENT_SECRET}" ]] && (echo "AZCLI_ALT_CLIENT_SECRET is empty"; exit 1)
 [[ -z "${TEST_MODE}" ]] && (echo "TEST_MODE is empty"; exit 1)
 [[ -z "${PARALLELISM}" ]] && (echo "PARALLELISM is empty"; exit 1)
+[[ -z "${CLI_TEST_MATRIX}" ]] && (echo "CLI_TEST_MATRIX is empty")
+[[ -z "${CLI_TEST_FILTER}" ]] && (echo "CLI_TEST_FILTER is empty")
+[[ -z "${CLI_TEST_COVERAGE}" ]] && (echo "CLI_TEST_COVERAGE is empty")
 
 # activate virtualenv
 source azEnv/bin/activate
diff --git a/src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh
index 9a8bf0cef4f..f90d0170f3b 100755
--- a/src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh
+++ b/src/aks-preview/azcli_aks_live_test/scripts/test_ext_live.sh
@@ -13,7 +13,6 @@ set -o xtrace
 [[ -z "${AZCLI_ALT_CLIENT_SECRET}" ]] && (echo "AZCLI_ALT_CLIENT_SECRET is empty"; exit 1)
 [[ -z "${TEST_MODE}" ]] && (echo "TEST_MODE is empty"; exit 1)
 [[ -z "${PARALLELISM}" ]] && (echo "PARALLELISM is empty"; exit 1)
-[[ -z "${TEST_CASES}" ]] && (echo "TEST_CASES is empty")
 [[ -z "${EXT_TEST_MATRIX}" ]] && (echo "EXT_TEST_MATRIX is empty")
 [[ -z "${EXT_TEST_FILTER}" ]] && (echo "EXT_TEST_FILTER is empty")
 [[ -z "${EXT_TEST_COVERAGE}" ]] && (echo "EXT_TEST_COVERAGE is empty")

From d6d66c354a4154840e7dbe95af7bcf381ebe987c Mon Sep 17 00:00:00 2001
From: Fuming Zhang <fumingzhang@microsoft.com>
Date: Thu, 24 Jun 2021 12:57:29 +0800
Subject: [PATCH 10/10] fix unit test report path

---
 src/aks-preview/azcli_aks_live_test/scripts/test_cli_unit.sh | 5 +++--
 src/aks-preview/azcli_aks_live_test/scripts/test_ext_unit.sh | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/src/aks-preview/azcli_aks_live_test/scripts/test_cli_unit.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_cli_unit.sh
index 3bad35afe1a..b32a46be1c3 100755
--- a/src/aks-preview/azcli_aks_live_test/scripts/test_cli_unit.sh
+++ b/src/aks-preview/azcli_aks_live_test/scripts/test_cli_unit.sh
@@ -28,10 +28,11 @@ if ! coverage run --source=. --omit=*/tests/* -p -m unittest discover; then
     acs_unit_test_failed="true"
 fi
 # generate & copy coverage report
-coverage combine && coverage json -o coverage_acs.json
+coverage combine
 coverage report -m
+coverage json -o coverage_acs.json
 popd
-cp ${acs_base_dir}/coverage_acs.json reports/
+mkdir -p reports/ && cp ${acs_base_dir}/coverage_acs.json reports/
 
 if [[ ${acs_unit_test_failed} == "true" ]]; then
     echo "Unit test failed!"
diff --git a/src/aks-preview/azcli_aks_live_test/scripts/test_ext_unit.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_ext_unit.sh
index e1daa0236fa..c76feba0607 100755
--- a/src/aks-preview/azcli_aks_live_test/scripts/test_ext_unit.sh
+++ b/src/aks-preview/azcli_aks_live_test/scripts/test_ext_unit.sh
@@ -27,10 +27,11 @@ if ! coverage run --source=. --omit=*/vendored_sdks/*,*/tests/* -p -m unittest d
     azext_aks_preview_unit_test_failed="true"
 fi
 # generate & copy coverage report
-coverage combine && coverage json -o coverage_azext_aks_preview.json
+coverage combine
 coverage report -m
+coverage json -o coverage_azext_aks_preview.json
 popd
-cp ${aks_preview_base_dir}/coverage_azext_aks_preview.json reports/
+mkdir -p reports/ && cp ${aks_preview_base_dir}/coverage_azext_aks_preview.json reports/
 
 if [[ ${azext_aks_preview_unit_test_failed} == "true" ]]; then
     echo "Unit test failed!"