{AKS} Update live test pipeline (Azure#4171)

* update pipeline * Trigger Build * fix sh * Trigger Build
AzureArcForKubernetes · Dec 1, 2021 · fa998ce · fa998ce
1 parent 0c092fe
commit fa998ce
Show file tree

Hide file tree

Showing 6 changed files with 281 additions and 85 deletions.
diff --git a/src/aks-preview/azcli_aks_live_test/scripts/test_cli_live.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_cli_live.sh
@@ -7,15 +7,27 @@ set -o pipefail
 set -o xtrace
 
 # check var
+[[ -z "${WIKI_LINK}" ]] && (echo "WIKI_LINK is empty"; exit 1)
 [[ -z "${TENANT_ID}" ]] && (echo "TENANT_ID is empty"; exit 1)
 [[ -z "${AZCLI_ALT_SUBSCRIPTION_ID}" ]] && (echo "AZCLI_ALT_SUBSCRIPTION_ID is empty"; exit 1)
 [[ -z "${AZCLI_ALT_CLIENT_ID}" ]] && (echo "AZCLI_ALT_CLIENT_ID is empty"; exit 1)
 [[ -z "${AZCLI_ALT_CLIENT_SECRET}" ]] && (echo "AZCLI_ALT_CLIENT_SECRET is empty"; exit 1)
+[[ -z "${IGNORE_EXIT_CODE}" ]] && (echo "IGNORE_EXIT_CODE is empty")
 [[ -z "${TEST_MODE}" ]] && (echo "TEST_MODE is empty"; exit 1)
-[[ -z "${PARALLELISM}" ]] && (echo "PARALLELISM is empty"; exit 1)
+[[ -z "${PARALLELISM}" ]] && (echo "PARALLELISM is empty")
+[[ -z "${RERUNS}" ]] && (echo "RERUNS is empty")
+[[ -z "${NON_BOXED}" ]] && (echo "NON_BOXED is empty")
+[[ -z "${LOG_LEVEL}" ]] && (echo "LOG_LEVEL is empty")
+[[ -z "${CAPTURE}" ]] && (echo "CAPTURE is empty")
+[[ -z "${NO_EXIT_FIRST}" ]] && (echo "NO_EXIT_FIRST is empty")
+[[ -z "${LAST_FAILED}" ]] && (echo "LAST_FAILED is empty")
 [[ -z "${CLI_TEST_MATRIX}" ]] && (echo "CLI_TEST_MATRIX is empty")
 [[ -z "${CLI_TEST_FILTER}" ]] && (echo "CLI_TEST_FILTER is empty")
 [[ -z "${CLI_TEST_COVERAGE}" ]] && (echo "CLI_TEST_COVERAGE is empty")
+[[ -z "${ENABLE_SELECTION}" ]] && (echo "ENABLE_SELECTION is empty")
+[[ -z "${SELECTION_MODE}" ]] && (echo "SELECTION_MODE is empty")
+[[ -z "${SELECTION_COUNT}" ]] && (echo "SELECTION_COUNT is empty")
+[[ -z "${UNFIX_RANDOM_SEED}" ]] && (echo "UNFIX_RANDOM_SEED is empty")
 
 # activate virtualenv
 source azEnv/bin/activate
@@ -31,17 +43,41 @@ az account set -s "${AZCLI_ALT_SUBSCRIPTION_ID}"
 az account show
 
 # prepare running options
-# base options
-base_options="-c --no-exitfirst --report-path ./reports --reruns 3 --capture=sys"
+# pytest options
+pytest_options="--report-path ./reports"
 # parallel
-if [[ ${PARALLELISM} -ge 2 ]]; then
-    base_options+=" -j ${PARALLELISM}"
+if [[ -n ${PARALLELISM} ]]; then
+    pytest_options+=" -j ${PARALLELISM}"
 else
-    base_options+=" -s"
+    pytest_options+=" -s"
+fi
+# reruns
+if [[ -n ${RERUNS} ]]; then
+    pytest_options+=" --reruns ${RERUNS}"
+fi
+# non boxed
+if [[ -n ${NON_BOXED} ]]; then
+    pytest_options+=" --non-boxed"
+fi
+# log level
+if [[ -n ${LOG_LEVEL} ]]; then
+    pytest_options+=" --log-level ${LOG_LEVEL}"
+fi
+# capture
+if [[ -n ${CAPTURE} ]]; then
+    pytest_options+=" --capture ${CAPTURE}"
+fi
+# no exit first
+if [[ -n ${NO_EXIT_FIRST} ]]; then
+    pytest_options+=" --no-exitfirst"
+fi
+# last failed
+if [[ -n ${LAST_FAILED} ]]; then
+    pytest_options+=" --last-failed"
 fi
 
 # filter options
-filter_options=""
+filter_options=" -c"
 # cli matrix
 if [[ -n ${CLI_TEST_MATRIX} ]]; then
     filter_options+=" -cm ./configs/${CLI_TEST_MATRIX}"
@@ -55,20 +91,41 @@ if [[ -n ${CLI_TEST_COVERAGE} ]]; then
     filter_options+=" -cc ${CLI_TEST_COVERAGE}"
 fi
 
+# case selection
+selection_options=""
+# enable selection
+if [[ -n ${ENABLE_SELECTION} ]]; then
+    selection_options+=" --enable-selection"
+fi
+# selection mode
+if [[ -n ${SELECTION_MODE} ]]; then
+    selection_options+=" --selection-mode ${SELECTION_MODE}"
+fi
+# selection count
+if [[ -n ${SELECTION_COUNT} ]]; then
+    selection_options+=" --selection-count ${SELECTION_COUNT}"
+fi
+# unfix random seed
+if [[ -n ${UNFIX_RANDOM_SEED} ]]; then
+    selection_options+=" --unfix-random-seed"
+fi
+
 # recording test
 if [[ ${TEST_MODE} == "recording" || ${TEST_MODE} == "all" ]]; then
     echo "Test in recording mode!"
     coverage_options=" -cc test_aks_commands.AzureKubernetesServiceScenarioTest"
-    recording_options="${base_options}${coverage_options}"
+    recording_options="${pytest_options}${coverage_options}${selection_options}"
     recording_options+=" --json-report-file=cli_recording_report.json"
     recording_options+=" --xml-file=cli_recording_result.xml"
     echo "Recording test options: ${recording_options}"
     test_result=0
     azaks ${recording_options} || test_result=$?
     if [[ ${test_result} -ne 0 ]]; then
         echo "Recording test failed!"
-        echo "Please refer to this wiki (https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/AZCLI-AKS-Live-Unit-Test-Pipeline) for troubleshooting guidelines."
-        exit ${test_result}
+        echo "Please refer to this wiki (${WIKI_LINK}) for troubleshooting guidelines."
+        if [[ -z ${IGNORE_EXIT_CODE} ]]; then
+            exit ${test_result}
+        fi
     else
         echo -e "Recording test passed!\n\n"
     fi
@@ -77,16 +134,18 @@ fi
 # live test
 if [[ ${TEST_MODE} == "live" || ${TEST_MODE} == "all" ]]; then
     echo "Test in live mode!"
-    live_options="${base_options}${filter_options}"
+    live_options="${pytest_options}${filter_options}${selection_options}"
     live_options+=" -l --json-report-file=cli_live_report.json"
     live_options+=" --xml-file=cli_live_result.xml"
     echo "Live test options: ${live_options}"
     test_result=0
     azaks ${live_options} || test_result=$?
     if [[ ${test_result} -ne 0 ]]; then
         echo "Live test failed!"
-        echo "Please refer to this wiki (https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/AZCLI-AKS-Live-Unit-Test-Pipeline) for troubleshooting guidelines."
-        exit ${test_result}
+        echo "Please refer to this wiki (${WIKI_LINK}) for troubleshooting guidelines."
+        if [[ -z ${IGNORE_EXIT_CODE} ]]; then
+            exit ${test_result}
+        fi
     else
         echo -e "Live test passed!\n\n"
     fi
@@ -95,16 +154,18 @@ fi
 # live-replay test
 if [[ ${TEST_MODE} == "live" || ${TEST_MODE} == "all" ]]; then
     echo "Test in live-replay mode!"
-    live_replay_options="${base_options}${filter_options}"
+    live_replay_options="${pytest_options}${filter_options}${selection_options}"
     live_replay_options+=" --json-report-file=cli_live_replay_report.json"
     live_replay_options+=" --xml-file=cli_live_replay_result.xml"
     echo "Live-replay test options: ${live_replay_options}"
     test_result=0
     azaks ${live_replay_options} || test_result=$?
     if [[ ${test_result} -ne 0 ]]; then
         echo "Live-replay test failed!"
-        echo "Please refer to this wiki (https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/AZCLI-AKS-Live-Unit-Test-Pipeline) for troubleshooting guidelines."
-        exit ${test_result}
+        echo "Please refer to this wiki (${WIKI_LINK}) for troubleshooting guidelines."
+        if [[ -z ${IGNORE_EXIT_CODE} ]]; then
+            exit ${test_result}
+        fi
     else
         echo -e "Live-replay test passed!\n\n"
     fi

diff --git a/src/aks-preview/azcli_aks_live_test/scripts/test_cli_unit.sh b/src/aks-preview/azcli_aks_live_test/scripts/test_cli_unit.sh
@@ -6,8 +6,9 @@ set -o nounset
 set -o pipefail
 set -o xtrace
 
-# const
-acs_base_dir="azure-cli/src/azure-cli/azure/cli/command_modules/acs"
+# check var
+[[ -z "${WIKI_LINK}" ]] && (echo "WIKI_LINK is empty"; exit 1)
+[[ -z "${ACS_BASE_DIR}" ]] && (echo "ACS_BASE_DIR is empty"; exit 1)
 
 # activate virtualenv
 source azEnv/bin/activate
@@ -18,7 +19,7 @@ removeAKSPreview
 
 # unit test & coverage report
 acs_unit_test_failed=""
-pushd ${acs_base_dir}
+pushd ${ACS_BASE_DIR}
 # clean existing coverage report
 (coverage combine || true) && (coverage erase || true)
 # perform unit test with module 'unittest'
@@ -32,10 +33,10 @@ coverage combine
 coverage report -m
 coverage json -o coverage_acs.json
 popd
-mkdir -p reports/ && cp ${acs_base_dir}/coverage_acs.json reports/
+mkdir -p reports/ && cp ${ACS_BASE_DIR}/coverage_acs.json reports/
 
 if [[ ${acs_unit_test_failed} == "true" ]]; then
     echo "Unit test failed!"
-    echo "Please refer to this wiki (https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/AZCLI-AKS-Live-Unit-Test-Pipeline) for troubleshooting guidelines."
+    echo "Please refer to this wiki (${WIKI_LINK}) for troubleshooting guidelines."
     exit 1
 fi