From 8989634bc990c1bcd5f25645ab06b09ea8788405 Mon Sep 17 00:00:00 2001 From: FumingZhang <81607949+FumingZhang@users.noreply.github.com> Date: Mon, 17 May 2021 12:32:19 +0800 Subject: [PATCH] [AKS] Add a live test pipeline for aks-preview PR check-in (#3292) --- src/aks-preview/az_aks_tool/.gitignore | 1 + src/aks-preview/az_aks_tool/cli.py | 42 +++ src/aks-preview/az_aks_tool/const.py | 17 + src/aks-preview/az_aks_tool/ext.py | 46 +++ src/aks-preview/az_aks_tool/filter.py | 120 ++++++++ src/aks-preview/az_aks_tool/index.py | 290 ++++++++++++++++++ src/aks-preview/az_aks_tool/log.py | 40 +++ src/aks-preview/az_aks_tool/main.py | 148 +++++++++ src/aks-preview/az_aks_tool/run.py | 199 ++++++++++++ src/aks-preview/az_aks_tool/tests/__init__.py | 0 src/aks-preview/az_aks_tool/tests/test_cli.py | 22 ++ src/aks-preview/az_aks_tool/tests/test_ext.py | 22 ++ .../az_aks_tool/tests/test_filter.py | 121 ++++++++ .../az_aks_tool/tests/test_index.py | 34 ++ src/aks-preview/az_aks_tool/tests/test_log.py | 34 ++ .../az_aks_tool/tests/test_main.py | 21 ++ src/aks-preview/az_aks_tool/tests/test_run.py | 33 ++ .../az_aks_tool/tests/test_utils.py | 54 ++++ .../az_aks_tool/tests/testdata.json | 18 ++ src/aks-preview/az_aks_tool/utils.py | 83 +++++ .../azcli_aks_live_test/.gitignore | 5 + .../azcli_aks_live_test/HISTORY.rst | 9 + src/aks-preview/azcli_aks_live_test/README.md | 9 + .../azcli_aks_live_test/clean_up.sh | 12 + .../azcli_aks_live_test/clone_repo.sh | 35 +++ .../ext_matrix_default.json | 39 +++ .../azcli_aks_live_test/prepare_image.sh | 9 + .../azcli_aks_live_test/setup_venv.sh | 38 +++ .../azcli_aks_live_test/start_container.sh | 24 ++ .../azcli_aks_live_test/test_cli_live.sh | 30 ++ .../azcli_aks_live_test/test_cli_unit.sh | 16 + .../azcli_aks_live_test/test_ext_live.sh | 80 +++++ .../azcli_aks_live_test/test_ext_unit.sh | 71 +++++ .../azcli_aks_live_test/transcribe_env.sh | 40 +++ .../vsts-azcli-aks-live-test.yaml | 56 ++++ .../vsts-azcli-aks-unit-test.yaml | 56 ++++ .../tests/latest/custom_preparers.py | 62 ++++ .../tests/latest/test_aks_commands.py | 90 +++--- 38 files changed, 1982 insertions(+), 44 deletions(-) create mode 100644 src/aks-preview/az_aks_tool/.gitignore create mode 100644 src/aks-preview/az_aks_tool/cli.py create mode 100644 src/aks-preview/az_aks_tool/const.py create mode 100644 src/aks-preview/az_aks_tool/ext.py create mode 100644 src/aks-preview/az_aks_tool/filter.py create mode 100644 src/aks-preview/az_aks_tool/index.py create mode 100644 src/aks-preview/az_aks_tool/log.py create mode 100644 src/aks-preview/az_aks_tool/main.py create mode 100644 src/aks-preview/az_aks_tool/run.py create mode 100644 src/aks-preview/az_aks_tool/tests/__init__.py create mode 100644 src/aks-preview/az_aks_tool/tests/test_cli.py create mode 100644 src/aks-preview/az_aks_tool/tests/test_ext.py create mode 100644 src/aks-preview/az_aks_tool/tests/test_filter.py create mode 100644 src/aks-preview/az_aks_tool/tests/test_index.py create mode 100644 src/aks-preview/az_aks_tool/tests/test_log.py create mode 100644 src/aks-preview/az_aks_tool/tests/test_main.py create mode 100644 src/aks-preview/az_aks_tool/tests/test_run.py create mode 100644 src/aks-preview/az_aks_tool/tests/test_utils.py create mode 100644 src/aks-preview/az_aks_tool/tests/testdata.json create mode 100644 src/aks-preview/az_aks_tool/utils.py create mode 100644 src/aks-preview/azcli_aks_live_test/.gitignore create mode 100644 src/aks-preview/azcli_aks_live_test/HISTORY.rst create mode 100644 src/aks-preview/azcli_aks_live_test/README.md create mode 100755 src/aks-preview/azcli_aks_live_test/clean_up.sh create mode 100755 src/aks-preview/azcli_aks_live_test/clone_repo.sh create mode 100644 src/aks-preview/azcli_aks_live_test/ext_matrix_default.json create mode 100755 src/aks-preview/azcli_aks_live_test/prepare_image.sh create mode 100755 src/aks-preview/azcli_aks_live_test/setup_venv.sh create mode 100755 src/aks-preview/azcli_aks_live_test/start_container.sh create mode 100755 src/aks-preview/azcli_aks_live_test/test_cli_live.sh create mode 100755 src/aks-preview/azcli_aks_live_test/test_cli_unit.sh create mode 100755 src/aks-preview/azcli_aks_live_test/test_ext_live.sh create mode 100755 src/aks-preview/azcli_aks_live_test/test_ext_unit.sh create mode 100755 src/aks-preview/azcli_aks_live_test/transcribe_env.sh create mode 100644 src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml create mode 100644 src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-unit-test.yaml create mode 100644 src/aks-preview/azext_aks_preview/tests/latest/custom_preparers.py diff --git a/src/aks-preview/az_aks_tool/.gitignore b/src/aks-preview/az_aks_tool/.gitignore new file mode 100644 index 00000000000..bee8a64b79a --- /dev/null +++ b/src/aks-preview/az_aks_tool/.gitignore @@ -0,0 +1 @@ +__pycache__ diff --git a/src/aks-preview/az_aks_tool/cli.py b/src/aks-preview/az_aks_tool/cli.py new file mode 100644 index 00000000000..ed21d2ce0a2 --- /dev/null +++ b/src/aks-preview/az_aks_tool/cli.py @@ -0,0 +1,42 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import glob +import os +import logging + +import az_aks_tool.const as const +import az_aks_tool.index as index +logger = logging.getLogger(__name__) + + +def get_cli_mod_data(mod_name=const.ACS_MOD_NAME, profile="latest"): + profile_split = profile.split('-') + profile_namespace = '_'.join([profile_split[-1]] + profile_split[:-1]) + + # key value pairs of all modules(in azcli & extention) and its absolute path, used later to find test indexes + path_table = index.get_path_table() + command_modules = path_table["mod"] + inverse_name_table = index.get_name_index(invert=True) + + # construct 'import_name' & mod_data', used later to find test indexes + acs_mod_path = command_modules[mod_name] + mod_data = { + "alt_name": "{}{}".format(const.COMMAND_MODULE_PREFIX, mod_name), + "filepath": os.path.join(acs_mod_path, "tests", profile_namespace), + "base_path": "azure.cli.command_modules.{}.tests.{}".format(mod_name, profile_namespace), + "files": {} + } + + cli_test = index.discover_module_tests(mod_name, mod_data) + return cli_test + + +def get_cli_test_index(module_data=None, mod_name=const.ACS_MOD_NAME, profile="latest"): + if mod_name in module_data: + mod_data = module_data[mod_name] + else: + mod_data = get_cli_mod_data(mod_name=mod_name, profile=profile) + return mod_data["files"] diff --git a/src/aks-preview/az_aks_tool/const.py b/src/aks-preview/az_aks_tool/const.py new file mode 100644 index 00000000000..8285bbb80c3 --- /dev/null +++ b/src/aks-preview/az_aks_tool/const.py @@ -0,0 +1,17 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import sys + +IS_WINDOWS = sys.platform.lower() in ['windows', 'win32'] + +CLI_REPO_NAME = "azure-cli" +EXT_REPO_NAME = 'azure-cli-extensions' +COMMAND_MODULE_PREFIX = 'azure-cli-' +EXTENSION_PREFIX = 'azext_' +ACS_MOD_NAME = "acs" +AKS_PREVIEW_MOD_NAME = EXTENSION_PREFIX + "aks_preview" # azext_aks_preview + +ENV_VAR_TEST_LIVE = 'AZURE_TEST_RUN_LIVE' # denotes that tests should be run live instead of played back diff --git a/src/aks-preview/az_aks_tool/ext.py b/src/aks-preview/az_aks_tool/ext.py new file mode 100644 index 00000000000..8fa39c1e242 --- /dev/null +++ b/src/aks-preview/az_aks_tool/ext.py @@ -0,0 +1,46 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import glob +import os +import logging + +import az_aks_tool.const as const +import az_aks_tool.index as index +logger = logging.getLogger(__name__) + + +def get_ext_mod_data(mod_name=const.AKS_PREVIEW_MOD_NAME, profile="latest"): + profile_split = profile.split('-') + profile_namespace = '_'.join([profile_split[-1]] + profile_split[:-1]) + + # key value pairs of all modules(in azcli & extention) and its absolute path, used later to find test indexes + path_table = index.get_path_table() + extensions = path_table["ext"] + inverse_name_table = index.get_name_index(invert=True) + + # construct 'import_name' & mod_data', used later to find test indexes + aks_preview_mod_path = extensions[mod_name] + glob_pattern = os.path.normcase( + os.path.join("{}*".format(const.EXTENSION_PREFIX))) + file_path = glob.glob(os.path.join(aks_preview_mod_path, glob_pattern))[0] + import_name = os.path.basename(file_path) + mod_data = { + "alt_name": inverse_name_table[mod_name], + "filepath": os.path.join(file_path, "tests", profile_namespace), + "base_path": "{}.tests.{}".format(import_name, profile_namespace), + "files": {} + } + + ext_test = index.discover_module_tests(import_name, mod_data) + return ext_test + + +def get_ext_test_index(module_data=None, mod_name=const.AKS_PREVIEW_MOD_NAME, profile="latest"): + if mod_name in module_data: + mod_data = module_data[mod_name] + else: + mod_data = get_ext_mod_data(mod_name=mod_name, profile=profile) + return mod_data["files"] diff --git a/src/aks-preview/az_aks_tool/filter.py b/src/aks-preview/az_aks_tool/filter.py new file mode 100644 index 00000000000..718a34d4058 --- /dev/null +++ b/src/aks-preview/az_aks_tool/filter.py @@ -0,0 +1,120 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import logging +from collections import Iterable +logger = logging.getLogger(__name__) + + +def extract_file_class_pairs(tag_list): + pairs = [] + for k in tag_list: + tags = k.split(".") + if len(tags) == 2: + pairs.append((tags[0], tags[1])) + return pairs + + +def filter_valid_file_class_pairs(pairs, test_index): + valid_pairs = [] + for pair in pairs: + if pair[0] in test_index and pair[1] in test_index[pair[0]]: + valid_pairs.append(pair) + logger.debug("Valid file & class pair: '{}'".format(pair)) + else: + logger.debug("Invalid file & class pair: '{}'".format(pair)) + return valid_pairs + + +def get_all_values_from_nested_dict(d): + for v in d.values(): + if isinstance(v, dict): + yield from get_all_values_from_nested_dict(v) + else: + yield v + + +def flatten_nested_list(lis): + for item in lis: + if isinstance(item, Iterable) and not isinstance(item, str): + for x in flatten_nested_list(item): + yield x + else: + yield item + + +def filter_valid_test_cases(test_cases, test_index): + valid_test_cases = [] + nested_test_cases = list(get_all_values_from_nested_dict(test_index)) + falttened_test_cases = list(flatten_nested_list(nested_test_cases)) + for test_case in test_cases: + if test_case in falttened_test_cases: + valid_test_cases.append(test_case) + logger.debug("Valid test case: '{}'".format(test_case)) + else: + logger.debug("Invalid test case: '{}'".format(test_case)) + return valid_test_cases + + +def get_test_cases(test_index, matrix, extra_coverage=None): + test_cases = [] + coverage = matrix.get("coverage", {}) + # default coverage + for fileName, className in coverage.items(): + for c in className: + test_cases.extend(test_index[fileName][c]) + # custom extra coverage + if extra_coverage: + # method 1: fileName.className + file_class_pairs = extract_file_class_pairs(extra_coverage) + valid_file_class_pairs = filter_valid_file_class_pairs( + file_class_pairs, test_index) + for valid_pair in valid_file_class_pairs: + test_cases.extend( + test_index[valid_pair[0]][valid_pair[1]]) + # method 2: test cases + test_cases.extend(filter_valid_test_cases( + extra_coverage, test_index)) + return list(set(test_cases)) + + +def get_exclude_test_cases(test_index, matrix, extra_filter=None): + exclude_test_cases = [] + exclude = matrix.get("exclude", {}) + # default exclude + if not extra_filter or "default" in extra_filter: + matrix_test_cases = [] + matrix_file_class_pairs = [] + for k, v in exclude.items(): + # method 1: reason -> test cases + matrix_test_cases.extend(v) + # method 2: fileName -> className + matrix_file_class_pairs.extend((k, x) for x in v) + # method 1: reason -> test cases + exclude_test_cases.extend( + filter_valid_test_cases(matrix_test_cases, test_index)) + # method 2: fileName -> className + valid_matrix_file_class_pairs = filter_valid_file_class_pairs( + matrix_file_class_pairs, test_index) + for valid_matrix_pair in valid_matrix_file_class_pairs: + exclude_test_cases.extend( + test_index[valid_matrix_pair[0]][valid_matrix_pair[1]]) + # custom extra_filter + if extra_filter: + # method 1: matrix exclude key + for k, v in exclude.items(): + if k in extra_filter: + exclude_test_cases.extend(v) + # method 2: fileName.className + file_class_pairs = extract_file_class_pairs(extra_filter) + valid_file_class_pairs = filter_valid_file_class_pairs( + file_class_pairs, test_index) + for valid_pair in valid_file_class_pairs: + exclude_test_cases.extend( + test_index[valid_pair[0]][valid_pair[1]]) + # method 3: test cases + exclude_test_cases.extend( + filter_valid_test_cases(extra_filter, test_index)) + return list(set(exclude_test_cases)) diff --git a/src/aks-preview/az_aks_tool/index.py b/src/aks-preview/az_aks_tool/index.py new file mode 100644 index 00000000000..11d7e405179 --- /dev/null +++ b/src/aks-preview/az_aks_tool/index.py @@ -0,0 +1,290 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import glob +import logging +import os +from importlib import import_module + +import az_aks_tool.utils as utils +import az_aks_tool.const as const +logger = logging.getLogger(__name__) + + +def get_repo_path(repo_name, root_path=None): + # find cache from environment variable + repo_path = os.environ.get("{}_PATH".format(repo_name), "") + if os.path.isdir(repo_path): + logger.info("Find cached '{}' repo path: '{}'".format(repo_name, repo_path)) + return repo_path + + # search from root_path + candidate_root_paths = [os.getcwd(), os.path.expanduser("~")] + valid_repo_paths = [] + repo_path = "" + if root_path is None or not os.path.isdir(root_path): + logger.warning("Invalid root path '{}'!".format(root_path)) + else: + candidate_root_paths = [root_path] + candidate_root_paths + logger.info("Setting root path from '{}'".format(candidate_root_paths)) + for candidate_root_path in candidate_root_paths: + root_path = candidate_root_path + logger.info("Searching from root path: '{}'".format(root_path)) + for path, _, _ in os.walk(root_path): + pattern = os.path.join(path, repo_name) + valid_repo_paths.extend(glob.glob(pattern)) + if len(valid_repo_paths) >= 1: + repo_path = valid_repo_paths[0] + if len(valid_repo_paths) >= 2: + logger.warning("Find {} '{}' repo paths: {}".format(len(valid_repo_paths), repo_name, valid_repo_paths)) + logger.info("Set '{}' repo path as '{}'".format(repo_name, repo_path)) + os.environ["{}_PATH".format(repo_name)] = str(repo_path) + return repo_path + else: + logger.warning("Could not find valid path to repo '{}' from '{}'".format(repo_name, root_path)) + return repo_path + +def find_files(root_paths, file_pattern): + """ Returns the paths to all files that match a given pattern. + + :returns: Paths ([str]) to files matching the given pattern. + """ + if isinstance(root_paths, str): + root_paths = [root_paths] + paths = [] + for root_path in root_paths: + for path, _, _ in os.walk(root_path): + pattern = os.path.join(path, file_pattern) + paths.extend(glob.glob(pattern)) + return paths + +def get_name_index(invert=False, include_whl_extensions=False): + """ Returns a dictionary containing the long and short names of modules and extensions is {SHORT:LONG} format or + {LONG:SHORT} format when invert=True. """ + from azure.cli.core.extension import EXTENSIONS_DIR # pylint: disable=import-error + + table = {} + cli_repo_path = get_repo_path(const.CLI_REPO_NAME) + ext_repo_paths = get_repo_path(const.EXT_REPO_NAME) + + # unified azure-cli package (2.0.68 and later) + paths = os.path.normcase( + os.path.join( + cli_repo_path, 'src', 'azure-cli', 'azure', 'cli', 'command_modules', '*', '__init__.py' + ) + ) + modules_paths = glob.glob(paths) + core_paths = glob.glob(os.path.normcase(os.path.join(cli_repo_path, 'src', '*', 'setup.py'))) + ext_paths = [x for x in find_files(ext_repo_paths, '*.*-info') if 'site-packages' not in x] + whl_ext_paths = [] + if include_whl_extensions: + whl_ext_paths = [x for x in find_files(EXTENSIONS_DIR, '*.*-info') if 'site-packages' not in x] + + def _update_table(paths, key): + folder = None + long_name = None + short_name = None + for path in paths: + folder = os.path.dirname(path) + base_name = os.path.basename(folder) + # determine long-names + if key == 'ext': + short_name = base_name + for item in os.listdir(folder): + if item.startswith(const.EXTENSION_PREFIX): + long_name = item + break + elif base_name.startswith(const.COMMAND_MODULE_PREFIX): + long_name = base_name + short_name = base_name.replace(const.COMMAND_MODULE_PREFIX, '') or '__main__' + else: + short_name = base_name + long_name = '{}{}'.format(const.COMMAND_MODULE_PREFIX, base_name) + if not invert: + table[short_name] = long_name + else: + table[long_name] = short_name + + _update_table(modules_paths, 'mod') + _update_table(core_paths, 'core') + _update_table(ext_paths, 'ext') + _update_table(whl_ext_paths, 'ext') + + return table + +# pylint: disable=too-many-statements +def get_path_table(include_only=None, include_whl_extensions=False): + """ Returns a table containing the long and short names of different modules and extensions and the path to them. + The structure looks like: + { + 'core': { + NAME: PATH, + ... + }, + 'mod': { + NAME: PATH, + ... + }, + 'ext': { + NAME: PATH, + ... + } + } + """ + from azure.cli.core.extension import EXTENSIONS_DIR # pylint: disable=import-error + + # determine whether the call will filter or return all + if isinstance(include_only, str): + include_only = [include_only] + get_all = not include_only + + table = {} + cli_repo_path = get_repo_path(const.CLI_REPO_NAME) + ext_repo_paths = get_repo_path(const.EXT_REPO_NAME) + + paths = os.path.normcase( + os.path.join( + cli_repo_path, 'src', 'azure-cli', 'azure', 'cli', 'command_modules', '*', '__init__.py' + ) + ) + modules_paths = glob.glob(paths) + core_paths = glob.glob(os.path.normcase(os.path.join(cli_repo_path, 'src', '*', 'setup.py'))) + ext_paths = [x for x in find_files(ext_repo_paths, '*.*-info') if 'site-packages' not in x] + whl_ext_paths = [x for x in find_files(EXTENSIONS_DIR, '*.*-info') if 'site-packages' not in x] + + def _update_table(package_paths, key): + if key not in table: + table[key] = {} + + for path in package_paths: + folder = os.path.dirname(path) + base_name = os.path.basename(folder) + + if key == 'ext': + short_name = base_name + long_name = next((item for item in os.listdir(folder) if item.startswith(const.EXTENSION_PREFIX)), None) + else: + short_name = base_name + long_name = '{}{}'.format(const.COMMAND_MODULE_PREFIX, base_name) + + if get_all: + table[key][long_name if key == 'ext' else short_name] = folder + elif not include_only: + return # nothing left to filter + else: + # check and update filter + if short_name in include_only: + include_only.remove(short_name) + table[key][short_name] = folder + if long_name in include_only: + # long name takes precedence to ensure path doesn't appear twice + include_only.remove(long_name) + table[key].pop(short_name, None) + table[key][long_name] = folder + + _update_table(modules_paths, 'mod') + _update_table(core_paths, 'core') + _update_table(ext_paths, 'ext') + if include_whl_extensions: + _update_table(whl_ext_paths, 'ext') + + if include_only: + whl_extensions = [mod for whl_ext_path in whl_ext_paths for mod in include_only if mod in whl_ext_path] + if whl_extensions: + err = 'extension(s): [ {} ] installed from a wheel may need --include-whl-extensions option'.format( + ', '.join(whl_extensions)) + raise Exception(err) + + raise Exception('unrecognized modules: [ {} ]'.format(', '.join(include_only))) + + return table + +def discover_module_tests(mod_name, mod_data): + + # get the list of test files in each module + total_tests = 0 + total_files = 0 + logger.info('Mod: %s', mod_name) + try: + contents = os.listdir(mod_data['filepath']) + test_files = { + x[:-len('.py')]: {} for x in contents if x.startswith('test_') and x.endswith('.py') + } + total_files = len(test_files) + except FileNotFoundError: + logger.info(' No test files found.') + return None + + for file_name in test_files: + mod_data['files'][file_name] = {} + test_file_path = mod_data['base_path'] + '.' + file_name + try: + module = import_module(test_file_path) + except ImportError as ex: + logger.info(' %s', ex) + continue + module_dict = module.__dict__ + possible_test_classes = {x: y for x, y in module_dict.items() if not x.startswith('_')} + for class_name, class_def in possible_test_classes.items(): + try: + class_dict = class_def.__dict__ + except AttributeError: + # skip non-class symbols in files like constants, imported methods, etc. + continue + if class_dict.get('__module__') == test_file_path: + tests = [x for x in class_def.__dict__ if x.startswith('test_')] + if tests: + mod_data['files'][file_name][class_name] = tests + total_tests += len(tests) + logger.info(' %s tests found in %s files.', total_tests, total_files) + return mod_data + + +def build_test_index(module_data): + test_index = {} + conflicted_keys = [] + + def add_to_index(key, path): + key = key or mod_name + if key in test_index: + if key not in conflicted_keys: + conflicted_keys.append(key) + mod1 = utils.extract_module_name(path) + mod2 = utils.extract_module_name(test_index[key]) + if mod1 != mod2: + # resolve conflicted keys by prefixing with the module name and a dot (.) + logger.warning("'%s' exists in both '%s' and '%s'. Resolve using `%s.%s` or `%s.%s`", + key, mod1, mod2, mod1, key, mod2, key) + test_index['{}.{}'.format(mod1, key)] = path + test_index['{}.{}'.format(mod2, key)] = test_index[key] + else: + logger.error("'%s' exists twice in the '%s' module", key, mod1) + else: + test_index[key] = path + + # build the index + for mod_name, mod_data in module_data.items(): + # don't add empty mods to the index + if not mod_data: + continue + + mod_path = mod_data['filepath'] + for file_name, file_data in mod_data['files'].items(): + file_path = os.path.join(mod_path, file_name) + '.py' + for class_name, test_list in file_data.items(): + for test_name in test_list: + test_path = '{}::{}::{}'.format(file_path, class_name, test_name) + add_to_index(test_name, test_path) + class_path = '{}::{}'.format(file_path, class_name) + add_to_index(class_name, class_path) + add_to_index(file_name, file_path) + add_to_index(mod_name, mod_path) + add_to_index(mod_data['alt_name'], mod_path) + + # remove the conflicted keys since they would arbitrarily point to a random implementation + for key in conflicted_keys: + del test_index[key] + + return test_index diff --git a/src/aks-preview/az_aks_tool/log.py b/src/aks-preview/az_aks_tool/log.py new file mode 100644 index 00000000000..a5d27925983 --- /dev/null +++ b/src/aks-preview/az_aks_tool/log.py @@ -0,0 +1,40 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import logging +import sys + + +def parse_module_name(levels=1): + module_name = None + module_levels = __name__.split(".") + if len(module_levels) < levels: + print("Failed to parse {}-level module name from '{}'".format(levels, __name__)) + else: + module_name = ".".join(module_levels[:levels]) + return module_name + + +def setup_logging(root_logger_name=None, log_path="az_aks_tool.log"): + if root_logger_name == "" or root_logger_name.isspace(): + root_logger_name = parse_module_name() + logger = logging.getLogger(root_logger_name) + logger.setLevel(level=logging.DEBUG) + + # Formatter + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + + # FileHandler + file_handler = logging.FileHandler(filename=log_path, mode="w") + file_handler.setFormatter(formatter) + file_handler.setLevel(level=logging.DEBUG) + logger.addHandler(file_handler) + + # StreamHandler + stream_handler = logging.StreamHandler(sys.stdout) + stream_handler.setFormatter(formatter) + stream_handler.setLevel(level=logging.INFO) + logger.addHandler(stream_handler) diff --git a/src/aks-preview/az_aks_tool/main.py b/src/aks-preview/az_aks_tool/main.py new file mode 100644 index 00000000000..9864eeaaa72 --- /dev/null +++ b/src/aks-preview/az_aks_tool/main.py @@ -0,0 +1,148 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import argparse +import os +import sys +import logging + +import az_aks_tool.const as const +import az_aks_tool.log as log +import az_aks_tool.utils as utils +import az_aks_tool.cli as cli +import az_aks_tool.ext as ext +import az_aks_tool.index as index +import az_aks_tool.run as run + + +def init_argparse(args): + parser = argparse.ArgumentParser() + parser.add_argument("-c", "--cli", action="store_true", default=False, + help="enbale cli test") + parser.add_argument("-e", "--ext", action="store_true", default=False, + help="enbale ext test") + parser.add_argument("-a", "--all", action="store_true", default=False, + help="enbale all tests (cli & ext)") + parser.add_argument("-t", "--tests", nargs='+', help="test case names") + parser.add_argument("-cm", "--cli-matrix", type=str, + help="full path to cli test matrix") + parser.add_argument("-cc", "--cli-coverage", nargs="+", + help="cli test extra coverage") + parser.add_argument("-cf", "--cli-filter", nargs="+", + help="cli test filter") + parser.add_argument("-em", "--ext-matrix", type=str, + help="full path to extension test matrix") + parser.add_argument("-ec", "--ext-coverage", nargs="+", + help="extension test extra coverage") + parser.add_argument("-ef", "--ext-filter", nargs="+", + help="extension test filter") + parser.add_argument("-s", "--series", action="store_true", + default=False, help="series test") + parser.add_argument("-l", "--live", action="store_true", + default=False, help="live test") + parser.add_argument("-ne", "--no-exitfirst", action="store_true", + default=False, help="no exit first") + parser.add_argument("-j", "--parallelism", type=str, + default="8", help="test parallelism") + parser.add_argument("--reruns", type=str, + default="3", help="rerun times") + parser.add_argument("--capture", type=str, + default="sys", help="test capture") + parser.add_argument("-p", "--report-path", type=str, + required=True, help="report path") + parser.add_argument("-f", "--json-report-file", type=str, + default="azcli_aks_runner_report.json", help="json report filename") + parser.add_argument("--xml-file", type=str, + default="azcli_aks_runner.xml", help="junit/xml report filename") + parser.add_argument("--log-file", type=str, + default="az_aks_tool.log", help="log filename") + args = parser.parse_args(args) + return args + + +def main(): + # parse args + print("raw args: {}".format(sys.argv)) + args = init_argparse(sys.argv[1:]) + + # setup logger + root_module_name = log.parse_module_name(levels=1) + log.setup_logging(root_module_name, os.path.join( + args.report_path, args.log_file)) + logger = logging.getLogger("{}.{}".format(root_module_name, __name__)) + + # # check test cases + test_cases = args.tests + ext_matrix_file_path = args.ext_matrix + cli_matrix_file_path = args.cli_matrix + + # prepare pytest args + pytest_args = [] + if not args.series and args.parallelism: + pytest_args.append("-n {}".format(args.parallelism)) + pytest_args.append("--json-report") + pytest_args.append("--reruns {}".format(args.reruns)) + pytest_args.append("--capture {}".format(args.capture)) + pytest_args = [" ".join(pytest_args)] + logger.info("pytest_args: {}".format(pytest_args)) + + # check mode & collect module data + enable_cli = False + enable_ext = False + module_data = {} + if args.cli or args.all: + enable_cli = True + module_data[const.ACS_MOD_NAME] = cli.get_cli_mod_data() + cli_test_index = cli.get_cli_test_index(module_data) + + if args.ext or args.all: + enable_ext = True + module_data[const.AKS_PREVIEW_MOD_NAME] = ext.get_ext_mod_data() + ext_test_index = ext.get_ext_test_index(module_data) + + # build test index + if enable_cli or enable_ext: + logger.info("Building test index...") + test_index = index.build_test_index(module_data) + else: + logger.error( + "Both modes 'cli' and 'ext' are not enabled! No test will be performed!") + logger.error( + "Please provide at least one of the following parameters (-a, -c, -e) to enable the test!") + + # cli matrix test + if enable_cli: + cli_qualified_test_cases = utils.get_fully_qualified_test_cases( + cli_test_index, cli_matrix_file_path, const.ACS_MOD_NAME, args.cli_coverage, args.cli_filter) + logger.info("Perform following cli tests: {}".format( + cli_qualified_test_cases)) + exit_code = run.run_tests(cli_qualified_test_cases, test_index, mode="cli", base_path=args.report_path, xml_file=args.xml_file, json_file=args.json_report_file, in_series=args.series, + run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args) + if exit_code != 0: + sys.exit("CLI test failed with exit code: {}".format(exit_code)) + + # ext matrix test + if enable_ext: + ext_qualified_test_cases = utils.get_fully_qualified_test_cases( + ext_test_index, ext_matrix_file_path, const.AKS_PREVIEW_MOD_NAME, args.ext_coverage, args.ext_filter) + logger.info("Perform following ext tests: {}".format( + ext_qualified_test_cases)) + exit_code = run.run_tests(ext_qualified_test_cases, test_index, mode="ext", base_path=args.report_path, xml_file=args.xml_file, json_file=args.json_report_file, in_series=args.series, + run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args) + if exit_code != 0: + sys.exit("EXT test failed with exit code: {}".format(exit_code)) + + # raw tests + if test_cases: + logger.info("Get {} cases!".format(len(test_cases))) + logger.info("Perform following raw tets: {}".format(test_cases)) + exit_code = run.run_tests(test_cases, test_index, mode="raw", base_path=args.report_path, xml_file=args.xml_file, json_file=args.json_report_file, in_series=args.series, + run_live=args.live, no_exit_first=args.no_exitfirst, pytest_args=pytest_args) + if exit_code != 0: + sys.exit("Raw test failed with exit code: {}".format(exit_code)) + + +if __name__ == "__main__": + main() diff --git a/src/aks-preview/az_aks_tool/run.py b/src/aks-preview/az_aks_tool/run.py new file mode 100644 index 00000000000..13c2f2ae426 --- /dev/null +++ b/src/aks-preview/az_aks_tool/run.py @@ -0,0 +1,199 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import logging +import os +import subprocess +import sys +import traceback +from knack.util import CommandResultItem + +from az_aks_tool.const import IS_WINDOWS, ENV_VAR_TEST_LIVE +from az_aks_tool.utils import heading +logger = logging.getLogger(__name__) + + +class ProfileContext: + def __init__(self, profile_name=None): + self.target_profile = profile_name + + self.origin_profile = current_profile() + + def __enter__(self): + if self.target_profile is None or self.target_profile == self.origin_profile: + logger.info('The tests are set to run against current profile "{}"'.format( + self.origin_profile)) + else: + result = cmd('az cloud update --profile {}'.format(self.target_profile), + 'Switching to target profile "{}"...'.format(self.target_profile)) + if result.exit_code != 0: + raise Exception(result.error.output.decode('utf-8')) + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.target_profile is not None and self.target_profile != self.origin_profile: + logger.info('Switching back to origin profile "{}"...'.format( + self.origin_profile)) + call('az cloud update --profile {}'.format(self.origin_profile)) + + if exc_tb: + traceback.print_exception(exc_type, exc_val, exc_tb) + + +def current_profile(): + return cmd('az cloud show --query profile -otsv', show_stderr=False).result + + +class CommandError(Exception): + + def __init__(self, output, exit_code, command): + message = "Command `{}` failed with exit code {}:\n{}".format( + command, exit_code, output) + self.exit_code = exit_code + self.output = output + self.command = command + super().__init__(message) + + +def call(command, **kwargs): + """ Run an arbitrary command but don't buffer the output. + + :param command: The entire command line to run. + :param kwargs: Any kwargs supported by subprocess.Popen + :returns: (int) process exit code. + """ + return subprocess.call( + command, + shell=True, + **kwargs) + + +def cmd(command, message=False, show_stderr=True, raise_error=False, **kwargs): + """ Run an arbitrary command. + + :param command: The entire command line to run. + :param message: A custom message to display, or True (bool) to use a default. + :param show_stderr: On error, display the contents of STDERR. + :param raise_error: On error, raise CommandError. + :param kwargs: Any kwargs supported by subprocess.Popen + :returns: CommandResultItem object. + """ + + # use default message if custom not provided + if message is True: + message = 'Running: {}\n'.format(command) + + if message: + logger.info(message) + + logger.info("Running: %s", command) + try: + output = subprocess.check_output( + command.split(), + stderr=subprocess.STDOUT if show_stderr else None, + shell=IS_WINDOWS, + **kwargs).decode('utf-8').strip() + logger.debug(output) + return CommandResultItem(output, exit_code=0, error=None) + except subprocess.CalledProcessError as err: + if raise_error: + raise CommandError(err.output.decode(), err.returncode, command) + return CommandResultItem(err.output, exit_code=err.returncode, error=err) + + +def get_test_runner(parallel, log_path, last_failed, no_exit_first, mark): + """Create a pytest execution method""" + def _run(test_paths, pytest_args): + + if os.name == 'posix': + arguments = ['-x', '-v', '--boxed', '-p no:warnings', + '--log-level=WARN', '--junit-xml', log_path] + else: + arguments = ['-x', '-v', '-p no:warnings', + '--log-level=WARN', '--junit-xml', log_path] + + if no_exit_first: + arguments.remove('-x') + + if mark: + arguments.append('-m "{}"'.format(mark)) + + arguments.extend(test_paths) + if parallel: + arguments += ['-n', 'auto'] + if last_failed: + arguments.append('--lf') + if pytest_args: + arguments += pytest_args + cmd = 'python -m pytest {}'.format(' '.join(arguments)) + logger.info('Running: %s', cmd) + return call(cmd) + + return _run + + +def run_tests(tests, test_index, mode, base_path, xml_file, json_file, in_series=False, + run_live=False, profile=None, last_failed=False, no_exit_first=False, mark=None, pytest_args=None): + + heading('Run Tests') + + # process file path + if not xml_file.startswith(mode): + xml_file = "{}_{}".format(mode, xml_file) + if not json_file.startswith(mode): + json_file = "{}_{}".format(mode, json_file) + xml_path = os.path.realpath(os.path.join(base_path, xml_file)) + json_path = os.path.realpath(os.path.join(base_path, json_file)) + pytest_args.append("--json-report-file {}".format(json_path)) + logger.info("junit/xml report file full path: {}".format(xml_path)) + logger.info("json report file full path: {}".format(json_path)) + + # process environment variables + if run_live: + logger.warning('RUNNING TESTS LIVE') + os.environ[ENV_VAR_TEST_LIVE] = 'True' + + def _find_test(index, name): + name_comps = name.split('.') + num_comps = len(name_comps) + key_error = KeyError() + + for i in range(num_comps): + check_name = '.'.join(name_comps[(-1 - i):]) + try: + match = index[check_name] + if check_name != name: + logger.info( + "Test found using just '%s'. The rest of the name was ignored.", check_name) + return match + except KeyError as ex: + key_error = ex + continue + raise key_error + + # lookup test paths from index + test_paths = [] + for t in tests: + try: + test_path = os.path.normpath(_find_test(test_index, t)) + test_paths.append(test_path) + except KeyError: + logger.warning("'%s' not found.", t) + continue + + # Tests have been collected. Now run them. + exit_code = 0 + if not test_paths: + logger.warning('No tests selected to run.') + return exit_code + + with ProfileContext(profile): + runner = get_test_runner(parallel=not in_series, + log_path=xml_path, + last_failed=last_failed, + no_exit_first=no_exit_first, + mark=mark) + exit_code = runner(test_paths=test_paths, pytest_args=pytest_args) + + return 0 if not exit_code else 1 diff --git a/src/aks-preview/az_aks_tool/tests/__init__.py b/src/aks-preview/az_aks_tool/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/aks-preview/az_aks_tool/tests/test_cli.py b/src/aks-preview/az_aks_tool/tests/test_cli.py new file mode 100644 index 00000000000..5b9000b7af6 --- /dev/null +++ b/src/aks-preview/az_aks_tool/tests/test_cli.py @@ -0,0 +1,22 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import unittest + +import az_aks_tool.utils as utils +import az_aks_tool.cli as cli + +THIS_FILE = os.path.abspath(__file__) +THIS_DIR = os.path.dirname(THIS_FILE) +PARENT_DIR = os.path.dirname(THIS_DIR) + +class CliTestCase(unittest.TestCase): + + def test_get_cli_mod_data(self): + pass + + def test_get_cli_test_index(self): + pass diff --git a/src/aks-preview/az_aks_tool/tests/test_ext.py b/src/aks-preview/az_aks_tool/tests/test_ext.py new file mode 100644 index 00000000000..ec194f16641 --- /dev/null +++ b/src/aks-preview/az_aks_tool/tests/test_ext.py @@ -0,0 +1,22 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import unittest + +import az_aks_tool.utils as utils +import az_aks_tool.ext as ext + +THIS_FILE = os.path.abspath(__file__) +THIS_DIR = os.path.dirname(THIS_FILE) +PARENT_DIR = os.path.dirname(THIS_DIR) + +class ExtTestCase(unittest.TestCase): + + def test_get_ext_mod_data(self): + pass + + def test_get_ext_test_index(self): + pass diff --git a/src/aks-preview/az_aks_tool/tests/test_filter.py b/src/aks-preview/az_aks_tool/tests/test_filter.py new file mode 100644 index 00000000000..d9da9e6a624 --- /dev/null +++ b/src/aks-preview/az_aks_tool/tests/test_filter.py @@ -0,0 +1,121 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import unittest + +import az_aks_tool.utils as utils +import az_aks_tool.filter as custom_filter + +THIS_FILE = os.path.abspath(__file__) +THIS_DIR = os.path.dirname(THIS_FILE) +PARENT_DIR = os.path.dirname(THIS_DIR) + + +class FilterTestCase(unittest.TestCase): + + def test_extract_file_class_pairs(self): + test_cases = ["abc", "a.b", "a.b.c"] + s = custom_filter.extract_file_class_pairs(test_cases) + self.assertEqual(s, [("a", "b")]) + + def test_filter_valid_file_class_pairs(self): + pairs = [("a", "b"), ("c", "d"), ("e", "f")] + test_index = { + "a": { + "b": ["aaa", "bbb"] + }, + "x": { + "y": ["xxx", "yyy"] + } + } + s = custom_filter.filter_valid_file_class_pairs(pairs, test_index) + self.assertEqual(s, [("a", "b")]) + + def test_get_all_values_from_nested_dict(self): + d = { + "a": { + "b": ["aaa", "bbb"] + }, + "x": ["xxx"] + } + s = list(custom_filter.get_all_values_from_nested_dict(d)) + self.assertEqual(s, [["aaa", "bbb"], ["xxx"]]) + + def test_flatten_nested_list(self): + lis = [["aaa", "bbb"], ["xxx"]] + s = list(custom_filter.flatten_nested_list(lis)) + self.assertEqual(s, ["aaa", "bbb", "xxx"]) + + def test_filter_valid_test_cases(self): + test_cases = ["abc", "a.b", "a.b.c", "aaa", "yyy"] + test_index = { + "a": { + "b": ["aaa", "bbb"] + }, + "x": { + "y": ["xxx", "yyy"] + } + } + s = custom_filter.filter_valid_test_cases(test_cases, test_index) + self.assertEqual(s, ["aaa", "yyy"]) + + def test_get_test_cases(self): + test_index = { + "test_validators": { + "TestValidateIPRanges": ["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces", "test_disable_authorized_ip_ranges", "test_local_ip_address", "test_invalid_ip", "test_IPv6"], + "TestClusterAutoscalerParamsValidators": ["test_empty_key_empty_value", "test_non_empty_key_empty_value", "test_two_empty_keys_empty_value", "test_one_empty_key_in_pair_one_non_empty", "test_invalid_key", "test_valid_parameters"], + "TestSubnetId": ["test_invalid_subnet_id", "test_valid_vnet_subnet_id", "test_none_vnet_subnet_id", "test_empty_vnet_subnet_id"] + } + } + matrix = utils.get_test_matrix( + os.path.join(THIS_DIR, "testdata.json")) + base = test_index["test_validators"] + s1 = sorted(custom_filter.get_test_cases(test_index, matrix, [])) + t1 = sorted((base["TestValidateIPRanges"] + + base["TestClusterAutoscalerParamsValidators"])) + s2 = sorted(custom_filter.get_test_cases(test_index, matrix, [ + "test_valid_vnet_subnet_id", "abc"])) + t2 = sorted((base["TestValidateIPRanges"] + base["TestClusterAutoscalerParamsValidators"] + + ["test_valid_vnet_subnet_id"])) + s3 = sorted(custom_filter.get_test_cases(test_index, matrix, [ + "test_validators.TestSubnetId"])) + t3 = sorted((base["TestValidateIPRanges"] + + base["TestClusterAutoscalerParamsValidators"] + base["TestSubnetId"])) + self.assertEqual(s1, t1) + self.assertEqual(s2, t2) + self.assertEqual(s3, t3) + + def test_get_exclude_test_cases(self): + test_index = { + "test_validators": { + "TestValidateIPRanges": ["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces", "test_disable_authorized_ip_ranges", "test_local_ip_address", "test_invalid_ip", "test_IPv6"], + "TestClusterAutoscalerParamsValidators": ["test_empty_key_empty_value", "test_non_empty_key_empty_value", "test_two_empty_keys_empty_value", "test_one_empty_key_in_pair_one_non_empty", "test_invalid_key", "test_valid_parameters"], + "TestSubnetId": ["test_invalid_subnet_id", "test_valid_vnet_subnet_id", "test_none_vnet_subnet_id", "test_empty_vnet_subnet_id"] + } + } + matrix = utils.get_test_matrix( + os.path.join(THIS_DIR, "testdata.json")) + base = test_index["test_validators"] + s1 = sorted(custom_filter.get_exclude_test_cases( + test_index, matrix, [])) + t1 = sorted(["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces", + "test_disable_authorized_ip_ranges"] + base["TestClusterAutoscalerParamsValidators"]) + s2 = sorted(custom_filter.get_exclude_test_cases(test_index, + matrix, ["iprange"])) + t2 = sorted(["test_simultaneous_allow_and_disallow_with_spaces", + "test_simultaneous_enable_and_disable_with_spaces", "test_disable_authorized_ip_ranges"]) + s3 = sorted(custom_filter.get_exclude_test_cases(test_index, + matrix, ["default", "test_invalid_subnet_id"])) + t3 = sorted(["test_simultaneous_allow_and_disallow_with_spaces", "test_simultaneous_enable_and_disable_with_spaces", + "test_disable_authorized_ip_ranges"] + base["TestClusterAutoscalerParamsValidators"] + ["test_invalid_subnet_id"]) + s4 = sorted(custom_filter.get_exclude_test_cases(test_index, matrix, [ + "test_valid_vnet_subnet_id", "abc", "test_validators.TestClusterAutoscalerParamsValidators"])) + t4 = sorted( + (base["TestClusterAutoscalerParamsValidators"] + ["test_valid_vnet_subnet_id"])) + self.assertEqual(s1, t1) + self.assertEqual(s2, t2) + self.assertEqual(s3, t3) + self.assertEqual(s4, t4) diff --git a/src/aks-preview/az_aks_tool/tests/test_index.py b/src/aks-preview/az_aks_tool/tests/test_index.py new file mode 100644 index 00000000000..68b5816db2f --- /dev/null +++ b/src/aks-preview/az_aks_tool/tests/test_index.py @@ -0,0 +1,34 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import unittest + +import az_aks_tool.utils as utils +import az_aks_tool.index as index + +THIS_FILE = os.path.abspath(__file__) +THIS_DIR = os.path.dirname(THIS_FILE) +PARENT_DIR = os.path.dirname(THIS_DIR) + +class IndexTestCase(unittest.TestCase): + + def test_get_repo_path(self): + pass + + def test_find_files(self): + pass + + def test_get_name_index(self): + pass + + def test_get_path_table(self): + pass + + def test_discover_module_tests(self): + pass + + def build_test_index(self): + pass diff --git a/src/aks-preview/az_aks_tool/tests/test_log.py b/src/aks-preview/az_aks_tool/tests/test_log.py new file mode 100644 index 00000000000..4caafc341ba --- /dev/null +++ b/src/aks-preview/az_aks_tool/tests/test_log.py @@ -0,0 +1,34 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import logging +import os +import unittest + +import az_aks_tool.log as log + +THIS_FILE = os.path.abspath(__file__) +THIS_DIR = os.path.dirname(THIS_FILE) +PARENT_DIR = os.path.dirname(THIS_DIR) + + +class LogTestCase(unittest.TestCase): + + def test_parse_module_name(self): + root_module_name = log.parse_module_name(levels=1) + error_module_name = log.parse_module_name(levels=5) + self.assertEqual(root_module_name, "az_aks_tool") + self.assertEqual(error_module_name, None) + + def test_setup_logging(self): + log.setup_logging("unittest", "unittest_log.log") + logger = logging.getLogger("unittest.test_setup_logging") + logger.debug("test setup logging") + logger.info("test setup logging") + logger.warning("test setup logging") + f = open("unittest_log.log", "r") + raw_logs = f.readlines() + f.close() + self.assertEqual(len(raw_logs), 3) diff --git a/src/aks-preview/az_aks_tool/tests/test_main.py b/src/aks-preview/az_aks_tool/tests/test_main.py new file mode 100644 index 00000000000..36f4becd0f4 --- /dev/null +++ b/src/aks-preview/az_aks_tool/tests/test_main.py @@ -0,0 +1,21 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import logging +import os +import unittest + +import az_aks_tool.main as main + +THIS_FILE = os.path.abspath(__file__) +THIS_DIR = os.path.dirname(THIS_FILE) +PARENT_DIR = os.path.dirname(THIS_DIR) + + +class MainTestCase(unittest.TestCase): + + def test_init_argparse(self): + args = main.init_argparse(["-p", "./"]) + self.assertEqual(args.report_path, "./") diff --git a/src/aks-preview/az_aks_tool/tests/test_run.py b/src/aks-preview/az_aks_tool/tests/test_run.py new file mode 100644 index 00000000000..d69375d491d --- /dev/null +++ b/src/aks-preview/az_aks_tool/tests/test_run.py @@ -0,0 +1,33 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import unittest + +import az_aks_tool.utils as utils +import az_aks_tool.cli as cli + +THIS_FILE = os.path.abspath(__file__) +THIS_DIR = os.path.dirname(THIS_FILE) +PARENT_DIR = os.path.dirname(THIS_DIR) + +class RunTestCase(unittest.TestCase): + + def test_current_profile(self): + pass + + def test_call(self): + pass + + def test_cmd(self): + pass + + def test_get_test_runner(self): + pass + + def run_tests(self): + pass + + diff --git a/src/aks-preview/az_aks_tool/tests/test_utils.py b/src/aks-preview/az_aks_tool/tests/test_utils.py new file mode 100644 index 00000000000..6bef0b49a0f --- /dev/null +++ b/src/aks-preview/az_aks_tool/tests/test_utils.py @@ -0,0 +1,54 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +import unittest + +import az_aks_tool.utils as utils + +THIS_FILE = os.path.abspath(__file__) +THIS_DIR = os.path.dirname(THIS_FILE) +PARENT_DIR = os.path.dirname(THIS_DIR) + + +class UtilsTestCase(unittest.TestCase): + + def test_check_file_existence(self): + s1 = utils.check_file_existence(None) + s2 = utils.check_file_existence(THIS_DIR) + s3 = utils.check_file_existence(THIS_FILE) + s4 = utils.check_file_existence( + os.path.join(THIS_DIR, "testdata.json")) + self.assertEqual(s1, False) + self.assertEqual(s2, False) + self.assertEqual(s3, True) + self.assertEqual(s4, True) + + def test_get_test_matrix(self): + s = utils.get_test_matrix(os.path.join(THIS_DIR, "testdata.json")) + self.assertEqual(len(s), 2) # number of keys + + def test_get_filted_test_cases(self): + test_cases = ["a", "b", "c", "d"] + s1 = utils.get_filted_test_cases(test_cases, []) + t1 = ["a", "b", "c", "d"] + s2 = utils.get_filted_test_cases(test_cases, ["a", "x"]) + t2 = ["b", "c", "d"] + self.assertEqual(s1, t1) + self.assertEqual(s2, t2) + + def test_add_qualified_prefix(self): + test_cases = ["a", "b", "c"] + s = utils.add_qualified_prefix(test_cases, "p") + self.assertEqual(s, ["p.a", "p.b", "p.c"]) + + def test_get_fully_qualified_test_cases(self): + pass + + def test_heading(self): + pass + + def test_extract_module_name(self): + pass diff --git a/src/aks-preview/az_aks_tool/tests/testdata.json b/src/aks-preview/az_aks_tool/tests/testdata.json new file mode 100644 index 00000000000..221e6291e3e --- /dev/null +++ b/src/aks-preview/az_aks_tool/tests/testdata.json @@ -0,0 +1,18 @@ +{ + "coverage": { + "test_validators": [ + "TestValidateIPRanges", + "TestClusterAutoscalerParamsValidators" + ] + }, + "exclude": { + "iprange": [ + "test_simultaneous_allow_and_disallow_with_spaces", + "test_simultaneous_enable_and_disable_with_spaces", + "test_disable_authorized_ip_ranges" + ], + "test_validators": [ + "TestClusterAutoscalerParamsValidators" + ] + } +} \ No newline at end of file diff --git a/src/aks-preview/az_aks_tool/utils.py b/src/aks-preview/az_aks_tool/utils.py new file mode 100644 index 00000000000..f92117bbf56 --- /dev/null +++ b/src/aks-preview/az_aks_tool/utils.py @@ -0,0 +1,83 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import json +import os +import sys +import re +import logging + +import az_aks_tool.const as const +import az_aks_tool.filter as custom_filter +logger = logging.getLogger(__name__) + + +def check_file_existence(file_path): + if file_path is not None and os.path.isfile(file_path): + return True + return False + + +def get_test_matrix(matrix_file_path): + test_matrix = {} + if check_file_existence(matrix_file_path): + json_file = open(matrix_file_path, 'r') + test_matrix = json.load(json_file) + json_file.close() + else: + logger.warning("Matrix file '{}' not exists!".format(matrix_file_path)) + return test_matrix + + +def get_filted_test_cases(test_cases, exclude_test_cases): + filtered_test_cases = [ + x for x in test_cases if x not in exclude_test_cases] + logger.info("Find {} cases, exclude {} cases, finally get {} cases!".format( + len(test_cases), len(exclude_test_cases), len(filtered_test_cases))) + return filtered_test_cases + + +def add_qualified_prefix(test_cases, prefix): + decorated_test_cases = ["{}.{}".format(prefix, x) for x in test_cases] + return decorated_test_cases + + +def get_fully_qualified_test_cases(test_index, matrix_file_path, mod_name, extra_coverage=None, extra_filter=None): + qualified_test_cases = [] + matrix = get_test_matrix(matrix_file_path) + test_cases = custom_filter.get_test_cases( + test_index, matrix, extra_coverage) + exclude_test_cases = custom_filter.get_exclude_test_cases(test_index, + matrix, extra_filter) + filtered_test_cases = get_filted_test_cases( + test_cases, exclude_test_cases) + # add prefix + qualified_test_cases = add_qualified_prefix( + filtered_test_cases, mod_name) + return qualified_test_cases + + +def heading(txt): + """ Create standard heading to stderr """ + line_len = len(txt) + 4 + print('\n' + '=' * line_len, file=sys.stderr) + print('| {} |'.format(txt), file=sys.stderr) + print('=' * line_len + '\n', file=sys.stderr) + + +def extract_module_name(path): + _CORE_NAME_REGEX = re.compile( + r'azure-cli-(?P[^/\\]+)[/\\]azure[/\\]cli') + _MOD_NAME_REGEX = re.compile( + r'azure-cli[/\\]azure[/\\]cli[/\\]command_modules[/\\](?P[^/\\]+)') + _EXT_NAME_REGEX = re.compile(r'.*(?Pazext_[^/\\]+).*') + + for expression in [_MOD_NAME_REGEX, _CORE_NAME_REGEX, _EXT_NAME_REGEX]: + match = re.search(expression, path) + if not match: + continue + return match.groupdict().get('name') + raise Exception( + 'unexpected error: unable to extract name from path: {}'.format(path)) diff --git a/src/aks-preview/azcli_aks_live_test/.gitignore b/src/aks-preview/azcli_aks_live_test/.gitignore new file mode 100644 index 00000000000..819270b0650 --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/.gitignore @@ -0,0 +1,5 @@ +env.list +*.json +*.xml +!ext_matrix_default.json +!testdata.json diff --git a/src/aks-preview/azcli_aks_live_test/HISTORY.rst b/src/aks-preview/azcli_aks_live_test/HISTORY.rst new file mode 100644 index 00000000000..93372ba3c64 --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/HISTORY.rst @@ -0,0 +1,9 @@ +.. :changelog: + +Release History +=============== + +0.1.0 (4/23/2021) +++++++ + +* Add live test pipeline for aks commands diff --git a/src/aks-preview/azcli_aks_live_test/README.md b/src/aks-preview/azcli_aks_live_test/README.md new file mode 100644 index 00000000000..206a5eae203 --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/README.md @@ -0,0 +1,9 @@ +# Azure CLI AKS Live Test Pipeline & Azure CLI AKS Unit Test Pipeline + +These pipelines are used to test newly added aks commands in module aks-preview (azure-cli-extensions) / acs (azure-cli, not covered by default). + +## How to use + +**By default**, these pipelines will be **triggered** when submitting a **PR** to the master branch of the official repo which involves modifying the files under src/aks-preview. Then they will test the aks-preview command group in azure-cli-extensions. For live test pipeline, the test will be performed in record mode first, and then in live mode. Due to some specific [reasons](https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/157433/Live-Test-Failures-in-aks-preview-(with-bare-sub)), some test cases would fail. These test cases have been filtered out by file 'ext_matrix_default.json'. For unit test pipeline, the test will be perfomed with 'unittest' and 'pytest' modules. A code coverage report will be generated after the unit tests. You can find test reports and coverage report from pipeline artifacts. + +You can also trigger this pipeline **manually**. For more details, you may refer to this [wiki](https://dev.azure.com/msazure/CloudNativeCompute/_wiki/wikis/CloudNativeCompute.wiki/156735/Azure-CLI-AKS-Live-Test-Pipeline). diff --git a/src/aks-preview/azcli_aks_live_test/clean_up.sh b/src/aks-preview/azcli_aks_live_test/clean_up.sh new file mode 100755 index 00000000000..4bbeb2f3520 --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/clean_up.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +# login +az login --service-principal -u $AZCLI_ALT_CLIENT_ID -p $AZCLI_ALT_CLIENT_SECRET -t $TENANT_ID +az account set -s $AZCLI_ALT_SUBSCRIPTION_ID +az account show + +# list the details of resource groups whose names start with "clitest" +az group list --query "([].name)[?starts_with(@, 'clitest')]" -o tsv | xargs -i az group show -n {} + +# delete all resource groups whose names start with "clitest" +# az group list --query "([].name)[?starts_with(@, 'clitest')]" -o tsv | xargs -i az group delete --no-wait -y -n {} diff --git a/src/aks-preview/azcli_aks_live_test/clone_repo.sh b/src/aks-preview/azcli_aks_live_test/clone_repo.sh new file mode 100755 index 00000000000..4dccf3260d4 --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/clone_repo.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash + +set -eux +pwd + +# clone azure-cli (default is the official repo) +# git clone https://github.com/Azure/azure-cli.git +git clone $CLI_REPO + +# ckeckout to a specific azure-cli branch (default is the dev branch) +pushd azure-cli/ +git branch -a +git checkout $CLI_BRANCH +popd + +# clone azure-cli-extensions when manually specify the extension repo +if [[ $MANUAL_EXT == true && -n $EXT_REPO && -n $EXT_BRANCH ]]; then + echo "Manually specify the extension repo, delete the current 'azure-cli-extensions' directory!" + rm -rf azure-cli-extensions/ + git clone $EXT_REPO + pushd azure-cli-extensions/ + git checkout $EXT_BRANCH + popd +fi + +# check current branch & commit logs in azure-cli-extensions +pushd azure-cli-extensions/ +git branch -a +git log -10 +popd + +# copy live test related files to the same level as the checkout directory ($(Agent.BuildDirectory)/s) +cp -rT azure-cli-extensions/src/aks-preview/azcli_aks_live_test/ ./ +cp -r azure-cli-extensions/src/aks-preview/az_aks_tool/ ./ +ls -alh diff --git a/src/aks-preview/azcli_aks_live_test/ext_matrix_default.json b/src/aks-preview/azcli_aks_live_test/ext_matrix_default.json new file mode 100644 index 00000000000..13d3d7295af --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/ext_matrix_default.json @@ -0,0 +1,39 @@ +{ + "coverage": { + "test_aks_commands": [ + "AzureKubernetesServiceScenarioTest" + ] + }, + "exclude": { + "need additional feature": [ + "test_aks_create_addon_with_azurekeyvaultsecretsprovider_with_secret_rotation", + "test_aks_create_with_auto_upgrade_channel", + "test_aks_create_with_pod_identity_enabled", + "test_aks_create_with_azurekeyvaultsecretsprovider_addon", + "test_aks_create_using_azurecni_with_pod_identity_enabled", + "test_aks_create_with_gitops_addon", + "test_aks_create_with_node_config", + "test_aks_custom_kubelet_identity", + "test_aks_disable_addon_gitops", + "test_aks_disable_addon_openservicemesh", + "test_aks_pod_identity_usage", + "test_aks_create_with_openservicemesh_addon", + "test_aks_update_azurekeyvaultsecretsprovider_with_secret_rotation", + "test_aks_enable_addon_with_azurekeyvaultsecretsprovider", + "test_aks_enable_addon_with_gitops", + "test_aks_create_with_fips" + ], + "unknown": [ + "test_aks_create_and_update_with_managed_aad_enable_azure_rbac", + "test_aks_create_with_virtual_node_addon" + ], + "code bug": [ + "test_aks_create_with_ingress_appgw_addon_with_deprecated_subet_prefix", + "test_aks_byo_subnet_with_ingress_appgw_addon", + "test_aks_nodepool_get_upgrades", + "test_aks_byo_appgw_with_ingress_appgw_addon", + "test_aks_enable_addon_with_openservicemesh", + "test_aks_create_with_ingress_appgw_addon" + ] + } +} \ No newline at end of file diff --git a/src/aks-preview/azcli_aks_live_test/prepare_image.sh b/src/aks-preview/azcli_aks_live_test/prepare_image.sh new file mode 100755 index 00000000000..32793cd90ee --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/prepare_image.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +set -eux +pwd + +# prepare docker image +echo "Pulling test image from '$IMAGE_PREFIX/$IMAGE_NAME:$IMAGE_TAG'..." +docker pull $IMAGE_PREFIX/$IMAGE_NAME:$IMAGE_TAG +docker tag $IMAGE_PREFIX/$IMAGE_NAME:$IMAGE_TAG $IMAGE_NAME:$IMAGE_TAG diff --git a/src/aks-preview/azcli_aks_live_test/setup_venv.sh b/src/aks-preview/azcli_aks_live_test/setup_venv.sh new file mode 100755 index 00000000000..642d9a7c436 --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/setup_venv.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +set -eux +pwd +ls -alh + +# delete existing venv +rm -rf azEnv || true + +# install python packages +python$PYTHON_VERSION -m venv azEnv +source azEnv/bin/activate +python -m pip install -U pip +# fixed azdev version to avoid call failure in az_aks_tool +pip install azdev==0.1.32 +# install pytest plugins +pip install pytest-json-report pytest-rerunfailures --upgrade +# pip install pytest-html --upgrade +# module for measuring code coverage +pip install coverage + +# pre-install: check existing az +which az || az version || az extension list || true + +# install az from cloned repos with azdev +azdev setup -c azure-cli/ -r azure-cli-extensions/ +deactivate +source azEnv/bin/activate + +# post-install: check installation result +which az +az version + +# mkdir to store reports +mkdir -p reports/ + +# export PYTHONPATH +export PYTHONPATH=$(pwd) diff --git a/src/aks-preview/azcli_aks_live_test/start_container.sh b/src/aks-preview/azcli_aks_live_test/start_container.sh new file mode 100755 index 00000000000..dee2c5278b3 --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/start_container.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -eux +pwd + +# transcribe environment variables into file 'env.list' +source ./transcribe_env.sh + +# take the first arg as container name +container_name=${1:-"azcli-aks-live-test-container"} + +# start container in backgroud with tty +# mount current directory ($(Agent.BuildDirectory)/s) to /opt in container +# set working directory as /opt in container +# pass secrects as environment variables directly (instead of storing in env.list) +# pass other environment variables with env.list +docker run -t -d -v $PWD:/opt -w /opt \ +-e AZCLI_ALT_CLIENT_SECRET=$MAPPED_AZCLI_ALT_CLIENT_SECRET \ +-e AZURE_CLI_TEST_DEV_SP_PASSWORD=$MAPPED_AZCLI_ALT_CLIENT_SECRET \ +--env-file ./env.list \ +--name $container_name $IMAGE_NAME:$IMAGE_TAG + +# remove env.list +rm ./env.list diff --git a/src/aks-preview/azcli_aks_live_test/test_cli_live.sh b/src/aks-preview/azcli_aks_live_test/test_cli_live.sh new file mode 100755 index 00000000000..56e7a7c085b --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/test_cli_live.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -eux +pwd + +# activate virtualenv +source azEnv/bin/activate + +# remove extension +echo "Remove existing aks-preview extension (if any)" +if az extension remove --name aks-preview || azdev extension remove aks-preview; then + deactivate + source azEnv/bin/activate +fi + +# test cli +if [[ $TEST_MODE == "record" || $TEST_MODE == "all" ]]; then + echo "Test in record mode!" + azdev test acs --no-exitfirst --xml-path cli_result.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=cli_report.json --reruns 3 --capture=sys" + cp *cli_report.json *cli_result.xml reports/ +fi + +if [[ $TEST_MODE == "live" || $TEST_MODE == "all" ]]; then + echo "Test in live mode!" + az login --service-principal -u $AZCLI_ALT_CLIENT_ID -p $AZCLI_ALT_CLIENT_SECRET -t $TENANT_ID + az account set -s $AZCLI_ALT_SUBSCRIPTION_ID + az account show + azdev test acs --live --no-exitfirst --xml-path cli_live_result.xml --discover -a "-n $PARALLELISM --json-report --json-report-file=cli_live_report.json --reruns 3 --capture=sys" + cp *cli_live_report.json *cli_live_result.xml reports/ +fi diff --git a/src/aks-preview/azcli_aks_live_test/test_cli_unit.sh b/src/aks-preview/azcli_aks_live_test/test_cli_unit.sh new file mode 100755 index 00000000000..8ef79d3e09c --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/test_cli_unit.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash + +set -eux +pwd + +# activate virtualenv +source azEnv/bin/activate + +# remove extension +echo "Remove existing aks-preview extension (if any)" +if az extension remove --name aks-preview || azdev extension remove aks-preview; then + deactivate + source azEnv/bin/activate +fi + +echo "Implementing, pass for now!" diff --git a/src/aks-preview/azcli_aks_live_test/test_ext_live.sh b/src/aks-preview/azcli_aks_live_test/test_ext_live.sh new file mode 100755 index 00000000000..97fbbfcf254 --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/test_ext_live.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +set -eux +pwd + +# activate virtualenv +source azEnv/bin/activate + +# remove extension +echo "Remove existing aks-preview extension (if any)" +if az extension remove --name aks-preview || azdev extension remove aks-preview; then + deactivate + source azEnv/bin/activate +fi + +# install latest extension +echo "Install the latest aks-preview extension and re-activate the virtualenv" +azdev extension add aks-preview +az extension list +azdev extension list | grep "aks-preview" -C 5 +deactivate +source azEnv/bin/activate + +# Ensure that the command index is updated by calling a specific command in aks-preview, so that all the commands defined in aks-preview are loaded correctly +# Otherwise, cold boot execution of azdev test may use the api version adopted by the acs command group in azure-cli (which may diverge from the api version used in current aks-preview) +retry_count=0 +while ! az aks maintenanceconfiguration show --help && [[ $retry_count < 3 ]] +do + retry_count=`expr $retry_count + 1` + echo $retry_count"th retry to install aks-preview..." + azdev extension add aks-preview --debug + az extension list --debug + azdev extension list --debug | grep "aks-preview" -C 5 + deactivate + source azEnv/bin/activate +done + +# prepare run flags +run_flags="-e -em ext_matrix_default.json --no-exitfirst --report-path ./ --reruns 3 --capture=sys" +# parallel +if [ $PARALLELISM -ge 2 ]; then + run_flags+=" -j $PARALLELISM" +else + run_flags+=" -s" +fi +# test cases +if [[ -n $TEST_CASES ]]; then + run_flags+=" -t $TEST_CASES" +fi +# ext extra filter +if [[ -n $EXT_TEST_FILTER ]]; then + run_flags+=" -ef $EXT_TEST_FILTER" +fi +# ext extra coverage +if [[ -n $EXT_TEST_COVERAGE ]]; then + run_flags+=" -ec $EXT_TEST_COVERAGE" +fi + +# recording test +if [[ $TEST_MODE == "record" || $TEST_MODE == "all" ]]; then + echo "Test in record mode!" + run_flags+=" --json-report-file=ext_report.json" + run_flags+=" --xml-file=ext_result.xml" + echo "run flags: ${run_flags}" + echo ${run_flags} | xargs python -u az_aks_tool/main.py + cp *ext_report.json *ext_result.xml reports/ +fi + +# live test +if [[ $TEST_MODE == "live" || $TEST_MODE == "all" ]]; then + echo "Test in live mode!" + az login --service-principal -u $AZCLI_ALT_CLIENT_ID -p $AZCLI_ALT_CLIENT_SECRET -t $TENANT_ID + az account set -s $AZCLI_ALT_SUBSCRIPTION_ID + az account show + run_flags+=" -l --json-report-file=ext_live_report.json" + run_flags+=" --xml-file=ext_live_result.xml" + echo "run flags: ${run_flags}" + echo ${run_flags} | xargs python -u az_aks_tool/main.py + cp *ext_live_report.json *ext_live_result.xml reports/ +fi diff --git a/src/aks-preview/azcli_aks_live_test/test_ext_unit.sh b/src/aks-preview/azcli_aks_live_test/test_ext_unit.sh new file mode 100755 index 00000000000..a83fa7568ea --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/test_ext_unit.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +set -eux +pwd + +# activate virtualenv +source azEnv/bin/activate + +# remove extension +echo "Remove existing aks-preview extension (if any)" +if az extension remove --name aks-preview || azdev extension remove aks-preview; then + deactivate + source azEnv/bin/activate +fi + +# install latest extension +echo "Install the latest aks-preview extension and re-activate the virtualenv" +azdev extension add aks-preview +az extension list +azdev extension list | grep "aks-preview" -C 5 +deactivate +source azEnv/bin/activate + +# Ensure that the command index is updated by calling a specific command in aks-preview, so that all the commands defined in aks-preview are loaded correctly +# Otherwise, cold boot execution of azdev test may use the api version adopted by the acs command group in azure-cli (which may diverge from the api version used in current aks-preview) +retry_count=0 +while ! az aks command invoke --help && [[ $retry_count < 3 ]] +do + retry_count=`expr $retry_count + 1` + echo $retry_count"th retry to install aks-preview..." + azdev extension add aks-preview --debug + az extension list --debug + azdev extension list --debug | grep "aks-preview" -C 5 + deactivate + source azEnv/bin/activate +done + +# unit test & coverage report +# az_aks_tool +az_aks_tool_unit_test_result="" +pushd azure-cli-extensions/src/aks-preview/az_aks_tool/ +# clean existing coverage report +(coverage combine || true) && (coverage erase || true) +if ! coverage run --source=. --omit=*/tests/* -p -m unittest discover; then + az_aks_tool_unit_test_result="error" +fi +# currently no test written in pytest format under 'az_aks_tool/' +# coverage run --source=. --omit=*/tests/* -p -m pytest +coverage combine && coverage json -o coverage_az_aks_tool.json +coverage report -m +popd +cp azure-cli-extensions/src/aks-preview/az_aks_tool/coverage_az_aks_tool.json reports/ + +# azext_aks_preview +azext_aks_preview_unit_test_result="" +pushd azure-cli-extensions/src/aks-preview/azext_aks_preview +# clean existing coverage report +(coverage combine || true) && (coverage erase || true) +# currently test using module 'unittest' is the same as module 'pytest', and test using 'pytest' is just recording test +if ! coverage run --source=. --omit=*/vendored_sdks/*,*/tests/* -p -m unittest discover || ! coverage run --source=. --omit=*/vendored_sdks/*,*/tests/* -p -m pytest; then + azext_aks_preview_unit_test_result="error" +fi +coverage combine && coverage json -o coverage_azext_aks_preview.json +coverage report -m +popd +cp azure-cli-extensions/src/aks-preview/azext_aks_preview/coverage_azext_aks_preview.json reports/ + +if [[ $az_aks_tool_unit_test_result == "error" || $azext_aks_preview_unit_test_result == "error" ]]; then + echo "Unit test failed!" + exit 1 +fi diff --git a/src/aks-preview/azcli_aks_live_test/transcribe_env.sh b/src/aks-preview/azcli_aks_live_test/transcribe_env.sh new file mode 100755 index 00000000000..228698838ee --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/transcribe_env.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# clear +cat /dev/null > env.list + +# tenant, sub, client +echo "TENANT_ID=$TENANT_ID" >> env.list +echo "AZCLI_ALT_SUBSCRIPTION_ID=$AZCLI_ALT_SUBSCRIPTION_ID" >> env.list +echo "AZCLI_ALT_CLIENT_ID=$AZCLI_ALT_CLIENT_ID" >> env.list + +# azdev env +echo "AZURE_CLI_TEST_DEV_SP_NAME=$AZCLI_ALT_CLIENT_ID" >> env.list +echo "AZURE_CLI_TEST_DEV_RESOURCE_GROUP_LOCATION=$TEST_LOCATION" >> env.list + +# predefined variables +echo "BUILD_REASON=$BUILD_REASON" >> env.list +echo "SYSTEM_PULLREQUEST_TARGETBRANCH=$SYSTEM_PULLREQUEST_TARGETBRANCH" >> env.list + +# test +echo "COVERAGE=$COVERAGE" >> env.list +echo "TEST_MODE=$TEST_MODE" >> env.list +echo "PARALLELISM=$PARALLELISM" >> env.list +echo "TEST_CASES=$TEST_CASES" >> env.list +echo "EXT_TEST_FILTER=$EXT_TEST_FILTER" >> env.list +echo "EXT_TEST_COVERAGE=$EXT_TEST_COVERAGE" >> env.list + +# repo +echo "CLI_REPO=$CLI_REPO" >> env.list +echo "CLI_BRANCH=$CLI_BRANCH" >> env.list +echo "MANUAL_EXT=$MANUAL_EXT" >> env.list +echo "EXT_REPO=$EXT_REPO" >> env.list +echo "EXT_BRANCH=$EXT_BRANCH" >> env.list + +# image +echo "IMAGE_PREFIX=$IMAGE_PREFIX" >> env.list +echo "IMAGE_NAME=$IMAGE_NAME" >> env.list +echo "IMAGE_TAG=$IMAGE_TAG" >> env.list + +# misc +echo "PYTHON_VERSION=$PYTHON_VERSION" >> env.list diff --git a/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml new file mode 100644 index 00000000000..22b34e0fe63 --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-live-test.yaml @@ -0,0 +1,56 @@ +name: $(Date:yyyyMMdd)$(Rev:.r)_Python$(PYTHON_VERSION)_Coverage-$(COVERAGE)_Mode-$(TEST_MODE)_Branch-$(Build.SourceBranchName) + +trigger: none + +jobs: +- job: LiveTest + pool: + vmImage: 'ubuntu-16.04' + timeoutInMinutes: 360 + displayName: "Live Test with Python" + steps: + - bash: | + pwd + ls -alh + mkdir azure-cli-extensions + shopt -s extglob dotglob + mv !(azure-cli-extensions) azure-cli-extensions + shopt -u extglob dotglob + ls -alh + displayName: "Move All Checkout Files to the Newly Created 'azure-cli-extensions' Directory" + - bash: | + source ./azure-cli-extensions/src/aks-preview/azcli_aks_live_test/clone_repo.sh + condition: succeeded() + displayName: "Clone GitHub Repo and Move Live Test Related Files" + - bash: | + source ./prepare_image.sh + condition: succeeded() + displayName: "Prepare Live Test Image" + - bash: | + source ./start_container.sh + env: + MAPPED_AZCLI_ALT_CLIENT_SECRET: $(AZCLI_ALT_CLIENT_SECRET) + BUILD_REASON: $(Build.Reason) + SYSTEM_PULLREQUEST_TARGETBRANCH: $(System.PullRequest.TargetBranch) + condition: succeeded() + displayName: "Start Container" + - bash: | + docker exec "azcli-aks-live-test-container" /opt/setup_venv.sh + condition: succeeded() + displayName: "Set up Virtual Environment" + - bash: | + docker exec "azcli-aks-live-test-container" /opt/test_cli_live.sh + condition: and(succeeded(), in(variables['COVERAGE'], 'cli', 'all')) + displayName: Perform Live Test for CLI + - bash: | + docker exec "azcli-aks-live-test-container" /opt/test_ext_live.sh + condition: and(succeededOrFailed(), in(variables['COVERAGE'], 'ext', 'all')) + displayName: Perform Live Test for EXT + - task: CopyFiles@2 + inputs: + contents: 'reports/**' + targetFolder: $(Build.ArtifactStagingDirectory) + - task: PublishBuildArtifacts@1 + inputs: + pathToPublish: $(Build.ArtifactStagingDirectory) + artifactName: 'live test reports' diff --git a/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-unit-test.yaml b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-unit-test.yaml new file mode 100644 index 00000000000..ce792eea0dd --- /dev/null +++ b/src/aks-preview/azcli_aks_live_test/vsts-azcli-aks-unit-test.yaml @@ -0,0 +1,56 @@ +name: $(Date:yyyyMMdd)$(Rev:.r)_Python$(PYTHON_VERSION)_Coverage-$(COVERAGE)_Mode-$(TEST_MODE)_Branch-$(Build.SourceBranchName) + +trigger: none + +jobs: +- job: UnitTest + pool: + vmImage: 'ubuntu-16.04' + timeoutInMinutes: 360 + displayName: "Unit Test with Python" + steps: + - bash: | + pwd + ls -alh + mkdir azure-cli-extensions + shopt -s extglob dotglob + mv !(azure-cli-extensions) azure-cli-extensions + shopt -u extglob dotglob + ls -alh + displayName: "Move All Checkout Files to the Newly Created 'azure-cli-extensions' Directory" + - bash: | + source ./azure-cli-extensions/src/aks-preview/azcli_aks_live_test/clone_repo.sh + condition: succeeded() + displayName: "Clone GitHub Repo and Move Test Related Files" + - bash: | + source ./prepare_image.sh + condition: succeeded() + displayName: "Prepare Test Image" + - bash: | + source ./start_container.sh "azcli-aks-unit-test-container" + env: + MAPPED_AZCLI_ALT_CLIENT_SECRET: $(AZCLI_ALT_CLIENT_SECRET) + BUILD_REASON: $(Build.Reason) + SYSTEM_PULLREQUEST_TARGETBRANCH: $(System.PullRequest.TargetBranch) + condition: succeeded() + displayName: "Start Container" + - bash: | + docker exec "azcli-aks-unit-test-container" /opt/setup_venv.sh + condition: succeeded() + displayName: "Set up Virtual Environment" + - bash: | + docker exec "azcli-aks-unit-test-container" /opt/test_cli_unit.sh + condition: and(succeeded(), in(variables['COVERAGE'], 'cli', 'all')) + displayName: Perform Unit Test for CLI + - bash: | + docker exec "azcli-aks-unit-test-container" /opt/test_ext_unit.sh + condition: and(succeededOrFailed(), in(variables['COVERAGE'], 'ext', 'all')) + displayName: Perform Unit Test for EXT + - task: CopyFiles@2 + inputs: + contents: 'reports/**' + targetFolder: $(Build.ArtifactStagingDirectory) + - task: PublishBuildArtifacts@1 + inputs: + pathToPublish: $(Build.ArtifactStagingDirectory) + artifactName: 'unit test reports' diff --git a/src/aks-preview/azext_aks_preview/tests/latest/custom_preparers.py b/src/aks-preview/azext_aks_preview/tests/latest/custom_preparers.py new file mode 100644 index 00000000000..f0d066b7e9b --- /dev/null +++ b/src/aks-preview/azext_aks_preview/tests/latest/custom_preparers.py @@ -0,0 +1,62 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +import os +from datetime import datetime + +from azure.cli.testsdk import CliTestError +from azure.cli.testsdk.preparers import NoTrafficRecordingPreparer +from azure_devtools.scenario_tests import SingleValueReplacer +from azure.cli.testsdk.reverse_dependency import get_dummy_cli + + +class AKSCustomResourceGroupPreparer(NoTrafficRecordingPreparer, SingleValueReplacer): + def __init__(self, name_prefix='clitest.rg', + parameter_name='resource_group', + parameter_name_for_location='resource_group_location', location='westus', + dev_setting_name='AZURE_CLI_TEST_DEV_RESOURCE_GROUP_NAME', + dev_setting_location='AZURE_CLI_TEST_DEV_RESOURCE_GROUP_LOCATION', + random_name_length=75, key='rg'): + if ' ' in name_prefix: + raise CliTestError( + 'Error: Space character in resource group name prefix \'%s\'' % name_prefix) + super(AKSCustomResourceGroupPreparer, self).__init__( + name_prefix, random_name_length) + self.cli_ctx = get_dummy_cli() + self.location = location + self.parameter_name = parameter_name + self.parameter_name_for_location = parameter_name_for_location + self.key = key + + self.dev_setting_name = os.environ.get(dev_setting_name, None) + # use environment variable to modify the default value of location + self.dev_setting_location = os.environ.get(dev_setting_location, None) + if self.dev_setting_location is not None and self.dev_setting_location != "": + self.location = self.dev_setting_location + + def create_resource(self, name, **kwargs): + if self.dev_setting_name: + self.test_class_instance.kwargs[self.key] = self.dev_setting_name + return {self.parameter_name: self.dev_setting_name, + self.parameter_name_for_location: self.dev_setting_location} + + tags = {'product': 'azurecli', 'cause': 'automation', + 'date': datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')} + if 'ENV_JOB_NAME' in os.environ: + tags['job'] = os.environ['ENV_JOB_NAME'] + tags = ' '.join(['{}={}'.format(key, value) + for key, value in tags.items()]) + template = 'az group create --location {} --name {} --tag ' + tags + self.live_only_execute( + self.cli_ctx, template.format(self.location, name)) + + self.test_class_instance.kwargs[self.key] = name + return {self.parameter_name: name, self.parameter_name_for_location: self.location} + + def remove_resource(self, name, **kwargs): + # delete group if test is being recorded and if the group is not a dev rg + if not self.dev_setting_name: + self.live_only_execute( + self.cli_ctx, 'az group delete --name {} --yes --no-wait'.format(name)) diff --git a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py index fba2c0da28b..a1eceef5f18 100644 --- a/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py +++ b/src/aks-preview/azext_aks_preview/tests/latest/test_aks_commands.py @@ -5,12 +5,14 @@ import unittest import os -from .recording_processors import KeyReplacer from azure.cli.testsdk import ( ResourceGroupPreparer, RoleBasedServicePrincipalPreparer, ScenarioTest, live_only) from azure_devtools.scenario_tests import AllowLargeResponse +from .recording_processors import KeyReplacer +from .custom_preparers import AKSCustomResourceGroupPreparer + def _get_test_data_file(filename): curr_dir = os.path.dirname(os.path.realpath(__file__)) @@ -45,7 +47,7 @@ def test_get_os_options(self): # without live only fails with needs .ssh fails (maybe generate-ssh-keys would fix) and maybe az login. @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_and_update_with_managed_aad(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -78,7 +80,7 @@ def test_aks_create_and_update_with_managed_aad(self, resource_group, resource_g # without live only fails with needs .ssh fails (maybe generate-ssh-keys would fix) and maybe az login. @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='canadacentral') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='canadacentral') def test_aks_create_aadv1_and_update_with_managed_aad(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -120,7 +122,7 @@ def test_aks_create_aadv1_and_update_with_managed_aad(self, resource_group, reso # without live only fails with needs .ssh fails (maybe generate-ssh-keys would fix) and maybe az login. @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='canadacentral') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='canadacentral') def test_aks_create_nonaad_and_update_with_managed_aad(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -152,7 +154,7 @@ def test_aks_create_nonaad_and_update_with_managed_aad(self, resource_group, res # without live only fails with needs .ssh fails (maybe generate-ssh-keys would fix) and maybe az login. @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_and_update_with_managed_aad_enable_azure_rbac(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -185,7 +187,7 @@ def test_aks_create_and_update_with_managed_aad_enable_azure_rbac(self, resource ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_ingress_appgw_addon(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -203,7 +205,7 @@ def test_aks_create_with_ingress_appgw_addon(self, resource_group, resource_grou ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_ingress_appgw_addon_with_deprecated_subet_prefix(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -222,7 +224,7 @@ def test_aks_create_with_ingress_appgw_addon_with_deprecated_subet_prefix(self, @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_byo_subnet_with_ingress_appgw_addon(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) vnet_name = self.create_random_name('cliakstest', 16) @@ -274,7 +276,7 @@ def test_aks_byo_subnet_with_ingress_appgw_addon(self, resource_group, resource_ @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_byo_appgw_with_ingress_appgw_addon(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) vnet_name = self.create_random_name('cliakstest', 16) @@ -345,7 +347,7 @@ def test_aks_byo_appgw_with_ingress_appgw_addon(self, resource_group, resource_g }) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_openservicemesh_addon(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -361,7 +363,7 @@ def test_aks_create_with_openservicemesh_addon(self, resource_group, resource_gr ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_enable_addon_with_openservicemesh(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -382,7 +384,7 @@ def test_aks_enable_addon_with_openservicemesh(self, resource_group, resource_gr ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_disable_addon_openservicemesh(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -405,7 +407,7 @@ def test_aks_disable_addon_openservicemesh(self, resource_group, resource_group_ ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_azurekeyvaultsecretsprovider_addon(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -430,7 +432,7 @@ def test_aks_create_with_azurekeyvaultsecretsprovider_addon(self, resource_group ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_addon_with_azurekeyvaultsecretsprovider_with_secret_rotation(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -455,7 +457,7 @@ def test_aks_create_addon_with_azurekeyvaultsecretsprovider_with_secret_rotation ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_enable_addon_with_azurekeyvaultsecretsprovider(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -503,7 +505,7 @@ def test_aks_enable_addon_with_azurekeyvaultsecretsprovider(self, resource_group ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_update_azurekeyvaultsecretsprovider_with_secret_rotation(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -546,7 +548,7 @@ def test_aks_update_azurekeyvaultsecretsprovider_with_secret_rotation(self, reso ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_confcom_addon(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -564,7 +566,7 @@ def test_aks_create_with_confcom_addon(self, resource_group, resource_group_loca ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_confcom_addon_helper_enabled(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -582,7 +584,7 @@ def test_aks_create_with_confcom_addon_helper_enabled(self, resource_group, reso ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_enable_addons_confcom_addon(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -606,7 +608,7 @@ def test_aks_enable_addons_confcom_addon(self, resource_group, resource_group_lo ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_disable_addons_confcom_addon(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -632,7 +634,7 @@ def test_aks_disable_addons_confcom_addon(self, resource_group, resource_group_l @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_virtual_node_addon(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) vnet_name = self.create_random_name('cliakstest', 16) @@ -686,7 +688,7 @@ def test_aks_create_with_virtual_node_addon(self, resource_group, resource_group @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_stop_and_start(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -706,7 +708,7 @@ def test_aks_stop_and_start(self, resource_group, resource_group_location): self.cmd(start_cmd) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_managed_disk(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -723,7 +725,7 @@ def test_aks_create_with_managed_disk(self, resource_group, resource_group_locat ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_ephemeral_disk(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -741,7 +743,7 @@ def test_aks_create_with_ephemeral_disk(self, resource_group, resource_group_loc ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_nodepool_get_upgrades(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) node_pool_name = self.create_random_name('c', 6) @@ -776,7 +778,7 @@ def test_aks_nodepool_get_upgrades(self, resource_group, resource_group_location 'aks delete -g {resource_group} -n {name} --yes --no-wait', checks=[self.is_empty()]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_upgrade_node_image_only_cluster(self, resource_group, resource_group_location): # kwargs for string formatting aks_name = self.create_random_name('cliakstest', 16) @@ -807,7 +809,7 @@ def test_aks_upgrade_node_image_only_cluster(self, resource_group, resource_grou ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_upgrade_node_image_only_nodepool(self, resource_group, resource_group_location): # kwargs for string formatting aks_name = self.create_random_name('cliakstest', 16) @@ -844,7 +846,7 @@ def test_aks_upgrade_node_image_only_nodepool(self, resource_group, resource_gro ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_windows(self, resource_group, resource_group_location): # reset the count so in replay mode the random names will start with 0 self.test_resources_count = 0 @@ -893,7 +895,7 @@ def test_aks_create_with_windows(self, resource_group, resource_group_location): 'aks delete -g {resource_group} -n {name} --yes --no-wait', checks=[self.is_empty()]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='eastus2euap') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='eastus2euap') def test_aks_create_with_fips(self, resource_group, resource_group_location): # reset the count so in replay mode the random names will start with 0 self.test_resources_count = 0 @@ -927,7 +929,7 @@ def test_aks_create_with_fips(self, resource_group, resource_group_location): 'aks delete -g {resource_group} -n {name} --yes --no-wait', checks=[self.is_empty()]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_ahub(self, resource_group, resource_group_location): # reset the count so in replay mode the random names will start with 0 self.test_resources_count = 0 @@ -978,7 +980,7 @@ def test_aks_create_with_ahub(self, resource_group, resource_group_location): @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westeurope') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westeurope') def test_aks_update_to_msi_cluster(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -1003,7 +1005,7 @@ def test_aks_update_to_msi_cluster(self, resource_group, resource_group_location @live_only() # without live only fails with need az login @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='eastus') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='eastus') def test_aks_create_with_gitops_addon(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -1019,7 +1021,7 @@ def test_aks_create_with_gitops_addon(self, resource_group, resource_group_locat @live_only() # without live only fails with need az login @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='eastus') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='eastus') def test_aks_enable_addon_with_gitops(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -1041,7 +1043,7 @@ def test_aks_enable_addon_with_gitops(self, resource_group, resource_group_locat @live_only() # without live only fails with need az login @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='eastus') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='eastus') def test_aks_disable_addon_gitops(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -1064,7 +1066,7 @@ def test_aks_disable_addon_gitops(self, resource_group, resource_group_location) @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westeurope') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westeurope') def test_aks_update_to_msi_cluster_with_addons(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -1088,7 +1090,7 @@ def test_aks_update_to_msi_cluster_with_addons(self, resource_group, resource_gr 'aks delete -g {resource_group} -n {name} --yes --no-wait', checks=[self.is_empty()]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_auto_upgrade_channel(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -1117,7 +1119,7 @@ def test_aks_create_with_auto_upgrade_channel(self, resource_group, resource_gro 'aks delete -g {resource_group} -n {name} --yes --no-wait', checks=[self.is_empty()]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_node_config(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -1150,7 +1152,7 @@ def test_aks_create_with_node_config(self, resource_group, resource_group_locati ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_none_private_dns_zone(self, resource_group, resource_group_location): # reset the count so in replay mode the random names will start with 0 self.test_resources_count = 0 @@ -1178,7 +1180,7 @@ def test_aks_create_none_private_dns_zone(self, resource_group, resource_group_l @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_fqdn_subdomain(self, resource_group, resource_group_location): # reset the count so in replay mode the random names will start with 0 self.test_resources_count = 0 @@ -1241,7 +1243,7 @@ def test_aks_create_fqdn_subdomain(self, resource_group, resource_group_location 'aks delete -g {resource_group} -n {name} --yes --no-wait', checks=[self.is_empty()]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_with_pod_identity_enabled(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -1322,7 +1324,7 @@ def test_aks_create_with_pod_identity_enabled(self, resource_group, resource_gro ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_create_using_azurecni_with_pod_identity_enabled(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) self.kwargs.update({ @@ -1404,7 +1406,7 @@ def test_aks_create_using_azurecni_with_pod_identity_enabled(self, resource_grou # for this case we cannot use recording to capture the fixture, therefore we need to mark it as live_only @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_pod_identity_usage(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) identity_name = self.create_random_name('id', 6) @@ -1494,7 +1496,7 @@ def test_aks_pod_identity_usage(self, resource_group, resource_group_location): ]) @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_update_with_windows_password(self, resource_group, resource_group_location): # reset the count so in replay mode the random names will start with 0 self.test_resources_count = 0 @@ -1540,7 +1542,7 @@ def test_aks_update_with_windows_password(self, resource_group, resource_group_l @live_only() @AllowLargeResponse() - @ResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') + @AKSCustomResourceGroupPreparer(random_name_length=17, name_prefix='clitest', location='westus2') def test_aks_custom_kubelet_identity(self, resource_group, resource_group_location): aks_name = self.create_random_name('cliakstest', 16) control_plane_identity_name = self.create_random_name('cliakstest', 16)